diff --git a/README.md b/README.md index fca5a184..7c2e3e4e 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,7 @@ Request JSON is not allowed to replace those trusted capabilities. - [Shared load state](examples/shared-load-state/README.md) - [Binary object storage and temporary S3 URLs](examples/object-storage/README.md) - [Context-owned HTTP with bounded retries and safe diagnostics](examples/http-resilience/README.md) +- [Transactional committed-event Outbox from a real SQLite graph save](examples/committed-events/README.md) - [Extension points](https://github.com/teaql/teaql-conformance/blob/main/design/runtime-customization-p0-p1-matrix.md) ## Safe troubleshooting diff --git a/examples/committed-events/README.md b/examples/committed-events/README.md new file mode 100644 index 00000000..aa2379cc --- /dev/null +++ b/examples/committed-events/README.md @@ -0,0 +1,143 @@ +# SQLite committed-event source example + +## Durable observations (CMR-14) + +The same security verifier records nine phases in `observations.json` using +public Context-participating source/target observation APIs. It intentionally +loses both snapshot and tail receipts **after actual target commit**. Target +watermark advances while source ACK/backlog stay behind; duplicate retry then +converges. Two durable journal rows retain six attempts, four retries/failures +and two publisher-observed permanent rejections. Target-wide audit counts five +rejections, including direct unauthorized-source attempts. Final dead letters +are zero, not deleted source facts. Typed observations and existing runtime +telemetry agree; exported attributes contain only counts and hashed routes. +No fake ACK, transport authentication claim or automatic retry worker is used. + +## Authenticated snapshot and tail (CMR-13) + +`bash examples/committed-events/verify-security.sh` executes the local runtime +example twice; the complete verifier includes it. Two audited Mutation saves, +not INSERT fixtures, produce baseline 1 and tail 2. A retained signed snapshot +bootstraps an independent target. A lost snapshot receipt leaves the source +cursor pending; retry gets a stable duplicate receipt. The signed update then +catches up without rerunning business methods. A source ID above JavaScript's +exact-integer range survives unchanged. + +Five real rejections cover bad snapshot tag, tampered tail with recomputed +checksum, unauthorized source, replay of snapshot-covered history and a newer +snapshot attempting to reset an active stream. Checks inspect rows, source +cursor, Inbox, checkpoint and durable hash-only audit, including a fresh target +Context/connection. Exact event/snapshot retries do not rewind the tail. + +Builds and disposable fixtures are removed on exit. Explicitly set +`TEAQL_CMR13_EVIDENCE_ROOT` to an absent absolute path to retain two small +SQLite pairs, result JSON and native logs/JUnit. For the complete gate, +`TEAQL_COMMITTED_EVIDENCE_DIR` similarly opts into retaining generated Q proof. +Never reuse/reset these directories. The key is public test material, not a +deployment secret. This is authenticated in-process ingress, not a network/TLS +server, receipt authentication, installed-package or production qualification. + +## Existing source and target flows + +Run `bash examples/committed-events/verify.sh`. It executes a small, real +`Order.save(context)` flow twice against a local Java source. Each round uses +fresh source/target SQLite files and an independent DuckDB target. The test +reopens the Outbox, delivers the two events through bounded publishers into +both model-checked targets, verifies their rows and the SQLite target duplicate +acknowledgement, then injects a later source write failure. The first SQLite +delivery deliberately loses its acknowledgement after target commit; a new +source executor replays sequence 1 as a duplicate and then applies sequence 2 +in order. This small fixture +itself requires no generated ERP model or remote service. + +The example configures `SqliteCommittedMutationRecorder` at trusted runtime +startup; `context.ensureSchema()` installs its infrastructure tables. A database trigger +sets `memo` after insert; the Outbox contains that authoritative readback, not +the pre-save entity value. Subsequent update produces sequence 2. When the +Outbox table is made unavailable, the attempted third save rolls back its +business update and emits no committed event. + +This is **not** authenticated network replication. The in-process sinks prove +local cursor/target behavior; transport identity, retry budgets, snapshot-plus- +tail and dynamic fields remain separate work. The included typed wire tests +cover selected scalar, null, date, instant and Decimal boundaries, not every +possible model type. + +The verifier also runs an application-owned generated Standard Order consumer +twice. It creates a Customer through generated audited Save on SQLite, delivers +one committed set to a model-checked DuckDB target, and reads the target through +generated Q. The checked-in generated library remains byte-for-byte unchanged; +public-package and production-network qualification are separate. + +## Real source-process crash (CMR-03) + +`bash examples/committed-events/verify-process-crash.sh` runs a separate bounded +fixture (also included by the complete verifier above). An actual child performs +a normal audited save, signals a post-commit/pre-publish barrier, and waits with +the runtime still open. Its owning test forcibly terminates and reaps it. A new +process recovers the durable Outbox, lands the event, simulates a lost target +acknowledgement, accepts the duplicate receipt and advances the source cursor. +A third process checks persistent rows, versions, Inbox and cursor again. + +The same source/target files are used for both rounds: their row/event/cursor +counts go from one to two, not back to zero. A business-invocation marker also +stays one-per-writer, not one-per-delivery. A negative control rejects normal +exit without a ready marker. Logs and waits are bounded; only owned children +are killed. No production callback, HTTP endpoint or new runtime API is added. + +Set `TEAQL_CMR03_EVIDENCE_DIR` to a **fresh** directory to retain the two small +databases, child logs, barriers and `result.json` at a known location. By default +an owned temporary evidence directory is printed and retained. Never delete or +reset a previous database to force success. Module `target` directories remain +ordinary reusable Maven build outputs; one-off gate owners clean their own +build directories after recording results. This proves controlled process death, +not machine/power failure, snapshot restart, all CMR-12 paths or public artifacts. + +## Consistent snapshot plus tail (CMR-11) + +Run `bash examples/committed-events/verify-snapshot.sh`. It copies the current +local source without build outputs, runs `CommittedSnapshotExampleTest` twice +offline and deletes its owned build/database scratch on exit. It never selects +a released runtime. The complete verifier also invokes this gate. + +The small fixture has two legacy orders (including a tombstone and an ID above +JavaScript's exact-integer range), a negative epoch timestamp, Boolean/NULL and +an empty entity catalog entry. Context `ensureSchema` creates its schema; +explicit fixture SQL represents old data with no captured journal. A separate +transaction simulates one captured tail. Actual business audited-save capture +remains demonstrated by `CommittedEventsExampleTest`, not by this fixture SQL. + +The example imports snapshot N=0, applies tail 1, retries both and proves that a +snapshot retry acknowledges 0 without rewinding checkpoint 1. Optional +`TEAQL_SNAPSHOT_EXPORT_DIR` writes only `snapshot.json`, `tail.json` and their +SHA-256 sidecars with create-new semantics. `TEAQL_SNAPSHOT_IMPORT_DIR` consumes +the same files, including those exported by the Rust example. Use separate +producer/consumer runs and fresh owned export directories; never overwrite old +evidence. These files do not define an authenticated network transport. No +CMR-12 process-crash or production-scale qualification is claimed. + +## Managed bootstrap process recovery (CMR-12 candidate) + +`bash examples/committed-events/verify-bootstrap-crash.sh` copies the **local** +source, compiles it offline and runs the process example twice. Each invocation +keeps one source database throughout two rounds (snapshot/tail 1/2, then 3/4). +The second target is a new consumer slot, not a reset database. The first target +then catches up to 4 without replacing its original snapshot or rewinding its +cursor. Ordinary business writes use audited saves; one retained legacy +tombstone predates capture. + +In each round the owner SIGKILLs and reaps seven distinct workers: during the +capture read transaction, publication retention, partial target installation, +after target commit/before snapshot receipt, between source cursor insert and +ACK commit, partial tail application, and after tail commit/before event receipt. +Fresh workers recover from the same files. SQL barriers exist only in this +test fixture; production has no fault hook. A normal exit without the expected +PID/phase barrier is rejected. Waits and logs are bounded. + +Set `TEAQL_CMR12_EVIDENCE_ROOT` to a **nonexistent absolute path** to retain +compact databases, child logs, PID barriers, JUnit XML and `result.json` for both +invocations. No existing path is reset. Otherwise successful disposable test +fixtures are removed after all owned workers exit; failed fixtures print their +diagnostic location. Temporary source/build outputs are removed by the verifier +on exit. This controlled Linux/SQLite **source** example is not installed-package, +power-loss, authenticated transport or whole-CMR-12 acceptance. diff --git a/examples/committed-events/pom.xml b/examples/committed-events/pom.xml new file mode 100644 index 00000000..55a4dded --- /dev/null +++ b/examples/committed-events/pom.xml @@ -0,0 +1,21 @@ + + 4.0.0 + + io.teaqlteaql-java-parent + 1.555-RELEASE../../pom.xml + + teaql-committed-events-example + + io.teaqlteaql-tfp-endpoint${project.version}test + io.teaqlteaql-runtime + io.teaqlteaql-sqlite + io.teaqlteaql-duckdb${project.version} + io.teaqlteaql-provider-jdbc + com.fasterxml.jackson.corejackson-databind + org.xerialsqlite-jdbc3.53.2.1 + org.duckdbduckdb_jdbc1.5.5.1 + junitjunittest + + diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedBootstrapCrashExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedBootstrapCrashExampleTest.java new file mode 100644 index 00000000..37653d92 --- /dev/null +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedBootstrapCrashExampleTest.java @@ -0,0 +1,386 @@ +package io.teaql.examples.committed; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.*; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.Function; +import org.sqlite.SQLiteDataSource; + +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; + +/** Real owned-process death at SQLite bootstrap/tail boundaries; not power-loss proof. */ +public class CommittedBootstrapCrashExampleTest { + private static final String SOURCE = "cmr12-java-source", MODEL = "sha256:" + "0".repeat(64); + private static final long LEGACY_ID = 9_007_199_254_740_993L; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final SqliteCommittedSnapshot.Limits LIMITS = new SqliteCommittedSnapshot.Limits(32, 65536); + private static final long WAIT_SECONDS = 30, MAX_LOG = 256 * 1024; + + private static String slot(int round) { return "analytics-" + round; } + private static String identity(int round) { return "bootstrap-" + round; } + private static long baseline(int round) { return 2L * round - 1; } + private static String digest(String text) throws Exception { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))); + } + private static void writeNew(Path file, Object value) throws Exception { + Files.writeString(file, JSON.writeValueAsString(value), StandardOpenOption.CREATE_NEW); + } + private static void pause(Path directory, String phase, int round, long visibleRows) { + try { + Path temporary = directory.resolve(phase + "-" + round + ".ready.tmp"); + // Flush diagnostics before announcing readiness; SIGKILL may follow the rename immediately. + System.out.println("CMR12_SQL_BARRIER phase=" + phase + " round=" + round + " visible_rows=" + visibleRows); + System.out.flush(); + writeNew(temporary, Map.of("pid", ProcessHandle.current().pid(), "phase", phase, + "round", round, "visibleRows", visibleRows)); + Files.move(temporary, directory.resolve(phase + "-" + round + ".ready.json"), StandardCopyOption.ATOMIC_MOVE); + for (;;) Thread.sleep(1000); // parent must SIGKILL; no graceful transaction cleanup + } catch (Exception failure) { throw new IllegalStateException(failure); } + } + + /** Test adapter only: delegates the real SELECT first, then pauses its active read transaction. */ + private static final class BarrierSql extends JdbcSqlExecutor { + final Path directory; + final String phase; + final int round; + boolean armed; + BarrierSql(SQLiteDataSource ds, Path directory, String phase, int round) { + super(ds); this.directory = directory; this.phase = phase; this.round = round; + addConnectionInitializer(connection -> { + try { + Function.create(connection, "cmr12_test_barrier", new Function() { + @Override protected void xFunc() throws java.sql.SQLException { + String wanted = value_text(0); + if (armed && phase.equals(wanted)) { + assertFalse("trigger must run inside a real transaction", connection.getAutoCommit()); + pause(directory, phase, round, value_long(1)); + } + result(0); + } + }); + } catch (java.sql.SQLException failure) { throw new IllegalStateException(failure); } + }); + } + @Override public List> queryForList(String statement, Object[] values) { + var rows = super.queryForList(statement, values); + if (armed && phase.equals("capture") && statement.startsWith("SELECT current_sequence")) { + assertTrue(hasActiveTransaction()); + assertEquals(baseline(round), ((Number) rows.get(0).get("current_sequence")).longValue()); + pause(directory, phase, round, -1); // sequence read is established; domain rows are not read yet + } + return rows; + } + } + private record Fixture(BarrierSql sql, DefaultUserContext context, TeaQLRuntime runtime) implements AutoCloseable { + SqliteCommittedSnapshot snapshots() { return new SqliteCommittedSnapshot(sql, SOURCE, MODEL); } + SqliteCommittedSnapshotPublisher bootstrap(int round) { return new SqliteCommittedSnapshotPublisher(sql, SOURCE, slot(round), MODEL); } + SqliteCommittedOutboxPublisher publisher(int round) { return new SqliteCommittedOutboxPublisher(sql, SOURCE, slot(round), MODEL); } + @Override public void close() { runtime.close(); } + void schema() { context.ensureSchema(); new SqliteCommittedEventStore(sql).ensureSchema(); } + } + private static Fixture fixture(Path directory, String phase, int round, boolean source, boolean capture) { + SQLiteDataSource ds = new SQLiteDataSource(); + ds.setUrl("jdbc:sqlite:" + directory.resolve(source ? "source.sqlite" : "target-" + round + ".sqlite")); + BarrierSql sql = new BarrierSql(ds, directory, phase, round); + SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + descriptor.setType("Order"); descriptor.setTargetType(CommittedEventsExampleTest.Order.class); + descriptor.setEntitySupplier(CommittedEventsExampleTest.Order::new); descriptor.setDataService("sqlite"); + for (String field : List.of("id", "version", "name", "memo")) { + boolean text = field.equals("name") || field.equals("memo"); + GenericSQLProperty property = (GenericSQLProperty) descriptor.addSimpleProperty(field, text ? String.class : Long.class); + property.setColumnType(text ? "VARCHAR(255)" : "BIGINT"); + } + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); metadata.register(descriptor); + var builder = TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", sql, ds)).executionLogging(false) + .idGenerationService((context, entity) -> { throw new AssertionError("fixture already assigns IDs; replication must not allocate"); }); + if (capture) builder.committedMutationRecorder(new SqliteCommittedMutationRecorder(sql, SOURCE, MODEL)); + TeaQLRuntime runtime = builder.build(); + return new Fixture(sql, new DefaultUserContext(runtime), runtime); + } + private static long count(Fixture f, String table) { + return ((Number) f.sql.queryForList("SELECT count(*) AS n FROM " + table, new Object[0]).get(0).get("n")).longValue(); + } + private static long checkpoint(Fixture f) { + var rows = f.sql.queryForList("SELECT source_sequence FROM teaql_event_checkpoint WHERE source_runtime_id=?", new Object[]{SOURCE}); + return rows.isEmpty() ? 0 : ((Number) rows.get(0).get("source_sequence")).longValue(); + } + private static SqliteCommittedSnapshot.SnapshotRecord retained(Fixture f, Path directory, int round) throws Exception { + var rows = f.sql.queryForList("SELECT * FROM teaql_event_snapshot_publication WHERE source_runtime_id=? AND target_id=?", new Object[]{SOURCE, slot(round)}); + assertEquals(1, rows.size()); var row = rows.get(0); + String payload = (String) row.get("payload_json"); + assertEquals(Files.readString(directory.resolve("snapshot-" + round + ".json")), payload); + assertEquals(Files.readString(directory.resolve("snapshot-" + round + ".sha256")), digest(payload)); + assertEquals(baseline(round), ((Number) row.get("source_sequence")).longValue()); + return new SqliteCommittedSnapshot.SnapshotRecord(SOURCE, baseline(round), identity(round), MODEL, digest(payload), payload); + } + private static void save(Fixture source, Path directory, int round, boolean tail) throws Exception { + Files.writeString(directory.resolve("business-calls.txt"), (tail ? "tail-" : "seed-") + round + "\n", + StandardOpenOption.CREATE, StandardOpenOption.APPEND); + var order = new CommittedEventsExampleTest.Order(); + order.__internalInitializeNewEntityId((tail ? 200L : 100L) + round); + order.updateProperty("name", (tail ? "tail-" : "seed-") + round); + order.updateProperty("memo", "业务事实"); + order.auditAs("persist source facts for controlled bootstrap recovery").save(source.context); + } + private static void sourceState(Path directory, int round, boolean job, boolean acked, boolean tail, boolean delivered) throws Exception { + try (Fixture source = fixture(directory, "read", round, true, false)) { + long n = baseline(round), high = n + (tail ? 1 : 0); + assertEquals(high, new SqliteCommittedEventStore(source.sql).outboxAfter(SOURCE, 0, 10).size()); + assertEquals(high + 1, count(source, "order_data")); + assertEquals(high, Files.readAllLines(directory.resolve("business-calls.txt")).size()); + var jobs = source.sql.queryForList("SELECT acknowledged FROM teaql_event_snapshot_publication WHERE target_id=?", new Object[]{slot(round)}); + assertEquals(job ? 1 : 0, jobs.size()); + if (job) { retained(source, directory, round); assertEquals(acked ? 1 : 0, ((Number) jobs.get(0).get("acknowledged")).intValue()); } + var cursors = source.sql.queryForList("SELECT acked_sequence FROM teaql_event_delivery_cursor WHERE target_id=?", new Object[]{slot(round)}); + assertEquals(acked ? 1 : 0, cursors.size()); + if (acked) assertEquals(delivered ? high : n, ((Number) cursors.get(0).get("acked_sequence")).longValue()); + } + } + private static void targetState(Path directory, int round, boolean installed, boolean applied) throws Exception { + try (Fixture target = fixture(directory, "read", round, false, false)) { + long n = baseline(round), position = installed ? n + (applied ? 1 : 0) : 0; + assertEquals(installed ? position + 1 : 0, count(target, "order_data")); + assertEquals(installed ? 1 : 0, count(target, "teaql_event_snapshot")); + assertEquals(installed ? 1 : 0, count(target, "teaql_event_checkpoint")); + assertEquals(position, checkpoint(target)); + assertEquals(applied ? 1 : 0, count(target, "teaql_event_inbox")); + assertEquals(applied ? 1 : 0, count(target, "teaql_event_applied_payload")); + if (installed) assertEquals(-4L, ((Number) target.sql.queryForList("SELECT version FROM order_data WHERE id=?", new Object[]{LEGACY_ID}).get(0).get("version")).longValue()); + } + } + + /** Actual fresh-connection readback after each owned worker exits; retained for deterministic audit. */ + private static Map observed(Path directory, int round) throws Exception { + Map state = new LinkedHashMap<>(); + try (Fixture source = fixture(directory, "read", round, true, false)) { + var jobs = source.sql.queryForList("SELECT source_sequence,acknowledged,payload_sha256 FROM teaql_event_snapshot_publication WHERE target_id=?", new Object[]{slot(round)}); + var cursors = source.sql.queryForList("SELECT acked_sequence FROM teaql_event_delivery_cursor WHERE target_id=?", new Object[]{slot(round)}); + state.put("source", Map.of("high", source.publisher(round).status().sourceHighWatermark(), + "domainRows", count(source, "order_data"), "businessCalls", Files.readAllLines(directory.resolve("business-calls.txt")).size(), + "snapshotSequence", jobs.isEmpty() ? -1 : ((Number) jobs.get(0).get("source_sequence")).longValue(), + "snapshotAcknowledged", jobs.isEmpty() ? -1 : ((Number) jobs.get(0).get("acknowledged")).intValue(), + "snapshotHash", jobs.isEmpty() ? "" : jobs.get(0).get("payload_sha256"), + "cursor", cursors.isEmpty() ? -1 : ((Number) cursors.get(0).get("acked_sequence")).longValue())); + } + if (Files.exists(directory.resolve("target-" + round + ".sqlite"))) { + try (Fixture target = fixture(directory, "read", round, false, false)) { + state.put("target", Map.of("present", true, "domainRows", count(target, "order_data"), + "snapshotRows", count(target, "teaql_event_snapshot"), "checkpoint", checkpoint(target), + "inbox", count(target, "teaql_event_inbox"), "applied", count(target, "teaql_event_applied_payload"))); + } + } else state.put("target", Map.of("present", false)); + return state; + } + + /** Test worker only. All barriers are in this fixture, not runtime production code. */ + public static void main(String[] args) throws Exception { + if (args.length != 3) throw new IllegalArgumentException("phase directory round required"); + String phase = args[0]; Path directory = Path.of(args[1]); int round = Integer.parseInt(args[2]); + if (phase.equals("premature")) return; + try (Fixture source = fixture(directory, phase, round, true, phase.equals("seed") || phase.equals("tail"))) { + if (phase.equals("seed")) { + source.schema(); + if (round == 1) source.sql.update("INSERT INTO order_data(id,version,name,memo) VALUES(?,-4,'旧记录',NULL)", new Object[]{LEGACY_ID}); // legacy pre-capture fixture only + save(source, directory, round, false); + } else if (phase.equals("tail")) { + save(source, directory, round, true); + } else if (Set.of("capture", "retain", "prepare").contains(phase)) { + if (phase.equals("retain")) source.sql.execute("CREATE TRIGGER retain_barrier_" + round + + " AFTER INSERT ON teaql_event_snapshot_publication WHEN NEW.target_id='" + slot(round) + + "' BEGIN SELECT cmr12_test_barrier('retain',(SELECT count(*) FROM teaql_event_snapshot_publication WHERE target_id=NEW.target_id)); END"); + source.sql.armed = true; + var prepared = source.bootstrap(round).prepare(source.context, identity(round), "capture bootstrap facts", "recover analytics without replay", LIMITS); + assertFalse(prepared.acknowledged()); assertEquals(baseline(round), prepared.snapshot().sourceSequence()); + Files.writeString(directory.resolve("snapshot-" + round + ".json"), prepared.snapshot().payloadJson(), StandardOpenOption.CREATE_NEW); + Files.writeString(directory.resolve("snapshot-" + round + ".sha256"), prepared.snapshot().payloadSha256(), StandardOpenOption.CREATE_NEW); + } else { + retained(source, directory, round); + try (Fixture target = fixture(directory, phase, round, false, false)) { + if (phase.equals("target_apply")) { + target.schema(); + target.sql.execute("CREATE TRIGGER snapshot_barrier AFTER INSERT ON order_data WHEN NEW.id=101 " + + "BEGIN SELECT cmr12_test_barrier('target_apply',(SELECT count(*) FROM order_data)); END"); + target.sql.armed = true; + } + if (phase.equals("source_ack")) { + source.sql.execute("CREATE TRIGGER ack_barrier_" + round + " AFTER INSERT ON teaql_event_delivery_cursor WHEN NEW.target_id='" + slot(round) + + "' BEGIN SELECT cmr12_test_barrier('source_ack',(SELECT count(*) FROM teaql_event_delivery_cursor WHERE target_id=NEW.target_id)); END"); + source.sql.armed = true; + } + if (Set.of("target_apply", "snapshot_receipt_lost", "source_ack", "ack").contains(phase)) { + var prepared = source.bootstrap(round).publish(source.context, identity(round), LIMITS, record -> { + var receipt = SqliteCommittedSnapshotPublisher.initializeAndAcknowledge(target.context, target.snapshots(), record, slot(round), LIMITS); + assertEquals(!phase.equals("snapshot_receipt_lost") && !phase.equals("target_apply"), receipt.duplicate()); + if (phase.equals("snapshot_receipt_lost")) pause(directory, phase, round, count(target, "order_data")); + return receipt; + }); + assertTrue(prepared.acknowledged()); + } else if (Set.of("tail_apply", "tail_receipt_lost", "tail_ack").contains(phase)) { + if (phase.equals("tail_apply")) { + target.sql.execute("CREATE TRIGGER tail_barrier AFTER INSERT ON order_data WHEN NEW.id=" + (200 + round) + + " BEGIN SELECT cmr12_test_barrier('tail_apply',(SELECT count(*) FROM order_data)); END"); + target.sql.armed = true; + } + var outcome = source.publisher(round).publishBatch(1, record -> { + assertEquals(baseline(round) + 1, record.sourceSequence()); + var result = new SqliteModelCheckedCommittedTarget(target.sql, SOURCE, MODEL).apply(target.context, record); + assertEquals(phase.equals("tail_ack"), result.duplicate()); + if (phase.equals("tail_receipt_lost")) pause(directory, phase, round, count(target, "order_data")); + return result; + }); + assertEquals(1, outcome.acknowledged()); + } else if (phase.equals("verify")) { + assertTrue(source.bootstrap(round).publish(source.context, identity(round), LIMITS, ignored -> { throw new AssertionError("known snapshot ACK must not redeliver"); }).acknowledged()); + assertEquals(0, source.publisher(round).publishBatch(1, ignored -> { throw new AssertionError("known tail ACK must not redeliver"); }).acknowledged()); + assertEquals(source.sql.queryForList("SELECT * FROM order_data ORDER BY id", new Object[0]), target.sql.queryForList("SELECT * FROM order_data ORDER BY id", new Object[0])); + } else if (phase.equals("catchup")) { + var batch = source.publisher(round).publishBatch(10, record -> new SqliteModelCheckedCommittedTarget(target.sql, SOURCE, MODEL).apply(target.context, record)); + assertEquals(2, batch.acknowledged()); assertEquals(4, batch.status().ackedSequence()); + assertEquals(1, source.bootstrap(round).publish(source.context, identity(round), LIMITS, ignored -> { throw new AssertionError("snapshot must not reset after catchup"); }).snapshot().sourceSequence()); + assertEquals(source.sql.queryForList("SELECT * FROM order_data ORDER BY id", new Object[0]), target.sql.queryForList("SELECT * FROM order_data ORDER BY id", new Object[0])); + } else throw new IllegalArgumentException("unknown phase " + phase); + } + } + System.out.println("CMR12_WORKER_PASS phase=" + phase + " round=" + round + " pid=" + ProcessHandle.current().pid()); + } + } + + private static Process start(String phase, Path directory, int round, List children) throws Exception { + String cp = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + Process child = new ProcessBuilder(Path.of(System.getProperty("java.home"), "bin", "java").toString(), "-ea", + "-Djava.io.tmpdir=" + System.getProperty("java.io.tmpdir"), "-cp", cp, + CommittedBootstrapCrashExampleTest.class.getName(), phase, directory.toString(), Integer.toString(round)) + .redirectErrorStream(true).redirectOutput(directory.resolve(phase + "-" + round + ".log").toFile()).start(); + children.add(child); return child; + } + private static void bounded(Process child, Path log, long deadline) throws Exception { + assertTrue("worker deadline: " + log, System.nanoTime() < deadline); + assertTrue("worker log budget: " + log, Files.size(log) <= MAX_LOG); + } + private static void reap(Process child) throws Exception { + if (child.isAlive()) child.destroyForcibly(); + assertTrue("owned child reaped", child.waitFor(10, TimeUnit.SECONDS)); + } + private static Map execute(String phase, Path directory, int round, boolean crash, List children) throws Exception { + Process child = start(phase, directory, round, children); + Path log = directory.resolve(phase + "-" + round + ".log"), ready = directory.resolve(phase + "-" + round + ".ready.json"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(WAIT_SECONDS); + try { + if (crash) { + while (!Files.exists(ready)) { + assertTrue("exited before SQL/commit barrier: " + log, child.isAlive()); + bounded(child, log, deadline); Thread.sleep(20); + } + var state = JSON.readTree(Files.readString(ready)); + assertEquals(child.pid(), state.path("pid").asLong()); assertEquals(phase, state.path("phase").asText()); + assertEquals(round, state.path("round").asInt()); assertTrue(child.isAlive()); + if (phase.equals("target_apply")) assertEquals(1, state.path("visibleRows").asLong()); + if (phase.equals("retain") || phase.equals("source_ack")) assertEquals(1, state.path("visibleRows").asLong()); + if (phase.equals("tail_apply")) assertEquals(baseline(round) + 2, state.path("visibleRows").asLong()); + child.destroyForcibly(); reap(child); assertEquals("SIGKILL, not orderly close", 137, child.exitValue()); + } else { + while (child.isAlive()) { bounded(child, log, deadline); Thread.sleep(20); } + assertEquals("worker failure: " + log, 0, child.exitValue()); + assertTrue(Files.readString(log).contains("CMR12_WORKER_PASS phase=" + phase + " round=" + round)); + } + assertTrue(Files.size(log) <= MAX_LOG); + return Map.of("phase", phase, "round", round, "pid", child.pid(), "exit", child.exitValue(), "killed", crash, + "logSha256", digest(Files.readString(log)), "observed", observed(directory, round)); + } finally { reap(child); } + } + private static void removeOwnedFixture(Path directory) throws Exception { + try (var paths = Files.walk(directory)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) Files.delete(path); + } + } + + @Test public void bootstrapAndTailSurviveSevenRealCrashBoundariesTwiceWithoutSourceReset() throws Exception { + assertTrue("controlled Linux SIGKILL gate", System.getProperty("os.name").startsWith("Linux")); + String sourceRoot = System.getProperty("cmr12.source.root"); + if (sourceRoot != null) for (Class type : List.of(SqliteCommittedSnapshotPublisher.class, + SqliteCommittedSnapshot.class, SqliteCommittedOutboxPublisher.class, TeaQLRuntime.class)) { + String origin = Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toRealPath().toString(); + assertTrue("archive source origin: " + origin, origin.startsWith(sourceRoot + "/") && origin.endsWith("/target/classes")); + System.out.println("CMR12_PROCESS_SOURCE_ORIGIN class=" + type.getName() + " path=" + origin); + } + String configured = System.getenv("TEAQL_CMR12_EVIDENCE_DIR"); + Path directory = configured == null ? Files.createTempDirectory("teaql-cmr12-java-") : Path.of(configured).toAbsolutePath(); + Files.createDirectories(directory); assertFalse("fresh fixture only; never reset an old DB", Files.exists(directory.resolve("source.sqlite"))); + writeNew(directory.resolve("owner.json"), Map.of("pid", ProcessHandle.current().pid(), "scope", "cmr12-java-process-example")); + List children = new ArrayList<>(); List> workers = new ArrayList<>(); boolean passed = false; + try { + for (int round = 1; round <= 2; round++) { + workers.add(execute("seed", directory, round, false, children)); + sourceState(directory, round, false, false, false, false); + for (String phase : List.of("capture", "retain")) { + workers.add(execute(phase, directory, round, true, children)); + sourceState(directory, round, false, false, false, false); + assertFalse(Files.exists(directory.resolve("snapshot-" + round + ".json"))); + } + workers.add(execute("prepare", directory, round, false, children)); + sourceState(directory, round, true, false, false, false); + workers.add(execute("tail", directory, round, false, children)); + sourceState(directory, round, true, false, true, false); + workers.add(execute("target_apply", directory, round, true, children)); + targetState(directory, round, false, false); sourceState(directory, round, true, false, true, false); + workers.add(execute("snapshot_receipt_lost", directory, round, true, children)); + targetState(directory, round, true, false); sourceState(directory, round, true, false, true, false); + workers.add(execute("source_ack", directory, round, true, children)); + targetState(directory, round, true, false); sourceState(directory, round, true, false, true, false); + workers.add(execute("ack", directory, round, false, children)); + sourceState(directory, round, true, true, true, false); + workers.add(execute("tail_apply", directory, round, true, children)); + targetState(directory, round, true, false); sourceState(directory, round, true, true, true, false); + workers.add(execute("tail_receipt_lost", directory, round, true, children)); + targetState(directory, round, true, true); sourceState(directory, round, true, true, true, false); + workers.add(execute("tail_ack", directory, round, false, children)); + workers.add(execute("verify", directory, round, false, children)); + targetState(directory, round, true, true); sourceState(directory, round, true, true, true, true); + System.out.println("CMR12_PROCESS_PASS language=java round=" + round + " baseline=" + baseline(round) + + " tail=" + (baseline(round) + 1) + " killed=7 same_source_database=1 original_bytes=1"); + } + workers.add(execute("catchup", directory, 1, false, children)); + assertEquals(workers.size(), workers.stream().map(w -> w.get("pid")).distinct().count()); + try (Fixture source = fixture(directory, "read", 2, true, false); + Fixture first = fixture(directory, "read", 1, false, false); + Fixture second = fixture(directory, "read", 2, false, false)) { + assertEquals(4, Files.readAllLines(directory.resolve("business-calls.txt")).size()); + assertEquals(2, count(source, "teaql_event_snapshot_publication")); + assertEquals(4, source.publisher(1).status().ackedSequence()); assertEquals(4, source.publisher(2).status().ackedSequence()); + assertEquals(4, checkpoint(first)); assertEquals(4, checkpoint(second)); + assertEquals(3, count(first, "teaql_event_inbox")); assertEquals(1, count(second, "teaql_event_inbox")); + assertEquals(source.sql.queryForList("SELECT * FROM order_data ORDER BY id", new Object[0]), first.sql.queryForList("SELECT * FROM order_data ORDER BY id", new Object[0])); + } + writeNew(directory.resolve("result.json"), Map.of("passed", true, "workers", workers, "rounds", 2, "kills", 14, + "sameSourceDatabase", true, "sourceHighWatermark", 4, "businessCalls", 4, "targetCheckpoints", List.of(4,4), + "scope", "controlled Linux SQLite process crash; not power loss, installed artifacts or full CMR-12 acceptance")); + passed = true; + } finally { + for (Process child : children) reap(child); + if (configured == null && passed) removeOwnedFixture(directory); + else System.out.println("CMR12_PROCESS_EVIDENCE language=java path=" + directory); + } + } + + @Test public void normalExitWithoutBarrierIsRejected() throws Exception { + Path directory = Files.createTempDirectory("teaql-cmr12-java-negative-"); List children = new ArrayList<>(); + try { + assertThrows(AssertionError.class, () -> execute("premature", directory, 1, true, children)); + assertFalse(Files.exists(directory.resolve("source.sqlite"))); + assertFalse(Files.exists(directory.resolve("premature-1.ready.json"))); + } finally { for (Process child : children) reap(child); removeOwnedFixture(directory); } + } +} diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java new file mode 100644 index 00000000..c6f7c5ff --- /dev/null +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java @@ -0,0 +1,223 @@ +package io.teaql.examples.committed; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.BaseEntity; +import io.teaql.core.duck.DuckDbCommittedInbox; +import io.teaql.core.duck.DuckDbModelCheckedCommittedTarget; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome; +import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Logger; + +import static org.junit.Assert.*; + +public class CommittedEventsExampleTest { + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { return DriverManager.getConnection(url); } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class type) throws SQLException { throw new SQLException("not a wrapper"); } + @Override public boolean isWrapperFor(Class type) { return false; } + } + + public static final class Order extends BaseEntity { + private final Map fields = new HashMap<>(); + + @Override public String typeName() { return "Order"; } + @Override public Object __internalGet(String field) { + return field.equals("name") || field.equals("memo") ? fields.get(field) : super.__internalGet(field); + } + @Override public void __internalSet(String field, Object value) { + if (field.equals("name") || field.equals("memo")) fields.put(field, value); + else super.__internalSet(field, value); + } + } + + @Test public void realSaveFreezesDatabaseResultsBeforeCommit() throws Exception { + Path path = Files.createTempFile("teaql-committed-example-", ".sqlite"); + path.toFile().deleteOnExit(); + SQLiteDataSource source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor sql = new JdbcSqlExecutor(source); + SqliteCommittedMutationRecorder recorder = new SqliteCommittedMutationRecorder( + sql, "example-java-source", "sha256:" + "0".repeat(64)); + + SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + descriptor.setType("Order"); + descriptor.setTargetType(Order.class); + descriptor.setEntitySupplier(Order::new); + descriptor.setDataService("sqlite"); + for (String field : new String[] {"id", "version", "name", "memo"}) { + boolean text = field.equals("name") || field.equals("memo"); + GenericSQLProperty property = (GenericSQLProperty) descriptor.addSimpleProperty( + field, text ? String.class : Long.class); + property.setColumnType(text ? "VARCHAR(255)" : "BIGINT"); + } + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + metadata.register(descriptor); + TeaQLRuntime runtime = TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", sql, source)) + .idGenerationService((caller, entity) -> 42L) + .committedMutationRecorder(recorder) + .executionLogging(false).build(); + DefaultUserContext context = new DefaultUserContext(runtime); + context.ensureSchema(); + sql.execute("CREATE TRIGGER order_memo AFTER INSERT ON order_data " + + "BEGIN UPDATE order_data SET memo = 'set by database' WHERE id = NEW.id; END"); + + Order order = new Order(); + order.__internalInitializeNewEntityId(42L); + order.updateProperty("name", "draft"); + order.auditAs("create example order").save(context); + var first = recorder.eventStore().outboxAfter("example-java-source", 0, 10); + assertEquals(1, first.size()); + assertEquals(1, first.get(0).sourceSequence()); + JsonNode created = new ObjectMapper().readTree(first.get(0).payloadJson()); + assertEquals("set by database", created.path("mutations").get(0).path("fields").path("memo").asText()); + assertEquals("create example order", created.path("audit").path("comment").asText()); + + order.updateProperty("name", "submitted"); + order.auditAs("submit example order").save(context); + JdbcSqlExecutor reopened = new JdbcSqlExecutor(source); + var durable = new SqliteCommittedEventStore(reopened) + .outboxAfter("example-java-source", 0, 10); + assertEquals(2, durable.size()); + JsonNode submitted = new ObjectMapper().readTree(durable.get(1).payloadJson()); + assertEquals("1", submitted.path("mutations").get(0).path("base_version").asText()); + assertEquals("2", submitted.path("mutations").get(0).path("committed_version").asText()); + + Path replicaPath = Files.createTempFile("teaql-committed-target-example-", ".sqlite"); + replicaPath.toFile().deleteOnExit(); + SQLiteDataSource replicaSource = new SQLiteDataSource(); + replicaSource.setUrl("jdbc:sqlite:" + replicaPath); + JdbcSqlExecutor replicaSql = new JdbcSqlExecutor(replicaSource); + TeaQLRuntime replicaRuntime = TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", replicaSql, replicaSource)) + .executionLogging(false).build(); + DefaultUserContext replicaContext = new DefaultUserContext(replicaRuntime); + replicaContext.ensureSchema(); + SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget( + replicaSql, "example-java-source", "sha256:" + "0".repeat(64)); + target.ensureSchema(); + var interruptedPublisher = new SqliteCommittedOutboxPublisher(reopened, + "example-java-source", "example-analytics", "sha256:" + "0".repeat(64)); + assertThrows(IllegalStateException.class, () -> interruptedPublisher.publishBatch(10, + record -> { + target.apply(replicaContext, record); + throw new IllegalStateException("simulate lost target acknowledgement"); + })); + assertEquals(0L, interruptedPublisher.status().ackedSequence()); + assertEquals(2L, interruptedPublisher.status().backlog()); + assertEquals(1, replicaSql.queryForList("SELECT * FROM teaql_event_inbox", + new Object[0]).size()); + + SQLiteDataSource restartedSource = new SQLiteDataSource(); + restartedSource.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor restarted = new JdbcSqlExecutor(restartedSource); + var published = new SqliteCommittedOutboxPublisher(restarted, "example-java-source", + "example-analytics", "sha256:" + "0".repeat(64)) + .publishBatch(10, record -> target.apply(replicaContext, record)); + assertEquals(2, published.acknowledged()); + assertEquals(0, published.status().backlog()); + assertEquals(2L, published.status().ackedSequence()); + var duplicate = target.apply(replicaContext, durable.get(0)); + assertTrue(duplicate.duplicate()); + assertEquals(1, duplicate.checkpoint()); + var replicated = replicaSql.queryForList("SELECT version, name, memo FROM order_data WHERE id = ?", + new Object[] {42L}).get(0); + assertEquals(2L, ((Number) replicated.get("version")).longValue()); + assertEquals("submitted", replicated.get("name")); + assertEquals("set by database", replicated.get("memo")); + assertEquals(2, replicaSql.queryForList("SELECT * FROM teaql_event_applied_payload", + new Object[0]).size()); + System.out.println("COMMITTED_DELIVERY_RECOVERY_PASS language=java ordered=2 " + + "lost_ack=1 restart=1 duplicate=1"); + replicaRuntime.close(); + + Path duckDirectory = Files.createTempDirectory("teaql-committed-duckdb-example-"); + duckDirectory.toFile().deleteOnExit(); + Path duckPath = duckDirectory.resolve("target.db"); + duckPath.toFile().deleteOnExit(); + JdbcSqlExecutor duckSql = new JdbcSqlExecutor( + new DriverManagerDataSource("jdbc:duckdb:" + duckPath)); + duckSql.execute("CREATE TABLE order_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "name VARCHAR, memo VARCHAR)"); + DuckDbModelCheckedCommittedTarget duckTarget = + new DuckDbModelCheckedCommittedTarget(duckSql, "example-java-source", + "sha256:" + "0".repeat(64)); + duckTarget.ensureInboxSchema(); + var duckPublished = new SqliteCommittedOutboxPublisher(reopened, "example-java-source", + "example-duckdb-analytics", "sha256:" + "0".repeat(64)) + .publishBatch(10, record -> { + var outcome = duckTarget.apply(context, new DuckDbCommittedInbox.Record( + record.sourceRuntimeId(), record.sourceSequence(), record.changeSetId(), + record.modelHash(), record.payloadSha256(), record.payloadJson())); + return new InboxOutcome(outcome.duplicate(), outcome.checkpoint()); + }); + assertEquals(2, duckPublished.acknowledged()); + assertEquals(0, duckPublished.status().backlog()); + var duckOrder = duckSql.queryForList("SELECT version, name, memo FROM order_data WHERE id = ?", + new Object[] {42L}).get(0); + assertEquals(2L, ((Number) duckOrder.get("version")).longValue()); + assertEquals("submitted", duckOrder.get("name")); + assertEquals("set by database", duckOrder.get("memo")); + assertEquals(2, duckSql.queryForList("SELECT * FROM teaql_event_applied_payload", + new Object[0]).size()); + + sql.execute("DROP TABLE teaql_event_outbox"); + order.updateProperty("name", "should roll back"); + assertThrows(RuntimeException.class, + () -> order.auditAs("reject when outbox unavailable").save(context)); + assertEquals("submitted", sql.queryForList("SELECT name FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("name")); + assertEquals(2L, ((Number) sql.queryForList("SELECT version FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("version")).longValue()); + + // Even if application code catches the inner save failure, the shared + // JDBC transaction must not commit a business row without its outbox. + order.updateProperty("name", "caught failure must roll back"); + assertThrows(IllegalStateException.class, () -> sql.executeInTransaction(() -> { + try { + order.auditAs("reject caught outbox failure").save(context); + } catch (RuntimeException expected) { + // Simulate a caller that handles the failed save and returns. + } + })); + assertEquals("submitted", sql.queryForList("SELECT name FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("name")); + assertEquals(2L, ((Number) sql.queryForList("SELECT version FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("version")).longValue()); + runtime.close(); + } +} diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedProcessCrashExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedProcessCrashExampleTest.java new file mode 100644 index 00000000..4d2c55ae --- /dev/null +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedProcessCrashExampleTest.java @@ -0,0 +1,259 @@ +package io.teaql.examples.committed; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.*; + +/** CMR-03: real source-process death, not an orderly in-process reopen. */ +public class CommittedProcessCrashExampleTest { + private static final String SOURCE = "cmr03-java-source"; + private static final String TARGET = "cmr03-java-target"; + private static final String MODEL = "sha256:" + "0".repeat(64); + private static final ObjectMapper JSON = new ObjectMapper(); + private static final Duration BUDGET = Duration.ofSeconds(30); + private static final long MAX_CHILD_LOG = 256 * 1024; + + private record Fixture(JdbcSqlExecutor sql, DefaultUserContext context, + TeaQLRuntime runtime) implements AutoCloseable { + @Override public void close() { runtime.close(); } + } + + private static Fixture fixture(Path file, boolean capture, int round) { + SQLiteDataSource ds = new SQLiteDataSource(); + ds.setUrl("jdbc:sqlite:" + file); + JdbcSqlExecutor sql = new JdbcSqlExecutor(ds); + SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + descriptor.setType("Order"); + descriptor.setTargetType(CommittedEventsExampleTest.Order.class); + descriptor.setEntitySupplier(CommittedEventsExampleTest.Order::new); + descriptor.setDataService("sqlite"); + for (String field : new String[] {"id", "version", "name", "memo"}) { + boolean text = field.equals("name") || field.equals("memo"); + GenericSQLProperty property = (GenericSQLProperty) descriptor.addSimpleProperty( + field, text ? String.class : Long.class); + property.setColumnType(text ? "VARCHAR(255)" : "BIGINT"); + } + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + metadata.register(descriptor); + TeaQLRuntime.Builder builder = TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", sql, ds)) + .executionLogging(false) + .idGenerationService((context, entity) -> { + if (!capture) throw new AssertionError("replication must not allocate entity IDs"); + return 100L + round; + }); + if (capture) builder.committedMutationRecorder(new SqliteCommittedMutationRecorder(sql, SOURCE, MODEL)); + TeaQLRuntime runtime = builder.build(); + return new Fixture(sql, new DefaultUserContext(runtime), runtime); + } + + private static SqliteCommittedOutboxPublisher publisher(JdbcSqlExecutor sql) { + return new SqliteCommittedOutboxPublisher(sql, SOURCE, TARGET, MODEL); + } + + private static long count(JdbcSqlExecutor sql, String trustedTable) { + return ((Number) sql.queryForList("SELECT COUNT(*) AS n FROM " + trustedTable, + new Object[0]).get(0).get("n")).longValue(); + } + + private static void sourceState(Path directory, int round, int ack) throws Exception { + try (Fixture source = fixture(directory.resolve("source.sqlite"), false, round)) { + assertEquals(round, count(source.sql(), "order_data")); + assertEquals(round, new SqliteCommittedEventStore(source.sql()).outboxAfter(SOURCE, 0, 10).size()); + var status = publisher(source.sql()).status(); + assertEquals(round, status.sourceHighWatermark()); + assertEquals(ack, status.ackedSequence()); + assertEquals(round - ack, status.backlog()); + assertEquals(round, Files.readAllLines(directory.resolve("business-invocations.txt")).size()); + } + } + + private static void targetState(Fixture target, int round) { + assertEquals(round, count(target.sql(), "order_data")); + assertEquals(round, count(target.sql(), "teaql_event_inbox")); + for (int r = 1; r <= round; r++) { + var row = target.sql().queryForList("SELECT version, name FROM order_data WHERE id = ?", + new Object[] {100L + r}).get(0); + assertEquals(1L, ((Number) row.get("version")).longValue()); + assertEquals("persisted-" + r, row.get("name")); + } + } + + /** Private test worker; not an application endpoint or runtime API. */ + public static void main(String[] args) throws Exception { + if (args.length != 3) throw new IllegalArgumentException("phase directory round required"); + String phase = args[0]; + Path directory = Path.of(args[1]); + int round = Integer.parseInt(args[2]); + if (phase.equals("premature")) return; // negative control: no commit/barrier + if (phase.equals("seed")) { + try (Fixture source = fixture(directory.resolve("source.sqlite"), true, round)) { + source.context().ensureSchema(); + Files.writeString(directory.resolve("business-invocations.txt"), "seed-" + round + "\n", + StandardOpenOption.CREATE, StandardOpenOption.APPEND); + var order = new CommittedEventsExampleTest.Order(); + order.__internalInitializeNewEntityId(100L + round); + order.updateProperty("name", "persisted-" + round); + order.auditAs("commit order before controlled source process crash").save(source.context()); + sourceState(directory, round, round - 1); + Path temporary = directory.resolve("ready-" + round + ".tmp"); + Files.writeString(temporary, JSON.writeValueAsString(Map.of("pid", ProcessHandle.current().pid(), + "round", round, "phase", "committed_before_publish")), StandardOpenOption.CREATE_NEW); + Files.move(temporary, directory.resolve("ready-" + round + ".json"), StandardCopyOption.ATOMIC_MOVE); + System.out.println("CMR03_BARRIER pid=" + ProcessHandle.current().pid() + " round=" + round); + System.out.flush(); + for (;;) Thread.sleep(1000); // parent must kill us; do not close runtime normally + } + } + if (!phase.equals("recover") && !phase.equals("verify")) throw new IllegalArgumentException("unknown phase"); + try (Fixture source = fixture(directory.resolve("source.sqlite"), false, round); + Fixture target = fixture(directory.resolve("target.sqlite"), false, round)) { + if (phase.equals("recover")) target.context().ensureSchema(); + var apply = new SqliteModelCheckedCommittedTarget(target.sql(), SOURCE, MODEL); + apply.ensureSchema(); + var record = new SqliteCommittedEventStore(source.sql()).outboxAfter(SOURCE, round - 1, 1).get(0); + assertEquals(round, record.sourceSequence()); + if (phase.equals("recover")) { + sourceState(directory, round, round - 1); + var applied = apply.apply(target.context(), record); // target commit; ack deliberately lost + assertFalse(applied.duplicate()); + assertEquals(round, applied.checkpoint()); + assertEquals(round - 1, publisher(source.sql()).status().ackedSequence()); + var batch = publisher(source.sql()).publishBatch(1, item -> { + var duplicate = apply.apply(target.context(), item); + assertTrue(duplicate.duplicate()); + assertEquals(round, duplicate.checkpoint()); + return duplicate; + }); + assertEquals(1, batch.acknowledged()); + assertEquals(round, batch.status().ackedSequence()); + assertEquals(0, batch.status().backlog()); + } else { + assertTrue(apply.apply(target.context(), record).duplicate()); // durable across another new process + assertEquals(0, publisher(source.sql()).publishBatch(1, item -> { + throw new AssertionError("acked events must not be sent again"); + }).acknowledged()); + } + sourceState(directory, round, round); + targetState(target, round); + System.out.println("CMR03_WORKER_PASS phase=" + phase + " pid=" + ProcessHandle.current().pid() + + " round=" + round + " ack=" + round + " inbox=" + round + " business_invocations=" + round); + } + } + + private static Process start(String phase, Path directory, int round) throws Exception { + String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + return new ProcessBuilder(Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-ea", "-cp", classpath, CommittedProcessCrashExampleTest.class.getName(), phase, + directory.toString(), Integer.toString(round)) + .redirectErrorStream(true).redirectOutput(directory.resolve(phase + "-" + round + ".log").toFile()).start(); + } + + private static void checkLog(Path directory, String phase, int round) throws Exception { + assertTrue("bounded worker log", Files.size(directory.resolve(phase + "-" + round + ".log")) <= MAX_CHILD_LOG); + } + + private static void stopAndReap(Process child) throws Exception { + if (child.isAlive()) child.destroyForcibly(); + assertTrue("owned child must be reaped", child.waitFor(10, TimeUnit.SECONDS)); + } + + private static void barrier(Process child, Path directory, int round) throws Exception { + long deadline = System.nanoTime() + BUDGET.toNanos(); + Path ready = directory.resolve("ready-" + round + ".json"); + while (!Files.exists(ready)) { + assertTrue("writer exited before durable barrier", child.isAlive()); + assertTrue("writer barrier deadline", System.nanoTime() < deadline); + checkLog(directory, "seed", round); + Thread.sleep(20); + } + var observed = JSON.readTree(Files.readString(ready)); + assertEquals(child.pid(), observed.path("pid").asLong()); + assertEquals(round, observed.path("round").asInt()); + assertEquals("committed_before_publish", observed.path("phase").asText()); + assertTrue("writer must still be waiting when killed", child.isAlive()); + } + + private static void finished(Process child, Path directory, String phase, int round) throws Exception { + long deadline = System.nanoTime() + BUDGET.toNanos(); + while (child.isAlive()) { + assertTrue("worker deadline", System.nanoTime() < deadline); + checkLog(directory, phase, round); + Thread.sleep(20); + } + assertEquals("worker log: " + directory.resolve(phase + "-" + round + ".log"), 0, child.exitValue()); + checkLog(directory, phase, round); + } + + @Test public void killedSourceRecoversInFreshProcessesTwiceWithoutDatabaseReset() throws Exception { + String configured = System.getenv("TEAQL_CMR03_EVIDENCE_DIR"); + Path directory = configured == null ? Files.createTempDirectory("teaql-cmr03-java-") : Path.of(configured).toAbsolutePath(); + Files.createDirectories(directory); + assertFalse(Files.exists(directory.resolve("source.sqlite"))); // never erase a prior database + Files.writeString(directory.resolve("owner.json"), JSON.writeValueAsString(Map.of("pid", ProcessHandle.current().pid(), "kind", "cmr03-example")), StandardOpenOption.CREATE_NEW); + List> rounds = new ArrayList<>(); + List children = new ArrayList<>(); + try { + for (int round = 1; round <= 2; round++) { + Process writer = start("seed", directory, round); children.add(writer); + barrier(writer, directory, round); + writer.destroyForcibly(); stopAndReap(writer); + assertNotEquals("forced source termination", 0, writer.exitValue()); + sourceState(directory, round, round - 1); // independent read after actual process death + Process recovered = start("recover", directory, round); children.add(recovered); + finished(recovered, directory, "recover", round); + Process verified = start("verify", directory, round); children.add(verified); + finished(verified, directory, "verify", round); + assertEquals(3, new HashSet<>(List.of(writer.pid(), recovered.pid(), verified.pid())).size()); + rounds.add(Map.of("round", round, "writerPid", writer.pid(), "writerExit", writer.exitValue(), + "recoveryPid", recovered.pid(), "verificationPid", verified.pid(), "ack", round, + "domainRows", round, "outboxRows", round, "inboxRows", round, "businessInvocations", round)); + System.out.println("CMR03_PROCESS_CRASH_PASS language=java round=" + round + " killed_pid=" + writer.pid() + + " killed_exit=" + writer.exitValue() + " recovery_pid=" + recovered.pid() + " verify_pid=" + verified.pid()); + } + Files.writeString(directory.resolve("result.json"), JSON.writeValueAsString(Map.of("passed", true, "rounds", rounds, + "sameDatabases", true, "scope", "controlled SQLite source-process crash, not power failure or snapshot crash"))); + } finally { + for (Process child : children) stopAndReap(child); + System.out.println("CMR03_EVIDENCE language=java path=" + directory); + } + } + + @Test public void normalWorkerExitWithoutBarrierCannotPass() throws Exception { + Path directory = Files.createTempDirectory("teaql-cmr03-java-negative-"); + Process child = start("premature", directory, 1); + // Use the seed log name expected by the barrier observer, without inventing a ready marker. + Files.writeString(directory.resolve("seed-1.log"), ""); + try { assertThrows(AssertionError.class, () -> barrier(child, directory, 1)); } + finally { stopAndReap(child); } + assertFalse(Files.exists(directory.resolve("source.sqlite"))); + assertFalse(Files.exists(directory.resolve("ready-1.json"))); + Files.delete(directory.resolve("seed-1.log")); + Files.delete(directory.resolve("premature-1.log")); + Files.delete(directory); // successful negative control has no DB or diagnostic to retain + } +} diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedReceiverExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedReceiverExampleTest.java new file mode 100644 index 00000000..acaead2e --- /dev/null +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedReceiverExampleTest.java @@ -0,0 +1,298 @@ +package io.teaql.examples.committed; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.BaseEntity; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.*; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.core.sqlite.SqliteCommittedReceiver.Delivery; +import io.teaql.core.sqlite.SqliteCommittedReceiver.SnapshotDelivery; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import io.teaql.runtime.RuntimeTelemetry; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; + +import static org.junit.Assert.*; + +/** Actual audited source saves, authenticated local delivery and durable readback. */ +public class CommittedReceiverExampleTest { + private static final String SOURCE = "cmr13-example-source", TARGET = "analytics"; + private static final String MODEL = "sha256:" + "0".repeat(64); + private static final String CANARY = "private-business-fact"; + // Public test material only. Deployments provision their own independent secret. + private static final byte[] KEY = "0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.US_ASCII); + private static final long ID = 9_007_199_254_740_993L; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final SqliteCommittedSnapshot.Limits LIMITS = new SqliteCommittedSnapshot.Limits(8, 65536); + + public static final class CustomerOrder extends BaseEntity { + private final Map fields = new HashMap<>(); + @Override public String typeName() { return "CustomerOrder"; } + @Override public Object __internalGet(String field) { + return field.equals("name") || field.equals("memo") ? fields.get(field) : super.__internalGet(field); + } + @Override public void __internalSet(String field, Object value) { + if (field.equals("name") || field.equals("memo")) fields.put(field, value); + else super.__internalSet(field, value); + } + } + private static final class Fixture implements AutoCloseable { + final JdbcSqlExecutor sql; + final TeaQLRuntime runtime; + final DefaultUserContext context; + final SqliteCommittedMutationRecorder recorder; + final List> observations = new ArrayList<>(); + Fixture(Path file, boolean capture) { + SQLiteDataSource data = new SQLiteDataSource(); data.setUrl("jdbc:sqlite:" + file); + sql = new JdbcSqlExecutor(data); + SQLEntityDescriptor entity = new SQLEntityDescriptor(); + entity.setType("CustomerOrder"); entity.setTargetType(CustomerOrder.class); + entity.setEntitySupplier(CustomerOrder::new); entity.setDataService("sqlite"); + for (String field : List.of("id", "version", "name", "memo")) { + boolean text = field.equals("name") || field.equals("memo"); + GenericSQLProperty property = (GenericSQLProperty) entity.addSimpleProperty(field, text ? String.class : Long.class); + property.setColumnType(text ? "VARCHAR(255)" : "BIGINT"); + } + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); metadata.register(entity); + recorder = capture ? new SqliteCommittedMutationRecorder(sql, SOURCE, MODEL) : null; + var builder = TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", sql, data)) + .idGenerationService((caller, value) -> ID).executionLogging(false) + .telemetry(operation -> { + if (operation.family().equals("replication")) observations.add(operation.attributes()); + return RuntimeTelemetry.NOOP.start(operation); + }); + if (capture) builder.committedMutationRecorder(recorder); + runtime = builder.build(); context = new DefaultUserContext(runtime); context.ensureSchema(); + } + SqliteCommittedReceiver receiver() { + var receiver = new SqliteCommittedReceiver(sql, SOURCE, TARGET, MODEL, Map.of("current", KEY)); + receiver.ensureSchema(context); return receiver; + } + @Override public void close() { runtime.close(); } + } + private static long scalar(JdbcSqlExecutor sql, String query) { + return ((Number) sql.queryForList(query, new Object[0]).get(0).values().iterator().next()).longValue(); + } + private static Map domain(JdbcSqlExecutor sql) { + var rows = sql.queryForList("SELECT CAST(id AS TEXT) AS id,version,name,memo FROM customer_order_data", new Object[0]); + assertEquals(1, rows.size()); return rows.get(0); + } + private static void position(Fixture target, long checkpoint, long inbox, long audits, String name) { + assertEquals(checkpoint, scalar(target.sql, "SELECT source_sequence FROM teaql_event_checkpoint")); + assertEquals(inbox, scalar(target.sql, "SELECT count(*) FROM teaql_event_inbox")); + assertEquals(audits, scalar(target.sql, "SELECT count(*) FROM teaql_event_rejection")); + assertEquals(name, domain(target.sql).get("name")); + } + private static void rejected(String code, Runnable call) { + var failure = assertThrows(SqliteCommittedReceiver.ReceiverException.class, call::run); + assertEquals(code, failure.code()); assertFalse(failure.toString().contains(CANARY)); + } + private static String digest(String value) throws Exception { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } + private static Map observe(String phase, Fixture source, Fixture target) throws Exception { + var sourceObserver = new SqliteCommittedObservation(source.sql,SOURCE,TARGET,MODEL); + var s = sourceObserver.source(source.context); + var t = new SqliteCommittedObservation(target.sql,SOURCE,TARGET,MODEL) + .target(target.context,OptionalLong.of(s.sourceHighWatermark())); + var src = new LinkedHashMap(); + src.put("sourceHighWatermark",s.sourceHighWatermark());src.put("ackedSequence",s.ackedSequence()); + src.put("sequenceLag",s.sequenceLag());src.put("backlog",s.backlog());src.put("pendingSnapshots",s.pendingSnapshots()); + src.put("attempts",s.attempts());src.put("retries",s.retries());src.put("failures",s.failures()); + src.put("rejections",s.rejections());src.put("deadLetters",s.deadLetters());src.put("unresolvedAttempts",s.unresolvedAttempts()); + long[] expected=switch(phase){ + case "prepared"->new long[]{1,0,1,1,1,0,0,0,0,0,0}; + case "snapshot_rejected"->new long[]{1,0,1,1,1,1,0,1,1,1,0}; + case "snapshot_lost_ack"->new long[]{1,0,1,1,1,2,1,2,1,1,0}; + case "snapshot_acknowledged"->new long[]{1,1,0,0,0,3,2,2,1,0,0}; + case "tail_ready"->new long[]{2,1,1,1,0,3,2,2,1,0,0}; + case "tail_rejected"->new long[]{2,1,1,1,0,4,2,3,2,1,0}; + case "tail_lost_ack"->new long[]{2,1,1,1,0,5,3,4,2,1,0}; + case "tail_acknowledged","reopened"->new long[]{2,2,0,0,0,6,4,4,2,0,0}; + default->throw new AssertionError("unknown observation phase"); + }; + assertArrayEquals(phase,expected,src.values().stream().mapToLong(v->((Number)v).longValue()).toArray()); + var dst = Map.of("checkpoint",t.checkpoint(),"snapshotBaseline",t.snapshotBaseline(),"inboxRows",t.inboxRows(), + "sequenceLag",t.sequenceLag().orElseThrow(),"rejections",t.rejections()); + var result = new LinkedHashMap();result.put("phase",phase);result.put("source",src);result.put("target",dst); + result.put("deadLetters",sourceObserver.deadLetters(source.context,8)); + result.put("sourceTelemetry",source.observations.get(source.observations.size()-1)); + result.put("targetTelemetry",target.observations.get(target.observations.size()-1)); + String text=JSON.writeValueAsString(result); + for(String forbidden:List.of(CANARY,"submitted",SOURCE,new String(KEY,StandardCharsets.US_ASCII)))assertFalse(text.contains(forbidden)); + return result; + } + private static void businessCall(Path directory, CustomerOrder order, DefaultUserContext context, String comment) throws Exception { + Files.writeString(directory.resolve("business-calls.txt"), comment + "\n", StandardOpenOption.CREATE, StandardOpenOption.APPEND); + order.auditAs(comment).save(context); + } + + @Test public void ordinaryTfpReadsButCannotWriteOrDispatchReplicationOnRealSqlite() throws Exception { + Path directory=Files.createTempDirectory("teaql-cmr15-tfp-"); + try(Fixture source=new Fixture(directory.resolve("source.sqlite"),true); + Fixture target=new Fixture(directory.resolve("target.sqlite"),false)) { + var receiver=target.receiver();var order=new CustomerOrder();order.__internalInitializeNewEntityId(ID); + order.updateProperty("name","original");order.updateProperty("memo",CANARY); + order.auditAs("seed real TFP boundary example through audited save").save(source.context); + var event=source.recorder.eventStore().outboxAfter(SOURCE,0,1).get(0); + var delivery=SqliteCommittedReceiver.sign("current",TARGET,KEY,event);receiver.receive(target.context,delivery); + var endpoint=new io.teaql.tfp.TfpEndpointHandler(new SqliteDataServiceExecutor("sqlite",target.sql,null),JSON); + var trusted=new io.teaql.tfp.TrustedFederalContext("id",ID,"example:analyst","verify actual target boundary",Set.of("CustomerOrder"), + Map.of("CustomerOrder",Map.of("id","id","version","version","name","name","memo","memo")), + Map.of("CustomerOrder",Map.of("name","name","memo","memo")),Map.of("CustomerOrder",Set.of("Update")),10); + var query=JSON.createObjectNode().put("entity","CustomerOrder").put("limitValue",1) + .put("commentText","read real analytical target").put("purposeText","verify bounded TFP target reads"); + query.putArray("selectItems").add("id").add("version").add("name").add("memo"); + var rows=(List)endpoint.handleQuery(target.context,trusted,JSON.writeValueAsBytes(query)).get("data"); + assertEquals(1,rows.size());assertEquals("original",((io.teaql.core.Entity)rows.get(0)).getProperty("name")); + var mutation=JSON.createObjectNode().put("entity","CustomerOrder").put("action","Update").put("id",ID) + .put("expectedVersion",1).put("comment","attempt ordinary TFP write on target"); + mutation.putObject("payload").put("name","must not save").put("memo",CANARY); + var denied=assertThrows(RuntimeException.class,()->endpoint.handleMutation(target.context,trusted,JSON.writeValueAsBytes(mutation))); + assertTrue(denied.toString(),denied.toString().contains("READ_ONLY_TARGET")); + mutation.put("action","Apply"); + assertEquals("TFP_POLICY_VIOLATION",assertThrows(io.teaql.tfp.TfpEndpointException.class,()->endpoint.handleMutation(target.context,trusted,JSON.writeValueAsBytes(mutation))).getCode()); + query.put("sql","DELETE FROM customer_order_data"); + assertEquals("TFP_INVALID_REQUEST",assertThrows(io.teaql.tfp.TfpEndpointException.class,()->endpoint.handleQuery(target.context,trusted,JSON.writeValueAsBytes(query))).getCode()); + var frame=JSON.writeValueAsBytes(Map.of("protocol_version",1,"source_sequence","1","source_runtime_id",SOURCE)); + assertEquals("TFP_INVALID_REQUEST",assertThrows(io.teaql.tfp.TfpEndpointException.class,()->endpoint.handleMutation(target.context,trusted,frame)).getCode()); + assertEquals("TFP_INVALID_REQUEST",assertThrows(io.teaql.tfp.TfpEndpointException.class,()->endpoint.handleQuery(target.context,trusted,frame)).getCode()); + assertEquals("original",domain(target.sql).get("name"));assertEquals(1,((Number)domain(target.sql).get("version")).longValue()); + assertEquals(1,scalar(target.sql,"SELECT count(*) FROM teaql_event_inbox"));assertEquals(0,scalar(target.sql,"SELECT count(*) FROM teaql_event_rejection")); + assertTrue(receiver.receive(target.context,delivery).duplicate()); + System.out.println("CMR15_TFP_SQLITE_E2E_PASS language=java bounded_query=1 readonly_mutation=1 fake_apply=1 raw_sql=1 protocol_frame=1 unchanged=1"); + } finally {try(var paths=Files.walk(directory)){for(var p:paths.sorted(Comparator.reverseOrder()).toList())Files.delete(p);}} + } + @Test public void auditedSaveSnapshotAndTailAuthenticateBeforeApplication() throws Exception { + String retained = System.getenv("TEAQL_CMR13_EVIDENCE_DIR"); + Path directory; + if (retained == null) directory = Files.createTempDirectory("teaql-java-cmr13-"); + else { directory = Path.of(retained); assertTrue(directory.isAbsolute()); assertFalse(Files.exists(directory)); Files.createDirectory(directory); } + boolean passed = false; + try (Fixture source = new Fixture(directory.resolve("source.sqlite"), true); + Fixture target = new Fixture(directory.resolve("target.sqlite"), false)) { + var receiver = target.receiver(); + var phases = new ArrayList>(); + CustomerOrder order = new CustomerOrder(); order.__internalInitializeNewEntityId(ID); + order.updateProperty("name", "draft"); order.updateProperty("memo", CANARY); + businessCall(directory, order, source.context, "create authenticated replication example"); + var first = source.recorder.eventStore().outboxAfter(SOURCE, 0, 8).get(0); + assertEquals(1, first.sourceSequence()); + var snapshots = new SqliteCommittedSnapshotPublisher(source.sql, SOURCE, TARGET, MODEL); + var prepared = snapshots.prepare(source.context, "baseline", "capture accepted order", "bootstrap analytics", LIMITS); + assertEquals(1, prepared.snapshot().sourceSequence()); assertFalse(prepared.acknowledged()); + phases.add(observe("prepared",source,target)); + SnapshotDelivery signed = SqliteCommittedReceiver.signSnapshot("current", TARGET, KEY, prepared.snapshot()); + byte[] invalid = signed.signature(); invalid[0] ^= 1; + rejected("INVALID_SIGNATURE", () -> snapshots.publish(source.context, "baseline", LIMITS, + record -> receiver.receiveSnapshot(target.context, new SnapshotDelivery(record, "current", TARGET, invalid)))); + assertEquals(0, scalar(source.sql, "SELECT count(*) FROM teaql_event_delivery_cursor")); + assertEquals(0, scalar(source.sql, "SELECT acknowledged FROM teaql_event_snapshot_publication")); + assertEquals(0, scalar(target.sql, "SELECT count(*) FROM customer_order_data")); + assertEquals(1, scalar(target.sql, "SELECT count(*) FROM teaql_event_rejection")); + phases.add(observe("snapshot_rejected",source,target)); + + // Delivery commits, but the source does not get its receipt. This is a lost ACK, + // not a process-crash claim (the complete gate has real SIGKILL cases separately). + assertThrows(IllegalStateException.class, () -> snapshots.publish(source.context, "baseline", LIMITS, record -> { + receiver.receiveSnapshot(target.context, SqliteCommittedReceiver.signSnapshot("current", TARGET, KEY, record)); + throw new IllegalStateException("example lost snapshot receipt"); + })); + assertEquals(0, scalar(source.sql, "SELECT count(*) FROM teaql_event_delivery_cursor")); + position(target, 1, 0, 1, "draft"); + phases.add(observe("snapshot_lost_ack",source,target)); + var recovered = snapshots.publish(source.context, "baseline", LIMITS, record -> { + var receipt = receiver.receiveSnapshot(target.context, SqliteCommittedReceiver.signSnapshot("current", TARGET, KEY, record)); + assertTrue(receipt.duplicate()); return receipt; + }); + assertTrue(recovered.acknowledged()); assertEquals(prepared.snapshot(), recovered.snapshot()); + assertEquals(1, scalar(source.sql, "SELECT acked_sequence FROM teaql_event_delivery_cursor")); + phases.add(observe("snapshot_acknowledged",source,target)); + + order.updateProperty("name", "submitted"); + businessCall(directory, order, source.context, "submit authenticated replication example"); + OutboxRecord tail = source.recorder.eventStore().outboxAfter(SOURCE, 1, 8).get(0); + assertEquals(2, tail.sourceSequence()); + phases.add(observe("tail_ready",source,target)); + Delivery original = SqliteCommittedReceiver.sign("current", TARGET, KEY, tail); + String forged = tail.payloadJson().replace("submitted", "forged"); assertNotEquals(tail.payloadJson(), forged); + var altered = new OutboxRecord(SOURCE, 2, tail.changeSetId(), MODEL, digest(forged), forged); + var publisher = new SqliteCommittedOutboxPublisher(source.sql, SOURCE, TARGET, MODEL); + rejected("INVALID_SIGNATURE", () -> publisher.publishBatch(8, record -> receiver.receive(target.context, + new Delivery(altered, "current", TARGET, original.signature())))); + position(target, 1, 0, 2, "draft"); assertEquals(1, publisher.status().ackedSequence()); + phases.add(observe("tail_rejected",source,target)); + var stranger = new OutboxRecord("untrusted-source", 2, tail.changeSetId(), MODEL, tail.payloadSha256(), tail.payloadJson()); + rejected("UNAUTHORIZED_SOURCE", () -> receiver.receive(target.context, + SqliteCommittedReceiver.sign("current", TARGET, KEY, stranger))); + position(target, 1, 0, 3, "draft"); + assertThrows(IllegalStateException.class,()->publisher.publishBatch(8,record->{ + receiver.receive(target.context,SqliteCommittedReceiver.sign("current",TARGET,KEY,record)); + throw new IllegalStateException("example lost tail receipt"); + })); + position(target,2,1,3,"submitted");assertEquals(1,publisher.status().ackedSequence()); + phases.add(observe("tail_lost_ack",source,target)); + var delivered = publisher.publishBatch(8, record -> receiver.receive(target.context, + SqliteCommittedReceiver.sign("current", TARGET, KEY, record))); + assertEquals(1, delivered.acknowledged()); assertEquals(2, delivered.status().ackedSequence()); + assertEquals(0, delivered.status().backlog()); assertTrue(receiver.receive(target.context, original).duplicate()); + phases.add(observe("tail_acknowledged",source,target)); + rejected("SEQUENCE_GAP", () -> receiver.receive(target.context, SqliteCommittedReceiver.sign("current", TARGET, KEY, first))); + assertTrue(receiver.receiveSnapshot(target.context, signed).duplicate()); + position(target, 2, 1, 4, "submitted"); + var newer = new SqliteCommittedSnapshot(source.sql, SOURCE, MODEL).exportSnapshot(source.context, + "replacement", "capture a newer baseline", "prove active stream cannot reset", LIMITS); + rejected("SNAPSHOT_CONFLICT", () -> receiver.receiveSnapshot(target.context, + SqliteCommittedReceiver.signSnapshot("current", TARGET, KEY, newer))); + position(target, 2, 1, 5, "submitted"); + + // Fresh contexts/connections inspect durable state; no business method is replayed. + try (Fixture reopened = new Fixture(directory.resolve("target.sqlite"), false)) { + assertTrue(reopened.receiver().receive(reopened.context, original).duplicate()); + assertTrue(reopened.receiver().receiveSnapshot(reopened.context, signed).duplicate()); + position(reopened, 2, 1, 5, "submitted"); assertEquals(domain(source.sql), domain(reopened.sql)); + var audits = reopened.sql.queryForList("SELECT * FROM teaql_event_rejection ORDER BY audit_id", new Object[0]); + // Stored audit namespaces snapshot codes; the public receiver code remains SNAPSHOT_CONFLICT. + assertEquals(List.of("SNAPSHOT_INVALID_SIGNATURE", "INVALID_SIGNATURE", "UNAUTHORIZED_SOURCE", "SEQUENCE_GAP", "SNAPSHOT_SNAPSHOT_CONFLICT"), + audits.stream().map(row -> row.get("reason_code")).toList()); + String auditText = JSON.writeValueAsString(audits); assertFalse(auditText.contains(CANARY)); + assertFalse(auditText.contains("submitted")); assertFalse(auditText.contains(new String(KEY, StandardCharsets.US_ASCII))); + for (var row : audits) for (String field : List.of("source_fingerprint", "change_fingerprint", "target_fingerprint", "payload_fingerprint")) + assertTrue(((String)row.get(field)).matches("[a-f0-9]{64}")); + long calls = Files.readAllLines(directory.resolve("business-calls.txt")).size(); assertEquals(2, calls); + try (Fixture reopenedSource = new Fixture(directory.resolve("source.sqlite"),true)) { + phases.add(observe("reopened",reopenedSource,reopened)); + } + var observationResult = Map.of("schema","teaql.cmr14-example/v1","language","java","phases",phases); + Files.writeString(directory.resolve("observations.json"),JSON.writeValueAsString(observationResult),StandardOpenOption.CREATE_NEW); + var result = new LinkedHashMap(); + result.put("schema", "teaql.cmr13-example/v1"); result.put("language", "java"); + result.put("source", domain(source.sql)); result.put("target", domain(reopened.sql)); + result.put("businessCalls", calls); result.put("sourceCursor", publisher.status().ackedSequence()); + result.put("targetCheckpoint", scalar(reopened.sql, "SELECT source_sequence FROM teaql_event_checkpoint")); + result.put("snapshotSequence", prepared.snapshot().sourceSequence()); result.put("snapshotHash", prepared.snapshot().payloadSha256()); + result.put("rejections", audits); result.put("outbox", source.recorder.eventStore().outboxAfter(SOURCE, 0, 8).size()); + result.put("inbox", scalar(reopened.sql, "SELECT count(*) FROM teaql_event_inbox")); + result.put("receiverOrigin", SqliteCommittedReceiver.class.getProtectionDomain().getCodeSource().getLocation().toString()); + Files.writeString(directory.resolve("result.json"), JSON.writeValueAsString(result), StandardOpenOption.CREATE_NEW); + } + passed = true; + System.out.println("CMR14_OBSERVABILITY_EXAMPLE_PASS language=java phases=9 attempts=6 retries=4 failures=4 source_rejections=2 target_rejections=5 dead_letters=0"); + System.out.println("CMR13_SECURITY_EXAMPLE_PASS language=java snapshot=1 tail=2 audited_saves=2 rejected=5 lost_ack=1 duplicate_no_rewind=1 reopen=1"); + } finally { + if (retained == null && passed) try (var files = Files.walk(directory)) { + for (Path file : files.sorted(Comparator.reverseOrder()).toList()) Files.delete(file); + } else if (!passed) System.err.println("CMR13 failed compact diagnostic: " + directory); + } + } +} diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedSnapshotExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedSnapshotExampleTest.java new file mode 100644 index 00000000..eef7978f --- /dev/null +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedSnapshotExampleTest.java @@ -0,0 +1,131 @@ +package io.teaql.examples.committed; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.*; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.security.MessageDigest; +import java.time.Instant; +import java.util.*; +import static org.junit.Assert.*; + +/** Trusted bootstrap example, not a remote mutation/restore API. */ +public class CommittedSnapshotExampleTest { + private static final String SOURCE = "snapshot-example", MODEL = "sha256:" + "0".repeat(64); + private static final long ID = 9_007_199_254_740_993L; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final SqliteCommittedSnapshot.Limits LIMITS = new SqliteCommittedSnapshot.Limits(10, 64 * 1024); + + private static final class Fixture implements AutoCloseable { + final Path directory = Files.createTempDirectory("teaql-snapshot-example-"); + final JdbcSqlExecutor sql; + final DefaultUserContext context; + Fixture(boolean legacy) throws Exception { + SQLiteDataSource dataSource = new SQLiteDataSource(); + dataSource.setUrl("jdbc:sqlite:" + directory.resolve("data.sqlite")); + sql = new JdbcSqlExecutor(dataSource); + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + metadata.register(entity("customer_order", true)); metadata.register(entity("empty_probe", false)); + context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", sql, dataSource)) + .executionLogging(false).build()); + try { + context.ensureSchema(); // Always through the Context, using the local runtime. + new SqliteCommittedEventStore(sql).ensureSchema(); + if (legacy) { + // Only a legacy database fixture: these predate capture, not a seed API. + sql.update("INSERT INTO customer_order_data(id,version,name,occurred_at,active,note) VALUES(?,7,'LEGACY',-315619199877,1,NULL)", new Object[] {ID}); + sql.update("INSERT INTO customer_order_data(id,version,name,occurred_at,active,note) VALUES(?,-4,'已删除',0,0,'retained tombstone')", new Object[] {ID + 1}); + } + } catch (Exception | Error failure) { + try { close(); } catch (Exception cleanup) { failure.addSuppressed(cleanup); } throw failure; + } + } + SqliteCommittedSnapshot snapshots() { return new SqliteCommittedSnapshot(sql, SOURCE, MODEL); } + @Override public void close() throws Exception { + for (String name : List.of("data.sqlite-wal", "data.sqlite-shm", "data.sqlite-journal", "data.sqlite")) Files.deleteIfExists(directory.resolve(name)); + Files.deleteIfExists(directory); + } + } + private static SQLEntityDescriptor entity(String name, boolean order) { + SQLEntityDescriptor entity = new SQLEntityDescriptor(); entity.setType(name); entity.setDataService("sqlite"); + for (String field : order ? List.of("id", "version", "name", "occurred_at", "active", "note") : List.of("id", "version", "name")) { + Class type = switch (field) { case "id", "version" -> Long.class; case "occurred_at" -> Instant.class; case "active" -> Boolean.class; default -> String.class; }; + GenericSQLProperty property = (GenericSQLProperty) entity.addSimpleProperty(field, type); + property.setColumnType(type == String.class ? "VARCHAR(255)" : "BIGINT"); + } + return entity; + } + private static String tail(long n) { + return """ + {"protocol_version":1,"change_set_id":"snapshot-example-tail","source_runtime_id":"snapshot-example", + "source_sequence":"%d","source_model_hash":"%s","source_transaction_id":"snapshot-example-tx", + "committed_at":"2026-10-22T00:00:00Z","audit":{"comment":"advance legacy order","purpose":"verify snapshot tail"}, + "mutations":[{"entity_type":"customer_order","entity_id":"9007199254740993","operation":"update", + "base_version":"7","committed_version":"8","fields":{"name":"AFTER SNAPSHOT"},"trace_chain":[]}]} + """.formatted(n, MODEL); + } + private static void commitFixtureTail(Fixture source) { + // Infrastructure simulation; normal application saves are demonstrated by + // CommittedEventsExampleTest and use the audited Mutation API instead. + source.sql.executeInTransaction(() -> { + assertEquals(1, source.sql.update("UPDATE customer_order_data SET version=8,name='AFTER SNAPSHOT' WHERE id=? AND version=7", new Object[] {ID})); + new SqliteCommittedEventStore(source.sql).append(SOURCE, "snapshot-example-tail", MODEL, CommittedSnapshotExampleTest::tail); + }); + } + private static String digest(String text) throws Exception { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))); } + private static void write(Path directory, String name, String text) throws Exception { + Files.createDirectories(directory); + Files.writeString(directory.resolve(name + ".json"), text, StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW); + Files.writeString(directory.resolve(name + ".sha256"), digest(text), StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW); + } + private static String read(Path directory, String name) throws Exception { + String text = Files.readString(directory.resolve(name + ".json"), StandardCharsets.UTF_8); + assertEquals("exchange digest", Files.readString(directory.resolve(name + ".sha256")).trim(), digest(text)); return text; + } + private static void assertTarget(Fixture target) { + var rows = target.sql.queryForList("SELECT * FROM customer_order_data ORDER BY id", new Object[0]); + assertEquals(2, rows.size()); assertEquals(ID, ((Number) rows.get(0).get("id")).longValue()); + assertEquals(8, ((Number) rows.get(0).get("version")).longValue()); assertEquals("AFTER SNAPSHOT", rows.get(0).get("name")); + assertEquals(-315619199877L, ((Number) rows.get(0).get("occurred_at")).longValue()); + assertEquals(1, ((Number) rows.get(0).get("active")).intValue()); assertNull(rows.get(0).get("note")); + assertEquals(-4, ((Number) rows.get(1).get("version")).longValue()); assertEquals("已删除", rows.get(1).get("name")); + assertEquals(0, ((Number) target.sql.queryForList("SELECT count(*) AS n FROM empty_probe_data", new Object[0]).get(0).get("n")).intValue()); + assertEquals(1, ((Number) target.sql.queryForList("SELECT source_sequence FROM teaql_event_checkpoint", new Object[0]).get(0).get("source_sequence")).intValue()); + } + + @Test public void legacySnapshotAndTailPreserveFactsAndStableRetry() throws Exception { + String input = System.getenv("TEAQL_SNAPSHOT_IMPORT_DIR"), output = System.getenv("TEAQL_SNAPSHOT_EXPORT_DIR"); + assertFalse("separate producer/consumer roles", input != null && output != null); + try (Fixture source = new Fixture(true); Fixture target = new Fixture(false)) { + SqliteCommittedSnapshot.SnapshotRecord snapshot; + SqliteCommittedEventStore.OutboxRecord event; + if (input == null) { + snapshot = source.snapshots().exportSnapshot(source.context, "example-snapshot", "copy legacy facts", "initialize analytics", LIMITS); + assertEquals(0, snapshot.sourceSequence()); + commitFixtureTail(source); event = new SqliteCommittedEventStore(source.sql).outboxAfter(SOURCE, 0, 10).get(0); + if (output != null) { write(Path.of(output), "snapshot", snapshot.payloadJson()); write(Path.of(output), "tail", event.payloadJson()); } + } else { + String text = read(Path.of(input), "snapshot"); JsonNode body = JSON.readTree(text); + snapshot = new SqliteCommittedSnapshot.SnapshotRecord(SOURCE, Long.parseLong(body.path("source_sequence").asText()), body.path("snapshot_id").asText(), MODEL, digest(text), text); + text = read(Path.of(input), "tail"); body = JSON.readTree(text); + event = new SqliteCommittedEventStore.OutboxRecord(SOURCE, Long.parseLong(body.path("source_sequence").asText()), body.path("change_set_id").asText(), MODEL, digest(text), text); + } + assertEquals(0, target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS).checkpoint()); + var applier = new SqliteModelCheckedCommittedTarget(target.sql, SOURCE, MODEL); + assertEquals(1, applier.apply(target.context, event).checkpoint()); assertTrue(applier.apply(target.context, event).duplicate()); + assertEquals(0, target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS).checkpoint()); assertTarget(target); + if (input == null) assertEquals(source.sql.queryForList("SELECT * FROM customer_order_data ORDER BY id", new Object[0]), target.sql.queryForList("SELECT * FROM customer_order_data ORDER BY id", new Object[0])); + System.out.println("CMR11_EXAMPLE_PASS language=java mode=" + (input == null ? "local" : "exchange") + " snapshot=0 tail=1 rows=2 catalog=2 duplicate_after_tail=1"); + } + } +} diff --git a/examples/committed-events/verify-bootstrap-crash.sh b/examples/committed-events/verify-bootstrap-crash.sh new file mode 100644 index 00000000..81462761 --- /dev/null +++ b/examples/committed-events/verify-bootstrap-crash.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +df -h "$repo" "${TMPDIR:-/tmp}" +scratch="$(mktemp -d -t teaql-java-bootstrap-crash.XXXXXXXX)" +maven_pid="" +stop_owned_build() { + if [[ -n "$maven_pid" ]]; then + # setsid gives this build (and its worker JVMs) its own process group. + kill -TERM -- "-$maven_pid" 2>/dev/null || true + kill -KILL -- "-$maven_pid" 2>/dev/null || true + wait "$maven_pid" 2>/dev/null || true + maven_pid="" + fi +} +cleanup() { status=$?; trap - EXIT; stop_owned_build; rm -rf -- "$scratch"; exit "$status"; } +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +command -v setsid >/dev/null +mkdir -p "$scratch/source" "$scratch/tmp" +# Local repository code only. Compatible compilations are reused for both runs. +tar -C "$repo" --exclude=.git --exclude=target -cf - . | tar -C "$scratch/source" -xf - +export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djava.io.tmpdir=$scratch/tmp" +if [[ -n "${TEAQL_CMR12_EVIDENCE_ROOT:-}" ]]; then + if [[ "$TEAQL_CMR12_EVIDENCE_ROOT" != /* ]]; then + echo 'CMR-12 evidence root must be absolute' >&2 + exit 1 + fi + if [[ -e "$TEAQL_CMR12_EVIDENCE_ROOT" ]]; then + echo 'CMR-12 evidence root must be fresh; never reset prior databases' >&2 + exit 1 + fi + mkdir -p "$TEAQL_CMR12_EVIDENCE_ROOT" +fi +cd "$scratch/source" +for run in 1 2; do + if [[ -n "${TEAQL_CMR12_EVIDENCE_ROOT:-}" ]]; then + export TEAQL_CMR12_EVIDENCE_DIR="$TEAQL_CMR12_EVIDENCE_ROOT/run-$run" + else + unset TEAQL_CMR12_EVIDENCE_DIR + fi + setsid mvn -B -q -o -pl examples/committed-events -am \ + -Dtest=CommittedBootstrapCrashExampleTest -Dsurefire.failIfNoSpecifiedTests=false \ + "-Dcmr12.source.root=$scratch/source" test > "$scratch/run-$run.log" 2>&1 & + maven_pid=$! + if wait "$maven_pid"; then result=0; else result=$?; fi + stop_owned_build + cat "$scratch/run-$run.log" + if [[ "$result" != 0 ]]; then exit "$result"; fi + for round in 1 2; do + rg -Fq "CMR12_PROCESS_PASS language=java round=$round baseline=$((2 * round - 1)) tail=$((2 * round)) killed=7 same_source_database=1 original_bytes=1" "$scratch/run-$run.log" + done + if [[ -n "${TEAQL_CMR12_EVIDENCE_ROOT:-}" ]]; then + mkdir -p "$TEAQL_CMR12_EVIDENCE_ROOT/run-$run/junit" + cp examples/committed-events/target/surefire-reports/TEST-io.teaql.examples.committed.CommittedBootstrapCrashExampleTest.xml \ + "$TEAQL_CMR12_EVIDENCE_ROOT/run-$run/junit/" + fi + echo "PASS Java managed bootstrap process example run $run rounds=2 killed=14" +done diff --git a/examples/committed-events/verify-process-crash.sh b/examples/committed-events/verify-process-crash.sh new file mode 100644 index 00000000..632c31ab --- /dev/null +++ b/examples/committed-events/verify-process-crash.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo" +# No copied build tree: Maven uses this checkout's stable module targets. +# The small, owned fixture DBs/logs are retained and their path is printed. +# TEAQL_CMR03_EVIDENCE_DIR, when supplied, must be a fresh evidence directory. +output="$(mvn -B -q -pl examples/committed-events -am \ + -Dtest=CommittedProcessCrashExampleTest -Dsurefire.failIfNoSpecifiedTests=false test 2>&1)" || { + printf '%s\n' "$output" >&2 + exit 1 +} +printf '%s\n' "$output" +for round in 1 2; do + if ! rg -q "^CMR03_PROCESS_CRASH_PASS language=java round=$round " <<< "$output"; then + echo "missing Java real-process crash round $round" >&2 + exit 1 + fi +done +echo 'PASS Java CMR-03 real source-process crash rounds=2 same_databases=1' diff --git a/examples/committed-events/verify-security.sh b/examples/committed-events/verify-security.sh new file mode 100644 index 00000000..66494cb7 --- /dev/null +++ b/examples/committed-events/verify-security.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +df -h "$repo" "${TMPDIR:-/tmp}" +scratch="$(mktemp -d -t teaql-java-cmr13.XXXXXXXX)" +task_pid="" +cleanup() { + status=$?; trap - EXIT + if [[ -n "$task_pid" ]]; then + kill -TERM -- "-$task_pid" 2>/dev/null || true + kill -KILL -- "-$task_pid" 2>/dev/null || true + wait "$task_pid" 2>/dev/null || true + fi + rm -rf -- "$scratch" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +command -v setsid >/dev/null +case_root="$scratch" +if [[ -n "${TEAQL_CMR13_EVIDENCE_ROOT:-}" ]]; then + if [[ "$TEAQL_CMR13_EVIDENCE_ROOT" != /* || -e "$TEAQL_CMR13_EVIDENCE_ROOT" ]]; then + echo 'CMR-13 evidence root must be an absent absolute path; never reset databases' >&2; exit 1 + fi + mkdir "$TEAQL_CMR13_EVIDENCE_ROOT" + case_root="$TEAQL_CMR13_EVIDENCE_ROOT" +fi +mkdir -p "$scratch/source" "$scratch/tmp" +tar -C "$repo" --exclude=.git --exclude=target -cf - . | tar -C "$scratch/source" -xf - +export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djava.io.tmpdir=$scratch/tmp" +cd "$scratch/source" +for round in 1 2; do + TEAQL_CMR13_EVIDENCE_DIR="$case_root/run-$round" setsid mvn -B -q -o \ + -pl examples/committed-events -am -Dtest=CommittedReceiverExampleTest \ + -Dsurefire.failIfNoSpecifiedTests=false test > "$scratch/round-$round.log" 2>&1 & + task_pid=$! + if wait "$task_pid"; then result=0; else result=$?; fi + task_pid="" + cat "$scratch/round-$round.log" + if [[ -n "${TEAQL_CMR13_EVIDENCE_ROOT:-}" ]]; then + cp "$scratch/round-$round.log" "$case_root/round-$round.log" + reports=examples/committed-events/target/surefire-reports + if [[ -e "$reports/TEST-io.teaql.examples.committed.CommittedReceiverExampleTest.xml" ]]; then + cp "$reports/TEST-io.teaql.examples.committed.CommittedReceiverExampleTest.xml" "$case_root/round-$round.xml" + fi + fi + if [[ "$result" != 0 ]]; then exit "$result"; fi + rg -Fq 'CMR13_SECURITY_EXAMPLE_PASS language=java snapshot=1 tail=2 audited_saves=2 rejected=5 lost_ack=1 duplicate_no_rewind=1 reopen=1' "$scratch/round-$round.log" + rg -Fq 'CMR14_OBSERVABILITY_EXAMPLE_PASS language=java phases=9 attempts=6 retries=4 failures=4 source_rejections=2 target_rejections=5 dead_letters=0' "$scratch/round-$round.log" + echo "PASS Java authenticated committed receiver example round $round" +done diff --git a/examples/committed-events/verify-snapshot.sh b/examples/committed-events/verify-snapshot.sh new file mode 100644 index 00000000..9a8fc61b --- /dev/null +++ b/examples/committed-events/verify-snapshot.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +df -h "$repo" "${TMPDIR:-/tmp}" +scratch="$(mktemp -d -t teaql-java-snapshot.XXXXXXXX)" +cleanup() { status=$?; trap - EXIT; rm -rf -- "$scratch"; exit "$status"; } +trap cleanup EXIT +mkdir -p "$scratch/source" "$scratch/tmp" +# Copy current local source, including edits, never a released dependency or +# stale HEAD. Exclude reproducible outputs, not generated source fixtures. +tar -C "$repo" --exclude=.git --exclude=target -cf - . | tar -C "$scratch/source" -xf - +export JAVA_TOOL_OPTIONS="${JAVA_TOOL_OPTIONS:-} -Djava.io.tmpdir=$scratch/tmp" +cd "$scratch/source" +for round in 1 2; do + mvn -B -q -o -pl examples/committed-events -am \ + -Dtest=CommittedSnapshotExampleTest -Dsurefire.failIfNoSpecifiedTests=false test \ + 2>&1 | tee "$scratch/round-$round.log" + rg -Fq 'CMR11_EXAMPLE_PASS language=java mode=local snapshot=0 tail=1 rows=2 catalog=2 duplicate_after_tail=1' "$scratch/round-$round.log" + echo "PASS Java committed snapshot example round $round" +done diff --git a/examples/committed-events/verify.sh b/examples/committed-events/verify.sh new file mode 100755 index 00000000..4763df0b --- /dev/null +++ b/examples/committed-events/verify.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo" +disposable_evidence="" +if [[ -n "${TEAQL_COMMITTED_EVIDENCE_DIR:-}" ]]; then + if [[ "$TEAQL_COMMITTED_EVIDENCE_DIR" != /* || -e "$TEAQL_COMMITTED_EVIDENCE_DIR" ]]; then + echo 'Committed evidence directory must be an absent absolute path' >&2; exit 1 + fi + evidence="$TEAQL_COMMITTED_EVIDENCE_DIR"; mkdir "$evidence" +else + evidence="$(mktemp -d -t teaql-java-committed-generated.XXXXXXXX)" + disposable_evidence="$evidence" +fi +cleanup() { status=$?; trap - EXIT; if [[ -n "$disposable_evidence" ]]; then rm -rf -- "$disposable_evidence"; fi; exit "$status"; } +trap cleanup EXIT +generated_library_hash() { + (cd "$repo/examples/standard-order" && rg --files generated/lib -g '!target/**' \ + | LC_ALL=C sort | xargs -d '\n' sha256sum) +} +generated_library_hash > "$evidence/generated-before.sha256" +for round in 1 2; do + mvn -B -q -pl examples/committed-events -am \ + -Dtest=CommittedEventsExampleTest -Dsurefire.failIfNoSpecifiedTests=false test \ + 2>&1 | tee "$evidence/runtime-$round.log" + rg -Fq 'COMMITTED_DELIVERY_RECOVERY_PASS language=java ordered=2 lost_ack=1 restart=1 duplicate=1' \ + "$evidence/runtime-$round.log" + echo "PASS Java committed-events example round $round" + mvn -B -q -Pruntime-examples -pl examples/standard-order -am \ + -Dtest=StandardOrderCommittedTargetTest -Dsurefire.failIfNoSpecifiedTests=false test \ + 2>&1 | tee "$evidence/generated-$round.log" + rg -Fq 'JAVA_COMMITTED_GENERATED_Q_PASS source=sqlite target=duckdb outbox=1' \ + "$evidence/generated-$round.log" + generated_library_hash > "$evidence/generated-after.sha256" + cmp "$evidence/generated-before.sha256" "$evidence/generated-after.sha256" + echo "PASS Java generated committed Q example round $round" +done +bash examples/committed-events/verify-process-crash.sh +bash examples/committed-events/verify-snapshot.sh +bash examples/committed-events/verify-bootstrap-crash.sh +bash examples/committed-events/verify-security.sh +if [[ -n "${TEAQL_COMMITTED_EVIDENCE_DIR:-}" ]]; then + echo "Java generated committed Q evidence: $evidence" +else + echo 'Java generated committed Q disposable evidence removed on exit' +fi diff --git a/examples/http-resilience/README.md b/examples/http-resilience/README.md index 1e8d848c..444a0c6d 100644 --- a/examples/http-resilience/README.md +++ b/examples/http-resilience/README.md @@ -3,9 +3,30 @@ Run `bash examples/http-resilience/verify.sh`. The verifier starts a loopback-only HTTP fixture, installs one named service profile on `DefaultUserContext`, and executes the same call through both the provider-neutral Context API and the -Agent HTTP tool. It runs twice without deleting or resetting any fixture. +Agent HTTP tool. Each native test runs two rounds on the same server, Context +and service: request/trace counters grow from two to four without resetting. +The verifier executes that native example twice. An optional fresh +`TEAQL_HTTP_EXAMPLE_EVIDENCE_DIR` retains compact per-execution JUnit reports; +it never stores build trees or credentials. Only trusted startup code registers an upstream base URI. Application code uses its alias and a relative path; it cannot substitute an arbitrary URL. Each call has a comment and purpose. By default it makes one physical attempt and never follows redirects. Wider retry/circuit/rate cases live in `teaql-tool-http` tests. + +## Profile-scoped private HTTPS trust + +Trusted startup may configure `.trustedRootPem(pem)` or +`.trustedRootsPem(List.of(oldRoot, newRoot))` on an HTTPS service profile. +The explicit one-to-four-root set replaces system trust **for that profile**; +certificate and hostname verification remain enabled. Each PEM input contains +one certificate and is at most 64 KiB. Empty, duplicate, bundled, malformed, +oversized or plaintext-profile inputs reject before use. Unconfigured profiles +continue to use ordinary system roots. + +Profiles are immutable snapshots. Rotate by constructing a new service with +old+new roots, then new-only roots after upstream rollover. This does not mutate +other clients, Contexts or the process trust store. The verification script also +runs the standalone `HttpProfileTrustTest` loopback HTTPS matrix, including two +rounds over both live peers without resetting their counters. It requires local +OpenSSL and removes generated private keys; no Vault or external service is used. diff --git a/examples/http-resilience/src/test/java/io/teaql/examples/http/HttpResilienceExampleTest.java b/examples/http-resilience/src/test/java/io/teaql/examples/http/HttpResilienceExampleTest.java index 13950e81..b5d6460b 100644 --- a/examples/http-resilience/src/test/java/io/teaql/examples/http/HttpResilienceExampleTest.java +++ b/examples/http-resilience/src/test/java/io/teaql/examples/http/HttpResilienceExampleTest.java @@ -19,6 +19,9 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.Test; import static org.junit.Assert.*; @@ -28,8 +31,18 @@ public class HttpResilienceExampleTest { @Test public void postedRequestWithStalledResponseRequiresReconciliation() throws Exception { AtomicInteger requests = new AtomicInteger(); + AtomicInteger warmups = new AtomicInteger(); + CountDownLatch firstByteSent = new CountDownLatch(1); + CountDownLatch releaseResponse = new CountDownLatch(1); + AtomicReference result = new AtomicReference<>(); + AtomicReference worker = new AtomicReference<>(); HttpServer server = HttpServer.create(new InetSocketAddress( InetAddress.getByAddress(new byte[] {127, 0, 0, 1}), 0), 0); + server.createContext("/api/warmup", exchange -> { + warmups.incrementAndGet(); + exchange.sendResponseHeaders(200, -1); + exchange.close(); + }); server.createContext("/api/payments", exchange -> { requests.incrementAndGet(); assertEquals("POST", exchange.getRequestMethod()); @@ -38,7 +51,8 @@ public class HttpResilienceExampleTest { try (var output = exchange.getResponseBody()) { output.write('x'); output.flush(); - try { Thread.sleep(1200); } catch (InterruptedException interrupted) { + firstByteSent.countDown(); + try { releaseResponse.await(10, TimeUnit.SECONDS); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } } catch (java.io.IOException ignored) { @@ -49,18 +63,34 @@ public class HttpResilienceExampleTest { try { var profile = HttpServiceProfile.builder("payments", URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/api/")) - .budgets(Duration.ofMillis(1500), Duration.ofMillis(700), Duration.ofMillis(1000)) + .budgets(Duration.ofSeconds(2), Duration.ofSeconds(2), Duration.ofSeconds(3)) .build(); var service = new JdkHttpRequestService(List.of(profile)); var context = new DefaultUserContext(null); context.putAttribute(HttpRequestService.class.getName(), service); + assertEquals(200, context.httpRequests().execute(context, HttpRequestService.Call.get( + "payments", "warmup", "prepare HTTP fixture", "exclude initialization from stalled POST proof")).status()); + assertEquals(1, warmups.get()); var call = HttpRequestService.Call.postText("payments", "payments", "PRIVATE_BODY_CANARY", "submit payment", "confirm external effect"); - assertEquals(HttpRequestFailure.Code.AMBIGUOUS_OUTCOME, - assertThrows(HttpRequestFailure.class, - () -> context.httpRequests().execute(context, call)).code()); + Thread request = new Thread(() -> { + try { + context.httpRequests().execute(context, call); + result.set(new AssertionError("incomplete response unexpectedly succeeded")); + } catch (Throwable failure) { result.set(failure); } + }); + worker.set(request); request.start(); + assertTrue("POST must reach the fixture before checking its stalled outcome", + firstByteSent.await(5, TimeUnit.SECONDS)); + request.join(profile.totalTimeout().plusSeconds(2).toMillis()); + assertFalse("expired HTTP request did not stop", request.isAlive()); + assertTrue(result.get() instanceof HttpRequestFailure); + assertEquals(HttpRequestFailure.Code.AMBIGUOUS_OUTCOME, ((HttpRequestFailure) result.get()).code()); assertEquals(1, requests.get()); } finally { + releaseResponse.countDown(); + Thread request = worker.get(); + if (request != null && request.isAlive()) { request.interrupt(); request.join(2000); } server.stop(0); } } @@ -94,31 +124,36 @@ public class HttpResilienceExampleTest { return (outcome, attempts, elapsedMillis) -> {}; }); - var response = context.httpRequests().execute(context, - HttpRequestService.Call.get("orders", "orders/1?token=PRIVATE_QUERY_CANARY", - "load order status", "show the order status") - .withParentTraceChain(List.of( - new TraceNode(TraceKind.PURPOSE, "Order", "render order details"), - new TraceNode(TraceKind.COMMENT, "Order", "load order graph")))); - assertEquals(200, response.status()); - assertEquals("submitted", ((java.util.Map) response.jsonValue()).get("status")); - assertEquals(1, response.attempts()); + for (int round = 1; round <= 2; round++) { + var response = context.httpRequests().execute(context, + HttpRequestService.Call.get("orders", "orders/1?token=PRIVATE_QUERY_CANARY", + "load order status", "show the order status") + .withParentTraceChain(List.of( + new TraceNode(TraceKind.PURPOSE, "Order", "render order details"), + new TraceNode(TraceKind.COMMENT, "Order", "load order graph")))); + assertEquals(200, response.status()); + assertEquals("submitted", ((java.util.Map) response.jsonValue()).get("status")); + assertEquals(1, response.attempts()); - String viaTool = new JdkHttpTool(context).get("orders", "orders/1") - .comment("load order status for agent") - .purpose("explain the order status") - .execute(); - assertTrue(viaTool.contains("submitted")); - assertEquals(2, requests.get()); - assertEquals(List.of(TRACEPARENT, TRACEPARENT), traceparents); - assertEquals(2, observed.size()); - var mapper = new com.fasterxml.jackson.databind.ObjectMapper(); - assertEquals(3, mapper.readTree(observed.get(0).traceChain()).size()); - assertEquals("load order graph", - mapper.readTree(observed.get(0).traceChain()).get(1).get("comment").asText()); - assertEquals(2, mapper.readTree(observed.get(1).traceChain()).size()); - assertFalse(observed.toString().contains("PRIVATE_QUERY_CANARY")); - assertFalse(observed.get(0).traceChain().contains("PRIVATE_QUERY_CANARY")); + String viaTool = new JdkHttpTool(context).get("orders", "orders/1") + .comment("load order status for agent") + .purpose("explain the order status") + .execute(); + assertTrue(viaTool.contains("submitted")); + assertEquals(2 * round, requests.get()); + assertEquals(java.util.Collections.nCopies(2 * round, TRACEPARENT), traceparents); + assertEquals(2 * round, observed.size()); + var mapper = new com.fasterxml.jackson.databind.ObjectMapper(); + int offset = 2 * (round - 1); + assertEquals(3, mapper.readTree(observed.get(offset).traceChain()).size()); + assertEquals("load order graph", + mapper.readTree(observed.get(offset).traceChain()).get(1).get("comment").asText()); + assertEquals(2, mapper.readTree(observed.get(offset + 1).traceChain()).size()); + assertFalse(observed.toString().contains("PRIVATE_QUERY_CANARY")); + assertFalse(observed.get(offset).traceChain().contains("PRIVATE_QUERY_CANARY")); + System.out.println("HTTP_SAME_FIXTURE_PASS language=java round=" + round + + " requests=" + requests.get() + " trace=" + observed.size() + " lineage=3"); + } } finally { server.stop(0); } diff --git a/examples/http-resilience/verify.sh b/examples/http-resilience/verify.sh index b40ea84a..0aa3fd44 100644 --- a/examples/http-resilience/verify.sh +++ b/examples/http-resilience/verify.sh @@ -2,8 +2,24 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$repo" +evidence="${TEAQL_HTTP_EXAMPLE_EVIDENCE_DIR:-}" +if [[ -n "$evidence" ]]; then + [[ ! -e "$evidence" ]] || { echo 'HTTP example evidence must be fresh' >&2; exit 1; } + mkdir -p "$evidence" +fi for round in 1 2; do echo "Java HTTP resilience example round $round" + result=0 mvn -B -q -pl examples/http-resilience -am \ - -Dtest=HttpResilienceExampleTest -Dsurefire.failIfNoSpecifiedTests=false test + -Dtest=HttpResilienceExampleTest -Dsurefire.failIfNoSpecifiedTests=false test || result=$? + if [[ -n "$evidence" && -f examples/http-resilience/target/surefire-reports/TEST-io.teaql.examples.http.HttpResilienceExampleTest.xml ]]; then + if ! cp examples/http-resilience/target/surefire-reports/TEST-io.teaql.examples.http.HttpResilienceExampleTest.xml \ + "$evidence/round-$round.xml"; then + (( result != 0 )) || result=1 + fi + fi + (( result == 0 )) || exit "$result" done +mvn -B -q -pl teaql-tool-http -am \ + -Dtest=HttpProfileTrustTest -Dsurefire.failIfNoSpecifiedTests=false test +echo 'HTTP_TRUST_EXAMPLE_PASS language=java tests=5' diff --git a/examples/standard-order/pom.xml b/examples/standard-order/pom.xml index 76ccc998..bab313d6 100644 --- a/examples/standard-order/pom.xml +++ b/examples/standard-order/pom.xml @@ -10,7 +10,9 @@ com.examplerequest-tree-order-service-core0.0.1-SNAPSHOT io.teaqlteaql-runtime io.teaqlteaql-sqlite + io.teaqlteaql-duckdb${project.version} io.teaqlteaql-provider-jdbc + org.duckdbduckdb_jdbc1.5.5.1 io.teaqlteaql-query-json io.teaqlteaql-opentelemetry${project.version}test junitjunittest diff --git a/examples/standard-order/src/test/java/com/example/requesttreeorderservice/StandardOrderCommittedTargetTest.java b/examples/standard-order/src/test/java/com/example/requesttreeorderservice/StandardOrderCommittedTargetTest.java new file mode 100644 index 00000000..0d098a95 --- /dev/null +++ b/examples/standard-order/src/test/java/com/example/requesttreeorderservice/StandardOrderCommittedTargetTest.java @@ -0,0 +1,142 @@ +package com.example.requesttreeorderservice; + +import io.teaql.core.InternalIdGenerationService; +import io.teaql.core.UserContext; +import io.teaql.core.duck.DuckDataServiceExecutor; +import io.teaql.core.duck.DuckDbCommittedInbox; +import io.teaql.core.duck.DuckDbModelCheckedCommittedTarget; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome; +import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import static org.junit.Assert.*; + +/** Application-owned generated Q/Save replay across a committed SQLite to DuckDB boundary. */ +public class StandardOrderCommittedTargetTest { + private static final String SOURCE = "generated-java-order-source"; + private static final String HASH = "sha256:" + "0".repeat(64); + + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { return DriverManager.getConnection(url); } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class type) throws SQLException { + throw new SQLException("not a wrapper"); + } + @Override public boolean isWrapperFor(Class type) { return false; } + } + + private static InternalIdGenerationService ids() { + var next = new AtomicLong(3000); + return new InternalIdGenerationService() { + @Override public Long generateId(UserContext context, io.teaql.core.Entity entity) { + return next.incrementAndGet(); + } + @Override public void ensureFloor(String type, long floor) { + next.accumulateAndGet(floor, Math::max); + } + }; + } + + private static TeaQLRuntime sqliteRuntime(SqliteDataServiceExecutor service, + InternalIdGenerationService idService, SqliteCommittedMutationRecorder recorder) { + var builder = TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .dataService("default", service).dataService("sqlite", service) + .idGenerationService(idService).executionLogging(false); + if (recorder != null) builder.committedMutationRecorder(recorder); + return builder.build().install(GeneratedRuntimeModule.module()); + } + + @Test public void generatedSaveReplicatesToDuckDbAndGeneratedQReadsIt() throws Exception { + var source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + Files.createTempDirectory("teaql-generated-commit-source-") + .resolve("source.sqlite")); + var sourceSql = new JdbcSqlExecutor(source); + var sourceService = new SqliteDataServiceExecutor("sqlite", sourceSql, source); + var idService = ids(); + try (var bootstrap = sqliteRuntime(sourceService, idService, null)) { + new DefaultUserContext(bootstrap).ensureSchema(); + } + var recorder = new SqliteCommittedMutationRecorder(sourceSql, SOURCE, HASH); + try (var sourceRuntime = sqliteRuntime(sourceService, idService, recorder)) { + var sourceContext = new DefaultUserContext(sourceRuntime); + sourceContext.ensureSchema(); + var platform = Q.platforms().limit(1).comment("load source Platform") + .purpose("attach committed Customer to domain root") + .executeForOne(sourceContext); + assertNotNull(platform); + var customer = Q.customers().comment("prepare committed Customer") + .purpose("verify generated audited save").newEntity(sourceContext); + customer.updateName("Committed target customer"); + customer.updatePlatform(platform); + customer.auditAs("create committed target customer").save(sourceContext); + var outbox = recorder.eventStore().outboxAfter(SOURCE, 0, 10); + assertEquals("bootstrap must not enter this Outbox", 1, outbox.size()); + + var targetPath = Files.createTempDirectory("teaql-generated-commit-target-") + .resolve("target.db"); + var targetSql = new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + targetPath)); + var targetService = new DuckDataServiceExecutor("sqlite", targetSql); + try (var targetRuntime = TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .dataService("default", targetService).dataService("sqlite", targetService) + .idGenerationService(ids()).executionLogging(false).build() + .install(GeneratedRuntimeModule.module())) { + var targetContext = new DefaultUserContext(targetRuntime); + targetContext.ensureSchema(); + var target = new DuckDbModelCheckedCommittedTarget(targetSql, SOURCE, HASH); + target.ensureInboxSchema(); + var delivered = new SqliteCommittedOutboxPublisher(sourceSql, SOURCE, + "generated-java-duckdb-target", HASH).publishBatch(10, record -> { + var outcome = target.apply(targetContext, new DuckDbCommittedInbox.Record( + record.sourceRuntimeId(), record.sourceSequence(), record.changeSetId(), + record.modelHash(), record.payloadSha256(), record.payloadJson())); + return new InboxOutcome(outcome.duplicate(), outcome.checkpoint()); + }); + assertEquals(1, delivered.acknowledged()); + assertEquals(0, delivered.status().backlog()); + var replicated = Q.customers().withIdIs(customer.getId()).limit(1) + .comment("load replicated Customer through generated Q") + .purpose("prove DuckDB target is queryable by application") + .executeForOne(targetContext); + assertNotNull(replicated); + assertEquals("Committed target customer", replicated.getName()); + assertEquals(customer.getVersion(), replicated.getVersion()); + replicated.updateName("must not modify the analytical target"); + var denied=assertThrows(RuntimeException.class,()->replicated.auditAs("attempt ordinary target write").save(targetContext)); + assertTrue(denied.toString(),denied.toString().contains("READ_ONLY_TARGET")); + var unchanged=Q.customers().withIdIs(customer.getId()).limit(1) + .comment("reload target after denied ordinary write") + .purpose("verify generated mutation did not change replicated facts") + .executeForOne(targetContext); + assertEquals("Committed target customer",unchanged.getName());assertEquals(customer.getVersion(),unchanged.getVersion()); + assertEquals(1,recorder.eventStore().outboxAfter(SOURCE,0,10).size()); + System.out.println("CMR15_GENERATED_READ_ONLY_PASS language=java backend=duckdb generated_audited_save=1 fresh_query=1 source_isolated=1"); + } + } + System.out.println("JAVA_COMMITTED_GENERATED_Q_PASS source=sqlite target=duckdb outbox=1"); + } +} diff --git a/examples/verify-runtime-examples.sh b/examples/verify-runtime-examples.sh index 2e072eab..20b75259 100755 --- a/examples/verify-runtime-examples.sh +++ b/examples/verify-runtime-examples.sh @@ -85,6 +85,7 @@ for repetition in 1 2; do done bash "$repo_dir/examples/trace-chain/verify.sh" +bash "$repo_dir/examples/committed-events/verify.sh" if [[ -n "${TEAQL_CODEGEN_DIR:-}" ]]; then bash "$repo_dir/examples/shared-load-state/verify.sh" --generate else diff --git a/pom.xml b/pom.xml index 9a3262bd..26f1b49f 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ teaql-object-storage-s3 examples/object-storage examples/http-resilience + examples/committed-events examples/business-id-runtime examples/security-foundations examples/round-trip-reference-runtime diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index 204f9989..6c5aec17 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -4,7 +4,7 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" verification_dir="$(mktemp -d)" trap 'rm -rf -- "$verification_dir"' EXIT -expected=(business-id-runtime conformance http-resilience object-storage order-management round-trip-reference-runtime school-management security-foundations shared-load-state standard-order trace-chain) +expected=(business-id-runtime committed-events conformance http-resilience object-storage order-management round-trip-reference-runtime school-management security-foundations shared-load-state standard-order trace-chain) mapfile -t actual < <(find "$repo/examples" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort) if [[ "${actual[*]}" != "${expected[*]}" ]]; then echo "example inventory changed; update scripts/verify-examples.sh: ${actual[*]}" >&2 @@ -14,6 +14,7 @@ fi cd "$repo" bash examples/object-storage/verify.sh bash examples/http-resilience/verify.sh +bash examples/committed-events/verify.sh mvn -q -DskipTests install mvn -q -pl teaql-sqlite \ -Dtest=DerivedQueryTraceSqliteTest,LikeIntentPrivacySqliteTest,TypedIntentPrivacySqliteTest test diff --git a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java index d2fa2f4f..f682479e 100644 --- a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java +++ b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java @@ -427,6 +427,7 @@ public EntityMutationLedger getEntityMutationLedger() { public boolean __internalHasMutationLedger() { return entityMutationLedger != null; } public void setEntityMutationLedger(EntityMutationLedger entityMutationLedger) { + if (this.entityMutationLedger != null) this.entityMutationLedger.requireSaveable(); this.entityMutationLedger = entityMutationLedger; if (entityMutationLedger != null && id != null && newItem()) { entityMutationLedger.markAsNew(new EntityKey(typeName(), id)); diff --git a/teaql-core/src/main/java/io/teaql/core/CommittedEventSchemaContributor.java b/teaql-core/src/main/java/io/teaql/core/CommittedEventSchemaContributor.java new file mode 100644 index 00000000..89a33160 --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/CommittedEventSchemaContributor.java @@ -0,0 +1,6 @@ +package io.teaql.core; + +/** Trusted infrastructure tables installed only in the Context schema lifecycle. */ +public interface CommittedEventSchemaContributor { + void ensureSchema(UserContext context, SchemaExecutor.Invocation invocation); +} diff --git a/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java b/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java index d1cdd4cd..84b49f8f 100644 --- a/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java +++ b/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java @@ -16,6 +16,8 @@ public class EntityMutationLedger { private final Set newKeys = new TreeSet<>(); private final Map> traceChains = new TreeMap<>(); private final Map originalVersions = new TreeMap<>(); + private boolean committedStateRequiresReload; + private boolean transactionOutcomeUnknownRequiresReload; // --- Change Set Stack --- @@ -41,6 +43,28 @@ public void clearCurrentChangeSet() { traceChains.clear(); } + /** A provider write completed, but the in-memory graph could not be reconciled. */ + public void markCommittedStateRequiresReload() { + committedStateRequiresReload = true; + } + + /** A provider could not confirm commit or rollback; this graph must not be replayed. */ + public void markTransactionOutcomeUnknownRequiresReload() { + transactionOutcomeUnknownRequiresReload = true; + } + + /** Prevents replay of a graph whose persistence outcome needs a fresh read. */ + public void requireSaveable() { + if (transactionOutcomeUnknownRequiresReload) { + throw new TeaQLRuntimeException( + "TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED: reload the entity graph before another save"); + } + if (committedStateRequiresReload) { + throw new TeaQLRuntimeException( + "COMMITTED_STATE_RELOAD_REQUIRED: reload the entity graph before another save"); + } + } + public void set(EntityKey key, String field, Object value) { changeSets.set(key, field, value); } @@ -141,12 +165,14 @@ public Long getOriginalVersion(EntityKey key) { * for a read-only entity, which must retain its private ledger ownership. */ public boolean mergeEntityFrom(EntityMutationLedger other, EntityKey key) { + requireSaveable(); if (other == null || other == this) return false; Map fields = other.currentChangeSet().changes().get(key); var dynamic = other.currentChangeSet().dynamicChanges().get(key); boolean pending = (fields != null && !fields.isEmpty()) || other.newKeys.contains(key) || other.deletedKeys.contains(key) || other.recoveredKeys.contains(key) || (dynamic != null && !dynamic.isEmpty()); if (!pending) return false; + other.requireSaveable(); if (fields != null) fields.forEach((field, value) -> set(key, field, value)); if (dynamic != null) dynamic.values().forEach(mutation -> setDynamic(key, mutation)); if (other.deletedKeys.contains(key)) markAsDelete(key); @@ -162,7 +188,9 @@ public boolean mergeEntityFrom(EntityMutationLedger other, EntityKey key) { * Used when saving an entity graph (e.g., Order + OrderItems). */ public void mergeFrom(EntityMutationLedger other) { + requireSaveable(); if (other == null) return; + other.requireSaveable(); // Merge change sets EntityChangeSet otherChangeSet = other.currentChangeSet(); diff --git a/teaql-core/src/main/java/io/teaql/core/TransactionOutcomeUnknownException.java b/teaql-core/src/main/java/io/teaql/core/TransactionOutcomeUnknownException.java new file mode 100644 index 00000000..f6ecceac --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/TransactionOutcomeUnknownException.java @@ -0,0 +1,23 @@ +package io.teaql.core; + +/** A provider could not confirm whether a transaction committed; reload before retrying. */ +public final class TransactionOutcomeUnknownException extends TeaQLRuntimeException { + private static final String MESSAGE = "TRANSACTION_OUTCOME_UNKNOWN: the provider could not confirm the transaction result; " + + "reload the affected graph and committed events before any retry"; + private final Error originalError; + + public TransactionOutcomeUnknownException() { + this(null); + } + + /** Retain a business Error without exposing it as a diagnostic cause. */ + public TransactionOutcomeUnknownException(Error originalError) { + super(MESSAGE); + this.originalError = originalError; + } + + /** The graph runtime rethrows this Error only after fencing the mutation ledger. */ + public Error originalError() { + return originalError; + } +} diff --git a/teaql-core/src/main/java/io/teaql/core/UserContext.java b/teaql-core/src/main/java/io/teaql/core/UserContext.java index b51bb1c2..ae3d6ff0 100644 --- a/teaql-core/src/main/java/io/teaql/core/UserContext.java +++ b/teaql-core/src/main/java/io/teaql/core/UserContext.java @@ -75,6 +75,11 @@ default void ensureSchema() { if (businessIdSchema != null) { businessIdSchema.ensureSchema(this); } + CommittedEventSchemaContributor committedEvents = + capability(CommittedEventSchemaContributor.class); + if (committedEvents != null) { + committedEvents.ensureSchema(this, SchemaExecutor.Invocation.contextOwned()); + } GeneratedSchemaBootstrap bootstrap = capability(GeneratedSchemaBootstrap.class); if (bootstrap != null) { Object previousActor = getAttribute(GeneratedSchemaBootstrap.AUDIT_ACTOR_ATTRIBUTE); diff --git a/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java b/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java index d95112d9..9f5476c3 100644 --- a/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java +++ b/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java @@ -13,6 +13,23 @@ public class EntityMutationLedgerTest { private static final EntityKey ORDER = new EntityKey("Order", 1L); private static final EntityKey OTHER_ORDER = new EntityKey("Order", 2L); + @Test + public void committedReconciliationFailureCannotBeClearedOrMergedIntoAnotherLedger() { + EntityMutationLedger failed = new EntityMutationLedger(); + failed.markAsNew(ORDER); + failed.markCommittedStateRequiresReload(); + assertThrows(TeaQLRuntimeException.class, failed::requireSaveable); + + EntityMutationLedger independent = new EntityMutationLedger(); + assertThrows(TeaQLRuntimeException.class, + () -> independent.mergeEntityFrom(failed, ORDER)); + assertThrows(TeaQLRuntimeException.class, () -> independent.mergeFrom(failed)); + + failed.clearCurrentChangeSet(); + assertThrows(TeaQLRuntimeException.class, failed::requireSaveable); + independent.requireSaveable(); + } + @Test public void markAsNewRecordsOnlyRequestedKeyAndIsIdempotent() { EntityMutationLedger root = new EntityMutationLedger(); diff --git a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java index d120e2ec..de3a17b1 100644 --- a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java +++ b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlDataServiceExecutor.java @@ -63,11 +63,13 @@ public io.teaql.core.QueryCursor queryForCur @Override public MutationResult mutate(UserContext context, PersistenceMutation request) { + executionAdapter.requireOrdinaryMutationAllowed(); return getPortableService(context).mutate(context, request); } @Override public java.util.List mutateBatch(UserContext context, io.teaql.core.MutationBatchRequest request) { + executionAdapter.requireOrdinaryMutationAllowed(); return getPortableService(context).mutateBatch(context, request); } diff --git a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java index 708d9faf..79614548 100644 --- a/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java +++ b/teaql-data-service-sql/src/main/java/io/teaql/dataservice/sql/SqlExecutionAdapter.java @@ -46,6 +46,14 @@ default void executeInTransaction(Runnable action) { } default boolean hasActiveTransaction() { return false; } + /** Ordinary typed writes only; trusted infrastructure SQL is a separate capability. */ + default void requireOrdinaryMutationAllowed() { } + + /** Monotonic startup restriction. Unsupported adapters must reject target wiring. */ + default void restrictOrdinaryMutations() { + throw new UnsupportedOperationException("SQL adapter does not support ordinary mutation isolation"); + } + /** Opaque, non-reused instance token for held storage views; never a URL or transaction handle. */ default Object storageIdentity() { return null; } } diff --git a/teaql-duckdb/pom.xml b/teaql-duckdb/pom.xml index 8b666852..5148dc24 100644 --- a/teaql-duckdb/pom.xml +++ b/teaql-duckdb/pom.xml @@ -23,6 +23,10 @@ io.teaql teaql-utils + + com.fasterxml.jackson.core + jackson-databind + org.duckdb duckdb_jdbc diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java index 573f5f55..c2be45f2 100644 --- a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; public class DuckDataServiceExecutor extends SqlDataServiceExecutor { @@ -58,6 +59,25 @@ public List> getTableColumns(String tableName) { + "FROM information_schema.columns WHERE table_name = ?"; return getExecutionAdapter().queryForList(sql, new Object[] {tableName}); } + + @Override + public Optional canonicalRelationIndexMatches(UserContext caller, + String tableName, String indexName, List orderedColumns) { + List> matches = getExecutionAdapter().queryForList( + "SELECT table_name, is_unique, sql FROM duckdb_indexes() " + + "WHERE index_name = ?", new Object[] {indexName}); + if (matches.isEmpty()) return Optional.of(false); + if (matches.size() != 1) { + throw new IllegalStateException("DuckDB canonical relation index name is ambiguous"); + } + Map index = matches.get(0); + if (!tableName.equals(index.get("table_name")) + || !Boolean.FALSE.equals(index.get("is_unique")) + || !sameSimpleOrderedColumns(index.get("sql"), orderedColumns)) { + throw new IllegalStateException("DuckDB canonical relation index has incompatible shape"); + } + return Optional.of(true); + } }; for (EntityDescriptor descriptor : descriptors) { @@ -65,4 +85,23 @@ public List> getTableColumns(String tableName) { repository.ensurePhysicalSchema(context); } } + + private static boolean sameSimpleOrderedColumns(Object definition, List expected) { + if (!(definition instanceof String ddl)) return false; + int open = ddl.indexOf('('); + int close = ddl.lastIndexOf(')'); + if (open < 0 || close <= open || !ddl.regionMatches(true, 0, "CREATE INDEX ", 0, 13)) { + return false; + } + String[] actual = ddl.substring(open + 1, close).split(",", -1); + if (actual.length != expected.size()) return false; + for (int i = 0; i < actual.length; i++) { + String column = actual[i].trim().replaceFirst("(?i)\\s+(ASC|DESC)$", ""); + if (column.startsWith("\"") && column.endsWith("\"") && column.length() >= 2) { + column = column.substring(1, column.length() - 1).replace("\"\"", "\""); + } + if (!column.equals(expected.get(i))) return false; + } + return true; + } } diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java new file mode 100644 index 00000000..042147c4 --- /dev/null +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java @@ -0,0 +1,214 @@ +package io.teaql.core.duck; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Transaction-owned target Inbox for already committed model facts. The + * projection writer is trusted infrastructure, never a remote SQL callback. + * DuckDB has no SAVEPOINT in the qualified versions, so apply owns the entire + * transaction and refuses to run inside an ambient one. + */ +public final class DuckDbCommittedInbox { + private static final ObjectMapper JSON = new ObjectMapper(); + private final SqlExecutionAdapter sql; + private final String sourceRuntimeId; + private final String modelHash; + + public record Record(String sourceRuntimeId, long sourceSequence, String changeSetId, + String modelHash, String payloadSha256, String payloadJson) { + @Override public String toString() { + return "Record[sourceRuntimeId=" + sourceRuntimeId + ", sourceSequence=" + + sourceSequence + ", payload=]"; + } + } + + public record Outcome(boolean duplicate, long checkpoint) { } + + public static final class Failure extends RuntimeException { + private final String code; + + Failure(String code, String message) { + super(message); + this.code = code; + } + + public String code() { return code; } + } + + @FunctionalInterface + interface TrustedProjectionWriter { + void apply(SqlExecutionAdapter transaction, JsonNode committedFacts); + } + + public DuckDbCommittedInbox(SqlExecutionAdapter sql, String sourceRuntimeId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (blank(sourceRuntimeId) || modelHash == null + || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw invalid("target source identity or model hash is invalid"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.modelHash = modelHash; + } + + public void ensureSchema() { + if (sql.hasActiveTransaction()) throw invalid("schema setup requires no active transaction"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_inbox (" + + "source_runtime_id VARCHAR NOT NULL, change_set_id VARCHAR NOT NULL, " + + "source_sequence BIGINT NOT NULL CHECK(source_sequence > 0), " + + "payload_sha256 VARCHAR NOT NULL, PRIMARY KEY(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_applied_payload (" + + "source_runtime_id VARCHAR NOT NULL, source_sequence BIGINT NOT NULL " + + "CHECK(source_sequence > 0), change_set_id VARCHAR NOT NULL, " + + "payload_json VARCHAR NOT NULL, PRIMARY KEY(source_runtime_id, source_sequence), " + + "UNIQUE(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_checkpoint (" + + "source_runtime_id VARCHAR PRIMARY KEY, source_sequence BIGINT NOT NULL " + + "CHECK(source_sequence >= 0), model_hash VARCHAR NOT NULL)"); + } + + Outcome apply(Record record, TrustedProjectionWriter writer) { + if (sql.hasActiveTransaction()) { + throw new Failure("AMBIENT_TRANSACTION", "DuckDB committed apply requires its own transaction"); + } + if (record == null || writer == null) throw invalid("missing record or projection writer"); + if (!sourceRuntimeId.equals(record.sourceRuntimeId())) { + throw new Failure("UNAUTHORIZED_SOURCE", "committed-event source is not authorized"); + } + if (!modelHash.equals(record.modelHash())) { + throw new Failure("MODEL_MISMATCH", "committed-event model hash mismatch"); + } + if (record.sourceSequence() < 1 || blank(record.changeSetId()) + || record.payloadSha256() == null + || !record.payloadSha256().matches("[0-9a-f]{64}")) { + throw invalid("record identity or digest is invalid"); + } + JsonNode payload = parse(record.payloadJson()); + validateHeader(record, payload); + if (!digest(record.payloadJson()).equals(record.payloadSha256())) { + throw invalid("payload digest mismatch"); + } + AtomicReference outcome = new AtomicReference<>(); + try { + sql.executeInTransaction(() -> outcome.set(applyInside(record, payload, writer))); + } catch (Failure failure) { + throw failure; + } catch (RuntimeException failure) { + // JDBC exceptions may include raw SQL and bound values. The + // caller receives a stable category, never provider diagnostics. + throw new Failure("TARGET_FAILURE", "committed target transaction failed"); + } + return outcome.get(); + } + + private Outcome applyInside(Record record, JsonNode payload, TrustedProjectionWriter writer) { + if (!sql.hasActiveTransaction()) { + throw new Failure("TRANSACTION_REQUIRED", "committed target transaction is unavailable"); + } + List> checkpoints = sql.queryForList( + "SELECT source_sequence, model_hash FROM teaql_event_checkpoint " + + "WHERE source_runtime_id = ?", new Object[] {sourceRuntimeId}); + long last = checkpoints.isEmpty() ? 0 : number(checkpoints.get(0).get("source_sequence")); + if (!checkpoints.isEmpty() && !modelHash.equals(checkpoints.get(0).get("model_hash"))) { + throw new Failure("MODEL_MISMATCH", "committed-event checkpoint model mismatch"); + } + List> prior = sql.queryForList( + "SELECT source_sequence, payload_sha256 FROM teaql_event_inbox " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {sourceRuntimeId, record.changeSetId()}); + if (!prior.isEmpty()) { + Map seen = prior.get(0); + List> retained = sql.queryForList( + "SELECT source_sequence, payload_json FROM teaql_event_applied_payload " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {sourceRuntimeId, record.changeSetId()}); + if (number(seen.get("source_sequence")) == record.sourceSequence() + && record.payloadSha256().equals(seen.get("payload_sha256")) + && retained.size() == 1 + && number(retained.get(0).get("source_sequence")) == record.sourceSequence() + && record.payloadSha256().equals(digest((String) retained.get(0).get("payload_json")))) { + return new Outcome(true, record.sourceSequence()); + } + throw new Failure("DUPLICATE_CONFLICT", "committed-event identity has conflicting content"); + } + if (last == Long.MAX_VALUE) { + throw new Failure("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + } + if (record.sourceSequence() != last + 1) { + throw new Failure("SEQUENCE_GAP", "committed-event source sequence is not contiguous"); + } + writer.apply(sql, payload); + one(sql.update("INSERT INTO teaql_event_applied_payload " + + "(source_runtime_id, source_sequence, change_set_id, payload_json) " + + "VALUES (?, ?, ?, ?)", + new Object[] {sourceRuntimeId, record.sourceSequence(), + record.changeSetId(), record.payloadJson()})); + one(sql.update("INSERT INTO teaql_event_inbox " + + "(source_runtime_id, change_set_id, source_sequence, payload_sha256) " + + "VALUES (?, ?, ?, ?)", + new Object[] {sourceRuntimeId, record.changeSetId(), + record.sourceSequence(), record.payloadSha256()})); + one(sql.update("INSERT INTO teaql_event_checkpoint " + + "(source_runtime_id, source_sequence, model_hash) VALUES (?, ?, ?) " + + "ON CONFLICT(source_runtime_id) DO UPDATE SET " + + "source_sequence = excluded.source_sequence, model_hash = excluded.model_hash", + new Object[] {sourceRuntimeId, record.sourceSequence(), modelHash})); + return new Outcome(false, record.sourceSequence()); + } + + private static void validateHeader(Record record, JsonNode payload) { + if (!payload.isObject() || !payload.path("protocol_version").isInt() + || payload.path("protocol_version").intValue() != 1 + || !payload.path("source_runtime_id").isTextual() + || !record.sourceRuntimeId().equals(payload.path("source_runtime_id").textValue()) + || !payload.path("source_sequence").isTextual() + || !Long.toString(record.sourceSequence()).equals(payload.path("source_sequence").textValue()) + || !payload.path("change_set_id").isTextual() + || !record.changeSetId().equals(payload.path("change_set_id").textValue()) + || !payload.path("source_model_hash").isTextual() + || !record.modelHash().equals(payload.path("source_model_hash").textValue()) + || !payload.path("mutations").isArray() + || payload.path("mutations").isEmpty()) { + throw invalid("required v1 header or mutations mismatch"); + } + } + + private static JsonNode parse(String json) { + if (json == null) throw invalid("missing payload"); + try { + JsonNode payload = JSON.readTree(json); + if (payload == null) throw invalid("payload JSON is empty"); + return payload; + } + catch (Exception failure) { throw invalid("payload JSON is invalid"); } + } + + private static String digest(String payload) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new AssertionError("JDK SHA-256 is unavailable", impossible); + } + } + + private static void one(int changed) { + if (changed != 1) throw new Failure("TARGET_FAILURE", "committed target write count mismatch"); + } + + private static long number(Object value) { return ((Number) value).longValue(); } + private static boolean blank(String value) { return value == null || value.isBlank(); } + private static Failure invalid(String reason) { + return new Failure("INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } +} diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTarget.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTarget.java new file mode 100644 index 00000000..d94e4c60 --- /dev/null +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTarget.java @@ -0,0 +1,486 @@ +package io.teaql.core.duck; + +import com.fasterxml.jackson.databind.JsonNode; +import io.teaql.core.UserContext; +import io.teaql.core.duck.DuckDbCommittedInbox.Failure; +import io.teaql.core.duck.DuckDbCommittedInbox.Record; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.meta.PropertyDescriptor; +import io.teaql.core.meta.Relation; +import io.teaql.core.sql.SQLColumn; +import io.teaql.core.sql.portable.SQLPropertyUtil; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Model-checked, one-source DuckDB target for committed v1 graph facts. + * It writes only declared model columns and verifies the physical result + * before the Inbox transaction is acknowledged. + */ +public final class DuckDbModelCheckedCommittedTarget { + private static final Set TRACE_KINDS = Set.of( + "OPERATION", "REQUEST", "RELATION", "ENTITY", "PROVIDER", "SQL", + "COMMENT", "PURPOSE", "AUDIT_REASON"); + private static final Pattern DECLARED_DECIMAL = Pattern.compile( + "(?i)\\s*(?:NUMERIC|DECIMAL)\\s*\\(\\s*([0-9]+)\\s*,\\s*([0-9]+)\\s*\\)\\s*"); + private final SqlExecutionAdapter sql; + private final DuckDbCommittedInbox inbox; + + private enum Kind { SCALAR, DECIMAL, DATE, INSTANT, BYTES } + private record Mapping(String table, String column, String declaredType) { } + private record Field(String column, Object value, Kind kind) { + String bindExpression() { return kind == Kind.INSTANT ? "make_timestamptz(? * 1000)" : "?"; } + String readExpression() { + String name = quoted(column); + return switch (kind) { + case DATE -> "CAST(" + name + " AS VARCHAR)"; + case INSTANT -> "epoch_ms(" + name + ")"; + default -> name; + }; + } + } + private record Mutation(String entityType, String table, String idColumn, + String versionColumn, long id, Long baseVersion, long committedVersion, + String operation, List fields) { } + + public DuckDbModelCheckedCommittedTarget(SqlExecutionAdapter sql, + String sourceRuntimeId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + this.inbox = new DuckDbCommittedInbox(sql, sourceRuntimeId, modelHash); + sql.restrictOrdinaryMutations(); + } + + /** Infrastructure schema only; domain schema belongs to Context.ensureSchema(). */ + public void ensureInboxSchema() { inbox.ensureSchema(); } + + public DuckDbCommittedInbox.Outcome apply(UserContext context, Record record) { + Objects.requireNonNull(context, "context"); + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + return inbox.apply(record, (transaction, payload) -> applyGraph(transaction, metadata, payload)); + } + + private static void applyGraph(SqlExecutionAdapter sql, EntityMetaFactory metadata, JsonNode payload) { + JsonNode audit = payload.path("audit"); + if (!audit.isObject() || !nonblank(audit.path("comment")) + || !nonblank(audit.path("purpose")) + || !nonblank(payload.path("source_transaction_id"))) { + throw invalid("source transaction or audit intent is missing"); + } + try { OffsetDateTime.parse(payload.path("committed_at").asText()); } + catch (RuntimeException failure) { throw invalid("commit time is invalid"); } + List mutations = new ArrayList<>(); + Set seen = new HashSet<>(); + for (JsonNode fact : payload.path("mutations")) { + Mutation mutation = decode(sql, metadata, fact); + if (!seen.add(mutation.entityType() + ":" + mutation.id())) { + throw invalid("one set mutates the same entity twice"); + } + mutations.add(mutation); + } + // Every fact is validated before the first domain write. A subsequent + // SQL or readback error escapes to the Inbox owner for whole rollback. + for (Mutation mutation : mutations) applyOne(sql, mutation); + } + + private static Mutation decode(SqlExecutionAdapter sql, EntityMetaFactory metadata, JsonNode fact) { + if (!fact.isObject() || !nonblank(fact.path("entity_type"))) { + throw invalid("mutation entity type is missing"); + } + validateTraceChain(fact.path("trace_chain")); + String entityType = fact.path("entity_type").textValue(); + EntityDescriptor entity = metadata.allEntityDescriptors().stream() + .filter(candidate -> entityType.equals(candidate.getType())).findFirst() + .orElseThrow(() -> invalid("unknown entity type")); + if (entity.getParent() != null || entity.hasChildren()) { + throw invalid("multi-table entity target mapping is not supported"); + } + PropertyDescriptor idProperty = entity.findIdProperty(); + PropertyDescriptor versionProperty = entity.findVersionProperty(); + if (idProperty == null || versionProperty == null) { + throw invalid("entity has no ID or version property"); + } + Mapping idMapping = mapping(idProperty); + Mapping versionMapping = mapping(versionProperty); + if (!idMapping.table().equals(versionMapping.table())) { + throw invalid("identity and version span multiple tables"); + } + requirePhysical(sql, idMapping, "BIGINT", null, null); + requirePhysical(sql, versionMapping, "BIGINT", null, null); + long id = positiveId(fact.path("entity_id")); + String operation = fact.path("operation").asText(""); + if (!List.of("create", "update", "delete", "recover").contains(operation)) { + throw invalid("unknown mutation operation"); + } + JsonNode baseNode = fact.path("base_version"); + Long base = "create".equals(operation) && baseNode.isNull() + ? null : signedVersion(baseNode); + long committed = signedVersion(fact.path("committed_version")); + boolean valid = switch (operation) { + case "create" -> base == null && committed == 1; + case "update" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == base + 1; + case "delete" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == -(base + 1); + case "recover" -> base != null && base < 0 && base != Long.MIN_VALUE + && -base < Long.MAX_VALUE && committed == -base + 1; + default -> false; + }; + if (!valid) throw invalid("invalid version transition"); + JsonNode supplied = fact.path("fields"); + if (!supplied.isObject()) throw invalid("mutation fields are missing"); + if ("delete".equals(operation) && !supplied.isEmpty()) { + throw invalid("delete must be a tombstone"); + } + List ownProperties = entity.getOwnProperties(); + supplied.fieldNames().forEachRemaining(name -> { + PropertyDescriptor property = entity.findProperty(name); + if (property == null || !ownProperties.contains(property) + || property.isId() || property.isVersion()) { + throw invalid("unknown or reserved model field"); + } + }); + List fields = new ArrayList<>(); + for (PropertyDescriptor property : ownProperties) { + if (property.isId() || property.isVersion()) continue; + JsonNode value = supplied.get(property.getName()); + if (value == null) { + if ("create".equals(operation)) throw invalid("create omitted a modeled field"); + continue; + } + Mapping column = mapping(property); + if (!idMapping.table().equals(column.table())) { + throw invalid("mutation field spans multiple tables"); + } + fields.add(decodeField(sql, property, column, value)); + } + return new Mutation(entityType, idMapping.table(), idMapping.column(), + versionMapping.column(), id, base, committed, operation, fields); + } + + private static Field decodeField(SqlExecutionAdapter sql, PropertyDescriptor property, + Mapping mapping, JsonNode value) { + if (value.isNull() && property.getBoolean("required", false)) { + throw invalid("non-null field is null"); + } + if (property instanceof Relation relation) { + requirePhysical(sql, mapping, "BIGINT", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isObject() || value.size() != 2 || relation.getReverseProperty() == null + || relation.getReverseProperty().getOwner() == null + || !relation.getReverseProperty().getOwner().getType() + .equals(value.path("entity_type").asText())) { + throw invalid("reference target does not match model"); + } + return new Field(mapping.column(), positiveId(value.path("entity_id")), Kind.SCALAR); + } + Class type = property.getType() == null ? null : property.getType().javaType(); + if (type == LocalDate.class || type == java.sql.Date.class) { + requirePhysical(sql, mapping, "DATE", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.DATE); + if (!value.isTextual()) throw invalid("Date field has wrong type"); + try { + LocalDate date = LocalDate.parse(value.textValue()); + if (!date.toString().equals(value.textValue())) throw invalid("Date field is not canonical"); + return new Field(mapping.column(), date, Kind.DATE); + } catch (java.time.format.DateTimeParseException failure) { + throw invalid("Date field has wrong type"); + } + } + if (type == java.time.Instant.class || type == java.util.Date.class + || type == java.sql.Timestamp.class) { + requirePhysical(sql, mapping, "TIMESTAMP WITH TIME ZONE", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.INSTANT); + long millis = canonicalLong(value, "Timestamp field has wrong type"); + try { Math.multiplyExact(millis, 1000L); } + catch (ArithmeticException overflow) { throw invalid("Timestamp exceeds DuckDB instant range"); } + return new Field(mapping.column(), millis, Kind.INSTANT); + } + if (type == java.time.LocalDateTime.class) { + throw invalid("zone-free LocalDateTime has no committed instant mapping"); + } + if (type == String.class || type == java.util.UUID.class || type != null && type.isEnum()) { + requireTextPhysical(sql, mapping); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isTextual()) throw invalid("text field has wrong type"); + String text = value.textValue(); + String maxText = property.getStr("max", null); + if (maxText != null) { + try { + if (text.codePointCount(0, text.length()) > Integer.parseInt(maxText)) { + throw invalid("text exceeds modeled length"); + } + } catch (NumberFormatException malformed) { throw invalid("modeled text length is invalid"); } + } + return new Field(mapping.column(), text, Kind.SCALAR); + } + if (type == Boolean.class || type == boolean.class) { + String physical = requireBooleanPhysical(sql, mapping); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isBoolean()) throw invalid("Boolean field has wrong type"); + return new Field(mapping.column(), "BOOLEAN".equals(physical) + ? value.booleanValue() : value.booleanValue() ? 1 : 0, Kind.SCALAR); + } + if (type == Long.class || type == long.class || type == Integer.class || type == int.class + || type == Short.class || type == short.class || type == Byte.class || type == byte.class) { + String physical = type == Long.class || type == long.class ? "BIGINT" + : type == Integer.class || type == int.class ? "INTEGER" + : type == Short.class || type == short.class ? "SMALLINT" : "TINYINT"; + requirePhysical(sql, mapping, physical, null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + long number = canonicalLong(value, "integer field has wrong type"); + if (type == Integer.class || type == int.class) { + if (number < Integer.MIN_VALUE || number > Integer.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Short.class || type == short.class) { + if (number < Short.MIN_VALUE || number > Short.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Byte.class || type == byte.class) { + if (number < Byte.MIN_VALUE || number > Byte.MAX_VALUE) throw invalid("integer overflows field"); + } + return new Field(mapping.column(), number, Kind.SCALAR); + } + if (type == BigDecimal.class) { + Matcher declaration = DECLARED_DECIMAL.matcher(mapping.declaredType()); + if (!declaration.matches()) throw invalid("Decimal lacks modeled precision/scale"); + int precision; + int scale; + try { + precision = Integer.parseInt(declaration.group(1)); + scale = Integer.parseInt(declaration.group(2)); + } catch (NumberFormatException malformed) { throw invalid("modeled Decimal precision/scale is invalid"); } + if (precision < 1 || scale > precision) throw invalid("modeled Decimal precision/scale is invalid"); + requirePhysical(sql, mapping, "DECIMAL", precision, scale); + if (value.isNull()) return new Field(mapping.column(), null, Kind.DECIMAL); + if (!value.isTextual()) throw invalid("Decimal field has wrong type"); + BigDecimal decimal; + try { decimal = new BigDecimal(value.textValue()); } + catch (NumberFormatException malformed) { throw invalid("Decimal field is invalid"); } + BigDecimal normalized = decimal.stripTrailingZeros(); + long integerDigits = decimal.signum() == 0 ? 0 + : Math.max(0L, (long) normalized.precision() - normalized.scale()); + long fractionalDigits = Math.max(0L, normalized.scale()); + if (integerDigits > precision - scale || fractionalDigits > scale) { + throw invalid("Decimal exceeds modeled precision/scale"); + } + return new Field(mapping.column(), decimal, Kind.DECIMAL); + } + if (type == Double.class || type == double.class) { + requirePhysical(sql, mapping, "DOUBLE", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isNumber() || !Double.isFinite(value.doubleValue())) { + throw invalid("floating field has wrong type"); + } + return new Field(mapping.column(), value.doubleValue(), Kind.SCALAR); + } + if (type == byte[].class) { + requirePhysical(sql, mapping, "BLOB", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.BYTES); + if (!value.isObject() || value.size() != 1 + || !value.path("$binary_base64").isTextual()) { + throw invalid("binary field has wrong type"); + } + try { + return new Field(mapping.column(), java.util.Base64.getDecoder() + .decode(value.path("$binary_base64").textValue()), Kind.BYTES); + } catch (IllegalArgumentException malformed) { throw invalid("binary field is invalid"); } + } + throw invalid("unsupported modeled field type"); + } + + private static void applyOne(SqlExecutionAdapter sql, Mutation mutation) { + String table = quoted(mutation.table()); + String idColumn = quoted(mutation.idColumn()); + String versionColumn = quoted(mutation.versionColumn()); + List> current = sql.queryForList("SELECT " + versionColumn + + " AS committed_version FROM " + table + " WHERE " + idColumn + " = ?", + new Object[] {mutation.id()}); + Long actualVersion = current.isEmpty() ? null + : ((Number) current.get(0).get("committed_version")).longValue(); + if (!Objects.equals(actualVersion, mutation.baseVersion())) { + throw new Failure("VERSION_CONFLICT", "committed-event predecessor version mismatch"); + } + List args = new ArrayList<>(); + if ("create".equals(mutation.operation())) { + List columns = new ArrayList<>(List.of(idColumn, versionColumn)); + List binds = new ArrayList<>(List.of("?", "?")); + args.add(mutation.id()); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + columns.add(quoted(field.column())); + binds.add(field.bindExpression()); + args.add(field.value()); + } + if (sql.update("INSERT INTO " + table + " (" + String.join(", ", columns) + + ") VALUES (" + String.join(", ", binds) + ")", args.toArray()) != 1) { + throw new Failure("TARGET_FAILURE", "committed target insert count mismatch"); + } + } else { + List assignments = new ArrayList<>(List.of(versionColumn + " = ?")); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + assignments.add(quoted(field.column()) + " = " + field.bindExpression()); + args.add(field.value()); + } + args.add(mutation.id()); + args.add(mutation.baseVersion()); + int changed = sql.update("UPDATE " + table + " SET " + String.join(", ", assignments) + + " WHERE " + idColumn + " = ? AND " + versionColumn + " = ?", args.toArray()); + if (changed != 1) { + throw new Failure("VERSION_CONFLICT", "committed-event predecessor version mismatch"); + } + } + verifyReadback(sql, mutation); + } + + private static void verifyReadback(SqlExecutionAdapter sql, Mutation mutation) { + List projections = new ArrayList<>(List.of( + quoted(mutation.versionColumn()) + " AS committed_version")); + for (int i = 0; i < mutation.fields().size(); i++) { + projections.add(mutation.fields().get(i).readExpression() + " AS field_" + i); + } + List> rows = sql.queryForList("SELECT " + String.join(", ", projections) + + " FROM " + quoted(mutation.table()) + " WHERE " + quoted(mutation.idColumn()) + " = ?", + new Object[] {mutation.id()}); + if (rows.size() != 1 || !Objects.equals(mutation.committedVersion(), + ((Number) rows.get(0).get("committed_version")).longValue())) { + throw invalid("target did not retain committed version"); + } + for (int i = 0; i < mutation.fields().size(); i++) { + Field field = mutation.fields().get(i); + if (!sameValue(field, rows.get(0).get("field_" + i))) { + throw invalid("target did not retain committed field value"); + } + } + } + + private static boolean sameValue(Field field, Object actual) { + Object expected = field.value(); + if (expected == null || actual == null) return expected == actual; + if (field.kind() == Kind.BYTES) { + return actual instanceof byte[] bytes && Arrays.equals((byte[]) expected, bytes); + } + if (field.kind() == Kind.DATE) return expected.toString().equals(actual.toString()); + if (field.kind() == Kind.INSTANT) { + return actual instanceof Number number && number.longValue() == (long) expected; + } + if (field.kind() == Kind.DECIMAL) { + return actual instanceof BigDecimal decimal + && ((BigDecimal) expected).compareTo(decimal) == 0; + } + if (expected instanceof Number number && actual instanceof Number other) { + return new BigDecimal(number.toString()).compareTo(new BigDecimal(other.toString())) == 0; + } + return Objects.equals(expected, actual); + } + + private static Mapping mapping(PropertyDescriptor property) { + List columns = SQLPropertyUtil.getColumns(property); + if (columns.size() != 1 || columns.get(0).getType() == null) { + throw invalid("property has no single declared SQL column"); + } + SQLColumn column = columns.get(0); + return new Mapping(column.getTableName(), column.getColumnName(), column.getType()); + } + + private static void requireTextPhysical(SqlExecutionAdapter sql, Mapping mapping) { + if (!mapping.declaredType().matches("(?i)\\s*(?:VARCHAR(?:\\([0-9]+\\))?|TEXT)\\s*")) { + throw invalid("text column declaration is unsupported"); + } + requirePhysical(sql, mapping, "VARCHAR", null, null); + } + + private static String requireBooleanPhysical(SqlExecutionAdapter sql, Mapping mapping) { + String type = mapping.declaredType().trim().toUpperCase(java.util.Locale.ROOT); + if (!type.equals("BOOLEAN") && !type.equals("TINYINT")) { + throw invalid("Boolean column declaration is unsupported"); + } + requirePhysical(sql, mapping, type, null, null); + return type; + } + + private static void requirePhysical(SqlExecutionAdapter sql, Mapping mapping, + String expectedType, Integer precision, Integer scale) { + String declared = mapping.declaredType().trim().toUpperCase(java.util.Locale.ROOT); + if (expectedType.equals("TIMESTAMP WITH TIME ZONE")) { + if (!declared.equals("TIMESTAMPTZ") && !declared.equals(expectedType)) { + throw invalid("model Instant column is not an absolute timestamp"); + } + } else if (!expectedType.equals("DECIMAL") && !expectedType.equals("VARCHAR") + && !declared.equals(expectedType)) { + throw invalid("modeled SQL type does not match field kind"); + } + List> rows = sql.queryForList( + "SELECT data_type, numeric_precision, numeric_scale FROM information_schema.columns " + + "WHERE table_name = ? AND column_name = ?", + new Object[] {mapping.table(), mapping.column()}); + if (rows.size() != 1) throw invalid("physical model column is missing or ambiguous"); + Map row = rows.get(0); + String physical = (String) row.get("data_type"); + if (physical == null || !(expectedType.equals("DECIMAL") + ? physical.startsWith("DECIMAL(") : physical.equals(expectedType))) { + throw invalid("physical column has incompatible SQL type"); + } + if (precision != null && (number(row.get("numeric_precision")) != precision + || number(row.get("numeric_scale")) != scale)) { + throw invalid("physical Decimal precision/scale differs from model"); + } + } + + private static void validateTraceChain(JsonNode chain) { + if (!chain.isArray()) throw invalid("mutation trace chain is missing"); + for (JsonNode node : chain) { + JsonNode id = node.path("entity_id"); + if (!node.isObject() || node.size() != 4 + || !TRACE_KINDS.contains(node.path("kind").asText("")) + || !nonblank(node.path("name")) + || !nonblank(node.path("comment")) + || !(id.isNull() || id.isTextual() && id.textValue().matches("[1-9][0-9]*"))) { + throw invalid("trace node has wrong shape"); + } + } + } + + private static long positiveId(JsonNode value) { + long id = canonicalLong(value, "identity is outside the DuckDB ID range"); + if (id <= 0) throw invalid("identity must be positive"); + return id; + } + + private static long signedVersion(JsonNode value) { + return canonicalLong(value, "version has wrong type"); + } + + private static long canonicalLong(JsonNode value, String reason) { + if (!value.isTextual() || !value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) throw invalid(reason); + try { return Long.parseLong(value.textValue()); } + catch (NumberFormatException overflow) { throw invalid(reason); } + } + + private static String quoted(String identifier) { + if (identifier == null || identifier.isBlank()) throw invalid("model has an empty SQL identifier"); + return "\"" + identifier.replace("\"", "\"\"") + "\""; + } + + private static long number(Object value) { + if (!(value instanceof Number number)) throw invalid("physical numeric metadata is missing"); + return number.longValue(); + } + private static boolean nonblank(JsonNode value) { + return value.isTextual() && !value.textValue().isBlank(); + } + private static Failure invalid(String reason) { + return new Failure("INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } +} diff --git a/teaql-duckdb/src/main/java/module-info.java b/teaql-duckdb/src/main/java/module-info.java index 17270eeb..7ff87a4a 100644 --- a/teaql-duckdb/src/main/java/module-info.java +++ b/teaql-duckdb/src/main/java/module-info.java @@ -2,6 +2,7 @@ requires transitive io.teaql.core; requires transitive io.teaql.dataservice.sql; requires transitive io.teaql.utils; + requires com.fasterxml.jackson.databind; requires java.sql; exports io.teaql.core.duck; } diff --git a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbCommittedInboxTest.java b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbCommittedInboxTest.java new file mode 100644 index 00000000..59d7dddd --- /dev/null +++ b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbCommittedInboxTest.java @@ -0,0 +1,167 @@ +package io.teaql.core.duck; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import io.teaql.core.duck.DuckDbCommittedInbox.Failure; +import io.teaql.core.duck.DuckDbCommittedInbox.Record; +import io.teaql.provider.jdbc.JdbcSqlExecutor; + +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.HexFormat; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class DuckDbCommittedInboxTest { + private static final String SOURCE = "duckdb-source"; + private static final String MODEL_HASH = "sha256:" + "a".repeat(64); + private Path directory; + private Path database; + private JdbcSqlExecutor sql; + private DuckDbCommittedInbox inbox; + + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { + return DriverManager.getConnection(url); + } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class iface) throws SQLException { + throw new SQLException("not a wrapper"); + } + @Override public boolean isWrapperFor(Class iface) { return false; } + } + + @Before public void setUp() throws Exception { + directory = Files.createTempDirectory("teaql-duckdb-inbox-"); + database = directory.resolve("target.db"); + sql = new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)); + inbox = new DuckDbCommittedInbox(sql, SOURCE, MODEL_HASH); + inbox.ensureSchema(); + sql.execute("CREATE TABLE replica_order (id BIGINT PRIMARY KEY, version BIGINT NOT NULL)"); + } + + @After public void tearDown() throws Exception { + Files.deleteIfExists(database); + Files.deleteIfExists(directory); + } + + @Test public void appliesOnceAndRetainsReplayEvidence() { + Record first = record(1, "set-1"); + AtomicInteger calls = new AtomicInteger(); + DuckDbCommittedInbox.TrustedProjectionWriter writer = (transaction, ignored) -> { + calls.incrementAndGet(); + transaction.update("INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L}); + }; + assertEquals(new DuckDbCommittedInbox.Outcome(false, 1), inbox.apply(first, writer)); + assertEquals(new DuckDbCommittedInbox.Outcome(true, 1), inbox.apply(first, writer)); + assertEquals(1, calls.get()); + assertEquals(1, count("replica_order")); + assertEquals(1, count("teaql_event_inbox")); + assertEquals(1, count("teaql_event_applied_payload")); + assertEquals(1, count("teaql_event_checkpoint")); + assertEquals(1L, checkpoint()); + + DuckDbCommittedInbox reopened = new DuckDbCommittedInbox( + new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)), + SOURCE, MODEL_HASH); + assertTrue(reopened.apply(first, (transaction, ignored) -> { + throw new AssertionError("duplicate reran projection"); + }).duplicate()); + } + + @Test public void projectionFailureRollsBackAllStateAndCanRetry() { + Record first = record(1, "set-1"); + Failure failure = assertThrows(Failure.class, () -> inbox.apply(first, (transaction, ignored) -> { + transaction.update("INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L}); + throw new IllegalStateException("private-sql-canary"); + })); + assertEquals("TARGET_FAILURE", failure.code()); + assertFalse(failure.getMessage().contains("private-sql-canary")); + for (String table : new String[] {"replica_order", "teaql_event_inbox", + "teaql_event_applied_payload", "teaql_event_checkpoint"}) { + assertEquals(table, 0, count(table)); + } + assertFalse(inbox.apply(first, (transaction, ignored) -> transaction.update( + "INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L})).duplicate()); + assertEquals(1, count("replica_order")); + } + + @Test public void gapTamperAndAmbientTransactionFailClosed() { + assertCode("SEQUENCE_GAP", () -> inbox.apply(record(2, "set-2"), + (transaction, ignored) -> { })); + assertCode("INVALID_ENVELOPE", () -> inbox.apply( + new Record(SOURCE, 1, "set-1", MODEL_HASH, "0".repeat(64), record(1, "set-1").payloadJson()), + (transaction, ignored) -> { })); + assertCode("UNAUTHORIZED_SOURCE", () -> inbox.apply( + new Record("other-source", 1, "set-1", MODEL_HASH, + record(1, "set-1").payloadSha256(), record(1, "set-1").payloadJson()), + (transaction, ignored) -> { })); + sql.executeInTransaction(() -> assertCode("AMBIENT_TRANSACTION", + () -> inbox.apply(record(1, "set-1"), (transaction, ignored) -> { }))); + assertEquals(0, count("teaql_event_inbox")); + + Record first = record(1, "set-1"); + inbox.apply(first, (transaction, ignored) -> transaction.update( + "INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L})); + Record changed = record(1, "set-1", "update"); + assertCode("DUPLICATE_CONFLICT", () -> inbox.apply(changed, + (transaction, ignored) -> { })); + assertEquals(1L, checkpoint()); + } + + private long checkpoint() { + return ((Number) sql.queryForList("SELECT source_sequence FROM teaql_event_checkpoint", + new Object[0]).get(0).get("source_sequence")).longValue(); + } + + private int count(String table) { + return ((Number) sql.queryForList("SELECT count(*) AS row_count FROM " + table, + new Object[0]).get(0).get("row_count")).intValue(); + } + + private static Record record(long sequence, String changeSetId) { + return record(sequence, changeSetId, "create"); + } + + private static Record record(long sequence, String changeSetId, String operation) { + String payload = "{\"protocol_version\":1,\"source_runtime_id\":\"" + SOURCE + + "\",\"source_sequence\":\"" + sequence + + "\",\"change_set_id\":\"" + changeSetId + + "\",\"source_model_hash\":\"" + MODEL_HASH + + "\",\"mutations\":[{\"operation\":\"" + operation + "\"}]}"; + try { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8))); + return new Record(SOURCE, sequence, changeSetId, MODEL_HASH, digest, payload); + } catch (Exception impossible) { + throw new AssertionError(impossible); + } + } + + private static void assertCode(String code, Runnable action) { + assertEquals(code, assertThrows(Failure.class, action::run).code()); + } +} diff --git a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java new file mode 100644 index 00000000..78a5aff6 --- /dev/null +++ b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java @@ -0,0 +1,359 @@ +package io.teaql.core.duck; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.BaseEntity; +import io.teaql.core.duck.DuckDbCommittedInbox.Failure; +import io.teaql.core.duck.DuckDbCommittedInbox.Record; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class DuckDbModelCheckedCommittedTargetTest { + private static final String SOURCE = "sqlite-order-source"; + private static final String HASH = "sha256:" + "0".repeat(64); + private static final long ORDER_ID = 9_007_199_254_740_993L; + private static final long ITEM_ID = ORDER_ID + 1; + + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { return DriverManager.getConnection(url); } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class type) throws SQLException { throw new SQLException("not a wrapper"); } + @Override public boolean isWrapperFor(Class type) { return false; } + } + + private Path directory; + private Path database; + private JdbcSqlExecutor sql; + private DefaultUserContext context; + private DuckDbModelCheckedCommittedTarget target; + + @Before public void setUp() throws Exception { + directory = Files.createTempDirectory("teaql-duckdb-model-target-"); + database = directory.resolve("target.db"); + sql = new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)); + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + EntityDescriptor order = new EntityDescriptor(); + order.setType("order"); + order.addSimpleProperty("id", Long.class); + order.addSimpleProperty("version", Long.class); + order.addSimpleProperty("order_number", String.class); + order.addSimpleProperty("status", String.class); + order.addSimpleProperty("planned_date", java.time.LocalDate.class).with("sqlType", "DATE"); + order.addSimpleProperty("occurred_at", java.time.Instant.class) + .with("sqlType", "TIMESTAMPTZ").with("required", "true"); + order.addSimpleProperty("amount", BigDecimal.class).with("sqlType", "DECIMAL(19,7)"); + order.addSimpleProperty("active", Boolean.class) + .with("sqlType", "BOOLEAN").with("required", "true"); + order.addSimpleProperty("ratio", Double.class).with("required", "true"); + order.addSimpleProperty("optional_note", String.class); + order.addSimpleProperty("optional_label", String.class); + metadata.register(order); + EntityDescriptor item = new EntityDescriptor(); + item.setType("order_item"); + item.addSimpleProperty("id", Long.class); + item.addSimpleProperty("version", Long.class); + item.addObjectProperty(metadata, "order", "order", "items", BaseEntity.class) + .with("columnName", "order_id"); + item.addSimpleProperty("quantity", Long.class); + metadata.register(item); + context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).build()); + sql.execute("CREATE TABLE order_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_number VARCHAR NOT NULL, status VARCHAR NOT NULL, planned_date DATE, " + + "occurred_at TIMESTAMPTZ NOT NULL, amount DECIMAL(19,7), active BOOLEAN NOT NULL, " + + "ratio DOUBLE NOT NULL, optional_note VARCHAR, optional_label VARCHAR)"); + sql.execute("CREATE TABLE order_item_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_id BIGINT NOT NULL REFERENCES order_data(id), quantity BIGINT NOT NULL)"); + target = new DuckDbModelCheckedCommittedTarget(sql, SOURCE, HASH); + target.ensureInboxSchema(); + } + + @After public void tearDown() throws Exception { + Files.deleteIfExists(database); + Files.deleteIfExists(directory); + } + + @Test public void appliesWholeOrderGraphTwiceAndRetainsExactTypedValuesAcrossRestart() { + Record first = record(1, "order-graph-0001", created()); + Record second = record(2, "order-graph-0002", submitted()); + assertEquals(new DuckDbCommittedInbox.Outcome(false, 1), target.apply(context, first)); + assertEquals(new DuckDbCommittedInbox.Outcome(true, 1), target.apply(context, first)); + assertEquals(new DuckDbCommittedInbox.Outcome(false, 2), target.apply(context, second)); + assertEquals(new DuckDbCommittedInbox.Outcome(true, 2), target.apply(context, second)); + assertEquals(1, count("order_data")); + assertEquals(1, count("order_item_data")); + assertEquals(2, count("teaql_event_inbox")); + assertEquals(2, count("teaql_event_applied_payload")); + assertEquals(1, count("teaql_event_checkpoint")); + Map order = sql.queryForList("SELECT id, version, status, " + + "CAST(planned_date AS VARCHAR) AS planned_date, epoch_ms(occurred_at) AS occurred_at, " + + "amount, active, ratio, optional_note, optional_label FROM order_data", new Object[0]).get(0); + assertEquals(ORDER_ID, ((Number) order.get("id")).longValue()); + assertEquals(2L, ((Number) order.get("version")).longValue()); + assertEquals("SUBMITTED", order.get("status")); + assertEquals("2026-10-16", order.get("planned_date")); + assertEquals(-315_619_199_877L, ((Number) order.get("occurred_at")).longValue()); + assertEquals(0, ((BigDecimal) order.get("amount")).compareTo(BigDecimal.ZERO)); + assertEquals(true, order.get("active")); + assertEquals(0.5, ((Number) order.get("ratio")).doubleValue(), 0.0); + assertEquals("submitted note", order.get("optional_note")); + assertEquals(null, order.get("optional_label")); + Map item = sql.queryForList("SELECT id, version, order_id, quantity " + + "FROM order_item_data", new Object[0]).get(0); + assertEquals(ITEM_ID, ((Number) item.get("id")).longValue()); + assertEquals(-2L, ((Number) item.get("version")).longValue()); + assertEquals(ORDER_ID, ((Number) item.get("order_id")).longValue()); + assertEquals(2L, ((Number) item.get("quantity")).longValue()); + assertEquals(first.payloadJson(), sql.queryForList("SELECT payload_json FROM " + + "teaql_event_applied_payload WHERE source_sequence = 1", new Object[0]) + .get(0).get("payload_json")); + DuckDbModelCheckedCommittedTarget reopened = new DuckDbModelCheckedCommittedTarget( + new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)), SOURCE, HASH); + assertTrue(reopened.apply(context, second).duplicate()); + } + + @Test public void appliesSharedConformanceFixtureWhenConfigured() throws Exception { + String fixturePath = System.getenv("TEAQL_COMMITTED_EVENT_FIXTURE"); + Assume.assumeTrue("shared fixture path was not supplied", + fixturePath != null && !fixturePath.isBlank()); + JsonNode fixture = new ObjectMapper().readTree(Files.readString(Path.of(fixturePath))); + assertEquals(1, fixture.path("fixture_version").asInt()); + assertEquals(2, fixture.path("sets").size()); + for (int index = 0; index < 2; index++) { + JsonNode set = fixture.path("sets").get(index); + long sequence = Long.parseLong(set.path("source_sequence").asText()); + Record source = recordBody(sequence, set.path("change_set_id").asText(), set.toString()); + assertEquals(sequence, target.apply(context, source).checkpoint()); + assertTrue(target.apply(context, source).duplicate()); + } + Map order = sql.queryForList("SELECT id, version, status, " + + "CAST(planned_date AS VARCHAR) AS planned_date, epoch_ms(occurred_at) AS occurred_at, " + + "amount, active, ratio, optional_note, optional_label FROM order_data", + new Object[0]).get(0); + assertEquals(ORDER_ID, ((Number) order.get("id")).longValue()); + assertEquals(2L, ((Number) order.get("version")).longValue()); + assertEquals("SUBMITTED", order.get("status")); + assertEquals("2026-10-16", order.get("planned_date")); + assertEquals(-315_619_199_877L, ((Number) order.get("occurred_at")).longValue()); + assertEquals(0, ((BigDecimal) order.get("amount")).compareTo(BigDecimal.ZERO)); + assertEquals(true, order.get("active")); + assertEquals(0.5, ((Number) order.get("ratio")).doubleValue(), 0.0); + assertEquals("submitted note", order.get("optional_note")); + assertEquals(null, order.get("optional_label")); + Map item = sql.queryForList("SELECT id, version, order_id, quantity " + + "FROM order_item_data", new Object[0]).get(0); + assertEquals(ITEM_ID, ((Number) item.get("id")).longValue()); + assertEquals(-2L, ((Number) item.get("version")).longValue()); + assertEquals(ORDER_ID, ((Number) item.get("order_id")).longValue()); + assertEquals(2L, ((Number) item.get("quantity")).longValue()); + assertEquals(2, count("teaql_event_inbox")); + assertEquals(2, count("teaql_event_applied_payload")); + assertEquals(1, count("teaql_event_checkpoint")); + } + + @Test public void contextEnsureSchemaInstallsCompatibleModelStorage() throws Exception { + Path installedDatabase = directory.resolve("installed-schema.db"); + try { + JdbcSqlExecutor installedSql = new JdbcSqlExecutor( + new DriverManagerDataSource("jdbc:duckdb:" + installedDatabase)); + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + metadata.allEntityDescriptors().forEach(entity -> entity.setDataService("duckdb")); + DefaultUserContext installedContext = new DefaultUserContext(TeaQLRuntime.builder() + .metadata(metadata) + .dataService("duckdb", new DuckDataServiceExecutor("duckdb", installedSql)) + .build()); + installedContext.ensureSchema(); + installedContext.ensureSchema(); + DuckDbModelCheckedCommittedTarget installed = + new DuckDbModelCheckedCommittedTarget(installedSql, SOURCE, HASH); + installed.ensureInboxSchema(); + assertEquals(1L, installed.apply(installedContext, + record(1, "installed-create", created())).checkpoint()); + assertEquals(2L, installed.apply(installedContext, + record(2, "installed-update", submitted())).checkpoint()); + List> columns = installedSql.queryForList( + "SELECT column_name, data_type FROM information_schema.columns " + + "WHERE table_name = 'order_data'", new Object[0]); + Map types = new java.util.HashMap<>(); + for (Map column : columns) { + types.put((String) column.get("column_name"), (String) column.get("data_type")); + } + assertEquals("DATE", types.get("planned_date")); + assertEquals("TIMESTAMP WITH TIME ZONE", types.get("occurred_at")); + assertEquals("DECIMAL(19,7)", types.get("amount")); + assertEquals("BOOLEAN", types.get("active")); + assertEquals("SUBMITTED", installedSql.queryForList( + "SELECT status FROM order_data", new Object[0]).get(0).get("status")); + assertEquals(-2L, ((Number) installedSql.queryForList( + "SELECT version FROM order_item_data", new Object[0]) + .get(0).get("version")).longValue()); + List> indexes = installedSql.queryForList( + "SELECT index_name FROM duckdb_indexes() " + + "WHERE table_name = 'order_item_data'", new Object[0]); + assertEquals(1, indexes.size()); + String indexName = (String) indexes.get(0).get("index_name"); + assertTrue(indexName.matches("[A-Za-z0-9_]+")); + installedSql.execute("DROP INDEX \"" + indexName + "\""); + installedSql.execute("CREATE INDEX \"" + indexName + + "\" ON order_item_data(quantity)"); + IllegalStateException mismatch = assertThrows(IllegalStateException.class, + installedContext::ensureSchema); + assertTrue(mismatch.getMessage().contains("canonical relation index")); + } finally { + Files.deleteIfExists(installedDatabase); + } + } + + @Test public void lateGraphFailureRollsBackDomainAndInbox() { + String broken = created().replace("\"quantity\":\"2\"", "\"quantity\":\"2\",\"unknown\":1"); + expect("INVALID_ENVELOPE", () -> target.apply(context, record(1, "bad", broken))); + assertEmptyTarget(); + String missingParent = created().replace("\"order\":{\"entity_type\":\"order\",\"entity_id\":\"9007199254740993\"}", + "\"order\":{\"entity_type\":\"order\",\"entity_id\":\"9007199254740995\"}"); + assertFalse(missingParent.equals(created())); + expect("TARGET_FAILURE", () -> target.apply(context, record(1, "bad", missingParent))); + assertEmptyTarget(); + assertFalse(target.apply(context, record(1, "good", created())).duplicate()); + } + + @Test public void invalidCodecAndPhysicalMappingFailClosed() { + expectRejectedCreate(created().replace("\"amount\":\"0\"", "\"amount\":\"0.00000001\"")); + expectRejectedCreate(created().replace("\"amount\":\"0\"", "\"amount\":\"1234567890123\"")); + expectRejectedCreate(created().replace("\"occurred_at\":\"-315619199877\"", + "\"occurred_at\":\"2026-10-16T00:00:00Z\"")); + expectRejectedCreate(created().replace("\"planned_date\":\"2026-10-16\"", + "\"planned_date\":\"2026-10-16T00:00:00Z\"")); + expectRejectedCreate(created().replace("\"active\":true", "\"active\":\"true\"")); + expectRejectedCreate(created().replace("\"quantity\":\"2\"", "\"quantity\":2")); + expectRejectedCreate(created().replace("\"trace_chain\":[]", "\"trace_chain\":null")); + sql.execute("DROP TABLE order_item_data"); + sql.execute("ALTER TABLE order_data ALTER COLUMN amount SET DATA TYPE DECIMAL(18,2)"); + sql.execute("CREATE TABLE order_item_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_id BIGINT NOT NULL REFERENCES order_data(id), quantity BIGINT NOT NULL)"); + expectRejectedCreate(created()); + } + + @Test public void exactFractionalDecimalSurvivesDeclaredDuckDbStorage() { + String fractional = created().replace("\"amount\":\"0\"", "\"amount\":\"0.1\""); + assertFalse(target.apply(context, record(1, "fractional", fractional)).duplicate()); + BigDecimal actual = (BigDecimal) sql.queryForList("SELECT amount FROM order_data", + new Object[0]).get(0).get("amount"); + assertEquals(0, actual.compareTo(new BigDecimal("0.1"))); + } + + @Test public void zoneFreePhysicalTimestampCannotReceiveCommittedInstant() { + sql.execute("DROP TABLE order_item_data"); + sql.execute("DROP TABLE order_data"); + sql.execute("CREATE TABLE order_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_number VARCHAR NOT NULL, status VARCHAR NOT NULL, planned_date DATE, " + + "occurred_at TIMESTAMP NOT NULL, amount DECIMAL(19,7), active BOOLEAN NOT NULL, " + + "ratio DOUBLE NOT NULL, optional_note VARCHAR, optional_label VARCHAR)"); + sql.execute("CREATE TABLE order_item_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_id BIGINT NOT NULL REFERENCES order_data(id), quantity BIGINT NOT NULL)"); + expectRejectedCreate(created()); + } + + private void expectRejectedCreate(String mutations) { + expect("INVALID_ENVELOPE", () -> target.apply(context, record(1, "bad", mutations))); + assertEmptyTarget(); + } + + private void assertEmptyTarget() { + for (String table : List.of("order_data", "order_item_data", "teaql_event_inbox", + "teaql_event_applied_payload", "teaql_event_checkpoint")) { + assertEquals(table, 0, count(table)); + } + } + + private long count(String table) { + return ((Number) sql.queryForList("SELECT count(*) AS row_count FROM " + table, + new Object[0]).get(0).get("row_count")).longValue(); + } + + private static Record record(long sequence, String changeSetId, String mutations) { + String payload = """ + {"protocol_version":1,"change_set_id":"%s","source_runtime_id":"%s", + "source_sequence":"%d","source_model_hash":"%s","source_transaction_id":"tx-%d", + "committed_at":"2026-10-16T00:00:00Z", + "audit":{"actor_ref":"fixture:operator","comment":"Apply order graph", + "purpose":"Verify target application","trace_id":"fixture-trace"}, + "mutations":%s} + """.formatted(changeSetId, SOURCE, sequence, HASH, sequence, mutations); + return recordBody(sequence, changeSetId, payload); + } + + private static Record recordBody(long sequence, String changeSetId, String payload) { + try { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8))); + return new Record(SOURCE, sequence, changeSetId, HASH, digest, payload); + } catch (Exception impossible) { throw new AssertionError(impossible); } + } + + private static String created() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order_number":"ORD-0001","status":"DRAFT", + "planned_date":"2026-10-16","occurred_at":"-315619199877", + "amount":"0","active":true,"ratio":0.5,"optional_note":null, + "optional_label":"initial label"},"trace_chain":[]}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, + "quantity":"2"},"trace_chain":[]}] + """; + } + + private static String submitted() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"update", + "base_version":"1","committed_version":"2", + "fields":{"status":"SUBMITTED","optional_note":"submitted note", + "optional_label":null},"trace_chain":[]}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"delete", + "base_version":"1","committed_version":"-2","fields":{},"trace_chain":[]}] + """; + } + + private static void expect(String code, Runnable action) { + assertEquals(code, assertThrows(Failure.class, action::run).code()); + } +} diff --git a/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java b/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java index 00fae153..c0da7f26 100644 --- a/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java +++ b/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java @@ -276,6 +276,48 @@ public void supportsFacetWindowAndTransactionSql() throws Exception { } } + @Test + public void jdbcAdapterRollsBackDomainInboxAndCheckpointTogether() { + sql.execute("CREATE TABLE cmr_atomic_data (id BIGINT PRIMARY KEY, amount DECIMAL(19,7))"); + sql.execute("CREATE TABLE cmr_atomic_inbox (change_set_id VARCHAR PRIMARY KEY)"); + sql.execute("CREATE TABLE cmr_atomic_checkpoint (source_id VARCHAR PRIMARY KEY, sequence BIGINT)"); + + boolean failed = false; + try { + sql.executeInTransaction(() -> { + sql.update("INSERT INTO cmr_atomic_data VALUES (?, ?)", + new Object[] {1L, new BigDecimal("0.1000000")}); + sql.update("INSERT INTO cmr_atomic_inbox VALUES (?)", new Object[] {"set-1"}); + sql.update("INSERT INTO cmr_atomic_checkpoint VALUES (?, ?)", + new Object[] {"source-1", 1L}); + sql.update("INSERT INTO cmr_atomic_data VALUES (?, ?)", + new Object[] {1L, BigDecimal.ONE}); + }); + } catch (RuntimeException expected) { + failed = true; + } + assertTrue("duplicate row must fail the transaction", failed); + assertEquals(0L, rowCount("cmr_atomic_data")); + assertEquals(0L, rowCount("cmr_atomic_inbox")); + assertEquals(0L, rowCount("cmr_atomic_checkpoint")); + + sql.executeInTransaction(() -> { + sql.update("INSERT INTO cmr_atomic_data VALUES (?, ?)", + new Object[] {1L, new BigDecimal("0.1000000")}); + sql.update("INSERT INTO cmr_atomic_inbox VALUES (?)", new Object[] {"set-1"}); + sql.update("INSERT INTO cmr_atomic_checkpoint VALUES (?, ?)", + new Object[] {"source-1", 1L}); + }); + assertEquals(1L, rowCount("cmr_atomic_data")); + assertEquals(1L, rowCount("cmr_atomic_inbox")); + assertEquals(1L, rowCount("cmr_atomic_checkpoint")); + } + + private static long rowCount(String table) { + return ((Number) sql.queryForList("SELECT count(*) AS row_count FROM " + table, + new Object[0]).get(0).get("row_count")).longValue(); + } + @Test public void validatesSchemaValueDomainAndNullability() throws Exception { try { diff --git a/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java b/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java index 5e390dc8..0b66caf1 100644 --- a/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java +++ b/teaql-dynamic-fields-jdbc/src/main/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProvider.java @@ -101,6 +101,7 @@ public void ensureSchema() { } @Override public void validateGraphMutation(DynamicFieldContext context, DynamicGraphMutation request) { + executor.requireOrdinaryMutationAllowed(); if (!request.sourceMetadata().matchesStorage(requireStorageIdentity(), context.scopeType(), context.scopeId(), request.owner().ownerType())) { throw new DynamicFieldException("DYNAMIC_FIELD_STORAGE_PROVENANCE_MISMATCH", "Reload the dynamic view through the current storage provider and scope"); } @@ -133,6 +134,7 @@ private static long exactLong(Number value) { * Registers a field definition. Assigns an ID via teaql_id_space if not set. */ public DynamicFieldDef registerFieldDef(DynamicFieldContext context, DynamicFieldDef def) { + executor.requireOrdinaryMutationAllowed(); Objects.requireNonNull(def, "def"); if (def.getId() == 0) { def.setId(context.nextId("DynamicFieldDef")); @@ -252,6 +254,7 @@ public Map loadValues( @Override public void saveValue(DynamicFieldContext context, DynamicSetCommand command) { + executor.requireOrdinaryMutationAllowed(); // Look up the field def to get its id DynamicFieldRef ref = DynamicFieldRef.of( DynamicFieldScope.of(context.scopeType(), context.scopeId()), @@ -313,6 +316,7 @@ public void saveValue(DynamicFieldContext context, DynamicSetCommand command) { @Override public void deleteValue(DynamicFieldContext context, DynamicValueRef valueRef) { + executor.requireOrdinaryMutationAllowed(); // We need scope info from context executor.update(SQL_DELETE_VALUE, new Object[]{ context.scopeType(), context.scopeId(), diff --git a/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java b/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java index 456eb18f..e7b7b4f9 100644 --- a/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java +++ b/teaql-dynamic-fields-jdbc/src/test/java/io/teaql/data/dynamic/jdbc/JdbcDynamicFieldsProviderTest.java @@ -20,6 +20,23 @@ public class JdbcDynamicFieldsProviderTest { private JdbcDynamicFieldsProvider provider; + @Test + public void restrictedTargetCannotMutateDynamicValuesOrDefinitionsButCanRead() { + var executor=new JdbcSqlExecutor(new SimpleDataSource("jdbc:h2:mem:readonly_"+System.nanoTime()+";DB_CLOSE_DELAY=-1")); + var target=new JdbcDynamicFieldsProvider(executor);target.ensureSchema();var context=globalCtx(); + var definition=target.registerFieldDef(context,createStringField("note","Note"));var owner=DynamicOwnerRef.of("Platform",1L); + target.saveValue(context,DynamicSetCommand.of(owner,"note",DynamicDataType.STRING,"original","verify","create control")); + executor.restrictOrdinaryMutations(); + for(Runnable action:List.of( + ()->target.saveValue(context,DynamicSetCommand.of(owner,"note",DynamicDataType.STRING,"changed","verify","attempt restricted save")), + ()->target.deleteValue(context,DynamicValueRef.of(owner,definition.getId())), + ()->target.registerFieldDef(context,createStringField("other","Other")))) { + var error=assertThrows(RuntimeException.class,()->action.run());assertTrue(error.toString(),error.toString().contains("READ_ONLY_TARGET")); + } + assertEquals("original",target.loadValues(context,owner,new DynamicFieldSelection().selectString("note")).getString("note")); + System.out.println("CMR15_DYNAMIC_READ_ONLY_PASS language=java save=1 delete=1 definition=1 read=1"); + } + @Test public void newEntityMetadataNeedsNoOwnerDataReadAndBindsToExecutorNotProvider() { DataSource dataSource = new SimpleDataSource("jdbc:h2:mem:metadata_" + System.nanoTime() + ";DB_CLOSE_DELAY=-1"); diff --git a/teaql-object-storage-s3/src/test/java/io/teaql/storage/s3/SignedSdkCoexistenceTest.java b/teaql-object-storage-s3/src/test/java/io/teaql/storage/s3/SignedSdkCoexistenceTest.java new file mode 100644 index 00000000..fded1c35 --- /dev/null +++ b/teaql-object-storage-s3/src/test/java/io/teaql/storage/s3/SignedSdkCoexistenceTest.java @@ -0,0 +1,91 @@ +package io.teaql.storage.s3; + +import com.sun.net.httpserver.HttpServer; +import io.teaql.core.http.*; +import io.teaql.core.storage.*; +import io.teaql.runtime.DefaultUserContext; +import java.net.*; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; +import static org.junit.Assert.*; +import software.amazon.awssdk.auth.credentials.*; +import software.amazon.awssdk.core.interceptor.*; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.*; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; + +public class SignedSdkCoexistenceTest { + record Snapshot(String method, String target, Map headers) {} + + @Test public void signedSdkTransportDoesNotUseContextHttpOrChangeSignedFields() throws Exception { + var signed = new CopyOnWriteArrayList(); + var received = new CopyOnWriteArrayList(); + var status = new AtomicInteger(200); + var runtimeCalls = new AtomicInteger(); + HttpRequestService trap = (context, call) -> { + runtimeCalls.incrementAndGet(); + throw new HttpRequestFailure(HttpRequestFailure.Code.PROFILE_NOT_CONFIGURED); + }; + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + var headers = new TreeMap(); + exchange.getRequestHeaders().forEach((key, values) -> headers.put(key.toLowerCase(Locale.ROOT), String.join(",", values))); + received.add(new Snapshot(exchange.getRequestMethod(), exchange.getRequestURI().toString(), headers)); + exchange.getResponseHeaders().set("Content-Length", "0"); + exchange.getResponseHeaders().set("ETag", "\"sdk-proof-etag\""); + exchange.sendResponseHeaders(status.get(), -1); + exchange.close(); + }); + server.start(); + URI endpoint = URI.create("http://127.0.0.1:" + server.getAddress().getPort()); + var credentials = StaticCredentialsProvider.create(AwsBasicCredentials.create("sdk-proof-access", "sdk-proof-secret")); + var configuration = S3Configuration.builder().pathStyleAccessEnabled(true).build(); + var capture = new ExecutionInterceptor() { + @Override public void beforeTransmission(Context.BeforeTransmission context, ExecutionAttributes attributes) { + var request = context.httpRequest(); + var headers = new TreeMap(); + request.headers().forEach((key, values) -> headers.put(key.toLowerCase(Locale.ROOT), String.join(",", values))); + headers.putIfAbsent("host", request.getUri().getRawAuthority()); + String target = request.getUri().getRawPath(); + if (request.getUri().getRawQuery() != null) target += "?" + request.getUri().getRawQuery(); + signed.add(new Snapshot(request.method().name(), target, headers)); + } + }; + try (var provider = new S3ObjectStorage(S3Client.builder().endpointOverride(endpoint).region(Region.US_EAST_1) + .credentialsProvider(credentials).serviceConfiguration(configuration) + .overrideConfiguration(b -> b.retryPolicy(RetryPolicy.none()).apiCallTimeout(Duration.ofSeconds(3)).addExecutionInterceptor(capture)).build(), + S3Presigner.builder().endpointOverride(endpoint).region(Region.US_EAST_1).credentialsProvider(credentials).serviceConfiguration(configuration).build(), "bucket")) { + var context = new DefaultUserContext(null); + context.putAttribute(ObjectStorageProvider.class.getName(), provider); + context.putAttribute(HttpRequestService.class.getName(), trap); + context.putAttribute(HttpPropagation.class.getName(), (HttpPropagation) carrier -> carrier.put("traceparent", "runtime-http-must-not-inject-this")); + assertSame(trap, context.httpRequests()); + assertThrows(HttpRequestFailure.class, () -> context.httpRequests().execute(context, + HttpRequestService.Call.get("trap", "control", "check SPI installation", "verify SDK isolation"))); + assertEquals(1, runtimeCalls.get()); + var storage = context.objectStorage("verify signed SDK isolation"); + var key = new ObjectStorage.Key("sdk-proof"); + assertTrue(storage.exists(key)); + status.set(503); + assertEquals(StorageException.Code.UNAVAILABLE, assertThrows(StorageException.class, () -> storage.exists(key)).code()); + assertEquals("runtime HTTP must not wrap or retry either SDK call", 1, runtimeCalls.get()); + assertEquals(2, signed.size()); assertEquals(2, received.size()); + for (int i = 0; i < 2; i++) { + var before = signed.get(i); var wire = received.get(i); + assertEquals(before.method(), wire.method()); assertEquals(before.target(), wire.target()); + String authorization = before.headers().get("authorization"); + assertNotNull(authorization); assertTrue(authorization.startsWith("AWS4-HMAC-SHA256 ")); + assertTrue("authorization changed after SDK signing", authorization.equals(wire.headers().get("authorization"))); + String names = authorization.split("SignedHeaders=", 2)[1].split(",", 2)[0]; + for (String name : names.split(";")) assertTrue("SDK signed header changed: " + name, + Objects.equals(before.headers().get(name), wire.headers().get(name))); + assertFalse(wire.headers().containsKey("traceparent")); + } + System.out.println("HTTP_SDK_COEXISTENCE_PASS language=java control_calls=1 runtime_sdk_calls=0 sdk_attempts=2 wire_requests=2 signatures=2"); + } finally { server.stop(0); } + } +} diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index c433a51e..71a9f6d2 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -1,5 +1,6 @@ package io.teaql.provider.jdbc; +import io.teaql.core.TransactionOutcomeUnknownException; import io.teaql.dataservice.sql.SqlExecutionAdapter; import io.teaql.dataservice.sql.SqlRowMapper; @@ -14,13 +15,18 @@ import java.util.stream.Stream; import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Consumer; +import java.util.logging.Logger; public class JdbcSqlExecutor implements SqlExecutionAdapter { + private static final Logger LOG = Logger.getLogger(JdbcSqlExecutor.class.getName()); + private final DataSource dataSource; + private volatile boolean ordinaryMutationsReadOnly; // A held view retains only this token, not the executor, connection or DataSource. private final Object storageIdentity = new Object(); private final ThreadLocal transactionConnection = new ThreadLocal<>(); + private final ThreadLocal transactionRollbackOnly = new ThreadLocal<>(); private final List> connectionInitializers = new CopyOnWriteArrayList<>(); public JdbcSqlExecutor(DataSource dataSource) { @@ -30,6 +36,13 @@ public JdbcSqlExecutor(DataSource dataSource) { @Override public Object storageIdentity() { return storageIdentity; } + @Override public void restrictOrdinaryMutations() { ordinaryMutationsReadOnly = true; } + + @Override public void requireOrdinaryMutationAllowed() { + if (ordinaryMutationsReadOnly) throw new io.teaql.core.TeaQLRuntimeException( + "READ_ONLY_TARGET: ordinary mutations are disabled on this replication target"); + } + public JdbcSqlExecutor addConnectionInitializer(Consumer initializer) { connectionInitializers.add(initializer); return this; @@ -183,7 +196,11 @@ public List> queryForList(String sql, Object[] params) { return result; } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException("JDBC query failed for parameterized SQL: " + sql, e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -209,7 +226,11 @@ public List query( } return result; } catch (SQLException exception) { + markTransactionRollbackOnly(); throw new RuntimeException("JDBC typed query failed for parameterized SQL: " + sql, exception); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -240,6 +261,13 @@ public V get(int columnIndex, Class type) { private static V convertColumn(Class type, Object value) { if (value == null || type.isInstance(value)) return (V) value; if (value instanceof Number number) { + if (type == java.time.Instant.class) { + if (!(number instanceof Long || number instanceof Integer + || number instanceof Short || number instanceof Byte)) { + throw new IllegalArgumentException("Instant JDBC value is not an exact integer"); + } + return (V) java.time.Instant.ofEpochMilli(number.longValue()); + } if (type == Long.class) return (V) Long.valueOf(number.longValue()); if (type == Integer.class) return (V) Integer.valueOf(number.intValue()); if (type == Double.class) return (V) Double.valueOf(number.doubleValue()); @@ -256,6 +284,12 @@ private static V convertColumn(Class type, Object value) { if (type == java.time.LocalDateTime.class && value instanceof java.sql.Timestamp timestamp) { return (V) timestamp.toLocalDateTime(); } + if (type == java.time.Instant.class && value instanceof java.sql.Timestamp timestamp) { + return (V) timestamp.toInstant(); + } + if (type == java.time.Instant.class && value instanceof CharSequence text) { + return (V) java.time.Instant.parse(text); + } if (type == java.time.LocalDateTime.class) { return (V) java.time.LocalDateTime.parse(String.valueOf(value).replace(' ', 'T')); } @@ -325,7 +359,11 @@ public void execute(String sql) { ps.execute(); } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException(e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -350,7 +388,11 @@ public int update(String sql, Object[] params) { return ps.executeUpdate(); } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException(e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -373,7 +415,11 @@ public int[] batchUpdate(String sql, List paramsList) { return ps.executeBatch(); } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException(e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -382,28 +428,154 @@ public int[] batchUpdate(String sql, List paramsList) { @Override public void executeInTransaction(Runnable action) { if (transactionConnection.get() != null) { - action.run(); + try { + action.run(); + } catch (RuntimeException | Error failure) { + // The nested action shares the outer JDBC transaction. Without a + // savepoint, its partial writes cannot be committed safely even if + // application code catches the exception before the outer returns. + transactionRollbackOnly.set(true); + throw failure; + } return; } - try (Connection connection = openConnection()) { + Connection connection; + try { + connection = openConnection(); + } catch (SQLException failure) { + throw new RuntimeException("JDBC transaction failed", failure); + } + boolean committed = false; + boolean outcomeUnknown = false; + boolean rollbackUnconfirmed = false; + try { boolean autoCommit = connection.getAutoCommit(); connection.setAutoCommit(false); transactionConnection.set(connection); + transactionRollbackOnly.set(false); try { - action.run(); - connection.commit(); - } catch (RuntimeException | Error failure) { - connection.rollback(); - throw failure; + try { + action.run(); + if (Boolean.TRUE.equals(transactionRollbackOnly.get())) { + throw new IllegalStateException( + "JDBC transaction is rollback-only after a failed operation"); + } + } catch (RuntimeException | Error failure) { + try { + connection.rollback(); + } catch (SQLException | RuntimeException | Error rollbackFailure) { + outcomeUnknown = true; + rollbackUnconfirmed = true; + warnTransactionCleanup("rollback after failed action", false, true); + if (failure instanceof Error) { + // The graph runtime must fence the ledger before + // rethrowing the original Error. A suppressed marker + // is unreliable because Error may disable suppression. + throw new TransactionOutcomeUnknownException((Error) failure); + } + if (rollbackFailure instanceof Error) { + throw new TransactionOutcomeUnknownException((Error) rollbackFailure); + } + throw new TransactionOutcomeUnknownException(); + } + throw failure; + } + try { + connection.commit(); + committed = true; + } catch (SQLException | RuntimeException | Error commitFailure) { + // The database may have committed before the driver lost the + // acknowledgement. Conversely, restoring autoCommit without a + // rollback can commit a transaction the driver never attempted. + outcomeUnknown = true; + warnTransactionCleanup("commit confirmation", false, true); + try { + connection.rollback(); + } catch (SQLException | RuntimeException | Error rollbackFailure) { + rollbackUnconfirmed = true; + warnTransactionCleanup("rollback after unconfirmed commit", false, true); + } + throw commitFailure instanceof Error + ? new TransactionOutcomeUnknownException((Error) commitFailure) + : new TransactionOutcomeUnknownException(); + } } finally { transactionConnection.remove(); - connection.setAutoCommit(autoCommit); + transactionRollbackOnly.remove(); + // A confirmed rollback leaves no pending write to commit. If + // rollback itself failed, changing autoCommit could commit it. + if (!rollbackUnconfirmed) { + try { + connection.setAutoCommit(autoCommit); + } catch (SQLException | RuntimeException | Error cleanupFailure) { + // A successful commit is already durable. A cleanup error + // must not make the caller replay that business mutation. + warnTransactionCleanup("autoCommit restoration", committed, outcomeUnknown); + } + } } } catch (SQLException e) { throw new RuntimeException("JDBC transaction failed", e); + } finally { + if (rollbackUnconfirmed) { + // JDBC leaves close() with an active transaction implementation-defined. + // A pool may even commit when close() returns the connection, so never + // return an unconfirmed transaction to it. A one-shot rollback fault + // can recover on retry, allowing a confirmed rollback before close. + boolean rollbackConfirmed = false; + try { + connection.rollback(); + rollbackConfirmed = true; + } catch (SQLException | RuntimeException | Error cleanupFailure) { + warnTransactionCleanup("rollback retry before connection disposal", committed, outcomeUnknown); + } + if (rollbackConfirmed) { + try { + connection.close(); + } catch (SQLException | RuntimeException | Error cleanupFailure) { + warnTransactionCleanup("connection close after confirmed rollback", committed, outcomeUnknown); + } + } else { + try { + connection.abort(Runnable::run); + // Some drivers incorrectly implement abort() as a no-op. + if (!connection.isClosed()) { + warnTransactionCleanup("connection abort completion", committed, outcomeUnknown); + } + } catch (SQLException | RuntimeException | Error cleanupFailure) { + // Do not fall back to close(): correctness takes precedence + // over reclaiming a still-active transaction. + warnTransactionCleanup("connection abort", committed, outcomeUnknown); + } + } + } else { + try { + connection.close(); + } catch (SQLException | RuntimeException | Error cleanupFailure) { + warnTransactionCleanup("connection close", committed, outcomeUnknown); + } + } } } + private static void warnTransactionCleanup(String operation, boolean committed, boolean outcomeUnknown) { + try { + String state = committed ? "committed" : outcomeUnknown ? "outcome unknown" : "unsuccessful"; + LOG.warning("JDBC transaction " + state + "; " + operation + " failed"); + } catch (RuntimeException | Error ignored) { + // A user-installed logging handler must not turn a committed write + // back into an apparently retryable transaction failure, even if it + // throws an Error with suppressed diagnostics disabled. + } + } + + private void markTransactionRollbackOnly() { + // A failed statement, mapper or adapter operation may have left earlier + // writes in the shared transaction. Without a savepoint, catching the + // failure cannot make the transaction safe to commit. + if (transactionConnection.get() != null) transactionRollbackOnly.set(true); + } + @Override public boolean hasActiveTransaction() { return transactionConnection.get() != null; } static void bind(PreparedStatement statement, int index, Object value) throws SQLException { @@ -442,6 +614,14 @@ static void bind(PreparedStatement statement, int index, Object value) throws SQ } else { statement.setTimestamp(index, java.sql.Timestamp.valueOf(dateTime)); } + } else if (value instanceof java.time.Instant instant && isSqlite(statement)) { + if (instant.getNano() % 1_000_000 != 0) { + throw new SQLException("Instant has precision finer than milliseconds"); + } + try { statement.setLong(index, instant.toEpochMilli()); } + catch (ArithmeticException overflow) { + throw new SQLException("Instant exceeds epoch-millisecond range", overflow); + } } else if (value instanceof java.time.LocalTime time) { if (isSqlite(statement)) { statement.setString(index, time.toString()); diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java index 7e311267..885414a1 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java @@ -18,6 +18,7 @@ import java.util.logging.Logger; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class JdbcSqlExecutorTest { @@ -62,6 +63,96 @@ public void testExecuteAndInsert() throws Exception { } } + @Test + public void nestedFailureCaughtByCallerStillRollsBackOuterTransaction() { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {91, "before failure", 20}); + try { + sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {92, "failed save", 20}); + throw new IllegalStateException("simulated outbox append failure"); + }); + } catch (IllegalStateException expected) { + // Application code may log a failed save and continue its outer callback. + } + })); + + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + + sqlExecutor.executeInTransaction(() -> sqlExecutor.executeInTransaction(() -> + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {93, "next transaction", 20}))); + assertEquals(1, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + + @Test + public void caughtJdbcWriteFailureStillRollsBackOuterTransaction() { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {94, "first write", 20}); + try { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {94, "duplicate key", 21}); + } catch (RuntimeException expected) { + // A caller may handle a low-level JDBC failure before returning. + } + })); + + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + + @Test + public void caughtStatementOrReadFailureCannotCommitEarlierWrites() { + List failures = List.of( + () -> sqlExecutor.execute("INSERT INTO test_user (id, name, age) VALUES (94, 'duplicate', 21)"), + () -> sqlExecutor.batchUpdate( + "INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + java.util.Collections.singletonList(new Object[] {94, "duplicate", 21})), + () -> sqlExecutor.queryForList("SELECT * FROM missing_test_table", new Object[0]), + () -> sqlExecutor.query("SELECT * FROM missing_test_table", new Object[0], + row -> new TypedRow())); + for (Runnable failedStatement : failures) { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {94, "must roll back", 20}); + try { + failedStatement.run(); + } catch (RuntimeException expected) { + // The failed statement must poison the shared transaction. + } + })); + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + } + + @Test + public void caughtTypedRowMapperFailureCannotCommitEarlierWrites() { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {95, "must roll back", 20}); + try { + sqlExecutor.query("SELECT id FROM test_user WHERE id = ?", new Object[] {95}, + row -> { throw new IllegalArgumentException("bad typed row"); }); + } catch (IllegalArgumentException expected) { + // The mapper failure escaped a query using the same transaction. + } + })); + + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + @Test public void testQueryForListNormalizesAliasesForPortableMapping() { List> rows = sqlExecutor.queryForList( diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/AuditDeliveryAfterCommitException.java b/teaql-runtime/src/main/java/io/teaql/runtime/AuditDeliveryAfterCommitException.java new file mode 100644 index 00000000..65245433 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/AuditDeliveryAfterCommitException.java @@ -0,0 +1,19 @@ +package io.teaql.runtime; + +import io.teaql.core.TeaQLRuntimeException; + +/** The graph committed, but one or more post-commit audit deliveries failed. */ +public final class AuditDeliveryAfterCommitException extends TeaQLRuntimeException { + private final int failureCount; + + public AuditDeliveryAfterCommitException(int failureCount) { + super("AUDIT_DELIVERY_AFTER_COMMIT: transaction committed but audit delivery failed; " + + "do not retry the mutation blindly"); + if (failureCount < 1) throw new IllegalArgumentException("failureCount must be positive"); + this.failureCount = failureCount; + } + + public int failureCount() { + return failureCount; + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationFact.java b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationFact.java new file mode 100644 index 00000000..a8ac2cc9 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationFact.java @@ -0,0 +1,29 @@ +package io.teaql.runtime; + +import io.teaql.core.TraceNode; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** One immutable, model-named fact frozen from a provider readback before commit. */ +public record CommittedMutationFact( + String entityType, String entityId, String operation, + String baseVersion, String committedVersion, + Map fields, List traceChain) { + + public CommittedMutationFact { + Objects.requireNonNull(entityType, "entityType"); + Objects.requireNonNull(entityId, "entityId"); + Objects.requireNonNull(operation, "operation"); + Objects.requireNonNull(committedVersion, "committedVersion"); + fields = Collections.unmodifiableMap(new LinkedHashMap<>(Objects.requireNonNull(fields, "fields"))); + traceChain = List.copyOf(Objects.requireNonNull(traceChain, "traceChain")); + } + + @Override public String toString() { + return "CommittedMutationFact[entityType=" + entityType + + ", operation=" + operation + ", fields=, traceChain=]"; + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationRecorder.java b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationRecorder.java new file mode 100644 index 00000000..2592608e --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationRecorder.java @@ -0,0 +1,16 @@ +package io.teaql.runtime; + +import io.teaql.core.MutationIntent; +import io.teaql.core.UserContext; +import java.util.List; + +/** + * Optional, trusted source-side Outbox capability. Called exactly once for a + * changed graph while the graph's provider transaction is still open. A + * failure must abort that transaction; delivery is never a post-commit hook. + */ +@FunctionalInterface +public interface CommittedMutationRecorder { + void record(UserContext context, Object transactionResource, + MutationIntent intent, List facts); +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/CommittedStateReconciliationException.java b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedStateReconciliationException.java new file mode 100644 index 00000000..39f57f21 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedStateReconciliationException.java @@ -0,0 +1,11 @@ +package io.teaql.runtime; + +import io.teaql.core.TeaQLRuntimeException; + +/** The provider write completed, but the in-memory graph requires a fresh load. */ +public final class CommittedStateReconciliationException extends TeaQLRuntimeException { + public CommittedStateReconciliationException() { + super("COMMITTED_STATE_RELOAD_REQUIRED: provider write completed but entity reconciliation failed; " + + "reload the graph and do not retry this instance"); + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java index 785c2558..b672a606 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java @@ -145,6 +145,9 @@ public T capability(Class capabilityType) { return null; } Object value = getObj(capabilityType.getName()); + if (value == null && capabilityType == RuntimeTelemetry.class) { + value = runtime == null ? RuntimeTelemetry.NOOP : runtime.getTelemetry(); + } if (value == null && capabilityType == SchemaExecutor.class) { value = runtime.getSchemaExecutor(); if (value == null) { @@ -172,6 +175,9 @@ public T capability(Class capabilityType) { if (value == null && capabilityType == BusinessIdSchemaContributor.class) { value = runtime.getBusinessIdSchemaContributor(); } + if (value == null && capabilityType == CommittedEventSchemaContributor.class) { + value = runtime.getCommittedEventSchemaContributor(); + } if (value == null && capabilityType == BusinessIdKeyProvider.class) { value = runtime.getBusinessIdKeyProvider(); } diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 88d60e55..02d6504d 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -4,6 +4,7 @@ import io.teaql.core.meta.EntityDescriptor; import io.teaql.core.meta.EntityMetaFactory; import io.teaql.core.meta.PropertyDescriptor; +import io.teaql.core.meta.Relation; import io.teaql.core.checker.CheckException; import io.teaql.core.checker.CheckResult; import io.teaql.core.checker.Checker; @@ -38,6 +39,7 @@ public class TeaQLRuntime implements AutoCloseable { private final BusinessIdKeyProvider businessIdKeyProvider; private final RoundTripReferenceCodec roundTripReferenceCodec; private final BusinessClock businessClock; + private final CommittedMutationRecorder committedMutationRecorder; private final Map> checkers = new java.util.concurrent.ConcurrentHashMap<>(); private final List generatedBootstraps = new java.util.concurrent.CopyOnWriteArrayList<>(); @@ -65,6 +67,7 @@ private TeaQLRuntime(Builder builder) { ? null : new RoundTripReferenceCodec(builder.roundTripReferenceProvider); this.businessClock = builder.businessClock; + this.committedMutationRecorder = builder.committedMutationRecorder; } public static Builder builder() { @@ -139,6 +142,11 @@ BusinessIdSchemaContributor getBusinessIdSchemaContributor() { return businessIdSchemaContributor; } + CommittedEventSchemaContributor getCommittedEventSchemaContributor() { + return committedMutationRecorder instanceof CommittedEventSchemaContributor contributor + ? contributor : null; + } + BusinessIdKeyProvider getBusinessIdKeyProvider() { return businessIdKeyProvider; } @@ -449,9 +457,10 @@ public void saveGraph(UserContext context, Entity entity) { "teaql.mutation.kind", "save"))); try { MutationIntent intent = MutationIntent.of(entity.getComment()); - checkAndFix(context, entity); // Get entity's own EntityMutationLedger EntityMutationLedger entityMutationLedger = ((BaseEntity) entity).getEntityMutationLedger(); + entityMutationLedger.requireSaveable(); + checkAndFix(context, entity); // Allocate identifiers before roots are merged. New children commonly // receive their field updates while their id is still null, so those @@ -503,6 +512,10 @@ public void saveGraph(UserContext context, Entity entity) { requireSingleMutationRoute(mutationPlan, route); MutationGovernanceSnapshot governance = reviewMutationPlan(context, mutationPlan); var dynamicChanges = entityMutationLedger.currentChangeSet().dynamicChanges(); + if (committedMutationRecorder != null && !dynamicChanges.isEmpty()) { + throw new TeaQLRuntimeException( + "Committed mutation capture cannot yet encode dynamic field changes"); + } var dynamicRequests = new ArrayList(); dynamicChanges.forEach((key, mutations) -> { if (!entityMutationLedger.deletedKeys().contains(key)) mutations.values().forEach(mutation -> dynamicRequests.add( @@ -514,6 +527,9 @@ public void saveGraph(UserContext context, Entity entity) { try { if (mutationExecutor instanceof TransactionExecutor transactionExecutor) { completed = transactionExecutor.executeInTransaction(context, () -> { + Map> committedPreimages = + committedMutationRecorder == null ? Map.of() + : captureCommittedPreimages(context, mutationExecutor, mutationPlan); io.teaql.data.dynamic.DynamicFieldsFacade dynamic = null; if (!dynamicRequests.isEmpty()) { dynamic = context.dynamicFields().comment(intent.comment()).purpose("runtime: persist audited graph extensions"); @@ -536,13 +552,26 @@ public void saveGraph(UserContext context, Entity entity) { dynamicReadbacks.put(entry.getKey(), value); } } + if (committedMutationRecorder != null && !results.isEmpty()) { + committedMutationRecorder.record(context, transactionExecutor.transactionResource(), + intent, freezeCommittedFacts(results, committedPreimages)); + } return results; }); } else { + if (committedMutationRecorder != null) { + throw new TeaQLRuntimeException("Committed mutation capture requires one provider transaction"); + } if (!dynamicRequests.isEmpty()) throw new TeaQLRuntimeException("DYNAMIC_FIELD_TRANSACTION_BINDING_REQUIRED: native graph executor has no transaction"); completed = executeLedgerPlan( context, entityMutationLedger, mutationExecutor, realEntities, governance, intent, traceScopes, graphScope); } + } catch (TransactionOutcomeUnknownException failure) { + // The JDBC commit acknowledgement can fail after durable commit. + // Restoring the old version would make this graph appear retryable. + entityMutationLedger.markTransactionOutcomeUnknownRequiresReload(); + if (failure.originalError() != null) throw failure.originalError(); + throw failure; } catch (RuntimeException | Error failure) { restoreGraphPersistenceState( entity, @@ -551,12 +580,30 @@ public void saveGraph(UserContext context, Entity entity) { Collections.newSetFromMap(new IdentityHashMap<>())); throw failure; } - dynamicReadbacks.forEach((key, values) -> { - BaseEntity target = realEntities.get(key); - target.__internalHydrateDynamicFields(values, target.__internalLoadState().withDynamicSelection(values.selectedCodes())); - }); - completeLedgerPlan(context, completed); - entityMutationLedger.clearCurrentChangeSet(); + CommittedStateReconciliationException reconciliationOutcome = null; + try { + dynamicReadbacks.forEach((key, values) -> { + BaseEntity target = realEntities.get(key); + target.__internalHydrateDynamicFields(values, target.__internalLoadState().withDynamicSelection(values.selectedCodes())); + }); + completeLedgerPlan(completed); + entityMutationLedger.clearCurrentChangeSet(); + } catch (RuntimeException reconciliationFailure) { + // The provider has returned from its transaction. The graph may be + // partially reconciled, so neither its pending ledger nor its + // entity instances are safe to save again without a fresh load. + entityMutationLedger.markCommittedStateRequiresReload(); + reconciliationOutcome = new CommittedStateReconciliationException(); + } + try { + // The provider has already committed. A failed in-memory readback + // must not silently suppress the independent audit delivery. + deliverCommittedAuditEvents(context, completed); + } catch (AuditDeliveryAfterCommitException auditFailure) { + if (reconciliationOutcome == null) throw auditFailure; + reconciliationOutcome.addSuppressed(auditFailure); + } + if (reconciliationOutcome != null) throw reconciliationOutcome; telemetryScope.success(); } catch (RuntimeException | Error error) { telemetryScope.failure(error); @@ -939,6 +986,7 @@ private List executeLedgerPlan( for (var batch : deleteBatches.entrySet()) { List requests = new ArrayList<>(); List targets = new ArrayList<>(); + List predecessorVersions = new ArrayList<>(); EntityDescriptor descriptor = metadata.resolveEntityDescriptor(batch.getKey()); if (descriptor == null) throw new TeaQLRuntimeException("No entity descriptor for: " + batch.getKey()); for (EntityKey key : batch.getValue()) { @@ -956,11 +1004,13 @@ private List executeLedgerPlan( deleteEntity, EntityPersistenceMutation.Action.DELETE, intent, mutationTrace(root, key, traceScopes, graphScope), target, graphRedactions); requests.add(mutationRequest); targets.add(target == null ? deleteEntity : target); + predecessorVersions.add(originalVersion); } List results = mutateBatchWithTelemetry( context, mutationExecutor, intent, requests, batch.getKey(), "delete"); for (int index = 0; index < requests.size(); index++) { completed.add(new PendingMutation(descriptor, targets.get(index), results.get(index), + predecessorVersions.get(index), MutationAuditKind.DELETED, Collections.emptyMap(), governance, intent, requests.get(index).getTraceChain(), graphRedactions)); } } @@ -1019,7 +1069,7 @@ private List executeLedgerPlan( context, mutationExecutor, intent, requests, entityName, "save"); for (int index = 0; index < requests.size(); index++) { completed.add(new PendingMutation( - descriptor, targets.get(index), results.get(index), + descriptor, targets.get(index), results.get(index), null, MutationAuditKind.CREATED, snapshots.get(index), governance, intent, requests.get(index).getTraceChain(), graphRedactions)); } } @@ -1037,6 +1087,7 @@ private List executeLedgerPlan( for (boolean recovering : List.of(false, true)) { List requests = new ArrayList<>(); List targets = new ArrayList<>(); + List predecessorVersions = new ArrayList<>(); List> snapshots = new ArrayList<>(); for (EntityKey key : keys) { if (root.recoveredKeys().contains(key) != recovering) continue; @@ -1060,6 +1111,7 @@ private List executeLedgerPlan( entity, EntityPersistenceMutation.Action.SAVE, intent, mutationTrace(root, key, traceScopes, graphScope), target, graphRedactions); requests.add(mutationRequest); targets.add(target == null ? entity : target); + predecessorVersions.add(version); snapshots.add(snapshotChanges(changesWithDynamic(changeSet, key))); } MutationAuditKind auditKind = recovering ? MutationAuditKind.RECOVERED : MutationAuditKind.UPDATED; @@ -1067,6 +1119,7 @@ private List executeLedgerPlan( context, mutationExecutor, intent, requests, entityName, auditKind.name().toLowerCase(Locale.ROOT)); for (int index = 0; index < requests.size(); index++) { completed.add(new PendingMutation(descriptor, targets.get(index), results.get(index), + predecessorVersions.get(index), auditKind, snapshots.get(index), governance, intent, requests.get(index).getTraceChain(), graphRedactions)); } } @@ -1074,18 +1127,34 @@ private List executeLedgerPlan( return completed; } - private void completeLedgerPlan(UserContext context, List completed) { + private void completeLedgerPlan(List completed) { for (PendingMutation mutation : completed) { applyPersistedEntity(mutation.descriptor(), mutation.target(), mutation.result()); - emitAuditEvent( - context, mutation.target(), mutation.auditKind(), mutation.changedValues(), - mutation.governance(), mutation.intent(), mutation.traceChain(), mutation.redactions()); mutation.target().clearUpdatedProperties(); - // A committed delete supersedes any staged extension edits too. - mutation.target().__internalClearDynamicMutations(); } } + private void deliverCommittedAuditEvents(UserContext context, List completed) { + int failures = 0; + for (PendingMutation mutation : completed) { + try { + failures += emitAuditEvent( + context, mutation.target(), mutation.auditKind(), mutation.changedValues(), + mutation.governance(), mutation.intent(), mutation.traceChain(), mutation.redactions(), + mutation.result().persistedEntity().getVersion()); + } catch (RuntimeException failure) { + // The provider transaction has already committed. Continue delivery + // for the other graph members and never expose a retryable save error. + failures++; + } finally { + // Preserve dynamic original values until the audit projection has + // observed them; then clear staged extension edits even on failure. + mutation.target().__internalClearDynamicMutations(); + } + } + if (failures != 0) throw new AuditDeliveryAfterCommitException(failures); + } + private Map snapshotChanges(Map changes) { return Collections.unmodifiableMap(new LinkedHashMap<>(changes)); } @@ -1109,11 +1178,175 @@ private record PendingMutation( EntityDescriptor descriptor, BaseEntity target, MutationResult result, + Long baseVersion, MutationAuditKind auditKind, Map changedValues, MutationGovernanceSnapshot governance, MutationIntent intent, List traceChain, SqlIntentRedactions redactions) {} + private Map> captureCommittedPreimages( + UserContext context, MutationExecutor mutationExecutor, MutationPlan plan) { + if (!(mutationExecutor instanceof QueryExecutor queryExecutor)) { + throw new TeaQLRuntimeException( + "Committed update capture requires query access on the same transaction executor"); + } + Map> preimages = new TreeMap<>(); + for (MutationOperation operation : plan.operations()) { + if (operation.kind() != MutationOperationKind.UPDATE + && operation.kind() != MutationOperationKind.RECOVER) continue; + EntityKey key = operation.entity(); + EntityDescriptor descriptor = metadata.resolveEntityDescriptor(key.entity()); + if (descriptor == null || descriptor.getTargetType() == null) { + throw new TeaQLRuntimeException("Committed preimage has no entity metadata: " + key.entity()); + } + BaseRequest.TempRequest request = new BaseRequest.TempRequest( + descriptor.getTargetType(), descriptor.getType(), descriptor::createEntity); + request.bindMetadata(metadata); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getOwnProperties()) { + request.selectProperty(property.getName()); + } + } + request.appendSearchCriteria(request.createBasicSearchCriteria( + BaseEntity.ID_PROPERTY, io.teaql.core.criteria.Operator.EQUAL, key.id())); + // Recovery reads a negative-version tombstone. Explicit version + // intent suppresses the normal visible-row default in SQL providers. + request.appendSearchCriteria(request.createBasicSearchCriteria( + BaseEntity.VERSION_PROPERTY, io.teaql.core.criteria.Operator.IS_NOT_NULL)); + request.top(2); + QueryIntent intent = QueryIntent.of( + "capture committed mutation preimage", "compare authoritative pre/post write fields"); + QueryResult result = queryExecutor.query(context, new DefaultQueryRequest(request, intent)); + if (!(result instanceof DefaultQueryResult rows) || rows.getResult().size() != 1 + || !(rows.getResult().get(0) instanceof BaseEntity persisted)) { + throw new TeaQLRuntimeException("Committed mutation preimage must contain one row: " + key.entity()); + } + Map fields = new LinkedHashMap<>(); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getOwnProperties()) { + String name = property.getName(); + if (property.isId() || property.isVersion() || fields.containsKey(name)) continue; + if (!persisted.isPropertyLoaded(name)) { + throw new TeaQLRuntimeException("Committed preimage omitted modeled field " + name); + } + fields.put(name, committedFieldValue(property, persisted.getProperty(name))); + } + } + preimages.put(key, fields); + } + return preimages; + } + + private List freezeCommittedFacts( + List completed, Map> preimages) { + List facts = new ArrayList<>(completed.size()); + for (PendingMutation pending : completed) { + if (pending.result() == null || !(pending.result().persistedEntity() instanceof BaseEntity persisted) + || persisted.getId() == null || persisted.getVersion() == null) { + throw new TeaQLRuntimeException("Committed mutation has no authoritative provider readback"); + } + boolean created = pending.auditKind() == MutationAuditKind.CREATED; + boolean deleted = pending.auditKind() == MutationAuditKind.DELETED; + Long baseVersion = pending.baseVersion(); + if (!created && baseVersion == null) { + throw new TeaQLRuntimeException("Committed mutation has no predecessor version"); + } + Map fields = new LinkedHashMap<>(); + EntityKey key = new EntityKey(persisted.typeName(), persisted.getId()); + Map prior = created || deleted ? null : preimages.get(key); + if (!created && !deleted && prior == null) { + throw new TeaQLRuntimeException("Committed update has no transaction preimage: " + key.entity()); + } + if (!deleted) { + for (EntityDescriptor current = pending.descriptor(); current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getOwnProperties()) { + String name = property.getName(); + if (property.isId() || property.isVersion() || fields.containsKey(name)) continue; + if (!persisted.isPropertyLoaded(name)) { + throw new TeaQLRuntimeException("Committed provider readback omitted modeled field " + name); + } + if (prior != null && !prior.containsKey(name)) { + throw new TeaQLRuntimeException("Committed update preimage omitted modeled field " + name); + } + Object committedValue = committedFieldValue(property, persisted.getProperty(name)); + if (prior == null || !Objects.equals(prior.get(name), committedValue)) { + fields.put(name, committedValue); + } + } + } + } + String operation = switch (pending.auditKind()) { + case CREATED -> "create"; + case UPDATED -> "update"; + case DELETED -> "delete"; + case RECOVERED -> "recover"; + }; + facts.add(new CommittedMutationFact(persisted.typeName(), persisted.getId().toString(), + operation, baseVersion == null ? null : baseVersion.toString(), + persisted.getVersion().toString(), fields, pending.traceChain())); + } + return List.copyOf(facts); + } + + private Object committedFieldValue(PropertyDescriptor property, Object value) { + if (value == null) return null; + if (property instanceof Relation relation) { + if (relation.getReverseProperty() == null || relation.getReverseProperty().getOwner() == null) { + throw new TeaQLRuntimeException("Committed relation has no target metadata: " + property.getName()); + } + String type = relation.getReverseProperty().getOwner().getType(); + Long id = value instanceof Entity entity ? entity.getId() + : value instanceof Number number ? number.longValue() : null; + if (id == null) throw new TeaQLRuntimeException("Committed relation has no scalar identity: " + property.getName()); + return Map.of("entity_type", type, "entity_id", id.toString()); + } + if (value instanceof Long || value instanceof Integer || value instanceof Short + || value instanceof Byte) return value.toString(); + if (value instanceof String || value instanceof Boolean) return value; + if (value instanceof java.math.BigDecimal decimal) return decimal.toPlainString(); + if (value instanceof java.math.BigInteger integer) return integer.toString(); + if (value instanceof Double number) { + if (!Double.isFinite(number)) throw new TeaQLRuntimeException("Non-finite committed numeric value"); + return number; + } + if (value instanceof Float number) { + if (!Float.isFinite(number)) throw new TeaQLRuntimeException("Non-finite committed numeric value"); + return number; + } + if (value instanceof java.time.Instant instant) return committedInstantMillis(instant); + if (value instanceof java.time.LocalDate date) return date.toString(); + if (value instanceof java.time.LocalDateTime) { + throw new TeaQLRuntimeException("Zone-free LocalDateTime has no committed instant mapping"); + } + if (value instanceof java.time.temporal.TemporalAccessor) { + throw new TeaQLRuntimeException("Committed temporal field has unsupported type: " + property.getName()); + } + if (value instanceof java.util.UUID || value instanceof Enum) return value.toString(); + if (value instanceof java.util.Date date) { + // JDBC date/time subclasses deliberately throw from toInstant(). + // Keep this model-level runtime independent of the java.sql module. + try { return committedInstantMillis(date.toInstant()); } + catch (UnsupportedOperationException unsupported) { + String calendarDate = date.toString(); + if (calendarDate.matches("[0-9]{4}-[0-9]{2}-[0-9]{2}")) return calendarDate; + throw new TeaQLRuntimeException("Committed temporal field has unsupported type: " + property.getName()); + } + } + if (value instanceof byte[] bytes) return Map.of("$binary_base64", + java.util.Base64.getEncoder().encodeToString(bytes)); + throw new TeaQLRuntimeException("Committed field has unsupported type: " + property.getName()); + } + + private static String committedInstantMillis(java.time.Instant instant) { + if (instant.getNano() % 1_000_000 != 0) { + throw new TeaQLRuntimeException("Committed instant has finer than millisecond precision"); + } + try { return Long.toString(instant.toEpochMilli()); } + catch (ArithmeticException overflow) { + throw new TeaQLRuntimeException("Committed instant exceeds epoch-millisecond range"); + } + } + private record PersistenceState( Long version, io.teaql.core.EntityStatus status, boolean versionLoaded) {} @@ -1219,13 +1452,13 @@ private List mutateBatchWithTelemetry( } } - private void emitAuditEvent( + private int emitAuditEvent( UserContext context, Entity entity, MutationAuditKind kind, Map changedValues, MutationGovernanceSnapshot governance, MutationIntent intent, List traceChain, - SqlIntentRedactions redactions) { + SqlIntentRedactions redactions, Long committedVersion) { List changes = new ArrayList<>(); if (changedValues != null) { for (Map.Entry entry : changedValues.entrySet()) { @@ -1255,7 +1488,7 @@ private void emitAuditEvent( context.getAttribute(GeneratedSchemaBootstrap.AUDIT_ACTOR_ATTRIBUTE, String.class), context.getAttribute(GeneratedSchemaBootstrap.AUDIT_CATEGORY_ATTRIBUTE, String.class), intent.auditReason(), - entity.getVersion(), + committedVersion, java.time.Instant.now(), governance); @@ -1265,18 +1498,29 @@ private void emitAuditEvent( "teaql.mutation.kind", kind.name().toLowerCase(Locale.ROOT), "teaql.audit.changed_field_count", changes.size()))); try { + int failures = 0; + // The server-owned and application sinks are independent delivery + // attempts. One failure must not suppress the other after commit. + if (logSink != null) { + try { + logSink.writeAuditEvent(context, + LogPrivacy.audit(rawEvent, LogPrivacy.plaintextEnabled(), redactions)); + } catch (RuntimeException failure) { + failures++; + } + } - // The standard sink is server-owned by TeaQLRuntime and cannot be replaced by - // dynamic input or an application capability registered on UserContext. - if (logSink != null) { - logSink.writeAuditEvent(context, LogPrivacy.audit(rawEvent, LogPrivacy.plaintextEnabled(), redactions)); - } - - AppAuditEventSink appSink = context.capability(AppAuditEventSink.class); - if (appSink != null) { - appSink.onAuditEvent(context, buildSafeAuditEvent(rawEvent, redactions)); - } - telemetryScope.success(); + AppAuditEventSink appSink = context.capability(AppAuditEventSink.class); + if (appSink != null) { + try { + appSink.onAuditEvent(context, buildSafeAuditEvent(rawEvent, redactions)); + } catch (RuntimeException failure) { + failures++; + } + } + if (failures == 0) telemetryScope.success(); + else telemetryScope.failure(new AuditDeliveryAfterCommitException(failures)); + return failures; } catch (RuntimeException | Error error) { telemetryScope.failure(error); throw error; @@ -1363,6 +1607,7 @@ public static class Builder { private BusinessIdKeyProvider businessIdKeyProvider; private RoundTripReferenceProvider roundTripReferenceProvider; private BusinessClock businessClock = SystemBusinessClock.INSTANCE; + private CommittedMutationRecorder committedMutationRecorder; public Builder metadata(EntityMetaFactory metadata) { this.metadata = metadata; @@ -1419,6 +1664,12 @@ public Builder businessClock(BusinessClock businessClock) { return this; } + /** Enables graph-atomic, pre-commit capture through a trusted provider recorder. */ + public Builder committedMutationRecorder(CommittedMutationRecorder recorder) { + this.committedMutationRecorder = Objects.requireNonNull(recorder, "recorder"); + return this; + } + /** * Installs one provider-backed allocator for generated Fix execution and * the explicit {@code context.ensureSchema()} lifecycle. diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index dc4da6c9..9d5e340a 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -1403,6 +1403,15 @@ static Object convertTemporalColumnValue(Class targetType, Object value) { && value instanceof java.sql.Timestamp timestamp) { return timestamp.toLocalDateTime(); } + if (targetType == java.time.Instant.class) { + if (value instanceof Long || value instanceof Integer || value instanceof Short + || value instanceof Byte) { + return java.time.Instant.ofEpochMilli(((Number) value).longValue()); + } + if (value instanceof java.sql.Timestamp timestamp) return timestamp.toInstant(); + if (value instanceof CharSequence text) return java.time.Instant.parse(text); + throw new IllegalArgumentException("Instant readback is not an exact timestamp value"); + } if (targetType == java.time.LocalDateTime.class) { // SQLite returns text and some JDBC drivers return vendor temporal // wrappers. Both expose the standard timestamp representation. @@ -1863,7 +1872,8 @@ private void ensureCanonicalRelationIndexes(UserContext context) { String indexName = canonicalRelationIndexName( table, relationColumn.getColumnName(), idColumn.getColumnName()); try { - if (database.indexExists(context, table, indexName).orElse(false)) continue; + if (database.canonicalRelationIndexMatches(context, table, indexName, + List.of(relationColumn.getColumnName(), idColumn.getColumnName())).orElse(false)) continue; } catch (Exception failure) { throw canonicalRelationIndexFailure("inspect", table, indexName, failure); } @@ -1880,7 +1890,8 @@ private void ensureCanonicalRelationIndexes(UserContext context) { try { // PostgreSQL index names are schema-wide. A duplicate name on a // different table must not be mistaken for this index being present. - if (database.indexExists(context, table, indexName).orElse(true)) { + if (database.canonicalRelationIndexMatches(context, table, indexName, + List.of(relationColumn.getColumnName(), idColumn.getColumnName())).orElse(true)) { continue; } } catch (Exception inspectionFailure) { diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java index d4f510fa..00439c9c 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java @@ -128,4 +128,16 @@ default Optional indexExists( io.teaql.core.UserContext context, String tableName, String indexName) { return Optional.empty(); } + + /** + * Check the ordered keys of a generated relation index when the provider + * exposes them. Existing adapters retain their established index check; + * adapters with index-definition metadata can reject a name collision + * whose keys do not match the generated relation/index shape. + */ + default Optional canonicalRelationIndexMatches( + io.teaql.core.UserContext context, String tableName, String indexName, + List orderedColumns) { + return indexExists(context, tableName, indexName); + } } diff --git a/teaql-sqlite/pom.xml b/teaql-sqlite/pom.xml index 54a85105..69ca81b5 100644 --- a/teaql-sqlite/pom.xml +++ b/teaql-sqlite/pom.xml @@ -28,11 +28,20 @@ sqlite-jdbc 3.53.2.1 + + com.fasterxml.jackson.core + jackson-databind + io.teaql teaql-provider-jdbc ${project.version} + + io.teaql + teaql-runtime + ${project.version} + io.teaql @@ -51,12 +60,6 @@ junit test - - io.teaql - teaql-runtime - ${project.version} - test - diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java new file mode 100644 index 00000000..fb86b2d0 --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java @@ -0,0 +1,292 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.LongFunction; + +/** + * SQLite durability boundary for already-committed logical mutation facts. + * + *

The caller must install the schema before use. Append and apply require the + * same active SQL transaction as business writes. This class deliberately does + * not infer mutations from a ledger, replay business logic, or expose a remote + * replication endpoint. Projection writers are trusted infrastructure code. + */ +public final class SqliteCommittedEventStore { + private static final ObjectMapper JSON = new ObjectMapper(); + private static final java.util.Set ENVELOPE_FIELDS = java.util.Set.of( + "protocol_version", "change_set_id", "source_runtime_id", "source_sequence", + "source_model_hash", "source_transaction_id", "committed_at", "audit", "mutations"); + private final SqlExecutionAdapter sql; + + public SqliteCommittedEventStore(SqlExecutionAdapter sql) { + this.sql = Objects.requireNonNull(sql, "sql"); + } + + public record OutboxRecord(String sourceRuntimeId, long sourceSequence, + String changeSetId, String modelHash, String payloadSha256, String payloadJson) { + @Override public String toString() { + return "OutboxRecord[sourceRuntimeId=" + sourceRuntimeId + + ", sourceSequence=" + sourceSequence + ", payload=]"; + } + } + + public record InboxOutcome(boolean duplicate, long checkpoint) { + public static InboxOutcome applied(long checkpoint) { return new InboxOutcome(false, checkpoint); } + public static InboxOutcome duplicate(long checkpoint) { return new InboxOutcome(true, checkpoint); } + } + + @FunctionalInterface + interface TrustedProjectionWriter { + void apply(SqlExecutionAdapter transaction, JsonNode committedFacts); + } + + public static final class CommittedEventException extends RuntimeException { + private final String code; + + CommittedEventException(String code, String message) { + super(message); + this.code = code; + } + + public String code() { return code; } + } + + public void ensureSchema() { + SqliteCommittedObservation.ensureSchema(sql); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_sequence (" + + "source_runtime_id TEXT PRIMARY KEY, " + + "current_sequence INTEGER NOT NULL CHECK(current_sequence >= 0))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_outbox (" + + "source_runtime_id TEXT NOT NULL, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence > 0), " + + "change_set_id TEXT NOT NULL, model_hash TEXT NOT NULL, " + + "payload_sha256 TEXT NOT NULL, payload_json TEXT NOT NULL, " + + "PRIMARY KEY(source_runtime_id, source_sequence), " + + "UNIQUE(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_inbox (" + + "source_runtime_id TEXT NOT NULL, change_set_id TEXT NOT NULL, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence > 0), " + + "payload_sha256 TEXT NOT NULL, PRIMARY KEY(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_applied_payload (" + + "source_runtime_id TEXT NOT NULL, source_sequence INTEGER NOT NULL CHECK(source_sequence > 0), " + + "change_set_id TEXT NOT NULL, payload_json TEXT NOT NULL, " + + "PRIMARY KEY(source_runtime_id, source_sequence), " + + "UNIQUE(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_checkpoint (" + + "source_runtime_id TEXT PRIMARY KEY, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence >= 0), " + + "model_hash TEXT NOT NULL)"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_delivery_cursor (" + + "source_runtime_id TEXT NOT NULL, target_id TEXT NOT NULL, " + + "acked_sequence INTEGER NOT NULL CHECK(acked_sequence >= 0), " + + "model_hash TEXT NOT NULL, PRIMARY KEY(source_runtime_id, target_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_snapshot (" + + "source_runtime_id TEXT PRIMARY KEY, snapshot_id TEXT NOT NULL, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence >= 0), model_hash TEXT NOT NULL, " + + "payload_sha256 TEXT NOT NULL, payload_json TEXT NOT NULL)"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_snapshot_publication (" + + "source_runtime_id TEXT NOT NULL, target_id TEXT NOT NULL, snapshot_id TEXT NOT NULL, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence >= 0), model_hash TEXT NOT NULL, " + + "payload_sha256 TEXT NOT NULL, payload_json TEXT NOT NULL, " + + "acknowledged INTEGER NOT NULL CHECK(acknowledged IN (0,1)), " + + "PRIMARY KEY(source_runtime_id,target_id))"); + } + + /** Bounded publisher read of one durable, ordered source stream. */ + public List outboxAfter(String sourceRuntimeId, long after, int limit) { + if (blank(sourceRuntimeId) || after < 0 || limit < 1 || limit > 1000) { + throw invalid("invalid source, position, or read limit"); + } + return sql.queryForList("SELECT source_runtime_id, source_sequence, change_set_id, " + + "model_hash, payload_sha256, payload_json FROM teaql_event_outbox " + + "WHERE source_runtime_id = ? AND source_sequence > ? " + + "ORDER BY source_sequence ASC LIMIT ?", new Object[] {sourceRuntimeId, after, limit}) + .stream().map(row -> new OutboxRecord((String) row.get("source_runtime_id"), + number(row.get("source_sequence")), (String) row.get("change_set_id"), + (String) row.get("model_hash"), (String) row.get("payload_sha256"), + (String) row.get("payload_json"))).toList(); + } + + /** Allocate and append within the caller's already-open business transaction. */ + public OutboxRecord append(String sourceRuntimeId, String changeSetId, String modelHash, + LongFunction payloadBuilder) { + requireTransaction(); + if (blank(sourceRuntimeId) || blank(changeSetId) || blank(modelHash) || payloadBuilder == null) { + throw invalid("missing committed-event identity or payload builder"); + } + return savepoint("teaql_event_append", () -> { + sql.update("INSERT INTO teaql_event_sequence(source_runtime_id, current_sequence) " + + "VALUES (?, 0) ON CONFLICT(source_runtime_id) DO NOTHING", + new Object[] {sourceRuntimeId}); + int updated = sql.update("UPDATE teaql_event_sequence SET current_sequence = current_sequence + 1 " + + "WHERE source_runtime_id = ? AND current_sequence < 9223372036854775807", + new Object[] {sourceRuntimeId}); + if (updated != 1) throw new CommittedEventException("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + long sequence = number(sql.queryForList("SELECT current_sequence FROM teaql_event_sequence " + + "WHERE source_runtime_id = ?", new Object[] {sourceRuntimeId}).get(0).get("current_sequence")); + String payloadJson = payloadBuilder.apply(sequence); + JsonNode payload = parse(payloadJson); + validateHeader(payload, sourceRuntimeId, sequence, changeSetId, modelHash); + String digest = digest(payloadJson); + sql.update("INSERT INTO teaql_event_outbox " + + "(source_runtime_id, source_sequence, change_set_id, model_hash, payload_sha256, payload_json) " + + "VALUES (?, ?, ?, ?, ?, ?)", + new Object[] {sourceRuntimeId, sequence, changeSetId, modelHash, digest, payloadJson}); + return new OutboxRecord(sourceRuntimeId, sequence, changeSetId, modelHash, digest, payloadJson); + }); + } + + /** + * Apply one set through a restricted projection writer. The writer must not + * call normal entity save, Checker/Fix, ID generation or business hooks. + */ + InboxOutcome apply(OutboxRecord record, String expectedModelHash, + TrustedProjectionWriter writer) { + requireTransaction(); + if (record == null || blank(expectedModelHash) || writer == null) { + throw invalid("missing committed event, model hash, or projection writer"); + } + if (!expectedModelHash.equals(record.modelHash())) { + throw new CommittedEventException("MODEL_MISMATCH", "committed-event model hash mismatch"); + } + JsonNode payload = parse(record.payloadJson()); + validateHeader(payload, record.sourceRuntimeId(), record.sourceSequence(), + record.changeSetId(), record.modelHash()); + if (!digest(record.payloadJson()).equals(record.payloadSha256())) { + throw invalid("payload digest mismatch"); + } + List> checkpoints = sql.queryForList( + "SELECT source_sequence, model_hash FROM teaql_event_checkpoint WHERE source_runtime_id = ?", + new Object[] {record.sourceRuntimeId()}); + long last = checkpoints.isEmpty() ? 0 : number(checkpoints.get(0).get("source_sequence")); + if (!checkpoints.isEmpty() && !expectedModelHash.equals(checkpoints.get(0).get("model_hash"))) { + throw new CommittedEventException("MODEL_MISMATCH", "committed-event checkpoint model mismatch"); + } + List> prior = sql.queryForList( + "SELECT source_sequence, payload_sha256 FROM teaql_event_inbox " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {record.sourceRuntimeId(), record.changeSetId()}); + if (!prior.isEmpty()) { + Map seen = prior.get(0); + if (number(seen.get("source_sequence")) == record.sourceSequence() + && record.payloadSha256().equals(seen.get("payload_sha256"))) { + List> retained = sql.queryForList( + "SELECT source_sequence, payload_json FROM teaql_event_applied_payload " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {record.sourceRuntimeId(), record.changeSetId()}); + if (retained.size() != 1 + || number(retained.get(0).get("source_sequence")) != record.sourceSequence() + || !record.payloadSha256().equals(digest((String) retained.get(0).get("payload_json")))) { + throw invalid("prior Inbox identity lacks retained source evidence"); + } + if (last < record.sourceSequence()) { + throw invalid("prior Inbox identity is ahead of target checkpoint"); + } + return InboxOutcome.duplicate(record.sourceSequence()); + } + throw new CommittedEventException("DUPLICATE_CONFLICT", "committed-event identity has conflicting content"); + } + if (last == Long.MAX_VALUE) { + throw new CommittedEventException("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + } + if (record.sourceSequence() != last + 1) { + throw new CommittedEventException("SEQUENCE_GAP", + "committed-event sequence gap: expected " + (last + 1) + ", received " + record.sourceSequence()); + } + return savepoint("teaql_event_apply", () -> { + writer.apply(sql, payload); + sql.update("INSERT INTO teaql_event_applied_payload " + + "(source_runtime_id, source_sequence, change_set_id, payload_json) VALUES (?, ?, ?, ?)", + new Object[] {record.sourceRuntimeId(), record.sourceSequence(), + record.changeSetId(), record.payloadJson()}); + sql.update("INSERT INTO teaql_event_inbox " + + "(source_runtime_id, change_set_id, source_sequence, payload_sha256) VALUES (?, ?, ?, ?)", + new Object[] {record.sourceRuntimeId(), record.changeSetId(), + record.sourceSequence(), record.payloadSha256()}); + sql.update("INSERT INTO teaql_event_checkpoint (source_runtime_id, source_sequence, model_hash) " + + "VALUES (?, ?, ?) ON CONFLICT(source_runtime_id) DO UPDATE SET " + + "source_sequence = excluded.source_sequence, model_hash = excluded.model_hash", + new Object[] {record.sourceRuntimeId(), record.sourceSequence(), record.modelHash()}); + return InboxOutcome.applied(record.sourceSequence()); + }); + } + + private static JsonNode parse(String payloadJson) { + if (payloadJson == null) throw invalid("missing payload"); + try { + return JSON.readTree(payloadJson); + } catch (Exception ignored) { + throw invalid("invalid payload JSON"); + } + } + + private static void validateHeader(JsonNode payload, String source, long sequence, + String changeSetId, String modelHash) { + if (blank(source) || sequence < 1 || blank(changeSetId) || blank(modelHash) + || payload == null || payload.path("protocol_version").asInt(-1) != 1 + || !source.equals(payload.path("source_runtime_id").asText()) + || !Long.toString(sequence).equals(payload.path("source_sequence").asText()) + || !changeSetId.equals(payload.path("change_set_id").asText()) + || !modelHash.equals(payload.path("source_model_hash").asText()) + || !payload.path("mutations").isArray() || payload.path("mutations").isEmpty()) { + throw invalid("required v1 header or mutations mismatch"); + } + rejectUnknownControls(payload, ENVELOPE_FIELDS); + } + + /** Check protocol control frames, never recursively scan modeled values or audit text. */ + static void rejectUnknownControls(JsonNode object, java.util.Set allowed) { + if (!object.isObject()) throw invalid("noncanonical protocol object"); + object.fieldNames().forEachRemaining(key -> { + if (!allowed.contains(key)) throw invalid("noncanonical protocol control field"); + }); + } + + private T savepoint(String name, java.util.function.Supplier action) { + sql.execute("SAVEPOINT " + name); + try { + T value = action.get(); + sql.execute("RELEASE SAVEPOINT " + name); + return value; + } catch (RuntimeException | Error failure) { + try { + sql.execute("ROLLBACK TO SAVEPOINT " + name); + sql.execute("RELEASE SAVEPOINT " + name); + } catch (RuntimeException rollbackFailure) { + // A failed savepoint rollback must not leave partial rows committable. + try { sql.execute("ROLLBACK"); } catch (RuntimeException ignored) { } + throw new CommittedEventException("ROLLBACK_FAILED", "committed-event rollback failed"); + } + throw failure; + } + } + + private void requireTransaction() { + if (!sql.hasActiveTransaction()) throw invalid("an active SQL transaction is required"); + } + + private static long number(Object value) { return ((Number) value).longValue(); } + private static boolean blank(String value) { return value == null || value.isBlank(); } + private static CommittedEventException invalid(String reason) { + return new CommittedEventException("INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } + private static String digest(String payload) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(hash); + } catch (NoSuchAlgorithmException impossible) { + throw new AssertionError("JDK SHA-256 is unavailable", impossible); + } + } +} diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java new file mode 100644 index 00000000..1e45db2f --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java @@ -0,0 +1,111 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.GeneratedSchemaBootstrap; +import io.teaql.core.CommittedEventSchemaContributor; +import io.teaql.core.MutationIntent; +import io.teaql.core.SchemaExecutor; +import io.teaql.core.TraceNode; +import io.teaql.core.UserContext; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import io.teaql.runtime.CommittedMutationFact; +import io.teaql.runtime.CommittedMutationRecorder; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +/** SQLite source adapter that records one graph-save result inside its write transaction. */ +public final class SqliteCommittedMutationRecorder + implements CommittedMutationRecorder, CommittedEventSchemaContributor { + private static final ObjectMapper JSON = new ObjectMapper(); + private final SqlExecutionAdapter adapter; + private final SqliteCommittedEventStore events; + private final String sourceRuntimeId; + private final String modelHash; + + public SqliteCommittedMutationRecorder(SqlExecutionAdapter adapter, + String sourceRuntimeId, String modelHash) { + this.adapter = Objects.requireNonNull(adapter, "adapter"); + this.events = new SqliteCommittedEventStore(adapter); + if (sourceRuntimeId == null || sourceRuntimeId.isBlank() + || modelHash == null || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw new IllegalArgumentException("A source runtime ID and accepted SHA-256 model hash are required"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.modelHash = modelHash; + } + + @Override + public void ensureSchema(UserContext context, SchemaExecutor.Invocation invocation) { + SchemaExecutor.Invocation.requireContextOwned(invocation); + Objects.requireNonNull(context, "context"); + events.ensureSchema(); + } + public SqliteCommittedEventStore eventStore() { return events; } + + @Override + public void record(UserContext context, Object transactionResource, + MutationIntent intent, List facts) { + Objects.requireNonNull(context, "context"); + Objects.requireNonNull(intent, "intent"); + if (transactionResource != adapter || !adapter.hasActiveTransaction()) { + throw new IllegalStateException("Committed graph capture requires the source provider transaction"); + } + if (facts == null || facts.isEmpty()) { + throw new IllegalArgumentException("Committed graph capture requires at least one fact"); + } + String changeSetId = UUID.randomUUID().toString(); + String committedAt = Instant.now().toString(); + String actor = context.getAttribute(GeneratedSchemaBootstrap.AUDIT_ACTOR_ATTRIBUTE, String.class); + events.append(sourceRuntimeId, changeSetId, modelHash, sequence -> { + Map envelope = new LinkedHashMap<>(); + envelope.put("protocol_version", 1); + envelope.put("change_set_id", changeSetId); + envelope.put("source_runtime_id", sourceRuntimeId); + envelope.put("source_sequence", Long.toString(sequence)); + envelope.put("source_model_hash", modelHash); + envelope.put("source_transaction_id", changeSetId); + envelope.put("committed_at", committedAt); + Map audit = new LinkedHashMap<>(); + audit.put("actor_ref", actor); + audit.put("comment", intent.comment()); + // MutationIntent currently has no separate business purpose field. + audit.put("purpose", "replicate committed graph"); + envelope.put("audit", audit); + List> mutations = new ArrayList<>(facts.size()); + for (CommittedMutationFact fact : facts) { + Map mutation = new LinkedHashMap<>(); + mutation.put("entity_type", fact.entityType()); + mutation.put("entity_id", fact.entityId()); + mutation.put("operation", fact.operation()); + mutation.put("base_version", fact.baseVersion()); + mutation.put("committed_version", fact.committedVersion()); + mutation.put("fields", fact.fields()); + List> chain = new ArrayList<>(); + for (TraceNode node : fact.traceChain()) { + Map step = new LinkedHashMap<>(); + step.put("kind", node.getKind().name()); + step.put("name", Objects.toString(node.getName(), "")); + step.put("entity_id", node.getEntityId() == null ? null : node.getEntityId().toString()); + step.put("comment", Objects.toString(node.getComment(), "")); + chain.add(step); + } + mutation.put("trace_chain", chain); + mutations.add(mutation); + } + envelope.put("mutations", mutations); + try { + return JSON.writeValueAsString(envelope); + } catch (JsonProcessingException failure) { + // Do not attach values or serializer causes to public errors. + throw new IllegalStateException("Committed mutation payload encoding failed"); + } + }); + } +} diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedObservation.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedObservation.java new file mode 100644 index 00000000..95e066f5 --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedObservation.java @@ -0,0 +1,115 @@ +package io.teaql.core.sqlite; + +import io.teaql.core.UserContext; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import io.teaql.runtime.RuntimeTelemetry; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +/** Durable infrastructure observations. Source ACK is never labeled target watermark. */ +public final class SqliteCommittedObservation { + public record SourceObservation(long sourceHighWatermark,long ackedSequence,long sequenceLag, + long backlog,long pendingSnapshots,long attempts,long retries,long failures, + long rejections,long deadLetters,long unresolvedAttempts) { } + public record TargetObservation(long checkpoint,long snapshotBaseline,long inboxRows, + OptionalLong sequenceLag,long rejections) { } + public record DeadLetter(String kind,long sequence,String identityFingerprint, + String payloadFingerprint,long attempts,String reason) { } + record Attempt(String kind,long sequence,long ordinal) { } + private final SqlExecutionAdapter sql; + private final String source,target,model; + private static final Set PERMANENT=Set.of("UNAUTHORIZED_SOURCE","UNAUTHORIZED_TARGET", + "UNKNOWN_KEY","INVALID_SIGNATURE","INVALID_ENVELOPE","MODEL_MISMATCH", + "SEQUENCE_GAP","DUPLICATE_CONFLICT","VERSION_CONFLICT","SEQUENCE_EXHAUSTED", + "INVALID_SNAPSHOT","SNAPSHOT_CONFLICT","TARGET_NOT_EMPTY"); + public SqliteCommittedObservation(SqlExecutionAdapter sql,String source,String target,String model){ + this.sql=Objects.requireNonNull(sql,"sql"); + if(source==null||source.isBlank()||target==null||target.isBlank()||model==null||!model.matches("sha256:[0-9a-f]{64}"))throw unavailable(); + this.source=source;this.target=target;this.model=model; + } + static void ensureSchema(SqlExecutionAdapter sql){ + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_delivery_journal (source_runtime_id TEXT NOT NULL,target_id TEXT NOT NULL,kind TEXT NOT NULL CHECK(kind IN ('event','snapshot')),source_sequence INTEGER NOT NULL CHECK(source_sequence>=0),model_hash TEXT NOT NULL,identity_fingerprint TEXT NOT NULL,payload_fingerprint TEXT NOT NULL,attempts INTEGER NOT NULL CHECK(typeof(attempts)='integer' AND attempts>0),failures INTEGER NOT NULL CHECK(typeof(failures)='integer' AND failures>=0),rejections INTEGER NOT NULL CHECK(typeof(rejections)='integer' AND rejections>=0),dead_letter INTEGER NOT NULL CHECK(dead_letter IN (0,1)),last_outcome TEXT NOT NULL CHECK(last_outcome IN ('STARTED','ACKNOWLEDGED','FAILED')),last_reason TEXT NOT NULL,dead_reason TEXT NOT NULL,PRIMARY KEY(source_runtime_id,target_id,kind,source_sequence))"); + } + Attempt begin(String kind,long sequence,String identity,String payload){ + if(!Set.of("event","snapshot").contains(kind)||sequence<0||identity==null||payload==null||!payload.matches("[0-9a-f]{64}"))throw unavailable(); + return transaction(()->{ + var rows=sql.queryForList("SELECT * FROM teaql_event_delivery_journal WHERE source_runtime_id=? AND target_id=? AND kind=? AND source_sequence=?",new Object[]{source,target,kind,sequence}); + long ordinal=1; + if(rows.isEmpty())sql.update("INSERT INTO teaql_event_delivery_journal VALUES(?,?,?,?,?,?,?,1,0,0,0,'STARTED','NONE','NONE')",new Object[]{source,target,kind,sequence,model,fingerprint(identity),payload}); + else{ + var row=rows.get(0);long prior=number(row.get("attempts")); + if(!model.equals(row.get("model_hash"))||!fingerprint(identity).equals(row.get("identity_fingerprint"))||!payload.equals(row.get("payload_fingerprint"))||"ACKNOWLEDGED".equals(row.get("last_outcome")))throw conflict(); + ordinal=Math.addExact(prior,1); + if(sql.update("UPDATE teaql_event_delivery_journal SET attempts=?,last_outcome='STARTED',last_reason='NONE' WHERE source_runtime_id=? AND target_id=? AND kind=? AND source_sequence=? AND attempts=?",new Object[]{ordinal,source,target,kind,sequence,prior})!=1)throw conflict(); + } + return new Attempt(kind,sequence,ordinal); + }); + } + void failed(Attempt attempt,RuntimeException error){ + String code=error instanceof SqliteCommittedReceiver.ReceiverException r?r.code():error instanceof SqliteCommittedEventStore.CommittedEventException e?e.code():"TRANSPORT_UNAVAILABLE"; + String normalized=code.startsWith("SNAPSHOT_")&&PERMANENT.contains(code.substring(9))?code.substring(9):code; + boolean permanent=PERMANENT.contains(normalized); + String safe=permanent?normalized:Set.of("ACK_MISMATCH","AUDIT_UNAVAILABLE","CURSOR_CONFLICT").contains(code)?code:"TRANSPORT_UNAVAILABLE"; + transaction(()->{if(sql.update("UPDATE teaql_event_delivery_journal SET failures=failures+1,rejections=rejections+?,dead_letter=CASE WHEN ?=1 THEN 1 ELSE dead_letter END,dead_reason=CASE WHEN ?=1 THEN ? ELSE dead_reason END,last_outcome='FAILED',last_reason=? WHERE source_runtime_id=? AND target_id=? AND kind=? AND source_sequence=? AND attempts=? AND last_outcome='STARTED'",new Object[]{permanent?1:0,permanent?1:0,permanent?1:0,safe,safe,source,target,attempt.kind(),attempt.sequence(),attempt.ordinal()})!=1)throw conflict();return null;}); + } + T acknowledged(Attempt attempt,Supplier cursorUpdate){ + return transaction(()->{ + T value=cursorUpdate.get(); + if(sql.update("UPDATE teaql_event_delivery_journal SET last_outcome='ACKNOWLEDGED',last_reason='NONE',dead_letter=0,dead_reason='NONE' WHERE source_runtime_id=? AND target_id=? AND kind=? AND source_sequence=? AND attempts=? AND last_outcome='STARTED'",new Object[]{source,target,attempt.kind(),attempt.sequence(),attempt.ordinal()})!=1)throw conflict(); + return value; + }); + } + public SourceObservation source(UserContext context){ + require(context); + SourceObservation value=transaction(()->{ + if(count("SELECT count(*) n FROM teaql_event_delivery_journal WHERE source_runtime_id=? AND target_id=? AND model_hash<>?",new Object[]{source,target,model})!=0)throw unavailable(); + var cursor=sql.queryForList("SELECT acked_sequence,model_hash FROM teaql_event_delivery_cursor WHERE source_runtime_id=? AND target_id=?",new Object[]{source,target}); + long ack=cursor.isEmpty()?0:number(cursor.get(0).get("acked_sequence")); + if(!cursor.isEmpty()&&!model.equals(cursor.get(0).get("model_hash")))throw unavailable(); + var highRows=sql.queryForList("SELECT current_sequence FROM teaql_event_sequence WHERE source_runtime_id=?",new Object[]{source});long high=highRows.isEmpty()?0:number(highRows.get(0).get("current_sequence")); + if(high?",new Object[]{source,ack}); + long pending=count("SELECT count(*) n FROM teaql_event_snapshot_publication WHERE source_runtime_id=? AND target_id=? AND model_hash=? AND acknowledged=0",new Object[]{source,target,model}); + return new SourceObservation(high,ack,high-ack,backlog,pending,number(sums.get("attempts")),number(sums.get("retries")),number(sums.get("failures")),number(sums.get("rejections")),number(sums.get("dead")),number(sums.get("unresolved"))); + }); + emit(context,"source",Map.of("source_watermark",value.sourceHighWatermark(),"source_ack",value.ackedSequence(),"sequence_lag",value.sequenceLag(),"backlog",value.backlog(),"pending_snapshots",value.pendingSnapshots(),"attempts",value.attempts(),"retries",value.retries(),"failures",value.failures(),"rejections",value.rejections(),"dead_letters",value.deadLetters()),Map.of("unresolved_attempts",value.unresolvedAttempts())); + return value; + } + public TargetObservation target(UserContext context,OptionalLong observedSourceHigh){ + require(context);Objects.requireNonNull(observedSourceHigh,"observedSourceHigh"); + TargetObservation value=transaction(()->{ + var rows=sql.queryForList("SELECT source_sequence,model_hash FROM teaql_event_checkpoint WHERE source_runtime_id=?",new Object[]{source}); + long checkpoint=rows.isEmpty()?0:number(rows.get(0).get("source_sequence")); + if(!rows.isEmpty()&&!model.equals(rows.get(0).get("model_hash")))throw unavailable(); + if(observedSourceHigh.isPresent()&&observedSourceHigh.getAsLong()checkpoint)throw unavailable(); + return new TargetObservation(checkpoint,n,count("SELECT count(*) n FROM teaql_event_inbox WHERE source_runtime_id=?",new Object[]{source}),observedSourceHigh.isPresent()?OptionalLong.of(observedSourceHigh.getAsLong()-checkpoint):OptionalLong.empty(),count("SELECT count(*) n FROM teaql_event_rejection WHERE target_fingerprint=?",new Object[]{fingerprint(target)})); + }); + var attrs=new LinkedHashMap();attrs.put("target_watermark",value.checkpoint());attrs.put("snapshot_baseline",value.snapshotBaseline());attrs.put("inbox_rows",value.inboxRows());attrs.put("rejections",value.rejections());if(value.sequenceLag().isPresent())attrs.put("sequence_lag",value.sequenceLag().getAsLong());emit(context,"target",attrs,Map.of());return value; + } + public List deadLetters(UserContext context,int limit){ + require(context);if(limit<1||limit>1000)throw unavailable(); + return transaction(()->sql.queryForList("SELECT kind,source_sequence,identity_fingerprint,payload_fingerprint,attempts,dead_reason FROM teaql_event_delivery_journal WHERE source_runtime_id=? AND target_id=? AND model_hash=? AND dead_letter=1 ORDER BY source_sequence,kind LIMIT ?",new Object[]{source,target,model,limit}).stream().map(r->new DeadLetter((String)r.get("kind"),number(r.get("source_sequence")),(String)r.get("identity_fingerprint"),(String)r.get("payload_fingerprint"),number(r.get("attempts")),(String)r.get("dead_reason"))).toList()); + } + private void emit(UserContext context,String name,Map values,Map extra){ + var attributes=new LinkedHashMap();attributes.put("teaql.replication.source_fingerprint",fingerprint(source));attributes.put("teaql.replication.target_fingerprint",fingerprint(target));values.forEach((k,v)->attributes.put("teaql.replication."+k,v));extra.forEach((k,v)->attributes.put("teaql.replication."+k,v)); + RuntimeTelemetry.startSafely(context.capability(RuntimeTelemetry.class),new RuntimeTelemetry.Operation("replication",name,attributes)).success(); + } + private long count(String query,Object[] args){return number(sql.queryForList(query,args).get(0).get("n"));} + private void require(UserContext context){Objects.requireNonNull(context,"context");EntityMetaFactory.requireFrom(context);if(sql.hasActiveTransaction())throw unavailable();} + private T transaction(Supplier action){ + if(sql.hasActiveTransaction())throw unavailable(); + var value=new AtomicReference();sql.executeInTransaction(()->{if(!sql.hasActiveTransaction())throw unavailable();value.set(action.get());});return value.get(); + } + private static long number(Object value){if(!(value instanceof Number n)||n instanceof Double||n instanceof Float||n.longValue()<0)throw unavailable();return n.longValue();} + static String fingerprint(String value){try{return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)));}catch(java.security.NoSuchAlgorithmException impossible){throw new IllegalStateException(impossible);}} + private static SqliteCommittedEventStore.CommittedEventException unavailable(){return new SqliteCommittedEventStore.CommittedEventException("OBSERVATION_UNAVAILABLE","committed delivery observation is unavailable");} + private static SqliteCommittedEventStore.CommittedEventException conflict(){return new SqliteCommittedEventStore.CommittedEventException("CURSOR_CONFLICT","committed delivery attempt changed concurrently");} +} diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedOutboxPublisher.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedOutboxPublisher.java new file mode 100644 index 00000000..9607ca6f --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedOutboxPublisher.java @@ -0,0 +1,143 @@ +package io.teaql.core.sqlite; + +import io.teaql.core.sqlite.SqliteCommittedEventStore.CommittedEventException; +import io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Bounded ordered publication from one durable SQLite Outbox stream. The + * supplied sink must own its authentication and network retry budget; this + * publisher never treats a callback as proof of remote identity. + */ +public final class SqliteCommittedOutboxPublisher { + @FunctionalInterface + public interface DeliverySink { + InboxOutcome deliver(OutboxRecord record); + } + + public record DeliveryStatus(long ackedSequence, long sourceHighWatermark, long backlog) { } + public record BatchOutcome(int acknowledged, DeliveryStatus status) { } + + private final SqlExecutionAdapter sql; + private final SqliteCommittedEventStore events; + private final String sourceRuntimeId; + private final String targetId; + private final String modelHash; + + public SqliteCommittedOutboxPublisher(SqlExecutionAdapter sql, String sourceRuntimeId, + String targetId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (sourceRuntimeId == null || sourceRuntimeId.isBlank() + || targetId == null || targetId.isBlank() + || modelHash == null || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw failure("INVALID_ENVELOPE", "delivery source, target or model identity is invalid"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.targetId = targetId; + this.modelHash = modelHash; + this.events = new SqliteCommittedEventStore(sql); + } + + public DeliveryStatus status() { + List> rows = sql.queryForList( + "SELECT acked_sequence, model_hash FROM teaql_event_delivery_cursor " + + "WHERE source_runtime_id = ? AND target_id = ?", + new Object[] {sourceRuntimeId, targetId}); + long acked = 0; + if (!rows.isEmpty()) { + if (!modelHash.equals(rows.get(0).get("model_hash"))) { + throw failure("MODEL_MISMATCH", "delivery cursor model hash mismatch"); + } + acked = number(rows.get(0).get("acked_sequence")); + } + List> sourceRows = sql.queryForList( + "SELECT current_sequence FROM teaql_event_sequence WHERE source_runtime_id = ?", + new Object[] {sourceRuntimeId}); + long highWatermark = sourceRows.isEmpty() ? 0 + : number(sourceRows.get(0).get("current_sequence")); + if (highWatermark < acked) { + throw failure("INVALID_ENVELOPE", "delivery cursor is ahead of source sequence"); + } + return new DeliveryStatus(acked, highWatermark, highWatermark - acked); + } + + /** + * One attempt per event, at most {@code limit} events. The source cursor + * advances only after an acknowledgement for the exact sequence. If the + * target applied an event but the response is lost, a later call/restart + * re-delivers it and the target Inbox supplies a duplicate acknowledgement. + */ + public BatchOutcome publishBatch(int limit, DeliverySink sink) { + Objects.requireNonNull(sink, "sink"); + if (limit < 1 || limit > 1000) throw failure("INVALID_ENVELOPE", "invalid delivery read limit"); + claimCursor(); + DeliveryStatus before = status(); + List records = events.outboxAfter(sourceRuntimeId, before.ackedSequence(), limit); + if (records.isEmpty() && before.backlog() > 0) { + throw failure("SEQUENCE_GAP", "source Outbox has an unacknowledged gap"); + } + long acked = before.ackedSequence(); + int count = 0; + for (OutboxRecord record : records) { + if (acked == Long.MAX_VALUE) { + throw failure("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + } + if (record.sourceSequence() != acked + 1) { + throw failure("SEQUENCE_GAP", "source Outbox sequence is not contiguous"); + } + if (!modelHash.equals(record.modelHash())) { + throw failure("MODEL_MISMATCH", "source Outbox model hash mismatch"); + } + var journal = new SqliteCommittedObservation(sql,sourceRuntimeId,targetId,modelHash); + var attempt = journal.begin("event",record.sourceSequence(),record.changeSetId(),record.payloadSha256()); + try { + InboxOutcome outcome = sink.deliver(record); + if (outcome == null || outcome.checkpoint() != record.sourceSequence()) + throw failure("ACK_MISMATCH", "target acknowledgement does not match delivered sequence"); + } catch (RuntimeException error) { journal.failed(attempt,error); throw error; } + long prior = acked; + journal.acknowledged(attempt, () -> { + int changed = sql.update("UPDATE teaql_event_delivery_cursor SET acked_sequence=? " + + "WHERE source_runtime_id=? AND target_id=? AND acked_sequence=? AND model_hash=?", + new Object[] {record.sourceSequence(),sourceRuntimeId,targetId,prior,modelHash}); + if (changed != 1) { + throw failure("CURSOR_CONFLICT", "delivery cursor changed concurrently"); + } + return null; + }); + acked = record.sourceSequence(); + count++; + } + return new BatchOutcome(count, status()); + } + + private void claimCursor() { + if (sql.hasActiveTransaction()) throw failure("INVALID_ENVELOPE", "publisher requires a standalone transaction"); + sql.executeInTransaction(() -> { + if (!sql.hasActiveTransaction()) throw failure("INVALID_ENVELOPE", "publisher adapter must provide an active SQL transaction"); + var prepared = sql.queryForList("SELECT source_sequence,model_hash,acknowledged FROM teaql_event_snapshot_publication WHERE source_runtime_id=? AND target_id=?", + new Object[]{sourceRuntimeId,targetId}); + if (!prepared.isEmpty()) { + var row = prepared.get(0); + if (number(row.get("acknowledged")) == 0) throw failure("SNAPSHOT_PENDING", "snapshot bootstrap must be acknowledged before incremental delivery"); + var cursor = sql.queryForList("SELECT acked_sequence,model_hash FROM teaql_event_delivery_cursor WHERE source_runtime_id=? AND target_id=?",new Object[]{sourceRuntimeId,targetId}); + if (number(row.get("acknowledged")) != 1 || !modelHash.equals(row.get("model_hash")) || cursor.size()!=1 + || !modelHash.equals(cursor.get(0).get("model_hash")) || number(cursor.get(0).get("acked_sequence")) REJECTIONS=Set.of("INVALID_ENVELOPE","MODEL_MISMATCH", + "SEQUENCE_GAP","DUPLICATE_CONFLICT","VERSION_CONFLICT","SEQUENCE_EXHAUSTED"); + private final SqlExecutionAdapter sql; + private final String source,target,model; + private final Map keys; + private final SqliteModelCheckedCommittedTarget delegate; + private final SqliteCommittedSnapshot snapshots; + private final SqliteCommittedSnapshot.Limits snapshotLimits; + private final SqliteCommittedSnapshot.ImportMode snapshotMode; + + public record Delivery(OutboxRecord event,String keyId,String targetId,byte[] signature) { + public Delivery {signature=signature==null?new byte[0]:signature.clone();} + @Override public byte[] signature(){return signature.clone();} + @Override public String toString(){return "CommittedDelivery[content=]";} + } + public record SnapshotDelivery(SqliteCommittedSnapshot.SnapshotRecord snapshot, + String keyId,String targetId,byte[] signature) { + public SnapshotDelivery {signature=signature==null?new byte[0]:signature.clone();} + @Override public byte[] signature(){return signature.clone();} + @Override public String toString(){return "CommittedSnapshotDelivery[content=]";} + } + public static final class ReceiverException extends RuntimeException { + private final String code; + private ReceiverException(String code){super("committed receiver: "+code);this.code=code;} + public String code(){return code;} + } + public SqliteCommittedReceiver(SqlExecutionAdapter sql,String source,String target, + String model,Map keys) { + this(sql,source,target,model,keys,SqliteCommittedSnapshot.Limits.defaults()); + } + /** Limits are trusted startup policy, never caller-controlled wire options. */ + public SqliteCommittedReceiver(SqlExecutionAdapter sql,String source,String target, + String model,Map keys,SqliteCommittedSnapshot.Limits snapshotLimits) { + this(sql,source,target,model,keys,snapshotLimits,SqliteCommittedSnapshot.ImportMode.REQUIRE_EMPTY); + } + /** Explicit local policy for bootstrap-populated targets, never a wire flag. */ + public SqliteCommittedReceiver(SqlExecutionAdapter sql,String source,String target, + String model,Map keys,SqliteCommittedSnapshot.Limits snapshotLimits, + SqliteCommittedSnapshot.ImportMode snapshotMode) { + this.sql=Objects.requireNonNull(sql,"sql"); + if(snapshotLimits==null||snapshotLimits.maxPayloadBytes()>MAX_SNAPSHOT_PAYLOAD||snapshotMode==null) + throw new IllegalArgumentException("invalid receiver snapshot limits"); + this.snapshotLimits=snapshotLimits; + this.snapshotMode=snapshotMode; + try { + field(source,MAX_ID);field(target,MAX_ID); + if(model==null||!model.matches("sha256:[0-9a-f]{64}")||keys==null||keys.isEmpty()||keys.size()>16)throw rejected("INVALID_ENVELOPE"); + Map copy=new HashMap<>(); + for(var entry:keys.entrySet()){field(entry.getKey(),MAX_ID);key(entry.getValue());copy.put(entry.getKey(),entry.getValue().clone());} + this.keys=Map.copyOf(copy); + }catch(ReceiverException failure){throw new IllegalArgumentException("invalid committed receiver configuration");} + this.source=source;this.target=target;this.model=model; + this.delegate=new SqliteModelCheckedCommittedTarget(sql,source,model); + this.snapshots=new SqliteCommittedSnapshot(sql,source,model); + } + /** Explicit provider-infrastructure installation; no domain-row seeding. */ + public void ensureSchema(UserContext context) { + EntityMetaFactory.requireFrom(Objects.requireNonNull(context,"context")); + if(sql.hasActiveTransaction())throw rejected("AMBIENT_TRANSACTION"); + delegate.ensureSchema(); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_rejection (audit_id INTEGER PRIMARY KEY, " + +"reason_code TEXT NOT NULL, source_sequence INTEGER NOT NULL, source_fingerprint TEXT NOT NULL, " + +"change_fingerprint TEXT NOT NULL,target_fingerprint TEXT NOT NULL,payload_fingerprint TEXT NOT NULL)"); + } + /** Original model facts remain unchanged; authenticated duplicate ACKs stay stable. */ + public SqliteCommittedEventStore.InboxOutcome receive(UserContext context,Delivery delivery) { + Objects.requireNonNull(context,"context"); + // Infrastructure wiring error: never join/commit a caller's transaction. + if(sql.hasActiveTransaction())throw rejected("AMBIENT_TRANSACTION"); + try { + validate(delivery); + OutboxRecord event=delivery.event(); + if(!source.equals(event.sourceRuntimeId()))throw rejected("UNAUTHORIZED_SOURCE"); + if(!target.equals(delivery.targetId()))throw rejected("UNAUTHORIZED_TARGET"); + if(!model.equals(event.modelHash()))throw rejected("MODEL_MISMATCH"); + byte[] secret=keys.get(delivery.keyId()); + if(secret==null)throw rejected("UNKNOWN_KEY"); + if(!MessageDigest.isEqual(tag(delivery.keyId(),delivery.targetId(),secret,event),delivery.signature))throw rejected("INVALID_SIGNATURE"); + return delegate.apply(context,event); + }catch(ReceiverException failure){ + audit(delivery,failure.code());throw failure; + }catch(SqliteCommittedEventStore.CommittedEventException failure){ + // Unknown/failed transaction outcomes are not security rejections. + if(!REJECTIONS.contains(failure.code()))throw failure; + audit(delivery,failure.code());throw rejected(failure.code()); + } + } + /** Source-side signing of original bytes; requires a deployment-owned secret. */ + public static Delivery sign(String keyId,String target,byte[] secret,OutboxRecord event) { + field(keyId,MAX_ID);field(target,MAX_ID);key(secret);record(event); + return new Delivery(event,keyId,target,tag(keyId,target,secret,event)); + } + /** Separate type domain; zero is a valid snapshot baseline, never an event sequence. */ + public static SnapshotDelivery signSnapshot(String keyId,String target,byte[] secret, + SqliteCommittedSnapshot.SnapshotRecord snapshot) { + field(keyId,MAX_ID);field(target,MAX_ID);key(secret);snapshot(snapshot,MAX_SNAPSHOT_PAYLOAD); + return new SnapshotDelivery(snapshot,keyId,target,snapshotTag(keyId,target,secret,snapshot)); + } + /** Authenticate before restore. Deliveries cannot select or relax startup restore policy. */ + public SqliteCommittedSnapshotPublisher.Receipt receiveSnapshot(UserContext context,SnapshotDelivery delivery) { + Objects.requireNonNull(context,"context"); + if(sql.hasActiveTransaction())throw rejected("AMBIENT_TRANSACTION"); + try { + if(delivery==null)throw rejected("INVALID_ENVELOPE"); + field(delivery.keyId(),MAX_ID);field(delivery.targetId(),MAX_ID); + snapshot(delivery.snapshot(),snapshotLimits.maxPayloadBytes()); + if(delivery.signature.length!=32)throw rejected("INVALID_ENVELOPE"); + var record=delivery.snapshot(); + if(!source.equals(record.sourceRuntimeId()))throw rejected("UNAUTHORIZED_SOURCE"); + if(!target.equals(delivery.targetId()))throw rejected("UNAUTHORIZED_TARGET"); + if(!model.equals(record.modelHash()))throw rejected("MODEL_MISMATCH"); + byte[] secret=keys.get(delivery.keyId());if(secret==null)throw rejected("UNKNOWN_KEY"); + if(!MessageDigest.isEqual(snapshotTag(delivery.keyId(),delivery.targetId(),secret,record),delivery.signature))throw rejected("INVALID_SIGNATURE"); + return SqliteCommittedSnapshotPublisher.initializeAndAcknowledge(context,snapshots,record,target,snapshotMode,snapshotLimits); + }catch(ReceiverException failure){audit(delivery,failure.code());throw failure;} + catch(SqliteCommittedEventStore.CommittedEventException failure){ + String code="INVALID_ENVELOPE".equals(failure.code())?"INVALID_SNAPSHOT":failure.code(); + if(!REJECTIONS.contains(code)&&!Set.of("INVALID_SNAPSHOT","SNAPSHOT_CONFLICT").contains(code))throw failure; + audit(delivery,code);throw rejected(code); + } + } + private static void validate(Delivery delivery){ + if(delivery==null)throw rejected("INVALID_ENVELOPE"); + field(delivery.keyId(),MAX_ID);field(delivery.targetId(),MAX_ID);record(delivery.event()); + if(delivery.signature.length!=32)throw rejected("INVALID_ENVELOPE"); + } + private static void record(OutboxRecord e){ + if(e==null||e.sourceSequence()<=0)throw rejected("INVALID_ENVELOPE"); + field(e.sourceRuntimeId(),MAX_ID);field(e.changeSetId(),MAX_ID);field(e.modelHash(),MAX_ID);field(e.payloadSha256(),MAX_ID);field(e.payloadJson(),MAX_PAYLOAD); + } + private static void snapshot(SqliteCommittedSnapshot.SnapshotRecord s,int byteLimit){ + if(s==null||s.sourceSequence()<0)throw rejected("INVALID_ENVELOPE"); + field(s.sourceRuntimeId(),MAX_ID);field(s.snapshotId(),MAX_ID);field(s.modelHash(),MAX_ID);field(s.payloadSha256(),MAX_ID);field(s.payloadJson(),byteLimit); + } + private static byte[] field(String text,int limit){ + if(text==null||text.isEmpty()||text.length()>limit||!StandardCharsets.UTF_8.newEncoder().canEncode(text))throw rejected("INVALID_ENVELOPE"); + byte[] bytes=text.getBytes(StandardCharsets.UTF_8);if(bytes.length>limit)throw rejected("INVALID_ENVELOPE");return bytes; + } + private static void key(byte[] secret){if(secret==null||secret.length<32||secret.length>128)throw rejected("INVALID_ENVELOPE");} + private static byte[] tag(String keyId,String target,byte[] secret,OutboxRecord event){ + return hmac(secret,"teaql.committed.receiver.v1\0",keyId,target,event.sourceRuntimeId(),Long.toString(event.sourceSequence()),event.changeSetId(),event.modelHash(),event.payloadSha256(),event.payloadJson()); + } + private static byte[] snapshotTag(String keyId,String target,byte[] secret,SqliteCommittedSnapshot.SnapshotRecord s){ + return hmac(secret,"teaql.committed.snapshot.receiver.v1\0",keyId,target,s.sourceRuntimeId(),Long.toString(s.sourceSequence()),s.snapshotId(),s.modelHash(),s.payloadSha256(),s.payloadJson()); + } + private static byte[] hmac(byte[] secret,String domain,String... fields){ + try { + Mac mac=Mac.getInstance("HmacSHA256");mac.init(new SecretKeySpec(secret,"HmacSHA256")); + mac.update(domain.getBytes(StandardCharsets.UTF_8)); + for(String value:fields){byte[] bytes=value.getBytes(StandardCharsets.UTF_8);mac.update(ByteBuffer.allocate(4).putInt(bytes.length).array());mac.update(bytes);} + return mac.doFinal(); + }catch(GeneralSecurityException failure){throw new IllegalStateException("standard HMAC-SHA256 unavailable");} + } + private void audit(Delivery delivery,String code){ + OutboxRecord e=delivery==null?null:delivery.event(); + audit(code,e==null?0:e.sourceSequence(),e==null?null:e.sourceRuntimeId(),e==null?null:e.changeSetId(),e==null?null:e.payloadJson(),MAX_PAYLOAD); + } + private void audit(SnapshotDelivery delivery,String code){ + var s=delivery==null?null:delivery.snapshot(); + audit("SNAPSHOT_"+code,s==null?0:s.sourceSequence(),s==null?null:s.sourceRuntimeId(),s==null?null:s.snapshotId(),s==null?null:s.payloadJson(),MAX_SNAPSHOT_PAYLOAD); + } + private void audit(String code,long sequence,String sourceId,String identity,String payload,int maxPayload){ + try { + if(sql.hasActiveTransaction())throw new IllegalStateException("audit transaction unavailable"); + Object[] values={code,Math.max(0,sequence),fingerprint(sourceId,MAX_ID),fingerprint(identity,MAX_ID),fingerprint(target,MAX_ID),fingerprint(payload,maxPayload)}; + sql.executeInTransaction(()->{if(sql.update("INSERT INTO teaql_event_rejection(reason_code,source_sequence,source_fingerprint,change_fingerprint,target_fingerprint,payload_fingerprint) VALUES (?,?,?,?,?,?)",values)!=1)throw new IllegalStateException("audit row missing");}); + }catch(RuntimeException failure){throw rejected("AUDIT_UNAVAILABLE");} + } + private static String fingerprint(String text,int limit){ + if(text==null)return "missing"; + byte[] bytes;try{bytes=field(text,limit);}catch(ReceiverException invalid){return "invalid_or_oversize";} + try{return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));} + catch(GeneralSecurityException impossible){throw new IllegalStateException("standard SHA-256 unavailable");} + } + private static ReceiverException rejected(String code){return new ReceiverException(code);} + @Override public String toString(){return "SqliteCommittedReceiver[authority=]";} +} diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedSnapshot.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedSnapshot.java new file mode 100644 index 00000000..8647210d --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedSnapshot.java @@ -0,0 +1,227 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.teaql.core.UserContext; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.meta.PropertyDescriptor; +import io.teaql.core.meta.Relation; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; +import static io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget.*; + +/** Trusted startup infrastructure. No ordinary business save or remote request routing. */ +public final class SqliteCommittedSnapshot { + private static final Set SNAPSHOT_FIELDS = Set.of("snapshot_version", "snapshot_id", + "source_runtime_id", "source_sequence", "source_model_hash", "audit", "entity_types", "rows"); + private static final Set AUDIT_FIELDS = Set.of("actor_ref", "comment", "purpose", "trace_id"); + private static final ObjectMapper JSON = new ObjectMapper() + .enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION) + .enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + public record Limits(int maxRows, int maxPayloadBytes) { + public Limits { if (maxRows < 1 || maxRows == Integer.MAX_VALUE || maxPayloadBytes < 1) + throw new IllegalArgumentException("positive bounded snapshot limits are required"); } + public static Limits defaults() { return new Limits(100_000, 16 * 1024 * 1024); } + } + public enum ImportMode { REQUIRE_EMPTY, REPLACE_UNINITIALIZED } + public record SnapshotRecord(String sourceRuntimeId, long sourceSequence, String snapshotId, + String modelHash, String payloadSha256, String payloadJson) { + @Override public String toString() { return "SnapshotRecord[sourceSequence=" + sourceSequence + ", payload=]"; } + } + private record Table(EntityDescriptor entity, List fields, + String table, String id, String version) { } + private final SqlExecutionAdapter sql; + private final String source; + private final String model; + + public SqliteCommittedSnapshot(SqlExecutionAdapter sql, String source, String model) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (blank(source) || model == null || !model.matches("sha256:[0-9a-f]{64}")) throw invalid("source/model is invalid"); + this.source = source; this.model = model; + } + + /** N and physical rows share one read transaction. Never export a caller's uncommitted writes. */ + public SnapshotRecord exportSnapshot(UserContext context, String snapshotId, + String comment, String purpose, Limits limits) { + Objects.requireNonNull(context, "context"); Objects.requireNonNull(limits, "limits"); + if (blank(snapshotId) || blank(comment) || blank(purpose)) throw invalid("snapshot identity and intent are required"); + standalone(); + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + AtomicReference result = new AtomicReference<>(); + sql.executeInTransaction(() -> { + requireTransaction(); + // The first SELECT establishes SQLite's read snapshot, including when N=0. + long n = position(); + List tables = tables(metadata); + ObjectNode body = JSON.createObjectNode(); + body.put("snapshot_version", 1); body.put("snapshot_id", snapshotId); + body.put("source_runtime_id", source); body.put("source_sequence", Long.toString(n)); + body.put("source_model_hash", model); + body.putObject("audit").put("comment", comment).put("purpose", purpose); + var catalog = body.putArray("entity_types"); var rows = body.putArray("rows"); + long usedBytes = 0; + for (Table table : tables) { + catalog.add(table.entity().getType()); + List columns = new ArrayList<>(List.of(quoted(table.id()) + " AS snapshot_id", quoted(table.version()) + " AS snapshot_version")); + for (int i = 0; i < table.fields().size(); i++) columns.add(quoted(mapping(table.fields().get(i)).column()) + " AS snapshot_field_" + i); + int remaining = limits.maxRows() - rows.size(); + for (Map values : sql.queryForList("SELECT " + String.join(",", columns) + + " FROM " + quoted(table.table()) + " ORDER BY " + quoted(table.id()) + " LIMIT ?", + new Object[] {remaining + 1})) { + if (rows.size() == limits.maxRows()) throw invalid("snapshot row budget exceeded"); + ObjectNode row = JSON.createObjectNode().put("entity_type", table.entity().getType()) + .put("entity_id", integer(values.get("snapshot_id")).toString()) + .put("version", integer(values.get("snapshot_version")).toString()); + ObjectNode fields = row.putObject("fields"); + for (int i = 0; i < table.fields().size(); i++) fields.set(table.fields().get(i).getName(), encode(table.fields().get(i), values.get("snapshot_field_" + i))); + // Same structural decoder/representability checks as the target, not Checker/Fix. + decodeSnapshotRow(metadata, row); + usedBytes += bytes(row.toString()); if (usedBytes > limits.maxPayloadBytes()) throw invalid("snapshot byte budget exceeded"); + rows.add(row); + } + } + String text = body.toString(); bounded(text, limits); + result.set(new SnapshotRecord(source, n, snapshotId, model, digest(text), text)); + }); + return result.get(); + } + + /** Explicit destructive replacement is allowed only before this target has any stream state. */ + public SqliteCommittedEventStore.InboxOutcome initializeSnapshot(UserContext context, SnapshotRecord record, Limits limits) { + return initializeSnapshot(context, record, ImportMode.REQUIRE_EMPTY, limits); + } + + /** Explicit destructive replacement is allowed only before this target has any stream state. */ + public SqliteCommittedEventStore.InboxOutcome initializeSnapshot(UserContext context, + SnapshotRecord record, ImportMode mode, Limits limits) { + Objects.requireNonNull(context, "context"); Objects.requireNonNull(mode, "mode"); Objects.requireNonNull(limits, "limits"); + sql.restrictOrdinaryMutations(); + standalone(); JsonNode body = validate(record, limits); + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + AtomicReference result = new AtomicReference<>(); + sql.executeInTransaction(() -> { + requireTransaction(); + List> prior = sql.queryForList("SELECT snapshot_id,source_sequence,model_hash,payload_sha256,payload_json " + + "FROM teaql_event_snapshot WHERE source_runtime_id=?", new Object[] {source}); + if (!prior.isEmpty()) { + Map old = prior.get(0); + if (!record.snapshotId().equals(old.get("snapshot_id")) || record.sourceSequence() != integer(old.get("source_sequence")) + || !model.equals(old.get("model_hash")) || !record.payloadSha256().equals(old.get("payload_sha256")) + || !record.payloadSha256().equals(digest((String) old.get("payload_json")))) throw conflict("snapshot identity/content conflicts"); + var checkpoint = sql.queryForList("SELECT source_sequence,model_hash FROM teaql_event_checkpoint WHERE source_runtime_id=?", new Object[] {source}); + if (checkpoint.size() != 1 || integer(checkpoint.get(0).get("source_sequence")) < record.sourceSequence() + || !model.equals(checkpoint.get(0).get("model_hash"))) throw invalid("snapshot checkpoint is inconsistent"); + result.set(SqliteCommittedEventStore.InboxOutcome.duplicate(record.sourceSequence())); return; + } + for (String table : List.of("teaql_event_snapshot", "teaql_event_checkpoint", "teaql_event_inbox", "teaql_event_applied_payload", "teaql_event_outbox", "teaql_event_sequence", "teaql_event_snapshot_publication", "teaql_event_delivery_journal")) + if (count(table) != 0) throw conflict("target already contains stream state"); + List
tables = tables(metadata); + Set catalog = new TreeSet<>(); + if (!body.path("entity_types").isArray()) throw invalid("snapshot catalog is missing"); + for (JsonNode type : body.path("entity_types")) if (!type.isTextual() || !catalog.add(type.textValue())) throw invalid("snapshot catalog is invalid"); + if (!catalog.equals(new TreeSet<>(tables.stream().map(t -> t.entity().getType()).toList()))) throw invalid("snapshot catalog differs from model"); + List mutations = new ArrayList<>(); Set identities = new HashSet<>(); + for (JsonNode row : body.path("rows")) { + Mutation mutation = decodeSnapshotRow(metadata, row); + if (!identities.add(mutation.table() + ":" + mutation.id())) throw invalid("duplicate snapshot row"); + mutations.add(mutation); + } + sql.execute("PRAGMA defer_foreign_keys=ON"); + for (Table table : tables) { + if (mode == ImportMode.REQUIRE_EMPTY && count(table.table()) != 0) throw conflict("target domain is not empty; explicit replacement required"); + if (mode == ImportMode.REPLACE_UNINITIALIZED) sql.execute("DELETE FROM " + quoted(table.table())); + } + for (Mutation mutation : mutations) applyOne(sql, mutation); + // Later triggers/rows must not rewrite earlier authoritative facts. + for (Mutation mutation : mutations) verifyReadback(sql, mutation); + for (Table table : tables) { + long expected = mutations.stream().filter(m -> m.table().equals(table.table())).count(); + if (count(table.table()) != expected) throw invalid("target snapshot row count differs"); + if (!sql.queryForList("PRAGMA foreign_key_check(" + quoted(table.table()) + ")", new Object[0]).isEmpty()) throw invalid("snapshot has a dangling reference"); + } + sql.update("INSERT INTO teaql_event_snapshot(source_runtime_id,snapshot_id,source_sequence,model_hash,payload_sha256,payload_json) VALUES (?,?,?,?,?,?)", + new Object[] {source, record.snapshotId(), record.sourceSequence(), model, record.payloadSha256(), record.payloadJson()}); + sql.update("INSERT INTO teaql_event_checkpoint(source_runtime_id,source_sequence,model_hash) VALUES (?,?,?)", new Object[] {source, record.sourceSequence(), model}); + result.set(SqliteCommittedEventStore.InboxOutcome.applied(record.sourceSequence())); + }); + return result.get(); + } + + JsonNode validate(SnapshotRecord record, Limits limits) { + if (record == null || !source.equals(record.sourceRuntimeId())) throw invalid("snapshot source is not authorized"); + if (!model.equals(record.modelHash())) throw invalid("snapshot model mismatch"); + if (blank(record.snapshotId()) || record.sourceSequence() < 0) throw invalid("snapshot identity/position is invalid"); + bounded(record.payloadJson(), limits); + if (!digest(record.payloadJson()).equals(record.payloadSha256())) throw invalid("snapshot digest mismatch"); + JsonNode body; + try { body = JSON.readTree(record.payloadJson()); } catch (Exception failure) { throw invalid("snapshot JSON is invalid"); } + if (!body.isObject() || body.size() != 8 || !body.path("snapshot_version").isInt() || body.path("snapshot_version").intValue() != 1 + || !record.snapshotId().equals(body.path("snapshot_id").textValue()) || !source.equals(body.path("source_runtime_id").textValue()) + || !Long.toString(record.sourceSequence()).equals(body.path("source_sequence").textValue()) || !model.equals(body.path("source_model_hash").textValue()) + || !body.path("rows").isArray() || body.path("rows").size() > limits.maxRows() + || blank(body.path("audit").path("comment").asText()) || blank(body.path("audit").path("purpose").asText())) throw invalid("snapshot header or rows are invalid"); + SqliteCommittedEventStore.rejectUnknownControls(body, SNAPSHOT_FIELDS); + SqliteCommittedEventStore.rejectUnknownControls(body.path("audit"), AUDIT_FIELDS); + return body; + } + private long position() { + var positions = sql.queryForList("SELECT current_sequence FROM teaql_event_sequence WHERE source_runtime_id=?", new Object[] {source}); + long n = positions.isEmpty() ? 0 : integer(positions.get(0).get("current_sequence")); + var latest = sql.queryForList("SELECT source_sequence,model_hash FROM teaql_event_outbox WHERE source_runtime_id=? ORDER BY source_sequence DESC LIMIT 1", new Object[] {source}); + if (n < 0 || n == 0 && !latest.isEmpty() || n > 0 && (latest.size() != 1 || integer(latest.get(0).get("source_sequence")) != n || !model.equals(latest.get(0).get("model_hash")))) throw invalid("source sequence/model is inconsistent"); + return n; + } + private static List
tables(EntityMetaFactory metadata) { + List
result = new ArrayList<>(); Set names = new HashSet<>(), physical = new HashSet<>(); + for (EntityDescriptor entity : metadata.allEntityDescriptors()) { + if (entity.getParent() != null || entity.hasChildren() || entity.findIdProperty() == null || entity.findVersionProperty() == null) throw invalid("unsupported snapshot entity mapping"); + Mapping id = mapping(entity.findIdProperty()), version = mapping(entity.findVersionProperty()); + String physicalName = id.table().toLowerCase(Locale.ROOT); + if (!names.add(entity.getType()) || !physical.add(physicalName) || physicalName.startsWith("teaql_event_") || !id.table().equals(version.table())) throw invalid("ambiguous/reserved snapshot table"); + var fields = entity.getOwnProperties().stream().filter(p -> !p.isId() && !p.isVersion()).toList(); + Set columns = new HashSet<>(List.of(id.column(), version.column())); + for (PropertyDescriptor field : fields) if (!id.table().equals(mapping(field).table()) || !columns.add(mapping(field).column())) throw invalid("unsupported snapshot field mapping"); + result.add(new Table(entity, fields, id.table(), id.column(), version.column())); + } + if (result.isEmpty()) throw invalid("snapshot requires an installed model"); + result.sort(Comparator.comparing(t -> t.entity().getType())); return result; + } + private static JsonNode encode(PropertyDescriptor property, Object value) { + JsonNode wire; + Class type = property.getType() == null ? null : property.getType().javaType(); + if (value == null) wire = JSON.nullNode(); + else if (property instanceof Relation relation) wire = JSON.createObjectNode() + .put("entity_type", relation.getReverseProperty().getOwner().getType()).put("entity_id", integer(value).toString()); + else if (type == Boolean.class || type == boolean.class) { + long number = integer(value); if (number != 0 && number != 1) throw invalid("source Boolean storage is invalid"); wire = JSON.valueToTree(number == 1); + } else if (type == BigDecimal.class || type == java.math.BigInteger.class) { + if (value instanceof Double || value instanceof Float) throw invalid("source Decimal is approximate REAL"); wire = JSON.valueToTree(value.toString()); + } else if (type == Float.class || type == float.class || type == Double.class || type == double.class) { + if (!(value instanceof Number number) || !Double.isFinite(number.doubleValue())) throw invalid("source floating storage is invalid"); wire = JSON.valueToTree(number.doubleValue()); + } else if (type == byte[].class && value instanceof byte[] bytes) wire = JSON.createObjectNode().put("$binary_base64", Base64.getEncoder().encodeToString(bytes)); + else if (value instanceof Byte || value instanceof Short || value instanceof Integer || value instanceof Long) wire = JSON.valueToTree(integer(value).toString()); + else if (value instanceof String text) wire = JSON.valueToTree(text); + else throw invalid("source storage representation is unsupported"); + Object decoded = decodeValue(property, wire); + if (!sameValue(new Field(mapping(property).column(), decoded, type == BigDecimal.class || type == java.math.BigInteger.class), value)) throw invalid("source field cannot be represented losslessly"); + return wire; + } + private long count(String table) { return integer(sql.queryForList("SELECT COUNT(*) AS n FROM " + quoted(table), new Object[0]).get(0).get("n")); } + private void standalone() { if (sql.hasActiveTransaction()) throw invalid("snapshot requires a standalone transaction"); } + private void requireTransaction() { if (!sql.hasActiveTransaction()) throw invalid("snapshot adapter must provide an active SQL transaction"); } + private static Long integer(Object value) { if (!(value instanceof Long || value instanceof Integer || value instanceof Short || value instanceof Byte)) throw invalid("source identity/integer storage is invalid"); return ((Number) value).longValue(); } + private static int bytes(String text) { return text.getBytes(StandardCharsets.UTF_8).length; } + private static void bounded(String text, Limits limits) { if (text == null || text.length() > limits.maxPayloadBytes() || bytes(text) > limits.maxPayloadBytes()) throw invalid("snapshot byte budget exceeded"); } + private static String digest(String text) { try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))); } catch (java.security.NoSuchAlgorithmException impossible) { throw new AssertionError(impossible); } } + private static boolean blank(String text) { return text == null || text.isBlank(); } + private static SqliteCommittedEventStore.CommittedEventException invalid(String message) { return new SqliteCommittedEventStore.CommittedEventException("INVALID_SNAPSHOT", message); } + private static SqliteCommittedEventStore.CommittedEventException conflict(String message) { return new SqliteCommittedEventStore.CommittedEventException("SNAPSHOT_CONFLICT", message); } +} diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedSnapshotPublisher.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedSnapshotPublisher.java new file mode 100644 index 00000000..040d9e23 --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedSnapshotPublisher.java @@ -0,0 +1,125 @@ +package io.teaql.core.sqlite; + +import io.teaql.core.UserContext; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedSnapshot.Limits; +import io.teaql.core.sqlite.SqliteCommittedSnapshot.SnapshotRecord; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +/** Durable startup bootstrap for one trusted source/target slot, not a remote API. */ +public final class SqliteCommittedSnapshotPublisher { + public record PreparedSnapshot(SnapshotRecord snapshot, boolean acknowledged) { } + /** Correlation, not authentication: the transport must authenticate its peer. */ + public record Receipt(String sourceRuntimeId, String targetId, String snapshotId, + long sourceSequence, String modelHash, String payloadSha256, boolean duplicate) { } + @FunctionalInterface public interface SnapshotSink { Receipt deliver(SnapshotRecord record); } + + private final SqlExecutionAdapter sql; + private final String source, target, model; + private final SqliteCommittedSnapshot snapshots; + public SqliteCommittedSnapshotPublisher(SqlExecutionAdapter sql, String source, String target, String model) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (target == null || target.isBlank()) throw failure("INVALID_SNAPSHOT", "target identity is required"); + this.source = source; this.target = target; this.model = model; + this.snapshots = new SqliteCommittedSnapshot(sql, source, model); + } + + /** Retain original bytes before delivery; retries never re-export a published identity. */ + public PreparedSnapshot prepare(UserContext context, String snapshotId, String comment, String purpose, Limits limits) { + require(context, snapshotId, limits); + if (comment == null || comment.isBlank() || purpose == null || purpose.isBlank()) + throw failure("INVALID_SNAPSHOT", "snapshot intent is required"); + PreparedSnapshot prior = transaction(() -> existing(snapshotId, limits)); + if (prior != null) return prior; + SnapshotRecord captured = snapshots.exportSnapshot(context, snapshotId, comment, purpose, limits); + return transaction(() -> { + PreparedSnapshot raced = existing(snapshotId, limits); + if (raced != null) return raced; // First durable record wins, never replace with a newer view. + if (!cursor().isEmpty()) throw failure("CURSOR_CONFLICT", "target stream has already started"); + sql.update("INSERT INTO teaql_event_snapshot_publication " + + "(source_runtime_id,target_id,snapshot_id,source_sequence,model_hash,payload_sha256,payload_json,acknowledged) " + + "VALUES (?,?,?,?,?,?,?,0)", new Object[] {source,target,captured.snapshotId(),captured.sourceSequence(),model,captured.payloadSha256(),captured.payloadJson()}); + return new PreparedSnapshot(captured, false); + }); + } + + /** One attempt. Lost or mismatching ACK leaves the retained job/cursor untouched. */ + public PreparedSnapshot publish(UserContext context, String snapshotId, Limits limits, SnapshotSink sink) { + require(context, snapshotId, limits); Objects.requireNonNull(sink, "sink"); + PreparedSnapshot prepared = transaction(() -> existing(snapshotId, limits)); + if (prepared == null) throw failure("INVALID_SNAPSHOT", "snapshot has not been durably prepared"); + if (prepared.acknowledged()) return prepared; + SnapshotRecord record = prepared.snapshot(); + var journal = new SqliteCommittedObservation(sql,source,target,model); + var attempt = journal.begin("snapshot",record.sourceSequence(),record.snapshotId(),record.payloadSha256()); + Receipt receipt; + try { + receipt = sink.deliver(record); // No source SQL transaction/connection held across transport. + if (receipt == null || !source.equals(receipt.sourceRuntimeId()) || !target.equals(receipt.targetId()) + || !record.snapshotId().equals(receipt.snapshotId()) || record.sourceSequence() != receipt.sourceSequence() + || !model.equals(receipt.modelHash()) || !record.payloadSha256().equals(receipt.payloadSha256())) + throw failure("ACK_MISMATCH", "snapshot receipt does not match this source/target record"); + } catch (RuntimeException error) { journal.failed(attempt,error); throw error; } + return journal.acknowledged(attempt, () -> { + PreparedSnapshot current = existing(snapshotId, limits); + if (current == null || !current.snapshot().equals(record)) throw failure("SNAPSHOT_CONFLICT", "prepared snapshot changed"); + if (current.acknowledged()) return current; // Tail may already have advanced; never rewind. + if (!cursor().isEmpty()) throw failure("CURSOR_CONFLICT", "bootstrap cursor changed concurrently"); + var status = new SqliteCommittedOutboxPublisher(sql,source,target,model).status(); + if (status.sourceHighWatermark() < record.sourceSequence()) throw failure("INVALID_SNAPSHOT", "snapshot is ahead of source"); + sql.update("INSERT INTO teaql_event_delivery_cursor(source_runtime_id,target_id,acked_sequence,model_hash) VALUES (?,?,?,?)", + new Object[] {source,target,record.sourceSequence(),model}); + if (sql.update("UPDATE teaql_event_snapshot_publication SET acknowledged=1 WHERE source_runtime_id=? AND target_id=? " + + "AND snapshot_id=? AND payload_sha256=? AND acknowledged=0", new Object[] {source,target,snapshotId,record.payloadSha256()}) != 1) + throw failure("CURSOR_CONFLICT", "bootstrap acknowledgement changed concurrently"); + return new PreparedSnapshot(record, true); + }); + } + + /** Target helper: construct a receipt only after the authoritative target transaction commits. */ + public static Receipt initializeAndAcknowledge(UserContext context, SqliteCommittedSnapshot target, + SnapshotRecord record, String targetId, Limits limits) { + return initializeAndAcknowledge(context,target,record,targetId,SqliteCommittedSnapshot.ImportMode.REQUIRE_EMPTY,limits); + } + public static Receipt initializeAndAcknowledge(UserContext context, SqliteCommittedSnapshot target, + SnapshotRecord record, String targetId, SqliteCommittedSnapshot.ImportMode mode, Limits limits) { + if (targetId == null || targetId.isBlank()) throw failure("INVALID_SNAPSHOT", "target identity is required"); + var outcome = target.initializeSnapshot(context,record,mode,limits); + if (outcome.checkpoint() != record.sourceSequence()) throw failure("ACK_MISMATCH", "target snapshot position differs"); + return new Receipt(record.sourceRuntimeId(),targetId,record.snapshotId(),record.sourceSequence(),record.modelHash(),record.payloadSha256(),outcome.duplicate()); + } + + private PreparedSnapshot existing(String snapshotId, Limits limits) { + var rows = sql.queryForList("SELECT snapshot_id,source_sequence,model_hash,payload_sha256,payload_json,acknowledged " + + "FROM teaql_event_snapshot_publication WHERE source_runtime_id=? AND target_id=?", new Object[] {source,target}); + if (rows.isEmpty()) return null; + var row = rows.get(0); + if (!snapshotId.equals(row.get("snapshot_id"))) throw failure("SNAPSHOT_CONFLICT", "target slot already has a different snapshot"); + var record = new SnapshotRecord(source,((Number)row.get("source_sequence")).longValue(),snapshotId, + (String)row.get("model_hash"),(String)row.get("payload_sha256"),(String)row.get("payload_json")); + snapshots.validate(record,limits); + int flag = ((Number)row.get("acknowledged")).intValue(); + if (flag != 0 && flag != 1) throw failure("INVALID_SNAPSHOT", "invalid acknowledgement state"); + boolean acked = flag == 1; + var cursors = cursor(); + if (!acked && !cursors.isEmpty() || acked && (cursors.size()!=1 || !model.equals(cursors.get(0).get("model_hash")) + || ((Number)cursors.get(0).get("acked_sequence")).longValue()> cursor() { return sql.queryForList( + "SELECT acked_sequence,model_hash FROM teaql_event_delivery_cursor WHERE source_runtime_id=? AND target_id=?",new Object[]{source,target}); } + private void require(UserContext context, String snapshotId, Limits limits) { + Objects.requireNonNull(context,"context"); Objects.requireNonNull(limits,"limits"); + EntityMetaFactory.requireFrom(context); + if (snapshotId == null || snapshotId.isBlank()) throw failure("INVALID_SNAPSHOT","snapshot identity is required"); + if (sql.hasActiveTransaction()) throw failure("INVALID_SNAPSHOT","bootstrap requires a standalone transaction"); + } + private T transaction(Supplier action) { AtomicReference value=new AtomicReference<>();sql.executeInTransaction(()->{ + if (!sql.hasActiveTransaction()) throw failure("INVALID_SNAPSHOT","bootstrap adapter must provide an active SQL transaction"); + value.set(action.get());});return value.get(); } + private static SqliteCommittedEventStore.CommittedEventException failure(String code,String message) { return new SqliteCommittedEventStore.CommittedEventException(code,message); } +} diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java new file mode 100644 index 00000000..7abb3316 --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -0,0 +1,445 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import io.teaql.core.UserContext; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.meta.PropertyDescriptor; +import io.teaql.core.meta.Relation; +import io.teaql.core.sql.SQLColumn; +import io.teaql.core.sql.portable.SQLPropertyUtil; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Startup-owned, one-source SQLite target for committed model facts. This is + * infrastructure, not a business-facing Mutation or remote query endpoint. + */ +public final class SqliteModelCheckedCommittedTarget { + private static final Set AUDIT_FIELDS = Set.of("actor_ref", "comment", "purpose", "trace_id"); + private static final Set MUTATION_FIELDS = Set.of("entity_type", "entity_id", "operation", + "base_version", "committed_version", "fields", "trace_chain"); + private static final Set TRACE_KINDS = Set.of( + "OPERATION", "REQUEST", "RELATION", "ENTITY", "PROVIDER", "SQL", + "COMMENT", "PURPOSE", "AUDIT_REASON"); + private static final Pattern DECLARED_DECIMAL = Pattern.compile( + "(?i)\\s*(?:NUMERIC|DECIMAL)\\s*\\(\\s*([0-9]+)\\s*,\\s*([0-9]+)\\s*\\)\\s*"); + private final SqlExecutionAdapter sql; + private final SqliteCommittedEventStore events; + private final String sourceRuntimeId; + private final String modelHash; + + public SqliteModelCheckedCommittedTarget(SqlExecutionAdapter sql, + String sourceRuntimeId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (sourceRuntimeId == null || sourceRuntimeId.isBlank() + || modelHash == null || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw invalid("target source identity or SHA-256 model hash is invalid"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.modelHash = modelHash; + sql.restrictOrdinaryMutations(); + this.events = new SqliteCommittedEventStore(sql); + } + + public void ensureSchema() { events.ensureSchema(); } + + /** One graph, its Inbox identity, original audit payload and checkpoint commit together. */ + public SqliteCommittedEventStore.InboxOutcome apply( + UserContext context, SqliteCommittedEventStore.OutboxRecord record) { + Objects.requireNonNull(context, "context"); + if (record == null) throw invalid("committed event is missing"); + if (!sourceRuntimeId.equals(record.sourceRuntimeId())) { + throw new SqliteCommittedEventStore.CommittedEventException( + "UNAUTHORIZED_SOURCE", "committed-event source is not authorized"); + } + // Resolve from this Context, never from the legacy process-global fallback. + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + AtomicReference result = new AtomicReference<>(); + sql.executeInTransaction(() -> result.set(events.apply(record, modelHash, + (transaction, payload) -> applyGraph(transaction, metadata, payload)))); + return result.get(); + } + + private static void applyGraph(SqlExecutionAdapter sql, EntityMetaFactory metadata, JsonNode payload) { + JsonNode audit = payload.path("audit"); + SqliteCommittedEventStore.rejectUnknownControls(audit, AUDIT_FIELDS); + if (!audit.isObject() || !nonblank(audit.path("comment")) + || !nonblank(audit.path("purpose")) + || !nonblank(payload.path("source_transaction_id"))) { + throw invalid("source transaction or audit intent is missing"); + } + try { OffsetDateTime.parse(payload.path("committed_at").asText()); } + catch (RuntimeException failure) { throw invalid("commit time is invalid"); } + List mutations = new ArrayList<>(); + Set seen = new HashSet<>(); + for (JsonNode fact : payload.path("mutations")) { + Mutation mutation = decode(metadata, fact); + if (!seen.add(mutation.entityType() + ":" + mutation.id())) { + throw invalid("one set mutates the same entity twice"); + } + mutations.add(mutation); + } + // Decode every fact before touching domain rows. The surrounding + // savepoint also rolls back a later FK or readback failure. + for (Mutation mutation : mutations) applyOne(sql, mutation); + } + + private static Mutation decode(EntityMetaFactory metadata, JsonNode fact) { + return decode(metadata, fact, false); + } + + /** Package-private physical restore mode; never accepted by ordinary event apply. */ + static Mutation decodeSnapshotRow(EntityMetaFactory metadata, JsonNode row) { + if (!row.isObject() || row.size() != 4 || !row.has("entity_type") + || !row.has("entity_id") || !row.has("version") || !row.has("fields")) { + throw invalid("snapshot row has wrong shape"); + } + long id = positiveId(row.path("entity_id")); + long version = signedVersion(row.path("version")); + if (!Long.toString(id).equals(row.path("entity_id").asText()) + || !Long.toString(version).equals(row.path("version").asText()) + || version == 0 || version == Long.MIN_VALUE) throw invalid("snapshot identity/version is invalid"); + var fact = row.deepCopy(); + var object = (com.fasterxml.jackson.databind.node.ObjectNode) fact; + object.remove("version"); object.put("committed_version", Long.toString(version)); + object.put("operation", "create"); object.putNull("base_version"); object.putArray("trace_chain"); + return decode(metadata, object, true); + } + + private static Mutation decode(EntityMetaFactory metadata, JsonNode fact, boolean snapshot) { + if (!fact.isObject() || !nonblank(fact.path("entity_type"))) { + throw invalid("mutation entity type is missing"); + } + SqliteCommittedEventStore.rejectUnknownControls(fact, MUTATION_FIELDS); + validateTraceChain(fact.path("trace_chain")); + String entityType = fact.path("entity_type").textValue(); + EntityDescriptor entity = metadata.allEntityDescriptors().stream() + .filter(candidate -> entityType.equals(candidate.getType())).findFirst() + .orElseThrow(() -> invalid("unknown entity type")); + // Multi-table inheritance needs an explicit target mapping. Reject it + // instead of silently writing only the leaf table. + if (entity.getParent() != null || entity.hasChildren()) { + throw invalid("multi-table entity target mapping is not supported"); + } + PropertyDescriptor idProperty = entity.findIdProperty(); + PropertyDescriptor versionProperty = entity.findVersionProperty(); + if (idProperty == null || versionProperty == null) { + throw invalid("entity has no ID or version property"); + } + Mapping idMapping = mapping(idProperty); + Mapping versionMapping = mapping(versionProperty); + if (!idMapping.table().equals(versionMapping.table())) { + throw invalid("identity and version span multiple tables"); + } + long id = positiveId(fact.path("entity_id")); + String operation = fact.path("operation").asText(""); + if (!List.of("create", "update", "delete", "recover").contains(operation)) { + throw invalid("unknown mutation operation"); + } + JsonNode baseNode = fact.path("base_version"); + Long base = "create".equals(operation) && baseNode.isNull() + ? null : signedVersion(baseNode); + long committed = signedVersion(fact.path("committed_version")); + boolean valid = switch (operation) { + case "create" -> base == null && (snapshot ? committed != 0 && committed != Long.MIN_VALUE : committed == 1); + case "update" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == base + 1; + case "delete" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == -(base + 1); + case "recover" -> base != null && base < 0 && base != Long.MIN_VALUE + && -base < Long.MAX_VALUE && committed == -base + 1; + default -> false; + }; + if (!valid) throw invalid("invalid version transition"); + JsonNode supplied = fact.path("fields"); + if (!supplied.isObject()) throw invalid("mutation fields are missing"); + if ("delete".equals(operation) && !supplied.isEmpty()) { + throw invalid("delete must be a tombstone"); + } + List ownProperties = entity.getOwnProperties(); + supplied.fieldNames().forEachRemaining(name -> { + PropertyDescriptor property = entity.findProperty(name); + if (property == null || !ownProperties.contains(property) + || property.isId() || property.isVersion()) { + throw invalid("unknown or reserved model field"); + } + }); + List fields = new ArrayList<>(); + for (PropertyDescriptor property : ownProperties) { + if (property.isId() || property.isVersion()) continue; + JsonNode value = supplied.get(property.getName()); + if (value == null) { + if ("create".equals(operation)) throw invalid("create omitted a modeled field"); + continue; + } + Mapping column = mapping(property); + if (!idMapping.table().equals(column.table())) { + throw invalid("mutation field spans multiple tables"); + } + Class type = property.getType() == null ? null : property.getType().javaType(); + fields.add(new Field(column.column(), decodeValue(property, value), + type == BigDecimal.class || type == java.math.BigInteger.class)); + } + return new Mutation(entityType, idMapping.table(), idMapping.column(), + versionMapping.column(), id, base, committed, operation, fields); + } + + private static void validateTraceChain(JsonNode chain) { + if (!chain.isArray()) throw invalid("mutation trace chain is missing"); + for (JsonNode node : chain) { + JsonNode id = node.path("entity_id"); + if (!node.isObject() || node.size() != 4 + || !TRACE_KINDS.contains(node.path("kind").asText("")) + || !node.path("name").isTextual() + || !node.path("comment").isTextual() + || !(id.isNull() || id.isTextual() && id.textValue().matches("[1-9][0-9]*"))) { + throw invalid("trace node has wrong shape"); + } + } + } + + static Object decodeValue(PropertyDescriptor property, JsonNode value) { + if (value.isNull()) { + if (property.getBoolean("required", false)) throw invalid("non-null field is null"); + return null; + } + if (property instanceof Relation relation) { + if (!value.isObject() || value.size() != 2 || relation.getReverseProperty() == null + || relation.getReverseProperty().getOwner() == null + || !relation.getReverseProperty().getOwner().getType() + .equals(value.path("entity_type").asText())) { + throw invalid("reference target does not match model"); + } + return positiveId(value.path("entity_id")); + } + Class type = property.getType() == null ? null : property.getType().javaType(); + if (type == java.time.LocalDate.class || type == java.sql.Date.class) { + if (!value.isTextual()) throw invalid("Date field has wrong type"); + String text = value.textValue(); + try { + if (!java.time.LocalDate.parse(text).toString().equals(text)) { + throw invalid("Date field has wrong type"); + } + } catch (java.time.format.DateTimeParseException failure) { + throw invalid("Date field has wrong type"); + } + return text; + } + if (type == java.time.Instant.class || type == java.util.Date.class + || type == java.sql.Timestamp.class) { + if (!value.isTextual() || !value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) { + throw invalid("Timestamp field has wrong type"); + } + try { return Long.parseLong(value.textValue()); } + catch (NumberFormatException failure) { throw invalid("Timestamp field has wrong type"); } + } + if (type == java.time.LocalDateTime.class) { + throw invalid("zone-free LocalDateTime has no committed instant mapping"); + } + if (type == String.class || type == java.util.UUID.class || type != null && type.isEnum()) { + if (!value.isTextual()) throw invalid("text field has wrong type"); + return value.textValue(); + } + if (type == Boolean.class || type == boolean.class) { + if (!value.isBoolean()) throw invalid("Boolean field has wrong type"); + return value.booleanValue() ? 1 : 0; + } + if (type == Long.class || type == long.class || type == Integer.class || type == int.class + || type == Short.class || type == short.class || type == Byte.class || type == byte.class) { + if (!value.isTextual() || !value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) { + throw invalid("integer field has wrong type"); + } + long number; + try { number = Long.parseLong(value.textValue()); } + catch (NumberFormatException failure) { throw invalid("integer field overflows"); } + if (type == Integer.class || type == int.class) { + if (number < Integer.MIN_VALUE || number > Integer.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Short.class || type == short.class) { + if (number < Short.MIN_VALUE || number > Short.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Byte.class || type == byte.class) { + if (number < Byte.MIN_VALUE || number > Byte.MAX_VALUE) throw invalid("integer overflows field"); + } + return number; + } + if (type == BigDecimal.class || type == java.math.BigInteger.class) { + if (!value.isTextual()) throw invalid("Decimal field has wrong type"); + try { + if (type == java.math.BigInteger.class) { + if (!value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) { + throw invalid("integer field has wrong type"); + } + return new java.math.BigInteger(value.textValue()).toString(); + } + BigDecimal decimal = new BigDecimal(value.textValue()); + if (!fitsDeclaredDecimal(property, decimal)) { + throw invalid("Decimal lacks or exceeds modeled precision/scale"); + } + return decimal.toPlainString(); + } + catch (NumberFormatException failure) { throw invalid("Decimal field is invalid"); } + } + if (type == Double.class || type == double.class || type == Float.class || type == float.class) { + if (!value.isNumber() || !Double.isFinite(value.doubleValue())) { + throw invalid("floating field has wrong type"); + } + return value.doubleValue(); + } + if (type == byte[].class && value.isObject() && value.size() == 1 + && value.path("$binary_base64").isTextual()) { + try { return java.util.Base64.getDecoder().decode(value.path("$binary_base64").textValue()); } + catch (IllegalArgumentException failure) { throw invalid("binary field is invalid"); } + } + throw invalid("unsupported modeled field type"); + } + + static void applyOne(SqlExecutionAdapter sql, Mutation mutation) { + String table = quoted(mutation.table()); + String idColumn = quoted(mutation.idColumn()); + String versionColumn = quoted(mutation.versionColumn()); + List> current = sql.queryForList("SELECT " + versionColumn + + " AS committed_version FROM " + table + " WHERE " + idColumn + " = ?", + new Object[] {mutation.id()}); + Long actualVersion = current.isEmpty() ? null : ((Number) current.get(0).get("committed_version")).longValue(); + if (!Objects.equals(actualVersion, mutation.baseVersion())) { + throw new SqliteCommittedEventStore.CommittedEventException("VERSION_CONFLICT", + "committed-event predecessor version mismatch"); + } + List args = new ArrayList<>(); + if ("create".equals(mutation.operation())) { + List columns = new ArrayList<>(List.of(idColumn, versionColumn)); + args.add(mutation.id()); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + columns.add(quoted(field.column())); + args.add(field.value()); + } + sql.update("INSERT INTO " + table + " (" + String.join(", ", columns) + ") VALUES (" + + String.join(", ", java.util.Collections.nCopies(args.size(), "?")) + ")", + args.toArray()); + } else { + List assignments = new ArrayList<>(List.of(versionColumn + " = ?")); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + assignments.add(quoted(field.column()) + " = ?"); + args.add(field.value()); + } + args.add(mutation.id()); + args.add(mutation.baseVersion()); + int changed = sql.update("UPDATE " + table + " SET " + String.join(", ", assignments) + + " WHERE " + idColumn + " = ? AND " + versionColumn + " = ?", args.toArray()); + if (changed != 1) { + throw new SqliteCommittedEventStore.CommittedEventException("VERSION_CONFLICT", + "committed-event predecessor version mismatch"); + } + } + verifyReadback(sql, mutation); + } + + static void verifyReadback(SqlExecutionAdapter sql, Mutation mutation) { + List projections = new ArrayList<>(List.of(quoted(mutation.versionColumn()) + " AS committed_version")); + for (int i = 0; i < mutation.fields().size(); i++) { + projections.add(quoted(mutation.fields().get(i).column()) + " AS field_" + i); + } + List> rows = sql.queryForList("SELECT " + String.join(", ", projections) + + " FROM " + quoted(mutation.table()) + " WHERE " + quoted(mutation.idColumn()) + " = ?", + new Object[] {mutation.id()}); + if (rows.size() != 1 || !Objects.equals(mutation.committedVersion(), + ((Number) rows.get(0).get("committed_version")).longValue())) { + throw invalid("target did not retain committed version"); + } + for (int i = 0; i < mutation.fields().size(); i++) { + if (!sameValue(mutation.fields().get(i), rows.get(0).get("field_" + i))) { + throw invalid("target did not retain committed field value"); + } + } + } + + static boolean sameValue(Field field, Object actual) { + Object expected = field.value(); + if (expected instanceof byte[] bytes) return actual instanceof byte[] other && Arrays.equals(bytes, other); + if (expected == null || actual == null) return expected == actual; + // SQLite REAL is approximate even when Double.toString() happens to + // match the exact source decimal (for example, 0.1). + if (field.decimal() && (actual instanceof Double || actual instanceof Float)) return false; + if (field.decimal() || expected instanceof Number && actual instanceof Number) { + try { return new BigDecimal(expected.toString()).compareTo(new BigDecimal(actual.toString())) == 0; } + catch (NumberFormatException failure) { return false; } + } + return Objects.equals(expected, actual); + } + + static Mapping mapping(PropertyDescriptor property) { + List columns = SQLPropertyUtil.getColumns(property); + if (columns.size() != 1) throw invalid("property has no single SQLite column"); + return new Mapping(columns.get(0).getTableName(), columns.get(0).getColumnName()); + } + + private static boolean fitsDeclaredDecimal(PropertyDescriptor property, BigDecimal value) { + List columns = SQLPropertyUtil.getColumns(property); + if (columns.size() != 1) return false; + String declaredType = columns.get(0).getType(); + if (declaredType == null) return false; + Matcher declaration = DECLARED_DECIMAL.matcher(declaredType); + if (!declaration.matches()) return false; + try { + int precision = Integer.parseInt(declaration.group(1)); + int scale = Integer.parseInt(declaration.group(2)); + if (precision < 1 || scale > precision) return false; + BigDecimal normalized = value.stripTrailingZeros(); + long integerDigits = value.signum() == 0 ? 0 + : Math.max(0L, (long) normalized.precision() - normalized.scale()); + long fractionalDigits = Math.max(0L, normalized.scale()); + return integerDigits <= precision - scale && fractionalDigits <= scale; + } catch (NumberFormatException invalidDeclaration) { + return false; + } + } + + static String quoted(String identifier) { + if (identifier == null || identifier.isBlank()) throw invalid("model has an empty SQL identifier"); + return "\"" + identifier.replace("\"", "\"\"") + "\""; + } + + private static long positiveId(JsonNode value) { + if (!value.isTextual()) throw invalid("identity must be a decimal string"); + try { + long id = Long.parseLong(value.textValue()); + if (id > 0) return id; + } catch (NumberFormatException ignored) { } + throw invalid("identity is outside the SQLite ID range"); + } + + private static long signedVersion(JsonNode value) { + if (!value.isTextual()) throw invalid("version must be a decimal string"); + try { return Long.parseLong(value.textValue()); } + catch (NumberFormatException failure) { throw invalid("version must be a decimal string"); } + } + + private static boolean nonblank(JsonNode value) { return value.isTextual() && !value.textValue().isBlank(); } + private static SqliteCommittedEventStore.CommittedEventException invalid(String reason) { + return new SqliteCommittedEventStore.CommittedEventException( + "INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } + + record Mapping(String table, String column) { } + record Field(String column, Object value, boolean decimal) { } + record Mutation(String entityType, String table, String idColumn, + String versionColumn, long id, Long baseVersion, long committedVersion, + String operation, List fields) { } +} diff --git a/teaql-sqlite/src/main/java/module-info.java b/teaql-sqlite/src/main/java/module-info.java index 98b73c4b..82c0d1ef 100644 --- a/teaql-sqlite/src/main/java/module-info.java +++ b/teaql-sqlite/src/main/java/module-info.java @@ -4,6 +4,8 @@ requires transitive io.teaql.utils; requires transitive io.teaql.dataservice.sql; requires io.teaql.provider.jdbc; + requires io.teaql.runtime; + requires com.fasterxml.jackson.databind; requires org.xerial.sqlitejdbc; requires java.sql; exports io.teaql.core.sqlite; diff --git a/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java new file mode 100644 index 00000000..5eb5c4f0 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java @@ -0,0 +1,301 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedEventStore.CommittedEventException; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +/** Small SQLite graph fixture shared in meaning with CMR v1; no generated ERP model. */ +public class SqliteCommittedEventStoreTest { + private static final String MODEL_HASH = "sha256:" + "0".repeat(64); + private static final String SOURCE = "sqlite-order-source"; + private static final long ORDER_ID = 9_007_199_254_740_993L; + private static final long ITEM_ID = ORDER_ID + 1; + + private record Database(JdbcSqlExecutor driver, SqliteCommittedEventStore events) {} + + private static Database database(Path path) { + SQLiteDataSource source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor driver = new JdbcSqlExecutor(source); + return new Database(driver, new SqliteCommittedEventStore(driver)); + } + + private static void graphSchema(SqlExecutionAdapter sql) { + sql.execute("CREATE TABLE orders (id INTEGER PRIMARY KEY, status TEXT NOT NULL, version INTEGER NOT NULL)"); + sql.execute("CREATE TABLE order_items (id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL, " + + "quantity INTEGER NOT NULL, version INTEGER NOT NULL)"); + } + + private static long count(SqlExecutionAdapter sql, String table) { + // This helper only receives the two fixed table names from this test. + return ((Number) sql.queryForList("SELECT count(*) AS total FROM " + table, + new Object[0]).get(0).get("total")).longValue(); + } + + private static String payload(long sequence, String change, boolean submitted) { + String operation = submitted ? "update" : "create"; + String itemOperation = submitted ? "delete" : "create"; + String status = submitted ? "SUBMITTED" : "DRAFT"; + String version = submitted ? "2" : "1"; + String itemVersion = submitted ? "-2" : "1"; + String baseVersion = submitted ? "\"1\"" : "null"; + return (""" + {"protocol_version":1,"source_runtime_id":"%s","source_sequence":"%d", + "change_set_id":"%s","source_model_hash":"%s","mutations":[ + {"entity_type":"order","entity_id":"%d","operation":"%s", + "base_version":%s,"committed_version":"%s","fields":{"status":"%s"}}, + {"entity_type":"order_item","entity_id":"%d","operation":"%s", + "base_version":%s,"committed_version":"%s", + "fields":{"order":{"entity_type":"order","entity_id":"%d"},"quantity":"2"}} + ]} + """).formatted(SOURCE, sequence, change, MODEL_HASH, ORDER_ID, operation, + baseVersion, version, status, ITEM_ID, itemOperation, baseVersion, + itemVersion, ORDER_ID); + } + + private static void applyGraph(SqlExecutionAdapter sql, JsonNode facts) { + for (JsonNode mutation : facts.path("mutations")) { + long id = Long.parseLong(mutation.path("entity_id").asText()); + long version = Long.parseLong(mutation.path("committed_version").asText()); + String operation = mutation.path("operation").asText(); + if (mutation.path("entity_type").asText().equals("order")) { + String status = mutation.path("fields").path("status").asText(); + if (operation.equals("create")) { + sql.update("INSERT INTO orders (id, status, version) VALUES (?, ?, ?)", + new Object[] {id, status, version}); + } else { + sql.update("UPDATE orders SET status = ?, version = ? WHERE id = ?", + new Object[] {status, version, id}); + } + } else if (operation.equals("create")) { + long parent = Long.parseLong(mutation.path("fields").path("order").path("entity_id").asText()); + int quantity = mutation.path("fields").path("quantity").asInt(); + sql.update("INSERT INTO order_items (id, order_id, quantity, version) VALUES (?, ?, ?, ?)", + new Object[] {id, parent, quantity, version}); + } else { + sql.update("UPDATE order_items SET version = ? WHERE id = ?", new Object[] {version, id}); + } + } + } + + private static OutboxRecord append(Database database, String change, boolean submitted) { + AtomicReference result = new AtomicReference<>(); + database.driver().executeInTransaction(() -> { + if (submitted) { + database.driver().update("UPDATE orders SET status = 'SUBMITTED', version = 2 WHERE id = ?", + new Object[] {ORDER_ID}); + database.driver().update("UPDATE order_items SET version = -2 WHERE id = ?", + new Object[] {ITEM_ID}); + } else { + database.driver().update("INSERT INTO orders (id, status, version) VALUES (?, 'DRAFT', 1)", + new Object[] {ORDER_ID}); + database.driver().update("INSERT INTO order_items (id, order_id, quantity, version) " + + "VALUES (?, ?, 2, 1)", new Object[] {ITEM_ID, ORDER_ID}); + } + result.set(database.events().append(SOURCE, change, MODEL_HASH, + sequence -> payload(sequence, change, submitted))); + }); + return result.get(); + } + + private static CommittedEventException expectCode(String code, Runnable action) { + try { + action.run(); + fail("expected " + code); + throw new AssertionError("unreachable"); + } catch (CommittedEventException failure) { + assertEquals(code, failure.code()); + return failure; + } + } + + @Test public void sourceRollbackAndRestartPreserveAtomicOrderedFacts() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-source-", ".sqlite"); + try { + Database source = database(path); + source.events().ensureSchema(); + graphSchema(source.driver()); + try { + source.driver().executeInTransaction(() -> { + source.driver().update("INSERT INTO orders (id, status, version) VALUES (?, 'DRAFT', 1)", + new Object[] {ORDER_ID}); + source.driver().update("INSERT INTO order_items (id, order_id, quantity, version) " + + "VALUES (?, ?, 2, 1)", new Object[] {ITEM_ID, ORDER_ID}); + source.events().append(SOURCE, "change-a", MODEL_HASH, + sequence -> payload(sequence, "change-a", false)); + throw new IllegalStateException("roll back complete graph"); + }); + fail("expected source rollback"); + } catch (IllegalStateException expected) { + assertEquals("roll back complete graph", expected.getMessage()); + } + assertEquals(0, count(source.driver(), "orders")); + assertEquals(0, count(source.driver(), "order_items")); + assertTrue(source.events().outboxAfter(SOURCE, 0, 10).isEmpty()); + + OutboxRecord first = append(source, "change-a", false); + OutboxRecord second = append(source, "change-b", true); + assertFalse("raw facts must not appear in ordinary formatting", first.toString().contains("DRAFT")); + assertFalse(first.toString().contains("order_item")); + assertEquals(1, first.sourceSequence()); + assertEquals(2, second.sourceSequence()); + + Database reopened = database(path); + reopened.events().ensureSchema(); + List rows = reopened.events().outboxAfter(SOURCE, 0, 10); + assertEquals(List.of(first, second), rows); + assertEquals(List.of(second), reopened.events().outboxAfter(SOURCE, 1, 1)); + assertEquals(1, count(reopened.driver(), "orders")); + assertEquals(1, count(reopened.driver(), "order_items")); + Map item = reopened.driver().queryForList( + "SELECT version FROM order_items WHERE id = ?", new Object[] {ITEM_ID}).get(0); + assertEquals(-2, ((Number) item.get("version")).longValue()); + } finally { + Files.deleteIfExists(path); + } + } + + @Test public void targetApplyDeduplicatesAndRollsBackWholeGraphOnFailure() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-target-", ".sqlite"); + try { + Database target = database(path); + target.events().ensureSchema(); + graphSchema(target.driver()); + OutboxRecord first = record(1, "change-a", false); + OutboxRecord second = record(2, "change-b", true); + + target.driver().executeInTransaction(() -> expectCode("SEQUENCE_GAP", + () -> target.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + assertEquals(0, count(target.driver(), "orders")); + assertEquals(0, count(target.driver(), "teaql_event_inbox")); + + target.driver().executeInTransaction(() -> { + var outcome = target.events().apply(first, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + assertFalse(outcome.duplicate()); + assertEquals(1, outcome.checkpoint()); + }); + target.driver().executeInTransaction(() -> { + expectCode("MODEL_MISMATCH", () -> target.events().apply(second, + "sha256:" + "1".repeat(64), SqliteCommittedEventStoreTest::applyGraph)); + try { + target.events().apply(second, MODEL_HASH, (sql, facts) -> { + sql.update("UPDATE orders SET status = 'PARTIAL' WHERE id = ?", new Object[] {ORDER_ID}); + sql.update("UPDATE order_items SET version = -99 WHERE id = ?", new Object[] {ITEM_ID}); + throw new IllegalStateException("injected second-table failure"); + }); + fail("expected injected failure"); + } catch (IllegalStateException expected) { + assertEquals("injected second-table failure", expected.getMessage()); + } + }); + assertEquals("DRAFT", target.driver().queryForList("SELECT status FROM orders", new Object[0]) + .get(0).get("status")); + assertEquals(1L, ((Number) target.driver().queryForList("SELECT version FROM order_items", + new Object[0]).get(0).get("version")).longValue()); + assertEquals(1, count(target.driver(), "teaql_event_inbox")); + assertEquals(1, count(target.driver(), "teaql_event_applied_payload")); + + Database reopened = database(path); + reopened.driver().executeInTransaction(() -> { + var outcome = reopened.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + assertFalse(outcome.duplicate()); + assertEquals(2, outcome.checkpoint()); + var duplicate = reopened.events().apply(second, MODEL_HASH, (sql, facts) -> { + throw new AssertionError("duplicate must not rematerialize"); + }); + assertTrue(duplicate.duplicate()); + assertEquals(2, duplicate.checkpoint()); + var olderDuplicate = reopened.events().apply(first, MODEL_HASH, (sql, facts) -> { + throw new AssertionError("an older duplicate must not rematerialize"); + }); + assertTrue(olderDuplicate.duplicate()); + assertEquals(1, olderDuplicate.checkpoint()); + }); + assertEquals("SUBMITTED", reopened.driver().queryForList("SELECT status FROM orders", new Object[0]) + .get(0).get("status")); + assertEquals(-2L, ((Number) reopened.driver().queryForList( + "SELECT version FROM order_items", new Object[0]).get(0).get("version")).longValue()); + assertEquals(2, count(reopened.driver(), "teaql_event_inbox")); + assertEquals(2, count(reopened.driver(), "teaql_event_applied_payload")); + assertEquals(2L, ((Number) reopened.driver().queryForList( + "SELECT source_sequence FROM teaql_event_checkpoint", new Object[0]) + .get(0).get("source_sequence")).longValue()); + + OutboxRecord conflicted = record(2, "change-b", false); + reopened.driver().executeInTransaction(() -> expectCode("DUPLICATE_CONFLICT", + () -> reopened.events().apply(conflicted, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + OutboxRecord invalidDigest = new OutboxRecord(second.sourceRuntimeId(), second.sourceSequence(), + second.changeSetId(), second.modelHash(), "f".repeat(64), second.payloadJson()); + reopened.driver().executeInTransaction(() -> expectCode("INVALID_ENVELOPE", + () -> reopened.events().apply(invalidDigest, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + } finally { + Files.deleteIfExists(path); + } + } + + @Test public void appendAndApplyRequireTheCallerTransaction() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-boundary-", ".sqlite"); + try { + Database database = database(path); + database.events().ensureSchema(); + OutboxRecord first = record(1, "change-a", false); + expectCode("INVALID_ENVELOPE", () -> database.events().append(SOURCE, "change-a", + MODEL_HASH, sequence -> payload(sequence, "change-a", false))); + expectCode("INVALID_ENVELOPE", () -> database.events().apply(first, + MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph)); + assertEquals(0, count(database.driver(), "teaql_event_outbox")); + } finally { + Files.deleteIfExists(path); + } + } + + @Test public void duplicateRequiresCheckpointCoveringRetainedInbox() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-checkpoint-", ".sqlite"); + try { + Database target = database(path); + target.events().ensureSchema(); + graphSchema(target.driver()); + OutboxRecord first = record(1, "change-a", false); + OutboxRecord second = record(2, "change-b", true); + target.driver().executeInTransaction(() -> { + target.events().apply(first, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + target.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + }); + + target.driver().update("UPDATE teaql_event_checkpoint SET source_sequence = 1 " + + "WHERE source_runtime_id = ?", new Object[] {SOURCE}); + target.driver().executeInTransaction(() -> expectCode("INVALID_ENVELOPE", + () -> target.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + + target.driver().update("DELETE FROM teaql_event_checkpoint WHERE source_runtime_id = ?", + new Object[] {SOURCE}); + target.driver().executeInTransaction(() -> expectCode("INVALID_ENVELOPE", + () -> target.events().apply(first, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + assertEquals(2, count(target.driver(), "teaql_event_inbox")); + assertEquals(2, count(target.driver(), "teaql_event_applied_payload")); + } finally { + Files.deleteIfExists(path); + } + } + + private static OutboxRecord record(long sequence, String change, boolean submitted) throws Exception { + String json = payload(sequence, change, submitted); + byte[] bytes = java.security.MessageDigest.getInstance("SHA-256") + .digest(json.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + return new OutboxRecord(SOURCE, sequence, change, MODEL_HASH, + java.util.HexFormat.of().formatHex(bytes), json); + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index c0159efb..ad981399 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -5,15 +5,29 @@ import io.teaql.core.sql.GenericSQLProperty; import io.teaql.core.sql.SQLEntityDescriptor; import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import io.teaql.provider.jdbc.JdbcSqlExecutor; import io.teaql.runtime.*; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import javax.sql.DataSource; import io.teaql.core.checker.Checker; import io.teaql.core.checker.CheckException; import io.teaql.core.checker.FixEvidence; @@ -27,13 +41,24 @@ public class GraphTraceSqliteTest { public static final class GraphEntity extends BaseEntity { private final String type; private final Map values = new HashMap<>(); + boolean failPostCommitReconcile; public GraphEntity(String type) { this.type = type; } @Override public String typeName() { return type; } @Override public Object __internalGet(String field) { - return field.equals("name") || field.equals("memo") || field.equals("children") ? values.get(field) : super.__internalGet(field); + return field.equals("name") || field.equals("memo") || field.equals("children") + || field.equals("business_counter") || field.equals("planned_date") + || field.equals("active") || field.equals("ratio") || field.equals("optional_note") + || field.equals("occurred_at") || field.equals("wall_time") + ? values.get(field) : super.__internalGet(field); } @Override public void __internalSet(String field, Object value) { - if (field.equals("name") || field.equals("memo") || field.equals("children")) values.put(field, value); + if (failPostCommitReconcile && field.equals("version")) { + throw new IllegalStateException("diagnostic reconciliation failure"); + } + if (field.equals("name") || field.equals("memo") || field.equals("children") + || field.equals("business_counter") || field.equals("planned_date") + || field.equals("active") || field.equals("ratio") || field.equals("optional_note") + || field.equals("occurred_at") || field.equals("wall_time")) values.put(field, value); else super.__internalSet(field, value); } } @@ -41,10 +66,12 @@ public static final class GraphEntity extends BaseEntity { static final class Fixture { final List sql = new CopyOnWriteArrayList<>(); final List audit = new CopyOnWriteArrayList<>(); + final List rawAudit = new CopyOnWriteArrayList<>(); final List commands = new CopyOnWriteArrayList<>(); final List results = new CopyOnWriteArrayList<>(); boolean queryLogging = true; boolean mutationLogging = true; + boolean failStandardAudit; final List itemInsertBatchSizes = new CopyOnWriteArrayList<>(); final List itemUpdateBatchSizes = new CopyOnWriteArrayList<>(); final List itemDeleteBatchSizes = new CopyOnWriteArrayList<>(); @@ -52,14 +79,24 @@ static final class Fixture { final JdbcSqlExecutor driver; final SQLiteDataSource dataSource; final DefaultUserContext context; + final SqliteCommittedMutationRecorder committedRecorder; volatile boolean failReadback; + volatile boolean errorReadback; + volatile boolean errorReadbackWithoutSuppression; volatile boolean unknownUpdateCounts; - Fixture() throws Exception { + Fixture() throws Exception { this(false); } + + Fixture(boolean captureCommittedMutations) throws Exception { + this(captureCommittedMutations, source -> source); + } + + Fixture(boolean captureCommittedMutations, + Function transactionSource) throws Exception { var ds = new SQLiteDataSource(); dataSource = ds; ds.setUrl("jdbc:sqlite:" + Files.createTempFile("teaql-graph-trace-", ".db")); - driver = new JdbcSqlExecutor(ds) { + driver = new JdbcSqlExecutor(transactionSource.apply(ds)) { @Override public int[] batchUpdate(String text, List rows) { if (text.startsWith("INSERT INTO order_item_data")) { itemInsertBatchSizes.add(rows.size()); @@ -79,6 +116,13 @@ static final class Fixture { return counts; } @Override public List> queryForList(String text, Object[] args) { + if (errorReadbackWithoutSuppression && text.startsWith("SELECT * FROM") + && text.contains("customer_order_data")) { + throw new NoSuppressionError(); + } + if (errorReadback && text.startsWith("SELECT * FROM") && text.contains("customer_order_data")) { + throw new AssertionError("synthetic Error after business write"); + } if (failReadback && text.startsWith("SELECT * FROM") && text.contains("customer_order_data")) { // A real driver failure after a successful write, inside the same transaction. execute("DROP TABLE customer_order_data"); @@ -86,6 +130,9 @@ static final class Fixture { return super.queryForList(text, args); } }; + committedRecorder = captureCommittedMutations + ? new SqliteCommittedMutationRecorder(driver, "java-graph-source", "sha256:" + "0".repeat(64)) + : null; var metadata = new SimpleEntityMetaFactory(); for (String type : List.of("CustomerOrder", "OrderItem", "Payment", "PaymentAttempt", "Shipment")) { var descriptor = new SQLEntityDescriptor(); @@ -100,6 +147,26 @@ static final class Fixture { textField ? String.class : Long.class); property.setColumnType(textField ? "VARCHAR(255)" : "BIGINT"); } + if (captureCommittedMutations && type.equals("CustomerOrder")) { + var counter = (GenericSQLProperty) descriptor.addSimpleProperty("business_counter", Long.class); + counter.setColumnType("BIGINT"); + var plannedDate = (GenericSQLProperty) descriptor.addSimpleProperty( + "planned_date", java.time.LocalDate.class); + plannedDate.setColumnType("DATE"); + var active = (GenericSQLProperty) descriptor.addSimpleProperty("active", Boolean.class); + active.setColumnType("BOOLEAN"); + var ratio = (GenericSQLProperty) descriptor.addSimpleProperty("ratio", Double.class); + ratio.setColumnType("DOUBLE"); + var optionalNote = (GenericSQLProperty) descriptor.addSimpleProperty( + "optional_note", String.class); + optionalNote.setColumnType("VARCHAR(255)"); + var occurredAt = (GenericSQLProperty) descriptor.addSimpleProperty( + "occurred_at", java.time.Instant.class); + occurredAt.setColumnType("BIGINT"); + var wallTime = (GenericSQLProperty) descriptor.addSimpleProperty( + "wall_time", java.time.LocalDateTime.class); + wallTime.setColumnType("TIMESTAMP"); + } var children = new Relation(); children.setName("children"); children.setOwner(descriptor); @@ -124,9 +191,20 @@ static final class Fixture { } }; var ids = new AtomicLong(1000); - var runtime = TeaQLRuntime.builder().metadata(metadata).dataService("sqlite", provider) + var builder = TeaQLRuntime.builder().metadata(metadata).dataService("sqlite", provider) .idGenerationService((caller, entity) -> ids.getAndIncrement()) - .logSink((caller, entry) -> sql.add(entry)).build(); + .logSink(new RuntimeLogSink() { + @Override public void writeExecutionLog(UserContext caller, ExecutionMetadata entry) { + sql.add(entry); + } + + @Override public void writeAuditEvent(UserContext caller, RawAuditEvent event) { + if (failStandardAudit) throw new IllegalStateException("standard audit sink failure"); + rawAudit.add(event); + } + }); + if (committedRecorder != null) builder.committedMutationRecorder(committedRecorder); + var runtime = builder.build(); context = new DefaultUserContext(runtime) { @Override public boolean isQueryExecutionLoggingEnabled() { return queryLogging; } @Override public boolean isMutationExecutionLoggingEnabled() { return mutationLogging; } @@ -137,7 +215,7 @@ static final class Fixture { } void clear() { - sql.clear(); audit.clear(); commands.clear(); results.clear(); itemInsertBatchSizes.clear(); + sql.clear(); audit.clear(); rawAudit.clear(); commands.clear(); results.clear(); itemInsertBatchSizes.clear(); itemUpdateBatchSizes.clear(); itemDeleteBatchSizes.clear(); itemRecoverBatchSizes.clear(); } @@ -150,6 +228,12 @@ GraphEntity create(String type, long id, String name) { } } + private static final class NoSuppressionError extends Error { + private NoSuppressionError() { + super("synthetic Error without suppressed diagnostics", null, false, false); + } + } + private static void await(CountDownLatch latch) { try { assertTrue("overlapping execution must reach its checkpoint", latch.await(10, TimeUnit.SECONDS)); @@ -445,6 +529,648 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertEquals(Long.valueOf(-2), removed.getVersion()); } + @Test public void graphSaveCapturesAuthoritativeRowsAndOutboxInTheSameTransaction() throws Exception { + var fixture = new Fixture(true); + assertThrows(SecurityException.class, () -> fixture.committedRecorder.ensureSchema(fixture.context, null)); + fixture.driver.execute("CREATE TRIGGER order_db_default AFTER INSERT ON customer_order_data " + + "BEGIN UPDATE customer_order_data SET memo = 'database trigger value' WHERE id = NEW.id; END"); + var order = fixture.create("CustomerOrder", 100, "root value"); + order.updateProperty("business_counter", 9_007_199_254_740_993L); + order.updateProperty("planned_date", java.time.LocalDate.of(2026, 10, 16)); + order.updateProperty("active", true); + order.updateProperty("ratio", 0.5); + order.updateProperty("optional_note", null); + order.updateProperty("occurred_at", java.time.Instant.parse("1960-01-01T00:00:00.123Z")); + var item = fixture.create("OrderItem", 201, "first item"); + item.setComment("create item"); + var payment = fixture.create("Payment", 300, "payment value"); + order.__internalSet("children", List.of(item, payment)); + order.auditAs("create complete order graph").save(fixture.context); + + var events = fixture.committedRecorder.eventStore(); + var first = events.outboxAfter("java-graph-source", 0, 10); + assertEquals("one graph save must produce one durable set", 1, first.size()); + JsonNode created = new ObjectMapper().readTree(first.get(0).payloadJson()); + assertEquals("create complete order graph", created.path("audit").path("comment").asText()); + assertEquals(3, created.path("mutations").size()); + JsonNode orderFact = null; + for (JsonNode fact : created.path("mutations")) { + assertEquals("create", fact.path("operation").asText()); + assertTrue(fact.path("base_version").isNull()); + assertEquals("1", fact.path("committed_version").asText()); + assertTrue(fact.path("trace_chain").isArray()); + for (JsonNode node : fact.path("trace_chain")) { + assertEquals(4, node.size()); + assertTrue(node.has("kind")); + assertTrue(node.has("name")); + assertTrue(node.has("entity_id")); + assertTrue(node.has("comment")); + assertTrue(node.path("kind").asText().matches("[A-Z_]+")); + } + if (fact.path("entity_type").asText().equals("CustomerOrder")) orderFact = fact; + } + assertNotNull(orderFact); + assertEquals("database trigger value", orderFact.path("fields").path("memo").asText()); + assertTrue(orderFact.path("fields").path("business_counter").isTextual()); + assertEquals("9007199254740993", orderFact.path("fields").path("business_counter").asText()); + assertEquals("2026-10-16", orderFact.path("fields").path("planned_date").asText()); + assertTrue(orderFact.path("fields").path("active").isBoolean()); + assertTrue(orderFact.path("fields").path("active").booleanValue()); + assertEquals(0.5, orderFact.path("fields").path("ratio").doubleValue(), 0.0); + assertTrue(orderFact.path("fields").path("optional_note").isNull()); + assertEquals("-315619199877", orderFact.path("fields").path("occurred_at").asText()); + assertFalse(new CommittedMutationFact("CustomerOrder", "100", "create", null, "1", + Map.of("memo", "database trigger value"), List.of()).toString().contains("database trigger value")); + assertEquals("database trigger value", fixture.driver.queryForList( + "SELECT memo FROM customer_order_data WHERE id = ?", new Object[] {100L}) + .get(0).get("memo")); + Map instant = fixture.driver.queryForList( + "SELECT typeof(occurred_at) AS storage_type, occurred_at " + + "FROM customer_order_data WHERE id = ?", new Object[] {100L}).get(0); + assertEquals("integer", instant.get("storage_type")); + assertEquals(-315_619_199_877L, ((Number) instant.get("occurred_at")).longValue()); + + item.updateProperty("name", "revised item"); + item.setComment("revise item"); + order.auditAs("revise order graph").save(fixture.context); + var second = events.outboxAfter("java-graph-source", 1, 10); + assertEquals(1, second.size()); + JsonNode changed = new ObjectMapper().readTree(second.get(0).payloadJson()); + assertEquals(1, changed.path("mutations").size()); + JsonNode changedItem = changed.path("mutations").get(0); + assertEquals("update", changedItem.path("operation").asText()); + assertEquals("1", changedItem.path("base_version").asText()); + assertEquals("2", changedItem.path("committed_version").asText()); + assertEquals("revised item", changedItem.path("fields").path("name").asText()); + assertEquals(1, changedItem.path("fields").size()); + assertFalse(changedItem.path("fields").has("memo")); + + payment.markForDeletion(); + payment.setComment("remove payment"); + order.auditAs("remove payment from order").save(fixture.context); + var third = events.outboxAfter("java-graph-source", 2, 10); + assertEquals(1, third.size()); + JsonNode tombstone = new ObjectMapper().readTree(third.get(0).payloadJson()).path("mutations").get(0); + assertEquals("delete", tombstone.path("operation").asText()); + assertEquals("1", tombstone.path("base_version").asText()); + assertEquals("-2", tombstone.path("committed_version").asText()); + assertEquals(0, tombstone.path("fields").size()); + + fixture.driver.execute("CREATE UNIQUE INDEX payment_name_unique_for_outbox ON payment_data(name)"); + var duplicate = fixture.create("Payment", 301, "payment value"); + order.__internalSet("children", List.of(item, payment, duplicate)); + assertThrows(RuntimeException.class, + () -> order.auditAs("reject duplicate payment graph").save(fixture.context)); + assertEquals("failed save must leave no publishable set", 3, + events.outboxAfter("java-graph-source", 0, 10).size()); + assertTrue(fixture.driver.queryForList("SELECT id FROM payment_data WHERE id = ?", + new Object[] {301L}).isEmpty()); + } + + @Test public void committedUpdateIncludesDatabaseTriggerChangeButOmitsUnchangedFields() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 909, "before"); + order.auditAs("create trigger comparison order").save(fixture.context); + fixture.driver.execute("CREATE TRIGGER order_update_memo AFTER UPDATE OF name ON customer_order_data " + + "WHEN NEW.name = 'after' BEGIN UPDATE customer_order_data " + + "SET memo = 'database revised memo' WHERE id = NEW.id; END"); + + order.updateProperty("name", "after"); + order.auditAs("update trigger comparison order").save(fixture.context); + var outbox = fixture.committedRecorder.eventStore().outboxAfter("java-graph-source", 1, 10); + assertEquals(1, outbox.size()); + JsonNode fact = new ObjectMapper().readTree(outbox.get(0).payloadJson()) + .path("mutations").get(0); + assertEquals("update", fact.path("operation").asText()); + assertEquals("after", fact.path("fields").path("name").asText()); + assertEquals("database revised memo", fact.path("fields").path("memo").asText()); + assertEquals(2, fact.path("fields").size()); + } + + @Test public void committedUpdateDistinguishesNullTransitionsFromUnchangedFields() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 911, "nullable order"); + order.updateProperty("optional_note", null); + order.auditAs("create nullable order").save(fixture.context); + + order.updateProperty("optional_note", "submitted note"); + order.auditAs("set optional note").save(fixture.context); + var outbox = fixture.committedRecorder.eventStore(); + JsonNode set = new ObjectMapper().readTree( + outbox.outboxAfter("java-graph-source", 1, 10).get(0).payloadJson()); + JsonNode fields = set.path("mutations").get(0).path("fields"); + assertEquals("submitted note", fields.path("optional_note").asText()); + assertEquals(1, fields.size()); + + order.updateProperty("optional_note", null); + order.auditAs("clear optional note").save(fixture.context); + JsonNode cleared = new ObjectMapper().readTree( + outbox.outboxAfter("java-graph-source", 2, 10).get(0).payloadJson()) + .path("mutations").get(0).path("fields"); + assertTrue(cleared.has("optional_note")); + assertTrue(cleared.path("optional_note").isNull()); + assertEquals(1, cleared.size()); + assertNull(fixture.driver.queryForList( + "SELECT optional_note FROM customer_order_data WHERE id = ?", new Object[] {911L}) + .get(0).get("optional_note")); + } + + @Test public void committedLocalDateTimeFailsClosedWithoutAnInstantMapping() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 912, "wall-clock order"); + order.updateProperty("wall_time", java.time.LocalDateTime.of(2026, 10, 16, 8, 0)); + var failure = assertThrows(RuntimeException.class, + () -> order.auditAs("reject ambiguous wall time").save(fixture.context)); + assertTrue(failure.getMessage().contains("LocalDateTime")); + assertTrue(fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {912L}).isEmpty()); + assertTrue(fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).isEmpty()); + } + + @Test public void committedInstantRejectsSubMillisecondPrecision() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 913, "precise order"); + order.updateProperty("occurred_at", java.time.Instant.parse("2026-10-16T00:00:00.123456Z")); + assertThrows(RuntimeException.class, + () -> order.auditAs("reject submillisecond instant").save(fixture.context)); + assertTrue(fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {913L}).isEmpty()); + assertTrue(fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).isEmpty()); + } + + @Test public void committedRecoveryKeepsPredecessorAndNeedsNoChangedScalarField() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 910, "recoverable"); + order.auditAs("create recoverable order").save(fixture.context); + order.markForDeletion(); + order.auditAs("delete recoverable order").save(fixture.context); + order.markAsRecover(); + order.auditAs("recover order").save(fixture.context); + + var outbox = fixture.committedRecorder.eventStore().outboxAfter("java-graph-source", 2, 10); + assertEquals(1, outbox.size()); + JsonNode fact = new ObjectMapper().readTree(outbox.get(0).payloadJson()) + .path("mutations").get(0); + assertEquals("recover", fact.path("operation").asText()); + assertEquals("-2", fact.path("base_version").asText()); + assertEquals("3", fact.path("committed_version").asText()); + assertEquals(0, fact.path("fields").size()); + } + + @Test public void brokenOutboxRollsBackNormalGraphSaveAndDoesNotDeliverAudit() throws Exception { + var fixture = new Fixture(true); + fixture.driver.execute("DROP TABLE teaql_event_outbox"); + var order = fixture.create("CustomerOrder", 100, "must roll back"); + var item = fixture.create("OrderItem", 201, "must roll back too"); + order.__internalSet("children", List.of(item)); + assertThrows(RuntimeException.class, + () -> order.auditAs("fail together when outbox unavailable").save(fixture.context)); + assertTrue(fixture.driver.queryForList("SELECT id FROM customer_order_data WHERE id = ?", + new Object[] {100L}).isEmpty()); + assertTrue(fixture.driver.queryForList("SELECT id FROM order_item_data WHERE id = ?", + new Object[] {201L}).isEmpty()); + assertTrue("no post-commit audit on rolled back save", fixture.audit.isEmpty()); + } + + @Test public void lostCommitAcknowledgementAfterDurableWriteRequiresReload() throws Exception { + assertCommitAcknowledgementUnknown(true); + } + + @Test public void failedCommitBeforeWriteRequiresTheSameConservativeReload() throws Exception { + assertCommitAcknowledgementUnknown(false); + } + + @Test public void commitErrorAfterDurableWriteRequiresReload() throws Exception { + var armed = new AtomicBoolean(); + var injected = new AtomicBoolean(); + var fixture = new Fixture(true, + source -> commitErrorAfterWrite(source, armed, injected)); + var order = fixture.create("CustomerOrder", 193, "commit Error after durable write"); + armed.set(true); + + assertThrows(NoSuppressionError.class, + () -> order.auditAs("probe uncertain commit Error").save(fixture.context)); + assertTrue("the commit Error must be injected", injected.get()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {193L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject replay after commit Error").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + } + + @Test public void commitFailureAndRollbackErrorMustNotRestoreAutoCommit() throws Exception { + var armed = new AtomicBoolean(); + var rollbackInjected = new AtomicBoolean(); + var restoredAutoCommit = new AtomicBoolean(); + var fixture = new Fixture(true, source -> commitRollbackErrorFault( + source, armed, rollbackInjected, restoredAutoCommit)); + var order = fixture.create("CustomerOrder", 195, "commit and rollback uncertain"); + armed.set(true); + + var failure = assertThrows(TransactionOutcomeUnknownException.class, + () -> order.auditAs("probe commit and rollback Error").save(fixture.context)); + assertTrue(failure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN")); + assertTrue("rollback Error must be injected", rollbackInjected.get()); + assertFalse("unconfirmed rollback must not restore autoCommit", restoredAutoCommit.get()); + assertTrue("failed commit must not materialize the business row", fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {195L}).isEmpty()); + assertTrue("failed commit must not materialize Outbox", fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).isEmpty()); + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject replay after commit and rollback Error").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + } + + @Test public void failedRollbackMustNotImplicitlyCommitOnAutoCommitRestoration() throws Exception { + var armed = new AtomicBoolean(); + var injected = new AtomicBoolean(); + var restoredAutoCommit = new AtomicBoolean(); + var fixture = new Fixture(true, + source -> rollbackFault(source, armed, injected, restoredAutoCommit)); + fixture.driver.execute("DROP TABLE teaql_event_outbox"); + armed.set(true); + var order = fixture.create("CustomerOrder", 187, "rollback failed after Outbox error"); + + var failure = assertThrows(RuntimeException.class, + () -> order.auditAs("probe uncertain rollback outcome").save(fixture.context)); + assertTrue("the rollback failure must be injected", injected.get()); + assertFalse("restoring autoCommit after failed rollback can commit pending writes", + restoredAutoCommit.get()); + assertEquals(0, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {187L}).size()); + assertTrue(failure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN")); + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject replay after failed rollback").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + } + + @Test public void rollbackErrorMustNotRestoreAutoCommitOrPermitReplay() throws Exception { + var armed = new AtomicBoolean(); + var injected = new AtomicBoolean(); + var restoredAutoCommit = new AtomicBoolean(); + var fixture = new Fixture(true, + source -> rollbackFault(source, armed, injected, restoredAutoCommit, true)); + fixture.driver.execute("DROP TABLE teaql_event_outbox"); + var order = fixture.create("CustomerOrder", 194, "rollback Error after Outbox failure"); + armed.set(true); + + assertThrows(NoSuppressionError.class, + () -> order.auditAs("probe rollback Error").save(fixture.context)); + assertTrue("the rollback Error must be injected", injected.get()); + assertFalse("unconfirmed rollback must not restore autoCommit", restoredAutoCommit.get()); + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject replay after rollback Error").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + } + + @Test public void failedRollbackAfterErrorRequiresReloadBeforeRetry() throws Exception { + var armed = new AtomicBoolean(); + var injected = new AtomicBoolean(); + var restoredAutoCommit = new AtomicBoolean(); + var fixture = new Fixture(false, + source -> rollbackFault(source, armed, injected, restoredAutoCommit)); + var order = fixture.create("CustomerOrder", 190, "error before rollback"); + fixture.errorReadback = true; + armed.set(true); + + var failure = assertThrows(AssertionError.class, + () -> order.auditAs("probe rollback failure after Error").save(fixture.context)); + assertEquals("synthetic Error after business write", failure.getMessage()); + assertTrue("rollback fault must be injected", injected.get()); + assertFalse("failed rollback must not restore autoCommit", restoredAutoCommit.get()); + fixture.errorReadback = false; + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject retry after unknown Error outcome").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + var independent = fixture.create("Payment", 191, "independent graph after Error"); + independent.auditAs("save independent graph after unknown Error outcome") + .save(fixture.context); + assertEquals(Long.valueOf(1), independent.getVersion()); + } + + @Test public void failedRollbackAfterUnsuppressibleErrorRequiresReloadBeforeRetry() throws Exception { + var armed = new AtomicBoolean(); + var injected = new AtomicBoolean(); + var restoredAutoCommit = new AtomicBoolean(); + var fixture = new Fixture(false, + source -> rollbackFault(source, armed, injected, restoredAutoCommit)); + var order = fixture.create("CustomerOrder", 192, "unsuppressible error before rollback"); + fixture.errorReadbackWithoutSuppression = true; + armed.set(true); + + assertThrows(NoSuppressionError.class, + () -> order.auditAs("probe unsuppressible Error rollback failure").save(fixture.context)); + assertTrue("rollback fault must be injected", injected.get()); + assertFalse("failed rollback must not restore autoCommit", restoredAutoCommit.get()); + fixture.errorReadbackWithoutSuppression = false; + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject retry after unsuppressible Error").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + } + + private static void assertCommitAcknowledgementUnknown(boolean afterActualCommit) throws Exception { + var armed = new AtomicBoolean(); + var injected = new AtomicBoolean(); + var fixture = new Fixture(true, + source -> commitAcknowledgementFault(source, armed, injected, afterActualCommit)); + armed.set(true); + var order = fixture.create("CustomerOrder", afterActualCommit ? 189 : 188, + "commit acknowledgement may be lost"); + var item = fixture.create("OrderItem", afterActualCommit ? 289 : 288, + "same uncertain transaction"); + order.__internalSet("children", List.of(item)); + + var warnings = new ArrayList(); + var logger = Logger.getLogger(JdbcSqlExecutor.class.getName()); + Handler capture = new Handler() { + @Override public void publish(LogRecord record) { warnings.add(record.getMessage()); } + @Override public void flush() { } + @Override public void close() { } + }; + logger.addHandler(capture); + RuntimeException failure; + try { + failure = assertThrows(RuntimeException.class, + () -> order.auditAs("probe uncertain commit outcome").save(fixture.context)); + } finally { + logger.removeHandler(capture); + } + assertTrue("fault must reach the actual commit boundary", injected.get()); + assertEquals(afterActualCommit ? 1 : 0, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {order.getId()}).size()); + assertEquals(afterActualCommit ? 1 : 0, fixture.driver.queryForList( + "SELECT id FROM order_item_data WHERE id = ?", new Object[] {item.getId()}).size()); + assertEquals(afterActualCommit ? 1 : 0, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + assertTrue("the outcome must not be reported as a known rollback", + failure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN")); + assertFalse("the public message must not contain the JDBC diagnostic", + failure.getMessage().contains("COMMIT-ACK-SECRET-CANARY")); + assertTrue(warnings.stream().anyMatch(message -> message.contains("outcome unknown"))); + assertTrue("the operational warning must not contain the JDBC diagnostic", + warnings.stream().noneMatch(message -> message.contains("COMMIT-ACK-SECRET-CANARY"))); + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject replay before reload").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + var childRetryFailure = assertThrows(TeaQLRuntimeException.class, + () -> item.auditAs("reject child replay before reload").save(fixture.context)); + assertTrue(childRetryFailure.getMessage().contains("TRANSACTION_OUTCOME_UNKNOWN_RELOAD_REQUIRED")); + assertThrows(TeaQLRuntimeException.class, + () -> order.setEntityMutationLedger(new EntityMutationLedger())); + } + + private static DataSource commitAcknowledgementFault(SQLiteDataSource source, + AtomicBoolean armed, AtomicBoolean injected, boolean afterActualCommit) { + return (DataSource) Proxy.newProxyInstance(DataSource.class.getClassLoader(), + new Class[] {DataSource.class}, (proxy, method, args) -> { + Object result = invoke(source, method, args); + if (!method.getName().equals("getConnection")) return result; + Connection actual = (Connection) result; + return Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] {Connection.class}, (connectionProxy, operation, parameters) -> { + if (operation.getName().equals("commit") && armed.get() + && injected.compareAndSet(false, true)) { + if (afterActualCommit) invoke(actual, operation, parameters); + throw new SQLException("COMMIT-ACK-SECRET-CANARY"); + } + return invoke(actual, operation, parameters); + }); + }); + } + + private static DataSource commitErrorAfterWrite(SQLiteDataSource source, + AtomicBoolean armed, AtomicBoolean injected) { + return (DataSource) Proxy.newProxyInstance(DataSource.class.getClassLoader(), + new Class[] {DataSource.class}, (proxy, method, args) -> { + Object result = invoke(source, method, args); + if (!method.getName().equals("getConnection")) return result; + Connection actual = (Connection) result; + return Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] {Connection.class}, (connectionProxy, operation, parameters) -> { + if (operation.getName().equals("commit") && armed.get() + && injected.compareAndSet(false, true)) { + invoke(actual, operation, parameters); + throw new NoSuppressionError(); + } + return invoke(actual, operation, parameters); + }); + }); + } + + private static DataSource commitRollbackErrorFault(SQLiteDataSource source, + AtomicBoolean armed, AtomicBoolean rollbackInjected, AtomicBoolean restoredAutoCommit) { + return (DataSource) Proxy.newProxyInstance(DataSource.class.getClassLoader(), + new Class[] {DataSource.class}, (proxy, method, args) -> { + Object result = invoke(source, method, args); + if (!method.getName().equals("getConnection")) return result; + Connection actual = (Connection) result; + return Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] {Connection.class}, (connectionProxy, operation, parameters) -> { + if (operation.getName().equals("commit") && armed.get()) { + throw new SQLException("COMMIT-ACK-SECRET-CANARY"); + } + if (operation.getName().equals("rollback") && armed.get() + && rollbackInjected.compareAndSet(false, true)) { + throw new NoSuppressionError(); + } + if (operation.getName().equals("setAutoCommit") + && rollbackInjected.get() && Boolean.TRUE.equals(parameters[0])) { + restoredAutoCommit.set(true); + } + return invoke(actual, operation, parameters); + }); + }); + } + + private static DataSource rollbackFault(SQLiteDataSource source, + AtomicBoolean armed, AtomicBoolean injected, AtomicBoolean restoredAutoCommit) { + return rollbackFault(source, armed, injected, restoredAutoCommit, false); + } + + private static DataSource rollbackFault(SQLiteDataSource source, + AtomicBoolean armed, AtomicBoolean injected, AtomicBoolean restoredAutoCommit, + boolean throwError) { + return (DataSource) Proxy.newProxyInstance(DataSource.class.getClassLoader(), + new Class[] {DataSource.class}, (proxy, method, args) -> { + Object result = invoke(source, method, args); + if (!method.getName().equals("getConnection")) return result; + Connection actual = (Connection) result; + return Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] {Connection.class}, (connectionProxy, operation, parameters) -> { + if (operation.getName().equals("rollback") && armed.get() + && injected.compareAndSet(false, true)) { + if (throwError) throw new NoSuppressionError(); + throw new SQLException("ROLLBACK-SECRET-CANARY"); + } + if (operation.getName().equals("setAutoCommit") && injected.get() + && Boolean.TRUE.equals(parameters[0])) { + restoredAutoCommit.set(true); + } + return invoke(actual, operation, parameters); + }); + }); + } + + private static Object invoke(Object target, Method method, Object[] args) throws Throwable { + try { + return method.invoke(target, args); + } catch (InvocationTargetException failure) { + throw failure.getCause(); + } + } + + @Test public void postCommitAuditFailureReportsCommittedOutcomeWithoutReplayingLedger() throws Exception { + var fixture = new Fixture(true); + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> { + throw new IllegalStateException("diagnostic audit sink failure"); + }); + var order = fixture.create("CustomerOrder", 199, "committed before audit delivery"); + + var failure = assertThrows(AuditDeliveryAfterCommitException.class, + () -> order.auditAs("diagnose post-commit audit failure").save(fixture.context)); + assertTrue(failure.getMessage().contains("AUDIT_DELIVERY_AFTER_COMMIT")); + assertEquals(1, failure.failureCount()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {199L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> fixture.audit.add(event)); + order.auditAs("retry after audit delivery failure").save(fixture.context); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + + @Test public void postCommitReconciliationFailureMustReportCommittedOutcome() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 196, "reconcile after commit"); + order.failPostCommitReconcile = true; + + var failure = assertThrows(CommittedStateReconciliationException.class, + () -> order.auditAs("diagnose committed reconciliation failure").save(fixture.context)); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {196L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals("committed mutations still require application audit delivery", 1, fixture.audit.size()); + assertEquals(1, fixture.rawAudit.size()); + assertEquals(Long.valueOf(1), fixture.rawAudit.get(0).resultingVersion()); + assertTrue("committed outcome must be explicit to prevent blind retry", + failure.getMessage().contains("provider write completed")); + order.failPostCommitReconcile = false; + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("retry committed reconciliation failure").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("COMMITTED_STATE_RELOAD_REQUIRED")); + assertThrows(TeaQLRuntimeException.class, + () -> order.setEntityMutationLedger(new EntityMutationLedger())); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals(1, fixture.audit.size()); + var independent = fixture.create("Payment", 195, "unrelated graph may still save"); + independent.auditAs("save independent graph after failure").save(fixture.context); + assertEquals(2, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + + @Test public void postCommitChildReconciliationFailurePoisonsOnlyItsSaveGraph() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 194, "root already committed"); + var item = fixture.create("OrderItem", 294, "child needs reload"); + item.failPostCommitReconcile = true; + order.__internalSet("children", List.of(item)); + + assertThrows(CommittedStateReconciliationException.class, + () -> order.auditAs("commit graph before child reconciliation").save(fixture.context)); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {194L}).size()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM order_item_data WHERE id = ?", new Object[] {294L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + + item.failPostCommitReconcile = false; + assertThrows(TeaQLRuntimeException.class, + () -> item.auditAs("unsafe child retry").save(fixture.context)); + assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("unsafe root retry").save(fixture.context)); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals("both committed graph members must still be audited", 2, fixture.audit.size()); + assertEquals(2, fixture.rawAudit.size()); + } + + @Test public void postCommitReconciliationAndAuditFailureRetainBothOutcomes() throws Exception { + var fixture = new Fixture(true); + var attempts = new java.util.concurrent.atomic.AtomicInteger(); + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> { + attempts.incrementAndGet(); + throw new IllegalStateException("diagnostic audit sink failure"); + }); + var order = fixture.create("CustomerOrder", 193, "committed despite two post-commit failures"); + order.failPostCommitReconcile = true; + + var failure = assertThrows(CommittedStateReconciliationException.class, + () -> order.auditAs("commit before reconciliation and audit failures").save(fixture.context)); + assertEquals(1, attempts.get()); + assertEquals(1, failure.getSuppressed().length); + assertTrue(failure.getSuppressed()[0] instanceof AuditDeliveryAfterCommitException); + assertEquals(1, fixture.rawAudit.size()); + assertEquals(Long.valueOf(1), fixture.rawAudit.get(0).resultingVersion()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {193L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + order.failPostCommitReconcile = false; + assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject retry after two post-commit failures").save(fixture.context)); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + + @Test public void postCommitAuditFailureStillAttemptsEveryGraphMember() throws Exception { + var fixture = new Fixture(true); + var attempts = new java.util.concurrent.atomic.AtomicInteger(); + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> { + attempts.incrementAndGet(); + throw new IllegalStateException("diagnostic audit sink failure"); + }); + var order = fixture.create("CustomerOrder", 198, "root audit fails"); + var item = fixture.create("OrderItem", 298, "child audit fails"); + order.__internalSet("children", List.of(item)); + + var failure = assertThrows(AuditDeliveryAfterCommitException.class, + () -> order.auditAs("commit graph before audit delivery").save(fixture.context)); + assertEquals(2, failure.failureCount()); + assertEquals(2, attempts.get()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {198L}).size()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM order_item_data WHERE id = ?", new Object[] {298L}).size()); + } + + @Test public void standardAuditSinkFailureDoesNotSuppressApplicationAuditDelivery() throws Exception { + var fixture = new Fixture(true); + fixture.failStandardAudit = true; + var order = fixture.create("CustomerOrder", 197, "both audit sinks are attempted"); + + var failure = assertThrows(AuditDeliveryAfterCommitException.class, + () -> order.auditAs("commit before standard audit delivery").save(fixture.context)); + assertEquals(1, failure.failureCount()); + assertEquals(1, fixture.audit.size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + @Test public void realSqliteProviderFailureKeepsAttemptedLineageWithoutCommittedAudit() throws Exception { var fixture = new Fixture(); fixture.driver.execute("CREATE UNIQUE INDEX payment_name_unique ON payment_data(name)"); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/JdbcCommittedCleanupSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/JdbcCommittedCleanupSqliteTest.java new file mode 100644 index 00000000..6c471370 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/JdbcCommittedCleanupSqliteTest.java @@ -0,0 +1,147 @@ +package io.teaql.sqlite; + +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** A committed JDBC write must not look like a rolled-back, retryable save. */ +public class JdbcCommittedCleanupSqliteTest { + private static final String DIAGNOSTIC_CANARY = "POST-COMMIT-CLEANUP-SECRET-CANARY"; + private enum Fault { AUTO_COMMIT_RESTORE, CLOSE, AUTO_COMMIT_RESTORE_ERROR, CLOSE_ERROR } + + private static final class NoSuppressionError extends Error { + private NoSuppressionError() { + super(DIAGNOSTIC_CANARY, null, false, false); + } + } + + @Test public void failedAutoCommitRestoreAfterCommitKeepsCommittedOutcome() throws Exception { + verifyCommittedCleanupFailure(Fault.AUTO_COMMIT_RESTORE); + } + + @Test public void failedCloseAfterCommitKeepsCommittedOutcome() throws Exception { + verifyCommittedCleanupFailure(Fault.CLOSE); + } + + @Test public void failedWarningHandlerAfterCommitKeepsCommittedOutcome() throws Exception { + verifyCommittedCleanupFailure(Fault.AUTO_COMMIT_RESTORE, true); + } + + @Test public void autoCommitRestoreErrorAfterCommitKeepsCommittedOutcome() throws Exception { + verifyCommittedCleanupFailure(Fault.AUTO_COMMIT_RESTORE_ERROR); + } + + @Test public void closeErrorAfterCommitKeepsCommittedOutcome() throws Exception { + verifyCommittedCleanupFailure(Fault.CLOSE_ERROR); + } + + @Test public void warningHandlerErrorAfterCommitKeepsCommittedOutcome() throws Exception { + verifyCommittedCleanupFailure(Fault.AUTO_COMMIT_RESTORE, false, true); + } + + private static void verifyCommittedCleanupFailure(Fault fault) throws Exception { + verifyCommittedCleanupFailure(fault, false); + } + + private static void verifyCommittedCleanupFailure(Fault fault, boolean warningHandlerFails) + throws Exception { + verifyCommittedCleanupFailure(fault, warningHandlerFails, false); + } + + private static void verifyCommittedCleanupFailure(Fault fault, boolean warningHandlerFails, + boolean warningHandlerThrowsError) throws Exception { + var backing = new SQLiteDataSource(); + backing.setUrl("jdbc:sqlite:" + Files.createTempFile("teaql-committed-cleanup-", ".db")); + try (var connection = backing.getConnection(); var statement = connection.createStatement()) { + statement.execute("CREATE TABLE committed_cleanup_probe (id INTEGER PRIMARY KEY)"); + } + var committed = new AtomicBoolean(); + DataSource injecting = (DataSource) Proxy.newProxyInstance( + DataSource.class.getClassLoader(), new Class[] {DataSource.class}, + (proxy, method, args) -> { + Object value = invoke(backing, method, args); + if (!method.getName().equals("getConnection")) return value; + Connection actual = (Connection) value; + return Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] {Connection.class}, (connectionProxy, operation, parameters) -> { + if (operation.getName().equals("setAutoCommit") + && Boolean.TRUE.equals(parameters[0]) && committed.get() + && (fault == Fault.AUTO_COMMIT_RESTORE + || fault == Fault.AUTO_COMMIT_RESTORE_ERROR)) { + if (fault == Fault.AUTO_COMMIT_RESTORE_ERROR) throw new NoSuppressionError(); + throw new SQLException(DIAGNOSTIC_CANARY); + } + Object result = invoke(actual, operation, parameters); + if (operation.getName().equals("commit")) committed.set(true); + if (operation.getName().equals("close") && committed.get() + && (fault == Fault.CLOSE || fault == Fault.CLOSE_ERROR)) { + if (fault == Fault.CLOSE_ERROR) throw new NoSuppressionError(); + throw new SQLException(DIAGNOSTIC_CANARY); + } + return result; + }); + }); + var executor = new JdbcSqlExecutor(injecting); + List warnings = new ArrayList<>(); + Logger logger = Logger.getLogger(JdbcSqlExecutor.class.getName()); + Handler capture = new Handler() { + @Override public void publish(LogRecord record) { + warnings.add(record.getMessage()); + if (warningHandlerThrowsError) throw new NoSuppressionError(); + if (warningHandlerFails) throw new IllegalStateException(DIAGNOSTIC_CANARY); + } + @Override public void flush() { } + @Override public void close() { } + }; + logger.addHandler(capture); + + Throwable reported = null; + try { + executor.executeInTransaction(() -> executor.update( + "INSERT INTO committed_cleanup_probe (id) VALUES (?)", new Object[] {1L})); + } catch (RuntimeException | Error failure) { + reported = failure; + } finally { + logger.removeHandler(capture); + } + + assertTrue("the injected failure must occur after the actual commit", committed.get()); + try (var connection = backing.getConnection(); + var statement = connection.prepareStatement("SELECT COUNT(*) FROM committed_cleanup_probe"); + var rows = statement.executeQuery()) { + assertTrue(rows.next()); + assertEquals(1, rows.getInt(1)); + } + assertTrue("cleanup failure needs an operational warning", warnings.stream() + .anyMatch(message -> message.contains("JDBC transaction committed"))); + assertTrue("the warning must not reveal the JDBC exception", warnings.stream() + .noneMatch(message -> message.contains(DIAGNOSTIC_CANARY))); + if (reported != null) { + throw new AssertionError("committed row was reported as a failed transaction", reported); + } + } + + private static Object invoke(Object target, Method method, Object[] args) throws Throwable { + try { + return method.invoke(target, args); + } catch (InvocationTargetException failure) { + throw failure.getCause(); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/JdbcRollbackCloseSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/JdbcRollbackCloseSqliteTest.java new file mode 100644 index 00000000..315fc085 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/JdbcRollbackCloseSqliteTest.java @@ -0,0 +1,189 @@ +package io.teaql.sqlite; + +import io.teaql.core.TransactionOutcomeUnknownException; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import javax.sql.DataSource; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +/** An unconfirmed rollback must not be followed by a close that can commit pending writes. */ +public class JdbcRollbackCloseSqliteTest { + private enum AbortBehavior { CLOSE, THROW, NO_OP } + + @Test public void failedRollbackDoesNotCommitPendingWriteOnClose() throws Exception { + verifyFailedRollback(AbortBehavior.CLOSE); + } + + @Test public void failedAbortDoesNotFallBackToUnsafeClose() throws Exception { + verifyFailedRollback(AbortBehavior.THROW); + } + + @Test public void noopAbortDoesNotFallBackToUnsafeClose() throws Exception { + verifyFailedRollback(AbortBehavior.NO_OP); + } + + @Test public void confirmedRollbackRetryAllowsSafeClose() throws Exception { + var backing = new SQLiteDataSource(); + backing.setUrl("jdbc:sqlite:" + Files.createTempFile("teaql-rollback-retry-", ".db")); + try (var connection = backing.getConnection(); var statement = connection.createStatement()) { + statement.execute("CREATE TABLE rollback_close_probe (id INTEGER PRIMARY KEY)"); + } + + var rollbackAttempts = new AtomicInteger(); + var rollbackConfirmed = new AtomicBoolean(); + var closedAfterRollback = new AtomicBoolean(); + var committedOnUnsafeClose = new AtomicBoolean(); + var aborted = new AtomicBoolean(); + var opened = new AtomicReference(); + DataSource injecting = (DataSource) Proxy.newProxyInstance( + DataSource.class.getClassLoader(), new Class[] {DataSource.class}, + (proxy, method, args) -> { + Object value = invoke(backing, method, args); + if (!method.getName().equals("getConnection")) return value; + Connection actual = (Connection) value; + opened.set(actual); + return Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] {Connection.class}, (connectionProxy, operation, parameters) -> { + if (operation.getName().equals("rollback")) { + if (rollbackAttempts.incrementAndGet() == 1) { + throw new SQLException("one-shot injected rollback failure"); + } + Object result = invoke(actual, operation, parameters); + rollbackConfirmed.set(true); + return result; + } + if (operation.getName().equals("close")) { + if (rollbackConfirmed.get()) { + closedAfterRollback.set(true); + } else { + // Model a pool that commits a still-active transaction on close. + actual.commit(); + committedOnUnsafeClose.set(true); + } + } + if (operation.getName().equals("abort")) aborted.set(true); + return invoke(actual, operation, parameters); + }); + }); + var executor = new JdbcSqlExecutor(injecting); + + try { + assertThrows(TransactionOutcomeUnknownException.class, () -> + executor.executeInTransaction(() -> { + executor.update("INSERT INTO rollback_close_probe (id) VALUES (?)", new Object[] {1L}); + throw new IllegalStateException("abort this transaction"); + })); + assertEquals("rollback must be retried exactly once", 2, rollbackAttempts.get()); + assertTrue("the retry must confirm rollback", rollbackConfirmed.get()); + assertTrue("close is safe after confirmed rollback", closedAfterRollback.get()); + assertFalse("abort is unnecessary after confirmed rollback", aborted.get()); + assertFalse("close must not commit pending writes", committedOnUnsafeClose.get()); + } finally { + Connection actual = opened.get(); + if (actual != null && !actual.isClosed()) { + actual.rollback(); + actual.close(); + } + } + try (var connection = backing.getConnection(); + var statement = connection.prepareStatement("SELECT COUNT(*) FROM rollback_close_probe"); + var rows = statement.executeQuery()) { + assertTrue(rows.next()); + assertEquals(0, rows.getInt(1)); + } + } + + private static void verifyFailedRollback(AbortBehavior abortBehavior) throws Exception { + var backing = new SQLiteDataSource(); + backing.setUrl("jdbc:sqlite:" + Files.createTempFile("teaql-rollback-close-", ".db")); + try (var connection = backing.getConnection(); var statement = connection.createStatement()) { + statement.execute("CREATE TABLE rollback_close_probe (id INTEGER PRIMARY KEY)"); + } + + var rollbackFailed = new AtomicBoolean(); + var committedOnClose = new AtomicBoolean(); + var aborted = new AtomicBoolean(); + var opened = new AtomicReference(); + DataSource injecting = (DataSource) Proxy.newProxyInstance( + DataSource.class.getClassLoader(), new Class[] {DataSource.class}, + (proxy, method, args) -> { + Object value = invoke(backing, method, args); + if (!method.getName().equals("getConnection")) return value; + Connection actual = (Connection) value; + opened.set(actual); + return Proxy.newProxyInstance(Connection.class.getClassLoader(), + new Class[] {Connection.class}, (connectionProxy, operation, parameters) -> { + if (operation.getName().equals("rollback")) { + rollbackFailed.set(true); + throw new SQLException("injected rollback failure"); + } + if (operation.getName().equals("abort")) { + aborted.set(true); + if (abortBehavior == AbortBehavior.THROW) { + throw new SQLException("injected abort failure"); + } + if (abortBehavior == AbortBehavior.NO_OP) return null; + actual.rollback(); + actual.close(); + return null; + } + if (operation.getName().equals("close") && rollbackFailed.get()) { + // JDBC leaves active-transaction close behavior implementation-defined. + // Model a driver/pool that commits when close returns the connection. + actual.commit(); + committedOnClose.set(true); + } + return invoke(actual, operation, parameters); + }); + }); + var executor = new JdbcSqlExecutor(injecting); + + try { + assertThrows(TransactionOutcomeUnknownException.class, () -> + executor.executeInTransaction(() -> { + executor.update("INSERT INTO rollback_close_probe (id) VALUES (?)", new Object[] {1L}); + throw new IllegalStateException("abort this transaction"); + })); + + assertTrue("the rollback failure must be injected", rollbackFailed.get()); + assertFalse("an unsafe close must not commit pending writes", committedOnClose.get()); + assertTrue("unconfirmed transactions must invalidate the connection", aborted.get()); + } finally { + // A failed/no-op abort leaves the backing connection open; release it + // explicitly so the test does not strand SQLite's write lock. + Connection actual = opened.get(); + if (actual != null && !actual.isClosed()) { + actual.rollback(); + actual.close(); + } + } + try (var connection = backing.getConnection(); + var statement = connection.prepareStatement("SELECT COUNT(*) FROM rollback_close_probe"); + var rows = statement.executeQuery()) { + assertTrue(rows.next()); + assertEquals(0, rows.getInt(1)); + } + } + + private static Object invoke(Object target, Method method, Object[] args) throws Throwable { + try { + return method.invoke(target, args); + } catch (InvocationTargetException failure) { + throw failure.getCause(); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedBoundaryTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedBoundaryTest.java new file mode 100644 index 00000000..8e59ca28 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedBoundaryTest.java @@ -0,0 +1,100 @@ +package io.teaql.sqlite; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.core.sqlite.SqliteCommittedReceiver; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.nio.charset.StandardCharsets; +import java.nio.file.*; +import java.security.MessageDigest; +import java.util.*; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; +import static org.junit.Assert.*; + +/** Small signed protocol fixture, not a high-level business save/example. */ +public class SqliteCommittedBoundaryTest { + static final ObjectMapper JSON=new ObjectMapper(); + static final String SOURCE="boundary-source",TARGET="boundary-target",MODEL="sha256:"+"0".repeat(64); + static final String SQL="INSERT INTO school_data(id,version,name) VALUES(99,1,'SQL-CANARY')"; + static final byte[] KEY="0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.US_ASCII); + static ObjectNode body()throws Exception{ + return (ObjectNode)JSON.readTree("{\"protocol_version\":1,\"change_set_id\":\"fact\",\"source_runtime_id\":\"boundary-source\",\"source_sequence\":\"1\",\"source_model_hash\":\""+MODEL+"\",\"source_transaction_id\":\"tx\",\"committed_at\":\"2026-10-22T00:00:00Z\",\"audit\":{\"comment\":\"receive school\",\"purpose\":\"check protocol boundary\"},\"mutations\":[{\"entity_type\":\"school\",\"entity_id\":\"7\",\"operation\":\"create\",\"base_version\":null,\"committed_version\":\"1\",\"fields\":{\"name\":\"original\"},\"trace_chain\":[]}]}"); + } + static ObjectNode snapshotBody()throws Exception{ + return (ObjectNode)JSON.readTree("{\"snapshot_version\":1,\"snapshot_id\":\"baseline\",\"source_runtime_id\":\"boundary-source\",\"source_sequence\":\"0\",\"source_model_hash\":\""+MODEL+"\",\"audit\":{\"comment\":\"receive baseline\",\"purpose\":\"check snapshot boundary\"},\"entity_types\":[\"school\"],\"rows\":[{\"entity_type\":\"school\",\"entity_id\":\"7\",\"version\":\"1\",\"fields\":{\"name\":\"original\"}}]}"); + } + static final class Fixture implements AutoCloseable{ + final Path owner=Files.createTempDirectory("cmr15-java-");final JdbcSqlExecutor sql;final DefaultUserContext context;final SqliteCommittedReceiver receiver; + Fixture()throws Exception{ + var ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+owner.resolve("target.sqlite"));sql=new JdbcSqlExecutor(ds); + var school=new EntityDescriptor();school.setType("school");school.addSimpleProperty("id",Long.class);school.addSimpleProperty("version",Long.class);school.addSimpleProperty("name",String.class); + var metadata=new SimpleEntityMetaFactory();metadata.register(school);context=new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).executionLogging(false).build()); + sql.execute("CREATE TABLE school_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,name TEXT NOT NULL)"); + receiver=new SqliteCommittedReceiver(sql,SOURCE,TARGET,MODEL,Map.of("current",KEY));receiver.ensureSchema(context); + } + long count(String table){return ((Number)sql.queryForList("SELECT count(*) n FROM "+table,new Object[0]).get(0).get("n")).longValue();} + io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome receive(ObjectNode body)throws Exception{ + String text=JSON.writeValueAsString(body),hash=HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))); + var event=new OutboxRecord(SOURCE,1,"fact",MODEL,hash,text); + return receiver.receive(context,SqliteCommittedReceiver.sign("current",TARGET,KEY,event)); + } + void reject(ObjectNode body)throws Exception{ + try{receive(body);fail("noncanonical controls must be rejected");}catch(SqliteCommittedReceiver.ReceiverException error){assertEquals("INVALID_ENVELOPE",error.code());assertFalse(error.toString().contains(SQL));} + assertEquals(0,count("school_data"));assertEquals(0,count("teaql_event_inbox"));assertEquals(0,count("teaql_event_checkpoint"));assertEquals(1,count("teaql_event_rejection")); + assertFalse(receive(body()).duplicate());assertEquals(1,count("school_data"));assertEquals(1,count("teaql_event_inbox")); + } + void receiveSnapshot(ObjectNode body)throws Exception{ + String text=JSON.writeValueAsString(body),hash=HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))); + var record=new io.teaql.core.sqlite.SqliteCommittedSnapshot.SnapshotRecord(SOURCE,0,"baseline",MODEL,hash,text); + receiver.receiveSnapshot(context,SqliteCommittedReceiver.signSnapshot("current",TARGET,KEY,record)); + } + void rejectSnapshot(ObjectNode body)throws Exception{ + try{receiveSnapshot(body);fail("noncanonical snapshot controls must be rejected");}catch(SqliteCommittedReceiver.ReceiverException error){assertEquals("INVALID_SNAPSHOT",error.code());assertFalse(error.toString().contains(SQL));} + for(String table:List.of("school_data","teaql_event_snapshot","teaql_event_inbox","teaql_event_checkpoint"))assertEquals(table,0,count(table)); + assertEquals(1,count("teaql_event_rejection"));receiveSnapshot(snapshotBody());assertEquals(1,count("school_data"));assertEquals(1,count("teaql_event_snapshot")); + } + @Override public void close()throws Exception{try(var all=Files.walk(owner)){for(var p:all.sorted(Comparator.reverseOrder()).toList())Files.delete(p);}} + } + @Test public void envelopeControlsCannotSmuggleSql()throws Exception{ + for(String key:List.of("sql","rawSql","raw_sql","statement","statements","provider","mutation_plan","replace"))try(var f=new Fixture()){var b=body();b.put(key,SQL);f.reject(b);} + System.out.println("CMR15_CONTROL_FRAMES_PASS language=java scope=envelope rejected=8"); + } + @Test public void mutationControlsCannotSmuggleSql()throws Exception{ + for(String key:List.of("sql","rawSql","raw_sql","statement","table","provider","physical_table","command"))try(var f=new Fixture()){var b=body();((ObjectNode)b.path("mutations").get(0)).put(key,SQL);f.reject(b);} + } + @Test public void auditControlsCannotSmuggleSql()throws Exception{ + for(String key:List.of("sql","rawSql","statement","provider","class","command","tenantId"))try(var f=new Fixture()){var b=body();((ObjectNode)b.path("audit")).put(key,SQL);f.reject(b);} + } + @Test public void traceControlsStayStrict()throws Exception{ + try(var f=new Fixture()){var b=body();((com.fasterxml.jackson.databind.node.ArrayNode)b.path("mutations").get(0).path("trace_chain")).addObject().put("kind","SQL").put("name","diagnostic").putNull("entity_id").put("comment","trace metadata").put("statement",SQL);f.reject(b);} + } + @Test public void sqlLookingBusinessAndAuditTextRemainBoundData()throws Exception{ + try(var f=new Fixture()){var b=body();((ObjectNode)b.path("mutations").get(0).path("fields")).put("name",SQL);((ObjectNode)b.path("audit")).put("comment",SQL).put("actor_ref","user:7").put("trace_id","trace:1"); + assertFalse(f.receive(b).duplicate());assertEquals(1,f.count("school_data"));assertEquals(SQL,f.sql.queryForList("SELECT name FROM school_data WHERE id=7",new Object[0]).get(0).get("name"));assertEquals(0,f.count("teaql_event_rejection")); + } + } + @Test public void ordinaryRequestsAreNotCommittedEnvelopes()throws Exception{ + for(String input:List.of("{\"sql\":\"SELECT 1\"}","{\"entity\":\"school\",\"limitValue\":1,\"commentText\":\"read\",\"purposeText\":\"inspect\"}","{\"entity\":\"school\",\"action\":\"Create\",\"payload\":{\"name\":\"business\"},\"comment\":\"create school\"}"))try(var f=new Fixture()){f.reject((ObjectNode)JSON.readTree(input));} + } + @Test public void snapshotContainerControlsStayStrict()throws Exception{ + for(String key:List.of("sql","rawSql","statement","provider","replace"))try(var f=new Fixture()){var b=snapshotBody();b.put(key,SQL);f.rejectSnapshot(b);} + } + @Test public void snapshotRowControlsStayStrict()throws Exception{ + for(String key:List.of("sql","statement","table","command","operation"))try(var f=new Fixture()){var b=snapshotBody();((ObjectNode)b.path("rows").get(0)).put(key,SQL);f.rejectSnapshot(b);} + } + @Test public void snapshotAuditControlsCannotSmuggleSql()throws Exception{ + for(String key:List.of("sql","rawSql","statement","provider","class","command","tenantId"))try(var f=new Fixture()){var b=snapshotBody();((ObjectNode)b.path("audit")).put(key,SQL);f.rejectSnapshot(b);} + System.out.println("CMR15_SNAPSHOT_CONTROLS_PASS language=java container=5 row=5 audit=7"); + } + @Test public void snapshotSqlLookingBusinessAndAuditTextRemainBoundData()throws Exception{ + try(var f=new Fixture()){var b=snapshotBody();((ObjectNode)b.path("rows").get(0).path("fields")).put("name",SQL);((ObjectNode)b.path("audit")).put("comment",SQL).put("actor_ref","user:7").put("trace_id","trace:1"); + f.receiveSnapshot(b);assertEquals(1,f.count("school_data"));assertEquals(SQL,f.sql.queryForList("SELECT name FROM school_data WHERE id=7",new Object[0]).get(0).get("name"));assertEquals(0,f.count("teaql_event_rejection")); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedObservationTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedObservationTest.java new file mode 100644 index 00000000..f119e7c5 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedObservationTest.java @@ -0,0 +1,162 @@ +package io.teaql.sqlite; + +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; +import io.teaql.core.sqlite.SqliteCommittedObservation; +import io.teaql.core.sqlite.SqliteCommittedReceiver; +import io.teaql.core.sqlite.SqliteCommittedSnapshot; +import io.teaql.core.sqlite.SqliteCommittedSnapshotPublisher; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.RuntimeTelemetry; +import io.teaql.runtime.TeaQLRuntime; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; +import static org.junit.Assert.*; + +/** Small infrastructure fixture; not an audited business-save/example claim. */ +public class SqliteCommittedObservationTest { + static final String SOURCE="source-CANARY", TARGET="target-CANARY", MODEL="sha256:"+"0".repeat(64); + static final class Fixture implements AutoCloseable { + final Path directory=Files.createTempDirectory("cmr14-java-"); + final JdbcSqlExecutor sql; final DefaultUserContext context; + final SqliteCommittedOutboxPublisher publisher; + final SqliteCommittedObservation observations; + final List> telemetry=new ArrayList<>(); + Fixture() throws Exception { + var ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+directory.resolve("source.sqlite")); + sql=new JdbcSqlExecutor(ds);new SqliteCommittedEventStore(sql).ensureSchema(); + RuntimeTelemetry t=op->{telemetry.add(op.attributes());return new RuntimeTelemetry.Scope(){ + public void success(Map attrs){telemetry.add(attrs);} + public void failure(Throwable ignored){fail("observation should succeed");}};}; + var school=new EntityDescriptor();school.setType("school");school.addSimpleProperty("id",Long.class);school.addSimpleProperty("version",Long.class);school.addSimpleProperty("name",String.class); + var metadata=new SimpleEntityMetaFactory();metadata.register(school); + context=new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).telemetry(t).build()); + sql.execute("CREATE TABLE school_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,name TEXT)"); + publisher=new SqliteCommittedOutboxPublisher(sql,SOURCE,TARGET,MODEL); + observations=new SqliteCommittedObservation(sql,SOURCE,TARGET,MODEL); + sql.update("INSERT INTO teaql_event_sequence VALUES(?,2)",new Object[]{SOURCE}); + for(int n=1;n<=2;n++)sql.update("INSERT INTO teaql_event_outbox VALUES(?,?,?,?,?,?)", + new Object[]{SOURCE,n,"fact-CANARY-"+n,MODEL,"a".repeat(64),"payload-CANARY"}); + } + @Override public void close()throws Exception{try(var paths=Files.walk(directory)){for(var p:paths.sorted(Comparator.reverseOrder()).toList())Files.delete(p);}} + } + @Test public void lostResponseRetryAndReopenKeepDurableFacts()throws Exception { + try(var f=new Fixture()){ + try{f.publisher.publishBatch(1,event->{throw new IllegalStateException("transport-CANARY");});fail();}catch(IllegalStateException expected){} + var a=f.observations.source(f.context);assertEquals(2,a.sourceHighWatermark());assertEquals(0,a.ackedSequence()); + assertEquals(2,a.sequenceLag());assertEquals(2,a.backlog());assertEquals(1,a.attempts());assertEquals(1,a.failures());assertEquals(0,a.retries());assertEquals(0,a.deadLetters()); + assertEquals(1,f.publisher.publishBatch(1,event->new InboxOutcome(false,1)).acknowledged()); + var reopened=new SQLiteDataSource();reopened.setUrl("jdbc:sqlite:"+f.directory.resolve("source.sqlite")); + var b=new SqliteCommittedObservation(new JdbcSqlExecutor(reopened),SOURCE,TARGET,MODEL).source(f.context); + assertEquals(1,b.ackedSequence());assertEquals(1,b.backlog());assertEquals(2,b.attempts());assertEquals(1,b.retries());assertEquals(1,b.failures()); + assertEquals(2,((Number)f.sql.queryForList("SELECT count(*) n FROM teaql_event_outbox",new Object[0]).get(0).get("n")).intValue()); + assertFalse(f.telemetry.toString().contains("CANARY"));assertTrue(f.telemetry.toString().contains("teaql.replication.retries")); + System.out.println("CMR14_SOURCE_PASS language=java retry=1 backlog=1"); + } + } + @Test public void pendingRowsAreNotSequenceDistance()throws Exception{ + try(var f=new Fixture()){ + f.sql.update("DELETE FROM teaql_event_outbox WHERE source_sequence=1",new Object[0]); + var s=f.observations.source(f.context);assertEquals(2,s.sequenceLag());assertEquals(1,s.backlog()); + try{f.publisher.publishBatch(2,e->new InboxOutcome(false,e.sourceSequence()));fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){assertEquals("SEQUENCE_GAP",expected.code());} + assertEquals(0,f.observations.source(f.context).attempts()); + } + } + @Test public void badAckDoesNotAcknowledgeOrHideFailure()throws Exception{ + try(var f=new Fixture()){ + try{f.publisher.publishBatch(1,e->new InboxOutcome(false,2));fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){assertEquals("ACK_MISMATCH",expected.code());} + var s=f.observations.source(f.context);assertEquals(0,s.ackedSequence());assertEquals(1,s.failures());assertEquals(0,s.deadLetters()); + } + } + @Test public void journalFailureRollsBackSourceAck()throws Exception{ + try(var f=new Fixture()){ + f.sql.execute("CREATE TRIGGER refuse_ack BEFORE UPDATE ON teaql_event_delivery_journal WHEN NEW.last_outcome='ACKNOWLEDGED' BEGIN SELECT RAISE(ABORT,'journal-CANARY'); END"); + try{f.publisher.publishBatch(1,e->new InboxOutcome(false,1));fail();}catch(RuntimeException expected){} + var s=f.observations.source(f.context);assertEquals(0,s.ackedSequence());assertEquals(1,s.unresolvedAttempts());assertEquals(0,s.failures()); + } + } + @Test public void observationsRejectAmbientAndInconsistentPositions()throws Exception{ + try(var f=new Fixture()){ + f.sql.executeInTransaction(()->{try{f.observations.source(f.context);fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){assertEquals("OBSERVATION_UNAVAILABLE",expected.code());}}); + f.sql.update("INSERT INTO teaql_event_delivery_cursor VALUES(?,?,3,?)",new Object[]{SOURCE,TARGET,MODEL}); + try{f.observations.source(f.context);fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){assertEquals("OBSERVATION_UNAVAILABLE",expected.code());} + } + } + @Test public void permanentRejectionIsInspectableAndResolvingDoesNotEraseHistory()throws Exception{ + try(var f=new Fixture()){ + var ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+f.directory.resolve("target.sqlite"));var target=new JdbcSqlExecutor(ds); + var receiver=new SqliteCommittedReceiver(target,SOURCE,TARGET,MODEL,Map.of("current",new byte[32]));receiver.ensureSchema(f.context); + try{f.publisher.publishBatch(1,e->receiver.receive(f.context,new SqliteCommittedReceiver.Delivery(e,"current",TARGET,new byte[32])));fail();}catch(SqliteCommittedReceiver.ReceiverException expected){assertEquals("INVALID_SIGNATURE",expected.code());} + var s=f.observations.source(f.context);assertEquals(1,s.rejections());assertEquals(1,s.deadLetters());assertEquals(0,s.ackedSequence()); + var failures=f.observations.deadLetters(f.context,10);assertEquals(1,failures.size());assertEquals("INVALID_SIGNATURE",failures.get(0).reason());assertFalse(failures.toString().contains("CANARY")); + for(int limit:new int[]{0,1001})try{f.observations.deadLetters(f.context,limit);fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){} + try{f.publisher.publishBatch(1,e->{throw new IllegalStateException("network temporarily down");});fail();}catch(IllegalStateException expected){} + assertEquals("INVALID_SIGNATURE",f.observations.deadLetters(f.context,10).get(0).reason()); + // This deliberately fake callback ACK is not evidence of target application. + f.publisher.publishBatch(1,e->new InboxOutcome(false,1)); + var after=f.observations.source(f.context);assertEquals(0,after.deadLetters());assertEquals(1,after.rejections());assertEquals(2,after.retries()); + var targetView=new SqliteCommittedObservation(target,SOURCE,TARGET,MODEL).target(f.context,OptionalLong.of(2));assertEquals(0,targetView.checkpoint());assertEquals(2,targetView.sequenceLag().getAsLong());assertEquals(1,targetView.rejections()); + } + } + @Test public void unauthorizedSourceAuditIsNotFilteredOutAndMissingLagStaysAbsent()throws Exception{ + try(var f=new Fixture()){ + var receiver=new SqliteCommittedReceiver(f.sql,SOURCE,TARGET,MODEL,Map.of("current",new byte[32]));receiver.ensureSchema(f.context); + var e=new SqliteCommittedEventStore.OutboxRecord("intruder",1,"x",MODEL,"a".repeat(64),"payload-CANARY"); + try{receiver.receive(f.context,new SqliteCommittedReceiver.Delivery(e,"current",TARGET,new byte[32]));fail();}catch(SqliteCommittedReceiver.ReceiverException expected){assertEquals("UNAUTHORIZED_SOURCE",expected.code());} + var s=f.observations.target(f.context,OptionalLong.empty());assertFalse(s.sequenceLag().isPresent());assertEquals(1,s.rejections()); + f.sql.update("INSERT INTO teaql_event_checkpoint VALUES(?,1,?)",new Object[]{SOURCE,MODEL}); + try{f.observations.target(f.context,OptionalLong.of(0));fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){assertEquals("OBSERVATION_UNAVAILABLE",expected.code());} + } + } + @Test public void staleAttemptCannotOverwriteNewerSuccessfulAttempt()throws Exception{ + try(var f=new Fixture()){ + try{f.publisher.publishBatch(1,e->{f.publisher.publishBatch(1,inner->new InboxOutcome(false,1));return new InboxOutcome(false,1);});fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){assertEquals("CURSOR_CONFLICT",expected.code());} + var s=f.observations.source(f.context);assertEquals(1,s.ackedSequence());assertEquals(2,s.attempts());assertEquals(0,s.failures());assertEquals(0,s.unresolvedAttempts()); + } + } + @Test public void abortedCallbackIsUnresolvedNotFailed()throws Exception{ + try(var f=new Fixture()){ + try{f.publisher.publishBatch(1,e->{throw new AssertionError("aborted-CANARY");});fail();}catch(AssertionError expected){} + var s=f.observations.source(f.context);assertEquals(1,s.unresolvedAttempts());assertEquals(0,s.failures());assertEquals(0,s.ackedSequence()); + f.publisher.publishBatch(1,e->new InboxOutcome(false,1));assertEquals(1,f.observations.source(f.context).retries()); + } + } + @Test public void snapshotZeroHasIndependentDurableAttempts()throws Exception{ + try(var f=new Fixture()){ + f.sql.update("DELETE FROM teaql_event_outbox",new Object[0]);f.sql.update("DELETE FROM teaql_event_sequence",new Object[0]); + var publisher=new SqliteCommittedSnapshotPublisher(f.sql,SOURCE,TARGET,MODEL);var limits=SqliteCommittedSnapshot.Limits.defaults(); + publisher.prepare(f.context,"baseline","capture empty baseline","verify observations",limits); + try{publisher.publish(f.context,"baseline",limits,e->{throw new IllegalStateException("lost response-CANARY");});fail();}catch(IllegalStateException expected){} + var before=f.observations.source(f.context);assertEquals(1,before.pendingSnapshots());assertEquals(1,before.attempts());assertEquals(1,before.failures()); + publisher.publish(f.context,"baseline",limits,e->new SqliteCommittedSnapshotPublisher.Receipt(SOURCE,TARGET,e.snapshotId(),e.sourceSequence(),MODEL,e.payloadSha256(),false)); + var after=f.observations.source(f.context);assertEquals(0,after.pendingSnapshots());assertEquals(0,after.sourceHighWatermark());assertEquals(0,after.ackedSequence());assertEquals(2,after.attempts());assertEquals(1,after.retries()); + publisher.publish(f.context,"baseline",limits,e->{throw new AssertionError("already acked");});assertEquals(2,f.observations.source(f.context).attempts()); + } + } + @Test public void counterOverflowAndWrongModelFailClosed()throws Exception{ + try(var f=new Fixture()){ + try{f.publisher.publishBatch(1,e->{throw new IllegalStateException();});fail();}catch(IllegalStateException expected){} + f.sql.update("UPDATE teaql_event_delivery_journal SET attempts=?",new Object[]{Long.MAX_VALUE}); + try{f.publisher.publishBatch(1,e->{fail("overflow must not invoke transport");return null;});fail();}catch(ArithmeticException expected){} + assertEquals(Long.MAX_VALUE,f.observations.source(f.context).attempts()); + f.sql.update("UPDATE teaql_event_delivery_journal SET model_hash=?",new Object[]{"sha256:"+"1".repeat(64)}); + try{f.observations.source(f.context);fail();}catch(SqliteCommittedEventStore.CommittedEventException expected){assertEquals("OBSERVATION_UNAVAILABLE",expected.code());} + } + } + @Test public void telemetryExporterCannotBreakObservation()throws Exception{ + try(var f=new Fixture()){ + RuntimeTelemetry broken=op->{throw new IllegalStateException("exporter unavailable");}; + var context=new DefaultUserContext(TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()).telemetry(broken).build()); + assertEquals(2,f.observations.source(context).backlog()); + RuntimeTelemetry brokenScope=op->new RuntimeTelemetry.Scope(){public void success(Map a){throw new IllegalStateException("exporter unavailable");}public void failure(Throwable t){throw new AssertionError();}}; + context=new DefaultUserContext(TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()).telemetry(brokenScope).build());assertEquals(2,f.observations.source(context).backlog()); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReadOnlyTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReadOnlyTest.java new file mode 100644 index 00000000..8f8313d0 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReadOnlyTest.java @@ -0,0 +1,62 @@ +package io.teaql.sqlite; + +import io.teaql.core.*; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.*; +import java.util.List; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; +import static org.junit.Assert.*; + +/** A real ordinary executor and the signed receiver deliberately share storage. */ +public class SqliteCommittedReadOnlyTest { + public static final class School extends BaseEntity { + private String name; + @Override public String typeName(){return "school";} + @Override public Object __internalGet(String field){return "name".equals(field)?name:super.__internalGet(field);} + @Override public void __internalSet(String field,Object value){if("name".equals(field))name=(String)value;else super.__internalSet(field,value);} + } + private record Application(DefaultUserContext context,SqliteDataServiceExecutor provider){} + private static Application application(JdbcSqlExecutor sql,io.teaql.core.meta.EntityMetaFactory metadata){ + var descriptor=metadata.resolveEntityDescriptor("school");descriptor.setTargetType(School.class);descriptor.withEntitySupplier(School::new); + var provider=new SqliteDataServiceExecutor("default",sql,null); + var context=new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).dataService("default",provider) + .idGenerationService((c,e)->99L).executionLogging(false).build()); + return new Application(context,provider); + } + private static School school(){var s=new School();s.updateProperty("name","ordinary write");s.setComment("verify default analytical target write boundary");return s;} + private static void readonly(Runnable action){var error=assertThrows("analytical target ordinary mutation must be read-only",RuntimeException.class,()->action.run());assertTrue(error.toString(),error.toString().contains("READ_ONLY_TARGET"));} + @Test public void graphSaveCannotWriteThroughReceiverAuthority()throws Exception{ + try(var f=new SqliteCommittedBoundaryTest.Fixture()){ + var app=application(f.sql,EntityMetaFactory.requireFrom(f.context)); + readonly(()->app.context().saveGraph(school()));assertEquals(0,f.count("school_data")); + assertFalse(f.receive(SqliteCommittedBoundaryTest.body()).duplicate());assertEquals(1,f.count("school_data")); + assertEquals(1,f.count("teaql_event_inbox"));assertEquals(0,f.count("teaql_event_rejection")); + var query=new BaseRequest.TempRequest(EntityMetaFactory.requireFrom(app.context()).resolveEntityDescriptor("school")); + query.selectProperty("name");query.getSlice().setSize(1); + var result=(DefaultQueryResult)app.provider().query(app.context(),new DefaultQueryRequest(query,QueryIntent.of("read analytical target","verify reads remain available"))); + assertEquals(1,result.getResult().size());assertEquals("original",((Entity)result.getResult().getData().get(0)).getProperty("name")); + } + } + @Test public void directAndBatchMutationCannotBypassReadOnlyTarget()throws Exception{ + for(boolean batch:List.of(false,true))try(var f=new SqliteCommittedBoundaryTest.Fixture()){ + var app=application(f.sql,EntityMetaFactory.requireFrom(f.context));var s=school();s.setProperty("id",99L); + var request=new EntityPersistenceMutation(s,EntityPersistenceMutation.Action.SAVE); + if(batch)readonly(()->app.provider().mutateBatch(app.context(),new MutationBatchRequest(request.intent(),List.of(request)))); + else readonly(()->app.provider().mutate(app.context(),request)); + assertEquals(0,f.count("school_data"));assertEquals(0,f.count("teaql_event_inbox")); + } + } + @Test public void ordinarySourceIsNotMadeReadOnlyByAnotherTargetsReceiver()throws Exception{ + try(var f=new SqliteCommittedBoundaryTest.Fixture()){ + var ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+f.owner.resolve("ordinary.sqlite"));var sql=new JdbcSqlExecutor(ds); + sql.execute("CREATE TABLE school_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,name TEXT NOT NULL)"); + var app=application(sql,EntityMetaFactory.requireFrom(f.context));app.context().saveGraph(school()); + assertEquals(1,((Number)sql.queryForList("SELECT count(*) n FROM school_data",new Object[0]).get(0).get("n")).intValue()); + assertEquals(0,f.count("school_data"));assertFalse(f.receive(SqliteCommittedBoundaryTest.body()).duplicate()); + System.out.println("CMR15_READ_ONLY_SOURCE_PASS language=java graph=1 direct=1 batch=1 source_isolated=1 replication=1"); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReceiverSnapshotTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReceiverSnapshotTest.java new file mode 100644 index 00000000..ffdee8da --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReceiverSnapshotTest.java @@ -0,0 +1,131 @@ +package io.teaql.sqlite; + +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.core.sqlite.SqliteCommittedReceiver; +import io.teaql.core.sqlite.SqliteCommittedReceiver.Delivery; +import io.teaql.core.sqlite.SqliteCommittedReceiver.SnapshotDelivery; +import io.teaql.core.sqlite.SqliteCommittedReceiver.ReceiverException; +import io.teaql.core.sqlite.SqliteCommittedSnapshot.Limits; +import io.teaql.core.sqlite.SqliteCommittedSnapshot.SnapshotRecord; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.Map; +import static org.junit.Assert.*; + +public class SqliteCommittedReceiverSnapshotTest { + private static final String SOURCE="snapshot-source",TARGET="snapshot-target",MODEL="sha256:"+"0".repeat(64); + private static final byte[] KEY="0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.UTF_8); + private static final String CANARY="SNAPSHOT_SECRET_never_in_rejection_audit"; + private static String digest(String s)throws Exception{return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(s.getBytes(StandardCharsets.UTF_8)));} + private static SnapshotRecord snapshot(long n,String id)throws Exception{ + String body="{\"snapshot_version\":1,\"snapshot_id\":\""+id+"\",\"source_runtime_id\":\""+SOURCE+"\",\"source_sequence\":\""+n+"\",\"source_model_hash\":\""+MODEL+"\",\"audit\":{\"comment\":\"load baseline\",\"purpose\":\"verify authenticated recovery\"},\"entity_types\":[\"school\"],\"rows\":[{\"entity_type\":\"school\",\"entity_id\":\"7\",\"version\":\"1\",\"fields\":{\"name\":\""+CANARY+"\"}}]}"; + return new SnapshotRecord(SOURCE,n,id,MODEL,digest(body),body); + } + private static OutboxRecord tail(long n)throws Exception{ + String body="{\"protocol_version\":1,\"change_set_id\":\"tail\",\"source_runtime_id\":\""+SOURCE+"\",\"source_sequence\":\""+n+"\",\"source_model_hash\":\""+MODEL+"\",\"source_transaction_id\":\"tx\",\"committed_at\":\"2026-10-22T00:00:00Z\",\"audit\":{\"comment\":\"apply tail\",\"purpose\":\"verify recovery\"},\"mutations\":[{\"entity_type\":\"school\",\"entity_id\":\"7\",\"operation\":\"update\",\"base_version\":\"1\",\"committed_version\":\"2\",\"fields\":{\"name\":\"updated\"},\"trace_chain\":[]}]}"; + return new OutboxRecord(SOURCE,n,"tail",MODEL,digest(body),body); + } + private static SnapshotDelivery signed(SnapshotRecord s){return SqliteCommittedReceiver.signSnapshot("current",TARGET,KEY,s);} + private static SnapshotDelivery replace(SnapshotDelivery d,SnapshotRecord s){return new SnapshotDelivery(s,d.keyId(),d.targetId(),d.signature());} + private static final class Fixture implements AutoCloseable{ + final Path dir=Files.createTempDirectory("cmr13-snapshot-java-");final JdbcSqlExecutor sql;final DefaultUserContext context;final SqliteCommittedReceiver receiver; + Fixture()throws Exception{ + SQLiteDataSource ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+dir.resolve("target.sqlite"));sql=new JdbcSqlExecutor(ds); + EntityDescriptor e=new EntityDescriptor();e.setType("school");e.addSimpleProperty("id",Long.class);e.addSimpleProperty("version",Long.class);e.addSimpleProperty("name",String.class); + SimpleEntityMetaFactory meta=new SimpleEntityMetaFactory();meta.register(e);context=new DefaultUserContext(TeaQLRuntime.builder().metadata(meta).build()); + sql.execute("CREATE TABLE school_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,name TEXT NOT NULL)");receiver=new SqliteCommittedReceiver(sql,SOURCE,TARGET,MODEL,Map.of("current",KEY));receiver.ensureSchema(context); + } + long count(String table){return ((Number)sql.queryForList("SELECT count(*) AS n FROM "+table,new Object[0]).get(0).get("n")).longValue();} + void empty(long audits){for(String t:new String[]{"school_data","teaql_event_snapshot","teaql_event_checkpoint","teaql_event_inbox"})assertEquals(t,0,count(t));assertEquals(audits,count("teaql_event_rejection"));} + void reject(String code,SnapshotDelivery d){try{receiver.receiveSnapshot(context,d);fail();}catch(ReceiverException e){assertEquals(code,e.code());assertNull(e.getCause());assertFalse(e.toString().contains(CANARY));}} + @Override public void close()throws Exception{try(var paths=Files.walk(dir)){for(Path p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}} + } + @Test public void zeroAndNonzeroSnapshotTailAndDuplicateNeverRewind()throws Exception{ + for(long n:new long[]{0,5})try(var f=new Fixture()){ + var d=signed(snapshot(n,"baseline"));var ack=f.receiver.receiveSnapshot(f.context,d);assertEquals(n,ack.sourceSequence());assertFalse(ack.duplicate());assertEquals(TARGET,ack.targetId());assertEquals(d.snapshot().payloadSha256(),ack.payloadSha256()); + f.receiver.receive(f.context,SqliteCommittedReceiver.sign("current",TARGET,KEY,tail(n+1)));var duplicate=f.receiver.receiveSnapshot(f.context,d);assertTrue(duplicate.duplicate());assertEquals(ack.sourceSequence(),duplicate.sourceSequence()); + assertEquals("updated",f.sql.queryForList("SELECT name FROM school_data",new Object[0]).get(0).get("name"));assertEquals(n+1,((Number)f.sql.queryForList("SELECT source_sequence FROM teaql_event_checkpoint",new Object[0]).get(0).get("source_sequence")).longValue());assertEquals(0,f.count("teaql_event_rejection")); + System.out.println("CMR13_SNAPSHOT_SOURCE_PASS language=java baseline="+n+" tail="+(n+1)+" duplicate_no_rewind=1"); + } + } + @Test public void sameEightFieldsCannotCrossEventSnapshotSignatureDomains()throws Exception{ + try(var f=new Fixture()){ + var s=snapshot(1,"same");var e=new OutboxRecord(s.sourceRuntimeId(),1,s.snapshotId(),s.modelHash(),s.payloadSha256(),s.payloadJson());var eventTag=SqliteCommittedReceiver.sign("current",TARGET,KEY,e).signature(); + f.reject("INVALID_SIGNATURE",new SnapshotDelivery(s,"current",TARGET,eventTag)); + try{f.receiver.receive(f.context,new Delivery(e,"current",TARGET,signed(s).signature()));fail();}catch(ReceiverException error){assertEquals("INVALID_SIGNATURE",error.code());} + f.empty(2); + } + } + @Test public void snapshotTamperRoutingAndEveryIdentityFieldReject()throws Exception{ + try(var f=new Fixture()){ + var s=snapshot(0,"baseline");var d=signed(s);String altered=s.payloadJson().replace(CANARY,"attacker"); + for(String hash:new String[]{s.payloadSha256(),digest(altered)})f.reject("INVALID_SIGNATURE",replace(d,new SnapshotRecord(SOURCE,0,s.snapshotId(),MODEL,hash,altered))); + f.reject("INVALID_SIGNATURE",replace(d,new SnapshotRecord(SOURCE,1,s.snapshotId(),MODEL,s.payloadSha256(),s.payloadJson()))); + f.reject("INVALID_SIGNATURE",replace(d,new SnapshotRecord(SOURCE,0,"other",MODEL,s.payloadSha256(),s.payloadJson()))); + f.reject("UNAUTHORIZED_SOURCE",replace(d,new SnapshotRecord("other",0,s.snapshotId(),MODEL,s.payloadSha256(),s.payloadJson()))); + f.reject("UNAUTHORIZED_TARGET",new SnapshotDelivery(s,"current","other",d.signature()));f.reject("UNKNOWN_KEY",new SnapshotDelivery(s,"retired",TARGET,d.signature())); + f.reject("MODEL_MISMATCH",replace(d,new SnapshotRecord(SOURCE,0,s.snapshotId(),"sha256:"+"1".repeat(64),s.payloadSha256(),s.payloadJson())));f.empty(8); + } + } + @Test public void authenticatedMalformedSnapshotAndConflictingBaselineAreAudited()throws Exception{ + try(var f=new Fixture()){ + var s=snapshot(0,"baseline");String invalid=s.payloadJson().replace("\"school\"","\"unknown\"");f.reject("INVALID_SNAPSHOT",signed(new SnapshotRecord(SOURCE,0,s.snapshotId(),MODEL,digest(invalid),invalid)));f.empty(1); + f.receiver.receiveSnapshot(f.context,signed(s));f.reject("SNAPSHOT_CONFLICT",signed(snapshot(1,"different")));assertEquals(1,f.count("school_data"));assertEquals(2,f.count("teaql_event_rejection")); + assertEquals(CANARY,f.sql.queryForList("SELECT name FROM school_data",new Object[0]).get(0).get("name")); + } + } + @Test public void snapshotAuditReopensAndAuditFailureNeverAcknowledges()throws Exception{ + try(var f=new Fixture()){ + var d=new SnapshotDelivery(snapshot(0,CANARY),"current",TARGET,new byte[32]);f.reject("INVALID_SIGNATURE",d);f.empty(1); + SQLiteDataSource ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+f.dir.resolve("target.sqlite"));var rows=new JdbcSqlExecutor(ds).queryForList("SELECT * FROM teaql_event_rejection",new Object[0]);assertEquals(1,rows.size());assertFalse(rows.toString().contains(CANARY));assertFalse(rows.toString().contains(new String(KEY,StandardCharsets.UTF_8))); + f.sql.execute("CREATE TRIGGER reject_audit BEFORE INSERT ON teaql_event_rejection BEGIN SELECT RAISE(ABORT,'"+CANARY+"'); END");f.reject("AUDIT_UNAVAILABLE",d);f.empty(1); + } + } + @Test public void receiverOwnedLimitsAndAmbientTransactionsFailClosed()throws Exception{ + try(var f=new Fixture()){ + var d=signed(snapshot(0,"baseline"));var strict=new SqliteCommittedReceiver(f.sql,SOURCE,TARGET,MODEL,Map.of("current",KEY),new Limits(1,16)); + try{strict.receiveSnapshot(f.context,d);fail();}catch(ReceiverException e){assertEquals("INVALID_ENVELOPE",e.code());}f.empty(1); + f.reject("INVALID_ENVELOPE",replace(d,new SnapshotRecord(SOURCE,-1,"baseline",MODEL,d.snapshot().payloadSha256(),d.snapshot().payloadJson()))); + f.reject("INVALID_ENVELOPE",new SnapshotDelivery(d.snapshot(),"current",TARGET,new byte[0]));f.empty(3); + f.sql.executeInTransaction(()->f.reject("AMBIENT_TRANSACTION",d));f.empty(3); + } + } + @Test public void knownKeyAliasDestinationAndLengthFramingAreBound()throws Exception{ + try(var f=new Fixture()){ + var s=snapshot(0,"baseline");var d=signed(s);var aliases=new SqliteCommittedReceiver(f.sql,SOURCE,TARGET,MODEL,Map.of("current",KEY,"alias",KEY)); + try{aliases.receiveSnapshot(f.context,new SnapshotDelivery(s,"alias",TARGET,d.signature()));fail();}catch(ReceiverException e){assertEquals("INVALID_SIGNATURE",e.code());} + var destination=new SqliteCommittedReceiver(f.sql,SOURCE,"other",MODEL,Map.of("current",KEY));try{destination.receiveSnapshot(f.context,new SnapshotDelivery(s,"current","other",d.signature()));fail();}catch(ReceiverException e){assertEquals("INVALID_SIGNATURE",e.code());} + assertFalse(Arrays.equals(SqliteCommittedReceiver.signSnapshot("a","bc",KEY,s).signature(),SqliteCommittedReceiver.signSnapshot("ab","c",KEY,s).signature()));f.empty(2); + } + } + @Test public void independentSnapshotGoldenAndDefensiveTagCopies()throws Exception{ + var s=new SnapshotRecord("source",0,"baseline",MODEL,"a".repeat(64),"{\"x\":\"雪\"}");var d=SqliteCommittedReceiver.signSnapshot("key","target",KEY,s); + assertEquals("21dd4226dec5f6513b30b82b9bc4b8a340421a5663e4635658c54291226fe8d9",HexFormat.of().formatHex(d.signature()));byte[] first=d.signature();Arrays.fill(first,(byte)0);assertFalse(Arrays.equals(first,d.signature()));assertFalse(d.toString().contains(s.payloadJson())); + } + @Test public void trustedStartupReplacementCannotResetAnActiveStream()throws Exception{ + try(var f=new Fixture()){ + // Pre-stream fixture, representing existing bootstrap/legacy rows; + // not a production seeding API or a wire-selectable restore mode. + f.sql.update("INSERT INTO school_data(id,version,name) VALUES(99,1,'pre-stream fixture')",new Object[0]); + var original=signed(snapshot(0,"baseline"));f.reject("SNAPSHOT_CONFLICT",original); + assertEquals(99L,((Number)f.sql.queryForList("SELECT id FROM school_data",new Object[0]).get(0).get("id")).longValue()); + var configured=new SqliteCommittedReceiver(f.sql,SOURCE,TARGET,MODEL,Map.of("current",KEY),new Limits(10,65536),io.teaql.core.sqlite.SqliteCommittedSnapshot.ImportMode.REPLACE_UNINITIALIZED); + assertFalse(configured.receiveSnapshot(f.context,original).duplicate()); + configured.receive(f.context,SqliteCommittedReceiver.sign("current",TARGET,KEY,tail(1))); + try{configured.receiveSnapshot(f.context,signed(snapshot(0,"different")));fail();}catch(ReceiverException e){assertEquals("SNAPSHOT_CONFLICT",e.code());} + assertEquals("updated",f.sql.queryForList("SELECT name FROM school_data",new Object[0]).get(0).get("name"));assertEquals(1,f.count("school_data"));assertEquals(1,f.count("teaql_event_inbox"));assertEquals(2,f.count("teaql_event_rejection")); + System.out.println("CMR13_STARTUP_POLICY_PASS language=java default_reject=1 explicit_replace=1 active_stream_fenced=1"); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReceiverTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReceiverTest.java new file mode 100644 index 00000000..b45adc14 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedReceiverTest.java @@ -0,0 +1,127 @@ +package io.teaql.sqlite; + +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.core.sqlite.SqliteCommittedReceiver; +import io.teaql.core.sqlite.SqliteCommittedReceiver.Delivery; +import io.teaql.core.sqlite.SqliteCommittedReceiver.ReceiverException; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.Map; +import static org.junit.Assert.*; + +/** Public receiver APIs only: reusable by isolated installed consumers. */ +public class SqliteCommittedReceiverTest { + private static final String SOURCE="receiver-source", TARGET="receiver-target"; + private static final String MODEL="sha256:"+"0".repeat(64); + private static final byte[] KEY="0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.UTF_8); + private static final String CANARY="BUSINESS_SECRET_must_not_appear_in_audit"; + + private static String digest(String text) throws Exception {return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8)));} + private static OutboxRecord event(long sequence,String change,String source,boolean update) throws Exception { + String body="{\"protocol_version\":1,\"change_set_id\":\""+change+"\",\"source_runtime_id\":\""+source+"\",\"source_sequence\":\""+sequence+"\",\"source_model_hash\":\""+MODEL+"\",\"source_transaction_id\":\"tx\",\"committed_at\":\"2026-10-22T00:00:00Z\",\"audit\":{\"comment\":\"receive school\",\"purpose\":\"verify signed delivery\"},\"mutations\":[{\"entity_type\":\"school\",\"entity_id\":\"7\",\"operation\":\""+(update?"update":"create")+"\",\"base_version\":"+(update?"\"1\"":"null")+",\"committed_version\":\""+(update?"2":"1")+"\",\"fields\":{\"name\":\""+(update?"updated":CANARY)+"\"},\"trace_chain\":[]}]}"; + return new OutboxRecord(source,sequence,change,MODEL,digest(body),body); + } + private static Delivery sign(OutboxRecord event){return SqliteCommittedReceiver.sign("current",TARGET,KEY,event);} + private static Delivery replace(Delivery original,OutboxRecord event){return new Delivery(event,original.keyId(),original.targetId(),original.signature());} + private static final class Fixture implements AutoCloseable { + final Path dir=Files.createTempDirectory("cmr13-java-"); + final JdbcSqlExecutor sql; final DefaultUserContext context; final SqliteCommittedReceiver receiver; + Fixture() throws Exception { + SQLiteDataSource ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+dir.resolve("target.sqlite"));sql=new JdbcSqlExecutor(ds); + EntityDescriptor school=new EntityDescriptor();school.setType("school");school.addSimpleProperty("id",Long.class);school.addSimpleProperty("version",Long.class);school.addSimpleProperty("name",String.class); + SimpleEntityMetaFactory metadata=new SimpleEntityMetaFactory();metadata.register(school);context=new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).build()); + sql.execute("CREATE TABLE school_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,name TEXT NOT NULL)"); + receiver=new SqliteCommittedReceiver(sql,SOURCE,TARGET,MODEL,Map.of("current",KEY));receiver.ensureSchema(context); + } + long count(String table){return ((Number)sql.queryForList("SELECT count(*) AS n FROM "+table,new Object[0]).get(0).get("n")).longValue();} + void unchanged(long rows,long checkpoint,long audits){assertEquals(rows,count("school_data"));assertEquals(checkpoint,count("teaql_event_inbox"));assertEquals(audits,count("teaql_event_rejection"));} + void rejected(String code,Delivery delivery) {try{receiver.receive(context,delivery);fail("receiver must reject");}catch(ReceiverException error){assertEquals(code,error.code());assertFalse(error.toString().contains(CANARY));assertNull(error.getCause());}} + @Override public void close()throws Exception{try(var paths=Files.walk(dir)){for(Path p:paths.sorted(java.util.Comparator.reverseOrder()).toList())Files.delete(p);}} + } + + @Test public void validSignedApplyUpdateAndRetainedDuplicate()throws Exception{ + try(var f=new Fixture()){ + var a=sign(event(1,"create",SOURCE,false));var b=sign(event(2,"update",SOURCE,true)); + assertFalse(f.receiver.receive(f.context,a).duplicate());assertFalse(f.receiver.receive(f.context,b).duplicate());assertTrue(f.receiver.receive(f.context,a).duplicate());assertTrue(f.receiver.receive(f.context,b).duplicate());f.unchanged(1,2,0); + assertEquals("updated",f.sql.queryForList("SELECT name FROM school_data",new Object[0]).get(0).get("name")); + String origin=SqliteCommittedReceiver.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + if(System.getProperty("cmr13.source.root")!=null)assertTrue(origin,origin.startsWith(System.getProperty("cmr13.source.root")+"/teaql-sqlite/target/classes")); + System.out.println("CMR13_SOURCE_PASS language=java origin="+origin+" valid=2 duplicates=2"); + } + } + @Test public void tamperingEvenWithRecomputedChecksumRejectsWithoutWrites()throws Exception{ + try(var f=new Fixture()){ + var record=event(1,"a",SOURCE,false);var signed=sign(record);String changed=record.payloadJson().replace(CANARY,"attacker-value"); + for(String hash:new String[]{record.payloadSha256(),digest(changed)})f.rejected("INVALID_SIGNATURE",replace(signed,new OutboxRecord(SOURCE,1,"a",MODEL,hash,changed))); + f.unchanged(0,0,2); + } + } + @Test public void routingAndKeyChangesRejectAndAudit()throws Exception{ + try(var f=new Fixture()){ + var e=event(1,"a",SOURCE,false);var d=sign(e); + f.rejected("UNAUTHORIZED_SOURCE",sign(event(1,"a","other-source",false))); + f.rejected("UNAUTHORIZED_TARGET",new Delivery(e,d.keyId(),"other-target",d.signature())); + f.rejected("UNKNOWN_KEY",new Delivery(e,"retired",TARGET,d.signature())); + f.rejected("INVALID_SIGNATURE",new Delivery(e,d.keyId(),TARGET,new byte[32])); + f.rejected("INVALID_ENVELOPE",new Delivery(e,d.keyId(),TARGET,new byte[0])); + f.rejected("INVALID_SIGNATURE",replace(d,new OutboxRecord(SOURCE,2,"a",MODEL,e.payloadSha256(),e.payloadJson()))); + f.rejected("INVALID_SIGNATURE",replace(d,new OutboxRecord(SOURCE,1,"b",MODEL,e.payloadSha256(),e.payloadJson()))); + f.rejected("MODEL_MISMATCH",replace(d,new OutboxRecord(SOURCE,1,"a","sha256:"+"1".repeat(64),e.payloadSha256(),e.payloadJson()))); + f.unchanged(0,0,8); + } + } + @Test public void signedOutOfStreamReplayAndConflictAreAudited()throws Exception{ + try(var f=new Fixture()){ + var e=event(1,"a",SOURCE,false);f.receiver.receive(f.context,sign(e));f.receiver.receive(f.context,sign(event(2,"b",SOURCE,true))); + f.rejected("SEQUENCE_GAP",sign(event(1,"old-unseen",SOURCE,false))); + f.rejected("SEQUENCE_GAP",sign(event(4,"gap",SOURCE,true))); + String body=e.payloadJson().replace(CANARY,"changed");f.rejected("DUPLICATE_CONFLICT",sign(new OutboxRecord(SOURCE,1,"a",MODEL,digest(body),body))); + f.unchanged(1,2,3); + assertEquals(2L,((Number)f.sql.queryForList("SELECT source_sequence FROM teaql_event_checkpoint",new Object[0]).get(0).get("source_sequence")).longValue()); + } + } + @Test public void auditIsDurableHashOnlyAndUnavailableIsDistinct()throws Exception{ + try(var f=new Fixture()){ + var e=event(1,CANARY,SOURCE,false);f.rejected("INVALID_SIGNATURE",new Delivery(e,"current",TARGET,new byte[32])); + SQLiteDataSource ds=new SQLiteDataSource();ds.setUrl("jdbc:sqlite:"+f.dir.resolve("target.sqlite"));var reopened=new JdbcSqlExecutor(ds); + var audit=reopened.queryForList("SELECT * FROM teaql_event_rejection",new Object[0]);assertEquals(1,audit.size());assertFalse(audit.toString().contains(CANARY));assertFalse(audit.toString().contains(new String(KEY,StandardCharsets.UTF_8)));assertEquals("INVALID_SIGNATURE",audit.get(0).get("reason_code")); + f.sql.execute("CREATE TRIGGER reject_audit BEFORE INSERT ON teaql_event_rejection BEGIN SELECT RAISE(ABORT,'"+CANARY+"'); END"); + f.rejected("AUDIT_UNAVAILABLE",new Delivery(e,"current",TARGET,new byte[32]));f.unchanged(0,0,1); + } + } + @Test public void immutableKeyAndTagCopiesAndRotation()throws Exception{ + try(var f=new Fixture()){ + byte[] original=KEY.clone();var receiver=new SqliteCommittedReceiver(f.sql,SOURCE,TARGET,MODEL,Map.of("current",original,"previous",KEY));Arrays.fill(original,(byte)0); + var d=sign(event(1,"a",SOURCE,false));Arrays.fill(d.signature(),(byte)0);assertFalse(receiver.receive(f.context,d).duplicate()); + var old=SqliteCommittedReceiver.sign("previous",TARGET,KEY,event(2,"b",SOURCE,true));assertFalse(receiver.receive(f.context,old).duplicate()); + f.rejected("UNKNOWN_KEY",old);f.unchanged(1,2,1);assertFalse(d.toString().contains(CANARY));assertFalse(receiver.toString().contains(new String(KEY,StandardCharsets.UTF_8))); + } + } + @Test public void limitsAndAmbientTransactionFailClosed()throws Exception{ + try(var f=new Fixture()){ + var e=event(1,"a",SOURCE,false);var d=sign(e); + f.rejected("INVALID_ENVELOPE",replace(d,new OutboxRecord(SOURCE,0,"a",MODEL,e.payloadSha256(),e.payloadJson()))); + f.rejected("INVALID_ENVELOPE",replace(d,new OutboxRecord(SOURCE,1,"a",MODEL,e.payloadSha256(),"x".repeat(1024*1024+1)))); + f.rejected("INVALID_ENVELOPE",new Delivery(e,"x".repeat(257),TARGET,d.signature()));f.unchanged(0,0,3); + f.sql.executeInTransaction(()->{f.rejected("AMBIENT_TRANSACTION",d);assertEquals(0,f.count("school_data"));});f.unchanged(0,0,3); + try{new SqliteCommittedReceiver(f.sql,SOURCE,TARGET,MODEL,Map.of("weak",new byte[8]));fail();}catch(IllegalArgumentException expected){assertFalse(expected.toString().contains(CANARY));} + } + } + @Test public void goldenTranscriptHasCrossLanguageStableTag()throws Exception{ + var e=new OutboxRecord("source",42,"change",MODEL,"a".repeat(64),"{\"x\":\"雪\"}"); + var signed=SqliteCommittedReceiver.sign("key","target",KEY,e); + // Independent Python stdlib HMAC fixture; neither runtime supplies this oracle. + assertEquals("f1405877cd5999425693e559a147aa27294240456f88e307208b7162ff0bd8ac",HexFormat.of().formatHex(signed.signature())); + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedSnapshotTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedSnapshotTest.java new file mode 100644 index 00000000..ed71accf --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedSnapshotTest.java @@ -0,0 +1,342 @@ +package io.teaql.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.teaql.core.BaseEntity; +import io.teaql.core.meta.*; +import io.teaql.core.sqlite.*; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.*; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; +import java.nio.file.*; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import static org.junit.Assert.*; + +public class SqliteCommittedSnapshotTest { + private static final String SOURCE = "snapshot-source", MODEL = "sha256:" + "0".repeat(64); + private static final long ORDER = 9_007_199_254_740_993L; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final SqliteCommittedSnapshot.Limits LIMITS = SqliteCommittedSnapshot.Limits.defaults(); + private static final class Fixture implements AutoCloseable { + final Path directory, path; + final SQLiteDataSource dataSource = new SQLiteDataSource(); + final JdbcSqlExecutor sql; + final SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + final DefaultUserContext context; + Fixture(boolean legacy) throws Exception { + directory = Files.createTempDirectory("teaql-snapshot-"); path = directory.resolve("data.sqlite"); + dataSource.setUrl("jdbc:sqlite:" + path); + sql = executor(dataSource); sql.execute("PRAGMA journal_mode=WAL"); + EntityDescriptor root = entity("platform", "name", String.class); + // Exercise JDBC label folding without changing the model identity. + root.findIdProperty().with("columnName", "Id"); root.findVersionProperty().with("columnName", "Version"); + metadata.register(root); + EntityDescriptor status = entity("order_status", "code", String.class); + status.addObjectProperty(metadata, "platform", "platform", "status_list", BaseEntity.class); + metadata.register(status); + EntityDescriptor order = entity("customer_order", "name", String.class); + order.addSimpleProperty("amount", java.math.BigDecimal.class).with("sqlType", "NUMERIC(19,7)"); + order.addObjectProperty(metadata, "platform", "platform", "order_list", BaseEntity.class); + order.addObjectProperty(metadata, "status", "order_status", "order_list", BaseEntity.class); + metadata.register(order); + root.addObjectProperty(metadata, "favorite_order", "customer_order", "favorite_platform", BaseEntity.class); + EntityDescriptor item = entity("order_item", "quantity", Long.class); + item.addObjectProperty(metadata, "customer_order", "customer_order", "item_list", BaseEntity.class); + item.addSimpleProperty("occurred_at", java.time.Instant.class); + item.addSimpleProperty("planned_date", java.time.LocalDate.class); + item.addSimpleProperty("active", Boolean.class); + item.addSimpleProperty("ratio", Double.class); + item.addSimpleProperty("note", String.class); + metadata.register(item); metadata.register(entity("empty_probe", "name", String.class)); + context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).build()); + sql.execute("CREATE TABLE platform_data(Id INTEGER PRIMARY KEY,Version INTEGER NOT NULL,name TEXT,favorite_order INTEGER REFERENCES customer_order_data(id))"); + sql.execute("CREATE TABLE order_status_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,code TEXT,platform INTEGER REFERENCES platform_data(Id))"); + sql.execute("CREATE TABLE customer_order_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,name TEXT,amount TEXT,platform INTEGER REFERENCES platform_data(Id),status INTEGER REFERENCES order_status_data(id))"); + sql.execute("CREATE TABLE order_item_data(id INTEGER PRIMARY KEY,version INTEGER NOT NULL,quantity INTEGER,customer_order INTEGER REFERENCES customer_order_data(id),occurred_at INTEGER,planned_date TEXT,active INTEGER,ratio REAL,note TEXT)"); + sql.execute("CREATE TABLE empty_probe_data(id INTEGER PRIMARY KEY,version INTEGER,name TEXT)"); + new SqliteCommittedEventStore(sql).ensureSchema(); + if (legacy) sql.executeInTransaction(() -> { + sql.execute("PRAGMA defer_foreign_keys=ON"); + sql.update("INSERT INTO platform_data VALUES(1,3,'SOURCE ROOT',?)", new Object[] {ORDER}); + sql.execute("INSERT INTO order_status_data VALUES(1001,2,'OPEN',1)"); + sql.update("INSERT INTO customer_order_data VALUES(?,7,'LEGACY','123.4500000',1,1001)", new Object[] {ORDER}); + sql.update("INSERT INTO order_item_data VALUES(?,-4,2,?,-315619199877,'2026-10-16',1,0.125,NULL)", new Object[] {ORDER + 1, ORDER}); + }); + } + SqliteCommittedSnapshot snapshots() { return new SqliteCommittedSnapshot(sql, SOURCE, MODEL); } + SqliteModelCheckedCommittedTarget target() { return new SqliteModelCheckedCommittedTarget(sql, SOURCE, MODEL); } + SqliteCommittedSnapshot.SnapshotRecord snapshot() { return snapshots().exportSnapshot(context, "snapshot-1", "copy canonical facts", "initialize analytics", LIMITS); } + @Override public void close() throws Exception { + // Only the exact owned fixture files; no shared DB/cache cleanup. + for (String file : List.of("data.sqlite-wal", "data.sqlite-shm", "data.sqlite-journal", "data.sqlite")) Files.deleteIfExists(directory.resolve(file)); + Files.deleteIfExists(directory); + } + } + private static EntityDescriptor entity(String name, String field, Class type) { + var entity = new EntityDescriptor(); entity.setType(name); + entity.addSimpleProperty("id", Long.class); entity.addSimpleProperty("version", Long.class); + entity.addSimpleProperty(field, type); return entity; + } + private static JdbcSqlExecutor executor(SQLiteDataSource dataSource) { + return new JdbcSqlExecutor(dataSource).addConnectionInitializer(connection -> { + try (var statement = connection.createStatement()) { statement.execute("PRAGMA foreign_keys=ON"); statement.execute("PRAGMA busy_timeout=5000"); } + catch (java.sql.SQLException failure) { throw new IllegalStateException("fixture connection setup failed", failure); } + }); + } + private static void expect(Runnable action) { try { action.run(); fail("expected snapshot rejection"); } catch (SqliteCommittedEventStore.CommittedEventException expected) { assertNotNull(expected.code()); } } + private static long scalar(Fixture f, String sql) { return ((Number) f.sql.queryForList(sql, new Object[0]).get(0).values().iterator().next()).longValue(); } + private static void sameState(Fixture a, Fixture b) { + for (String table : List.of("platform_data", "order_status_data", "customer_order_data", "order_item_data", "empty_probe_data")) + assertEquals(table, a.sql.queryForList("SELECT * FROM " + table + " ORDER BY 1", new Object[0]), b.sql.queryForList("SELECT * FROM " + table + " ORDER BY 1", new Object[0])); + } + private static String tail(long sequence) { + return """ + {"protocol_version":1,"change_set_id":"tail-1","source_runtime_id":"snapshot-source", + "source_sequence":"%d","source_model_hash":"%s","source_transaction_id":"tx-tail", + "committed_at":"2026-10-22T00:00:00Z","audit":{"comment":"advance source","purpose":"verify snapshot tail"}, + "mutations":[{"entity_type":"customer_order","entity_id":"9007199254740993","operation":"update","base_version":"7","committed_version":"8","fields":{"name":"AFTER N","amount":"125.0000000"},"trace_chain":[]}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"recover","base_version":"-4","committed_version":"5","fields":{"quantity":"3"},"trace_chain":[]}]} + """.formatted(sequence, MODEL); + } + private static void commitTail(Fixture f) { + JdbcSqlExecutor writer = executor(f.dataSource); // A second real connection while export is reading. + writer.executeInTransaction(() -> { + writer.update("UPDATE customer_order_data SET version=8,name='AFTER N',amount='125.0000000' WHERE id=?", new Object[] {ORDER}); + writer.update("UPDATE order_item_data SET version=5,quantity=3 WHERE id=?", new Object[] {ORDER + 1}); + new SqliteCommittedEventStore(writer).append(SOURCE, "tail-1", MODEL, SqliteCommittedSnapshotTest::tail); + }); + } + private static SqliteCommittedSnapshot.SnapshotRecord changed(SqliteCommittedSnapshot.SnapshotRecord original, java.util.function.Consumer change) throws Exception { + ObjectNode body = (ObjectNode) JSON.readTree(original.payloadJson()); change.accept(body); String text = body.toString(); + return new SqliteCommittedSnapshot.SnapshotRecord(original.sourceRuntimeId(), original.sourceSequence(), original.snapshotId(), original.modelHash(), + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))), text); + } + + @Test public void legacySnapshotPlusConcurrentTailMatchesSourceAndDoesNotRewind() throws Exception { + try (Fixture source = new Fixture(true); Fixture target = new Fixture(false)) { + AtomicBoolean once = new AtomicBoolean(); + EntityMetaFactory intercepted = new EntityMetaFactory() { + public EntityDescriptor resolveEntityDescriptor(String type) { return source.metadata.resolveEntityDescriptor(type); } + public void register(EntityDescriptor type) { throw new AssertionError("no model changes"); } + public List allEntityDescriptors() { if (once.compareAndSet(false, true)) commitTail(source); return source.metadata.allEntityDescriptors(); } + }; + DefaultUserContext context = new DefaultUserContext(TeaQLRuntime.builder().metadata(intercepted).build()); + var snapshot = source.snapshots().exportSnapshot(context, "snapshot-1", "copy legacy state", "verify consistent bootstrap", LIMITS); + assertTrue(once.get()); assertEquals(0, snapshot.sourceSequence()); + JsonNode body = JSON.readTree(snapshot.payloadJson()); assertEquals(5, body.path("entity_types").size()); assertEquals(4, body.path("rows").size()); + assertTrue(snapshot.payloadJson().contains("LEGACY")); assertFalse(snapshot.payloadJson().contains("AFTER N")); + assertTrue(snapshot.payloadJson().contains("\"version\":\"-4\"")); + assertEquals(0, target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS).checkpoint()); + var events = new SqliteCommittedEventStore(source.sql).outboxAfter(SOURCE, snapshot.sourceSequence(), 10); + assertEquals(1, events.size()); assertEquals(1, events.get(0).sourceSequence()); + assertEquals(1, target.target().apply(target.context, events.get(0)).checkpoint()); sameState(source, target); + assertTrue(target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS).duplicate()); + assertEquals(0, target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS).checkpoint()); + assertEquals(1, scalar(target, "SELECT source_sequence FROM teaql_event_checkpoint")); sameState(source, target); + assertTrue(target.target().apply(target.context, events.get(0)).duplicate()); + expect(() -> target.snapshots().initializeSnapshot(target.context, source.snapshot(), SqliteCommittedSnapshot.ImportMode.REPLACE_UNINITIALIZED, LIMITS)); + System.out.println("CMR11_SNAPSHOT_PASS language=java snapshot=0 tail=1 rows=4 catalog=5 concurrent_writer=1 duplicate_after_tail=1"); + } + } + @Test public void explicitReplacementHandlesBootstrapRowsButDefaultCannotOverwrite() throws Exception { + try (Fixture source = new Fixture(true); Fixture target = new Fixture(true)) { + target.sql.execute("UPDATE platform_data SET name='BOOTSTRAP'"); var snapshot = source.snapshot(); + expect(() -> target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS)); + assertEquals(0, scalar(target, "SELECT count(*) FROM teaql_event_snapshot")); + target.snapshots().initializeSnapshot(target.context, snapshot, SqliteCommittedSnapshot.ImportMode.REPLACE_UNINITIALIZED, LIMITS); sameState(source, target); + } + } + @Test public void nonzeroSnapshotStartsAtNPlusOneWithoutReplayingOlderHistory() throws Exception { + try (Fixture source = new Fixture(true); Fixture target = new Fixture(false)) { + commitTail(source); var snapshot = source.snapshot(); assertEquals(1, snapshot.sourceSequence()); + target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS); sameState(source, target); + var old = new SqliteCommittedEventStore(source.sql).outboxAfter(SOURCE, 0, 10).get(0); + expect(() -> target.target().apply(target.context, old)); // No fabricated pre-snapshot Inbox. + source.sql.executeInTransaction(() -> { + assertEquals(1, source.sql.update("UPDATE customer_order_data SET version=9,name='AT TWO' WHERE id=? AND version=8", new Object[] {ORDER})); + new SqliteCommittedEventStore(source.sql).append(SOURCE, "tail-2", MODEL, sequence -> { + try { + ObjectNode body = (ObjectNode) JSON.readTree(tail(sequence)); body.put("change_set_id", "tail-2"); + var mutations = body.withArray("mutations"); ObjectNode mutation = (ObjectNode) mutations.get(0); + mutation.put("base_version", "8"); mutation.put("committed_version", "9"); + mutation.with("fields").removeAll().put("name", "AT TWO"); mutations.remove(1); + return body.toString(); + } catch (Exception failure) { throw new AssertionError(failure); } + }); + }); + var next = new SqliteCommittedEventStore(source.sql).outboxAfter(SOURCE, 1, 10); + assertEquals(1, next.size()); assertEquals(2, next.get(0).sourceSequence()); + target.target().apply(target.context, next.get(0)); sameState(source, target); + assertEquals(1, target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS).checkpoint()); + assertEquals(2, scalar(target, "SELECT source_sequence FROM teaql_event_checkpoint")); + System.out.println("CMR11_NONZERO_PASS language=java snapshot=1 tail=2 legacy_history_absent=1"); + } + } + @Test public void lateStorageFailureRollsBackReplacementRowsSnapshotAndCheckpoint() throws Exception { + try (Fixture source = new Fixture(true); Fixture target = new Fixture(true)) { + target.sql.execute("UPDATE platform_data SET name='BOOTSTRAP'"); + target.sql.execute("CREATE TRIGGER reject_snapshot BEFORE INSERT ON order_item_data BEGIN SELECT RAISE(ABORT,'snapshot fault'); END"); + var snapshot = source.snapshot(); + try { target.snapshots().initializeSnapshot(target.context, snapshot, SqliteCommittedSnapshot.ImportMode.REPLACE_UNINITIALIZED, LIMITS); fail("expected storage failure"); } catch (RuntimeException expected) { } + assertEquals("BOOTSTRAP", target.sql.queryForList("SELECT name FROM platform_data", new Object[0]).get(0).get("name")); + assertEquals(0, scalar(target, "SELECT count(*) FROM teaql_event_snapshot")); assertEquals(0, scalar(target, "SELECT count(*) FROM teaql_event_checkpoint")); + target.sql.execute("DROP TRIGGER reject_snapshot"); target.snapshots().initializeSnapshot(target.context, snapshot, SqliteCommittedSnapshot.ImportMode.REPLACE_UNINITIALIZED, LIMITS); sameState(source, target); + } + } + @Test public void strictCatalogRowsVersionsProtocolAndBudgetsRejectBeforeAnyTargetChange() throws Exception { + try (Fixture source = new Fixture(true); Fixture target = new Fixture(false)) { + var snapshot = source.snapshot(); + for (var invalid : List.of( + changed(snapshot, b -> b.withArray("entity_types").remove(0)), + changed(snapshot, b -> ((ObjectNode) b.withArray("rows").get(0)).with("fields").put("unknown", "bad")), + changed(snapshot, b -> ((ObjectNode) b.withArray("rows").get(0)).put("version", "0")), + changed(snapshot, b -> b.put("snapshot_version", 4294967297L)), + changed(snapshot, b -> b.withArray("rows").add(b.withArray("rows").get(0).deepCopy())))) + expect(() -> target.snapshots().initializeSnapshot(target.context, invalid, LIMITS)); + expect(() -> target.snapshots().initializeSnapshot(target.context, snapshot, new SqliteCommittedSnapshot.Limits(3, LIMITS.maxPayloadBytes()))); + expect(() -> target.snapshots().initializeSnapshot(target.context, snapshot, new SqliteCommittedSnapshot.Limits(10, 1))); + expect(() -> source.snapshots().exportSnapshot(source.context, "small", "copy", "budget", new SqliteCommittedSnapshot.Limits(3, 100_000))); + expect(() -> source.snapshots().exportSnapshot(source.context, "small", "copy", "budget", new SqliteCommittedSnapshot.Limits(10, 1))); + assertEquals(0, scalar(target, "SELECT count(*) FROM customer_order_data")); assertEquals(0, scalar(target, "SELECT count(*) FROM teaql_event_snapshot")); + assertFalse(snapshot.toString().contains("SOURCE ROOT")); + } + } + @Test public void callerTransactionsAndMissingIntentAreRejected() throws Exception { + try (Fixture f = new Fixture(true)) { + expect(() -> f.snapshots().exportSnapshot(f.context, "x", " ", "purpose", LIMITS)); + f.sql.executeInTransaction(() -> expect(() -> f.snapshot())); + } + } + @Test public void ordinaryEventCannotUseSnapshotVersionEscapeHatch() throws Exception { + try (Fixture source = new Fixture(true); Fixture target = new Fixture(false)) { + String body = tail(1).replace("\"operation\":\"update\",\"base_version\":\"7\",\"committed_version\":\"8\"", "\"operation\":\"create\",\"base_version\":null,\"committed_version\":\"8\""); + var record = new SqliteCommittedEventStore.OutboxRecord(SOURCE, 1, "tail-1", MODEL, HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(body.getBytes(StandardCharsets.UTF_8))), body); + expect(() -> target.target().apply(target.context, record)); assertEquals(0, scalar(target, "SELECT count(*) FROM customer_order_data")); + } + } + + private static SqliteCommittedSnapshotPublisher publisher(Fixture source, String target) { return new SqliteCommittedSnapshotPublisher(source.sql,SOURCE,target,MODEL); } + private static void secondTail(Fixture source) { + source.sql.executeInTransaction(() -> { + assertEquals(1,source.sql.update("UPDATE customer_order_data SET version=9,name='AT TWO' WHERE id=? AND version=8",new Object[]{ORDER})); + new SqliteCommittedEventStore(source.sql).append(SOURCE,"tail-2",MODEL,sequence->{ + try { ObjectNode body=(ObjectNode)JSON.readTree(tail(sequence));body.put("change_set_id","tail-2");var mutations=body.withArray("mutations");var row=(ObjectNode)mutations.get(0);row.put("base_version","8").put("committed_version","9");row.with("fields").removeAll().put("name","AT TWO");mutations.remove(1);return body.toString(); } + catch(Exception failure){throw new AssertionError(failure);} + }); + }); + } + @Test public void durableBootstrapKeepsOriginalBytesAndResumesOrdinaryPublisherAfterLostAck() throws Exception { + try(Fixture source=new Fixture(true);Fixture target=new Fixture(false)) { + commitTail(source);var pub=publisher(source,"analytics"); + var prepared=pub.prepare(source.context,"bootstrap","copy facts","initialize target",LIMITS); + assertEquals(1,prepared.snapshot().sourceSequence());assertFalse(prepared.acknowledged());secondTail(source); + assertEquals(prepared,pub.prepare(source.context,"bootstrap","retry copy","retry bootstrap",LIMITS)); + var normal=new SqliteCommittedOutboxPublisher(source.sql,SOURCE,"analytics",MODEL); + expect(()->normal.publishBatch(10,row->{throw new AssertionError("pending bootstrap must block tail");})); + assertThrows(IllegalStateException.class,()->pub.publish(source.context,"bootstrap",LIMITS,row->{ + assertFalse(source.sql.hasActiveTransaction()); + var receipt=SqliteCommittedSnapshotPublisher.initializeAndAcknowledge(target.context,target.snapshots(),row,"analytics",LIMITS); + assertFalse(receipt.duplicate());throw new IllegalStateException("fixture loses committed target ACK"); + })); + assertEquals(0,scalar(source,"SELECT count(*) FROM teaql_event_delivery_cursor")); + var restarted=publisher(source,"analytics"); + assertTrue(restarted.publish(source.context,"bootstrap",LIMITS,row->{ + var receipt=SqliteCommittedSnapshotPublisher.initializeAndAcknowledge(target.context,target.snapshots(),row,"analytics",LIMITS);assertTrue(receipt.duplicate());return receipt; + }).acknowledged()); + assertEquals(1,normal.status().ackedSequence()); + assertEquals(1,normal.publishBatch(10,row->{assertEquals(2,row.sourceSequence());return target.target().apply(target.context,row);}).acknowledged()); + sameState(source,target);assertEquals(2,normal.status().ackedSequence()); + assertTrue(restarted.publish(source.context,"bootstrap",LIMITS,row->{throw new AssertionError("known ACK must not redeliver");}).acknowledged()); + assertEquals(2,normal.status().ackedSequence());expect(()->pub.prepare(source.context,"different","copy","bootstrap",LIMITS)); + System.out.println("CMR12_BOOTSTRAP_PASS language=java baseline=1 tail=2 retained_bytes=1 lost_ack=1 pending_gate=1 no_rewind=1"); + } + } + @Test public void acknowledgementAndCursorAreAtomicAndEveryReceiptIdentityIsBound() throws Exception { + try(Fixture source=new Fixture(true);Fixture target=new Fixture(false)) { + commitTail(source);var pub=publisher(source,"analytics");var snapshot=pub.prepare(source.context,"bootstrap","copy","bootstrap",LIMITS).snapshot(); + var ack=SqliteCommittedSnapshotPublisher.initializeAndAcknowledge(target.context,target.snapshots(),snapshot,"analytics",LIMITS); + for(var bad:List.of( + new SqliteCommittedSnapshotPublisher.Receipt("other",ack.targetId(),ack.snapshotId(),1,MODEL,ack.payloadSha256(),false), + new SqliteCommittedSnapshotPublisher.Receipt(SOURCE,"other",ack.snapshotId(),1,MODEL,ack.payloadSha256(),false), + new SqliteCommittedSnapshotPublisher.Receipt(SOURCE,ack.targetId(),"other",1,MODEL,ack.payloadSha256(),false), + new SqliteCommittedSnapshotPublisher.Receipt(SOURCE,ack.targetId(),ack.snapshotId(),2,MODEL,ack.payloadSha256(),false), + new SqliteCommittedSnapshotPublisher.Receipt(SOURCE,ack.targetId(),ack.snapshotId(),1,"sha256:"+"1".repeat(64),ack.payloadSha256(),false), + new SqliteCommittedSnapshotPublisher.Receipt(SOURCE,ack.targetId(),ack.snapshotId(),1,MODEL,"0".repeat(64),false))) + expect(()->pub.publish(source.context,"bootstrap",LIMITS,row->bad)); + source.sql.execute("CREATE TRIGGER reject_bootstrap_ack BEFORE UPDATE ON teaql_event_snapshot_publication WHEN NEW.acknowledged=1 BEGIN SELECT RAISE(ABORT,'fixture rejects ACK after cursor insert'); END"); + assertThrows(RuntimeException.class,()->pub.publish(source.context,"bootstrap",LIMITS,row->ack)); + assertEquals(0,scalar(source,"SELECT count(*) FROM teaql_event_delivery_cursor")); + assertEquals(0,scalar(source,"SELECT acknowledged FROM teaql_event_snapshot_publication")); + source.sql.execute("DROP TRIGGER reject_bootstrap_ack");assertTrue(pub.publish(source.context,"bootstrap",LIMITS,row->ack).acknowledged()); + source.sql.execute("DELETE FROM teaql_event_delivery_cursor"); + expect(()->new SqliteCommittedOutboxPublisher(source.sql,SOURCE,"analytics",MODEL).publishBatch(10,row->{throw new AssertionError("corrupt ACK/cursor must fail before delivery");})); + System.out.println("CMR12_ACK_ATOMIC_PASS language=java bound_fields=6 rollback=1 retry=1 corrupt_cursor=1"); + } + } + @Test public void bootstrapModeIsExclusiveAndDoesNotJoinCallerTransactionsOrResetPublicationState() throws Exception { + try(Fixture source=new Fixture(true);Fixture target=new Fixture(false)) { + var normal=new SqliteCommittedOutboxPublisher(source.sql,SOURCE,"incremental",MODEL); + expect(()->normal.publishBatch(0,row->{throw new AssertionError();}));assertEquals(0,scalar(source,"SELECT count(*) FROM teaql_event_delivery_cursor")); + assertEquals(0,normal.publishBatch(1,row->{throw new AssertionError();}).acknowledged()); + expect(()->publisher(source,"incremental").prepare(source.context,"bootstrap","copy","bootstrap",LIMITS)); + assertEquals(0,scalar(source,"SELECT count(*) FROM teaql_event_snapshot_publication")); + source.sql.executeInTransaction(()->expect(()->publisher(source,"new-target").prepare(source.context,"bootstrap","copy","bootstrap",LIMITS))); + var targetPub=publisher(target,"another-target");targetPub.prepare(target.context,"bootstrap","copy","bootstrap",LIMITS); + expect(()->target.snapshots().initializeSnapshot(target.context,source.snapshot(),SqliteCommittedSnapshot.ImportMode.REPLACE_UNINITIALIZED,LIMITS)); + assertEquals(1,scalar(target,"SELECT count(*) FROM teaql_event_snapshot_publication")); + } + } + @Test public void nontransactionalAdaptersCannotPretendToProvideAtomicBootstrap() throws Exception { + try(Fixture source=new Fixture(true);Fixture target=new Fixture(false)) { + var queries=new java.util.concurrent.atomic.AtomicInteger(); + var unsafe=(io.teaql.dataservice.sql.SqlExecutionAdapter)java.lang.reflect.Proxy.newProxyInstance( + getClass().getClassLoader(),new Class[]{io.teaql.dataservice.sql.SqlExecutionAdapter.class},(proxy,method,args)->{ + if(method.getName().equals("executeInTransaction")){((Runnable)args[0]).run();return null;} + if(method.getName().startsWith("query")||method.getName().equals("update")||method.getName().equals("execute"))queries.incrementAndGet(); + try{return method.invoke(target.sql,args);}catch(java.lang.reflect.InvocationTargetException failure){throw failure.getCause();} + }); + var snapshot=source.snapshot(); + expect(()->new SqliteCommittedSnapshot(unsafe,SOURCE,MODEL).exportSnapshot(target.context,"test","copy","bootstrap",LIMITS)); + expect(()->new SqliteCommittedSnapshot(unsafe,SOURCE,MODEL).initializeSnapshot(target.context,snapshot,LIMITS)); + expect(()->new SqliteCommittedSnapshotPublisher(unsafe,SOURCE,"target",MODEL).prepare(target.context,"test","copy","bootstrap",LIMITS)); + expect(()->new SqliteCommittedOutboxPublisher(unsafe,SOURCE,"target",MODEL).publishBatch(1,row->{throw new AssertionError();})); + assertEquals(0,queries.get()); + } + } + @Test public void snapshotHashAuthorityParserReferencesAndConflictingRetryFailClosed() throws Exception { + try (Fixture source = new Fixture(true); Fixture target = new Fixture(false)) { + var snapshot = source.snapshot(); + for (var invalid : List.of( + new SqliteCommittedSnapshot.SnapshotRecord("another-source", 0, snapshot.snapshotId(), MODEL, snapshot.payloadSha256(), snapshot.payloadJson()), + new SqliteCommittedSnapshot.SnapshotRecord(SOURCE, 0, snapshot.snapshotId(), "sha256:" + "1".repeat(64), snapshot.payloadSha256(), snapshot.payloadJson()), + new SqliteCommittedSnapshot.SnapshotRecord(SOURCE, 0, snapshot.snapshotId(), MODEL, "0".repeat(64), snapshot.payloadJson()), + changed(snapshot, body -> body.put("source_sequence", "00")), + changed(snapshot, body -> { + for (JsonNode row : body.withArray("rows")) if (row.path("entity_type").asText().equals("customer_order")) + ((ObjectNode) row.path("fields").path("platform")).put("entity_id", "999"); + }))) expect(() -> target.snapshots().initializeSnapshot(target.context, invalid, LIMITS)); + for (String text : List.of(snapshot.payloadJson() + " {}", + snapshot.payloadJson().replace("\"snapshot_version\":1", "\"snapshot_version\":1,\"snapshot_version\":1"))) { + var invalid = new SqliteCommittedSnapshot.SnapshotRecord(SOURCE, 0, snapshot.snapshotId(), MODEL, + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8))), text); + expect(() -> target.snapshots().initializeSnapshot(target.context, invalid, LIMITS)); + } + assertEquals(0, scalar(target, "SELECT count(*) FROM platform_data")); + assertEquals(0, scalar(target, "SELECT count(*) FROM customer_order_data")); + assertEquals(0, scalar(target, "SELECT count(*) FROM teaql_event_snapshot")); + assertEquals(0, scalar(target, "SELECT count(*) FROM teaql_event_checkpoint")); + target.sql.executeInTransaction(() -> expect(() -> target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS))); + target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS); sameState(source, target); + var conflict = changed(snapshot, body -> ((ObjectNode) body.path("audit")).put("comment", "different bytes for same snapshot")); + expect(() -> target.snapshots().initializeSnapshot(target.context, conflict, LIMITS)); + assertTrue(target.snapshots().initializeSnapshot(target.context, snapshot, LIMITS).duplicate()); + sameState(source, target); + System.out.println("CMR11_GUARDS_PASS language=java source=1 model=1 digest=1 parser=2 dangling_fk=1 conflict=1 atomic=1 retry=1"); + } + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java new file mode 100644 index 00000000..31bd12e4 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -0,0 +1,674 @@ +package io.teaql.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.BaseEntity; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedEventStore.CommittedEventException; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.junit.Assume; +import org.sqlite.SQLiteDataSource; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.Map; + +import static org.junit.Assert.*; + +public class SqliteModelCheckedCommittedTargetTest { + private static final String SOURCE = "sqlite-order-source"; + private static final String HASH = "sha256:" + "0".repeat(64); + private static final long ORDER_ID = 9_007_199_254_740_993L; + private static final long ITEM_ID = ORDER_ID + 1; + + private record Target(JdbcSqlExecutor sql, SqliteModelCheckedCommittedTarget target, + DefaultUserContext context) { } + + private static Target target(Path path) { + return target(path, "NUMERIC(19,7)"); + } + + private static Target target(Path path, String amountType) { + SQLiteDataSource source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor sql = new JdbcSqlExecutor(source); + EntityDescriptor order = new EntityDescriptor(); + order.setType("order"); + order.addSimpleProperty("id", Long.class); + order.addSimpleProperty("version", Long.class); + order.addSimpleProperty("order_number", String.class); + order.addSimpleProperty("status", String.class); + order.addSimpleProperty("planned_date", java.time.LocalDate.class); + order.addSimpleProperty("occurred_at", java.time.Instant.class).with("required", "true"); + var amount = order.addSimpleProperty("amount", java.math.BigDecimal.class); + if (amountType != null) amount.with("sqlType", amountType); + order.addSimpleProperty("active", Boolean.class).with("required", "true"); + order.addSimpleProperty("ratio", Double.class).with("required", "true"); + order.addSimpleProperty("optional_note", String.class); + order.addSimpleProperty("optional_label", String.class); + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + metadata.register(order); + EntityDescriptor item = new EntityDescriptor(); + item.setType("order_item"); + item.addSimpleProperty("id", Long.class); + item.addSimpleProperty("version", Long.class); + item.addObjectProperty(metadata, "order", "order", "items", BaseEntity.class) + .with("columnName", "order_id"); + item.addSimpleProperty("quantity", Long.class); + metadata.register(item); + DefaultUserContext context = new DefaultUserContext( + TeaQLRuntime.builder().metadata(metadata).build()); + sql.execute("CREATE TABLE order_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE, " + + "occurred_at INTEGER NOT NULL, amount NUMERIC, " + + "active INTEGER NOT NULL, ratio REAL NOT NULL, optional_note TEXT, optional_label TEXT)"); + sql.execute("CREATE TABLE order_item_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + + "order_id INTEGER NOT NULL, quantity INTEGER NOT NULL)"); + SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget(sql, SOURCE, HASH); + target.ensureSchema(); + return new Target(sql, target, context); + } + + private static String payload(long sequence, String change, String mutations) { + return """ + {"protocol_version":1,"change_set_id":"%s","source_runtime_id":"%s", + "source_sequence":"%d","source_model_hash":"%s","source_transaction_id":"tx-%d", + "committed_at":"2026-10-16T00:00:00Z", + "audit":{"actor_ref":"fixture:operator","comment":"Apply order graph", + "purpose":"Verify target application","trace_id":"fixture-trace"}, + "mutations":%s} + """.formatted(change, SOURCE, sequence, HASH, sequence, mutations); + } + + private static String created() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order_number":"ORD-0001","status":"DRAFT", + "planned_date":"2026-10-16","occurred_at":"-315619199877", + "amount":"0", + "active":true,"ratio":0.5,"optional_note":null, + "optional_label":"initial label"},"trace_chain":[]}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, + "quantity":"2"},"trace_chain":[]}] + """; + } + + private static String submitted() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"update", + "base_version":"1","committed_version":"2", + "fields":{"status":"SUBMITTED","optional_note":"submitted note", + "optional_label":null},"trace_chain":[]}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"delete", + "base_version":"1","committed_version":"-2","fields":{},"trace_chain":[]}] + """; + } + + private static OutboxRecord record(long sequence, String change, String mutations) { + return recordBody(sequence, change, payload(sequence, change, mutations)); + } + + private static OutboxRecord recordBody(long sequence, String change, String body) { + try { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(body.getBytes(StandardCharsets.UTF_8))); + return new OutboxRecord(SOURCE, sequence, change, HASH, digest, body); + } catch (java.security.NoSuchAlgorithmException impossible) { + throw new AssertionError("JDK SHA-256 is unavailable", impossible); + } + } + + private static void expect(String code, Runnable action) { + try { + action.run(); + fail("expected " + code); + } catch (CommittedEventException failure) { + assertEquals(code, failure.code()); + } + } + + private static long count(JdbcSqlExecutor sql, String table) { + return ((Number) sql.queryForList("SELECT COUNT(*) AS n FROM " + table, + new Object[0]).get(0).get("n")).longValue(); + } + + private static final class ForbiddenBusinessApi extends RuntimeException { + ForbiddenBusinessApi(String method) { super("business API reached: " + method); } + } + + /** CMR-09: deny every Context method except its actual metadata capability. */ + @Test public void committedTargetUsesOnlyMetadataAndNeverReplaysBusinessApis() throws Exception { + Path path = Files.createTempFile("teaql-target-nonreplay-", ".sqlite"); + try { + Target t = target(path); + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(t.context()); + var unexpected = new java.util.ArrayList(); + var metadataReads = new java.util.concurrent.atomic.AtomicInteger(); + io.teaql.core.UserContext guarded = (io.teaql.core.UserContext) + java.lang.reflect.Proxy.newProxyInstance(io.teaql.core.UserContext.class.getClassLoader(), + new Class[] {io.teaql.core.UserContext.class}, (proxy, method, args) -> { + if (method.getName().equals("capability") && args.length == 1 + && args[0] == EntityMetaFactory.class) { + metadataReads.incrementAndGet(); + return metadata; + } + unexpected.add(method.getName()); + throw new ForbiddenBusinessApi(method.getName()); + }); + // Positive controls: the guard must intercept real entrypoints, + // not count zero because a test double was never wired in. + Runnable[] controls = { + () -> guarded.capability(io.teaql.core.checker.Checker.class), + () -> guarded.beginFixEvidence(), + () -> guarded.saveGraph((io.teaql.core.Entity) null), + () -> guarded.businessTime(), + () -> guarded.capability(io.teaql.core.InternalIdGenerationService.class), + () -> guarded.httpRequests(), + () -> guarded.extension("llm"), + () -> guarded.extension("notifications") + }; + for (Runnable control : controls) { + boolean intercepted = false; + try { control.run(); } catch (ForbiddenBusinessApi expected) { intercepted = true; } + assertTrue("positive control did not reach the Context guard", intercepted); + } + assertEquals(8, unexpected.size()); + unexpected.clear(); + String deleted = """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"delete", + "base_version":"2","committed_version":"-3","fields":{},"trace_chain":[]}] + """; + String recovered = """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"recover", + "base_version":"-3","committed_version":"4", + "fields":{"status":"RECOVERED"},"trace_chain":[]}] + """; + String[] facts = {created(), submitted(), deleted, recovered}; + for (int index = 0; index < facts.length; index++) { + OutboxRecord event = record(index + 1, "nonreplay-" + index, facts[index]); + assertEquals(index + 1, t.target().apply(guarded, event).checkpoint()); + assertTrue(t.target().apply(guarded, event).duplicate()); + assertTrue("target invoked a forbidden business API: " + unexpected, unexpected.isEmpty()); + } + Map row = t.sql().queryForList( + "SELECT id,version,order_number,status,occurred_at FROM order_data WHERE id=?", + new Object[] {ORDER_ID}).get(0); + assertEquals(ORDER_ID, ((Number) row.get("id")).longValue()); + assertEquals(4, ((Number) row.get("version")).longValue()); + assertEquals("ORD-0001", row.get("order_number")); + assertEquals("RECOVERED", row.get("status")); + assertEquals(-315_619_199_877L, ((Number) row.get("occurred_at")).longValue()); + assertEquals(4, count(t.sql(), "teaql_event_inbox")); + assertEquals(4, count(t.sql(), "teaql_event_applied_payload")); + assertEquals(8, metadataReads.get()); + System.out.println("CMR_NONREPLAY_PASS language=java controls=8 business_calls=0 metadata_reads=8 operations=4 duplicates=4"); + } finally { Files.deleteIfExists(path); } + } + + @Test public void appliesSharedConformanceFixtureWhenConfigured() throws Exception { + String fixturePath = System.getenv("TEAQL_COMMITTED_EVENT_FIXTURE"); + Assume.assumeTrue("shared fixture path was not supplied", + fixturePath != null && !fixturePath.isBlank()); + JsonNode fixture = new ObjectMapper().readTree(Files.readString(Path.of(fixturePath))); + assertEquals(1, fixture.path("fixture_version").asInt()); + assertEquals(2, fixture.path("sets").size()); + Path path = Files.createTempFile("teaql-shared-model-target-", ".sqlite"); + try { + Target t = target(path); + for (int index = 0; index < 2; index++) { + JsonNode set = fixture.path("sets").get(index); + long sequence = Long.parseLong(set.path("source_sequence").asText()); + OutboxRecord record = recordBody(sequence, + set.path("change_set_id").asText(), set.toString()); + assertEquals(sequence, t.target().apply(t.context(), record).checkpoint()); + } + assertEquals("SUBMITTED", t.sql().queryForList( + "SELECT status FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("status")); + assertEquals("2026-10-16", t.sql().queryForList( + "SELECT planned_date FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("planned_date")); + Map instant = t.sql().queryForList( + "SELECT typeof(occurred_at) AS storage_type, occurred_at FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals("integer", instant.get("storage_type")); + assertEquals(-315_619_199_877L, ((Number) instant.get("occurred_at")).longValue()); + Map scalars = t.sql().queryForList( + "SELECT active, ratio, optional_note, optional_label FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(1L, ((Number) scalars.get("active")).longValue()); + assertEquals(0.5, ((Number) scalars.get("ratio")).doubleValue(), 0.0); + assertEquals("submitted note", scalars.get("optional_note")); + assertNull(scalars.get("optional_label")); + assertEquals(-2L, ((Number) t.sql().queryForList( + "SELECT version FROM order_item_data WHERE id = ?", new Object[] {ITEM_ID}) + .get(0).get("version")).longValue()); + assertEquals(2, count(t.sql(), "teaql_event_applied_payload")); + } finally { Files.deleteIfExists(path); } + } + + @Test public void appliesWholeOrderGraphAndReturnsStableDuplicateAcks() throws Exception { + Path path = Files.createTempFile("teaql-model-target-", ".sqlite"); + try { + Target t = target(path); + OutboxRecord first = record(1, "order-graph-0001", created()); + OutboxRecord second = record(2, "order-graph-0002", submitted()); + OutboxRecord otherSource = new OutboxRecord("other-source", first.sourceSequence(), + first.changeSetId(), first.modelHash(), first.payloadSha256(), first.payloadJson()); + expect("UNAUTHORIZED_SOURCE", () -> t.target().apply(t.context(), otherSource)); + OutboxRecord otherModel = new OutboxRecord(first.sourceRuntimeId(), first.sourceSequence(), + first.changeSetId(), "sha256:" + "1".repeat(64), first.payloadSha256(), first.payloadJson()); + expect("MODEL_MISMATCH", () -> t.target().apply(t.context(), otherModel)); + String unsupported = payload(1, "unsupported", created()) + .replace("\"protocol_version\":1", "\"protocol_version\":2"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + recordBody(1, "unsupported", unsupported))); + expect("SEQUENCE_GAP", () -> t.target().apply(t.context(), second)); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(1, t.target().apply(t.context(), first).checkpoint()); + assertTrue(t.target().apply(t.context(), first).duplicate()); + assertEquals(2, t.target().apply(t.context(), second).checkpoint()); + var oldAck = t.target().apply(t.context(), first); + assertTrue(oldAck.duplicate()); + assertEquals(1, oldAck.checkpoint()); + Map order = t.sql().queryForList( + "SELECT version, order_number, status FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(2L, ((Number) order.get("version")).longValue()); + assertEquals("ORD-0001", order.get("order_number")); + assertEquals("SUBMITTED", order.get("status")); + Map item = t.sql().queryForList( + "SELECT version, order_id, quantity FROM order_item_data WHERE id = ?", + new Object[] {ITEM_ID}).get(0); + assertEquals(-2L, ((Number) item.get("version")).longValue()); + assertEquals(ORDER_ID, ((Number) item.get("order_id")).longValue()); + assertEquals(2L, ((Number) item.get("quantity")).longValue()); + assertEquals(2, count(t.sql(), "teaql_event_inbox")); + assertEquals(2, count(t.sql(), "teaql_event_applied_payload")); + SQLiteDataSource reopenedSource = new SQLiteDataSource(); + reopenedSource.setUrl("jdbc:sqlite:" + path); + SqliteModelCheckedCommittedTarget reopened = new SqliteModelCheckedCommittedTarget( + new JdbcSqlExecutor(reopenedSource), SOURCE, HASH); + var retryAfterRestart = reopened.apply(t.context(), first); + assertTrue(retryAfterRestart.duplicate()); + assertEquals(1, retryAfterRestart.checkpoint()); + String retained = (String) t.sql().queryForList( + "SELECT payload_json FROM teaql_event_applied_payload WHERE source_sequence = 2", + new Object[0]).get(0).get("payload_json"); + assertTrue(retained.contains("fixture-trace")); + } finally { Files.deleteIfExists(path); } + } + + /** CMR-05/06: all three operations in one set, not separate successful sets. */ + @Test public void mixedCreateUpdateDeleteRollsBackThenRetriesAndDeduplicates() throws Exception { + Path path = Files.createTempFile("teaql-mixed-target-", ".sqlite"); + try { + Target t = target(path); + assertEquals(1, t.target().apply(t.context(), record(1, "mixed-baseline", created())).checkpoint()); + var mutations = (com.fasterxml.jackson.databind.node.ArrayNode) new ObjectMapper().readTree(submitted()); + mutations.add(new ObjectMapper().readTree(""" + {"entity_type":"order_item","entity_id":"9007199254740995","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, + "quantity":"3"},"trace_chain":[]} + """)); + OutboxRecord mixed = record(2, "mixed-three-operations", mutations.toString()); + // The fault fires only if update and delete were already executed. + // A failure before those writes cannot masquerade as rollback proof. + t.sql().execute("CREATE TRIGGER reject_mixed_create BEFORE INSERT ON order_item_data " + + "WHEN NEW.id = 9007199254740995 " + + "AND (SELECT version FROM order_data WHERE id = 9007199254740993) = 2 " + + "AND (SELECT version FROM order_item_data WHERE id = 9007199254740994) = -2 " + + "BEGIN SELECT RAISE(ABORT, 'fixture rejects final create'); END"); + try { + t.target().apply(t.context(), mixed); + fail("expected final-create failure after update and delete"); + } catch (RuntimeException expected) { + var order = t.sql().queryForList("SELECT version,status FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(1L, ((Number) order.get("version")).longValue()); + assertEquals("DRAFT", order.get("status")); + assertEquals(1L, ((Number) t.sql().queryForList( + "SELECT version FROM order_item_data WHERE id = ?", new Object[] {ITEM_ID}) + .get(0).get("version")).longValue()); + assertTrue(t.sql().queryForList("SELECT id FROM order_item_data WHERE id = ?", + new Object[] {ITEM_ID + 1}).isEmpty()); + assertEquals(1, count(t.sql(), "teaql_event_inbox")); + assertEquals(1, count(t.sql(), "teaql_event_applied_payload")); + assertEquals(1L, ((Number) t.sql().queryForList( + "SELECT source_sequence FROM teaql_event_checkpoint WHERE source_runtime_id = ?", + new Object[] {SOURCE}).get(0).get("source_sequence")).longValue()); + } + t.sql().execute("DROP TRIGGER reject_mixed_create"); // disarm fault; do not reset data + assertEquals(2, t.target().apply(t.context(), mixed).checkpoint()); + var duplicate = t.target().apply(t.context(), mixed); + assertTrue(duplicate.duplicate()); + assertEquals(2, duplicate.checkpoint()); + var order = t.sql().queryForList("SELECT version,status FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(2L, ((Number) order.get("version")).longValue()); + assertEquals("SUBMITTED", order.get("status")); + assertEquals(-2L, ((Number) t.sql().queryForList( + "SELECT version FROM order_item_data WHERE id = ?", new Object[] {ITEM_ID}) + .get(0).get("version")).longValue()); + var added = t.sql().queryForList("SELECT version,order_id,quantity FROM order_item_data WHERE id = ?", + new Object[] {ITEM_ID + 1}).get(0); + assertEquals(1L, ((Number) added.get("version")).longValue()); + assertEquals(ORDER_ID, ((Number) added.get("order_id")).longValue()); + assertEquals(3L, ((Number) added.get("quantity")).longValue()); + assertEquals(1, count(t.sql(), "order_data")); + assertEquals(2, count(t.sql(), "order_item_data")); + assertEquals(2, count(t.sql(), "teaql_event_inbox")); + assertEquals(2, count(t.sql(), "teaql_event_applied_payload")); + assertEquals(2L, ((Number) t.sql().queryForList( + "SELECT source_sequence FROM teaql_event_checkpoint WHERE source_runtime_id = ?", + new Object[] {SOURCE}).get(0).get("source_sequence")).longValue()); + String retained = (String) t.sql().queryForList( + "SELECT payload_json FROM teaql_event_applied_payload WHERE source_sequence = 2", + new Object[0]).get(0).get("payload_json"); + assertEquals(3, new ObjectMapper().readTree(retained).path("mutations").size()); + System.out.println("MIXED_TARGET_ATOMIC_PASS language=java mixed=3 rollback=1 retry=1 duplicate=1"); + } finally { Files.deleteIfExists(path); } + } + + @Test public void rejectsUnknownModelAndRollsBackWhenSecondObjectFails() throws Exception { + Path path = Files.createTempFile("teaql-model-target-negative-", ".sqlite"); + try { + Target t = target(path); + OutboxRecord first = record(1, "order-graph-0001", created()); + String unknownEntity = created().replace("\"entity_type\":\"order_item\"", + "\"entity_type\":\"not_in_model\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), record(1, "unknown-entity", unknownEntity))); + String unknown = created().replace("\"quantity\":\"2\"", + "\"quantity\":\"2\",\"not_in_model\":1"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), record(1, "unknown", unknown))); + String reverseRelation = created().replace("\"status\":\"DRAFT\"", + "\"status\":\"DRAFT\",\"items\":[]"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "reverse-relation", reverseRelation))); + String invalidTrace = created().replace("\"trace_chain\":[]", + "\"trace_chain\":[{\"kind\":\"AuditReason\",\"entity_type\":\"order\"," + + "\"entity_id\":\"9007199254740993\",\"comment\":\"Create order\"}]"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-trace", invalidTrace))); + String missingTrace = created().replace(",\"trace_chain\":[]", ""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "missing-trace", missingTrace))); + t.sql().execute("CREATE TRIGGER reject_item BEFORE INSERT ON order_item_data " + + "BEGIN SELECT RAISE(ABORT, 'reject item'); END"); + try { + t.target().apply(t.context(), first); + fail("expected second-object failure"); + } catch (RuntimeException expected) { + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "order_item_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + assertEquals(0, count(t.sql(), "teaql_event_applied_payload")); + assertEquals(0, count(t.sql(), "teaql_event_checkpoint")); + } + } finally { Files.deleteIfExists(path); } + } + + @Test public void preservesLargeLongFieldWithoutJsonNumberRounding() throws Exception { + Path path = Files.createTempFile("teaql-model-target-large-long-", ".sqlite"); + try { + Target t = target(path); + String numeric = created().replace("\"quantity\":\"2\"", + "\"quantity\":9007199254740993"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "unsafe-json-number", numeric))); + for (String invalid : new String[] {"02", "-0", "9223372036854775808"}) { + String malformed = created().replace("\"quantity\":\"2\"", + "\"quantity\":\"" + invalid + "\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-integer", malformed))); + } + assertEquals(0, count(t.sql(), "order_data")); + String textual = created().replace("\"quantity\":\"2\"", + "\"quantity\":\"9007199254740993\""); + t.target().apply(t.context(), record(1, "lossless-long", textual)); + Object stored = t.sql().queryForList("SELECT quantity FROM order_item_data WHERE id = ?", + new Object[] {ITEM_ID}).get(0).get("quantity"); + assertEquals(9_007_199_254_740_993L, ((Number) stored).longValue()); + } finally { Files.deleteIfExists(path); } + } + + @Test public void acceptsOnlyCanonicalCalendarDate() throws Exception { + Path path = Files.createTempFile("teaql-model-target-date-", ".sqlite"); + try { + Target t = target(path); + for (String invalid : new String[] {"2026-2-3", "2026-02-30", "2026-10-16T00:00:00Z"}) { + String malformed = created().replace("\"planned_date\":\"2026-10-16\"", + "\"planned_date\":\"" + invalid + "\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-date", malformed))); + } + String numeric = created().replace("\"planned_date\":\"2026-10-16\"", + "\"planned_date\":20261016"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "numeric-date", numeric))); + assertEquals(0, count(t.sql(), "order_data")); + t.target().apply(t.context(), record(1, "valid-date", created())); + assertEquals("2026-10-16", t.sql().queryForList( + "SELECT planned_date FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("planned_date")); + } finally { Files.deleteIfExists(path); } + } + + @Test public void acceptsOnlyLosslessInstantMillisecondText() throws Exception { + Path path = Files.createTempFile("teaql-model-target-instant-", ".sqlite"); + try { + Target t = target(path); + for (String invalid : new String[] { + "-315619199877", "\"1960-01-01T00:00:00.123Z\"", + "\"1960-01-01T00:00:00.123\"", "\"-0315619199877\"", + "\"-0\"", "\"9223372036854775808\""}) { + String malformed = created().replace("\"occurred_at\":\"-315619199877\"", + "\"occurred_at\":" + invalid); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-instant", malformed))); + } + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + t.target().apply(t.context(), record(1, "valid-instant", created())); + Map stored = t.sql().queryForList( + "SELECT typeof(occurred_at) AS storage_type, occurred_at FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals("integer", stored.get("storage_type")); + assertEquals(-315_619_199_877L, ((Number) stored.get("occurred_at")).longValue()); + } finally { Files.deleteIfExists(path); } + } + + @Test public void rejectsSqliteRealCoercionOfExactDecimal() throws Exception { + Path path = Files.createTempFile("teaql-model-target-decimal-", ".sqlite"); + try { + Target t = target(path); + for (String decimal : new String[] { + "0.1", "123456789012345.6789", "0.12345678", "1234567890123"}) { + String inexact = created().replace("\"amount\":\"0\"", + "\"amount\":\"" + decimal + "\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "inexact-decimal", inexact))); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + assertEquals(0, count(t.sql(), "teaql_event_checkpoint")); + assertEquals(0, count(t.sql(), "teaql_event_applied_payload")); + } + t.target().apply(t.context(), record(1, "exact-decimal", created())); + Map stored = t.sql().queryForList( + "SELECT typeof(amount) AS storage_type, amount FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals("integer", stored.get("storage_type")); + assertEquals(0L, ((Number) stored.get("amount")).longValue()); + } finally { Files.deleteIfExists(path); } + } + + @Test public void rejectsDecimalWithoutExplicitModeledPrecisionAndScale() throws Exception { + Path path = Files.createTempFile("teaql-model-target-untyped-decimal-", ".sqlite"); + try { + Target t = target(path, null); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "untyped-decimal", created()))); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + } finally { Files.deleteIfExists(path); } + } + + @Test public void preservesBooleanFiniteFloatAndExplicitNull() throws Exception { + Path path = Files.createTempFile("teaql-model-target-scalar-kinds-", ".sqlite"); + try { + Target t = target(path); + String textBoolean = created().replace("\"active\":true", "\"active\":\"true\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "text-boolean", textBoolean))); + String textFloat = created().replace("\"ratio\":0.5", "\"ratio\":\"0.5\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "text-float", textFloat))); + String nullBoolean = created().replace("\"active\":true", "\"active\":null"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "null-boolean", nullBoolean))); + String omitted = created().replace(",\"optional_note\":null", ""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "omitted-note", omitted))); + assertEquals(0, count(t.sql(), "order_data")); + + t.target().apply(t.context(), record(1, "valid-scalar-kinds", created())); + Map stored = t.sql().queryForList( + "SELECT active, ratio, optional_note FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(1L, ((Number) stored.get("active")).longValue()); + assertEquals(0.5, ((Number) stored.get("ratio")).doubleValue(), 0.0); + assertNull(stored.get("optional_note")); + } finally { Files.deleteIfExists(path); } + } + + @Test public void appliesNullableTransitionsWithoutTreatingOmissionAsNull() throws Exception { + Path path = Files.createTempFile("teaql-model-target-null-transition-", ".sqlite"); + try { + Target t = target(path); + t.target().apply(t.context(), record(1, "created-null-transition", created())); + String nullRequired = submitted().replace("\"status\":\"SUBMITTED\"", + "\"status\":\"SUBMITTED\",\"active\":null"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(2, "null-required-update", nullRequired))); + String wrongKind = submitted().replace("\"optional_note\":\"submitted note\"", + "\"optional_note\":123"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(2, "wrong-note-kind", wrongKind))); + assertEquals(1, count(t.sql(), "teaql_event_inbox")); + Map unchanged = t.sql().queryForList( + "SELECT version, optional_note, optional_label FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(1L, ((Number) unchanged.get("version")).longValue()); + assertNull(unchanged.get("optional_note")); + assertEquals("initial label", unchanged.get("optional_label")); + + t.target().apply(t.context(), record(2, "changed-null-transition", submitted())); + Map stored = t.sql().queryForList( + "SELECT version, active, optional_note, optional_label FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(2L, ((Number) stored.get("version")).longValue()); + assertEquals(1L, ((Number) stored.get("active")).longValue()); + assertEquals("submitted note", stored.get("optional_note")); + assertNull(stored.get("optional_label")); + } finally { Files.deleteIfExists(path); } + } + + @Test public void rejectsTriggerRewriteAndStalePredecessor() throws Exception { + Path path = Files.createTempFile("teaql-model-target-trigger-", ".sqlite"); + try { + Target t = target(path); + OutboxRecord first = record(1, "order-graph-0001", created()); + t.sql().execute("CREATE TRIGGER rewrite_order AFTER INSERT ON order_data " + + "BEGIN UPDATE order_data SET status = 'TARGET_OVERRIDE' WHERE id = NEW.id; END"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), first)); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + t.sql().execute("DROP TRIGGER rewrite_order"); + t.target().apply(t.context(), first); + String stale = submitted().replaceFirst("\\\"base_version\\\":\\\"1\\\"", + "\"base_version\":\"9\"").replaceFirst("\\\"committed_version\\\":\\\"2\\\"", + "\"committed_version\":\"10\""); + expect("VERSION_CONFLICT", () -> t.target().apply(t.context(), record(2, "stale", stale))); + assertEquals(1, count(t.sql(), "teaql_event_inbox")); + } finally { Files.deleteIfExists(path); } + } + + @Test public void orderedPublisherRecoversAmbiguousAckAfterSourceReopen() throws Exception { + Path sourcePath = Files.createTempFile("teaql-source-delivery-", ".sqlite"); + Path targetPath = Files.createTempFile("teaql-target-delivery-", ".sqlite"); + try { + SQLiteDataSource sourceDataSource = new SQLiteDataSource(); + sourceDataSource.setUrl("jdbc:sqlite:" + sourcePath); + JdbcSqlExecutor sourceSql = new JdbcSqlExecutor(sourceDataSource); + SqliteCommittedEventStore sourceEvents = new SqliteCommittedEventStore(sourceSql); + sourceEvents.ensureSchema(); + for (String change : new String[] {"order-graph-0001", "order-graph-0002"}) { + String mutations = change.endsWith("0001") ? created() : submitted(); + sourceSql.executeInTransaction(() -> sourceEvents.append(SOURCE, change, HASH, + sequence -> record(sequence, change, mutations).payloadJson())); + } + Target target = target(targetPath); + SqliteCommittedOutboxPublisher publisher = new SqliteCommittedOutboxPublisher( + sourceSql, SOURCE, "analytics", HASH); + assertEquals(2, publisher.status().backlog()); + expect("ACK_MISMATCH", () -> publisher.publishBatch(2, + record -> SqliteCommittedEventStore.InboxOutcome.applied(99))); + assertEquals(0, publisher.status().ackedSequence()); + try { + publisher.publishBatch(2, record -> { + target.target().apply(target.context(), record); + throw new IllegalStateException("acknowledgement lost after target commit"); + }); + fail("expected ambiguous acknowledgement"); + } catch (IllegalStateException expected) { + assertEquals("acknowledgement lost after target commit", expected.getMessage()); + } + assertEquals(0, publisher.status().ackedSequence()); + assertEquals(1, count(target.sql(), "teaql_event_inbox")); + + SqliteCommittedOutboxPublisher restarted = new SqliteCommittedOutboxPublisher( + new JdbcSqlExecutor(sourceDataSource), SOURCE, "analytics", HASH); + var outcome = restarted.publishBatch(2, + record -> target.target().apply(target.context(), record)); + assertEquals(2, outcome.acknowledged()); + assertEquals(2, outcome.status().ackedSequence()); + assertEquals(0, outcome.status().backlog()); + assertEquals(0, restarted.publishBatch(2, record -> { + throw new AssertionError("acked event must not be delivered again"); + }).acknowledged()); + assertEquals(2, count(target.sql(), "teaql_event_inbox")); + assertEquals("SUBMITTED", target.sql().queryForList( + "SELECT status FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("status")); + assertEquals(2, new SqliteCommittedOutboxPublisher(sourceSql, SOURCE, + "second-target", HASH).status().backlog()); + expect("MODEL_MISMATCH", () -> new SqliteCommittedOutboxPublisher(sourceSql, + SOURCE, "analytics", "sha256:" + "1".repeat(64)).status()); + sourceSql.update("DELETE FROM teaql_event_outbox WHERE source_runtime_id = ? " + + "AND source_sequence = 1", new Object[] {SOURCE}); + expect("SEQUENCE_GAP", () -> new SqliteCommittedOutboxPublisher(sourceSql, + SOURCE, "second-target", HASH).publishBatch(2, record -> { + throw new AssertionError("gap must be rejected before delivery"); + })); + } finally { + Files.deleteIfExists(sourcePath); + Files.deleteIfExists(targetPath); + } + } +} diff --git a/teaql-tfp-endpoint/src/test/java/io/teaql/tfp/TfpEndpointPolicyTest.java b/teaql-tfp-endpoint/src/test/java/io/teaql/tfp/TfpEndpointPolicyTest.java index 1b3e663d..a68c810d 100644 --- a/teaql-tfp-endpoint/src/test/java/io/teaql/tfp/TfpEndpointPolicyTest.java +++ b/teaql-tfp-endpoint/src/test/java/io/teaql/tfp/TfpEndpointPolicyTest.java @@ -77,6 +77,46 @@ public void requiresTrustedContextAndNeverDropsFilter() throws Exception { org.junit.Assert.assertTrue(response.get("data") instanceof java.util.List); } + @Test + public void committedReplicationControlsNeverReachOrdinaryExecutors() throws Exception { + TfpEndpointHandler handler = handler(); + ObjectMapper json = new ObjectMapper(); + String sql = "INSERT INTO probe_data(id,status) VALUES(99,'SQL-CANARY')"; + for (String control : java.util.List.of("sql", "rawSql", "statement", "statements", + "committed_mutation_set", "source_sequence", "replication", "apply")) { + var query = (com.fasterxml.jackson.databind.node.ObjectNode) json.readTree(query("id")); + query.put(control, sql); + assertCode("TFP_INVALID_REQUEST", () -> handler.handleQuery(context, trusted(), json.writeValueAsBytes(query))); + var mutation = json.createObjectNode().put("entity", "Probe").put("action", "Create") + .put("comment", "inspect ordinary entry boundary").put(control, sql); + mutation.putObject("payload").put("status", "PAID"); + assertCode("TFP_INVALID_REQUEST", () -> handler.handleMutation(context, trusted(), json.writeValueAsBytes(mutation))); + org.junit.Assert.assertNull(capturedQuery); + org.junit.Assert.assertNull(capturedMutation); + } + // Neither a protocol-shaped document nor an action name creates dispatch authority. + byte[] committed = json.writeValueAsBytes(json.createObjectNode().put("protocol_version", 1) + .put("source_runtime_id", "untrusted-source").put("source_sequence", "1")); + assertCode("TFP_INVALID_REQUEST", () -> handler.handleQuery(context, trusted(), committed)); + assertCode("TFP_INVALID_REQUEST", () -> handler.handleMutation(context, trusted(), committed)); + for (String action : java.util.List.of("Apply", "Replicate", "CommittedMutationSet", "receive", "rawSql")) { + var mutation = json.createObjectNode().put("entity", "Probe").put("action", action) + .put("comment", "attempt replication dispatch"); + mutation.putObject("payload").put("status", "PAID"); + assertCode("TFP_POLICY_VIOLATION", () -> handler.handleMutation(context, trusted(), json.writeValueAsBytes(mutation))); + org.junit.Assert.assertNull(capturedMutation); + } + // Ordinary modeled values are not scanned for SQL keywords. + handler.handleQuery(context, trusted(), query("id")); + org.junit.Assert.assertNotNull(capturedQuery); + var allowed = json.createObjectNode().put("entity", "Probe").put("action", "Create") + .put("comment", "save SQL-looking ordinary text"); + allowed.putObject("payload").put("status", sql); + handler.handleMutation(context, trusted(), json.writeValueAsBytes(allowed)); + org.junit.Assert.assertEquals(sql, ((Probe) ((EntityPersistenceMutation) capturedMutation).getEntity()).getStatus()); + System.out.println("CMR15_ENDPOINT_SOURCE_PASS language=java controls=8 actions=5 executor_spy=true database=false"); + } + @Test public void rejectsGovernedRequestsWithoutAnInvokingContext() { io.teaql.core.TeaQLRuntimeException error = assertThrows( diff --git a/teaql-tool-http/src/main/java/io/teaql/tools/http/HttpServiceProfile.java b/teaql-tool-http/src/main/java/io/teaql/tools/http/HttpServiceProfile.java index 00e6da37..5495abdb 100644 --- a/teaql-tool-http/src/main/java/io/teaql/tools/http/HttpServiceProfile.java +++ b/teaql-tool-http/src/main/java/io/teaql/tools/http/HttpServiceProfile.java @@ -1,9 +1,14 @@ package io.teaql.tools.http; import io.teaql.core.http.HttpRequestService; +import java.io.ByteArrayInputStream; import java.net.URI; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; import java.time.Duration; +import java.util.ArrayList; import java.util.EnumSet; +import java.util.List; import java.util.Objects; import java.util.Set; @@ -11,6 +16,7 @@ public final class HttpServiceProfile { private final String alias; private final URI baseUri; + private final List trustedRoots; private final Duration connectTimeout; private final Duration requestTimeout; private final Duration totalTimeout; @@ -26,6 +32,7 @@ public final class HttpServiceProfile { private HttpServiceProfile(Builder b) { this.alias = b.alias; this.baseUri = b.baseUri; + this.trustedRoots = b.trustedRoots; this.connectTimeout = b.connectTimeout; this.requestTimeout = b.requestTimeout; this.totalTimeout = b.totalTimeout; @@ -42,6 +49,7 @@ private HttpServiceProfile(Builder b) { public static Builder builder(String alias, URI baseUri) { return new Builder(alias, baseUri); } public String alias() { return alias; } public URI baseUri() { return baseUri; } + List trustedRoots() { return trustedRoots; } public Duration connectTimeout() { return connectTimeout; } public Duration requestTimeout() { return requestTimeout; } public Duration totalTimeout() { return totalTimeout; } @@ -58,6 +66,7 @@ private HttpServiceProfile(Builder b) { public static final class Builder { private final String alias; private final URI baseUri; + private List trustedRoots = List.of(); private Duration connectTimeout = Duration.ofSeconds(2); private Duration requestTimeout = Duration.ofSeconds(5); private Duration totalTimeout = Duration.ofSeconds(10); @@ -94,6 +103,39 @@ public Builder budgets(Duration connect, Duration request, Duration total) { return this; } public Builder maxResponseBytes(int bytes) { this.maxResponseBytes = bytes; return this; } + /** Replace this profile's custom trust set with exactly one PEM root. */ + public Builder trustedRootPem(byte[] pem) { + if (pem == null) throw new IllegalArgumentException("invalid trusted TLS root"); + return trustedRootsPem(List.of(pem)); + } + + /** Explicit short overlap for CA rotation; custom roots replace system roots for this profile. */ + public Builder trustedRootsPem(List pems) { + if (!"https".equalsIgnoreCase(baseUri.getScheme()) || pems == null + || pems.isEmpty() || pems.size() > 4) { + throw new IllegalArgumentException("invalid trusted TLS root"); + } + try { + var parsed = new ArrayList(pems.size()); + var factory = CertificateFactory.getInstance("X.509"); + for (byte[] pem : pems) { + if (pem == null || pem.length == 0 || pem.length > 64 * 1024) { + throw new IllegalArgumentException("invalid trusted TLS root"); + } + var certificates = factory.generateCertificates(new ByteArrayInputStream(pem.clone())); + if (certificates.size() != 1 + || !(certificates.iterator().next() instanceof X509Certificate root) + || parsed.contains(root)) { + throw new IllegalArgumentException("invalid trusted TLS root"); + } + parsed.add(root); + } + trustedRoots = List.copyOf(parsed); + } catch (java.security.cert.CertificateException failure) { + throw new IllegalArgumentException("invalid trusted TLS root"); + } + return this; + } public Builder retry(int attempts, Duration delay, Set methods) { this.maxAttempts = attempts; this.retryDelay = positive(delay); diff --git a/teaql-tool-http/src/main/java/io/teaql/tools/http/JdkHttpRequestService.java b/teaql-tool-http/src/main/java/io/teaql/tools/http/JdkHttpRequestService.java index 81ff9949..8e82b94f 100644 --- a/teaql-tool-http/src/main/java/io/teaql/tools/http/JdkHttpRequestService.java +++ b/teaql-tool-http/src/main/java/io/teaql/tools/http/JdkHttpRequestService.java @@ -12,6 +12,8 @@ import java.net.http.HttpResponse; import java.net.http.HttpTimeoutException; import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.security.KeyStore; import java.time.Duration; import java.util.LinkedHashMap; import java.util.List; @@ -24,6 +26,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.logging.Logger; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; /** Shared JDK transport for startup registered HTTP profiles. */ public final class JdkHttpRequestService implements HttpRequestService { @@ -265,11 +269,30 @@ private static final class State { this.profile = profile; this.clock = clock; this.refilledAt = clock.nanoTime(); - this.client = HttpClient.newBuilder() + HttpClient.Builder builder = HttpClient.newBuilder() .connectTimeout(profile.connectTimeout()) - .followRedirects(HttpClient.Redirect.NEVER).build(); + .followRedirects(HttpClient.Redirect.NEVER); + if (!profile.trustedRoots().isEmpty()) builder.sslContext(trustedRootContext(profile)); + this.client = builder.build(); this.tokens = profile.ratePerSecond(); } + private static SSLContext trustedRootContext(HttpServiceProfile profile) { + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, null); + for (int index = 0; index < profile.trustedRoots().size(); index++) { + store.setCertificateEntry("root-" + index, profile.trustedRoots().get(index)); + } + TrustManagerFactory managers = TrustManagerFactory.getInstance( + TrustManagerFactory.getDefaultAlgorithm()); + managers.init(store); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, managers.getTrustManagers(), null); + return context; + } catch (GeneralSecurityException | java.io.IOException failure) { + throw new IllegalArgumentException("invalid trusted TLS root"); + } + } synchronized boolean acquireCircuit() { if (profile.circuitFailures() == 0) return false; if (openUntil == 0) return false; diff --git a/teaql-tool-http/src/test/java/io/teaql/tools/http/HttpProfileTrustTest.java b/teaql-tool-http/src/test/java/io/teaql/tools/http/HttpProfileTrustTest.java new file mode 100644 index 00000000..34e76316 --- /dev/null +++ b/teaql-tool-http/src/test/java/io/teaql/tools/http/HttpProfileTrustTest.java @@ -0,0 +1,243 @@ +package io.teaql.tools.http; + +import io.teaql.core.UserContext; +import io.teaql.core.http.HttpRequestFailure; +import io.teaql.core.http.HttpRequestService; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Proxy; +import java.net.InetAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyFactory; +import java.security.KeyStore; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.security.spec.PKCS8EncodedKeySpec; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import static org.junit.Assert.*; + +/** Dedicated private-CA proof; no Secret Provider or external service is involved. */ +public class HttpProfileTrustTest { + private static Path directory; + private static final List roots = new ArrayList<>(); + + private static void openssl(String... args) throws Exception { + var command = new ArrayList(); command.add("openssl"); command.addAll(List.of(args)); + Process child = new ProcessBuilder(command).directory(directory.toFile()) + .redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start(); + try { + assertTrue("TLS fixture generator exceeded budget", child.waitFor(30, TimeUnit.SECONDS)); + assertEquals("TLS fixture generator failed", 0, child.exitValue()); + } finally { + if (child.isAlive()) { child.destroyForcibly(); child.waitFor(); } + } + } + + @BeforeClass public static void createCertificates() throws Exception { + directory = Files.createTempDirectory("teaql-http-trust-"); + for (int i = 0; i < 4; i++) { + openssl("req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=TeaQL HTTP fixture root " + i, + "-addext", "basicConstraints=critical,CA:TRUE", + "-addext", "keyUsage=critical,keyCertSign,cRLSign", + "-keyout", "root-" + i + ".key", "-out", "root-" + i + ".pem"); + roots.add(Files.readAllBytes(directory.resolve("root-" + i + ".pem"))); + } + openssl("req", "-new", "-newkey", "rsa:2048", "-nodes", "-subj", "/CN=HTTP fixture", + "-addext", "subjectAltName=IP:127.0.0.1", "-keyout", "server.key", "-out", "server.csr"); + for (int i = 0; i < 2; i++) openssl("x509", "-req", "-days", "1", "-CAcreateserial", + "-copy_extensions", "copy", "-in", "server.csr", "-CA", "root-" + i + ".pem", + "-CAkey", "root-" + i + ".key", "-out", "server-" + i + ".pem"); + openssl("req", "-new", "-key", "server.key", "-subj", "/CN=HTTP wrong host fixture", + "-addext", "subjectAltName=IP:127.0.0.2", "-out", "wrong-host.csr"); + openssl("x509", "-req", "-days", "1", "-CAcreateserial", "-copy_extensions", "copy", + "-in", "wrong-host.csr", "-CA", "root-0.pem", "-CAkey", "root-0.key", + "-out", "wrong-host.pem"); + } + + @AfterClass public static void removeOwnedPrivateKeys() throws Exception { + if (directory == null) return; + try (var files = Files.walk(directory)) { + for (Path file : files.sorted(Comparator.reverseOrder()).toList()) Files.delete(file); + } + } + + private static HttpServiceProfile.Builder profile(String alias, URI uri) { + return HttpServiceProfile.builder(alias, uri) + .budgets(Duration.ofSeconds(3), Duration.ofSeconds(3), Duration.ofSeconds(5)); + } + + private static UserContext context() { + return (UserContext) Proxy.newProxyInstance(UserContext.class.getClassLoader(), + new Class[] { UserContext.class }, (proxy, method, args) -> { + if (method.isDefault()) return InvocationHandler.invokeDefault(proxy, method, args); + return null; + }); + } + + private static void success(JdkHttpRequestService service, String alias) { + var response = service.execute(context(), HttpRequestService.Call.get(alias, "status", + "load private upstream status", "verify profile-scoped HTTPS trust")); + assertEquals(200, response.status()); assertArrayEquals("ok".getBytes(StandardCharsets.US_ASCII), response.body()); + } + + private static void rejected(JdkHttpRequestService service, String alias) { + var failure = assertThrows(HttpRequestFailure.class, () -> success(service, alias)); + assertEquals(HttpRequestFailure.Code.UPSTREAM_FAILURE, failure.code()); assertNull(failure.getCause()); + } + + @Test public void rejectsInvalidTrustInputsAndAcceptsOneToFourDistinctRoots() { + URI https = URI.create("https://127.0.0.1/"); + byte[] old = roots.get(0), next = roots.get(1); + byte[] bundled = new byte[old.length + next.length]; + System.arraycopy(old, 0, bundled, 0, old.length); System.arraycopy(next, 0, bundled, old.length, next.length); + var invalid = List.of(List.of(), List.of(new byte[0]), List.of(new byte[65537]), + List.of("PRIVATE_ROOT_CANARY".getBytes(StandardCharsets.US_ASCII)), List.of(bundled), + List.of(old, old), List.of(old, new String(old, StandardCharsets.US_ASCII) + .replace("\n", "\r\n").getBytes(StandardCharsets.US_ASCII)), + List.of(old, next, roots.get(2), roots.get(3), old)); + for (var input : invalid) { + var failure = assertThrows(IllegalArgumentException.class, + () -> profile("upstream", https).trustedRootsPem(input)); + assertEquals("invalid trusted TLS root", failure.getMessage()); assertNull(failure.getCause()); + } + assertThrows(IllegalArgumentException.class, () -> profile("upstream", https).trustedRootPem(null)); + assertThrows(IllegalArgumentException.class, () -> profile("upstream", https).trustedRootsPem(null)); + assertThrows(IllegalArgumentException.class, () -> profile("upstream", https).trustedRootsPem(Arrays.asList(old, null))); + assertThrows(IllegalArgumentException.class, + () -> profile("upstream", URI.create("http://127.0.0.1/")).trustedRootPem(old)); + for (int count = 1; count <= 4; count++) assertEquals(count, + profile("upstream", https).trustedRootsPem(roots.subList(0, count)).build().trustedRoots().size()); + System.out.println("HTTP_TRUST_INPUT_PASS language=java bounded=1 malformed=1 duplicate=1 bundle=1 https_only=1"); + } + + @Test public void customRootsStayProfileLocalAndDoNotDisableHostnameChecks() throws Exception { + try (var peer = new Peer("server-0.pem", 0); var wrong = new Peer("wrong-host.pem", 0)) { + var service = new JdkHttpRequestService(List.of(profile("system", peer.uri()).build(), + profile("trusted", peer.uri()).trustedRootPem(roots.get(0)).build(), + profile("wrong_host", wrong.uri()).trustedRootPem(roots.get(0)).build())); + rejected(service, "system"); assertEquals(0, peer.requests.get()); + success(service, "trusted"); assertEquals(1, peer.requests.get()); + rejected(service, "system"); assertEquals(1, peer.requests.get()); + rejected(service, "wrong_host"); assertEquals(0, wrong.requests.get()); + System.out.println("HTTP_TRUST_ISOLATION_PASS language=java system_reject=2 custom_success=1 hostname_reject=1"); + } + } + + @Test public void overlappingRootsRotateAcrossBothLivePeersWithoutReset() throws Exception { + try (var old = new Peer("server-0.pem", 0); var next = new Peer("server-1.pem", 1)) { + var oldOnly = new JdkHttpRequestService(List.of(profile("old", old.uri()).trustedRootPem(roots.get(0)).build(), + profile("next", next.uri()).trustedRootPem(roots.get(0)).build())); + success(oldOnly, "old"); rejected(oldOnly, "next"); assertEquals(0, next.requests.get()); + var overlap = new JdkHttpRequestService(List.of(profile("old", old.uri()).trustedRootsPem(roots.subList(0, 2)).build(), + profile("next", next.uri()).trustedRootsPem(roots.subList(0, 2)).build())); + var newOnly = new JdkHttpRequestService(List.of(profile("old", old.uri()).trustedRootPem(roots.get(1)).build(), + profile("next", next.uri()).trustedRootPem(roots.get(1)).build())); + for (int round = 1; round <= 2; round++) { + success(overlap, "old"); success(overlap, "next"); rejected(newOnly, "old"); success(newOnly, "next"); + assertEquals(1 + round, old.requests.get()); assertEquals(2 * round, next.requests.get()); + System.out.println("HTTP_TRUST_ROTATION_PASS language=java round=" + round + " old_requests=" + + old.requests.get() + " new_requests=" + next.requests.get()); + } + } + } + + @Test public void builtProfilesSnapshotInputsAndFailedReplacementIsAtomic() throws Exception { + try (var peer = new Peer("server-0.pem", 0)) { + byte[] copy = roots.get(0).clone(); var input = new ArrayList<>(List.of(copy)); + var builder = profile("trusted", peer.uri()).trustedRootsPem(input); var before = builder.build(); + input.clear(); Arrays.fill(copy, (byte) 0); + assertThrows(IllegalArgumentException.class, () -> builder.trustedRootsPem(List.of(roots.get(1), new byte[0]))); + var afterFailure = builder.build(); builder.trustedRootPem(roots.get(1)); + success(new JdkHttpRequestService(List.of(before)), "trusted"); + success(new JdkHttpRequestService(List.of(afterFailure)), "trusted"); + rejected(new JdkHttpRequestService(List.of(builder.build())), "trusted"); + assertEquals(2, peer.requests.get()); + } + } + + @Test public void profileLimitsAndDuplicateAliasesFailBeforeUse() { + URI uri = URI.create("https://127.0.0.1/"); + for (Duration invalid : List.of(Duration.ZERO, Duration.ofNanos(-1), Duration.ofHours(1).plusNanos(1))) { + assertThrows(IllegalArgumentException.class, () -> profile("valid", uri).budgets(invalid, Duration.ofSeconds(1), Duration.ofSeconds(1))); + assertThrows(IllegalArgumentException.class, () -> profile("valid", uri).budgets(Duration.ofSeconds(1), invalid, Duration.ofSeconds(1))); + assertThrows(IllegalArgumentException.class, () -> profile("valid", uri).budgets(Duration.ofSeconds(1), Duration.ofSeconds(1), invalid)); + } + for (int bytes : List.of(0, 16 * 1024 * 1024 + 1)) assertThrows(IllegalArgumentException.class, + () -> profile("valid", uri).maxResponseBytes(bytes).build()); + for (int attempts : List.of(0, 6)) assertThrows(IllegalArgumentException.class, + () -> profile("valid", uri).retry(attempts, Duration.ofMillis(1), Set.of()).build()); + assertThrows(IllegalArgumentException.class, () -> profile("valid", uri).circuit(1, Duration.ofSeconds(1), 0).build()); + assertThrows(IllegalArgumentException.class, () -> new JdkHttpRequestService(List.of(profile("same", uri).build(), profile("same", uri).build()))); + } + + private static final class Peer implements AutoCloseable { + final SSLServerSocket server; + final Thread worker; + final AtomicBoolean closed = new AtomicBoolean(); + final AtomicReference error = new AtomicReference<>(); + final AtomicInteger requests = new AtomicInteger(); + Peer(String certificateFile, int rootIndex) throws Exception { + var factory = CertificateFactory.getInstance("X.509"); + var certificate = (X509Certificate) factory.generateCertificate( + new java.io.ByteArrayInputStream(Files.readAllBytes(directory.resolve(certificateFile)))); + var root = (X509Certificate) factory.generateCertificate(new java.io.ByteArrayInputStream(roots.get(rootIndex))); + certificate.checkValidity(); certificate.verify(root.getPublicKey()); + String key = Files.readString(directory.resolve("server.key")).replace("-----BEGIN PRIVATE KEY-----", "") + .replace("-----END PRIVATE KEY-----", "").replaceAll("\\s", ""); + var privateKey = KeyFactory.getInstance("RSA").generatePrivate(new PKCS8EncodedKeySpec(Base64.getDecoder().decode(key))); + var store = KeyStore.getInstance("PKCS12"); store.load(null, null); + char[] password = "fixture-only".toCharArray(); + store.setKeyEntry("server", privateKey, password, new java.security.cert.Certificate[] {certificate, root}); + var managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); managers.init(store, password); + var tls = SSLContext.getInstance("TLS"); tls.init(managers.getKeyManagers(), null, null); + server = (SSLServerSocket) tls.getServerSocketFactory().createServerSocket(0, 16, InetAddress.getByName("127.0.0.1")); + server.setSoTimeout(100); + worker = new Thread(() -> { + while (!closed.get()) { + try (var socket = (SSLSocket) server.accept()) { + socket.setSoTimeout(2000); + var reader = new java.io.BufferedReader(new java.io.InputStreamReader(socket.getInputStream(), StandardCharsets.US_ASCII)); + String first = reader.readLine(); + if (first == null) continue; + assertTrue(first.startsWith("GET /status HTTP/1.1")); + String line; + while ((line = reader.readLine()) != null && !line.isEmpty()) {} + if (line == null) continue; + requests.incrementAndGet(); + socket.getOutputStream().write("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".getBytes(StandardCharsets.US_ASCII)); + socket.getOutputStream().flush(); + } catch (java.io.IOException rejectedOrStopped) { + // Unknown CA and hostname failures cannot enter HTTP. + } catch (Throwable unexpected) { error.compareAndSet(null, unexpected); return; } + } + }, "teaql-http-trust-peer"); + worker.start(); + } + URI uri() { return URI.create("https://127.0.0.1:" + server.getLocalPort() + "/"); } + @Override public void close() throws Exception { + closed.set(true); server.close(); worker.join(5000); + assertFalse("owned TLS peer did not stop", worker.isAlive()); + if (error.get() != null) throw new AssertionError("owned TLS peer failed", error.get()); + } + } +} diff --git a/teaql-tool-http/src/test/java/io/teaql/tools/http/HttpResilienceTest.java b/teaql-tool-http/src/test/java/io/teaql/tools/http/HttpResilienceTest.java index 1b0a9d91..e16b1f24 100644 --- a/teaql-tool-http/src/test/java/io/teaql/tools/http/HttpResilienceTest.java +++ b/teaql-tool-http/src/test/java/io/teaql/tools/http/HttpResilienceTest.java @@ -137,6 +137,80 @@ private static HttpRequestService.Call call(HttpRequestService.Method method, St assertEquals(3, server.requests.get()); } + @Test public void missingContextOrIntentCannotStartNetworkIo() { + var service = new JdkHttpRequestService(java.util.List.of(profile().build())); + assertEquals(HttpRequestFailure.Code.INVALID_INPUT, assertThrows(HttpRequestFailure.class, + () -> service.execute(null, get("ok"))).code()); + for (String[] intent : new String[][] {{" ", "render order"}, {"load order", " \t"}}) { + assertEquals(HttpRequestFailure.Code.INVALID_INPUT, assertThrows(HttpRequestFailure.class, + () -> service.execute(context(service), HttpRequestService.Call.get( + "orders", "ok", intent[0], intent[1]))).code()); + } + assertEquals(0, server.requests.get()); + System.out.println("HTTP_NUMBERED_PASS language=java case=HTTP-01 requests=0"); + } + + @Test public void alwaysFailingEndpointStopsAtConfiguredAttemptCeiling() { + for (int ceiling : new int[] {1, 2, 3, 5}) { + var service = new JdkHttpRequestService(java.util.List.of(profile() + .retry(ceiling, Duration.ofMillis(1), Set.of(HttpRequestService.Method.GET)).build())); + int before = server.requests.get(); + var response = service.execute(context(service), call(HttpRequestService.Method.GET, + "fail", HttpRequestService.RetryMode.CONFIRMED_IDEMPOTENT)); + assertEquals(503, response.status()); + assertEquals(ceiling, response.attempts()); + assertEquals(ceiling, server.requests.get() - before); + } + System.out.println("HTTP_NUMBERED_PASS language=java case=HTTP-07 ceilings=1,2,3,5"); + } + + @Test public void oversizedIncompleteResponseReleasesPeerBeforeNextCall() throws Exception { + var service = new JdkHttpRequestService(java.util.List.of(profile().maxResponseBytes(16).build())); + assertEquals(HttpRequestFailure.Code.RESPONSE_TOO_LARGE, assertThrows(HttpRequestFailure.class, + () -> service.execute(context(service), get("overflow-open"))).code()); + assertTrue("incomplete oversized response did not close at the peer", + server.overflowReleased.await(2, TimeUnit.SECONDS)); + assertEquals(200, service.execute(context(service), get("ok")).status()); + assertEquals(2, server.requests.get()); + System.out.println("HTTP_NUMBERED_PASS language=java case=HTTP-09 peer_closed=1 next_call=1"); + } + + @Test public void successfulHalfOpenProbeClosesCircuitForConcurrentCalls() throws Exception { + AtomicLong monotonic = new AtomicLong(1); + HttpOperationalClock clock = new HttpOperationalClock() { + @Override public long nanoTime() { return monotonic.get(); } + @Override public void pauseNanos(long nanos) { monotonic.addAndGet(nanos); } + }; + var service = new JdkHttpRequestService(java.util.List.of(profile() + .circuit(1, Duration.ofSeconds(5), 1).build()), clock); + assertEquals(503, service.execute(context(service), get("fail")).status()); + assertEquals(HttpRequestFailure.Code.CIRCUIT_OPEN, assertThrows(HttpRequestFailure.class, + () -> service.execute(context(service), get("ok"))).code()); + monotonic.addAndGet(Duration.ofSeconds(5).toNanos()); + assertEquals(200, service.execute(context(service), get("ok")).status()); + AtomicReference failure = new AtomicReference<>(); + AtomicInteger completed = new AtomicInteger(); + Runnable request = () -> { + try { + assertEquals(200, service.execute(context(service), get("gated")).status()); + completed.incrementAndGet(); + } catch (Throwable error) { failure.compareAndSet(null, error); } + }; + Thread first = new Thread(request), second = new Thread(request); + first.start(); second.start(); + try { + // Neither response is released until both physical requests arrive. + assertTrue("successful probe left circuit in one-at-a-time half-open state", + server.gatedEntered.await(3, TimeUnit.SECONDS)); + } finally { + server.releaseGated.countDown(); first.join(2000); second.join(2000); + } + assertFalse(first.isAlive()); assertFalse(second.isAlive()); + if (failure.get() != null) throw new AssertionError(failure.get()); + assertEquals(2, completed.get()); assertEquals(4, server.requests.get()); + System.out.println("HTTP_NUMBERED_PASS language=java case=HTTP-11 concurrent_after_probe=2"); + } + @Test public void retryIsExplicitAndPostTransportFailureIsAmbiguous() { HttpServiceProfile configured = profile().retry(3, Duration.ofMillis(1), Set.of(HttpRequestService.Method.GET)).build(); @@ -632,6 +706,9 @@ private static final class Fixture implements AutoCloseable { private final AtomicInteger retryPost = new AtomicInteger(); private final ConcurrentLinkedQueue traceparents = new ConcurrentLinkedQueue<>(); private final ConcurrentLinkedQueue retryBodies = new ConcurrentLinkedQueue<>(); + private final CountDownLatch overflowReleased = new CountDownLatch(1); + private final CountDownLatch gatedEntered = new CountDownLatch(2); + private final CountDownLatch releaseGated = new CountDownLatch(1); private volatile boolean closed; Fixture() throws IOException { socket = new ServerSocket(0, 32, InetAddress.getLoopbackAddress()); @@ -681,6 +758,20 @@ private void handle(Socket socket) { if (target.contains("/retry-post")) retryBodies.add(new String(body)); } if (target.contains("/close")) return; + if (target.endsWith("/overflow-open")) { + client.setSoTimeout(3000); + client.getOutputStream().write(("HTTP/1.1 200 OK\r\nContent-Length: 4096\r\n" + + "Connection: close\r\n\r\n" + "L".repeat(32)).getBytes(StandardCharsets.UTF_8)); + client.getOutputStream().flush(); + // Never finish the body: only the client can release this connection. + try { if (reader.read() == -1) overflowReleased.countDown(); } + catch (java.net.SocketException closedByPeer) { overflowReleased.countDown(); } + return; + } + if (target.endsWith("/gated")) { + gatedEntered.countDown(); + if (!releaseGated.await(4, TimeUnit.SECONDS)) return; + } if (target.endsWith("/slow")) Thread.sleep(300); int status = target.contains("/private-response") ? 503 : target.contains("/redirect") ? 302 : target.contains("/missing") ? 404