From c12bb437e0c9429202559f989bd304ab401e568b Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 01:34:30 +0800 Subject: [PATCH 01/29] feat(sqlite): add transactional committed-event outbox and inbox store --- teaql-sqlite/pom.xml | 4 + .../sqlite/SqliteCommittedEventStore.java | 239 ++++++++++++++++ teaql-sqlite/src/main/java/module-info.java | 1 + .../sqlite/SqliteCommittedEventStoreTest.java | 263 ++++++++++++++++++ 4 files changed, 507 insertions(+) create mode 100644 teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java create mode 100644 teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java diff --git a/teaql-sqlite/pom.xml b/teaql-sqlite/pom.xml index 54a85105..e1133aa9 100644 --- a/teaql-sqlite/pom.xml +++ b/teaql-sqlite/pom.xml @@ -28,6 +28,10 @@ sqlite-jdbc 3.53.2.1 + + com.fasterxml.jackson.core + jackson-databind + io.teaql teaql-provider-jdbc diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java new file mode 100644 index 00000000..5ad9207a --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java @@ -0,0 +1,239 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.LongFunction; + +/** + * SQLite durability boundary for already-committed logical mutation facts. + * + *

The caller must install the schema before use. Append and apply require the + * same active SQL transaction as business writes. This class deliberately does + * not infer mutations from a ledger, replay business logic, or expose a remote + * replication endpoint. Projection writers are trusted infrastructure code. + */ +public final class SqliteCommittedEventStore { + private static final ObjectMapper JSON = new ObjectMapper(); + private final SqlExecutionAdapter sql; + + public SqliteCommittedEventStore(SqlExecutionAdapter sql) { + this.sql = Objects.requireNonNull(sql, "sql"); + } + + public record OutboxRecord(String sourceRuntimeId, long sourceSequence, + String changeSetId, String modelHash, String payloadSha256, String payloadJson) {} + + public record InboxOutcome(boolean duplicate, long checkpoint) { + public static InboxOutcome applied(long checkpoint) { return new InboxOutcome(false, checkpoint); } + public static InboxOutcome duplicate(long checkpoint) { return new InboxOutcome(true, checkpoint); } + } + + @FunctionalInterface + public interface TrustedProjectionWriter { + void apply(SqlExecutionAdapter transaction, JsonNode committedFacts); + } + + public static final class CommittedEventException extends RuntimeException { + private final String code; + + private CommittedEventException(String code, String message) { + super(message); + this.code = code; + } + + public String code() { return code; } + } + + public void ensureSchema() { + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_sequence (" + + "source_runtime_id TEXT PRIMARY KEY, " + + "current_sequence INTEGER NOT NULL CHECK(current_sequence >= 0))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_outbox (" + + "source_runtime_id TEXT NOT NULL, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence > 0), " + + "change_set_id TEXT NOT NULL, model_hash TEXT NOT NULL, " + + "payload_sha256 TEXT NOT NULL, payload_json TEXT NOT NULL, " + + "PRIMARY KEY(source_runtime_id, source_sequence), " + + "UNIQUE(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_inbox (" + + "source_runtime_id TEXT NOT NULL, change_set_id TEXT NOT NULL, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence > 0), " + + "payload_sha256 TEXT NOT NULL, PRIMARY KEY(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_checkpoint (" + + "source_runtime_id TEXT PRIMARY KEY, " + + "source_sequence INTEGER NOT NULL CHECK(source_sequence >= 0), " + + "model_hash TEXT NOT NULL)"); + } + + /** Bounded publisher read of one durable, ordered source stream. */ + public List outboxAfter(String sourceRuntimeId, long after, int limit) { + if (blank(sourceRuntimeId) || after < 0 || limit < 1 || limit > 1000) { + throw invalid("invalid source, position, or read limit"); + } + return sql.queryForList("SELECT source_runtime_id, source_sequence, change_set_id, " + + "model_hash, payload_sha256, payload_json FROM teaql_event_outbox " + + "WHERE source_runtime_id = ? AND source_sequence > ? " + + "ORDER BY source_sequence ASC LIMIT ?", new Object[] {sourceRuntimeId, after, limit}) + .stream().map(row -> new OutboxRecord((String) row.get("source_runtime_id"), + number(row.get("source_sequence")), (String) row.get("change_set_id"), + (String) row.get("model_hash"), (String) row.get("payload_sha256"), + (String) row.get("payload_json"))).toList(); + } + + /** Allocate and append within the caller's already-open business transaction. */ + public OutboxRecord append(String sourceRuntimeId, String changeSetId, String modelHash, + LongFunction payloadBuilder) { + requireTransaction(); + if (blank(sourceRuntimeId) || blank(changeSetId) || blank(modelHash) || payloadBuilder == null) { + throw invalid("missing committed-event identity or payload builder"); + } + return savepoint("teaql_event_append", () -> { + sql.update("INSERT INTO teaql_event_sequence(source_runtime_id, current_sequence) " + + "VALUES (?, 0) ON CONFLICT(source_runtime_id) DO NOTHING", + new Object[] {sourceRuntimeId}); + int updated = sql.update("UPDATE teaql_event_sequence SET current_sequence = current_sequence + 1 " + + "WHERE source_runtime_id = ? AND current_sequence < 9223372036854775807", + new Object[] {sourceRuntimeId}); + if (updated != 1) throw new CommittedEventException("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + long sequence = number(sql.queryForList("SELECT current_sequence FROM teaql_event_sequence " + + "WHERE source_runtime_id = ?", new Object[] {sourceRuntimeId}).get(0).get("current_sequence")); + String payloadJson = payloadBuilder.apply(sequence); + JsonNode payload = parse(payloadJson); + validateHeader(payload, sourceRuntimeId, sequence, changeSetId, modelHash); + String digest = digest(payloadJson); + sql.update("INSERT INTO teaql_event_outbox " + + "(source_runtime_id, source_sequence, change_set_id, model_hash, payload_sha256, payload_json) " + + "VALUES (?, ?, ?, ?, ?, ?)", + new Object[] {sourceRuntimeId, sequence, changeSetId, modelHash, digest, payloadJson}); + return new OutboxRecord(sourceRuntimeId, sequence, changeSetId, modelHash, digest, payloadJson); + }); + } + + /** + * Apply one set through a restricted projection writer. The writer must not + * call normal entity save, Checker/Fix, ID generation or business hooks. + */ + public InboxOutcome apply(OutboxRecord record, String expectedModelHash, + TrustedProjectionWriter writer) { + requireTransaction(); + if (record == null || blank(expectedModelHash) || writer == null) { + throw invalid("missing committed event, model hash, or projection writer"); + } + if (!expectedModelHash.equals(record.modelHash())) { + throw new CommittedEventException("MODEL_MISMATCH", "committed-event model hash mismatch"); + } + JsonNode payload = parse(record.payloadJson()); + validateHeader(payload, record.sourceRuntimeId(), record.sourceSequence(), + record.changeSetId(), record.modelHash()); + if (!digest(record.payloadJson()).equals(record.payloadSha256())) { + throw invalid("payload digest mismatch"); + } + List> checkpoints = sql.queryForList( + "SELECT source_sequence, model_hash FROM teaql_event_checkpoint WHERE source_runtime_id = ?", + new Object[] {record.sourceRuntimeId()}); + long last = checkpoints.isEmpty() ? 0 : number(checkpoints.get(0).get("source_sequence")); + if (!checkpoints.isEmpty() && !expectedModelHash.equals(checkpoints.get(0).get("model_hash"))) { + throw new CommittedEventException("MODEL_MISMATCH", "committed-event checkpoint model mismatch"); + } + List> prior = sql.queryForList( + "SELECT source_sequence, payload_sha256 FROM teaql_event_inbox " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {record.sourceRuntimeId(), record.changeSetId()}); + if (!prior.isEmpty()) { + Map seen = prior.get(0); + if (number(seen.get("source_sequence")) == record.sourceSequence() + && record.payloadSha256().equals(seen.get("payload_sha256"))) { + return InboxOutcome.duplicate(last); + } + throw new CommittedEventException("DUPLICATE_CONFLICT", "committed-event identity has conflicting content"); + } + if (last == Long.MAX_VALUE) { + throw new CommittedEventException("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + } + if (record.sourceSequence() != last + 1) { + throw new CommittedEventException("SEQUENCE_GAP", + "committed-event sequence gap: expected " + (last + 1) + ", received " + record.sourceSequence()); + } + return savepoint("teaql_event_apply", () -> { + writer.apply(sql, payload); + sql.update("INSERT INTO teaql_event_inbox " + + "(source_runtime_id, change_set_id, source_sequence, payload_sha256) VALUES (?, ?, ?, ?)", + new Object[] {record.sourceRuntimeId(), record.changeSetId(), + record.sourceSequence(), record.payloadSha256()}); + sql.update("INSERT INTO teaql_event_checkpoint (source_runtime_id, source_sequence, model_hash) " + + "VALUES (?, ?, ?) ON CONFLICT(source_runtime_id) DO UPDATE SET " + + "source_sequence = excluded.source_sequence, model_hash = excluded.model_hash", + new Object[] {record.sourceRuntimeId(), record.sourceSequence(), record.modelHash()}); + return InboxOutcome.applied(record.sourceSequence()); + }); + } + + private static JsonNode parse(String payloadJson) { + if (payloadJson == null) throw invalid("missing payload"); + try { + return JSON.readTree(payloadJson); + } catch (Exception ignored) { + throw invalid("invalid payload JSON"); + } + } + + private static void validateHeader(JsonNode payload, String source, long sequence, + String changeSetId, String modelHash) { + if (blank(source) || sequence < 1 || blank(changeSetId) || blank(modelHash) + || payload == null || payload.path("protocol_version").asInt(-1) != 1 + || !source.equals(payload.path("source_runtime_id").asText()) + || !Long.toString(sequence).equals(payload.path("source_sequence").asText()) + || !changeSetId.equals(payload.path("change_set_id").asText()) + || !modelHash.equals(payload.path("source_model_hash").asText()) + || !payload.path("mutations").isArray() || payload.path("mutations").isEmpty()) { + throw invalid("required v1 header or mutations mismatch"); + } + } + + private T savepoint(String name, java.util.function.Supplier action) { + sql.execute("SAVEPOINT " + name); + try { + T value = action.get(); + sql.execute("RELEASE SAVEPOINT " + name); + return value; + } catch (RuntimeException | Error failure) { + try { + sql.execute("ROLLBACK TO SAVEPOINT " + name); + sql.execute("RELEASE SAVEPOINT " + name); + } catch (RuntimeException rollbackFailure) { + // A failed savepoint rollback must not leave partial rows committable. + try { sql.execute("ROLLBACK"); } catch (RuntimeException ignored) { } + throw new CommittedEventException("ROLLBACK_FAILED", "committed-event rollback failed"); + } + throw failure; + } + } + + private void requireTransaction() { + if (!sql.hasActiveTransaction()) throw invalid("an active SQL transaction is required"); + } + + private static long number(Object value) { return ((Number) value).longValue(); } + private static boolean blank(String value) { return value == null || value.isBlank(); } + private static CommittedEventException invalid(String reason) { + return new CommittedEventException("INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } + private static String digest(String payload) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(hash); + } catch (NoSuchAlgorithmException impossible) { + throw new AssertionError("JDK SHA-256 is unavailable", impossible); + } + } +} diff --git a/teaql-sqlite/src/main/java/module-info.java b/teaql-sqlite/src/main/java/module-info.java index 98b73c4b..5028c4b3 100644 --- a/teaql-sqlite/src/main/java/module-info.java +++ b/teaql-sqlite/src/main/java/module-info.java @@ -4,6 +4,7 @@ requires transitive io.teaql.utils; requires transitive io.teaql.dataservice.sql; requires io.teaql.provider.jdbc; + requires com.fasterxml.jackson.databind; requires org.xerial.sqlitejdbc; requires java.sql; exports io.teaql.core.sqlite; diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java new file mode 100644 index 00000000..0b5780e0 --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java @@ -0,0 +1,263 @@ +package io.teaql.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedEventStore.CommittedEventException; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.Assert.*; + +/** Small SQLite graph fixture shared in meaning with CMR v1; no generated ERP model. */ +public class SqliteCommittedEventStoreTest { + private static final String MODEL_HASH = "sha256:" + "0".repeat(64); + private static final String SOURCE = "sqlite-order-source"; + private static final long ORDER_ID = 9_007_199_254_740_993L; + private static final long ITEM_ID = ORDER_ID + 1; + + private record Database(JdbcSqlExecutor driver, SqliteCommittedEventStore events) {} + + private static Database database(Path path) { + SQLiteDataSource source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor driver = new JdbcSqlExecutor(source); + return new Database(driver, new SqliteCommittedEventStore(driver)); + } + + private static void graphSchema(SqlExecutionAdapter sql) { + sql.execute("CREATE TABLE orders (id INTEGER PRIMARY KEY, status TEXT NOT NULL, version INTEGER NOT NULL)"); + sql.execute("CREATE TABLE order_items (id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL, " + + "quantity INTEGER NOT NULL, version INTEGER NOT NULL)"); + } + + private static long count(SqlExecutionAdapter sql, String table) { + // This helper only receives the two fixed table names from this test. + return ((Number) sql.queryForList("SELECT count(*) AS total FROM " + table, + new Object[0]).get(0).get("total")).longValue(); + } + + private static String payload(long sequence, String change, boolean submitted) { + String operation = submitted ? "update" : "create"; + String itemOperation = submitted ? "delete" : "create"; + String status = submitted ? "SUBMITTED" : "DRAFT"; + String version = submitted ? "2" : "1"; + String itemVersion = submitted ? "-2" : "1"; + String baseVersion = submitted ? "\"1\"" : "null"; + return (""" + {"protocol_version":1,"source_runtime_id":"%s","source_sequence":"%d", + "change_set_id":"%s","source_model_hash":"%s","mutations":[ + {"entity_type":"order","entity_id":"%d","operation":"%s", + "base_version":%s,"committed_version":"%s","fields":{"status":"%s"}}, + {"entity_type":"order_item","entity_id":"%d","operation":"%s", + "base_version":%s,"committed_version":"%s", + "fields":{"order":{"entity_type":"order","entity_id":"%d"},"quantity":2}} + ]} + """).formatted(SOURCE, sequence, change, MODEL_HASH, ORDER_ID, operation, + baseVersion, version, status, ITEM_ID, itemOperation, baseVersion, + itemVersion, ORDER_ID); + } + + private static void applyGraph(SqlExecutionAdapter sql, JsonNode facts) { + for (JsonNode mutation : facts.path("mutations")) { + long id = Long.parseLong(mutation.path("entity_id").asText()); + long version = Long.parseLong(mutation.path("committed_version").asText()); + String operation = mutation.path("operation").asText(); + if (mutation.path("entity_type").asText().equals("order")) { + String status = mutation.path("fields").path("status").asText(); + if (operation.equals("create")) { + sql.update("INSERT INTO orders (id, status, version) VALUES (?, ?, ?)", + new Object[] {id, status, version}); + } else { + sql.update("UPDATE orders SET status = ?, version = ? WHERE id = ?", + new Object[] {status, version, id}); + } + } else if (operation.equals("create")) { + long parent = Long.parseLong(mutation.path("fields").path("order").path("entity_id").asText()); + int quantity = mutation.path("fields").path("quantity").asInt(); + sql.update("INSERT INTO order_items (id, order_id, quantity, version) VALUES (?, ?, ?, ?)", + new Object[] {id, parent, quantity, version}); + } else { + sql.update("UPDATE order_items SET version = ? WHERE id = ?", new Object[] {version, id}); + } + } + } + + private static OutboxRecord append(Database database, String change, boolean submitted) { + AtomicReference result = new AtomicReference<>(); + database.driver().executeInTransaction(() -> { + if (submitted) { + database.driver().update("UPDATE orders SET status = 'SUBMITTED', version = 2 WHERE id = ?", + new Object[] {ORDER_ID}); + database.driver().update("UPDATE order_items SET version = -2 WHERE id = ?", + new Object[] {ITEM_ID}); + } else { + database.driver().update("INSERT INTO orders (id, status, version) VALUES (?, 'DRAFT', 1)", + new Object[] {ORDER_ID}); + database.driver().update("INSERT INTO order_items (id, order_id, quantity, version) " + + "VALUES (?, ?, 2, 1)", new Object[] {ITEM_ID, ORDER_ID}); + } + result.set(database.events().append(SOURCE, change, MODEL_HASH, + sequence -> payload(sequence, change, submitted))); + }); + return result.get(); + } + + private static CommittedEventException expectCode(String code, Runnable action) { + try { + action.run(); + fail("expected " + code); + throw new AssertionError("unreachable"); + } catch (CommittedEventException failure) { + assertEquals(code, failure.code()); + return failure; + } + } + + @Test public void sourceRollbackAndRestartPreserveAtomicOrderedFacts() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-source-", ".sqlite"); + try { + Database source = database(path); + source.events().ensureSchema(); + graphSchema(source.driver()); + try { + source.driver().executeInTransaction(() -> { + source.driver().update("INSERT INTO orders (id, status, version) VALUES (?, 'DRAFT', 1)", + new Object[] {ORDER_ID}); + source.driver().update("INSERT INTO order_items (id, order_id, quantity, version) " + + "VALUES (?, ?, 2, 1)", new Object[] {ITEM_ID, ORDER_ID}); + source.events().append(SOURCE, "change-a", MODEL_HASH, + sequence -> payload(sequence, "change-a", false)); + throw new IllegalStateException("roll back complete graph"); + }); + fail("expected source rollback"); + } catch (IllegalStateException expected) { + assertEquals("roll back complete graph", expected.getMessage()); + } + assertEquals(0, count(source.driver(), "orders")); + assertEquals(0, count(source.driver(), "order_items")); + assertTrue(source.events().outboxAfter(SOURCE, 0, 10).isEmpty()); + + OutboxRecord first = append(source, "change-a", false); + OutboxRecord second = append(source, "change-b", true); + assertEquals(1, first.sourceSequence()); + assertEquals(2, second.sourceSequence()); + + Database reopened = database(path); + reopened.events().ensureSchema(); + List rows = reopened.events().outboxAfter(SOURCE, 0, 10); + assertEquals(List.of(first, second), rows); + assertEquals(List.of(second), reopened.events().outboxAfter(SOURCE, 1, 1)); + assertEquals(1, count(reopened.driver(), "orders")); + assertEquals(1, count(reopened.driver(), "order_items")); + Map item = reopened.driver().queryForList( + "SELECT version FROM order_items WHERE id = ?", new Object[] {ITEM_ID}).get(0); + assertEquals(-2, ((Number) item.get("version")).longValue()); + } finally { + Files.deleteIfExists(path); + } + } + + @Test public void targetApplyDeduplicatesAndRollsBackWholeGraphOnFailure() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-target-", ".sqlite"); + try { + Database target = database(path); + target.events().ensureSchema(); + graphSchema(target.driver()); + OutboxRecord first = record(1, "change-a", false); + OutboxRecord second = record(2, "change-b", true); + + target.driver().executeInTransaction(() -> expectCode("SEQUENCE_GAP", + () -> target.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + assertEquals(0, count(target.driver(), "orders")); + assertEquals(0, count(target.driver(), "teaql_event_inbox")); + + target.driver().executeInTransaction(() -> { + var outcome = target.events().apply(first, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + assertFalse(outcome.duplicate()); + assertEquals(1, outcome.checkpoint()); + }); + target.driver().executeInTransaction(() -> { + expectCode("MODEL_MISMATCH", () -> target.events().apply(second, + "sha256:" + "1".repeat(64), SqliteCommittedEventStoreTest::applyGraph)); + try { + target.events().apply(second, MODEL_HASH, (sql, facts) -> { + sql.update("UPDATE orders SET status = 'PARTIAL' WHERE id = ?", new Object[] {ORDER_ID}); + sql.update("UPDATE order_items SET version = -99 WHERE id = ?", new Object[] {ITEM_ID}); + throw new IllegalStateException("injected second-table failure"); + }); + fail("expected injected failure"); + } catch (IllegalStateException expected) { + assertEquals("injected second-table failure", expected.getMessage()); + } + }); + assertEquals("DRAFT", target.driver().queryForList("SELECT status FROM orders", new Object[0]) + .get(0).get("status")); + assertEquals(1L, ((Number) target.driver().queryForList("SELECT version FROM order_items", + new Object[0]).get(0).get("version")).longValue()); + assertEquals(1, count(target.driver(), "teaql_event_inbox")); + + Database reopened = database(path); + reopened.driver().executeInTransaction(() -> { + var outcome = reopened.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + assertFalse(outcome.duplicate()); + assertEquals(2, outcome.checkpoint()); + var duplicate = reopened.events().apply(second, MODEL_HASH, (sql, facts) -> { + throw new AssertionError("duplicate must not rematerialize"); + }); + assertTrue(duplicate.duplicate()); + assertEquals(2, duplicate.checkpoint()); + }); + assertEquals("SUBMITTED", reopened.driver().queryForList("SELECT status FROM orders", new Object[0]) + .get(0).get("status")); + assertEquals(-2L, ((Number) reopened.driver().queryForList( + "SELECT version FROM order_items", new Object[0]).get(0).get("version")).longValue()); + assertEquals(2, count(reopened.driver(), "teaql_event_inbox")); + assertEquals(2L, ((Number) reopened.driver().queryForList( + "SELECT source_sequence FROM teaql_event_checkpoint", new Object[0]) + .get(0).get("source_sequence")).longValue()); + + OutboxRecord conflicted = record(2, "change-b", false); + reopened.driver().executeInTransaction(() -> expectCode("DUPLICATE_CONFLICT", + () -> reopened.events().apply(conflicted, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + OutboxRecord invalidDigest = new OutboxRecord(second.sourceRuntimeId(), second.sourceSequence(), + second.changeSetId(), second.modelHash(), "f".repeat(64), second.payloadJson()); + reopened.driver().executeInTransaction(() -> expectCode("INVALID_ENVELOPE", + () -> reopened.events().apply(invalidDigest, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + } finally { + Files.deleteIfExists(path); + } + } + + @Test public void appendAndApplyRequireTheCallerTransaction() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-boundary-", ".sqlite"); + try { + Database database = database(path); + database.events().ensureSchema(); + OutboxRecord first = record(1, "change-a", false); + expectCode("INVALID_ENVELOPE", () -> database.events().append(SOURCE, "change-a", + MODEL_HASH, sequence -> payload(sequence, "change-a", false))); + expectCode("INVALID_ENVELOPE", () -> database.events().apply(first, + MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph)); + assertEquals(0, count(database.driver(), "teaql_event_outbox")); + } finally { + Files.deleteIfExists(path); + } + } + + private static OutboxRecord record(long sequence, String change, boolean submitted) throws Exception { + String json = payload(sequence, change, submitted); + byte[] bytes = java.security.MessageDigest.getInstance("SHA-256") + .digest(json.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + return new OutboxRecord(SOURCE, sequence, change, MODEL_HASH, + java.util.HexFormat.of().formatHex(bytes), json); + } +} From e2950aff36a01ba11e3f2b85529032bed0213105 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 01:50:42 +0800 Subject: [PATCH 02/29] Capture authoritative graph mutations in SQLite outbox --- README.md | 1 + examples/committed-events/README.md | 16 +++ examples/committed-events/pom.xml | 18 +++ .../committed/CommittedEventsExampleTest.java | 102 ++++++++++++++++ examples/committed-events/verify.sh | 9 ++ examples/verify-runtime-examples.sh | 1 + pom.xml | 1 + scripts/verify-examples.sh | 3 +- .../core/CommittedEventSchemaContributor.java | 6 + .../main/java/io/teaql/core/UserContext.java | 5 + .../teaql/runtime/CommittedMutationFact.java | 29 +++++ .../runtime/CommittedMutationRecorder.java | 16 +++ .../io/teaql/runtime/DefaultUserContext.java | 3 + .../java/io/teaql/runtime/TeaQLRuntime.java | 111 +++++++++++++++++- teaql-sqlite/pom.xml | 11 +- .../sqlite/SqliteCommittedEventStore.java | 7 +- .../SqliteCommittedMutationRecorder.java | 111 ++++++++++++++++++ teaql-sqlite/src/main/java/module-info.java | 1 + .../io/teaql/sqlite/GraphTraceSqliteTest.java | 101 +++++++++++++++- .../sqlite/SqliteCommittedEventStoreTest.java | 2 + 20 files changed, 542 insertions(+), 12 deletions(-) create mode 100644 examples/committed-events/README.md create mode 100644 examples/committed-events/pom.xml create mode 100644 examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java create mode 100755 examples/committed-events/verify.sh create mode 100644 teaql-core/src/main/java/io/teaql/core/CommittedEventSchemaContributor.java create mode 100644 teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationFact.java create mode 100644 teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationRecorder.java create mode 100644 teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java diff --git a/README.md b/README.md index fca5a184..7c2e3e4e 100644 --- a/README.md +++ b/README.md @@ -322,6 +322,7 @@ Request JSON is not allowed to replace those trusted capabilities. - [Shared load state](examples/shared-load-state/README.md) - [Binary object storage and temporary S3 URLs](examples/object-storage/README.md) - [Context-owned HTTP with bounded retries and safe diagnostics](examples/http-resilience/README.md) +- [Transactional committed-event Outbox from a real SQLite graph save](examples/committed-events/README.md) - [Extension points](https://github.com/teaql/teaql-conformance/blob/main/design/runtime-customization-p0-p1-matrix.md) ## Safe troubleshooting diff --git a/examples/committed-events/README.md b/examples/committed-events/README.md new file mode 100644 index 00000000..fce32b05 --- /dev/null +++ b/examples/committed-events/README.md @@ -0,0 +1,16 @@ +# SQLite committed-event source example + +Run `bash examples/committed-events/verify.sh`. It executes a small, real +`Order.save(context)` flow twice against local Java source. Each round uses a +fresh SQLite file, and each test reopens the Outbox before injecting a later +write failure. No generated ERP model or remote service is required. + +The example configures `SqliteCommittedMutationRecorder` at trusted runtime +startup; `context.ensureSchema()` installs its infrastructure tables. A database trigger +sets `memo` after insert; the Outbox contains that authoritative readback, not +the pre-save entity value. Subsequent update produces sequence 2. When the +Outbox table is made unavailable, the attempted third save rolls back its +business update and emits no committed audit event. + +This is **not** a completed replication example: it does not publish the row, +validate a target model, apply to another runtime, or cover dynamic fields. diff --git a/examples/committed-events/pom.xml b/examples/committed-events/pom.xml new file mode 100644 index 00000000..2dc08742 --- /dev/null +++ b/examples/committed-events/pom.xml @@ -0,0 +1,18 @@ + + 4.0.0 + + io.teaqlteaql-java-parent + 1.555-RELEASE../../pom.xml + + teaql-committed-events-example + + io.teaqlteaql-runtime + io.teaqlteaql-sqlite + io.teaqlteaql-provider-jdbc + com.fasterxml.jackson.corejackson-databind + org.xerialsqlite-jdbc3.53.2.1 + junitjunittest + + diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java new file mode 100644 index 00000000..be4c0da7 --- /dev/null +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java @@ -0,0 +1,102 @@ +package io.teaql.examples.committed; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.BaseEntity; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sql.GenericSQLProperty; +import io.teaql.core.sql.SQLEntityDescriptor; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.*; + +public class CommittedEventsExampleTest { + public static final class Order extends BaseEntity { + private final Map fields = new HashMap<>(); + + @Override public String typeName() { return "Order"; } + @Override public Object __internalGet(String field) { + return field.equals("name") || field.equals("memo") ? fields.get(field) : super.__internalGet(field); + } + @Override public void __internalSet(String field, Object value) { + if (field.equals("name") || field.equals("memo")) fields.put(field, value); + else super.__internalSet(field, value); + } + } + + @Test public void realSaveFreezesDatabaseResultsBeforeCommit() throws Exception { + Path path = Files.createTempFile("teaql-committed-example-", ".sqlite"); + path.toFile().deleteOnExit(); + SQLiteDataSource source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor sql = new JdbcSqlExecutor(source); + SqliteCommittedMutationRecorder recorder = new SqliteCommittedMutationRecorder( + sql, "example-java-source", "sha256:" + "0".repeat(64)); + + SQLEntityDescriptor descriptor = new SQLEntityDescriptor(); + descriptor.setType("Order"); + descriptor.setTargetType(Order.class); + descriptor.setEntitySupplier(Order::new); + descriptor.setDataService("sqlite"); + for (String field : new String[] {"id", "version", "name", "memo"}) { + boolean text = field.equals("name") || field.equals("memo"); + GenericSQLProperty property = (GenericSQLProperty) descriptor.addSimpleProperty( + field, text ? String.class : Long.class); + property.setColumnType(text ? "VARCHAR(255)" : "BIGINT"); + } + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + metadata.register(descriptor); + TeaQLRuntime runtime = TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", sql, source)) + .idGenerationService((caller, entity) -> 42L) + .committedMutationRecorder(recorder) + .executionLogging(false).build(); + DefaultUserContext context = new DefaultUserContext(runtime); + context.ensureSchema(); + sql.execute("CREATE TRIGGER order_memo AFTER INSERT ON order_data " + + "BEGIN UPDATE order_data SET memo = 'set by database' WHERE id = NEW.id; END"); + + Order order = new Order(); + order.__internalInitializeNewEntityId(42L); + order.updateProperty("name", "draft"); + order.auditAs("create example order").save(context); + var first = recorder.eventStore().outboxAfter("example-java-source", 0, 10); + assertEquals(1, first.size()); + assertEquals(1, first.get(0).sourceSequence()); + JsonNode created = new ObjectMapper().readTree(first.get(0).payloadJson()); + assertEquals("set by database", created.path("mutations").get(0).path("fields").path("memo").asText()); + assertEquals("create example order", created.path("audit").path("comment").asText()); + + order.updateProperty("name", "submitted"); + order.auditAs("submit example order").save(context); + JdbcSqlExecutor reopened = new JdbcSqlExecutor(source); + var durable = new SqliteCommittedEventStore(reopened) + .outboxAfter("example-java-source", 0, 10); + assertEquals(2, durable.size()); + JsonNode submitted = new ObjectMapper().readTree(durable.get(1).payloadJson()); + assertEquals("1", submitted.path("mutations").get(0).path("base_version").asText()); + assertEquals("2", submitted.path("mutations").get(0).path("committed_version").asText()); + + sql.execute("DROP TABLE teaql_event_outbox"); + order.updateProperty("name", "should roll back"); + assertThrows(RuntimeException.class, + () -> order.auditAs("reject when outbox unavailable").save(context)); + assertEquals("submitted", sql.queryForList("SELECT name FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("name")); + assertEquals(2L, ((Number) sql.queryForList("SELECT version FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("version")).longValue()); + runtime.close(); + } +} diff --git a/examples/committed-events/verify.sh b/examples/committed-events/verify.sh new file mode 100755 index 00000000..dc4f3da2 --- /dev/null +++ b/examples/committed-events/verify.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo" +for round in 1 2; do + echo "Java committed-events example round $round" + mvn -B -q -pl examples/committed-events -am \ + -Dtest=CommittedEventsExampleTest -Dsurefire.failIfNoSpecifiedTests=false test +done diff --git a/examples/verify-runtime-examples.sh b/examples/verify-runtime-examples.sh index 2e072eab..20b75259 100755 --- a/examples/verify-runtime-examples.sh +++ b/examples/verify-runtime-examples.sh @@ -85,6 +85,7 @@ for repetition in 1 2; do done bash "$repo_dir/examples/trace-chain/verify.sh" +bash "$repo_dir/examples/committed-events/verify.sh" if [[ -n "${TEAQL_CODEGEN_DIR:-}" ]]; then bash "$repo_dir/examples/shared-load-state/verify.sh" --generate else diff --git a/pom.xml b/pom.xml index 9a3262bd..26f1b49f 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,7 @@ teaql-object-storage-s3 examples/object-storage examples/http-resilience + examples/committed-events examples/business-id-runtime examples/security-foundations examples/round-trip-reference-runtime diff --git a/scripts/verify-examples.sh b/scripts/verify-examples.sh index 204f9989..6c5aec17 100755 --- a/scripts/verify-examples.sh +++ b/scripts/verify-examples.sh @@ -4,7 +4,7 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" verification_dir="$(mktemp -d)" trap 'rm -rf -- "$verification_dir"' EXIT -expected=(business-id-runtime conformance http-resilience object-storage order-management round-trip-reference-runtime school-management security-foundations shared-load-state standard-order trace-chain) +expected=(business-id-runtime committed-events conformance http-resilience object-storage order-management round-trip-reference-runtime school-management security-foundations shared-load-state standard-order trace-chain) mapfile -t actual < <(find "$repo/examples" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort) if [[ "${actual[*]}" != "${expected[*]}" ]]; then echo "example inventory changed; update scripts/verify-examples.sh: ${actual[*]}" >&2 @@ -14,6 +14,7 @@ fi cd "$repo" bash examples/object-storage/verify.sh bash examples/http-resilience/verify.sh +bash examples/committed-events/verify.sh mvn -q -DskipTests install mvn -q -pl teaql-sqlite \ -Dtest=DerivedQueryTraceSqliteTest,LikeIntentPrivacySqliteTest,TypedIntentPrivacySqliteTest test diff --git a/teaql-core/src/main/java/io/teaql/core/CommittedEventSchemaContributor.java b/teaql-core/src/main/java/io/teaql/core/CommittedEventSchemaContributor.java new file mode 100644 index 00000000..89a33160 --- /dev/null +++ b/teaql-core/src/main/java/io/teaql/core/CommittedEventSchemaContributor.java @@ -0,0 +1,6 @@ +package io.teaql.core; + +/** Trusted infrastructure tables installed only in the Context schema lifecycle. */ +public interface CommittedEventSchemaContributor { + void ensureSchema(UserContext context, SchemaExecutor.Invocation invocation); +} diff --git a/teaql-core/src/main/java/io/teaql/core/UserContext.java b/teaql-core/src/main/java/io/teaql/core/UserContext.java index b51bb1c2..ae3d6ff0 100644 --- a/teaql-core/src/main/java/io/teaql/core/UserContext.java +++ b/teaql-core/src/main/java/io/teaql/core/UserContext.java @@ -75,6 +75,11 @@ default void ensureSchema() { if (businessIdSchema != null) { businessIdSchema.ensureSchema(this); } + CommittedEventSchemaContributor committedEvents = + capability(CommittedEventSchemaContributor.class); + if (committedEvents != null) { + committedEvents.ensureSchema(this, SchemaExecutor.Invocation.contextOwned()); + } GeneratedSchemaBootstrap bootstrap = capability(GeneratedSchemaBootstrap.class); if (bootstrap != null) { Object previousActor = getAttribute(GeneratedSchemaBootstrap.AUDIT_ACTOR_ATTRIBUTE); diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationFact.java b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationFact.java new file mode 100644 index 00000000..a8ac2cc9 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationFact.java @@ -0,0 +1,29 @@ +package io.teaql.runtime; + +import io.teaql.core.TraceNode; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** One immutable, model-named fact frozen from a provider readback before commit. */ +public record CommittedMutationFact( + String entityType, String entityId, String operation, + String baseVersion, String committedVersion, + Map fields, List traceChain) { + + public CommittedMutationFact { + Objects.requireNonNull(entityType, "entityType"); + Objects.requireNonNull(entityId, "entityId"); + Objects.requireNonNull(operation, "operation"); + Objects.requireNonNull(committedVersion, "committedVersion"); + fields = Collections.unmodifiableMap(new LinkedHashMap<>(Objects.requireNonNull(fields, "fields"))); + traceChain = List.copyOf(Objects.requireNonNull(traceChain, "traceChain")); + } + + @Override public String toString() { + return "CommittedMutationFact[entityType=" + entityType + + ", operation=" + operation + ", fields=, traceChain=]"; + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationRecorder.java b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationRecorder.java new file mode 100644 index 00000000..2592608e --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedMutationRecorder.java @@ -0,0 +1,16 @@ +package io.teaql.runtime; + +import io.teaql.core.MutationIntent; +import io.teaql.core.UserContext; +import java.util.List; + +/** + * Optional, trusted source-side Outbox capability. Called exactly once for a + * changed graph while the graph's provider transaction is still open. A + * failure must abort that transaction; delivery is never a post-commit hook. + */ +@FunctionalInterface +public interface CommittedMutationRecorder { + void record(UserContext context, Object transactionResource, + MutationIntent intent, List facts); +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java index 785c2558..d826b9b0 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/DefaultUserContext.java @@ -172,6 +172,9 @@ public T capability(Class capabilityType) { if (value == null && capabilityType == BusinessIdSchemaContributor.class) { value = runtime.getBusinessIdSchemaContributor(); } + if (value == null && capabilityType == CommittedEventSchemaContributor.class) { + value = runtime.getCommittedEventSchemaContributor(); + } if (value == null && capabilityType == BusinessIdKeyProvider.class) { value = runtime.getBusinessIdKeyProvider(); } diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 88d60e55..ec6b1551 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -4,6 +4,7 @@ import io.teaql.core.meta.EntityDescriptor; import io.teaql.core.meta.EntityMetaFactory; import io.teaql.core.meta.PropertyDescriptor; +import io.teaql.core.meta.Relation; import io.teaql.core.checker.CheckException; import io.teaql.core.checker.CheckResult; import io.teaql.core.checker.Checker; @@ -38,6 +39,7 @@ public class TeaQLRuntime implements AutoCloseable { private final BusinessIdKeyProvider businessIdKeyProvider; private final RoundTripReferenceCodec roundTripReferenceCodec; private final BusinessClock businessClock; + private final CommittedMutationRecorder committedMutationRecorder; private final Map> checkers = new java.util.concurrent.ConcurrentHashMap<>(); private final List generatedBootstraps = new java.util.concurrent.CopyOnWriteArrayList<>(); @@ -65,6 +67,7 @@ private TeaQLRuntime(Builder builder) { ? null : new RoundTripReferenceCodec(builder.roundTripReferenceProvider); this.businessClock = builder.businessClock; + this.committedMutationRecorder = builder.committedMutationRecorder; } public static Builder builder() { @@ -139,6 +142,11 @@ BusinessIdSchemaContributor getBusinessIdSchemaContributor() { return businessIdSchemaContributor; } + CommittedEventSchemaContributor getCommittedEventSchemaContributor() { + return committedMutationRecorder instanceof CommittedEventSchemaContributor contributor + ? contributor : null; + } + BusinessIdKeyProvider getBusinessIdKeyProvider() { return businessIdKeyProvider; } @@ -503,6 +511,10 @@ public void saveGraph(UserContext context, Entity entity) { requireSingleMutationRoute(mutationPlan, route); MutationGovernanceSnapshot governance = reviewMutationPlan(context, mutationPlan); var dynamicChanges = entityMutationLedger.currentChangeSet().dynamicChanges(); + if (committedMutationRecorder != null && !dynamicChanges.isEmpty()) { + throw new TeaQLRuntimeException( + "Committed mutation capture cannot yet encode dynamic field changes"); + } var dynamicRequests = new ArrayList(); dynamicChanges.forEach((key, mutations) -> { if (!entityMutationLedger.deletedKeys().contains(key)) mutations.values().forEach(mutation -> dynamicRequests.add( @@ -536,9 +548,16 @@ public void saveGraph(UserContext context, Entity entity) { dynamicReadbacks.put(entry.getKey(), value); } } + if (committedMutationRecorder != null && !results.isEmpty()) { + committedMutationRecorder.record(context, transactionExecutor.transactionResource(), + intent, freezeCommittedFacts(results)); + } return results; }); } else { + if (committedMutationRecorder != null) { + throw new TeaQLRuntimeException("Committed mutation capture requires one provider transaction"); + } if (!dynamicRequests.isEmpty()) throw new TeaQLRuntimeException("DYNAMIC_FIELD_TRANSACTION_BINDING_REQUIRED: native graph executor has no transaction"); completed = executeLedgerPlan( context, entityMutationLedger, mutationExecutor, realEntities, governance, intent, traceScopes, graphScope); @@ -939,6 +958,7 @@ private List executeLedgerPlan( for (var batch : deleteBatches.entrySet()) { List requests = new ArrayList<>(); List targets = new ArrayList<>(); + List predecessorVersions = new ArrayList<>(); EntityDescriptor descriptor = metadata.resolveEntityDescriptor(batch.getKey()); if (descriptor == null) throw new TeaQLRuntimeException("No entity descriptor for: " + batch.getKey()); for (EntityKey key : batch.getValue()) { @@ -956,11 +976,13 @@ private List executeLedgerPlan( deleteEntity, EntityPersistenceMutation.Action.DELETE, intent, mutationTrace(root, key, traceScopes, graphScope), target, graphRedactions); requests.add(mutationRequest); targets.add(target == null ? deleteEntity : target); + predecessorVersions.add(originalVersion); } List results = mutateBatchWithTelemetry( context, mutationExecutor, intent, requests, batch.getKey(), "delete"); for (int index = 0; index < requests.size(); index++) { completed.add(new PendingMutation(descriptor, targets.get(index), results.get(index), + predecessorVersions.get(index), MutationAuditKind.DELETED, Collections.emptyMap(), governance, intent, requests.get(index).getTraceChain(), graphRedactions)); } } @@ -1019,7 +1041,7 @@ private List executeLedgerPlan( context, mutationExecutor, intent, requests, entityName, "save"); for (int index = 0; index < requests.size(); index++) { completed.add(new PendingMutation( - descriptor, targets.get(index), results.get(index), + descriptor, targets.get(index), results.get(index), null, MutationAuditKind.CREATED, snapshots.get(index), governance, intent, requests.get(index).getTraceChain(), graphRedactions)); } } @@ -1037,6 +1059,7 @@ private List executeLedgerPlan( for (boolean recovering : List.of(false, true)) { List requests = new ArrayList<>(); List targets = new ArrayList<>(); + List predecessorVersions = new ArrayList<>(); List> snapshots = new ArrayList<>(); for (EntityKey key : keys) { if (root.recoveredKeys().contains(key) != recovering) continue; @@ -1060,6 +1083,7 @@ private List executeLedgerPlan( entity, EntityPersistenceMutation.Action.SAVE, intent, mutationTrace(root, key, traceScopes, graphScope), target, graphRedactions); requests.add(mutationRequest); targets.add(target == null ? entity : target); + predecessorVersions.add(version); snapshots.add(snapshotChanges(changesWithDynamic(changeSet, key))); } MutationAuditKind auditKind = recovering ? MutationAuditKind.RECOVERED : MutationAuditKind.UPDATED; @@ -1067,6 +1091,7 @@ private List executeLedgerPlan( context, mutationExecutor, intent, requests, entityName, auditKind.name().toLowerCase(Locale.ROOT)); for (int index = 0; index < requests.size(); index++) { completed.add(new PendingMutation(descriptor, targets.get(index), results.get(index), + predecessorVersions.get(index), auditKind, snapshots.get(index), governance, intent, requests.get(index).getTraceChain(), graphRedactions)); } } @@ -1109,11 +1134,88 @@ private record PendingMutation( EntityDescriptor descriptor, BaseEntity target, MutationResult result, + Long baseVersion, MutationAuditKind auditKind, Map changedValues, MutationGovernanceSnapshot governance, MutationIntent intent, List traceChain, SqlIntentRedactions redactions) {} + private List freezeCommittedFacts(List completed) { + List facts = new ArrayList<>(completed.size()); + for (PendingMutation pending : completed) { + if (pending.result() == null || !(pending.result().persistedEntity() instanceof BaseEntity persisted) + || persisted.getId() == null || persisted.getVersion() == null) { + throw new TeaQLRuntimeException("Committed mutation has no authoritative provider readback"); + } + boolean created = pending.auditKind() == MutationAuditKind.CREATED; + boolean deleted = pending.auditKind() == MutationAuditKind.DELETED; + Long baseVersion = pending.baseVersion(); + if (!created && baseVersion == null) { + throw new TeaQLRuntimeException("Committed mutation has no predecessor version"); + } + Map fields = new LinkedHashMap<>(); + if (!deleted) { + for (EntityDescriptor current = pending.descriptor(); current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getOwnProperties()) { + String name = property.getName(); + if (property.isId() || property.isVersion() || fields.containsKey(name)) continue; + if (!persisted.isPropertyLoaded(name)) { + throw new TeaQLRuntimeException("Committed provider readback omitted modeled field " + name); + } + fields.put(name, committedFieldValue(property, persisted.getProperty(name))); + } + } + } + String operation = switch (pending.auditKind()) { + case CREATED -> "create"; + case UPDATED -> "update"; + case DELETED -> "delete"; + case RECOVERED -> "recover"; + }; + facts.add(new CommittedMutationFact(persisted.typeName(), persisted.getId().toString(), + operation, baseVersion == null ? null : baseVersion.toString(), + persisted.getVersion().toString(), fields, pending.traceChain())); + } + return List.copyOf(facts); + } + + private Object committedFieldValue(PropertyDescriptor property, Object value) { + if (value == null) return null; + if (property instanceof Relation relation) { + if (relation.getReverseProperty() == null || relation.getReverseProperty().getOwner() == null) { + throw new TeaQLRuntimeException("Committed relation has no target metadata: " + property.getName()); + } + String type = relation.getReverseProperty().getOwner().getType(); + Long id = value instanceof Entity entity ? entity.getId() + : value instanceof Number number ? number.longValue() : null; + if (id == null) throw new TeaQLRuntimeException("Committed relation has no scalar identity: " + property.getName()); + return Map.of("entity_type", type, "entity_id", id.toString()); + } + if (value instanceof String || value instanceof Boolean || value instanceof Long + || value instanceof Integer || value instanceof Short || value instanceof Byte) return value; + if (value instanceof java.math.BigDecimal decimal) return decimal.toPlainString(); + if (value instanceof java.math.BigInteger integer) return integer.toString(); + if (value instanceof Double number) { + if (!Double.isFinite(number)) throw new TeaQLRuntimeException("Non-finite committed numeric value"); + return number; + } + if (value instanceof Float number) { + if (!Float.isFinite(number)) throw new TeaQLRuntimeException("Non-finite committed numeric value"); + return number; + } + if (value instanceof java.time.temporal.TemporalAccessor || value instanceof java.util.UUID + || value instanceof Enum) return value.toString(); + if (value instanceof java.util.Date date) { + // JDBC date/time subclasses deliberately throw from toInstant(). + // Keep this model-level runtime independent of the java.sql module. + try { return date.toInstant().toString(); } + catch (UnsupportedOperationException unsupported) { return date.toString(); } + } + if (value instanceof byte[] bytes) return Map.of("$binary_base64", + java.util.Base64.getEncoder().encodeToString(bytes)); + throw new TeaQLRuntimeException("Committed field has unsupported type: " + property.getName()); + } + private record PersistenceState( Long version, io.teaql.core.EntityStatus status, boolean versionLoaded) {} @@ -1363,6 +1465,7 @@ public static class Builder { private BusinessIdKeyProvider businessIdKeyProvider; private RoundTripReferenceProvider roundTripReferenceProvider; private BusinessClock businessClock = SystemBusinessClock.INSTANCE; + private CommittedMutationRecorder committedMutationRecorder; public Builder metadata(EntityMetaFactory metadata) { this.metadata = metadata; @@ -1419,6 +1522,12 @@ public Builder businessClock(BusinessClock businessClock) { return this; } + /** Enables graph-atomic, pre-commit capture through a trusted provider recorder. */ + public Builder committedMutationRecorder(CommittedMutationRecorder recorder) { + this.committedMutationRecorder = Objects.requireNonNull(recorder, "recorder"); + return this; + } + /** * Installs one provider-backed allocator for generated Fix execution and * the explicit {@code context.ensureSchema()} lifecycle. diff --git a/teaql-sqlite/pom.xml b/teaql-sqlite/pom.xml index e1133aa9..69ca81b5 100644 --- a/teaql-sqlite/pom.xml +++ b/teaql-sqlite/pom.xml @@ -37,6 +37,11 @@ teaql-provider-jdbc ${project.version} + + io.teaql + teaql-runtime + ${project.version} + io.teaql @@ -55,12 +60,6 @@ junit test - - io.teaql - teaql-runtime - ${project.version} - test - diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java index 5ad9207a..bc2cb268 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java @@ -30,7 +30,12 @@ public SqliteCommittedEventStore(SqlExecutionAdapter sql) { } public record OutboxRecord(String sourceRuntimeId, long sourceSequence, - String changeSetId, String modelHash, String payloadSha256, String payloadJson) {} + String changeSetId, String modelHash, String payloadSha256, String payloadJson) { + @Override public String toString() { + return "OutboxRecord[sourceRuntimeId=" + sourceRuntimeId + + ", sourceSequence=" + sourceSequence + ", payload=]"; + } + } public record InboxOutcome(boolean duplicate, long checkpoint) { public static InboxOutcome applied(long checkpoint) { return new InboxOutcome(false, checkpoint); } diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java new file mode 100644 index 00000000..bb8e1d88 --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java @@ -0,0 +1,111 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.core.GeneratedSchemaBootstrap; +import io.teaql.core.CommittedEventSchemaContributor; +import io.teaql.core.MutationIntent; +import io.teaql.core.SchemaExecutor; +import io.teaql.core.TraceNode; +import io.teaql.core.UserContext; +import io.teaql.dataservice.sql.SqlExecutionAdapter; +import io.teaql.runtime.CommittedMutationFact; +import io.teaql.runtime.CommittedMutationRecorder; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +/** SQLite source adapter that records one graph-save result inside its write transaction. */ +public final class SqliteCommittedMutationRecorder + implements CommittedMutationRecorder, CommittedEventSchemaContributor { + private static final ObjectMapper JSON = new ObjectMapper(); + private final SqlExecutionAdapter adapter; + private final SqliteCommittedEventStore events; + private final String sourceRuntimeId; + private final String modelHash; + + public SqliteCommittedMutationRecorder(SqlExecutionAdapter adapter, + String sourceRuntimeId, String modelHash) { + this.adapter = Objects.requireNonNull(adapter, "adapter"); + this.events = new SqliteCommittedEventStore(adapter); + if (sourceRuntimeId == null || sourceRuntimeId.isBlank() + || modelHash == null || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw new IllegalArgumentException("A source runtime ID and accepted SHA-256 model hash are required"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.modelHash = modelHash; + } + + @Override + public void ensureSchema(UserContext context, SchemaExecutor.Invocation invocation) { + SchemaExecutor.Invocation.requireContextOwned(invocation); + Objects.requireNonNull(context, "context"); + events.ensureSchema(); + } + public SqliteCommittedEventStore eventStore() { return events; } + + @Override + public void record(UserContext context, Object transactionResource, + MutationIntent intent, List facts) { + Objects.requireNonNull(context, "context"); + Objects.requireNonNull(intent, "intent"); + if (transactionResource != adapter || !adapter.hasActiveTransaction()) { + throw new IllegalStateException("Committed graph capture requires the source provider transaction"); + } + if (facts == null || facts.isEmpty()) { + throw new IllegalArgumentException("Committed graph capture requires at least one fact"); + } + String changeSetId = UUID.randomUUID().toString(); + String committedAt = Instant.now().toString(); + String actor = context.getAttribute(GeneratedSchemaBootstrap.AUDIT_ACTOR_ATTRIBUTE, String.class); + events.append(sourceRuntimeId, changeSetId, modelHash, sequence -> { + Map envelope = new LinkedHashMap<>(); + envelope.put("protocol_version", 1); + envelope.put("change_set_id", changeSetId); + envelope.put("source_runtime_id", sourceRuntimeId); + envelope.put("source_sequence", Long.toString(sequence)); + envelope.put("source_model_hash", modelHash); + envelope.put("source_transaction_id", changeSetId); + envelope.put("committed_at", committedAt); + Map audit = new LinkedHashMap<>(); + audit.put("actor_ref", actor); + audit.put("comment", intent.comment()); + // MutationIntent currently has no separate business purpose field. + audit.put("purpose", "replicate committed graph"); + envelope.put("audit", audit); + List> mutations = new ArrayList<>(facts.size()); + for (CommittedMutationFact fact : facts) { + Map mutation = new LinkedHashMap<>(); + mutation.put("entity_type", fact.entityType()); + mutation.put("entity_id", fact.entityId()); + mutation.put("operation", fact.operation()); + mutation.put("base_version", fact.baseVersion()); + mutation.put("committed_version", fact.committedVersion()); + mutation.put("fields", fact.fields()); + List> chain = new ArrayList<>(); + for (TraceNode node : fact.traceChain()) { + Map step = new LinkedHashMap<>(); + step.put("kind", node.getKind().name()); + step.put("name", node.getName()); + step.put("entity_id", node.getEntityId() == null ? null : node.getEntityId().toString()); + step.put("comment", node.getComment()); + chain.add(step); + } + mutation.put("trace_chain", chain); + mutations.add(mutation); + } + envelope.put("mutations", mutations); + try { + return JSON.writeValueAsString(envelope); + } catch (JsonProcessingException failure) { + // Do not attach values or serializer causes to public errors. + throw new IllegalStateException("Committed mutation payload encoding failed"); + } + }); + } +} diff --git a/teaql-sqlite/src/main/java/module-info.java b/teaql-sqlite/src/main/java/module-info.java index 5028c4b3..82c0d1ef 100644 --- a/teaql-sqlite/src/main/java/module-info.java +++ b/teaql-sqlite/src/main/java/module-info.java @@ -4,6 +4,7 @@ requires transitive io.teaql.utils; requires transitive io.teaql.dataservice.sql; requires io.teaql.provider.jdbc; + requires io.teaql.runtime; requires com.fasterxml.jackson.databind; requires org.xerial.sqlitejdbc; requires java.sql; diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index c0159efb..a2634c98 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -5,6 +5,9 @@ import io.teaql.core.sql.GenericSQLProperty; import io.teaql.core.sql.SQLEntityDescriptor; import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import io.teaql.provider.jdbc.JdbcSqlExecutor; import io.teaql.runtime.*; import java.nio.file.Files; @@ -52,10 +55,13 @@ static final class Fixture { final JdbcSqlExecutor driver; final SQLiteDataSource dataSource; final DefaultUserContext context; + final SqliteCommittedMutationRecorder committedRecorder; volatile boolean failReadback; volatile boolean unknownUpdateCounts; - Fixture() throws Exception { + Fixture() throws Exception { this(false); } + + Fixture(boolean captureCommittedMutations) throws Exception { var ds = new SQLiteDataSource(); dataSource = ds; ds.setUrl("jdbc:sqlite:" + Files.createTempFile("teaql-graph-trace-", ".db")); @@ -86,6 +92,9 @@ static final class Fixture { return super.queryForList(text, args); } }; + committedRecorder = captureCommittedMutations + ? new SqliteCommittedMutationRecorder(driver, "java-graph-source", "sha256:" + "0".repeat(64)) + : null; var metadata = new SimpleEntityMetaFactory(); for (String type : List.of("CustomerOrder", "OrderItem", "Payment", "PaymentAttempt", "Shipment")) { var descriptor = new SQLEntityDescriptor(); @@ -124,9 +133,11 @@ static final class Fixture { } }; var ids = new AtomicLong(1000); - var runtime = TeaQLRuntime.builder().metadata(metadata).dataService("sqlite", provider) + var builder = TeaQLRuntime.builder().metadata(metadata).dataService("sqlite", provider) .idGenerationService((caller, entity) -> ids.getAndIncrement()) - .logSink((caller, entry) -> sql.add(entry)).build(); + .logSink((caller, entry) -> sql.add(entry)); + if (committedRecorder != null) builder.committedMutationRecorder(committedRecorder); + var runtime = builder.build(); context = new DefaultUserContext(runtime) { @Override public boolean isQueryExecutionLoggingEnabled() { return queryLogging; } @Override public boolean isMutationExecutionLoggingEnabled() { return mutationLogging; } @@ -445,6 +456,90 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertEquals(Long.valueOf(-2), removed.getVersion()); } + @Test public void graphSaveCapturesAuthoritativeRowsAndOutboxInTheSameTransaction() throws Exception { + var fixture = new Fixture(true); + assertThrows(SecurityException.class, () -> fixture.committedRecorder.ensureSchema(fixture.context, null)); + fixture.driver.execute("CREATE TRIGGER order_db_default AFTER INSERT ON customer_order_data " + + "BEGIN UPDATE customer_order_data SET memo = 'database trigger value' WHERE id = NEW.id; END"); + var order = fixture.create("CustomerOrder", 100, "root value"); + var item = fixture.create("OrderItem", 201, "first item"); + item.setComment("create item"); + var payment = fixture.create("Payment", 300, "payment value"); + order.__internalSet("children", List.of(item, payment)); + order.auditAs("create complete order graph").save(fixture.context); + + var events = fixture.committedRecorder.eventStore(); + var first = events.outboxAfter("java-graph-source", 0, 10); + assertEquals("one graph save must produce one durable set", 1, first.size()); + JsonNode created = new ObjectMapper().readTree(first.get(0).payloadJson()); + assertEquals("create complete order graph", created.path("audit").path("comment").asText()); + assertEquals(3, created.path("mutations").size()); + JsonNode orderFact = null; + for (JsonNode fact : created.path("mutations")) { + assertEquals("create", fact.path("operation").asText()); + assertTrue(fact.path("base_version").isNull()); + assertEquals("1", fact.path("committed_version").asText()); + assertTrue(fact.path("trace_chain").isArray()); + if (fact.path("entity_type").asText().equals("CustomerOrder")) orderFact = fact; + } + assertNotNull(orderFact); + assertEquals("database trigger value", orderFact.path("fields").path("memo").asText()); + assertFalse(new CommittedMutationFact("CustomerOrder", "100", "create", null, "1", + Map.of("memo", "database trigger value"), List.of()).toString().contains("database trigger value")); + assertEquals("database trigger value", fixture.driver.queryForList( + "SELECT memo FROM customer_order_data WHERE id = ?", new Object[] {100L}) + .get(0).get("memo")); + + item.updateProperty("name", "revised item"); + item.setComment("revise item"); + order.auditAs("revise order graph").save(fixture.context); + var second = events.outboxAfter("java-graph-source", 1, 10); + assertEquals(1, second.size()); + JsonNode changed = new ObjectMapper().readTree(second.get(0).payloadJson()); + assertEquals(1, changed.path("mutations").size()); + JsonNode changedItem = changed.path("mutations").get(0); + assertEquals("update", changedItem.path("operation").asText()); + assertEquals("1", changedItem.path("base_version").asText()); + assertEquals("2", changedItem.path("committed_version").asText()); + assertEquals("revised item", changedItem.path("fields").path("name").asText()); + + payment.markForDeletion(); + payment.setComment("remove payment"); + order.auditAs("remove payment from order").save(fixture.context); + var third = events.outboxAfter("java-graph-source", 2, 10); + assertEquals(1, third.size()); + JsonNode tombstone = new ObjectMapper().readTree(third.get(0).payloadJson()).path("mutations").get(0); + assertEquals("delete", tombstone.path("operation").asText()); + assertEquals("1", tombstone.path("base_version").asText()); + assertEquals("-2", tombstone.path("committed_version").asText()); + assertEquals(0, tombstone.path("fields").size()); + + fixture.driver.execute("CREATE UNIQUE INDEX payment_name_unique_for_outbox ON payment_data(name)"); + var duplicate = fixture.create("Payment", 301, "payment value"); + order.__internalSet("children", List.of(item, payment, duplicate)); + assertThrows(RuntimeException.class, + () -> order.auditAs("reject duplicate payment graph").save(fixture.context)); + assertEquals("failed save must leave no publishable set", 3, + events.outboxAfter("java-graph-source", 0, 10).size()); + assertTrue(fixture.driver.queryForList("SELECT id FROM payment_data WHERE id = ?", + new Object[] {301L}).isEmpty()); + } + + @Test public void brokenOutboxRollsBackNormalGraphSaveAndDoesNotDeliverAudit() throws Exception { + var fixture = new Fixture(true); + fixture.driver.execute("DROP TABLE teaql_event_outbox"); + var order = fixture.create("CustomerOrder", 100, "must roll back"); + var item = fixture.create("OrderItem", 201, "must roll back too"); + order.__internalSet("children", List.of(item)); + assertThrows(RuntimeException.class, + () -> order.auditAs("fail together when outbox unavailable").save(fixture.context)); + assertTrue(fixture.driver.queryForList("SELECT id FROM customer_order_data WHERE id = ?", + new Object[] {100L}).isEmpty()); + assertTrue(fixture.driver.queryForList("SELECT id FROM order_item_data WHERE id = ?", + new Object[] {201L}).isEmpty()); + assertTrue("no post-commit audit on rolled back save", fixture.audit.isEmpty()); + } + @Test public void realSqliteProviderFailureKeepsAttemptedLineageWithoutCommittedAudit() throws Exception { var fixture = new Fixture(); fixture.driver.execute("CREATE UNIQUE INDEX payment_name_unique ON payment_data(name)"); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java index 0b5780e0..67b12649 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java @@ -148,6 +148,8 @@ private static CommittedEventException expectCode(String code, Runnable action) OutboxRecord first = append(source, "change-a", false); OutboxRecord second = append(source, "change-b", true); + assertFalse("raw facts must not appear in ordinary formatting", first.toString().contains("DRAFT")); + assertFalse(first.toString().contains("order_item")); assertEquals(1, first.sourceSequence()); assertEquals(2, second.sourceSequence()); From 52231533f7513b6d60d065ddb43d0ce1ab1f42d2 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 02:18:07 +0800 Subject: [PATCH 03/29] Freeze Java committed update facts from transaction preimages --- .../java/io/teaql/runtime/TeaQLRuntime.java | 74 ++++++++++++++++++- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 41 ++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index ec6b1551..142de097 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -526,6 +526,9 @@ public void saveGraph(UserContext context, Entity entity) { try { if (mutationExecutor instanceof TransactionExecutor transactionExecutor) { completed = transactionExecutor.executeInTransaction(context, () -> { + Map> committedPreimages = + committedMutationRecorder == null ? Map.of() + : captureCommittedPreimages(context, mutationExecutor, mutationPlan); io.teaql.data.dynamic.DynamicFieldsFacade dynamic = null; if (!dynamicRequests.isEmpty()) { dynamic = context.dynamicFields().comment(intent.comment()).purpose("runtime: persist audited graph extensions"); @@ -550,7 +553,7 @@ public void saveGraph(UserContext context, Entity entity) { } if (committedMutationRecorder != null && !results.isEmpty()) { committedMutationRecorder.record(context, transactionExecutor.transactionResource(), - intent, freezeCommittedFacts(results)); + intent, freezeCommittedFacts(results, committedPreimages)); } return results; }); @@ -1140,7 +1143,61 @@ private record PendingMutation( MutationGovernanceSnapshot governance, MutationIntent intent, List traceChain, SqlIntentRedactions redactions) {} - private List freezeCommittedFacts(List completed) { + private Map> captureCommittedPreimages( + UserContext context, MutationExecutor mutationExecutor, MutationPlan plan) { + if (!(mutationExecutor instanceof QueryExecutor queryExecutor)) { + throw new TeaQLRuntimeException( + "Committed update capture requires query access on the same transaction executor"); + } + Map> preimages = new TreeMap<>(); + for (MutationOperation operation : plan.operations()) { + if (operation.kind() != MutationOperationKind.UPDATE + && operation.kind() != MutationOperationKind.RECOVER) continue; + EntityKey key = operation.entity(); + EntityDescriptor descriptor = metadata.resolveEntityDescriptor(key.entity()); + if (descriptor == null || descriptor.getTargetType() == null) { + throw new TeaQLRuntimeException("Committed preimage has no entity metadata: " + key.entity()); + } + BaseRequest.TempRequest request = new BaseRequest.TempRequest( + descriptor.getTargetType(), descriptor.getType(), descriptor::createEntity); + request.bindMetadata(metadata); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getOwnProperties()) { + request.selectProperty(property.getName()); + } + } + request.appendSearchCriteria(request.createBasicSearchCriteria( + BaseEntity.ID_PROPERTY, io.teaql.core.criteria.Operator.EQUAL, key.id())); + // Recovery reads a negative-version tombstone. Explicit version + // intent suppresses the normal visible-row default in SQL providers. + request.appendSearchCriteria(request.createBasicSearchCriteria( + BaseEntity.VERSION_PROPERTY, io.teaql.core.criteria.Operator.IS_NOT_NULL)); + request.top(2); + QueryIntent intent = QueryIntent.of( + "capture committed mutation preimage", "compare authoritative pre/post write fields"); + QueryResult result = queryExecutor.query(context, new DefaultQueryRequest(request, intent)); + if (!(result instanceof DefaultQueryResult rows) || rows.getResult().size() != 1 + || !(rows.getResult().get(0) instanceof BaseEntity persisted)) { + throw new TeaQLRuntimeException("Committed mutation preimage must contain one row: " + key.entity()); + } + Map fields = new LinkedHashMap<>(); + for (EntityDescriptor current = descriptor; current != null; current = current.getParent()) { + for (PropertyDescriptor property : current.getOwnProperties()) { + String name = property.getName(); + if (property.isId() || property.isVersion() || fields.containsKey(name)) continue; + if (!persisted.isPropertyLoaded(name)) { + throw new TeaQLRuntimeException("Committed preimage omitted modeled field " + name); + } + fields.put(name, committedFieldValue(property, persisted.getProperty(name))); + } + } + preimages.put(key, fields); + } + return preimages; + } + + private List freezeCommittedFacts( + List completed, Map> preimages) { List facts = new ArrayList<>(completed.size()); for (PendingMutation pending : completed) { if (pending.result() == null || !(pending.result().persistedEntity() instanceof BaseEntity persisted) @@ -1154,6 +1211,11 @@ private List freezeCommittedFacts(List c throw new TeaQLRuntimeException("Committed mutation has no predecessor version"); } Map fields = new LinkedHashMap<>(); + EntityKey key = new EntityKey(persisted.typeName(), persisted.getId()); + Map prior = created || deleted ? null : preimages.get(key); + if (!created && !deleted && prior == null) { + throw new TeaQLRuntimeException("Committed update has no transaction preimage: " + key.entity()); + } if (!deleted) { for (EntityDescriptor current = pending.descriptor(); current != null; current = current.getParent()) { for (PropertyDescriptor property : current.getOwnProperties()) { @@ -1162,7 +1224,13 @@ private List freezeCommittedFacts(List c if (!persisted.isPropertyLoaded(name)) { throw new TeaQLRuntimeException("Committed provider readback omitted modeled field " + name); } - fields.put(name, committedFieldValue(property, persisted.getProperty(name))); + if (prior != null && !prior.containsKey(name)) { + throw new TeaQLRuntimeException("Committed update preimage omitted modeled field " + name); + } + Object committedValue = committedFieldValue(property, persisted.getProperty(name)); + if (prior == null || !Objects.equals(prior.get(name), committedValue)) { + fields.put(name, committedValue); + } } } } diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index a2634c98..2d533aac 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -502,6 +502,8 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertEquals("1", changedItem.path("base_version").asText()); assertEquals("2", changedItem.path("committed_version").asText()); assertEquals("revised item", changedItem.path("fields").path("name").asText()); + assertEquals(1, changedItem.path("fields").size()); + assertFalse(changedItem.path("fields").has("memo")); payment.markForDeletion(); payment.setComment("remove payment"); @@ -525,6 +527,45 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri new Object[] {301L}).isEmpty()); } + @Test public void committedUpdateIncludesDatabaseTriggerChangeButOmitsUnchangedFields() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 909, "before"); + order.auditAs("create trigger comparison order").save(fixture.context); + fixture.driver.execute("CREATE TRIGGER order_update_memo AFTER UPDATE OF name ON customer_order_data " + + "WHEN NEW.name = 'after' BEGIN UPDATE customer_order_data " + + "SET memo = 'database revised memo' WHERE id = NEW.id; END"); + + order.updateProperty("name", "after"); + order.auditAs("update trigger comparison order").save(fixture.context); + var outbox = fixture.committedRecorder.eventStore().outboxAfter("java-graph-source", 1, 10); + assertEquals(1, outbox.size()); + JsonNode fact = new ObjectMapper().readTree(outbox.get(0).payloadJson()) + .path("mutations").get(0); + assertEquals("update", fact.path("operation").asText()); + assertEquals("after", fact.path("fields").path("name").asText()); + assertEquals("database revised memo", fact.path("fields").path("memo").asText()); + assertEquals(2, fact.path("fields").size()); + } + + @Test public void committedRecoveryKeepsPredecessorAndNeedsNoChangedScalarField() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 910, "recoverable"); + order.auditAs("create recoverable order").save(fixture.context); + order.markForDeletion(); + order.auditAs("delete recoverable order").save(fixture.context); + order.markAsRecover(); + order.auditAs("recover order").save(fixture.context); + + var outbox = fixture.committedRecorder.eventStore().outboxAfter("java-graph-source", 2, 10); + assertEquals(1, outbox.size()); + JsonNode fact = new ObjectMapper().readTree(outbox.get(0).payloadJson()) + .path("mutations").get(0); + assertEquals("recover", fact.path("operation").asText()); + assertEquals("-2", fact.path("base_version").asText()); + assertEquals("3", fact.path("committed_version").asText()); + assertEquals(0, fact.path("fields").size()); + } + @Test public void brokenOutboxRollsBackNormalGraphSaveAndDoesNotDeliverAudit() throws Exception { var fixture = new Fixture(true); fixture.driver.execute("DROP TABLE teaql_event_outbox"); From 1413c3385d4ad90168fd595814c3550d2aed2872 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 02:40:07 +0800 Subject: [PATCH 04/29] Apply committed events through model-checked Java SQLite target --- .../committed/CommittedEventsExampleTest.java | 28 ++ .../sqlite/SqliteCommittedEventStore.java | 26 +- .../SqliteModelCheckedCommittedTarget.java | 339 ++++++++++++++++++ .../sqlite/SqliteCommittedEventStoreTest.java | 9 +- ...SqliteModelCheckedCommittedTargetTest.java | 222 ++++++++++++ 5 files changed, 619 insertions(+), 5 deletions(-) create mode 100644 teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java rename teaql-sqlite/src/test/java/io/teaql/{ => core}/sqlite/SqliteCommittedEventStoreTest.java (96%) create mode 100644 teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java index be4c0da7..b1b54cc4 100644 --- a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java @@ -8,6 +8,7 @@ import io.teaql.core.sql.SQLEntityDescriptor; import io.teaql.core.sqlite.SqliteCommittedEventStore; import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; import io.teaql.core.sqlite.SqliteDataServiceExecutor; import io.teaql.provider.jdbc.JdbcSqlExecutor; import io.teaql.runtime.DefaultUserContext; @@ -89,6 +90,33 @@ public static final class Order extends BaseEntity { assertEquals("1", submitted.path("mutations").get(0).path("base_version").asText()); assertEquals("2", submitted.path("mutations").get(0).path("committed_version").asText()); + Path replicaPath = Files.createTempFile("teaql-committed-target-example-", ".sqlite"); + replicaPath.toFile().deleteOnExit(); + SQLiteDataSource replicaSource = new SQLiteDataSource(); + replicaSource.setUrl("jdbc:sqlite:" + replicaPath); + JdbcSqlExecutor replicaSql = new JdbcSqlExecutor(replicaSource); + TeaQLRuntime replicaRuntime = TeaQLRuntime.builder().metadata(metadata) + .dataService("sqlite", new SqliteDataServiceExecutor("sqlite", replicaSql, replicaSource)) + .executionLogging(false).build(); + DefaultUserContext replicaContext = new DefaultUserContext(replicaRuntime); + replicaContext.ensureSchema(); + SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget( + replicaSql, "example-java-source", "sha256:" + "0".repeat(64)); + target.ensureSchema(); + assertEquals(1, target.apply(replicaContext, durable.get(0)).checkpoint()); + assertEquals(2, target.apply(replicaContext, durable.get(1)).checkpoint()); + var duplicate = target.apply(replicaContext, durable.get(0)); + assertTrue(duplicate.duplicate()); + assertEquals(1, duplicate.checkpoint()); + var replicated = replicaSql.queryForList("SELECT version, name, memo FROM order_data WHERE id = ?", + new Object[] {42L}).get(0); + assertEquals(2L, ((Number) replicated.get("version")).longValue()); + assertEquals("submitted", replicated.get("name")); + assertEquals("set by database", replicated.get("memo")); + assertEquals(2, replicaSql.queryForList("SELECT * FROM teaql_event_applied_payload", + new Object[0]).size()); + replicaRuntime.close(); + sql.execute("DROP TABLE teaql_event_outbox"); order.updateProperty("name", "should roll back"); assertThrows(RuntimeException.class, diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java index bc2cb268..a644df67 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java @@ -43,14 +43,14 @@ public record InboxOutcome(boolean duplicate, long checkpoint) { } @FunctionalInterface - public interface TrustedProjectionWriter { + interface TrustedProjectionWriter { void apply(SqlExecutionAdapter transaction, JsonNode committedFacts); } public static final class CommittedEventException extends RuntimeException { private final String code; - private CommittedEventException(String code, String message) { + CommittedEventException(String code, String message) { super(message); this.code = code; } @@ -73,6 +73,11 @@ public void ensureSchema() { + "source_runtime_id TEXT NOT NULL, change_set_id TEXT NOT NULL, " + "source_sequence INTEGER NOT NULL CHECK(source_sequence > 0), " + "payload_sha256 TEXT NOT NULL, PRIMARY KEY(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_applied_payload (" + + "source_runtime_id TEXT NOT NULL, source_sequence INTEGER NOT NULL CHECK(source_sequence > 0), " + + "change_set_id TEXT NOT NULL, payload_json TEXT NOT NULL, " + + "PRIMARY KEY(source_runtime_id, source_sequence), " + + "UNIQUE(source_runtime_id, change_set_id))"); sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_checkpoint (" + "source_runtime_id TEXT PRIMARY KEY, " + "source_sequence INTEGER NOT NULL CHECK(source_sequence >= 0), " @@ -127,7 +132,7 @@ public OutboxRecord append(String sourceRuntimeId, String changeSetId, String mo * Apply one set through a restricted projection writer. The writer must not * call normal entity save, Checker/Fix, ID generation or business hooks. */ - public InboxOutcome apply(OutboxRecord record, String expectedModelHash, + InboxOutcome apply(OutboxRecord record, String expectedModelHash, TrustedProjectionWriter writer) { requireTransaction(); if (record == null || blank(expectedModelHash) || writer == null) { @@ -157,7 +162,16 @@ public InboxOutcome apply(OutboxRecord record, String expectedModelHash, Map seen = prior.get(0); if (number(seen.get("source_sequence")) == record.sourceSequence() && record.payloadSha256().equals(seen.get("payload_sha256"))) { - return InboxOutcome.duplicate(last); + List> retained = sql.queryForList( + "SELECT source_sequence, payload_json FROM teaql_event_applied_payload " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {record.sourceRuntimeId(), record.changeSetId()}); + if (retained.size() != 1 + || number(retained.get(0).get("source_sequence")) != record.sourceSequence() + || !record.payloadSha256().equals(digest((String) retained.get(0).get("payload_json")))) { + throw invalid("prior Inbox identity lacks retained source evidence"); + } + return InboxOutcome.duplicate(record.sourceSequence()); } throw new CommittedEventException("DUPLICATE_CONFLICT", "committed-event identity has conflicting content"); } @@ -170,6 +184,10 @@ public InboxOutcome apply(OutboxRecord record, String expectedModelHash, } return savepoint("teaql_event_apply", () -> { writer.apply(sql, payload); + sql.update("INSERT INTO teaql_event_applied_payload " + + "(source_runtime_id, source_sequence, change_set_id, payload_json) VALUES (?, ?, ?, ?)", + new Object[] {record.sourceRuntimeId(), record.sourceSequence(), + record.changeSetId(), record.payloadJson()}); sql.update("INSERT INTO teaql_event_inbox " + "(source_runtime_id, change_set_id, source_sequence, payload_sha256) VALUES (?, ?, ?, ?)", new Object[] {record.sourceRuntimeId(), record.changeSetId(), diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java new file mode 100644 index 00000000..1098f55d --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -0,0 +1,339 @@ +package io.teaql.core.sqlite; + +import com.fasterxml.jackson.databind.JsonNode; +import io.teaql.core.UserContext; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.meta.PropertyDescriptor; +import io.teaql.core.meta.Relation; +import io.teaql.core.sql.SQLColumn; +import io.teaql.core.sql.portable.SQLPropertyUtil; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Startup-owned, one-source SQLite target for committed model facts. This is + * infrastructure, not a business-facing Mutation or remote query endpoint. + */ +public final class SqliteModelCheckedCommittedTarget { + private final SqlExecutionAdapter sql; + private final SqliteCommittedEventStore events; + private final String sourceRuntimeId; + private final String modelHash; + + public SqliteModelCheckedCommittedTarget(SqlExecutionAdapter sql, + String sourceRuntimeId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (sourceRuntimeId == null || sourceRuntimeId.isBlank() + || modelHash == null || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw invalid("target source identity or SHA-256 model hash is invalid"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.modelHash = modelHash; + this.events = new SqliteCommittedEventStore(sql); + } + + public void ensureSchema() { events.ensureSchema(); } + + /** One graph, its Inbox identity, original audit payload and checkpoint commit together. */ + public SqliteCommittedEventStore.InboxOutcome apply( + UserContext context, SqliteCommittedEventStore.OutboxRecord record) { + Objects.requireNonNull(context, "context"); + if (record == null) throw invalid("committed event is missing"); + if (!sourceRuntimeId.equals(record.sourceRuntimeId())) { + throw new SqliteCommittedEventStore.CommittedEventException( + "UNAUTHORIZED_SOURCE", "committed-event source is not authorized"); + } + // Resolve from this Context, never from the legacy process-global fallback. + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + AtomicReference result = new AtomicReference<>(); + sql.executeInTransaction(() -> result.set(events.apply(record, modelHash, + (transaction, payload) -> applyGraph(transaction, metadata, payload)))); + return result.get(); + } + + private static void applyGraph(SqlExecutionAdapter sql, EntityMetaFactory metadata, JsonNode payload) { + JsonNode audit = payload.path("audit"); + if (!audit.isObject() || !nonblank(audit.path("comment")) + || !nonblank(audit.path("purpose")) + || !nonblank(payload.path("source_transaction_id"))) { + throw invalid("source transaction or audit intent is missing"); + } + try { OffsetDateTime.parse(payload.path("committed_at").asText()); } + catch (RuntimeException failure) { throw invalid("commit time is invalid"); } + List mutations = new ArrayList<>(); + Set seen = new HashSet<>(); + for (JsonNode fact : payload.path("mutations")) { + Mutation mutation = decode(metadata, fact); + if (!seen.add(mutation.entityType() + ":" + mutation.id())) { + throw invalid("one set mutates the same entity twice"); + } + mutations.add(mutation); + } + // Decode every fact before touching domain rows. The surrounding + // savepoint also rolls back a later FK or readback failure. + for (Mutation mutation : mutations) applyOne(sql, mutation); + } + + private static Mutation decode(EntityMetaFactory metadata, JsonNode fact) { + if (!fact.isObject() || !nonblank(fact.path("entity_type"))) { + throw invalid("mutation entity type is missing"); + } + String entityType = fact.path("entity_type").textValue(); + EntityDescriptor entity = metadata.allEntityDescriptors().stream() + .filter(candidate -> entityType.equals(candidate.getType())).findFirst() + .orElseThrow(() -> invalid("unknown entity type")); + // Multi-table inheritance needs an explicit target mapping. Reject it + // instead of silently writing only the leaf table. + if (entity.getParent() != null || entity.hasChildren()) { + throw invalid("multi-table entity target mapping is not supported"); + } + PropertyDescriptor idProperty = entity.findIdProperty(); + PropertyDescriptor versionProperty = entity.findVersionProperty(); + if (idProperty == null || versionProperty == null) { + throw invalid("entity has no ID or version property"); + } + Mapping idMapping = mapping(idProperty); + Mapping versionMapping = mapping(versionProperty); + if (!idMapping.table().equals(versionMapping.table())) { + throw invalid("identity and version span multiple tables"); + } + long id = positiveId(fact.path("entity_id")); + String operation = fact.path("operation").asText(""); + if (!List.of("create", "update", "delete", "recover").contains(operation)) { + throw invalid("unknown mutation operation"); + } + JsonNode baseNode = fact.path("base_version"); + Long base = "create".equals(operation) && baseNode.isNull() + ? null : signedVersion(baseNode); + long committed = signedVersion(fact.path("committed_version")); + boolean valid = switch (operation) { + case "create" -> base == null && committed == 1; + case "update" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == base + 1; + case "delete" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == -(base + 1); + case "recover" -> base != null && base < 0 && base != Long.MIN_VALUE + && -base < Long.MAX_VALUE && committed == -base + 1; + default -> false; + }; + if (!valid) throw invalid("invalid version transition"); + JsonNode supplied = fact.path("fields"); + if (!supplied.isObject()) throw invalid("mutation fields are missing"); + if ("delete".equals(operation) && !supplied.isEmpty()) { + throw invalid("delete must be a tombstone"); + } + List ownProperties = entity.getOwnProperties(); + supplied.fieldNames().forEachRemaining(name -> { + PropertyDescriptor property = entity.findProperty(name); + if (property == null || !ownProperties.contains(property) + || property.isId() || property.isVersion()) { + throw invalid("unknown or reserved model field"); + } + }); + List fields = new ArrayList<>(); + for (PropertyDescriptor property : ownProperties) { + if (property.isId() || property.isVersion()) continue; + JsonNode value = supplied.get(property.getName()); + if (value == null) { + if ("create".equals(operation)) throw invalid("create omitted a modeled field"); + continue; + } + Mapping column = mapping(property); + if (!idMapping.table().equals(column.table())) { + throw invalid("mutation field spans multiple tables"); + } + Class type = property.getType() == null ? null : property.getType().javaType(); + fields.add(new Field(column.column(), decodeValue(property, value), + type == BigDecimal.class || type == java.math.BigInteger.class)); + } + return new Mutation(entityType, idMapping.table(), idMapping.column(), + versionMapping.column(), id, base, committed, operation, fields); + } + + private static Object decodeValue(PropertyDescriptor property, JsonNode value) { + if (value.isNull()) { + if (property.getBoolean("required", false)) throw invalid("non-null field is null"); + return null; + } + if (property instanceof Relation relation) { + if (!value.isObject() || value.size() != 2 || relation.getReverseProperty() == null + || relation.getReverseProperty().getOwner() == null + || !relation.getReverseProperty().getOwner().getType() + .equals(value.path("entity_type").asText())) { + throw invalid("reference target does not match model"); + } + return positiveId(value.path("entity_id")); + } + Class type = property.getType() == null ? null : property.getType().javaType(); + if (type == String.class || type == java.util.UUID.class || type == java.time.LocalDate.class + || type == java.time.LocalDateTime.class || type == java.time.Instant.class + || type == java.util.Date.class || type == java.sql.Date.class + || type == java.sql.Timestamp.class || type != null && type.isEnum()) { + if (!value.isTextual()) throw invalid("text/date field has wrong type"); + return value.textValue(); + } + if (type == Boolean.class || type == boolean.class) { + if (!value.isBoolean()) throw invalid("Boolean field has wrong type"); + return value.booleanValue() ? 1 : 0; + } + if (type == Long.class || type == long.class || type == Integer.class || type == int.class + || type == Short.class || type == short.class || type == Byte.class || type == byte.class) { + if (!value.isIntegralNumber() || !value.canConvertToLong()) { + throw invalid("integer field has wrong type"); + } + long number = value.longValue(); + if (type == Integer.class || type == int.class) { + if (number < Integer.MIN_VALUE || number > Integer.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Short.class || type == short.class) { + if (number < Short.MIN_VALUE || number > Short.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Byte.class || type == byte.class) { + if (number < Byte.MIN_VALUE || number > Byte.MAX_VALUE) throw invalid("integer overflows field"); + } + return number; + } + if (type == BigDecimal.class || type == java.math.BigInteger.class) { + if (!value.isTextual()) throw invalid("Decimal field has wrong type"); + try { + if (type == java.math.BigInteger.class) { + return new java.math.BigInteger(value.textValue()).toString(); + } + return new BigDecimal(value.textValue()).toPlainString(); + } + catch (NumberFormatException failure) { throw invalid("Decimal field is invalid"); } + } + if (type == Double.class || type == double.class || type == Float.class || type == float.class) { + if (!value.isNumber() || !Double.isFinite(value.doubleValue())) { + throw invalid("floating field has wrong type"); + } + return value.doubleValue(); + } + if (type == byte[].class && value.isObject() && value.size() == 1 + && value.path("$binary_base64").isTextual()) { + try { return java.util.Base64.getDecoder().decode(value.path("$binary_base64").textValue()); } + catch (IllegalArgumentException failure) { throw invalid("binary field is invalid"); } + } + throw invalid("unsupported modeled field type"); + } + + private static void applyOne(SqlExecutionAdapter sql, Mutation mutation) { + String table = quoted(mutation.table()); + String idColumn = quoted(mutation.idColumn()); + String versionColumn = quoted(mutation.versionColumn()); + List> current = sql.queryForList("SELECT " + versionColumn + + " AS committed_version FROM " + table + " WHERE " + idColumn + " = ?", + new Object[] {mutation.id()}); + Long actualVersion = current.isEmpty() ? null : ((Number) current.get(0).get("committed_version")).longValue(); + if (!Objects.equals(actualVersion, mutation.baseVersion())) { + throw new SqliteCommittedEventStore.CommittedEventException("VERSION_CONFLICT", + "committed-event predecessor version mismatch"); + } + List args = new ArrayList<>(); + if ("create".equals(mutation.operation())) { + List columns = new ArrayList<>(List.of(idColumn, versionColumn)); + args.add(mutation.id()); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + columns.add(quoted(field.column())); + args.add(field.value()); + } + sql.update("INSERT INTO " + table + " (" + String.join(", ", columns) + ") VALUES (" + + String.join(", ", java.util.Collections.nCopies(args.size(), "?")) + ")", + args.toArray()); + } else { + List assignments = new ArrayList<>(List.of(versionColumn + " = ?")); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + assignments.add(quoted(field.column()) + " = ?"); + args.add(field.value()); + } + args.add(mutation.id()); + args.add(mutation.baseVersion()); + int changed = sql.update("UPDATE " + table + " SET " + String.join(", ", assignments) + + " WHERE " + idColumn + " = ? AND " + versionColumn + " = ?", args.toArray()); + if (changed != 1) { + throw new SqliteCommittedEventStore.CommittedEventException("VERSION_CONFLICT", + "committed-event predecessor version mismatch"); + } + } + verifyReadback(sql, mutation); + } + + private static void verifyReadback(SqlExecutionAdapter sql, Mutation mutation) { + List projections = new ArrayList<>(List.of(quoted(mutation.versionColumn()) + " AS committed_version")); + for (int i = 0; i < mutation.fields().size(); i++) { + projections.add(quoted(mutation.fields().get(i).column()) + " AS field_" + i); + } + List> rows = sql.queryForList("SELECT " + String.join(", ", projections) + + " FROM " + quoted(mutation.table()) + " WHERE " + quoted(mutation.idColumn()) + " = ?", + new Object[] {mutation.id()}); + if (rows.size() != 1 || !Objects.equals(mutation.committedVersion(), + ((Number) rows.get(0).get("committed_version")).longValue())) { + throw invalid("target did not retain committed version"); + } + for (int i = 0; i < mutation.fields().size(); i++) { + if (!sameValue(mutation.fields().get(i), rows.get(0).get("field_" + i))) { + throw invalid("target did not retain committed field value"); + } + } + } + + private static boolean sameValue(Field field, Object actual) { + Object expected = field.value(); + if (expected instanceof byte[] bytes) return actual instanceof byte[] other && Arrays.equals(bytes, other); + if (expected == null || actual == null) return expected == actual; + if (field.decimal() || expected instanceof Number && actual instanceof Number) { + try { return new BigDecimal(expected.toString()).compareTo(new BigDecimal(actual.toString())) == 0; } + catch (NumberFormatException failure) { return false; } + } + return Objects.equals(expected, actual); + } + + private static Mapping mapping(PropertyDescriptor property) { + List columns = SQLPropertyUtil.getColumns(property); + if (columns.size() != 1) throw invalid("property has no single SQLite column"); + return new Mapping(columns.get(0).getTableName(), columns.get(0).getColumnName()); + } + + private static String quoted(String identifier) { + if (identifier == null || identifier.isBlank()) throw invalid("model has an empty SQL identifier"); + return "\"" + identifier.replace("\"", "\"\"") + "\""; + } + + private static long positiveId(JsonNode value) { + if (!value.isTextual()) throw invalid("identity must be a decimal string"); + try { + long id = Long.parseLong(value.textValue()); + if (id > 0) return id; + } catch (NumberFormatException ignored) { } + throw invalid("identity is outside the SQLite ID range"); + } + + private static long signedVersion(JsonNode value) { + if (!value.isTextual()) throw invalid("version must be a decimal string"); + try { return Long.parseLong(value.textValue()); } + catch (NumberFormatException failure) { throw invalid("version must be a decimal string"); } + } + + private static boolean nonblank(JsonNode value) { return value.isTextual() && !value.textValue().isBlank(); } + private static SqliteCommittedEventStore.CommittedEventException invalid(String reason) { + return new SqliteCommittedEventStore.CommittedEventException( + "INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } + + private record Mapping(String table, String column) { } + private record Field(String column, Object value, boolean decimal) { } + private record Mutation(String entityType, String table, String idColumn, + String versionColumn, long id, Long baseVersion, long committedVersion, + String operation, List fields) { } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java similarity index 96% rename from teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java rename to teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java index 67b12649..a0525620 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteCommittedEventStoreTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java @@ -1,4 +1,4 @@ -package io.teaql.sqlite; +package io.teaql.core.sqlite; import com.fasterxml.jackson.databind.JsonNode; import io.teaql.core.sqlite.SqliteCommittedEventStore; @@ -206,6 +206,7 @@ private static CommittedEventException expectCode(String code, Runnable action) assertEquals(1L, ((Number) target.driver().queryForList("SELECT version FROM order_items", new Object[0]).get(0).get("version")).longValue()); assertEquals(1, count(target.driver(), "teaql_event_inbox")); + assertEquals(1, count(target.driver(), "teaql_event_applied_payload")); Database reopened = database(path); reopened.driver().executeInTransaction(() -> { @@ -217,12 +218,18 @@ private static CommittedEventException expectCode(String code, Runnable action) }); assertTrue(duplicate.duplicate()); assertEquals(2, duplicate.checkpoint()); + var olderDuplicate = reopened.events().apply(first, MODEL_HASH, (sql, facts) -> { + throw new AssertionError("an older duplicate must not rematerialize"); + }); + assertTrue(olderDuplicate.duplicate()); + assertEquals(1, olderDuplicate.checkpoint()); }); assertEquals("SUBMITTED", reopened.driver().queryForList("SELECT status FROM orders", new Object[0]) .get(0).get("status")); assertEquals(-2L, ((Number) reopened.driver().queryForList( "SELECT version FROM order_items", new Object[0]).get(0).get("version")).longValue()); assertEquals(2, count(reopened.driver(), "teaql_event_inbox")); + assertEquals(2, count(reopened.driver(), "teaql_event_applied_payload")); assertEquals(2L, ((Number) reopened.driver().queryForList( "SELECT source_sequence FROM teaql_event_checkpoint", new Object[0]) .get(0).get("source_sequence")).longValue()); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java new file mode 100644 index 00000000..0f3456ad --- /dev/null +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -0,0 +1,222 @@ +package io.teaql.sqlite; + +import io.teaql.core.BaseEntity; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedEventStore.CommittedEventException; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.Map; + +import static org.junit.Assert.*; + +public class SqliteModelCheckedCommittedTargetTest { + private static final String SOURCE = "sqlite-order-source"; + private static final String HASH = "sha256:" + "0".repeat(64); + private static final long ORDER_ID = 9_007_199_254_740_993L; + private static final long ITEM_ID = ORDER_ID + 1; + + private record Target(JdbcSqlExecutor sql, SqliteModelCheckedCommittedTarget target, + DefaultUserContext context) { } + + private static Target target(Path path) { + SQLiteDataSource source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor sql = new JdbcSqlExecutor(source); + EntityDescriptor order = new EntityDescriptor(); + order.setType("order"); + order.addSimpleProperty("id", Long.class); + order.addSimpleProperty("version", Long.class); + order.addSimpleProperty("order_number", String.class); + order.addSimpleProperty("status", String.class); + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + metadata.register(order); + EntityDescriptor item = new EntityDescriptor(); + item.setType("order_item"); + item.addSimpleProperty("id", Long.class); + item.addSimpleProperty("version", Long.class); + item.addObjectProperty(metadata, "order", "order", "items", BaseEntity.class) + .with("columnName", "order_id"); + item.addSimpleProperty("quantity", Long.class); + metadata.register(item); + DefaultUserContext context = new DefaultUserContext( + TeaQLRuntime.builder().metadata(metadata).build()); + sql.execute("CREATE TABLE order_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + + "order_number TEXT NOT NULL, status TEXT NOT NULL)"); + sql.execute("CREATE TABLE order_item_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + + "order_id INTEGER NOT NULL, quantity INTEGER NOT NULL)"); + SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget(sql, SOURCE, HASH); + target.ensureSchema(); + return new Target(sql, target, context); + } + + private static String payload(long sequence, String change, String mutations) { + return """ + {"protocol_version":1,"change_set_id":"%s","source_runtime_id":"%s", + "source_sequence":"%d","source_model_hash":"%s","source_transaction_id":"tx-%d", + "committed_at":"2026-10-16T00:00:00Z", + "audit":{"actor_ref":"fixture:operator","comment":"Apply order graph", + "purpose":"Verify target application","trace_id":"fixture-trace"}, + "mutations":%s} + """.formatted(change, SOURCE, sequence, HASH, sequence, mutations); + } + + private static String created() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order_number":"ORD-0001","status":"DRAFT"}}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, + "quantity":2}}] + """; + } + + private static String submitted() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"update", + "base_version":"1","committed_version":"2","fields":{"status":"SUBMITTED"}}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"delete", + "base_version":"1","committed_version":"-2","fields":{}}] + """; + } + + private static OutboxRecord record(long sequence, String change, String mutations) { + return recordBody(sequence, change, payload(sequence, change, mutations)); + } + + private static OutboxRecord recordBody(long sequence, String change, String body) { + try { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(body.getBytes(StandardCharsets.UTF_8))); + return new OutboxRecord(SOURCE, sequence, change, HASH, digest, body); + } catch (java.security.NoSuchAlgorithmException impossible) { + throw new AssertionError("JDK SHA-256 is unavailable", impossible); + } + } + + private static void expect(String code, Runnable action) { + try { + action.run(); + fail("expected " + code); + } catch (CommittedEventException failure) { + assertEquals(code, failure.code()); + } + } + + private static long count(JdbcSqlExecutor sql, String table) { + return ((Number) sql.queryForList("SELECT COUNT(*) AS n FROM " + table, + new Object[0]).get(0).get("n")).longValue(); + } + + @Test public void appliesWholeOrderGraphAndReturnsStableDuplicateAcks() throws Exception { + Path path = Files.createTempFile("teaql-model-target-", ".sqlite"); + try { + Target t = target(path); + OutboxRecord first = record(1, "order-graph-0001", created()); + OutboxRecord second = record(2, "order-graph-0002", submitted()); + OutboxRecord otherSource = new OutboxRecord("other-source", first.sourceSequence(), + first.changeSetId(), first.modelHash(), first.payloadSha256(), first.payloadJson()); + expect("UNAUTHORIZED_SOURCE", () -> t.target().apply(t.context(), otherSource)); + OutboxRecord otherModel = new OutboxRecord(first.sourceRuntimeId(), first.sourceSequence(), + first.changeSetId(), "sha256:" + "1".repeat(64), first.payloadSha256(), first.payloadJson()); + expect("MODEL_MISMATCH", () -> t.target().apply(t.context(), otherModel)); + String unsupported = payload(1, "unsupported", created()) + .replace("\"protocol_version\":1", "\"protocol_version\":2"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + recordBody(1, "unsupported", unsupported))); + expect("SEQUENCE_GAP", () -> t.target().apply(t.context(), second)); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(1, t.target().apply(t.context(), first).checkpoint()); + assertTrue(t.target().apply(t.context(), first).duplicate()); + assertEquals(2, t.target().apply(t.context(), second).checkpoint()); + var oldAck = t.target().apply(t.context(), first); + assertTrue(oldAck.duplicate()); + assertEquals(1, oldAck.checkpoint()); + Map order = t.sql().queryForList( + "SELECT version, order_number, status FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(2L, ((Number) order.get("version")).longValue()); + assertEquals("ORD-0001", order.get("order_number")); + assertEquals("SUBMITTED", order.get("status")); + Map item = t.sql().queryForList( + "SELECT version, order_id, quantity FROM order_item_data WHERE id = ?", + new Object[] {ITEM_ID}).get(0); + assertEquals(-2L, ((Number) item.get("version")).longValue()); + assertEquals(ORDER_ID, ((Number) item.get("order_id")).longValue()); + assertEquals(2L, ((Number) item.get("quantity")).longValue()); + assertEquals(2, count(t.sql(), "teaql_event_inbox")); + assertEquals(2, count(t.sql(), "teaql_event_applied_payload")); + SQLiteDataSource reopenedSource = new SQLiteDataSource(); + reopenedSource.setUrl("jdbc:sqlite:" + path); + SqliteModelCheckedCommittedTarget reopened = new SqliteModelCheckedCommittedTarget( + new JdbcSqlExecutor(reopenedSource), SOURCE, HASH); + var retryAfterRestart = reopened.apply(t.context(), first); + assertTrue(retryAfterRestart.duplicate()); + assertEquals(1, retryAfterRestart.checkpoint()); + String retained = (String) t.sql().queryForList( + "SELECT payload_json FROM teaql_event_applied_payload WHERE source_sequence = 2", + new Object[0]).get(0).get("payload_json"); + assertTrue(retained.contains("fixture-trace")); + } finally { Files.deleteIfExists(path); } + } + + @Test public void rejectsUnknownModelAndRollsBackWhenSecondObjectFails() throws Exception { + Path path = Files.createTempFile("teaql-model-target-negative-", ".sqlite"); + try { + Target t = target(path); + OutboxRecord first = record(1, "order-graph-0001", created()); + String unknown = created().replace("\"quantity\":2", "\"quantity\":2,\"not_in_model\":1"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), record(1, "unknown", unknown))); + String reverseRelation = created().replace("\"status\":\"DRAFT\"", + "\"status\":\"DRAFT\",\"items\":[]"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "reverse-relation", reverseRelation))); + t.sql().execute("CREATE TRIGGER reject_item BEFORE INSERT ON order_item_data " + + "BEGIN SELECT RAISE(ABORT, 'reject item'); END"); + try { + t.target().apply(t.context(), first); + fail("expected second-object failure"); + } catch (RuntimeException expected) { + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "order_item_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + assertEquals(0, count(t.sql(), "teaql_event_applied_payload")); + assertEquals(0, count(t.sql(), "teaql_event_checkpoint")); + } + } finally { Files.deleteIfExists(path); } + } + + @Test public void rejectsTriggerRewriteAndStalePredecessor() throws Exception { + Path path = Files.createTempFile("teaql-model-target-trigger-", ".sqlite"); + try { + Target t = target(path); + OutboxRecord first = record(1, "order-graph-0001", created()); + t.sql().execute("CREATE TRIGGER rewrite_order AFTER INSERT ON order_data " + + "BEGIN UPDATE order_data SET status = 'TARGET_OVERRIDE' WHERE id = NEW.id; END"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), first)); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + t.sql().execute("DROP TRIGGER rewrite_order"); + t.target().apply(t.context(), first); + String stale = submitted().replaceFirst("\\\"base_version\\\":\\\"1\\\"", + "\"base_version\":\"9\"").replaceFirst("\\\"committed_version\\\":\\\"2\\\"", + "\"committed_version\":\"10\""); + expect("VERSION_CONFLICT", () -> t.target().apply(t.context(), record(2, "stale", stale))); + assertEquals(1, count(t.sql(), "teaql_event_inbox")); + } finally { Files.deleteIfExists(path); } + } +} From 409a8c00e2f6e0dbc9ea0b332988b061ef6e7e6f Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 02:42:20 +0800 Subject: [PATCH 05/29] Replay shared committed Order fixture in Java target test --- ...SqliteModelCheckedCommittedTargetTest.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index 0f3456ad..5149680b 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -1,5 +1,7 @@ package io.teaql.sqlite; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import io.teaql.core.BaseEntity; import io.teaql.core.meta.EntityDescriptor; import io.teaql.core.meta.SimpleEntityMetaFactory; @@ -11,6 +13,7 @@ import io.teaql.runtime.DefaultUserContext; import io.teaql.runtime.TeaQLRuntime; import org.junit.Test; +import org.junit.Assume; import org.sqlite.SQLiteDataSource; import java.nio.charset.StandardCharsets; @@ -122,6 +125,33 @@ private static long count(JdbcSqlExecutor sql, String table) { new Object[0]).get(0).get("n")).longValue(); } + @Test public void appliesSharedConformanceFixtureWhenConfigured() throws Exception { + String fixturePath = System.getenv("TEAQL_COMMITTED_EVENT_FIXTURE"); + Assume.assumeTrue("shared fixture path was not supplied", + fixturePath != null && !fixturePath.isBlank()); + JsonNode fixture = new ObjectMapper().readTree(Files.readString(Path.of(fixturePath))); + assertEquals(1, fixture.path("fixture_version").asInt()); + assertEquals(2, fixture.path("sets").size()); + Path path = Files.createTempFile("teaql-shared-model-target-", ".sqlite"); + try { + Target t = target(path); + for (int index = 0; index < 2; index++) { + JsonNode set = fixture.path("sets").get(index); + long sequence = Long.parseLong(set.path("source_sequence").asText()); + OutboxRecord record = recordBody(sequence, + set.path("change_set_id").asText(), set.toString()); + assertEquals(sequence, t.target().apply(t.context(), record).checkpoint()); + } + assertEquals("SUBMITTED", t.sql().queryForList( + "SELECT status FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("status")); + assertEquals(-2L, ((Number) t.sql().queryForList( + "SELECT version FROM order_item_data WHERE id = ?", new Object[] {ITEM_ID}) + .get(0).get("version")).longValue()); + assertEquals(2, count(t.sql(), "teaql_event_applied_payload")); + } finally { Files.deleteIfExists(path); } + } + @Test public void appliesWholeOrderGraphAndReturnsStableDuplicateAcks() throws Exception { Path path = Files.createTempFile("teaql-model-target-", ".sqlite"); try { From 18850e2811cd74e747582bd7a1edd099ba6a4fa8 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 02:51:03 +0800 Subject: [PATCH 06/29] Add durable ordered Java committed-event publication --- examples/committed-events/README.md | 15 ++- .../committed/CommittedEventsExampleTest.java | 8 +- .../sqlite/SqliteCommittedEventStore.java | 4 + .../SqliteCommittedOutboxPublisher.java | 118 ++++++++++++++++++ ...SqliteModelCheckedCommittedTargetTest.java | 64 ++++++++++ 5 files changed, 201 insertions(+), 8 deletions(-) create mode 100644 teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedOutboxPublisher.java diff --git a/examples/committed-events/README.md b/examples/committed-events/README.md index fce32b05..9debbca8 100644 --- a/examples/committed-events/README.md +++ b/examples/committed-events/README.md @@ -1,16 +1,19 @@ # SQLite committed-event source example Run `bash examples/committed-events/verify.sh`. It executes a small, real -`Order.save(context)` flow twice against local Java source. Each round uses a -fresh SQLite file, and each test reopens the Outbox before injecting a later -write failure. No generated ERP model or remote service is required. +`Order.save(context)` flow twice against a local Java source. Each round uses +fresh source and target SQLite files. The test reopens the Outbox, delivers +the two events through a bounded publisher into a model-checked target, +verifies the target rows and duplicate acknowledgement, then injects a later +source write failure. No generated ERP model or remote service is required. The example configures `SqliteCommittedMutationRecorder` at trusted runtime startup; `context.ensureSchema()` installs its infrastructure tables. A database trigger sets `memo` after insert; the Outbox contains that authoritative readback, not the pre-save entity value. Subsequent update produces sequence 2. When the Outbox table is made unavailable, the attempted third save rolls back its -business update and emits no committed audit event. +business update and emits no committed event. -This is **not** a completed replication example: it does not publish the row, -validate a target model, apply to another runtime, or cover dynamic fields. +This is **not** authenticated network replication. The in-process sink proves +local cursor/target behavior; transport identity, retry budgets, snapshot-plus- +tail, cross-language codec breadth and dynamic fields remain separate work. diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java index b1b54cc4..2c469969 100644 --- a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java @@ -9,6 +9,7 @@ import io.teaql.core.sqlite.SqliteCommittedEventStore; import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; import io.teaql.core.sqlite.SqliteDataServiceExecutor; import io.teaql.provider.jdbc.JdbcSqlExecutor; import io.teaql.runtime.DefaultUserContext; @@ -103,8 +104,11 @@ public static final class Order extends BaseEntity { SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget( replicaSql, "example-java-source", "sha256:" + "0".repeat(64)); target.ensureSchema(); - assertEquals(1, target.apply(replicaContext, durable.get(0)).checkpoint()); - assertEquals(2, target.apply(replicaContext, durable.get(1)).checkpoint()); + var published = new SqliteCommittedOutboxPublisher(reopened, "example-java-source", + "example-analytics", "sha256:" + "0".repeat(64)) + .publishBatch(10, record -> target.apply(replicaContext, record)); + assertEquals(2, published.acknowledged()); + assertEquals(0, published.status().backlog()); var duplicate = target.apply(replicaContext, durable.get(0)); assertTrue(duplicate.duplicate()); assertEquals(1, duplicate.checkpoint()); diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java index a644df67..dc555f19 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java @@ -82,6 +82,10 @@ public void ensureSchema() { + "source_runtime_id TEXT PRIMARY KEY, " + "source_sequence INTEGER NOT NULL CHECK(source_sequence >= 0), " + "model_hash TEXT NOT NULL)"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_delivery_cursor (" + + "source_runtime_id TEXT NOT NULL, target_id TEXT NOT NULL, " + + "acked_sequence INTEGER NOT NULL CHECK(acked_sequence >= 0), " + + "model_hash TEXT NOT NULL, PRIMARY KEY(source_runtime_id, target_id))"); } /** Bounded publisher read of one durable, ordered source stream. */ diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedOutboxPublisher.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedOutboxPublisher.java new file mode 100644 index 00000000..82b18a64 --- /dev/null +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedOutboxPublisher.java @@ -0,0 +1,118 @@ +package io.teaql.core.sqlite; + +import io.teaql.core.sqlite.SqliteCommittedEventStore.CommittedEventException; +import io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome; +import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Bounded ordered publication from one durable SQLite Outbox stream. The + * supplied sink must own its authentication and network retry budget; this + * publisher never treats a callback as proof of remote identity. + */ +public final class SqliteCommittedOutboxPublisher { + @FunctionalInterface + public interface DeliverySink { + InboxOutcome deliver(OutboxRecord record); + } + + public record DeliveryStatus(long ackedSequence, long sourceHighWatermark, long backlog) { } + public record BatchOutcome(int acknowledged, DeliveryStatus status) { } + + private final SqlExecutionAdapter sql; + private final SqliteCommittedEventStore events; + private final String sourceRuntimeId; + private final String targetId; + private final String modelHash; + + public SqliteCommittedOutboxPublisher(SqlExecutionAdapter sql, String sourceRuntimeId, + String targetId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (sourceRuntimeId == null || sourceRuntimeId.isBlank() + || targetId == null || targetId.isBlank() + || modelHash == null || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw failure("INVALID_ENVELOPE", "delivery source, target or model identity is invalid"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.targetId = targetId; + this.modelHash = modelHash; + this.events = new SqliteCommittedEventStore(sql); + } + + public DeliveryStatus status() { + List> rows = sql.queryForList( + "SELECT acked_sequence, model_hash FROM teaql_event_delivery_cursor " + + "WHERE source_runtime_id = ? AND target_id = ?", + new Object[] {sourceRuntimeId, targetId}); + long acked = 0; + if (!rows.isEmpty()) { + if (!modelHash.equals(rows.get(0).get("model_hash"))) { + throw failure("MODEL_MISMATCH", "delivery cursor model hash mismatch"); + } + acked = number(rows.get(0).get("acked_sequence")); + } + List> sourceRows = sql.queryForList( + "SELECT current_sequence FROM teaql_event_sequence WHERE source_runtime_id = ?", + new Object[] {sourceRuntimeId}); + long highWatermark = sourceRows.isEmpty() ? 0 + : number(sourceRows.get(0).get("current_sequence")); + if (highWatermark < acked) { + throw failure("INVALID_ENVELOPE", "delivery cursor is ahead of source sequence"); + } + return new DeliveryStatus(acked, highWatermark, highWatermark - acked); + } + + /** + * One attempt per event, at most {@code limit} events. The source cursor + * advances only after an acknowledgement for the exact sequence. If the + * target applied an event but the response is lost, a later call/restart + * re-delivers it and the target Inbox supplies a duplicate acknowledgement. + */ + public BatchOutcome publishBatch(int limit, DeliverySink sink) { + Objects.requireNonNull(sink, "sink"); + DeliveryStatus before = status(); + List records = events.outboxAfter(sourceRuntimeId, before.ackedSequence(), limit); + if (records.isEmpty() && before.backlog() > 0) { + throw failure("SEQUENCE_GAP", "source Outbox has an unacknowledged gap"); + } + long acked = before.ackedSequence(); + int count = 0; + for (OutboxRecord record : records) { + if (acked == Long.MAX_VALUE) { + throw failure("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + } + if (record.sourceSequence() != acked + 1) { + throw failure("SEQUENCE_GAP", "source Outbox sequence is not contiguous"); + } + if (!modelHash.equals(record.modelHash())) { + throw failure("MODEL_MISMATCH", "source Outbox model hash mismatch"); + } + InboxOutcome outcome = sink.deliver(record); + if (outcome == null || outcome.checkpoint() != record.sourceSequence()) { + throw failure("ACK_MISMATCH", "target acknowledgement does not match delivered sequence"); + } + int changed = sql.update("INSERT INTO teaql_event_delivery_cursor " + + "(source_runtime_id, target_id, acked_sequence, model_hash) VALUES (?, ?, ?, ?) " + + "ON CONFLICT(source_runtime_id, target_id) DO UPDATE SET " + + "acked_sequence = excluded.acked_sequence " + + "WHERE teaql_event_delivery_cursor.acked_sequence = ? " + + "AND teaql_event_delivery_cursor.model_hash = excluded.model_hash", + new Object[] {sourceRuntimeId, targetId, record.sourceSequence(), modelHash, acked}); + if (changed != 1) { + throw failure("CURSOR_CONFLICT", "delivery cursor changed concurrently"); + } + acked = record.sourceSequence(); + count++; + } + return new BatchOutcome(count, status()); + } + + private static long number(Object value) { return ((Number) value).longValue(); } + private static CommittedEventException failure(String code, String message) { + return new CommittedEventException(code, "committed-event " + message); + } +} diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index 5149680b..b60b4b3e 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -9,6 +9,7 @@ import io.teaql.core.sqlite.SqliteCommittedEventStore.CommittedEventException; import io.teaql.core.sqlite.SqliteCommittedEventStore.OutboxRecord; import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; import io.teaql.provider.jdbc.JdbcSqlExecutor; import io.teaql.runtime.DefaultUserContext; import io.teaql.runtime.TeaQLRuntime; @@ -249,4 +250,67 @@ private static long count(JdbcSqlExecutor sql, String table) { assertEquals(1, count(t.sql(), "teaql_event_inbox")); } finally { Files.deleteIfExists(path); } } + + @Test public void orderedPublisherRecoversAmbiguousAckAfterSourceReopen() throws Exception { + Path sourcePath = Files.createTempFile("teaql-source-delivery-", ".sqlite"); + Path targetPath = Files.createTempFile("teaql-target-delivery-", ".sqlite"); + try { + SQLiteDataSource sourceDataSource = new SQLiteDataSource(); + sourceDataSource.setUrl("jdbc:sqlite:" + sourcePath); + JdbcSqlExecutor sourceSql = new JdbcSqlExecutor(sourceDataSource); + SqliteCommittedEventStore sourceEvents = new SqliteCommittedEventStore(sourceSql); + sourceEvents.ensureSchema(); + for (String change : new String[] {"order-graph-0001", "order-graph-0002"}) { + String mutations = change.endsWith("0001") ? created() : submitted(); + sourceSql.executeInTransaction(() -> sourceEvents.append(SOURCE, change, HASH, + sequence -> record(sequence, change, mutations).payloadJson())); + } + Target target = target(targetPath); + SqliteCommittedOutboxPublisher publisher = new SqliteCommittedOutboxPublisher( + sourceSql, SOURCE, "analytics", HASH); + assertEquals(2, publisher.status().backlog()); + expect("ACK_MISMATCH", () -> publisher.publishBatch(2, + record -> SqliteCommittedEventStore.InboxOutcome.applied(99))); + assertEquals(0, publisher.status().ackedSequence()); + try { + publisher.publishBatch(2, record -> { + target.target().apply(target.context(), record); + throw new IllegalStateException("acknowledgement lost after target commit"); + }); + fail("expected ambiguous acknowledgement"); + } catch (IllegalStateException expected) { + assertEquals("acknowledgement lost after target commit", expected.getMessage()); + } + assertEquals(0, publisher.status().ackedSequence()); + assertEquals(1, count(target.sql(), "teaql_event_inbox")); + + SqliteCommittedOutboxPublisher restarted = new SqliteCommittedOutboxPublisher( + new JdbcSqlExecutor(sourceDataSource), SOURCE, "analytics", HASH); + var outcome = restarted.publishBatch(2, + record -> target.target().apply(target.context(), record)); + assertEquals(2, outcome.acknowledged()); + assertEquals(2, outcome.status().ackedSequence()); + assertEquals(0, outcome.status().backlog()); + assertEquals(0, restarted.publishBatch(2, record -> { + throw new AssertionError("acked event must not be delivered again"); + }).acknowledged()); + assertEquals(2, count(target.sql(), "teaql_event_inbox")); + assertEquals("SUBMITTED", target.sql().queryForList( + "SELECT status FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("status")); + assertEquals(2, new SqliteCommittedOutboxPublisher(sourceSql, SOURCE, + "second-target", HASH).status().backlog()); + expect("MODEL_MISMATCH", () -> new SqliteCommittedOutboxPublisher(sourceSql, + SOURCE, "analytics", "sha256:" + "1".repeat(64)).status()); + sourceSql.update("DELETE FROM teaql_event_outbox WHERE source_runtime_id = ? " + + "AND source_sequence = 1", new Object[] {SOURCE}); + expect("SEQUENCE_GAP", () -> new SqliteCommittedOutboxPublisher(sourceSql, + SOURCE, "second-target", HASH).publishBatch(2, record -> { + throw new AssertionError("gap must be rejected before delivery"); + })); + } finally { + Files.deleteIfExists(sourcePath); + Files.deleteIfExists(targetPath); + } + } } From b03a35d1c4297848725458ca6dfe3470e6ce99b4 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 06:51:04 +0800 Subject: [PATCH 07/29] Prove DuckDB JDBC adapter rolls back replica state atomically --- .../teaql/duckdb/DuckDbIntegrationTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java b/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java index 00fae153..c0da7f26 100644 --- a/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java +++ b/teaql-duckdb/src/test/java/io/teaql/duckdb/DuckDbIntegrationTest.java @@ -276,6 +276,48 @@ public void supportsFacetWindowAndTransactionSql() throws Exception { } } + @Test + public void jdbcAdapterRollsBackDomainInboxAndCheckpointTogether() { + sql.execute("CREATE TABLE cmr_atomic_data (id BIGINT PRIMARY KEY, amount DECIMAL(19,7))"); + sql.execute("CREATE TABLE cmr_atomic_inbox (change_set_id VARCHAR PRIMARY KEY)"); + sql.execute("CREATE TABLE cmr_atomic_checkpoint (source_id VARCHAR PRIMARY KEY, sequence BIGINT)"); + + boolean failed = false; + try { + sql.executeInTransaction(() -> { + sql.update("INSERT INTO cmr_atomic_data VALUES (?, ?)", + new Object[] {1L, new BigDecimal("0.1000000")}); + sql.update("INSERT INTO cmr_atomic_inbox VALUES (?)", new Object[] {"set-1"}); + sql.update("INSERT INTO cmr_atomic_checkpoint VALUES (?, ?)", + new Object[] {"source-1", 1L}); + sql.update("INSERT INTO cmr_atomic_data VALUES (?, ?)", + new Object[] {1L, BigDecimal.ONE}); + }); + } catch (RuntimeException expected) { + failed = true; + } + assertTrue("duplicate row must fail the transaction", failed); + assertEquals(0L, rowCount("cmr_atomic_data")); + assertEquals(0L, rowCount("cmr_atomic_inbox")); + assertEquals(0L, rowCount("cmr_atomic_checkpoint")); + + sql.executeInTransaction(() -> { + sql.update("INSERT INTO cmr_atomic_data VALUES (?, ?)", + new Object[] {1L, new BigDecimal("0.1000000")}); + sql.update("INSERT INTO cmr_atomic_inbox VALUES (?)", new Object[] {"set-1"}); + sql.update("INSERT INTO cmr_atomic_checkpoint VALUES (?, ?)", + new Object[] {"source-1", 1L}); + }); + assertEquals(1L, rowCount("cmr_atomic_data")); + assertEquals(1L, rowCount("cmr_atomic_inbox")); + assertEquals(1L, rowCount("cmr_atomic_checkpoint")); + } + + private static long rowCount(String table) { + return ((Number) sql.queryForList("SELECT count(*) AS row_count FROM " + table, + new Object[0]).get(0).get("row_count")).longValue(); + } + @Test public void validatesSchemaValueDomainAndNullability() throws Exception { try { From e09249f9fd7329727e2f4448fb77b6bdb4318596 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 06:58:05 +0800 Subject: [PATCH 08/29] Add transaction-owned DuckDB committed Inbox core --- teaql-duckdb/pom.xml | 4 + .../teaql/core/duck/DuckDbCommittedInbox.java | 214 ++++++++++++++++++ teaql-duckdb/src/main/java/module-info.java | 1 + .../core/duck/DuckDbCommittedInboxTest.java | 167 ++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java create mode 100644 teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbCommittedInboxTest.java diff --git a/teaql-duckdb/pom.xml b/teaql-duckdb/pom.xml index 8b666852..5148dc24 100644 --- a/teaql-duckdb/pom.xml +++ b/teaql-duckdb/pom.xml @@ -23,6 +23,10 @@ io.teaql teaql-utils + + com.fasterxml.jackson.core + jackson-databind + org.duckdb duckdb_jdbc diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java new file mode 100644 index 00000000..22119130 --- /dev/null +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java @@ -0,0 +1,214 @@ +package io.teaql.core.duck; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Transaction-owned target Inbox for already committed model facts. The + * projection writer is trusted infrastructure, never a remote SQL callback. + * DuckDB has no SAVEPOINT in the qualified versions, so apply owns the entire + * transaction and refuses to run inside an ambient one. + */ +public final class DuckDbCommittedInbox { + private static final ObjectMapper JSON = new ObjectMapper(); + private final SqlExecutionAdapter sql; + private final String sourceRuntimeId; + private final String modelHash; + + public record Record(String sourceRuntimeId, long sourceSequence, String changeSetId, + String modelHash, String payloadSha256, String payloadJson) { + @Override public String toString() { + return "Record[sourceRuntimeId=" + sourceRuntimeId + ", sourceSequence=" + + sourceSequence + ", payload=]"; + } + } + + public record Outcome(boolean duplicate, long checkpoint) { } + + public static final class Failure extends RuntimeException { + private final String code; + + private Failure(String code, String message) { + super(message); + this.code = code; + } + + public String code() { return code; } + } + + @FunctionalInterface + interface TrustedProjectionWriter { + void apply(SqlExecutionAdapter transaction, JsonNode committedFacts); + } + + public DuckDbCommittedInbox(SqlExecutionAdapter sql, String sourceRuntimeId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + if (blank(sourceRuntimeId) || modelHash == null + || !modelHash.matches("sha256:[0-9a-f]{64}")) { + throw invalid("target source identity or model hash is invalid"); + } + this.sourceRuntimeId = sourceRuntimeId; + this.modelHash = modelHash; + } + + public void ensureSchema() { + if (sql.hasActiveTransaction()) throw invalid("schema setup requires no active transaction"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_inbox (" + + "source_runtime_id VARCHAR NOT NULL, change_set_id VARCHAR NOT NULL, " + + "source_sequence BIGINT NOT NULL CHECK(source_sequence > 0), " + + "payload_sha256 VARCHAR NOT NULL, PRIMARY KEY(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_applied_payload (" + + "source_runtime_id VARCHAR NOT NULL, source_sequence BIGINT NOT NULL " + + "CHECK(source_sequence > 0), change_set_id VARCHAR NOT NULL, " + + "payload_json VARCHAR NOT NULL, PRIMARY KEY(source_runtime_id, source_sequence), " + + "UNIQUE(source_runtime_id, change_set_id))"); + sql.execute("CREATE TABLE IF NOT EXISTS teaql_event_checkpoint (" + + "source_runtime_id VARCHAR PRIMARY KEY, source_sequence BIGINT NOT NULL " + + "CHECK(source_sequence >= 0), model_hash VARCHAR NOT NULL)"); + } + + Outcome apply(Record record, TrustedProjectionWriter writer) { + if (sql.hasActiveTransaction()) { + throw new Failure("AMBIENT_TRANSACTION", "DuckDB committed apply requires its own transaction"); + } + if (record == null || writer == null) throw invalid("missing record or projection writer"); + if (!sourceRuntimeId.equals(record.sourceRuntimeId())) { + throw new Failure("UNAUTHORIZED_SOURCE", "committed-event source is not authorized"); + } + if (!modelHash.equals(record.modelHash())) { + throw new Failure("MODEL_MISMATCH", "committed-event model hash mismatch"); + } + if (record.sourceSequence() < 1 || blank(record.changeSetId()) + || record.payloadSha256() == null + || !record.payloadSha256().matches("[0-9a-f]{64}")) { + throw invalid("record identity or digest is invalid"); + } + JsonNode payload = parse(record.payloadJson()); + validateHeader(record, payload); + if (!digest(record.payloadJson()).equals(record.payloadSha256())) { + throw invalid("payload digest mismatch"); + } + AtomicReference outcome = new AtomicReference<>(); + try { + sql.executeInTransaction(() -> outcome.set(applyInside(record, payload, writer))); + } catch (Failure failure) { + throw failure; + } catch (RuntimeException failure) { + // JDBC exceptions may include raw SQL and bound values. The + // caller receives a stable category, never provider diagnostics. + throw new Failure("TARGET_FAILURE", "committed target transaction failed"); + } + return outcome.get(); + } + + private Outcome applyInside(Record record, JsonNode payload, TrustedProjectionWriter writer) { + if (!sql.hasActiveTransaction()) { + throw new Failure("TRANSACTION_REQUIRED", "committed target transaction is unavailable"); + } + List> checkpoints = sql.queryForList( + "SELECT source_sequence, model_hash FROM teaql_event_checkpoint " + + "WHERE source_runtime_id = ?", new Object[] {sourceRuntimeId}); + long last = checkpoints.isEmpty() ? 0 : number(checkpoints.get(0).get("source_sequence")); + if (!checkpoints.isEmpty() && !modelHash.equals(checkpoints.get(0).get("model_hash"))) { + throw new Failure("MODEL_MISMATCH", "committed-event checkpoint model mismatch"); + } + List> prior = sql.queryForList( + "SELECT source_sequence, payload_sha256 FROM teaql_event_inbox " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {sourceRuntimeId, record.changeSetId()}); + if (!prior.isEmpty()) { + Map seen = prior.get(0); + List> retained = sql.queryForList( + "SELECT source_sequence, payload_json FROM teaql_event_applied_payload " + + "WHERE source_runtime_id = ? AND change_set_id = ?", + new Object[] {sourceRuntimeId, record.changeSetId()}); + if (number(seen.get("source_sequence")) == record.sourceSequence() + && record.payloadSha256().equals(seen.get("payload_sha256")) + && retained.size() == 1 + && number(retained.get(0).get("source_sequence")) == record.sourceSequence() + && record.payloadSha256().equals(digest((String) retained.get(0).get("payload_json")))) { + return new Outcome(true, record.sourceSequence()); + } + throw new Failure("DUPLICATE_CONFLICT", "committed-event identity has conflicting content"); + } + if (last == Long.MAX_VALUE) { + throw new Failure("SEQUENCE_EXHAUSTED", "source sequence exhausted"); + } + if (record.sourceSequence() != last + 1) { + throw new Failure("SEQUENCE_GAP", "committed-event source sequence is not contiguous"); + } + writer.apply(sql, payload); + one(sql.update("INSERT INTO teaql_event_applied_payload " + + "(source_runtime_id, source_sequence, change_set_id, payload_json) " + + "VALUES (?, ?, ?, ?)", + new Object[] {sourceRuntimeId, record.sourceSequence(), + record.changeSetId(), record.payloadJson()})); + one(sql.update("INSERT INTO teaql_event_inbox " + + "(source_runtime_id, change_set_id, source_sequence, payload_sha256) " + + "VALUES (?, ?, ?, ?)", + new Object[] {sourceRuntimeId, record.changeSetId(), + record.sourceSequence(), record.payloadSha256()})); + one(sql.update("INSERT INTO teaql_event_checkpoint " + + "(source_runtime_id, source_sequence, model_hash) VALUES (?, ?, ?) " + + "ON CONFLICT(source_runtime_id) DO UPDATE SET " + + "source_sequence = excluded.source_sequence, model_hash = excluded.model_hash", + new Object[] {sourceRuntimeId, record.sourceSequence(), modelHash})); + return new Outcome(false, record.sourceSequence()); + } + + private static void validateHeader(Record record, JsonNode payload) { + if (!payload.isObject() || !payload.path("protocol_version").isInt() + || payload.path("protocol_version").intValue() != 1 + || !payload.path("source_runtime_id").isTextual() + || !record.sourceRuntimeId().equals(payload.path("source_runtime_id").textValue()) + || !payload.path("source_sequence").isTextual() + || !Long.toString(record.sourceSequence()).equals(payload.path("source_sequence").textValue()) + || !payload.path("change_set_id").isTextual() + || !record.changeSetId().equals(payload.path("change_set_id").textValue()) + || !payload.path("source_model_hash").isTextual() + || !record.modelHash().equals(payload.path("source_model_hash").textValue()) + || !payload.path("mutations").isArray() + || payload.path("mutations").isEmpty()) { + throw invalid("required v1 header or mutations mismatch"); + } + } + + private static JsonNode parse(String json) { + if (json == null) throw invalid("missing payload"); + try { + JsonNode payload = JSON.readTree(json); + if (payload == null) throw invalid("payload JSON is empty"); + return payload; + } + catch (Exception failure) { throw invalid("payload JSON is invalid"); } + } + + private static String digest(String payload) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException impossible) { + throw new AssertionError("JDK SHA-256 is unavailable", impossible); + } + } + + private static void one(int changed) { + if (changed != 1) throw new Failure("TARGET_FAILURE", "committed target write count mismatch"); + } + + private static long number(Object value) { return ((Number) value).longValue(); } + private static boolean blank(String value) { return value == null || value.isBlank(); } + private static Failure invalid(String reason) { + return new Failure("INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } +} diff --git a/teaql-duckdb/src/main/java/module-info.java b/teaql-duckdb/src/main/java/module-info.java index 17270eeb..7ff87a4a 100644 --- a/teaql-duckdb/src/main/java/module-info.java +++ b/teaql-duckdb/src/main/java/module-info.java @@ -2,6 +2,7 @@ requires transitive io.teaql.core; requires transitive io.teaql.dataservice.sql; requires transitive io.teaql.utils; + requires com.fasterxml.jackson.databind; requires java.sql; exports io.teaql.core.duck; } diff --git a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbCommittedInboxTest.java b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbCommittedInboxTest.java new file mode 100644 index 00000000..59d7dddd --- /dev/null +++ b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbCommittedInboxTest.java @@ -0,0 +1,167 @@ +package io.teaql.core.duck; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import io.teaql.core.duck.DuckDbCommittedInbox.Failure; +import io.teaql.core.duck.DuckDbCommittedInbox.Record; +import io.teaql.provider.jdbc.JdbcSqlExecutor; + +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.HexFormat; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class DuckDbCommittedInboxTest { + private static final String SOURCE = "duckdb-source"; + private static final String MODEL_HASH = "sha256:" + "a".repeat(64); + private Path directory; + private Path database; + private JdbcSqlExecutor sql; + private DuckDbCommittedInbox inbox; + + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { + return DriverManager.getConnection(url); + } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class iface) throws SQLException { + throw new SQLException("not a wrapper"); + } + @Override public boolean isWrapperFor(Class iface) { return false; } + } + + @Before public void setUp() throws Exception { + directory = Files.createTempDirectory("teaql-duckdb-inbox-"); + database = directory.resolve("target.db"); + sql = new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)); + inbox = new DuckDbCommittedInbox(sql, SOURCE, MODEL_HASH); + inbox.ensureSchema(); + sql.execute("CREATE TABLE replica_order (id BIGINT PRIMARY KEY, version BIGINT NOT NULL)"); + } + + @After public void tearDown() throws Exception { + Files.deleteIfExists(database); + Files.deleteIfExists(directory); + } + + @Test public void appliesOnceAndRetainsReplayEvidence() { + Record first = record(1, "set-1"); + AtomicInteger calls = new AtomicInteger(); + DuckDbCommittedInbox.TrustedProjectionWriter writer = (transaction, ignored) -> { + calls.incrementAndGet(); + transaction.update("INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L}); + }; + assertEquals(new DuckDbCommittedInbox.Outcome(false, 1), inbox.apply(first, writer)); + assertEquals(new DuckDbCommittedInbox.Outcome(true, 1), inbox.apply(first, writer)); + assertEquals(1, calls.get()); + assertEquals(1, count("replica_order")); + assertEquals(1, count("teaql_event_inbox")); + assertEquals(1, count("teaql_event_applied_payload")); + assertEquals(1, count("teaql_event_checkpoint")); + assertEquals(1L, checkpoint()); + + DuckDbCommittedInbox reopened = new DuckDbCommittedInbox( + new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)), + SOURCE, MODEL_HASH); + assertTrue(reopened.apply(first, (transaction, ignored) -> { + throw new AssertionError("duplicate reran projection"); + }).duplicate()); + } + + @Test public void projectionFailureRollsBackAllStateAndCanRetry() { + Record first = record(1, "set-1"); + Failure failure = assertThrows(Failure.class, () -> inbox.apply(first, (transaction, ignored) -> { + transaction.update("INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L}); + throw new IllegalStateException("private-sql-canary"); + })); + assertEquals("TARGET_FAILURE", failure.code()); + assertFalse(failure.getMessage().contains("private-sql-canary")); + for (String table : new String[] {"replica_order", "teaql_event_inbox", + "teaql_event_applied_payload", "teaql_event_checkpoint"}) { + assertEquals(table, 0, count(table)); + } + assertFalse(inbox.apply(first, (transaction, ignored) -> transaction.update( + "INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L})).duplicate()); + assertEquals(1, count("replica_order")); + } + + @Test public void gapTamperAndAmbientTransactionFailClosed() { + assertCode("SEQUENCE_GAP", () -> inbox.apply(record(2, "set-2"), + (transaction, ignored) -> { })); + assertCode("INVALID_ENVELOPE", () -> inbox.apply( + new Record(SOURCE, 1, "set-1", MODEL_HASH, "0".repeat(64), record(1, "set-1").payloadJson()), + (transaction, ignored) -> { })); + assertCode("UNAUTHORIZED_SOURCE", () -> inbox.apply( + new Record("other-source", 1, "set-1", MODEL_HASH, + record(1, "set-1").payloadSha256(), record(1, "set-1").payloadJson()), + (transaction, ignored) -> { })); + sql.executeInTransaction(() -> assertCode("AMBIENT_TRANSACTION", + () -> inbox.apply(record(1, "set-1"), (transaction, ignored) -> { }))); + assertEquals(0, count("teaql_event_inbox")); + + Record first = record(1, "set-1"); + inbox.apply(first, (transaction, ignored) -> transaction.update( + "INSERT INTO replica_order VALUES (?, ?)", new Object[] {7L, 1L})); + Record changed = record(1, "set-1", "update"); + assertCode("DUPLICATE_CONFLICT", () -> inbox.apply(changed, + (transaction, ignored) -> { })); + assertEquals(1L, checkpoint()); + } + + private long checkpoint() { + return ((Number) sql.queryForList("SELECT source_sequence FROM teaql_event_checkpoint", + new Object[0]).get(0).get("source_sequence")).longValue(); + } + + private int count(String table) { + return ((Number) sql.queryForList("SELECT count(*) AS row_count FROM " + table, + new Object[0]).get(0).get("row_count")).intValue(); + } + + private static Record record(long sequence, String changeSetId) { + return record(sequence, changeSetId, "create"); + } + + private static Record record(long sequence, String changeSetId, String operation) { + String payload = "{\"protocol_version\":1,\"source_runtime_id\":\"" + SOURCE + + "\",\"source_sequence\":\"" + sequence + + "\",\"change_set_id\":\"" + changeSetId + + "\",\"source_model_hash\":\"" + MODEL_HASH + + "\",\"mutations\":[{\"operation\":\"" + operation + "\"}]}"; + try { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8))); + return new Record(SOURCE, sequence, changeSetId, MODEL_HASH, digest, payload); + } catch (Exception impossible) { + throw new AssertionError(impossible); + } + } + + private static void assertCode(String code, Runnable action) { + assertEquals(code, assertThrows(Failure.class, action::run).code()); + } +} From 842323ed7810eb73f09f4bc37ab1f8579fc72795 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 07:07:25 +0800 Subject: [PATCH 09/29] feat(duckdb): apply committed graph facts through model-checked target --- .../teaql/core/duck/DuckDbCommittedInbox.java | 2 +- .../DuckDbModelCheckedCommittedTarget.java | 485 ++++++++++++++++++ ...DuckDbModelCheckedCommittedTargetTest.java | 259 ++++++++++ 3 files changed, 745 insertions(+), 1 deletion(-) create mode 100644 teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTarget.java create mode 100644 teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java index 22119130..042147c4 100644 --- a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbCommittedInbox.java @@ -38,7 +38,7 @@ public record Outcome(boolean duplicate, long checkpoint) { } public static final class Failure extends RuntimeException { private final String code; - private Failure(String code, String message) { + Failure(String code, String message) { super(message); this.code = code; } diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTarget.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTarget.java new file mode 100644 index 00000000..81ea85e6 --- /dev/null +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTarget.java @@ -0,0 +1,485 @@ +package io.teaql.core.duck; + +import com.fasterxml.jackson.databind.JsonNode; +import io.teaql.core.UserContext; +import io.teaql.core.duck.DuckDbCommittedInbox.Failure; +import io.teaql.core.duck.DuckDbCommittedInbox.Record; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; +import io.teaql.core.meta.PropertyDescriptor; +import io.teaql.core.meta.Relation; +import io.teaql.core.sql.SQLColumn; +import io.teaql.core.sql.portable.SQLPropertyUtil; +import io.teaql.dataservice.sql.SqlExecutionAdapter; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Model-checked, one-source DuckDB target for committed v1 graph facts. + * It writes only declared model columns and verifies the physical result + * before the Inbox transaction is acknowledged. + */ +public final class DuckDbModelCheckedCommittedTarget { + private static final Set TRACE_KINDS = Set.of( + "OPERATION", "REQUEST", "RELATION", "ENTITY", "PROVIDER", "SQL", + "COMMENT", "PURPOSE", "AUDIT_REASON"); + private static final Pattern DECLARED_DECIMAL = Pattern.compile( + "(?i)\\s*(?:NUMERIC|DECIMAL)\\s*\\(\\s*([0-9]+)\\s*,\\s*([0-9]+)\\s*\\)\\s*"); + private final SqlExecutionAdapter sql; + private final DuckDbCommittedInbox inbox; + + private enum Kind { SCALAR, DECIMAL, DATE, INSTANT, BYTES } + private record Mapping(String table, String column, String declaredType) { } + private record Field(String column, Object value, Kind kind) { + String bindExpression() { return kind == Kind.INSTANT ? "make_timestamptz(? * 1000)" : "?"; } + String readExpression() { + String name = quoted(column); + return switch (kind) { + case DATE -> "CAST(" + name + " AS VARCHAR)"; + case INSTANT -> "epoch_ms(" + name + ")"; + default -> name; + }; + } + } + private record Mutation(String entityType, String table, String idColumn, + String versionColumn, long id, Long baseVersion, long committedVersion, + String operation, List fields) { } + + public DuckDbModelCheckedCommittedTarget(SqlExecutionAdapter sql, + String sourceRuntimeId, String modelHash) { + this.sql = Objects.requireNonNull(sql, "sql"); + this.inbox = new DuckDbCommittedInbox(sql, sourceRuntimeId, modelHash); + } + + /** Infrastructure schema only; domain schema belongs to Context.ensureSchema(). */ + public void ensureInboxSchema() { inbox.ensureSchema(); } + + public DuckDbCommittedInbox.Outcome apply(UserContext context, Record record) { + Objects.requireNonNull(context, "context"); + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + return inbox.apply(record, (transaction, payload) -> applyGraph(transaction, metadata, payload)); + } + + private static void applyGraph(SqlExecutionAdapter sql, EntityMetaFactory metadata, JsonNode payload) { + JsonNode audit = payload.path("audit"); + if (!audit.isObject() || !nonblank(audit.path("comment")) + || !nonblank(audit.path("purpose")) + || !nonblank(payload.path("source_transaction_id"))) { + throw invalid("source transaction or audit intent is missing"); + } + try { OffsetDateTime.parse(payload.path("committed_at").asText()); } + catch (RuntimeException failure) { throw invalid("commit time is invalid"); } + List mutations = new ArrayList<>(); + Set seen = new HashSet<>(); + for (JsonNode fact : payload.path("mutations")) { + Mutation mutation = decode(sql, metadata, fact); + if (!seen.add(mutation.entityType() + ":" + mutation.id())) { + throw invalid("one set mutates the same entity twice"); + } + mutations.add(mutation); + } + // Every fact is validated before the first domain write. A subsequent + // SQL or readback error escapes to the Inbox owner for whole rollback. + for (Mutation mutation : mutations) applyOne(sql, mutation); + } + + private static Mutation decode(SqlExecutionAdapter sql, EntityMetaFactory metadata, JsonNode fact) { + if (!fact.isObject() || !nonblank(fact.path("entity_type"))) { + throw invalid("mutation entity type is missing"); + } + validateTraceChain(fact.path("trace_chain")); + String entityType = fact.path("entity_type").textValue(); + EntityDescriptor entity = metadata.allEntityDescriptors().stream() + .filter(candidate -> entityType.equals(candidate.getType())).findFirst() + .orElseThrow(() -> invalid("unknown entity type")); + if (entity.getParent() != null || entity.hasChildren()) { + throw invalid("multi-table entity target mapping is not supported"); + } + PropertyDescriptor idProperty = entity.findIdProperty(); + PropertyDescriptor versionProperty = entity.findVersionProperty(); + if (idProperty == null || versionProperty == null) { + throw invalid("entity has no ID or version property"); + } + Mapping idMapping = mapping(idProperty); + Mapping versionMapping = mapping(versionProperty); + if (!idMapping.table().equals(versionMapping.table())) { + throw invalid("identity and version span multiple tables"); + } + requirePhysical(sql, idMapping, "BIGINT", null, null); + requirePhysical(sql, versionMapping, "BIGINT", null, null); + long id = positiveId(fact.path("entity_id")); + String operation = fact.path("operation").asText(""); + if (!List.of("create", "update", "delete", "recover").contains(operation)) { + throw invalid("unknown mutation operation"); + } + JsonNode baseNode = fact.path("base_version"); + Long base = "create".equals(operation) && baseNode.isNull() + ? null : signedVersion(baseNode); + long committed = signedVersion(fact.path("committed_version")); + boolean valid = switch (operation) { + case "create" -> base == null && committed == 1; + case "update" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == base + 1; + case "delete" -> base != null && base > 0 && base < Long.MAX_VALUE + && committed == -(base + 1); + case "recover" -> base != null && base < 0 && base != Long.MIN_VALUE + && -base < Long.MAX_VALUE && committed == -base + 1; + default -> false; + }; + if (!valid) throw invalid("invalid version transition"); + JsonNode supplied = fact.path("fields"); + if (!supplied.isObject()) throw invalid("mutation fields are missing"); + if ("delete".equals(operation) && !supplied.isEmpty()) { + throw invalid("delete must be a tombstone"); + } + List ownProperties = entity.getOwnProperties(); + supplied.fieldNames().forEachRemaining(name -> { + PropertyDescriptor property = entity.findProperty(name); + if (property == null || !ownProperties.contains(property) + || property.isId() || property.isVersion()) { + throw invalid("unknown or reserved model field"); + } + }); + List fields = new ArrayList<>(); + for (PropertyDescriptor property : ownProperties) { + if (property.isId() || property.isVersion()) continue; + JsonNode value = supplied.get(property.getName()); + if (value == null) { + if ("create".equals(operation)) throw invalid("create omitted a modeled field"); + continue; + } + Mapping column = mapping(property); + if (!idMapping.table().equals(column.table())) { + throw invalid("mutation field spans multiple tables"); + } + fields.add(decodeField(sql, property, column, value)); + } + return new Mutation(entityType, idMapping.table(), idMapping.column(), + versionMapping.column(), id, base, committed, operation, fields); + } + + private static Field decodeField(SqlExecutionAdapter sql, PropertyDescriptor property, + Mapping mapping, JsonNode value) { + if (value.isNull() && property.getBoolean("required", false)) { + throw invalid("non-null field is null"); + } + if (property instanceof Relation relation) { + requirePhysical(sql, mapping, "BIGINT", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isObject() || value.size() != 2 || relation.getReverseProperty() == null + || relation.getReverseProperty().getOwner() == null + || !relation.getReverseProperty().getOwner().getType() + .equals(value.path("entity_type").asText())) { + throw invalid("reference target does not match model"); + } + return new Field(mapping.column(), positiveId(value.path("entity_id")), Kind.SCALAR); + } + Class type = property.getType() == null ? null : property.getType().javaType(); + if (type == LocalDate.class || type == java.sql.Date.class) { + requirePhysical(sql, mapping, "DATE", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.DATE); + if (!value.isTextual()) throw invalid("Date field has wrong type"); + try { + LocalDate date = LocalDate.parse(value.textValue()); + if (!date.toString().equals(value.textValue())) throw invalid("Date field is not canonical"); + return new Field(mapping.column(), date, Kind.DATE); + } catch (java.time.format.DateTimeParseException failure) { + throw invalid("Date field has wrong type"); + } + } + if (type == java.time.Instant.class || type == java.util.Date.class + || type == java.sql.Timestamp.class) { + requirePhysical(sql, mapping, "TIMESTAMP WITH TIME ZONE", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.INSTANT); + long millis = canonicalLong(value, "Timestamp field has wrong type"); + try { Math.multiplyExact(millis, 1000L); } + catch (ArithmeticException overflow) { throw invalid("Timestamp exceeds DuckDB instant range"); } + return new Field(mapping.column(), millis, Kind.INSTANT); + } + if (type == java.time.LocalDateTime.class) { + throw invalid("zone-free LocalDateTime has no committed instant mapping"); + } + if (type == String.class || type == java.util.UUID.class || type != null && type.isEnum()) { + requireTextPhysical(sql, mapping); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isTextual()) throw invalid("text field has wrong type"); + String text = value.textValue(); + String maxText = property.getStr("max", null); + if (maxText != null) { + try { + if (text.codePointCount(0, text.length()) > Integer.parseInt(maxText)) { + throw invalid("text exceeds modeled length"); + } + } catch (NumberFormatException malformed) { throw invalid("modeled text length is invalid"); } + } + return new Field(mapping.column(), text, Kind.SCALAR); + } + if (type == Boolean.class || type == boolean.class) { + String physical = requireBooleanPhysical(sql, mapping); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isBoolean()) throw invalid("Boolean field has wrong type"); + return new Field(mapping.column(), "BOOLEAN".equals(physical) + ? value.booleanValue() : value.booleanValue() ? 1 : 0, Kind.SCALAR); + } + if (type == Long.class || type == long.class || type == Integer.class || type == int.class + || type == Short.class || type == short.class || type == Byte.class || type == byte.class) { + String physical = type == Long.class || type == long.class ? "BIGINT" + : type == Integer.class || type == int.class ? "INTEGER" + : type == Short.class || type == short.class ? "SMALLINT" : "TINYINT"; + requirePhysical(sql, mapping, physical, null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + long number = canonicalLong(value, "integer field has wrong type"); + if (type == Integer.class || type == int.class) { + if (number < Integer.MIN_VALUE || number > Integer.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Short.class || type == short.class) { + if (number < Short.MIN_VALUE || number > Short.MAX_VALUE) throw invalid("integer overflows field"); + } else if (type == Byte.class || type == byte.class) { + if (number < Byte.MIN_VALUE || number > Byte.MAX_VALUE) throw invalid("integer overflows field"); + } + return new Field(mapping.column(), number, Kind.SCALAR); + } + if (type == BigDecimal.class) { + Matcher declaration = DECLARED_DECIMAL.matcher(mapping.declaredType()); + if (!declaration.matches()) throw invalid("Decimal lacks modeled precision/scale"); + int precision; + int scale; + try { + precision = Integer.parseInt(declaration.group(1)); + scale = Integer.parseInt(declaration.group(2)); + } catch (NumberFormatException malformed) { throw invalid("modeled Decimal precision/scale is invalid"); } + if (precision < 1 || scale > precision) throw invalid("modeled Decimal precision/scale is invalid"); + requirePhysical(sql, mapping, "DECIMAL", precision, scale); + if (value.isNull()) return new Field(mapping.column(), null, Kind.DECIMAL); + if (!value.isTextual()) throw invalid("Decimal field has wrong type"); + BigDecimal decimal; + try { decimal = new BigDecimal(value.textValue()); } + catch (NumberFormatException malformed) { throw invalid("Decimal field is invalid"); } + BigDecimal normalized = decimal.stripTrailingZeros(); + long integerDigits = decimal.signum() == 0 ? 0 + : Math.max(0L, (long) normalized.precision() - normalized.scale()); + long fractionalDigits = Math.max(0L, normalized.scale()); + if (integerDigits > precision - scale || fractionalDigits > scale) { + throw invalid("Decimal exceeds modeled precision/scale"); + } + return new Field(mapping.column(), decimal, Kind.DECIMAL); + } + if (type == Double.class || type == double.class) { + requirePhysical(sql, mapping, "DOUBLE", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.SCALAR); + if (!value.isNumber() || !Double.isFinite(value.doubleValue())) { + throw invalid("floating field has wrong type"); + } + return new Field(mapping.column(), value.doubleValue(), Kind.SCALAR); + } + if (type == byte[].class) { + requirePhysical(sql, mapping, "BLOB", null, null); + if (value.isNull()) return new Field(mapping.column(), null, Kind.BYTES); + if (!value.isObject() || value.size() != 1 + || !value.path("$binary_base64").isTextual()) { + throw invalid("binary field has wrong type"); + } + try { + return new Field(mapping.column(), java.util.Base64.getDecoder() + .decode(value.path("$binary_base64").textValue()), Kind.BYTES); + } catch (IllegalArgumentException malformed) { throw invalid("binary field is invalid"); } + } + throw invalid("unsupported modeled field type"); + } + + private static void applyOne(SqlExecutionAdapter sql, Mutation mutation) { + String table = quoted(mutation.table()); + String idColumn = quoted(mutation.idColumn()); + String versionColumn = quoted(mutation.versionColumn()); + List> current = sql.queryForList("SELECT " + versionColumn + + " AS committed_version FROM " + table + " WHERE " + idColumn + " = ?", + new Object[] {mutation.id()}); + Long actualVersion = current.isEmpty() ? null + : ((Number) current.get(0).get("committed_version")).longValue(); + if (!Objects.equals(actualVersion, mutation.baseVersion())) { + throw new Failure("VERSION_CONFLICT", "committed-event predecessor version mismatch"); + } + List args = new ArrayList<>(); + if ("create".equals(mutation.operation())) { + List columns = new ArrayList<>(List.of(idColumn, versionColumn)); + List binds = new ArrayList<>(List.of("?", "?")); + args.add(mutation.id()); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + columns.add(quoted(field.column())); + binds.add(field.bindExpression()); + args.add(field.value()); + } + if (sql.update("INSERT INTO " + table + " (" + String.join(", ", columns) + + ") VALUES (" + String.join(", ", binds) + ")", args.toArray()) != 1) { + throw new Failure("TARGET_FAILURE", "committed target insert count mismatch"); + } + } else { + List assignments = new ArrayList<>(List.of(versionColumn + " = ?")); + args.add(mutation.committedVersion()); + for (Field field : mutation.fields()) { + assignments.add(quoted(field.column()) + " = " + field.bindExpression()); + args.add(field.value()); + } + args.add(mutation.id()); + args.add(mutation.baseVersion()); + int changed = sql.update("UPDATE " + table + " SET " + String.join(", ", assignments) + + " WHERE " + idColumn + " = ? AND " + versionColumn + " = ?", args.toArray()); + if (changed != 1) { + throw new Failure("VERSION_CONFLICT", "committed-event predecessor version mismatch"); + } + } + verifyReadback(sql, mutation); + } + + private static void verifyReadback(SqlExecutionAdapter sql, Mutation mutation) { + List projections = new ArrayList<>(List.of( + quoted(mutation.versionColumn()) + " AS committed_version")); + for (int i = 0; i < mutation.fields().size(); i++) { + projections.add(mutation.fields().get(i).readExpression() + " AS field_" + i); + } + List> rows = sql.queryForList("SELECT " + String.join(", ", projections) + + " FROM " + quoted(mutation.table()) + " WHERE " + quoted(mutation.idColumn()) + " = ?", + new Object[] {mutation.id()}); + if (rows.size() != 1 || !Objects.equals(mutation.committedVersion(), + ((Number) rows.get(0).get("committed_version")).longValue())) { + throw invalid("target did not retain committed version"); + } + for (int i = 0; i < mutation.fields().size(); i++) { + Field field = mutation.fields().get(i); + if (!sameValue(field, rows.get(0).get("field_" + i))) { + throw invalid("target did not retain committed field value"); + } + } + } + + private static boolean sameValue(Field field, Object actual) { + Object expected = field.value(); + if (expected == null || actual == null) return expected == actual; + if (field.kind() == Kind.BYTES) { + return actual instanceof byte[] bytes && Arrays.equals((byte[]) expected, bytes); + } + if (field.kind() == Kind.DATE) return expected.toString().equals(actual.toString()); + if (field.kind() == Kind.INSTANT) { + return actual instanceof Number number && number.longValue() == (long) expected; + } + if (field.kind() == Kind.DECIMAL) { + return actual instanceof BigDecimal decimal + && ((BigDecimal) expected).compareTo(decimal) == 0; + } + if (expected instanceof Number number && actual instanceof Number other) { + return new BigDecimal(number.toString()).compareTo(new BigDecimal(other.toString())) == 0; + } + return Objects.equals(expected, actual); + } + + private static Mapping mapping(PropertyDescriptor property) { + List columns = SQLPropertyUtil.getColumns(property); + if (columns.size() != 1 || columns.get(0).getType() == null) { + throw invalid("property has no single declared SQL column"); + } + SQLColumn column = columns.get(0); + return new Mapping(column.getTableName(), column.getColumnName(), column.getType()); + } + + private static void requireTextPhysical(SqlExecutionAdapter sql, Mapping mapping) { + if (!mapping.declaredType().matches("(?i)\\s*(?:VARCHAR(?:\\([0-9]+\\))?|TEXT)\\s*")) { + throw invalid("text column declaration is unsupported"); + } + requirePhysical(sql, mapping, "VARCHAR", null, null); + } + + private static String requireBooleanPhysical(SqlExecutionAdapter sql, Mapping mapping) { + String type = mapping.declaredType().trim().toUpperCase(java.util.Locale.ROOT); + if (!type.equals("BOOLEAN") && !type.equals("TINYINT")) { + throw invalid("Boolean column declaration is unsupported"); + } + requirePhysical(sql, mapping, type, null, null); + return type; + } + + private static void requirePhysical(SqlExecutionAdapter sql, Mapping mapping, + String expectedType, Integer precision, Integer scale) { + String declared = mapping.declaredType().trim().toUpperCase(java.util.Locale.ROOT); + if (expectedType.equals("TIMESTAMP WITH TIME ZONE")) { + if (!declared.equals("TIMESTAMPTZ") && !declared.equals(expectedType)) { + throw invalid("model Instant column is not an absolute timestamp"); + } + } else if (!expectedType.equals("DECIMAL") && !expectedType.equals("VARCHAR") + && !declared.equals(expectedType)) { + throw invalid("modeled SQL type does not match field kind"); + } + List> rows = sql.queryForList( + "SELECT data_type, numeric_precision, numeric_scale FROM information_schema.columns " + + "WHERE table_name = ? AND column_name = ?", + new Object[] {mapping.table(), mapping.column()}); + if (rows.size() != 1) throw invalid("physical model column is missing or ambiguous"); + Map row = rows.get(0); + String physical = (String) row.get("data_type"); + if (physical == null || !(expectedType.equals("DECIMAL") + ? physical.startsWith("DECIMAL(") : physical.equals(expectedType))) { + throw invalid("physical column has incompatible SQL type"); + } + if (precision != null && (number(row.get("numeric_precision")) != precision + || number(row.get("numeric_scale")) != scale)) { + throw invalid("physical Decimal precision/scale differs from model"); + } + } + + private static void validateTraceChain(JsonNode chain) { + if (!chain.isArray()) throw invalid("mutation trace chain is missing"); + for (JsonNode node : chain) { + JsonNode id = node.path("entity_id"); + if (!node.isObject() || node.size() != 4 + || !TRACE_KINDS.contains(node.path("kind").asText("")) + || !nonblank(node.path("name")) + || !nonblank(node.path("comment")) + || !(id.isNull() || id.isTextual() && id.textValue().matches("[1-9][0-9]*"))) { + throw invalid("trace node has wrong shape"); + } + } + } + + private static long positiveId(JsonNode value) { + long id = canonicalLong(value, "identity is outside the DuckDB ID range"); + if (id <= 0) throw invalid("identity must be positive"); + return id; + } + + private static long signedVersion(JsonNode value) { + return canonicalLong(value, "version has wrong type"); + } + + private static long canonicalLong(JsonNode value, String reason) { + if (!value.isTextual() || !value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) throw invalid(reason); + try { return Long.parseLong(value.textValue()); } + catch (NumberFormatException overflow) { throw invalid(reason); } + } + + private static String quoted(String identifier) { + if (identifier == null || identifier.isBlank()) throw invalid("model has an empty SQL identifier"); + return "\"" + identifier.replace("\"", "\"\"") + "\""; + } + + private static long number(Object value) { + if (!(value instanceof Number number)) throw invalid("physical numeric metadata is missing"); + return number.longValue(); + } + private static boolean nonblank(JsonNode value) { + return value.isTextual() && !value.textValue().isBlank(); + } + private static Failure invalid(String reason) { + return new Failure("INVALID_ENVELOPE", "invalid committed-event envelope: " + reason); + } +} diff --git a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java new file mode 100644 index 00000000..a507b105 --- /dev/null +++ b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java @@ -0,0 +1,259 @@ +package io.teaql.core.duck; + +import io.teaql.core.BaseEntity; +import io.teaql.core.duck.DuckDbCommittedInbox.Failure; +import io.teaql.core.duck.DuckDbCommittedInbox.Record; +import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +public class DuckDbModelCheckedCommittedTargetTest { + private static final String SOURCE = "sqlite-order-source"; + private static final String HASH = "sha256:" + "0".repeat(64); + private static final long ORDER_ID = 9_007_199_254_740_993L; + private static final long ITEM_ID = ORDER_ID + 1; + + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { return DriverManager.getConnection(url); } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class type) throws SQLException { throw new SQLException("not a wrapper"); } + @Override public boolean isWrapperFor(Class type) { return false; } + } + + private Path directory; + private Path database; + private JdbcSqlExecutor sql; + private DefaultUserContext context; + private DuckDbModelCheckedCommittedTarget target; + + @Before public void setUp() throws Exception { + directory = Files.createTempDirectory("teaql-duckdb-model-target-"); + database = directory.resolve("target.db"); + sql = new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)); + SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); + EntityDescriptor order = new EntityDescriptor(); + order.setType("order"); + order.addSimpleProperty("id", Long.class); + order.addSimpleProperty("version", Long.class); + order.addSimpleProperty("order_number", String.class); + order.addSimpleProperty("status", String.class); + order.addSimpleProperty("planned_date", java.time.LocalDate.class).with("sqlType", "DATE"); + order.addSimpleProperty("occurred_at", java.time.Instant.class) + .with("sqlType", "TIMESTAMPTZ").with("required", "true"); + order.addSimpleProperty("amount", BigDecimal.class).with("sqlType", "DECIMAL(19,7)"); + order.addSimpleProperty("active", Boolean.class) + .with("sqlType", "BOOLEAN").with("required", "true"); + order.addSimpleProperty("ratio", Double.class).with("required", "true"); + order.addSimpleProperty("optional_note", String.class); + order.addSimpleProperty("optional_label", String.class); + metadata.register(order); + EntityDescriptor item = new EntityDescriptor(); + item.setType("order_item"); + item.addSimpleProperty("id", Long.class); + item.addSimpleProperty("version", Long.class); + item.addObjectProperty(metadata, "order", "order", "items", BaseEntity.class) + .with("columnName", "order_id"); + item.addSimpleProperty("quantity", Long.class); + metadata.register(item); + context = new DefaultUserContext(TeaQLRuntime.builder().metadata(metadata).build()); + sql.execute("CREATE TABLE order_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_number VARCHAR NOT NULL, status VARCHAR NOT NULL, planned_date DATE, " + + "occurred_at TIMESTAMPTZ NOT NULL, amount DECIMAL(19,7), active BOOLEAN NOT NULL, " + + "ratio DOUBLE NOT NULL, optional_note VARCHAR, optional_label VARCHAR)"); + sql.execute("CREATE TABLE order_item_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_id BIGINT NOT NULL REFERENCES order_data(id), quantity BIGINT NOT NULL)"); + target = new DuckDbModelCheckedCommittedTarget(sql, SOURCE, HASH); + target.ensureInboxSchema(); + } + + @After public void tearDown() throws Exception { + Files.deleteIfExists(database); + Files.deleteIfExists(directory); + } + + @Test public void appliesWholeOrderGraphTwiceAndRetainsExactTypedValuesAcrossRestart() { + Record first = record(1, "order-graph-0001", created()); + Record second = record(2, "order-graph-0002", submitted()); + assertEquals(new DuckDbCommittedInbox.Outcome(false, 1), target.apply(context, first)); + assertEquals(new DuckDbCommittedInbox.Outcome(true, 1), target.apply(context, first)); + assertEquals(new DuckDbCommittedInbox.Outcome(false, 2), target.apply(context, second)); + assertEquals(new DuckDbCommittedInbox.Outcome(true, 2), target.apply(context, second)); + assertEquals(1, count("order_data")); + assertEquals(1, count("order_item_data")); + assertEquals(2, count("teaql_event_inbox")); + assertEquals(2, count("teaql_event_applied_payload")); + assertEquals(1, count("teaql_event_checkpoint")); + Map order = sql.queryForList("SELECT id, version, status, " + + "CAST(planned_date AS VARCHAR) AS planned_date, epoch_ms(occurred_at) AS occurred_at, " + + "amount, active, ratio, optional_note, optional_label FROM order_data", new Object[0]).get(0); + assertEquals(ORDER_ID, ((Number) order.get("id")).longValue()); + assertEquals(2L, ((Number) order.get("version")).longValue()); + assertEquals("SUBMITTED", order.get("status")); + assertEquals("2026-10-16", order.get("planned_date")); + assertEquals(-315_619_199_877L, ((Number) order.get("occurred_at")).longValue()); + assertEquals(0, ((BigDecimal) order.get("amount")).compareTo(BigDecimal.ZERO)); + assertEquals(true, order.get("active")); + assertEquals(0.5, ((Number) order.get("ratio")).doubleValue(), 0.0); + assertEquals("submitted note", order.get("optional_note")); + assertEquals(null, order.get("optional_label")); + Map item = sql.queryForList("SELECT id, version, order_id, quantity " + + "FROM order_item_data", new Object[0]).get(0); + assertEquals(ITEM_ID, ((Number) item.get("id")).longValue()); + assertEquals(-2L, ((Number) item.get("version")).longValue()); + assertEquals(ORDER_ID, ((Number) item.get("order_id")).longValue()); + assertEquals(2L, ((Number) item.get("quantity")).longValue()); + assertEquals(first.payloadJson(), sql.queryForList("SELECT payload_json FROM " + + "teaql_event_applied_payload WHERE source_sequence = 1", new Object[0]) + .get(0).get("payload_json")); + DuckDbModelCheckedCommittedTarget reopened = new DuckDbModelCheckedCommittedTarget( + new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + database)), SOURCE, HASH); + assertTrue(reopened.apply(context, second).duplicate()); + } + + @Test public void lateGraphFailureRollsBackDomainAndInbox() { + String broken = created().replace("\"quantity\":\"2\"", "\"quantity\":\"2\",\"unknown\":1"); + expect("INVALID_ENVELOPE", () -> target.apply(context, record(1, "bad", broken))); + assertEmptyTarget(); + String missingParent = created().replace("\"order\":{\"entity_type\":\"order\",\"entity_id\":\"9007199254740993\"}", + "\"order\":{\"entity_type\":\"order\",\"entity_id\":\"9007199254740995\"}"); + assertFalse(missingParent.equals(created())); + expect("TARGET_FAILURE", () -> target.apply(context, record(1, "bad", missingParent))); + assertEmptyTarget(); + assertFalse(target.apply(context, record(1, "good", created())).duplicate()); + } + + @Test public void invalidCodecAndPhysicalMappingFailClosed() { + expectRejectedCreate(created().replace("\"amount\":\"0\"", "\"amount\":\"0.00000001\"")); + expectRejectedCreate(created().replace("\"amount\":\"0\"", "\"amount\":\"1234567890123\"")); + expectRejectedCreate(created().replace("\"occurred_at\":\"-315619199877\"", + "\"occurred_at\":\"2026-10-16T00:00:00Z\"")); + expectRejectedCreate(created().replace("\"planned_date\":\"2026-10-16\"", + "\"planned_date\":\"2026-10-16T00:00:00Z\"")); + expectRejectedCreate(created().replace("\"active\":true", "\"active\":\"true\"")); + expectRejectedCreate(created().replace("\"quantity\":\"2\"", "\"quantity\":2")); + expectRejectedCreate(created().replace("\"trace_chain\":[]", "\"trace_chain\":null")); + sql.execute("DROP TABLE order_item_data"); + sql.execute("ALTER TABLE order_data ALTER COLUMN amount SET DATA TYPE DECIMAL(18,2)"); + sql.execute("CREATE TABLE order_item_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_id BIGINT NOT NULL REFERENCES order_data(id), quantity BIGINT NOT NULL)"); + expectRejectedCreate(created()); + } + + @Test public void exactFractionalDecimalSurvivesDeclaredDuckDbStorage() { + String fractional = created().replace("\"amount\":\"0\"", "\"amount\":\"0.1\""); + assertFalse(target.apply(context, record(1, "fractional", fractional)).duplicate()); + BigDecimal actual = (BigDecimal) sql.queryForList("SELECT amount FROM order_data", + new Object[0]).get(0).get("amount"); + assertEquals(0, actual.compareTo(new BigDecimal("0.1"))); + } + + @Test public void zoneFreePhysicalTimestampCannotReceiveCommittedInstant() { + sql.execute("DROP TABLE order_item_data"); + sql.execute("DROP TABLE order_data"); + sql.execute("CREATE TABLE order_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_number VARCHAR NOT NULL, status VARCHAR NOT NULL, planned_date DATE, " + + "occurred_at TIMESTAMP NOT NULL, amount DECIMAL(19,7), active BOOLEAN NOT NULL, " + + "ratio DOUBLE NOT NULL, optional_note VARCHAR, optional_label VARCHAR)"); + sql.execute("CREATE TABLE order_item_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "order_id BIGINT NOT NULL REFERENCES order_data(id), quantity BIGINT NOT NULL)"); + expectRejectedCreate(created()); + } + + private void expectRejectedCreate(String mutations) { + expect("INVALID_ENVELOPE", () -> target.apply(context, record(1, "bad", mutations))); + assertEmptyTarget(); + } + + private void assertEmptyTarget() { + for (String table : List.of("order_data", "order_item_data", "teaql_event_inbox", + "teaql_event_applied_payload", "teaql_event_checkpoint")) { + assertEquals(table, 0, count(table)); + } + } + + private long count(String table) { + return ((Number) sql.queryForList("SELECT count(*) AS row_count FROM " + table, + new Object[0]).get(0).get("row_count")).longValue(); + } + + private static Record record(long sequence, String changeSetId, String mutations) { + String payload = """ + {"protocol_version":1,"change_set_id":"%s","source_runtime_id":"%s", + "source_sequence":"%d","source_model_hash":"%s","source_transaction_id":"tx-%d", + "committed_at":"2026-10-16T00:00:00Z", + "audit":{"actor_ref":"fixture:operator","comment":"Apply order graph", + "purpose":"Verify target application","trace_id":"fixture-trace"}, + "mutations":%s} + """.formatted(changeSetId, SOURCE, sequence, HASH, sequence, mutations); + try { + String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(payload.getBytes(StandardCharsets.UTF_8))); + return new Record(SOURCE, sequence, changeSetId, HASH, digest, payload); + } catch (Exception impossible) { throw new AssertionError(impossible); } + } + + private static String created() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order_number":"ORD-0001","status":"DRAFT", + "planned_date":"2026-10-16","occurred_at":"-315619199877", + "amount":"0","active":true,"ratio":0.5,"optional_note":null, + "optional_label":"initial label"},"trace_chain":[]}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", + "base_version":null,"committed_version":"1", + "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, + "quantity":"2"},"trace_chain":[]}] + """; + } + + private static String submitted() { + return """ + [{"entity_type":"order","entity_id":"9007199254740993","operation":"update", + "base_version":"1","committed_version":"2", + "fields":{"status":"SUBMITTED","optional_note":"submitted note", + "optional_label":null},"trace_chain":[]}, + {"entity_type":"order_item","entity_id":"9007199254740994","operation":"delete", + "base_version":"1","committed_version":"-2","fields":{},"trace_chain":[]}] + """; + } + + private static void expect(String code, Runnable action) { + assertEquals(code, assertThrows(Failure.class, action::run).code()); + } +} From b37911e96fec80a3786e43ac632ef2b4e8385ca1 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 07:11:04 +0800 Subject: [PATCH 10/29] test(duckdb): replay shared committed Order fixture --- ...DuckDbModelCheckedCommittedTargetTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java index a507b105..10a5fc8c 100644 --- a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java +++ b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java @@ -1,5 +1,7 @@ package io.teaql.core.duck; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import io.teaql.core.BaseEntity; import io.teaql.core.duck.DuckDbCommittedInbox.Failure; import io.teaql.core.duck.DuckDbCommittedInbox.Record; @@ -9,6 +11,7 @@ import io.teaql.runtime.DefaultUserContext; import io.teaql.runtime.TeaQLRuntime; import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -145,6 +148,45 @@ private record DriverManagerDataSource(String url) implements DataSource { assertTrue(reopened.apply(context, second).duplicate()); } + @Test public void appliesSharedConformanceFixtureWhenConfigured() throws Exception { + String fixturePath = System.getenv("TEAQL_COMMITTED_EVENT_FIXTURE"); + Assume.assumeTrue("shared fixture path was not supplied", + fixturePath != null && !fixturePath.isBlank()); + JsonNode fixture = new ObjectMapper().readTree(Files.readString(Path.of(fixturePath))); + assertEquals(1, fixture.path("fixture_version").asInt()); + assertEquals(2, fixture.path("sets").size()); + for (int index = 0; index < 2; index++) { + JsonNode set = fixture.path("sets").get(index); + long sequence = Long.parseLong(set.path("source_sequence").asText()); + Record source = recordBody(sequence, set.path("change_set_id").asText(), set.toString()); + assertEquals(sequence, target.apply(context, source).checkpoint()); + assertTrue(target.apply(context, source).duplicate()); + } + Map order = sql.queryForList("SELECT id, version, status, " + + "CAST(planned_date AS VARCHAR) AS planned_date, epoch_ms(occurred_at) AS occurred_at, " + + "amount, active, ratio, optional_note, optional_label FROM order_data", + new Object[0]).get(0); + assertEquals(ORDER_ID, ((Number) order.get("id")).longValue()); + assertEquals(2L, ((Number) order.get("version")).longValue()); + assertEquals("SUBMITTED", order.get("status")); + assertEquals("2026-10-16", order.get("planned_date")); + assertEquals(-315_619_199_877L, ((Number) order.get("occurred_at")).longValue()); + assertEquals(0, ((BigDecimal) order.get("amount")).compareTo(BigDecimal.ZERO)); + assertEquals(true, order.get("active")); + assertEquals(0.5, ((Number) order.get("ratio")).doubleValue(), 0.0); + assertEquals("submitted note", order.get("optional_note")); + assertEquals(null, order.get("optional_label")); + Map item = sql.queryForList("SELECT id, version, order_id, quantity " + + "FROM order_item_data", new Object[0]).get(0); + assertEquals(ITEM_ID, ((Number) item.get("id")).longValue()); + assertEquals(-2L, ((Number) item.get("version")).longValue()); + assertEquals(ORDER_ID, ((Number) item.get("order_id")).longValue()); + assertEquals(2L, ((Number) item.get("quantity")).longValue()); + assertEquals(2, count("teaql_event_inbox")); + assertEquals(2, count("teaql_event_applied_payload")); + assertEquals(1, count("teaql_event_checkpoint")); + } + @Test public void lateGraphFailureRollsBackDomainAndInbox() { String broken = created().replace("\"quantity\":\"2\"", "\"quantity\":\"2\",\"unknown\":1"); expect("INVALID_ENVELOPE", () -> target.apply(context, record(1, "bad", broken))); @@ -220,6 +262,10 @@ private static Record record(long sequence, String changeSetId, String mutations "purpose":"Verify target application","trace_id":"fixture-trace"}, "mutations":%s} """.formatted(changeSetId, SOURCE, sequence, HASH, sequence, mutations); + return recordBody(sequence, changeSetId, payload); + } + + private static Record recordBody(long sequence, String changeSetId, String payload) { try { String digest = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") .digest(payload.getBytes(StandardCharsets.UTF_8))); From 895d462c110b6708d91600b6bff351ec2f2d3b16 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 07:18:17 +0800 Subject: [PATCH 11/29] fix(duckdb): verify canonical relation indexes on repeated schema --- .../core/duck/DuckDataServiceExecutor.java | 39 ++++++++++++++ ...DuckDbModelCheckedCommittedTargetTest.java | 54 +++++++++++++++++++ .../sql/portable/PortableSQLRepository.java | 6 ++- .../core/sql/portable/TeaQLDatabase.java | 12 +++++ 4 files changed, 109 insertions(+), 2 deletions(-) diff --git a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java index 573f5f55..c2be45f2 100644 --- a/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java +++ b/teaql-duckdb/src/main/java/io/teaql/core/duck/DuckDataServiceExecutor.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; +import java.util.Optional; public class DuckDataServiceExecutor extends SqlDataServiceExecutor { @@ -58,6 +59,25 @@ public List> getTableColumns(String tableName) { + "FROM information_schema.columns WHERE table_name = ?"; return getExecutionAdapter().queryForList(sql, new Object[] {tableName}); } + + @Override + public Optional canonicalRelationIndexMatches(UserContext caller, + String tableName, String indexName, List orderedColumns) { + List> matches = getExecutionAdapter().queryForList( + "SELECT table_name, is_unique, sql FROM duckdb_indexes() " + + "WHERE index_name = ?", new Object[] {indexName}); + if (matches.isEmpty()) return Optional.of(false); + if (matches.size() != 1) { + throw new IllegalStateException("DuckDB canonical relation index name is ambiguous"); + } + Map index = matches.get(0); + if (!tableName.equals(index.get("table_name")) + || !Boolean.FALSE.equals(index.get("is_unique")) + || !sameSimpleOrderedColumns(index.get("sql"), orderedColumns)) { + throw new IllegalStateException("DuckDB canonical relation index has incompatible shape"); + } + return Optional.of(true); + } }; for (EntityDescriptor descriptor : descriptors) { @@ -65,4 +85,23 @@ public List> getTableColumns(String tableName) { repository.ensurePhysicalSchema(context); } } + + private static boolean sameSimpleOrderedColumns(Object definition, List expected) { + if (!(definition instanceof String ddl)) return false; + int open = ddl.indexOf('('); + int close = ddl.lastIndexOf(')'); + if (open < 0 || close <= open || !ddl.regionMatches(true, 0, "CREATE INDEX ", 0, 13)) { + return false; + } + String[] actual = ddl.substring(open + 1, close).split(",", -1); + if (actual.length != expected.size()) return false; + for (int i = 0; i < actual.length; i++) { + String column = actual[i].trim().replaceFirst("(?i)\\s+(ASC|DESC)$", ""); + if (column.startsWith("\"") && column.endsWith("\"") && column.length() >= 2) { + column = column.substring(1, column.length() - 1).replace("\"\"", "\""); + } + if (!column.equals(expected.get(i))) return false; + } + return true; + } } diff --git a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java index 10a5fc8c..78a5aff6 100644 --- a/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java +++ b/teaql-duckdb/src/test/java/io/teaql/core/duck/DuckDbModelCheckedCommittedTargetTest.java @@ -6,6 +6,7 @@ import io.teaql.core.duck.DuckDbCommittedInbox.Failure; import io.teaql.core.duck.DuckDbCommittedInbox.Record; import io.teaql.core.meta.EntityDescriptor; +import io.teaql.core.meta.EntityMetaFactory; import io.teaql.core.meta.SimpleEntityMetaFactory; import io.teaql.provider.jdbc.JdbcSqlExecutor; import io.teaql.runtime.DefaultUserContext; @@ -187,6 +188,59 @@ private record DriverManagerDataSource(String url) implements DataSource { assertEquals(1, count("teaql_event_checkpoint")); } + @Test public void contextEnsureSchemaInstallsCompatibleModelStorage() throws Exception { + Path installedDatabase = directory.resolve("installed-schema.db"); + try { + JdbcSqlExecutor installedSql = new JdbcSqlExecutor( + new DriverManagerDataSource("jdbc:duckdb:" + installedDatabase)); + EntityMetaFactory metadata = EntityMetaFactory.requireFrom(context); + metadata.allEntityDescriptors().forEach(entity -> entity.setDataService("duckdb")); + DefaultUserContext installedContext = new DefaultUserContext(TeaQLRuntime.builder() + .metadata(metadata) + .dataService("duckdb", new DuckDataServiceExecutor("duckdb", installedSql)) + .build()); + installedContext.ensureSchema(); + installedContext.ensureSchema(); + DuckDbModelCheckedCommittedTarget installed = + new DuckDbModelCheckedCommittedTarget(installedSql, SOURCE, HASH); + installed.ensureInboxSchema(); + assertEquals(1L, installed.apply(installedContext, + record(1, "installed-create", created())).checkpoint()); + assertEquals(2L, installed.apply(installedContext, + record(2, "installed-update", submitted())).checkpoint()); + List> columns = installedSql.queryForList( + "SELECT column_name, data_type FROM information_schema.columns " + + "WHERE table_name = 'order_data'", new Object[0]); + Map types = new java.util.HashMap<>(); + for (Map column : columns) { + types.put((String) column.get("column_name"), (String) column.get("data_type")); + } + assertEquals("DATE", types.get("planned_date")); + assertEquals("TIMESTAMP WITH TIME ZONE", types.get("occurred_at")); + assertEquals("DECIMAL(19,7)", types.get("amount")); + assertEquals("BOOLEAN", types.get("active")); + assertEquals("SUBMITTED", installedSql.queryForList( + "SELECT status FROM order_data", new Object[0]).get(0).get("status")); + assertEquals(-2L, ((Number) installedSql.queryForList( + "SELECT version FROM order_item_data", new Object[0]) + .get(0).get("version")).longValue()); + List> indexes = installedSql.queryForList( + "SELECT index_name FROM duckdb_indexes() " + + "WHERE table_name = 'order_item_data'", new Object[0]); + assertEquals(1, indexes.size()); + String indexName = (String) indexes.get(0).get("index_name"); + assertTrue(indexName.matches("[A-Za-z0-9_]+")); + installedSql.execute("DROP INDEX \"" + indexName + "\""); + installedSql.execute("CREATE INDEX \"" + indexName + + "\" ON order_item_data(quantity)"); + IllegalStateException mismatch = assertThrows(IllegalStateException.class, + installedContext::ensureSchema); + assertTrue(mismatch.getMessage().contains("canonical relation index")); + } finally { + Files.deleteIfExists(installedDatabase); + } + } + @Test public void lateGraphFailureRollsBackDomainAndInbox() { String broken = created().replace("\"quantity\":\"2\"", "\"quantity\":\"2\",\"unknown\":1"); expect("INVALID_ENVELOPE", () -> target.apply(context, record(1, "bad", broken))); diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index dc4da6c9..8cf465b4 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -1863,7 +1863,8 @@ private void ensureCanonicalRelationIndexes(UserContext context) { String indexName = canonicalRelationIndexName( table, relationColumn.getColumnName(), idColumn.getColumnName()); try { - if (database.indexExists(context, table, indexName).orElse(false)) continue; + if (database.canonicalRelationIndexMatches(context, table, indexName, + List.of(relationColumn.getColumnName(), idColumn.getColumnName())).orElse(false)) continue; } catch (Exception failure) { throw canonicalRelationIndexFailure("inspect", table, indexName, failure); } @@ -1880,7 +1881,8 @@ private void ensureCanonicalRelationIndexes(UserContext context) { try { // PostgreSQL index names are schema-wide. A duplicate name on a // different table must not be mistaken for this index being present. - if (database.indexExists(context, table, indexName).orElse(true)) { + if (database.canonicalRelationIndexMatches(context, table, indexName, + List.of(relationColumn.getColumnName(), idColumn.getColumnName())).orElse(true)) { continue; } } catch (Exception inspectionFailure) { diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java index d4f510fa..00439c9c 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/TeaQLDatabase.java @@ -128,4 +128,16 @@ default Optional indexExists( io.teaql.core.UserContext context, String tableName, String indexName) { return Optional.empty(); } + + /** + * Check the ordered keys of a generated relation index when the provider + * exposes them. Existing adapters retain their established index check; + * adapters with index-definition metadata can reject a name collision + * whose keys do not match the generated relation/index shape. + */ + default Optional canonicalRelationIndexMatches( + io.teaql.core.UserContext context, String tableName, String indexName, + List orderedColumns) { + return indexExists(context, tableName, indexName); + } } From 1a073c57b4bff7eeab808a205e88c0717df20da4 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 04:34:14 +0800 Subject: [PATCH 12/29] Validate committed-event trace wire nodes --- .../SqliteCommittedMutationRecorder.java | 4 ++-- .../SqliteModelCheckedCommittedTarget.java | 18 ++++++++++++++++++ .../io/teaql/sqlite/GraphTraceSqliteTest.java | 8 ++++++++ .../SqliteModelCheckedCommittedTargetTest.java | 16 ++++++++++++---- 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java index bb8e1d88..1e45db2f 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedMutationRecorder.java @@ -91,9 +91,9 @@ public void record(UserContext context, Object transactionResource, for (TraceNode node : fact.traceChain()) { Map step = new LinkedHashMap<>(); step.put("kind", node.getKind().name()); - step.put("name", node.getName()); + step.put("name", Objects.toString(node.getName(), "")); step.put("entity_id", node.getEntityId() == null ? null : node.getEntityId().toString()); - step.put("comment", node.getComment()); + step.put("comment", Objects.toString(node.getComment(), "")); chain.add(step); } mutation.put("trace_chain", chain); diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java index 1098f55d..191751ec 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -26,6 +26,9 @@ * infrastructure, not a business-facing Mutation or remote query endpoint. */ public final class SqliteModelCheckedCommittedTarget { + private static final Set TRACE_KINDS = Set.of( + "OPERATION", "REQUEST", "RELATION", "ENTITY", "PROVIDER", "SQL", + "COMMENT", "PURPOSE", "AUDIT_REASON"); private final SqlExecutionAdapter sql; private final SqliteCommittedEventStore events; private final String sourceRuntimeId; @@ -89,6 +92,7 @@ private static Mutation decode(EntityMetaFactory metadata, JsonNode fact) { if (!fact.isObject() || !nonblank(fact.path("entity_type"))) { throw invalid("mutation entity type is missing"); } + validateTraceChain(fact.path("trace_chain")); String entityType = fact.path("entity_type").textValue(); EntityDescriptor entity = metadata.allEntityDescriptors().stream() .filter(candidate -> entityType.equals(candidate.getType())).findFirst() @@ -161,6 +165,20 @@ private static Mutation decode(EntityMetaFactory metadata, JsonNode fact) { versionMapping.column(), id, base, committed, operation, fields); } + private static void validateTraceChain(JsonNode chain) { + if (!chain.isArray()) throw invalid("mutation trace chain is missing"); + for (JsonNode node : chain) { + JsonNode id = node.path("entity_id"); + if (!node.isObject() || node.size() != 4 + || !TRACE_KINDS.contains(node.path("kind").asText("")) + || !node.path("name").isTextual() + || !node.path("comment").isTextual() + || !(id.isNull() || id.isTextual() && id.textValue().matches("[1-9][0-9]*"))) { + throw invalid("trace node has wrong shape"); + } + } + } + private static Object decodeValue(PropertyDescriptor property, JsonNode value) { if (value.isNull()) { if (property.getBoolean("required", false)) throw invalid("non-null field is null"); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index 2d533aac..afb36382 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -480,6 +480,14 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertTrue(fact.path("base_version").isNull()); assertEquals("1", fact.path("committed_version").asText()); assertTrue(fact.path("trace_chain").isArray()); + for (JsonNode node : fact.path("trace_chain")) { + assertEquals(4, node.size()); + assertTrue(node.has("kind")); + assertTrue(node.has("name")); + assertTrue(node.has("entity_id")); + assertTrue(node.has("comment")); + assertTrue(node.path("kind").asText().matches("[A-Z_]+")); + } if (fact.path("entity_type").asText().equals("CustomerOrder")) orderFact = fact; } assertNotNull(orderFact); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index b60b4b3e..b56ea43e 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -81,20 +81,20 @@ private static String created() { return """ [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", "base_version":null,"committed_version":"1", - "fields":{"order_number":"ORD-0001","status":"DRAFT"}}, + "fields":{"order_number":"ORD-0001","status":"DRAFT"},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, - "quantity":2}}] + "quantity":2},"trace_chain":[]}] """; } private static String submitted() { return """ [{"entity_type":"order","entity_id":"9007199254740993","operation":"update", - "base_version":"1","committed_version":"2","fields":{"status":"SUBMITTED"}}, + "base_version":"1","committed_version":"2","fields":{"status":"SUBMITTED"},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"delete", - "base_version":"1","committed_version":"-2","fields":{}}] + "base_version":"1","committed_version":"-2","fields":{},"trace_chain":[]}] """; } @@ -216,6 +216,14 @@ private static long count(JdbcSqlExecutor sql, String table) { "\"status\":\"DRAFT\",\"items\":[]"); expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), record(1, "reverse-relation", reverseRelation))); + String invalidTrace = created().replace("\"trace_chain\":[]", + "\"trace_chain\":[{\"kind\":\"AuditReason\",\"entity_type\":\"order\"," + + "\"entity_id\":\"9007199254740993\",\"comment\":\"Create order\"}]"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-trace", invalidTrace))); + String missingTrace = created().replace(",\"trace_chain\":[]", ""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "missing-trace", missingTrace))); t.sql().execute("CREATE TRIGGER reject_item BEFORE INSERT ON order_item_data " + "BEGIN SELECT RAISE(ABORT, 'reject item'); END"); try { From 9580529a91f401b8236e2fa6d56750acdfcab2e2 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 04:46:50 +0800 Subject: [PATCH 13/29] Encode committed integral fields losslessly --- .../java/io/teaql/runtime/TeaQLRuntime.java | 5 ++-- .../SqliteModelCheckedCommittedTarget.java | 11 +++++-- .../sqlite/SqliteCommittedEventStoreTest.java | 2 +- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 13 +++++++-- ...SqliteModelCheckedCommittedTargetTest.java | 29 +++++++++++++++++-- 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 142de097..53e89521 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -1259,8 +1259,9 @@ private Object committedFieldValue(PropertyDescriptor property, Object value) { if (id == null) throw new TeaQLRuntimeException("Committed relation has no scalar identity: " + property.getName()); return Map.of("entity_type", type, "entity_id", id.toString()); } - if (value instanceof String || value instanceof Boolean || value instanceof Long - || value instanceof Integer || value instanceof Short || value instanceof Byte) return value; + if (value instanceof Long || value instanceof Integer || value instanceof Short + || value instanceof Byte) return value.toString(); + if (value instanceof String || value instanceof Boolean) return value; if (value instanceof java.math.BigDecimal decimal) return decimal.toPlainString(); if (value instanceof java.math.BigInteger integer) return integer.toString(); if (value instanceof Double number) { diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java index 191751ec..3cf1acc1 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -207,10 +207,13 @@ private static Object decodeValue(PropertyDescriptor property, JsonNode value) { } if (type == Long.class || type == long.class || type == Integer.class || type == int.class || type == Short.class || type == short.class || type == Byte.class || type == byte.class) { - if (!value.isIntegralNumber() || !value.canConvertToLong()) { + if (!value.isTextual() || !value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) { throw invalid("integer field has wrong type"); } - long number = value.longValue(); + long number; + try { number = Long.parseLong(value.textValue()); } + catch (NumberFormatException failure) { throw invalid("integer field overflows"); } if (type == Integer.class || type == int.class) { if (number < Integer.MIN_VALUE || number > Integer.MAX_VALUE) throw invalid("integer overflows field"); } else if (type == Short.class || type == short.class) { @@ -224,6 +227,10 @@ private static Object decodeValue(PropertyDescriptor property, JsonNode value) { if (!value.isTextual()) throw invalid("Decimal field has wrong type"); try { if (type == java.math.BigInteger.class) { + if (!value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) { + throw invalid("integer field has wrong type"); + } return new java.math.BigInteger(value.textValue()).toString(); } return new BigDecimal(value.textValue()).toPlainString(); diff --git a/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java index a0525620..b35609e4 100644 --- a/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java @@ -59,7 +59,7 @@ private static String payload(long sequence, String change, boolean submitted) { "base_version":%s,"committed_version":"%s","fields":{"status":"%s"}}, {"entity_type":"order_item","entity_id":"%d","operation":"%s", "base_version":%s,"committed_version":"%s", - "fields":{"order":{"entity_type":"order","entity_id":"%d"},"quantity":2}} + "fields":{"order":{"entity_type":"order","entity_id":"%d"},"quantity":"2"}} ]} """).formatted(SOURCE, sequence, change, MODEL_HASH, ORDER_ID, operation, baseVersion, version, status, ITEM_ID, itemOperation, baseVersion, diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index afb36382..4294105a 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -33,10 +33,12 @@ public static final class GraphEntity extends BaseEntity { public GraphEntity(String type) { this.type = type; } @Override public String typeName() { return type; } @Override public Object __internalGet(String field) { - return field.equals("name") || field.equals("memo") || field.equals("children") ? values.get(field) : super.__internalGet(field); + return field.equals("name") || field.equals("memo") || field.equals("children") + || field.equals("business_counter") ? values.get(field) : super.__internalGet(field); } @Override public void __internalSet(String field, Object value) { - if (field.equals("name") || field.equals("memo") || field.equals("children")) values.put(field, value); + if (field.equals("name") || field.equals("memo") || field.equals("children") + || field.equals("business_counter")) values.put(field, value); else super.__internalSet(field, value); } } @@ -109,6 +111,10 @@ static final class Fixture { textField ? String.class : Long.class); property.setColumnType(textField ? "VARCHAR(255)" : "BIGINT"); } + if (captureCommittedMutations && type.equals("CustomerOrder")) { + var counter = (GenericSQLProperty) descriptor.addSimpleProperty("business_counter", Long.class); + counter.setColumnType("BIGINT"); + } var children = new Relation(); children.setName("children"); children.setOwner(descriptor); @@ -462,6 +468,7 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri fixture.driver.execute("CREATE TRIGGER order_db_default AFTER INSERT ON customer_order_data " + "BEGIN UPDATE customer_order_data SET memo = 'database trigger value' WHERE id = NEW.id; END"); var order = fixture.create("CustomerOrder", 100, "root value"); + order.updateProperty("business_counter", 9_007_199_254_740_993L); var item = fixture.create("OrderItem", 201, "first item"); item.setComment("create item"); var payment = fixture.create("Payment", 300, "payment value"); @@ -492,6 +499,8 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri } assertNotNull(orderFact); assertEquals("database trigger value", orderFact.path("fields").path("memo").asText()); + assertTrue(orderFact.path("fields").path("business_counter").isTextual()); + assertEquals("9007199254740993", orderFact.path("fields").path("business_counter").asText()); assertFalse(new CommittedMutationFact("CustomerOrder", "100", "create", null, "1", Map.of("memo", "database trigger value"), List.of()).toString().contains("database trigger value")); assertEquals("database trigger value", fixture.driver.queryForList( diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index b56ea43e..213f2091 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -85,7 +85,7 @@ private static String created() { {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, - "quantity":2},"trace_chain":[]}] + "quantity":"2"},"trace_chain":[]}] """; } @@ -210,7 +210,8 @@ private static long count(JdbcSqlExecutor sql, String table) { try { Target t = target(path); OutboxRecord first = record(1, "order-graph-0001", created()); - String unknown = created().replace("\"quantity\":2", "\"quantity\":2,\"not_in_model\":1"); + String unknown = created().replace("\"quantity\":\"2\"", + "\"quantity\":\"2\",\"not_in_model\":1"); expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), record(1, "unknown", unknown))); String reverseRelation = created().replace("\"status\":\"DRAFT\"", "\"status\":\"DRAFT\",\"items\":[]"); @@ -239,6 +240,30 @@ private static long count(JdbcSqlExecutor sql, String table) { } finally { Files.deleteIfExists(path); } } + @Test public void preservesLargeLongFieldWithoutJsonNumberRounding() throws Exception { + Path path = Files.createTempFile("teaql-model-target-large-long-", ".sqlite"); + try { + Target t = target(path); + String numeric = created().replace("\"quantity\":\"2\"", + "\"quantity\":9007199254740993"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "unsafe-json-number", numeric))); + for (String invalid : new String[] {"02", "-0", "9223372036854775808"}) { + String malformed = created().replace("\"quantity\":\"2\"", + "\"quantity\":\"" + invalid + "\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-integer", malformed))); + } + assertEquals(0, count(t.sql(), "order_data")); + String textual = created().replace("\"quantity\":\"2\"", + "\"quantity\":\"9007199254740993\""); + t.target().apply(t.context(), record(1, "lossless-long", textual)); + Object stored = t.sql().queryForList("SELECT quantity FROM order_item_data WHERE id = ?", + new Object[] {ITEM_ID}).get(0).get("quantity"); + assertEquals(9_007_199_254_740_993L, ((Number) stored).longValue()); + } finally { Files.deleteIfExists(path); } + } + @Test public void rejectsTriggerRewriteAndStalePredecessor() throws Exception { Path path = Files.createTempFile("teaql-model-target-trigger-", ".sqlite"); try { From 25e2368e0a8aacefe88f305c78358cd297220d9a Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 04:52:57 +0800 Subject: [PATCH 14/29] Validate committed calendar dates across runtimes --- .../SqliteModelCheckedCommittedTarget.java | 16 ++++++++-- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 10 ++++-- ...SqliteModelCheckedCommittedTargetTest.java | 31 +++++++++++++++++-- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java index 3cf1acc1..3c760415 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -194,9 +194,21 @@ private static Object decodeValue(PropertyDescriptor property, JsonNode value) { return positiveId(value.path("entity_id")); } Class type = property.getType() == null ? null : property.getType().javaType(); - if (type == String.class || type == java.util.UUID.class || type == java.time.LocalDate.class + if (type == java.time.LocalDate.class || type == java.sql.Date.class) { + if (!value.isTextual()) throw invalid("Date field has wrong type"); + String text = value.textValue(); + try { + if (!java.time.LocalDate.parse(text).toString().equals(text)) { + throw invalid("Date field has wrong type"); + } + } catch (java.time.format.DateTimeParseException failure) { + throw invalid("Date field has wrong type"); + } + return text; + } + if (type == String.class || type == java.util.UUID.class || type == java.time.LocalDateTime.class || type == java.time.Instant.class - || type == java.util.Date.class || type == java.sql.Date.class + || type == java.util.Date.class || type == java.sql.Timestamp.class || type != null && type.isEnum()) { if (!value.isTextual()) throw invalid("text/date field has wrong type"); return value.textValue(); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index 4294105a..a81440cf 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -34,11 +34,12 @@ public static final class GraphEntity extends BaseEntity { @Override public String typeName() { return type; } @Override public Object __internalGet(String field) { return field.equals("name") || field.equals("memo") || field.equals("children") - || field.equals("business_counter") ? values.get(field) : super.__internalGet(field); + || field.equals("business_counter") || field.equals("planned_date") + ? values.get(field) : super.__internalGet(field); } @Override public void __internalSet(String field, Object value) { if (field.equals("name") || field.equals("memo") || field.equals("children") - || field.equals("business_counter")) values.put(field, value); + || field.equals("business_counter") || field.equals("planned_date")) values.put(field, value); else super.__internalSet(field, value); } } @@ -114,6 +115,9 @@ static final class Fixture { if (captureCommittedMutations && type.equals("CustomerOrder")) { var counter = (GenericSQLProperty) descriptor.addSimpleProperty("business_counter", Long.class); counter.setColumnType("BIGINT"); + var plannedDate = (GenericSQLProperty) descriptor.addSimpleProperty( + "planned_date", java.time.LocalDate.class); + plannedDate.setColumnType("DATE"); } var children = new Relation(); children.setName("children"); @@ -469,6 +473,7 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri + "BEGIN UPDATE customer_order_data SET memo = 'database trigger value' WHERE id = NEW.id; END"); var order = fixture.create("CustomerOrder", 100, "root value"); order.updateProperty("business_counter", 9_007_199_254_740_993L); + order.updateProperty("planned_date", java.time.LocalDate.of(2026, 10, 16)); var item = fixture.create("OrderItem", 201, "first item"); item.setComment("create item"); var payment = fixture.create("Payment", 300, "payment value"); @@ -501,6 +506,7 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertEquals("database trigger value", orderFact.path("fields").path("memo").asText()); assertTrue(orderFact.path("fields").path("business_counter").isTextual()); assertEquals("9007199254740993", orderFact.path("fields").path("business_counter").asText()); + assertEquals("2026-10-16", orderFact.path("fields").path("planned_date").asText()); assertFalse(new CommittedMutationFact("CustomerOrder", "100", "create", null, "1", Map.of("memo", "database trigger value"), List.of()).toString().contains("database trigger value")); assertEquals("database trigger value", fixture.driver.queryForList( diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index 213f2091..379621c5 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -45,6 +45,7 @@ private static Target target(Path path) { order.addSimpleProperty("version", Long.class); order.addSimpleProperty("order_number", String.class); order.addSimpleProperty("status", String.class); + order.addSimpleProperty("planned_date", java.time.LocalDate.class); SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); metadata.register(order); EntityDescriptor item = new EntityDescriptor(); @@ -58,7 +59,7 @@ private static Target target(Path path) { DefaultUserContext context = new DefaultUserContext( TeaQLRuntime.builder().metadata(metadata).build()); sql.execute("CREATE TABLE order_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " - + "order_number TEXT NOT NULL, status TEXT NOT NULL)"); + + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE)"); sql.execute("CREATE TABLE order_item_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + "order_id INTEGER NOT NULL, quantity INTEGER NOT NULL)"); SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget(sql, SOURCE, HASH); @@ -81,7 +82,8 @@ private static String created() { return """ [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", "base_version":null,"committed_version":"1", - "fields":{"order_number":"ORD-0001","status":"DRAFT"},"trace_chain":[]}, + "fields":{"order_number":"ORD-0001","status":"DRAFT", + "planned_date":"2026-10-16"},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, @@ -146,6 +148,9 @@ private static long count(JdbcSqlExecutor sql, String table) { assertEquals("SUBMITTED", t.sql().queryForList( "SELECT status FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) .get(0).get("status")); + assertEquals("2026-10-16", t.sql().queryForList( + "SELECT planned_date FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("planned_date")); assertEquals(-2L, ((Number) t.sql().queryForList( "SELECT version FROM order_item_data WHERE id = ?", new Object[] {ITEM_ID}) .get(0).get("version")).longValue()); @@ -264,6 +269,28 @@ private static long count(JdbcSqlExecutor sql, String table) { } finally { Files.deleteIfExists(path); } } + @Test public void acceptsOnlyCanonicalCalendarDate() throws Exception { + Path path = Files.createTempFile("teaql-model-target-date-", ".sqlite"); + try { + Target t = target(path); + for (String invalid : new String[] {"2026-2-3", "2026-02-30", "2026-10-16T00:00:00Z"}) { + String malformed = created().replace("\"planned_date\":\"2026-10-16\"", + "\"planned_date\":\"" + invalid + "\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-date", malformed))); + } + String numeric = created().replace("\"planned_date\":\"2026-10-16\"", + "\"planned_date\":20261016"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "numeric-date", numeric))); + assertEquals(0, count(t.sql(), "order_data")); + t.target().apply(t.context(), record(1, "valid-date", created())); + assertEquals("2026-10-16", t.sql().queryForList( + "SELECT planned_date FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) + .get(0).get("planned_date")); + } finally { Files.deleteIfExists(path); } + } + @Test public void rejectsTriggerRewriteAndStalePredecessor() throws Exception { Path path = Files.createTempFile("teaql-model-target-trigger-", ".sqlite"); try { From a034eb0b4f4e9727f4818942e30ca402e114728a Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 04:57:47 +0800 Subject: [PATCH 15/29] Reject inexact SQLite decimal replication --- .../SqliteModelCheckedCommittedTarget.java | 3 ++ ...SqliteModelCheckedCommittedTargetTest.java | 28 +++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java index 3c760415..73c0a490 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -329,6 +329,9 @@ private static boolean sameValue(Field field, Object actual) { Object expected = field.value(); if (expected instanceof byte[] bytes) return actual instanceof byte[] other && Arrays.equals(bytes, other); if (expected == null || actual == null) return expected == actual; + // SQLite REAL is approximate even when Double.toString() happens to + // match the exact source decimal (for example, 0.1). + if (field.decimal() && (actual instanceof Double || actual instanceof Float)) return false; if (field.decimal() || expected instanceof Number && actual instanceof Number) { try { return new BigDecimal(expected.toString()).compareTo(new BigDecimal(actual.toString())) == 0; } catch (NumberFormatException failure) { return false; } diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index 379621c5..5653af49 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -46,6 +46,7 @@ private static Target target(Path path) { order.addSimpleProperty("order_number", String.class); order.addSimpleProperty("status", String.class); order.addSimpleProperty("planned_date", java.time.LocalDate.class); + order.addSimpleProperty("amount", java.math.BigDecimal.class); SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); metadata.register(order); EntityDescriptor item = new EntityDescriptor(); @@ -59,7 +60,7 @@ private static Target target(Path path) { DefaultUserContext context = new DefaultUserContext( TeaQLRuntime.builder().metadata(metadata).build()); sql.execute("CREATE TABLE order_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " - + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE)"); + + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE, amount NUMERIC)"); sql.execute("CREATE TABLE order_item_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + "order_id INTEGER NOT NULL, quantity INTEGER NOT NULL)"); SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget(sql, SOURCE, HASH); @@ -83,7 +84,7 @@ private static String created() { [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order_number":"ORD-0001","status":"DRAFT", - "planned_date":"2026-10-16"},"trace_chain":[]}, + "planned_date":"2026-10-16","amount":"0"},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, @@ -291,6 +292,29 @@ private static long count(JdbcSqlExecutor sql, String table) { } finally { Files.deleteIfExists(path); } } + @Test public void rejectsSqliteRealCoercionOfExactDecimal() throws Exception { + Path path = Files.createTempFile("teaql-model-target-decimal-", ".sqlite"); + try { + Target t = target(path); + for (String decimal : new String[] {"0.1", "123456789012345.6789"}) { + String inexact = created().replace("\"amount\":\"0\"", + "\"amount\":\"" + decimal + "\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "inexact-decimal", inexact))); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + assertEquals(0, count(t.sql(), "teaql_event_checkpoint")); + assertEquals(0, count(t.sql(), "teaql_event_applied_payload")); + } + t.target().apply(t.context(), record(1, "exact-decimal", created())); + Map stored = t.sql().queryForList( + "SELECT typeof(amount) AS storage_type, amount FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals("integer", stored.get("storage_type")); + assertEquals(0L, ((Number) stored.get("amount")).longValue()); + } finally { Files.deleteIfExists(path); } + } + @Test public void rejectsTriggerRewriteAndStalePredecessor() throws Exception { Path path = Files.createTempFile("teaql-model-target-trigger-", ".sqlite"); try { From 5f185b79e1c2ecb4f1e61bbc8bf5bcc942067bd0 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 05:02:51 +0800 Subject: [PATCH 16/29] Qualify committed scalar and null wire kinds --- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 18 +++++++- ...SqliteModelCheckedCommittedTargetTest.java | 43 ++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index a81440cf..3e5daf7c 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -35,11 +35,13 @@ public static final class GraphEntity extends BaseEntity { @Override public Object __internalGet(String field) { return field.equals("name") || field.equals("memo") || field.equals("children") || field.equals("business_counter") || field.equals("planned_date") + || field.equals("active") || field.equals("ratio") || field.equals("optional_note") ? values.get(field) : super.__internalGet(field); } @Override public void __internalSet(String field, Object value) { if (field.equals("name") || field.equals("memo") || field.equals("children") - || field.equals("business_counter") || field.equals("planned_date")) values.put(field, value); + || field.equals("business_counter") || field.equals("planned_date") + || field.equals("active") || field.equals("ratio") || field.equals("optional_note")) values.put(field, value); else super.__internalSet(field, value); } } @@ -118,6 +120,13 @@ static final class Fixture { var plannedDate = (GenericSQLProperty) descriptor.addSimpleProperty( "planned_date", java.time.LocalDate.class); plannedDate.setColumnType("DATE"); + var active = (GenericSQLProperty) descriptor.addSimpleProperty("active", Boolean.class); + active.setColumnType("BOOLEAN"); + var ratio = (GenericSQLProperty) descriptor.addSimpleProperty("ratio", Double.class); + ratio.setColumnType("DOUBLE"); + var optionalNote = (GenericSQLProperty) descriptor.addSimpleProperty( + "optional_note", String.class); + optionalNote.setColumnType("VARCHAR(255)"); } var children = new Relation(); children.setName("children"); @@ -474,6 +483,9 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri var order = fixture.create("CustomerOrder", 100, "root value"); order.updateProperty("business_counter", 9_007_199_254_740_993L); order.updateProperty("planned_date", java.time.LocalDate.of(2026, 10, 16)); + order.updateProperty("active", true); + order.updateProperty("ratio", 0.5); + order.updateProperty("optional_note", null); var item = fixture.create("OrderItem", 201, "first item"); item.setComment("create item"); var payment = fixture.create("Payment", 300, "payment value"); @@ -507,6 +519,10 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertTrue(orderFact.path("fields").path("business_counter").isTextual()); assertEquals("9007199254740993", orderFact.path("fields").path("business_counter").asText()); assertEquals("2026-10-16", orderFact.path("fields").path("planned_date").asText()); + assertTrue(orderFact.path("fields").path("active").isBoolean()); + assertTrue(orderFact.path("fields").path("active").booleanValue()); + assertEquals(0.5, orderFact.path("fields").path("ratio").doubleValue(), 0.0); + assertTrue(orderFact.path("fields").path("optional_note").isNull()); assertFalse(new CommittedMutationFact("CustomerOrder", "100", "create", null, "1", Map.of("memo", "database trigger value"), List.of()).toString().contains("database trigger value")); assertEquals("database trigger value", fixture.driver.queryForList( diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index 5653af49..13ec01a4 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -47,6 +47,9 @@ private static Target target(Path path) { order.addSimpleProperty("status", String.class); order.addSimpleProperty("planned_date", java.time.LocalDate.class); order.addSimpleProperty("amount", java.math.BigDecimal.class); + order.addSimpleProperty("active", Boolean.class).with("required", "true"); + order.addSimpleProperty("ratio", Double.class).with("required", "true"); + order.addSimpleProperty("optional_note", String.class); SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); metadata.register(order); EntityDescriptor item = new EntityDescriptor(); @@ -60,7 +63,8 @@ private static Target target(Path path) { DefaultUserContext context = new DefaultUserContext( TeaQLRuntime.builder().metadata(metadata).build()); sql.execute("CREATE TABLE order_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " - + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE, amount NUMERIC)"); + + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE, amount NUMERIC, " + + "active INTEGER NOT NULL, ratio REAL NOT NULL, optional_note TEXT)"); sql.execute("CREATE TABLE order_item_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + "order_id INTEGER NOT NULL, quantity INTEGER NOT NULL)"); SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget(sql, SOURCE, HASH); @@ -84,7 +88,8 @@ private static String created() { [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order_number":"ORD-0001","status":"DRAFT", - "planned_date":"2026-10-16","amount":"0"},"trace_chain":[]}, + "planned_date":"2026-10-16","amount":"0", + "active":true,"ratio":0.5,"optional_note":null},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, @@ -152,6 +157,12 @@ private static long count(JdbcSqlExecutor sql, String table) { assertEquals("2026-10-16", t.sql().queryForList( "SELECT planned_date FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) .get(0).get("planned_date")); + Map scalars = t.sql().queryForList( + "SELECT active, ratio, optional_note FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(1L, ((Number) scalars.get("active")).longValue()); + assertEquals(0.5, ((Number) scalars.get("ratio")).doubleValue(), 0.0); + assertNull(scalars.get("optional_note")); assertEquals(-2L, ((Number) t.sql().queryForList( "SELECT version FROM order_item_data WHERE id = ?", new Object[] {ITEM_ID}) .get(0).get("version")).longValue()); @@ -315,6 +326,34 @@ private static long count(JdbcSqlExecutor sql, String table) { } finally { Files.deleteIfExists(path); } } + @Test public void preservesBooleanFiniteFloatAndExplicitNull() throws Exception { + Path path = Files.createTempFile("teaql-model-target-scalar-kinds-", ".sqlite"); + try { + Target t = target(path); + String textBoolean = created().replace("\"active\":true", "\"active\":\"true\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "text-boolean", textBoolean))); + String textFloat = created().replace("\"ratio\":0.5", "\"ratio\":\"0.5\""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "text-float", textFloat))); + String nullBoolean = created().replace("\"active\":true", "\"active\":null"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "null-boolean", nullBoolean))); + String omitted = created().replace(",\"optional_note\":null", ""); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "omitted-note", omitted))); + assertEquals(0, count(t.sql(), "order_data")); + + t.target().apply(t.context(), record(1, "valid-scalar-kinds", created())); + Map stored = t.sql().queryForList( + "SELECT active, ratio, optional_note FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(1L, ((Number) stored.get("active")).longValue()); + assertEquals(0.5, ((Number) stored.get("ratio")).doubleValue(), 0.0); + assertNull(stored.get("optional_note")); + } finally { Files.deleteIfExists(path); } + } + @Test public void rejectsTriggerRewriteAndStalePredecessor() throws Exception { Path path = Files.createTempFile("teaql-model-target-trigger-", ".sqlite"); try { From 6ec1f4fd665532463c4cc19939f6cf41fa42da16 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 05:09:08 +0800 Subject: [PATCH 17/29] Qualify committed nullable field transitions --- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 28 +++++++++++ ...SqliteModelCheckedCommittedTargetTest.java | 47 +++++++++++++++++-- 2 files changed, 70 insertions(+), 5 deletions(-) diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index 3e5daf7c..e7dfb76f 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -586,6 +586,34 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertEquals(2, fact.path("fields").size()); } + @Test public void committedUpdateDistinguishesNullTransitionsFromUnchangedFields() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 911, "nullable order"); + order.updateProperty("optional_note", null); + order.auditAs("create nullable order").save(fixture.context); + + order.updateProperty("optional_note", "submitted note"); + order.auditAs("set optional note").save(fixture.context); + var outbox = fixture.committedRecorder.eventStore(); + JsonNode set = new ObjectMapper().readTree( + outbox.outboxAfter("java-graph-source", 1, 10).get(0).payloadJson()); + JsonNode fields = set.path("mutations").get(0).path("fields"); + assertEquals("submitted note", fields.path("optional_note").asText()); + assertEquals(1, fields.size()); + + order.updateProperty("optional_note", null); + order.auditAs("clear optional note").save(fixture.context); + JsonNode cleared = new ObjectMapper().readTree( + outbox.outboxAfter("java-graph-source", 2, 10).get(0).payloadJson()) + .path("mutations").get(0).path("fields"); + assertTrue(cleared.has("optional_note")); + assertTrue(cleared.path("optional_note").isNull()); + assertEquals(1, cleared.size()); + assertNull(fixture.driver.queryForList( + "SELECT optional_note FROM customer_order_data WHERE id = ?", new Object[] {911L}) + .get(0).get("optional_note")); + } + @Test public void committedRecoveryKeepsPredecessorAndNeedsNoChangedScalarField() throws Exception { var fixture = new Fixture(true); var order = fixture.create("CustomerOrder", 910, "recoverable"); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index 13ec01a4..b3c1dc05 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -50,6 +50,7 @@ private static Target target(Path path) { order.addSimpleProperty("active", Boolean.class).with("required", "true"); order.addSimpleProperty("ratio", Double.class).with("required", "true"); order.addSimpleProperty("optional_note", String.class); + order.addSimpleProperty("optional_label", String.class); SimpleEntityMetaFactory metadata = new SimpleEntityMetaFactory(); metadata.register(order); EntityDescriptor item = new EntityDescriptor(); @@ -64,7 +65,7 @@ private static Target target(Path path) { TeaQLRuntime.builder().metadata(metadata).build()); sql.execute("CREATE TABLE order_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE, amount NUMERIC, " - + "active INTEGER NOT NULL, ratio REAL NOT NULL, optional_note TEXT)"); + + "active INTEGER NOT NULL, ratio REAL NOT NULL, optional_note TEXT, optional_label TEXT)"); sql.execute("CREATE TABLE order_item_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + "order_id INTEGER NOT NULL, quantity INTEGER NOT NULL)"); SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget(sql, SOURCE, HASH); @@ -89,7 +90,8 @@ private static String created() { "base_version":null,"committed_version":"1", "fields":{"order_number":"ORD-0001","status":"DRAFT", "planned_date":"2026-10-16","amount":"0", - "active":true,"ratio":0.5,"optional_note":null},"trace_chain":[]}, + "active":true,"ratio":0.5,"optional_note":null, + "optional_label":"initial label"},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order":{"entity_type":"order","entity_id":"9007199254740993"}, @@ -100,7 +102,9 @@ private static String created() { private static String submitted() { return """ [{"entity_type":"order","entity_id":"9007199254740993","operation":"update", - "base_version":"1","committed_version":"2","fields":{"status":"SUBMITTED"},"trace_chain":[]}, + "base_version":"1","committed_version":"2", + "fields":{"status":"SUBMITTED","optional_note":"submitted note", + "optional_label":null},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"delete", "base_version":"1","committed_version":"-2","fields":{},"trace_chain":[]}] """; @@ -158,11 +162,12 @@ private static long count(JdbcSqlExecutor sql, String table) { "SELECT planned_date FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) .get(0).get("planned_date")); Map scalars = t.sql().queryForList( - "SELECT active, ratio, optional_note FROM order_data WHERE id = ?", + "SELECT active, ratio, optional_note, optional_label FROM order_data WHERE id = ?", new Object[] {ORDER_ID}).get(0); assertEquals(1L, ((Number) scalars.get("active")).longValue()); assertEquals(0.5, ((Number) scalars.get("ratio")).doubleValue(), 0.0); - assertNull(scalars.get("optional_note")); + assertEquals("submitted note", scalars.get("optional_note")); + assertNull(scalars.get("optional_label")); assertEquals(-2L, ((Number) t.sql().queryForList( "SELECT version FROM order_item_data WHERE id = ?", new Object[] {ITEM_ID}) .get(0).get("version")).longValue()); @@ -354,6 +359,38 @@ private static long count(JdbcSqlExecutor sql, String table) { } finally { Files.deleteIfExists(path); } } + @Test public void appliesNullableTransitionsWithoutTreatingOmissionAsNull() throws Exception { + Path path = Files.createTempFile("teaql-model-target-null-transition-", ".sqlite"); + try { + Target t = target(path); + t.target().apply(t.context(), record(1, "created-null-transition", created())); + String nullRequired = submitted().replace("\"status\":\"SUBMITTED\"", + "\"status\":\"SUBMITTED\",\"active\":null"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(2, "null-required-update", nullRequired))); + String wrongKind = submitted().replace("\"optional_note\":\"submitted note\"", + "\"optional_note\":123"); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(2, "wrong-note-kind", wrongKind))); + assertEquals(1, count(t.sql(), "teaql_event_inbox")); + Map unchanged = t.sql().queryForList( + "SELECT version, optional_note, optional_label FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(1L, ((Number) unchanged.get("version")).longValue()); + assertNull(unchanged.get("optional_note")); + assertEquals("initial label", unchanged.get("optional_label")); + + t.target().apply(t.context(), record(2, "changed-null-transition", submitted())); + Map stored = t.sql().queryForList( + "SELECT version, active, optional_note, optional_label FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals(2L, ((Number) stored.get("version")).longValue()); + assertEquals(1L, ((Number) stored.get("active")).longValue()); + assertEquals("submitted note", stored.get("optional_note")); + assertNull(stored.get("optional_label")); + } finally { Files.deleteIfExists(path); } + } + @Test public void rejectsTriggerRewriteAndStalePredecessor() throws Exception { Path path = Files.createTempFile("teaql-model-target-trigger-", ".sqlite"); try { From 26d4b75d8b8234a50221459998b09c486a2b0be6 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 05:20:31 +0800 Subject: [PATCH 18/29] Qualify committed absolute instant codec on SQLite --- .../teaql/provider/jdbc/JdbcSqlExecutor.java | 21 ++++++++++ .../java/io/teaql/runtime/TeaQLRuntime.java | 29 +++++++++++-- .../sql/portable/PortableSQLRepository.java | 9 ++++ .../SqliteModelCheckedCommittedTarget.java | 19 ++++++--- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 42 ++++++++++++++++++- ...SqliteModelCheckedCommittedTargetTest.java | 36 +++++++++++++++- 6 files changed, 144 insertions(+), 12 deletions(-) diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index c433a51e..77222765 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -240,6 +240,13 @@ public V get(int columnIndex, Class type) { private static V convertColumn(Class type, Object value) { if (value == null || type.isInstance(value)) return (V) value; if (value instanceof Number number) { + if (type == java.time.Instant.class) { + if (!(number instanceof Long || number instanceof Integer + || number instanceof Short || number instanceof Byte)) { + throw new IllegalArgumentException("Instant JDBC value is not an exact integer"); + } + return (V) java.time.Instant.ofEpochMilli(number.longValue()); + } if (type == Long.class) return (V) Long.valueOf(number.longValue()); if (type == Integer.class) return (V) Integer.valueOf(number.intValue()); if (type == Double.class) return (V) Double.valueOf(number.doubleValue()); @@ -256,6 +263,12 @@ private static V convertColumn(Class type, Object value) { if (type == java.time.LocalDateTime.class && value instanceof java.sql.Timestamp timestamp) { return (V) timestamp.toLocalDateTime(); } + if (type == java.time.Instant.class && value instanceof java.sql.Timestamp timestamp) { + return (V) timestamp.toInstant(); + } + if (type == java.time.Instant.class && value instanceof CharSequence text) { + return (V) java.time.Instant.parse(text); + } if (type == java.time.LocalDateTime.class) { return (V) java.time.LocalDateTime.parse(String.valueOf(value).replace(' ', 'T')); } @@ -442,6 +455,14 @@ static void bind(PreparedStatement statement, int index, Object value) throws SQ } else { statement.setTimestamp(index, java.sql.Timestamp.valueOf(dateTime)); } + } else if (value instanceof java.time.Instant instant && isSqlite(statement)) { + if (instant.getNano() % 1_000_000 != 0) { + throw new SQLException("Instant has precision finer than milliseconds"); + } + try { statement.setLong(index, instant.toEpochMilli()); } + catch (ArithmeticException overflow) { + throw new SQLException("Instant exceeds epoch-millisecond range", overflow); + } } else if (value instanceof java.time.LocalTime time) { if (isSqlite(statement)) { statement.setString(index, time.toString()); diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 53e89521..55613bb8 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -1272,19 +1272,40 @@ private Object committedFieldValue(PropertyDescriptor property, Object value) { if (!Float.isFinite(number)) throw new TeaQLRuntimeException("Non-finite committed numeric value"); return number; } - if (value instanceof java.time.temporal.TemporalAccessor || value instanceof java.util.UUID - || value instanceof Enum) return value.toString(); + if (value instanceof java.time.Instant instant) return committedInstantMillis(instant); + if (value instanceof java.time.LocalDate date) return date.toString(); + if (value instanceof java.time.LocalDateTime) { + throw new TeaQLRuntimeException("Zone-free LocalDateTime has no committed instant mapping"); + } + if (value instanceof java.time.temporal.TemporalAccessor) { + throw new TeaQLRuntimeException("Committed temporal field has unsupported type: " + property.getName()); + } + if (value instanceof java.util.UUID || value instanceof Enum) return value.toString(); if (value instanceof java.util.Date date) { // JDBC date/time subclasses deliberately throw from toInstant(). // Keep this model-level runtime independent of the java.sql module. - try { return date.toInstant().toString(); } - catch (UnsupportedOperationException unsupported) { return date.toString(); } + try { return committedInstantMillis(date.toInstant()); } + catch (UnsupportedOperationException unsupported) { + String calendarDate = date.toString(); + if (calendarDate.matches("[0-9]{4}-[0-9]{2}-[0-9]{2}")) return calendarDate; + throw new TeaQLRuntimeException("Committed temporal field has unsupported type: " + property.getName()); + } } if (value instanceof byte[] bytes) return Map.of("$binary_base64", java.util.Base64.getEncoder().encodeToString(bytes)); throw new TeaQLRuntimeException("Committed field has unsupported type: " + property.getName()); } + private static String committedInstantMillis(java.time.Instant instant) { + if (instant.getNano() % 1_000_000 != 0) { + throw new TeaQLRuntimeException("Committed instant has finer than millisecond precision"); + } + try { return Long.toString(instant.toEpochMilli()); } + catch (ArithmeticException overflow) { + throw new TeaQLRuntimeException("Committed instant exceeds epoch-millisecond range"); + } + } + private record PersistenceState( Long version, io.teaql.core.EntityStatus status, boolean versionLoaded) {} diff --git a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java index 8cf465b4..9d5e340a 100644 --- a/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java +++ b/teaql-sql-portable/src/main/java/io/teaql/core/sql/portable/PortableSQLRepository.java @@ -1403,6 +1403,15 @@ static Object convertTemporalColumnValue(Class targetType, Object value) { && value instanceof java.sql.Timestamp timestamp) { return timestamp.toLocalDateTime(); } + if (targetType == java.time.Instant.class) { + if (value instanceof Long || value instanceof Integer || value instanceof Short + || value instanceof Byte) { + return java.time.Instant.ofEpochMilli(((Number) value).longValue()); + } + if (value instanceof java.sql.Timestamp timestamp) return timestamp.toInstant(); + if (value instanceof CharSequence text) return java.time.Instant.parse(text); + throw new IllegalArgumentException("Instant readback is not an exact timestamp value"); + } if (targetType == java.time.LocalDateTime.class) { // SQLite returns text and some JDBC drivers return vendor temporal // wrappers. Both expose the standard timestamp representation. diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java index 73c0a490..24d72d74 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -206,11 +206,20 @@ private static Object decodeValue(PropertyDescriptor property, JsonNode value) { } return text; } - if (type == String.class || type == java.util.UUID.class - || type == java.time.LocalDateTime.class || type == java.time.Instant.class - || type == java.util.Date.class - || type == java.sql.Timestamp.class || type != null && type.isEnum()) { - if (!value.isTextual()) throw invalid("text/date field has wrong type"); + if (type == java.time.Instant.class || type == java.util.Date.class + || type == java.sql.Timestamp.class) { + if (!value.isTextual() || !value.textValue().matches("-?(0|[1-9][0-9]*)") + || value.textValue().equals("-0")) { + throw invalid("Timestamp field has wrong type"); + } + try { return Long.parseLong(value.textValue()); } + catch (NumberFormatException failure) { throw invalid("Timestamp field has wrong type"); } + } + if (type == java.time.LocalDateTime.class) { + throw invalid("zone-free LocalDateTime has no committed instant mapping"); + } + if (type == String.class || type == java.util.UUID.class || type != null && type.isEnum()) { + if (!value.isTextual()) throw invalid("text field has wrong type"); return value.textValue(); } if (type == Boolean.class || type == boolean.class) { diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index e7dfb76f..90c06f07 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -36,12 +36,14 @@ public static final class GraphEntity extends BaseEntity { return field.equals("name") || field.equals("memo") || field.equals("children") || field.equals("business_counter") || field.equals("planned_date") || field.equals("active") || field.equals("ratio") || field.equals("optional_note") + || field.equals("occurred_at") || field.equals("wall_time") ? values.get(field) : super.__internalGet(field); } @Override public void __internalSet(String field, Object value) { if (field.equals("name") || field.equals("memo") || field.equals("children") || field.equals("business_counter") || field.equals("planned_date") - || field.equals("active") || field.equals("ratio") || field.equals("optional_note")) values.put(field, value); + || field.equals("active") || field.equals("ratio") || field.equals("optional_note") + || field.equals("occurred_at") || field.equals("wall_time")) values.put(field, value); else super.__internalSet(field, value); } } @@ -127,6 +129,12 @@ static final class Fixture { var optionalNote = (GenericSQLProperty) descriptor.addSimpleProperty( "optional_note", String.class); optionalNote.setColumnType("VARCHAR(255)"); + var occurredAt = (GenericSQLProperty) descriptor.addSimpleProperty( + "occurred_at", java.time.Instant.class); + occurredAt.setColumnType("BIGINT"); + var wallTime = (GenericSQLProperty) descriptor.addSimpleProperty( + "wall_time", java.time.LocalDateTime.class); + wallTime.setColumnType("TIMESTAMP"); } var children = new Relation(); children.setName("children"); @@ -486,6 +494,7 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri order.updateProperty("active", true); order.updateProperty("ratio", 0.5); order.updateProperty("optional_note", null); + order.updateProperty("occurred_at", java.time.Instant.parse("1960-01-01T00:00:00.123Z")); var item = fixture.create("OrderItem", 201, "first item"); item.setComment("create item"); var payment = fixture.create("Payment", 300, "payment value"); @@ -523,11 +532,17 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertTrue(orderFact.path("fields").path("active").booleanValue()); assertEquals(0.5, orderFact.path("fields").path("ratio").doubleValue(), 0.0); assertTrue(orderFact.path("fields").path("optional_note").isNull()); + assertEquals("-315619199877", orderFact.path("fields").path("occurred_at").asText()); assertFalse(new CommittedMutationFact("CustomerOrder", "100", "create", null, "1", Map.of("memo", "database trigger value"), List.of()).toString().contains("database trigger value")); assertEquals("database trigger value", fixture.driver.queryForList( "SELECT memo FROM customer_order_data WHERE id = ?", new Object[] {100L}) .get(0).get("memo")); + Map instant = fixture.driver.queryForList( + "SELECT typeof(occurred_at) AS storage_type, occurred_at " + + "FROM customer_order_data WHERE id = ?", new Object[] {100L}).get(0); + assertEquals("integer", instant.get("storage_type")); + assertEquals(-315_619_199_877L, ((Number) instant.get("occurred_at")).longValue()); item.updateProperty("name", "revised item"); item.setComment("revise item"); @@ -614,6 +629,31 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri .get(0).get("optional_note")); } + @Test public void committedLocalDateTimeFailsClosedWithoutAnInstantMapping() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 912, "wall-clock order"); + order.updateProperty("wall_time", java.time.LocalDateTime.of(2026, 10, 16, 8, 0)); + var failure = assertThrows(RuntimeException.class, + () -> order.auditAs("reject ambiguous wall time").save(fixture.context)); + assertTrue(failure.getMessage().contains("LocalDateTime")); + assertTrue(fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {912L}).isEmpty()); + assertTrue(fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).isEmpty()); + } + + @Test public void committedInstantRejectsSubMillisecondPrecision() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 913, "precise order"); + order.updateProperty("occurred_at", java.time.Instant.parse("2026-10-16T00:00:00.123456Z")); + assertThrows(RuntimeException.class, + () -> order.auditAs("reject submillisecond instant").save(fixture.context)); + assertTrue(fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {913L}).isEmpty()); + assertTrue(fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).isEmpty()); + } + @Test public void committedRecoveryKeepsPredecessorAndNeedsNoChangedScalarField() throws Exception { var fixture = new Fixture(true); var order = fixture.create("CustomerOrder", 910, "recoverable"); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index b3c1dc05..c2884d19 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -46,6 +46,7 @@ private static Target target(Path path) { order.addSimpleProperty("order_number", String.class); order.addSimpleProperty("status", String.class); order.addSimpleProperty("planned_date", java.time.LocalDate.class); + order.addSimpleProperty("occurred_at", java.time.Instant.class).with("required", "true"); order.addSimpleProperty("amount", java.math.BigDecimal.class); order.addSimpleProperty("active", Boolean.class).with("required", "true"); order.addSimpleProperty("ratio", Double.class).with("required", "true"); @@ -64,7 +65,8 @@ private static Target target(Path path) { DefaultUserContext context = new DefaultUserContext( TeaQLRuntime.builder().metadata(metadata).build()); sql.execute("CREATE TABLE order_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " - + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE, amount NUMERIC, " + + "order_number TEXT NOT NULL, status TEXT NOT NULL, planned_date DATE, " + + "occurred_at INTEGER NOT NULL, amount NUMERIC, " + "active INTEGER NOT NULL, ratio REAL NOT NULL, optional_note TEXT, optional_label TEXT)"); sql.execute("CREATE TABLE order_item_data (id INTEGER PRIMARY KEY, version INTEGER NOT NULL, " + "order_id INTEGER NOT NULL, quantity INTEGER NOT NULL)"); @@ -89,7 +91,8 @@ private static String created() { [{"entity_type":"order","entity_id":"9007199254740993","operation":"create", "base_version":null,"committed_version":"1", "fields":{"order_number":"ORD-0001","status":"DRAFT", - "planned_date":"2026-10-16","amount":"0", + "planned_date":"2026-10-16","occurred_at":"-315619199877", + "amount":"0", "active":true,"ratio":0.5,"optional_note":null, "optional_label":"initial label"},"trace_chain":[]}, {"entity_type":"order_item","entity_id":"9007199254740994","operation":"create", @@ -161,6 +164,11 @@ private static long count(JdbcSqlExecutor sql, String table) { assertEquals("2026-10-16", t.sql().queryForList( "SELECT planned_date FROM order_data WHERE id = ?", new Object[] {ORDER_ID}) .get(0).get("planned_date")); + Map instant = t.sql().queryForList( + "SELECT typeof(occurred_at) AS storage_type, occurred_at FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals("integer", instant.get("storage_type")); + assertEquals(-315_619_199_877L, ((Number) instant.get("occurred_at")).longValue()); Map scalars = t.sql().queryForList( "SELECT active, ratio, optional_note, optional_label FROM order_data WHERE id = ?", new Object[] {ORDER_ID}).get(0); @@ -308,6 +316,30 @@ private static long count(JdbcSqlExecutor sql, String table) { } finally { Files.deleteIfExists(path); } } + @Test public void acceptsOnlyLosslessInstantMillisecondText() throws Exception { + Path path = Files.createTempFile("teaql-model-target-instant-", ".sqlite"); + try { + Target t = target(path); + for (String invalid : new String[] { + "-315619199877", "\"1960-01-01T00:00:00.123Z\"", + "\"1960-01-01T00:00:00.123\"", "\"-0315619199877\"", + "\"-0\"", "\"9223372036854775808\""}) { + String malformed = created().replace("\"occurred_at\":\"-315619199877\"", + "\"occurred_at\":" + invalid); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "invalid-instant", malformed))); + } + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + t.target().apply(t.context(), record(1, "valid-instant", created())); + Map stored = t.sql().queryForList( + "SELECT typeof(occurred_at) AS storage_type, occurred_at FROM order_data WHERE id = ?", + new Object[] {ORDER_ID}).get(0); + assertEquals("integer", stored.get("storage_type")); + assertEquals(-315_619_199_877L, ((Number) stored.get("occurred_at")).longValue()); + } finally { Files.deleteIfExists(path); } + } + @Test public void rejectsSqliteRealCoercionOfExactDecimal() throws Exception { Path path = Files.createTempFile("teaql-model-target-decimal-", ".sqlite"); try { From d3f89ccd9df662a0f9cdf3e4a0af562a737e6b0a Mon Sep 17 00:00:00 2001 From: Philip Z Date: Fri, 16 Oct 2026 06:49:21 +0800 Subject: [PATCH 19/29] Reject unmodeled and out-of-range replicated Decimal values --- .../SqliteModelCheckedCommittedTarget.java | 31 ++++++++++++++++++- ...SqliteModelCheckedCommittedTargetTest.java | 21 +++++++++++-- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java index 24d72d74..941e3761 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteModelCheckedCommittedTarget.java @@ -20,6 +20,8 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Startup-owned, one-source SQLite target for committed model facts. This is @@ -29,6 +31,8 @@ public final class SqliteModelCheckedCommittedTarget { private static final Set TRACE_KINDS = Set.of( "OPERATION", "REQUEST", "RELATION", "ENTITY", "PROVIDER", "SQL", "COMMENT", "PURPOSE", "AUDIT_REASON"); + private static final Pattern DECLARED_DECIMAL = Pattern.compile( + "(?i)\\s*(?:NUMERIC|DECIMAL)\\s*\\(\\s*([0-9]+)\\s*,\\s*([0-9]+)\\s*\\)\\s*"); private final SqlExecutionAdapter sql; private final SqliteCommittedEventStore events; private final String sourceRuntimeId; @@ -254,7 +258,11 @@ private static Object decodeValue(PropertyDescriptor property, JsonNode value) { } return new java.math.BigInteger(value.textValue()).toString(); } - return new BigDecimal(value.textValue()).toPlainString(); + BigDecimal decimal = new BigDecimal(value.textValue()); + if (!fitsDeclaredDecimal(property, decimal)) { + throw invalid("Decimal lacks or exceeds modeled precision/scale"); + } + return decimal.toPlainString(); } catch (NumberFormatException failure) { throw invalid("Decimal field is invalid"); } } @@ -354,6 +362,27 @@ private static Mapping mapping(PropertyDescriptor property) { return new Mapping(columns.get(0).getTableName(), columns.get(0).getColumnName()); } + private static boolean fitsDeclaredDecimal(PropertyDescriptor property, BigDecimal value) { + List columns = SQLPropertyUtil.getColumns(property); + if (columns.size() != 1) return false; + String declaredType = columns.get(0).getType(); + if (declaredType == null) return false; + Matcher declaration = DECLARED_DECIMAL.matcher(declaredType); + if (!declaration.matches()) return false; + try { + int precision = Integer.parseInt(declaration.group(1)); + int scale = Integer.parseInt(declaration.group(2)); + if (precision < 1 || scale > precision) return false; + BigDecimal normalized = value.stripTrailingZeros(); + long integerDigits = value.signum() == 0 ? 0 + : Math.max(0L, (long) normalized.precision() - normalized.scale()); + long fractionalDigits = Math.max(0L, normalized.scale()); + return integerDigits <= precision - scale && fractionalDigits <= scale; + } catch (NumberFormatException invalidDeclaration) { + return false; + } + } + private static String quoted(String identifier) { if (identifier == null || identifier.isBlank()) throw invalid("model has an empty SQL identifier"); return "\"" + identifier.replace("\"", "\"\"") + "\""; diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java index c2884d19..a49f170e 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/SqliteModelCheckedCommittedTargetTest.java @@ -36,6 +36,10 @@ private record Target(JdbcSqlExecutor sql, SqliteModelCheckedCommittedTarget tar DefaultUserContext context) { } private static Target target(Path path) { + return target(path, "NUMERIC(19,7)"); + } + + private static Target target(Path path, String amountType) { SQLiteDataSource source = new SQLiteDataSource(); source.setUrl("jdbc:sqlite:" + path); JdbcSqlExecutor sql = new JdbcSqlExecutor(source); @@ -47,7 +51,8 @@ private static Target target(Path path) { order.addSimpleProperty("status", String.class); order.addSimpleProperty("planned_date", java.time.LocalDate.class); order.addSimpleProperty("occurred_at", java.time.Instant.class).with("required", "true"); - order.addSimpleProperty("amount", java.math.BigDecimal.class); + var amount = order.addSimpleProperty("amount", java.math.BigDecimal.class); + if (amountType != null) amount.with("sqlType", amountType); order.addSimpleProperty("active", Boolean.class).with("required", "true"); order.addSimpleProperty("ratio", Double.class).with("required", "true"); order.addSimpleProperty("optional_note", String.class); @@ -344,7 +349,8 @@ private static long count(JdbcSqlExecutor sql, String table) { Path path = Files.createTempFile("teaql-model-target-decimal-", ".sqlite"); try { Target t = target(path); - for (String decimal : new String[] {"0.1", "123456789012345.6789"}) { + for (String decimal : new String[] { + "0.1", "123456789012345.6789", "0.12345678", "1234567890123"}) { String inexact = created().replace("\"amount\":\"0\"", "\"amount\":\"" + decimal + "\""); expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), @@ -363,6 +369,17 @@ private static long count(JdbcSqlExecutor sql, String table) { } finally { Files.deleteIfExists(path); } } + @Test public void rejectsDecimalWithoutExplicitModeledPrecisionAndScale() throws Exception { + Path path = Files.createTempFile("teaql-model-target-untyped-decimal-", ".sqlite"); + try { + Target t = target(path, null); + expect("INVALID_ENVELOPE", () -> t.target().apply(t.context(), + record(1, "untyped-decimal", created()))); + assertEquals(0, count(t.sql(), "order_data")); + assertEquals(0, count(t.sql(), "teaql_event_inbox")); + } finally { Files.deleteIfExists(path); } + } + @Test public void preservesBooleanFiniteFloatAndExplicitNull() throws Exception { Path path = Files.createTempFile("teaql-model-target-scalar-kinds-", ".sqlite"); try { From e1ac957a6f2958f6e0165d466eeeb7c5c6d8769c Mon Sep 17 00:00:00 2001 From: Philip Z Date: Sun, 18 Oct 2026 04:08:09 +0800 Subject: [PATCH 20/29] Exercise committed SQLite-to-DuckDB delivery in runtime example --- examples/committed-events/README.md | 15 +++-- examples/committed-events/pom.xml | 2 + .../committed/CommittedEventsExampleTest.java | 56 +++++++++++++++++++ examples/committed-events/verify.sh | 2 +- 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/examples/committed-events/README.md b/examples/committed-events/README.md index 9debbca8..77e34679 100644 --- a/examples/committed-events/README.md +++ b/examples/committed-events/README.md @@ -2,10 +2,11 @@ Run `bash examples/committed-events/verify.sh`. It executes a small, real `Order.save(context)` flow twice against a local Java source. Each round uses -fresh source and target SQLite files. The test reopens the Outbox, delivers -the two events through a bounded publisher into a model-checked target, -verifies the target rows and duplicate acknowledgement, then injects a later -source write failure. No generated ERP model or remote service is required. +fresh source/target SQLite files and an independent DuckDB target. The test +reopens the Outbox, delivers the two events through bounded publishers into +both model-checked targets, verifies their rows and the SQLite target duplicate +acknowledgement, then injects a later source write failure. No generated ERP +model or remote service is required. The example configures `SqliteCommittedMutationRecorder` at trusted runtime startup; `context.ensureSchema()` installs its infrastructure tables. A database trigger @@ -14,6 +15,8 @@ the pre-save entity value. Subsequent update produces sequence 2. When the Outbox table is made unavailable, the attempted third save rolls back its business update and emits no committed event. -This is **not** authenticated network replication. The in-process sink proves +This is **not** authenticated network replication. The in-process sinks prove local cursor/target behavior; transport identity, retry budgets, snapshot-plus- -tail, cross-language codec breadth and dynamic fields remain separate work. +tail and dynamic fields remain separate work. The included typed wire tests +cover selected scalar, null, date, instant and Decimal boundaries, not every +possible model type. diff --git a/examples/committed-events/pom.xml b/examples/committed-events/pom.xml index 2dc08742..18c0e613 100644 --- a/examples/committed-events/pom.xml +++ b/examples/committed-events/pom.xml @@ -10,9 +10,11 @@ io.teaqlteaql-runtime io.teaqlteaql-sqlite + io.teaqlteaql-duckdb${project.version} io.teaqlteaql-provider-jdbc com.fasterxml.jackson.corejackson-databind org.xerialsqlite-jdbc3.53.2.1 + org.duckdbduckdb_jdbc1.5.5.1 junitjunittest diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java index 2c469969..7fbf5eb1 100644 --- a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java @@ -3,10 +3,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.teaql.core.BaseEntity; +import io.teaql.core.duck.DuckDbCommittedInbox; +import io.teaql.core.duck.DuckDbModelCheckedCommittedTarget; import io.teaql.core.meta.SimpleEntityMetaFactory; import io.teaql.core.sql.GenericSQLProperty; import io.teaql.core.sql.SQLEntityDescriptor; import io.teaql.core.sqlite.SqliteCommittedEventStore; +import io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome; import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; import io.teaql.core.sqlite.SqliteModelCheckedCommittedTarget; import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; @@ -17,14 +20,37 @@ import org.junit.Test; import org.sqlite.SQLiteDataSource; +import javax.sql.DataSource; +import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; import java.util.HashMap; import java.util.Map; +import java.util.logging.Logger; import static org.junit.Assert.*; public class CommittedEventsExampleTest { + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { return DriverManager.getConnection(url); } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class type) throws SQLException { throw new SQLException("not a wrapper"); } + @Override public boolean isWrapperFor(Class type) { return false; } + } + public static final class Order extends BaseEntity { private final Map fields = new HashMap<>(); @@ -121,6 +147,36 @@ public static final class Order extends BaseEntity { new Object[0]).size()); replicaRuntime.close(); + Path duckDirectory = Files.createTempDirectory("teaql-committed-duckdb-example-"); + duckDirectory.toFile().deleteOnExit(); + Path duckPath = duckDirectory.resolve("target.db"); + duckPath.toFile().deleteOnExit(); + JdbcSqlExecutor duckSql = new JdbcSqlExecutor( + new DriverManagerDataSource("jdbc:duckdb:" + duckPath)); + duckSql.execute("CREATE TABLE order_data (id BIGINT PRIMARY KEY, version BIGINT NOT NULL, " + + "name VARCHAR, memo VARCHAR)"); + DuckDbModelCheckedCommittedTarget duckTarget = + new DuckDbModelCheckedCommittedTarget(duckSql, "example-java-source", + "sha256:" + "0".repeat(64)); + duckTarget.ensureInboxSchema(); + var duckPublished = new SqliteCommittedOutboxPublisher(reopened, "example-java-source", + "example-duckdb-analytics", "sha256:" + "0".repeat(64)) + .publishBatch(10, record -> { + var outcome = duckTarget.apply(context, new DuckDbCommittedInbox.Record( + record.sourceRuntimeId(), record.sourceSequence(), record.changeSetId(), + record.modelHash(), record.payloadSha256(), record.payloadJson())); + return new InboxOutcome(outcome.duplicate(), outcome.checkpoint()); + }); + assertEquals(2, duckPublished.acknowledged()); + assertEquals(0, duckPublished.status().backlog()); + var duckOrder = duckSql.queryForList("SELECT version, name, memo FROM order_data WHERE id = ?", + new Object[] {42L}).get(0); + assertEquals(2L, ((Number) duckOrder.get("version")).longValue()); + assertEquals("submitted", duckOrder.get("name")); + assertEquals("set by database", duckOrder.get("memo")); + assertEquals(2, duckSql.queryForList("SELECT * FROM teaql_event_applied_payload", + new Object[0]).size()); + sql.execute("DROP TABLE teaql_event_outbox"); order.updateProperty("name", "should roll back"); assertThrows(RuntimeException.class, diff --git a/examples/committed-events/verify.sh b/examples/committed-events/verify.sh index dc4f3da2..cfe2255f 100755 --- a/examples/committed-events/verify.sh +++ b/examples/committed-events/verify.sh @@ -3,7 +3,7 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$repo" for round in 1 2; do - echo "Java committed-events example round $round" mvn -B -q -pl examples/committed-events -am \ -Dtest=CommittedEventsExampleTest -Dsurefire.failIfNoSpecifiedTests=false test + echo "PASS Java committed-events example round $round" done From 35abd8291f80dbc1f34d112775fdc03cf6e97aa3 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Sun, 18 Oct 2026 04:32:58 +0800 Subject: [PATCH 21/29] Verify committed DuckDB target through generated Q API --- examples/committed-events/README.md | 10 +- examples/committed-events/verify.sh | 15 ++ examples/standard-order/pom.xml | 2 + .../StandardOrderCommittedTargetTest.java | 132 ++++++++++++++++++ 4 files changed, 157 insertions(+), 2 deletions(-) create mode 100644 examples/standard-order/src/test/java/com/example/requesttreeorderservice/StandardOrderCommittedTargetTest.java diff --git a/examples/committed-events/README.md b/examples/committed-events/README.md index 77e34679..c1929b8a 100644 --- a/examples/committed-events/README.md +++ b/examples/committed-events/README.md @@ -5,8 +5,8 @@ Run `bash examples/committed-events/verify.sh`. It executes a small, real fresh source/target SQLite files and an independent DuckDB target. The test reopens the Outbox, delivers the two events through bounded publishers into both model-checked targets, verifies their rows and the SQLite target duplicate -acknowledgement, then injects a later source write failure. No generated ERP -model or remote service is required. +acknowledgement, then injects a later source write failure. This small fixture +itself requires no generated ERP model or remote service. The example configures `SqliteCommittedMutationRecorder` at trusted runtime startup; `context.ensureSchema()` installs its infrastructure tables. A database trigger @@ -20,3 +20,9 @@ local cursor/target behavior; transport identity, retry budgets, snapshot-plus- tail and dynamic fields remain separate work. The included typed wire tests cover selected scalar, null, date, instant and Decimal boundaries, not every possible model type. + +The verifier also runs an application-owned generated Standard Order consumer +twice. It creates a Customer through generated audited Save on SQLite, delivers +one committed set to a model-checked DuckDB target, and reads the target through +generated Q. The checked-in generated library remains byte-for-byte unchanged; +public-package and production-network qualification are separate. diff --git a/examples/committed-events/verify.sh b/examples/committed-events/verify.sh index cfe2255f..192b9394 100755 --- a/examples/committed-events/verify.sh +++ b/examples/committed-events/verify.sh @@ -2,8 +2,23 @@ set -euo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$repo" +evidence="$(mktemp -d -t teaql-java-committed-generated.XXXXXXXX)" +generated_library_hash() { + (cd "$repo/examples/standard-order" && rg --files generated/lib -g '!target/**' \ + | LC_ALL=C sort | xargs -d '\n' sha256sum) +} +generated_library_hash > "$evidence/generated-before.sha256" for round in 1 2; do mvn -B -q -pl examples/committed-events -am \ -Dtest=CommittedEventsExampleTest -Dsurefire.failIfNoSpecifiedTests=false test echo "PASS Java committed-events example round $round" + mvn -B -q -Pruntime-examples -pl examples/standard-order -am \ + -Dtest=StandardOrderCommittedTargetTest -Dsurefire.failIfNoSpecifiedTests=false test \ + 2>&1 | tee "$evidence/generated-$round.log" + rg -Fq 'JAVA_COMMITTED_GENERATED_Q_PASS source=sqlite target=duckdb outbox=1' \ + "$evidence/generated-$round.log" + generated_library_hash > "$evidence/generated-after.sha256" + cmp "$evidence/generated-before.sha256" "$evidence/generated-after.sha256" + echo "PASS Java generated committed Q example round $round" done +echo "Java generated committed Q evidence: $evidence" diff --git a/examples/standard-order/pom.xml b/examples/standard-order/pom.xml index 76ccc998..bab313d6 100644 --- a/examples/standard-order/pom.xml +++ b/examples/standard-order/pom.xml @@ -10,7 +10,9 @@ com.examplerequest-tree-order-service-core0.0.1-SNAPSHOT io.teaqlteaql-runtime io.teaqlteaql-sqlite + io.teaqlteaql-duckdb${project.version} io.teaqlteaql-provider-jdbc + org.duckdbduckdb_jdbc1.5.5.1 io.teaqlteaql-query-json io.teaqlteaql-opentelemetry${project.version}test junitjunittest diff --git a/examples/standard-order/src/test/java/com/example/requesttreeorderservice/StandardOrderCommittedTargetTest.java b/examples/standard-order/src/test/java/com/example/requesttreeorderservice/StandardOrderCommittedTargetTest.java new file mode 100644 index 00000000..fdcb3c3f --- /dev/null +++ b/examples/standard-order/src/test/java/com/example/requesttreeorderservice/StandardOrderCommittedTargetTest.java @@ -0,0 +1,132 @@ +package com.example.requesttreeorderservice; + +import io.teaql.core.InternalIdGenerationService; +import io.teaql.core.UserContext; +import io.teaql.core.duck.DuckDataServiceExecutor; +import io.teaql.core.duck.DuckDbCommittedInbox; +import io.teaql.core.duck.DuckDbModelCheckedCommittedTarget; +import io.teaql.core.meta.SimpleEntityMetaFactory; +import io.teaql.core.sqlite.SqliteCommittedEventStore.InboxOutcome; +import io.teaql.core.sqlite.SqliteCommittedMutationRecorder; +import io.teaql.core.sqlite.SqliteCommittedOutboxPublisher; +import io.teaql.core.sqlite.SqliteDataServiceExecutor; +import io.teaql.provider.jdbc.JdbcSqlExecutor; +import io.teaql.runtime.DefaultUserContext; +import io.teaql.runtime.TeaQLRuntime; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; +import javax.sql.DataSource; +import org.junit.Test; +import org.sqlite.SQLiteDataSource; + +import static org.junit.Assert.*; + +/** Application-owned generated Q/Save replay across a committed SQLite to DuckDB boundary. */ +public class StandardOrderCommittedTargetTest { + private static final String SOURCE = "generated-java-order-source"; + private static final String HASH = "sha256:" + "0".repeat(64); + + private record DriverManagerDataSource(String url) implements DataSource { + @Override public Connection getConnection() throws SQLException { return DriverManager.getConnection(url); } + @Override public Connection getConnection(String user, String password) throws SQLException { + return getConnection(); + } + @Override public PrintWriter getLogWriter() { return null; } + @Override public void setLogWriter(PrintWriter writer) { } + @Override public void setLoginTimeout(int seconds) { } + @Override public int getLoginTimeout() { return 0; } + @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { + throw new SQLFeatureNotSupportedException(); + } + @Override public T unwrap(Class type) throws SQLException { + throw new SQLException("not a wrapper"); + } + @Override public boolean isWrapperFor(Class type) { return false; } + } + + private static InternalIdGenerationService ids() { + var next = new AtomicLong(3000); + return new InternalIdGenerationService() { + @Override public Long generateId(UserContext context, io.teaql.core.Entity entity) { + return next.incrementAndGet(); + } + @Override public void ensureFloor(String type, long floor) { + next.accumulateAndGet(floor, Math::max); + } + }; + } + + private static TeaQLRuntime sqliteRuntime(SqliteDataServiceExecutor service, + InternalIdGenerationService idService, SqliteCommittedMutationRecorder recorder) { + var builder = TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .dataService("default", service).dataService("sqlite", service) + .idGenerationService(idService).executionLogging(false); + if (recorder != null) builder.committedMutationRecorder(recorder); + return builder.build().install(GeneratedRuntimeModule.module()); + } + + @Test public void generatedSaveReplicatesToDuckDbAndGeneratedQReadsIt() throws Exception { + var source = new SQLiteDataSource(); + source.setUrl("jdbc:sqlite:" + Files.createTempDirectory("teaql-generated-commit-source-") + .resolve("source.sqlite")); + var sourceSql = new JdbcSqlExecutor(source); + var sourceService = new SqliteDataServiceExecutor("sqlite", sourceSql, source); + var idService = ids(); + try (var bootstrap = sqliteRuntime(sourceService, idService, null)) { + new DefaultUserContext(bootstrap).ensureSchema(); + } + var recorder = new SqliteCommittedMutationRecorder(sourceSql, SOURCE, HASH); + try (var sourceRuntime = sqliteRuntime(sourceService, idService, recorder)) { + var sourceContext = new DefaultUserContext(sourceRuntime); + sourceContext.ensureSchema(); + var platform = Q.platforms().limit(1).comment("load source Platform") + .purpose("attach committed Customer to domain root") + .executeForOne(sourceContext); + assertNotNull(platform); + var customer = Q.customers().comment("prepare committed Customer") + .purpose("verify generated audited save").newEntity(sourceContext); + customer.updateName("Committed target customer"); + customer.updatePlatform(platform); + customer.auditAs("create committed target customer").save(sourceContext); + var outbox = recorder.eventStore().outboxAfter(SOURCE, 0, 10); + assertEquals("bootstrap must not enter this Outbox", 1, outbox.size()); + + var targetPath = Files.createTempDirectory("teaql-generated-commit-target-") + .resolve("target.db"); + var targetSql = new JdbcSqlExecutor(new DriverManagerDataSource("jdbc:duckdb:" + targetPath)); + var targetService = new DuckDataServiceExecutor("sqlite", targetSql); + try (var targetRuntime = TeaQLRuntime.builder().metadata(new SimpleEntityMetaFactory()) + .dataService("default", targetService).dataService("sqlite", targetService) + .idGenerationService(ids()).executionLogging(false).build() + .install(GeneratedRuntimeModule.module())) { + var targetContext = new DefaultUserContext(targetRuntime); + targetContext.ensureSchema(); + var target = new DuckDbModelCheckedCommittedTarget(targetSql, SOURCE, HASH); + target.ensureInboxSchema(); + var delivered = new SqliteCommittedOutboxPublisher(sourceSql, SOURCE, + "generated-java-duckdb-target", HASH).publishBatch(10, record -> { + var outcome = target.apply(targetContext, new DuckDbCommittedInbox.Record( + record.sourceRuntimeId(), record.sourceSequence(), record.changeSetId(), + record.modelHash(), record.payloadSha256(), record.payloadJson())); + return new InboxOutcome(outcome.duplicate(), outcome.checkpoint()); + }); + assertEquals(1, delivered.acknowledged()); + assertEquals(0, delivered.status().backlog()); + var replicated = Q.customers().withIdIs(customer.getId()).limit(1) + .comment("load replicated Customer through generated Q") + .purpose("prove DuckDB target is queryable by application") + .executeForOne(targetContext); + assertNotNull(replicated); + assertEquals("Committed target customer", replicated.getName()); + assertEquals(customer.getVersion(), replicated.getVersion()); + } + } + System.out.println("JAVA_COMMITTED_GENERATED_Q_PASS source=sqlite target=duckdb outbox=1"); + } +} From 6c85892b4461c2e77e14b8ea4809167d22329c78 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Sun, 18 Oct 2026 05:15:24 +0800 Subject: [PATCH 22/29] Prove ordered lost-ack recovery in committed example --- examples/committed-events/README.md | 5 ++++- .../committed/CommittedEventsExampleTest.java | 20 ++++++++++++++++++- examples/committed-events/verify.sh | 5 ++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/examples/committed-events/README.md b/examples/committed-events/README.md index c1929b8a..1124deed 100644 --- a/examples/committed-events/README.md +++ b/examples/committed-events/README.md @@ -5,7 +5,10 @@ Run `bash examples/committed-events/verify.sh`. It executes a small, real fresh source/target SQLite files and an independent DuckDB target. The test reopens the Outbox, delivers the two events through bounded publishers into both model-checked targets, verifies their rows and the SQLite target duplicate -acknowledgement, then injects a later source write failure. This small fixture +acknowledgement, then injects a later source write failure. The first SQLite +delivery deliberately loses its acknowledgement after target commit; a new +source executor replays sequence 1 as a duplicate and then applies sequence 2 +in order. This small fixture itself requires no generated ERP model or remote service. The example configures `SqliteCommittedMutationRecorder` at trusted runtime diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java index 7fbf5eb1..43f9b2e2 100644 --- a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java @@ -130,11 +130,27 @@ public static final class Order extends BaseEntity { SqliteModelCheckedCommittedTarget target = new SqliteModelCheckedCommittedTarget( replicaSql, "example-java-source", "sha256:" + "0".repeat(64)); target.ensureSchema(); - var published = new SqliteCommittedOutboxPublisher(reopened, "example-java-source", + var interruptedPublisher = new SqliteCommittedOutboxPublisher(reopened, + "example-java-source", "example-analytics", "sha256:" + "0".repeat(64)); + assertThrows(IllegalStateException.class, () -> interruptedPublisher.publishBatch(10, + record -> { + target.apply(replicaContext, record); + throw new IllegalStateException("simulate lost target acknowledgement"); + })); + assertEquals(0L, interruptedPublisher.status().ackedSequence()); + assertEquals(2L, interruptedPublisher.status().backlog()); + assertEquals(1, replicaSql.queryForList("SELECT * FROM teaql_event_inbox", + new Object[0]).size()); + + SQLiteDataSource restartedSource = new SQLiteDataSource(); + restartedSource.setUrl("jdbc:sqlite:" + path); + JdbcSqlExecutor restarted = new JdbcSqlExecutor(restartedSource); + var published = new SqliteCommittedOutboxPublisher(restarted, "example-java-source", "example-analytics", "sha256:" + "0".repeat(64)) .publishBatch(10, record -> target.apply(replicaContext, record)); assertEquals(2, published.acknowledged()); assertEquals(0, published.status().backlog()); + assertEquals(2L, published.status().ackedSequence()); var duplicate = target.apply(replicaContext, durable.get(0)); assertTrue(duplicate.duplicate()); assertEquals(1, duplicate.checkpoint()); @@ -145,6 +161,8 @@ public static final class Order extends BaseEntity { assertEquals("set by database", replicated.get("memo")); assertEquals(2, replicaSql.queryForList("SELECT * FROM teaql_event_applied_payload", new Object[0]).size()); + System.out.println("COMMITTED_DELIVERY_RECOVERY_PASS language=java ordered=2 " + + "lost_ack=1 restart=1 duplicate=1"); replicaRuntime.close(); Path duckDirectory = Files.createTempDirectory("teaql-committed-duckdb-example-"); diff --git a/examples/committed-events/verify.sh b/examples/committed-events/verify.sh index 192b9394..72dae31e 100755 --- a/examples/committed-events/verify.sh +++ b/examples/committed-events/verify.sh @@ -10,7 +10,10 @@ generated_library_hash() { generated_library_hash > "$evidence/generated-before.sha256" for round in 1 2; do mvn -B -q -pl examples/committed-events -am \ - -Dtest=CommittedEventsExampleTest -Dsurefire.failIfNoSpecifiedTests=false test + -Dtest=CommittedEventsExampleTest -Dsurefire.failIfNoSpecifiedTests=false test \ + 2>&1 | tee "$evidence/runtime-$round.log" + rg -Fq 'COMMITTED_DELIVERY_RECOVERY_PASS language=java ordered=2 lost_ack=1 restart=1 duplicate=1' \ + "$evidence/runtime-$round.log" echo "PASS Java committed-events example round $round" mvn -B -q -Pruntime-examples -pl examples/standard-order -am \ -Dtest=StandardOrderCommittedTargetTest -Dsurefire.failIfNoSpecifiedTests=false test \ From 7cfe9ffbc4de816c9c617dfcb9bf247d526a7833 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Sun, 18 Oct 2026 15:51:52 +0800 Subject: [PATCH 23/29] Reject duplicate inbox after target checkpoint rewind --- .../sqlite/SqliteCommittedEventStore.java | 3 ++ .../sqlite/SqliteCommittedEventStoreTest.java | 29 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java index dc555f19..f91dada1 100644 --- a/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java +++ b/teaql-sqlite/src/main/java/io/teaql/core/sqlite/SqliteCommittedEventStore.java @@ -175,6 +175,9 @@ InboxOutcome apply(OutboxRecord record, String expectedModelHash, || !record.payloadSha256().equals(digest((String) retained.get(0).get("payload_json")))) { throw invalid("prior Inbox identity lacks retained source evidence"); } + if (last < record.sourceSequence()) { + throw invalid("prior Inbox identity is ahead of target checkpoint"); + } return InboxOutcome.duplicate(record.sourceSequence()); } throw new CommittedEventException("DUPLICATE_CONFLICT", "committed-event identity has conflicting content"); diff --git a/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java index b35609e4..5eb5c4f0 100644 --- a/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/core/sqlite/SqliteCommittedEventStoreTest.java @@ -262,6 +262,35 @@ private static CommittedEventException expectCode(String code, Runnable action) } } + @Test public void duplicateRequiresCheckpointCoveringRetainedInbox() throws Exception { + Path path = Files.createTempFile("teaql-java-cmr-checkpoint-", ".sqlite"); + try { + Database target = database(path); + target.events().ensureSchema(); + graphSchema(target.driver()); + OutboxRecord first = record(1, "change-a", false); + OutboxRecord second = record(2, "change-b", true); + target.driver().executeInTransaction(() -> { + target.events().apply(first, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + target.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph); + }); + + target.driver().update("UPDATE teaql_event_checkpoint SET source_sequence = 1 " + + "WHERE source_runtime_id = ?", new Object[] {SOURCE}); + target.driver().executeInTransaction(() -> expectCode("INVALID_ENVELOPE", + () -> target.events().apply(second, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + + target.driver().update("DELETE FROM teaql_event_checkpoint WHERE source_runtime_id = ?", + new Object[] {SOURCE}); + target.driver().executeInTransaction(() -> expectCode("INVALID_ENVELOPE", + () -> target.events().apply(first, MODEL_HASH, SqliteCommittedEventStoreTest::applyGraph))); + assertEquals(2, count(target.driver(), "teaql_event_inbox")); + assertEquals(2, count(target.driver(), "teaql_event_applied_payload")); + } finally { + Files.deleteIfExists(path); + } + } + private static OutboxRecord record(long sequence, String change, boolean submitted) throws Exception { String json = payload(sequence, change, submitted); byte[] bytes = java.security.MessageDigest.getInstance("SHA-256") From 7f99d1ea0e47fa3ebc31ea29a55b1ba1d017f5c6 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Sun, 18 Oct 2026 17:04:09 +0800 Subject: [PATCH 24/29] Keep nested JDBC save failures rollback-only --- .../committed/CommittedEventsExampleTest.java | 15 ++++++++++ .../teaql/provider/jdbc/JdbcSqlExecutor.java | 17 ++++++++++- .../provider/jdbc/JdbcSqlExecutorTest.java | 29 +++++++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java index 43f9b2e2..c6f7c5ff 100644 --- a/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java +++ b/examples/committed-events/src/test/java/io/teaql/examples/committed/CommittedEventsExampleTest.java @@ -203,6 +203,21 @@ record -> { new Object[] {42L}).get(0).get("name")); assertEquals(2L, ((Number) sql.queryForList("SELECT version FROM order_data WHERE id = ?", new Object[] {42L}).get(0).get("version")).longValue()); + + // Even if application code catches the inner save failure, the shared + // JDBC transaction must not commit a business row without its outbox. + order.updateProperty("name", "caught failure must roll back"); + assertThrows(IllegalStateException.class, () -> sql.executeInTransaction(() -> { + try { + order.auditAs("reject caught outbox failure").save(context); + } catch (RuntimeException expected) { + // Simulate a caller that handles the failed save and returns. + } + })); + assertEquals("submitted", sql.queryForList("SELECT name FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("name")); + assertEquals(2L, ((Number) sql.queryForList("SELECT version FROM order_data WHERE id = ?", + new Object[] {42L}).get(0).get("version")).longValue()); runtime.close(); } } diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index 77222765..f513d5af 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -21,6 +21,7 @@ public class JdbcSqlExecutor implements SqlExecutionAdapter { // A held view retains only this token, not the executor, connection or DataSource. private final Object storageIdentity = new Object(); private final ThreadLocal transactionConnection = new ThreadLocal<>(); + private final ThreadLocal transactionRollbackOnly = new ThreadLocal<>(); private final List> connectionInitializers = new CopyOnWriteArrayList<>(); public JdbcSqlExecutor(DataSource dataSource) { @@ -395,21 +396,35 @@ public int[] batchUpdate(String sql, List paramsList) { @Override public void executeInTransaction(Runnable action) { if (transactionConnection.get() != null) { - action.run(); + try { + action.run(); + } catch (RuntimeException | Error failure) { + // The nested action shares the outer JDBC transaction. Without a + // savepoint, its partial writes cannot be committed safely even if + // application code catches the exception before the outer returns. + transactionRollbackOnly.set(true); + throw failure; + } return; } try (Connection connection = openConnection()) { boolean autoCommit = connection.getAutoCommit(); connection.setAutoCommit(false); transactionConnection.set(connection); + transactionRollbackOnly.set(false); try { action.run(); + if (Boolean.TRUE.equals(transactionRollbackOnly.get())) { + throw new IllegalStateException( + "JDBC transaction is rollback-only after a nested failure"); + } connection.commit(); } catch (RuntimeException | Error failure) { connection.rollback(); throw failure; } finally { transactionConnection.remove(); + transactionRollbackOnly.remove(); connection.setAutoCommit(autoCommit); } } catch (SQLException e) { diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java index 7e311267..2b8b36d9 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java @@ -18,6 +18,7 @@ import java.util.logging.Logger; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; public class JdbcSqlExecutorTest { @@ -62,6 +63,34 @@ public void testExecuteAndInsert() throws Exception { } } + @Test + public void nestedFailureCaughtByCallerStillRollsBackOuterTransaction() { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {91, "before failure", 20}); + try { + sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {92, "failed save", 20}); + throw new IllegalStateException("simulated outbox append failure"); + }); + } catch (IllegalStateException expected) { + // Application code may log a failed save and continue its outer callback. + } + })); + + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + + sqlExecutor.executeInTransaction(() -> sqlExecutor.executeInTransaction(() -> + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {93, "next transaction", 20}))); + assertEquals(1, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + @Test public void testQueryForListNormalizesAliasesForPortableMapping() { List> rows = sqlExecutor.queryForList( From b81d9751d06ba44cccba60ae32b4f5ea8d9a83da Mon Sep 17 00:00:00 2001 From: Philip Z Date: Sun, 18 Oct 2026 17:56:07 +0800 Subject: [PATCH 25/29] Mark JDBC transaction rollback-only after caught SQL failures --- .../teaql/provider/jdbc/JdbcSqlExecutor.java | 12 +++++ .../provider/jdbc/JdbcSqlExecutorTest.java | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index f513d5af..f32c03df 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -184,6 +184,7 @@ public List> queryForList(String sql, Object[] params) { return result; } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException("JDBC query failed for parameterized SQL: " + sql, e); } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } @@ -210,6 +211,7 @@ public List query( } return result; } catch (SQLException exception) { + markTransactionRollbackOnly(); throw new RuntimeException("JDBC typed query failed for parameterized SQL: " + sql, exception); } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } @@ -339,6 +341,7 @@ public void execute(String sql) { ps.execute(); } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException(e); } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } @@ -364,6 +367,7 @@ public int update(String sql, Object[] params) { return ps.executeUpdate(); } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException(e); } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } @@ -387,6 +391,7 @@ public int[] batchUpdate(String sql, List paramsList) { return ps.executeBatch(); } } catch (SQLException e) { + markTransactionRollbackOnly(); throw new RuntimeException(e); } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } @@ -432,6 +437,13 @@ public void executeInTransaction(Runnable action) { } } + private void markTransactionRollbackOnly() { + // A failed statement may have left earlier writes in the shared + // transaction. Without a savepoint, catching that failure cannot + // make the transaction safe to commit. + if (transactionConnection.get() != null) transactionRollbackOnly.set(true); + } + @Override public boolean hasActiveTransaction() { return transactionConnection.get() != null; } static void bind(PreparedStatement statement, int index, Object value) throws SQLException { diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java index 2b8b36d9..d26e6b2d 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java @@ -91,6 +91,50 @@ public void nestedFailureCaughtByCallerStillRollsBackOuterTransaction() { .get(0).get("total")).intValue()); } + @Test + public void caughtJdbcWriteFailureStillRollsBackOuterTransaction() { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {94, "first write", 20}); + try { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {94, "duplicate key", 21}); + } catch (RuntimeException expected) { + // A caller may handle a low-level JDBC failure before returning. + } + })); + + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + + @Test + public void caughtStatementOrReadFailureCannotCommitEarlierWrites() { + List failures = List.of( + () -> sqlExecutor.execute("INSERT INTO test_user (id, name, age) VALUES (94, 'duplicate', 21)"), + () -> sqlExecutor.batchUpdate( + "INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + java.util.Collections.singletonList(new Object[] {94, "duplicate", 21})), + () -> sqlExecutor.queryForList("SELECT * FROM missing_test_table", new Object[0]), + () -> sqlExecutor.query("SELECT * FROM missing_test_table", new Object[0], + row -> new TypedRow())); + for (Runnable failedStatement : failures) { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {94, "must roll back", 20}); + try { + failedStatement.run(); + } catch (RuntimeException expected) { + // The failed statement must poison the shared transaction. + } + })); + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + } + @Test public void testQueryForListNormalizesAliasesForPortableMapping() { List> rows = sqlExecutor.queryForList( From 54d0f117feffdd7ebc97f6c121d9043fb271cbd3 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Sun, 18 Oct 2026 18:32:24 +0800 Subject: [PATCH 26/29] Rollback JDBC transaction after caught mapper failures --- .../teaql/provider/jdbc/JdbcSqlExecutor.java | 23 +++++++++++++++---- .../provider/jdbc/JdbcSqlExecutorTest.java | 18 +++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java index f32c03df..23a476c4 100644 --- a/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java +++ b/teaql-provider-jdbc/src/main/java/io/teaql/provider/jdbc/JdbcSqlExecutor.java @@ -186,6 +186,9 @@ public List> queryForList(String sql, Object[] params) { } catch (SQLException e) { markTransactionRollbackOnly(); throw new RuntimeException("JDBC query failed for parameterized SQL: " + sql, e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -213,6 +216,9 @@ public List query( } catch (SQLException exception) { markTransactionRollbackOnly(); throw new RuntimeException("JDBC typed query failed for parameterized SQL: " + sql, exception); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -343,6 +349,9 @@ public void execute(String sql) { } catch (SQLException e) { markTransactionRollbackOnly(); throw new RuntimeException(e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -369,6 +378,9 @@ public int update(String sql, Object[] params) { } catch (SQLException e) { markTransactionRollbackOnly(); throw new RuntimeException(e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -393,6 +405,9 @@ public int[] batchUpdate(String sql, List paramsList) { } catch (SQLException e) { markTransactionRollbackOnly(); throw new RuntimeException(e); + } catch (RuntimeException | Error failure) { + markTransactionRollbackOnly(); + throw failure; } finally { if (owned && connection != null) try { connection.close(); } catch (SQLException ignored) { } } @@ -421,7 +436,7 @@ public void executeInTransaction(Runnable action) { action.run(); if (Boolean.TRUE.equals(transactionRollbackOnly.get())) { throw new IllegalStateException( - "JDBC transaction is rollback-only after a nested failure"); + "JDBC transaction is rollback-only after a failed operation"); } connection.commit(); } catch (RuntimeException | Error failure) { @@ -438,9 +453,9 @@ public void executeInTransaction(Runnable action) { } private void markTransactionRollbackOnly() { - // A failed statement may have left earlier writes in the shared - // transaction. Without a savepoint, catching that failure cannot - // make the transaction safe to commit. + // A failed statement, mapper or adapter operation may have left earlier + // writes in the shared transaction. Without a savepoint, catching the + // failure cannot make the transaction safe to commit. if (transactionConnection.get() != null) transactionRollbackOnly.set(true); } diff --git a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java index d26e6b2d..885414a1 100644 --- a/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java +++ b/teaql-provider-jdbc/src/test/java/io/teaql/provider/jdbc/JdbcSqlExecutorTest.java @@ -135,6 +135,24 @@ public void caughtStatementOrReadFailureCannotCommitEarlierWrites() { } } + @Test + public void caughtTypedRowMapperFailureCannotCommitEarlierWrites() { + assertThrows(IllegalStateException.class, () -> sqlExecutor.executeInTransaction(() -> { + sqlExecutor.update("INSERT INTO test_user (id, name, age) VALUES (?, ?, ?)", + new Object[] {95, "must roll back", 20}); + try { + sqlExecutor.query("SELECT id FROM test_user WHERE id = ?", new Object[] {95}, + row -> { throw new IllegalArgumentException("bad typed row"); }); + } catch (IllegalArgumentException expected) { + // The mapper failure escaped a query using the same transaction. + } + })); + + assertEquals(0, ((Number) sqlExecutor.queryForList( + "SELECT count(*) AS total FROM test_user", new Object[0]) + .get(0).get("total")).intValue()); + } + @Test public void testQueryForListNormalizesAliasesForPortableMapping() { List> rows = sqlExecutor.queryForList( From 938940707bc5f0605e31e2ea17b978f21732c811 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Mon, 19 Oct 2026 02:34:04 +0800 Subject: [PATCH 27/29] fix: distinguish post-commit audit delivery failures --- .../AuditDeliveryAfterCommitException.java | 19 +++++ .../java/io/teaql/runtime/TeaQLRuntime.java | 65 ++++++++++++----- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 72 ++++++++++++++++++- 3 files changed, 136 insertions(+), 20 deletions(-) create mode 100644 teaql-runtime/src/main/java/io/teaql/runtime/AuditDeliveryAfterCommitException.java diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/AuditDeliveryAfterCommitException.java b/teaql-runtime/src/main/java/io/teaql/runtime/AuditDeliveryAfterCommitException.java new file mode 100644 index 00000000..65245433 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/AuditDeliveryAfterCommitException.java @@ -0,0 +1,19 @@ +package io.teaql.runtime; + +import io.teaql.core.TeaQLRuntimeException; + +/** The graph committed, but one or more post-commit audit deliveries failed. */ +public final class AuditDeliveryAfterCommitException extends TeaQLRuntimeException { + private final int failureCount; + + public AuditDeliveryAfterCommitException(int failureCount) { + super("AUDIT_DELIVERY_AFTER_COMMIT: transaction committed but audit delivery failed; " + + "do not retry the mutation blindly"); + if (failureCount < 1) throw new IllegalArgumentException("failureCount must be positive"); + this.failureCount = failureCount; + } + + public int failureCount() { + return failureCount; + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 55613bb8..c8b5ee91 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -577,8 +577,9 @@ public void saveGraph(UserContext context, Entity entity) { BaseEntity target = realEntities.get(key); target.__internalHydrateDynamicFields(values, target.__internalLoadState().withDynamicSelection(values.selectedCodes())); }); - completeLedgerPlan(context, completed); + completeLedgerPlan(completed); entityMutationLedger.clearCurrentChangeSet(); + deliverCommittedAuditEvents(context, completed); telemetryScope.success(); } catch (RuntimeException | Error error) { telemetryScope.failure(error); @@ -1102,18 +1103,33 @@ private List executeLedgerPlan( return completed; } - private void completeLedgerPlan(UserContext context, List completed) { + private void completeLedgerPlan(List completed) { for (PendingMutation mutation : completed) { applyPersistedEntity(mutation.descriptor(), mutation.target(), mutation.result()); - emitAuditEvent( - context, mutation.target(), mutation.auditKind(), mutation.changedValues(), - mutation.governance(), mutation.intent(), mutation.traceChain(), mutation.redactions()); mutation.target().clearUpdatedProperties(); - // A committed delete supersedes any staged extension edits too. - mutation.target().__internalClearDynamicMutations(); } } + private void deliverCommittedAuditEvents(UserContext context, List completed) { + int failures = 0; + for (PendingMutation mutation : completed) { + try { + failures += emitAuditEvent( + context, mutation.target(), mutation.auditKind(), mutation.changedValues(), + mutation.governance(), mutation.intent(), mutation.traceChain(), mutation.redactions()); + } catch (RuntimeException failure) { + // The provider transaction has already committed. Continue delivery + // for the other graph members and never expose a retryable save error. + failures++; + } finally { + // Preserve dynamic original values until the audit projection has + // observed them; then clear staged extension edits even on failure. + mutation.target().__internalClearDynamicMutations(); + } + } + if (failures != 0) throw new AuditDeliveryAfterCommitException(failures); + } + private Map snapshotChanges(Map changes) { return Collections.unmodifiableMap(new LinkedHashMap<>(changes)); } @@ -1411,7 +1427,7 @@ private List mutateBatchWithTelemetry( } } - private void emitAuditEvent( + private int emitAuditEvent( UserContext context, Entity entity, MutationAuditKind kind, @@ -1457,18 +1473,29 @@ private void emitAuditEvent( "teaql.mutation.kind", kind.name().toLowerCase(Locale.ROOT), "teaql.audit.changed_field_count", changes.size()))); try { + int failures = 0; + // The server-owned and application sinks are independent delivery + // attempts. One failure must not suppress the other after commit. + if (logSink != null) { + try { + logSink.writeAuditEvent(context, + LogPrivacy.audit(rawEvent, LogPrivacy.plaintextEnabled(), redactions)); + } catch (RuntimeException failure) { + failures++; + } + } - // The standard sink is server-owned by TeaQLRuntime and cannot be replaced by - // dynamic input or an application capability registered on UserContext. - if (logSink != null) { - logSink.writeAuditEvent(context, LogPrivacy.audit(rawEvent, LogPrivacy.plaintextEnabled(), redactions)); - } - - AppAuditEventSink appSink = context.capability(AppAuditEventSink.class); - if (appSink != null) { - appSink.onAuditEvent(context, buildSafeAuditEvent(rawEvent, redactions)); - } - telemetryScope.success(); + AppAuditEventSink appSink = context.capability(AppAuditEventSink.class); + if (appSink != null) { + try { + appSink.onAuditEvent(context, buildSafeAuditEvent(rawEvent, redactions)); + } catch (RuntimeException failure) { + failures++; + } + } + if (failures == 0) telemetryScope.success(); + else telemetryScope.failure(new AuditDeliveryAfterCommitException(failures)); + return failures; } catch (RuntimeException | Error error) { telemetryScope.failure(error); throw error; diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index 90c06f07..c9c833e8 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -55,6 +55,7 @@ static final class Fixture { final List results = new CopyOnWriteArrayList<>(); boolean queryLogging = true; boolean mutationLogging = true; + boolean failStandardAudit; final List itemInsertBatchSizes = new CopyOnWriteArrayList<>(); final List itemUpdateBatchSizes = new CopyOnWriteArrayList<>(); final List itemDeleteBatchSizes = new CopyOnWriteArrayList<>(); @@ -162,7 +163,15 @@ static final class Fixture { var ids = new AtomicLong(1000); var builder = TeaQLRuntime.builder().metadata(metadata).dataService("sqlite", provider) .idGenerationService((caller, entity) -> ids.getAndIncrement()) - .logSink((caller, entry) -> sql.add(entry)); + .logSink(new RuntimeLogSink() { + @Override public void writeExecutionLog(UserContext caller, ExecutionMetadata entry) { + sql.add(entry); + } + + @Override public void writeAuditEvent(UserContext caller, RawAuditEvent event) { + if (failStandardAudit) throw new IllegalStateException("standard audit sink failure"); + } + }); if (committedRecorder != null) builder.committedMutationRecorder(committedRecorder); var runtime = builder.build(); context = new DefaultUserContext(runtime) { @@ -688,6 +697,67 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri assertTrue("no post-commit audit on rolled back save", fixture.audit.isEmpty()); } + @Test public void postCommitAuditFailureReportsCommittedOutcomeWithoutReplayingLedger() throws Exception { + var fixture = new Fixture(true); + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> { + throw new IllegalStateException("diagnostic audit sink failure"); + }); + var order = fixture.create("CustomerOrder", 199, "committed before audit delivery"); + + var failure = assertThrows(AuditDeliveryAfterCommitException.class, + () -> order.auditAs("diagnose post-commit audit failure").save(fixture.context)); + assertTrue(failure.getMessage().contains("AUDIT_DELIVERY_AFTER_COMMIT")); + assertEquals(1, failure.failureCount()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {199L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> fixture.audit.add(event)); + order.auditAs("retry after audit delivery failure").save(fixture.context); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + + @Test public void postCommitAuditFailureStillAttemptsEveryGraphMember() throws Exception { + var fixture = new Fixture(true); + var attempts = new java.util.concurrent.atomic.AtomicInteger(); + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> { + attempts.incrementAndGet(); + throw new IllegalStateException("diagnostic audit sink failure"); + }); + var order = fixture.create("CustomerOrder", 198, "root audit fails"); + var item = fixture.create("OrderItem", 298, "child audit fails"); + order.__internalSet("children", List.of(item)); + + var failure = assertThrows(AuditDeliveryAfterCommitException.class, + () -> order.auditAs("commit graph before audit delivery").save(fixture.context)); + assertEquals(2, failure.failureCount()); + assertEquals(2, attempts.get()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {198L}).size()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM order_item_data WHERE id = ?", new Object[] {298L}).size()); + } + + @Test public void standardAuditSinkFailureDoesNotSuppressApplicationAuditDelivery() throws Exception { + var fixture = new Fixture(true); + fixture.failStandardAudit = true; + var order = fixture.create("CustomerOrder", 197, "both audit sinks are attempted"); + + var failure = assertThrows(AuditDeliveryAfterCommitException.class, + () -> order.auditAs("commit before standard audit delivery").save(fixture.context)); + assertEquals(1, failure.failureCount()); + assertEquals(1, fixture.audit.size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + @Test public void realSqliteProviderFailureKeepsAttemptedLineageWithoutCommittedAudit() throws Exception { var fixture = new Fixture(); fixture.driver.execute("CREATE UNIQUE INDEX payment_name_unique ON payment_data(name)"); From 628991a778f12a61d421b1519f0bc80f66b3d7e8 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Mon, 19 Oct 2026 03:09:48 +0800 Subject: [PATCH 28/29] fix: classify committed graph reconciliation failures --- .../main/java/io/teaql/core/BaseEntity.java | 1 + .../io/teaql/core/EntityMutationLedger.java | 18 ++++++ .../teaql/core/EntityMutationLedgerTest.java | 17 ++++++ ...CommittedStateReconciliationException.java | 11 ++++ .../java/io/teaql/runtime/TeaQLRuntime.java | 23 +++++--- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 56 +++++++++++++++++++ 6 files changed, 119 insertions(+), 7 deletions(-) create mode 100644 teaql-runtime/src/main/java/io/teaql/runtime/CommittedStateReconciliationException.java diff --git a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java index d2fa2f4f..f682479e 100644 --- a/teaql-core/src/main/java/io/teaql/core/BaseEntity.java +++ b/teaql-core/src/main/java/io/teaql/core/BaseEntity.java @@ -427,6 +427,7 @@ public EntityMutationLedger getEntityMutationLedger() { public boolean __internalHasMutationLedger() { return entityMutationLedger != null; } public void setEntityMutationLedger(EntityMutationLedger entityMutationLedger) { + if (this.entityMutationLedger != null) this.entityMutationLedger.requireSaveable(); this.entityMutationLedger = entityMutationLedger; if (entityMutationLedger != null && id != null && newItem()) { entityMutationLedger.markAsNew(new EntityKey(typeName(), id)); diff --git a/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java b/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java index d1cdd4cd..18bb83af 100644 --- a/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java +++ b/teaql-core/src/main/java/io/teaql/core/EntityMutationLedger.java @@ -16,6 +16,7 @@ public class EntityMutationLedger { private final Set newKeys = new TreeSet<>(); private final Map> traceChains = new TreeMap<>(); private final Map originalVersions = new TreeMap<>(); + private boolean committedStateRequiresReload; // --- Change Set Stack --- @@ -41,6 +42,19 @@ public void clearCurrentChangeSet() { traceChains.clear(); } + /** A provider write completed, but the in-memory graph could not be reconciled. */ + public void markCommittedStateRequiresReload() { + committedStateRequiresReload = true; + } + + /** Prevents a stale graph from replaying a write whose outcome is already committed. */ + public void requireSaveable() { + if (committedStateRequiresReload) { + throw new TeaQLRuntimeException( + "COMMITTED_STATE_RELOAD_REQUIRED: reload the entity graph before another save"); + } + } + public void set(EntityKey key, String field, Object value) { changeSets.set(key, field, value); } @@ -141,12 +155,14 @@ public Long getOriginalVersion(EntityKey key) { * for a read-only entity, which must retain its private ledger ownership. */ public boolean mergeEntityFrom(EntityMutationLedger other, EntityKey key) { + requireSaveable(); if (other == null || other == this) return false; Map fields = other.currentChangeSet().changes().get(key); var dynamic = other.currentChangeSet().dynamicChanges().get(key); boolean pending = (fields != null && !fields.isEmpty()) || other.newKeys.contains(key) || other.deletedKeys.contains(key) || other.recoveredKeys.contains(key) || (dynamic != null && !dynamic.isEmpty()); if (!pending) return false; + other.requireSaveable(); if (fields != null) fields.forEach((field, value) -> set(key, field, value)); if (dynamic != null) dynamic.values().forEach(mutation -> setDynamic(key, mutation)); if (other.deletedKeys.contains(key)) markAsDelete(key); @@ -162,7 +178,9 @@ public boolean mergeEntityFrom(EntityMutationLedger other, EntityKey key) { * Used when saving an entity graph (e.g., Order + OrderItems). */ public void mergeFrom(EntityMutationLedger other) { + requireSaveable(); if (other == null) return; + other.requireSaveable(); // Merge change sets EntityChangeSet otherChangeSet = other.currentChangeSet(); diff --git a/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java b/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java index d95112d9..9f5476c3 100644 --- a/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java +++ b/teaql-core/src/test/java/io/teaql/core/EntityMutationLedgerTest.java @@ -13,6 +13,23 @@ public class EntityMutationLedgerTest { private static final EntityKey ORDER = new EntityKey("Order", 1L); private static final EntityKey OTHER_ORDER = new EntityKey("Order", 2L); + @Test + public void committedReconciliationFailureCannotBeClearedOrMergedIntoAnotherLedger() { + EntityMutationLedger failed = new EntityMutationLedger(); + failed.markAsNew(ORDER); + failed.markCommittedStateRequiresReload(); + assertThrows(TeaQLRuntimeException.class, failed::requireSaveable); + + EntityMutationLedger independent = new EntityMutationLedger(); + assertThrows(TeaQLRuntimeException.class, + () -> independent.mergeEntityFrom(failed, ORDER)); + assertThrows(TeaQLRuntimeException.class, () -> independent.mergeFrom(failed)); + + failed.clearCurrentChangeSet(); + assertThrows(TeaQLRuntimeException.class, failed::requireSaveable); + independent.requireSaveable(); + } + @Test public void markAsNewRecordsOnlyRequestedKeyAndIsIdempotent() { EntityMutationLedger root = new EntityMutationLedger(); diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/CommittedStateReconciliationException.java b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedStateReconciliationException.java new file mode 100644 index 00000000..39f57f21 --- /dev/null +++ b/teaql-runtime/src/main/java/io/teaql/runtime/CommittedStateReconciliationException.java @@ -0,0 +1,11 @@ +package io.teaql.runtime; + +import io.teaql.core.TeaQLRuntimeException; + +/** The provider write completed, but the in-memory graph requires a fresh load. */ +public final class CommittedStateReconciliationException extends TeaQLRuntimeException { + public CommittedStateReconciliationException() { + super("COMMITTED_STATE_RELOAD_REQUIRED: provider write completed but entity reconciliation failed; " + + "reload the graph and do not retry this instance"); + } +} diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index c8b5ee91..6513fdd3 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -457,9 +457,10 @@ public void saveGraph(UserContext context, Entity entity) { "teaql.mutation.kind", "save"))); try { MutationIntent intent = MutationIntent.of(entity.getComment()); - checkAndFix(context, entity); // Get entity's own EntityMutationLedger EntityMutationLedger entityMutationLedger = ((BaseEntity) entity).getEntityMutationLedger(); + entityMutationLedger.requireSaveable(); + checkAndFix(context, entity); // Allocate identifiers before roots are merged. New children commonly // receive their field updates while their id is still null, so those @@ -573,12 +574,20 @@ public void saveGraph(UserContext context, Entity entity) { Collections.newSetFromMap(new IdentityHashMap<>())); throw failure; } - dynamicReadbacks.forEach((key, values) -> { - BaseEntity target = realEntities.get(key); - target.__internalHydrateDynamicFields(values, target.__internalLoadState().withDynamicSelection(values.selectedCodes())); - }); - completeLedgerPlan(completed); - entityMutationLedger.clearCurrentChangeSet(); + try { + dynamicReadbacks.forEach((key, values) -> { + BaseEntity target = realEntities.get(key); + target.__internalHydrateDynamicFields(values, target.__internalLoadState().withDynamicSelection(values.selectedCodes())); + }); + completeLedgerPlan(completed); + entityMutationLedger.clearCurrentChangeSet(); + } catch (RuntimeException reconciliationFailure) { + // The provider has returned from its transaction. The graph may be + // partially reconciled, so neither its pending ledger nor its + // entity instances are safe to save again without a fresh load. + entityMutationLedger.markCommittedStateRequiresReload(); + throw new CommittedStateReconciliationException(); + } deliverCommittedAuditEvents(context, completed); telemetryScope.success(); } catch (RuntimeException | Error error) { diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index c9c833e8..5a88fed4 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -30,6 +30,7 @@ public class GraphTraceSqliteTest { public static final class GraphEntity extends BaseEntity { private final String type; private final Map values = new HashMap<>(); + boolean failPostCommitReconcile; public GraphEntity(String type) { this.type = type; } @Override public String typeName() { return type; } @Override public Object __internalGet(String field) { @@ -40,6 +41,9 @@ public static final class GraphEntity extends BaseEntity { ? values.get(field) : super.__internalGet(field); } @Override public void __internalSet(String field, Object value) { + if (failPostCommitReconcile && field.equals("version")) { + throw new IllegalStateException("diagnostic reconciliation failure"); + } if (field.equals("name") || field.equals("memo") || field.equals("children") || field.equals("business_counter") || field.equals("planned_date") || field.equals("active") || field.equals("ratio") || field.equals("optional_note") @@ -721,6 +725,58 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri .outboxAfter("java-graph-source", 0, 10).size()); } + @Test public void postCommitReconciliationFailureMustReportCommittedOutcome() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 196, "reconcile after commit"); + order.failPostCommitReconcile = true; + + var failure = assertThrows(CommittedStateReconciliationException.class, + () -> order.auditAs("diagnose committed reconciliation failure").save(fixture.context)); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {196L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + assertTrue("committed outcome must be explicit to prevent blind retry", + failure.getMessage().contains("provider write completed")); + order.failPostCommitReconcile = false; + var retryFailure = assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("retry committed reconciliation failure").save(fixture.context)); + assertTrue(retryFailure.getMessage().contains("COMMITTED_STATE_RELOAD_REQUIRED")); + assertThrows(TeaQLRuntimeException.class, + () -> order.setEntityMutationLedger(new EntityMutationLedger())); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + var independent = fixture.create("Payment", 195, "unrelated graph may still save"); + independent.auditAs("save independent graph after failure").save(fixture.context); + assertEquals(2, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + + @Test public void postCommitChildReconciliationFailurePoisonsOnlyItsSaveGraph() throws Exception { + var fixture = new Fixture(true); + var order = fixture.create("CustomerOrder", 194, "root already committed"); + var item = fixture.create("OrderItem", 294, "child needs reload"); + item.failPostCommitReconcile = true; + order.__internalSet("children", List.of(item)); + + assertThrows(CommittedStateReconciliationException.class, + () -> order.auditAs("commit graph before child reconciliation").save(fixture.context)); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {194L}).size()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM order_item_data WHERE id = ?", new Object[] {294L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + + item.failPostCommitReconcile = false; + assertThrows(TeaQLRuntimeException.class, + () -> item.auditAs("unsafe child retry").save(fixture.context)); + assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("unsafe root retry").save(fixture.context)); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + } + @Test public void postCommitAuditFailureStillAttemptsEveryGraphMember() throws Exception { var fixture = new Fixture(true); var attempts = new java.util.concurrent.atomic.AtomicInteger(); From 092e483d28f16de0d435449327b6d777cc2ce621 Mon Sep 17 00:00:00 2001 From: Philip Z Date: Mon, 19 Oct 2026 03:26:57 +0800 Subject: [PATCH 29/29] fix: deliver committed audits after reconciliation failure --- .../java/io/teaql/runtime/TeaQLRuntime.java | 20 +++++++--- .../io/teaql/sqlite/GraphTraceSqliteTest.java | 39 ++++++++++++++++++- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java index 6513fdd3..39d15d19 100644 --- a/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java +++ b/teaql-runtime/src/main/java/io/teaql/runtime/TeaQLRuntime.java @@ -574,6 +574,7 @@ public void saveGraph(UserContext context, Entity entity) { Collections.newSetFromMap(new IdentityHashMap<>())); throw failure; } + CommittedStateReconciliationException reconciliationOutcome = null; try { dynamicReadbacks.forEach((key, values) -> { BaseEntity target = realEntities.get(key); @@ -586,9 +587,17 @@ public void saveGraph(UserContext context, Entity entity) { // partially reconciled, so neither its pending ledger nor its // entity instances are safe to save again without a fresh load. entityMutationLedger.markCommittedStateRequiresReload(); - throw new CommittedStateReconciliationException(); + reconciliationOutcome = new CommittedStateReconciliationException(); } - deliverCommittedAuditEvents(context, completed); + try { + // The provider has already committed. A failed in-memory readback + // must not silently suppress the independent audit delivery. + deliverCommittedAuditEvents(context, completed); + } catch (AuditDeliveryAfterCommitException auditFailure) { + if (reconciliationOutcome == null) throw auditFailure; + reconciliationOutcome.addSuppressed(auditFailure); + } + if (reconciliationOutcome != null) throw reconciliationOutcome; telemetryScope.success(); } catch (RuntimeException | Error error) { telemetryScope.failure(error); @@ -1125,7 +1134,8 @@ private void deliverCommittedAuditEvents(UserContext context, List changedValues, MutationGovernanceSnapshot governance, MutationIntent intent, List traceChain, - SqlIntentRedactions redactions) { + SqlIntentRedactions redactions, Long committedVersion) { List changes = new ArrayList<>(); if (changedValues != null) { for (Map.Entry entry : changedValues.entrySet()) { @@ -1472,7 +1482,7 @@ private int emitAuditEvent( context.getAttribute(GeneratedSchemaBootstrap.AUDIT_ACTOR_ATTRIBUTE, String.class), context.getAttribute(GeneratedSchemaBootstrap.AUDIT_CATEGORY_ATTRIBUTE, String.class), intent.auditReason(), - entity.getVersion(), + committedVersion, java.time.Instant.now(), governance); diff --git a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java index 5a88fed4..e8b366eb 100644 --- a/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java +++ b/teaql-sqlite/src/test/java/io/teaql/sqlite/GraphTraceSqliteTest.java @@ -55,6 +55,7 @@ public static final class GraphEntity extends BaseEntity { static final class Fixture { final List sql = new CopyOnWriteArrayList<>(); final List audit = new CopyOnWriteArrayList<>(); + final List rawAudit = new CopyOnWriteArrayList<>(); final List commands = new CopyOnWriteArrayList<>(); final List results = new CopyOnWriteArrayList<>(); boolean queryLogging = true; @@ -174,6 +175,7 @@ static final class Fixture { @Override public void writeAuditEvent(UserContext caller, RawAuditEvent event) { if (failStandardAudit) throw new IllegalStateException("standard audit sink failure"); + rawAudit.add(event); } }); if (committedRecorder != null) builder.committedMutationRecorder(committedRecorder); @@ -188,7 +190,7 @@ static final class Fixture { } void clear() { - sql.clear(); audit.clear(); commands.clear(); results.clear(); itemInsertBatchSizes.clear(); + sql.clear(); audit.clear(); rawAudit.clear(); commands.clear(); results.clear(); itemInsertBatchSizes.clear(); itemUpdateBatchSizes.clear(); itemDeleteBatchSizes.clear(); itemRecoverBatchSizes.clear(); } @@ -736,6 +738,9 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {196L}).size()); assertEquals(1, fixture.committedRecorder.eventStore() .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals("committed mutations still require application audit delivery", 1, fixture.audit.size()); + assertEquals(1, fixture.rawAudit.size()); + assertEquals(Long.valueOf(1), fixture.rawAudit.get(0).resultingVersion()); assertTrue("committed outcome must be explicit to prevent blind retry", failure.getMessage().contains("provider write completed")); order.failPostCommitReconcile = false; @@ -746,6 +751,7 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri () -> order.setEntityMutationLedger(new EntityMutationLedger())); assertEquals(1, fixture.committedRecorder.eventStore() .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals(1, fixture.audit.size()); var independent = fixture.create("Payment", 195, "unrelated graph may still save"); independent.auditAs("save independent graph after failure").save(fixture.context); assertEquals(2, fixture.committedRecorder.eventStore() @@ -775,6 +781,37 @@ private static CheckedSave checkedSave(Fixture fixture, GraphEntity entity, Stri () -> order.auditAs("unsafe root retry").save(fixture.context)); assertEquals(1, fixture.committedRecorder.eventStore() .outboxAfter("java-graph-source", 0, 10).size()); + assertEquals("both committed graph members must still be audited", 2, fixture.audit.size()); + assertEquals(2, fixture.rawAudit.size()); + } + + @Test public void postCommitReconciliationAndAuditFailureRetainBothOutcomes() throws Exception { + var fixture = new Fixture(true); + var attempts = new java.util.concurrent.atomic.AtomicInteger(); + fixture.context.putAttribute(AppAuditEventSink.class.getName(), + (AppAuditEventSink) (caller, event) -> { + attempts.incrementAndGet(); + throw new IllegalStateException("diagnostic audit sink failure"); + }); + var order = fixture.create("CustomerOrder", 193, "committed despite two post-commit failures"); + order.failPostCommitReconcile = true; + + var failure = assertThrows(CommittedStateReconciliationException.class, + () -> order.auditAs("commit before reconciliation and audit failures").save(fixture.context)); + assertEquals(1, attempts.get()); + assertEquals(1, failure.getSuppressed().length); + assertTrue(failure.getSuppressed()[0] instanceof AuditDeliveryAfterCommitException); + assertEquals(1, fixture.rawAudit.size()); + assertEquals(Long.valueOf(1), fixture.rawAudit.get(0).resultingVersion()); + assertEquals(1, fixture.driver.queryForList( + "SELECT id FROM customer_order_data WHERE id = ?", new Object[] {193L}).size()); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); + order.failPostCommitReconcile = false; + assertThrows(TeaQLRuntimeException.class, + () -> order.auditAs("reject retry after two post-commit failures").save(fixture.context)); + assertEquals(1, fixture.committedRecorder.eventStore() + .outboxAfter("java-graph-source", 0, 10).size()); } @Test public void postCommitAuditFailureStillAttemptsEveryGraphMember() throws Exception {