diff --git a/database/src/main/java/io/hstore/db/query/Ast.java b/database/src/main/java/io/hstore/db/query/Ast.java index 4324c51..a09b09e 100644 --- a/database/src/main/java/io/hstore/db/query/Ast.java +++ b/database/src/main/java/io/hstore/db/query/Ast.java @@ -314,6 +314,9 @@ record ShowViews() implements Statement { record Checkpoint() implements Statement { } + record CheckpointAt(OptionalLong bytes) implements Statement { + } + record Compact() implements Statement { } } diff --git a/database/src/main/java/io/hstore/db/query/Executor.java b/database/src/main/java/io/hstore/db/query/Executor.java index a5837f5..76446ee 100644 --- a/database/src/main/java/io/hstore/db/query/Executor.java +++ b/database/src/main/java/io/hstore/db/query/Executor.java @@ -37,6 +37,7 @@ import io.hstore.db.value.Value; import io.hstore.db.view.MaterializedViews; import io.hstore.db.view.ViewCell; +import io.hstore.engine.CheckpointThreshold; import io.hstore.engine.EngineStats; import io.hstore.engine.HStoreException; import io.hstore.engine.catalog.Branch; @@ -316,6 +317,7 @@ QueryResult run(Statement statement) { var result = database().engine().checkpoint(); yield QueryResult.message("checkpoint at generation " + result.generation() + ", lsn " + result.lsn()); } + case Statement.CheckpointAt(OptionalLong bytes) -> QueryResult.message(describe(database().engine().checkpointAt(bytes))); case Statement.Compact _ -> { var report = database().engine().compact(); yield QueryResult.message("compacted segments " + report.compacted() + " at generation " + report.generation()); @@ -330,7 +332,7 @@ private static boolean administrative(Statement statement) { return switch (statement) { case Statement.CreateBranch _, Statement.DropBranch _, Statement.MergeBranch _, Statement.Sample _, Statement.DiffGenerations _, Statement.DiffBranches _, Statement.History _, Statement.Stats _, - Statement.Checkpoint _, Statement.Compact _, Statement.CreateTenant _, Statement.AlterTenant _, + Statement.Checkpoint _, Statement.CheckpointAt _, Statement.Compact _, Statement.CreateTenant _, Statement.AlterTenant _, Statement.CreateUser _, Statement.AlterUser _, Statement.DropUser _, Statement.ShowTenants _, Statement.ShowUsers _, Statement.CreateView _, Statement.RefreshView _, Statement.ShowView _, Statement.CreateType _, Statement.CreateIndex _, Statement.CreateJsonIndex _ -> true; @@ -338,6 +340,14 @@ private static boolean administrative(Statement statement) { }; } + private static String describe(CheckpointThreshold threshold) { + return "checkpoint after " + threshold.bytes() + " bytes of log, " + switch (threshold.source()) { + case AUTOMATIC -> "sized from the store"; + case DATABASE -> "set by CHECKPOINT AT"; + case CONFIGURATION -> "set by configuration"; + }; + } + private static Cell limit(long value) { return value == Long.MAX_VALUE ? QueryResult.text("unlimited") : QueryResult.number(value); } @@ -612,6 +622,9 @@ private QueryResult stats() { metrics.put("data bytes written", QueryResult.number(engine.dataBytesWritten())); metrics.put("wal bytes", QueryResult.number(engine.walBytes())); metrics.put("wal segments", QueryResult.number(engine.walSegments())); + CheckpointThreshold threshold = database().engine().checkpointThreshold(); + metrics.put("checkpoint after wal bytes", QueryResult.number(threshold.bytes())); + metrics.put("checkpoint threshold", QueryResult.text(threshold.source().name().toLowerCase())); metrics.put("cache hit rate", QueryResult.number(Math.round(engine.cacheHitRate() * 1000) / 1000.0)); metrics.put("data segments", QueryResult.number(engine.segments().size())); metrics.put("feed bytes", QueryResult.number(engine.feedBytes())); diff --git a/database/src/main/java/io/hstore/db/query/Parser.java b/database/src/main/java/io/hstore/db/query/Parser.java index f0abef3..768cc78 100644 --- a/database/src/main/java/io/hstore/db/query/Parser.java +++ b/database/src/main/java/io/hstore/db/query/Parser.java @@ -393,6 +393,9 @@ private Statement statement() { return new Statement.ShowViews(); } if (accept("CHECKPOINT")) { + if (accept("AT")) { + return new Statement.CheckpointAt(accept("AUTO") ? OptionalLong.empty() : OptionalLong.of(size())); + } return new Statement.Checkpoint(); } if (accept("COMPACT")) { @@ -931,6 +934,32 @@ private BigDecimal number() { } } + private long size() { + BigDecimal amount = number(); + Token unit = next(); + long scale = unit.kind() != Token.Kind.WORD ? 0 : switch (unit.text().toUpperCase()) { + case "B" -> 1; + case "KB", "KIB" -> 1L << 10; + case "MB", "MIB" -> 1L << 20; + case "GB", "GIB" -> 1L << 30; + case "TB", "TIB" -> 1L << 40; + default -> 0; + }; + if (scale == 0) { + throw error("expected a size unit (B, KB, MB, GB or TB) but found " + unit); + } + long bytes; + try { + bytes = amount.multiply(BigDecimal.valueOf(scale)).longValueExact(); + } catch (ArithmeticException e) { + throw error(amount.toPlainString() + " " + unit.text() + " is not a whole number of bytes"); + } + if (bytes < 1) { + throw error("a size must be positive, not " + amount.toPlainString() + " " + unit.text()); + } + return bytes; + } + private long longValue() { try { return number().longValueExact(); diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 18d025b..1a369cc 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -5,6 +5,7 @@ import io.hstore.db.query.QueryResult; import io.hstore.db.query.Session; +import io.hstore.engine.CheckpointThreshold; import io.hstore.engine.EngineOptions; import io.hstore.engine.HStoreException; @@ -126,6 +127,9 @@ void rolesAndAuthenticationAreEnforced() { assertThrows(HStoreException.InvalidSchema.class, () -> auditor.execute("INSERT NODE Patient 'x' {mrn: 'x'}")); assertThrows(HStoreException.InvalidSchema.class, () -> alice.execute("CREATE NODE TYPE Forbidden")); assertThrows(HStoreException.InvalidSchema.class, () -> alice.execute("USE TENANT globex")); + HStoreException.InvalidSchema denied = assertThrows(HStoreException.InvalidSchema.class, () -> alice.execute("CHECKPOINT AT 64MB")); + assertTrue(denied.getMessage().contains("ADMIN"), denied.getMessage()); + assertEquals(CheckpointThreshold.Source.AUTOMATIC, database.engine().checkpointThreshold().source()); assertEquals("acme", alice.execute("WHOAMI").rows().getFirst().get(1).render()); } assertTrue(database.requiresAuthentication()); diff --git a/database/src/test/java/io/hstore/db/query/QueryLanguageTest.java b/database/src/test/java/io/hstore/db/query/QueryLanguageTest.java index b83ab0a..a39f5e9 100644 --- a/database/src/test/java/io/hstore/db/query/QueryLanguageTest.java +++ b/database/src/test/java/io/hstore/db/query/QueryLanguageTest.java @@ -17,6 +17,8 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.OptionalLong; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -253,4 +255,39 @@ void syntaxErrorsAreReportedWithOffsets() { HStoreException.InvalidSchema error = assertThrows(HStoreException.InvalidSchema.class, () -> session.execute("MATCH EDGE c:Claim WHERE")); assertTrue(error.getMessage().contains("offset"), error.getMessage()); } + + @Test + void checkpointAtTakesSizesInAnyCaseOrAuto() { + Map sizes = Map.of("CHECKPOINT AT 64MB", 64L << 20, "checkpoint at 64 mb", 64L << 20, "CHECKPOINT AT 1GB", 1L << 30, + "CHECKPOINT AT 512KB", 512L << 10, "CHECKPOINT AT 1.5 GiB", 3L << 29, "Checkpoint At 4096 b", 4096L); + sizes.forEach((text, bytes) -> assertEquals(new Ast.Statement.CheckpointAt(OptionalLong.of(bytes)), Parser.parse(text), text)); + assertEquals(new Ast.Statement.CheckpointAt(OptionalLong.empty()), Parser.parse("CHECKPOINT AT AUTO")); + assertEquals(new Ast.Statement.CheckpointAt(OptionalLong.empty()), Parser.parse("checkpoint at auto")); + assertEquals(new Ast.Statement.Checkpoint(), Parser.parse("CHECKPOINT")); + for (String bad : List.of("CHECKPOINT AT", "CHECKPOINT AT 64", "CHECKPOINT AT 64 PB", "CHECKPOINT AT 0MB", "CHECKPOINT AT -1MB", + "CHECKPOINT AT 0.5B", "CHECKPOINT AT MB", "CHECKPOINT AT 99999999999TB", "CHECKPOINT AT 'auto'", "CHECKPOINT AT 64MB 2")) { + HStoreException.InvalidSchema error = assertThrows(HStoreException.InvalidSchema.class, () -> session.execute(bad), bad); + assertTrue(error.getMessage().contains("syntax error"), error.getMessage()); + } + } + + @Test + void checkpointAtIsKeptAcrossReopenAndAutoClearsIt() { + assertTrue(session.execute("CHECKPOINT AT 64MB").message().contains("67108864 bytes of log, set by CHECKPOINT AT")); + assertEquals(List.of("67108864", "database"), checkpointStats()); + session.close(); + database.close(); + database = HypergraphDatabase.open(directory, DatabaseOptions.defaults() + .withEngine(EngineOptions.defaults().withPageSize(4096).withHistoryLimit(500))); + session = new Session(database); + assertEquals(List.of("67108864", "database"), checkpointStats()); + assertTrue(session.execute("CHECKPOINT AT AUTO").message().contains("sized from the store")); + assertEquals(List.of(String.valueOf(2L << 20), "automatic"), checkpointStats()); + } + + private List checkpointStats() { + Map metrics = session.execute("STATS").rows().stream() + .collect(Collectors.toMap(row -> row.get(0).render(), row -> row.get(1).render())); + return List.of(metrics.get("checkpoint after wal bytes"), metrics.get("checkpoint threshold")); + } } diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 6c874b0..4f2f178 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -210,7 +210,7 @@ The commit's tree nodes stay in memory. A checkpoint later writes every node rea |---|---|---|---| | Caller threads | any (the server uses virtual threads) | the caller | Run reads without locks. Run a transaction's operations without locks. Before committing, wait if the write buffer is full, then take `commitLock` only for append (encoding the key changes, WAL append, and page writes for the rare tree past 4,096 changes). They then block in `GroupCommitter.await` outside the lock. | | (none; commit leader) | the committing thread | `GroupCommitter.await` | There is no group-commit thread. The first committer waiting on an unpublished commit leads: it takes every queued `Pending` as one batch and calls `makeDurable(batchSize, wrotePages)` once (data sync in `PAGE_REFERENCES` mode if a commit in the batch wrote pages, then WAL sync, both only under `Durability.SYNC`). It then runs each publication in generation order and wakes the waiters. One fsync pair covers the whole batch (`averageGroupCommit` in the statistics). A failure poisons the pipeline, and every later commit fails until restart. | -| `hstore-maintenance` | virtual | `StorageEngine` constructor | Parks for `maintenanceInterval` (default 500 ms), or until a commit that finds the write buffer full unparks it. Then, under the `maintenance` lock (`tryLock`): checkpoint once the write buffer passes half of `write_buffer_mb` or the WAL or the pages written since the last checkpoint exceed `checkpoint_wal_mb`, compact one segment if any has fallen below `compaction_live_ratio`, then reclaim retired segments that no pinned reader can still see. A background checkpoint writes the pages, counts references and publishes the catalog image on this thread. | +| `hstore-maintenance` | virtual | `StorageEngine` constructor | Parks for `maintenanceInterval` (default 500 ms), or until a commit that finds the write buffer full, or that passes the checkpoint threshold, unparks it. Then, under the `maintenance` lock (`tryLock`): checkpoint once the write buffer passes half of `write_buffer_mb` or the WAL or the pages written since the last checkpoint exceed the checkpoint threshold (`CHECKPOINT AT`, else `checkpoint_wal_mb`, else a quarter of the live data between 2 MiB and 256 MiB), compact one segment if any has fallen below `compaction_live_ratio`, then reclaim retired segments that no pinned reader can still see. A background checkpoint writes the pages, counts references and publishes the catalog image on this thread. | | `hstore-references` | platform, daemon | `StorageEngine` constructor | Marks dead the pages a transaction spilled and then gave up (abort, conflict, page budget, duplicate request). The queue holds up to 4 096 entries; past that, the caller waits. Counting itself happens at checkpoints, on the checkpointing thread. | | `feed-subscriber` | virtual, one per subscription | `ChangeFeed.subscribe` | Tails the feed segments, waiting on a condition signalled by `append` (with a 250 ms timeout). Subscribers are `SemanticPlane`, `MaterializedViews` and `Statistics`. | | Server connections | virtual, one per socket | `Server` (`Executors.newVirtualThreadPerTaskExecutor`) | Each connection runs a `Session`. | diff --git a/docs/database/hql.md b/docs/database/hql.md index cce3c9f..6d84518 100644 --- a/docs/database/hql.md +++ b/docs/database/hql.md @@ -83,7 +83,7 @@ Values and their storage are described in [data model](data-model.md#values). | [Views](#materialized-views) | `CREATE VIEW`, `REFRESH VIEW`, `VIEW`, `SHOW VIEWS` | | [Tenancy](#tenancy-and-users) | `CREATE TENANT`, `ALTER TENANT`, `SHOW TENANTS`, `CREATE USER`, `ALTER USER`, `DROP USER`, `SHOW USERS`, `USE TENANT`, `AUTHENTICATE` | | [Session](#session-and-output) | `WHOAMI`, `FORMAT` | -| [Maintenance](#maintenance) | `CHECKPOINT`, `COMPACT`, `STATS` | +| [Maintenance](#maintenance) | `CHECKPOINT`, `CHECKPOINT AT`, `COMPACT`, `STATS` | Statements marked **ADMIN** are rejected for non-admin principals before execution (`Executor.administrative`). Write statements additionally require the WRITER role (`Principal.requireWrite`). @@ -702,20 +702,38 @@ Atom cells render as `{"id", "label"}`, list cells as arrays, `NULL` as `null`, ## Maintenance ```ebnf -checkpoint = "CHECKPOINT" ; (* ADMIN *) -compact = "COMPACT" ; (* ADMIN *) -stats = "STATS" ; (* ADMIN *) +checkpoint = "CHECKPOINT" ; (* ADMIN *) +checkpoint_at = "CHECKPOINT" "AT" ( "AUTO" | number unit ) ; (* ADMIN *) +unit = "B" | "KB" | "MB" | "GB" | "TB" | "KiB" | "MiB" | "GiB" | "TiB" ; (* any case *) +compact = "COMPACT" ; (* ADMIN *) +stats = "STATS" ; (* ADMIN *) ``` ```sql -CHECKPOINT; -- checkpoint at generation 246, lsn 222166 -COMPACT; -- compacted segments [] at generation 246 +CHECKPOINT; -- checkpoint at generation 246, lsn 222166 +CHECKPOINT AT 64MB; -- checkpoint after 67108864 bytes of log, set by CHECKPOINT AT +CHECKPOINT AT 1 gb; -- checkpoint after 1073741824 bytes of log, set by CHECKPOINT AT +CHECKPOINT AT AUTO; -- checkpoint after 2097152 bytes of log, sized from the store +COMPACT; -- compacted segments [] at generation 246 STATS; ``` +`CHECKPOINT AT` sets how much WAL, or how many bytes of pages, the background maintenance thread lets build up +before it takes a checkpoint. Units are powers of 1024 (`1KB` is 1,024 bytes) and the space before the unit is +optional. The size must be a whole, positive number of bytes; a size without a unit is a syntax error. The value +is stored in the database and committed like any other change, so it survives a restart or a crash, and it takes +effect on the next maintenance pass. It wins over the `checkpoint_wal_mb` setting. `CHECKPOINT AT AUTO` removes +it, and the threshold goes back to `checkpoint_wal_mb`, or to the automatic rule when that is `auto` (see +[configuration](../operations/configuration.md#reference)). The statement commits on its own, outside any open +transaction, and needs the ADMIN role. + +The value is kept in an engine slot that builds without `CHECKPOINT AT` don't know. Once a database has run +`CHECKPOINT AT `, older builds can no longer open it: they fail with `no slot registered with id 8`. + `STATS` columns `metric, value`: catalog statistics (see [statistics](views-signals-stats.md#statistics)), engine counters (`commits`, `rebased commits`, `conflicts`, `pages read`, `pages written`, `data bytes written`, -`wal bytes`, `wal segments`, `cache hit rate`, `data segments`, `feed bytes`), the `semantic index generation` +`wal bytes`, `wal segments`, `checkpoint after wal bytes`, `checkpoint threshold` (`automatic`, `database` or +`configuration`), `cache hit rate`, `data segments`, `feed bytes`), the `semantic index generation` and one `planner feedback ` row per access-path kind with a learned correction factor. ## Errors diff --git a/docs/database/security.md b/docs/database/security.md index be0eaf7..6e49231 100644 --- a/docs/database/security.md +++ b/docs/database/security.md @@ -17,7 +17,7 @@ enum Role { ADMIN, WRITER, READER } | read atoms of its tenant | yes | yes | yes | | insert, update, delete atoms, members, properties, embeddings, evidence, state, signals | yes | yes | no | | DDL (`CREATE … TYPE`, `CREATE INDEX`, `CREATE JSON INDEX`) | yes | no | no | -| branches (`CREATE/DROP/MERGE BRANCH`, `SAMPLE`, `DIFF`), `HISTORY`, `STATS`, `CHECKPOINT`, `COMPACT` | yes | no | no | +| branches (`CREATE/DROP/MERGE BRANCH`, `SAMPLE`, `DIFF`), `HISTORY`, `STATS`, `CHECKPOINT`, `CHECKPOINT AT`, `COMPACT` | yes | no | no | | tenants and users | yes | no | no | | create, refresh and read materialized views (`SHOW VIEWS` lists names for every role) | yes | no | no | | switch tenant (`USE TENANT`, `Principal.inTenant`) | yes | no | no | diff --git a/docs/operations/configuration.md b/docs/operations/configuration.md index 0dde46d..ec4fec5 100644 --- a/docs/operations/configuration.md +++ b/docs/operations/configuration.md @@ -93,7 +93,7 @@ max_connections = 200 | `wal_mode` | `HSTORE_WAL_MODE` | `references` | Applies to the pages a commit writes itself: spilled pages of bulk loads and trees that changed in more than 4,096 places. `references`: the WAL logs `PageRef(pageId, address, length, crc32c)` records and those pages are synced before the WAL. `images`: the WAL logs their full images. Ordinary commits log only the keys they changed, and pages written at checkpoints are not logged in either mode. See [WAL and recovery](../transactions/wal-and-recovery.md). | | `node_cache_mb` | `HSTORE_NODE_CACHE_MB` | `256` | Megabytes of Java heap for decoded tree nodes, each counted at an estimate of the memory it takes. | | `max_open_segment_files` | `HSTORE_MAX_OPEN_SEGMENT_FILES` | `256` | Segment files kept open at once. A segment file is opened when a page in it is written or read without its memory map, and once the limit is reached the least recently used file that no thread is using is closed to make room, after its unsynced writes are forced to disk. Must be between 1 and 2,147,483,647. | -| `checkpoint_wal_mb` | `HSTORE_CHECKPOINT_WAL_MB` | `256` | WAL volume, or volume of pages written, since the last checkpoint that triggers a background checkpoint. | +| `checkpoint_wal_mb` | `HSTORE_CHECKPOINT_WAL_MB` | `auto` | WAL volume, or volume of pages written, since the last checkpoint that triggers a background checkpoint. `auto` sizes it from the store: a quarter of the live bytes in the data segments, at least 2 MiB and at most 256 MiB. A number is megabytes, between 1 and 16,777,216. A threshold stored with `CHECKPOINT AT` overrides this setting. | | `write_buffer_mb` | `HSTORE_WRITE_BUFFER_MB` | `64` | Megabytes of Java heap held by committed tree nodes that no checkpoint has written yet. A background checkpoint starts once more than half of it is in use; a commit that finds it full waits for a checkpoint, for at most 30 seconds. Must be between 1 and 16,777,216. | | `history_limit` | `HSTORE_HISTORY_LIMIT` | `64` | Committed generations kept addressable for `AT GENERATION`, `AS OF`, `HISTORY` and the Studio time slider. History is kept in memory only: after a restart it holds the checkpointed generation and the generations replayed from the WAL after it. | | `compaction_live_ratio` | `HSTORE_COMPACTION_LIVE_RATIO` | `0.5` | Sealed segments whose live bytes fall below this fraction of their allocated bytes are emptied by compaction, which moves their live pages to the active segment: one segment (the emptiest) per background pass whenever one is below the ratio, or all of them with `COMPACT`. Pages count as free once nothing refers to them and no retained generation or open reader can reach them; a checkpoint releases them. Time-travel history is preserved. Emptied segments are deleted after the next checkpoint, once no reader that started before the move is still open. Must satisfy 0 < r < 1; other values fail at startup. | @@ -171,6 +171,20 @@ at the cost of more frequent catalog writes. Recovery time scales roughly linear last checkpoint. The same volume of page writes also triggers a checkpoint, which is what releases dead pages during a bulk load. +With `auto` (the default) the threshold follows the store. A store with 12 MiB of live data checkpoints every +3 MiB of log, one with 512 MiB every 128 MiB, and one of 1 GiB or more every 256 MiB, the fixed default before +`auto`. The rule is in [maintenance](../storage/maintenance.md#checkpoints). + +The threshold in use is decided in this order: + +1. a value stored in the database with `CHECKPOINT AT ` ([HQL](../database/hql.md#maintenance)), which + persists across restarts and crashes and is removed with `CHECKPOINT AT AUTO`; +2. a number given for `checkpoint_wal_mb` (`EngineOptions.withCheckpointWalBytes` for embedded use); +3. the automatic rule. + +`STATS` and the Studio dashboard show the value in use and which of the three set it. Changing it with +`CHECKPOINT AT` takes effect on the next maintenance pass, without a restart. + **`write_buffer_mb`.** Commits keep their new tree nodes on the heap until a checkpoint writes them, and this setting bounds that heap. A key that many commits change is written once per checkpoint, so a larger buffer means fewer page writes for hot data and fewer checkpoints. The cost is heap, which the collector has to walk like the diff --git a/docs/operations/logging.md b/docs/operations/logging.md index 4de882b..4242973 100644 --- a/docs/operations/logging.md +++ b/docs/operations/logging.md @@ -159,7 +159,7 @@ protocol (origin `connection {n}`) and Studio (origin `studio`). A statement is holds: 1. it matches `log_statement`: - * `ddl`: `CREATE … TYPE/INDEX/JSON INDEX/VIEW/BRANCH`, `DROP BRANCH`, `CREATE/ALTER TENANT`, `CREATE/ALTER/DROP USER`; + * `ddl`: `CREATE … TYPE/INDEX/JSON INDEX/VIEW/BRANCH`, `DROP BRANCH`, `CREATE/ALTER TENANT`, `CREATE/ALTER/DROP USER`, `CHECKPOINT AT`; * `mod`: everything that is not a read. Reads are queries, `EXPLAIN`, `MEMBERS`, `INCIDENT TO`, `DESCRIBE`, HORA operators, `HISTORY`, `STATS`, `SHOW …`, `WHOAMI`, `DIFF …`, signal resolution and `TRACE`; * `all`: every statement; diff --git a/docs/operations/studio.md b/docs/operations/studio.md index 9f84e15..e7a2ea8 100644 --- a/docs/operations/studio.md +++ b/docs/operations/studio.md @@ -141,7 +141,7 @@ Errors use one JSON shape: | `GET /api/graph` | yes | Hypergraph neighbourhood for visualisation; see below. | | `GET /api/schema` | yes | Types visible to the session: `id`, `name`, `kind`, `version`, `count`, `roles`, `properties[{name,type,indexed,required}]`, `jsonIndexes[{path,type}]`, plus `branches`. | | `GET /api/history` | yes | `{"current": G, "generations": [{"id","wallTime","txnId"}]}` for the retained generations. | -| `GET /api/stats` | ADMIN | Engine counters, segments (`id`, `state`, `pages` = node images, `bytes`, `live`), recovery outcome, configuration (`pageSize`, `segmentBytes` (the largest segment), `durability`, `walMode`, `nodeCacheBytes`, `historyLimit`, `checkpointWalBytes`), semantic index generation, Studio session count, branches. | +| `GET /api/stats` | ADMIN | Engine counters, segments (`id`, `state`, `pages` = node images, `bytes`, `live`), recovery outcome, configuration (`pageSize`, `segmentBytes` (the largest segment), `durability`, `walMode`, `nodeCacheBytes`, `historyLimit`, `checkpointWalBytes` = the threshold in use, `checkpointSource` = `automatic`, `database` or `configuration`), semantic index generation, Studio session count, branches. | Example `GET /api/info`: diff --git a/docs/storage/catalog-and-generations.md b/docs/storage/catalog-and-generations.md index 15f3a92..acab268 100644 --- a/docs/storage/catalog-and-generations.md +++ b/docs/storage/catalog-and-generations.md @@ -28,6 +28,7 @@ A **root vector** (`catalog/RootVector.java`) is an immutable `Map | 5 | `symbols` | primary | #11 | symbol id → role name or role set | | 6 | `symbol-lookup` | derived | #12 / #13 | `symbol.hash()` → symbol id | | 7 | `requests` | primary | #14 | `hash(requestId)` → `RequestRecord` (idempotent commit keys) | +| 8 | `settings` | primary | #15 | setting number → value; key 1 is the threshold stored by `CHECKPOINT AT` ([maintenance.md](maintenance.md#checkpoints)) | Tree schema ids 2–4 are the nested topology schemas (`set-members`, `ordered-members`, `order-index`). They never appear as slot roots; they are reached only through edge records. diff --git a/docs/storage/maintenance.md b/docs/storage/maintenance.md index a4a29f6..30cb006 100644 --- a/docs/storage/maintenance.md +++ b/docs/storage/maintenance.md @@ -15,7 +15,7 @@ Sources are under `engine/src/main/java/io/hstore/engine/maintenance/`, plus `St | Trigger | Action | Code | |---|---|---| | Engine open, after recovery | checkpoint, then reclaim | `StorageEngine` constructor | -| `hstore-maintenance` virtual thread, every `maintenanceInterval` (default 500 ms) or when a commit wakes it because the write buffer is full (skipped when the `maintenance` lock is busy) | checkpoint if the write buffer holds more than half of `write_buffer_mb`, more than `checkpointWalBytes` of WAL or of pages were written since the last one, or more than 1 048 576 counts changed; compact one segment if any is below `compaction_live_ratio`, sealing the active one first if it is at least a quarter full and that far below (followed by a checkpoint); then reclaim | `StorageEngine.maintain` | +| `hstore-maintenance` virtual thread, every `maintenanceInterval` (default 500 ms) or when a commit wakes it because the write buffer is full or the checkpoint threshold was passed (skipped when the `maintenance` lock is busy) | checkpoint if the write buffer holds more than half of `write_buffer_mb`, more than the checkpoint threshold of WAL or of pages were written since the last one, or more than 1 048 576 counts changed; compact one segment if any is below `compaction_live_ratio`, sealing the active one first if it is at least a quarter full and that far below (followed by a checkpoint); then reclaim | `StorageEngine.maintain` | | `CHECKPOINT;` (HQL), `StorageEngine.checkpoint()` | checkpoint | `timedCheckpoint` | | `COMPACT;` (HQL), `StorageEngine.compact()` | checkpoint, compact, checkpoint, reclaim | `StorageEngine.compact` | | Every checkpoint | count references between the previous checkpointed generation and the new one | `ReferenceCounter.checkpointed` | @@ -51,7 +51,25 @@ An ordinary commit doesn't write tree pages. It logs the keys it changed ([wal-a If a step after the catalog is published fails and the engine keeps running, the journal on disk still belongs to the published catalog. The next checkpoint applies that journal before it writes its own, so the file never falls more than one checkpoint behind the catalog. If publishing the catalog itself fails, it isn't known whether the new catalog reached the disk, so `pages.ref` is marked for rebuilding and the next open recounts. The full ordering, and what a crash at each step leaves behind, is in [wal-and-recovery.md](../transactions/wal-and-recovery.md#9-checkpoints). -**Trigger threshold.** `walSinceCheckpoint() = wal.end() − lastLsn` is compared with `checkpoint_wal_mb` (default 256 MiB, `EngineOptions.checkpointWalBytes`), and so is the volume of pages written since the last checkpoint. A commit logs its changed keys rather than its pages, so the threshold corresponds to a lot of committed work, and replay after a crash stays short. +**Trigger threshold.** `walSinceCheckpoint() = wal.end() − lastLsn` is compared with the checkpoint threshold, and so is the volume of pages written since the last checkpoint. `StorageEngine.checkpointThreshold()` resolves the threshold on every maintenance pass ([`CheckpointThreshold`](../../engine/src/main/java/io/hstore/engine/CheckpointThreshold.java)): + +1. a value stored with `CHECKPOINT AT ` (`StorageEngine.checkpointAt`) wins; +2. otherwise a fixed `checkpoint_wal_mb` (`EngineOptions.checkpointWalBytes`); +3. otherwise, with `checkpoint_wal_mb = auto` (the default, `EngineOptions.AUTOMATIC_CHECKPOINTS`), a quarter of the live bytes in the data segments, held between 2 MiB and 256 MiB. + +| Live data | Automatic threshold | +|---|---| +| up to 8 MiB | 2 MiB | +| 12 MiB | 3 MiB | +| 64 MiB | 16 MiB | +| 512 MiB | 128 MiB | +| 1 GiB and more | 256 MiB | + +The rule weighs what recovery has to redo against what a checkpoint costs. Recovery replays every commit logged since the last checkpoint and then writes every page those commits changed, so the replayed log should stay small next to the store. A checkpoint writes the pages changed since the previous one, which is at most the whole store (the write buffer trigger also starts one once unwritten nodes pass half of `write_buffer_mb`). Checkpointing once the log reaches a quarter of the live data keeps replay to a quarter of the store's size, and keeps page writes from checkpoints to at most about four bytes per byte of log even when every checkpoint rewrites the whole store. The 2 MiB floor keeps a nearly empty store from paying the fixed cost of a checkpoint (a catalog image, a reference journal and several fsyncs) every few hundred kilobytes of log, while a store being loaded from empty still checkpoints within its first few megabytes. The ceiling is the fixed 256 MiB threshold used before. + +**Waking on commit.** The maintenance thread also looks at the threshold right after a commit passes it, not only every `maintenanceInterval`. At the end of each maintenance pass `StorageEngine.armWake` turns the threshold into two marks: the WAL byte count and the page byte count at which it will be passed. `TransactionManager` runs `StorageEngine.wakeIfDue` after every commit, which compares `wal.bytesAppended()` and `pages.bytesWritten()` with the marks and unparks the maintenance thread when either is passed. The checkpoint still runs on the maintenance thread. The first commit to pass a mark sets a flag that the next maintenance pass clears, so the commits that follow while that checkpoint is pending or running don't wake the thread again. + +The stored value lives in the engine slot `settings` (slot 8, key 1), written by a system transaction on main like interned symbols, so it is logged, checkpointed and replayed with everything else. The slot has no root until a value is stored, so a store that never ran `CHECKPOINT AT` is unchanged on disk. A build from before the slot existed cannot open a store that has a stored value: it fails with `no slot registered with id 8`. `CHECKPOINT AT AUTO` deletes the key. **What recovery gains.** Recovery loads the newest valid catalog image and replays only the commits logged after `checkpointLsn` ([wal-and-recovery.md](../transactions/wal-and-recovery.md)). Its segment table restores the allocator position (`units`) and the compaction state machine. History is not in the image, so time travel after a restart reaches only the generations recovery rebuilt ([catalog-and-generations.md](catalog-and-generations.md#history-retention-and-time-travel)). @@ -149,7 +167,7 @@ transaction that a crash cut short are found on the next open (see below). not touched. A checkpoint journals them and then applies them (steps 6 and 11 above), so a crash at any point leaves the file as of one checkpoint or the next, never a mix. The background thread checkpoints early when more than 1 048 576 counts have changed, which keeps that map to a few tens of megabytes. It also checkpoints after every -`checkpoint_wal_mb` of pages written, because a bulk load writes many pages for little WAL and its dead pages would +checkpoint threshold's worth of pages written, because a bulk load writes many pages for little WAL and its dead pages would otherwise wait a long time to be released. **On open**, `PageStore.open` applies `pages.ref.journal` if it was written by the checkpoint the catalog points at @@ -309,7 +327,7 @@ A failure names the page, for example `HStoreException.corrupt(pageId, "checksum | Setting | Default | Effect | |---|---|---| -| `checkpoint_wal_mb` | 256 | WAL or page volume since the last checkpoint that triggers a background checkpoint | +| `checkpoint_wal_mb` | auto | WAL or page volume since the last checkpoint that triggers a background checkpoint. `auto` is a quarter of the live data, between 2 MiB and 256 MiB; `CHECKPOINT AT` overrides it | | `write_buffer_mb` | 64 | Heap held by nodes no checkpoint has written yet. A background checkpoint starts above half of it; commits wait while it is over the limit | | `history_limit` | 64 | Generations kept for time travel (more while pinned) | | `compaction_live_ratio` | 0.5 | A sealed segment is compacted when fewer than this fraction of its images are live | diff --git a/docs/storage/topology.md b/docs/storage/topology.md index ffd5dce..f5c44e3 100644 --- a/docs/storage/topology.md +++ b/docs/storage/topology.md @@ -190,6 +190,7 @@ MATCH EDGE c:Claim WHERE c CONTAINS ($alice, $bob) RETURN c.amount, card(c); | 5 `symbols` (#11) | primary | symbol id | `Symbol.Name` / `Symbol.Group` | `Dictionary.intern` (system transaction) | | 6 `symbol-lookup` (#12/#13, inline 8) | derived | `symbol.hash()` | symbol ids | `SYMBOL_LOOKUP` | | 7 `requests` (#14) | primary | `Hashing.of(requestId)` | `RequestRecord(requestId, txnId, generation)` | idempotent commits (`TxnOptions.requestId`) | +| 8 `settings` (#15) | primary | setting number (1 = checkpoint threshold) | the value as a varint | `StorageEngine.checkpointAt` (system transaction on main) | Hash-keyed indexes (`canonical`, `symbol-lookup`, `requests`) store the full original value in the posting and compare it on lookup (`Views.resolve`, `Dictionary.find`, `priorRequest`). A 64-bit collision is therefore detected and never confused with a match. diff --git a/docs/transactions/wal-and-recovery.md b/docs/transactions/wal-and-recovery.md index ec0c366..b8e4bad 100644 --- a/docs/transactions/wal-and-recovery.md +++ b/docs/transactions/wal-and-recovery.md @@ -321,7 +321,7 @@ The checkpoint LSN is the end of `g`'s `Commit` record, not the position of the | Trigger | Where | |---|---| | Every open, right after recovery | `StorageEngine` constructor | -| The write buffer holds more than half of `writeBufferBytes` (default 64 MiB); WAL or page writes grew by more than `checkpointWalBytes` (default 256 MiB) since the last checkpoint; or more than 1 048 576 reference counts changed. Checked every `maintenanceInterval` (default 500 ms), or sooner when a commit finds the write buffer full | `StorageEngine.maintain` | +| The write buffer holds more than half of `writeBufferBytes` (default 64 MiB); WAL or page writes grew by more than the checkpoint threshold since the last checkpoint (a value stored with `CHECKPOINT AT`, else `checkpointWalBytes`, else by default a quarter of the live data between 2 MiB and 256 MiB; see [maintenance](../storage/maintenance.md#checkpoints)); or more than 1 048 576 reference counts changed. Checked every `maintenanceInterval` (default 500 ms), or sooner when a commit finds the write buffer full or passes the checkpoint threshold | `StorageEngine.maintain` | | Compaction, before choosing victims and again after moving pages, before deleting segments | `StorageEngine.compact` | | `verifyReferences()`, before it compares the counts | `StorageEngine.verifyReferences` | | Clean shutdown | `StorageEngine.close` | diff --git a/engine/src/main/java/io/hstore/engine/CheckpointThreshold.java b/engine/src/main/java/io/hstore/engine/CheckpointThreshold.java new file mode 100644 index 0000000..0abb72d --- /dev/null +++ b/engine/src/main/java/io/hstore/engine/CheckpointThreshold.java @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Metacog Labs +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.engine; + +import java.util.OptionalLong; + +public record CheckpointThreshold(long bytes, Source source) { + + public enum Source { AUTOMATIC, DATABASE, CONFIGURATION } + + public static final long FLOOR = 2L << 20; + public static final long CEILING = 256L << 20; + public static final int STORE_FRACTION = 4; + + public CheckpointThreshold { + if (bytes < 1) { + throw new IllegalArgumentException("a checkpoint threshold must be at least one byte: " + bytes); + } + } + + public static long automatic(long liveBytes) { + return Math.clamp(liveBytes / STORE_FRACTION, FLOOR, CEILING); + } + + public static CheckpointThreshold resolve(OptionalLong stored, long configured, long liveBytes) { + if (stored.isPresent()) { + return new CheckpointThreshold(stored.getAsLong(), Source.DATABASE); + } + if (configured != EngineOptions.AUTOMATIC_CHECKPOINTS) { + return new CheckpointThreshold(configured, Source.CONFIGURATION); + } + return new CheckpointThreshold(automatic(liveBytes), Source.AUTOMATIC); + } +} diff --git a/engine/src/main/java/io/hstore/engine/EngineOptions.java b/engine/src/main/java/io/hstore/engine/EngineOptions.java index c309de6..a5b2758 100644 --- a/engine/src/main/java/io/hstore/engine/EngineOptions.java +++ b/engine/src/main/java/io/hstore/engine/EngineOptions.java @@ -30,6 +30,8 @@ public record EngineOptions( List derivations, CrashPoint.Injector faults) { + public static final long AUTOMATIC_CHECKPOINTS = 0; + public EngineOptions { if (Integer.bitCount(pageSize) != 1 || pageSize < 1024 || pageSize > 1 << 20) { throw new IllegalArgumentException("page size must be a power of two between 1 KiB and 1 MiB: " + pageSize); @@ -43,6 +45,9 @@ public record EngineOptions( if (nodeCacheBytes < 1) { throw new IllegalArgumentException("node cache must be at least one byte: " + nodeCacheBytes); } + if (checkpointWalBytes < 0) { + throw new IllegalArgumentException("the checkpoint threshold cannot be negative: " + checkpointWalBytes); + } if (writeBufferBytes < 1) { throw new IllegalArgumentException("the write buffer must be at least one byte: " + writeBufferBytes); } @@ -58,7 +63,7 @@ public record EngineOptions( public static EngineOptions defaults() { return new EngineOptions(16 * 1024, 16 * 1024, 256, 64L << 20, 256L << 20, Durability.SYNC, WalMode.PAGE_REFERENCES, 64, - 256L << 20, 64L << 20, 0.5, 100_000, Duration.ofMillis(500), List.of(), List.of(), CrashPoint.Injector.NONE); + AUTOMATIC_CHECKPOINTS, 64L << 20, 0.5, 100_000, Duration.ofMillis(500), List.of(), List.of(), CrashPoint.Injector.NONE); } public EngineOptions withPageSize(int bytes) { @@ -101,6 +106,10 @@ public EngineOptions withCheckpointWalBytes(long bytes) { bytes, writeBufferBytes, compactionLiveRatio, feedRetention, maintenanceInterval, slots, derivations, faults); } + public EngineOptions withAutomaticCheckpoints() { + return withCheckpointWalBytes(AUTOMATIC_CHECKPOINTS); + } + public EngineOptions withWriteBufferBytes(long bytes) { return new EngineOptions(pageSize, pagesPerSegment, maxOpenSegmentFiles, walSegmentBytes, nodeCacheBytes, durability, walMode, historyLimit, checkpointWalBytes, bytes, compactionLiveRatio, feedRetention, maintenanceInterval, slots, derivations, faults); diff --git a/engine/src/main/java/io/hstore/engine/StorageEngine.java b/engine/src/main/java/io/hstore/engine/StorageEngine.java index 7fbdab9..1fc3c6e 100644 --- a/engine/src/main/java/io/hstore/engine/StorageEngine.java +++ b/engine/src/main/java/io/hstore/engine/StorageEngine.java @@ -6,6 +6,7 @@ import io.hstore.engine.catalog.Branch; import io.hstore.engine.catalog.CatalogImage; import io.hstore.engine.catalog.CatalogStore; +import io.hstore.engine.catalog.EngineSlots; import io.hstore.engine.catalog.SlotRegistry; import io.hstore.engine.feed.ChangeFeed; import io.hstore.engine.feed.FeedCodec; @@ -38,9 +39,12 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; import java.util.Properties; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.LockSupport; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -69,9 +73,14 @@ public final class StorageEngine implements AutoCloseable { private final Dictionary dictionary; private final Recovery.Outcome recovery; private final ReentrantLock maintenance = new ReentrantLock(); + private final ReentrantLock settings = new ReentrantLock(); private final Thread maintainer; private final List checkpointListeners = new CopyOnWriteArrayList<>(); private long writtenAtCheckpoint; + private volatile OptionalLong checkpointAt; + private final AtomicBoolean wakePending = new AtomicBoolean(); + private volatile long wakeAtWal = Long.MAX_VALUE; + private volatile long wakeAtPages = Long.MAX_VALUE; private volatile boolean closed; private StorageEngine(Path directory, EngineOptions options, FileChannel lockChannel, FileLock lock) { @@ -100,13 +109,16 @@ private StorageEngine(Path directory, EngineOptions options, FileChannel lockCha checkpoint.current(), recovery.image().checkpointLsn()); this.compactor = new Compactor(transactions, pages, checkpointer, options.faults()); this.dictionary = new Dictionary(transactions); + this.checkpointAt = storedCheckpointAt(); checkpointer.checkpoint(); compactor.reclaim(); references.start(); LOG.log(System.Logger.Level.INFO, "database at {0} is ready: generation {1}, {2} data segments, page size {3}, {4} durability, {5} wal", directory, transactions.current().id(), pages.segments().size(), options.pageSize(), options.durability(), options.walMode()); + armWake(); this.maintainer = Thread.ofVirtual().name("hstore-maintenance").start(this::maintain); transactions.whenFull(() -> LockSupport.unpark(maintainer)); + transactions.afterCommit(this::wakeIfDue); } public static StorageEngine open(Path directory, EngineOptions options) { @@ -317,6 +329,37 @@ public Map liveness() { return pages.liveBytes(); } + public CheckpointThreshold checkpointThreshold() { + return CheckpointThreshold.resolve(checkpointAt, options.checkpointWalBytes(), + pages.liveBytes().values().stream().mapToLong(Long::longValue).sum()); + } + + public CheckpointThreshold checkpointAt(OptionalLong bytes) { + requireOpen(); + if (bytes.isPresent() && bytes.getAsLong() < 1) { + throw HStoreException.invalid("a checkpoint threshold must be at least one byte, not " + bytes.getAsLong()); + } + settings.lock(); + try { + transactions.system(Branch.MAIN, workspace -> { + workspace.write(EngineSlots.SETTINGS, EngineSlots.CHECKPOINT_WAL_BYTES, + _ -> bytes.isPresent() ? Optional.of(bytes.getAsLong()) : Optional.empty()); + return null; + }); + checkpointAt = bytes; + } finally { + settings.unlock(); + } + LockSupport.unpark(maintainer); + return checkpointThreshold(); + } + + private OptionalLong storedCheckpointAt() { + try (Snapshot snapshot = transactions.snapshot(Branch.MAIN)) { + return snapshot.get(EngineSlots.SETTINGS, EngineSlots.CHECKPOINT_WAL_BYTES).stream().mapToLong(Long::longValue).findFirst(); + } + } + public ReferenceCounter.Verification verifyReferences() { requireOpen(); maintenance.lock(); @@ -351,9 +394,10 @@ private void maintain() { continue; } try { + long threshold = checkpointThreshold().bytes(); if (transactions.recountUnwritten() > options.writeBufferBytes() / 2 - || checkpointer.walSinceCheckpoint() > options.checkpointWalBytes() - || pages.bytesWritten() - writtenAtCheckpoint > options.checkpointWalBytes() + || checkpointer.walSinceCheckpoint() > threshold + || pages.bytesWritten() - writtenAtCheckpoint > threshold || references.pending() > CHANGED_COUNTS_LIMIT) { timedCheckpoint(); } @@ -368,6 +412,7 @@ private void maintain() { if (!reclaimed.isEmpty()) { LOG.log(System.Logger.Level.INFO, "reclaimed retired segments {0}", reclaimed); } + armWake(); } finally { maintenance.unlock(); } @@ -379,6 +424,23 @@ private void maintain() { } } + private void armWake() { + long threshold = checkpointThreshold().bytes(); + wakeAtWal = saturatedSum(wal.bytesAppended(), Math.max(0, threshold - checkpointer.walSinceCheckpoint())); + wakeAtPages = saturatedSum(writtenAtCheckpoint, threshold); + wakePending.set(false); + } + + private void wakeIfDue() { + if ((wal.bytesAppended() > wakeAtWal || pages.bytesWritten() > wakeAtPages) && wakePending.compareAndSet(false, true)) { + LockSupport.unpark(maintainer); + } + } + + private static long saturatedSum(long base, long increment) { + return increment > Long.MAX_VALUE - base ? Long.MAX_VALUE : base + increment; + } + private void requireOpen() { if (closed) { throw new IllegalStateException("storage engine at " + directory + " is closed"); diff --git a/engine/src/main/java/io/hstore/engine/catalog/EngineSlots.java b/engine/src/main/java/io/hstore/engine/catalog/EngineSlots.java index eaf4603..7de2162 100644 --- a/engine/src/main/java/io/hstore/engine/catalog/EngineSlots.java +++ b/engine/src/main/java/io/hstore/engine/catalog/EngineSlots.java @@ -50,6 +50,11 @@ public record RequestRecord(String requestId, long txnId, long generation) { in -> new RequestRecord(in.getString(), in.getVarLong(), in.getVarLong())), EntryMeasure.keyed(request -> Hashing.of(request.requestId()))); + public static final TreeSchema SETTING_SCHEMA = new TreeSchema<>(15, "settings", FingerprintMode.SET, + ValueCodec.rows(_ -> 10, ByteCursor::putVarLong, ByteCursor::getVarLong), EntryMeasure.keyed(Hashing::of)); + + public static final long CHECKPOINT_WAL_BYTES = 1; + public static final Slot CATALOG = Slot.primary(1, "catalog", ATOM_SCHEMA); public static final Slot> REVERSE = Slot.derived(2, "reverse", REVERSE_INDEX.schema()); public static final Slot> TYPES = Slot.derived(3, "type-index", TYPE_INDEX.schema()); @@ -58,7 +63,9 @@ public record RequestRecord(String requestId, long txnId, long generation) { public static final Slot> SYMBOL_LOOKUP = Slot.derived(6, "symbol-lookup", SYMBOL_INDEX.schema()); public static final Slot REQUESTS = Slot.primary(7, "requests", REQUEST_SCHEMA); - public static final List> ALL = List.of(CATALOG, REVERSE, TYPES, CANONICAL, SYMBOLS, SYMBOL_LOOKUP, REQUESTS); + public static final Slot SETTINGS = Slot.primary(8, "settings", SETTING_SCHEMA); + + public static final List> ALL = List.of(CATALOG, REVERSE, TYPES, CANONICAL, SYMBOLS, SYMBOL_LOOKUP, REQUESTS, SETTINGS); private EngineSlots() { } diff --git a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java index 9766af5..b036782 100644 --- a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java +++ b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java @@ -89,6 +89,8 @@ private record PageWrite(long pageId, MemorySegment image) { private final Condition room = roomLock.newCondition(); private volatile Runnable full = () -> { }; + private volatile Runnable committed = () -> { + }; private volatile boolean stopped; private final AtomicLong atoms; private final AtomicLong txns; @@ -251,6 +253,7 @@ CommitResult commit(Transaction txn) { } Generation published = committer.await(appended.pending()); commits.increment(); + committed.run(); if (!fast) { rebases.increment(); } @@ -481,6 +484,10 @@ public void whenFull(Runnable action) { full = action; } + public void afterCommit(Runnable action) { + committed = action; + } + public long recountUnwritten() { long bytes = head.get().branches().values().stream() .flatMap(branch -> branch.roots().roots().values().stream()) diff --git a/engine/src/test/java/io/hstore/engine/CheckpointThresholdTest.java b/engine/src/test/java/io/hstore/engine/CheckpointThresholdTest.java new file mode 100644 index 0000000..42e6c0d --- /dev/null +++ b/engine/src/test/java/io/hstore/engine/CheckpointThresholdTest.java @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: Metacog Labs +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.engine; + +import io.hstore.engine.txn.CrashPoint; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class CheckpointThresholdTest { + + @TempDir + Path directory; + + @Test + void smallStoresGetTheFloorAndLargeStoresTheCeiling() { + assertEquals(2L << 20, CheckpointThreshold.automatic(0)); + assertEquals(2L << 20, CheckpointThreshold.automatic(8L << 20)); + assertEquals(3L << 20, CheckpointThreshold.automatic(12L << 20)); + assertEquals(4L << 20, CheckpointThreshold.automatic(16L << 20)); + assertEquals(8L << 20, CheckpointThreshold.automatic(32L << 20)); + assertEquals(128L << 20, CheckpointThreshold.automatic(512L << 20)); + assertEquals(256L << 20, CheckpointThreshold.automatic(1L << 30)); + assertEquals(256L << 20, CheckpointThreshold.automatic(100L << 30)); + } + + @Test + void aStoredValueWinsOverTheConfigurationWhichWinsOverTheRule() { + assertEquals(new CheckpointThreshold(64L << 20, CheckpointThreshold.Source.DATABASE), + CheckpointThreshold.resolve(OptionalLong.of(64L << 20), 128L << 20, 1L << 30)); + assertEquals(new CheckpointThreshold(128L << 20, CheckpointThreshold.Source.CONFIGURATION), + CheckpointThreshold.resolve(OptionalLong.empty(), 128L << 20, 1L << 30)); + assertEquals(new CheckpointThreshold(10L << 20, CheckpointThreshold.Source.AUTOMATIC), + CheckpointThreshold.resolve(OptionalLong.empty(), EngineOptions.AUTOMATIC_CHECKPOINTS, 40L << 20)); + assertEquals(EngineOptions.AUTOMATIC_CHECKPOINTS, EngineOptions.defaults().checkpointWalBytes()); + assertEquals(EngineOptions.AUTOMATIC_CHECKPOINTS, EngineOptions.defaults().withCheckpointWalBytes(1 << 20).withAutomaticCheckpoints().checkpointWalBytes()); + } + + @Test + void aSmallStoreWithDefaultOptionsCheckpointsOnceItsLogPassesTheFloor() throws InterruptedException { + EngineOptions options = EngineTest.small().withWriteBufferBytes(1L << 30).withMaintenanceInterval(Duration.ofMillis(50)); + AtomicInteger checkpoints = new AtomicInteger(); + int commits = 0; + StorageEngine engine = StorageEngine.open(directory, options); + engine.onCheckpoint(checkpoints::incrementAndGet); + String padding = "x".repeat(200); + while (engine.stats().walBytes() < CheckpointThreshold.FLOOR + (1 << 20)) { + int batch = commits++; + engine.write(txn -> { + for (int i = 0; i < 200; i++) { + txn.createNode(EngineTest.NODE, "node-" + batch + "-" + i + padding); + } + return null; + }); + } + long deadline = System.currentTimeMillis() + 10_000; + while (checkpoints.get() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + CheckpointThreshold threshold = engine.checkpointThreshold(); + engine.halt(); + assertTrue(checkpoints.get() > 0, "no background checkpoint after " + commits + " commits"); + assertEquals(CheckpointThreshold.Source.AUTOMATIC, threshold.source()); + assertTrue(threshold.bytes() < CheckpointThreshold.CEILING, "threshold " + threshold); + try (StorageEngine recovered = StorageEngine.open(directory, options)) { + assertTrue(recovered.recovery().replayedCommits() < commits, + recovered.recovery().replayedCommits() + " of " + commits + " commits replayed"); + assertEquals(200L * commits, (long) recovered.read(snapshot -> snapshot.countOfType(0, EngineTest.NODE))); + } + } + + @Test + void aStoredThresholdSurvivesCloseAndReopenAndAutoClearsIt() { + EngineOptions configured = EngineTest.small().withCheckpointWalBytes(128L << 20); + try (StorageEngine engine = StorageEngine.open(directory, configured)) { + assertEquals(CheckpointThreshold.Source.CONFIGURATION, engine.checkpointThreshold().source()); + assertEquals(new CheckpointThreshold(64L << 20, CheckpointThreshold.Source.DATABASE), engine.checkpointAt(OptionalLong.of(64L << 20))); + assertThrows(HStoreException.class, () -> engine.checkpointAt(OptionalLong.of(0))); + } + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + assertEquals(new CheckpointThreshold(64L << 20, CheckpointThreshold.Source.DATABASE), engine.checkpointThreshold()); + assertEquals(CheckpointThreshold.Source.AUTOMATIC, engine.checkpointAt(OptionalLong.empty()).source()); + } + try (StorageEngine engine = StorageEngine.open(directory, configured)) { + assertEquals(new CheckpointThreshold(128L << 20, CheckpointThreshold.Source.CONFIGURATION), engine.checkpointThreshold()); + } + } + + @ParameterizedTest + @EnumSource(value = CrashPoint.class, names = {"WAL_SYNC", "CHECKPOINT_PAGES", "CHECKPOINT_CATALOG", "CHECKPOINT_JOURNAL"}) + void aStoredThresholdSurvivesACrash(CrashPoint point) { + AtomicBoolean armed = new AtomicBoolean(false); + EngineOptions options = EngineTest.small().withFaults(reached -> { + if (reached == point && armed.compareAndSet(true, false)) { + throw new CrashPoint.SimulatedCrash(point); + } + }); + StorageEngine engine = StorageEngine.open(directory, options); + engine.write(txn -> txn.createNode(EngineTest.NODE, "before")); + engine.checkpointAt(OptionalLong.of(64L << 20)); + armed.set(true); + if (point == CrashPoint.WAL_SYNC) { + assertThrows(CrashPoint.SimulatedCrash.class, () -> engine.write(txn -> txn.createNode(EngineTest.NODE, "doomed"))); + } else { + engine.write(txn -> txn.createNode(EngineTest.NODE, "after")); + assertThrows(CrashPoint.SimulatedCrash.class, engine::checkpoint); + } + engine.halt(); + try (StorageEngine recovered = StorageEngine.open(directory, EngineTest.small())) { + assertEquals(new CheckpointThreshold(64L << 20, CheckpointThreshold.Source.DATABASE), recovered.checkpointThreshold()); + } + } + + @Test + void aStoredThresholdTakesEffectWithoutARestartAndWinsOverTheConfiguration() throws InterruptedException { + EngineOptions options = EngineTest.small().withCheckpointWalBytes(Long.MAX_VALUE).withWriteBufferBytes(1L << 30) + .withMaintenanceInterval(Duration.ofMillis(50)); + AtomicInteger checkpoints = new AtomicInteger(); + try (StorageEngine engine = StorageEngine.open(directory, options)) { + engine.onCheckpoint(checkpoints::incrementAndGet); + engine.checkpointAt(OptionalLong.of(1L << 20)); + String padding = "x".repeat(200); + for (int batch = 0; engine.stats().walBytes() < 2L << 20; batch++) { + int current = batch; + engine.write(txn -> { + for (int i = 0; i < 200; i++) { + txn.createNode(EngineTest.NODE, "node-" + current + "-" + i + padding); + } + return null; + }); + } + long deadline = System.currentTimeMillis() + 10_000; + while (checkpoints.get() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + assertTrue(checkpoints.get() > 0, "no background checkpoint after " + engine.stats().walBytes() + " bytes of log"); + } + } + + @Test + void aCommitThatPassesTheThresholdWakesTheMaintenanceThread() throws InterruptedException { + EngineOptions options = EngineTest.small().withWriteBufferBytes(1L << 30).withMaintenanceInterval(Duration.ofHours(1)); + AtomicInteger checkpoints = new AtomicInteger(); + try (StorageEngine engine = StorageEngine.open(directory, options)) { + engine.onCheckpoint(checkpoints::incrementAndGet); + String padding = "x".repeat(200); + for (int batch = 0; engine.stats().walBytes() <= CheckpointThreshold.FLOOR; batch++) { + int current = batch; + engine.write(txn -> { + for (int i = 0; i < 200; i++) { + txn.createNode(EngineTest.NODE, "node-" + current + "-" + i + padding); + } + return null; + }); + } + long deadline = System.currentTimeMillis() + 10_000; + while (checkpoints.get() == 0 && System.currentTimeMillis() < deadline) { + Thread.sleep(10); + } + assertEquals(1, checkpoints.get(), "background checkpoints after " + engine.stats().walBytes() + " bytes of log"); + } + } +} diff --git a/server/src/main/java/io/hstore/server/ServerConfig.java b/server/src/main/java/io/hstore/server/ServerConfig.java index 66b65ad..63300d8 100644 --- a/server/src/main/java/io/hstore/server/ServerConfig.java +++ b/server/src/main/java/io/hstore/server/ServerConfig.java @@ -115,6 +115,17 @@ int positive(Setting setting) { return (int) value; } + private long checkpointWalBytes() { + String value = string(Setting.CHECKPOINT_WAL_MB); + if (value.equalsIgnoreCase("auto")) { + return EngineOptions.AUTOMATIC_CHECKPOINTS; + } + if (!value.matches("-?[0-9]+")) { + throw new IllegalArgumentException("checkpoint_wal_mb must be auto or a number of megabytes, not " + value); + } + return megabytes(Setting.CHECKPOINT_WAL_MB); + } + boolean flag(Setting setting) { return switch (values.get(setting).toLowerCase()) { case "on", "true", "yes", "1" -> true; @@ -134,7 +145,7 @@ EngineOptions engineOptions() { .withWalMode(string(Setting.WAL_MODE).equalsIgnoreCase("images") ? WalMode.PAGE_IMAGES : WalMode.PAGE_REFERENCES) .withNodeCacheBytes(megabytes(Setting.NODE_CACHE_MB)) .withMaxOpenSegmentFiles(positive(Setting.MAX_OPEN_SEGMENT_FILES)) - .withCheckpointWalBytes(number(Setting.CHECKPOINT_WAL_MB) << 20) + .withCheckpointWalBytes(checkpointWalBytes()) .withWriteBufferBytes(megabytes(Setting.WRITE_BUFFER_MB)) .withHistoryLimit(integer(Setting.HISTORY_LIMIT)) .withCompactionLiveRatio(Double.parseDouble(string(Setting.COMPACTION_LIVE_RATIO))) diff --git a/server/src/main/java/io/hstore/server/Setting.java b/server/src/main/java/io/hstore/server/Setting.java index 0104cf6..033550b 100644 --- a/server/src/main/java/io/hstore/server/Setting.java +++ b/server/src/main/java/io/hstore/server/Setting.java @@ -26,7 +26,7 @@ enum Setting { WAL_MODE("wal_mode", "references", "references logs page checksums and syncs data first; images logs full page images"), NODE_CACHE_MB("node_cache_mb", "256", "megabytes of heap for decoded tree nodes, each counted at an estimate of its size"), MAX_OPEN_SEGMENT_FILES("max_open_segment_files", "256", "segment files kept open at once; opening another closes the least recently used"), - CHECKPOINT_WAL_MB("checkpoint_wal_mb", "256", "write-ahead log volume that triggers a background checkpoint"), + CHECKPOINT_WAL_MB("checkpoint_wal_mb", "auto", "megabytes of write-ahead log that trigger a background checkpoint; auto sizes it from the store"), WRITE_BUFFER_MB("write_buffer_mb", "64", "megabytes of committed tree nodes kept in memory until a checkpoint writes them"), HISTORY_LIMIT("history_limit", "64", "committed generations retained for time travel"), COMPACTION_LIVE_RATIO("compaction_live_ratio", "0.5", "sealed segments whose live node images fall below this ratio are compacted"), diff --git a/server/src/main/java/io/hstore/server/StatementLogger.java b/server/src/main/java/io/hstore/server/StatementLogger.java index 21cec93..48e7b1c 100644 --- a/server/src/main/java/io/hstore/server/StatementLogger.java +++ b/server/src/main/java/io/hstore/server/StatementLogger.java @@ -47,7 +47,7 @@ private static Category category(Statement statement) { return switch (statement) { case Statement.CreateType _, Statement.CreateIndex _, Statement.CreateJsonIndex _, Statement.CreateView _, Statement.CreateBranch _, Statement.DropBranch _, Statement.CreateTenant _, Statement.AlterTenant _, - Statement.CreateUser _, Statement.AlterUser _, Statement.DropUser _ -> Category.DDL; + Statement.CreateUser _, Statement.AlterUser _, Statement.DropUser _, Statement.CheckpointAt _ -> Category.DDL; case Statement.Query _, Statement.Explain _, Statement.Members _, Statement.IncidentTo _, Statement.Describe _, Statement.Gather _, Statement.Reduce _, Statement.Scatter _, Statement.Propagate _, Statement.Neighbors _, Statement.OverlapJoin _, Statement.Closure _, Statement.PatternQuery _, Statement.Expand _, Statement.History _, diff --git a/server/src/main/java/io/hstore/server/studio/Studio.java b/server/src/main/java/io/hstore/server/studio/Studio.java index 6ffbdc2..80a675e 100644 --- a/server/src/main/java/io/hstore/server/studio/Studio.java +++ b/server/src/main/java/io/hstore/server/studio/Studio.java @@ -14,6 +14,7 @@ import io.hstore.db.security.Role; import io.hstore.db.security.Security; import io.hstore.db.value.Json; +import io.hstore.engine.CheckpointThreshold; import io.hstore.engine.EngineOptions; import io.hstore.engine.EngineStats; import io.hstore.engine.HStoreException; @@ -441,6 +442,7 @@ private Response stats(Request request) { TransactionManager.Statistics transactions = database.engine().transactions().statistics(); Recovery.Outcome recovery = database.engine().recovery(); EngineOptions configured = database.engine().options(); + CheckpointThreshold checkpoint = database.engine().checkpointThreshold(); return Response.json(JsonFields.object() .put("sampledAt", System.currentTimeMillis()) .put("startedAt", startedAt) @@ -481,7 +483,8 @@ private Response stats(Request request) { .put("walMode", configured.walMode().name()) .put("nodeCacheBytes", configured.nodeCacheBytes()) .put("historyLimit", configured.historyLimit()) - .put("checkpointWalBytes", configured.checkpointWalBytes()) + .put("checkpointWalBytes", checkpoint.bytes()) + .put("checkpointSource", checkpoint.source().name().toLowerCase()) .build()) .put("semanticGeneration", database.semantic().indexedGeneration()) .put("studioSessions", sessions.size()) diff --git a/server/src/main/resources/studio/js/views/dashboard.js b/server/src/main/resources/studio/js/views/dashboard.js index 29a9d80..b21aaf5 100644 --- a/server/src/main/resources/studio/js/views/dashboard.js +++ b/server/src/main/resources/studio/js/views/dashboard.js @@ -8,6 +8,11 @@ import { bytes, count, duration, number, plural, timestamp } from '../format.js' import { theme } from '../palette.js'; const INTERVAL = 2000; +const CHECKPOINT_SOURCES = { + automatic: 'automatic', + database: 'set by CHECKPOINT AT', + configuration: 'set by checkpoint_wal_mb' +}; export class DashboardPage { constructor(studio) { @@ -125,7 +130,7 @@ export class DashboardPage { pair('WAL mode', configuration.walMode), pair('Node cache', bytes(configuration.nodeCacheBytes)), pair('History limit', `${count(configuration.historyLimit)} generations`), - pair('Checkpoint after', bytes(configuration.checkpointWalBytes)), + pair('Checkpoint after', `${bytes(configuration.checkpointWalBytes)} of WAL · ${CHECKPOINT_SOURCES[configuration.checkpointSource] ?? configuration.checkpointSource}`), pair('Change feed', bytes(engine.feedBytes)), pair('Studio sessions', count(stats.studioSessions)), pair('Replayed at start', `${count(recovery.replayedCommits)} commits`), diff --git a/server/src/test/java/io/hstore/server/ServerTest.java b/server/src/test/java/io/hstore/server/ServerTest.java index c6b8f1f..181d62c 100644 --- a/server/src/test/java/io/hstore/server/ServerTest.java +++ b/server/src/test/java/io/hstore/server/ServerTest.java @@ -180,6 +180,20 @@ void openSegmentFilesComeFromTheFileTheEnvironmentOrAFlag(@TempDir Path data) th } } + @Test + void checkpointWalMbIsAutomaticUnlessGivenInMegabytes() { + assertEquals(EngineOptions.AUTOMATIC_CHECKPOINTS, ServerConfig.load(Optional.empty(), Map.of(), Map.of()).engineOptions().checkpointWalBytes()); + assertEquals(EngineOptions.AUTOMATIC_CHECKPOINTS, + ServerConfig.load(Optional.empty(), Map.of("HSTORE_CHECKPOINT_WAL_MB", "AUTO"), Map.of()).engineOptions().checkpointWalBytes()); + assertEquals(64L << 20, ServerConfig.load(Optional.empty(), Map.of("HSTORE_CHECKPOINT_WAL_MB", "64"), Map.of()).engineOptions().checkpointWalBytes()); + assertEquals(512L << 20, ServerConfig.load(Optional.empty(), Map.of(), Map.of("checkpoint-wal-mb", "512")).engineOptions().checkpointWalBytes()); + for (String invalid : List.of("0", "-1", "99999999999999", "sometimes")) { + IllegalArgumentException failure = assertThrows(IllegalArgumentException.class, + () -> ServerConfig.load(Optional.empty(), Map.of("HSTORE_CHECKPOINT_WAL_MB", invalid), Map.of()).engineOptions()); + assertTrue(failure.getMessage().contains("checkpoint_wal_mb"), failure.getMessage()); + } + } + @Test void tlsWithoutACertificateIsRejectedAtStartup() { ServerConfig config = ServerConfig.load(Optional.empty(), Map.of(), Map.of("tls", "on")); diff --git a/server/src/test/java/io/hstore/server/studio/StudioTest.java b/server/src/test/java/io/hstore/server/studio/StudioTest.java index eb13a49..d26f1fa 100644 --- a/server/src/test/java/io/hstore/server/studio/StudioTest.java +++ b/server/src/test/java/io/hstore/server/studio/StudioTest.java @@ -178,6 +178,14 @@ void dashboardStatisticsAreAvailableToAdministrators() throws Exception { Map stats = object(get("/api/stats", cookie).body()); assertTrue(stats.containsKey("engine")); assertTrue(stats.containsKey("segments")); + Map configuration = ((Json.Obj) stats.get("configuration")).fields(); + assertEquals(new Json.Str("automatic"), configuration.get("checkpointSource")); + try (Session session = new Session(database)) { + session.execute("CHECKPOINT AT 64MB"); + } + configuration = ((Json.Obj) object(get("/api/stats", cookie).body()).get("configuration")).fields(); + assertEquals(new Json.Str("database"), configuration.get("checkpointSource")); + assertEquals(64L << 20, ((Json.Number) configuration.get("checkpointWalBytes")).value().longValueExact()); Map history = object(get("/api/history", cookie).body()); assertTrue(((Json.Array) history.get("generations")).items().size() > 1); }