From 4610ceca5c4997a749accb7ed9dc9cf019f056ce Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 12:15:25 +0200 Subject: [PATCH 1/9] feat(sqlite): open the site's SQLite file with its functions and core tables --- go.mod | 9 + go.sum | 36 +- internal/sqlite/functions.go | 20 + internal/sqlite/functions_internal_test.go | 21 + internal/sqlite/functions_test.go | 97 ++++ internal/sqlite/helpers_test.go | 75 ++++ internal/sqlite/libc_test.go | 15 + internal/sqlite/migrate.go | 82 ++++ internal/sqlite/migrate_internal_test.go | 75 ++++ internal/sqlite/migrate_test.go | 249 ++++++++++ internal/sqlite/migrations.go | 10 + .../migrations/00026_create_core_tables.sql | 207 +++++++++ internal/sqlite/open.go | 131 ++++++ internal/sqlite/open_internal_test.go | 15 + internal/sqlite/open_test.go | 325 ++++++++++++++ internal/sqlite/schema_test.go | 424 ++++++++++++++++++ internal/sqlite/strip.go | 196 ++++++++ internal/sqlite/strip_test.go | 183 ++++++++ 18 files changed, 2166 insertions(+), 4 deletions(-) create mode 100644 internal/sqlite/functions.go create mode 100644 internal/sqlite/functions_internal_test.go create mode 100644 internal/sqlite/functions_test.go create mode 100644 internal/sqlite/helpers_test.go create mode 100644 internal/sqlite/libc_test.go create mode 100644 internal/sqlite/migrate.go create mode 100644 internal/sqlite/migrate_internal_test.go create mode 100644 internal/sqlite/migrate_test.go create mode 100644 internal/sqlite/migrations.go create mode 100644 internal/sqlite/migrations/00026_create_core_tables.sql create mode 100644 internal/sqlite/open.go create mode 100644 internal/sqlite/open_internal_test.go create mode 100644 internal/sqlite/open_test.go create mode 100644 internal/sqlite/schema_test.go create mode 100644 internal/sqlite/strip.go create mode 100644 internal/sqlite/strip_test.go diff --git a/go.mod b/go.mod index 4df68844..15fcd940 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gopherium/framework/dbkit v0.2.0 github.com/gopherium/framework/dbkit/postgres v0.3.0 + github.com/gopherium/framework/dbkit/sqlite v0.1.0 github.com/gopherium/framework/goncierge v0.1.0 github.com/gopherium/framework/gonsole v0.7.0 github.com/gopherium/framework/gonsole/auth v0.7.0 @@ -23,6 +24,7 @@ require ( github.com/gopherium/gouncer/authkit v0.19.0 github.com/gopherium/gouncer/authkit/postgres v0.13.0 github.com/gopherium/gouncer/authkit/ratelimit v0.3.1 + github.com/gopherium/gouncer/authkit/sqlite v0.1.0 github.com/jackc/pgx/v5 v5.11.0 github.com/joho/godotenv v1.5.1 github.com/microcosm-cc/bluemonday v1.0.27 @@ -45,6 +47,7 @@ require ( github.com/cucumber/gherkin/go/v42 v42.0.0 // indirect github.com/cucumber/messages/go/v34 v34.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.18.0 // indirect github.com/fatih/structtag v1.2.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect @@ -69,6 +72,7 @@ require ( github.com/mfridman/interpolate v0.0.2 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/ncruces/go-sqlite3 v0.32.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/julianday v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pganalyze/pg_query_go/v6 v6.2.2 // indirect @@ -76,6 +80,7 @@ require ( github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect github.com/pingcap/log v1.1.0 // indirect github.com/pingcap/tidb/pkg/parser v0.0.0-20260418072757-ce92298d1124 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/riza-io/grpc-go v0.2.0 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/sethvargo/go-retry v0.4.0 // indirect @@ -107,6 +112,10 @@ require ( google.golang.org/protobuf v1.36.12 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.77.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect + modernc.org/sqlite v1.60.1 // indirect ) replace github.com/gopherium/gophenberg/sdk => ./sdk diff --git a/go.sum b/go.sum index 174cb332..bd9c724a 100644 --- a/go.sum +++ b/go.sum @@ -58,12 +58,16 @@ github.com/google/cel-go v0.29.0 h1:fEG+Ja3YRwNOqnQxTyJwoByAUAvTuxUGiro/jhrm4F4= github.com/google/cel-go v0.29.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherium/framework/dbkit v0.2.0 h1:V+Di+B8oGvdmQKPx5q/a/PbgfHOvRAHhCFfQ5g8DojM= github.com/gopherium/framework/dbkit v0.2.0/go.mod h1:sGsNkpbHe1LRW7k1qEjTzqbeHVYOnoCI5e1l/81TAvQ= github.com/gopherium/framework/dbkit/postgres v0.3.0 h1:W9SoXXu9Y3YRum5vIG10iBHlMMIhmbuUmOPLmZEnW3U= github.com/gopherium/framework/dbkit/postgres v0.3.0/go.mod h1:ZUsvp2lOkhX+08sstBErt+jN6FM5HkhxV7hXgToRd+A= +github.com/gopherium/framework/dbkit/sqlite v0.1.0 h1:NmfcEuhlV5/SpqqZHHeN1qtA6FBDwERGmYFCHMXy5YA= +github.com/gopherium/framework/dbkit/sqlite v0.1.0/go.mod h1:43gIZC9JkU/PeAvKLVjEubbe6TOz+1rjIANf615QEOI= github.com/gopherium/framework/goncierge v0.1.0 h1:9Zn2zxAuWTrhKeMEHNSncAb1jmjFrkEpVYXGdZrxTG8= github.com/gopherium/framework/goncierge v0.1.0/go.mod h1:ekTJKyCbU0s659tjHvH/sjEZfPp2S4iuOGaLRRULZuA= github.com/gopherium/framework/gonsole v0.7.0 h1:JluS72WcE6LXcBihJB+5zdhX7IUiLZxv0g059z8T3/U= @@ -80,6 +84,8 @@ github.com/gopherium/gouncer/authkit/postgres v0.13.0 h1:9uN7xZirufk0osM/LxMj0K1 github.com/gopherium/gouncer/authkit/postgres v0.13.0/go.mod h1:wnA/EmvSqV3w8n3L4EYTejY1bnYjP7Cd7O+sRtf9bkw= github.com/gopherium/gouncer/authkit/ratelimit v0.3.1 h1:l+YGVBqG1FDxIl/G1WaoIPS/jbsUicQRg0p1NYWxk9s= github.com/gopherium/gouncer/authkit/ratelimit v0.3.1/go.mod h1:BXea+ZZJUypviIcWmb788rQSBpqW5VcGvovlJPOblqw= +github.com/gopherium/gouncer/authkit/sqlite v0.1.0 h1:pyPOkXVy5CzYR+tTkqL/6FuVcwBKf9FXCZe2XDcgwl0= +github.com/gopherium/gouncer/authkit/sqlite v0.1.0/go.mod h1:Rd+jedhpfi4GaQ34RrdmlqQgIvVB4FHN4ftGCAa+gfs= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b h1:wDUNC2eKiL35DbLvsDhiblTUXHxcOPwQSCzi7xpQUN4= @@ -94,6 +100,8 @@ github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb h1:PGufWXXDq9yaev6xX1YQauaO1MV90e6Mpoq1I7Lz/VM= github.com/hectane/go-acl v0.0.0-20230122075934-ca0b05cb1adb/go.mod h1:QiyDdbZLaJ/mZP4Zwc9g2QsfaEA4o7XvvgZegSci5/E= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -290,11 +298,31 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= -modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= +modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= +modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= +modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= -modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= +modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/sqlite/functions.go b/internal/sqlite/functions.go new file mode 100644 index 00000000..e8e6cf04 --- /dev/null +++ b/internal/sqlite/functions.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import "github.com/gopherium/framework/dbkit" + +// functions lists the Go functions every SQLite handle of the site registers, built once per process. +var functions = mustFunctions(dbkit.NewFunctionList( + dbkit.CaseFold(), + dbkit.Function{Name: "strip_field_path", Args: 2, Deterministic: true, Call: stripFieldPath}, + dbkit.Function{Name: "strip_layout", Args: 2, Deterministic: true, Call: stripLayout}, +)) + +// mustFunctions returns list and panics with err when dbkit could not build it. +func mustFunctions(list *dbkit.FunctionList, err error) *dbkit.FunctionList { + if err != nil { + panic(err) + } + return list +} diff --git a/internal/sqlite/functions_internal_test.go b/internal/sqlite/functions_internal_test.go new file mode 100644 index 00000000..9de01714 --- /dev/null +++ b/internal/sqlite/functions_internal_test.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "errors" + "testing" +) + +func TestMustFunctionsPanicsOnAListDbkitCannotBuild(t *testing.T) { + t.Parallel() + + failed := errors.New("no function list") + defer func() { + if recovered := recover(); recovered != failed { + t.Fatalf("recovered %v, want the error", recovered) + } + }() + + mustFunctions(nil, failed) +} diff --git a/internal/sqlite/functions_test.go b/internal/sqlite/functions_test.go new file mode 100644 index 00000000..a30ec6f1 --- /dev/null +++ b/internal/sqlite/functions_test.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite_test + +import ( + "testing" + + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" +) + +func TestEveryFreshConnectionCallsTheSiteFunctions(t *testing.T) { + t.Parallel() + + settings := testSettings() + db := sqlitetest.Open(t, settings.Options()) + + for i, conn := range fullPool(t, db, settings.MaxConns) { + var folded, field, layout string + err := conn.QueryRowContext(t.Context(), `SELECT casefold('ÉCOLE'), `+ + `strip_field_path('{"a":1,"b":2}', '["a"]'), strip_layout('{"rows":["a","b"]}', '["rows","a"]')`). + Scan(&folded, &field, &layout) + if err != nil { + t.Fatalf("calling the site functions on connection %d error = %v, want nil", i, err) + } + if folded != "école" || field != `{"b":2}` || layout != `{"rows":["b"]}` { + t.Errorf("the site functions on connection %d = %q, %q and %q, want école, {\"b\":2} and {\"rows\":[\"b\"]}", + i, folded, field, layout) + } + } +} + +func TestOpenRegistersTheSiteFunctions(t *testing.T) { + t.Parallel() + + address, _ := emptyFile(t) + db := open(t, address, testSettings()) + + var folded string + if err := db.QueryRowContext(t.Context(), "SELECT casefold('Straße')").Scan(&folded); err != nil { + t.Fatalf("SELECT casefold() error = %v, want nil", err) + } + if folded != "straße" { + t.Errorf("casefold('Straße') = %q, want straße", folded) + } +} + +func TestTheSiteFunctionsServeAnIndexExpression(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + mustExec(t, db, "CREATE TABLE docs (doc TEXT)") + + for name, expression := range map[string]string{ + "folded": "casefold(doc)", "fields": `strip_field_path(doc, '["a"]')`, "layout": `strip_layout(doc, '["a"]')`, + } { + statement := "CREATE INDEX docs_" + name + "_idx ON docs (" + expression + ")" + if _, err := db.ExecContext(t.Context(), statement); err != nil { + t.Errorf("%s error = %v, want a function deterministic enough to index", statement, err) + } + } +} + +func TestCasefoldFoldsTextByUnicodeSimpleFolding(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + + for text, want := range map[string]string{ + "ÉCOLE": "école", "Straße": "straße", "K": "k", "ΣΑΣ": "σασ", "already folded": "already folded", + } { + var got string + if err := db.QueryRowContext(t.Context(), "SELECT casefold(?)", text).Scan(&got); err != nil { + t.Fatalf("casefold(%q) error = %v, want nil", text, err) + } + if got != want { + t.Errorf("casefold(%q) = %q, want %q", text, got, want) + } + } +} + +func TestCasefoldKeepsAValueThatIsNotText(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + + var number, kind string + var null *string + err := db.QueryRowContext(t.Context(), "SELECT casefold(42), typeof(casefold(42)), casefold(NULL)"). + Scan(&number, &kind, &null) + if err != nil { + t.Fatalf("casefold() on a number and NULL error = %v, want nil", err) + } + if number != "42" || kind != "integer" || null != nil { + t.Errorf("casefold(42) = %s of type %s and casefold(NULL) = %v, want 42 of type integer and NULL", + number, kind, null) + } +} diff --git a/internal/sqlite/helpers_test.go b/internal/sqlite/helpers_test.go new file mode 100644 index 00000000..db85a812 --- /dev/null +++ b/internal/sqlite/helpers_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite_test + +import ( + "context" + "database/sql" + "fmt" + "os" + "testing" + "time" + + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" + authkitsqlite "github.com/gopherium/gouncer/authkit/sqlite" + + "github.com/gopherium/gophenberg/internal/sqlite" +) + +// testMigrate is how every test run of Migrate waits for the migration lock. +var testMigrate = sqlite.MigrateOptions{LockWait: 5 * time.Second, LockPoll: 10 * time.Millisecond} + +// template is the database with the account and core tables that each test opens a copy of. +var template *sqlitetest.Template + +// migrateAccounts applies the account tables the core tables point at to db. +func migrateAccounts(ctx context.Context, db *sql.DB) error { + return authkitsqlite.Migrate(ctx, db, authkitsqlite.MigrateOptions{ + LockWait: testMigrate.LockWait, LockPoll: testMigrate.LockPoll, + }) +} + +// migrateAll applies the account tables, then the core tables, to db. +func migrateAll(ctx context.Context, db *sql.DB) error { + if err := migrateAccounts(ctx, db); err != nil { + return err + } + return sqlite.Migrate(ctx, db, testMigrate) +} + +// migrated returns a handle on a fresh copy of the migrated template, closed when t ends. +func migrated(t *testing.T) *sql.DB { + t.Helper() + return template.Open(t) +} + +// mustExec runs statement with args on db and fails t when it errs. +func mustExec(t *testing.T, db *sql.DB, statement string, args ...any) { + t.Helper() + if _, err := db.ExecContext(t.Context(), statement, args...); err != nil { + t.Fatalf("%s: %v", statement, err) + } +} + +// count returns the single integer lookup with args answers on db. +func count(t *testing.T, db *sql.DB, lookup string, args ...any) int64 { + t.Helper() + var n int64 + if err := db.QueryRowContext(t.Context(), lookup, args...).Scan(&n); err != nil { + t.Fatalf("%s: %v", lookup, err) + } + return n +} + +// TestMain builds the migrated template, runs the tests, then removes the template. +func TestMain(m *testing.M) { + built, err := sqlitetest.NewTemplate(context.Background(), testSettings().Options(), migrateAll) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, "building the migrated template:", err) + os.Exit(1) + } + template = built + code := m.Run() + _ = template.Close() + os.Exit(code) +} diff --git a/internal/sqlite/libc_test.go b/internal/sqlite/libc_test.go new file mode 100644 index 00000000..21b1ac07 --- /dev/null +++ b/internal/sqlite/libc_test.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite_test + +import ( + "testing" + + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" +) + +func TestTheBuildRunsTheLibcTheDriverWasMadeFor(t *testing.T) { + t.Parallel() + + sqlitetest.CheckLibc(t) +} diff --git a/internal/sqlite/migrate.go b/internal/sqlite/migrate.go new file mode 100644 index 00000000..7091f98d --- /dev/null +++ b/internal/sqlite/migrate.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "time" + + dbsqlite "github.com/gopherium/framework/dbkit/sqlite" + "github.com/gopherium/framework/gonsole" +) + +const ( + // versionTable is the goose version table of the core tables, beside the version tables of the other owners. + versionTable = "core_goose_db_version" + // accountsLookup reports whether the database holds the account table the core tables point at. + accountsLookup = "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'auth_users')" +) + +// migrationSource holds the core migration files at its root. +var migrationSource = mustSub(migrations, "migrations") + +// ErrNoAccounts reports that the core tables were to be migrated before the account tables they point at. +var ErrNoAccounts = errors.New("sqlite: the core tables need the account tables, so migrate the accounts first") + +// MigrateOptions sets how a run of Migrate waits for the migration lock of the database file. +type MigrateOptions struct { + LockWait time.Duration + LockPoll time.Duration +} + +// MigrateOptionsFrom reads the migration lock settings from env, each one left unset taking its documented default. +func MigrateOptionsFrom(env gonsole.Env) (MigrateOptions, error) { + wait, err := env.Duration("SQLITE_MIGRATE_LOCK_WAIT", 30*time.Second) + if err != nil { + return MigrateOptions{}, err + } + poll, err := env.Duration("SQLITE_MIGRATE_LOCK_POLL", 250*time.Millisecond) + if err != nil { + return MigrateOptions{}, err + } + return MigrateOptions{LockWait: wait, LockPoll: poll}, nil +} + +// Migrate applies the core table migrations to db under the migration lock of its file, after the account tables. +func Migrate(ctx context.Context, db *sql.DB, opts MigrateOptions) error { + if err := checkAccounts(ctx, db); err != nil { + return err + } + err := dbsqlite.Migrate(ctx, db, dbsqlite.Migrations{ + Table: versionTable, FS: migrationSource, LockWait: opts.LockWait, LockPoll: opts.LockPoll, + }) + if err != nil { + return fmt.Errorf("sqlite: apply migrations: %w", dbsqlite.Classify(err)) + } + return nil +} + +// checkAccounts returns ErrNoAccounts when db holds no account table for the core tables to point at. +func checkAccounts(ctx context.Context, db *sql.DB) error { + var found bool + if err := db.QueryRowContext(ctx, accountsLookup).Scan(&found); err != nil { + return fmt.Errorf("sqlite: look for the account tables: %w", err) + } + if !found { + return ErrNoAccounts + } + return nil +} + +// mustSub returns the subtree of fsys at dir, and panics when dir is not a valid path. +func mustSub(fsys fs.FS, dir string) fs.FS { + sub, err := fs.Sub(fsys, dir) + if err != nil { + panic(err) + } + return sub +} diff --git a/internal/sqlite/migrate_internal_test.go b/internal/sqlite/migrate_internal_test.go new file mode 100644 index 00000000..96c7911f --- /dev/null +++ b/internal/sqlite/migrate_internal_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "testing" + "time" + + dbsqlite "github.com/gopherium/framework/dbkit/sqlite" + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" + authkitsqlite "github.com/gopherium/gouncer/authkit/sqlite" + "github.com/pressly/goose/v3" +) + +func TestMustSubPanicsOnAFolderOutsideTheTree(t *testing.T) { + t.Parallel() + + defer func() { + if recover() == nil { + t.Fatal("mustSub() did not panic, want a panic") + } + }() + + mustSub(migrations, "..") +} + +func TestTheBaselineRollsBackOverAStoredPostAndAppliesAgain(t *testing.T) { + t.Parallel() + + settings := Settings{BusyTimeout: 5 * time.Second, CacheSize: 2048, MaxConns: 4, Synchronous: dbsqlite.SynchronousFull} + opts := MigrateOptions{LockWait: 5 * time.Second, LockPoll: 10 * time.Millisecond} + db := sqlitetest.Open(t, settings.Options()) + if err := authkitsqlite.Migrate(t.Context(), db, authkitsqlite.MigrateOptions(opts)); err != nil { + t.Fatalf("migrating the account tables: %v", err) + } + if err := Migrate(t.Context(), db, opts); err != nil { + t.Fatalf("Migrate() error = %v, want nil", err) + } + for _, statement := range []string{ + "INSERT INTO auth_users (id, email, name, password_hash, disabled, created_at) VALUES " + + "('019a0000-0000-7000-8000-000000000001', 'maria@example.com', 'Maria Perez', 'placeholder-hash', 0, 1)", + "INSERT INTO core_content (id, type, status, slug, author_id, created_at, updated_at, path) VALUES " + + "('019a0000-0000-7000-8000-0000000000c1', 'post', 'draft', 'hello', " + + "'019a0000-0000-7000-8000-000000000001', 1, 1, 'hello')", + } { + if _, err := db.ExecContext(t.Context(), statement); err != nil { + t.Fatalf("%s: %v", statement, err) + } + } + provider, err := goose.NewProvider(goose.DialectSQLite3, db, migrationSource, + goose.WithTableName(versionTable), goose.WithDisableGlobalRegistry(true)) + if err != nil { + t.Fatalf("building the migration runner: %v", err) + } + tables := `SELECT count(*) FILTER (WHERE name LIKE 'core\_%' ESCAPE '\' AND name <> ?), ` + + `count(*) FILTER (WHERE name = 'auth_users') FROM sqlite_schema WHERE type = 'table'` + + if _, err := provider.DownTo(t.Context(), 0); err != nil { + t.Fatalf("DownTo(0) over a stored post error = %v, want each table dropped after the tables that point at it", err) + } + + var core, accounts int + if err := db.QueryRowContext(t.Context(), tables, versionTable).Scan(&core, &accounts); err != nil { + t.Fatalf("counting the tables: %v", err) + } + if core != 0 || accounts != 1 { + t.Errorf("after the rollback %d core tables and %d account tables are left, want none and the one", core, accounts) + } + if err := Migrate(t.Context(), db, opts); err != nil { + t.Fatalf("Migrate() after the rollback error = %v, want a clean reapply", err) + } + if err := db.QueryRowContext(t.Context(), tables, versionTable).Scan(&core, &accounts); err != nil || core != 9 { + t.Errorf("after the reapply %d core tables are there with error %v, want all 9", core, err) + } +} diff --git a/internal/sqlite/migrate_test.go b/internal/sqlite/migrate_test.go new file mode 100644 index 00000000..a4bd6f44 --- /dev/null +++ b/internal/sqlite/migrate_test.go @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux || darwin + +package sqlite_test + +import ( + "database/sql" + "errors" + "os" + "strings" + "syscall" + "testing" + "time" + + "github.com/gopherium/framework/dbkit" + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" + + "github.com/gopherium/gophenberg/internal/sqlite" +) + +// schemaText returns every table and index definition db holds, in name order. +func schemaText(t *testing.T, db *sql.DB) string { + t.Helper() + var text string + lookup := "SELECT group_concat(sql, char(10)) FROM (SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL ORDER BY name)" + if err := db.QueryRowContext(t.Context(), lookup).Scan(&text); err != nil { + t.Fatalf("%s: %v", lookup, err) + } + return text +} + +// withAccounts returns a handle on a new file that holds only the account tables, closed when t ends. +func withAccounts(t *testing.T) *sql.DB { + t.Helper() + db := sqlitetest.Open(t, testSettings().Options()) + if err := migrateAccounts(t.Context(), db); err != nil { + t.Fatalf("migrating the account tables: %v", err) + } + return db +} + +// coreTables returns how many core tables db holds beside the core version table. +func coreTables(t *testing.T, db *sql.DB) int64 { + t.Helper() + return count(t, db, `SELECT count(*) FROM sqlite_schema WHERE type = 'table' `+ + `AND name LIKE 'core\_%' ESCAPE '\' AND name <> 'core_goose_db_version'`) +} + +// mainFile returns the path of the main database file of db. +func mainFile(t *testing.T, db *sql.DB) string { + t.Helper() + var path string + lookup := "SELECT file FROM pragma_database_list WHERE name = 'main'" + if err := db.QueryRowContext(t.Context(), lookup).Scan(&path); err != nil { + t.Fatalf("%s: %v", lookup, err) + } + return path +} + +// migrationLock holds the migration lock file beside the main file of db until t ends. +func migrationLock(t *testing.T, db *sql.DB) { + t.Helper() + lock, err := os.OpenFile(mainFile(t, db)+".migrate.lock", os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + t.Fatalf("opening the migration lock: %v", err) + } + t.Cleanup(func() { _ = lock.Close() }) + if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil { + t.Fatalf("holding the migration lock: %v", err) + } +} + +func TestMigrateOptionsFallBackToTheDocumentedDefaults(t *testing.T) { + t.Parallel() + + got, err := sqlite.MigrateOptionsFrom(environment(nil)) + if err != nil { + t.Fatalf("MigrateOptionsFrom() error = %v, want nil", err) + } + + want := sqlite.MigrateOptions{LockWait: 30 * time.Second, LockPoll: 250 * time.Millisecond} + if got != want { + t.Errorf("MigrateOptionsFrom() = %+v, want %+v", got, want) + } +} + +func TestMigrateOptionsReadEveryValueFromTheEnvironment(t *testing.T) { + t.Parallel() + + got, err := sqlite.MigrateOptionsFrom(environment(map[string]string{ + "GOPHENBERG_SQLITE_MIGRATE_LOCK_WAIT": "2m", + "GOPHENBERG_SQLITE_MIGRATE_LOCK_POLL": "40ms", + })) + if err != nil { + t.Fatalf("MigrateOptionsFrom() error = %v, want nil", err) + } + + want := sqlite.MigrateOptions{LockWait: 2 * time.Minute, LockPoll: 40 * time.Millisecond} + if got != want { + t.Errorf("MigrateOptionsFrom() = %+v, want %+v", got, want) + } +} + +func TestMigrateOptionsRefuseABadValueByItsName(t *testing.T) { + t.Parallel() + + for _, bad := range []struct{ name, value string }{ + {"GOPHENBERG_SQLITE_MIGRATE_LOCK_WAIT", "0s"}, + {"GOPHENBERG_SQLITE_MIGRATE_LOCK_WAIT", "-1s"}, + {"GOPHENBERG_SQLITE_MIGRATE_LOCK_WAIT", "soon"}, + {"GOPHENBERG_SQLITE_MIGRATE_LOCK_POLL", "0s"}, + {"GOPHENBERG_SQLITE_MIGRATE_LOCK_POLL", "-5ms"}, + {"GOPHENBERG_SQLITE_MIGRATE_LOCK_POLL", "often"}, + } { + _, err := sqlite.MigrateOptionsFrom(environment(map[string]string{bad.name: bad.value})) + if err == nil || !strings.Contains(err.Error(), bad.name) { + t.Errorf("MigrateOptionsFrom() with %s=%s error = %v, want one naming the setting", bad.name, bad.value, err) + } + } +} + +func TestMigrateRecordsTheBaselineInTheCoreVersionTable(t *testing.T) { + t.Parallel() + + db := migrated(t) + + core := count(t, db, "SELECT max(version_id) FROM core_goose_db_version") + auth := count(t, db, "SELECT max(version_id) FROM auth_goose_db_version") + if core != 26 || auth != 4 { + t.Errorf("the highest versions are core %d and auth %d, want the 00026 core baseline beside the 00004 auth one", + core, auth) + } +} + +func TestMigrateDoesNothingOnASecondRun(t *testing.T) { + t.Parallel() + + db := migrated(t) + schema := schemaText(t, db) + versions := count(t, db, "SELECT count(*) FROM core_goose_db_version") + + err := sqlite.Migrate(t.Context(), db, testMigrate) + + if err != nil { + t.Fatalf("a second Migrate() error = %v, want nil", err) + } + if got := count(t, db, "SELECT count(*) FROM core_goose_db_version"); got != versions { + t.Errorf("core_goose_db_version holds %d rows after a second run, want the %d it held", got, versions) + } + if got := count(t, db, "SELECT count(*) FROM core_content_types"); got != 1 { + t.Errorf("core_content_types holds %d rows after a second run, want the one seeded type", got) + } + if got := schemaText(t, db); got != schema { + t.Errorf("the schema after a second run = %s, want it unchanged from %s", got, schema) + } +} + +func TestMigrateRefusesAFileWithoutTheAccountTables(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + + err := sqlite.Migrate(t.Context(), db, testMigrate) + + if !errors.Is(err, sqlite.ErrNoAccounts) { + t.Errorf("Migrate() before the account tables error = %v, want ErrNoAccounts", err) + } + if got := coreTables(t, db); got != 0 { + t.Errorf("Migrate() before the account tables created %d core tables, want none", got) + } +} + +func TestMigrateReportsAHandleItCannotRead(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + if err := db.Close(); err != nil { + t.Fatalf("Close() error = %v, want nil", err) + } + + err := sqlite.Migrate(t.Context(), db, testMigrate) + + if err == nil || errors.Is(err, sqlite.ErrNoAccounts) || + !strings.HasPrefix(err.Error(), "sqlite: look for the account tables: ") { + t.Errorf("Migrate() on a closed handle error = %v, want the account lookup that failed named", err) + } +} + +func TestMigrateReportsAFileAnotherWriterHoldsAsBusy(t *testing.T) { + t.Parallel() + + db := withAccounts(t) + writer, err := db.Conn(t.Context()) + if err != nil { + t.Fatalf("Conn() error = %v, want nil", err) + } + t.Cleanup(func() { _ = writer.Close() }) + if _, err := writer.ExecContext(t.Context(), "BEGIN IMMEDIATE"); err != nil { + t.Fatalf("BEGIN IMMEDIATE error = %v, want nil", err) + } + impatient := testSettings() + impatient.BusyTimeout = time.Millisecond + other := open(t, "sqlite:"+mainFile(t, db), impatient) + + err = sqlite.Migrate(t.Context(), other, testMigrate) + + if !errors.Is(err, dbkit.ErrBusy) { + t.Errorf("Migrate() behind another writer error = %v, want dbkit.ErrBusy", err) + } +} + +func TestMigrateRefusesLockSettingsAtZero(t *testing.T) { + t.Parallel() + + db := withAccounts(t) + + err := sqlite.Migrate(t.Context(), db, sqlite.MigrateOptions{}) + + if err == nil || !strings.HasPrefix(err.Error(), "sqlite: apply migrations: ") { + t.Errorf("Migrate() with no lock settings error = %v, want it refused under the sqlite prefix", err) + } + if got := coreTables(t, db); got != 0 { + t.Errorf("Migrate() with no lock settings created %d core tables, want none", got) + } +} + +func TestMigrateGivesUpOnAMigrationLockHeldPastItsWait(t *testing.T) { + t.Parallel() + + db := withAccounts(t) + migrationLock(t, db) + + start := time.Now() + err := sqlite.Migrate(t.Context(), db, sqlite.MigrateOptions{ + LockWait: 100 * time.Millisecond, LockPoll: 10 * time.Millisecond, + }) + waited := time.Since(start) + + if err == nil || !strings.Contains(err.Error(), "migration lock") { + t.Errorf("Migrate() behind a held lock error = %v, want the migration lock named", err) + } + if got := coreTables(t, db); got != 0 { + t.Errorf("Migrate() behind a held lock created %d core tables, want none", got) + } + if waited < 100*time.Millisecond { + t.Errorf("Migrate() gave up after %v, want it to wait out the lock wait of 100ms", waited) + } +} diff --git a/internal/sqlite/migrations.go b/internal/sqlite/migrations.go new file mode 100644 index 00000000..121dec78 --- /dev/null +++ b/internal/sqlite/migrations.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import "embed" + +// migrations holds the core table migration files applied by goose. +// +//go:embed migrations/*.sql +var migrations embed.FS diff --git a/internal/sqlite/migrations/00026_create_core_tables.sql b/internal/sqlite/migrations/00026_create_core_tables.sql new file mode 100644 index 00000000..13299185 --- /dev/null +++ b/internal/sqlite/migrations/00026_create_core_tables.sql @@ -0,0 +1,207 @@ +-- SPDX-License-Identifier: Apache-2.0 + +-- +goose Up +CREATE TABLE core_content_types ( + key TEXT NOT NULL PRIMARY KEY, + singular_label TEXT NOT NULL, + plural_label TEXT NOT NULL, + route_word TEXT NOT NULL, + hierarchical BOOLEAN NOT NULL DEFAULT 0 CHECK (hierarchical IN (0, 1)), + revisions BOOLEAN NOT NULL DEFAULT 1 CHECK (revisions IN (0, 1)), + revision_cap INTEGER NOT NULL DEFAULT 0, + page_kind TEXT NOT NULL CONSTRAINT content_types_page_kind_check CHECK (page_kind IN ('single', 'archive')), + is_default BOOLEAN NOT NULL DEFAULT 0 CHECK (is_default IN (0, 1)), + active BOOLEAN NOT NULL DEFAULT 1 CHECK (active IN (0, 1)), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + origin TEXT, + description TEXT NOT NULL DEFAULT '', + CONSTRAINT content_types_root_is_the_default CHECK (route_word <> '' OR is_default), + CONSTRAINT content_types_default_stays_active CHECK (active OR NOT is_default) +); + +CREATE UNIQUE INDEX core_content_types_one_default_idx ON core_content_types (is_default) + WHERE is_default; + +CREATE UNIQUE INDEX core_content_types_route_word_idx ON core_content_types (route_word) + WHERE route_word <> ''; + +INSERT INTO core_content_types ( + key, singular_label, plural_label, route_word, hierarchical, revisions, + revision_cap, page_kind, is_default, active, created_at, updated_at, description +) +VALUES ( + 'post', 'Post', 'Posts', '', 0, 1, + 100, 'single', 1, 1, + CAST(round(unixepoch('subsec') * 1000) AS INTEGER) * 1000, + CAST(round(unixepoch('subsec') * 1000) AS INTEGER) * 1000, + 'Manage the posts on this site.' +); + +CREATE TABLE core_content ( + id TEXT NOT NULL PRIMARY KEY, + type TEXT NOT NULL CONSTRAINT content_type_fkey REFERENCES core_content_types (key) ON DELETE RESTRICT, + status TEXT NOT NULL CONSTRAINT content_status_check CHECK ( + status IN ('draft', 'pending', 'private', 'scheduled', 'published', 'trash') + ), + slug TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', + excerpt TEXT NOT NULL DEFAULT '', + author_id TEXT NOT NULL CONSTRAINT content_author_id_fkey REFERENCES auth_users (id) ON DELETE RESTRICT, + published_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + parent_id TEXT CONSTRAINT content_parent_id_fkey REFERENCES core_content (id) ON DELETE RESTRICT, + path TEXT NOT NULL, + fields TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(fields)), + CONSTRAINT content_published_needs_date CHECK ( + status NOT IN ('scheduled', 'published') OR published_at IS NOT NULL + ), + CONSTRAINT content_path_is_addressable CHECK (path <> '') +); + +CREATE UNIQUE INDEX core_content_path_unique ON core_content (path); + +CREATE INDEX core_content_type_status_published_at_idx ON core_content (type, status, published_at DESC); + +CREATE INDEX core_content_author_id_idx ON core_content (author_id); + +CREATE INDEX core_content_parent_id_idx ON core_content (parent_id); + +CREATE TABLE core_content_revisions ( + id TEXT NOT NULL PRIMARY KEY, + content_id TEXT NOT NULL CONSTRAINT content_revisions_content_id_fkey + REFERENCES core_content (id) ON DELETE CASCADE, + kind TEXT NOT NULL CONSTRAINT content_revisions_kind_check CHECK (kind IN ('revision', 'autosave')), + author_id TEXT NOT NULL CONSTRAINT content_revisions_author_id_fkey + REFERENCES auth_users (id) ON DELETE RESTRICT, + title TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', + excerpt TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + fields TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(fields)) +); + +CREATE INDEX core_content_revisions_content_id_created_at_idx + ON core_content_revisions (content_id, created_at DESC); + +CREATE INDEX core_content_revisions_author_id_idx ON core_content_revisions (author_id); + +CREATE UNIQUE INDEX core_content_revisions_one_autosave_per_author_idx + ON core_content_revisions (content_id, author_id) WHERE kind = 'autosave'; + +CREATE TABLE core_settings ( + key TEXT NOT NULL PRIMARY KEY, + value TEXT NOT NULL +); + +CREATE TABLE core_media ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + media_type TEXT NOT NULL CONSTRAINT media_media_type_check CHECK (media_type IN ('image', 'file')), + file TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + alt_text TEXT NOT NULL DEFAULT '', + caption TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + mime_type TEXT NOT NULL, + width INTEGER NOT NULL DEFAULT 0, + height INTEGER NOT NULL DEFAULT 0, + filesize INTEGER NOT NULL DEFAULT 0, + sizes TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(sizes)), + author_id TEXT NOT NULL CONSTRAINT media_author_id_fkey REFERENCES auth_users (id) ON DELETE RESTRICT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +); + +CREATE INDEX core_media_created_at_idx ON core_media (created_at DESC, id DESC); + +CREATE INDEX core_media_media_type_idx ON core_media (media_type); + +CREATE INDEX core_media_author_id_idx ON core_media (author_id); + +CREATE TABLE core_user_settings ( + user_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + CONSTRAINT user_settings_pkey PRIMARY KEY (user_id, key) +); + +CREATE TABLE core_field_groups ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + location TEXT NOT NULL DEFAULT '[]', + position INTEGER NOT NULL DEFAULT 0, + active BOOLEAN NOT NULL DEFAULT 1 CHECK (active IN (0, 1)), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + origin TEXT, + key TEXT NOT NULL, + CONSTRAINT field_groups_location_is_array CHECK (json_valid(location) AND json_type(location) = 'array') +); + +CREATE UNIQUE INDEX core_field_groups_key_unique ON core_field_groups (key); + +CREATE TABLE core_content_fields ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + key TEXT NOT NULL, + label TEXT NOT NULL, + kind TEXT NOT NULL, + relates_to TEXT CONSTRAINT content_fields_relates_to_fkey REFERENCES core_content_types (key), + many BOOLEAN NOT NULL DEFAULT 0 CHECK (many IN (0, 1)), + required BOOLEAN NOT NULL DEFAULT 0 CHECK (required IN (0, 1)), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + position INTEGER NOT NULL DEFAULT 0, + group_id INTEGER NOT NULL CONSTRAINT content_fields_group_fkey + REFERENCES core_field_groups (id) ON DELETE RESTRICT, + settings TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(settings)), + parent_field_id INTEGER CONSTRAINT content_fields_parent_fkey + REFERENCES core_content_fields (id) ON DELETE CASCADE, + depth INTEGER NOT NULL DEFAULT 0, + origin TEXT, + CONSTRAINT content_fields_relation_target CHECK ((kind = 'relation') = (relates_to IS NOT NULL)), + CONSTRAINT content_fields_many_relational CHECK (many = 0 OR kind IN ('relation', 'media')) +); + +CREATE UNIQUE INDEX core_content_fields_scope_key_unique + ON core_content_fields (group_id, coalesce(parent_field_id, 0), key); + +CREATE UNIQUE INDEX core_content_fields_container_key_unique + ON core_content_fields (parent_field_id, key) WHERE parent_field_id IS NOT NULL; + +CREATE TABLE core_content_relations ( + from_id TEXT NOT NULL CONSTRAINT content_relations_from_id_fkey + REFERENCES core_content (id) ON DELETE CASCADE, + field_id INTEGER NOT NULL CONSTRAINT content_relations_field_id_fkey + REFERENCES core_content_fields (id) ON DELETE CASCADE, + to_id TEXT NOT NULL CONSTRAINT content_relations_to_id_fkey + REFERENCES core_content (id) ON DELETE CASCADE, + position INTEGER NOT NULL DEFAULT 1, + sort_at INTEGER NOT NULL, + visible BOOLEAN NOT NULL DEFAULT 0 CHECK (visible IN (0, 1)), + CONSTRAINT content_relations_pkey PRIMARY KEY (from_id, field_id, to_id) +); + +CREATE INDEX core_content_relations_target_idx + ON core_content_relations (to_id, sort_at DESC, from_id, field_id) + WHERE visible; + +-- +goose Down +DROP TABLE core_content_relations; + +DROP TABLE core_content_fields; + +DROP TABLE core_field_groups; + +DROP TABLE core_user_settings; + +DROP TABLE core_media; + +DROP TABLE core_settings; + +DROP TABLE core_content_revisions; + +DROP TABLE core_content; + +DROP TABLE core_content_types; diff --git a/internal/sqlite/open.go b/internal/sqlite/open.go new file mode 100644 index 00000000..888c59f6 --- /dev/null +++ b/internal/sqlite/open.go @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package sqlite provides the SQLite data layer for the CMS core. +package sqlite + +import ( + "database/sql" + "errors" + "fmt" + "io/fs" + "math" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/gopherium/framework/dbkit" + dbsqlite "github.com/gopherium/framework/dbkit/sqlite" + "github.com/gopherium/framework/gonsole" +) + +// maxBusyTimeout is the longest busy timeout SQLite keeps. +const maxBusyTimeout = math.MaxInt32 * time.Millisecond + +// ErrNoFile reports that the database file a sqlite: address names does not exist. +var ErrNoFile = errors.New("sqlite: the database file does not exist") + +// Settings are the values every connection of the site's SQLite handle runs with. +type Settings struct { + BusyTimeout time.Duration + CacheSize int + MaxConns int + Synchronous dbsqlite.Synchronous + JournalSizeLimit *int64 +} + +// SettingsFrom reads the SQLite settings from env, each one left unset taking its documented default. +func SettingsFrom(env gonsole.Env) (Settings, error) { + maxConns, err := connectionCap(env) + if err != nil { + return Settings{}, err + } + busyTimeout, err := env.Duration("SQLITE_BUSY_TIMEOUT", 5*time.Second, + gonsole.WholeMilliseconds(), gonsole.AtMost(int64(maxBusyTimeout))) + if err != nil { + return Settings{}, err + } + cacheSize, err := env.Count("SQLITE_CACHE_SIZE", 2000, gonsole.AtMost(math.MaxInt32)) + if err != nil { + return Settings{}, err + } + synchronous, err := gonsole.Parse(env, "SQLITE_SYNCHRONOUS", dbsqlite.SynchronousFull, parseSynchronous) + if err != nil { + return Settings{}, err + } + journalSizeLimit, err := gonsole.Parse[*int64](env, "SQLITE_JOURNAL_SIZE_LIMIT", nil, parseJournalSizeLimit) + if err != nil { + return Settings{}, err + } + return Settings{ + BusyTimeout: busyTimeout, CacheSize: cacheSize, MaxConns: maxConns, + Synchronous: synchronous, JournalSizeLimit: journalSizeLimit, + }, nil +} + +// connectionCap reads the DATABASE_MAX_CONNS setting, defaultMaxConns of the CPU count when unset, and refuses 1. +func connectionCap(env gonsole.Env) (int, error) { + conns, err := env.Count("DATABASE_MAX_CONNS", defaultMaxConns(runtime.NumCPU())) + if err != nil { + return 0, err + } + if conns < 2 { + return 0, fmt.Errorf("%s: must be 2 or more on SQLite, got %q", env.Key("DATABASE_MAX_CONNS"), strconv.Itoa(conns)) + } + return conns, nil +} + +// defaultMaxConns returns the connection cap a machine with cpus processors gets, the larger of 4 and cpus. +func defaultMaxConns(cpus int) int { + return max(4, cpus) +} + +// parseSynchronous reads full or normal, in any case, as the Synchronous it names. +func parseSynchronous(value string) (dbsqlite.Synchronous, error) { + switch strings.ToLower(value) { + case "full": + return dbsqlite.SynchronousFull, nil + case "normal": + return dbsqlite.SynchronousNormal, nil + } + return 0, fmt.Errorf("must be full or normal, got %q", value) +} + +// parseJournalSizeLimit reads a whole number of bytes above zero. +func parseJournalSizeLimit(value string) (*int64, error) { + limit, err := strconv.ParseInt(value, 10, 64) + if err != nil || limit < 1 { + return nil, fmt.Errorf("must be a whole number of bytes above zero, got %q", value) + } + return &limit, nil +} + +// Options returns the dbkit options of s with every Go function of the site registered. +func (s Settings) Options() dbsqlite.Options { + return dbsqlite.Options{ + BusyTimeout: s.BusyTimeout, CacheSize: s.CacheSize, MaxConns: s.MaxConns, + Synchronous: s.Synchronous, JournalSizeLimit: s.JournalSizeLimit, Functions: functions, + } +} + +// Open returns a handle on the site's SQLite file at address under s with every rule on, and connects to nothing. +func Open(address string, s Settings) (*sql.DB, error) { + if err := checkExists(address); err != nil { + return nil, err + } + return dbsqlite.Open(address, s.Options()) +} + +// checkExists returns ErrNoFile when the absolute path of address names no file, and leaves every other check to dbkit. +func checkExists(address string) error { + path, err := dbkit.SQLitePath(address) + if err != nil || !filepath.IsAbs(path) { + return nil + } + if _, err := os.Lstat(path); errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("%w: %s, create an empty file at that path to start a new site", ErrNoFile, path) + } + return nil +} diff --git a/internal/sqlite/open_internal_test.go b/internal/sqlite/open_internal_test.go new file mode 100644 index 00000000..a6037aca --- /dev/null +++ b/internal/sqlite/open_internal_test.go @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import "testing" + +func TestDefaultMaxConnsKeepsAFloorOfFour(t *testing.T) { + t.Parallel() + + for cpus, want := range map[int]int{1: 4, 4: 4, 16: 16} { + if got := defaultMaxConns(cpus); got != want { + t.Errorf("defaultMaxConns(%d) = %d, want %d", cpus, got, want) + } + } +} diff --git a/internal/sqlite/open_test.go b/internal/sqlite/open_test.go new file mode 100644 index 00000000..f4f9adda --- /dev/null +++ b/internal/sqlite/open_test.go @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite_test + +import ( + "database/sql" + "errors" + "io/fs" + "math" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/gopherium/framework/dbkit" + dbsqlite "github.com/gopherium/framework/dbkit/sqlite" + "github.com/gopherium/framework/gonsole" + "github.com/gopherium/framework/gonsole/testkit" + + "github.com/gopherium/gophenberg/internal/sqlite" +) + +// testSettings returns the settings a test handle opens with, four connections and a five second busy timeout. +func testSettings() sqlite.Settings { + return sqlite.Settings{ + BusyTimeout: 5 * time.Second, CacheSize: 2048, MaxConns: 4, Synchronous: dbsqlite.SynchronousFull, + } +} + +// environment returns the Gophenberg settings reader over values, each key unset when values lacks it. +func environment(values map[string]string) gonsole.Env { + return gonsole.Env{Prefix: "GOPHENBERG_", Getenv: testkit.Getenv(values)} +} + +// emptyFile creates an empty database file in a fresh folder of t and returns its address and path. +func emptyFile(t *testing.T) (address, path string) { + t.Helper() + path = filepath.Join(t.TempDir(), "site.db") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("creating the empty database file: %v", err) + } + return "sqlite:" + path, path +} + +// open returns the handle Open gives on address under settings, closed when t ends. +func open(t *testing.T, address string, settings sqlite.Settings) *sql.DB { + t.Helper() + db, err := sqlite.Open(address, settings) + if err != nil { + t.Fatalf("Open() error = %v, want nil", err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Errorf("Close() error = %v, want nil", err) + } + }) + return db +} + +// fullPool takes every connection of db up to size, each opened after the ones before, closed when t ends. +func fullPool(t *testing.T, db *sql.DB, size int) []*sql.Conn { + t.Helper() + conns := make([]*sql.Conn, size) + for i := range conns { + conn, err := db.Conn(t.Context()) + if err != nil { + t.Fatalf("Conn() error = %v, want nil", err) + } + t.Cleanup(func() { _ = conn.Close() }) + conns[i] = conn + } + if got := db.Stats().OpenConnections; got != size { + t.Fatalf("Stats().OpenConnections = %d, want %d fresh connections", got, size) + } + return conns +} + +func TestSettingsFallBackToTheDocumentedDefaults(t *testing.T) { + t.Parallel() + + got, err := sqlite.SettingsFrom(environment(nil)) + if err != nil { + t.Fatalf("SettingsFrom() error = %v, want nil", err) + } + + want := sqlite.Settings{ + BusyTimeout: 5 * time.Second, CacheSize: 2000, MaxConns: max(4, runtime.NumCPU()), + Synchronous: dbsqlite.SynchronousFull, + } + if got != want { + t.Errorf("SettingsFrom() = %+v, want %+v", got, want) + } +} + +func TestSettingsReadEveryValueFromTheEnvironment(t *testing.T) { + t.Parallel() + + got, err := sqlite.SettingsFrom(environment(map[string]string{ + "GOPHENBERG_DATABASE_MAX_CONNS": "3", + "GOPHENBERG_SQLITE_BUSY_TIMEOUT": "1500ms", + "GOPHENBERG_SQLITE_CACHE_SIZE": "4096", + "GOPHENBERG_SQLITE_SYNCHRONOUS": "normal", + "GOPHENBERG_SQLITE_JOURNAL_SIZE_LIMIT": "3145728", + })) + if err != nil { + t.Fatalf("SettingsFrom() error = %v, want nil", err) + } + + if got.JournalSizeLimit == nil || *got.JournalSizeLimit != 3145728 { + t.Fatalf("SettingsFrom().JournalSizeLimit = %v, want 3145728", got.JournalSizeLimit) + } + got.JournalSizeLimit = nil + want := sqlite.Settings{ + BusyTimeout: 1500 * time.Millisecond, CacheSize: 4096, MaxConns: 3, Synchronous: dbsqlite.SynchronousNormal, + } + if got != want { + t.Errorf("SettingsFrom() = %+v, want %+v", got, want) + } +} + +func TestSettingsTakeTheEdgeValues(t *testing.T) { + t.Parallel() + + got, err := sqlite.SettingsFrom(environment(map[string]string{ + "GOPHENBERG_DATABASE_MAX_CONNS": "2", + "GOPHENBERG_SQLITE_BUSY_TIMEOUT": "596h31m23.647s", + "GOPHENBERG_SQLITE_CACHE_SIZE": "2147483647", + "GOPHENBERG_SQLITE_JOURNAL_SIZE_LIMIT": "1", + })) + if err != nil { + t.Fatalf("SettingsFrom() error = %v, want nil", err) + } + + if got.JournalSizeLimit == nil || *got.JournalSizeLimit != 1 { + t.Fatalf("SettingsFrom().JournalSizeLimit = %v, want 1", got.JournalSizeLimit) + } + got.JournalSizeLimit = nil + want := sqlite.Settings{ + BusyTimeout: math.MaxInt32 * time.Millisecond, CacheSize: math.MaxInt32, MaxConns: 2, + Synchronous: dbsqlite.SynchronousFull, + } + if got != want { + t.Errorf("SettingsFrom() = %+v, want %+v", got, want) + } +} + +func TestSettingsQuoteTheConnectionCapTheyRefuse(t *testing.T) { + t.Parallel() + + _, err := sqlite.SettingsFrom(environment(map[string]string{"GOPHENBERG_DATABASE_MAX_CONNS": "1"})) + + if err == nil || !strings.HasSuffix(err.Error(), `got "1"`) { + t.Errorf("SettingsFrom() with one connection error = %v, want the value quoted as the other settings quote it", err) + } +} + +func TestSettingsReadSynchronousInAnyCase(t *testing.T) { + t.Parallel() + + for value, want := range map[string]dbsqlite.Synchronous{ + "FULL": dbsqlite.SynchronousFull, "Normal": dbsqlite.SynchronousNormal, + } { + got, err := sqlite.SettingsFrom(environment(map[string]string{"GOPHENBERG_SQLITE_SYNCHRONOUS": value})) + if err != nil { + t.Fatalf("SettingsFrom() with synchronous %q error = %v, want nil", value, err) + } + if got.Synchronous != want { + t.Errorf("SettingsFrom() with synchronous %q = %d, want %d", value, got.Synchronous, want) + } + } +} + +func TestSettingsRefuseABadValueByItsName(t *testing.T) { + t.Parallel() + + for _, bad := range []struct{ name, value string }{ + {"GOPHENBERG_DATABASE_MAX_CONNS", "1"}, + {"GOPHENBERG_DATABASE_MAX_CONNS", "many"}, + {"GOPHENBERG_SQLITE_BUSY_TIMEOUT", "0s"}, + {"GOPHENBERG_SQLITE_BUSY_TIMEOUT", "1500us"}, + {"GOPHENBERG_SQLITE_BUSY_TIMEOUT", "596h31m23.648s"}, + {"GOPHENBERG_SQLITE_CACHE_SIZE", "0"}, + {"GOPHENBERG_SQLITE_CACHE_SIZE", "2147483648"}, + {"GOPHENBERG_SQLITE_SYNCHRONOUS", "off"}, + {"GOPHENBERG_SQLITE_JOURNAL_SIZE_LIMIT", "0"}, + {"GOPHENBERG_SQLITE_JOURNAL_SIZE_LIMIT", "lots"}, + {"GOPHENBERG_SQLITE_JOURNAL_SIZE_LIMIT", "9223372036854775808"}, + } { + _, err := sqlite.SettingsFrom(environment(map[string]string{bad.name: bad.value})) + if err == nil || !strings.Contains(err.Error(), bad.name) { + t.Errorf("SettingsFrom() with %s=%s error = %v, want one naming the setting", bad.name, bad.value, err) + } + } +} + +func TestOpenRunsEveryFreshConnectionUnderTheSettingsAndTheRules(t *testing.T) { + t.Parallel() + + address, _ := emptyFile(t) + limit := int64(3145728) + settings := sqlite.Settings{ + BusyTimeout: 1234 * time.Millisecond, CacheSize: 3000, MaxConns: 3, + Synchronous: dbsqlite.SynchronousNormal, JournalSizeLimit: &limit, + } + + db := open(t, address, settings) + + conns := fullPool(t, db, settings.MaxConns) + for pragma, want := range map[string]string{ + "busy_timeout": "1234", "cache_size": "-3000", "synchronous": "1", "journal_size_limit": "3145728", + "foreign_keys": "1", "journal_mode": "wal", "temp_store": "2", + } { + for i, conn := range conns { + var got string + if err := conn.QueryRowContext(t.Context(), "PRAGMA "+pragma).Scan(&got); err != nil { + t.Fatalf("PRAGMA %s on connection %d error = %v, want nil", pragma, i, err) + } + if got != want { + t.Errorf("PRAGMA %s on connection %d = %s, want %s", pragma, i, got, want) + } + } + } + if got := db.Stats().MaxOpenConnections; got != settings.MaxConns { + t.Errorf("Stats().MaxOpenConnections = %d, want %d", got, settings.MaxConns) + } +} + +func TestOpenRunsFullSynchronousByDefault(t *testing.T) { + t.Parallel() + + address, _ := emptyFile(t) + + db := open(t, address, testSettings()) + + var got string + if err := db.QueryRowContext(t.Context(), "PRAGMA synchronous").Scan(&got); err != nil { + t.Fatalf("PRAGMA synchronous error = %v, want nil", err) + } + if got != "2" { + t.Errorf("PRAGMA synchronous = %s, want 2 for full", got) + } +} + +func TestOpenConnectsToNothingBeforeTheFirstQuery(t *testing.T) { + t.Parallel() + + address, path := emptyFile(t) + + open(t, address, testSettings()) + + if _, err := os.Lstat(path + "-wal"); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("Lstat(%s-wal) error = %v, want no write-ahead log before the first query", path, err) + } +} + +func TestOpenRefusesAMissingFileAndSaysToCreateIt(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "site.db") + + _, err := sqlite.Open("sqlite:"+path, testSettings()) + + if !errors.Is(err, sqlite.ErrNoFile) { + t.Fatalf("Open() error = %v, want ErrNoFile", err) + } + if !strings.Contains(err.Error(), path) || !strings.Contains(err.Error(), "create an empty file") { + t.Errorf("Open() error = %q, want the path and how to start a new site", err) + } + if _, statErr := os.Lstat(path); !errors.Is(statErr, fs.ErrNotExist) { + t.Errorf("Lstat() error = %v, want the missing file left uncreated", statErr) + } +} + +func TestOpenLeavesALinkToAMissingFileToDbkit(t *testing.T) { + t.Parallel() + + folder := t.TempDir() + link := filepath.Join(folder, "site.db") + if err := os.Symlink(filepath.Join(folder, "gone.db"), link); err != nil { + t.Fatalf("Symlink() error = %v, want nil", err) + } + + _, err := sqlite.Open("sqlite:"+link, testSettings()) + + if err == nil || errors.Is(err, sqlite.ErrNoFile) { + t.Errorf("Open() on a link to a missing file error = %v, want the link refused by dbkit", err) + } +} + +func TestOpenLeavesAPathUnderAPlainFileToDbkit(t *testing.T) { + t.Parallel() + + plain := filepath.Join(t.TempDir(), "plain") + if err := os.WriteFile(plain, nil, 0o600); err != nil { + t.Fatalf("WriteFile() error = %v, want nil", err) + } + + _, err := sqlite.Open("sqlite:"+filepath.Join(plain, "site.db"), testSettings()) + + if err == nil || errors.Is(err, sqlite.ErrNoFile) { + t.Errorf("Open() on a path under a plain file error = %v, want the path refused by dbkit", err) + } +} + +func TestOpenRefusesAnAddressThatNamesNoSQLiteFile(t *testing.T) { + t.Parallel() + + _, err := sqlite.Open("postgres://localhost/site", testSettings()) + + if !errors.Is(err, dbkit.ErrAddress) { + t.Errorf("Open() error = %v, want dbkit.ErrAddress", err) + } +} + +func TestOpenRefusesARelativePath(t *testing.T) { + t.Parallel() + + _, err := sqlite.Open("sqlite:site.db", testSettings()) + + if err == nil || errors.Is(err, sqlite.ErrNoFile) { + t.Errorf("Open() error = %v, want the relative path refused", err) + } +} diff --git a/internal/sqlite/schema_test.go b/internal/sqlite/schema_test.go new file mode 100644 index 00000000..30e94058 --- /dev/null +++ b/internal/sqlite/schema_test.go @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite_test + +import ( + "database/sql" + "errors" + "maps" + "slices" + "strings" + "testing" + "time" + + "github.com/gopherium/framework/dbkit" + dbsqlite "github.com/gopherium/framework/dbkit/sqlite" +) + +const ( + // accountID is the id of the account the schema tests insert. + accountID = "019a0000-0000-7000-8000-000000000001" + // otherAccountID is the id of a second account the schema tests insert. + otherAccountID = "019a0000-0000-7000-8000-000000000002" + // contentID is the id of the content row the schema tests insert. + contentID = "019a0000-0000-7000-8000-0000000000c1" + // insertAccount stores an enabled account row with every column that has no default. + insertAccount = "INSERT INTO auth_users (id, email, name, password_hash, disabled, created_at) " + + "VALUES (?, ? || '@example.com', 'Maria Perez', 'placeholder-hash', 0, 1767225600000000)" + // insertGroup stores a field group under a key and returns its id. + insertGroup = "INSERT INTO core_field_groups (title, key, created_at, updated_at) " + + "VALUES ('Post fields', ?, 1767225600000000, 1767225600000000) RETURNING id" + // insertField stores a top-level text field of the first group under a key and returns its id. + insertField = "INSERT INTO core_content_fields (key, label, kind, group_id, created_at, updated_at) " + + "VALUES (?, 'Subtitle', 'text', 1, 1767225600000000, 1767225600000000) RETURNING id" + // insertMedia stores an image of the first account under a file name and returns its id. + insertMedia = "INSERT INTO core_media (media_type, file, mime_type, author_id, created_at, updated_at) " + + "VALUES ('image', ?, 'image/png', '" + accountID + "', 1767225600000000, 1767225600000000) RETURNING id" + // insertContent stores a draft post with an id, an author and a path. + insertContent = "INSERT INTO core_content (id, type, status, slug, author_id, created_at, updated_at, path) " + + "VALUES (?, 'post', 'draft', ?, ?, 1767225600000000, 1767225600000000, ?)" + // insertRevision stores a revision of the first content row by an author. + insertRevision = "INSERT INTO core_content_revisions (id, content_id, kind, author_id, created_at) " + + "VALUES ('019a0000-0000-7000-8000-0000000000e1', '" + contentID + "', 'revision', ?, 1767225600000000)" + // insertType stores a content type with a route word, a default flag and an active flag. + insertType = "INSERT INTO core_content_types (key, singular_label, plural_label, route_word, page_kind, " + + "is_default, active, created_at, updated_at) " + + "VALUES ('page', 'Page', 'Pages', ?, 'single', ?, ?, 1767225600000000, 1767225600000000)" +) + +// seededType is the row of the type the baseline seeds, without its stamps. +type seededType struct { + key, singular, plural, route, pageKind, description string + hierarchical, revisions, isDefault, active bool + revisionCap int64 + origin *string +} + +// classed returns the error statement with args answers on db, classed by dbkit. +func classed(t *testing.T, db *sql.DB, statement string, args ...any) error { + t.Helper() + _, err := db.ExecContext(t.Context(), statement, args...) + return dbsqlite.Classify(err) +} + +// inserted runs the insert statement with args on db and returns the id it answers. +func inserted(t *testing.T, db *sql.DB, statement string, args ...any) int64 { + t.Helper() + return count(t, db, statement, args...) +} + +// withRows returns a migrated copy holding the first account, a field group, a field and a draft post. +func withRows(t *testing.T) *sql.DB { + t.Helper() + db := migrated(t) + mustExec(t, db, insertAccount, accountID, "maria") + inserted(t, db, insertGroup, "post-fields") + inserted(t, db, insertField, "subtitle") + mustExec(t, db, insertContent, contentID, "hello", accountID, "hello") + return db +} + +// names returns the first column of every row lookup answers on db, each keyed to the text of its second column. +func names(t *testing.T, db *sql.DB, lookup string) map[string]string { + t.Helper() + rows, err := db.QueryContext(t.Context(), lookup) + if err != nil { + t.Fatalf("%s: %v", lookup, err) + } + defer func() { _ = rows.Close() }() + found := map[string]string{} + for rows.Next() { + var name, definition string + if err := rows.Scan(&name, &definition); err != nil { + t.Fatalf("%s: %v", lookup, err) + } + found[name] = definition + } + if err := rows.Err(); err != nil { + t.Fatalf("%s: %v", lookup, err) + } + return found +} + +func TestSchemaNamesEveryIndexWithTheCorePrefix(t *testing.T) { + t.Parallel() + + db := migrated(t) + + indexes := names(t, db, `SELECT name, tbl_name FROM sqlite_schema WHERE type = 'index' AND sql IS NOT NULL `+ + `AND tbl_name LIKE 'core\_%' ESCAPE '\'`) + + want := []string{ + "core_content_author_id_idx", "core_content_fields_container_key_unique", + "core_content_fields_scope_key_unique", "core_content_parent_id_idx", "core_content_path_unique", + "core_content_relations_target_idx", "core_content_revisions_author_id_idx", + "core_content_revisions_content_id_created_at_idx", "core_content_revisions_one_autosave_per_author_idx", + "core_content_type_status_published_at_idx", "core_content_types_one_default_idx", + "core_content_types_route_word_idx", "core_field_groups_key_unique", "core_media_author_id_idx", + "core_media_created_at_idx", "core_media_media_type_idx", + } + if got := slices.Sorted(maps.Keys(indexes)); !slices.Equal(got, want) { + t.Errorf("the named indexes of the core tables = %v, want the %d of %v", got, len(want), want) + } +} + +func TestSchemaKeepsTheFivePartialIndexesWithTheirPredicates(t *testing.T) { + t.Parallel() + + db := migrated(t) + + partial := names(t, db, `SELECT i.name, s.sql FROM sqlite_schema AS t JOIN pragma_index_list(t.name) AS i `+ + `JOIN sqlite_schema AS s ON s.name = i.name `+ + `WHERE t.type = 'table' AND t.name LIKE 'core\_%' ESCAPE '\' AND i.partial = 1`) + + want := map[string]string{ + "core_content_revisions_one_autosave_per_author_idx": "WHERE kind = 'autosave'", + "core_content_types_one_default_idx": "WHERE is_default", + "core_content_types_route_word_idx": "WHERE route_word <> ''", + "core_content_relations_target_idx": "WHERE visible", + "core_content_fields_container_key_unique": "WHERE parent_field_id IS NOT NULL", + } + got, wantNames := slices.Sorted(maps.Keys(partial)), slices.Sorted(maps.Keys(want)) + if !slices.Equal(got, wantNames) { + t.Fatalf("the partial indexes = %v, want %v", got, wantNames) + } + for name, predicate := range want { + if !strings.HasSuffix(partial[name], predicate) { + t.Errorf("index %s = %s, want it to end with %s", name, partial[name], predicate) + } + } +} + +func TestSchemaNamesEachCheckAsPostgreSQLDoes(t *testing.T) { + t.Parallel() + + db := withRows(t) + + for name, broken := range map[string]struct { + statement string + args []any + }{ + "content_status_check": { + "INSERT INTO core_content (id, type, status, slug, author_id, created_at, updated_at, path) " + + "VALUES ('019a0000-0000-7000-8000-0000000000c2', 'post', 'archived', 'b', ?, 1, 1, 'b')", + []any{accountID}, + }, + "content_published_needs_date": {"UPDATE core_content SET status = 'published'", nil}, + "content_path_is_addressable": {"UPDATE core_content SET path = ''", nil}, + "content_revisions_kind_check": { + "INSERT INTO core_content_revisions (id, content_id, kind, author_id, created_at) " + + "VALUES ('019a0000-0000-7000-8000-0000000000e2', ?, 'draft', ?, 1)", + []any{contentID, accountID}, + }, + "content_types_page_kind_check": {"UPDATE core_content_types SET page_kind = 'list'", nil}, + "content_types_root_is_the_default": {insertType, []any{"", 0, 1}}, + "content_types_default_stays_active": {"UPDATE core_content_types SET active = 0", nil}, + "media_media_type_check": { + "INSERT INTO core_media (media_type, file, mime_type, author_id, created_at, updated_at) " + + "VALUES ('video', 'clip.mp4', 'video/mp4', ?, 1, 1)", + []any{accountID}, + }, + "content_fields_relation_target": {"UPDATE core_content_fields SET kind = 'relation'", nil}, + "content_fields_many_relational": {"UPDATE core_content_fields SET many = 1", nil}, + "field_groups_location_is_array": {"UPDATE core_field_groups SET location = '{}'", nil}, + } { + err := classed(t, db, broken.statement, broken.args...) + if !errors.Is(err, dbkit.ErrCheck) || !strings.Contains(err.Error(), "CHECK constraint failed: "+name) { + t.Errorf("%s = %v, want dbkit.ErrCheck naming %s", broken.statement, err, name) + } + } +} + +func TestSchemaTakesEveryValueItsChecksAllow(t *testing.T) { + t.Parallel() + + db := withRows(t) + + for _, status := range []string{"draft", "pending", "private", "scheduled", "published", "trash"} { + mustExec(t, db, "UPDATE core_content SET status = ?, published_at = 1", status) + } + mustExec(t, db, insertRevision, accountID) + mustExec(t, db, "INSERT INTO core_content_revisions (id, content_id, kind, author_id, created_at) "+ + "VALUES ('019a0000-0000-7000-8000-0000000000e2', ?, 'autosave', ?, 1)", contentID, accountID) + mustExec(t, db, "UPDATE core_content_types SET page_kind = 'archive'") + inserted(t, db, insertMedia, "photo.png") + mustExec(t, db, "INSERT INTO core_media (media_type, file, mime_type, author_id, created_at, updated_at) "+ + "VALUES ('file', 'a.pdf', 'application/pdf', ?, 1, 1)", accountID) + mustExec(t, db, "UPDATE core_content_fields SET kind = 'media', many = 1") + mustExec(t, db, "UPDATE core_content_fields SET kind = 'relation', relates_to = 'post', many = 1") +} + +func TestSchemaRefusesAScheduledRowWithNoDate(t *testing.T) { + t.Parallel() + + db := withRows(t) + + err := classed(t, db, "UPDATE core_content SET status = 'scheduled'") + + if !errors.Is(err, dbkit.ErrCheck) || !strings.Contains(err.Error(), "content_published_needs_date") { + t.Errorf("a scheduled row with no date = %v, want dbkit.ErrCheck naming content_published_needs_date", err) + } +} + +func TestSchemaRefusesTextThatIsNotJSONInAJSONColumn(t *testing.T) { + t.Parallel() + + db := withRows(t) + inserted(t, db, insertMedia, "photo.png") + mustExec(t, db, insertRevision, accountID) + + for _, column := range []string{ + "core_content.fields", "core_content_revisions.fields", "core_media.sizes", "core_content_fields.settings", + "core_field_groups.location", + } { + table, name, _ := strings.Cut(column, ".") + for _, text := range []string{"not json", "[1,]", "['a']", "[0x10]"} { + err := classed(t, db, "UPDATE "+table+" SET "+name+" = ?", text) + if !errors.Is(err, dbkit.ErrCheck) { + t.Errorf("%s set to %s = %v, want dbkit.ErrCheck", column, text, err) + } + } + } +} + +func TestSchemaFillsTheColumnsAnInsertLeavesOut(t *testing.T) { + t.Parallel() + + db := withRows(t) + mustExec(t, db, insertType, "pages", 0, 1) + mustExec(t, db, "INSERT INTO core_content_relations (from_id, field_id, to_id, sort_at) VALUES (?, 1, ?, 1)", + contentID, contentID) + + var hierarchical, revisions, visible bool + var revisionCap, position int64 + var description, fields string + err := db.QueryRowContext(t.Context(), "SELECT t.hierarchical, t.revisions, t.revision_cap, t.description, "+ + "r.position, r.visible, c.fields FROM core_content_types AS t, core_content_relations AS r, core_content AS c "+ + "WHERE t.key = 'page'").Scan(&hierarchical, &revisions, &revisionCap, &description, &position, &visible, &fields) + if err != nil { + t.Fatalf("reading the rows: %v", err) + } + + if hierarchical || !revisions || revisionCap != 0 || description != "" { + t.Errorf("the type defaults = %t, %t, %d, %q, want false, true, 0 and empty", hierarchical, revisions, + revisionCap, description) + } + if position != 1 || visible || fields != "{}" { + t.Errorf("the relation defaults = %d, %t and the content fields %s, want 1, false and {}", position, visible, + fields) + } +} + +func TestSchemaRefusesABooleanOtherThanZeroOrOne(t *testing.T) { + t.Parallel() + + db := withRows(t) + + for _, broken := range []struct { + column string + statement string + args []any + }{ + {"hierarchical", "UPDATE core_content_types SET hierarchical = 2", nil}, + {"revisions", "UPDATE core_content_types SET revisions = 2", nil}, + {"is_default", "UPDATE core_content_types SET is_default = 2", nil}, + {"active", "UPDATE core_content_types SET active = 2", nil}, + {"active", "UPDATE core_field_groups SET active = 2", nil}, + {"many", "UPDATE core_content_fields SET many = 2, kind = 'media'", nil}, + {"required", "UPDATE core_content_fields SET required = 2", nil}, + { + "visible", + "INSERT INTO core_content_relations (from_id, field_id, to_id, sort_at, visible) VALUES (?, 1, ?, 1, 2)", + []any{contentID, contentID}, + }, + } { + err := classed(t, db, broken.statement, broken.args...) + if !errors.Is(err, dbkit.ErrCheck) || !strings.Contains(err.Error(), broken.column+" IN (0, 1)") { + t.Errorf("%s = %v, want dbkit.ErrCheck on %s IN (0, 1)", broken.statement, err, broken.column) + } + } +} + +func TestSchemaKeepsAnAuthorWhoHoldsContentARevisionOrMedia(t *testing.T) { + t.Parallel() + + for table, hold := range map[string]func(t *testing.T, db *sql.DB){ + "core_content": func(t *testing.T, db *sql.DB) { + mustExec(t, db, insertContent, contentID, "hello", accountID, "hello") + }, + "core_content_revisions": func(t *testing.T, db *sql.DB) { + mustExec(t, db, insertContent, contentID, "hello", otherAccountID, "hello") + mustExec(t, db, insertRevision, accountID) + }, + "core_media": func(t *testing.T, db *sql.DB) { + inserted(t, db, insertMedia, "photo.png") + }, + } { + t.Run(table, func(t *testing.T) { + t.Parallel() + db := migrated(t) + mustExec(t, db, insertAccount, accountID, "maria") + mustExec(t, db, insertAccount, otherAccountID, "maria.perez") + hold(t, db) + + err := classed(t, db, "DELETE FROM auth_users WHERE id = ?", accountID) + + action := count(t, db, `SELECT count(*) FROM pragma_foreign_key_list(?) `+ + `WHERE "from" = 'author_id' AND "table" = 'auth_users' AND on_delete = 'RESTRICT'`, table) + left := count(t, db, "SELECT count(*) FROM auth_users WHERE id = ?", accountID) + if !errors.Is(err, dbkit.ErrForeignKey) || left != 1 || action != 1 { + t.Errorf("deleting an author held by %s = %v with %d accounts left and %d RESTRICT keys, "+ + "want dbkit.ErrForeignKey, the account kept and one RESTRICT key", table, err, left, action) + } + }) + } +} + +func TestSchemaNeverHandsOutTheIdentityOfADeletedRowAgain(t *testing.T) { + t.Parallel() + + for table, insert := range map[string]string{ + "core_media": insertMedia, "core_field_groups": insertGroup, "core_content_fields": insertField, + } { + t.Run(table, func(t *testing.T) { + t.Parallel() + db := withRows(t) + inserted(t, db, insert, "first") + deleted := inserted(t, db, insert, "second") + mustExec(t, db, "DELETE FROM "+table+" WHERE id = ?", deleted) + + if again := inserted(t, db, insert, "third"); again <= deleted { + t.Errorf("the row stored after id %d was deleted from %s got id %d, want a higher one", + deleted, table, again) + } + }) + } +} + +func TestSchemaRefusesASecondTopLevelFieldWithOneKeyInAGroup(t *testing.T) { + t.Parallel() + + db := withRows(t) + + err := classed(t, db, insertField, "subtitle") + + if !errors.Is(err, dbkit.ErrUnique) { + t.Errorf("a second top-level field keyed subtitle in one group = %v, want dbkit.ErrUnique", err) + } +} + +func TestSchemaRefusesASecondContentRowOnOnePath(t *testing.T) { + t.Parallel() + + db := withRows(t) + + err := classed(t, db, insertContent, "019a0000-0000-7000-8000-0000000000c2", "hello-again", accountID, "hello") + + if !errors.Is(err, dbkit.ErrUnique) { + t.Errorf("a second content row on the path hello = %v, want dbkit.ErrUnique", err) + } +} + +func TestSchemaHoldsNoPrimaryKeyColumnThatTakesNull(t *testing.T) { + t.Parallel() + + db := migrated(t) + + nullable := count(t, db, `SELECT count(*) FROM sqlite_schema AS t JOIN pragma_table_info(t.name) AS c `+ + `WHERE t.type = 'table' AND t.name LIKE 'core\_%' ESCAPE '\' AND t.name <> 'core_goose_db_version' `+ + `AND c.pk > 0 AND c."notnull" = 0`) + + if nullable != 0 { + t.Errorf("%d primary key columns of the core tables take NULL, want none", nullable) + } +} + +func TestSchemaSeedsTheDefaultPostType(t *testing.T) { + t.Parallel() + + db := migrated(t) + var got seededType + var created, updated int64 + err := db.QueryRowContext(t.Context(), "SELECT key, singular_label, plural_label, route_word, hierarchical, "+ + "revisions, revision_cap, page_kind, is_default, active, origin, description, created_at, updated_at "+ + "FROM core_content_types").Scan(&got.key, &got.singular, &got.plural, &got.route, &got.hierarchical, + &got.revisions, &got.revisionCap, &got.pageKind, &got.isDefault, &got.active, &got.origin, &got.description, + &created, &updated) + if err != nil { + t.Fatalf("reading the seeded type: %v", err) + } + + want := seededType{ + key: "post", singular: "Post", plural: "Posts", revisions: true, revisionCap: 100, pageKind: "single", + isDefault: true, active: true, description: "Manage the posts on this site.", + } + if got != want { + t.Errorf("the seeded type = %+v, want %+v", got, want) + } + earliest := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC).UnixMicro() + if created != updated || created < earliest || created > time.Now().Add(time.Hour).UnixMicro() || + created%1000 != 0 { + t.Errorf("the seeded type was stamped %d and %d, want one whole millisecond of the build in microseconds", + created, updated) + } +} diff --git a/internal/sqlite/strip.go b/internal/sqlite/strip.go new file mode 100644 index 00000000..289e024d --- /dev/null +++ b/internal/sqlite/strip.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite + +import ( + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strings" +) + +// keyPath is the object keys a strip function walks down, read from a JSON array of text. +type keyPath []string + +// stripFieldPath answers strip_field_path(doc, path), doc without the value at path, the same path in every array. +func stripFieldPath(args []driver.Value) (driver.Value, error) { + return strip("strip_field_path", args, dropKey) +} + +// stripLayout answers strip_layout(doc, path), doc without the layout rows that hold the last key of path. +func stripLayout(args []driver.Value) (driver.Value, error) { + return strip("strip_layout", args, dropHolders) +} + +// strip answers fn(doc, path) with last applied where path has one key left, NULL in either giving NULL. +func strip(fn string, args []driver.Value, last func(doc any, key string) any) (driver.Value, error) { + if args[0] == nil || args[1] == nil { + return nil, nil + } + doc, path, err := decodeArgs(args) + if err != nil { + return nil, fmt.Errorf("%s: %w", fn, err) + } + return encode(cutAt(doc, path, last)) +} + +// decodeArgs returns the document and the path a strip function was called with. +func decodeArgs(args []driver.Value) (any, keyPath, error) { + doc, err := decodeDocument(args[0]) + if err != nil { + return nil, nil, err + } + path, err := decodePath(args[1]) + if err != nil { + return nil, nil, err + } + return doc, path, nil +} + +// textOf returns a SQL text or blob value as text, and false for any other value. +func textOf(value driver.Value) (string, bool) { + switch v := value.(type) { + case string: + return v, true + case []byte: + return string(v), true + } + return "", false +} + +// decodeDocument returns the one JSON value of value with every number kept as written. +func decodeDocument(value driver.Value) (any, error) { + text, ok := textOf(value) + if !ok { + return nil, fmt.Errorf("the document must be JSON text, got %T", value) + } + decoder := json.NewDecoder(strings.NewReader(text)) + decoder.UseNumber() + var doc any + if err := decoder.Decode(&doc); err != nil { + return nil, fmt.Errorf("the document is not JSON: %w", err) + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return nil, errors.New("the document holds more after its JSON value") + } + return doc, nil +} + +// decodePath returns the keys of value, a JSON array of text. +func decodePath(value driver.Value) (keyPath, error) { + text, ok := textOf(value) + if !ok { + return nil, fmt.Errorf("the path must be JSON text, got %T", value) + } + path, ok := keysOf(text) + if !ok { + return nil, fmt.Errorf("the path must be a JSON array of text, got %s", text) + } + return path, nil +} + +// keysOf returns the keys of text, and false when text is not a JSON array of text with no null in it. +func keysOf(text string) (keyPath, bool) { + var keys []*string + if err := json.Unmarshal([]byte(text), &keys); err != nil || keys == nil { + return nil, false + } + path := make(keyPath, len(keys)) + for i, key := range keys { + if key == nil { + return nil, false + } + path[i] = *key + } + return path, true +} + +// encode returns doc as compact JSON text with its keys sorted and its markup left unescaped. +func encode(doc any) (driver.Value, error) { + var text strings.Builder + encoder := json.NewEncoder(&text) + encoder.SetEscapeHTML(false) + err := encoder.Encode(doc) + return strings.TrimSuffix(text.String(), "\n"), err +} + +// cutAt returns doc with last applied where path has one key left, walking every array with the same path. +func cutAt(doc any, path keyPath, last func(doc any, key string) any) any { + if len(path) == 1 { + return last(doc, path[0]) + } + switch v := doc.(type) { + case []any: + return mapElements(v, func(element any) any { return cutAt(element, path, last) }) + case map[string]any: + return descend(v, path, last) + } + return doc +} + +// descend returns object with the value under the first key of path cut at the rest of path, when both exist. +func descend(object map[string]any, path keyPath, last func(doc any, key string) any) any { + if len(path) == 0 { + return object + } + if inner, ok := object[path[0]]; ok { + object[path[0]] = cutAt(inner, path[1:], last) + } + return object +} + +// mapElements returns the elements of array, each passed through cut. +func mapElements(array []any, cut func(element any) any) []any { + cutArray := make([]any, len(array)) + for i, element := range array { + cutArray[i] = cut(element) + } + return cutArray +} + +// dropKey returns doc without key, from an object or from every object an array holds at any depth. +func dropKey(doc any, key string) any { + switch v := doc.(type) { + case []any: + return mapElements(v, func(element any) any { return dropKey(element, key) }) + case map[string]any: + delete(v, key) + } + return doc +} + +// dropHolders returns an array without its rows that hold key, and any other value unchanged. +func dropHolders(doc any, key string) any { + rows, ok := doc.([]any) + if !ok { + return doc + } + kept := make([]any, 0, len(rows)) + for _, row := range rows { + if !holds(row, key) { + kept = append(kept, row) + } + } + return kept +} + +// holds reports whether jsonb's ? operator finds key in value, as an object key, a text element or the text itself. +func holds(value any, key string) bool { + switch v := value.(type) { + case map[string]any: + _, ok := v[key] + return ok + case []any: + return slices.ContainsFunc(v, func(element any) bool { return holdsText(element, key) }) + } + return holdsText(value, key) +} + +// holdsText reports whether value is the text key. +func holdsText(value any, key string) bool { + text, ok := value.(string) + return ok && text == key +} diff --git a/internal/sqlite/strip_test.go b/internal/sqlite/strip_test.go new file mode 100644 index 00000000..cc49e0cf --- /dev/null +++ b/internal/sqlite/strip_test.go @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sqlite_test + +import ( + "database/sql" + "strings" + "testing" + + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" +) + +// stripFunctions names both field strip functions SQL calls. +var stripFunctions = []string{"strip_field_path", "strip_layout"} + +// stripCase is one document and path a strip function is called with and the JSON text it answers. +type stripCase struct { + name string + doc string + path string + want string +} + +// stripped returns the text the SQL function fn answers for doc and path on db. +func stripped(t *testing.T, db *sql.DB, fn string, doc, path any) string { + t.Helper() + var got string + if err := db.QueryRowContext(t.Context(), "SELECT "+fn+"(?, ?)", doc, path).Scan(&got); err != nil { + t.Fatalf("%s(%v, %v) error = %v, want nil", fn, doc, path, err) + } + return got +} + +// checkStripped fails t for every case whose answer from the SQL function fn differs from the case's want. +func checkStripped(t *testing.T, fn string, cases []stripCase) { + t.Helper() + db := sqlitetest.Open(t, testSettings().Options()) + for _, c := range cases { + if got := stripped(t, db, fn, c.doc, c.path); got != c.want { + t.Errorf("%s on %s (%s, %s) = %s, want %s", fn, c.name, c.doc, c.path, got, c.want) + } + } +} + +func TestStripFieldPathRemovesTheValueAtThePath(t *testing.T) { + t.Parallel() + + checkStripped(t, "strip_field_path", []stripCase{ + {"a top level key", `{"a":1,"b":2}`, `["a"]`, `{"b":2}`}, + {"a nested key", `{"a":{"x":1,"y":2},"b":3}`, `["a","x"]`, `{"a":{"y":2},"b":3}`}, + {"every element of an array", `[{"a":1,"b":2},{"a":3},5]`, `["a"]`, `[{"b":2},{},5]`}, + {"every element of a nested array", `{"rows":[{"a":1,"b":2},{"a":3}]}`, `["rows","a"]`, `{"rows":[{"b":2},{}]}`}, + {"every element of an array inside an array", `[[{"a":1,"b":2}],[]]`, `["a"]`, `[[{"b":2}],[]]`}, + {"an empty key", `{"":1,"a":2}`, `[""]`, `{"a":2}`}, + {"the last of a repeated key", `{"a":1,"a":2,"b":3}`, `["b"]`, `{"a":2}`}, + {"a key beside an integer above 2^53", `{"n":9007199254740993,"a":1}`, `["a"]`, `{"n":9007199254740993}`}, + {"a key beside markup and accents", `{"t":"é & co","a":1}`, `["a"]`, `{"t":"é & co"}`}, + }) +} + +func TestStripFieldPathLeavesADocumentWithNothingAtThePathUnchanged(t *testing.T) { + t.Parallel() + + checkStripped(t, "strip_field_path", []stripCase{ + {"an empty array", `[]`, `["a"]`, `[]`}, + {"text", `"a"`, `["a"]`, `"a"`}, + {"a number", `5`, `["a"]`, `5`}, + {"true", `true`, `["a"]`, `true`}, + {"JSON null", `null`, `["a"]`, `null`}, + {"an empty path", `{"a":1}`, `[]`, `{"a":1}`}, + {"a missing key", `{"a":1.50,"b":1e2}`, `["c"]`, `{"a":1.50,"b":1e2}`}, + {"a missing parent key", `{"a":1}`, `["b","a"]`, `{"a":1}`}, + {"a path longer than the document", `{"a":5}`, `["a","b"]`, `{"a":5}`}, + }) +} + +func TestStripLayoutDropsTheRowsThatHoldTheKey(t *testing.T) { + t.Parallel() + + checkStripped(t, "strip_layout", []stripCase{ + {"an object row with the key", `{"blocks":[{"hero":{}},{"text":1}]}`, `["blocks","hero"]`, + `{"blocks":[{"text":1}]}`}, + {"a bare text row equal to the key", `{"blocks":["hero","other"]}`, `["blocks","hero"]`, + `{"blocks":["other"]}`}, + {"an array row holding the key as text", `{"blocks":[["x","hero"],["x",["hero"]]]}`, `["blocks","hero"]`, + `{"blocks":[["x",["hero"]]]}`}, + {"no number row that reads as the key", `[7,"7"]`, `["7"]`, `[7]`}, + {"rows of a top level array", `[{"a":1},{"b":2},null,true]`, `["a"]`, `[{"b":2},null,true]`}, + {"every row", `{"rows":[{"a":1}]}`, `["rows","a"]`, `{"rows":[]}`}, + {"rows under every element", `[{"rows":[{"a":1},{"b":1}]},3]`, `["rows","a"]`, `[{"rows":[{"b":1}]},3]`}, + {"rows three levels down", `{"l":{"rows":[{"a":1},{"b":2}]}}`, `["l","rows","a"]`, `{"l":{"rows":[{"b":2}]}}`}, + {"rows beside an integer above 2^53", `{"n":9007199254740993,"rows":[{"a":1}]}`, `["rows","a"]`, + `{"n":9007199254740993,"rows":[]}`}, + }) +} + +func TestStripLayoutLeavesADocumentWithNoRowsAtThePathUnchanged(t *testing.T) { + t.Parallel() + + checkStripped(t, "strip_layout", []stripCase{ + {"an object under a path of one", `{"a":1}`, `["a"]`, `{"a":1}`}, + {"an object under an empty path", `{"a":1}`, `[]`, `{"a":1}`}, + {"an array under an empty path", `[{"a":1}]`, `[]`, `[{"a":1}]`}, + {"an object where the rows belong", `{"rows":{"a":1}}`, `["rows","a"]`, `{"rows":{"a":1}}`}, + {"a missing key", `{"x":[],"y":1.50}`, `["rows","a"]`, `{"x":[],"y":1.50}`}, + {"text", `"rows"`, `["rows","a"]`, `"rows"`}, + {"an empty array", `[]`, `["a"]`, `[]`}, + {"rows that are not text under the empty key", `[1,null,true,[2],{"b":1}]`, `[""]`, `[1,null,true,[2],{"b":1}]`}, + }) +} + +func TestTheStripFunctionsAnswerNullForANullDocumentOrPath(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + + for _, fn := range stripFunctions { + var docNull, pathNull bool + query := "SELECT " + fn + `(NULL, '["a"]') IS NULL, ` + fn + "('{}', NULL) IS NULL" + if err := db.QueryRowContext(t.Context(), query).Scan(&docNull, &pathNull); err != nil { + t.Fatalf("%s with NULL error = %v, want nil", fn, err) + } + if !docNull || !pathNull { + t.Errorf("%s answers NULL for a NULL document %t and for a NULL path %t, want both", fn, docNull, pathNull) + } + } +} + +func TestTheStripFunctionsReadJSONTextHeldInABlob(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + + if got := stripped(t, db, "strip_field_path", []byte(`{"a":1,"b":2}`), []byte(`["a"]`)); got != `{"b":2}` { + t.Errorf("strip_field_path on blobs = %s, want {\"b\":2}", got) + } + if got := stripped(t, db, "strip_layout", []byte(`{"r":["a","b"]}`), []byte(`["r","a"]`)); got != `{"r":["b"]}` { + t.Errorf("strip_layout on blobs = %s, want {\"r\":[\"b\"]}", got) + } +} + +func TestTheStripFunctionsRefuseADocumentOrAPathThatIsNotJSONText(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + + for _, fn := range stripFunctions { + for _, bad := range []struct { + name string + doc, path any + }{ + {"a document that is not JSON", `{`, `["a"]`}, + {"a document with text after its value", `{} {}`, `["a"]`}, + {"an empty document", ``, `["a"]`}, + {"a document that is a number", int64(5), `["a"]`}, + {"a path that is not JSON", `{}`, `[`}, + {"a path that holds a number", `{}`, `["a",1]`}, + {"a path that holds JSON null", `{}`, `["a",null]`}, + {"a path of JSON null", `{}`, `null`}, + {"a path that is an object", `{}`, `{"a":1}`}, + {"a path that is a number", `{}`, int64(1)}, + } { + var got sql.NullString + err := db.QueryRowContext(t.Context(), "SELECT "+fn+"(?, ?)", bad.doc, bad.path).Scan(&got) + if err == nil || !strings.Contains(err.Error(), fn+": the ") { + t.Errorf("%s on %s error = %v, want the function naming what it was handed", fn, bad.name, err) + } + } + } +} + +func TestTheStripFunctionsTakeTwoArguments(t *testing.T) { + t.Parallel() + + db := sqlitetest.Open(t, testSettings().Options()) + + for _, fn := range stripFunctions { + _, err := db.ExecContext(t.Context(), "SELECT "+fn+`('{}', '[]', 'x')`) + if err == nil || !strings.Contains(err.Error(), "wrong number of arguments") { + t.Errorf("%s with three arguments error = %v, want the argument count refused", fn, err) + } + } +} From e6ba68fd833a32ef016b26b115a7b91e06e94f6b Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 12:15:25 +0200 Subject: [PATCH 2/9] test(postgres): compare the core schema and the strip functions with SQLite --- internal/postgres/engine_parity_test.go | 878 ++++++++++++++++++++++++ 1 file changed, 878 insertions(+) create mode 100644 internal/postgres/engine_parity_test.go diff --git a/internal/postgres/engine_parity_test.go b/internal/postgres/engine_parity_test.go new file mode 100644 index 00000000..f1720fa0 --- /dev/null +++ b/internal/postgres/engine_parity_test.go @@ -0,0 +1,878 @@ +// SPDX-License-Identifier: Apache-2.0 + +package postgres_test + +import ( + "database/sql" + "fmt" + "maps" + "reflect" + "regexp" + "slices" + "strings" + "testing" + + "github.com/gopherium/framework/dbkit/sqlite/sqlitetest" + "github.com/gopherium/framework/gonsole" + authkitsqlite "github.com/gopherium/gouncer/authkit/sqlite" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/gopherium/gophenberg/internal/sqlite" +) + +const ( + // postgresColumns lists every column of the core tables in order, each table named schema_table. + postgresColumns = `SELECT c.table_schema || '_' || c.table_name, c.column_name::text, c.data_type::text, + c.is_nullable = 'NO', coalesce(c.column_default, ''), c.is_identity = 'YES' + FROM information_schema.columns AS c + JOIN information_schema.tables AS t USING (table_schema, table_name) + WHERE c.table_schema = 'core' AND t.table_type = 'BASE TABLE' + ORDER BY c.table_name, c.ordinal_position` + // postgresIndexSlots lists every slot of every index on the core tables in order, each name taking the schema. + postgresIndexSlots = `SELECT ns.nspname || '_' || tc.relname, ns.nspname || '_' || ic.relname, + i.indisprimary, i.indisunique, i.indnullsnotdistinct, NOT i.indimmediate, k.n <= i.indnkeyatts, + i.indkey[k.n - 1] = 0, + coalesce(a.attname::text, ''), coalesce((i.indoption[k.n - 1] & 1) = 1, false), + coalesce(pg_get_expr(i.indpred, i.indrelid), '') + FROM pg_index AS i + JOIN pg_class AS ic ON ic.oid = i.indexrelid + JOIN pg_class AS tc ON tc.oid = i.indrelid + JOIN pg_namespace AS ns ON ns.oid = tc.relnamespace + CROSS JOIN LATERAL generate_series(1, i.indnatts) AS k (n) + LEFT JOIN pg_attribute AS a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[k.n - 1] + WHERE ns.nspname = 'core' + ORDER BY ic.relname, k.n` + // postgresForeignKeys lists every foreign key of the core tables with its columns, target and actions. + postgresForeignKeys = `SELECT ns.nspname || '_' || tc.relname, rn.nspname || '_' || rt.relname, + con.confdeltype::text, con.confupdtype::text, + ARRAY(SELECT a.attname::text FROM unnest(con.conkey) WITH ORDINALITY AS k (attnum, at) + JOIN pg_attribute AS a ON a.attrelid = con.conrelid AND a.attnum = k.attnum ORDER BY k.at), + ARRAY(SELECT a.attname::text FROM unnest(con.confkey) WITH ORDINALITY AS k (attnum, at) + JOIN pg_attribute AS a ON a.attrelid = con.confrelid AND a.attnum = k.attnum ORDER BY k.at) + FROM pg_constraint AS con + JOIN pg_class AS tc ON tc.oid = con.conrelid + JOIN pg_namespace AS ns ON ns.oid = tc.relnamespace + JOIN pg_class AS rt ON rt.oid = con.confrelid + JOIN pg_namespace AS rn ON rn.oid = rt.relnamespace + WHERE ns.nspname = 'core' AND con.contype = 'f'` + // postgresChecks lists the name and the definition of every CHECK constraint of the core tables. + postgresChecks = `SELECT ns.nspname || '_' || tc.relname, con.conname::text, pg_get_constraintdef(con.oid) + FROM pg_constraint AS con + JOIN pg_class AS tc ON tc.oid = con.conrelid + JOIN pg_namespace AS ns ON ns.oid = tc.relnamespace + WHERE ns.nspname = 'core' AND con.contype = 'c'` + // sqliteTables lists the core tables of a SQLite file, leaving out the core version table. + sqliteTables = `SELECT name FROM sqlite_schema WHERE type = 'table' + AND name LIKE 'core\_%' ESCAPE '\' AND name <> 'core_goose_db_version'` + // sqliteColumns lists the columns of one SQLite table in order. + sqliteColumns = `SELECT name, upper(type), "notnull", pk, coalesce(dflt_value, '') + FROM pragma_table_info(?) ORDER BY cid` + // sqliteIndexSlots lists every key slot of every index of one SQLite table but its primary key, in order. + sqliteIndexSlots = `SELECT il.name, il."unique", coalesce(s.sql, ''), x.cid = -2, coalesce(x.name, ''), x."desc" + FROM pragma_index_list(?) AS il + JOIN pragma_index_xinfo(il.name) AS x ON x."key" = 1 + LEFT JOIN sqlite_schema AS s ON s.type = 'index' AND s.name = il.name + WHERE il.origin <> 'pk' + ORDER BY il.name, x.seqno` + // sqliteForeignKeys lists every column of every foreign key of one SQLite table, in order. + sqliteForeignKeys = `SELECT id, "table", "from", coalesce("to", ''), on_update, on_delete + FROM pragma_foreign_key_list(?) ORDER BY id, seq` + // sqliteTableText returns the statement that created one SQLite table. + sqliteTableText = `SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?` + // postgresStrip calls a core strip function on JSON text and compares its answer with other JSON text as jsonb. + postgresStrip = `WITH answer AS ( + SELECT core.%s($1::text::jsonb, ARRAY(SELECT jsonb_array_elements_text($2::text::jsonb))) AS doc + ) + SELECT doc::text, doc IS NOT DISTINCT FROM $3::text::jsonb FROM answer` +) + +var ( + // neutralTypes maps each PostgreSQL column type of the core tables to the SQLite type that stores it. + neutralTypes = map[string]string{ + "uuid": "TEXT", "text": "TEXT", "jsonb": jsonText, + "timestamp with time zone": "INTEGER", "integer": "INTEGER", "bigint": "INTEGER", + "boolean": "BOOLEAN", + } + // defaultTwins maps each column default PostgreSQL prints in another form to the form SQLite reports. + defaultTwins = map[string]string{ + "''::text": "''", "'{}'::jsonb": "'{}'", "'[]'::jsonb": "'[]'", "false": "0", "true": "1", + } + // foreignKeyActions maps each PostgreSQL foreign key action code to the action SQLite reports. + foreignKeyActions = map[string]string{ + "a": "NO ACTION", "r": "RESTRICT", "c": "CASCADE", "n": "SET NULL", "d": "SET DEFAULT", + } + // checkTwins holds each named check with its definition as PostgreSQL prints it and as the SQLite baseline writes it. + checkTwins = map[string]sqlTwin{ + "content_status_check": { + postgres: "CHECK ((status = ANY (ARRAY['draft'::text, 'pending'::text, 'private'::text, " + + "'scheduled'::text, 'published'::text, 'trash'::text])))", + sqlite: "status IN ('draft', 'pending', 'private', 'scheduled', 'published', 'trash')", + }, + "content_published_needs_date": { + postgres: "CHECK (((status <> ALL (ARRAY['scheduled'::text, 'published'::text])) OR " + + "(published_at IS NOT NULL)))", + sqlite: "status NOT IN ('scheduled', 'published') OR published_at IS NOT NULL", + }, + "content_path_is_addressable": {postgres: "CHECK ((path <> ''::text))", sqlite: "path <> ''"}, + "content_revisions_kind_check": { + postgres: "CHECK ((kind = ANY (ARRAY['revision'::text, 'autosave'::text])))", + sqlite: "kind IN ('revision', 'autosave')", + }, + "content_types_page_kind_check": { + postgres: "CHECK ((page_kind = ANY (ARRAY['single'::text, 'archive'::text])))", + sqlite: "page_kind IN ('single', 'archive')", + }, + "content_types_root_is_the_default": { + postgres: "CHECK (((route_word <> ''::text) OR is_default))", sqlite: "route_word <> '' OR is_default", + }, + "content_types_default_stays_active": { + postgres: "CHECK ((active OR (NOT is_default)))", sqlite: "active OR NOT is_default", + }, + "media_media_type_check": { + postgres: "CHECK ((media_type = ANY (ARRAY['image'::text, 'file'::text])))", + sqlite: "media_type IN ('image', 'file')", + }, + "content_fields_relation_target": { + postgres: "CHECK (((kind = 'relation'::text) = (relates_to IS NOT NULL)))", + sqlite: "(kind = 'relation') = (relates_to IS NOT NULL)", + }, + "content_fields_many_relational": { + postgres: "CHECK (((many = false) OR (kind = ANY (ARRAY['relation'::text, 'media'::text]))))", + sqlite: "many = 0 OR kind IN ('relation', 'media')", + }, + "field_groups_location_is_array": { + postgres: "CHECK ((jsonb_typeof(location) = 'array'::text))", + sqlite: "json_valid(location) AND json_type(location) = 'array'", + }, + } + // partialPredicates holds each partial index by name with its predicate as PostgreSQL prints it and SQLite keeps it. + partialPredicates = map[string]sqlTwin{ + "core_content_types_one_default_idx": {postgres: "is_default", sqlite: "is_default"}, + "core_content_types_route_word_idx": {postgres: "(route_word <> ''::text)", sqlite: "route_word <> ''"}, + "core_content_revisions_one_autosave_per_author_idx": { + postgres: "(kind = 'autosave'::text)", sqlite: "kind = 'autosave'", + }, + "core_content_fields_container_key_unique": { + postgres: "(parent_field_id IS NOT NULL)", sqlite: "parent_field_id IS NOT NULL", + }, + "core_content_relations_target_idx": {postgres: "visible", sqlite: "visible"}, + } + // allowedDifferences are the indexes the SQLite baseline writes differently from PostgreSQL on purpose. + allowedDifferences = []allowedDifference{ + { + why: "a GIN index over the field values is a PostgreSQL speed-up with no SQLite twin", + table: "core_content", index: "core_content_field_values_idx", twin: noTwin, + }, + { + why: "the unique path index serves the prefix scan that text_pattern_ops serves on PostgreSQL", + table: "core_content", index: "core_content_path_prefix_idx", twin: noTwin, + }, + { + why: "SQLite has no INCLUDE, so the included column trails the key columns", + table: "core_content_relations", index: "core_content_relations_target_idx", twin: includedAsKeys, + }, + { + why: "SQLite has no NULLS NOT DISTINCT, so the parent slot is an expression that turns NULL into 0", + table: "core_content_fields", index: "core_content_fields_scope_key_unique", twin: coalescedParent, + }, + { + why: "SQLite checks a unique index row by row and cannot defer it, so its path rewrite runs as two updates", + table: "core_content", index: "core_content_path_unique", twin: checkedRowByRow, + }, + } + // fieldPathInputs are the documents and paths both engines run strip_field_path over. + fieldPathInputs = []stripInput{ + {`{"a":1,"b":2}`, `["a"]`}, + {`{"a":{"x":1,"y":2},"b":3}`, `["a","x"]`}, + {`[{"a":1,"b":2},{"a":3},5]`, `["a"]`}, + {`[[{"a":1,"b":2}],[]]`, `["a"]`}, + {`{"rows":[{"a":1,"b":2},{"a":3}]}`, `["rows","a"]`}, + {`{"a":[{"b":1},{"c":2}],"b":3}`, `["a","b"]`}, + {`{"a":{"a":1,"b":2}}`, `["a","a"]`}, + {`{"":1,"a":2}`, `[""]`}, + {`{"a":1,"a":2,"b":3}`, `["b"]`}, + {`{"é":1,"e":2}`, `["é"]`}, + {`{"n":9007199254740993,"a":1}`, `["a"]`}, + {`{"t":"é & co","a":1}`, `["a"]`}, + {`[]`, `["a"]`}, + {`"a"`, `["a"]`}, + {`5`, `["a"]`}, + {`true`, `["a"]`}, + {`null`, `["a"]`}, + {`{"a":1}`, `[]`}, + {`[1,{"a":2}]`, `[]`}, + {`{"a":1.50,"b":1e2}`, `["c"]`}, + {`{"a":1}`, `["b","a"]`}, + {`{"a":5}`, `["a","b"]`}, + {`{"a":null}`, `["a","b"]`}, + {nil, `["a"]`}, + {nil, `["a","b"]`}, + } + // layoutInputs are the documents and paths both engines run strip_layout over. + layoutInputs = []stripInput{ + {`{"blocks":[{"hero":{}},{"text":1}]}`, `["blocks","hero"]`}, + {`{"blocks":["hero","other"]}`, `["blocks","hero"]`}, + {`{"blocks":[["x","hero"],["x",["hero"]]]}`, `["blocks","hero"]`}, + {`{"rows":[[{"a":1}],["a"]]}`, `["rows","a"]`}, + {`[7,"7"]`, `["7"]`}, + {`[{"a":1},{"b":2},null,true]`, `["a"]`}, + {`[{"a":null},{"b":1}]`, `["a"]`}, + {`{"rows":[{"a":1}]}`, `["rows","a"]`}, + {`{"rows":["a","b"]}`, `["rows","a"]`}, + {`{"rows":[{"a":1}],"a":2}`, `["rows","a"]`}, + {`[1,null,true,[2],{"b":1}]`, `[""]`}, + {`[{"rows":[{"a":1},{"b":1}]},3]`, `["rows","a"]`}, + {`[[{"rows":[{"a":1}]}]]`, `["rows","a"]`}, + {`{"l":{"rows":[{"a":1},{"b":2}]}}`, `["l","rows","a"]`}, + {`{"n":9007199254740993,"rows":[{"a":1}]}`, `["rows","a"]`}, + {`{"a":1}`, `["a"]`}, + {`{"a":1}`, `[]`}, + {`[{"a":1}]`, `[]`}, + {`{"rows":{"a":1}}`, `["rows","a"]`}, + {`{"rows":null}`, `["rows","a"]`}, + {`{"x":[],"y":1.50}`, `["rows","a"]`}, + {`"rows"`, `["rows","a"]`}, + {`[]`, `["a"]`}, + {nil, `["a"]`}, + {nil, `["rows","a"]`}, + } + // indexPredicate finds the predicate after the key list of a CREATE INDEX statement. + indexPredicate = regexp.MustCompile(`(?s)\)\s+WHERE\s+(.+)$`) + // checkStart finds the name of every named CHECK constraint in a CREATE TABLE statement and the parenthesis after it. + checkStart = regexp.MustCompile(`(?i)\bCONSTRAINT\s+(\w+)\s+CHECK\s*\(`) +) + +// jsonText is the shared type of a JSON column, jsonb on PostgreSQL and text that passes json_valid on SQLite. +const jsonText = "JSON TEXT" + +// schemaShape is the core tables of one engine in the shape both catalogs are read into, keyed by table name. +type schemaShape map[string]*tableShape + +// tableShape is one core table with its columns, primary key, other indexes, foreign keys and named checks. +type tableShape struct { + order []string + columns map[string]columnShape + primaryKey []string + indexes map[string]indexShape + foreignKeys map[string]foreignKeyShape + checks map[string]string +} + +// columnShape is the type of one column, whether it refuses NULL, its default and whether it counts its ids up. +type columnShape struct { + kind string + notNull bool + fallback string + counted bool +} + +// indexShape is one index other than the primary key, its predicate empty when it covers every row. +type indexShape struct { + unique bool + nullsNotDistinct bool + deferrable bool + keys []indexKey + included []string + predicate string +} + +// indexKey is one key slot of an index, a column or an expression, in ascending or descending order. +type indexKey struct { + column string + expression bool + descending bool +} + +// foreignKeyShape is the table and columns one foreign key points at and its delete and update actions. +type foreignKeyShape struct { + target string + targetColumns []string + onDelete string + onUpdate string +} + +// sqlTwin is one piece of schema text as PostgreSQL prints it and as the SQLite baseline writes it. +type sqlTwin struct { + postgres string + sqlite string +} + +// allowedDifference is one index the SQLite baseline writes differently on purpose, why, and the SQLite form it takes. +type allowedDifference struct { + why string + table string + index string + twin func(index indexShape) (indexShape, bool) +} + +// stripInput is one document and path both engines strip, the document nil for SQL NULL. +type stripInput struct { + doc any + path string +} + +// table returns the shape of the named table, adding an empty one when the schema holds none yet. +func (s schemaShape) table(name string) *tableShape { + if s[name] == nil { + s[name] = &tableShape{ + columns: map[string]columnShape{}, indexes: map[string]indexShape{}, + foreignKeys: map[string]foreignKeyShape{}, checks: map[string]string{}, + } + } + return s[name] +} + +// eachPostgresRow scans every row lookup answers on pool into dest and calls take after each one. +func eachPostgresRow(t *testing.T, pool *pgxpool.Pool, lookup string, dest []any, take func()) { + t.Helper() + rows, err := pool.Query(t.Context(), lookup) + if err != nil { + t.Fatalf("%s: %v", lookup, err) + } + _, err = pgx.ForEachRow(rows, dest, func() error { + take() + return nil + }) + if err != nil { + t.Fatalf("%s: %v", lookup, err) + } +} + +// eachSQLiteRow scans every row lookup answers for args on db into dest and calls take after each one. +func eachSQLiteRow(t *testing.T, db *sql.DB, lookup string, args []any, dest []any, take func()) { + t.Helper() + rows, err := db.QueryContext(t.Context(), lookup, args...) + if err != nil { + t.Fatalf("%s: %v", lookup, err) + } + defer func() { _ = rows.Close() }() + for rows.Next() { + if err := rows.Scan(dest...); err != nil { + t.Fatalf("%s: %v", lookup, err) + } + take() + } + if err := rows.Err(); err != nil { + t.Fatalf("%s: %v", lookup, err) + } +} + +// migratedSQLite returns a handle on a new SQLite file under the default settings, the account and core tables applied. +func migratedSQLite(t *testing.T) *sql.DB { + t.Helper() + env := gonsole.Env{Prefix: "GOPHENBERG_"} + settings, err := sqlite.SettingsFrom(env) + if err != nil { + t.Fatalf("reading the default SQLite settings: %v", err) + } + opts, err := sqlite.MigrateOptionsFrom(env) + if err != nil { + t.Fatalf("reading the default migration lock settings: %v", err) + } + db := sqlitetest.Open(t, settings.Options()) + accounts := authkitsqlite.MigrateOptions{LockWait: opts.LockWait, LockPoll: opts.LockPoll} + if err := authkitsqlite.Migrate(t.Context(), db, accounts); err != nil { + t.Fatalf("migrating the SQLite account tables: %v", err) + } + if err := sqlite.Migrate(t.Context(), db, opts); err != nil { + t.Fatalf("migrating the SQLite core tables: %v", err) + } + return db +} + +// postgresShape reads the core tables of pool into the shared shape, PostgreSQL types mapped to SQLite ones. +func postgresShape(t *testing.T, pool *pgxpool.Pool) schemaShape { + t.Helper() + shape := schemaShape{} + var table, name, dataType, fallback string + var notNull, identity bool + dest := []any{&table, &name, &dataType, ¬Null, &fallback, &identity} + eachPostgresRow(t, pool, postgresColumns, dest, func() { + kind, ok := neutralTypes[dataType] + if !ok { + kind = "PostgreSQL type " + dataType + " with no SQLite type" + } + addColumn(shape.table(table), name, columnShape{ + kind: kind, notNull: notNull, fallback: sqliteDefault(fallback), counted: identity, + }) + }) + readPostgresIndexes(t, pool, shape) + readPostgresForeignKeys(t, pool, shape) + var check, definition string + eachPostgresRow(t, pool, postgresChecks, []any{&table, &check, &definition}, func() { + shape.table(table).checks[check] = sqliteCheck(check, definition) + }) + return shape +} + +// readPostgresIndexes adds every index of the core tables on pool to shape, the primary key as its columns. +func readPostgresIndexes(t *testing.T, pool *pgxpool.Pool, shape schemaShape) { + t.Helper() + var table, name, column, predicate string + var primary, unique, nullsNotDistinct, deferrable, isKey, expression, descending bool + dest := []any{ + &table, &name, &primary, &unique, &nullsNotDistinct, &deferrable, &isKey, &expression, &column, &descending, + &predicate, + } + eachPostgresRow(t, pool, postgresIndexSlots, dest, func() { + owner := shape.table(table) + if primary { + owner.primaryKey = append(owner.primaryKey, column) + return + } + index := owner.indexes[name] + index.unique, index.nullsNotDistinct, index.predicate = unique, nullsNotDistinct, predicate + index.deferrable = deferrable + if isKey { + index.keys = append(index.keys, slot(column, expression, descending)) + } else { + index.included = append(index.included, column) + } + owner.indexes[name] = index + }) +} + +// readPostgresForeignKeys adds every foreign key of the core tables on pool to shape, keyed by its columns. +func readPostgresForeignKeys(t *testing.T, pool *pgxpool.Pool, shape schemaShape) { + t.Helper() + var table, target, onDelete, onUpdate string + var columns, targetColumns []string + dest := []any{&table, &target, &onDelete, &onUpdate, &columns, &targetColumns} + eachPostgresRow(t, pool, postgresForeignKeys, dest, func() { + shape.table(table).foreignKeys[strings.Join(columns, ", ")] = foreignKeyShape{ + target: target, targetColumns: targetColumns, + onDelete: foreignKeyActions[onDelete], onUpdate: foreignKeyActions[onUpdate], + } + }) +} + +// sqliteShape reads the core tables of db into the shared shape. +func sqliteShape(t *testing.T, db *sql.DB) schemaShape { + t.Helper() + shape := schemaShape{} + var table string + eachSQLiteRow(t, db, sqliteTables, nil, []any{&table}, func() { shape.table(table) }) + for name, owner := range shape { + var statement string + if err := db.QueryRowContext(t.Context(), sqliteTableText, name).Scan(&statement); err != nil { + t.Fatalf("reading the statement of %s: %v", name, err) + } + readSQLiteColumns(t, db, name, statement, owner) + readSQLiteIndexes(t, db, name, owner) + readSQLiteForeignKeys(t, db, name, owner) + readSQLiteChecks(statement, owner) + } + return shape +} + +// readSQLiteColumns adds the columns and the primary key of the named table on db, created by statement, to owner. +func readSQLiteColumns(t *testing.T, db *sql.DB, table, statement string, owner *tableShape) { + t.Helper() + var name, kind, fallback string + var notNull bool + var position int + keyAt := map[int]string{} + dest := []any{&name, &kind, ¬Null, &position, &fallback} + eachSQLiteRow(t, db, sqliteColumns, []any{table}, dest, func() { + addColumn(owner, name, columnShape{ + kind: sqliteKind(kind, name, statement), notNull: notNull, fallback: fallback, + counted: position > 0 && strings.Contains(statement, "AUTOINCREMENT"), + }) + if position > 0 { + keyAt[position] = name + } + }) + for position := 1; position <= len(keyAt); position++ { + owner.primaryKey = append(owner.primaryKey, keyAt[position]) + } +} + +// readSQLiteIndexes adds every index of the named table on db but its primary key to owner. +func readSQLiteIndexes(t *testing.T, db *sql.DB, table string, owner *tableShape) { + t.Helper() + var name, statement, column string + var unique, expression, descending bool + dest := []any{&name, &unique, &statement, &expression, &column, &descending} + eachSQLiteRow(t, db, sqliteIndexSlots, []any{table}, dest, func() { + index := owner.indexes[name] + index.unique, index.predicate = unique, predicateOf(statement) + index.keys = append(index.keys, slot(column, expression, descending)) + owner.indexes[name] = index + }) +} + +// readSQLiteForeignKeys adds every foreign key of the named table on db to owner, keyed by its columns. +func readSQLiteForeignKeys(t *testing.T, db *sql.DB, table string, owner *tableShape) { + t.Helper() + var id int + var target, from, to, onUpdate, onDelete string + columns := map[int][]string{} + keys := map[int]foreignKeyShape{} + dest := []any{&id, &target, &from, &to, &onUpdate, &onDelete} + eachSQLiteRow(t, db, sqliteForeignKeys, []any{table}, dest, func() { + columns[id] = append(columns[id], from) + key := keys[id] + keys[id] = foreignKeyShape{ + target: target, targetColumns: append(key.targetColumns, to), onDelete: onDelete, onUpdate: onUpdate, + } + }) + for id, key := range keys { + owner.foreignKeys[strings.Join(columns[id], ", ")] = key + } +} + +// readSQLiteChecks adds every named CHECK constraint of the table statement creates to owner, with its spaces folded. +func readSQLiteChecks(statement string, owner *tableShape) { + for _, at := range checkStart.FindAllStringSubmatchIndex(statement, -1) { + owner.checks[statement[at[2]:at[3]]] = enclosed(statement[at[1]-1:]) + } +} + +// enclosed returns the text inside the parentheses text opens with, its spaces folded. +func enclosed(text string) string { + depth := 0 + for i, r := range text { + switch r { + case '(': + depth++ + case ')': + depth-- + } + if depth == 0 { + return strings.Join(strings.Fields(text[1:i]), " ") + } + } + return "an unclosed check " + text +} + +// sqliteKind returns the shared type of the named SQLite column, jsonText when statement checks it with json_valid. +func sqliteKind(kind, name, statement string) string { + if kind == "TEXT" && strings.Contains(statement, "json_valid("+name+")") { + return jsonText + } + return kind +} + +// sqliteDefault returns the form SQLite reports for a column default as PostgreSQL prints it. +func sqliteDefault(fallback string) string { + if twin, ok := defaultTwins[fallback]; ok { + return twin + } + return fallback +} + +// sqliteCheck returns the SQLite form of the PostgreSQL definition of the named check, or names it untranslated. +func sqliteCheck(name, definition string) string { + if twin, ok := checkTwins[name]; ok && twin.postgres == definition { + return twin.sqlite + } + return "PostgreSQL check " + definition + " with no SQLite form in the check table" +} + +// addColumn appends the named column to the order of owner and records its shape. +func addColumn(owner *tableShape, name string, column columnShape) { + owner.order = append(owner.order, name) + owner.columns[name] = column +} + +// slot returns the index key slot over column, or over an expression when expression is set. +func slot(column string, expression, descending bool) indexKey { + if expression { + return indexKey{expression: true, descending: descending} + } + return indexKey{column: column, descending: descending} +} + +// predicateOf returns the predicate of a CREATE INDEX statement with its spaces folded, empty when it has none. +func predicateOf(statement string) string { + match := indexPredicate.FindStringSubmatch(statement) + if match == nil { + return "" + } + return strings.Join(strings.Fields(match[1]), " ") +} + +// expectedOnSQLite returns the PostgreSQL shape as the SQLite baseline must write it, every allowed difference applied. +func expectedOnSQLite(t *testing.T, shape schemaShape) schemaShape { + t.Helper() + for _, owner := range shape { + translatePredicates(owner) + } + for _, difference := range allowedDifferences { + applyDifference(t, shape, difference) + } + return shape +} + +// translatePredicates replaces the PostgreSQL predicate of every partial index of owner with its SQLite form. +func translatePredicates(owner *tableShape) { + for name, index := range owner.indexes { + index.predicate = sqlitePredicate(name, index.predicate) + owner.indexes[name] = index + } +} + +// applyDifference puts the SQLite form of the index difference names into shape, failing t when shape lacks it. +func applyDifference(t *testing.T, shape schemaShape, difference allowedDifference) { + t.Helper() + index, ok := shape.index(difference.table, difference.index) + if !ok { + t.Errorf("the allowed difference %q names %s, which PostgreSQL no longer has", difference.why, difference.index) + return + } + indexes := shape[difference.table].indexes + if twin, kept := difference.twin(index); kept { + indexes[difference.index] = twin + return + } + delete(indexes, difference.index) +} + +// index returns the named index of the named table and whether the schema holds it. +func (s schemaShape) index(table, name string) (indexShape, bool) { + owner := s[table] + if owner == nil { + return indexShape{}, false + } + index, ok := owner.indexes[name] + return index, ok +} + +// sqlitePredicate returns the SQLite form of the PostgreSQL predicate of the named index, or names it untranslated. +func sqlitePredicate(index, predicate string) string { + if predicate == "" { + return "" + } + if twin, ok := partialPredicates[index]; ok && twin.postgres == predicate { + return twin.sqlite + } + return "PostgreSQL predicate " + predicate + " with no SQLite form in the predicate table" +} + +// noTwin drops an index the SQLite baseline leaves out. +func noTwin(indexShape) (indexShape, bool) { + return indexShape{}, false +} + +// includedAsKeys returns index with its included columns moved after its key columns. +func includedAsKeys(index indexShape) (indexShape, bool) { + keys := slices.Clone(index.keys) + for _, column := range index.included { + keys = append(keys, indexKey{column: column}) + } + index.keys, index.included = keys, nil + return index, true +} + +// coalescedParent returns index with its NULLS NOT DISTINCT parent slot turned into an expression slot. +func coalescedParent(index indexShape) (indexShape, bool) { + if !index.nullsNotDistinct || len(index.keys) < 2 || index.keys[1] != (indexKey{column: "parent_field_id"}) { + return index, true + } + keys := slices.Clone(index.keys) + keys[1] = indexKey{expression: true} + index.keys, index.nullsNotDistinct = keys, false + return index, true +} + +// checkedRowByRow returns index with its deferrable check made one that runs on every row. +func checkedRowByRow(index indexShape) (indexShape, bool) { + index.deferrable = false + return index, true +} + +// String returns the column as a failure reads it, its type, NOT NULL, its default and whether it counts its ids up. +func (c columnShape) String() string { + text := c.kind + if c.notNull { + text += " NOT NULL" + } + if c.fallback != "" { + text += " DEFAULT " + c.fallback + } + if c.counted { + text += " counting its ids up" + } + return text +} + +// String returns the index as a failure reads it, its keys in order with the included columns and the predicate. +func (i indexShape) String() string { + keys := make([]string, len(i.keys)) + for n, key := range i.keys { + keys[n] = key.String() + } + text := "(" + strings.Join(keys, ", ") + ")" + if i.unique { + text = "UNIQUE " + text + } + if i.nullsNotDistinct { + text += " NULLS NOT DISTINCT" + } + if i.deferrable { + text += " DEFERRABLE" + } + if len(i.included) > 0 { + text += " INCLUDE (" + strings.Join(i.included, ", ") + ")" + } + if i.predicate != "" { + text += " WHERE " + i.predicate + } + return text +} + +// String returns the key slot as a failure reads it, a column name or an expression, with DESC when it descends. +func (k indexKey) String() string { + text := k.column + if k.expression { + text = "an expression" + } + if k.descending { + text += " DESC" + } + return text +} + +// String returns the foreign key as a failure reads it, its target and both actions. +func (f foreignKeyShape) String() string { + return f.target + " (" + strings.Join(f.targetColumns, ", ") + ") ON DELETE " + f.onDelete + + " ON UPDATE " + f.onUpdate +} + +// compareShapes fails t for every core table, column, key, index, foreign key and check SQLite does not twin. +func compareShapes(t *testing.T, onSQLite, onPostgres schemaShape) { + t.Helper() + for _, name := range unionKeys(onSQLite, onPostgres) { + got, want := onSQLite[name], onPostgres[name] + switch { + case got == nil: + t.Errorf("table %s is on PostgreSQL and missing on SQLite", name) + case want == nil: + t.Errorf("table %s is on SQLite and missing on PostgreSQL", name) + default: + compareTables(t, name, got, want) + } + } +} + +// compareTables fails t for every way the named table on SQLite differs from its PostgreSQL twin. +func compareTables(t *testing.T, name string, got, want *tableShape) { + t.Helper() + compareNamed(t, name, "column", got.columns, want.columns) + gotOrder, wantOrder := inBoth(got.order, want.columns), inBoth(want.order, got.columns) + if !slices.Equal(gotOrder, wantOrder) { + t.Errorf("%s columns stand as %v on SQLite, want %v as on PostgreSQL", name, gotOrder, wantOrder) + } + if !slices.Equal(got.primaryKey, want.primaryKey) { + t.Errorf("%s primary key is %v on SQLite, want %v as on PostgreSQL", name, got.primaryKey, want.primaryKey) + } + compareNamed(t, name, "index", got.indexes, want.indexes) + compareNamed(t, name, "foreign key on", got.foreignKeys, want.foreignKeys) + compareNamed(t, name, "check", got.checks, want.checks) +} + +// compareNamed fails t for every entry of what in the named table that is missing from one engine or differs. +func compareNamed[V any](t *testing.T, table, what string, got, want map[string]V) { + t.Helper() + for _, name := range unionKeys(got, want) { + gotValue, onSQLite := got[name] + wantValue, onPostgres := want[name] + switch { + case !onSQLite: + t.Errorf("%s %s %s is on PostgreSQL and missing on SQLite", table, what, name) + case !onPostgres: + t.Errorf("%s %s %s is on SQLite and missing on PostgreSQL", table, what, name) + case !reflect.DeepEqual(gotValue, wantValue): + t.Errorf("%s %s %s is %v on SQLite, want %v as on PostgreSQL", table, what, name, gotValue, wantValue) + } + } +} + +// inBoth returns the names of order that other also holds, in the order they stand. +func inBoth(order []string, other map[string]columnShape) []string { + return slices.DeleteFunc(slices.Clone(order), func(name string) bool { + _, ok := other[name] + return !ok + }) +} + +// unionKeys returns every key of a and b once, sorted. +func unionKeys[V any](a, b map[string]V) []string { + keys := slices.Collect(maps.Keys(a)) + for key := range b { + if _, ok := a[key]; !ok { + keys = append(keys, key) + } + } + slices.Sort(keys) + return keys +} + +// sqliteStripped returns the answer of the SQL function fn on db for in. +func sqliteStripped(t *testing.T, db *sql.DB, fn string, in stripInput) sql.NullString { + t.Helper() + var answer sql.NullString + if err := db.QueryRowContext(t.Context(), "SELECT "+fn+"(?, ?)", in.doc, in.path).Scan(&answer); err != nil { + t.Fatalf("%s(%v) on SQLite: %v", fn, in, err) + } + return answer +} + +// postgresStripped returns the answer of core.fn on pool for in and whether it equals other as jsonb. +func postgresStripped( + t *testing.T, pool *pgxpool.Pool, fn string, in stripInput, other sql.NullString, +) (sql.NullString, bool) { + t.Helper() + var answer sql.NullString + var same bool + err := pool.QueryRow(t.Context(), fmt.Sprintf(postgresStrip, fn), in.doc, in.path, other).Scan(&answer, &same) + if err != nil { + t.Fatalf("%s(%v) on PostgreSQL: %v", fn, in, err) + } + return answer, same +} + +// String returns the input as a failure reads it, its document NULL when nil. +func (in stripInput) String() string { + if in.doc == nil { + return "NULL, " + in.path + } + return fmt.Sprintf("%s, %s", in.doc, in.path) +} + +// shown returns answer as a failure reads it, NULL when it is SQL NULL. +func shown(answer sql.NullString) string { + if !answer.Valid { + return "NULL" + } + return answer.String +} + +func TestEngineParityKeepsTheCoreSchemaAlikeOnBothEngines(t *testing.T) { + t.Parallel() + + want := expectedOnSQLite(t, postgresShape(t, migratedPool(t))) + got := sqliteShape(t, migratedSQLite(t)) + + compareShapes(t, got, want) +} + +func TestEngineParityStripsEveryDocumentAlikeOnBothEngines(t *testing.T) { + t.Parallel() + + pool, db := migratedPool(t), migratedSQLite(t) + + for _, run := range []struct { + fn string + inputs []stripInput + }{{"strip_field_path", fieldPathInputs}, {"strip_layout", layoutInputs}} { + for _, in := range run.inputs { + onSQLite := sqliteStripped(t, db, run.fn, in) + onPostgres, same := postgresStripped(t, pool, run.fn, in, onSQLite) + if !same { + t.Errorf("%s(%v) is %s on SQLite and %s on PostgreSQL", run.fn, in, shown(onSQLite), shown(onPostgres)) + } + } + } +} From 116435fbdf879f8061593e40ecbd62b52cede6c1 Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 12:15:26 +0200 Subject: [PATCH 3/9] chore(lint): let only dbkit import a SQLite driver or call sql.Open --- .golangci.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 15eb1539..2fcd7bec 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -5,6 +5,7 @@ linters: enable: - cyclop - depguard + - forbidigo - gocognit - lll - misspell @@ -56,6 +57,26 @@ linters: deny: - pkg: github.com/gopherium/gophenberg/internal desc: plugins may import only the sdk package from Gophenberg + one-sqlite-driver: + deny: + - pkg: modernc.org/sqlite + desc: open SQLite through github.com/gopherium/framework/dbkit/sqlite + - pkg: modernc.org/libc + desc: open SQLite through github.com/gopherium/framework/dbkit/sqlite + - pkg: github.com/mattn/go-sqlite3 + desc: open SQLite through github.com/gopherium/framework/dbkit/sqlite + - pkg: github.com/ncruces/go-sqlite3 + desc: open SQLite through github.com/gopherium/framework/dbkit/sqlite + - pkg: github.com/glebarez/go-sqlite + desc: open SQLite through github.com/gopherium/framework/dbkit/sqlite + - pkg: turso.tech/database/tursogo + desc: open SQLite through github.com/gopherium/framework/dbkit/sqlite + forbidigo: + analyze-types: true + forbid: + - pattern: ^sql\.Open(DB)?$ + pkg: ^database/sql$ + msg: open the database through dbkit exclusions: rules: - path: _test\.go @@ -66,6 +87,12 @@ linters: linters: - cyclop - gocognit + - path: ^internal/postgres/(migrate|migrate_test|export_test|content(_types|_paths)?_migration_test)\.go$ + linters: + - forbidigo + - path: ^internal/testdb/testdb_test\.go$ + linters: + - forbidigo formatters: enable: From 10b369511a8e72ccfec199d59ede903d17f6f68a Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 12:15:26 +0200 Subject: [PATCH 4/9] chore(ci): run the SQLite tests in their own job under the race detector --- .github/workflows/ci.yml | 16 ++++++ Makefile | 18 ++++++- codecov.yml | 5 ++ test/makefile/coversqlite_test.go | 86 +++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 test/makefile/coversqlite_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b020465..d1bc4e74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,22 @@ jobs: fail_ci_if_error: true version: v11.3.0 + test-sqlite: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - run: make cover-sqlite GOTESTFLAGS=-race + - uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 + with: + files: .covdata/sqlite/cover.out + flags: sqlite + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true + version: v11.3.0 + frontend: runs-on: ubuntu-latest defaults: diff --git a/Makefile b/Makefile index 1b66a166..2e2381be 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ -.PHONY: peers dev seed test test-race cover cover-html mutate lint fmt generate outdated db-up db-down pot catalogs \ +.PHONY: peers dev seed test test-race cover cover-sqlite cover-html mutate lint fmt generate outdated db-up db-down pot catalogs \ translations \ translations-push translations-retire \ e2e e2e-build e2e-theme e2e-serve e2e-db-reset e2e-seed e2e-reset bump bump-kit sdk-compat \ brick-link brick-sync brick-pack brick-unlink -COVERPKGS = $(shell go list ./... | grep -v -e /internal/postgres/db -e /internal/testdb -e /internal/content/contenttest -e /internal/media/mediatest) +COVERPKGS = $(shell go list ./... | grep -v -e /internal/postgres/db -e /internal/sqlite -e /internal/testdb -e /internal/content/contenttest -e /internal/media/mediatest) +SQLITE_COVERPKGS = $(shell go list ./internal/sqlite/... | grep -v -e /internal/sqlite/db) # Where a linked package's restore point is kept while it is linked. BRICK_HELD = .brick-held @@ -144,6 +145,19 @@ cover: cover-html: cover go tool cover -html=$(COVERDATA)/cover.out +SQLITE_COVERDATA = $(COVERDATA)/sqlite + +cover-sqlite: + rm -rf $(SQLITE_COVERDATA) + mkdir -p $(SQLITE_COVERDATA)/counters + go test -cover -covermode=atomic $(GOTESTFLAGS) $(SQLITE_COVERPKGS) \ + -args -test.gocoverdir=$(CURDIR)/$(SQLITE_COVERDATA)/counters + @echo "=== SQLite coverage ===" + go tool covdata percent -i=$(SQLITE_COVERDATA)/counters + @echo + @go tool covdata textfmt -i=$(SQLITE_COVERDATA)/counters -o $(SQLITE_COVERDATA)/cover.out + @go tool cover -func=$(SQLITE_COVERDATA)/cover.out | tail -1 + MUTATE_PKG ?= . MUTATE_FLAGS ?= MUTATE_SDK_FLAGS ?= diff --git a/codecov.yml b/codecov.yml index f0ff15aa..3b7bf213 100644 --- a/codecov.yml +++ b/codecov.yml @@ -12,12 +12,17 @@ coverage: flags: - kit target: 100% + sqlite: + flags: + - sqlite + target: 100% patch: default: target: 100% ignore: - internal/postgres/db + - internal/sqlite/db - internal/testdb - internal/content/contenttest - internal/media/mediatest diff --git a/test/makefile/coversqlite_test.go b/test/makefile/coversqlite_test.go new file mode 100644 index 00000000..68ffd3d1 --- /dev/null +++ b/test/makefile/coversqlite_test.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 + +package makefile_test + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// sqliteRepo returns a module repo that also holds a SQLite package whose test passes or fails, and its generated code. +func sqliteRepo(t *testing.T, sqlitePasses bool) string { + t.Helper() + root := moduleRepo(t, true, false) + verdict := `t.Error("the sqlite package ran and failed")` + if sqlitePasses { + verdict = `t.Log("the sqlite package ran")` + } + sqliteTest := "package sqlite\n\nimport \"testing\"\n\nfunc TestOpen(t *testing.T) {\n\tOpen()\n\t" + verdict + "\n}\n" + files := map[string]string{ + "internal/sqlite/sqlite.go": "package sqlite\n\nfunc Open() bool { return true }\n", + "internal/sqlite/sqlite_test.go": sqliteTest, + "internal/sqlite/db/db.go": "package db\n\nfunc Query() bool { return true }\n", + } + for path, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(root, path)), 0o755); err != nil { + t.Fatalf("making the folder of %s: %v", path, err) + } + if err := os.WriteFile(filepath.Join(root, path), []byte(body), 0o644); err != nil { + t.Fatalf("writing %s: %v", path, err) + } + } + return root +} + +// profileOf runs make with the target in the directory and returns what it printed and the profile it wrote. +func profileOf(t *testing.T, root, target string, profile ...string) (string, string, error) { + t.Helper() + out, err := makeIn(t, root, false, target) + written, readErr := os.ReadFile(filepath.Join(append([]string{root}, profile...)...)) + if err == nil { + err = readErr + } + return out, string(written), err +} + +func TestMakeCoverSQLiteMeasuresOnlyTheSQLitePackages(t *testing.T) { + t.Parallel() + + out, profile, err := profileOf(t, sqliteRepo(t, true), "cover-sqlite", ".covdata", "sqlite", "cover.out") + + if err != nil || !strings.Contains(profile, "example.com/site/internal/sqlite/sqlite.go") { + t.Fatalf("make cover-sqlite = %v with output %q, want the SQLite package in its own profile", err, out) + } + for _, left := range []string{ + "example.com/site/internal/sqlite/db", "example.com/site/internal/app", "example.com/site/sdk", + } { + if strings.Contains(profile, left) || strings.Contains(out, left) { + t.Errorf("make cover-sqlite measured %s, want only the SQLite packages outside the generated code", left) + } + } +} + +func TestMakeCoverSQLiteFailsWhenASQLiteTestFails(t *testing.T) { + t.Parallel() + + out, err := makeIn(t, sqliteRepo(t, false), false, "cover-sqlite") + + if err == nil || !strings.Contains(out, "the sqlite package ran and failed") { + t.Errorf("make cover-sqlite = %v with output %q, want it to fail on the SQLite package's failing test", err, out) + } +} + +func TestMakeCoverLeavesTheSQLitePackagesToTheirOwnJob(t *testing.T) { + t.Parallel() + + out, profile, err := profileOf(t, sqliteRepo(t, true), "cover", ".covdata", "cover.out") + + if err != nil || !strings.Contains(profile, "example.com/site/sdk/sdk.go") { + t.Fatalf("make cover = %v with output %q, want the merged profile written", err, out) + } + if strings.Contains(profile, "example.com/site/internal/sqlite/") { + t.Errorf("make cover measured the SQLite packages, want them left to make cover-sqlite") + } +} From 07c8d5d74e41d113cddabe8fb303fbef13f43333 Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 12:15:27 +0200 Subject: [PATCH 5/9] chore(deps): leave the SQLite driver and its libc to dbkit releases --- .github/dependabot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17416c41..04b95e2b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,6 +10,9 @@ updates: directory: / schedule: interval: weekly + ignore: + - dependency-name: "modernc.org/sqlite" + - dependency-name: "modernc.org/libc" - package-ecosystem: npm directory: / versioning-strategy: increase-if-necessary From 544e5d156dfd43c001cfc35be9a106c466aa261d Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 12:15:27 +0200 Subject: [PATCH 6/9] chore(build): keep local SQLite files out of git and the image --- .dockerignore | 4 ++++ .gitignore | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.dockerignore b/.dockerignore index 79bc989e..d630111b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,10 @@ frontend/coverage **/*_test.go .env .env.* +**/*.db +**/*.db-wal +**/*.db-shm +**/*.migrate.lock .covdata cover.out /gophenberg diff --git a/.gitignore b/.gitignore index 0f6b30ea..04923b99 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,10 @@ test/theme/dist/ /reports/ .stryker-tmp/ .dev-media/ +*.db +*.db-wal +*.db-shm +*.migrate.lock node_modules/ .env /pluginwire From ad6a7e9bdda000f47de5786f3110f5cc8efc439f Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 14:33:28 +0200 Subject: [PATCH 7/9] test(sqlite): store a sub field that shares a key with a top-level field --- internal/sqlite/schema_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/sqlite/schema_test.go b/internal/sqlite/schema_test.go index 30e94058..a4bc99ae 100644 --- a/internal/sqlite/schema_test.go +++ b/internal/sqlite/schema_test.go @@ -367,6 +367,20 @@ func TestSchemaRefusesASecondTopLevelFieldWithOneKeyInAGroup(t *testing.T) { } } +func TestSchemaTakesASubFieldThatSharesAKeyWithATopLevelFieldOfItsGroup(t *testing.T) { + t.Parallel() + + db := withRows(t) + inserted(t, db, insertField, "title") + + err := classed(t, db, "INSERT INTO core_content_fields (key, label, kind, group_id, parent_field_id, depth, "+ + "created_at, updated_at) VALUES ('title', 'Title', 'text', 1, 1, 1, 1767225600000000, 1767225600000000)") + + if err != nil { + t.Errorf("a sub field of field 1 keyed title beside a top-level title = %v, want it stored", err) + } +} + func TestSchemaRefusesASecondContentRowOnOnePath(t *testing.T) { t.Parallel() From 77be5d05a2f64103a3d10ab176cf30ead7385f0e Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 14:33:29 +0200 Subject: [PATCH 8/9] test(postgres): compare the text of each index expression with SQLite --- internal/postgres/engine_parity_test.go | 63 +++++++++++++++++++------ 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/internal/postgres/engine_parity_test.go b/internal/postgres/engine_parity_test.go index f1720fa0..28d1a68f 100644 --- a/internal/postgres/engine_parity_test.go +++ b/internal/postgres/engine_parity_test.go @@ -32,7 +32,7 @@ const ( // postgresIndexSlots lists every slot of every index on the core tables in order, each name taking the schema. postgresIndexSlots = `SELECT ns.nspname || '_' || tc.relname, ns.nspname || '_' || ic.relname, i.indisprimary, i.indisunique, i.indnullsnotdistinct, NOT i.indimmediate, k.n <= i.indnkeyatts, - i.indkey[k.n - 1] = 0, + CASE WHEN i.indkey[k.n - 1] = 0 THEN pg_get_indexdef(i.indexrelid, k.n, true) ELSE '' END, coalesce(a.attname::text, ''), coalesce((i.indoption[k.n - 1] & 1) = 1, false), coalesce(pg_get_expr(i.indpred, i.indrelid), '') FROM pg_index AS i @@ -69,7 +69,8 @@ const ( sqliteColumns = `SELECT name, upper(type), "notnull", pk, coalesce(dflt_value, '') FROM pragma_table_info(?) ORDER BY cid` // sqliteIndexSlots lists every key slot of every index of one SQLite table but its primary key, in order. - sqliteIndexSlots = `SELECT il.name, il."unique", coalesce(s.sql, ''), x.cid = -2, coalesce(x.name, ''), x."desc" + sqliteIndexSlots = `SELECT il.name, il."unique", coalesce(s.sql, ''), x.cid = -2, x.seqno, coalesce(x.name, ''), + x."desc" FROM pragma_index_list(?) AS il JOIN pragma_index_xinfo(il.name) AS x ON x."key" = 1 LEFT JOIN sqlite_schema AS s ON s.type = 'index' AND s.name = il.name @@ -239,6 +240,8 @@ var ( } // indexPredicate finds the predicate after the key list of a CREATE INDEX statement. indexPredicate = regexp.MustCompile(`(?s)\)\s+WHERE\s+(.+)$`) + // indexKeyList finds the parenthesis that opens the key list of a CREATE INDEX statement. + indexKeyList = regexp.MustCompile(`(?is)\bON\s+\w+\s*\(`) // checkStart finds the name of every named CHECK constraint in a CREATE TABLE statement and the parenthesis after it. checkStart = regexp.MustCompile(`(?i)\bCONSTRAINT\s+(\w+)\s+CHECK\s*\(`) ) @@ -277,10 +280,10 @@ type indexShape struct { predicate string } -// indexKey is one key slot of an index, a column or an expression, in ascending or descending order. +// indexKey is one key slot of an index, a column or the text of an expression, in ascending or descending order. type indexKey struct { column string - expression bool + expression string descending bool } @@ -409,8 +412,8 @@ func postgresShape(t *testing.T, pool *pgxpool.Pool) schemaShape { // readPostgresIndexes adds every index of the core tables on pool to shape, the primary key as its columns. func readPostgresIndexes(t *testing.T, pool *pgxpool.Pool, shape schemaShape) { t.Helper() - var table, name, column, predicate string - var primary, unique, nullsNotDistinct, deferrable, isKey, expression, descending bool + var table, name, expression, column, predicate string + var primary, unique, nullsNotDistinct, deferrable, isKey, descending bool dest := []any{ &table, &name, &primary, &unique, &nullsNotDistinct, &deferrable, &isKey, &expression, &column, &descending, &predicate, @@ -492,16 +495,46 @@ func readSQLiteColumns(t *testing.T, db *sql.DB, table, statement string, owner func readSQLiteIndexes(t *testing.T, db *sql.DB, table string, owner *tableShape) { t.Helper() var name, statement, column string - var unique, expression, descending bool - dest := []any{&name, &unique, &statement, &expression, &column, &descending} + var unique, isExpression, descending bool + var position int + dest := []any{&name, &unique, &statement, &isExpression, &position, &column, &descending} eachSQLiteRow(t, db, sqliteIndexSlots, []any{table}, dest, func() { index := owner.indexes[name] index.unique, index.predicate = unique, predicateOf(statement) + expression := "" + if isExpression { + expression = expressionAt(statement, position) + } index.keys = append(index.keys, slot(column, expression, descending)) owner.indexes[name] = index }) } +// expressionAt returns the key slot at position of a CREATE INDEX statement, its spaces folded and its order dropped. +func expressionAt(statement string, position int) string { + start := indexKeyList.FindStringIndex(statement) + item := topLevelItems(enclosed(statement[start[1]-1:]))[position] + return strings.TrimSuffix(strings.TrimSuffix(item, " DESC"), " ASC") +} + +// topLevelItems returns the parts of text between the commas that stand outside parentheses, each trimmed. +func topLevelItems(text string) []string { + var items []string + depth, from := 0, 0 + for i, r := range text { + switch { + case r == '(': + depth++ + case r == ')': + depth-- + case r == ',' && depth == 0: + items = append(items, strings.TrimSpace(text[from:i])) + from = i + 1 + } + } + return append(items, strings.TrimSpace(text[from:])) +} + // readSQLiteForeignKeys adds every foreign key of the named table on db to owner, keyed by its columns. func readSQLiteForeignKeys(t *testing.T, db *sql.DB, table string, owner *tableShape) { t.Helper() @@ -576,10 +609,10 @@ func addColumn(owner *tableShape, name string, column columnShape) { owner.columns[name] = column } -// slot returns the index key slot over column, or over an expression when expression is set. -func slot(column string, expression, descending bool) indexKey { - if expression { - return indexKey{expression: true, descending: descending} +// slot returns the index key slot over column, or over the text of expression when it is not empty. +func slot(column, expression string, descending bool) indexKey { + if expression != "" { + return indexKey{expression: expression, descending: descending} } return indexKey{column: column, descending: descending} } @@ -671,7 +704,7 @@ func coalescedParent(index indexShape) (indexShape, bool) { return index, true } keys := slices.Clone(index.keys) - keys[1] = indexKey{expression: true} + keys[1] = indexKey{expression: "coalesce(parent_field_id, 0)"} index.keys, index.nullsNotDistinct = keys, false return index, true } @@ -725,8 +758,8 @@ func (i indexShape) String() string { // String returns the key slot as a failure reads it, a column name or an expression, with DESC when it descends. func (k indexKey) String() string { text := k.column - if k.expression { - text = "an expression" + if k.expression != "" { + text = k.expression } if k.descending { text += " DESC" From 33331d32c7316c7df1b163a6812eba0c92779648 Mon Sep 17 00:00:00 2001 From: SirLouen Date: Tue, 13 Oct 2026 14:33:29 +0200 Subject: [PATCH 9/9] chore(ci): limit the workflow token to read access --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1bc4e74..c6c2745e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest