diff --git a/alembic/versions/af0fea8d6317_add_owned_bookmark_sync.py b/alembic/versions/af0fea8d6317_add_owned_bookmark_sync.py new file mode 100644 index 0000000..4adb77e --- /dev/null +++ b/alembic/versions/af0fea8d6317_add_owned_bookmark_sync.py @@ -0,0 +1,48 @@ +"""add owned bookmark sync + +Revision ID: af0fea8d6317 +Revises: dcd3b384e6a4 +Create Date: 2026-09-06 00:15:06.254846 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "af0fea8d6317" +down_revision: str | Sequence[str] | None = "dcd3b384e6a4" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add owned bookmarks without changing existing library data.""" + op.create_table( + "bookmarks", + sa.Column("bookmark_id", sa.Uuid(), nullable=False), + sa.Column("book_id", sa.Uuid(), nullable=False), + sa.Column("position", sa.Float(), server_default="0", nullable=False), + sa.Column("page_number", sa.Integer(), nullable=True), + sa.Column("chapter_title", sa.Text(), nullable=True), + sa.Column("note", sa.Text(), nullable=True), + sa.Column("color_hex", sa.Text(), server_default="#FF5722", nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.CheckConstraint("position >= 0 AND position <= 1", name="ck_bookmarks_position_range"), + sa.ForeignKeyConstraint(["book_id"], ["books.book_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("bookmark_id"), + ) + op.create_index(op.f("ix_bookmarks_book_id"), "bookmarks", ["book_id"], unique=False) + op.create_index(op.f("ix_bookmarks_owner_user_id"), "bookmarks", ["owner_user_id"], unique=False) + + +def downgrade() -> None: + """Remove bookmark storage; export bookmarks before downgrading.""" + op.drop_index(op.f("ix_bookmarks_owner_user_id"), table_name="bookmarks") + op.drop_index(op.f("ix_bookmarks_book_id"), table_name="bookmarks") + op.drop_table("bookmarks") diff --git a/alembic/versions/dcd3b384e6a4_add_owned_library_sync_and_promoted_.py b/alembic/versions/dcd3b384e6a4_add_owned_library_sync_and_promoted_.py new file mode 100644 index 0000000..aa0721c --- /dev/null +++ b/alembic/versions/dcd3b384e6a4_add_owned_library_sync_and_promoted_.py @@ -0,0 +1,227 @@ +"""add owned library sync and promoted book metadata + +Revision ID: dcd3b384e6a4 +Revises: a0c1456470b0 +Create Date: 2026-09-05 22:48:08.042605 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +revision: str = "dcd3b384e6a4" +down_revision: str | Sequence[str] | None = "a0c1456470b0" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "shelves", + sa.Column("shelf_id", sa.Uuid(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("color_hex", sa.Text(), nullable=True), + sa.Column("icon_code_point", sa.Integer(), nullable=True), + sa.Column("icon_font_family", sa.Text(), nullable=True), + sa.Column("icon_font_package", sa.Text(), nullable=True), + sa.Column("icon_match_text_direction", sa.Boolean(), server_default="false", nullable=False), + sa.Column("parent_shelf_id", sa.Uuid(), nullable=True), + sa.Column("is_smart", sa.Boolean(), server_default="false", nullable=False), + sa.Column("smart_query", sa.Text(), nullable=True), + sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["parent_shelf_id"], ["shelves.shelf_id"], ondelete="SET NULL"), + sa.PrimaryKeyConstraint("shelf_id"), + ) + op.create_index(op.f("ix_shelves_owner_user_id"), "shelves", ["owner_user_id"], unique=False) + op.create_table( + "sync_tombstones", + sa.Column("table_name", sa.Text(), nullable=False), + sa.Column("entity_id", sa.Uuid(), nullable=False), + sa.Column("deleted_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("table_name", "entity_id"), + ) + op.create_index(op.f("ix_sync_tombstones_owner_user_id"), "sync_tombstones", ["owner_user_id"], unique=False) + op.create_table( + "tags", + sa.Column("tag_id", sa.Uuid(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("color_hex", sa.Text(), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("tag_id"), + ) + op.create_index(op.f("ix_tags_owner_user_id"), "tags", ["owner_user_id"], unique=False) + op.create_table( + "annotations", + sa.Column("annotation_id", sa.Uuid(), nullable=False), + sa.Column("book_id", sa.Uuid(), nullable=False), + sa.Column("selected_text", sa.Text(), nullable=False), + sa.Column("color", sa.Text(), server_default="yellow", nullable=False), + sa.Column("location", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("note", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(["book_id"], ["books.book_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("annotation_id"), + ) + op.create_index(op.f("ix_annotations_book_id"), "annotations", ["book_id"], unique=False) + op.create_index(op.f("ix_annotations_owner_user_id"), "annotations", ["owner_user_id"], unique=False) + op.create_table( + "book_shelves", + sa.Column("id", sa.Text(), nullable=False), + sa.Column("book_id", sa.Uuid(), nullable=False), + sa.Column("shelf_id", sa.Uuid(), nullable=False), + sa.Column("added_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("sort_order", sa.Integer(), server_default="0", nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.CheckConstraint("id = book_id::text || ':' || shelf_id::text", name="ck_book_shelves_pair_id"), + sa.ForeignKeyConstraint(["book_id"], ["books.book_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["shelf_id"], ["shelves.shelf_id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("book_id", "shelf_id"), + ) + op.create_index(op.f("ix_book_shelves_book_id"), "book_shelves", ["book_id"], unique=False) + op.create_index(op.f("ix_book_shelves_owner_user_id"), "book_shelves", ["owner_user_id"], unique=False) + op.create_index(op.f("ix_book_shelves_shelf_id"), "book_shelves", ["shelf_id"], unique=False) + op.create_table( + "book_tags", + sa.Column("id", sa.Text(), nullable=False), + sa.Column("book_id", sa.Uuid(), nullable=False), + sa.Column("tag_id", sa.Uuid(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.CheckConstraint("id = book_id::text || ':' || tag_id::text", name="ck_book_tags_pair_id"), + sa.ForeignKeyConstraint(["book_id"], ["books.book_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["tag_id"], ["tags.tag_id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("book_id", "tag_id"), + ) + op.create_index(op.f("ix_book_tags_book_id"), "book_tags", ["book_id"], unique=False) + op.create_index(op.f("ix_book_tags_owner_user_id"), "book_tags", ["owner_user_id"], unique=False) + op.create_index(op.f("ix_book_tags_tag_id"), "book_tags", ["tag_id"], unique=False) + op.create_table( + "notes", + sa.Column("note_id", sa.Uuid(), nullable=False), + sa.Column("book_id", sa.Uuid(), nullable=False), + sa.Column("title", sa.Text(), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("location", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("tags", postgresql.JSONB(astext_type=sa.Text()), server_default="[]", nullable=False), + sa.Column("is_pinned", sa.Boolean(), server_default="false", nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("owner_user_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint(["book_id"], ["books.book_id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["owner_user_id"], ["users.user_id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("note_id"), + ) + op.create_index(op.f("ix_notes_book_id"), "notes", ["book_id"], unique=False) + op.create_index(op.f("ix_notes_owner_user_id"), "notes", ["owner_user_id"], unique=False) + op.add_column("books", sa.Column("publication_date", sa.DateTime(timezone=True), nullable=True)) + op.add_column("books", sa.Column("file_format", sa.Text(), nullable=True)) + op.add_column("books", sa.Column("file_size", sa.BigInteger(), nullable=True)) + op.add_column("books", sa.Column("file_hash", sa.Text(), nullable=True)) + op.add_column("books", sa.Column("is_physical", sa.Boolean(), server_default="false", nullable=False)) + op.add_column("books", sa.Column("physical_location", sa.Text(), nullable=True)) + op.add_column("books", sa.Column("lent_to", sa.Text(), nullable=True)) + op.add_column("books", sa.Column("lent_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("books", sa.Column("series_id", sa.Text(), nullable=True)) + op.add_column("books", sa.Column("series_name", sa.Text(), nullable=True)) + op.add_column("books", sa.Column("series_number", sa.Float(), nullable=True)) + op.add_column("books", sa.Column("started_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("books", sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("books", sa.Column("last_read_at", sa.DateTime(timezone=True), nullable=True)) + + _backfill_book_metadata() + + +def _backfill_book_metadata() -> None: + """Promote valid legacy values without rewriting the original envelope. + + PostgreSQL 17 is the deployment baseline. Invalid historical values stay + available in custom_metadata instead of preventing the migration. Naive + timestamps were emitted by older clients and are interpreted as UTC. + """ + op.execute("SET LOCAL TIME ZONE 'UTC'") + types = { + "publication_date": "timestamptz", + "file_format": "text", + "file_size": "bigint", + "file_hash": "text", + "is_physical": "boolean", + "physical_location": "text", + "lent_to": "text", + "lent_at": "timestamptz", + "series_id": "text", + "series_name": "text", + "series_number": "double precision", + "started_at": "timestamptz", + "completed_at": "timestamptz", + "last_read_at": "timestamptz", + } + + for field, sql_type in types.items(): + value = f"custom_metadata ->> '{field}'" + condition = ( + f"jsonb_typeof(custom_metadata -> '{field}') = 'string'" + if sql_type == "text" + else f"pg_input_is_valid({value}, '{sql_type}')" + ) + op.execute(f"UPDATE books SET {field} = ({value})::{sql_type} WHERE {value} IS NOT NULL AND {condition}") + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_column("books", "last_read_at") + op.drop_column("books", "completed_at") + op.drop_column("books", "started_at") + op.drop_column("books", "series_number") + op.drop_column("books", "series_name") + op.drop_column("books", "series_id") + op.drop_column("books", "lent_at") + op.drop_column("books", "lent_to") + op.drop_column("books", "physical_location") + op.drop_column("books", "is_physical") + op.drop_column("books", "file_hash") + op.drop_column("books", "file_size") + op.drop_column("books", "file_format") + op.drop_column("books", "publication_date") + op.drop_index(op.f("ix_notes_owner_user_id"), table_name="notes") + op.drop_index(op.f("ix_notes_book_id"), table_name="notes") + op.drop_table("notes") + op.drop_index(op.f("ix_book_tags_tag_id"), table_name="book_tags") + op.drop_index(op.f("ix_book_tags_owner_user_id"), table_name="book_tags") + op.drop_index(op.f("ix_book_tags_book_id"), table_name="book_tags") + op.drop_table("book_tags") + op.drop_index(op.f("ix_book_shelves_shelf_id"), table_name="book_shelves") + op.drop_index(op.f("ix_book_shelves_owner_user_id"), table_name="book_shelves") + op.drop_index(op.f("ix_book_shelves_book_id"), table_name="book_shelves") + op.drop_table("book_shelves") + op.drop_index(op.f("ix_annotations_owner_user_id"), table_name="annotations") + op.drop_index(op.f("ix_annotations_book_id"), table_name="annotations") + op.drop_table("annotations") + op.drop_index(op.f("ix_tags_owner_user_id"), table_name="tags") + op.drop_table("tags") + op.drop_index(op.f("ix_sync_tombstones_owner_user_id"), table_name="sync_tombstones") + op.drop_table("sync_tombstones") + op.drop_index(op.f("ix_shelves_owner_user_id"), table_name="shelves") + op.drop_table("shelves") diff --git a/docs/powersync-sandbox.md b/docs/powersync-sandbox.md index 39f0acf..0425e74 100644 --- a/docs/powersync-sandbox.md +++ b/docs/powersync-sandbox.md @@ -75,3 +75,136 @@ npm --prefix frontend/dev-pages run dev ``` Clear browser storage for `http://localhost:8080/__dev/powersync-sandbox`. + +## Library sync + +The production Flutter library uses `POST /v1/sync/powersync-upload`. Its automatic, +owner-filtered streams contain `books`, `shelves`, `tags`, `notes`, `annotations`, `bookmarks`, +`book_shelves`, and `book_tags`. The demo stream and sandbox remain available. +The other library REST routers are placeholders; use PowerSync uploads for these +persisted domains. + +Apply revision `af0fea8d6317` before starting an API or PowerSync version that uses +these streams, then refresh the source publication and restart PowerSync: + +```bash +uv run alembic upgrade head +./scripts/setup_local_powersync.sh +docker compose restart powersync +``` + +Roll out the server migration and upload handlers first, refresh the publication, +then activate the expanded sync configuration before updating the client. A +running container alone does not prove replication is ready. Inspect startup +logs for activation and subsequent checkpoint progress: + +```bash +docker compose logs --since 5m powersync +``` + +Confirm the new sync configuration becomes active, then make a disposable test +write and confirm a later replication checkpoint and delivery to another client. +`PSYNC_S2302: No sync config available` indicates that configuration has not +become active; inspect replication/configuration errors before changing keys. +This upgrade requires no key regeneration or database reset. The reset procedure +above is destructive and is only for intentionally discarding a local sandbox. + +The additive migration promotes legacy book metadata to columns and leaves the +original `custom_metadata` envelope intact. Invalid historical field values stay +in that envelope and do not abort the backfill. PostgreSQL 17 is the supported +local baseline. Downgrading removes the new library tables, tombstones, and +promoted columns; back up library data before a downgrade. + +Each upload retains the existing transaction envelope: + +```json +{ + "batch": [ + { + "type": "shelves", + "op": "PUT", + "id": "00c7dcac-8fc2-40d7-8558-9b5c55b20f25", + "data": {"name": "Reading", "sort_order": 0} + } + ] +} +``` + +Entity IDs are UUIDs. Membership IDs are canonical lowercase +`:` or `:` strings; reference fields must +match that pair. Book-shelf memberships carry `added_at` and `sort_order`; +book-tag memberships carry `created_at`. Membership removal is a hard delete, +and adding the pair again is supported. + +Payloads use snake_case. Shelf icons carry `icon_code_point`, +`icon_font_family`, `icon_font_package`, and `icon_match_text_direction`. +Notes and annotations use a `location` object containing `page_number`, optional +`chapter`, optional `chapter_title`, and optional `percentage`. A note location +can be null. Annotation colors are `yellow`, `green`, `blue`, `pink`, `purple`, +and `orange`. Note tags remain a list of free-text strings. + +Bookmarks use UUID `bookmark_id` source keys and reference an owned `book_id`. +Their `position` is a number between 0 and 1, defaulting to 0. The optional +`page_number`, `chapter_title`, and `note` fields can be cleared with null. +`color_hex` defaults to `#FF5722`; `created_at` is preserved from the client, +and `updated_at` is controlled by the server. Bookmark deletions leave durable +tombstones, including bookmarks removed by book deletion. Revision `af0fea8d6317` +adds this table without modifying existing library records. Its downgrade +removes bookmark data, so export bookmarks before downgrading. + +Book columns additionally contain `publication_date`, `file_format`, `file_size`, +`file_hash`, `is_physical`, `physical_location`, `lent_to`, `lent_at`, `series_id`, +`series_name`, `series_number`, `started_at`, `completed_at`, and `last_read_at`. +`series_id` is a text descriptor, not a foreign key. New clients retain user +metadata as `{"custom_metadata": {"custom_metadata": {"key": "value"}}}` in upload +data. Legacy queued envelopes are also accepted, with explicitly supplied +promoted fields taking precedence. Local file paths are not part of the contract; +media references use the existing media upload and download endpoints. + +The server derives ownership from authentication and controls `updated_at`. +Timestamps are stored as UTC instants; uploads accept ISO 8601 strings and treat +legacy timestamps without an offset as UTC. PUT upserts only supplied fields; +PATCH updates only supplied fields and accepts explicit null for nullable fields. +A PATCH for an absent entity is acknowledged without creating a placeholder. +Unknown fields, invalid values, missing live references, foreign references, and +shelf cycles reject the transaction; all earlier mutations in that batch roll +back. Transactions for one owner serialize to preserve unrelated concurrent +field changes. + +Entity deletion wins over stale offline writes through durable, server-only +`sync_tombstones`. Deleting a book removes its notes, annotations, bookmarks, memberships, +and existing media; physical files are removed only after commit. Deleting a +shelf reparents its immediate children to the root and removes its memberships. +Deleting a tag removes its memberships. Delayed entity writes and writes with +a tombstoned parent are acknowledged as no-ops so a device can drain its queue. +Do not purge tombstones while offline clients may still upload old changes. + +For two-client library validation, use the same account in two independent +Flutter browser profiles. Create a shelf, tag, note, annotation, bookmark, and memberships +on one client, and confirm all appear on the other. Disconnect one client, edit +an unrelated field on each client, reconnect, and verify both changes survive. +Repeat with deletion on the connected client and a stale edit on the offline +client; the deleted entity must stay absent. A separate account must never see +or modify the first account's library. + +The automated live check creates two independent native PowerSync databases for +one disposable account and a third database for another account. It exercises +all synchronized domains, queued writes across restart, different-field merges, +server-order conflicts, null clearing, and deletion against stale offline edits. +It removes its domain records and disables its disposable accounts afterward. +With the local API on port 8080 and PowerSync running, execute from `client/app/`: + +```bash +PAPYRUS_LIVE_SYNC=1 flutter test test/powersync/library_live_sync_test.dart --reporter expanded +``` + +The test also covers physical-book bookmarks and favorites across an offline +restart. With the default upload rate limit, the extra transactions may take +over a minute to drain; HTTP 429 responses are retried with the queue preserved. + +Client schema expansion preserves existing book databases and queued uploads. +A one-time local migration promotes only compatible legacy metadata values; +explicit column nulls remain cleared. Guest tables are local-only, and switching +account or server invalidates the old repository handles and clears library +views before loading the selected database. Previously memory-only shelves, +topics, notes, annotations, and bookmarks are not automatically assigned to any account. diff --git a/papyrus/models/__init__.py b/papyrus/models/__init__.py index a94957f..843fe4f 100644 --- a/papyrus/models/__init__.py +++ b/papyrus/models/__init__.py @@ -1,12 +1,30 @@ from papyrus.core.database import Base from papyrus.models.acquisition import AcquisitionEndpoint, AcquisitionJob, AcquisitionRule from papyrus.models.auth import AuthExchangeCode, AuthSession, EmailActionToken, PasswordCredential, UserIdentity +from papyrus.models.library import ( + SyncAnnotation, + SyncBookmark, + SyncBookShelf, + SyncBookTag, + SyncNote, + SyncShelf, + SyncTag, + SyncTombstone, +) from papyrus.models.media import MediaAsset from papyrus.models.powersync_demo import PowerSyncDemoItem from papyrus.models.sync import SyncBook from papyrus.models.user import User __all__ = [ + "SyncShelf", + "SyncTag", + "SyncNote", + "SyncAnnotation", + "SyncBookmark", + "SyncBookShelf", + "SyncBookTag", + "SyncTombstone", "AcquisitionEndpoint", "AcquisitionJob", "AcquisitionRule", diff --git a/papyrus/models/library.py b/papyrus/models/library.py new file mode 100644 index 0000000..890f362 --- /dev/null +++ b/papyrus/models/library.py @@ -0,0 +1,127 @@ +"""Owned library rows and durable entity deletion markers.""" + +from datetime import datetime +from uuid import UUID, uuid4 + +from sqlalchemy import ( + Boolean, + CheckConstraint, + DateTime, + Float, + ForeignKey, + Integer, + Text, + UniqueConstraint, + Uuid, + func, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column + +from papyrus.core.database import Base + + +class OwnedLibraryRow: + owner_user_id: Mapped[UUID] = mapped_column(ForeignKey("users.user_id", ondelete="CASCADE"), index=True) + + +class LibraryEntity(OwnedLibraryRow): + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +class SyncShelf(LibraryEntity, Base): + __tablename__ = "shelves" + + shelf_id: Mapped[UUID] = mapped_column(Uuid, primary_key=True, default=uuid4) + name: Mapped[str] = mapped_column(Text) + description: Mapped[str | None] = mapped_column(Text) + color_hex: Mapped[str | None] = mapped_column(Text) + icon_code_point: Mapped[int | None] = mapped_column(Integer) + icon_font_family: Mapped[str | None] = mapped_column(Text) + icon_font_package: Mapped[str | None] = mapped_column(Text) + icon_match_text_direction: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + parent_shelf_id: Mapped[UUID | None] = mapped_column(ForeignKey("shelves.shelf_id", ondelete="SET NULL")) + is_smart: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + smart_query: Mapped[str | None] = mapped_column(Text) + sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class SyncTag(LibraryEntity, Base): + __tablename__ = "tags" + + tag_id: Mapped[UUID] = mapped_column(Uuid, primary_key=True, default=uuid4) + name: Mapped[str] = mapped_column(Text) + color_hex: Mapped[str] = mapped_column(Text) + description: Mapped[str | None] = mapped_column(Text) + + +class SyncNote(LibraryEntity, Base): + __tablename__ = "notes" + + note_id: Mapped[UUID] = mapped_column(Uuid, primary_key=True, default=uuid4) + book_id: Mapped[UUID] = mapped_column(ForeignKey("books.book_id", ondelete="CASCADE"), index=True) + title: Mapped[str] = mapped_column(Text) + content: Mapped[str] = mapped_column(Text) + location: Mapped[dict[str, object] | None] = mapped_column(JSONB) + tags: Mapped[list[str]] = mapped_column(JSONB, default=list, server_default="[]") + is_pinned: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") + + +class SyncAnnotation(LibraryEntity, Base): + __tablename__ = "annotations" + + annotation_id: Mapped[UUID] = mapped_column(Uuid, primary_key=True, default=uuid4) + book_id: Mapped[UUID] = mapped_column(ForeignKey("books.book_id", ondelete="CASCADE"), index=True) + selected_text: Mapped[str] = mapped_column(Text) + color: Mapped[str] = mapped_column(Text, default="yellow", server_default="yellow") + location: Mapped[dict[str, object]] = mapped_column(JSONB) + note: Mapped[str | None] = mapped_column(Text) + + +class SyncBookmark(LibraryEntity, Base): + __tablename__ = "bookmarks" + __table_args__ = (CheckConstraint("position >= 0 AND position <= 1", name="ck_bookmarks_position_range"),) + + bookmark_id: Mapped[UUID] = mapped_column(Uuid, primary_key=True, default=uuid4) + book_id: Mapped[UUID] = mapped_column(ForeignKey("books.book_id", ondelete="CASCADE"), index=True) + position: Mapped[float] = mapped_column(Float, default=0.0, server_default="0") + page_number: Mapped[int | None] = mapped_column(Integer) + chapter_title: Mapped[str | None] = mapped_column(Text) + note: Mapped[str | None] = mapped_column(Text) + color_hex: Mapped[str] = mapped_column(Text, default="#FF5722", server_default="#FF5722") + + +class SyncBookShelf(OwnedLibraryRow, Base): + __tablename__ = "book_shelves" + __table_args__ = ( + UniqueConstraint("book_id", "shelf_id"), + CheckConstraint("id = book_id::text || ':' || shelf_id::text", name="ck_book_shelves_pair_id"), + ) + + id: Mapped[str] = mapped_column(Text, primary_key=True) + book_id: Mapped[UUID] = mapped_column(ForeignKey("books.book_id", ondelete="CASCADE"), index=True) + shelf_id: Mapped[UUID] = mapped_column(ForeignKey("shelves.shelf_id", ondelete="CASCADE"), index=True) + added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + + +class SyncBookTag(OwnedLibraryRow, Base): + __tablename__ = "book_tags" + __table_args__ = ( + UniqueConstraint("book_id", "tag_id"), + CheckConstraint("id = book_id::text || ':' || tag_id::text", name="ck_book_tags_pair_id"), + ) + + id: Mapped[str] = mapped_column(Text, primary_key=True) + book_id: Mapped[UUID] = mapped_column(ForeignKey("books.book_id", ondelete="CASCADE"), index=True) + tag_id: Mapped[UUID] = mapped_column(ForeignKey("tags.tag_id", ondelete="CASCADE"), index=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + +class SyncTombstone(OwnedLibraryRow, Base): + __tablename__ = "sync_tombstones" + + table_name: Mapped[str] = mapped_column(Text, primary_key=True) + entity_id: Mapped[UUID] = mapped_column(Uuid, primary_key=True) + deleted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) diff --git a/papyrus/models/sync.py b/papyrus/models/sync.py index 63872f9..b1ce833 100644 --- a/papyrus/models/sync.py +++ b/papyrus/models/sync.py @@ -5,7 +5,7 @@ from datetime import datetime from uuid import UUID, uuid4 -from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Integer, String, Text, Uuid, func +from sqlalchemy import BigInteger, Boolean, DateTime, Float, ForeignKey, Integer, String, Text, Uuid, func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column @@ -43,3 +43,17 @@ class SyncBook(Base): custom_metadata: Mapped[dict[str, object] | None] = mapped_column(JSONB, nullable=True) added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + publication_date: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + file_format: Mapped[str | None] = mapped_column(Text, nullable=True) + file_size: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + file_hash: Mapped[str | None] = mapped_column(Text, nullable=True) + is_physical: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") + physical_location: Mapped[str | None] = mapped_column(Text, nullable=True) + lent_to: Mapped[str | None] = mapped_column(Text, nullable=True) + lent_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + series_id: Mapped[str | None] = mapped_column(Text, nullable=True) + series_name: Mapped[str | None] = mapped_column(Text, nullable=True) + series_number: Mapped[float | None] = mapped_column(Float, nullable=True) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_read_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/papyrus/schemas/sync.py b/papyrus/schemas/sync.py index 295de0d..1f098b6 100644 --- a/papyrus/schemas/sync.py +++ b/papyrus/schemas/sync.py @@ -2,7 +2,7 @@ from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator BOOK_UPLOAD_FIELDS = frozenset( { @@ -33,27 +33,72 @@ ) +BOOK_UPLOAD_FIELDS |= frozenset( + { + "publication_date", + "file_format", + "file_size", + "file_hash", + "is_physical", + "physical_location", + "lent_to", + "lent_at", + "series_id", + "series_name", + "series_number", + "started_at", + "completed_at", + "last_read_at", + } +) +ENTITY_FIELDS = frozenset({"owner_user_id", "created_at", "updated_at"}) +UPLOAD_FIELDS = { + "books": BOOK_UPLOAD_FIELDS, + "shelves": ENTITY_FIELDS + | { + "name", + "description", + "color_hex", + "icon_code_point", + "icon_font_family", + "icon_font_package", + "icon_match_text_direction", + "parent_shelf_id", + "is_smart", + "smart_query", + "sort_order", + }, + "tags": ENTITY_FIELDS | {"name", "color_hex", "description"}, + "notes": ENTITY_FIELDS | {"book_id", "title", "content", "location", "tags", "is_pinned"}, + "annotations": ENTITY_FIELDS | {"book_id", "selected_text", "color", "location", "note"}, + "bookmarks": ENTITY_FIELDS | {"book_id", "position", "page_number", "chapter_title", "note", "color_hex"}, + "book_shelves": {"owner_user_id", "book_id", "shelf_id", "added_at", "sort_order"}, + "book_tags": {"owner_user_id", "book_id", "tag_id", "created_at"}, +} + + class PowerSyncCrudMutation(BaseModel): - """Single books-table mutation uploaded from the PowerSync queue.""" + """One owned library mutation uploaded from the PowerSync queue.""" model_config = ConfigDict(populate_by_name=True) - table: Literal["books"] = Field(alias="type") + table: Literal["books", "shelves", "tags", "notes", "annotations", "bookmarks", "book_shelves", "book_tags"] = ( + Field(alias="type") + ) op: Literal["PUT", "PATCH", "DELETE", "put", "patch", "delete"] id: str op_id: int | None = Field(default=None, alias="op_id") tx_id: int | None = None op_data: dict[str, Any] | None = Field(default=None, alias="data") - @field_validator("op_data") - @classmethod - def reject_unknown_book_fields(cls, value: dict[str, Any] | None) -> dict[str, Any] | None: - if value is None: - return None - unknown = value.keys() - BOOK_UPLOAD_FIELDS + @model_validator(mode="after") + def reject_unknown_fields(self) -> "PowerSyncCrudMutation": + unknown = (self.op_data or {}).keys() - UPLOAD_FIELDS[self.table] + if unknown: - raise ValueError(f"Unsupported book fields: {', '.join(sorted(unknown))}") - return value + raise ValueError(f"Unsupported {self.table} fields: {', '.join(sorted(unknown))}") + + return self class PowerSyncUploadRequest(BaseModel): diff --git a/papyrus/services/library_sync.py b/papyrus/services/library_sync.py new file mode 100644 index 0000000..ef2f5dd --- /dev/null +++ b/papyrus/services/library_sync.py @@ -0,0 +1,261 @@ +"""Ownership, references, and offline deletion semantics for library mutations.""" + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from uuid import UUID + +from sqlalchemy import delete, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from papyrus.core.exceptions import ForbiddenError, ValidationError +from papyrus.models import ( + SyncAnnotation, + SyncBook, + SyncBookmark, + SyncBookShelf, + SyncBookTag, + SyncNote, + SyncShelf, + SyncTag, + SyncTombstone, +) +from papyrus.schemas.sync import PowerSyncCrudMutation +from papyrus.services import media as media_service +from papyrus.services.library_validation import convert_value, normalize_book_payload, uuid_value + +MODELS: dict[str, Any] = { + "books": SyncBook, + "shelves": SyncShelf, + "tags": SyncTag, + "notes": SyncNote, + "annotations": SyncAnnotation, + "bookmarks": SyncBookmark, + "book_shelves": SyncBookShelf, + "book_tags": SyncBookTag, +} +PRIMARY_KEYS = { + "books": "book_id", + "shelves": "shelf_id", + "tags": "tag_id", + "notes": "note_id", + "annotations": "annotation_id", + "bookmarks": "bookmark_id", + "book_shelves": "id", + "book_tags": "id", +} +REFERENCES = {"book_id": "books", "shelf_id": "shelves", "tag_id": "tags", "parent_shelf_id": "shelves"} +MEMBERSHIPS = {"book_shelves", "book_tags"} + + +async def owned_row(session: AsyncSession, user_id: UUID, table: str, row_id: UUID | str) -> Any: + row = await session.get(MODELS[table], row_id) + + if row is not None and row.owner_user_id != user_id: + raise ForbiddenError(f"Cannot access another user's {table} row") + + return row + + +async def tombstoned(session: AsyncSession, user_id: UUID, table: str, row_id: UUID) -> bool: + marker = await session.get(SyncTombstone, (table, row_id)) + + if marker is not None and marker.owner_user_id != user_id: + raise ForbiddenError("Cannot access another user's deleted entity") + + return marker is not None + + +async def mark_deleted(session: AsyncSession, user_id: UUID, table: str, row_id: UUID) -> None: + if not await tombstoned(session, user_id, table, row_id): + session.add(SyncTombstone(table_name=table, entity_id=row_id, owner_user_id=user_id)) + await session.flush() + + +async def delete_entity(session: AsyncSession, user_id: UUID, table: str, row_id: UUID, row: Any) -> list[Path]: + """Record deletion before cascading so delayed entity writes cannot revive it.""" + await mark_deleted(session, user_id, table, row_id) + paths: list[Path] = [] + + if row is None: + return paths + + if table == "books": + for child_table in ("notes", "annotations", "bookmarks"): + model = MODELS[child_table] + result = await session.execute(select(model).where(model.book_id == row_id)) + + for child in result.scalars(): + await mark_deleted(session, user_id, child_table, getattr(child, PRIMARY_KEYS[child_table])) + + paths = await media_service.delete_book_media(session, user_id, row_id) + + if table == "shelves": + await session.execute( + update(SyncShelf) + .where(SyncShelf.parent_shelf_id == row_id) + .values(parent_shelf_id=None, updated_at=datetime.now(UTC)) + ) + + await session.delete(row) + await session.flush() + return paths + + +async def validate_references( + session: AsyncSession, + user_id: UUID, + table: str, + row_id: UUID | str, + values: dict[str, Any], + row: Any, + *, + deleting: bool = False, +) -> bool: + """Check every reference even when another parent has a deletion marker.""" + stale = False + + for field, target in REFERENCES.items(): + if field not in MODELS[table].__table__.columns or field == PRIMARY_KEYS[table]: + continue + + ref = values.get(field, getattr(row, field, None)) + + if ref is None: + continue + + parent = await owned_row(session, user_id, target, ref) + deleted_parent = await tombstoned(session, user_id, target, ref) + stale |= deleted_parent + + if parent is None and not deleted_parent and not deleting: + raise ValidationError(f"{field} was not found") + + if field == "parent_shelf_id" and not deleted_parent and not deleting: + visited = {row_id} + + while parent is not None: + if parent.shelf_id in visited: + raise ValidationError("Shelf hierarchy cannot contain cycles") + + visited.add(parent.shelf_id) + parent = ( + await owned_row(session, user_id, "shelves", parent.parent_shelf_id) + if parent.parent_shelf_id is not None + else None + ) + + return stale + + +def membership_values(table: str, raw_id: str, payload: dict[str, Any]) -> dict[str, Any]: + parts = raw_id.split(":") + + if len(parts) != 2: + raise ValidationError("Membership id must be ':'") + + book_id, target_id = (uuid_value(part, "id") for part in parts) + + if raw_id != f"{book_id}:{target_id}": + raise ValidationError("Membership id must use canonical UUIDs") + + target_key = "shelf_id" if table == "book_shelves" else "tag_id" + pair = {"book_id": book_id, target_key: target_id} + + for key, expected in pair.items(): + if key in payload and uuid_value(payload[key], key) != expected: + raise ValidationError("Membership fields must match its deterministic id") + + return {**payload, **pair} + + +async def apply_library_mutation( + session: AsyncSession, + user_id: UUID, + mutation: PowerSyncCrudMutation, +) -> tuple[int, list[Path]]: + table = mutation.table + model = MODELS[table] + is_membership = table in MEMBERSHIPS + row_id = mutation.id if is_membership else uuid_value(mutation.id, "id") + payload = dict(mutation.op_data or {}) + payload = {key: value for key, value in payload.items() if key not in {"owner_user_id", "updated_at"}} + + if table == "books": + payload = normalize_book_payload(payload) + + if is_membership: + payload = membership_values(table, mutation.id, payload) + + row = await owned_row(session, user_id, table, row_id) + + if not is_membership and await tombstoned(session, user_id, table, uuid_value(row_id, "id")): + return 0, [] + + if mutation.op.upper() == "PATCH" and row is None: + return 0, [] + + if mutation.op.upper() == "DELETE": + if is_membership: + await validate_references(session, user_id, table, row_id, payload, row, deleting=True) + + if row is not None: + await session.execute(delete(model).where(model.id == row_id)) + + return int(row is not None), [] + + paths = await delete_entity(session, user_id, table, uuid_value(row_id, "id"), row) + return int(row is not None), paths + + values = {key: convert_value(model.__table__.columns[key], value) for key, value in payload.items()} + stale_parent = await validate_references(session, user_id, table, row_id, values, row) + + if stale_parent: + return 0, [] + + if table == "bookmarks" and "position" in values and not 0 <= values["position"] <= 1: + raise ValidationError("position must be between 0 and 1") + + if table == "annotations" and values.get("color", "yellow") not in { + "yellow", + "green", + "blue", + "pink", + "purple", + "orange", + }: + raise ValidationError("Unsupported annotation color") + + if table == "books": + for key, kind in (("file_media_id", "book_file"), ("cover_media_id", "cover_image")): + if key in values: + values[key] = await media_service.validate_media_reference( + session, user_id, uuid_value(row_id, "id"), values[key], field_name=key, expected_kind=kind + ) + + if row is None: + if table == "books": + values.setdefault("title", "Untitled Book") + + for column in model.__table__.columns: + if ( + not column.nullable + and not column.primary_key + and column.name != "owner_user_id" + and column.default is None + and column.server_default is None + and column.name not in values + ): + raise ValidationError(f"{column.name} is required") + + row = model(**{PRIMARY_KEYS[table]: row_id, "owner_user_id": user_id}, **values) + session.add(row) + else: + for key, value in values.items(): + setattr(row, key, value) + + if "updated_at" in model.__table__.columns: + row.updated_at = datetime.now(UTC) + + await session.flush() + return 1, [] diff --git a/papyrus/services/library_validation.py b/papyrus/services/library_validation.py new file mode 100644 index 0000000..dc68380 --- /dev/null +++ b/papyrus/services/library_validation.py @@ -0,0 +1,167 @@ +"""Type conversion and backwards compatibility for library queue payloads.""" + +from datetime import UTC, datetime +from math import isfinite +from typing import Any +from uuid import UUID + +from sqlalchemy import BigInteger, Boolean, DateTime, Float, Integer, String, Uuid +from sqlalchemy.dialects.postgresql import JSONB + +from papyrus.core.exceptions import ValidationError +from papyrus.models.sync import SyncBook + +PROMOTED_BOOK_FIELDS = frozenset( + { + "publication_date", + "file_format", + "file_size", + "file_hash", + "is_physical", + "physical_location", + "lent_to", + "lent_at", + "series_id", + "series_name", + "series_number", + "started_at", + "completed_at", + "last_read_at", + } +) + + +def uuid_value(value: object, name: str) -> UUID: + try: + return UUID(str(value)) + except ValueError as exc: + raise ValidationError(f"{name} must be a valid UUID") from exc + + +def normalize_book_payload(payload: dict[str, Any]) -> dict[str, Any]: + """Keep the legacy envelope intact while promoting queued legacy values.""" + envelope = payload.get("custom_metadata") + + if isinstance(envelope, dict): + promoted = {} + + for key, value in envelope.items(): + if key not in PROMOTED_BOOK_FIELDS or key in payload: + continue + + try: + convert_value(SyncBook.__table__.columns[key], value) + except ValidationError: + continue + + promoted[key] = value + + return {**promoted, **payload} + + return payload + + +def finite_number(value: Any) -> bool: + if not isinstance(value, int | float) or isinstance(value, bool): + return False + + try: + return isfinite(value) + except OverflowError: + return False + + +def validate_location(value: dict[str, Any]) -> dict[str, Any]: + if value.keys() - {"chapter", "chapter_title", "page_number", "percentage"}: + raise ValidationError("Unsupported location fields") + + page = value.get("page_number") + + if not isinstance(page, int) or isinstance(page, bool): + raise ValidationError("location.page_number must be an integer") + + chapter = value.get("chapter") + + if chapter is not None and (not isinstance(chapter, int) or isinstance(chapter, bool)): + raise ValidationError("location.chapter must be an integer") + + title = value.get("chapter_title") + + if title is not None and not isinstance(title, str): + raise ValidationError("location.chapter_title must be text") + + percentage = value.get("percentage") + + if percentage is not None and not finite_number(percentage): + raise ValidationError("location.percentage must be a finite number") + + return value + + +def convert_value(column: Any, value: Any) -> Any: + """Validate each present value without manufacturing absent PATCH fields.""" + key = column.name + + if value is None: + if not column.nullable: + raise ValidationError(f"{key} cannot be null") + + return None + + column_type = column.type + + if isinstance(column_type, Uuid): + return uuid_value(value, key) + + if isinstance(column_type, Boolean): + if value in (True, False, 0, 1) and isinstance(value, bool | int): + return bool(value) + + raise ValidationError(f"{key} must be a boolean") + + if isinstance(column_type, Integer | BigInteger): + if not isinstance(value, int) or isinstance(value, bool): + raise ValidationError(f"{key} must be an integer") + + limit = 2**63 if isinstance(column_type, BigInteger) else 2**31 + + if not -limit <= value < limit: + raise ValidationError(f"{key} is out of range") + + return value + + if isinstance(column_type, Float): + if not finite_number(value): + raise ValidationError(f"{key} must be a finite number") + + return float(value) + + if isinstance(column_type, DateTime): + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed.astimezone(UTC) + except (ValueError, OverflowError) as exc: + raise ValidationError(f"{key} must be an ISO datetime") from exc + + if isinstance(column_type, JSONB): + if key in {"tags", "co_authors"}: + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ValidationError(f"{key} must be a list of strings") + + return value + + if not isinstance(value, dict): + raise ValidationError(f"{key} must be an object") + + return validate_location(value) if key == "location" else value + + if isinstance(column_type, String): + if not isinstance(value, str): + raise ValidationError(f"{key} must be text") + + if column_type.length is not None and len(value) > column_type.length: + raise ValidationError(f"{key} is too long") + + return value + + raise ValidationError(f"Unsupported field: {key}") diff --git a/papyrus/services/sync.py b/papyrus/services/sync.py index b327729..87d283c 100644 --- a/papyrus/services/sync.py +++ b/papyrus/services/sync.py @@ -1,169 +1,15 @@ -"""Books-only PowerSync upload service.""" +"""Atomic PowerSync uploads for the user's owned library.""" -from __future__ import annotations - -from datetime import UTC, datetime from pathlib import Path from uuid import UUID +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from papyrus.core.exceptions import ForbiddenError, ValidationError -from papyrus.models import SyncBook +from papyrus.models import User from papyrus.schemas.sync import PowerSyncCrudMutation from papyrus.services import media as media_service - -BOOK_FIELDS = frozenset( - { - "title", - "subtitle", - "author", - "co_authors", - "isbn", - "isbn13", - "publisher", - "language", - "page_count", - "description", - "cover_image_url", - "file_media_id", - "cover_media_id", - "reading_status", - "current_page", - "current_position", - "current_cfi", - "is_favorite", - "rating", - "custom_metadata", - "added_at", - "owner_user_id", - "updated_at", - } -) -SERVER_CONTROLLED_FIELDS = frozenset({"owner_user_id", "updated_at"}) - - -def _now() -> datetime: - return datetime.now(UTC) - - -def _uuid(value: object, field_name: str) -> UUID: - try: - return UUID(str(value)) - except ValueError as exc: - raise ValidationError(f"{field_name} must be a valid UUID") from exc - - -def _validate_payload(payload: dict[str, object]) -> dict[str, object]: - unknown = payload.keys() - BOOK_FIELDS - if unknown: - raise ValidationError(f"Unsupported book fields: {', '.join(sorted(unknown))}") - return {key: value for key, value in payload.items() if key not in SERVER_CONTROLLED_FIELDS} - - -def _optional_text(payload: dict[str, object], key: str, default: str | None = None) -> str | None: - if key not in payload: - return default - value = payload[key] - return None if value is None else str(value) - - -def _optional_uuid(payload: dict[str, object], key: str, default: UUID | None = None) -> UUID | None: - if key not in payload: - return default - value = payload[key] - if value is None: - return None - return _uuid(value, key) - - -def _required_text(payload: dict[str, object], key: str, default: str | None = None) -> str: - value = _optional_text(payload, key, default) - if value is None or not value: - raise ValidationError(f"{key} is required") - return value - - -def _optional_int(payload: dict[str, object], key: str, default: int | None = None) -> int | None: - if key not in payload: - return default - value = payload[key] - if value is None: - return None - if not isinstance(value, int | float | str) or isinstance(value, bool): - raise ValidationError(f"{key} must be an integer") - try: - return int(value) - except (TypeError, ValueError) as exc: - raise ValidationError(f"{key} must be an integer") from exc - - -def _optional_float(payload: dict[str, object], key: str, default: float | None = None) -> float | None: - if key not in payload: - return default - value = payload[key] - if value is None: - return None - if not isinstance(value, int | float | str) or isinstance(value, bool): - raise ValidationError(f"{key} must be a number") - try: - return float(value) - except (TypeError, ValueError) as exc: - raise ValidationError(f"{key} must be a number") from exc - - -def _optional_bool(payload: dict[str, object], key: str, default: bool = False) -> bool: - if key not in payload: - return default - value = payload[key] - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() in {"1", "true", "yes", "on"} - return bool(value) - - -def _optional_datetime(payload: dict[str, object], key: str, default: datetime) -> datetime: - if key not in payload: - return default - value = payload[key] - if isinstance(value, datetime): - return value - try: - return datetime.fromisoformat(str(value).replace("Z", "+00:00")) - except ValueError as exc: - raise ValidationError(f"{key} must be an ISO datetime") from exc - - -def _optional_string_list(payload: dict[str, object], key: str, default: list[str] | None = None) -> list[str] | None: - if key not in payload: - return default - value = payload[key] - if value is None: - return None - if not isinstance(value, list): - raise ValidationError(f"{key} must be a list") - return [str(item) for item in value] - - -def _optional_json_object( - payload: dict[str, object], key: str, default: dict[str, object] | None = None -) -> dict[str, object] | None: - if key not in payload: - return default - value = payload[key] - if value is None: - return None - if not isinstance(value, dict): - raise ValidationError(f"{key} must be an object") - return value - - -async def _get_owned_book(session: AsyncSession, user_id: UUID, book_id: UUID) -> SyncBook | None: - book = await session.get(SyncBook, book_id) - if book is not None and book.owner_user_id != user_id: - raise ForbiddenError("Cannot access another user's book") - return book +from papyrus.services.library_sync import apply_library_mutation async def apply_powersync_upload_batch( @@ -171,85 +17,22 @@ async def apply_powersync_upload_batch( user_id: UUID, batch: list[PowerSyncCrudMutation], ) -> int: - """Apply one PowerSync CRUD transaction and commit it atomically.""" + """Serialize each owner's queue transactions and commit mixed batches atomically.""" applied_count = 0 media_paths_to_delete: list[Path] = [] + try: + await session.execute(select(User.user_id).where(User.user_id == user_id).with_for_update()) + for mutation in batch: - applied, deleted_media_paths = await _apply_book_mutation(session, user_id, mutation) + applied, deleted_media_paths = await apply_library_mutation(session, user_id, mutation) applied_count += applied media_paths_to_delete.extend(deleted_media_paths) + await session.commit() except Exception: await session.rollback() raise + media_service.delete_physical_paths(media_paths_to_delete) return applied_count - - -async def _apply_book_mutation( - session: AsyncSession, - user_id: UUID, - mutation: PowerSyncCrudMutation, -) -> tuple[int, list[Path]]: - book_id = _uuid(mutation.id, "id") - operation = mutation.op.upper() - - if operation == "DELETE": - book = await _get_owned_book(session, user_id, book_id) - if book is None: - return 0, [] - deleted_media_paths = await media_service.delete_book_media(session, user_id, book_id) - await session.delete(book) - return 1, deleted_media_paths - - payload = _validate_payload(mutation.op_data or {}) - book = await _get_owned_book(session, user_id, book_id) - now = _now() - - if book is None: - book = SyncBook( - book_id=book_id, - owner_user_id=user_id, - title=_required_text(payload, "title", "Untitled Book"), - added_at=_optional_datetime(payload, "added_at", now), - updated_at=now, - ) - session.add(book) - - book.title = _required_text(payload, "title", book.title) - book.subtitle = _optional_text(payload, "subtitle", book.subtitle) - book.author = _optional_text(payload, "author", book.author) - book.co_authors = _optional_string_list(payload, "co_authors", book.co_authors) - book.isbn = _optional_text(payload, "isbn", book.isbn) - book.isbn13 = _optional_text(payload, "isbn13", book.isbn13) - book.publisher = _optional_text(payload, "publisher", book.publisher) - book.language = _optional_text(payload, "language", book.language) - book.page_count = _optional_int(payload, "page_count", book.page_count) - book.description = _optional_text(payload, "description", book.description) - book.cover_image_url = _optional_text(payload, "cover_image_url", book.cover_image_url) - book.file_media_id = await media_service.validate_media_reference( - session, - user_id, - book.book_id, - _optional_uuid(payload, "file_media_id", book.file_media_id), - field_name="file_media_id", - expected_kind="book_file", - ) - book.cover_media_id = await media_service.validate_media_reference( - session, - user_id, - book.book_id, - _optional_uuid(payload, "cover_media_id", book.cover_media_id), - field_name="cover_media_id", - expected_kind="cover_image", - ) - book.reading_status = _optional_text(payload, "reading_status", book.reading_status) - book.current_page = _optional_int(payload, "current_page", book.current_page) - book.current_position = _optional_float(payload, "current_position", book.current_position) - book.current_cfi = _optional_text(payload, "current_cfi", book.current_cfi) - book.is_favorite = _optional_bool(payload, "is_favorite", book.is_favorite) - book.rating = _optional_int(payload, "rating", book.rating) - book.custom_metadata = _optional_json_object(payload, "custom_metadata", book.custom_metadata) - book.updated_at = now - return 1, [] diff --git a/powersync/sync-config.yaml b/powersync/sync-config.yaml index 2992e1c..7b53085 100644 --- a/powersync/sync-config.yaml +++ b/powersync/sync-config.yaml @@ -29,10 +29,135 @@ streams: rating, custom_metadata::text AS custom_metadata, added_at::text AS added_at, - updated_at::text AS updated_at + updated_at::text AS updated_at, + publication_date::text AS publication_date, + file_format, + file_size, + file_hash, + is_physical, + physical_location, + lent_to, + lent_at::text AS lent_at, + series_id, + series_name, + series_number, + started_at::text AS started_at, + completed_at::text AS completed_at, + last_read_at::text AS last_read_at FROM books WHERE owner_user_id::text = auth.user_id() + shelves: + auto_subscribe: true + query: | + SELECT + shelf_id AS id, + name, + description, + color_hex, + icon_code_point, + icon_font_family, + icon_font_package, + icon_match_text_direction, + parent_shelf_id::text AS parent_shelf_id, + is_smart, + smart_query, + sort_order, + created_at::text AS created_at, + updated_at::text AS updated_at, + owner_user_id::text AS owner_user_id + FROM shelves + WHERE owner_user_id::text = auth.user_id() + + tags: + auto_subscribe: true + query: | + SELECT + tag_id AS id, + name, + color_hex, + description, + created_at::text AS created_at, + updated_at::text AS updated_at, + owner_user_id::text AS owner_user_id + FROM tags + WHERE owner_user_id::text = auth.user_id() + + notes: + auto_subscribe: true + query: | + SELECT + note_id AS id, + book_id::text AS book_id, + title, + content, + location::text AS location, + tags::text AS tags, + is_pinned, + created_at::text AS created_at, + updated_at::text AS updated_at, + owner_user_id::text AS owner_user_id + FROM notes + WHERE owner_user_id::text = auth.user_id() + + annotations: + auto_subscribe: true + query: | + SELECT + annotation_id AS id, + book_id::text AS book_id, + selected_text, + color, + location::text AS location, + note, + created_at::text AS created_at, + updated_at::text AS updated_at, + owner_user_id::text AS owner_user_id + FROM annotations + WHERE owner_user_id::text = auth.user_id() + + bookmarks: + auto_subscribe: true + query: | + SELECT + bookmark_id AS id, + book_id::text AS book_id, + position, + page_number, + chapter_title, + note, + color_hex, + created_at::text AS created_at, + updated_at::text AS updated_at, + owner_user_id::text AS owner_user_id + FROM bookmarks + WHERE owner_user_id::text = auth.user_id() + + book_shelves: + auto_subscribe: true + query: | + SELECT + id, + book_id::text AS book_id, + shelf_id::text AS shelf_id, + added_at::text AS added_at, + sort_order, + owner_user_id::text AS owner_user_id + FROM book_shelves + WHERE owner_user_id::text = auth.user_id() + + book_tags: + auto_subscribe: true + query: | + SELECT + id, + book_id::text AS book_id, + tag_id::text AS tag_id, + created_at::text AS created_at, + owner_user_id::text AS owner_user_id + FROM book_tags + WHERE owner_user_id::text = auth.user_id() + demo_items: auto_subscribe: true query: | diff --git a/scripts/setup_local_powersync.sh b/scripts/setup_local_powersync.sh index 5a5ae88..7d120c0 100755 --- a/scripts/setup_local_powersync.sh +++ b/scripts/setup_local_powersync.sh @@ -29,16 +29,15 @@ END \$\$; GRANT USAGE ON SCHEMA public TO "${POWERSYNC_SOURCE_ROLE}"; -GRANT SELECT ON TABLE public.books TO "${POWERSYNC_SOURCE_ROLE}"; -GRANT SELECT ON TABLE public.powersync_demo_items TO "${POWERSYNC_SOURCE_ROLE}"; +GRANT SELECT ON TABLE public.books, public.shelves, public.tags, public.notes, public.annotations, public.bookmarks, public.book_shelves, public.book_tags, public.powersync_demo_items TO "${POWERSYNC_SOURCE_ROLE}"; ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO "${POWERSYNC_SOURCE_ROLE}"; DO \$\$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_publication WHERE pubname = 'powersync') THEN - CREATE PUBLICATION powersync FOR TABLE public.books, public.powersync_demo_items; + CREATE PUBLICATION powersync FOR TABLE public.books, public.shelves, public.tags, public.notes, public.annotations, public.bookmarks, public.book_shelves, public.book_tags, public.powersync_demo_items; ELSE - ALTER PUBLICATION powersync SET TABLE public.books, public.powersync_demo_items; + ALTER PUBLICATION powersync SET TABLE public.books, public.shelves, public.tags, public.notes, public.annotations, public.bookmarks, public.book_shelves, public.book_tags, public.powersync_demo_items; END IF; END \$\$; diff --git a/tests/api/routes/test_bookmark_sync.py b/tests/api/routes/test_bookmark_sync.py new file mode 100644 index 0000000..9fccb17 --- /dev/null +++ b/tests/api/routes/test_bookmark_sync.py @@ -0,0 +1,157 @@ +"""Bookmark queue contracts and offline deletion regressions.""" + +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import text + +from papyrus.models import SyncBook, User +from tests.api.routes.test_library_sync import mutation, upload + + +async def test_bookmark_roundtrip_retry_partial_update_and_null_clear(client, auth_headers, auth_user, db_session): + book_id, bookmark_id = uuid4(), uuid4() + created_at = "2025-01-02T03:04:05Z" + bookmark = mutation( + "bookmarks", + bookmark_id, + { + "book_id": str(book_id), + "position": 0.25, + "page_number": 12, + "chapter_title": "Chapter", + "note": "Remember", + "color_hex": "#ABCDEF", + "created_at": created_at, + "owner_user_id": str(uuid4()), + "updated_at": created_at, + }, + ) + response = await upload(client, auth_headers, mutation("books", book_id, {"title": "Book"}), bookmark) + assert response.status_code == 200, response.text + assert (await upload(client, auth_headers, bookmark)).status_code == 200 + response = await upload( + client, + auth_headers, + mutation( + "bookmarks", + bookmark_id, + { + "note": None, + "page_number": None, + "chapter_title": None, + }, + "PATCH", + ), + ) + assert response.status_code == 200, response.text + row = (await db_session.execute(text("SELECT * FROM bookmarks"))).mappings().one() + assert row["position"] == 0.25 + assert row["color_hex"] == "#ABCDEF" + assert row["note"] is row["page_number"] is row["chapter_title"] is None + assert row["owner_user_id"] == UUID(auth_user["user_id"]) + assert row["created_at"] == datetime(2025, 1, 2, 3, 4, 5, tzinfo=UTC) + assert row["updated_at"] > row["created_at"] + + +async def test_bookmark_defaults_and_missing_patch(client, auth_headers, db_session): + book_id, bookmark_id = uuid4(), uuid4() + response = await upload( + client, + auth_headers, + mutation("books", book_id, {"title": "Book"}), + mutation("bookmarks", bookmark_id, {"book_id": str(book_id)}), + mutation("bookmarks", uuid4(), {"note": "Missing"}, "PATCH"), + ) + assert response.status_code == 200, response.text + row = (await db_session.execute(text("SELECT position, color_hex FROM bookmarks"))).one() + assert row == (0.0, "#FF5722") + + +@pytest.mark.parametrize("delete_parent", [False, True]) +async def test_bookmark_deletion_wins_over_stale_writes(client, auth_headers, db_session, delete_parent): + book_id, bookmark_id = uuid4(), uuid4() + bookmark = mutation("bookmarks", bookmark_id, {"book_id": str(book_id), "position": 0.5}) + assert ( + await upload(client, auth_headers, mutation("books", book_id, {"title": "Book"}), bookmark) + ).status_code == 200 + deletion = mutation( + "books" if delete_parent else "bookmarks", book_id if delete_parent else bookmark_id, op="DELETE" + ) + for _ in range(2): + assert (await upload(client, auth_headers, deletion)).status_code == 200 + stale = [bookmark, mutation("bookmarks", bookmark_id, {"note": "Stale"}, "PATCH")] + if delete_parent: + stale.append(mutation("bookmarks", uuid4(), {"book_id": str(book_id), "position": 0.7})) + assert (await upload(client, auth_headers, *stale)).status_code == 200 + assert (await db_session.execute(text("SELECT count(*) FROM bookmarks"))).scalar_one() == 0 + assert ( + await db_session.execute(text("SELECT count(*) FROM sync_tombstones WHERE table_name = 'bookmarks'")) + ).scalar_one() == 1 + + +async def test_bookmark_rejects_foreign_book_and_rolls_back(client, auth_headers, db_session): + owner = User( + display_name="Other", + primary_email="bookmark-owner@example.com", + primary_email_verified=True, + last_login_at=datetime.now(UTC), + ) + db_session.add(owner) + await db_session.flush() + book = SyncBook(book_id=uuid4(), owner_user_id=owner.user_id, title="Foreign") + db_session.add(book) + await db_session.commit() + foreign_id = book.book_id + response = await upload( + client, + auth_headers, + mutation("shelves", uuid4(), {"name": "Rollback"}), + mutation("bookmarks", uuid4(), {"book_id": str(foreign_id), "position": 0.5}), + ) + assert response.status_code == 403, response.text + assert (await db_session.execute(text("SELECT count(*) FROM shelves"))).scalar_one() == 0 + + +async def test_bookmark_rejects_foreign_entity_mutations(client, auth_headers, db_session): + from papyrus.models import SyncBookmark + + owner = User( + display_name="Other", + primary_email="bookmark-entity@example.com", + primary_email_verified=True, + last_login_at=datetime.now(UTC), + ) + db_session.add(owner) + await db_session.flush() + book = SyncBook(book_id=uuid4(), owner_user_id=owner.user_id, title="Foreign") + db_session.add(book) + await db_session.flush() + bookmark = SyncBookmark(bookmark_id=uuid4(), owner_user_id=owner.user_id, book_id=book.book_id) + db_session.add(bookmark) + await db_session.commit() + bookmark_id = bookmark.bookmark_id + for op in ("PUT", "PATCH", "DELETE"): + response = await upload(client, auth_headers, mutation("bookmarks", bookmark_id, {"note": "Forbidden"}, op)) + assert response.status_code == 403, response.text + + +@pytest.mark.parametrize("position", [-0.1, 1.1, None, "0.5", True]) +async def test_bookmark_invalid_position_rolls_back(client, auth_headers, db_session, position): + book_id = uuid4() + response = await upload( + client, + auth_headers, + mutation("books", book_id, {"title": "Rollback"}), + mutation("bookmarks", uuid4(), {"book_id": str(book_id), "position": position}), + ) + assert response.status_code == 400, response.text + assert (await db_session.execute(text("SELECT count(*) FROM books"))).scalar_one() == 0 + + +async def test_bookmark_requires_live_book_and_rejects_unknown_fields(client, auth_headers): + response = await upload(client, auth_headers, mutation("bookmarks", uuid4(), {"book_id": str(uuid4())})) + assert response.status_code == 400 + response = await upload(client, auth_headers, mutation("bookmarks", uuid4(), {"local_path": "/tmp/book"})) + assert response.status_code == 422 diff --git a/tests/api/routes/test_library_sync.py b/tests/api/routes/test_library_sync.py new file mode 100644 index 0000000..8781846 --- /dev/null +++ b/tests/api/routes/test_library_sync.py @@ -0,0 +1,244 @@ +"""Library upload transaction and offline conflict regression tests.""" + +from uuid import uuid4 + +from sqlalchemy import text + + +def mutation(table, row_id, data=None, op="PUT"): + return {"type": table, "op": op, "id": str(row_id), "data": data} + + +async def upload(client, auth_headers, *batch): + return await client.post("/v1/sync/powersync-upload", headers=auth_headers, json={"batch": batch}) + + +async def test_mixed_roundtrip_and_null_patch(client, auth_headers, db_session): + book, shelf, tag, note, annotation = [uuid4() for _ in range(5)] + location = {"chapter": 3, "chapter_title": "Chapter", "page_number": 12, "percentage": 0.4} + batch = [ + mutation("books", book, {"title": "Book", "series_id": "legacy-series", "is_physical": True}), + mutation("shelves", shelf, {"name": "Shelf", "icon_code_point": 123, "icon_font_family": "MaterialIcons"}), + mutation("tags", tag, {"name": "Tag", "color_hex": "#FFFFFF"}), + mutation( + "notes", + note, + { + "book_id": str(book), + "title": "Note", + "content": "Body", + "location": location, + "tags": ["free text"], + "is_pinned": True, + }, + ), + mutation( + "annotations", + annotation, + {"book_id": str(book), "selected_text": "Quote", "location": location, "color": "green", "note": "Comment"}, + ), + mutation("book_shelves", f"{book}:{shelf}", {"book_id": str(book), "shelf_id": str(shelf), "sort_order": 2}), + mutation("book_tags", f"{book}:{tag}", {"book_id": str(book), "tag_id": str(tag)}), + ] + for _ in range(2): + response = await upload(client, auth_headers, *batch) + assert response.status_code == 200, response.text + response = await upload( + client, auth_headers, mutation("notes", note, {"location": None, "content": "Edited"}, "PATCH") + ) + assert response.status_code == 200 + row = (await db_session.execute(text("SELECT title, content, location, tags, is_pinned FROM notes"))).one() + assert row == ("Note", "Edited", None, ["free text"], True) + assert (await db_session.execute(text("SELECT count(*) FROM book_tags"))).scalar_one() == 1 + + +async def test_delete_wins_and_cascades(client, auth_headers, db_session): + book, shelf, note = [uuid4() for _ in range(3)] + response = await upload( + client, + auth_headers, + mutation("books", book, {"title": "Book"}), + mutation("shelves", shelf, {"name": "Shelf"}), + mutation("notes", note, {"book_id": str(book), "title": "Note", "content": "Body"}), + mutation("book_shelves", f"{book}:{shelf}", {"book_id": str(book), "shelf_id": str(shelf)}), + ) + assert response.status_code == 200, response.text + assert (await upload(client, auth_headers, mutation("books", book, op="DELETE"))).status_code == 200 + response = await upload( + client, + auth_headers, + mutation("books", book, {"title": "Stale"}), + mutation("notes", note, {"book_id": str(book), "title": "Stale", "content": "Body"}), + mutation("notes", uuid4(), {"book_id": str(book), "title": "Late", "content": "Body"}), + mutation("book_shelves", f"{book}:{shelf}", {"book_id": str(book), "shelf_id": str(shelf)}), + ) + assert response.status_code == 200, response.text + for table in ("books", "notes", "book_shelves"): + assert (await db_session.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0 + + +async def test_shelf_cycle_rolls_back_and_delete_reparents(client, auth_headers, db_session): + parent, child = uuid4(), uuid4() + assert ( + await upload( + client, + auth_headers, + mutation("shelves", parent, {"name": "Parent"}), + mutation("shelves", child, {"name": "Child", "parent_shelf_id": str(parent)}), + ) + ).status_code == 200 + response = await upload( + client, + auth_headers, + mutation("shelves", child, {"name": "Wrong"}, "PATCH"), + mutation("shelves", parent, {"parent_shelf_id": str(child)}, "PATCH"), + ) + assert response.status_code == 400 + assert ( + await db_session.execute(text("SELECT name FROM shelves WHERE shelf_id = :id"), {"id": child}) + ).scalar_one() == "Child" + await db_session.rollback() + assert (await upload(client, auth_headers, mutation("shelves", parent, op="DELETE"))).status_code == 200 + assert (await db_session.execute(text("SELECT parent_shelf_id FROM shelves"))).scalar_one() is None + + +async def test_membership_remove_readd_and_pair_validation(client, auth_headers): + book, tag = uuid4(), uuid4() + membership = mutation("book_tags", f"{book}:{tag}", {"book_id": str(book), "tag_id": str(tag)}) + assert ( + await upload( + client, + auth_headers, + mutation("books", book, {"title": "Book"}), + mutation("tags", tag, {"name": "Tag", "color_hex": "#FFFFFF"}), + membership, + ) + ).status_code == 200 + assert (await upload(client, auth_headers, mutation("book_tags", membership["id"], op="DELETE"))).status_code == 200 + assert (await upload(client, auth_headers, membership)).status_code == 200 + assert ( + await upload(client, auth_headers, mutation("book_tags", f"{book}:{uuid4()}", membership["data"])) + ).status_code == 400 + + +async def test_legacy_envelope_normalizes_and_preserves_metadata(client, auth_headers, db_session): + envelope = { + "publication_date": "2020-01-01T00:00:00Z", + "series_id": "old-id", + "file_size": 42, + "is_physical": True, + "custom_metadata": {"key": "value"}, + } + response = await upload( + client, + auth_headers, + mutation("books", uuid4(), {"title": "Book", "custom_metadata": envelope, "series_id": "explicit"}), + ) + assert response.status_code == 200, response.text + row = (await db_session.execute(text("SELECT series_id, file_size, is_physical, custom_metadata FROM books"))).one() + assert row == ("explicit", 42, True, envelope) + + +async def test_foreign_reference_rolls_back_mixed_batch(client, auth_headers, db_session): + from datetime import UTC, datetime + + from papyrus.models import SyncBook, User + + other = User( + display_name="Other", + primary_email="other-library@example.com", + primary_email_verified=True, + last_login_at=datetime.now(UTC), + ) + db_session.add(other) + await db_session.flush() + foreign_book = SyncBook(book_id=uuid4(), owner_user_id=other.user_id, title="Foreign") + db_session.add(foreign_book) + await db_session.commit() + for table, data in ( + ("notes", {"book_id": str(foreign_book.book_id), "title": "Note", "content": "Body"}), + ( + "annotations", + {"book_id": str(foreign_book.book_id), "selected_text": "Quote", "location": {"page_number": 1}}, + ), + ): + shelf_id = uuid4() + response = await upload( + client, + auth_headers, + mutation("shelves", shelf_id, {"name": "Must roll back"}), + mutation(table, uuid4(), data), + ) + assert response.status_code == 403, response.text + assert (await db_session.execute(text("SELECT count(*) FROM shelves"))).scalar_one() == 0 + await db_session.rollback() + + +async def test_delete_before_create_and_missing_patch_do_not_create(client, auth_headers, db_session): + for table in ("books", "shelves", "tags", "notes", "annotations"): + row_id = uuid4() + assert (await upload(client, auth_headers, mutation(table, row_id, op="DELETE"))).status_code == 200 + assert (await upload(client, auth_headers, mutation(table, row_id, {}))).status_code == 200 + assert (await upload(client, auth_headers, mutation(table, uuid4(), {}, "PATCH"))).status_code == 200 + assert (await db_session.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0 + await db_session.rollback() + + +async def test_deleting_annotation_or_tag_blocks_stale_recreation(client, auth_headers, db_session): + book, tag, annotation = uuid4(), uuid4(), uuid4() + batch = [ + mutation("books", book, {"title": "Book"}), + mutation("tags", tag, {"name": "Tag", "color_hex": "red"}), + mutation( + "annotations", annotation, {"book_id": str(book), "selected_text": "Quote", "location": {"page_number": 1}} + ), + mutation("book_tags", f"{book}:{tag}", {"book_id": str(book), "tag_id": str(tag)}), + ] + assert (await upload(client, auth_headers, *batch)).status_code == 200 + assert ( + await upload( + client, auth_headers, mutation("tags", tag, op="DELETE"), mutation("annotations", annotation, op="DELETE") + ) + ).status_code == 200 + assert (await upload(client, auth_headers, *batch[1:])).status_code == 200 + for table in ("tags", "annotations", "book_tags"): + assert (await db_session.execute(text(f"SELECT count(*) FROM {table}"))).scalar_one() == 0 + + +async def test_concurrent_patches_preserve_unrelated_values(client, auth_headers, db_session): + import asyncio + + book = uuid4() + assert ( + await upload( + client, + auth_headers, + mutation( + "books", + book, + {"title": "Book", "author": "Author", "custom_metadata": {"custom_metadata": {"keep": True}}}, + ), + ) + ).status_code == 200 + results = await asyncio.gather( + upload(client, auth_headers, mutation("books", book, {"title": "New title"}, "PATCH")), + upload(client, auth_headers, mutation("books", book, {"author": "New author"}, "PATCH")), + ) + assert [result.status_code for result in results] == [200, 200] + row = (await db_session.execute(text("SELECT title, author, custom_metadata FROM books"))).one() + assert row == ("New title", "New author", {"custom_metadata": {"keep": True}}) + + +async def test_library_field_validation_is_atomic(client, auth_headers, db_session): + for table, data in ( + ("shelves", {"name": "Shelf", "is_smart": "maybe"}), + ("notes", {"title": "Note", "content": "Body", "location": {"page_number": "bad"}}), + ("annotations", {"location": {"page_number": 1, "unknown": True}}), + ("books", {"title": "Book", "file_size": 1.5}), + ): + response = await upload( + client, auth_headers, mutation("books", uuid4(), {"title": "Rollback"}), mutation(table, uuid4(), data) + ) + assert response.status_code == 400, response.text + assert (await db_session.execute(text("SELECT count(*) FROM books"))).scalar_one() == 0 + await db_session.rollback() diff --git a/tests/api/routes/test_sync.py b/tests/api/routes/test_sync.py index f5f5d88..57abac0 100644 --- a/tests/api/routes/test_sync.py +++ b/tests/api/routes/test_sync.py @@ -161,7 +161,7 @@ async def test_powersync_upload_rejects_unsupported_table( response = await client.post( "/v1/sync/powersync-upload", headers=auth_headers, - json={"batch": [{"type": "shelves", "op": "PUT", "id": str(uuid4()), "data": {"name": "Shelf"}}]}, + json={"batch": [{"type": "reading_sessions", "op": "PUT", "id": str(uuid4()), "data": {"name": "Shelf"}}]}, ) assert response.status_code == 422 @@ -170,8 +170,8 @@ async def test_powersync_upload_rejects_partial_future_tables( client: AsyncClient, auth_headers: dict[str, str], ): - """Annotations and reading sessions are not part of the books-only contract.""" - for table in ("annotations", "reading_sessions"): + """Reading sessions and goals are outside the library sync contract.""" + for table in ("reading_sessions", "goals"): response = await client.post( "/v1/sync/powersync-upload", headers=auth_headers, diff --git a/tests/services/test_library_validation.py b/tests/services/test_library_validation.py new file mode 100644 index 0000000..c7ea19d --- /dev/null +++ b/tests/services/test_library_validation.py @@ -0,0 +1,31 @@ +import pytest + +from papyrus.core.exceptions import ValidationError +from papyrus.models import SyncAnnotation, SyncBook +from papyrus.services.library_validation import convert_value, normalize_book_payload + + +@pytest.mark.parametrize( + ("column", "value"), + [ + (SyncBook.__table__.c.series_number, 10**500), + (SyncBook.__table__.c.publication_date, "0001-01-01T00:00:00+01:00"), + (SyncAnnotation.__table__.c.location, {"page_number": 1, "percentage": 10**500}), + ], +) +def test_overflows_are_controlled_validation_errors(column, value): + with pytest.raises(ValidationError): + convert_value(column, value) + + +def test_legacy_invalid_values_remain_in_envelope_without_blocking_queue(): + envelope = {"is_physical": None, "series_number": "invalid", "file_size": 10**500, "lent_to": "Reader"} + normalized = normalize_book_payload({"custom_metadata": envelope}) + assert normalized == {"custom_metadata": envelope, "lent_to": "Reader"} + + +def test_explicit_promoted_value_wins_and_stays_subject_to_validation(): + normalized = normalize_book_payload({"custom_metadata": {"is_physical": True}, "is_physical": None}) + assert normalized["is_physical"] is None + with pytest.raises(ValidationError): + convert_value(SyncBook.__table__.c.is_physical, normalized["is_physical"]) diff --git a/tests/test_bookmark_migration.py b/tests/test_bookmark_migration.py new file mode 100644 index 0000000..4832c4f --- /dev/null +++ b/tests/test_bookmark_migration.py @@ -0,0 +1,35 @@ +"""Verify bookmark migration structure in the isolated pytest database.""" + +import importlib.util +from pathlib import Path + +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import inspect + +from papyrus.models import Base + + +async def test_bookmark_revision_upgrade_downgrade_and_metadata(db_session): + path = Path(__file__).parents[1] / "alembic/versions/af0fea8d6317_add_owned_bookmark_sync.py" + spec = importlib.util.spec_from_file_location("bookmark_revision", path) + revision = importlib.util.module_from_spec(spec) + spec.loader.exec_module(revision) + + def migrate(session, fn): + with Operations.context(MigrationContext.configure(session.connection())): + fn() + + await db_session.run_sync(lambda session: migrate(session, revision.downgrade)) + assert not await db_session.run_sync(lambda session: inspect(session.connection()).has_table("bookmarks")) + await db_session.run_sync(lambda session: migrate(session, revision.upgrade)) + differences = await db_session.run_sync( + lambda session: compare_metadata(MigrationContext.configure(session.connection()), Base.metadata) + ) + assert differences == [] + constraints = await db_session.run_sync( + lambda session: inspect(session.connection()).get_check_constraints("bookmarks") + ) + assert {constraint["name"] for constraint in constraints} == {"ck_bookmarks_position_range"} + await db_session.commit() diff --git a/tests/test_library_migration.py b/tests/test_library_migration.py new file mode 100644 index 0000000..62c1917 --- /dev/null +++ b/tests/test_library_migration.py @@ -0,0 +1,50 @@ +"""Run the library revision against the isolated pytest database only.""" + +import importlib.util +from pathlib import Path +from uuid import uuid4 + +from alembic.autogenerate import compare_metadata +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy import text + +from papyrus.models import Base + + +async def test_library_revision_backfill_and_metadata(db_session, auth_user): + path = Path(__file__).parents[1] / "alembic/versions/dcd3b384e6a4_add_owned_library_sync_and_promoted_.py" + spec = importlib.util.spec_from_file_location("library_revision", path) + revision = importlib.util.module_from_spec(spec) + spec.loader.exec_module(revision) + + def migrate(session, fn): + with Operations.context(MigrationContext.configure(session.connection())): + fn() + + await db_session.run_sync(lambda session: migrate(session, revision.downgrade)) + envelope = '{"publication_date":"2020-01-02T03:04:05Z","file_size":42,"is_physical":true,"series_id":"descriptor","custom_metadata":{"keep":"value"}}' + await db_session.execute( + text( + 'INSERT INTO books (book_id, owner_user_id, title, custom_metadata) VALUES (:id, :owner, \'Legacy\', CAST(:metadata AS jsonb)), (:bad_id, :owner, \'Invalid\', \'{"publication_date":"invalid","file_size":"nope"}\'::jsonb)' + ), + {"id": uuid4(), "bad_id": uuid4(), "owner": auth_user["user_id"], "metadata": envelope}, + ) + await db_session.run_sync(lambda session: migrate(session, revision.upgrade)) + row = ( + await db_session.execute( + text( + "SELECT file_size, is_physical, series_id, custom_metadata->'custom_metadata' FROM books WHERE title = 'Legacy'" + ) + ) + ).one() + assert row == (42, True, "descriptor", {"keep": "value"}) + invalid = ( + await db_session.execute(text("SELECT publication_date, file_size FROM books WHERE title = 'Invalid'")) + ).one() + assert invalid == (None, None) + differences = await db_session.run_sync( + lambda session: compare_metadata(MigrationContext.configure(session.connection()), Base.metadata) + ) + assert differences == [] + await db_session.commit() diff --git a/tests/test_models.py b/tests/test_models.py index 0f53630..ada8282 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -52,10 +52,7 @@ def test_managed_acquisition_models_expose_download_lifecycle() -> None: "ix_acquisition_jobs_next_poll_at", } ) - assert any( - foreign_key.target_fullname == "books.book_id" - for foreign_key in job_table.c.book_id.foreign_keys - ) + assert any(foreign_key.target_fullname == "books.book_id" for foreign_key in job_table.c.book_id.foreign_keys) def test_media_asset_kind_is_unique_per_book() -> None: @@ -106,5 +103,6 @@ def test_auth_models_are_registered_with_metadata() -> None: "updated_at", } assert {"book_id", "owner_user_id", "title", "updated_at"}.issubset(books_table.columns.keys()) - assert "annotations" not in Base.metadata.tables + for table_name in ("shelves", "tags", "notes", "annotations", "book_shelves", "book_tags"): + assert "owner_user_id" in Base.metadata.tables[table_name].columns assert "reading_sessions" not in Base.metadata.tables diff --git a/tests/test_powersync_sync_config.py b/tests/test_powersync_sync_config.py index a363c23..08624a5 100644 --- a/tests/test_powersync_sync_config.py +++ b/tests/test_powersync_sync_config.py @@ -7,3 +7,19 @@ def test_books_stream_downloads_media_references() -> None: assert "file_media_id" in books_select assert "cover_media_id" in books_select + + +def test_library_streams_filter_owners_and_setup_publishes_tables() -> None: + root = Path(__file__).parents[1] + config = (root / "powersync/sync-config.yaml").read_text() + setup = (root / "scripts/setup_local_powersync.sh").read_text() + for table in ("books", "shelves", "tags", "notes", "annotations", "bookmarks", "book_shelves", "book_tags"): + stream = config.split(f" {table}:\n", 1)[1].split("\n\n", 1)[0] + assert "auto_subscribe: true" in stream + assert "WHERE owner_user_id::text = auth.user_id()" in stream + assert f"public.{table}" in setup + assert " demo_items:" in config + assert "sync_tombstones" not in config + assert "file_path" not in config + assert "series_id" in config + assert "location::text AS location" in config