Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions .github/workflows/shared-state-postgres.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
name: Shared-state PostgreSQL acceptance

on:
pull_request:
paths:
- "src/youtube_extension/services/shared_sql_state.py"
- "apps/web/supabase/migrations/**"
- "scripts/testing/verify_shared_state_postgres.py"
- "tests/unit/test_shared_sql_state.py"
- ".github/workflows/shared-state-postgres.yml"

permissions:
contents: read

concurrency:
group: shared-state-postgres-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
disposable-postgres:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install bounded acceptance dependencies
run: >-
python -m pip install
pgserver==0.1.4 SQLAlchemy==2.1.8 'psycopg[binary]==3.3.9'
pydantic==2.14.3 aiohttp==3.14.5 requests==2.34.2 redis==8.1.0
- name: Verify temporary PostgreSQL hardening and persistence
env:
PYTHONPATH: src
DATABASE_URL: ""
EVENTRELAY_SHARED_STATE_DATABASE_URL: ""
run: python scripts/testing/verify_shared_state_postgres.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
-- Shared SQL state for session timelines and agent-to-agent (A2A) messages.
-- Source models: src/youtube_extension/services/shared_sql_state.py (SharedSQLStateStore).
-- Columns, types, nullability, primary keys and indexes match the SQLAlchemy Table
-- definitions exactly (SQLAlchemy emits SERIAL for the integer autoincrement keys and
-- ix_<table>_<column> names for index=True). There are no foreign keys in the models.
-- Idempotent: these tables may already exist in a database where an older build ran
-- metadata.create_all(), so tables, indexes, RLS and grants are all safe to re-apply.
-- Access is service-role only, mirroring public.studio_chats: RLS enabled and forced,
-- no policies, nothing granted to public, anon or authenticated.

create table if not exists public.shared_sessions (
session_id varchar(255) not null,
payload text not null,
version integer not null,
created_at timestamptz not null,
updated_at timestamptz not null,
constraint shared_sessions_pkey primary key (session_id)
);

create table if not exists public.shared_timeline_events (
id serial not null,
session_id varchar(255) not null,
"timestamp" varchar(64) not null,
summary text not null,
content text not null,
constraint shared_timeline_events_pkey primary key (id)
);

create index if not exists ix_shared_timeline_events_session_id on public.shared_timeline_events (session_id);

create table if not exists public.shared_a2a_messages (
id serial not null,
sender varchar(255) not null,
recipient varchar(255) not null,
conversation_id varchar(255) not null,
"timestamp" varchar(64) not null,
content text not null,
constraint shared_a2a_messages_pkey primary key (id)
);

create index if not exists ix_shared_a2a_messages_conversation_id on public.shared_a2a_messages (conversation_id);

alter table public.shared_sessions enable row level security;
alter table public.shared_sessions force row level security;
revoke all on table public.shared_sessions from public, anon, authenticated;
grant select, insert, update, delete on table public.shared_sessions to service_role;
Comment thread
groupthinking marked this conversation as resolved.
Resolved

alter table public.shared_timeline_events enable row level security;
alter table public.shared_timeline_events force row level security;
revoke all on table public.shared_timeline_events from public, anon, authenticated;
grant select, insert, update, delete on table public.shared_timeline_events to service_role;

alter table public.shared_a2a_messages enable row level security;
alter table public.shared_a2a_messages force row level security;
revoke all on table public.shared_a2a_messages from public, anon, authenticated;
grant select, insert, update, delete on table public.shared_a2a_messages to service_role;

-- The serial keys own sequences; inserts need them. Same role boundary as the tables.
revoke all on sequence public.shared_timeline_events_id_seq from public, anon, authenticated;
grant usage, select on sequence public.shared_timeline_events_id_seq to service_role;
revoke all on sequence public.shared_a2a_messages_id_seq from public, anon, authenticated;
grant usage, select on sequence public.shared_a2a_messages_id_seq to service_role;
7 changes: 7 additions & 0 deletions apps/web/vercel.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"git": {
"deploymentEnabled": {
"fix/2169-reconcile-models-migrations": false
}
}
}
119 changes: 112 additions & 7 deletions docs/MIGRATION_TRUTH.md
Original file line number Diff line number Diff line change
@@ -1,15 +1,120 @@
# SQL model and migration truth (AUDIT-007 / #2169)

## Current repository evidence — 2026-10-08
## Current state — 2026-11-09

The previous inventory in this file named backend model modules and three Alembic revisions. That inventory is obsolete: the current recursive repository tree does not contain `src/youtube_extension/backend/models/`, `src/youtube_extension/backend/migrations/`, or `tests/unit/test_migration_truth.py`. `alembic.ini` still points to the missing migration directory. Do not treat the old count guard or directory list as verified current coverage.
Machine-readable inventory: [`docs/schema_inventory.json`](schema_inventory.json). It is
guarded by `tests/unit/test_schema_inventory.py`, which fails if a listed source or
migration file is missing, if the shared-state migration drifts from the SQLAlchemy
models (columns, types, nullability, primary keys, indexes, foreign keys), or if a
migrated table stops forcing row-level security.

One SQL migration is present at `apps/web/supabase/migrations/20260912214530_create_private_studio_chat_ownership.sql`. Its presence alone does not prove that all live tables, tenant policies, deployed revisions, or Python persistence models are reconciled.
Migrations live in `apps/web/supabase/migrations/` and are named
`<UTC yyyymmddhhmmss>_<snake_case_description>.sql`.

## Acceptance and blocking requirements
| Table | Model / owner | Migration | RLS | Access |
| --- | --- | --- | --- | --- |
| `studio_chats` | `apps/web/src/lib/studio/chat-store.ts` | `20260912214530_create_private_studio_chat_ownership.sql` | enabled + forced | `service_role` only |
| `shared_sessions` | `src/youtube_extension/services/shared_sql_state.py` | `20261109062000_create_service_role_shared_state_tables.sql` | enabled + forced | `service_role` only |
| `shared_timeline_events` | same | same | enabled + forced | `service_role` only |
| `shared_a2a_messages` | same | same | enabled + forced | `service_role` only |

Issue #2169 remains open. Before creating migrations, identify the actual active database and owning service, inventory current persistence models and deployed revision/RLS state using approved read-only access, and map each model to its authoritative migration history. The missing legacy directory needs an explicit retirement or restoration decision; do not silently manufacture a replacement migration chain.
### Schema ownership for the shared-state tables

No live schema comparison, database access, migration generation, or migration application was performed in this correction. A destructive migration requires separate approval. New tables/columns must accompany the matching migration and fixture validation in the same PR.
- **PostgreSQL / Supabase:** the migration is the only schema source.
`SharedSQLStateStore` no longer runs `metadata.create_all()` there; it inspects the
database and raises `SharedStateSchemaError` (naming the migration) if a table or
column is missing, or PostgreSQL tables lack forced RLS, deny-all client ACLs,
required service-role grants or their secured serial sequences. Existing tables
from `create_all()` therefore cannot pass by having the right columns alone.
`AgentOrchestrator` already catches this and falls back to its
in-memory A2A log with a warning; `scripts/maintenance/session_orchestration_manager.py`
fails loudly.
- **SQLite (local default, tests):** `create_all()` still creates the tables, so local
runs need no setup.
- The migration is idempotent (`create table if not exists`, `create index if not
exists`), so it can be applied to a database where an older build already created
these tables with `create_all()`. That path was checked against PostgreSQL 17: the
resulting schema is identical to a fresh apply.

Historical claim retained for audit: the previous file asserted three committed revisions (`001`–`003`) and a `test_migration_truth.py` guard. That claim is not valid for the inspected current tree.
### Tenancy and RLS strategy

Tables written only by trusted server code are service-role only: RLS enabled and
forced, no policies, all privileges revoked from `public`, `anon` and `authenticated`,
and `select, insert, update, delete` granted to `service_role` (sequences: `usage,
select`). The connecting role must be `service_role` or another role with `BYPASSRLS`
(on Supabase, `postgres` has it). A role without `BYPASSRLS` — including a table owner,
because RLS is forced — reads zero rows and cannot write.

## Not yet reconciled

- `supabase/supabase_setup.sql` (projects, chats, files, media, mcp_logs, nods_page,
nods_page_section): a Docker init script, not a migration; RLS enabled but not forced,
with owner/authenticated policies. Needs its own migrate-or-retire decision.
- `infrastructure/database/query_optimization.sql` (video_processing_results,
api_usage_optimized, performance_metrics_optimized): unmanaged script, no application
model, no migration. `index_analysis.py` inspects video_processing_results and
performance_metrics_optimized.
- `supabase/setup_mcp_db.js` attempts to create `mcp_contexts`; bootstrap script,
no checked-in migration or RLS.
- `src/integration/supabase_db.py` `generate_schema_sql()` emits template tables
(profiles, subscriptions) for generated apps; out of scope for EventRelay's own schema.
- The Supabase CLI config is `supabase/config.toml` (project `nsfrhirwsjqwhagtuaxx`),
while migrations are in `apps/web/supabase/migrations/`. Applying them is a manual
step until the two are aligned.

## Deployment

Applying migrations to the live Supabase project is a human step; nothing in this
repository applies them automatically. Before applying
`20261109062000_create_service_role_shared_state_tables.sql`, check whether any
deployed service's `DATABASE_URL` / `EVENTRELAY_SHARED_STATE_DATABASE_URL` already
created these tables via `create_all()` — if so they currently have no RLS and default
grants, and the migration will lock them down in place.

### Plain PostgreSQL prerequisites

Supabase supplies `anon`, `authenticated`, and `service_role`. A plain PostgreSQL
administrator must provision these roles **before applying the migration**. Run the
following only for roles confirmed absent (existing roles must be inspected, not replaced):

```sql
CREATE ROLE anon NOLOGIN;
CREATE ROLE authenticated NOLOGIN;
CREATE ROLE service_role NOLOGIN BYPASSRLS;
GRANT USAGE ON SCHEMA public TO service_role;
```

Then apply the checked-in migration as the database owner or administrator. Connect
the server with a dedicated login role that has `BYPASSRLS` and inherits the
`service_role` object grants, or use an appropriately configured `service_role`
login. `BYPASSRLS` itself is not inherited through role membership. Provision login
authentication through the database's secret-management process; never commit it.
Client-facing roles must have neither `BYPASSRLS` nor membership in privileged
server roles. No application start-up path creates roles or applies migrations.

## History

The October 2026 version of this file noted that the backend model modules and the three
Alembic revisions (`001`–`003`) named in the original audit, and
`tests/unit/test_migration_truth.py`, are absent from the tree. That remains true; the
legacy Alembic chain is treated as retired, not restored.

## Disposable PostgreSQL acceptance

`scripts/testing/verify_shared_state_postgres.py` starts its own temporary PostgreSQL
cluster (optional `pgserver` and `psycopg` packages), refuses IP listeners, and tests
legacy tables, effective client privileges, role attributes, sequence access,
server-role writes, fresh-store readback and a hostile `search_path`. Run from the
repository root with `PYTHONPATH=src python scripts/testing/verify_shared_state_postgres.py`.
It never reads deployment database environment variables.

The November 12, 2026 local attempt was blocked before server startup by
`PermissionError: [Errno 1] Operation not permitted` when creating the private
Unix socket. The same script subsequently passed all **13 cases on PostgreSQL
16.2** in the isolated GitHub runner at source head
`190fc17f7b38a19978094e08aaf3deac5f745c59`:
[acceptance run](https://github.com/groupthinking/EventRelay/actions/runs/43347589645/job/131630572786).
The runner verified no IP listener and stopped and deleted the temporary cluster.
This establishes disposable PostgreSQL acceptance for that source, not application
of the migration or acceptance against any production database. The separate
environment-gated real-PostgreSQL unit test remains skipped in local unit runs.
97 changes: 97 additions & 0 deletions docs/schema_inventory.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
{
"issue": "https://github.com/groupthinking/EventRelay/issues/2169",
"updated": "2026-11-09",
"migrations_dir": "apps/web/supabase/migrations",
"supabase_project_ref": "nsfrhirwsjqwhagtuaxx",
"notes": "Repository inventory only. No live database was read or changed. 'migrated' means a checked-in migration defines the table; whether it is applied to a given database must be confirmed separately.",
"tables": [
{
"name": "studio_chats",
"schema": "public",
"model_source": "apps/web/src/lib/studio/chat-store.ts",
"migration": "apps/web/supabase/migrations/20260912214530_create_private_studio_chat_ownership.sql",
"status": "migrated",
"rls": "forced",
"access": "service_role only"
},
{
"name": "shared_sessions",
"schema": "public",
"model_source": "src/youtube_extension/services/shared_sql_state.py",
"migration": "apps/web/supabase/migrations/20261109062000_create_service_role_shared_state_tables.sql",
"status": "migrated",
"rls": "forced",
"access": "service_role only"
},
{
"name": "shared_timeline_events",
"schema": "public",
"model_source": "src/youtube_extension/services/shared_sql_state.py",
"migration": "apps/web/supabase/migrations/20261109062000_create_service_role_shared_state_tables.sql",
"status": "migrated",
"rls": "forced",
"access": "service_role only"
},
{
"name": "shared_a2a_messages",
"schema": "public",
"model_source": "src/youtube_extension/services/shared_sql_state.py",
"migration": "apps/web/supabase/migrations/20261109062000_create_service_role_shared_state_tables.sql",
"status": "migrated",
"rls": "forced",
"access": "service_role only"
}
],
"unmigrated_sql_sources": [
{
"source": "supabase/supabase_setup.sql",
"tables": [
"projects",
"chats",
"files",
"media",
"mcp_logs",
"nods_page",
"nods_page_section"
],
"status": "setup_script_not_migration",
"rls": "enabled_not_forced",
"note": "Docker init script with owner/authenticated policies; not part of the migrations directory. Needs its own migrate-or-retire decision."
},
{
"source": "infrastructure/database/query_optimization.sql",
"tables": [
"video_processing_results",
"api_usage_optimized",
"performance_metrics_optimized"
],
"status": "unmanaged_sql_script",
"rls": "none",
"note": "infrastructure/database/index_analysis.py inspects video_processing_results and performance_metrics_optimized; api_usage_optimized is also defined by this SQL script. No application model and no migration."
},
{
"source": "supabase/setup_mcp_db.js",
"tables": ["mcp_contexts"],
"status": "setup_script_not_migration",
"rls": "none",
"note": "JavaScript bootstrap attempts to create mcp_contexts; not part of the migrations directory. Needs its own migrate-or-retire decision."
},
{
"source": "src/integration/supabase_db.py (generate_schema_sql)",
"tables": [
"profiles",
"subscriptions"
],
"status": "out_of_scope_template",
"rls": "enabled_not_forced",
"note": "SQL template emitted for generated apps, not EventRelay's own schema."
}
],
"retired": [
{
"source": "src/youtube_extension/backend/models/ and src/youtube_extension/backend/migrations/versions/001-003",
"status": "absent_from_tree",
"note": "Named in the original audit; not present in the current repository."
}
]
}
Loading
Loading