ENG-2359 Speed up my_contents and the other access views to prevent plugin timeouts

packages/database/features/step-definitions/stepdefs.ts

@@ -199,6 +199,22 @@ Given(
   },
 );
 
+// Access does not depend on the vector, so every embedding gets the same one.
+Given("these contents have embeddings:", async (table: DataTable) => {
+  const client = getServiceClient();
+  const localRefs = (world.localRefs || {}) as LocalRefsType;
+  const vector = JSON.stringify(Array<number>(1536).fill(0.1));
+  const rows = table.hashes().map(({ content, obsolete }) => {
+    const targetId = localRefs[content!];
+    if (typeof targetId !== "number") assert.fail(`unknown content ${content}`);
+    return { target_id: targetId, vector, obsolete: obsolete === "true" };
+  });
+  const response = await client
+    .from("ContentEmbedding_openai_text_embedding_3_small_1536")
+    .insert(rows);
+  assert.equal(response.error, null);
+});
+
 const userEmail = (userAccountId: string) => `${userAccountId}@example.com`;
 
 // Invoke the edge function to log an account into a database.
@@ -334,26 +350,51 @@ Then(
 
 const INSUFFICIENT_PRIVILEGE = "42501";
 
-// Exact list comparison, so duplicated rows fail.
-const expectVisibleConcepts = async ({
-  viewerSpaceName,
-  filterSpaceName,
-  inFilterSpace,
-  table,
-}: {
+// The column that identifies a row of each view in a visibility table.
+const CONTENT_VIEW_KEYS = {
+  my_documents: "id",
+  my_contents: "id",
+  my_file_references: "filepath",
+  my_contents_with_embedding_openai_text_embedding_3_small_1536: "id",
+} as const;
+type ContentView = keyof typeof CONTENT_VIEW_KEYS;
+
+type VisibilityCheck = {
   viewerSpaceName: string;
   filterSpaceName: string;
   inFilterSpace: boolean;
   table: DataTable;
-}): Promise<void> => {
+};
+
+const getViewerClient = async (
+  viewerSpaceName: string,
+  filterSpaceName: string,
+): Promise<{
+  filterSpaceId: number;
+  client: Awaited<ReturnType<typeof getLoggedinDatabase>>;
+}> => {
   const localRefs = (world.localRefs || {}) as LocalRefsType;
   const viewerSpaceId = localRefs[viewerSpaceName];
   const filterSpaceId = localRefs[filterSpaceName];
   if (typeof viewerSpaceId !== "number")
     assert.fail("viewer spaceId not a number");
   if (typeof filterSpaceId !== "number")
     assert.fail("filter spaceId not a number");
-  const client = await getLoggedinDatabase(viewerSpaceId);
+  return { filterSpaceId, client: await getLoggedinDatabase(viewerSpaceId) };
+};
+
+// Exact list comparison, so duplicated rows fail.
+const expectVisibleConcepts = async ({
+  viewerSpaceName,
+  filterSpaceName,
+  inFilterSpace,
+  table,
+}: VisibilityCheck): Promise<void> => {
+  const localRefs = (world.localRefs || {}) as LocalRefsType;
+  const { filterSpaceId, client } = await getViewerClient(
+    viewerSpaceName,
+    filterSpaceName,
+  );
   const query = client.from("my_concepts").select("id");
   const response = await (inFilterSpace
     ? query.eq("space_id", filterSpaceId)
@@ -380,6 +421,64 @@ Then(
     }),
 );
 
+// Exact list comparison per view, so duplicated rows fail; a view absent from the table must show no rows.
+// A row is a local reference, or the literal key value when it is not one.
+const expectVisibleContentViewRows = async ({
+  viewerSpaceName,
+  filterSpaceName,
+  inFilterSpace,
+  table,
+}: VisibilityCheck): Promise<void> => {
+  const localRefs = (world.localRefs || {}) as LocalRefsType;
+  const { filterSpaceId, client } = await getViewerClient(
+    viewerSpaceName,
+    filterSpaceName,
+  );
+  const views = Object.keys(CONTENT_VIEW_KEYS) as ContentView[];
+  const expected = Object.fromEntries(
+    views.map((view) => [view, [] as string[]]),
+  ) as Record<ContentView, string[]>;
+  for (const { view, row } of table.hashes()) {
+    if (!Object.hasOwn(CONTENT_VIEW_KEYS, view!))
+      assert.fail(`unknown view ${view}`);
+    expected[view as ContentView].push(String(localRefs[row!] ?? row));
+  }
+  for (const view of views) {
+    const key = CONTENT_VIEW_KEYS[view];
+    const query = client.from(view).select(key);
+    const response = await (inFilterSpace
+      ? query.eq("space_id", filterSpaceId)
+      : query.neq("space_id", filterSpaceId));
+    assert.equal(response.error, null, view);
+    // With view ranging over four views, the generated select types do not resolve.
+    const rows = (response.data || []) as unknown as Record<string, unknown>[];
+    const visible = rows.map((row) => String(row[key]));
+    assert.deepEqual(visible.sort(), expected[view].sort(), view);
+  }
+};
+
+Then(
+  "a user logged in space {word} should see these content view rows in space {word}:",
+  (viewerSpaceName: string, filterSpaceName: string, table: DataTable) =>
+    expectVisibleContentViewRows({
+      viewerSpaceName,
+      filterSpaceName,
+      inFilterSpace: true,
+      table,
+    }),
+);
+
+Then(
+  "a user logged in space {word} should see these content view rows outside space {word}:",
+  (viewerSpaceName: string, filterSpaceName: string, table: DataTable) =>
+    expectVisibleContentViewRows({
+      viewerSpaceName,
+      filterSpaceName,
+      inFilterSpace: false,
+      table,
+    }),
+);
+
 type ViewWrites = Record<
   "insert" | "update" | "delete",
   () => PromiseLike<PostgrestSingleResponse<null>>
@@ -471,6 +570,76 @@ Then(
   },
 );
 
+Then(
+  "a user logged in space {word} cannot {word} documents through my_documents",
+  async (spaceName: string, operation: string) => {
+    const { spaceId, client } = await getSpaceClient(spaceName);
+    const view = client.from("my_documents");
+    const now = new Date().toISOString();
+    await expectWriteRejected(operation, {
+      insert: () =>
+        view.insert({
+          source_local_id: "written-through-my-documents",
+          space_id: spaceId,
+          created: now,
+          last_modified: now,
+        }),
+      update: () =>
+        view
+          .update({ url: "https://example.com/written-through-my-documents" })
+          .eq("space_id", spaceId),
+      delete: () => view.delete().eq("space_id", spaceId),
+    });
+  },
+);
+
+Then(
+  "a user logged in space {word} cannot {word} contents through my_contents",
+  async (spaceName: string, operation: string) => {
+    const { spaceId, client } = await getSpaceClient(spaceName);
+    const view = client.from("my_contents");
+    const now = new Date().toISOString();
+    await expectWriteRejected(operation, {
+      insert: () =>
+        view.insert({
+          text: "written through my_contents",
+          space_id: spaceId,
+          created: now,
+          last_modified: now,
+        }),
+      update: () =>
+        view
+          .update({ text: "written through my_contents" })
+          .eq("space_id", spaceId),
+      delete: () => view.delete().eq("space_id", spaceId),
+    });
+  },
+);
+
+Then(
+  "a user logged in space {word} cannot {word} file references through my_file_references",
+  async (spaceName: string, operation: string) => {
+    const { spaceId, client } = await getSpaceClient(spaceName);
+    const view = client.from("my_file_references");
+    const now = new Date().toISOString();
+    await expectWriteRejected(operation, {
+      insert: () =>
+        view.insert({
+          filepath: "written-through-my-file-references.png",
+          filehash: "written-through-my-file-references",
+          space_id: spaceId,
+          created: now,
+          last_modified: now,
+        }),
+      update: () =>
+        view
+          .update({ filehash: "written-through-my-file-references" })
+          .eq("space_id", spaceId),
+      delete: () => view.delete().eq("space_id", spaceId),
+    });
+  },
+);
+
 Then(
   "a user logged in space {word} should see these concepts outside space {word}:",
   (viewerSpaceName: string, filterSpaceName: string, table: DataTable) =>

packages/database/src/dbTypes.ts

@@ -1122,6 +1122,40 @@ export type Database = {
           text: string | null
           variant: Database["public"]["Enums"]["ContentVariant"] | null
         }
+        Insert: {
+          author_id?: number | null
+          content_type?: string | null
+          created?: string | null
+          creator_id?: number | null
+          document_id?: number | null
+          id?: number | null
+          last_modified?: string | null
+          metadata?: Json | null
+          original?: boolean | null
+          part_of_id?: number | null
+          scale?: Database["public"]["Enums"]["Scale"] | null
+          source_local_id?: string | null
+          space_id?: number | null
+          text?: string | null
+          variant?: Database["public"]["Enums"]["ContentVariant"] | null
+        }
+        Update: {
+          author_id?: number | null
+          content_type?: string | null
+          created?: string | null
+          creator_id?: number | null
+          document_id?: number | null
+          id?: number | null
+          last_modified?: string | null
+          metadata?: Json | null
+          original?: boolean | null
+          part_of_id?: number | null
+          scale?: Database["public"]["Enums"]["Scale"] | null
+          source_local_id?: string | null
+          space_id?: number | null
+          text?: string | null
+          variant?: Database["public"]["Enums"]["ContentVariant"] | null
+        }
         Relationships: [
           {
             foreignKeyName: "Content_author_id_fkey"
@@ -1343,6 +1377,30 @@ export type Database = {
           space_id: number | null
           url: string | null
         }
+        Insert: {
+          author_id?: number | null
+          content_type?: string | null
+          contents?: unknown
+          created?: string | null
+          id?: number | null
+          last_modified?: string | null
+          metadata?: Json | null
+          source_local_id?: string | null
+          space_id?: number | null
+          url?: string | null
+        }
+        Update: {
+          author_id?: number | null
+          content_type?: string | null
+          contents?: unknown
+          created?: string | null
+          id?: number | null
+          last_modified?: string | null
+          metadata?: Json | null
+          source_local_id?: string | null
+          space_id?: number | null
+          url?: string | null
+        }
         Relationships: [
           {
             foreignKeyName: "Document_author_id_fkey"
@@ -1391,6 +1449,24 @@ export type Database = {
           source_path: string | null
           space_id: number | null
         }
+        Insert: {
+          created?: string | null
+          filehash?: string | null
+          filepath?: string | null
+          last_modified?: string | null
+          source_local_id?: string | null
+          source_path?: string | null
+          space_id?: number | null
+        }
+        Update: {
+          created?: string | null
+          filehash?: string | null
+          filepath?: string | null
+          last_modified?: string | null
+          source_local_id?: string | null
+          source_path?: string | null
+          space_id?: number | null
+        }
         Relationships: []
       }
       my_groups: {