From 14093dd3a15295f9b579ee4f87dd49ea1843177f Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Tue, 6 Oct 2026 09:33:40 -0400 Subject: [PATCH] ENG-2340 Import cross-space relations into Roam --- .../components/DiscoverSharedNodesDialog.tsx | 36 ++- .../__tests__/discoverSharedRelations.test.ts | 53 +++ .../__tests__/importSharedRelations.test.ts | 269 +++++++++++++++- .../__tests__/refreshAllImportedNodes.test.ts | 55 +++- .../__tests__/refreshImportedNode.test.ts | 89 +++++- .../__tests__/roamDefaultNodeTypeIds.test.ts | 11 + .../roam/src/utils/discoverSharedRelations.ts | 223 +++++++++---- apps/roam/src/utils/importSharedRelations.ts | 301 ++++++++++++++---- .../roam/src/utils/refreshAllImportedNodes.ts | 18 +- apps/roam/src/utils/refreshImportedNode.ts | 44 ++- 10 files changed, 960 insertions(+), 139 deletions(-) create mode 100644 apps/roam/src/utils/__tests__/discoverSharedRelations.test.ts create mode 100644 apps/roam/src/utils/__tests__/roamDefaultNodeTypeIds.test.ts diff --git a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx index 0eb4884854..654c015bb6 100644 --- a/apps/roam/src/components/DiscoverSharedNodesDialog.tsx +++ b/apps/roam/src/components/DiscoverSharedNodesDialog.tsx @@ -154,12 +154,14 @@ const ImportedNodeLink = ({ const ImportResultsSummary = ({ results, relationsFailed, + hasRelationFailures, retryingRelations, onOpenInMainWindow, onRetryRelations, }: { results: SharedNodeImportItem[]; relationsFailed: boolean; + hasRelationFailures: boolean; retryingRelations: boolean; onOpenInMainWindow: () => void; onRetryRelations: () => void; @@ -179,7 +181,10 @@ const ImportResultsSummary = ({ 0 || warningCount > 0 || relationsFailed + failedImports.length > 0 || + warningCount > 0 || + relationsFailed || + hasRelationFailures ? Intent.WARNING : Intent.SUCCESS } @@ -220,6 +225,21 @@ const ImportResultsSummary = ({ ); }; +const RelationFailuresSummary = ({ failures }: { failures: string[] }) => ( + +
These relations are retried on the next import.
+
    + {failures.map((failure) => ( +
  • {failure}
    • + ))} +
+ +); + const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { const [nodes, setNodes] = useState([]); const [importedRids, setImportedRids] = useState>(new Set()); @@ -238,6 +258,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { >(null); const [relationsFailed, setRelationsFailed] = useState(false); const [retryingRelations, setRetryingRelations] = useState(false); + const [relationFailures, setRelationFailures] = useState([]); const importing = importProgress !== null; const loadNodes = useCallback(async (): Promise => { @@ -245,6 +266,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { setError(""); setSelectedRids(new Set()); setImportResults(null); + setRelationFailures([]); try { const context = await getSupabaseContext(); if (!context) throw new Error(CONNECTION_ERROR_MESSAGE); @@ -326,7 +348,11 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { const importRelations = async (client: DGSupabaseClient): Promise => { try { - await importSharedRelations(client, spaceId); + const { failures, skipped } = await importSharedRelations( + client, + spaceId, + ); + setRelationFailures([...skipped, ...failures]); setRelationsFailed(false); } catch (relationsError) { setRelationsFailed(true); @@ -363,6 +389,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { const selectedNodes = nodes.filter((node) => selectedRids.has(node.rid)); setImportResults(null); + setRelationFailures([]); setImportProgress({ current: 0, total: selectedNodes.length }); try { const client = await getLoggedInClient(); @@ -381,6 +408,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { newlyImportedRids.forEach((rid) => next.add(rid)); return next; }); + // Shown before importing relations, so a relation import error keeps the node summary. setImportResults(results); const failedImports = results.filter(isFailedSharedNodeImport); setSelectedRids( @@ -472,6 +500,7 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { ) : ( importResults && ( 0} onOpenInMainWindow={onClose} onRetryRelations={() => void retryRelations()} relationsFailed={relationsFailed} @@ -480,6 +509,9 @@ const DiscoverSharedNodesDialog = ({ onClose }: { onClose: () => void }) => { /> ) )} + {relationFailures.length > 0 && ( + + )} {loading ? (
diff --git a/apps/roam/src/utils/__tests__/discoverSharedRelations.test.ts b/apps/roam/src/utils/__tests__/discoverSharedRelations.test.ts new file mode 100644 index 0000000000..5bcfbfb43c --- /dev/null +++ b/apps/roam/src/utils/__tests__/discoverSharedRelations.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from "vitest"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { discoverSharedRelations } from "~/utils/discoverSharedRelations"; + +vi.mock("~/utils/importedSourceIdentity", () => ({ + getImportedSourceRids: () => + Promise.resolve( + new Set(["orn:obsidian.note:vault-a/x", "orn:obsidian.note:vault-a/y"]), + ), +})); +vi.mock("@repo/database/lib/dbToCrossAppConverters", () => ({ + getSpaceMap: () => Promise.resolve({ 20: "obsidian:vault-a" }), +})); + +// Each `from` call answers with the next queued result, whatever the filters. +const fakeClient = (results: unknown[][]): DGSupabaseClient => { + const from = () => { + const data = results.shift(); + const query: Record = {}; + for (const method of ["select", "neq", "eq", "in", "overlaps"]) + query[method] = () => query; + query.then = (resolve: (value: unknown) => unknown) => + resolve({ data, error: null }); + return query; + }; + return { from } as unknown as DGSupabaseClient; +}; + +describe("discoverSharedRelations", () => { + it("skips a relation whose schema is not visible and reports it", async () => { + const relation = { + id: 300, + space_id: 20, + source_local_id: "rel-1", + schema_id: 99, + is_schema: false, + is_relation: true, + reference_content: { source: 1, destination: 2 }, + concepts_of_relation: [ + { id: 1, space_id: 20, source_local_id: "x", schema_id: 10 }, + { id: 2, space_id: 20, source_local_id: "y", schema_id: 11 }, + ], + }; + const client = fakeClient([[relation], []]); + + const result = await discoverSharedRelations(client, 7); + + expect(result.relations).toEqual([]); + expect(result.skippedRelations).toEqual([ + "orn:obsidian.relation:vault-a/rel-1: its relation type is not visible", + ]); + }); +}); diff --git a/apps/roam/src/utils/__tests__/importSharedRelations.test.ts b/apps/roam/src/utils/__tests__/importSharedRelations.test.ts index f14274edd1..bb67c4bc48 100644 --- a/apps/roam/src/utils/__tests__/importSharedRelations.test.ts +++ b/apps/roam/src/utils/__tests__/importSharedRelations.test.ts @@ -8,6 +8,14 @@ import getDiscourseRelations from "~/utils/getDiscourseRelations"; import { createRelationSchema } from "~/utils/createRelationSchema"; import { createDiscourseNodeType } from "~/components/settings/utils/accessors"; import { discoverSharedRelations } from "~/utils/discoverSharedRelations"; +import type { DiscoverSharedRelationsResult } from "~/utils/discoverSharedRelations"; +import { + findImportedNodeUidBySourceRid, + getImportedSourceRids, +} from "~/utils/importedSourceIdentity"; +import { createReifiedRelation } from "~/utils/createReifiedBlock"; +import findDiscourseNode from "~/utils/findDiscourseNode"; +import internalError from "~/utils/internalError"; vi.hoisted(() => { vi.stubGlobal("window", { roamAlphaAPI: { graph: { name: "local" } } }); @@ -17,8 +25,13 @@ vi.mock("~/utils/getDiscourseRelations", () => ({ default: vi.fn() })); vi.mock("~/utils/getDiscourseNodes", () => ({ default: () => [{ type: "local-claim", text: "Claim" }], })); +vi.mock("~/utils/internalError", () => ({ default: vi.fn() })); +vi.mock("~/utils/findTargetUid", () => ({ + findTargetUid: (localOrRid: string) => Promise.resolve(`page-${localOrRid}`), +})); +vi.mock("~/utils/findDiscourseNode", () => ({ default: vi.fn() })); vi.mock("~/utils/importedSourceIdentity", () => ({ - getImportedSourceRids: () => Promise.resolve(new Set()), + getImportedSourceRids: vi.fn(() => Promise.resolve(new Set())), findImportedNodeUidBySourceRid: vi.fn(), writeImportedSourceIdentity: vi.fn(), })); @@ -29,7 +42,7 @@ vi.mock("~/utils/createRelationSchema", () => ({ createRelationSchema: vi.fn(), })); vi.mock("~/utils/createReifiedBlock", () => ({ - getReifiedRelations: () => Promise.resolve([]), + getReifiedRelations: vi.fn(() => Promise.resolve([])), createReifiedRelation: vi.fn(), })); vi.mock("roamjs-components/writes", () => ({ deleteBlock: vi.fn() })); @@ -59,6 +72,9 @@ vi.mock("~/utils/discoverSharedRelations", () => ({ createdAt: new Date("2026-09-07"), }, ], + tripleCandidatesByRelationType: {}, + matchedTripleByRelation: {}, + skippedRelations: [], }), ), })); @@ -104,7 +120,10 @@ describe("importSharedRelations schema matching", () => { triples: [["source", "is in page", "destination"]], }, ]); - await expect(importSharedRelations(client, 7)).resolves.toBeUndefined(); + await expect(importSharedRelations(client, 7)).resolves.toEqual({ + failures: [], + skipped: [], + }); expect(createRelationSchema).not.toHaveBeenCalled(); }); @@ -125,6 +144,9 @@ describe("importSharedRelations schema matching", () => { relations: [], relTypeSchemas: [], relTripleSchemas: [], + tripleCandidatesByRelationType: {}, + matchedTripleByRelation: {}, + skippedRelations: [], nodeSchemas: [ { localId: "evidence", @@ -168,3 +190,244 @@ describe("importSharedRelations schema matching", () => { }); }); }); + +describe("importSharedRelations for relations that point to a relation type", () => { + const relationTypeRid = "orn:obsidian.schema:remote/rel-supports"; + const tripleRid = "orn:obsidian.schema:remote/claim-supports-evidence"; + const relationRid = (localId: string) => + `orn:obsidian.relation:remote/${localId}`; + const crossSpaceRelation = (localId: string) => ({ + rid: relationRid(localId), + localId, + authorId: "author", + createdAt: new Date("2026-10-01"), + relationType: "rel-supports", + source: `x-${localId}`, + destination: `y-${localId}`, + }); + const discovered = ( + overrides: Partial = {}, + ): DiscoverSharedRelationsResult => ({ + relations: [crossSpaceRelation("one")], + relTypeSchemas: [ + { + localId: "rel-supports", + rid: relationTypeRid, + label: "Supports", + complement: "Supported by", + authorId: "author", + createdAt: new Date("2026-10-01"), + }, + ], + relTripleSchemas: [], + nodeSchemas: [], + tripleCandidatesByRelationType: { + [relationTypeRid]: [ + { + rid: tripleRid, + label: "Supports", + complement: "Is supported by", + modifiedAt: new Date("2026-10-02"), + }, + ], + }, + matchedTripleByRelation: {}, + skippedRelations: [], + ...overrides, + }); + const localTriple = (id: string, label = "Supports"): DiscourseRelation => ({ + id, + label, + complement: "Supported by", + source: "local-claim", + destination: "local-evidence", + triples: [], + }); + + beforeEach(() => { + vi.mocked(findDiscourseNode).mockImplementation(({ uid }) => + uid.startsWith("page-x") + ? ({ type: "local-claim" } as ReturnType) + : ({ type: "local-evidence" } as ReturnType), + ); + vi.mocked(createReifiedRelation).mockResolvedValue("new-relation"); + vi.mocked(getImportedSourceRids).mockResolvedValue(new Set()); + vi.mocked(findImportedNodeUidBySourceRid).mockResolvedValue(null); + }); + + it("uses the local triple that fits the pages' node types", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([ + localTriple("local-supports"), + ]); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce( + discovered({ + matchedTripleByRelation: { [relationRid("one")]: tripleRid }, + }), + ); + + await expect(importSharedRelations(client, 7)).resolves.toEqual({ + failures: [], + skipped: [], + }); + + expect(createRelationSchema).not.toHaveBeenCalled(); + expect(createReifiedRelation).toHaveBeenCalledWith({ + sourceUid: "page-x-one", + destinationUid: "page-y-one", + relationBlockUid: "local-supports", + tentative: true, + }); + }); + + it("finds a triple imported earlier and renamed locally by its source identity", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([ + localTriple("renamed-supports", "Backs"), + ]); + vi.mocked(getImportedSourceRids).mockResolvedValue(new Set([tripleRid])); + vi.mocked(findImportedNodeUidBySourceRid).mockResolvedValue( + "renamed-supports", + ); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce(discovered()); + + await importSharedRelations(client, 7); + + expect(createRelationSchema).not.toHaveBeenCalled(); + expect(createReifiedRelation).toHaveBeenCalledWith( + expect.objectContaining({ relationBlockUid: "renamed-supports" }), + ); + }); + + it("imports the matched triple with the pages' node types as its ends", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([]); + vi.mocked(createRelationSchema).mockResolvedValue("imported-supports"); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce( + discovered({ + matchedTripleByRelation: { [relationRid("one")]: tripleRid }, + }), + ); + + await importSharedRelations(client, 7); + + expect(createRelationSchema).toHaveBeenCalledWith({ + label: "Supports", + complement: "Is supported by", + source: "local-claim", + destination: "local-evidence", + triples: [], + }); + expect(writeImportedSourceIdentity).toHaveBeenCalledWith({ + pageUid: "imported-supports", + sourceNodeRid: tripleRid, + sourceModifiedAt: "2026-10-02T00:00:00.000Z", + }); + expect(createReifiedRelation).toHaveBeenCalledWith( + expect.objectContaining({ relationBlockUid: "imported-supports" }), + ); + }); + + it("does not give a second triple the source RID of an imported copy that does not fit", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([ + { ...localTriple("imported-elsewhere"), source: "local-question" }, + ]); + vi.mocked(getImportedSourceRids).mockResolvedValue(new Set([tripleRid])); + vi.mocked(findImportedNodeUidBySourceRid).mockResolvedValue( + "imported-elsewhere", + ); + vi.mocked(createRelationSchema).mockResolvedValue("second-supports"); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce( + discovered({ + matchedTripleByRelation: { [relationRid("one")]: tripleRid }, + }), + ); + + await importSharedRelations(client, 7); + + expect(createRelationSchema).toHaveBeenCalledWith( + expect.objectContaining({ complement: "Is supported by" }), + ); + expect(writeImportedSourceIdentity).not.toHaveBeenCalledWith( + expect.objectContaining({ pageUid: "second-supports" }), + ); + expect(createReifiedRelation).toHaveBeenCalledWith( + expect.objectContaining({ relationBlockUid: "second-supports" }), + ); + }); + + it("creates a triple with no source identity when no candidate matched", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([]); + vi.mocked(createRelationSchema).mockResolvedValue("created-supports"); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce(discovered()); + + await importSharedRelations(client, 7); + + expect(createRelationSchema).toHaveBeenCalledWith( + expect.objectContaining({ + label: "Supports", + complement: "Supported by", + }), + ); + expect(writeImportedSourceIdentity).not.toHaveBeenCalledWith( + expect.objectContaining({ pageUid: "created-supports" }), + ); + }); + + it("creates the triple once for several relations that need it", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([]); + vi.mocked(createRelationSchema).mockResolvedValue("created-supports"); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce( + discovered({ + relations: [crossSpaceRelation("one"), crossSpaceRelation("two")], + }), + ); + + await importSharedRelations(client, 7); + + expect(createRelationSchema).toHaveBeenCalledOnce(); + expect(createReifiedRelation).toHaveBeenCalledTimes(2); + }); + + it("reports relations skipped for a hidden schema and imports the rest", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([ + localTriple("local-supports"), + ]); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce( + discovered({ + skippedRelations: ["hidden: its relation type is not visible"], + }), + ); + + const result = await importSharedRelations(client, 7); + + expect(result).toEqual({ + failures: [], + skipped: ["hidden: its relation type is not visible"], + }); + expect(createReifiedRelation).toHaveBeenCalledOnce(); + expect(internalError).not.toHaveBeenCalled(); + }); + + it("imports the relations after one that fails", async () => { + vi.mocked(getDiscourseRelations).mockReturnValue([ + localTriple("local-supports"), + ]); + vi.mocked(discoverSharedRelations).mockResolvedValueOnce( + discovered({ + relations: [ + { ...crossSpaceRelation("broken"), relationType: "unknown" }, + crossSpaceRelation("two"), + ], + }), + ); + + const { failures } = await importSharedRelations(client, 7); + + expect(failures).toEqual([ + `${relationRid("broken")}: Missing relation type: unknown`, + ]); + expect(createReifiedRelation).toHaveBeenCalledOnce(); + expect(createReifiedRelation).toHaveBeenCalledWith( + expect.objectContaining({ sourceUid: "page-x-two" }), + ); + expect(internalError).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts b/apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts index f8799419c2..eb9071a89f 100644 --- a/apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts +++ b/apps/roam/src/utils/__tests__/refreshAllImportedNodes.test.ts @@ -1,20 +1,33 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { getImportedNodeUids } from "~/utils/importedSourceIdentity"; import { refreshAllImportedNodes } from "~/utils/refreshAllImportedNodes"; -import { refreshImportedNode } from "~/utils/refreshImportedNode"; +import { + importRelationsAfterRefresh, + refreshImportedNode, +} from "~/utils/refreshImportedNode"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { getLoggedInClient } from "~/utils/supabaseContext"; vi.mock("~/utils/importedSourceIdentity", () => ({ getImportedNodeUids: vi.fn(), })); vi.mock("~/utils/refreshImportedNode", () => ({ refreshImportedNode: vi.fn(), + importRelationsAfterRefresh: vi.fn(), })); +vi.mock("~/utils/supabaseContext", () => ({ getLoggedInClient: vi.fn() })); const mockedGetImportedNodeUids = vi.mocked(getImportedNodeUids); const mockedRefreshImportedNode = vi.mocked(refreshImportedNode); +const mockedImportRelationsAfterRefresh = vi.mocked( + importRelationsAfterRefresh, +); +const client = {} as DGSupabaseClient; beforeEach(() => { vi.clearAllMocks(); + vi.mocked(getLoggedInClient).mockResolvedValue(client); + mockedImportRelationsAfterRefresh.mockResolvedValue(undefined); }); describe("refreshAllImportedNodes", () => { @@ -35,11 +48,43 @@ describe("refreshAllImportedNodes", () => { warnings: [], }); expect(mockedRefreshImportedNode.mock.calls).toEqual([ - [{ pageUid: "uid-1", force: false }], - [{ pageUid: "uid-2", force: false }], - [{ pageUid: "uid-3", force: false }], - [{ pageUid: "uid-4", force: false }], + [{ pageUid: "uid-1", force: false, importRelations: false }], + [{ pageUid: "uid-2", force: false, importRelations: false }], + [{ pageUid: "uid-3", force: false, importRelations: false }], + [{ pageUid: "uid-4", force: false, importRelations: false }], ]); + expect(mockedImportRelationsAfterRefresh).toHaveBeenCalledOnce(); + expect(mockedImportRelationsAfterRefresh).toHaveBeenCalledWith(client); + }); + + it("adds the relation import warning to the refresh warnings", async () => { + mockedGetImportedNodeUids.mockResolvedValue(new Set(["uid-1"])); + mockedRefreshImportedNode.mockResolvedValue({ + status: "skipped", + message: "Up to date.", + }); + mockedImportRelationsAfterRefresh.mockResolvedValue( + "1 relation could not be imported.", + ); + + await expect(refreshAllImportedNodes()).resolves.toEqual({ + refreshed: 0, + skipped: 1, + failed: 0, + warnings: ["1 relation could not be imported."], + }); + }); + + it("does not import relations when every refresh failed", async () => { + mockedGetImportedNodeUids.mockResolvedValue(new Set(["uid-1"])); + mockedRefreshImportedNode.mockResolvedValue({ + status: "failed", + message: "Not shared.", + }); + + await refreshAllImportedNodes(); + + expect(mockedImportRelationsAfterRefresh).not.toHaveBeenCalled(); }); it("returns zero counts when the graph has no imported nodes", async () => { diff --git a/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts b/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts index 657abdb3a6..987fdc92c4 100644 --- a/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts +++ b/apps/roam/src/utils/__tests__/refreshImportedNode.test.ts @@ -11,7 +11,8 @@ import internalError from "~/utils/internalError"; import { materializeSharedNode } from "~/utils/materializeSharedNode"; import { refreshImportedNode } from "~/utils/refreshImportedNode"; import { resolveSharedNodeTypes } from "~/utils/resolveSharedNodeTypes"; -import { getLoggedInClient } from "~/utils/supabaseContext"; +import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; +import { importSharedRelations } from "~/utils/importSharedRelations"; vi.mock("roamjs-components/queries/getPageTitleByPageUid", () => ({ default: vi.fn(), @@ -34,6 +35,10 @@ vi.mock("~/utils/resolveSharedNodeTypes", () => ({ })); vi.mock("~/utils/supabaseContext", () => ({ getLoggedInClient: vi.fn(), + getSupabaseContext: vi.fn(), +})); +vi.mock("~/utils/importSharedRelations", () => ({ + importSharedRelations: vi.fn(), })); // Runs before the imports above: getDiscourseNodes calls generateUID at module load. @@ -50,6 +55,8 @@ const mockedInternalError = vi.mocked(internalError); const mockedMaterializeSharedNode = vi.mocked(materializeSharedNode); const mockedGetLoggedInClient = vi.mocked(getLoggedInClient); const mockedResolveSharedNodeTypes = vi.mocked(resolveSharedNodeTypes); +const mockedGetSupabaseContext = vi.mocked(getSupabaseContext); +const mockedImportSharedRelations = vi.mocked(importSharedRelations); const NODE_TYPE: DiscourseNode = { text: "Evidence", @@ -92,6 +99,10 @@ beforeEach(() => { sourceNodeRid: sharedNode.rid, }); mockedGetLoggedInClient.mockResolvedValue(client); + mockedGetSupabaseContext.mockResolvedValue({ spaceId: 5 } as Awaited< + ReturnType + >); + mockedImportSharedRelations.mockResolvedValue({ failures: [], skipped: [] }); mockedGetSharedNodeByRid.mockResolvedValue(sharedNode); mockedResolveSharedNodeTypes.mockResolvedValue(new Map()); mockedMaterializeSharedNode.mockResolvedValue({ @@ -104,6 +115,82 @@ beforeEach(() => { }); describe("refreshImportedNode", () => { + it("imports relations after refreshing, including when the node was up to date", async () => { + await refreshImportedNode({ pageUid: PAGE_UID, force: true }); + expect(mockedImportSharedRelations).toHaveBeenCalledWith(client, 5); + + mockedImportSharedRelations.mockClear(); + mockedMaterializeSharedNode.mockResolvedValue({ + success: true, + action: "skipped", + pageUid: PAGE_UID, + sourceModifiedAt: sharedNode.lastModified, + sourceNodeRid: sharedNode.rid, + }); + await refreshImportedNode({ pageUid: PAGE_UID, force: false }); + expect(mockedImportSharedRelations).toHaveBeenCalledWith(client, 5); + }); + + it("does not import relations when asked not to", async () => { + await refreshImportedNode({ + pageUid: PAGE_UID, + force: false, + importRelations: false, + }); + expect(mockedImportSharedRelations).not.toHaveBeenCalled(); + }); + + it("reports relations that failed to import as a warning, not a failure", async () => { + mockedImportSharedRelations.mockResolvedValue({ + failures: ["a: broken", "b: broken"], + skipped: ["c: hidden"], + }); + mockedMaterializeSharedNode.mockResolvedValue({ + success: true, + action: "updated", + pageUid: PAGE_UID, + sourceModifiedAt: sharedNode.lastModified, + sourceNodeRid: sharedNode.rid, + warning: "No source was published with this node.", + }); + + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "refreshed", + message: 'Refreshed "EVD - REM sleep and recall" from Research vault.', + warning: + "No source was published with this node. 2 relations could not be imported.", + }); + }); + + it("does not warn about relations skipped for a hidden schema", async () => { + mockedImportSharedRelations.mockResolvedValue({ + failures: [], + skipped: ["c: hidden"], + }); + + const result = await refreshImportedNode({ + pageUid: PAGE_UID, + force: true, + }); + + expect(result.status).toBe("refreshed"); + expect(result).not.toHaveProperty("warning"); + }); + + it("keeps the refresh when the relation import throws", async () => { + mockedImportSharedRelations.mockRejectedValue(new Error("network down")); + + await expect( + refreshImportedNode({ pageUid: PAGE_UID, force: true }), + ).resolves.toEqual({ + status: "refreshed", + message: 'Refreshed "EVD - REM sleep and recall" from Research vault.', + warning: "Could not import relations: network down", + }); + }); + it("refreshes the page from its stored source identity", async () => { await expect( refreshImportedNode({ pageUid: PAGE_UID, force: true }), diff --git a/apps/roam/src/utils/__tests__/roamDefaultNodeTypeIds.test.ts b/apps/roam/src/utils/__tests__/roamDefaultNodeTypeIds.test.ts new file mode 100644 index 0000000000..7631d1e2d3 --- /dev/null +++ b/apps/roam/src/utils/__tests__/roamDefaultNodeTypeIds.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; +import { ROAM_DEFAULT_NODE_TYPE_IDS } from "@repo/database/lib/tripleMatching"; +import INITIAL_NODE_VALUES from "~/data/defaultDiscourseNodes"; + +describe("ROAM_DEFAULT_NODE_TYPE_IDS", () => { + it("lists exactly the ids of Roam's default node types", () => { + expect(new Set(ROAM_DEFAULT_NODE_TYPE_IDS)).toEqual( + new Set(INITIAL_NODE_VALUES.map(({ type }) => type)), + ); + }); +}); diff --git a/apps/roam/src/utils/discoverSharedRelations.ts b/apps/roam/src/utils/discoverSharedRelations.ts index b9d19f86dc..5b397db252 100644 --- a/apps/roam/src/utils/discoverSharedRelations.ts +++ b/apps/roam/src/utils/discoverSharedRelations.ts @@ -16,14 +16,33 @@ import { import { Tables } from "@repo/database/dbTypes"; import { spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; import { getImportedSourceRids } from "./importedSourceIdentity"; +import { + type NodeTypeIdentity, + parseTripleEndNames, + pickTripleCandidate, +} from "@repo/database/lib/tripleMatching"; type Concept = Tables<"Concept">; +// A visible triple of an Obsidian relation's relation type. +export type TripleCandidate = { + rid: string; + label?: string; + complement?: string; + modifiedAt?: Date; +}; + export type DiscoverSharedRelationsResult = { relations: CrossAppRelation[]; relTripleSchemas: CrossAppRelationTripleSchema[]; relTypeSchemas: CrossAppRelationTypeSchema[]; nodeSchemas: CrossAppNodeSchema[]; + // By relation type RID. + tripleCandidatesByRelationType: Record; + // By relation RID; set only when exactly one candidate matches. + matchedTripleByRelation: Record; + // One failure message per relation dropped because its schema is not visible. + skippedRelations: string[]; }; export const discoverSharedRelations = async ( @@ -35,6 +54,9 @@ export const discoverSharedRelations = async ( relTripleSchemas: [], relTypeSchemas: [], nodeSchemas: [], + tripleCandidatesByRelationType: {}, + matchedTripleByRelation: {}, + skippedRelations: [], }; // TODO: paginate const { data: dbAllImportableRelations, error: relError } = await client @@ -53,6 +75,8 @@ export const discoverSharedRelations = async ( .map((r) => r.concepts_of_relation) .flat(); const spaceIds = new Set(relatedNodeInfo.map(({ space_id }) => space_id!)); + // A relation between two imported nodes shares a space with neither end. + dbAllImportableRelations.forEach(({ space_id }) => spaceIds.add(space_id!)); spaceIds.add(spaceId); const spaceMap = await getSpaceMap(client, [...spaceIds]); const toRid = (spaceId: number, localId: string) => @@ -81,7 +105,18 @@ export const discoverSharedRelations = async ( .map(({ id }) => id), ); const importedNodeRids = await getImportedSourceRids(); - const dbRelations = dbAllImportableRelations.filter((r) => { + const relationRid = (r: { + space_id: number | null; + source_local_id: string | null; + }) => + r.space_id !== null && r.space_id in spaceMap + ? spaceUriAndLocalIdToRid( + spaceMap[r.space_id], + r.source_local_id!, + "relation", + ) + : `${r.space_id}/${r.source_local_id}`; + const relatedRelations = dbAllImportableRelations.filter((r) => { const references = (r.reference_content || {}) as Record; const sourceId = references["source"]; const destinationId = references["destination"]; @@ -94,22 +129,35 @@ export const discoverSharedRelations = async ( ); }); const relationSchemaIds = new Set( - dbRelations.map((r) => r.schema_id).filter((r) => r !== null), + relatedRelations.map((r) => r.schema_id).filter((r) => r !== null), ); - if (relationSchemaIds.size === 0) return response; - const { data: dbRelSchemas, error: relSchError } = await client - .from("my_concepts") - .select() - .in("id", [...relationSchemaIds]); + const { data: dbRelSchemas, error: relSchError } = + relationSchemaIds.size === 0 + ? { data: [], error: null } + : await client + .from("my_concepts") + .select() + .in("id", [...relationSchemaIds]); if (relSchError) throw relSchError; if (!dbRelSchemas) throw new Error("Missing schemas"); + // A group can see a relation without its schema, for instance one published before + // relation types and triples were granted with it. Such a relation cannot be + // converted; it is retried on every import, so it arrives once the schema is granted. + const visibleSchemaIds = new Set(dbRelSchemas.map(({ id }) => id)); + const dbRelations = relatedRelations.filter( + (r) => r.schema_id !== null && visibleSchemaIds.has(r.schema_id), + ); + response.skippedRelations = relatedRelations + .filter((r) => !dbRelations.includes(r)) + .map((r) => `${relationRid(r)}: its relation type is not visible`); + if (dbRelations.length === 0) return response; const dbRelTripleSchemasDirect = dbRelSchemas.filter( (r) => r.refs !== null && r.refs.length > 0, ) as Concept[]; const dbRelTypeSchemasDirect = dbRelSchemas.filter( (r) => r.refs === null || r.refs.length === 0, ) as Concept[]; - let dbRelTripleSchemas = dbRelTripleSchemasDirect; + const dbRelTripleSchemas = dbRelTripleSchemasDirect; let dbRelTypeSchemas = dbRelTypeSchemasDirect; const missingRelationTypeSchemaIds = new Set( @@ -132,11 +180,9 @@ export const discoverSharedRelations = async ( if (!data) throw new Error("Missing relation type schemas"); dbRelTypeSchemas = [...dbRelTypeSchemasDirect, ...(data as Concept[])]; } - // Obsidian relation instances point to the RelationType, - // so we need to filter relevant RelationTriples - // according to the relation instance's type signatures + // Obsidian relations point to a relation type. The importer matches the triple + // locally; these candidates only supply provenance. if (dbRelTypeSchemasDirect.length) { - // Fetch all corresponding triples and filter const relTypeIds = dbRelTypeSchemasDirect.map((r) => r.id); const { data, error: trsError } = await client .from("my_concepts") @@ -146,52 +192,116 @@ export const discoverSharedRelations = async ( .overlaps("refs", relTypeIds); if (trsError) throw trsError; if (!data) throw new Error("Missing relation triple schemas"); - const triplesBySchemaId: Record = Object.fromEntries( - relTypeIds.map((id) => [id, []]), + const candidateTriples = data as Concept[]; + const refsOf = (c: { reference_content: unknown }) => + (c.reference_content ?? {}) as Record; + const schemaRid = (c: { + space_id: number | null; + source_local_id: string | null; + }) => + c.space_id !== null && + c.space_id in spaceMap && + c.source_local_id !== null + ? spaceUriAndLocalIdToRid( + spaceMap[c.space_id], + c.source_local_id, + "schema", + ) + : undefined; + + // Triple end types may be hidden; relation end types are visible with the relation. + const endSchemaIds = new Set( + [ + ...candidateTriples.flatMap((t) => [ + refsOf(t).source, + refsOf(t).destination, + ]), + ...dbRelations.flatMap((r) => + r.concepts_of_relation.map(({ schema_id }) => schema_id), + ), + ].filter((id): id is number => typeof id === "number"), ); - data.forEach((c) => { - const triplesArray = - triplesBySchemaId[ - ((c.reference_content ?? {}) as Record)[ - "relation_type" - ] - ]; - if (triplesArray !== undefined) triplesArray.push(c as Concept); - }); - const tripleIds = new Set(); - for (const relation of dbRelations) { - const potentialTriples = triplesBySchemaId[relation.schema_id || 0]; - if (potentialTriples === undefined) continue; - const refs = (relation.reference_content || {}) as Record; - const sourceContent = relation.concepts_of_relation.filter( - (cr) => cr.id === refs["source"], - ); - const destinationContent = relation.concepts_of_relation.filter( - (cr) => cr.id === refs["destination"], - ); - if (sourceContent.length !== 1 || destinationContent.length !== 1) - continue; - const matches = potentialTriples.filter( - (triple) => - ((triple.reference_content ?? {}) as Record)[ - "source" - ] === sourceContent[0].schema_id && - ((triple.reference_content ?? {}) as Record)[ - "destination" - ] === destinationContent[0].schema_id, + const { data: endSchemas, error: esError } = await client + .from("my_concepts") + .select() + .in("id", [...endSchemaIds]); + if (esError) throw esError; + const identityById: Record = {}; + for (const s of endSchemas ?? []) { + if (s.id === null) continue; + const literal = (s.literal_content ?? {}) as Record; + identityById[s.id] = { + rid: schemaRid(s), + localId: s.source_local_id ?? undefined, + importedFromRid: + typeof literal.importedFromRid === "string" + ? literal.importedFromRid + : undefined, + name: s.name ?? undefined, + }; + } + + const relTypeById = Object.fromEntries( + dbRelTypeSchemasDirect.map((r) => [r.id, r]), + ); + const candidatesByRelationTypeId: Record< + number, + { + candidate: TripleCandidate; + source: NodeTypeIdentity; + destination: NodeTypeIdentity; + }[] + > = {}; + for (const triple of candidateTriples) { + const refs = refsOf(triple); + const relationType = relTypeById[refs.relation_type]; + const rid = schemaRid(triple); + if (relationType === undefined || rid === undefined) continue; + const literal = (triple.literal_content ?? {}) as Record; + const label = + typeof literal.label === "string" ? literal.label : relationType.name; + const complement = + typeof literal.complement === "string" ? literal.complement : undefined; + const names = parseTripleEndNames(triple.name, label); + (candidatesByRelationTypeId[relationType.id] ??= []).push({ + candidate: { + rid, + label, + complement, + modifiedAt: new Date(triple.last_modified + "Z"), + }, + source: identityById[refs.source] ?? { name: names?.source }, + destination: identityById[refs.destination] ?? { + name: names?.destination, + }, + }); + } + for (const [relationTypeId, candidates] of Object.entries( + candidatesByRelationTypeId, + )) { + const relationTypeRid = schemaRid(relTypeById[Number(relationTypeId)]); + if (relationTypeRid === undefined) continue; + response.tripleCandidatesByRelationType[relationTypeRid] = candidates.map( + ({ candidate }) => candidate, ); - if (matches.length === 1) { - const relationTripleSchemaId = matches[0].id; - tripleIds.add(relationTripleSchemaId); - // prentend that the obsidian relation referred to the triple - // for when we convert - relation.schema_id = relationTripleSchemaId; - } } - dbRelTripleSchemas = [ - ...dbRelTripleSchemasDirect, - ...(data as Concept[]).filter((tr) => tripleIds.has(tr.id || 0)), - ]; + + for (const relation of dbRelations) { + const candidates = candidatesByRelationTypeId[relation.schema_id ?? 0]; + if (candidates === undefined) continue; + const refs = refsOf(relation); + const endIdentity = (id: number): NodeTypeIdentity => { + const ends = relation.concepts_of_relation.filter((cr) => cr.id === id); + if (ends.length !== 1 || ends[0].schema_id === null) return {}; + return identityById[ends[0].schema_id] ?? {}; + }; + const picked = pickTripleCandidate(candidates, { + source: endIdentity(refs.source), + destination: endIdentity(refs.destination), + }); + if (picked === undefined) continue; + response.matchedTripleByRelation[relationRid(relation)] = picked.rid; + } } const nodeTypeSchemaIds = new Set( dbRelTripleSchemas @@ -246,5 +356,8 @@ export const discoverSharedRelations = async ( relTripleSchemas, relTypeSchemas, nodeSchemas, + tripleCandidatesByRelationType: response.tripleCandidatesByRelationType, + matchedTripleByRelation: response.matchedTripleByRelation, + skippedRelations: response.skippedRelations, }; }; diff --git a/apps/roam/src/utils/importSharedRelations.ts b/apps/roam/src/utils/importSharedRelations.ts index fbf5a8007b..f8576bf1e7 100644 --- a/apps/roam/src/utils/importSharedRelations.ts +++ b/apps/roam/src/utils/importSharedRelations.ts @@ -24,7 +24,14 @@ import { createReifiedRelation, getReifiedRelations, } from "./createReifiedBlock"; -import { discoverSharedRelations } from "./discoverSharedRelations"; +import { + discoverSharedRelations, + type DiscoverSharedRelationsResult, + type TripleCandidate, +} from "./discoverSharedRelations"; +import findDiscourseNode from "./findDiscourseNode"; +import internalError from "./internalError"; +import { getErrorMessage } from "./getErrorMessage"; import { DGSupabaseClient } from "@repo/database/lib/client"; import { deleteBlock } from "roamjs-components/writes"; import refreshConfigTree from "./refreshConfigTree"; @@ -79,13 +86,18 @@ const matchImportedNodeSchemas = async ( return result; }; -const matchImportedRelationSchemas = async ( - nodeSchemaRidToLocalId: Record, - relationTypeSchemas: CrossAppRelationTypeSchema[], - relationTripleSchemas: CrossAppRelationTripleSchema[], -): Promise> => { +const matchImportedRelationSchemas = async ({ + nodeSchemaRidToLocalId, + relationTypeSchemas, + relationTripleSchemas, + relationSchemas, +}: { + nodeSchemaRidToLocalId: Record; + relationTypeSchemas: CrossAppRelationTypeSchema[]; + relationTripleSchemas: CrossAppRelationTripleSchema[]; + relationSchemas: DiscourseRelation[]; +}): Promise> => { const result: Record = {}; - const relationSchemas = getDiscourseRelations(); const existing = await getImportedSourceRids(); const relationTypeSchemasByRid = Object.fromEntries( relationTypeSchemas.map((s) => [s.rid!, s]), @@ -178,83 +190,234 @@ const matchImportedRelationSchemas = async ( return result; }; +type RelationTripleContext = { + relationSchemas: DiscourseRelation[]; + importedRids: Set; +}; + +// A local triple fitting the pages' node types is correct by construction, so it wins; +// the source space's triple only supplies provenance for a new one. +const findOrImportRelationTriple = async ({ + relationType, + sourceUid, + destinationUid, + candidates, + matchedTripleRid, + context, +}: { + relationType: CrossAppRelationTypeSchema; + sourceUid: string; + destinationUid: string; + candidates: TripleCandidate[]; + matchedTripleRid?: string; + context: RelationTripleContext; +}): Promise => { + const sourceType = findDiscourseNode({ uid: sourceUid }); + if (!sourceType) throw new Error(`No node type for page: ${sourceUid}`); + const destinationType = findDiscourseNode({ uid: destinationUid }); + if (!destinationType) + throw new Error(`No node type for page: ${destinationUid}`); + const fitsPages = (r: DiscourseRelation) => + r.source === sourceType.type && r.destination === destinationType.type; + + const label = relationType.label; + // Each query pattern can produce a match for the same local schema. + const matchIds = [ + ...new Set( + context.relationSchemas + .filter( + (r) => fitsPages(r) && r.label.toLowerCase() === label.toLowerCase(), + ) + .map(({ id }) => id), + ), + ]; + if (matchIds.length > 1) throw new Error("multiple matches"); + if (matchIds.length === 1) return matchIds[0]; + + // Finds an imported triple even after a local rename. + for (const candidate of candidates) { + if (!context.importedRids.has(candidate.rid)) continue; + const uid = await findImportedNodeUidBySourceRid(candidate.rid); + const imported = context.relationSchemas.find((r) => r.id === uid); + if (imported && fitsPages(imported)) return imported.id; + } + + // Ends come from the pages: the source space's node types may be hidden. + const matched = candidates.find(({ rid }) => rid === matchedTripleRid); + const newRelation: Omit = { + label: matched?.label ?? label, + complement: matched?.complement ?? relationType.complement, + source: sourceType.type, + destination: destinationType.type, + triples: [], + }; + const id = await createRelationSchema(newRelation); + // One source RID names one local triple: the lookup above returns a single block. + if (matched && !context.importedRids.has(matched.rid)) { + await writeImportedSourceIdentity({ + pageUid: id, + sourceNodeRid: matched.rid, + sourceModifiedAt: (matched.modifiedAt ?? new Date()).toISOString(), + }); + context.importedRids.add(matched.rid); + } + context.relationSchemas.push({ ...newRelation, id }); + return id; +}; + +const importRelation = async ({ + relation, + schemaRidToLocalId, + discovered, + existing, + allRelations, + context, +}: { + relation: CrossAppRelation; + schemaRidToLocalId: Record; + discovered: DiscoverSharedRelationsResult; + existing: Set; + allRelations: Awaited>; + context: RelationTripleContext; +}): Promise => { + const { rid: sourceNodeRid, source, destination, relationType } = relation; + if (sourceNodeRid === undefined) return; + const { spaceUri } = ridToSpaceUriAndLocalId(sourceNodeRid); + const schemaRid = spaceUriAndLocalIdToRid(spaceUri, relationType, "schema"); + const sourceUid = await findTargetUid(source, spaceUri); + if (sourceUid === null) throw new Error(`Missing relation source: ${source}`); + const destinationUid = await findTargetUid(destination, spaceUri); + if (destinationUid === null) + throw new Error(`Missing relation destination: ${destination}`); + let relationBlockUid = schemaRidToLocalId[schemaRid]; + if (relationBlockUid === undefined) { + const relationTypeSchema = discovered.relTypeSchemas.find( + ({ rid }) => rid === schemaRid, + ); + if (relationTypeSchema === undefined) + throw new Error(`Missing relation type: ${relationType}`); + relationBlockUid = await findOrImportRelationTriple({ + relationType: relationTypeSchema, + sourceUid, + destinationUid, + candidates: discovered.tripleCandidatesByRelationType[schemaRid] ?? [], + matchedTripleRid: discovered.matchedTripleByRelation[sourceNodeRid], + context, + }); + } + if (existing.has(sourceNodeRid)) { + // Update existing + const existingRelUid = await findImportedNodeUidBySourceRid(sourceNodeRid); + if (existingRelUid === null) + throw new Error("Could not get imported block"); + const existingRel = allRelations.find( + (r) => r.relationId === existingRelUid, + ); + if (existingRel === undefined) throw new Error("Could not find relation"); + if ( + existingRel.hasSchema === relationBlockUid && + existingRel.sourceUid === sourceUid && + existingRel.destinationUid === destinationUid + ) + return; + // It was imported and modified. We could update, but easier to delete and recreate. + await deleteBlock(existingRelUid); + } + + const existingRel = allRelations.filter( + (r) => + r.hasSchema === relationBlockUid && + r.sourceUid === sourceUid && + r.destinationUid === destinationUid, + ); + if (existingRel.length > 1) throw new Error("Multiple matching relations"); + if (existingRel.length === 0) { + const uid = await createReifiedRelation({ + sourceUid, + destinationUid, + relationBlockUid, + tentative: true, + }); + await writeImportedSourceIdentity({ + pageUid: uid, + sourceNodeRid, + sourceModifiedAt: (relation.modifiedAt ?? new Date()).toISOString(), + }); + } +}; + const importRelations = async ( schemaRidToLocalId: Record, - relations: CrossAppRelation[], -): Promise => { + discovered: DiscoverSharedRelationsResult, + relationSchemas: DiscourseRelation[], +): Promise => { const existing = await getImportedSourceRids(); const allRelations = await getReifiedRelations(); - for (const relation of relations) { - const { rid: sourceNodeRid, source, destination, relationType } = relation; - if (sourceNodeRid === undefined) continue; - const { spaceUri } = ridToSpaceUriAndLocalId(sourceNodeRid); - const schemaRid = spaceUriAndLocalIdToRid(spaceUri, relationType, "schema"); - const relationBlockUid = schemaRidToLocalId[schemaRid]; - if (relationBlockUid === undefined) - throw new Error(`Missing relation type: ${relationType}`); - const sourceUid = await findTargetUid(source, spaceUri); - if (sourceUid === null) - throw new Error(`Missing relation source: ${source}`); - const destinationUid = await findTargetUid(destination, spaceUri); - if (destinationUid === null) - throw new Error(`Missing relation destination: ${destination}`); - if (existing.has(sourceNodeRid)) { - // Update existing - const existingRelUid = - await findImportedNodeUidBySourceRid(sourceNodeRid); - if (existingRelUid === null) - throw new Error("Could not get imported block"); - const existingRel = allRelations.find( - (r) => r.relationId === existingRelUid, - ); - if (existingRel === undefined) throw new Error("Could not find relation"); - if ( - existingRel.hasSchema === relationBlockUid && - existingRel.sourceUid === sourceUid && - existingRel.destinationUid === destinationUid - ) - continue; - // It was imported and modified. We could update, but easier to delete and recreate. - await deleteBlock(existingRelUid); - } - - const existingRel = allRelations.filter( - (r) => - r.hasSchema === relationBlockUid && - r.sourceUid === sourceUid && - r.destinationUid === destinationUid, - ); - if (existingRel.length > 1) throw new Error("Multiple matching relations"); - if (existingRel.length === 0) { - const uid = await createReifiedRelation({ - sourceUid, - destinationUid, - relationBlockUid, - tentative: true, - }); - await writeImportedSourceIdentity({ - pageUid: uid, - sourceNodeRid, - sourceModifiedAt: (relation.modifiedAt ?? new Date()).toISOString(), + const context: RelationTripleContext = { + relationSchemas, + importedRids: existing, + }; + const failures: string[] = []; + for (const relation of discovered.relations) { + try { + await importRelation({ + relation, + schemaRidToLocalId, + discovered, + existing, + allRelations, + context, }); + } catch (error) { + failures.push( + `${relation.rid ?? relation.localId}: ${getErrorMessage(error)}`, + ); } } + if (failures.length > 0) + internalError({ + error: new Error( + `${failures.length} of ${discovered.relations.length} shared relation imports failed`, + ), + type: "Shared relation import failed", + context: { + operation: "import-shared-relations", + failureMessages: failures, + }, + sendEmail: false, + }); + return failures; +}; + +// One message per relation: `failures` failed to import, `skipped` have a schema the +// reader cannot see. +export type SharedRelationImportResult = { + failures: string[]; + skipped: string[]; }; export const importSharedRelations = async ( client: DGSupabaseClient, spaceId: number, -) => { - const { relations, relTripleSchemas, relTypeSchemas, nodeSchemas } = - await discoverSharedRelations(client, spaceId); +): Promise => { + const discovered = await discoverSharedRelations(client, spaceId); + const { relTripleSchemas, relTypeSchemas, nodeSchemas } = discovered; + // Shared by both passes, so a triple the first creates is matched by the second. + const relationSchemas = getDiscourseRelations(); let ridToLocalId = await matchImportedNodeSchemas(nodeSchemas); - const relationSchemaMap = await matchImportedRelationSchemas( + const relationSchemaMap = await matchImportedRelationSchemas({ + nodeSchemaRidToLocalId: ridToLocalId, + relationTypeSchemas: relTypeSchemas, + relationTripleSchemas: relTripleSchemas, + relationSchemas, + }); + ridToLocalId = { ...ridToLocalId, ...relationSchemaMap }; + const failures = await importRelations( ridToLocalId, - relTypeSchemas, - relTripleSchemas, + discovered, + relationSchemas, ); - ridToLocalId = { ...ridToLocalId, ...relationSchemaMap }; - await importRelations(ridToLocalId, relations); // Legacy settings read the cached grammar, including newly imported schemas. refreshConfigTree(); + return { failures, skipped: discovered.skippedRelations }; }; diff --git a/apps/roam/src/utils/refreshAllImportedNodes.ts b/apps/roam/src/utils/refreshAllImportedNodes.ts index 785d1db392..6ba426bcad 100644 --- a/apps/roam/src/utils/refreshAllImportedNodes.ts +++ b/apps/roam/src/utils/refreshAllImportedNodes.ts @@ -1,5 +1,9 @@ import { getImportedNodeUids } from "./importedSourceIdentity"; -import { refreshImportedNode } from "./refreshImportedNode"; +import { + importRelationsAfterRefresh, + refreshImportedNode, +} from "./refreshImportedNode"; +import { getLoggedInClient } from "./supabaseContext"; type RefreshAllImportedNodesResult = { refreshed: number; @@ -18,10 +22,20 @@ export const refreshAllImportedNodes = warnings: [], }; for (const pageUid of pageUids) { - const result = await refreshImportedNode({ pageUid, force: false }); + const result = await refreshImportedNode({ + pageUid, + force: false, + importRelations: false, + }); counts[result.status] += 1; if (result.warning) counts.warnings.push(`${result.message} ${result.warning}`); } + if (counts.refreshed + counts.skipped === 0) return counts; + const client = await getLoggedInClient(); + const relationWarning = client + ? await importRelationsAfterRefresh(client) + : "Could not import relations: could not connect to shared persistence."; + if (relationWarning) counts.warnings.push(relationWarning); return counts; }; diff --git a/apps/roam/src/utils/refreshImportedNode.ts b/apps/roam/src/utils/refreshImportedNode.ts index ac97ee54b8..4fe766abde 100644 --- a/apps/roam/src/utils/refreshImportedNode.ts +++ b/apps/roam/src/utils/refreshImportedNode.ts @@ -5,7 +5,9 @@ import internalError from "./internalError"; import { getErrorMessage } from "./getErrorMessage"; import { materializeSharedNode } from "./materializeSharedNode"; import { resolveSharedNodeTypes } from "./resolveSharedNodeTypes"; -import { getLoggedInClient } from "./supabaseContext"; +import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; +import { importSharedRelations } from "./importSharedRelations"; +import type { DGSupabaseClient } from "@repo/database/lib/client"; export const REFRESH_ERROR_TYPE = "Imported node refresh failed"; const REFRESH_ERROR_OPERATION = "refresh-imported-node"; @@ -16,12 +18,46 @@ type RefreshImportedNodeResult = { warning?: string; }; +// A relation published after both of its ends were imported arrives only with a later +// import or refresh. Returns a warning when some relations could not be imported. +export const importRelationsAfterRefresh = async ( + client: DGSupabaseClient, +): Promise => { + try { + const context = await getSupabaseContext(); + if (!context) + return "Could not import relations: could not connect to shared persistence."; + // Skipped relations are not reported: they need not involve the refreshed node. + const { failures } = await importSharedRelations(client, context.spaceId); + if (failures.length === 0) return undefined; + return failures.length === 1 + ? "1 relation could not be imported." + : `${failures.length} relations could not be imported.`; + } catch (error) { + internalError({ + error, + type: REFRESH_ERROR_TYPE, + context: { operation: "import-relations-after-refresh" }, + sendEmail: false, + }); + return `Could not import relations: ${getErrorMessage(error)}`; + } +}; + +const joinWarnings = (...warnings: (string | undefined)[]) => { + const present = warnings.filter((w) => w !== undefined); + return present.length > 0 ? { warning: present.join(" ") } : {}; +}; + +// Refresh-all passes `importRelations: false` and imports relations once at the end. export const refreshImportedNode = async ({ pageUid, force, + importRelations = true, }: { pageUid: string; force: boolean; + importRelations?: boolean; }): Promise => { try { const title = getPageTitleByPageUid(pageUid); @@ -77,15 +113,19 @@ export const refreshImportedNode = async ({ status: "failed", message: `A different page ("${getPageTitleByPageUid(result.pageUid)}") is linked to the same source and was refreshed instead.`, }; + const relationWarning = importRelations + ? await importRelationsAfterRefresh(client) + : undefined; if (result.action === "skipped") return { status: "skipped", message: `"${sharedNode.title}" is already up to date.`, + ...joinWarnings(relationWarning), }; return { status: "refreshed", message: `Refreshed "${sharedNode.title}" from ${sharedNode.spaceName}.`, - ...(result.warning ? { warning: result.warning } : {}), + ...joinWarnings(result.warning, relationWarning), }; } catch (error) { internalError({