From dc415ea94577f71d05180b142e36c4706c998aab Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Tue, 6 Oct 2026 23:29:27 -0400 Subject: [PATCH] ENG-2372 Record the source triple's RID on triples created by Obsidian relation import --- .../utils/__tests__/repairTripleRids.test.ts | 41 ++++ .../src/utils/__tests__/sourceTriple.test.ts | 147 ++++++++++++ apps/obsidian/src/utils/importRelations.ts | 89 +++++-- apps/obsidian/src/utils/repairTripleRids.ts | 97 ++++++++ apps/obsidian/src/utils/sourceTriple.ts | 223 ++++++++++++++++++ .../src/utils/syncDgNodesToSupabase.ts | 6 + 6 files changed, 579 insertions(+), 24 deletions(-) create mode 100644 apps/obsidian/src/utils/__tests__/repairTripleRids.test.ts create mode 100644 apps/obsidian/src/utils/__tests__/sourceTriple.test.ts create mode 100644 apps/obsidian/src/utils/repairTripleRids.ts create mode 100644 apps/obsidian/src/utils/sourceTriple.ts diff --git a/apps/obsidian/src/utils/__tests__/repairTripleRids.test.ts b/apps/obsidian/src/utils/__tests__/repairTripleRids.test.ts new file mode 100644 index 0000000000..6416dce246 --- /dev/null +++ b/apps/obsidian/src/utils/__tests__/repairTripleRids.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vitest"; +import type { DiscourseRelation } from "~/types"; +import { findTriplesWithRelationRid } from "~/utils/repairTripleRids"; + +const triple = (id: string, importedFromRid?: string): DiscourseRelation => ({ + id, + sourceId: "clm", + destinationId: "evd", + relationshipTypeId: "sup", + created: 0, + modified: 0, + importedFromRid, +}); + +describe("findTriplesWithRelationRid", () => { + it("finds a triple whose RID has the relation subtype", () => { + const bad = triple("t1", "orn:obsidian.relation:vault-a/rel-1"); + expect(findTriplesWithRelationRid([bad], new Set())).toEqual([bad]); + }); + + it("finds a triple whose RID is an imported relation instance's", () => { + const roamRid = "https://roamresearch.com/#/app/graph/page/relUid"; + const bad = triple("t1", roamRid); + expect(findTriplesWithRelationRid([bad], new Set([roamRid]))).toEqual([ + bad, + ]); + }); + + it("leaves local triples and triples with a schema RID alone", () => { + expect( + findTriplesWithRelationRid( + [ + triple("t1"), + triple("t2", "orn:obsidian.schema:vault-a/triple-1"), + triple("t3", "https://roamresearch.com/#/app/graph/page/tripleUid"), + ], + new Set(["orn:obsidian.relation:vault-a/other"]), + ), + ).toEqual([]); + }); +}); diff --git a/apps/obsidian/src/utils/__tests__/sourceTriple.test.ts b/apps/obsidian/src/utils/__tests__/sourceTriple.test.ts new file mode 100644 index 0000000000..9ca54563f2 --- /dev/null +++ b/apps/obsidian/src/utils/__tests__/sourceTriple.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it } from "vitest"; +import type { RemoteRelationInstance } from "~/utils/importRelations"; +import { type SchemaRow, selectSourceTripleRids } from "~/utils/sourceTriple"; + +const A = 10; +const B = 20; +const spaceUriById = new Map([ + [A, "obsidian:vault-a"], + [B, "obsidian:vault-b"], +]); + +const schema = ( + row: Pick & + Partial, +): SchemaRow => ({ literal_content: {}, reference_content: {}, ...row }); + +const claim = schema({ + id: 1, + space_id: A, + source_local_id: "clm", + name: "Claim", +}); +const evidenceA = schema({ + id: 2, + space_id: A, + source_local_id: "evd-a", + name: "Evidence", + literal_content: { importedFromRid: "orn:obsidian.schema:vault-b/evd-b" }, +}); +const evidenceB = schema({ + id: 3, + space_id: B, + source_local_id: "evd-b", + name: "Evidence", +}); +const supports = schema({ + id: 4, + space_id: A, + source_local_id: "sup", + name: "Supports", + literal_content: { label: "supports" }, +}); +const triple = (id: number, localId: string, refs: Record) => + schema({ + id, + space_id: A, + source_local_id: localId, + name: "Claim -supports-> Evidence", + reference_content: { relation_type: supports.id, ...refs }, + }); + +// X is A's own claim; Y is B's evidence node, which A imported. +const relation = (schemaId: number): RemoteRelationInstance => ({ + id: 100, + source_local_id: "rel-1", + schema_id: schemaId, + reference_content: { source: 50, destination: 60 }, + refs: [50, 60], + created: null, + last_modified: null, + author_id: null, + concepts_of_relation: [ + { id: 50, space_id: A, source_local_id: "x", schema_id: claim.id }, + { id: 60, space_id: B, source_local_id: "y", schema_id: evidenceB.id }, + ], +}); + +const byId = (...rows: SchemaRow[]) => new Map(rows.map((r) => [r.id, r])); + +describe("selectSourceTripleRids", () => { + it("uses the schema itself when the relation's schema is a triple", () => { + const roamTriple = triple(5, "roam-triple", { source: 1, destination: 3 }); + expect( + selectSourceTripleRids({ + relations: [relation(roamTriple.id)], + schemasById: byId(roamTriple, claim, evidenceB), + candidateTriples: [], + spaceUriById, + }), + ).toEqual(new Map([[100, "orn:obsidian.schema:vault-a/roam-triple"]])); + }); + + it("picks the relation space's triple whose ends match the relation's end types", () => { + const fits = triple(5, "t-fits", { source: 1, destination: 2 }); + const other = triple(6, "t-other", { source: 1, destination: 1 }); + expect( + selectSourceTripleRids({ + relations: [relation(supports.id)], + schemasById: byId(supports, claim, evidenceA, evidenceB), + candidateTriples: [fits, other], + spaceUriById, + }), + ).toEqual(new Map([[100, "orn:obsidian.schema:vault-a/t-fits"]])); + }); + + it("matches a hidden end type by the name in the triple's name", () => { + const hiddenEnd = triple(5, "t-hidden", { source: 1, destination: 2 }); + expect( + selectSourceTripleRids({ + relations: [relation(supports.id)], + schemasById: byId(supports, claim, evidenceB), + candidateTriples: [hiddenEnd], + spaceUriById, + }), + ).toEqual(new Map([[100, "orn:obsidian.schema:vault-a/t-hidden"]])); + }); + + it("ignores triples from other spaces", () => { + const elsewhere = { + ...triple(5, "t-b", { source: 1, destination: 3 }), + space_id: B, + }; + expect( + selectSourceTripleRids({ + relations: [relation(supports.id)], + schemasById: byId(supports, claim, evidenceB), + candidateTriples: [elsewhere], + spaceUriById, + }).size, + ).toBe(0); + }); + + it("records nothing when two triples match equally", () => { + expect( + selectSourceTripleRids({ + relations: [relation(supports.id)], + schemasById: byId(supports, claim, evidenceB), + candidateTriples: [ + triple(5, "t-one", { source: 1, destination: 90 }), + triple(6, "t-two", { source: 1, destination: 91 }), + ], + spaceUriById, + }).size, + ).toBe(0); + }); + + it("records nothing when the relation's schema is not visible", () => { + expect( + selectSourceTripleRids({ + relations: [relation(supports.id)], + schemasById: byId(claim, evidenceB), + candidateTriples: [triple(5, "t", { source: 1, destination: 2 })], + spaceUriById, + }).size, + ).toBe(0); + }); +}); diff --git a/apps/obsidian/src/utils/importRelations.ts b/apps/obsidian/src/utils/importRelations.ts index 51480e9e7d..45ebe8d66a 100644 --- a/apps/obsidian/src/utils/importRelations.ts +++ b/apps/obsidian/src/utils/importRelations.ts @@ -16,11 +16,13 @@ import { findExistingTriple, findLocalRelationTypeMatch, } from "./schemaMatching"; +import { fetchSourceTripleRids } from "./sourceTriple"; type ConceptInRelation = { id: number; space_id: number; source_local_id: string; + schema_id: number | null; }; export type RemoteRelationInstance = { @@ -168,24 +170,29 @@ const findOrCreateTriple = async ({ }; /** - * Fetch relation instances from a remote space. Relation instances are concepts with - * is_schema=false and schema_id pointing to a relation type + * Fetch relation instances from a remote space, or only those with the given local ids. + * Relation instances are concepts with is_schema=false and schema_id pointing to a relation + * type (Obsidian) or a triple (Roam). */ export const fetchRelationInstancesFromSpace = async ({ client, spaceId, + sourceLocalIds, }: { client: DGSupabaseClient; spaceId: number; + sourceLocalIds?: string[]; }): Promise => { - const { data: instances, error } = await client + let query = client .from("my_concepts") .select( - "id, source_local_id, schema_id, reference_content, refs, created, last_modified, author_id, concepts_of_relation!inner(id, space_id, source_local_id)", + "id, source_local_id, schema_id, reference_content, refs, created, last_modified, author_id, concepts_of_relation!inner(id, space_id, source_local_id, schema_id)", ) .eq("space_id", spaceId) .eq("is_schema", false) .eq("is_relation", true); + if (sourceLocalIds) query = query.in("source_local_id", sourceLocalIds); + const { data: instances, error } = await query; if (error || !instances) { console.warn("Error fetching relation instances:", error); @@ -195,6 +202,36 @@ export const fetchRelationInstancesFromSpace = async ({ return instances as unknown as RemoteRelationInstance[]; }; +const resolveRelationEnds = ( + rel: RemoteRelationInstance, + keyToRelationEndpointId: Map, +): + | { + sourceData: ConceptInRelation; + destData: ConceptInRelation; + sourceEndpointId: string; + destEndpointId: string; + } + | undefined => { + const refs = rel.reference_content as Record; + const sourceData = rel.concepts_of_relation.find( + (cor) => cor.id === refs.source, + ); + const destData = rel.concepts_of_relation.find( + (cor) => cor.id === refs.destination, + ); + if (!sourceData || !destData) return undefined; + + const sourceEndpointId = keyToRelationEndpointId.get( + `${sourceData.space_id}:${sourceData.source_local_id}`, + ); + const destEndpointId = keyToRelationEndpointId.get( + `${destData.space_id}:${destData.source_local_id}`, + ); + if (!sourceEndpointId || !destEndpointId) return undefined; + return { sourceData, destData, sourceEndpointId, destEndpointId }; +}; + /** * Import relations where both source and destination resolve in this vault (imported or local). * keyToRelationEndpointId maps "spaceId:source_local_id" -> endpoint id (RID or nodeInstanceId) to store in RelationInstance. @@ -248,27 +285,28 @@ export const importRelationsForImportedNodes = async ({ } } - for (const rel of relationInstances) { - const sourceData = rel.concepts_of_relation.find( - (cor) => - cor.id === - (rel.reference_content as Record).source, - ); - const destData = rel.concepts_of_relation.find( - (cor) => - cor.id === - (rel.reference_content as Record) - .destination, - ); - if (!sourceData || !destData) continue; - - const sourceKey = `${sourceData.space_id}:${sourceData.source_local_id}`; - const destKey = `${destData.space_id}:${destData.source_local_id}`; + const importable = relationInstances.flatMap((rel) => { + const ends = resolveRelationEnds(rel, keyToRelationEndpointId); + return ends ? [{ rel, ...ends }] : []; + }); - const sourceEndpointId = keyToRelationEndpointId.get(sourceKey); - const destEndpointId = keyToRelationEndpointId.get(destKey); - if (!sourceEndpointId || !destEndpointId) continue; + let sourceTripleRids = new Map(); + try { + sourceTripleRids = await fetchSourceTripleRids({ + client, + relations: importable.map(({ rel }) => rel), + }); + } catch (error) { + console.warn("Could not look up the source triples of relations:", error); + } + for (const { + rel, + sourceData, + destData, + sourceEndpointId, + destEndpointId, + } of importable) { if (!rel.schema_id) continue; const sourceRelationTypeId = schemaMap.get(rel.schema_id); @@ -360,7 +398,10 @@ export const importRelationsForImportedNodes = async ({ relationTypeId: mappedTypeId, importedCreatedAt, importedModifiedAt, - importedFromRid: relationImportedFromRid, + // Without a unique source triple, keep the relation's RID: an imported triple + // needs some importedFromRid, or isAcceptedSchema treats it as a local one. + importedFromRid: + sourceTripleRids.get(rel.id) ?? relationImportedFromRid, authorId, }); } diff --git a/apps/obsidian/src/utils/repairTripleRids.ts b/apps/obsidian/src/utils/repairTripleRids.ts new file mode 100644 index 0000000000..75fe3d5211 --- /dev/null +++ b/apps/obsidian/src/utils/repairTripleRids.ts @@ -0,0 +1,97 @@ +import { ridToSpaceUriAndLocalId } from "@repo/database/lib/rid"; +import type DiscourseGraphPlugin from "~/index"; +import type { DiscourseRelation } from "~/types"; +import { fetchRelationInstancesFromSpace } from "./importRelations"; +import { loadRelations } from "./relationsStore"; +import { fetchSourceTripleRids } from "./sourceTriple"; +import { getSpaceIdsBySpaceUris } from "./spaceFromRid"; +import { getLoggedInClient } from "./supabaseContext"; + +// URL RIDs (Roam spaces) have no subtype. +const ridSubtype = (rid: string): string | undefined => + rid.match(/^orn:\w+\.(\w+):/)?.[1]; + +/** + * Triples that older relation imports stamped with the creating instance's RID: a + * `relation` RID, or, where RIDs carry no subtype, an imported instance's RID. + */ +export const findTriplesWithRelationRid = ( + triples: DiscourseRelation[], + relationInstanceRids: Set, +): DiscourseRelation[] => + triples.filter( + ({ importedFromRid }) => + importedFromRid !== undefined && + (ridSubtype(importedFromRid) === "relation" || + relationInstanceRids.has(importedFromRid)), + ); + +/** + * A triple with no unique source triple keeps its RID, as in relation import. Repaired + * triples get a new `modified` so the next sync uploads the accepted ones. + */ +export const repairImportedTripleRids = async ( + plugin: DiscourseGraphPlugin, +): Promise => { + const relationsData = await loadRelations(plugin); + const relationInstanceRids = new Set( + Object.values(relationsData.relations).flatMap(({ importedFromRid }) => + importedFromRid ? [importedFromRid] : [], + ), + ); + const badTriples = findTriplesWithRelationRid( + plugin.settings.discourseRelations ?? [], + relationInstanceRids, + ); + if (badTriples.length === 0) return; + + const client = await getLoggedInClient(plugin); + if (!client) return; + + const badRids = [...new Set(badTriples.map((t) => t.importedFromRid!))]; + const parsed = badRids.map((rid) => ({ + rid, + ...ridToSpaceUriAndLocalId(rid), + })); + const spaceIdsByUri = await getSpaceIdsBySpaceUris( + client, + parsed.map(({ spaceUri }) => spaceUri), + ); + + const replacements = new Map(); + for (const [spaceUri, spaceId] of spaceIdsByUri) { + const inSpace = parsed.filter((p) => p.spaceUri === spaceUri); + const relations = await fetchRelationInstancesFromSpace({ + client, + spaceId, + sourceLocalIds: inSpace.map(({ sourceLocalId }) => sourceLocalId), + }); + const tripleRids = await fetchSourceTripleRids({ client, relations }); + for (const rel of relations) { + const tripleRid = tripleRids.get(rel.id); + const badRid = inSpace.find( + ({ sourceLocalId }) => sourceLocalId === rel.source_local_id, + )?.rid; + if (tripleRid !== undefined && badRid !== undefined) + replacements.set(badRid, tripleRid); + } + } + console.debug( + `Triple RID repair: ${badRids.length} relation RID(s) found on triples, ${replacements.size} matched a source triple`, + ); + if (replacements.size === 0) return; + + const now = Date.now(); + plugin.settings.discourseRelations = plugin.settings.discourseRelations.map( + (triple) => { + const tripleRid = + triple.importedFromRid === undefined + ? undefined + : replacements.get(triple.importedFromRid); + return tripleRid === undefined + ? triple + : { ...triple, importedFromRid: tripleRid, modified: now }; + }, + ); + await plugin.saveSettings(); +}; diff --git a/apps/obsidian/src/utils/sourceTriple.ts b/apps/obsidian/src/utils/sourceTriple.ts new file mode 100644 index 0000000000..ba285e7cc4 --- /dev/null +++ b/apps/obsidian/src/utils/sourceTriple.ts @@ -0,0 +1,223 @@ +import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { + type NodeTypeIdentity, + parseTripleEndNames, + pickTripleCandidate, +} from "@repo/database/lib/tripleMatching"; +import { getSpaceInfoFromIds } from "./importNodes"; +import type { RemoteRelationInstance } from "./importRelations"; +import { buildSchemaRid } from "./schemaMatching"; + +export type SchemaRow = { + id: number; + space_id: number; + source_local_id: string; + name: string; + literal_content: unknown; + reference_content: unknown; +}; + +const SCHEMA_COLUMNS = + "id, space_id, source_local_id, name, literal_content, reference_content"; + +const refsOf = (row: { reference_content: unknown }): Record => + typeof row.reference_content === "object" && row.reference_content !== null + ? (row.reference_content as Record) + : {}; + +const literalOf = (row: { + literal_content: unknown; +}): Record => + typeof row.literal_content === "object" && row.literal_content !== null + ? (row.literal_content as Record) + : {}; + +// A triple references its ends; a relation type references nothing. +const isTriple = (row: SchemaRow): boolean => + refsOf(row).source !== undefined && refsOf(row).destination !== undefined; + +const endSchemaIds = (rel: RemoteRelationInstance): number[] => + rel.concepts_of_relation.flatMap(({ schema_id }) => + schema_id === null ? [] : [schema_id], + ); + +/** + * Returns source triple RIDs by relation concept id, for relations with a unique match. + * A Roam relation's schema is already a triple; an Obsidian relation's schema is a + * relation type, so its space's triples of that type are matched on the end types. + */ +export const selectSourceTripleRids = ({ + relations, + schemasById, + candidateTriples, + spaceUriById, +}: { + relations: RemoteRelationInstance[]; + schemasById: Map; + candidateTriples: SchemaRow[]; + spaceUriById: Map; +}): Map => { + const ridOf = (row: SchemaRow): string | undefined => { + const spaceUri = spaceUriById.get(row.space_id); + return spaceUri === undefined + ? undefined + : buildSchemaRid({ spaceUri, localId: row.source_local_id }); + }; + const identityOf = (row: SchemaRow): NodeTypeIdentity => { + const importedFromRid = literalOf(row).importedFromRid; + return { + rid: ridOf(row), + localId: row.source_local_id, + importedFromRid: + typeof importedFromRid === "string" ? importedFromRid : undefined, + name: row.name, + }; + }; + + const result = new Map(); + for (const rel of relations) { + const schema = + rel.schema_id === null ? undefined : schemasById.get(rel.schema_id); + if (schema === undefined) continue; + if (isTriple(schema)) { + const rid = ridOf(schema); + if (rid !== undefined) result.set(rel.id, rid); + continue; + } + + const endIdentity = (conceptId: number | undefined): NodeTypeIdentity => { + const end = rel.concepts_of_relation.find(({ id }) => id === conceptId); + const endSchema = + end?.schema_id == null ? undefined : schemasById.get(end.schema_id); + return endSchema === undefined ? {} : identityOf(endSchema); + }; + const relRefs = refsOf(rel); + const label = literalOf(schema).label; + const candidates = candidateTriples + .filter( + (triple) => + triple.space_id === schema.space_id && + refsOf(triple).relation_type === schema.id, + ) + .flatMap((triple) => { + const rid = ridOf(triple); + if (rid === undefined) return []; + // An end type the reader cannot see is known only by its name in the triple's name. + const names = parseTripleEndNames( + triple.name, + typeof label === "string" ? label : schema.name, + ); + const end = (role: "source" | "destination"): NodeTypeIdentity => { + const endSchema = schemasById.get(refsOf(triple)[role] ?? -1); + return endSchema === undefined + ? { name: names?.[role] } + : identityOf(endSchema); + }; + return [ + { + candidate: rid, + source: end("source"), + destination: end("destination"), + }, + ]; + }); + const picked = pickTripleCandidate(candidates, { + source: endIdentity(relRefs.source), + destination: endIdentity(relRefs.destination), + }); + if (picked !== undefined) result.set(rel.id, picked); + } + return result; +}; + +const fetchSchemas = async ( + client: DGSupabaseClient, + ids: number[], +): Promise => { + if (ids.length === 0) return []; + const { data, error } = await client + .from("my_concepts") + .select(SCHEMA_COLUMNS) + .in("id", ids); + if (error) throw error; + return (data ?? []) as SchemaRow[]; +}; + +export const fetchSourceTripleRids = async ({ + client, + relations, +}: { + client: DGSupabaseClient; + relations: RemoteRelationInstance[]; +}): Promise> => { + if (relations.length === 0) return new Map(); + const schemasById = new Map(); + const addSchemas = (rows: SchemaRow[]) => { + for (const row of rows) schemasById.set(row.id, row); + }; + + addSchemas( + await fetchSchemas(client, [ + ...new Set( + relations.flatMap((rel) => [ + ...(rel.schema_id === null ? [] : [rel.schema_id]), + ...endSchemaIds(rel), + ]), + ), + ]), + ); + + const relationTypeIds = [ + ...new Set( + relations.flatMap((rel) => { + const schema = + rel.schema_id === null ? undefined : schemasById.get(rel.schema_id); + return schema === undefined || isTriple(schema) ? [] : [schema.id]; + }), + ), + ]; + let candidateTriples: SchemaRow[] = []; + if (relationTypeIds.length > 0) { + const { data, error } = await client + .from("my_concepts") + .select(SCHEMA_COLUMNS) + .eq("is_schema", true) + .eq("is_relation", true) + .overlaps("refs", relationTypeIds); + if (error) throw error; + candidateTriples = (data ?? []) as SchemaRow[]; + addSchemas( + await fetchSchemas(client, [ + ...new Set( + candidateTriples + .flatMap((triple) => [ + refsOf(triple).source, + refsOf(triple).destination, + ]) + .filter( + (id): id is number => + typeof id === "number" && !schemasById.has(id), + ), + ), + ]), + ); + } + + const spaceInfo = await getSpaceInfoFromIds(client, [ + ...new Set( + [...schemasById.values(), ...candidateTriples].map( + ({ space_id }) => space_id, + ), + ), + ]); + const spaceUriById = new Map( + [...spaceInfo].map(([id, { url }]) => [id, url]), + ); + + return selectSourceTripleRids({ + relations, + schemasById, + candidateTriples, + spaceUriById, + }); +}; diff --git a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts index 454cb9c1bf..40880fc696 100644 --- a/apps/obsidian/src/utils/syncDgNodesToSupabase.ts +++ b/apps/obsidian/src/utils/syncDgNodesToSupabase.ts @@ -36,6 +36,7 @@ import { collectDiscourseNodesFromVault, } from "./getDiscourseNodes"; import { isAcceptedSchema } from "./typeUtils"; +import { repairImportedTripleRids } from "./repairTripleRids"; import { diffSchemaIds, findIdsMissingSchema } from "./schemaReconciliation"; import { getTemplatePluginInfo } from "./templates"; import { difference } from "@repo/utils/setOperations"; @@ -1269,6 +1270,11 @@ export const initializeSupabaseSync = async ( ); } + // Before the sync, so it uploads the repaired triples. + await repairImportedTripleRids(plugin).catch((error) => { + console.error("Failed to repair imported triple RIDs:", error); + }); + await syncAllNodesAndRelations(plugin, context).catch((error) => { new Notice(`Initial sync failed: ${error}`); console.error("Initial sync failed:", error);