ENG-1734 Add Roam semantic search to Advanced Node Search

apps/roam/src/components/AdvancedNodeSearchDialog/AdvancedSearchDialog.tsx

@@ -171,7 +171,7 @@ const AdvancedNodeSearchDialog = ({
     discourseNodes.map((node) => [node.type, node]),
   );
 
-  const results = useAdvancedNodeSearchResults({
+  const { results, isSearching } = useAdvancedNodeSearchResults({
     debouncedSearchTerm,
     selectedNodeTypeIds,
     sort,
@@ -291,7 +291,7 @@ const AdvancedNodeSearchDialog = ({
 
   const contentState = indexError
     ? "error"
-    : isIndexLoading
+    : isIndexLoading || isSearching
       ? "indexing"
       : !debouncedSearchTerm && selectedNodeTypeIds.length === 0
         ? "initial"

apps/roam/src/components/AdvancedNodeSearchDialog/AdvancedSearchSidebarPanel.tsx

@@ -226,7 +226,7 @@ export const AdvancedSearchSidebarPanel = ({
     };
   }, [discourseNodes]);
 
-  const results = useAdvancedNodeSearchResults({
+  const { results, isSearching } = useAdvancedNodeSearchResults({
     debouncedSearchTerm,
     selectedNodeTypeIds,
     sort,
@@ -306,11 +306,14 @@ export const AdvancedSearchSidebarPanel = ({
                     results={results}
                   />
                 )}
-                {debouncedSearchTerm && !results.length && !isIndexLoading && (
-                  <p className="px-2 py-3 text-sm text-gray-500">
-                    No matches. Try another keyword.
-                  </p>
-                )}
+                {debouncedSearchTerm &&
+                  !results.length &&
+                  !isIndexLoading &&
+                  !isSearching && (
+                    <p className="px-2 py-3 text-sm text-gray-500">
+                      No matches. Try another keyword.
+                    </p>
+                  )}
               </>
             )}
           </>

apps/roam/src/components/AdvancedNodeSearchDialog/useAdvancedNodeSearchResults.ts

@@ -1,8 +1,11 @@
-import { useMemo } from "react";
+import { useEffect, useMemo, useState } from "react";
 import MiniSearch from "minisearch";
+import getDiscourseNodes from "~/utils/getDiscourseNodes";
+import { searchDiscourseNodes } from "~/utils/searchDiscourseNodes";
 import {
-  searchIndexedNodes,
+  searchDiscourseNodesWithMiniSearch,
   sortSearchResults,
+  type ScoredSearchResult,
   type SearchResult,
   type SortConfig,
 } from "./utils";
@@ -32,37 +35,110 @@ export const useAdvancedNodeSearchResults = ({
   searchIndex,
   dockedQuery,
   dockedResults,
-}: UseAdvancedNodeSearchResultsArgs): SearchResult[] =>
-  useMemo(() => {
-    if (!debouncedSearchTerm) return [];
-
-    const isDockedQuery =
+}: UseAdvancedNodeSearchResultsArgs): {
+  results: SearchResult[];
+  isSearching: boolean;
+} => {
+  // An empty persisted result set is not a usable cache: the docked panel also
+  // persists the transient [] published while an async search is in flight.
+  const hasUsableDockedResults = useMemo(
+    () =>
       dockedQuery !== undefined &&
-      debouncedSearchTerm.trim() === dockedQuery.trim();
+      debouncedSearchTerm.trim() === dockedQuery.trim() &&
+      !!dockedResults &&
+      dockedResults.length > 0,
+    [debouncedSearchTerm, dockedQuery, dockedResults],
+  );
+
+  const [unsortedScoredResults, setUnsortedScoredResults] = useState<
+    ScoredSearchResult[]
+  >([]);
+  const [isSearching, setIsSearching] = useState(false);
 
-    if (isDockedQuery && dockedResults) {
-      return dockedResults;
+  useEffect(() => {
+    if (hasUsableDockedResults) {
+      setUnsortedScoredResults([]);
+      setIsSearching(false);
+      return;
+    }
+
+    if (!debouncedSearchTerm) {
+      setUnsortedScoredResults([]);
+      setIsSearching(false);
+      return;
     }
 
     if (isIndexLoading || indexError || !searchIndex) {
-      return [];
+      setUnsortedScoredResults([]);
+      setIsSearching(false);
+      return;
     }
 
-    const scoredHits = searchIndexedNodes({
-      miniSearch: searchIndex.miniSearch,
-      allResults: searchIndex.allResults,
-      searchTerm: debouncedSearchTerm,
-      typeFilter: selectedNodeTypeIds.length ? selectedNodeTypeIds : undefined,
-    });
+    setUnsortedScoredResults([]);
+    setIsSearching(true);
+    let cancelled = false;
+    const typeFilter = selectedNodeTypeIds.length
+      ? selectedNodeTypeIds
+      : undefined;
+    const discourseNodes = getDiscourseNodes().filter(
+      (node) =>
+        node.backedBy === "user" &&
+        (!typeFilter || typeFilter.includes(node.type)),
+    );
+    const resultsByUid = new Map(
+      searchIndex.allResults.map((result) => [result.uid, result]),
+    );
+
+    const runMiniSearch = (): ScoredSearchResult[] =>
+      searchDiscourseNodesWithMiniSearch({
+        miniSearch: searchIndex.miniSearch,
+        allResults: searchIndex.allResults,
+        searchTerm: debouncedSearchTerm,
+        typeFilter,
+      });
 
-    return sortSearchResults({ hits: scoredHits, sort });
+    void searchDiscourseNodes({
+      nodeTypes: discourseNodes,
+      query: debouncedSearchTerm,
+      resultsByUid,
+      runMiniSearch,
+    })
+      .then((results) => {
+        if (cancelled) return;
+        setUnsortedScoredResults(results);
+      })
+      .catch(() => {
+        if (cancelled) return;
+        try {
+          setUnsortedScoredResults(runMiniSearch());
+        } catch {
+          setUnsortedScoredResults([]);
+        }
+      })
+      .finally(() => {
+        if (!cancelled) setIsSearching(false);
+      });
+
+    return () => {
+      cancelled = true;
+    };
   }, [
     debouncedSearchTerm,
-    dockedQuery,
-    dockedResults,
+    hasUsableDockedResults,
     indexError,
     isIndexLoading,
     searchIndex,
     selectedNodeTypeIds,
-    sort,
   ]);
+
+  const results = useMemo(
+    () => sortSearchResults({ scoredResults: unsortedScoredResults, sort }),
+    [unsortedScoredResults, sort],
+  );
+
+  if (hasUsableDockedResults && dockedResults) {
+    return { results: dockedResults, isSearching: false };
+  }
+
+  return { results, isSearching };
+};

apps/roam/src/components/AdvancedNodeSearchDialog/utils.ts

@@ -9,9 +9,19 @@ import {
   getPulledDiscourseNodeUid,
   queryDiscourseNodesByFormat,
 } from "~/utils/discourseNodeSearch";
+import {
+  MAX_RESULTS,
+  type ScoredSearchResult,
+  type SearchResult,
+} from "~/utils/discourseNodeSearchTypes";
+
+export type {
+  ScoredSearchResult,
+  SearchResult,
+} from "~/utils/discourseNodeSearchTypes";
 
 export const DEBOUNCE_MS = 250;
-export const MAX_RESULTS = 50;
+export { MAX_RESULTS };
 
 export const getSearchKeywords = (searchTerm: string): string[] =>
   searchTerm.split(/\s+/).filter(Boolean);
@@ -36,17 +46,6 @@ export const SORT_FIELD_LABELS: Record<SortField, string> = {
   author: "Author",
 };
 
-export type SearchResult = {
-  uid: string;
-  title: string;
-  type: string;
-  nodeTypeLabel: string;
-  excerpt: string;
-  createdAt: string;
-  lastModified: string;
-  authorName: string;
-};
-
 export type DockedSearchState = {
   query: string;
   results: SearchResult[];
@@ -56,11 +55,6 @@ export type DockedSearchState = {
   dgSearchId?: string;
 };
 
-export type ScoredSearchHit = {
-  result: SearchResult;
-  score: number;
-};
-
 type MiniSearchDocument = SearchResult & {
   id: string;
 };
@@ -214,23 +208,28 @@ export const isNonDefaultSort = (sort: SortConfig): boolean =>
   sort.direction !== DEFAULT_SORT_CONFIG.direction;
 
 export const sortSearchResults = ({
-  hits,
+  scoredResults,
   sort,
 }: {
-  hits: ScoredSearchHit[];
+  scoredResults: ScoredSearchResult[];
   sort: SortConfig;
 }): SearchResult[] => {
-  const sorted = [...hits];
+  // Semantic and MiniSearch results already arrive in their provider's relevance
+  // order, so relevance sorting only has to reverse it for ascending.
+  if (sort.field === "relevance") {
+    const ordered =
+      sort.direction === "asc" ? [...scoredResults].reverse() : scoredResults;
+    return ordered.map((entry) => entry.result);
+  }
 
-  sorted.sort((aHit, bHit) => {
-    const a = aHit.result;
-    const b = bHit.result;
+  const sorted = [...scoredResults];
+
+  sorted.sort((aEntry, bEntry) => {
+    const a = aEntry.result;
+    const b = bEntry.result;
     let comparison = 0;
 
     switch (sort.field) {
-      case "relevance":
-        comparison = compareNumbers(aHit.score, bHit.score, sort.direction);
-        break;
       case "alphabetical":
         comparison = compareStrings(
           getSortableTitle(a),
@@ -259,10 +258,10 @@ export const sortSearchResults = ({
     return comparison || a.uid.localeCompare(b.uid);
   });
 
-  return sorted.map((hit) => hit.result);
+  return sorted.map((entry) => entry.result);
 };
 
-export const searchIndexedNodes = ({
+export const searchDiscourseNodesWithMiniSearch = ({
   miniSearch,
   allResults,
   searchTerm,
@@ -272,7 +271,7 @@ export const searchIndexedNodes = ({
   allResults: SearchResult[];
   searchTerm: string;
   typeFilter?: string[];
-}): ScoredSearchHit[] => {
+}): ScoredSearchResult[] => {
   const resultsByUid = new Map(
     allResults.map((result) => [result.uid, result]),
   );
@@ -291,7 +290,11 @@ export const searchIndexedNodes = ({
     .map((result) => {
       const searchResult = resultsByUid.get(String(result.id));
       if (!searchResult) return null;
-      return { result: searchResult, score: result.score };
+      return {
+        result: searchResult,
+        score: result.score,
+        source: "miniSearch",
+      };
     })
-    .filter((hit): hit is ScoredSearchHit => !!hit);
+    .filter((entry): entry is ScoredSearchResult => !!entry);
 };