diff --git a/README.md b/README.md index 83960ee..9b378b1 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Encrypted vector search for LangChain using Envector, powered by homomorphic enc - LangChain `VectorStore` interface with `similarity_search`, `from_texts`, etc. - Optional `VectorStoreRetriever` helper for quick RAG integrations. - Client-side encryption handled transparently by the SDK, including score thresholds and filtering. -- In-place `delete`, `update_documents` and `upsert_documents` by item ID, plus named partitions. +- In-place `delete`, `update_documents` and `upsert_documents` by item ID, `get_by_ids` to read documents back, plus named partitions. Requires `pyenvector >= 1.6.2`. @@ -41,8 +41,10 @@ Key dataclasses live in `libs/envector/config.py`: - Client-side filtering requires the JSON envelope to include an object under `metadata`. ## Limitations -- Item IDs are issued by the server (positive integers, returned as strings). Pass them back to `delete`, `update_documents`, `upsert_documents`, or as `ids` to `add_documents` to update in place — any numeric id is taken to be one of them. Other IDs cannot be created; such rows get server-issued IDs and a `UserWarning`. -- Fetch-by-ID (`get_by_ids`) is unsupported, and so is LangChain's `indexing` API, which depends on its own IDs. +- Item IDs are issued by the server (positive integers, returned as strings). Pass them back — as those strings or as ints — to `get_by_ids`, `delete`, `update_documents`, `upsert_documents`, or as `ids` to `add_documents` to update in place. Other IDs, such as UUIDs, cannot be created; such rows get server-issued IDs and a `UserWarning`, and `get_by_ids` never finds them. +- LangChain's `indexing` API is unsupported, since it depends on its own IDs. +- Item IDs are unique within a partition only; pass `partition_name` to `get_by_ids` for documents added to a named partition. +- `get_by_ids` needs a pyenvector release that provides `Index.get_by_ids`; with an earlier pyenvector it raises `NotImplementedError`. - Embeddings must be unit norm: scores are inner products computed under encryption, and vectors with components outside [-1, 1] rank incorrectly. - A row deleted moments ago can still take a top-k slot briefly, so a search right after `delete` may return fewer than `k`; pass `fetch_k` to over-fetch. - Filtering happens client-side after the server returns `k` hits, so filtered results can be fewer than `k`; set `fetch_k` (or `IndexSettings.fetch_k`) to over-fetch. @@ -177,6 +179,13 @@ result = store.upsert_documents( print(result["inserted_item_ids"]) # IDs issued for the ID-less entries ``` +### Fetch by ID + +```python +docs = store.get_by_ids(ids) # Documents for the IDs that exist; missing IDs are skipped +docs = store.get_by_ids(ids, partition_name="tenant_a") # rows in a named partition +``` + ### Delete ```python diff --git a/libs/envector/langchain_envector/vectorstore.py b/libs/envector/langchain_envector/vectorstore.py index a8b64ea..0019d11 100644 --- a/libs/envector/langchain_envector/vectorstore.py +++ b/libs/envector/langchain_envector/vectorstore.py @@ -1,17 +1,21 @@ from __future__ import annotations import warnings -from typing import Any, Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple from langchain_core.documents import Document from langchain_core.vectorstores import VectorStore from pyenvector import UpdateItem, UpsertItem from pyenvector.index.index import MAX_MUTATION_ITEMS_PER_CALL +from pyenvector.index.index import Index as _SdkIndex from .config import EnvectorConfig from .client import EnvectorClient from .types import Embeddings, as_embeddings, pack_metadata, unpack_metadata +# Index.get_by_ids is newer than the oldest pyenvector this package accepts. +SDK_HAS_GET_BY_IDS = hasattr(_SdkIndex, "get_by_ids") + def _mutation_items( item_ids: List[Any], label: str, *, dedupe: bool = False @@ -66,6 +70,34 @@ def _split_caller_ids(ids: List[Any]) -> Tuple[List[Optional[int]], List[Any]]: return item_ids, foreign +# Item IDs travel as proto int64; a larger value cannot name a row and the +# SDK would fail to encode it. +_MAX_ITEM_ID = 2**63 - 1 + + +def _readable_item_id(value: Any) -> Optional[int]: + """The item ID ``value`` names exactly, or ``None`` when it names none. + + For `get_by_ids`, which must never read an item the caller did not name: + only a positive ``int`` within int64 (the server issues item IDs as + ``int64``), or an ASCII decimal string of one, counts. ``bool`` and + ``float`` are not item IDs — ``int(True)`` is 1 and ``int(3.9)`` is 3, so + coercing them would return a different document — and neither are + non-ASCII digits such as ``"٣"`` or ``"３"``, which ``str.isdecimal`` + accepts. + """ + if isinstance(value, bool): + return None + if isinstance(value, int): + return value if 0 < value <= _MAX_ITEM_ID else None + if isinstance(value, str): + text = value.strip() + if text.isascii() and text.isdigit(): + item_id = int(text) + return item_id if 0 < item_id <= _MAX_ITEM_ID else None + return None + + def _one_embedding_arg(embedding: Any, embeddings: Any) -> Any: """Resolve the standard positional ``embedding`` and our older ``embeddings=`` keyword into one value, rejecting conflicting pairs.""" @@ -90,6 +122,41 @@ def _chunked(items: List[Any], size: int) -> Iterable[List[Any]]: yield items[start : start + size] +def _stored_document(item: Dict[str, Any]) -> Optional[Document]: + """Turn an SDK result dict (search hit or ``get_by_ids`` entry) into a Document. + + The payload is the JSON envelope ``{"text": ..., "metadata": {...}}`` that + `add_texts` stores; any other string is taken as the document text. Returns + ``None`` when the payload is not a document at all — ``text`` present but + not a string, or ``metadata`` present but not a dict, as a row written by + another client might be — so callers skip the row instead of failing the + whole call on a pydantic error. + """ + # Metadata encryption/decryption is handled by the SDK. Envector stores a + # single string per item; `unpack_metadata` also accepts the dict the SDK + # returns once it has decrypted and parsed that string. + md_obj = unpack_metadata(item.get("metadata")) + if "_raw" in md_obj: + raw = md_obj["_raw"] + text = raw if isinstance(raw, str) else "" + metadata: Dict[str, Any] = {} + else: + text = md_obj.get("text", "") + metadata = md_obj.get("metadata", {}) + if text is None: + text = "" + if metadata is None: + metadata = {} + if not isinstance(text, str) or not isinstance(metadata, dict): + return None + doc_id = item.get("id") + return Document( + page_content=text, + metadata=metadata, + id=str(doc_id) if doc_id is not None else None, + ) + + def _is_empty_shard_list_error(exc: Exception) -> bool: """True for the backend's "index has no shards" answer to a search. @@ -394,6 +461,53 @@ def delete( ) return True + def get_by_ids( + self, ids: Sequence[str], /, *, partition_name: Optional[str] = None + ) -> List[Document]: + """Read documents by item ID, without a search. + + Takes the IDs `add_texts` / `add_documents` return, or a search + result's ``Document.id``. Every live item comes back as a ``Document`` + whose ``id`` is its item ID, in the order of ``ids``; repeated IDs are + read once. IDs that match no live row — never issued, deleted, or not + enVector item IDs at all — are left out rather than raised, as + LangChain's contract asks. + + Liveness here is the row's own state, not search visibility. A document + is readable as soon as `add_texts` returns, before any merge, so it can + come back from ``get_by_ids`` while ``similarity_search`` does not find + it yet. A deleted document stops being readable as soon as `delete` + returns. After an `update_documents` that replaces the vector and is + not awaited (``await_completion=False``), this returns the new content + while search may leave the document out until the new vector is + searchable. These differences are expected. + + Item IDs are unique within a partition only: pass the ``partition_name`` + a document was added under. Without it the default partition is read, + where the same ID may be a different document. + """ + index = self.client.index + if not hasattr(index, "get_by_ids"): + raise NotImplementedError( + "Envector.get_by_ids needs a pyenvector release that provides " + "Index.get_by_ids; upgrade pyenvector." + ) + item_ids = list( + dict.fromkeys( + i for i in (_readable_item_id(x) for x in ids) if i is not None + ) + ) + if not item_ids: + return [] + # The SDK splits the request at the server's per-call cap. + items = index.get_by_ids( + item_ids, + output_fields=self.config.index.output_fields, + partition_name=partition_name, + ) + docs = (_stored_document(item) for item in items) + return [doc for doc in docs if doc is not None] + # ------------------------------- # In-place mutation (pyenvector >= 1.6.0) # ------------------------------- @@ -729,38 +843,23 @@ def _similarity_search_with_scores( for item in result: # item = {"id": ..., "score": float, "metadata": [str] or {...}} score = float(item.get("score", 0.0)) - md_obj_raw = item.get("metadata") - if md_obj_raw in (None, "", [], {}): + if item.get("metadata") in (None, "", [], {}): # Skip placeholder/empty hits returned by the backend. continue - - # Metadata encryption/decryption is handled by the SDK. - # Envector currently supports a single associated data field (string). - # Convention: if the string is JSON like {"text": str, "metadata": {...}}, - # we unpack it; otherwise, we treat the raw string as the document text. - md_obj = unpack_metadata(md_obj_raw) - - text = md_obj.get("text", "") if "_raw" not in md_obj else md_obj["_raw"] - metadata = md_obj.get("metadata", {}) if "_raw" not in md_obj else {} - if not text and not metadata: - # Treat empty text+metadata as no result. + doc = _stored_document(item) + if doc is None or not (doc.page_content or doc.metadata): + # Not a document, or empty text+metadata: no result. continue # client-side filter if filter: # simple dict-equality filter on top-level user metadata - matched = all(metadata.get(k) == v for k, v in filter.items()) + matched = all(doc.metadata.get(k) == v for k, v in filter.items()) if not matched: continue if score_threshold is not None and score < score_threshold: continue - doc_id = item.get("id") - doc = Document( - page_content=text, - metadata=metadata, - id=str(doc_id) if doc_id is not None else None, - ) docs_with_scores.append((doc, score)) # Trim to k after filtering diff --git a/tests/conftest.py b/tests/conftest.py index 2695286..d6789a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,9 @@ class FakeIndex: load_calls: int = 0 next_item_id: int = 1 row_count: int = 0 + # (partition_name, item_id) -> stored metadata string, for get_by_ids. + stored: Dict[Any, str] = field(default_factory=dict) + fetched: List[Dict[str, Any]] = field(default_factory=list) def load(self): self.load_calls += 1 @@ -87,7 +90,10 @@ def insert( if request_ids is not None: request_ids.append(f"req-ins-{len(self.inserted)}") self.row_count += len(metadata) - return self._issue_ids(len(metadata)) + ids = self._issue_ids(len(metadata)) + for i, m in zip(ids, metadata): + self.stored[(partition_name, i)] = m + return ids def wait_for_insert_stage( self, @@ -124,8 +130,33 @@ def delete( } ) self.row_count = max(0, self.row_count - len(item_ids)) + for i in item_ids: + self.stored.pop((partition_name, i), None) return f"req-del-{len(self.deleted)}" + def get_by_ids( + self, + item_ids: List[int], + output_fields: Optional[List[str]] = None, + partition_name: Optional[str] = None, + ) -> List[Dict[str, Any]]: + self.fetched.append( + { + "item_ids": list(item_ids), + "output_fields": output_fields, + "partition_name": partition_name, + } + ) + return [ + { + "id": i, + "metadata": self.stored[(partition_name, i)] if output_fields else "", + "partition_name": partition_name or "", + } + for i in item_ids + if (partition_name, i) in self.stored + ] + def update( self, items: List[Any], diff --git a/tests/integration_tests/test_get_by_ids.py b/tests/integration_tests/test_get_by_ids.py new file mode 100644 index 0000000..73946ab --- /dev/null +++ b/tests/integration_tests/test_get_by_ids.py @@ -0,0 +1,117 @@ +"""Integration coverage for `get_by_ids` against a live server. + +The LangChain standard tests cover the plain read-back. This adds what they do +not: reading right after an un-awaited insert, a delete becoming invisible as +soon as it returns, named partitions, and an index with metadata encryption. + +Run with: + ENVECTOR_ADDRESS=host:port ENVECTOR_KEY_PATH=./keys ENVECTOR_KEY_ID=my_key \\ + pytest -q -m integration tests/integration_tests/test_get_by_ids.py +""" + +from __future__ import annotations + +import os +import secrets +from typing import Generator, List + +import pytest + +from langchain_envector.config import ( + ConnectionConfig, + EnvectorConfig, + IndexSettings, + KeyConfig, + WriteSettings, +) +from langchain_envector.vectorstore import SDK_HAS_GET_BY_IDS, Document, Envector + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not SDK_HAS_GET_BY_IDS, reason="installed pyenvector has no Index.get_by_ids" + ), +] + +DIM = 32 + + +def _require_env(name: str) -> str: + value = os.environ.get(name) + if not value: + pytest.skip(f"Set {name} to enable integration test") + return value + + +def _unit_vector(pos: int) -> List[float]: + vec = [0.0] * DIM + vec[pos % DIM] = 1.0 + return vec + + +@pytest.fixture(params=[False, True], ids=["plain-metadata", "encrypted-metadata"]) +def store(request) -> Generator[Envector, None, None]: + name = f"lc_gbi_{secrets.token_hex(4)}" + cfg = EnvectorConfig( + connection=ConnectionConfig(address=_require_env("ENVECTOR_ADDRESS")), + key=KeyConfig( + key_path=_require_env("ENVECTOR_KEY_PATH"), + key_id=_require_env("ENVECTOR_KEY_ID"), + ), + index=IndexSettings( + index_name=name, dim=DIM, metadata_encryption=request.param + ), + # Nothing here waits for a merge: get_by_ids must not need one. + write=WriteSettings(await_insert=False, await_delete=False), + create_if_missing=True, + ) + store = Envector(config=cfg) + try: + yield store + finally: + try: + store.client.ev.delete_index(name) + except Exception: + pass + + +def _add(store: Envector, texts: List[str], **kwargs) -> List[str]: + return store.add_texts( + texts, + metadatas=[{"n": i} for i in range(len(texts))], + vectors=[_unit_vector(i) for i in range(len(texts))], + **kwargs, + ) + + +def test_readable_right_after_insert_and_gone_right_after_delete( + store: Envector, +) -> None: + ids = _add(store, ["a", "b", "c"]) + + docs = store.get_by_ids(ids) + assert docs == [ + Document(page_content=t, metadata={"n": i}, id=ids[i]) + for i, t in enumerate("abc") + ] + + # Missing and foreign ids are left out; order follows the request. + got = store.get_by_ids([ids[2], "999999", "not-an-id", ids[0]]) + assert [d.id for d in got] == [ids[2], ids[0]] + + store.delete([ids[1]]) + assert [d.id for d in store.get_by_ids(ids)] == [ids[0], ids[2]] + + +def test_named_partition_needs_partition_name(store: Envector) -> None: + default_ids = _add(store, ["default doc"]) + store.create_partition("tenant_a") + tenant_ids = _add(store, ["tenant doc"], partition_name="tenant_a") + + in_tenant = store.get_by_ids(tenant_ids, partition_name="tenant_a") + assert [d.page_content for d in in_tenant] == ["tenant doc"] + + # Item ids restart per partition: the same number in the default partition + # is a different document, which is why partition_name matters. + assert tenant_ids == default_ids + assert [d.page_content for d in store.get_by_ids(tenant_ids)] == ["default doc"] diff --git a/tests/integration_tests/test_vectorstore.py b/tests/integration_tests/test_vectorstore.py index 6c4aede..92f9e72 100644 --- a/tests/integration_tests/test_vectorstore.py +++ b/tests/integration_tests/test_vectorstore.py @@ -17,7 +17,7 @@ IndexSettings, KeyConfig, ) -from langchain_envector.vectorstore import Envector +from langchain_envector.vectorstore import SDK_HAS_GET_BY_IDS, Envector pytestmark = pytest.mark.integration @@ -64,8 +64,8 @@ def has_async(self) -> bool: @property def has_get_by_ids(self) -> bool: - # Envector does not yet support get by IDs. - return False + # Follows the installed SDK: skipped where pyenvector lacks Index.get_by_ids. + return SDK_HAS_GET_BY_IDS @pytest.fixture() def vectorstore(self) -> Generator[VectorStore, None, None]: # type: ignore[override] @@ -117,5 +117,12 @@ def test_deleting_documents(self, vectorstore: VectorStore) -> None: def test_deleting_bulk_documents(self, vectorstore: VectorStore) -> None: super().test_deleting_bulk_documents(vectorstore) + @pytest.mark.xfail( + reason="enVector issues item IDs; a caller-chosen id such as 'foo' " + "cannot be created, so it is not among the returned ids." + ) + def test_add_documents_with_existing_ids(self, vectorstore: VectorStore) -> None: + super().test_add_documents_with_existing_ids(vectorstore) + # Async standard tests are not overridden: has_async=False makes the base # class skip them. diff --git a/tests/test_vectorstore.py b/tests/test_vectorstore.py index 29a21e5..8cbd299 100644 --- a/tests/test_vectorstore.py +++ b/tests/test_vectorstore.py @@ -1269,3 +1269,224 @@ async def test_async_relevance_scores_match_sync(): assert [(d.page_content, s) for d, s in async_pairs] == [ (d.page_content, s) for d, s in sync_pairs ] + + +def test_get_by_ids_reads_documents_back_in_request_order(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + ids = store.add_texts(["a", "b", "c"], metadatas=[{"k": 1}, {"k": 2}, {"k": 3}]) + + docs = store.get_by_ids([ids[2], ids[0]]) + + assert docs == [ + LC_Document(page_content="c", metadata={"k": 3}, id=ids[2]), + LC_Document(page_content="a", metadata={"k": 1}, id=ids[0]), + ] + assert client.index.fetched[-1]["output_fields"] == ["metadata"] + + +def test_get_by_ids_leaves_out_ids_it_cannot_find_without_raising(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + ids = store.add_texts(["a"]) + + docs = store.get_by_ids(["uuid-like", "0", "-3", "999", ids[0], ids[0], 1]) + + assert [d.id for d in docs] == [ids[0]] + # Non-item IDs never reach the SDK, and repeats are sent once. + assert client.index.fetched[-1]["item_ids"] == [999, 1] + + +def test_get_by_ids_empty_or_foreign_only_makes_no_call(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + assert store.get_by_ids([]) == [] + assert store.get_by_ids(["foo", "bar"]) == [] + assert client.index.fetched == [] + + +def test_get_by_ids_does_not_see_deleted_documents(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + ids = store.add_texts(["a", "b"]) + store.delete([ids[0]]) + assert [d.id for d in store.get_by_ids(ids)] == [ids[1]] + + +def test_get_by_ids_reads_the_named_partition(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + ids = store.add_texts(["tenant doc"], partition_name="tenant_a") + + assert store.get_by_ids(ids) == [] + docs = store.get_by_ids(ids, partition_name="tenant_a") + assert [d.page_content for d in docs] == ["tenant doc"] + assert client.index.fetched[-1]["partition_name"] == "tenant_a" + + +def test_get_by_ids_keeps_a_live_row_with_no_stored_content(): + client = FakeClient() + index = client.index + index.stored[(None, 7)] = "" + index.stored[(None, 8)] = "not an envelope" + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + + docs = store.get_by_ids(["7", "8"]) + + assert docs == [ + LC_Document(page_content="", metadata={}, id="7"), + LC_Document(page_content="not an envelope", metadata={}, id="8"), + ] + + +def test_get_by_ids_accepts_already_decrypted_payloads(): + # With metadata encryption on, the SDK hands back the parsed envelope. + client = FakeClient() + client.index.get_by_ids = lambda item_ids, **kw: [ + {"id": 5, "metadata": {"text": "t", "metadata": {"m": 1}}, "partition_name": ""} + ] + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + assert store.get_by_ids(["5"]) == [ + LC_Document(page_content="t", metadata={"m": 1}, id="5") + ] + + +def test_get_by_ids_does_not_load_the_index(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + client.index.stored[(None, 1)] = '{"text": "x", "metadata": {}}' + store.get_by_ids(["1"]) + assert client.index.load_calls == 0 + + +def test_stored_null_text_reads_as_empty_document(): + # An envelope whose text is JSON null (a foreign writer) yields an empty page, + # not a pydantic error, in both search and get_by_ids. + client = FakeClient() + client.index.stored[(None, 1)] = '{"text": null, "metadata": {"k": 1}}' + client.index.search_payload = [ + [{"id": 1, "score": 0.5, "metadata": client.index.stored[(None, 1)]}] + ] + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + + expected = [LC_Document(page_content="", metadata={"k": 1}, id="1")] + assert store.get_by_ids(["1"]) == expected + assert store.similarity_search("q", k=1) == expected + + +def test_get_by_ids_needs_an_sdk_that_has_it(): + # A pyenvector release without Index.get_by_ids gets a clear + # NotImplementedError, LangChain's usual answer, not an AttributeError. + client = FakeClient() + + class _OldIndex(FakeIndex): + get_by_ids = property(lambda self: (_ for _ in ()).throw(AttributeError)) + + client._index = _OldIndex() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + with pytest.raises(NotImplementedError, match="upgrade pyenvector"): + store.get_by_ids(["1"]) + + +@pytest.mark.parametrize( + "not_an_id", + [ + True, + False, + 3.9, + 3.0, + "3.0", + "-3", + "0", + 0, + -3, + " ", + None, + b"3", + [3], + "\u0663", # Arabic-Indic three: str.isdecimal() accepts it + "\uff13", # fullwidth three + 2**63, # past int64: cannot name a row, would fail proto encoding + str(2**63), + ], +) +def test_get_by_ids_never_reads_an_item_the_caller_did_not_name(not_an_id): + # int(True) is 1 and int(3.9) is 3: coercing would return another document. + # Such values name no item, so they are left out, as LangChain asks. + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + store.add_texts(["a", "b", "c"]) + + assert store.get_by_ids([not_an_id]) == [] + assert client.index.fetched == [] + + +def test_get_by_ids_accepts_ints_and_decimal_strings(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + store.add_texts(["a", "b", "c"]) + + docs = store.get_by_ids([3, " 2 ", "1", 3.9, True]) + assert [d.page_content for d in docs] == ["c", "b", "a"] + assert client.index.fetched[-1]["item_ids"] == [3, 2, 1] + + +def test_get_by_ids_takes_the_largest_int64_id(): + client = FakeClient() + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + client.index.stored[(None, 2**63 - 1)] = '{"text": "big", "metadata": {}}' + assert [d.page_content for d in store.get_by_ids([str(2**63 - 1)])] == ["big"] + + +@pytest.mark.parametrize( + "payload", + [ + '{"text": 123, "metadata": {}}', + '{"text": ["x"], "metadata": {}}', + '{"text": "x", "metadata": "m"}', + '{"text": "x", "metadata": [1]}', + ], +) +def test_rows_that_are_not_documents_are_skipped_not_raised(payload): + # A row written by another client can carry an envelope with the right + # keys and wrong types. Neither search nor get_by_ids may fail the whole + # call on it; search also must not raise before its threshold check. + # A JSON null for either key is "absent", not a wrong type: see + # test_stored_null_text_reads_as_empty_document and the test below. + client = FakeClient() + client.index.stored[(None, 1)] = '{"text": "a", "metadata": {}}' + client.index.stored[(None, 2)] = payload + client.index.search_payload = [ + [ + {"id": 2, "score": 0.1, "metadata": payload}, + {"id": 1, "score": 0.9, "metadata": client.index.stored[(None, 1)]}, + ] + ] + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + + assert [d.id for d in store.get_by_ids(["2", "1"])] == ["1"] + assert [d.id for d in store.similarity_search("q", k=2, score_threshold=0.5)] == [ + "1" + ] + + +def test_metadata_null_in_envelope_reads_as_empty_metadata(): + client = FakeClient() + client.index.stored[(None, 1)] = '{"text": "a", "metadata": null}' + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + assert store.get_by_ids(["1"]) == [ + LC_Document(page_content="a", metadata={}, id="1") + ] + + +def test_search_skips_a_hit_with_empty_text_and_metadata(): + # Moved into _stored_document by this change; pin the behaviour. + client = FakeClient() + client.index.search_payload = [ + [ + {"id": 1, "score": 0.9, "metadata": '{"text": "", "metadata": {}}'}, + {"id": 2, "score": 0.8, "metadata": '{"text": "b", "metadata": {}}'}, + ] + ] + store = Envector(config=_cfg(), embeddings=FakeEmbeddings(dim=4), client=client) + assert [d.id for d in store.similarity_search("q", k=2)] == ["2"]