feat: add get_by_ids, reading documents back by item ID

README.md

@@ -6,7 +6,7 @@ Encrypted vector search for LangChain using Envector, powered by homomorphic enc
 - LangChain `VectorStore` interface with `similarity_search`, `from_texts`, etc.
 - Optional `VectorStoreRetriever` helper for quick RAG integrations.
 - Client-side encryption handled transparently by the SDK, including score thresholds and filtering.
-- In-place `delete`, `update_documents` and `upsert_documents` by item ID, plus named partitions.
+- In-place `delete`, `update_documents` and `upsert_documents` by item ID, `get_by_ids` to read documents back, plus named partitions.
 
 Requires `pyenvector >= 1.6.2`.
 
@@ -41,8 +41,10 @@ Key dataclasses live in `libs/envector/config.py`:
 - Client-side filtering requires the JSON envelope to include an object under `metadata`.
 
 ## Limitations
-- Item IDs are issued by the server (positive integers, returned as strings). Pass them back to `delete`, `update_documents`, `upsert_documents`, or as `ids` to `add_documents` to update in place — any numeric id is taken to be one of them. Other IDs cannot be created; such rows get server-issued IDs and a `UserWarning`.
-- Fetch-by-ID (`get_by_ids`) is unsupported, and so is LangChain's `indexing` API, which depends on its own IDs.
+- Item IDs are issued by the server (positive integers, returned as strings). Pass them back — as those strings or as ints — to `get_by_ids`, `delete`, `update_documents`, `upsert_documents`, or as `ids` to `add_documents` to update in place. Other IDs, such as UUIDs, cannot be created; such rows get server-issued IDs and a `UserWarning`, and `get_by_ids` never finds them.
+- LangChain's `indexing` API is unsupported, since it depends on its own IDs.
+- Item IDs are unique within a partition only; pass `partition_name` to `get_by_ids` for documents added to a named partition.
+- `get_by_ids` needs a pyenvector release that provides `Index.get_by_ids`; with an earlier pyenvector it raises `NotImplementedError`.
 - Embeddings must be unit norm: scores are inner products computed under encryption, and vectors with components outside [-1, 1] rank incorrectly.
 - A row deleted moments ago can still take a top-k slot briefly, so a search right after `delete` may return fewer than `k`; pass `fetch_k` to over-fetch.
 - Filtering happens client-side after the server returns `k` hits, so filtered results can be fewer than `k`; set `fetch_k` (or `IndexSettings.fetch_k`) to over-fetch.
@@ -177,6 +179,13 @@ result = store.upsert_documents(
 print(result["inserted_item_ids"])  # IDs issued for the ID-less entries
 ```
 
+### Fetch by ID
+
+```python
+docs = store.get_by_ids(ids)  # Documents for the IDs that exist; missing IDs are skipped
+docs = store.get_by_ids(ids, partition_name="tenant_a")  # rows in a named partition
+```
+
 ### Delete
 
 ```python

libs/envector/langchain_envector/vectorstore.py

@@ -1,17 +1,21 @@
 from __future__ import annotations
 
 import warnings
-from typing import Any, Dict, Iterable, List, Optional, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
 
 from langchain_core.documents import Document
 from langchain_core.vectorstores import VectorStore
 from pyenvector import UpdateItem, UpsertItem
 from pyenvector.index.index import MAX_MUTATION_ITEMS_PER_CALL
+from pyenvector.index.index import Index as _SdkIndex
 
 from .config import EnvectorConfig
 from .client import EnvectorClient
 from .types import Embeddings, as_embeddings, pack_metadata, unpack_metadata
 
+# Index.get_by_ids is newer than the oldest pyenvector this package accepts.
+SDK_HAS_GET_BY_IDS = hasattr(_SdkIndex, "get_by_ids")
+
 
 def _mutation_items(
     item_ids: List[Any], label: str, *, dedupe: bool = False
@@ -66,6 +70,26 @@ def _split_caller_ids(ids: List[Any]) -> Tuple[List[Optional[int]], List[Any]]:
     return item_ids, foreign
 
 
+def _readable_item_id(value: Any) -> Optional[int]:
+    """The item ID ``value`` names exactly, or ``None`` when it names none.
+
+    For `get_by_ids`, which must never read an item the caller did not name:
+    only a positive ``int`` or a decimal string of one counts. ``bool`` and
+    ``float`` are not item IDs — ``int(True)`` is 1 and ``int(3.9)`` is 3, so
+    coercing them would return a different document.
+    """
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value if value > 0 else None
+    if isinstance(value, str):
+        text = value.strip()
+        if text.isdecimal():
+            item_id = int(text)
+            return item_id if item_id > 0 else None
+    return None
+
+
 def _one_embedding_arg(embedding: Any, embeddings: Any) -> Any:
     """Resolve the standard positional ``embedding`` and our older
     ``embeddings=`` keyword into one value, rejecting conflicting pairs."""
@@ -90,6 +114,30 @@ def _chunked(items: List[Any], size: int) -> Iterable[List[Any]]:
         yield items[start : start + size]
 
 
+def _stored_document(item: Dict[str, Any]) -> Tuple[Document, bool]:
+    """Turn an SDK result dict (search hit or ``get_by_ids`` entry) into a Document.
+
+    The payload is the JSON envelope ``{"text": ..., "metadata": {...}}`` that
+    `add_texts` stores; anything else is taken as the document text. Returns the
+    Document and whether the payload carried any content.
+    """
+    # Metadata encryption/decryption is handled by the SDK. Envector stores a
+    # single string per item; `unpack_metadata` also accepts the dict the SDK
+    # returns once it has decrypted and parsed that string.
+    md_obj = unpack_metadata(item.get("metadata"))
+    text = md_obj.get("text", "") if "_raw" not in md_obj else md_obj["_raw"]
+    metadata = md_obj.get("metadata", {}) if "_raw" not in md_obj else {}
+    if text is None:
+        text = ""
+    doc_id = item.get("id")
+    doc = Document(
+        page_content=text,
+        metadata=metadata,
+        id=str(doc_id) if doc_id is not None else None,
+    )
+    return doc, bool(text or metadata)
+
+
 def _is_empty_shard_list_error(exc: Exception) -> bool:
     """True for the backend's "index has no shards" answer to a search.
 
@@ -394,6 +442,52 @@ def delete(
         )
         return True
 
+    def get_by_ids(
+        self, ids: Sequence[str], /, *, partition_name: Optional[str] = None
+    ) -> List[Document]:
+        """Read documents by item ID, without a search.
+
+        Takes the IDs `add_texts` / `add_documents` return, or a search
+        result's ``Document.id``. Every live item comes back as a ``Document``
+        whose ``id`` is its item ID, in the order of ``ids``; repeated IDs are
+        read once. IDs that match no live row — never issued, deleted, or not
+        enVector item IDs at all — are left out rather than raised, as
+        LangChain's contract asks.
+
+        Liveness here is the row's own state, not search visibility. A document
+        is readable as soon as `add_texts` returns, before any merge, so it can
+        come back from ``get_by_ids`` while ``similarity_search`` does not find
+        it yet. A deleted document stops being readable as soon as `delete`
+        returns. After an `update_documents` that replaces the vector and is
+        not awaited (``await_completion=False``), this returns the new content
+        while search may leave the document out until the new vector is
+        searchable. These differences are expected.
+
+        Item IDs are unique within a partition only: pass the ``partition_name``
+        a document was added under. Without it the default partition is read,
+        where the same ID may be a different document.
+        """
+        index = self.client.index
+        if not hasattr(index, "get_by_ids"):
+            raise NotImplementedError(
+                "Envector.get_by_ids needs a pyenvector release that provides "
+                "Index.get_by_ids; upgrade pyenvector."
+            )
+        item_ids = list(
+            dict.fromkeys(
+                i for i in (_readable_item_id(x) for x in ids) if i is not None
+            )
+        )
+        if not item_ids:
+            return []
+        # The SDK splits the request at the server's per-call cap.
+        items = index.get_by_ids(
+            item_ids,
+            output_fields=self.config.index.output_fields,
+            partition_name=partition_name,
+        )
+        return [_stored_document(item)[0] for item in items]
+
     # -------------------------------
     # In-place mutation (pyenvector >= 1.6.0)
     # -------------------------------
@@ -729,38 +823,23 @@ def _similarity_search_with_scores(
         for item in result:
             # item = {"id": ..., "score": float, "metadata": [str] or {...}}
             score = float(item.get("score", 0.0))
-            md_obj_raw = item.get("metadata")
-            if md_obj_raw in (None, "", [], {}):
+            if item.get("metadata") in (None, "", [], {}):
                 # Skip placeholder/empty hits returned by the backend.
                 continue
-
-            # Metadata encryption/decryption is handled by the SDK.
-            # Envector currently supports a single associated data field (string).
-            # Convention: if the string is JSON like {"text": str, "metadata": {...}},
-            # we unpack it; otherwise, we treat the raw string as the document text.
-            md_obj = unpack_metadata(md_obj_raw)
-
-            text = md_obj.get("text", "") if "_raw" not in md_obj else md_obj["_raw"]
-            metadata = md_obj.get("metadata", {}) if "_raw" not in md_obj else {}
-            if not text and not metadata:
+            doc, has_content = _stored_document(item)
+            if not has_content:
                 # Treat empty text+metadata as no result.
                 continue
 
             # client-side filter
             if filter:
                 # simple dict-equality filter on top-level user metadata
-                matched = all(metadata.get(k) == v for k, v in filter.items())
+                matched = all(doc.metadata.get(k) == v for k, v in filter.items())
                 if not matched:
                     continue
             if score_threshold is not None and score < score_threshold:
                 continue
 
-            doc_id = item.get("id")
-            doc = Document(
-                page_content=text,
-                metadata=metadata,
-                id=str(doc_id) if doc_id is not None else None,
-            )
             docs_with_scores.append((doc, score))
 
         # Trim to k after filtering

tests/conftest.py

@@ -38,6 +38,9 @@ class FakeIndex:
     load_calls: int = 0
     next_item_id: int = 1
     row_count: int = 0
+    # (partition_name, item_id) -> stored metadata string, for get_by_ids.
+    stored: Dict[Any, str] = field(default_factory=dict)
+    fetched: List[Dict[str, Any]] = field(default_factory=list)
 
     def load(self):
         self.load_calls += 1
@@ -87,7 +90,10 @@ def insert(
         if request_ids is not None:
             request_ids.append(f"req-ins-{len(self.inserted)}")
         self.row_count += len(metadata)
-        return self._issue_ids(len(metadata))
+        ids = self._issue_ids(len(metadata))
+        for i, m in zip(ids, metadata):
+            self.stored[(partition_name, i)] = m
+        return ids
 
     def wait_for_insert_stage(
         self,
@@ -124,8 +130,33 @@ def delete(
             }
         )
         self.row_count = max(0, self.row_count - len(item_ids))
+        for i in item_ids:
+            self.stored.pop((partition_name, i), None)
         return f"req-del-{len(self.deleted)}"
 
+    def get_by_ids(
+        self,
+        item_ids: List[int],
+        output_fields: Optional[List[str]] = None,
+        partition_name: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        self.fetched.append(
+            {
+                "item_ids": list(item_ids),
+                "output_fields": output_fields,
+                "partition_name": partition_name,
+            }
+        )
+        return [
+            {
+                "id": i,
+                "metadata": self.stored[(partition_name, i)] if output_fields else "",
+                "partition_name": partition_name or "",
+            }
+            for i in item_ids
+            if (partition_name, i) in self.stored
+        ]
+
     def update(
         self,
         items: List[Any],

tests/integration_tests/test_get_by_ids.py

@@ -0,0 +1,117 @@
+"""Integration coverage for `get_by_ids` against a live server.
+
+The LangChain standard tests cover the plain read-back. This adds what they do
+not: reading right after an un-awaited insert, a delete becoming invisible as
+soon as it returns, named partitions, and an index with metadata encryption.
+
+Run with:
+    ENVECTOR_ADDRESS=host:port ENVECTOR_KEY_PATH=./keys ENVECTOR_KEY_ID=my_key \\
+        pytest -q -m integration tests/integration_tests/test_get_by_ids.py
+"""
+
+from __future__ import annotations
+
+import os
+import secrets
+from typing import Generator, List
+
+import pytest
+
+from langchain_envector.config import (
+    ConnectionConfig,
+    EnvectorConfig,
+    IndexSettings,
+    KeyConfig,
+    WriteSettings,
+)
+from langchain_envector.vectorstore import SDK_HAS_GET_BY_IDS, Document, Envector
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(
+        not SDK_HAS_GET_BY_IDS, reason="installed pyenvector has no Index.get_by_ids"
+    ),
+]
+
+DIM = 32
+
+
+def _require_env(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        pytest.skip(f"Set {name} to enable integration test")
+    return value
+
+
+def _unit_vector(pos: int) -> List[float]:
+    vec = [0.0] * DIM
+    vec[pos % DIM] = 1.0
+    return vec
+
+
+@pytest.fixture(params=[False, True], ids=["plain-metadata", "encrypted-metadata"])
+def store(request) -> Generator[Envector, None, None]:
+    name = f"lc_gbi_{secrets.token_hex(4)}"
+    cfg = EnvectorConfig(
+        connection=ConnectionConfig(address=_require_env("ENVECTOR_ADDRESS")),
+        key=KeyConfig(
+            key_path=_require_env("ENVECTOR_KEY_PATH"),
+            key_id=_require_env("ENVECTOR_KEY_ID"),
+        ),
+        index=IndexSettings(
+            index_name=name, dim=DIM, metadata_encryption=request.param
+        ),
+        # Nothing here waits for a merge: get_by_ids must not need one.
+        write=WriteSettings(await_insert=False, await_delete=False),
+        create_if_missing=True,
+    )
+    store = Envector(config=cfg)
+    try:
+        yield store
+    finally:
+        try:
+            store.client.ev.delete_index(name)
+        except Exception:
+            pass
+
+
+def _add(store: Envector, texts: List[str], **kwargs) -> List[str]:
+    return store.add_texts(
+        texts,
+        metadatas=[{"n": i} for i in range(len(texts))],
+        vectors=[_unit_vector(i) for i in range(len(texts))],
+        **kwargs,
+    )
+
+
+def test_readable_right_after_insert_and_gone_right_after_delete(
+    store: Envector,
+) -> None:
+    ids = _add(store, ["a", "b", "c"])
+
+    docs = store.get_by_ids(ids)
+    assert docs == [
+        Document(page_content=t, metadata={"n": i}, id=ids[i])
+        for i, t in enumerate("abc")
+    ]
+
+    # Missing and foreign ids are left out; order follows the request.
+    got = store.get_by_ids([ids[2], "999999", "not-an-id", ids[0]])
+    assert [d.id for d in got] == [ids[2], ids[0]]
+
+    store.delete([ids[1]])
+    assert [d.id for d in store.get_by_ids(ids)] == [ids[0], ids[2]]
+
+
+def test_named_partition_needs_partition_name(store: Envector) -> None:
+    default_ids = _add(store, ["default doc"])
+    store.create_partition("tenant_a")
+    tenant_ids = _add(store, ["tenant doc"], partition_name="tenant_a")
+
+    in_tenant = store.get_by_ids(tenant_ids, partition_name="tenant_a")
+    assert [d.page_content for d in in_tenant] == ["tenant doc"]
+
+    # Item ids restart per partition: the same number in the default partition
+    # is a different document, which is why partition_name matters.
+    assert tenant_ids == default_ids
+    assert [d.page_content for d in store.get_by_ids(tenant_ids)] == ["default doc"]