diff --git a/examples/agentic_vectorless_rag_demo.py b/examples/agentic_vectorless_rag_demo.py index fef4f794e..126d1ef06 100644 --- a/examples/agentic_vectorless_rag_demo.py +++ b/examples/agentic_vectorless_rag_demo.py @@ -50,10 +50,11 @@ def query_agent(client: PageIndexLocalClient, doc_id: str, prompt: str, verbose: """ agent = Agent( **client.openai_agent_config( - doc_id=doc_id, # model_settings=ModelSettings(reasoning={"effort": "low", "summary": "auto"}), # from agents.model_settings import ModelSettings ), ) + # Document targeting is conversation content: it leads the first message. + prompt = client.document_context(doc_id) + "\n\n" + prompt async def _run(): streamed_run = Runner.run_streamed(agent, prompt) diff --git a/pageindex/agent_tools.py b/pageindex/agent_tools.py index f2794ad03..671625112 100644 --- a/pageindex/agent_tools.py +++ b/pageindex/agent_tools.py @@ -1519,27 +1519,14 @@ def _require_doc_selection(doc_ids) -> None: "doc_id to give the agent the whole library.") -def _require_local_scope(client, doc_ids) -> None: - """The allowlist is enforced in-process; cloud tools take none, so - accepting doc_ids there would be advisory-only — refuse loudly.""" - _require_doc_selection(doc_ids) - if doc_ids is not None and getattr(client, "api_key", None): - raise PageIndexAPIError( - "doc_ids scoping applies to local tools only — the managed " - "cloud chat scopes doc_id server-side, and own-model chat " - "over cloud documents targets documents at the prompt level, " - "without a tool-layer allowlist." - ) - - def _tool_specs(client, include_management: bool = False, doc_ids=None, ) -> "list[tuple[str, str, dict, Callable[[dict], tuple[list, bool]]]]": """(name, description, schema, invoke) per tool, for adapters that take the wire schema verbatim. ``invoke`` returns (content blocks, is_error): the MCP content as the server sent it, one text block from the local tools. Schemas are copies (frameworks keep the dict by reference). - ``doc_ids`` is the local chat scope.""" - _require_local_scope(client, doc_ids) + ``doc_ids`` is the local chat scope, already validated and dropped on + cloud by ``_local_doc_scope``.""" if getattr(client, "api_key", None): bridge = _cloud_bridge(client, gated=not include_management) tools_meta = bridge.list_tools() @@ -1570,7 +1557,7 @@ def invoke(arguments: dict) -> tuple[list, bool]: def build_agent_tools(client, include_management: bool = False, - doc_ids=None) -> list[Callable[..., str]]: + ) -> list[Callable[..., str]]: """Plain synchronous functions bound to `client`. Cloud: one function per tool of the live cloud MCP tool set, signatures @@ -1583,11 +1570,10 @@ def build_agent_tools(client, include_management: bool = False, USAGE_LIMIT_REACHED tool error, which re-raise PageIndexAPIError (cloud-only parameters are absent from the local signatures; the call_tool path answers them with the guided envelope). - ``doc_ids`` is the local allowlist, as in ``_tool_specs``. """ return [_make_tool_function(name, description, schema, invoke) for name, description, schema, invoke - in _tool_specs(client, include_management, doc_ids)] + in _tool_specs(client, include_management)] # ── agent instructions ── @@ -1717,18 +1703,17 @@ def _base_instructions(client, include_management: bool = False) -> str: return instructions -def doc_targeting_block(client, doc_id, scoped: bool = False) -> Optional[str]: - """The doc_id targeting text: names, metadata, and the directive to work - within those documents. Shared by agent_instructions and the local chat - surfaces (a leading conversation item on the OpenAI surfaces, a system - block on the Messages lane). Raises when a doc_id's name is shadowed by a - newer - same-name document — the name-addressed tools could not reach it. With - ``scoped`` (surfaces whose tools resolve names inside the doc_id - allowlist) only a same-name duplicate within the targeted set - shadows.""" +def doc_targeting_block(client, doc_id) -> Optional[str]: + """The doc_id targeting text, rendered as the cloud's managed chat + renders its own: the documents' metadata rows and the directive to + work within them. Conversation content, never system prompt: the chat + lanes prepend it as the first user message, and document_context() + hands it to callers who own the conversation.""" if doc_id is None: return None + if not isinstance(doc_id, (str, list)): + raise PageIndexAPIError("doc_id must be a string or a list of " + "strings.") doc_ids = [doc_id] if isinstance(doc_id, str) else list(doc_id) _require_doc_selection(doc_ids) details = [] @@ -1745,51 +1730,59 @@ def doc_targeting_block(client, doc_id, scoped: bool = False) -> Optional[str]: if missing: raise PageIndexAPIError( "Documents not found or access denied: " + ", ".join(missing)) - # Scoped: the listing only backfills the target docs' metadata (list - # entries carry it, get_document does not — cloud parity), so paging - # can stop at those ids. Unscoped needs it all for the shadow check. - listing = _all_documents(client, stop_ids=doc_ids if scoped else None) - documents = ([{**detail, "id": one_id} - for one_id, detail in zip(doc_ids, details)] - if scoped else listing) - for one_id, detail in zip(doc_ids, details): - entry, _ = _resolve_document(client, str(detail.get("name")), - documents=documents) - if entry is not None and entry.get("id") != one_id: - raise PageIndexAPIError( - f'Document "{detail.get("name")}" (doc_id: {one_id}) is ' - "shadowed by a newer document with the same name (doc_id: " - f'{entry.get("id")}). The tools address documents by name ' - "and would read the newer one. Rename or remove the " - "duplicate, or pass the newer doc_id." - ) - by_id = {doc.get("id"): doc for doc in listing} - for one_id, detail in zip(doc_ids, details): - if detail.get("metadata") is None: - tags = _flat_metadata(by_id.get(one_id, {}).get("metadata")) - if tags is not None: - detail["metadata"] = tags - context = json.dumps(details, ensure_ascii=False) if len(details) == 1: return ( f"The user has specified document: {details[0].get('name')}\n" - f"Document metadata: {context}\n" + f"Document metadata: {json.dumps(details[0], ensure_ascii=False)}\n" "Use this document's name to retrieve its content with " "get_document_structure() and get_page_content()." ) names = ", ".join(str(item.get("name")) for item in details) return ( f"The user has specified documents: {names}\n" - f"Documents metadata: {context}\n" + f"Documents metadata: {json.dumps(details, ensure_ascii=False)}\n" "Use these documents' names to retrieve their content with " "get_document_structure() and get_page_content()." ) -def build_agent_instructions(client, doc_id=None, scoped: bool = False, - include_management: bool = False) -> str: - """Orchestration guidance for document QA agents; with doc_id, appends - the target documents and directs the agent to work within them.""" - base = _base_instructions(client, include_management) - block = doc_targeting_block(client, doc_id, scoped=scoped) - return base if block is None else base + "\n\n" + block +def folder_targeting_block(client, folder_id) -> Optional[str]: + """The folder_id targeting text, rendered as the cloud's managed chat + renders its own: the folder's name and metadata and the directive to + discover its documents there. None for no folder — None, "", and + "root", the library itself, which the managed chat leaves untargeted. + A folder proper is cloud-only: local libraries have none.""" + if folder_id is None: + return None + if not isinstance(folder_id, str): + raise PageIndexAPIError("folder_id must be a string.") + if folder_id in ("", "root"): + return None + if not getattr(client, "api_key", None): + raise PageIndexAPIError( + "folder_id is cloud-only — folders are not supported in local " + "mode. Create the client with an api_key to use folders.") + folders = client.list_folders().get("folders") or [] + folder = next((f for f in folders if f.get("id") == folder_id), None) + if folder is None: + raise PageIndexAPIError( + f"Folder not found or access denied: {folder_id}") + metadata = {key: folder[key] for key in ("id", "name", "description") + if folder.get(key)} + return ( + f"The user has specified folder: {folder.get('name')}\n" + f"Folder metadata: {json.dumps(metadata, ensure_ascii=False)}\n" + "Discover its documents with " + f'browse_documents(folder_id="{folder_id}", recursive=true) ' + f'or search_documents(query, folder_id="{folder_id}", ' + "recursive=true)." + ) + + +def targeting_block(client, doc_id, folder_id=None) -> Optional[str]: + """The chat lanes' leading user message: the folder block, then the + document block, joined as the managed chat joins them; None when + there is nothing to place.""" + blocks = [folder_targeting_block(client, folder_id), + doc_targeting_block(client, doc_id)] + return "\n\n".join(block for block in blocks if block) or None diff --git a/pageindex/client.py b/pageindex/client.py index 09878f060..a36124cba 100644 --- a/pageindex/client.py +++ b/pageindex/client.py @@ -595,8 +595,8 @@ def submit_document( beta_headers (list[str], optional): Cloud-only beta feature headers. folder_id (str, optional): Cloud-only folder (workspace) ID. metadata (dict, optional): Your own JSON-serializable tags for the - document; returned in get_tree/get_ocr responses and - list_documents entries (both modes). + document; returned in get_document/get_tree/get_ocr responses + and list_documents entries (both modes). wait (bool): Return only once the document is ready for use. Cloud: polls status until "completed" (raises on "failed" or after 30 minutes). Local: indexing is synchronous already, so @@ -792,6 +792,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: None = None, instructions: Optional[Union[str, list[dict[str, Any]]]] = None, citations: bool = False, @@ -811,6 +812,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: None = None, instructions: Optional[Union[str, list[dict[str, Any]]]] = None, citations: bool = False, @@ -830,6 +832,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: Literal["chat_completions", "responses", "messages"], instructions: Optional[Union[str, list[dict[str, Any]]]] = None, citations: bool = False, @@ -849,6 +852,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: Literal["chat_completions", "responses"], instructions: Optional[Union[str, list[dict[str, Any]]]] = None, citations: bool = False, @@ -868,6 +872,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: Literal["messages"], instructions: Optional[Union[str, list[dict[str, Any]]]] = None, citations: bool = False, @@ -887,6 +892,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: None = None, instructions: Optional[Union[str, list[dict[str, Any]]]] = None, citations: bool = False, @@ -906,6 +912,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: Optional[Literal["chat_completions", "responses", "messages"]] = None, instructions: Optional[Union[str, list[dict[str, Any]]]] = None, @@ -925,6 +932,7 @@ def chat( model: Optional[str] = None, reasoning_effort: Optional[str] = None, show_process: Union[bool, Mapping[str, Any], None] = None, + folder_id: Optional[str] = None, protocol: Optional[Literal["chat_completions", "responses", "messages"]] = None, instructions: Optional[Union[str, list[dict[str, Any]]]] = None, @@ -977,6 +985,12 @@ def chat( documents: also enforced at the tool layer, not just prompted. Cloud documents: the managed chat scopes server-side; own-model chat targets at the prompt level. + folder_id: Folder ID to steer discovery toward that folder's + documents. Cloud-only. The managed chat scopes it + server-side; own-model chat leads the conversation with + the folder's targeting text (``folder_context``), ahead + of the document block. ``"root"`` is the whole library. + Keep it identical across a conversation's calls. stream: Answer lane: return a ``ChatStream`` — iterate it for the answer as text chunks as they are produced (``show_process`` is on by default, so the run's process @@ -1155,6 +1169,7 @@ def chat( **given.get("reasoning", {})}} return self._responses( messages, model=model, stream=stream, doc_id=doc_id, + folder_id=folder_id, instructions=cast(Optional[str], instructions), max_turns=max_turns, extra_body=body, extra_headers=extra_headers, backend=backend) @@ -1173,6 +1188,7 @@ def chat( **given.get("output_config", {})}} return self._messages( messages, model=model, stream=stream, doc_id=doc_id, + folder_id=folder_id, system=instructions, max_turns=max_turns, extra_body=body, extra_headers=extra_headers, backend=backend) if instructions: @@ -1191,7 +1207,7 @@ def chat( if protocol == "chat_completions": return self.chat_completions( messages, stream=stream, stream_metadata=True, doc_id=doc_id, - enable_citations=enable_citations, + enable_citations=enable_citations, folder_id=folder_id, model=model, max_turns=max_turns, reasoning_effort=reasoning_effort, extra_body=extra_body, extra_headers=extra_headers, backend=backend) @@ -1201,7 +1217,7 @@ def chat( if self._local_chat: from .local_chat import run_chat_stream return run_chat_stream(self, messages, doc_id=doc_id, - model=model, + folder_id=folder_id, model=model, reasoning_effort=reasoning_effort, show_process=resolved, max_turns=max_turns, backend=backend, @@ -1212,6 +1228,7 @@ def chat( stream_metadata=True, enable_citations=enable_citations, doc_id=doc_id, model=model, + folder_id=folder_id, reasoning_effort=reasoning_effort, max_turns=max_turns, backend=backend, @@ -1221,6 +1238,7 @@ def chat( cast(Iterator[dict[str, Any]], chunks), resolved) result = self.chat_completions(messages, doc_id=doc_id, model=model, enable_citations=enable_citations, + folder_id=folder_id, reasoning_effort=reasoning_effort, max_turns=max_turns, backend=backend, extra_headers=extra_headers, @@ -1249,6 +1267,8 @@ def chat_completions( extra_body: Optional[dict[str, Any]] = None, extra_headers: Optional[dict[str, str]] = None, backend: Optional[dict[str, Any]] = None, + *, + folder_id: Optional[str] = None, ) -> Union[dict[str, Any], Iterator[str], Iterator[dict[str, Any]]]: """ Kept for existing code — new code calls ``chat()``. Everything @@ -1296,6 +1316,11 @@ def chat_completions( enforced at the tool layer, not just prompted. Cloud documents: the managed chat scopes server-side; own-model chat targets at the prompt level. + folder_id: Folder ID to steer discovery toward that folder's + documents (cloud-only): the managed chat scopes it + server-side; own-model chat leads the conversation with + the folder's targeting text, ahead of the document block. + ``"root"`` is the whole library. temperature: Sampling temperature, passed through to the model. stream_metadata: With stream=True, yield chunk dicts instead of text pieces. @@ -1358,6 +1383,7 @@ def chat_completions( from .local_chat import run_chat_completions return run_chat_completions( self, messages, stream=stream, doc_id=doc_id, + folder_id=folder_id, temperature=temperature, stream_metadata=stream_metadata, enable_citations=enable_citations, model=model, max_turns=max_turns, top_p=top_p, max_tokens=max_tokens, @@ -1383,6 +1409,7 @@ def chat_completions( from .cloud_api import CloudAPI return cast(CloudAPI, self._api).chat_completions( messages=messages, stream=stream, doc_id=doc_id, + folder_id=folder_id, temperature=temperature, stream_metadata=stream_metadata, enable_citations=enable_citations, extra_body=extra_body, ) @@ -1402,6 +1429,8 @@ def _responses( extra_body: Optional[dict[str, Any]] = None, extra_headers: Optional[dict[str, str]] = None, backend: Optional[dict[str, Any]] = None, + *, + folder_id: Optional[str] = None, ) -> Union[dict[str, Any], Iterator[dict[str, Any]]]: """ The engine behind ``chat(protocol="responses")``: document QA over @@ -1444,6 +1473,10 @@ def _responses( of the cached prompt prefix. Local documents: also enforced at the tool layer; cloud documents: prompt-level targeting only. + folder_id: Folder ID to steer discovery toward that folder's + documents (cloud-only), as the leading targeting text + ahead of the document block. ``"root"`` is the whole + library. instructions: Appended to the managed system prompt. temperature / top_p: Passed through to the model. max_turns: Cap on agent turns per call. @@ -1471,6 +1504,7 @@ def _responses( from .local_chat import run_responses return run_responses( self, input, model=model, stream=stream, doc_id=doc_id, + folder_id=folder_id, instructions=instructions, temperature=temperature, top_p=top_p, max_turns=max_turns, max_output_tokens=max_output_tokens, reasoning=reasoning, extra_body=extra_body, @@ -1494,6 +1528,8 @@ def _messages( extra_body: Optional[dict[str, Any]] = None, extra_headers: Optional[dict[str, str]] = None, backend: Optional[dict[str, Any]] = None, + *, + folder_id: Optional[str] = None, ) -> Union[dict[str, Any], Iterator[Any]]: """ The engine behind ``chat(protocol="messages")``: document QA over @@ -1532,6 +1568,10 @@ def _messages( targeting block it adds is re-set each call. Local documents: also enforced at the tool layer; cloud documents: prompt-level targeting only. + folder_id: Folder ID to steer discovery toward that folder's + documents (cloud-only), as the leading targeting text + ahead of the document block. ``"root"`` is the whole + library. system: Appended after the managed system blocks. temperature / top_p / top_k / stop_sequences: Passed through. max_turns: Cap on agent turns per call (default 10, like the @@ -1557,7 +1597,7 @@ def _messages( from .local_chat import run_messages return run_messages( self, messages, model=model, max_tokens=max_tokens, - stream=stream, doc_id=doc_id, system=system, + stream=stream, doc_id=doc_id, folder_id=folder_id, system=system, temperature=temperature, top_p=top_p, top_k=top_k, stop_sequences=stop_sequences, max_turns=max_turns, thinking=thinking, extra_body=extra_body, @@ -1569,9 +1609,10 @@ def _messages( def get_document(self, doc_id: str) -> dict[str, Any]: """ Get document metadata: {'id', 'name', 'description', 'status', - 'createdAt', 'pageNum', 'folderId'}. Status is one of "queued", - "processing", "completed", "failed" (local documents are - always "completed"; local 'folderId' is always None). + 'createdAt', 'pageNum', 'folderId', 'metadata'}. Status is one of + "queued", "processing", "completed", "failed" (local documents are + always "completed"; local 'folderId' is always None). 'metadata' + is your own tags from ``submit_document``, or None. 'createdAt' is UTC with no timezone marker, in both modes. To show it in the user's timezone:: @@ -1615,7 +1656,6 @@ def list_documents( def agent_tools( self, include_management: bool = False, - doc_id: Optional[Union[str, list[str]]] = None, ) -> list[Callable[..., str]]: """ Plain functions for any agent framework (LangChain, PydanticAI, ...). @@ -1644,16 +1684,12 @@ def agent_tools( is the gate — the default serves what the read-only endpoint (``?tools=read``) registers; True connects to the full ``/mcp`` list (upload, delete, ...). - doc_id: Local only — restrict the tools to this document ID - (or list of IDs), enforced at the tool layer: out-of-scope - lookups return NOT_FOUND. Raises on cloud. """ from .agent_tools import build_agent_tools - return build_agent_tools(self, include_management, doc_ids=doc_id) + return build_agent_tools(self, include_management) def as_openai_tools(self, include_management: bool = False, - hosted: bool = False, - doc_id: Optional[Union[str, list[str]]] = None) -> list: + hosted: bool = False) -> list: """ Tools for the OpenAI Agents SDK — pass to ``Agent(tools=...)`` (or ``openai_agent_config()`` for all the Agent slots in one @@ -1692,19 +1728,14 @@ def as_openai_tools(self, include_management: bool = False, True switches to the full ``/mcp`` list. hosted (bool): Cloud only — hand the MCP connection to OpenAI for server-side tool execution (OpenAI models only). - doc_id: Local only — restrict the tools to this document ID - (or list of IDs), enforced at the tool layer: out-of-scope - lookups return NOT_FOUND. Raises on cloud. """ from .integrations.openai_agents import build_openai_tools - return build_openai_tools(self, include_management, hosted, - doc_ids=doc_id) + return build_openai_tools(self, include_management, hosted) def _local_doc_scope(self, doc_id): """doc_id for the tool layer: passed through locally (structural allowlist), dropped on cloud — its tools take no allowlist, so - own-model chat and the config helpers target at the prompt level - only.""" + own-model chat targets at the prompt level only.""" from .agent_tools import _require_doc_selection _require_doc_selection(doc_id) if not getattr(self, "api_key", None): @@ -1713,7 +1744,7 @@ def _local_doc_scope(self, doc_id): def openai_agent_config( self, - doc_id: Optional[Union[str, list[str]]] = None, + *, include_management: bool = False, model: Optional[str] = None, model_settings: Optional[Any] = None, @@ -1725,15 +1756,16 @@ def openai_agent_config( agent = Agent(**client.openai_agent_config()) - Sugar over the explicit form — ``agent_instructions`` (with - ``doc_id`` targeting) as the instructions and - ``as_openai_tools`` as the tools; clients with a configured - ``chat_model`` — local mode, or cloud with ``chat_model=`` — - also carry it (a plain cloud client omits ``model`` so the - framework default applies). To customize further, switch to - those methods directly. You run this config in your own - environment, so its model auth comes from there — - ``chat_backend`` does not travel with it. + Sugar over the explicit form — ``agent_instructions`` as the + instructions and ``as_openai_tools`` as the tools; clients with a + configured ``chat_model`` — local mode, or cloud with + ``chat_model=`` — also carry it (a plain cloud client omits + ``model`` so the framework default applies). To target a folder + or documents, prepend ``folder_context(folder_id)`` / + ``document_context(doc_id)`` to your first message; to + customize further, switch to those methods directly. You run this + config in your own environment, so its model auth comes from + there — ``chat_backend`` does not travel with it. Prompt caching: OpenAI models cache server-side on their own; LiteLLM-routed Claude (Anthropic, Bedrock, Vertex) gets its @@ -1743,9 +1775,6 @@ def openai_agent_config( wholesale drops the marks instead. Args: - doc_id: Document ID or list of IDs to target, as in - ``agent_instructions``. Local: also enforced at the tool - layer, not just prompted. Cloud: prompt-level targeting. include_management (bool): Also expose tools that modify the library. model: Backend model name; overrides the local default. Same @@ -1757,14 +1786,11 @@ def openai_agent_config( name (str): Agent display name; in composition it also seeds the SDK-derived handoff and ``as_tool`` names. """ - from .agent_tools import build_agent_instructions - scope = self._local_doc_scope(doc_id) + from .agent_tools import _base_instructions config: dict[str, Any] = { "name": name, - "instructions": build_agent_instructions( - self, doc_id, scoped=scope is not None, - include_management=include_management), - "tools": self.as_openai_tools(include_management, doc_id=scope), + "instructions": _base_instructions(self, include_management), + "tools": self.as_openai_tools(include_management), } model = model or (self.chat_model if self._local_chat else None) if model: @@ -1793,9 +1819,7 @@ def openai_agent_config( return config def as_anthropic_tools(self, include_management: bool = False, - asynchronous: bool = False, - doc_id: Optional[Union[str, list[str]]] = None, - ) -> list: + asynchronous: bool = False) -> list: """ Runnable tools for the Anthropic SDK's tool runner — pass to ``client.beta.messages.tool_runner(tools=...)`` (or @@ -1838,18 +1862,14 @@ def as_anthropic_tools(self, include_management: bool = False, ``AsyncAnthropic`` (each tool call runs in a worker thread, keeping blocking I/O off your event loop). The sync and async runners each accept only their own flavor. - doc_id: Local only — restrict the tools to this document ID - (or list of IDs), enforced at the tool layer: out-of-scope - lookups return NOT_FOUND. Raises on cloud. """ from .integrations.anthropic_sdk import build_anthropic_tools - return build_anthropic_tools(self, include_management, asynchronous, - doc_ids=doc_id) + return build_anthropic_tools(self, include_management, asynchronous) def anthropic_runner_config( self, model: str, - doc_id: Optional[Union[str, list[str]]] = None, + *, include_management: bool = False, asynchronous: bool = False, max_tokens: Optional[int] = None, @@ -1865,24 +1885,22 @@ def anthropic_runner_config( messages=[{"role": "user", "content": "..."}], ) - Sugar over the explicit form — ``agent_instructions`` (with - ``doc_id`` targeting) as the system prompt and - ``as_anthropic_tools`` as the tools — plus the ``max_tokens`` - default and 10-turn ``max_iterations`` bound + Sugar over the explicit form — ``agent_instructions`` as the + system prompt and ``as_anthropic_tools`` as the tools — plus the + ``max_tokens`` default and 10-turn ``max_iterations`` bound ``chat(protocol="messages")`` uses, and a top-level ``cache_control`` so each loop turn re-reads the growing prompt from cache (pop the key if you place your own breakpoints — the API allows four). Unlike the chat lane, ``system`` here is the bare instructions string, without the chat - header or its block-level breakpoint. To customize further, - switch to those methods directly. + header or its block-level breakpoint. To target a folder or + documents, prepend ``folder_context(folder_id)`` / + ``document_context(doc_id)`` to your first message; to + customize further, switch to those methods directly. Args: model: Backend model name (also resolves the ``max_tokens`` default). - doc_id: Document ID or list of IDs to target, as in - ``agent_instructions``. Local: also enforced at the tool - layer, not just prompted. Cloud: prompt-level targeting. include_management (bool): Also expose tools that modify the library. asynchronous (bool): Build async runnables for @@ -1895,26 +1913,21 @@ def anthropic_runner_config( default above it. Pass it here, not alongside the unpacked config, so the default stays valid. """ - from .agent_tools import build_agent_instructions + from .agent_tools import _base_instructions from .local_chat import _default_max_tokens, _validate_max_turns _validate_max_turns(max_turns) - scope = self._local_doc_scope(doc_id) return { "model": model, "max_tokens": (max_tokens if max_tokens is not None else _default_max_tokens(model, thinking)), - "system": build_agent_instructions( - self, doc_id, scoped=scope is not None, - include_management=include_management), - "tools": self.as_anthropic_tools(include_management, asynchronous, - doc_id=scope), + "system": _base_instructions(self, include_management), + "tools": self.as_anthropic_tools(include_management, asynchronous), "max_iterations": max_turns if max_turns is not None else 10, **({"thinking": thinking} if thinking is not None else {}), "cache_control": {"type": "ephemeral"}, } - def as_claude_mcp(self, include_management: bool = False, - doc_id: Optional[Union[str, list[str]]] = None, + def as_claude_mcp(self, include_management: bool = False, *, server_name: str = "pageindex"): """ ``mcp_servers`` entry for the Claude Agent SDK. @@ -1926,17 +1939,15 @@ def as_claude_mcp(self, include_management: bool = False, ``True`` connects to the full tool set. Local: returns an in-process SDK MCP server exposing the agent tools, gated the same way at registration (requires ``claude-agent-sdk``; - ``pip install 'pageindex[claude]'``). ``doc_id`` (local only) - restricts those tools to that document ID (or list), enforced at - the tool layer; it raises on cloud. - ``server_name`` names the in-process server — match it to the key - you register the entry under (cloud entries carry no name). + ``pip install 'pageindex[claude]'``). ``server_name`` names the + in-process server — match it to the key you register the entry + under (cloud entries carry no name). Cloud hosts that surface MCP server instructions receive the same guidance ``agent_instructions()`` returns natively — passing both duplicates the text (harmless). ``system_prompt`` stays the - recommended channel: it is guaranteed delivery, carries ``doc_id`` - targeting, and is the only channel local mode has. + recommended channel: it is guaranteed delivery, and the only + channel local mode has. Usage (or ``claude_agent_config()`` for all three slots in one call):: @@ -1949,12 +1960,12 @@ def as_claude_mcp(self, include_management: bool = False, ) """ from .integrations.claude_agent_sdk import build_claude_mcp - return build_claude_mcp(self, include_management, doc_ids=doc_id, + return build_claude_mcp(self, include_management, server_name=server_name) def claude_agent_config( self, - doc_id: Optional[Union[str, list[str]]] = None, + *, include_management: bool = False, server_name: str = "pageindex", ) -> dict[str, Any]: @@ -1967,35 +1978,28 @@ def claude_agent_config( (``agent_instructions``) and the server entry (``as_claude_mcp``, itself the tool gate) with its ``allowed_tools`` pre-approval, one ``include_management`` and ``server_name`` applied - everywhere. To customize (your own system prompt, extra + everywhere. To target a folder or documents, prepend + ``folder_context(folder_id)`` / ``document_context(doc_id)`` to + your prompt; to customize (your own system prompt, extra servers), switch to those methods directly. Args: - doc_id: Document ID or list of IDs to target, as in - ``agent_instructions``. Local: also enforced at the tool - layer, not just prompted. Cloud: prompt-level targeting. include_management (bool): Also allow tools that modify the library. server_name (str): Key the server is registered under; locally also the name the SDK server declares. """ - from .agent_tools import build_agent_instructions - scope = self._local_doc_scope(doc_id) + from .agent_tools import _base_instructions return { - "system_prompt": build_agent_instructions( - self, doc_id, scoped=scope is not None, - include_management=include_management), + "system_prompt": _base_instructions(self, include_management), "mcp_servers": {server_name: self.as_claude_mcp( - include_management, doc_id=scope, server_name=server_name)}, + include_management, server_name=server_name)}, # Pre-approval only — the server itself is already gated (the # read-only endpoint on cloud, the registered set locally). "allowed_tools": [f"mcp__{server_name}"], } - def agent_instructions( - self, doc_id: Optional[Union[str, list[str]]] = None, - include_management: bool = False, - ) -> str: + def agent_instructions(self, *, include_management: bool = False) -> str: """ Orchestration guidance for document QA agents — pass as the agent's system prompt (or append to your own). @@ -2006,21 +2010,41 @@ def agent_instructions( SDK release. Raises PageIndexAPIError if the server cannot be reached. Local: the built-in guidance for the in-process tools. - With ``doc_id`` (str or list, same shape as ``chat``), - appends the target documents' names and metadata and directs the - agent to work within them. Raises PageIndexAPIError if a doc_id - does not exist, or if its name is shadowed by a newer same-name - document — the name-addressed tools could not reach it (the - ``*_agent_config`` bundles, whose tools carry the doc_id scope, - relax this to duplicates within the targeted set). + Static by design: document targeting is conversation content, not + guidance — see ``document_context()``. ``include_management``: fetch the guidance for the full tool set, matching tools built with ``include_management=True`` (cloud; local guidance is a single set). """ - from .agent_tools import build_agent_instructions - return build_agent_instructions( - self, doc_id, include_management=include_management) + from .agent_tools import _base_instructions + return _base_instructions(self, include_management) + + def document_context(self, doc_id: Union[str, list[str]]) -> str: + """ + Document targeting text for the first user message: the target + documents' names and metadata, and the directive to work within + them. ``chat(doc_id=...)`` places it for you; on the framework + routes you own the conversation, so lead with it yourself:: + + Runner.run_sync(agent, [ + {"role": "user", "content": client.document_context(doc_id)}, + {"role": "user", "content": question}, + ]) + + (or prepend it to the prompt text where the framework takes a + string). Conversation content, not system prompt: it varies per + request, so keeping it out of the system prompt leaves the cached + prefix stable. + + ``doc_id``: a document ID or list of IDs, as in ``chat``. Raises + PageIndexAPIError if a document does not exist. + """ + from .agent_tools import doc_targeting_block + if doc_id is None: + raise PageIndexAPIError("doc_id must be a string or a list of " + "strings.") + return cast(str, doc_targeting_block(self, doc_id)) def citation_prompt(self, format: str = "cite") -> str: """ @@ -2044,6 +2068,23 @@ def citation_prompt(self, format: str = "cite") -> str: from .agent_tools import fetch_citation_prompt return fetch_citation_prompt(self, format or "cite") + def folder_context(self, folder_id: str) -> str: + """ + Folder targeting text for the first user message, placed as + ``document_context`` is (and ahead of it, the managed chat's + order): the folder's name and metadata, and the directive to + discover its documents there, rendered as the managed chat renders + its own ``folder_id``. ``chat(folder_id=...)`` places it for you. + Cloud-only: local libraries have no folders. ``"root"`` is the + library itself: ``""``, nothing to place, as the managed chat + places nothing for it. Raises PageIndexAPIError if the folder does + not exist. + """ + from .agent_tools import folder_targeting_block + if folder_id is None: + raise PageIndexAPIError("folder_id must be a string.") + return folder_targeting_block(self, folder_id) or "" + # ---------- FOLDER MANAGEMENT ---------- def create_folder( diff --git a/pageindex/cloud_api.py b/pageindex/cloud_api.py index a25f72606..94e2cfd39 100644 --- a/pageindex/cloud_api.py +++ b/pageindex/cloud_api.py @@ -199,6 +199,7 @@ def chat_completions( stream_metadata: bool = False, enable_citations: bool = False, extra_body: Optional[Dict[str, Any]] = None, + folder_id: Optional[str] = None, ) -> Union[Dict[str, Any], Iterator[str], Iterator[Dict[str, Any]]]: """ PageIndex Chat Completions. Optionally scoped to specific PageIndex documents. @@ -211,6 +212,7 @@ def chat_completions( stream_metadata (bool, optional): If True and stream=True, return raw chunks with metadata instead of just text. Default is False. enable_citations (bool, optional): Enable citation instructions in responses. Default is False. extra_body (Optional[Dict[str, Any]], optional): Extra request fields, merged into the payload last. + folder_id (Optional[str], optional): Folder ID to steer discovery toward one folder; "root" means the whole library. Returns: Union[Dict[str, Any], Iterator[str], Iterator[Dict[str, Any]]]: @@ -226,6 +228,9 @@ def chat_completions( if doc_id is not None: payload["doc_id"] = doc_id + if folder_id: + payload["folder_id"] = folder_id + if temperature is not None: payload["temperature"] = temperature @@ -316,7 +321,7 @@ def _stream_chat_response_raw(self, response: requests.Response) -> Iterator[Dic def get_document(self, doc_id: str) -> Dict[str, Any]: """ - Get document metadata including id, name, description, status, createdAt, and pageNum. + Get document metadata. Args: doc_id (str): Document ID. @@ -329,6 +334,8 @@ def get_document(self, doc_id: str) -> Dict[str, Any]: - status (str): Processing status (e.g., "queued", "processing", "completed", "failed") - createdAt (str): Creation timestamp in ISO format - pageNum (int): Number of pages in the document + - folderId (str | None): Containing folder ID + - metadata (dict | None): Your own tags from submit_document """ response = requests.get( f"{self.BASE_URL}/doc/{_enc(doc_id)}/metadata/", diff --git a/pageindex/integrations/claude_agent_sdk.py b/pageindex/integrations/claude_agent_sdk.py index 80b3b321b..bea250935 100644 --- a/pageindex/integrations/claude_agent_sdk.py +++ b/pageindex/integrations/claude_agent_sdk.py @@ -15,10 +15,8 @@ from ..errors import PageIndexAPIError -def build_claude_mcp(client, include_management: bool = False, doc_ids=None, +def build_claude_mcp(client, include_management: bool = False, *, server_name: str = "pageindex"): - from ..agent_tools import _require_local_scope - _require_local_scope(client, doc_ids) if getattr(client, "api_key", None): # include_management picks the endpoint — the URL itself is the # gate (?tools=read serves only readOnlyHint-annotated tools). @@ -61,7 +59,7 @@ def tool_kwargs(name: str) -> dict: tool(name, description, schema, **tool_kwargs(name))(make_handler(invoke)) for name, description, schema, invoke - in _tool_specs(client, include_management, doc_ids) + in _tool_specs(client, include_management) ] return create_sdk_mcp_server(name=server_name, version=sdk_version(), tools=tools) diff --git a/pageindex/integrations/openai_agents.py b/pageindex/integrations/openai_agents.py index ed69f1f0d..a9902e732 100644 --- a/pageindex/integrations/openai_agents.py +++ b/pageindex/integrations/openai_agents.py @@ -84,8 +84,6 @@ def build_openai_tools(client, include_management: bool = False, "as_openai_tools requires the OpenAI Agents SDK — " "pip install openai-agents." ) from exc - from ..agent_tools import _require_local_scope - _require_local_scope(client, doc_ids) if getattr(client, "api_key", None) and hosted: # include_management picks the endpoint — the URL itself is the # gate (?tools=read serves only readOnlyHint-annotated tools), so diff --git a/pageindex/local_api.py b/pageindex/local_api.py index 40aa3aedc..e60433932 100644 --- a/pageindex/local_api.py +++ b/pageindex/local_api.py @@ -335,7 +335,8 @@ def get_document(self, doc_id: str) -> dict[str, Any]: if meta is None: raise PageIndexAPIError("Failed to get document metadata: Document not found") return {key: meta.get(key) for key in - ("id", "name", "description", "status", "createdAt", "pageNum", "folderId")} + ("id", "name", "description", "status", "createdAt", "pageNum", + "folderId", "metadata")} def delete_document(self, doc_id: str) -> dict[str, Any]: if not self._store.delete_document(doc_id): diff --git a/pageindex/local_chat.py b/pageindex/local_chat.py index fc981731a..f60df2c26 100644 --- a/pageindex/local_chat.py +++ b/pageindex/local_chat.py @@ -13,7 +13,7 @@ import uuid from typing import Any, Iterator, Mapping, Optional, Union -from .agent_tools import _base_instructions, doc_targeting_block +from .agent_tools import _base_instructions, targeting_block from .chat_stream import ChatStream from .errors import PageIndexAPIError, _pageindex_cause @@ -32,19 +32,6 @@ def _managed_instructions(client, extra_system: list[str]) -> str: return "\n\n".join([CHAT_HEADER, base, *extra_system]) -def _doc_block(client, doc_id, scoped: bool) -> Optional[str]: - if doc_id is None: - return None - if not isinstance(doc_id, (str, list)): - raise PageIndexAPIError("doc_id must be a string or a list of " - "strings.") - # scoped: local surfaces also pass doc_id into the tool layer, so name - # resolution happens inside the allowlist — only a duplicate name - # within the targeted set shadows. Cloud tools take no allowlist - # (targeting is prompt-level), so the whole library shadows. - return doc_targeting_block(client, doc_id, scoped=scoped) - - def _system_text(content: Any) -> str: """Text of a system/developer message: a string, or text parts joined.""" if isinstance(content, str): @@ -417,7 +404,7 @@ def _validate_max_turns(max_turns) -> None: def _conversation_cache_key(model_name: str, instructions: str, doc_id, - items) -> str: + items, folder_id=None) -> str: """Stable per-conversation cache-routing key, sent as the OpenAI ``prompt_cache_key`` through ModelSettings.extra_body (openai-agents 0.20 no longer derives it from RunConfig.group_id — verified against a @@ -427,10 +414,12 @@ def _conversation_cache_key(model_name: str, instructions: str, doc_id, Callers pass the conversation's own items, never the SDK-prepended doc-targeting block: that block is byte-identical for every conversation about a document and would pool them all under one key. - doc_id carries the targeting identity instead — the same opening - question against different documents is different conversations.""" + doc_id and folder_id carry the targeting identity instead — the same + opening question against different documents is different + conversations.""" scope = [doc_id] if isinstance(doc_id, str) else doc_id seed = json.dumps([model_name, instructions, scope, + *([folder_id] if folder_id else []), items[0] if items else None], sort_keys=True, default=str) return "pageindex-" + hashlib.sha256(seed.encode()).hexdigest()[:16] @@ -644,19 +633,21 @@ def _responses_usage(raw_responses) -> dict: def _chat_agent(client, messages, doc_id, model, temperature=None, top_p=None, reasoning_effort=None, extra_body=None, max_tokens=None, backend=None, extra_headers=None, + folder_id=None, ) -> "tuple[Any, list, str]": """The chat lane's shared prologue: validated history, doc targeting, and the configured agent. Returns (agent, input items, model name).""" system_texts, history = _split_chat_messages(messages) scope = client._local_doc_scope(doc_id) - block = _doc_block(client, doc_id, scoped=scope is not None) + block = targeting_block(client, doc_id, folder_id) items = ([{"role": "user", "content": block}] if block else []) + history model_name = model or client.chat_model managed = _managed_instructions(client, system_texts) agent = _openai_agent(client, "chat", model_name, managed, temperature, top_p, doc_ids=scope, cache_key=_conversation_cache_key( - model_name, managed, doc_id, history), + model_name, managed, doc_id, history, + folder_id), reasoning_effort=reasoning_effort, extra_body=extra_body, max_tokens=max_tokens, backend=_merged_backend(client, backend), @@ -914,7 +905,7 @@ def run_chat_stream(client, messages, doc_id=None, model=None, reasoning_effort=None, show_process: Union[bool, Mapping[str, Any]] = False, max_turns=None, backend=None, extra_headers=None, - extra_body=None, + extra_body=None, folder_id=None, ) -> ChatStream: """chat(stream=True): validation and the agent build run here, eagerly; the run itself starts when the returned stream's chosen view is first @@ -932,7 +923,8 @@ def run_chat_stream(client, messages, doc_id=None, model=None, agent, items, _ = _chat_agent(client, messages, doc_id, model, reasoning_effort=reasoning_effort, extra_body=extra_body, backend=backend, - extra_headers=extra_headers) + extra_headers=extra_headers, + folder_id=folder_id) run_kwargs = _run_kwargs(max_turns) def events(): @@ -954,6 +946,7 @@ def run_chat_completions(client, messages, stream: bool = False, extra_body: Optional[dict] = None, extra_headers: Optional[dict] = None, backend: Optional[dict] = None, + folder_id: Optional[str] = None, ) -> Union[dict, Iterator[str], Iterator[dict]]: if enable_citations: raise PageIndexAPIError( @@ -967,7 +960,7 @@ def run_chat_completions(client, messages, stream: bool = False, client, messages, doc_id, model, temperature=temperature, top_p=top_p, reasoning_effort=reasoning_effort, extra_body=extra_body, max_tokens=max_tokens, backend=backend, - extra_headers=extra_headers) + extra_headers=extra_headers, folder_id=folder_id) reported_model = _reported_model(model_name) recorded: dict = {} _record_chat_finish(agent, recorded) @@ -1054,6 +1047,7 @@ def run_responses(client, input, model: Optional[str] = None, extra_body: Optional[dict] = None, extra_headers: Optional[dict] = None, backend: Optional[dict] = None, + folder_id: Optional[str] = None, ) -> Union[dict, Iterator[dict]]: _require_openai_agents("chat(protocol='responses')") _validate_max_turns(max_turns) @@ -1066,7 +1060,7 @@ def run_responses(client, input, model: Optional[str] = None, raise PageIndexAPIError("messages must be a non-empty string or list " "of item dicts.") scope = client._local_doc_scope(doc_id) - block = _doc_block(client, doc_id, scoped=scope is not None) + block = targeting_block(client, doc_id, folder_id) conversation = items if block: items = [{"role": "user", "content": block}] + items @@ -1076,7 +1070,8 @@ def run_responses(client, input, model: Optional[str] = None, agent = _openai_agent(client, "responses", model_name, managed, temperature, top_p, doc_ids=scope, cache_key=_conversation_cache_key( - model_name, managed, doc_id, conversation), + model_name, managed, doc_id, conversation, + folder_id), reasoning=reasoning, extra_body=extra_body, max_tokens=max_output_tokens, backend=_merged_backend(client, backend), @@ -1267,16 +1262,13 @@ def _anthropic_client(backend=None): return client -def _anthropic_system(client, extra_system, block: Optional[str]) -> list[dict]: +def _anthropic_system(client, extra_system) -> list[dict]: """System blocks: cache_control marks the stable managed prefix only - (the API allows 4 breakpoints total — the varying doc block and caller - blocks must not consume the budget); the doc block and caller system - content follow as their own blocks.""" + (the API allows 4 breakpoints total — caller blocks must not consume + the budget); caller system content follows as its own blocks.""" blocks = [{"type": "text", "text": CHAT_HEADER + "\n\n" + _base_instructions(client), "cache_control": {"type": "ephemeral"}}] - if block: - blocks.append({"type": "text", "text": block}) if extra_system is None: return blocks if isinstance(extra_system, str): @@ -1370,6 +1362,7 @@ def run_messages(client, messages, model: str, extra_body: Optional[dict] = None, extra_headers: Optional[dict] = None, backend: Optional[dict] = None, + folder_id: Optional[str] = None, ) -> Union[dict, Iterator[Any]]: from .integrations.anthropic_sdk import build_anthropic_tools @@ -1384,14 +1377,16 @@ def run_messages(client, messages, model: str, raise PageIndexAPIError("messages must be a non-empty string or a " "list of message dicts.") scope = client._local_doc_scope(doc_id) - block = _doc_block(client, doc_id, scoped=scope is not None) + block = targeting_block(client, doc_id, folder_id) prepared = [dict(message) for message in messages] + if block: + prepared = [{"role": "user", "content": block}] + prepared passthrough = {key: value for key, value in { "temperature": temperature, "top_p": top_p, "top_k": top_k, "stop_sequences": stop_sequences, "thinking": thinking, "extra_body": extra_body, "extra_headers": extra_headers, }.items() if value is not None} - system_blocks = _anthropic_system(client, system, block) + system_blocks = _anthropic_system(client, system) # Top-level cache_control: the server re-marks the newest block each # turn, so the loop re-reads the growing conversation from cache. # Counts toward the 4-breakpoint limit (live-verified 400 past it). diff --git a/tests/test_agent_tools.py b/tests/test_agent_tools.py index 1d3af6b5e..3de8384ab 100644 --- a/tests/test_agent_tools.py +++ b/tests/test_agent_tools.py @@ -738,11 +738,10 @@ def test_claude_agent_config_is_sugar_over_the_explicit_form( assert renamed["allowed_tools"] == ["mcp__docs"] -def test_claude_agent_config_local(client, store_path): +def test_claude_agent_config_local(client): pytest.importorskip("claude_agent_sdk") - seed_doc(store_path, "pi-a", "report.pdf") - config = client.claude_agent_config(doc_id="pi-a") - assert "report.pdf" in config["system_prompt"] + config = client.claude_agent_config() + assert config["system_prompt"] == AGENT_INSTRUCTIONS assert config["allowed_tools"] == ["mcp__pageindex"] assert config["mcp_servers"]["pageindex"]["name"] == "pageindex" # The SDK server's declared identity follows the registration key. @@ -751,13 +750,12 @@ def test_claude_agent_config_local(client, store_path): assert renamed["allowed_tools"] == ["mcp__docs"] -def test_openai_agent_config_local(client, store_path): +def test_openai_agent_config_local(client): pytest.importorskip("agents") from agents import Agent - seed_doc(store_path, "pi-a", "report.pdf") - config = client.openai_agent_config(doc_id="pi-a") + config = client.openai_agent_config() assert config["name"] == "PageIndex" - assert "report.pdf" in config["instructions"] + assert config["instructions"] == AGENT_INSTRUCTIONS assert [tool.name for tool in config["tools"]] == list(tool_names()) assert config["model"] == client.retrieve_model assert client.openai_agent_config(model="gpt-x")["model"] == "gpt-x" @@ -891,17 +889,15 @@ def test_openai_agent_config_cloud_omits_model(cloud_with_fake_bridge): "get_document"] -def test_anthropic_runner_config_shapes(client, store_path): +def test_anthropic_runner_config_shapes(client): pytest.importorskip("anthropic") import anthropic from anthropic.lib.tools import BetaAsyncFunctionTool - seed_doc(store_path, "pi-a", "report.pdf") - config = client.anthropic_runner_config(model="claude-3-opus-20240229", - doc_id="pi-a") + config = client.anthropic_runner_config(model="claude-3-opus-20240229") assert config["max_tokens"] == 4096 assert config["max_iterations"] == 10 assert config["cache_control"] == {"type": "ephemeral"} - assert "report.pdf" in config["system"] + assert config["system"] == AGENT_INSTRUCTIONS assert [tool.name for tool in config["tools"]] == list(tool_names()) assert (client.anthropic_runner_config(model="claude-sonnet-4-5") ["max_tokens"] == 8192) @@ -929,104 +925,6 @@ def test_anthropic_runner_config_cloud(cloud_with_fake_bridge): "get_document"] -# ── config helpers: doc_id is structural in the tools, not just prompted ── - -def test_openai_agent_config_doc_scope_enforced_in_tools(client, store_path): - pytest.importorskip("agents") - seed_doc(store_path, "pi-a", "report.pdf") - seed_doc(store_path, "pi-b", "payroll.pdf", - created_at="2026-08-02T10:00:00.123000") - tools = {tool.name: tool - for tool in client.openai_agent_config(doc_id="pi-a")["tools"]} - out = asyncio.run(tools["get_page_content"].on_invoke_tool( - None, json.dumps({"doc_name": "payroll.pdf", "pages": "1"}))) - assert json.loads(out["text"])["errorCode"] == "NOT_FOUND" - out = asyncio.run(tools["browse_documents"].on_invoke_tool(None, "{}")) - assert [doc["name"] - for doc in json.loads(out["text"])["documents"]] == ["report.pdf"] - - -def test_anthropic_runner_config_doc_scope_enforced_in_tools(client, - store_path): - pytest.importorskip("anthropic") - from anthropic.lib.tools import ToolError - seed_doc(store_path, "pi-a", "report.pdf") - seed_doc(store_path, "pi-b", "payroll.pdf", - created_at="2026-08-02T10:00:00.123000") - config = client.anthropic_runner_config(model="claude-sonnet-4-5", - doc_id="pi-a") - tools = {tool.name: tool for tool in config["tools"]} - with pytest.raises(ToolError, match="NOT_FOUND"): - tools["get_page_content"].call({"doc_name": "payroll.pdf", - "pages": "1"}) - browse = json.loads(tools["browse_documents"].call({})[0]["text"]) - assert [doc["name"] for doc in browse["documents"]] == ["report.pdf"] - - -def test_claude_agent_config_doc_scope_enforced_in_tools(client, store_path, - monkeypatch): - """claude_agent_config(doc_id=...) must wire scope all the way into the - handlers it registers — an out-of-scope document returns NOT_FOUND. The - assertion has to drive those handlers, or build_claude_mcp's doc_ids - pass-through goes unguarded.""" - claude_agent_sdk = pytest.importorskip("claude_agent_sdk") - seed_doc(store_path, "pi-a", "report.pdf") - seed_doc(store_path, "pi-b", "payroll.pdf", - created_at="2026-08-02T10:00:00.123000") - - registered = {} - create_server = claude_agent_sdk.create_sdk_mcp_server - - def capture(**kwargs): - registered.update(kwargs) - return create_server(**kwargs) - - monkeypatch.setattr(claude_agent_sdk, "create_sdk_mcp_server", capture) - config = client.claude_agent_config(doc_id="pi-a") - assert "report.pdf" in config["system_prompt"] - handlers = {spec.name: spec.handler for spec in registered["tools"]} - result = asyncio.run(handlers["get_page_content"]( - {"doc_name": "payroll.pdf", "pages": "1"})) - assert result.get("is_error") - assert json.loads(result["content"][0]["text"])["errorCode"] == "NOT_FOUND" - browse = asyncio.run(handlers["browse_documents"]({})) - listed = json.loads(browse["content"][0]["text"])["documents"] - assert [doc["name"] for doc in listed] == ["report.pdf"] - - -def test_openai_agent_config_scoped_shadow_check(client, store_path): - """The bundles' tools resolve names inside the allowlist, so a same-name - document outside the target set must not block — only an in-set - duplicate shadows.""" - pytest.importorskip("agents") - seed_doc(store_path, "pi-old", "report.pdf") - seed_doc(store_path, "pi-new", "report.pdf", - created_at="2026-08-02T10:00:00.123000") - config = client.openai_agent_config(doc_id="pi-old") - assert "report.pdf" in config["instructions"] - with pytest.raises(PageIndexAPIError, match="shadowed"): - client.openai_agent_config(doc_id=["pi-old", "pi-new"]) - - -def test_anthropic_runner_config_scoped_shadow_check(client, store_path): - pytest.importorskip("anthropic") - seed_doc(store_path, "pi-old", "report.pdf") - seed_doc(store_path, "pi-new", "report.pdf", - created_at="2026-08-02T10:00:00.123000") - config = client.anthropic_runner_config(model="claude-sonnet-4-5", - doc_id="pi-old") - assert "report.pdf" in config["system"] - - -def test_claude_agent_config_scoped_shadow_check(client, store_path): - pytest.importorskip("claude_agent_sdk") - seed_doc(store_path, "pi-old", "report.pdf") - seed_doc(store_path, "pi-new", "report.pdf", - created_at="2026-08-02T10:00:00.123000") - config = client.claude_agent_config(doc_id="pi-old") - assert "report.pdf" in config["system_prompt"] - - def test_anthropic_runner_config_thinking_lifts_max_tokens(client): pytest.importorskip("anthropic") config = client.anthropic_runner_config( @@ -1112,30 +1010,6 @@ def instructions(self): assert len(created) == 2 # cached per gate -def test_doc_scope_rejected_on_cloud_openai(): - pytest.importorskip("agents") - cloud = PageIndexCloudClient(api_key="pi-test-key") - with pytest.raises(PageIndexAPIError, match="server-side"): - cloud.as_openai_tools(doc_id="pi-a") - # The hosted branch returns before _tool_specs — it must reject too, - # not silently drop the allowlist. - with pytest.raises(PageIndexAPIError, match="server-side"): - cloud.as_openai_tools(hosted=True, doc_id="pi-a") - - -def test_doc_scope_rejected_on_cloud_anthropic(): - pytest.importorskip("anthropic") - cloud = PageIndexCloudClient(api_key="pi-test-key") - with pytest.raises(PageIndexAPIError, match="server-side"): - cloud.as_anthropic_tools(doc_id="pi-a") - - -def test_doc_scope_rejected_on_cloud_claude(): - cloud = PageIndexCloudClient(api_key="pi-test-key") - with pytest.raises(PageIndexAPIError, match="server-side"): - cloud.as_claude_mcp(doc_id="pi-a") - - def test_as_anthropic_tools_missing_dependency(client, monkeypatch): monkeypatch.setitem(sys.modules, "anthropic", None) with pytest.raises(PageIndexAPIError, match="anthropic"): @@ -2405,18 +2279,37 @@ def test_agent_instructions_default(client): assert 'sort="relevance"' not in text # cloud-side capability -def test_agent_instructions_with_doc_id(client, store_path): +def test_document_context(client, store_path): + """Targeting is conversation content; the instructions stay static.""" seed_doc(store_path, "pi-a", "report.pdf") - text = client.agent_instructions(doc_id="pi-a") - assert text.startswith(AGENT_INSTRUCTIONS) + text = client.document_context("pi-a") assert "The user has specified document: report.pdf" in text + with pytest.raises(TypeError): + client.agent_instructions(doc_id="pi-a") seed_doc(store_path, "pi-b", "other.pdf") - multi = client.agent_instructions(doc_id=["pi-a", "pi-b"]) + multi = client.document_context(["pi-a", "pi-b"]) assert "The user has specified documents: report.pdf, other.pdf" in multi with pytest.raises(PageIndexAPIError): - client.agent_instructions(doc_id="pi-missing") + client.document_context("pi-missing") + for bad in (None, 123): + with pytest.raises(PageIndexAPIError, match="string or a list"): + client.document_context(bad) + + +def test_removed_doc_id_positional_slot_raises(client): + """A stale positional doc_id raises instead of landing on the next slot.""" + from pageindex.integrations.claude_agent_sdk import build_claude_mcp + for stale in (lambda: client.agent_instructions("pi-a"), + lambda: client.openai_agent_config("pi-a"), + lambda: client.anthropic_runner_config( + "claude-sonnet-4-5", "pi-a"), + lambda: client.claude_agent_config("pi-a"), + lambda: client.as_claude_mcp(False, "pi-a"), + lambda: build_claude_mcp(client, False, "pi-a")): + with pytest.raises(TypeError): + stale() def test_local_instructions_name_only_local_tools(): @@ -2731,16 +2624,12 @@ def denied(doc_id): assert "Processing continues" not in str(err.value) -def test_config_helpers_reject_empty_doc_id_on_cloud(): - """An explicitly empty scope must not silently widen to the whole - library — cloud has no tool-layer allowlist to enforce it.""" +def test_document_context_rejects_empty_doc_id(): + """An explicitly empty selection must not silently widen to the whole + library.""" cloud = PageIndexCloudClient(api_key="pi-test-key") with pytest.raises(PageIndexAPIError, match="doc_id is empty"): - cloud.openai_agent_config(doc_id=[]) - with pytest.raises(PageIndexAPIError, match="doc_id is empty"): - cloud.anthropic_runner_config(model="claude-sonnet-4-5", doc_id=[]) - with pytest.raises(PageIndexAPIError, match="doc_id is empty"): - cloud.claude_agent_config(doc_id=[]) + cloud.document_context([]) def test_doc_targeting_keeps_transport_errors_out_of_not_found(): @@ -2765,6 +2654,27 @@ def get_document(self, doc_id): agent_tools_module.doc_targeting_block(Stub(status), "pi-a") +def test_doc_targeting_is_one_lookup_per_document(): + """One get_document per id, rendered like the cloud's managed chat.""" + calls = [] + + class Client: + def get_document(self, doc_id): + calls.append(doc_id) + return {"id": doc_id, "name": f"{doc_id}.pdf", + "status": "completed", + "metadata": {"quarter": "Q3", "nested": {"x": 1}}} + + single = agent_tools_module.doc_targeting_block(Client(), "pi-a") + assert calls == ["pi-a"] + assert "Document metadata: {" in single + assert '"quarter": "Q3"' in single and '"nested": {"x": 1}' in single + block = agent_tools_module.doc_targeting_block(Client(), ["pi-a", "pi-b"]) + assert calls == ["pi-a", "pi-a", "pi-b"] + assert "The user has specified documents: pi-a.pdf, pi-b.pdf" in block + assert "Documents metadata: [" in block + + def test_call_tool_coerces_string_booleans(client, store_path, monkeypatch): """Models routinely send booleans as JSON strings — "false" must not read as True (a full wait_for_completion stall).""" @@ -2838,16 +2748,15 @@ def list_documents(self, limit, offset): assert payload["has_more"] is False and payload["next_offset"] is None -def test_agent_instructions_carry_user_metadata(client, store_path): - """The targeting block promises names and metadata; local get_document - keeps the 7-key detail wire shape, so the tags come from the listing.""" +def test_document_context_carries_user_metadata(client, store_path): + """The targeting block carries the user's tags from get_document.""" seed_doc(store_path, "pi-1", "report.pdf", metadata={"quarter": "Q3", "year": 2025}) - text = client.agent_instructions(doc_id="pi-1") + text = client.document_context("pi-1") assert '"quarter": "Q3"' in text and '"year": 2025' in text -# ── wait-poll resilience, instruction scoping, agent_tools doc_id ── +# ── wait-poll resilience, document targeting ── def test_await_completion_polls_through_transient_refetch_failure(monkeypatch): """A refetch that fails once must not end the wait early — the caller @@ -2868,36 +2777,6 @@ def get_document(self, doc_id): assert calls["n"] == 2 -def test_agent_instructions_doc_id_shadow_check(client, store_path): - seed_doc(store_path, "pi-old", "report.pdf", - created_at="2026-08-01T10:00:00.123000") - seed_doc(store_path, "pi-new", "report.pdf", - created_at="2026-08-05T10:00:00.123000") - # standalone instructions get the strict check; only the *_agent_config - # bundles (which build the tools too) relax it - with pytest.raises(PageIndexAPIError, match="shadowed"): - client.agent_instructions(doc_id="pi-old") - - -def test_agent_tools_doc_id_scopes_the_functions(client, store_path): - seed_doc(store_path, "pi-a", "alpha.pdf") - seed_doc(store_path, "pi-b", "secret.pdf") - funcs = {fn.__name__: fn for fn in client.agent_tools(doc_id="pi-a")} - blocked = json.loads(funcs["get_page_content"](doc_name="secret.pdf", - pages="1")) - assert "success" not in blocked - assert blocked["errorCode"] == "NOT_FOUND" - allowed = json.loads(funcs["get_page_content"](doc_name="alpha.pdf", - pages="1")) - assert allowed["success"] is True - - -def test_agent_tools_doc_id_refused_on_cloud(): - cloud = PageIndexCloudClient(api_key="pi-test-key") - with pytest.raises(PageIndexAPIError, match="local tools only"): - cloud.agent_tools(doc_id="pi-a") - - def test_cloud_tool_list_empty_raises(monkeypatch): """An empty tools/list must raise like empty instructions does: a zero-tool agent answers from the model's own knowledge instead of diff --git a/tests/test_client.py b/tests/test_client.py index 51eefcda9..260838045 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,6 +1,7 @@ """SDK surface tests: PageIndexClient in local and cloud mode.""" import asyncio import importlib +import inspect import json import os import re @@ -857,7 +858,7 @@ def test_submit_with_metadata(local_client, sample_pdf, monkeypatch): assert local_client.get_tree(doc_id)["metadata"] == tags assert local_client.get_ocr(doc_id)["metadata"] == tags assert local_client.list_documents()["documents"][0]["metadata"] == tags - assert "metadata" not in local_client.get_document(doc_id) + assert local_client.get_document(doc_id)["metadata"] == tags def test_submit_metadata_validation(local_client, sample_pdf, monkeypatch): @@ -1349,6 +1350,9 @@ def test_folders_are_cloud_only(local_client): local_client.create_folder("team") with pytest.raises(PageIndexAPIError, match="cloud-only"): local_client.list_folders() + with pytest.raises(PageIndexAPIError, match="cloud-only"): + local_client.folder_context("f") + assert local_client.folder_context("root") == "" # the library itself # ── local: retrieval endpoints are cloud-only ── @@ -1475,6 +1479,7 @@ def test_cloud_errors_carry_status_code(cloud, monkeypatch, sample_pdf): lambda: client.list_documents(), lambda: client.create_folder("f"), lambda: client.list_folders(), + lambda: client.folder_context("f"), ] for attempt in attempts: with pytest.raises(PageIndexAPIError) as err: @@ -1602,6 +1607,28 @@ def test_cloud_chat_rejects_extra_body_doc_id_before_request( assert calls == [] +def test_cloud_chat_folder_id_rides_the_wire(cloud): + """The managed chat scopes folder_id server-side: it goes out as the + request's own field, with no folder lookup on this side.""" + client, calls, fake = cloud + fake.payload = {"choices": [{"message": {"content": "ok"}}]} + assert client.chat("q", folder_id="f-1") == "ok" + assert calls[-1]["url"].endswith("/chat/completions/") + assert calls[-1]["json"]["folder_id"] == "f-1" + assert len(calls) == 1 + client.chat_completions("q", folder_id="") + assert "folder_id" not in calls[-1]["json"] + client.chat("q", protocol="chat_completions", folder_id="f-1") + assert calls[-1]["json"]["folder_id"] == "f-1" + + +def test_folder_id_is_keyword_only_on_every_chat_surface(): + for method in (PageIndexClient.chat, PageIndexClient.chat_completions, + PageIndexClient._responses, PageIndexClient._messages): + param = inspect.signature(method).parameters["folder_id"] + assert param.kind is inspect.Parameter.KEYWORD_ONLY, method.__name__ + + def test_parse_pages_overlap_counts_union(): from pageindex.client import _parse_pages pages = _parse_pages("1-5000,2000-9000") @@ -1614,6 +1641,35 @@ def test_parse_pages_overlap_counts_union(): _parse_pages("0-3") +def test_folder_context(cloud): + """Folder targeting renders as the managed chat renders folder_id: the + name, an id/name/description metadata row (empty description dropped), + and the discovery directive carrying the id.""" + client, calls, fake = cloud + fake.payload = {"folders": [ + {"id": "f-1", "name": "Research", "description": "", + "parent_folder_id": None, "file_count": 2, "children_count": 1}, + {"id": "f-2", "name": "Q3", "description": "quarterly", + "parent_folder_id": "f-1", "file_count": 1, "children_count": 0}, + ], "total": 2} + text = client.folder_context("f-2") + assert calls[-1]["url"] == "https://api.pageindex.ai/folders/" + assert text.startswith("The user has specified folder: Q3\n") + assert ('Folder metadata: {"id": "f-2", "name": "Q3", ' + '"description": "quarterly"}\n') in text + assert 'browse_documents(folder_id="f-2", recursive=true)' in text + assert ('Folder metadata: {"id": "f-1", "name": "Research"}\n' + in client.folder_context("f-1")) + with pytest.raises(PageIndexAPIError, match="not found"): + client.folder_context("f-9") + with pytest.raises(PageIndexAPIError, match="must be a string"): + client.folder_context(["f-1"]) + calls.clear() + assert client.folder_context("root") == "" + assert client.folder_context("") == "" + assert calls == [] + + # ── backend: the indexing lane ── def test_backend_scopes_the_index_lane(tmp_path, monkeypatch): diff --git a/tests/test_local_chat.py b/tests/test_local_chat.py index dac8d8125..204aacf72 100644 --- a/tests/test_local_chat.py +++ b/tests/test_local_chat.py @@ -1004,8 +1004,8 @@ def test_doc_id_conversations_get_distinct_cache_keys(client, store_path, keys = [] real = local_chat._conversation_cache_key - def spy(model_name, instructions, doc_id, items): - key = real(model_name, instructions, doc_id, items) + def spy(model_name, instructions, doc_id, items, folder_id=None): + key = real(model_name, instructions, doc_id, items, folder_id) keys.append(key) return key @@ -1036,6 +1036,16 @@ def spy(model_name, instructions, doc_id, items): assert keys[5] != keys[0] # same opener, different doc: no pooling +def test_folder_less_cache_key_is_the_pre_folder_key(): + """Adding folder_id to the seed must not rotate every existing + conversation's prompt_cache_key on upgrade.""" + items = [{"role": "user", "content": "hi"}] + key = local_chat._conversation_cache_key("m", "sys", "d1", items) + assert key == "pageindex-b0ab095344ee8f89" + assert local_chat._conversation_cache_key( + "m", "sys", "d1", items, "f-1") != key + + @needs_agents def test_responses_stream_passthrough(client, store_path, fake_model): seed_doc(store_path, "pi-a", "report.pdf") @@ -1245,15 +1255,25 @@ def test_messages_end_to_end(client, store_path, fake_anthropic): @needs_anthropic def test_messages_doc_block_and_system(client, store_path, fake_anthropic): + """Doc block leads as a user message; system keeps the cached header.""" doc_id = seed_doc(store_path, "pi-a", "report.pdf") calls = fake_anthropic([ + _anthropic_message( + [{"type": "tool_use", "id": "tu_1", "name": "get_document", + "input": {"doc_name": "report.pdf"}}], "tool_use"), _anthropic_message([{"type": "text", "text": "ok"}], "end_turn"), ]) - client._messages([{"role": "user", "content": "hi"}], model="claude-test", - max_tokens=100, doc_id=doc_id, system="Answer in French.") + result = client._messages([{"role": "user", "content": "hi"}], + model="claude-test", max_tokens=100, + doc_id=doc_id, system="Answer in French.") + first, second = calls[0]["messages"][:2] + assert first["role"] == "user" + assert "The user has specified document: report.pdf" in first["content"] + assert second == {"role": "user", "content": "hi"} system = calls[0]["system"] - assert "The user has specified document: report.pdf" in system[1]["text"] - assert system[-1]["text"] == "Answer in French." + assert [block["text"] for block in system[1:]] == ["Answer in French."] + roles = [message["role"] for message in result["messages"]] + assert roles == ["assistant", "user", "assistant"] @needs_anthropic @@ -1751,9 +1771,7 @@ def test_empty_doc_id_is_refused(client, store_path): with pytest.raises(PageIndexAPIError, match="doc_id is empty"): client.chat_completions("q", doc_id=[]) with pytest.raises(PageIndexAPIError, match="doc_id is empty"): - client.as_openai_tools(doc_id=[]) - with pytest.raises(PageIndexAPIError, match="doc_id is empty"): - client.agent_instructions(doc_id=[]) + client.document_context([]) @needs_agents @@ -3017,18 +3035,12 @@ def test_process_display_elides_image_payloads(bridge_client, fake_model): def test_bridge_doc_id_targets_at_prompt_level(bridge_client, fake_model, monkeypatch): """On cloud tools there is no local allowlist: doc_id becomes the - prompt-level targeting block only. (Had the tool layer received the - doc_ids, _require_local_scope would raise on a cloud client — this - call succeeding is the proof it did not.)""" + prompt-level targeting block only.""" client, _ = bridge_client monkeypatch.setattr(client, "get_document", lambda doc_id: {"name": "r.pdf", "description": "d", "status": "completed", "metadata": None}) - monkeypatch.setattr( - client, "list_documents", - lambda **kw: {"documents": [{"id": "pi-a", "name": "r.pdf"}], - "total": 1}) fake = fake_model([[_msg_item("ok")]]) client.chat_completions("q", doc_id="pi-a") first = fake.inputs[0][0] @@ -3036,6 +3048,86 @@ def test_bridge_doc_id_targets_at_prompt_level(bridge_client, fake_model, assert "r.pdf" in first["content"] +def test_targeting_block_orders_folder_before_documents(bridge_client, + monkeypatch): + """The folder block leads the document block, joined as the managed + chat joins them; "root" and no folder place nothing of their own.""" + from pageindex.agent_tools import targeting_block + client, _ = bridge_client + monkeypatch.setattr(client, "list_folders", lambda: {"folders": [ + {"id": "f-1", "name": "Team", "description": None}]}) + monkeypatch.setattr(client, "get_document", + lambda doc_id: {"name": "r.pdf", "status": "completed"}) + both = targeting_block(client, "pi-a", "f-1") + assert both is not None + folder, doc = both.split("\n\n") + assert folder.startswith("The user has specified folder: Team\n") + assert 'Folder metadata: {"id": "f-1", "name": "Team"}\n' in folder + assert doc.startswith("The user has specified document: r.pdf\n") + assert targeting_block(client, "pi-a", "root") == doc + assert targeting_block(client, None, "f-1") == folder + assert targeting_block(client, None, None) is None + + +@needs_agents +def test_bridge_folder_id_targets_ahead_of_documents(bridge_client, fake_model, + monkeypatch): + """folder_id is prompt-level targeting on cloud tools, one leading + user message with the folder block ahead of the document block.""" + client, _ = bridge_client + monkeypatch.setattr(client, "list_folders", lambda: {"folders": [ + {"id": "f-1", "name": "Team", "description": "shared"}]}) + monkeypatch.setattr(client, "get_document", + lambda doc_id: {"name": "r.pdf", "status": "completed"}) + fake = fake_model([[_msg_item("ok")]]) + with pytest.raises(PageIndexAPIError, match="not found"): + client.chat_completions("q", folder_id="f-9") + client.chat_completions("q", doc_id="pi-a", folder_id="f-1") + first, question = fake.inputs[0][:2] + assert first["content"].startswith("The user has specified folder: Team\n") + assert "The user has specified document: r.pdf" in first["content"] + assert question == {"role": "user", "content": "q"} + + +@needs_agents +def test_bridge_folder_id_reaches_the_protocol_lanes(bridge_client, fake_model, + monkeypatch): + """chat(protocol="responses") threads folder_id to its engine.""" + client, _ = bridge_client + monkeypatch.setattr(client, "list_folders", lambda: {"folders": [ + {"id": "f-1", "name": "Team"}]}) + fake = fake_model([[_msg_item("ok")]]) + client.chat("q", protocol="responses", folder_id="f-1") + assert fake.inputs[0][0]["content"].startswith( + "The user has specified folder: Team\n") + assert fake.inputs[0][1] == {"role": "user", "content": "q"} + + +@needs_anthropic +def test_bridge_folder_id_reaches_the_messages_lane(bridge_client, + fake_anthropic, + monkeypatch): + """chat(protocol="messages") threads folder_id to its engine.""" + client, _ = bridge_client + monkeypatch.setattr(client, "list_folders", lambda: {"folders": [ + {"id": "f-1", "name": "Team"}]}) + calls = fake_anthropic([ + _anthropic_message([{"type": "text", "text": "ok"}], "end_turn")]) + client.chat("q", protocol="messages", model="claude-test", + folder_id="f-1") + first, second = calls[0]["messages"][:2] + assert first["content"].startswith("The user has specified folder: Team\n") + assert second == {"role": "user", "content": "q"} + + +@needs_agents +def test_folder_id_is_cloud_only(client): + """A local library has no folders: folder_id refuses before any model + call, like the folder methods.""" + with pytest.raises(PageIndexAPIError, match="cloud-only"): + client.chat_completions("q", folder_id="f-1") + + def test_bridge_gate_and_citations(monkeypatch): from pageindex import PageIndexClient client = PageIndexClient(api_key="pi-k", chat_model="m") @@ -3253,7 +3345,7 @@ def test_chat_protocol_chat_completions_serves_managed_cloud(monkeypatch): assert seen[-1] == {"messages": [{"role": "user", "content": "q"}], "stream": False, "doc_id": None, "temperature": None, "stream_metadata": True, "enable_citations": False, - "extra_body": None} + "extra_body": None, "folder_id": None} cloud.chat("q", protocol="chat_completions", extra_body={"temperature": 0.2, "enable_citations": True}) assert seen[-1]["extra_body"] == {"temperature": 0.2,