DRU-650 -- Other files reach the chat agent as links

backend/druks/chat/service.py

@@ -4,6 +4,7 @@
 import logging
 from contextlib import suppress
 from datetime import timedelta
+from pathlib import PurePosixPath
 from urllib.parse import urlsplit
 
 import asyncssh
@@ -286,30 +287,47 @@ async def send_turn(
             "prompt",
             conversationId=conversation.id,
             messageId=delivered_messages[-1].id,
-            content=await get_turn_content(delivered_messages),
+            content=await get_turn_content(host, conversation_root, delivered_messages),
             timeout=timeout,
         )
         await publish(conversation.id, {"type": "messages"})
         return delivered_messages[-1]
     return
 
 
-async def get_turn_content(messages: list[Message]) -> list[dict]:
-    """The ACP content blocks the agent reads: each message's text, then its image."""
+async def get_turn_content(
+    host: Host, conversation_root: str, messages: list[Message]
+) -> list[dict]:
+    """The ACP content blocks the agent reads: each message's text, then its file. An
+    image travels in the prompt. Audio adds nothing. Any other file goes to the
+    conversation's folder in the sandbox, and the agent gets a link to it."""
     content = []
     for message in messages:
         if message.body:
             content.append({"type": "text", "text": message.body})
-        file = message.file
-        if file and file.content_type.startswith("image/") and file.size <= MAX_UPLOAD_BYTES:
-            image = await asyncio.to_thread(get_file_storage().open, file.id)
-            content.append(
-                {
-                    "type": "image",
-                    "data": base64.b64encode(image).decode(),
-                    "mimeType": file.content_type,
-                }
-            )
+        if file := message.file:
+            if file.content_type.startswith("image/") and file.size <= MAX_UPLOAD_BYTES:
+                image = get_file_storage().open(file.id)
+                content.append(
+                    {
+                        "type": "image",
+                        "data": base64.b64encode(image).decode(),
+                        "mimeType": file.content_type,
+                    }
+                )
+            elif not file.content_type.startswith("audio/"):
+                # The sender names the file. Only the base name joins the sandbox path.
+                name = PurePosixPath(file.name).name
+                remote = f"{conversation_root}/files/{file.id}/{name}"
+                await host.upload_file(local=get_file_storage().path(file.id), remote=remote)
+                content.append(
+                    {
+                        "type": "resource_link",
+                        "uri": PurePosixPath(remote).as_uri(),
+                        "name": name,
+                        "mimeType": file.content_type,
+                    }
+                )
     return content
 
 

backend/tests/test_whatsapp.py

@@ -169,6 +169,10 @@ def message_event(chat, body, *, key="MSG1", from_me=False, session_name="sessio
     }
 
 
+def waha_media(name, mimetype):
+    return {"url": f"http://waha.test/api/files/{name}", "mimetype": mimetype}
+
+
 def status_event(me_id=NUMBER, session_name="session_one", status="WORKING", engine="GOWS"):
     return {
         "id": "evt_status",
@@ -341,11 +345,16 @@ async def test_one_turn_answers_every_pending_message_and_knows_its_own_reply(
     monkeypatch.setattr(WahaClient, "download", AsyncMock(return_value=b"\x89PNG"))
     connection = await link(druks_db, await bot_account(druks_db))
     photo = message_event(ANA, "", key="M2")
-    media = {"url": "http://waha.test/api/files/photo.png", "mimetype": "image/png"}
-    photo["payload"].update(hasMedia=True, media=media)
+    photo["payload"].update(hasMedia=True, media=waha_media("photo.png", "image/png"))
+    form = message_event(ANA, "Here is the form", key="M4")
+    form["payload"].update(hasMedia=True, media=waha_media("form.pdf", "application/pdf"))
+    note = message_event(ANA, "", key="M5")
+    note["payload"].update(hasMedia=True, media=waha_media("note.oga", "audio/ogg"))
     await receive(connection, message_event(ANA, "Hello", key="M1"))
     await receive(connection, photo)
     await receive(connection, message_event(ANA, "And the second one?", key="M3"))
+    await receive(connection, form)
+    await receive(connection, note)
     [conversation] = await Conversation.list_for_connection(druks_db, connection.id)
     config = SimpleNamespace(
         harness_class=ClaudeHarness,
@@ -361,7 +370,9 @@ async def test_one_turn_answers_every_pending_message_and_knows_its_own_reply(
         "druks.mcp.inbound.load_settings",
         lambda: SimpleNamespace(urls=Urls(webhook_host="hooks.test", endpoint="")),
     )
-    host = SimpleNamespace(id="bot-sandbox", ssh_username="druks", aclose=AsyncMock())
+    host = SimpleNamespace(
+        id="bot-sandbox", ssh_username="druks", aclose=AsyncMock(), upload_file=AsyncMock()
+    )
     monkeypatch.setattr(service, "get_sandbox", AsyncMock(return_value=(host, SimpleNamespace())))
     monkeypatch.setattr(service, "sandbox_client", SimpleNamespace(set_expiry=AsyncMock()))
     monkeypatch.setattr(Bridge, "reply", AsyncMock(return_value=("Your ticket is open.", [])))
@@ -392,19 +403,29 @@ async def copy_arrives_first() -> None:
 
     await service.deliver_pending(druks_db, conversation)
 
+    await druks_db.refresh(conversation, ["messages"])
+    *asked, reply = conversation.messages
+    copy = f"/home/druks/work/chat/{conversation.id}/files/{asked[3].file.id}/form.pdf"
     [prompt] = [values for method, values in requests if method == "prompt"]
     assert prompt["content"] == [
         {"type": "text", "text": "Hello"},
         {"type": "image", "data": base64.b64encode(b"\x89PNG").decode(), "mimeType": "image/png"},
         {"type": "text", "text": "And the second one?"},
+        {"type": "text", "text": "Here is the form"},
+        {
+            "type": "resource_link",
+            "uri": f"file://{copy}",
+            "name": "form.pdf",
+            "mimeType": "application/pdf",
+        },
     ]
     assert prompt["timeout"] == 60
+    [upload] = host.upload_file.await_args_list
+    assert (upload.kwargs["local"].is_file(), upload.kwargs["remote"]) == (True, copy)
     [start] = [values for method, values in requests if method == "start"]
     assert start["headers"] == [{"name": CONVERSATION_HEADER, "value": conversation.id}]
     assert start["meta"]["claudeCode"]["options"]["systemPrompt"] == "Be kind."
-    await druks_db.refresh(conversation, ["messages"])
-    *asked, reply = conversation.messages
-    assert [message.state for message in asked] == [MessageState.REPLIED] * 3
+    assert [message.state for message in asked] == [MessageState.REPLIED] * 5
     assert reply.source_id == "REPLY1"
     sends = [body for method, path, body in waha.calls if path == "/api/sendText"]
     assert sends == [

docs/chat.md

@@ -226,11 +226,14 @@ agents and runs bill the default account.
 
 All pending messages of a WhatsApp conversation go into one turn. Druks saves
 media as a Druks file on its message. An image reaches the agent with its
-message. Druks drops a repeated message by its WhatsApp ID. Druks writes to a
-person only to answer them, one reply for each turn. It never starts a chat and
-never sends in bulk. Before it sends a reply, Druks takes a new message ID from
-WAHA and records it. WAHA's copy of the sent message then carries a known ID,
-even when the copy arrives before the send returns.
+message. Any other file except audio reaches it as a link to a copy in the
+sandbox. An operator's agent opens the copy with its tools. A number's bot
+agents have no file tools, so they see the link and the file's name. Druks
+drops a repeated message by its WhatsApp ID. Druks writes to a person only to
+answer them, one reply for each turn. It never starts a chat and never sends in
+bulk. Before it sends a reply, Druks takes a new message ID from WAHA and
+records it. WAHA's copy of the sent message then carries a known ID, even when
+the copy arrives before the send returns.
 
 Druks also adds **internal messages** to a conversation. Each one comes from a
 fixed template. An internal message starts a turn like any message, and it
@@ -315,8 +318,11 @@ people from other workspaces.
 
 A file you send to the bot, in a direct message or in a thread you joined,
 becomes a Druks file on its message. A message with only a file starts a turn
-like any other. An image reaches the agent with its message. Each further file
-in one Slack message gets a message of its own. The agent sends no files back.
+like any other. An image reaches the agent with its message. Any other file
+except audio reaches the agent as a link to a copy in the sandbox. The agent
+opens the copy with its tools. An image over 25 MiB goes as a link too. Each
+further file in one Slack message gets a message of its own. The agent sends no
+files back.
 
 ### Rooms