From 12c91f01ea43e192c3bcf413e4652ed5f0a43152 Mon Sep 17 00:00:00 2001 From: Paulo Date: Sat, 26 Sep 2026 21:30:36 +0200 Subject: [PATCH] DRU-650 -- Send other chat files to the agent as links --- backend/druks/chat/service.py | 44 ++++++++++++++++++++++++---------- backend/tests/test_whatsapp.py | 33 ++++++++++++++++++++----- docs/chat.md | 20 ++++++++++------ 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/backend/druks/chat/service.py b/backend/druks/chat/service.py index 5ebefebd..0703ca6e 100644 --- a/backend/druks/chat/service.py +++ b/backend/druks/chat/service.py @@ -4,6 +4,7 @@ import logging from contextlib import suppress from datetime import timedelta +from pathlib import PurePosixPath from urllib.parse import urlsplit import asyncssh @@ -286,7 +287,7 @@ async def send_turn( "prompt", conversationId=conversation.id, messageId=delivered_messages[-1].id, - content=await get_turn_content(delivered_messages), + content=await get_turn_content(host, conversation_root, delivered_messages), timeout=timeout, ) await publish(conversation.id, {"type": "messages"}) @@ -294,22 +295,39 @@ async def send_turn( return -async def get_turn_content(messages: list[Message]) -> list[dict]: - """The ACP content blocks the agent reads: each message's text, then its image.""" +async def get_turn_content( + host: Host, conversation_root: str, messages: list[Message] +) -> list[dict]: + """The ACP content blocks the agent reads: each message's text, then its file. An + image travels in the prompt. Audio adds nothing. Any other file goes to the + conversation's folder in the sandbox, and the agent gets a link to it.""" content = [] for message in messages: if message.body: content.append({"type": "text", "text": message.body}) - file = message.file - if file and file.content_type.startswith("image/") and file.size <= MAX_UPLOAD_BYTES: - image = await asyncio.to_thread(get_file_storage().open, file.id) - content.append( - { - "type": "image", - "data": base64.b64encode(image).decode(), - "mimeType": file.content_type, - } - ) + if file := message.file: + if file.content_type.startswith("image/") and file.size <= MAX_UPLOAD_BYTES: + image = get_file_storage().open(file.id) + content.append( + { + "type": "image", + "data": base64.b64encode(image).decode(), + "mimeType": file.content_type, + } + ) + elif not file.content_type.startswith("audio/"): + # The sender names the file. Only the base name joins the sandbox path. + name = PurePosixPath(file.name).name + remote = f"{conversation_root}/files/{file.id}/{name}" + await host.upload_file(local=get_file_storage().path(file.id), remote=remote) + content.append( + { + "type": "resource_link", + "uri": PurePosixPath(remote).as_uri(), + "name": name, + "mimeType": file.content_type, + } + ) return content diff --git a/backend/tests/test_whatsapp.py b/backend/tests/test_whatsapp.py index 4d9aa74b..421f15fa 100644 --- a/backend/tests/test_whatsapp.py +++ b/backend/tests/test_whatsapp.py @@ -169,6 +169,10 @@ def message_event(chat, body, *, key="MSG1", from_me=False, session_name="sessio } +def waha_media(name, mimetype): + return {"url": f"http://waha.test/api/files/{name}", "mimetype": mimetype} + + def status_event(me_id=NUMBER, session_name="session_one", status="WORKING", engine="GOWS"): return { "id": "evt_status", @@ -341,11 +345,16 @@ async def test_one_turn_answers_every_pending_message_and_knows_its_own_reply( monkeypatch.setattr(WahaClient, "download", AsyncMock(return_value=b"\x89PNG")) connection = await link(druks_db, await bot_account(druks_db)) photo = message_event(ANA, "", key="M2") - media = {"url": "http://waha.test/api/files/photo.png", "mimetype": "image/png"} - photo["payload"].update(hasMedia=True, media=media) + photo["payload"].update(hasMedia=True, media=waha_media("photo.png", "image/png")) + form = message_event(ANA, "Here is the form", key="M4") + form["payload"].update(hasMedia=True, media=waha_media("form.pdf", "application/pdf")) + note = message_event(ANA, "", key="M5") + note["payload"].update(hasMedia=True, media=waha_media("note.oga", "audio/ogg")) await receive(connection, message_event(ANA, "Hello", key="M1")) await receive(connection, photo) await receive(connection, message_event(ANA, "And the second one?", key="M3")) + await receive(connection, form) + await receive(connection, note) [conversation] = await Conversation.list_for_connection(druks_db, connection.id) config = SimpleNamespace( harness_class=ClaudeHarness, @@ -361,7 +370,9 @@ async def test_one_turn_answers_every_pending_message_and_knows_its_own_reply( "druks.mcp.inbound.load_settings", lambda: SimpleNamespace(urls=Urls(webhook_host="hooks.test", endpoint="")), ) - host = SimpleNamespace(id="bot-sandbox", ssh_username="druks", aclose=AsyncMock()) + host = SimpleNamespace( + id="bot-sandbox", ssh_username="druks", aclose=AsyncMock(), upload_file=AsyncMock() + ) monkeypatch.setattr(service, "get_sandbox", AsyncMock(return_value=(host, SimpleNamespace()))) monkeypatch.setattr(service, "sandbox_client", SimpleNamespace(set_expiry=AsyncMock())) monkeypatch.setattr(Bridge, "reply", AsyncMock(return_value=("Your ticket is open.", []))) @@ -392,19 +403,29 @@ async def copy_arrives_first() -> None: await service.deliver_pending(druks_db, conversation) + await druks_db.refresh(conversation, ["messages"]) + *asked, reply = conversation.messages + copy = f"/home/druks/work/chat/{conversation.id}/files/{asked[3].file.id}/form.pdf" [prompt] = [values for method, values in requests if method == "prompt"] assert prompt["content"] == [ {"type": "text", "text": "Hello"}, {"type": "image", "data": base64.b64encode(b"\x89PNG").decode(), "mimeType": "image/png"}, {"type": "text", "text": "And the second one?"}, + {"type": "text", "text": "Here is the form"}, + { + "type": "resource_link", + "uri": f"file://{copy}", + "name": "form.pdf", + "mimeType": "application/pdf", + }, ] assert prompt["timeout"] == 60 + [upload] = host.upload_file.await_args_list + assert (upload.kwargs["local"].is_file(), upload.kwargs["remote"]) == (True, copy) [start] = [values for method, values in requests if method == "start"] assert start["headers"] == [{"name": CONVERSATION_HEADER, "value": conversation.id}] assert start["meta"]["claudeCode"]["options"]["systemPrompt"] == "Be kind." - await druks_db.refresh(conversation, ["messages"]) - *asked, reply = conversation.messages - assert [message.state for message in asked] == [MessageState.REPLIED] * 3 + assert [message.state for message in asked] == [MessageState.REPLIED] * 5 assert reply.source_id == "REPLY1" sends = [body for method, path, body in waha.calls if path == "/api/sendText"] assert sends == [ diff --git a/docs/chat.md b/docs/chat.md index d9e371fb..e0d584c3 100644 --- a/docs/chat.md +++ b/docs/chat.md @@ -226,11 +226,14 @@ agents and runs bill the default account. All pending messages of a WhatsApp conversation go into one turn. Druks saves media as a Druks file on its message. An image reaches the agent with its -message. Druks drops a repeated message by its WhatsApp ID. Druks writes to a -person only to answer them, one reply for each turn. It never starts a chat and -never sends in bulk. Before it sends a reply, Druks takes a new message ID from -WAHA and records it. WAHA's copy of the sent message then carries a known ID, -even when the copy arrives before the send returns. +message. Any other file except audio reaches it as a link to a copy in the +sandbox. An operator's agent opens the copy with its tools. A number's bot +agents have no file tools, so they see the link and the file's name. Druks +drops a repeated message by its WhatsApp ID. Druks writes to a person only to +answer them, one reply for each turn. It never starts a chat and never sends in +bulk. Before it sends a reply, Druks takes a new message ID from WAHA and +records it. WAHA's copy of the sent message then carries a known ID, even when +the copy arrives before the send returns. Druks also adds **internal messages** to a conversation. Each one comes from a fixed template. An internal message starts a turn like any message, and it @@ -315,8 +318,11 @@ people from other workspaces. A file you send to the bot, in a direct message or in a thread you joined, becomes a Druks file on its message. A message with only a file starts a turn -like any other. An image reaches the agent with its message. Each further file -in one Slack message gets a message of its own. The agent sends no files back. +like any other. An image reaches the agent with its message. Any other file +except audio reaches the agent as a link to a copy in the sandbox. The agent +opens the copy with its tools. An image over 25 MiB goes as a link too. Each +further file in one Slack message gets a message of its own. The agent sends no +files back. ### Rooms