From d86753ad6d96e5796a9e25407124aa0fd4db3a3f Mon Sep 17 00:00:00 2001 From: Tanishq Date: Thu, 12 Nov 2026 22:17:31 +0530 Subject: [PATCH] fix(screenshot): add the capture origin to the image-to-screen conversion Image pixel (0, 0) is the top-left corner of what was captured: the region, the selected displays, or the virtual screen, which starts left of or above (0, 0) when a monitor sits left of or above the primary one. The Screenshot Coordinate Scale note told the agent to multiply image pixels by the scale and nothing else, so for those captures a point worked out from the image missed by that corner's position, often onto another monitor (#453). A capture at full size that did not start at (0, 0) came with no conversion at all. The note now adds the corner when it is not (0, 0), in its instruction and in its worked example, and a full-size capture that starts elsewhere gets a Screenshot Origin line. Captures that start at (0, 0) read as before. The Screenshot description now says to add the corner too. --- manifest.json | 2 +- src/windows_mcp/tools/_snapshot_helpers.py | 36 +++- src/windows_mcp/tools/snapshot.py | 2 +- tests/test_screenshot_coordinate_origin.py | 215 +++++++++++++++++++++ 4 files changed, 251 insertions(+), 4 deletions(-) create mode 100644 tests/test_screenshot_coordinate_origin.py diff --git a/manifest.json b/manifest.json index ade32a4c..435006c2 100755 --- a/manifest.json +++ b/manifest.json @@ -97,7 +97,7 @@ }, { "name": "Screenshot", - "description": "Captures a fast screenshot-first desktop snapshot with cursor position, active/open windows, and an image. Skips UI tree extraction for speed and should be the default first call when you mainly need visual context. Supports display=[0] or display=[0,1] using zero-based active Windows display indices. Note: the returned image may be downscaled for efficiency; when it is, multiply image coordinates by the ratio of original size to displayed size to get the actual screen coordinates for mouse actions (Click, Move, etc.)." + "description": "Captures a fast screenshot-first desktop snapshot with cursor position, active/open windows, and an image. Skips UI tree extraction for speed and should be the default first call when you mainly need visual context. Supports display=[0] or display=[0,1] using zero-based active Windows display indices. Note: the returned image may be downscaled for efficiency, and it starts at the top-left corner of the captured area, not necessarily at screen (0, 0). To get the screen coordinates for mouse actions (Click, Move, etc.), multiply image coordinates by the ratio of original size to displayed size, then add the screen coordinate of image pixel (0, 0) when the response gives one." }, { "name": "Snapshot", diff --git a/src/windows_mcp/tools/_snapshot_helpers.py b/src/windows_mcp/tools/_snapshot_helpers.py index 9ed05ee2..a92eabf1 100644 --- a/src/windows_mcp/tools/_snapshot_helpers.py +++ b/src/windows_mcp/tools/_snapshot_helpers.py @@ -14,6 +14,7 @@ from textwrap import dedent from windows_mcp.desktop.service import Desktop, Size from windows_mcp.desktop.utils import remove_private_use_chars, repair_surrogates +from windows_mcp.desktop.views import DesktopState logger = logging.getLogger(__name__) @@ -48,6 +49,25 @@ def _as_region(value: list | str | None) -> list | None: WINDOWS_NOT_LISTED = "Not listed: Screenshot skips window enumeration. Call Snapshot for windows." +def _image_origin(desktop_state: DesktopState) -> tuple[int, int]: + """Return the screen coordinate of the screenshot's top-left pixel. + + A region or display capture starts at the corner of its rectangle. A full capture + covers the virtual screen, which starts left of or above (0, 0) when a monitor sits + left of or above the primary one. + """ + region = desktop_state.screenshot_region + if region is not None: + return region.left, region.top + displays = desktop_state.available_displays + if not displays: + return 0, 0 + return ( + min(display.bounding_box.left for display in displays), + min(display.bounding_box.top for display in displays), + ) + + def capture_desktop_state( desktop: Desktop, *, @@ -192,17 +212,29 @@ def display_to_string(display): if desktop_state.screenshot_original_size: orig = desktop_state.screenshot_original_size scale = desktop_state.screenshot_scale or 1.0 + origin_x, origin_y = _image_origin(desktop_state) + origin = f"({origin_x},{origin_y})" if scale < 1.0: coord_scale = round(1.0 / scale, 6) + add_origin = "" + if (origin_x, origin_y) != (0, 0): + add_origin = f", then add {origin}, the screen coordinate of image pixel (0, 0)," metadata_text += ( f"Screenshot Original Size: {orig.to_string()}\n" f"Screenshot Coordinate Scale: {coord_scale} " f"— image pixels are downscaled; multiply every image pixel coordinate by " - f"{coord_scale} before passing to Click, Move, Scroll, or any loc= argument " - f"(e.g. image pixel (200, 150) → screen coordinate ({round(200 * coord_scale)}, {round(150 * coord_scale)}))\n" + f"{coord_scale}{add_origin} before passing to Click, Move, Scroll, or any loc= argument " + f"(e.g. image pixel (200, 150) → screen coordinate ({origin_x + round(200 * coord_scale)}, {origin_y + round(150 * coord_scale)}))\n" ) else: metadata_text += f"Screenshot Size: {orig.to_string()}\n" + if (origin_x, origin_y) != (0, 0): + metadata_text += ( + f"Screenshot Origin: {origin} — image pixel (0, 0) is screen coordinate {origin}; " + f"add {origin} to every image pixel coordinate before passing to Click, Move, " + f"Scroll, or any loc= argument " + f"(e.g. image pixel (200, 150) → screen coordinate ({origin_x + 200}, {origin_y + 150}))\n" + ) if desktop_state.available_displays: metadata_text += "Visible Displays: " metadata_text += "; ".join( diff --git a/src/windows_mcp/tools/snapshot.py b/src/windows_mcp/tools/snapshot.py index 76cde8ff..0eb1ca5c 100644 --- a/src/windows_mcp/tools/snapshot.py +++ b/src/windows_mcp/tools/snapshot.py @@ -85,7 +85,7 @@ def _state_tool( @mcp.tool( name='Screenshot', - description="Captures a fast screenshot-first desktop snapshot with cursor position, desktop/window summaries, and an image. This path skips UI tree extraction for speed. Use Snapshot when you need interactive element ids, scrollable regions, or browser DOM extraction. Set display=[0] or display=[0,1] using zero-based active Windows display indices to capture only those monitors. Set region=[left, top, right, bottom] in virtual-desktop pixel coordinates to capture only that rectangle instead of the whole screen/display — useful when you already know which area matters and want to save tokens; region takes precedence over display when both are given, and an invalid or out-of-bounds region raises an error rather than silently capturing something else. Set save_path to an absolute file path (or an existing directory for a timestamped name) to also write the returned image as a PNG file (same pixels as the returned image); the saved path is reported in the response and existing files are never overwritten. Note: the returned image may be downscaled for efficiency; when it is, multiply image coordinates by the ratio of original size to displayed size to get the actual screen coordinates for mouse actions (Click, Move, etc.).", + description="Captures a fast screenshot-first desktop snapshot with cursor position, desktop/window summaries, and an image. This path skips UI tree extraction for speed. Use Snapshot when you need interactive element ids, scrollable regions, or browser DOM extraction. Set display=[0] or display=[0,1] using zero-based active Windows display indices to capture only those monitors. Set region=[left, top, right, bottom] in virtual-desktop pixel coordinates to capture only that rectangle instead of the whole screen/display — useful when you already know which area matters and want to save tokens; region takes precedence over display when both are given, and an invalid or out-of-bounds region raises an error rather than silently capturing something else. Set save_path to an absolute file path (or an existing directory for a timestamped name) to also write the returned image as a PNG file (same pixels as the returned image); the saved path is reported in the response and existing files are never overwritten. Note: the returned image may be downscaled for efficiency, and it starts at the top-left corner of the captured area, not necessarily at screen (0, 0). To get the screen coordinates for mouse actions (Click, Move, etc.), multiply image coordinates by the ratio of original size to displayed size, then add the screen coordinate of image pixel (0, 0) when the response gives one.", annotations=ToolAnnotations( title="Screenshot", readOnlyHint=True, diff --git a/tests/test_screenshot_coordinate_origin.py b/tests/test_screenshot_coordinate_origin.py new file mode 100644 index 00000000..b5b99950 --- /dev/null +++ b/tests/test_screenshot_coordinate_origin.py @@ -0,0 +1,215 @@ +"""Screenshot and Snapshot must say where the returned image starts on the screen. + +Image pixel (0, 0) is the top-left corner of what was captured: the region or the +displays asked for, else the virtual screen, which starts left of or above (0, 0) +when a monitor sits left of or above the primary one. The response told the agent to +turn image pixels into screen coordinates by scaling alone (#453), so a point worked +out from the image missed by that corner's position, often onto another monitor. + +Nothing here touches the real desktop: the displays, the virtual screen, the cursor, +the virtual desktop list and the capture are stand-ins. The capture is cut out of a +synthetic virtual desktop with a red button on it, the way the pillow and mss +backends cut a region out of the virtual screen. +""" + +import io +import re +from unittest.mock import MagicMock, patch + +import pytest +from PIL import Image, ImageDraw + +from windows_mcp.desktop.service import Desktop +from windows_mcp.tools._snapshot_helpers import build_snapshot_response, capture_desktop_state +from windows_mcp.tree.views import BoundingBox +from windows_mcp.uia import DisplayInfo, Rect + +EXAMPLE = re.compile(r"image pixel \(200, 150\) → screen coordinate \((-?\d+), (-?\d+)\)") +SCALE = re.compile(r"Screenshot Coordinate Scale: ([0-9.]+)") + + +@pytest.fixture(autouse=True) +def _default_screenshot_scale(monkeypatch) -> None: + monkeypatch.delenv("WINDOWS_MCP_SCREENSHOT_SCALE", raising=False) + + +def _screenshot( + monitors: list[Rect], + button: tuple[int, int], + *, + display: list[int] | None = None, + region: list[int] | None = None, +) -> tuple[str, Image.Image]: + """Run the Screenshot tool's capture; return the response text and the image.""" + left = min(monitor.left for monitor in monitors) + top = min(monitor.top for monitor in monitors) + right = max(monitor.right for monitor in monitors) + bottom = max(monitor.bottom for monitor in monitors) + + virtual_screen = Image.new("RGB", (right - left, bottom - top), (40, 40, 40)) + x, y = button[0] - left, button[1] - top + ImageDraw.Draw(virtual_screen).rectangle([x - 5, y - 5, x + 5, y + 5], fill=(255, 0, 0)) + + def get_screenshot(capture_rect: Rect | None = None) -> Image.Image: + if capture_rect is None: + return virtual_screen.copy() + return virtual_screen.crop( + ( + capture_rect.left - left, + capture_rect.top - top, + capture_rect.right - left, + capture_rect.bottom - top, + ) + ) + + with patch.object(Desktop, "__init__", lambda self: None): + desktop = Desktop() + desktop.desktop_state = None + desktop.tree = MagicMock() + desktop.tree.screen_box = BoundingBox( + left=left, top=top, right=right, bottom=bottom, width=right - left, height=bottom - top + ) + desktop.get_displays = lambda: [ + DisplayInfo( + index=index, + device_name=f"DISPLAY{index + 1}", + rect=monitor, + primary=(monitor.left, monitor.top) == (0, 0), + ) + for index, monitor in enumerate(monitors) + ] + desktop.get_cursor_location = MagicMock(return_value=(0, 0)) + desktop.get_screenshot = get_screenshot + + with ( + patch( + "windows_mcp.desktop.service.uia.GetVirtualScreenRect", + return_value=(left, top, right - left, bottom - top), + ), + patch( + "windows_mcp.desktop.service.get_current_desktop", + return_value={"name": "Desktop 1"}, + ), + patch( + "windows_mcp.desktop.service.get_all_desktops", + return_value=[{"name": "Desktop 1"}], + ), + ): + # The arguments the Screenshot tool passes (tools/snapshot.py). + capture = capture_desktop_state( + desktop, + use_vision=True, + use_dom=False, + use_annotation=False, + use_ui_tree=False, + width_reference_line=None, + height_reference_line=None, + display=display, + region=region, + tool_name="Screenshot tool", + ) + text, image = build_snapshot_response(capture, include_ui_details=False) + return text, Image.open(io.BytesIO(image.data)).convert("RGB") + + +def _button_pixel(image: Image.Image) -> tuple[float, float]: + """Centre of the red button in the returned image.""" + red = [ + (x, y) + for y in range(image.height) + for x in range(image.width) + if image.getpixel((x, y))[0] > 200 and image.getpixel((x, y))[1] < 80 + ] + assert red, "the button is not in the returned image" + return sum(x for x, _ in red) / len(red), sum(y for _, y in red) / len(red) + + +def _to_screen(text: str, pixel: tuple[float, float]) -> tuple[float, float]: + """Map an image pixel to the screen the way the response's worked example does.""" + example = EXAMPLE.search(text) + assert example, "the response says nothing about turning image pixels into screen coordinates" + scale = SCALE.search(text) + factor = float(scale.group(1)) if scale else 1.0 + return ( + int(example.group(1)) + (pixel[0] - 200) * factor, + int(example.group(2)) + (pixel[1] - 150) * factor, + ) + + +@pytest.mark.parametrize( + ("monitors", "button", "display", "region", "screenshot_scale"), + [ + pytest.param( + [Rect(0, 0, 1920, 1080), Rect(-1920, 0, 0, 1080)], + (-1500, 300), + None, + None, + None, + id="full-capture-monitor-left-of-primary", + ), + pytest.param( + [Rect(0, 0, 1920, 1080), Rect(0, -1080, 1920, 0)], + (700, -400), + None, + None, + None, + id="full-capture-monitor-above-primary", + ), + pytest.param( + [Rect(0, 0, 1920, 1080), Rect(1920, 0, 4480, 1440)], + (2500, 600), + [1], + None, + None, + id="display-right-of-primary", + ), + pytest.param( + [Rect(0, 0, 3840, 2160)], + (2400, 1300), + None, + [2000, 1000, 3000, 1600], + "0.5", + id="region-downscaled", + ), + pytest.param( + [Rect(0, 0, 1920, 1080)], + (300, 250), + None, + [100, 100, 900, 750], + None, + id="region-full-size", + ), + ], +) +def test_image_pixel_maps_to_its_screen_coordinate( + monkeypatch, monitors, button, display, region, screenshot_scale +) -> None: + if screenshot_scale is not None: + monkeypatch.setenv("WINDOWS_MCP_SCREENSHOT_SCALE", screenshot_scale) + + text, image = _screenshot(monitors, button, display=display, region=region) + x, y = _to_screen(text, _button_pixel(image)) + + assert abs(x - button[0]) <= 2 and abs(y - button[1]) <= 2, ( + f"the response maps the button's image pixel to ({x:.0f}, {y:.0f}), " + f"but the button is at {button}" + ) + + +def test_downscaled_capture_at_screen_origin_keeps_its_note() -> None: + text, image = _screenshot([Rect(0, 0, 3840, 2160)], (2400, 1300)) + + assert image.size == (1920, 1080) + assert ( + "Screenshot Coordinate Scale: 2.0 — image pixels are downscaled; multiply every image " + "pixel coordinate by 2.0 before passing to Click, Move, Scroll, or any loc= argument " + "(e.g. image pixel (200, 150) → screen coordinate (400, 300))" + ) in text + + +def test_full_size_capture_at_screen_origin_needs_no_conversion() -> None: + text, image = _screenshot([Rect(0, 0, 1920, 1080)], (300, 250)) + + assert image.size == (1920, 1080) + assert "Screenshot Size: (1920,1080)" in text + assert "image pixel (200, 150)" not in text