Skip to content

Commit 674e14c

Browse files
NikhilShahiNikhilShahi
authored
Expose cursor position and modifier keys for computer actions (#136)
* Expose desktop cursor position and pointer modifier keys * Bump Python SDK minor version to 1.10.0 * Preserve existing computer-action typing contracts * Revert "Preserve existing computer-action typing contracts" This reverts commit 9e92e34. * Restore cursor helpers and dedicated cursor response typing * Include cursor-relative scrolling in computer action params * Use shared response for all computer actions * Remove standalone computer-action README section
1 parent d0efea3 commit 674e14c

11 files changed

Lines changed: 565 additions & 13 deletions

File tree

‎hyperbrowser/client/managers/async_manager/computer_action.py‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
PressKeysActionParams,
1313
MoveMouseActionParams,
1414
ScreenshotActionParams,
15+
CursorPositionActionParams,
16+
ScrollAtCursorActionParams,
1517
ScrollActionParams,
1618
TypeTextActionParams,
1719
Coordinate,
@@ -38,6 +40,7 @@
3840
ComputerAction.MOVE_MOUSE.value: MoveMouseActionParams,
3941
ComputerAction.PRESS_KEYS.value: PressKeysActionParams,
4042
ComputerAction.SCREENSHOT.value: ScreenshotActionParams,
43+
ComputerAction.CURSOR_POSITION.value: CursorPositionActionParams,
4144
ComputerAction.SCROLL.value: ScrollActionParams,
4245
ComputerAction.TYPE_TEXT.value: TypeTextActionParams,
4346
ComputerAction.GET_CLIPBOARD_TEXT.value: GetClipboardTextActionParams,
@@ -47,6 +50,9 @@
4750

4851

4952
def _action_param_model(params):
53+
if isinstance(params, ScrollAtCursorActionParams):
54+
return ScrollAtCursorActionParams
55+
5056
for model in _ACTION_PARAM_MODELS.values():
5157
if isinstance(params, model):
5258
return model
@@ -56,6 +62,8 @@ def _action_param_model(params):
5662
if isinstance(action, ComputerAction):
5763
action = action.value
5864
model = _ACTION_PARAM_MODELS.get(action)
65+
if model is ScrollActionParams and "x" not in params and "y" not in params:
66+
return ScrollAtCursorActionParams
5967
if model is not None:
6068
return model
6169

@@ -71,18 +79,13 @@ async def _execute_request(
7179
session: Union[SessionDetail, str],
7280
params: Union[ComputerActionParamsDict, ComputerActionParams],
7381
) -> ComputerActionResponse:
82+
payload = dump_request(params, _action_param_model(params), name="params")
7483
if isinstance(session, str):
7584
session = await self._client.sessions.get(session)
7685

7786
if not session.computer_action_endpoint:
7887
raise ValueError("Computer action endpoint not available for this session")
7988

80-
payload = dump_request(
81-
params,
82-
_action_param_model(params),
83-
name="params",
84-
)
85-
8689
response = await self._client.transport.post(
8790
session.computer_action_endpoint,
8891
data=payload,
@@ -97,12 +100,14 @@ async def click(
97100
button: ComputerActionMouseButton = "left",
98101
num_clicks: int = 1,
99102
return_screenshot: bool = False,
103+
keys: Optional[List[str]] = None,
100104
) -> ComputerActionResponse:
101105
params = ClickActionParams(
102106
x=x,
103107
y=y,
104108
button=button,
105109
num_clicks=num_clicks,
110+
keys=keys,
106111
return_screenshot=return_screenshot,
107112
)
108113
return await self._execute_request(session, params)
@@ -123,6 +128,14 @@ async def screenshot(
123128
params = ScreenshotActionParams()
124129
return await self._execute_request(session, params)
125130

131+
async def cursor_position(
132+
self,
133+
session: Union[SessionDetail, str],
134+
return_screenshot: bool = False,
135+
) -> ComputerActionResponse:
136+
params = CursorPositionActionParams(return_screenshot=return_screenshot)
137+
return await self._execute_request(session, params)
138+
126139
async def press_keys(
127140
self,
128141
session: Union[SessionDetail, str],
@@ -169,13 +182,15 @@ async def drag(
169182
session: Union[SessionDetail, str],
170183
path: List[Union[CoordinateDict, Coordinate]],
171184
return_screenshot: bool = False,
185+
keys: Optional[List[str]] = None,
172186
) -> ComputerActionResponse:
173187
params = DragActionParams(
174188
path=[
175189
coerce_request(coordinate, Coordinate, name="coordinate")
176190
for coordinate in path
177191
],
178192
return_screenshot=return_screenshot,
193+
keys=keys,
179194
)
180195
return await self._execute_request(session, params)
181196

@@ -197,12 +212,30 @@ async def scroll(
197212
scroll_x: int,
198213
scroll_y: int,
199214
return_screenshot: bool = False,
215+
keys: Optional[List[str]] = None,
200216
) -> ComputerActionResponse:
201217
params = ScrollActionParams(
202218
x=x,
203219
y=y,
204220
scroll_x=scroll_x,
205221
scroll_y=scroll_y,
222+
keys=keys,
223+
return_screenshot=return_screenshot,
224+
)
225+
return await self._execute_request(session, params)
226+
227+
async def scroll_at_cursor(
228+
self,
229+
session: Union[SessionDetail, str],
230+
scroll_x: int = 0,
231+
scroll_y: int = 0,
232+
return_screenshot: bool = False,
233+
keys: Optional[List[str]] = None,
234+
) -> ComputerActionResponse:
235+
params = ScrollAtCursorActionParams(
236+
scroll_x=scroll_x,
237+
scroll_y=scroll_y,
238+
keys=keys,
206239
return_screenshot=return_screenshot,
207240
)
208241
return await self._execute_request(session, params)

‎hyperbrowser/client/managers/sync_manager/computer_action.py‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
PressKeysActionParams,
1313
MoveMouseActionParams,
1414
ScreenshotActionParams,
15+
CursorPositionActionParams,
16+
ScrollAtCursorActionParams,
1517
ScrollActionParams,
1618
TypeTextActionParams,
1719
Coordinate,
@@ -38,6 +40,7 @@
3840
ComputerAction.MOVE_MOUSE.value: MoveMouseActionParams,
3941
ComputerAction.PRESS_KEYS.value: PressKeysActionParams,
4042
ComputerAction.SCREENSHOT.value: ScreenshotActionParams,
43+
ComputerAction.CURSOR_POSITION.value: CursorPositionActionParams,
4144
ComputerAction.SCROLL.value: ScrollActionParams,
4245
ComputerAction.TYPE_TEXT.value: TypeTextActionParams,
4346
ComputerAction.GET_CLIPBOARD_TEXT.value: GetClipboardTextActionParams,
@@ -47,6 +50,9 @@
4750

4851

4952
def _action_param_model(params):
53+
if isinstance(params, ScrollAtCursorActionParams):
54+
return ScrollAtCursorActionParams
55+
5056
for model in _ACTION_PARAM_MODELS.values():
5157
if isinstance(params, model):
5258
return model
@@ -56,6 +62,8 @@ def _action_param_model(params):
5662
if isinstance(action, ComputerAction):
5763
action = action.value
5864
model = _ACTION_PARAM_MODELS.get(action)
65+
if model is ScrollActionParams and "x" not in params and "y" not in params:
66+
return ScrollAtCursorActionParams
5967
if model is not None:
6068
return model
6169

@@ -71,18 +79,13 @@ def _execute_request(
7179
session: Union[SessionDetail, str],
7280
params: Union[ComputerActionParamsDict, ComputerActionParams],
7381
) -> ComputerActionResponse:
82+
payload = dump_request(params, _action_param_model(params), name="params")
7483
if isinstance(session, str):
7584
session = self._client.sessions.get(session)
7685

7786
if not session.computer_action_endpoint:
7887
raise ValueError("Computer action endpoint not available for this session")
7988

80-
payload = dump_request(
81-
params,
82-
_action_param_model(params),
83-
name="params",
84-
)
85-
8689
response = self._client.transport.post(
8790
session.computer_action_endpoint,
8891
data=payload,
@@ -97,12 +100,14 @@ def click(
97100
button: ComputerActionMouseButton = "left",
98101
num_clicks: int = 1,
99102
return_screenshot: bool = False,
103+
keys: Optional[List[str]] = None,
100104
) -> ComputerActionResponse:
101105
params = ClickActionParams(
102106
x=x,
103107
y=y,
104108
button=button,
105109
num_clicks=num_clicks,
110+
keys=keys,
106111
return_screenshot=return_screenshot,
107112
)
108113
return self._execute_request(session, params)
@@ -123,6 +128,14 @@ def screenshot(
123128
params = ScreenshotActionParams()
124129
return self._execute_request(session, params)
125130

131+
def cursor_position(
132+
self,
133+
session: Union[SessionDetail, str],
134+
return_screenshot: bool = False,
135+
) -> ComputerActionResponse:
136+
params = CursorPositionActionParams(return_screenshot=return_screenshot)
137+
return self._execute_request(session, params)
138+
126139
def press_keys(
127140
self,
128141
session: Union[SessionDetail, str],
@@ -169,13 +182,15 @@ def drag(
169182
session: Union[SessionDetail, str],
170183
path: List[Union[CoordinateDict, Coordinate]],
171184
return_screenshot: bool = False,
185+
keys: Optional[List[str]] = None,
172186
) -> ComputerActionResponse:
173187
params = DragActionParams(
174188
path=[
175189
coerce_request(coordinate, Coordinate, name="coordinate")
176190
for coordinate in path
177191
],
178192
return_screenshot=return_screenshot,
193+
keys=keys,
179194
)
180195
return self._execute_request(session, params)
181196

@@ -197,12 +212,30 @@ def scroll(
197212
scroll_x: int,
198213
scroll_y: int,
199214
return_screenshot: bool = False,
215+
keys: Optional[List[str]] = None,
200216
) -> ComputerActionResponse:
201217
params = ScrollActionParams(
202218
x=x,
203219
y=y,
204220
scroll_x=scroll_x,
205221
scroll_y=scroll_y,
222+
keys=keys,
223+
return_screenshot=return_screenshot,
224+
)
225+
return self._execute_request(session, params)
226+
227+
def scroll_at_cursor(
228+
self,
229+
session: Union[SessionDetail, str],
230+
scroll_x: int = 0,
231+
scroll_y: int = 0,
232+
return_screenshot: bool = False,
233+
keys: Optional[List[str]] = None,
234+
) -> ComputerActionResponse:
235+
params = ScrollAtCursorActionParams(
236+
scroll_x=scroll_x,
237+
scroll_y=scroll_y,
238+
keys=keys,
206239
return_screenshot=return_screenshot,
207240
)
208241
return self._execute_request(session, params)

‎hyperbrowser/models/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,8 @@
274274
MoveMouseActionParams,
275275
PressKeysActionParams,
276276
ScreenshotActionParams,
277+
CursorPositionActionParams,
278+
ScrollAtCursorActionParams,
277279
ScrollActionParams,
278280
TypeTextActionParams,
279281
ComputerActionMouseButton,
@@ -283,6 +285,7 @@
283285
ComputerActionWindow,
284286
ComputerActionResponseDataClipboardText,
285287
ComputerActionResponseDataListWindows,
288+
ComputerActionResponseDataCursorPosition,
286289
ComputerActionResponseData,
287290
)
288291
from .session import (
@@ -779,6 +782,8 @@
779782
"MoveMouseActionParams",
780783
"PressKeysActionParams",
781784
"ScreenshotActionParams",
785+
"CursorPositionActionParams",
786+
"ScrollAtCursorActionParams",
782787
"ScrollActionParams",
783788
"TypeTextActionParams",
784789
"ComputerActionMouseButton",
@@ -788,6 +793,7 @@
788793
"ComputerActionWindow",
789794
"ComputerActionResponseDataClipboardText",
790795
"ComputerActionResponseDataListWindows",
796+
"ComputerActionResponseDataCursorPosition",
791797
"ComputerActionResponseData",
792798
# web
793799
"StartBatchFetchJobParams",

‎hyperbrowser/models/computer_action.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ class ComputerAction(str, Enum):
1414
MOVE_MOUSE = "move_mouse"
1515
PRESS_KEYS = "press_keys"
1616
SCREENSHOT = "screenshot"
17+
CURSOR_POSITION = "cursor_position"
1718
SCROLL = "scroll"
1819
TYPE_TEXT = "type_text"
1920
GET_CLIPBOARD_TEXT = "get_clipboard_text"
@@ -43,6 +44,7 @@ class ClickActionParams(BaseModel):
4344
y: Optional[int] = Field(default=None)
4445
button: ComputerActionMouseButton = Field(default="left")
4546
num_clicks: int = Field(serialization_alias="numClicks", default=1)
47+
keys: Optional[List[str]] = None
4648
return_screenshot: bool = Field(
4749
serialization_alias="returnScreenshot", default=False
4850
)
@@ -55,6 +57,7 @@ class DragActionParams(BaseModel):
5557

5658
action: Literal[ComputerAction.DRAG] = ComputerAction.DRAG
5759
path: List[Coordinate]
60+
keys: Optional[List[str]] = None
5861
return_screenshot: bool = Field(
5962
serialization_alias="returnScreenshot", default=False
6063
)
@@ -130,6 +133,17 @@ class ScreenshotActionParams(BaseModel):
130133
action: Literal[ComputerAction.SCREENSHOT] = ComputerAction.SCREENSHOT
131134

132135

136+
class CursorPositionActionParams(BaseModel):
137+
"""Parameters for reading the desktop cursor position."""
138+
139+
model_config = ConfigDict(use_enum_values=True)
140+
141+
action: Literal[ComputerAction.CURSOR_POSITION] = ComputerAction.CURSOR_POSITION
142+
return_screenshot: bool = Field(
143+
serialization_alias="returnScreenshot", default=False
144+
)
145+
146+
133147
class ScrollActionParams(BaseModel):
134148
"""Parameters for scroll action."""
135149

@@ -138,8 +152,23 @@ class ScrollActionParams(BaseModel):
138152
action: Literal[ComputerAction.SCROLL] = ComputerAction.SCROLL
139153
x: int
140154
y: int
155+
keys: Optional[List[str]] = None
156+
scroll_x: int = Field(serialization_alias="scrollX")
157+
scroll_y: int = Field(serialization_alias="scrollY")
158+
return_screenshot: bool = Field(
159+
serialization_alias="returnScreenshot", default=False
160+
)
161+
162+
163+
class ScrollAtCursorActionParams(BaseModel):
164+
"""Parameters for scrolling without moving the desktop cursor."""
165+
166+
model_config = ConfigDict(use_enum_values=True)
167+
168+
action: Literal[ComputerAction.SCROLL] = ComputerAction.SCROLL
141169
scroll_x: int = Field(serialization_alias="scrollX")
142170
scroll_y: int = Field(serialization_alias="scrollY")
171+
keys: Optional[List[str]] = None
143172
return_screenshot: bool = Field(
144173
serialization_alias="returnScreenshot", default=False
145174
)
@@ -201,7 +230,9 @@ class ListWindowsActionParams(BaseModel):
201230
PressKeysActionParams,
202231
MoveMouseActionParams,
203232
ScreenshotActionParams,
233+
CursorPositionActionParams,
204234
ScrollActionParams,
235+
ScrollAtCursorActionParams,
205236
TypeTextActionParams,
206237
HoldKeyActionParams,
207238
MouseDownActionParams,
@@ -237,7 +268,15 @@ class ComputerActionResponseDataListWindows(BaseModel):
237268
windows: List[ComputerActionWindow] = Field(default_factory=list)
238269

239270

271+
class ComputerActionResponseDataCursorPosition(BaseModel):
272+
"""Desktop cursor coordinates in physical screen pixels."""
273+
274+
x: int
275+
y: int
276+
277+
240278
ComputerActionResponseData = Union[
279+
ComputerActionResponseDataCursorPosition,
241280
ComputerActionResponseDataListWindows,
242281
ComputerActionResponseDataClipboardText,
243282
]

0 commit comments

Comments
 (0)