Fix Python SDK authorization and resource paths

.github/workflows/ci.yml

@@ -3,15 +3,19 @@ on:
   push:
     branches: [main]
   pull_request:
+permissions:
+  contents: read
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python: ['3.9', '3.13']
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         with:
           python-version: ${{ matrix.python }}
       - run: pip install -e '.[dev]'

README.md

@@ -376,7 +376,7 @@ print(check.checked_at)   # ISO timestamp
 
 ### X (Twitter) monitoring
 
-Monitor X for new tweets matching a profile, search, list, or reply thread, and fire a webhook on new matches — plus export an account's followers or following. These are **paid-only** features (a free/lapsed account gets `AuthenticationError` for 403). Monitors and audience export are billed per X request at your plan rate (Starter 5, Growth 3, Pro 2, Scale 1 credits). Max 50 monitors per user.
+Monitor X for new tweets matching a profile, search, list, or reply thread, and fire a webhook on new matches — plus export an account's followers or following. These are **paid-only** features (a free/lapsed account gets `ScopeError` for 403). `ScopeError` remains an `AuthenticationError` subclass for backward-compatible catches. Monitors and audience export are billed per X request at your plan rate (Starter 5, Growth 3, Pro 2, Scale 1 credits). Max 50 monitors per user.
 
 **Create a monitor:**
 
@@ -474,13 +474,16 @@ All errors inherit from `WebclawError`, which carries the HTTP status code when
 from webclaw import (
     WebclawError,
     AuthenticationError,
+    ScopeError,
     NotFoundError,
     RateLimitError,
     TimeoutError,
 )
 
 try:
     result = client.scrape("https://example.com")
+except ScopeError:
+    print("API key is missing the required plan or scope")
 except AuthenticationError:
     print("Invalid or missing API key")
 except RateLimitError:
@@ -495,7 +498,8 @@ except WebclawError as e:
 
 | Exception | HTTP Status | When |
 |-----------|-------------|------|
-| `AuthenticationError` | 401 / 403 | Invalid or missing API key |
+| `AuthenticationError` | 401 | Invalid or missing API key |
+| `ScopeError` | 403 | Authenticated but missing the required plan or scope |
 | `NotFoundError` | 404 | Resource does not exist |
 | `RateLimitError` | 429 | Too many requests |
 | `TimeoutError` | -- | Crawl/research polling exceeded timeout |

tests/test_async_client.py

@@ -9,6 +9,7 @@
     AuthenticationError,
     NotFoundError,
     RateLimitError,
+    ScopeError,
     TimeoutError,
     WebclawError,
 )
@@ -468,6 +469,17 @@ async def test_auth_error(client: AsyncWebclaw):
         await client.scrape("https://example.com")
 
 
+@respx.mock
+async def test_scope_error(client: AsyncWebclaw):
+    respx.post(f"{BASE}/v1/scrape").mock(
+        return_value=httpx.Response(403, json={"error": "Paid plan required"})
+    )
+    with pytest.raises(ScopeError) as exc:
+        await client.scrape("https://example.com")
+    assert exc.value.status_code == 403
+    assert isinstance(exc.value, AuthenticationError)
+
+
 @respx.mock
 async def test_not_found_error(client: AsyncWebclaw):
     respx.get(f"{BASE}/v1/crawl/nope").mock(

tests/test_client.py

@@ -8,6 +8,7 @@
     AuthenticationError,
     NotFoundError,
     RateLimitError,
+    ScopeError,
     TimeoutError,
     Webclaw,
     WebclawError,
@@ -16,6 +17,13 @@
 BASE = "https://api.webclaw.io"
 
 
+def test_opaque_ids_are_encoded_as_single_path_segments():
+    from webclaw import _endpoints as ep
+
+    assert ep.path_segment("job/with space?") == "job%2Fwith%20space%3F"
+    assert ep.x_monitor_path("monitor/one") == "/v1/x/monitors/monitor%2Fone"
+
+
 @pytest.fixture()
 def client():
     c = Webclaw("test-key", base_url=BASE)
@@ -699,12 +707,14 @@ def test_auth_error(client: Webclaw):
 
 
 @respx.mock
-def test_auth_error_403(client: Webclaw):
+def test_scope_error_403(client: Webclaw):
     respx.post(f"{BASE}/v1/scrape").mock(
         return_value=httpx.Response(403, json={"error": "Forbidden"})
     )
-    with pytest.raises(AuthenticationError):
+    with pytest.raises(ScopeError) as exc:
         client.scrape("https://example.com")
+    assert exc.value.status_code == 403
+    assert isinstance(exc.value, AuthenticationError)
 
 
 @respx.mock

tests/test_x.py

@@ -14,6 +14,7 @@
 from webclaw import (
     AsyncWebclaw,
     AuthenticationError,
+    ScopeError,
     Webclaw,
     XAudienceResponse,
     XMonitor,
@@ -294,12 +295,14 @@ def test_export_x_audience_null_cursor_means_done(client: Webclaw):
 
 
 @respx.mock
-def test_create_x_monitor_403_is_auth_error(client: Webclaw):
+def test_create_x_monitor_403_is_scope_error(client: Webclaw):
     respx.post(f"{BASE}/v1/x/monitors").mock(
         return_value=httpx.Response(403, json={"error": "X monitoring requires a paid plan"})
     )
-    with pytest.raises(AuthenticationError, match="paid plan"):
+    with pytest.raises(ScopeError, match="paid plan") as exc:
         client.create_x_monitor("profile", "@handle")
+    assert exc.value.status_code == 403
+    assert isinstance(exc.value, AuthenticationError)
 
 
 # -- async mirror -------------------------------------------------------------
@@ -360,5 +363,5 @@ async def test_async_create_x_monitor_403(aclient: AsyncWebclaw):
     respx.post(f"{BASE}/v1/x/monitors").mock(
         return_value=httpx.Response(403, json={"error": "paid only"})
     )
-    with pytest.raises(AuthenticationError):
+    with pytest.raises(ScopeError):
         await aclient.create_x_monitor("search", "rust")

webclaw/__init__.py

@@ -6,6 +6,7 @@
     AuthenticationError,
     NotFoundError,
     RateLimitError,
+    ScopeError,
     TimeoutError,
     WebclawError,
 )
@@ -88,6 +89,7 @@
     # errors
     "WebclawError",
     "AuthenticationError",
+    "ScopeError",
     "RateLimitError",
     "NotFoundError",
     "TimeoutError",

webclaw/_endpoints.py

@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 from typing import Any, Sequence
+from urllib.parse import quote
 
 from .errors import WebclawError
 from .types import (
@@ -49,12 +50,17 @@
 X_AUDIENCE_PATH = "/v1/x/audience"
 
 
+def path_segment(value: str) -> str:
+    """Percent-encode an opaque identifier as exactly one URL path segment."""
+    return quote(value, safe="")
+
+
 def x_monitor_path(monitor_id: str) -> str:
-    return f"{X_MONITORS_PATH}/{monitor_id}"
+    return f"{X_MONITORS_PATH}/{path_segment(monitor_id)}"
 
 
 def x_monitor_check_path(monitor_id: str) -> str:
-    return f"{X_MONITORS_PATH}/{monitor_id}/check"
+    return f"{X_MONITORS_PATH}/{path_segment(monitor_id)}/check"
 
 # Job lifecycle states shared by crawl and research polling.
 #

webclaw/async_client.py

@@ -103,7 +103,7 @@ async def crawl(
 
     async def get_crawl_status(self, job_id: str) -> CrawlStatus:
         """Get current status of a crawl job."""
-        return ep.parse_crawl_status(await self._request("GET", f"/v1/crawl/{job_id}"))
+        return ep.parse_crawl_status(await self._request("GET", f"/v1/crawl/{ep.path_segment(job_id)}"))
 
     async def map(self, url: str) -> MapResponse:
         """Discover URLs from a site's sitemap."""
@@ -161,7 +161,7 @@ async def lead_batch(self, urls: list[str], *, no_cache: bool = False) -> LeadBa
 
     async def get_lead_batch(self, job_id: str) -> LeadBatchStatus:
         """Get status/results of a lead batch job without polling."""
-        return ep.parse_lead_batch_status(await self._request("GET", f"/v1/lead/batch/{job_id}"))
+        return ep.parse_lead_batch_status(await self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}"))
 
     async def wait_for_lead_batch(
         self, job_id: str, *, interval: float = 2.0, timeout: float = 600.0,
@@ -172,7 +172,7 @@ async def wait_for_lead_batch(
         :meth:`wait_for_research` / :meth:`wait_for_crawl`.
         """
         return await _async_poll_until_done(
-            fetcher=lambda: self._request("GET", f"/v1/lead/batch/{job_id}"),
+            fetcher=lambda: self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}"),
             parser=ep.parse_lead_batch_status,
             label=f"Lead batch {job_id}",
             interval=interval,
@@ -238,7 +238,7 @@ async def research(
         body = ep.build_research_body(query, deep=deep, max_sources=max_sources, max_iterations=max_iterations, topic=topic)
         job_id = (await self._request("POST", "/v1/research", json=body))["id"]
         return await _async_poll_until_done(
-            fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"),
+            fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"),
             parser=ep.parse_research,
             label=f"Research {job_id}",
             interval=2.0,
@@ -247,7 +247,7 @@ async def research(
 
     async def get_research_status(self, job_id: str) -> ResearchStatusResponse:
         """Get status/results of a research job without polling."""
-        return ep.parse_research(await self._request("GET", f"/v1/research/{job_id}"))
+        return ep.parse_research(await self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"))
 
     async def wait_for_research(
         self, job_id: str, *, interval: float = 2.0, timeout: float = 1200.0,
@@ -260,7 +260,7 @@ async def wait_for_research(
         restarting the job.
         """
         return await _async_poll_until_done(
-            fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"),
+            fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"),
             parser=ep.parse_research,
             label=f"Research {job_id}",
             interval=interval,
@@ -298,21 +298,21 @@ async def watch_list(self, *, limit: int = 50, offset: int = 0) -> WatchListResp
 
     async def watch_get(self, watch_id: str) -> WatchEntry:
         """Get a single watch monitor by ID."""
-        return ep.parse_watch_entry(await self._request("GET", f"/v1/watch/{watch_id}"))
+        return ep.parse_watch_entry(await self._request("GET", f"/v1/watch/{ep.path_segment(watch_id)}"))
 
     async def watch_delete(self, watch_id: str) -> None:
         """Delete a watch monitor."""
-        await self._request("DELETE", f"/v1/watch/{watch_id}")
+        await self._request("DELETE", f"/v1/watch/{ep.path_segment(watch_id)}")
 
     async def watch_check(self, watch_id: str) -> WatchCheckResponse:
         """Trigger an immediate check for a watch monitor."""
-        return ep.parse_watch_check(await self._request("POST", f"/v1/watch/{watch_id}/check"))
+        return ep.parse_watch_check(await self._request("POST", f"/v1/watch/{ep.path_segment(watch_id)}/check"))
 
     # -- X (Twitter) monitoring -----------------------------------------------
     #
     # Async mirror of the sync X endpoints. The X analog of watch: a monitor
     # polls X on a schedule and fires a webhook on new matches. Paid-only --
-    # the server returns 403 (AuthenticationError) for free/lapsed accounts.
+    # the server returns 403 (ScopeError) for free/lapsed accounts.
     # Monitors cost 1 credit per check; audience export 1 credit per page.
 
     async def create_x_monitor(

webclaw/client.py

@@ -10,7 +10,7 @@
 import httpx
 
 from . import _endpoints as ep
-from .errors import AuthenticationError, NotFoundError, RateLimitError, TimeoutError, WebclawError
+from .errors import AuthenticationError, NotFoundError, RateLimitError, ScopeError, TimeoutError, WebclawError
 from .types import (
     BatchResponse, BrandResponse, CrawlStatus, EndpointsResponse,
     ExtractResponse, LeadBatchJob, LeadBatchStatus, LeadResponse, MapResponse,
@@ -91,7 +91,7 @@ def crawl(
 
     def get_crawl_status(self, job_id: str) -> CrawlStatus:
         """Get current status of a crawl job."""
-        return ep.parse_crawl_status(self._request("GET", f"/v1/crawl/{job_id}"))
+        return ep.parse_crawl_status(self._request("GET", f"/v1/crawl/{ep.path_segment(job_id)}"))
 
     def map(self, url: str) -> MapResponse:
         """Discover URLs from a site's sitemap."""
@@ -161,7 +161,7 @@ def lead_batch(self, urls: list[str], *, no_cache: bool = False) -> LeadBatchJob
 
     def get_lead_batch(self, job_id: str) -> LeadBatchStatus:
         """Get status/results of a lead batch job without polling."""
-        return ep.parse_lead_batch_status(self._request("GET", f"/v1/lead/batch/{job_id}"))
+        return ep.parse_lead_batch_status(self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}"))
 
     def wait_for_lead_batch(
         self, job_id: str, *, interval: float = 2.0, timeout: float = 600.0,
@@ -172,7 +172,7 @@ def wait_for_lead_batch(
         capped backoff and terminal/unknown-status fail-fast behaviour.
         """
         return _poll_until_done(
-            fetcher=lambda: self._request("GET", f"/v1/lead/batch/{job_id}"),
+            fetcher=lambda: self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}"),
             parser=ep.parse_lead_batch_status,
             label=f"Lead batch {job_id}",
             interval=interval,
@@ -249,7 +249,7 @@ def research(
         body = ep.build_research_body(query, deep=deep, max_sources=max_sources, max_iterations=max_iterations, topic=topic)
         job_id = self._request("POST", "/v1/research", json=body)["id"]
         return _poll_until_done(
-            fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"),
+            fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"),
             parser=ep.parse_research,
             label=f"Research {job_id}",
             interval=2.0,
@@ -258,7 +258,7 @@ def research(
 
     def get_research_status(self, job_id: str) -> ResearchStatusResponse:
         """Get status/results of a research job without polling."""
-        return ep.parse_research(self._request("GET", f"/v1/research/{job_id}"))
+        return ep.parse_research(self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"))
 
     def wait_for_research(
         self, job_id: str, *, interval: float = 2.0, timeout: float = 1200.0,
@@ -275,7 +275,7 @@ def wait_for_research(
         shorter waits.
         """
         return _poll_until_done(
-            fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"),
+            fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"),
             parser=ep.parse_research,
             label=f"Research {job_id}",
             interval=interval,
@@ -314,21 +314,21 @@ def watch_list(self, *, limit: int = 50, offset: int = 0) -> WatchListResponse:
 
     def watch_get(self, watch_id: str) -> WatchEntry:
         """Get a single watch monitor by ID."""
-        return ep.parse_watch_entry(self._request("GET", f"/v1/watch/{watch_id}"))
+        return ep.parse_watch_entry(self._request("GET", f"/v1/watch/{ep.path_segment(watch_id)}"))
 
     def watch_delete(self, watch_id: str) -> None:
         """Delete a watch monitor."""
-        self._request("DELETE", f"/v1/watch/{watch_id}")
+        self._request("DELETE", f"/v1/watch/{ep.path_segment(watch_id)}")
 
     def watch_check(self, watch_id: str) -> WatchCheckResponse:
         """Trigger an immediate check for a watch monitor."""
-        return ep.parse_watch_check(self._request("POST", f"/v1/watch/{watch_id}/check"))
+        return ep.parse_watch_check(self._request("POST", f"/v1/watch/{ep.path_segment(watch_id)}/check"))
 
     # -- X (Twitter) monitoring -----------------------------------------------
     #
     # The X analog of the watch endpoints: a monitor polls X on a schedule and
     # fires a webhook on new matches. Paid-only -- the server returns 403 for
-    # free/lapsed accounts (surfaced as AuthenticationError). Monitors cost 1
+    # free/lapsed accounts (surfaced as ScopeError). Monitors cost 1
     # credit per check; audience export costs 1 credit per page fetched. Max 50
     # monitors per user.
 
@@ -494,8 +494,10 @@ def _raise_for_status(response: httpx.Response) -> None:
     else:
         detail = response.text
 
-    if response.status_code in (401, 403):
+    if response.status_code == 401:
         raise AuthenticationError(str(detail))
+    if response.status_code == 403:
+        raise ScopeError(str(detail))
     if response.status_code == 404:
         raise NotFoundError(str(detail))
     if response.status_code == 429: