diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a08cfe..d043f80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,8 @@ on: push: branches: [main] pull_request: +permissions: + contents: read jobs: test: runs-on: ubuntu-latest @@ -10,8 +12,10 @@ jobs: matrix: python: ['3.9', '3.13'] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python }} - run: pip install -e '.[dev]' diff --git a/README.md b/README.md index 754b130..3303ab8 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,7 @@ print(check.checked_at) # ISO timestamp ### X (Twitter) monitoring -Monitor X for new tweets matching a profile, search, list, or reply thread, and fire a webhook on new matches — plus export an account's followers or following. These are **paid-only** features (a free/lapsed account gets `AuthenticationError` for 403). Monitors and audience export are billed per X request at your plan rate (Starter 5, Growth 3, Pro 2, Scale 1 credits). Max 50 monitors per user. +Monitor X for new tweets matching a profile, search, list, or reply thread, and fire a webhook on new matches — plus export an account's followers or following. These are **paid-only** features (a free/lapsed account gets `ScopeError` for 403). `ScopeError` remains an `AuthenticationError` subclass for backward-compatible catches. Monitors and audience export are billed per X request at your plan rate (Starter 5, Growth 3, Pro 2, Scale 1 credits). Max 50 monitors per user. **Create a monitor:** @@ -474,6 +474,7 @@ All errors inherit from `WebclawError`, which carries the HTTP status code when from webclaw import ( WebclawError, AuthenticationError, + ScopeError, NotFoundError, RateLimitError, TimeoutError, @@ -481,6 +482,8 @@ from webclaw import ( try: result = client.scrape("https://example.com") +except ScopeError: + print("API key is missing the required plan or scope") except AuthenticationError: print("Invalid or missing API key") except RateLimitError: @@ -495,7 +498,8 @@ except WebclawError as e: | Exception | HTTP Status | When | |-----------|-------------|------| -| `AuthenticationError` | 401 / 403 | Invalid or missing API key | +| `AuthenticationError` | 401 | Invalid or missing API key | +| `ScopeError` | 403 | Authenticated but missing the required plan or scope | | `NotFoundError` | 404 | Resource does not exist | | `RateLimitError` | 429 | Too many requests | | `TimeoutError` | -- | Crawl/research polling exceeded timeout | diff --git a/tests/test_async_client.py b/tests/test_async_client.py index 769e6ed..da2afeb 100644 --- a/tests/test_async_client.py +++ b/tests/test_async_client.py @@ -9,6 +9,7 @@ AuthenticationError, NotFoundError, RateLimitError, + ScopeError, TimeoutError, WebclawError, ) @@ -468,6 +469,17 @@ async def test_auth_error(client: AsyncWebclaw): await client.scrape("https://example.com") +@respx.mock +async def test_scope_error(client: AsyncWebclaw): + respx.post(f"{BASE}/v1/scrape").mock( + return_value=httpx.Response(403, json={"error": "Paid plan required"}) + ) + with pytest.raises(ScopeError) as exc: + await client.scrape("https://example.com") + assert exc.value.status_code == 403 + assert isinstance(exc.value, AuthenticationError) + + @respx.mock async def test_not_found_error(client: AsyncWebclaw): respx.get(f"{BASE}/v1/crawl/nope").mock( diff --git a/tests/test_client.py b/tests/test_client.py index 81b501c..9f55b86 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -8,6 +8,7 @@ AuthenticationError, NotFoundError, RateLimitError, + ScopeError, TimeoutError, Webclaw, WebclawError, @@ -16,6 +17,13 @@ BASE = "https://api.webclaw.io" +def test_opaque_ids_are_encoded_as_single_path_segments(): + from webclaw import _endpoints as ep + + assert ep.path_segment("job/with space?") == "job%2Fwith%20space%3F" + assert ep.x_monitor_path("monitor/one") == "/v1/x/monitors/monitor%2Fone" + + @pytest.fixture() def client(): c = Webclaw("test-key", base_url=BASE) @@ -699,12 +707,14 @@ def test_auth_error(client: Webclaw): @respx.mock -def test_auth_error_403(client: Webclaw): +def test_scope_error_403(client: Webclaw): respx.post(f"{BASE}/v1/scrape").mock( return_value=httpx.Response(403, json={"error": "Forbidden"}) ) - with pytest.raises(AuthenticationError): + with pytest.raises(ScopeError) as exc: client.scrape("https://example.com") + assert exc.value.status_code == 403 + assert isinstance(exc.value, AuthenticationError) @respx.mock diff --git a/tests/test_x.py b/tests/test_x.py index 9816d5b..a0dca67 100644 --- a/tests/test_x.py +++ b/tests/test_x.py @@ -14,6 +14,7 @@ from webclaw import ( AsyncWebclaw, AuthenticationError, + ScopeError, Webclaw, XAudienceResponse, XMonitor, @@ -294,12 +295,14 @@ def test_export_x_audience_null_cursor_means_done(client: Webclaw): @respx.mock -def test_create_x_monitor_403_is_auth_error(client: Webclaw): +def test_create_x_monitor_403_is_scope_error(client: Webclaw): respx.post(f"{BASE}/v1/x/monitors").mock( return_value=httpx.Response(403, json={"error": "X monitoring requires a paid plan"}) ) - with pytest.raises(AuthenticationError, match="paid plan"): + with pytest.raises(ScopeError, match="paid plan") as exc: client.create_x_monitor("profile", "@handle") + assert exc.value.status_code == 403 + assert isinstance(exc.value, AuthenticationError) # -- async mirror ------------------------------------------------------------- @@ -360,5 +363,5 @@ async def test_async_create_x_monitor_403(aclient: AsyncWebclaw): respx.post(f"{BASE}/v1/x/monitors").mock( return_value=httpx.Response(403, json={"error": "paid only"}) ) - with pytest.raises(AuthenticationError): + with pytest.raises(ScopeError): await aclient.create_x_monitor("search", "rust") diff --git a/webclaw/__init__.py b/webclaw/__init__.py index 8816c1c..f34d8b0 100644 --- a/webclaw/__init__.py +++ b/webclaw/__init__.py @@ -6,6 +6,7 @@ AuthenticationError, NotFoundError, RateLimitError, + ScopeError, TimeoutError, WebclawError, ) @@ -88,6 +89,7 @@ # errors "WebclawError", "AuthenticationError", + "ScopeError", "RateLimitError", "NotFoundError", "TimeoutError", diff --git a/webclaw/_endpoints.py b/webclaw/_endpoints.py index 91e6393..742d6e2 100644 --- a/webclaw/_endpoints.py +++ b/webclaw/_endpoints.py @@ -8,6 +8,7 @@ from __future__ import annotations from typing import Any, Sequence +from urllib.parse import quote from .errors import WebclawError from .types import ( @@ -49,12 +50,17 @@ X_AUDIENCE_PATH = "/v1/x/audience" +def path_segment(value: str) -> str: + """Percent-encode an opaque identifier as exactly one URL path segment.""" + return quote(value, safe="") + + def x_monitor_path(monitor_id: str) -> str: - return f"{X_MONITORS_PATH}/{monitor_id}" + return f"{X_MONITORS_PATH}/{path_segment(monitor_id)}" def x_monitor_check_path(monitor_id: str) -> str: - return f"{X_MONITORS_PATH}/{monitor_id}/check" + return f"{X_MONITORS_PATH}/{path_segment(monitor_id)}/check" # Job lifecycle states shared by crawl and research polling. # diff --git a/webclaw/async_client.py b/webclaw/async_client.py index 62fa8a1..028dec1 100644 --- a/webclaw/async_client.py +++ b/webclaw/async_client.py @@ -103,7 +103,7 @@ async def crawl( async def get_crawl_status(self, job_id: str) -> CrawlStatus: """Get current status of a crawl job.""" - return ep.parse_crawl_status(await self._request("GET", f"/v1/crawl/{job_id}")) + return ep.parse_crawl_status(await self._request("GET", f"/v1/crawl/{ep.path_segment(job_id)}")) async def map(self, url: str) -> MapResponse: """Discover URLs from a site's sitemap.""" @@ -161,7 +161,7 @@ async def lead_batch(self, urls: list[str], *, no_cache: bool = False) -> LeadBa async def get_lead_batch(self, job_id: str) -> LeadBatchStatus: """Get status/results of a lead batch job without polling.""" - return ep.parse_lead_batch_status(await self._request("GET", f"/v1/lead/batch/{job_id}")) + return ep.parse_lead_batch_status(await self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}")) async def wait_for_lead_batch( self, job_id: str, *, interval: float = 2.0, timeout: float = 600.0, @@ -172,7 +172,7 @@ async def wait_for_lead_batch( :meth:`wait_for_research` / :meth:`wait_for_crawl`. """ return await _async_poll_until_done( - fetcher=lambda: self._request("GET", f"/v1/lead/batch/{job_id}"), + fetcher=lambda: self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}"), parser=ep.parse_lead_batch_status, label=f"Lead batch {job_id}", interval=interval, @@ -238,7 +238,7 @@ async def research( body = ep.build_research_body(query, deep=deep, max_sources=max_sources, max_iterations=max_iterations, topic=topic) job_id = (await self._request("POST", "/v1/research", json=body))["id"] return await _async_poll_until_done( - fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"), + fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"), parser=ep.parse_research, label=f"Research {job_id}", interval=2.0, @@ -247,7 +247,7 @@ async def research( async def get_research_status(self, job_id: str) -> ResearchStatusResponse: """Get status/results of a research job without polling.""" - return ep.parse_research(await self._request("GET", f"/v1/research/{job_id}")) + return ep.parse_research(await self._request("GET", f"/v1/research/{ep.path_segment(job_id)}")) async def wait_for_research( self, job_id: str, *, interval: float = 2.0, timeout: float = 1200.0, @@ -260,7 +260,7 @@ async def wait_for_research( restarting the job. """ return await _async_poll_until_done( - fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"), + fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"), parser=ep.parse_research, label=f"Research {job_id}", interval=interval, @@ -298,21 +298,21 @@ async def watch_list(self, *, limit: int = 50, offset: int = 0) -> WatchListResp async def watch_get(self, watch_id: str) -> WatchEntry: """Get a single watch monitor by ID.""" - return ep.parse_watch_entry(await self._request("GET", f"/v1/watch/{watch_id}")) + return ep.parse_watch_entry(await self._request("GET", f"/v1/watch/{ep.path_segment(watch_id)}")) async def watch_delete(self, watch_id: str) -> None: """Delete a watch monitor.""" - await self._request("DELETE", f"/v1/watch/{watch_id}") + await self._request("DELETE", f"/v1/watch/{ep.path_segment(watch_id)}") async def watch_check(self, watch_id: str) -> WatchCheckResponse: """Trigger an immediate check for a watch monitor.""" - return ep.parse_watch_check(await self._request("POST", f"/v1/watch/{watch_id}/check")) + return ep.parse_watch_check(await self._request("POST", f"/v1/watch/{ep.path_segment(watch_id)}/check")) # -- X (Twitter) monitoring ----------------------------------------------- # # Async mirror of the sync X endpoints. The X analog of watch: a monitor # polls X on a schedule and fires a webhook on new matches. Paid-only -- - # the server returns 403 (AuthenticationError) for free/lapsed accounts. + # the server returns 403 (ScopeError) for free/lapsed accounts. # Monitors cost 1 credit per check; audience export 1 credit per page. async def create_x_monitor( diff --git a/webclaw/client.py b/webclaw/client.py index 7a7e586..8cce1d7 100644 --- a/webclaw/client.py +++ b/webclaw/client.py @@ -10,7 +10,7 @@ import httpx from . import _endpoints as ep -from .errors import AuthenticationError, NotFoundError, RateLimitError, TimeoutError, WebclawError +from .errors import AuthenticationError, NotFoundError, RateLimitError, ScopeError, TimeoutError, WebclawError from .types import ( BatchResponse, BrandResponse, CrawlStatus, EndpointsResponse, ExtractResponse, LeadBatchJob, LeadBatchStatus, LeadResponse, MapResponse, @@ -91,7 +91,7 @@ def crawl( def get_crawl_status(self, job_id: str) -> CrawlStatus: """Get current status of a crawl job.""" - return ep.parse_crawl_status(self._request("GET", f"/v1/crawl/{job_id}")) + return ep.parse_crawl_status(self._request("GET", f"/v1/crawl/{ep.path_segment(job_id)}")) def map(self, url: str) -> MapResponse: """Discover URLs from a site's sitemap.""" @@ -161,7 +161,7 @@ def lead_batch(self, urls: list[str], *, no_cache: bool = False) -> LeadBatchJob def get_lead_batch(self, job_id: str) -> LeadBatchStatus: """Get status/results of a lead batch job without polling.""" - return ep.parse_lead_batch_status(self._request("GET", f"/v1/lead/batch/{job_id}")) + return ep.parse_lead_batch_status(self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}")) def wait_for_lead_batch( self, job_id: str, *, interval: float = 2.0, timeout: float = 600.0, @@ -172,7 +172,7 @@ def wait_for_lead_batch( capped backoff and terminal/unknown-status fail-fast behaviour. """ return _poll_until_done( - fetcher=lambda: self._request("GET", f"/v1/lead/batch/{job_id}"), + fetcher=lambda: self._request("GET", f"/v1/lead/batch/{ep.path_segment(job_id)}"), parser=ep.parse_lead_batch_status, label=f"Lead batch {job_id}", interval=interval, @@ -249,7 +249,7 @@ def research( body = ep.build_research_body(query, deep=deep, max_sources=max_sources, max_iterations=max_iterations, topic=topic) job_id = self._request("POST", "/v1/research", json=body)["id"] return _poll_until_done( - fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"), + fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"), parser=ep.parse_research, label=f"Research {job_id}", interval=2.0, @@ -258,7 +258,7 @@ def research( def get_research_status(self, job_id: str) -> ResearchStatusResponse: """Get status/results of a research job without polling.""" - return ep.parse_research(self._request("GET", f"/v1/research/{job_id}")) + return ep.parse_research(self._request("GET", f"/v1/research/{ep.path_segment(job_id)}")) def wait_for_research( self, job_id: str, *, interval: float = 2.0, timeout: float = 1200.0, @@ -275,7 +275,7 @@ def wait_for_research( shorter waits. """ return _poll_until_done( - fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"), + fetcher=lambda: self._request("GET", f"/v1/research/{ep.path_segment(job_id)}"), parser=ep.parse_research, label=f"Research {job_id}", interval=interval, @@ -314,21 +314,21 @@ def watch_list(self, *, limit: int = 50, offset: int = 0) -> WatchListResponse: def watch_get(self, watch_id: str) -> WatchEntry: """Get a single watch monitor by ID.""" - return ep.parse_watch_entry(self._request("GET", f"/v1/watch/{watch_id}")) + return ep.parse_watch_entry(self._request("GET", f"/v1/watch/{ep.path_segment(watch_id)}")) def watch_delete(self, watch_id: str) -> None: """Delete a watch monitor.""" - self._request("DELETE", f"/v1/watch/{watch_id}") + self._request("DELETE", f"/v1/watch/{ep.path_segment(watch_id)}") def watch_check(self, watch_id: str) -> WatchCheckResponse: """Trigger an immediate check for a watch monitor.""" - return ep.parse_watch_check(self._request("POST", f"/v1/watch/{watch_id}/check")) + return ep.parse_watch_check(self._request("POST", f"/v1/watch/{ep.path_segment(watch_id)}/check")) # -- X (Twitter) monitoring ----------------------------------------------- # # The X analog of the watch endpoints: a monitor polls X on a schedule and # fires a webhook on new matches. Paid-only -- the server returns 403 for - # free/lapsed accounts (surfaced as AuthenticationError). Monitors cost 1 + # free/lapsed accounts (surfaced as ScopeError). Monitors cost 1 # credit per check; audience export costs 1 credit per page fetched. Max 50 # monitors per user. @@ -494,8 +494,10 @@ def _raise_for_status(response: httpx.Response) -> None: else: detail = response.text - if response.status_code in (401, 403): + if response.status_code == 401: raise AuthenticationError(str(detail)) + if response.status_code == 403: + raise ScopeError(str(detail)) if response.status_code == 404: raise NotFoundError(str(detail)) if response.status_code == 429: diff --git a/webclaw/errors.py b/webclaw/errors.py index 6a90fea..1dc1bae 100644 --- a/webclaw/errors.py +++ b/webclaw/errors.py @@ -12,12 +12,24 @@ def __init__(self, message: str, status_code: int | None = None) -> None: class AuthenticationError(WebclawError): - """Raised on 401/403 responses -- invalid or missing API key.""" + """Raised on 401 responses -- invalid or missing API key.""" def __init__(self, message: str = "Invalid or missing API key") -> None: super().__init__(message, status_code=401) +class ScopeError(AuthenticationError): + """Raised on 403 responses -- authenticated but not authorized. + + This subclasses :class:`AuthenticationError` for compatibility with code + that previously caught 401 and 403 together, while preserving the real + HTTP status and allowing callers to distinguish plan/scope failures. + """ + + def __init__(self, message: str = "Insufficient permissions") -> None: + WebclawError.__init__(self, message, status_code=403) + + class RateLimitError(WebclawError): """Raised on 429 responses -- too many requests."""