diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3a0ad7b..b58b1d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,9 @@ on: jobs: offline: - runs-on: ubuntu-latest + # Pinned, not ubuntu-latest: the 26.04 image has no Python 3.9, so the + # 3.9 leg of the matrix died in setup-python. 3.9 is the README's floor. + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: diff --git a/CHANGELOG.md b/CHANGELOG.md index 7322220..c6b1b49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,31 @@ burying it. ## [Unreleased] +### Fixed + +- `--url` with a listing address from the site: `experience-level=` and `employment-type=` now become endpoint filters, and any other query parameter (for example `remote=yes`) is refused with exit 2 naming it, instead of being dropped while the run looked filtered. +- The Docker image installs `libatomic1`, which Playwright's bundled node driver needs; the build failed without it (arm64, and amd64 with Playwright 1.65). +- Ctrl+C between pages ends the run as `interrupted` (`status: partial`, exit 6) and writes the pages already fetched, in all three engines; before, they were lost to a traceback. +- Arguments refused before the first request, with exit 2: `--pages` below 1, `--out` in a directory that does not exist, `--slugs-file` that does not exist. +- Configuration precedence: an explicit empty flag (`--proxy ""`, `--url ""`, `--cdp-endpoint ""`) now overrides the `.env` / environment value instead of falling through to it. +- A lone `-` (the site's "no street") is no longer written as `'-` in CSV. +- `--allow-empty` with a finished, genuinely empty run writes `status: complete` instead of `failed`. +- `--mode offer`: a queue shorter than `--pages` ends as `queue_exhausted`, not `page_cap_reached`; `--concurrency` now applies to an offer queue. +- Solve allowance: a page's one paid solve is spent when a task is actually created, not when the captcha check merely ran, so a challenge that appears one step later is no longer refused. +- `--fingerprint` in Selenium and Puppeteer: the user agent, screen size and timezone from the fingerprint are applied; Puppeteer's init script was a syntax error and ran nothing. The WebGL vendor and renderer are read from `glParameters` in every engine. +- `diff_runs.py`: refuses `--route api` against `--route ssr`; warns when a listings run took less than 100% of its query; says "no such file" for a missing input. +- `scraper_api_client.py`: a listing page (`/job-offers/...`) is parsed as the rendered page; an offer page, a non-justjoin.it host, `--timeout` outside 1-120 and `--mode` mismatches are refused before the paid request; `--mode facets` defaults to the facets endpoint; the help no longer carries text from another scraper. +- A mistyped `--proxy` entry is shown in the error instead of `?://?`. +- A proxy that refuses its login (HTTP 407 to a CONNECT) is named at once and exits 2, instead of two minutes of timeouts ending in `page_load_timeout`. The password is not echoed. +- Puppeteer with a proxy login: credentials are answered over the CDP `Fetch` domain. `page.authenticate()` needs a method current Chromium no longer has, so the browser died before the first request. Checked against a local authenticating proxy and Chromium 154. +- A refused sitemap in `--mode offer` (HTTP 403/429 or a challenge page) now moves to the next exit like a refused listing page does, and ends as exit 3 (`blocked_sitemap`) instead of `enumeration_empty` ("the sitemap is empty"). +- On the endpoint route a challenge page that carries its marker only in the markup is classified as blocked (exit 3, not 4/5), and a refused page is saved with its markup instead of 41 bytes of `innerText`. +- A dead proxy ends the run as `proxy_unreachable` in the sidecar, not `page_load_timeout`. +- A reCAPTCHA token is no longer thrown away by a page reload: the page's own callback runs, or the form holding the token field is submitted, and the page is left as it is. Checked on a local page; no paid solve was bought for this. +- Puppeteer without a usable Chromium (Apple Silicon's bundled x86_64 one) prints a `--chromium-path` hint instead of a traceback. +- A misspelt `--category` is flagged with a close match; the `html` category, which the site no longer serves, is gone from the hint list and the README loop. `sample_output.*` was regenerated from a live run and now shows rows with several contract types. +- The `--route ssr` help and warnings say about 50 rows, not 100; the README loop no longer lists the `html` category, and states that only the Playwright engine fetches pages in parallel. + ### Changed - CI only: the canary now runs weekly (Mondays 05:40 UTC) instead of daily, and on a manual dispatch. diff --git a/Dockerfile b/Dockerfile index 02cae73..594575a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,13 @@ FROM python:3.12-slim WORKDIR /app +# Playwright's bundled node driver links libatomic.so.1, which python:*-slim +# does not ship (observed on arm64 with every release, and on amd64 with +# Playwright 1.65), and it runs before `--with-deps` gets a chance to apt-get. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libatomic1 \ + && rm -rf /var/lib/apt/lists/* + COPY requirements.txt requirements-playwright.txt ./ RUN pip install --no-cache-dir -r requirements.txt -r requirements-playwright.txt \ # Playwright's own apt-get for Chromium's shared-library dependencies — diff --git a/README.md b/README.md index 54de70b..9b9b9d0 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ for r in json.load(open('board.json')): # then one run per category for c in data java analytics pm devops erp architecture testing ai security \ other python net admin javascript support mobile c ux php go game \ - scala ruby html; do + scala ruby; do python3 playwright_scraper.py --category "$c" --pages 20 --per-page 100 \ --out "board_$c" done @@ -436,7 +436,8 @@ Known limits, stated rather than left to be discovered: `user:pass` URL is doing something. * **pyppeteer is effectively unmaintained** and its own README points at Playwright. It gains `--chromium-path`, which its twins do not need - because they do not ship a browser. + because they do not ship a browser. On Apple Silicon you must pass it: + the Chromium pyppeteer downloads is x86_64 and does not start. There is also `scraper_api_client.py`, which renders a page on 2Captcha's infrastructure and feeds the result to the same parser. @@ -464,9 +465,11 @@ endpoint paginates by numeric offset, so page 5's address is knowable without fetching page 4. It is clamped to 1 where it would be pointless — `--mode facets` (one -request) and `--route ssr` (one page) — and refused with `--cdp-endpoint`, -because a Scraping Browser profile allows one live connection and workers -would collide. Use several `pid`s, one run each. +request) and `--route ssr` (one page) — and ignored, with a warning, with +`--cdp-endpoint`, because a Scraping Browser +profile allows one live connection and workers would collide. Use several `pid`s, one run each. +Only `playwright_scraper.py` fetches pages in parallel; the Selenium and +Puppeteer engines accept the flag and run one page at a time. Without a proxy pool, N workers send N times the traffic from one address. The scraper warns and continues. diff --git a/captcha_solver.py b/captcha_solver.py index 6d5eea4..d50865b 100644 --- a/captcha_solver.py +++ b/captcha_solver.py @@ -135,6 +135,7 @@ import base64 import re +import threading import time import logging from dataclasses import dataclass @@ -881,6 +882,19 @@ def _solve_with_2captcha_v1(api_key: str, challenge: CaptchaChallenge, raise TimeoutError("2captcha.com did not return a token in time") +# Purchases STARTED by this thread. `solve_recaptcha` is the one place a paid +# task is created, so a caller that wants "how many did I buy" reads the +# difference across its call. Counting at the call site instead (before it +# knew whether there was anything to solve) charged the page's one-solve +# allowance for a no-op check, and a challenge that appeared a step later was +# then refused although nothing had been bought. +_attempts = threading.local() + + +def solve_attempts() -> int: + return getattr(_attempts, "n", 0) + + def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[str], api_version: str = "v2", min_score: float = 0.7) -> str: """Public entry point: solve `challenge` through 2captcha and return the token. @@ -909,6 +923,7 @@ def solve_recaptcha(challenge: CaptchaChallenge, twocaptcha_api_key: Optional[st "Turnstile is only implemented on the v2 API here. Drop " "--captcha-api v1 (v2 is the default) to solve it.") solver = _solve_with_2captcha_v1 if api_version == "v1" else _solve_with_2captcha_v2 + _attempts.n = solve_attempts() + 1 return solver(twocaptcha_api_key, challenge, min_score=min_score) @@ -1161,20 +1176,41 @@ def turnstile_task_for(challenge: CaptchaChallenge) -> dict: } el.value = token; el.innerHTML = token; + let called = 0; try { if (window.___grecaptcha_cfg && window.___grecaptcha_cfg.clients) { Object.values(window.___grecaptcha_cfg.clients).forEach((client) => { Object.values(client).forEach((prop) => { if (prop && typeof prop === 'object') { Object.values(prop).forEach((cb) => { - if (typeof cb === 'function') { try { cb(token); } catch (e) {} } + if (typeof cb === 'function') { + try { cb(token); called += 1; } catch (e) {} + } }); } }); }); } } catch (e) { /* best effort, non-fatal */ } - return true; + return called; +} +""" + +# Submits the form that holds the token field, WITHOUT reloading. A reload +# discards `g-recaptcha-response` (verified on Google's reCAPTCHA demo: the +# field was empty after `reload()` and the form failed; with the token left in +# place it passed), so the engines inject, let the page's own callback run or +# submit this form, and carry on from the page as it is. Returns whether a +# form was submitted. +SUBMIT_RECAPTCHA_FORM_JS = """ +() => { + const el = document.getElementById('g-recaptcha-response'); + const form = el && el.form; + if (!form) return false; + try { + if (form.requestSubmit) { form.requestSubmit(); } else { form.submit(); } + return true; + } catch (e) { return false; } } """ diff --git a/diff_runs.py b/diff_runs.py index 2064225..0ed085d 100644 --- a/diff_runs.py +++ b/diff_runs.py @@ -74,6 +74,7 @@ import argparse import json +import os import pathlib import re import sys @@ -476,6 +477,25 @@ def _check_comparable(args) -> bool: f"`added` and `removed` would describe the filter change " f"rather than the board.") + # The route changes which rows exist and which columns are filled: the + # rendered page is ONE page of about 50 rows and publishes no `applyUrl`, + # so an api run against an ssr run reports the missing pages as `removed` + # and every apply_url as `changed`. `scraper_api` reads the same endpoint + # as `api`, so those two stay comparable. + routes = {} + for label, path in (("--old", args.old), ("--new", args.new)): + _, meta = _run_status(path) + route = (meta or {}).get("route") + if route: + routes[label] = "api" if route == "scraper_api" else route + if len(routes) == 2 and routes["--old"] != routes["--new"]: + problems.append( + f"the two runs read different routes (old={routes['--old']}, " + f"new={routes['--new']}). The rendered listing is one page of " + f"about 50 rows without an apply_url, so the difference would be " + f"reported as `removed` rows and changed apply_url values rather " + f"than as anything that happened on the board.") + # And whether either run was CAPPED, which changes what `removed` means. for label, path in (("--old", args.old), ("--new", args.new)): _, meta = _run_status(path) @@ -488,6 +508,14 @@ def _check_comparable(args) -> bool: f"and a slice. A `removed` line may mean the slice moved " f"rather than that an offer was taken down. Use --pages " f"{enumerated} on both runs to compare a full census.") + share = meta.get("share_of_query_pct") + if (meta.get("mode") == "listings" and not meta.get("capped_by_site") + and isinstance(share, (int, float)) and share < 100): + print(f"[i] {label} ({path}) took {share}% of its query — a " + f"complete run, and a slice. A `removed` line may mean a " + f"newer posting pushed the offer past the last page " + f"fetched rather than that it was taken down. Fetch every " + f"page on both runs to compare a full census.") if meta.get("capped_by_site"): # The third meaning of `removed`. A capped run # holds a moving window over a board that is published to @@ -539,6 +567,10 @@ def parse_args(): def main() -> int: args = parse_args() + for label, path in (("--old", args.old), ("--new", args.new)): + if not os.path.isfile(path): + print(f"[!] {label} {path}: no such file.") + return 2 if not args.force and not _check_comparable(args): return 2 diff --git a/env_config.py b/env_config.py index b11f8cb..c3fc76d 100644 --- a/env_config.py +++ b/env_config.py @@ -201,7 +201,9 @@ def apply(args, keys=None, quiet=False): for env_name, dest in (keys or ENV_KEYS).items(): if not hasattr(args, dest): continue - if getattr(args, dest): + # `is not None`, not truthiness: an explicit empty flag (`--proxy ""`) + # is the user switching a paid path off, and must beat the .env value. + if getattr(args, dest) is not None: continue value = env_value(env_name) if value: diff --git a/fingerprint_client.py b/fingerprint_client.py index 459b9f4..3efa357 100644 --- a/fingerprint_client.py +++ b/fingerprint_client.py @@ -31,9 +31,9 @@ Chromium-format response (the fields this module actually uses): id, country, screen: {width, height} - userAgent: {value} + userAgent: {userAgent} navigator: {platform, hardwareConcurrency, deviceMemory} - webgl: {vendor, renderer} + webgl: {glParameters: {"37445": vendor, "37446": renderer}} speechSynthesis: {voices: [...]} What can and cannot be applied @@ -282,6 +282,30 @@ def playwright_context_kwargs(fp: dict) -> dict: return kwargs +def cdp_emulation(fp: dict) -> dict: + """What Selenium and Puppeteer must set over CDP, because they have no + `new_context(screen=..., timezone_id=...)` the way Playwright does. + + `metrics` is the argument of `Emulation.setDeviceMetricsOverride` and + `timezone` of `Emulation.setTimezoneOverride`; either is absent when the + fingerprint does not state it. The numbers are the ones + `playwright_context_kwargs` uses, so the engines present one identity. + """ + out: dict = {} + kwargs = playwright_context_kwargs(fp) + viewport, screen = kwargs.get("viewport"), kwargs.get("screen") + if viewport and screen: + out["metrics"] = { + "width": viewport["width"], "height": viewport["height"], + "screenWidth": screen["width"], "screenHeight": screen["height"], + "deviceScaleFactor": kwargs.get("device_scale_factor", 1.0), + "mobile": False, + } + if kwargs.get("timezone_id"): + out["timezone"] = kwargs["timezone_id"] + return out + + def playwright_init_script(fp: dict) -> str: """JS to run before page scripts, patching what Playwright can't set. @@ -290,12 +314,19 @@ def playwright_init_script(fp: dict) -> str: """ nav = fp.get("navigator") or {} webgl = fp.get("webgl") or {} + # The `chromium` format keeps the strings under webgl.glParameters, keyed + # by the GL enum (37445 = UNMASKED_VENDOR, 37446 = UNMASKED_RENDERER); + # `webgl.vendor`/`.renderer` are the older spelling and stay as fallback. + # Reading only the old keys left both null, so nothing was patched and a + # Windows fingerprint reported the host's own GPU. + gl = webgl.get("glParameters") + gl = gl if isinstance(gl, dict) else {} payload = json.dumps({ "platform": nav.get("platform"), "hardwareConcurrency": nav.get("hardwareConcurrency"), "deviceMemory": nav.get("deviceMemory"), - "webglVendor": webgl.get("vendor"), - "webglRenderer": webgl.get("renderer"), + "webglVendor": gl.get("37445") or webgl.get("vendor"), + "webglRenderer": gl.get("37446") or webgl.get("renderer"), }) return """ (() => { @@ -359,8 +390,9 @@ def main() -> int: p.add_argument("--tags", default=None, metavar="TAG", help="ONE OS-family tag, not a list: Windows, " "Microsoft Windows or Android. Chrome/Desktop/Mobile " - "are rejected by the API with 400, and no combination " - "is accepted. Use --country to narrow further.") + "were each accepted by the API on 2026-11-12; " + "combinations are not tried. Use --country to narrow " + "further.") p.add_argument("--country", default=None, help="ISO 3166-1 alpha-2, e.g. us") p.add_argument("--min-browser-version", type=int, default=None) p.add_argument("--browser-version", type=int, default=None) diff --git a/output_writer.py b/output_writer.py index d9a2441..14a4171 100644 --- a/output_writer.py +++ b/output_writer.py @@ -514,6 +514,11 @@ def _csv_value(v: Any) -> Any: def _csv_cell(v: Any) -> Tuple[Any, bool]: v = _csv_value(v) if isinstance(v, str) and v.startswith(CSV_FORMULA_PREFIXES): + # A lone "-" is the site's placeholder for "no street" (53 cells in + # 300 rows) and cannot be a formula; "-5" would be a number and "-1+2" + # is not worth the exception, so only the bare dash is spared. + if v.strip() == "-": + return v, False return "'" + v, True return v, False @@ -832,6 +837,9 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, # one page so a run should never reach it, and it is left in so that a route # which starts clamping is noticed rather than silently re-collected. # +# "queue_exhausted": `--mode offer` fetched every offer it was given, and the +# queue was shorter than `--pages`. It is not the site's page cap. +# # "single_page_route" is complete by construction for the two routes that # have one page: `--mode facets` is one request, and `--route ssr` serves # one page and ignores `?page=N`. @@ -844,7 +852,7 @@ def save(rows: Sequence[Any], out_prefix: str, fmt: str, # the other. COMPLETE_STOP_REASONS = ("completed", "pagination_exhausted", "no_new_products", "page_cap_reached", "page_echo_mismatch", - "single_page_route") + "single_page_route", "queue_exhausted") def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, @@ -889,8 +897,14 @@ def finish_run(rows: Sequence[Any], out_prefix: str, fmt: str, wrote_output = bool(rows) or allow_empty if wrote_output: - status = "complete" if (rows and complete) else ( - "partial" if rows else "failed") + if rows: + status = "complete" if complete else "partial" + elif allow_empty and complete: + # The caller declared that an empty result is a valid answer, so + # a finished run with nothing in it is not a failure. + status = "complete" + else: + status = "failed" write_run_meta(out_prefix, run_meta( status=status, stop_reason=stop_reason, pages_requested=pages_requested, pages_completed=pages_completed, diff --git a/page_flow.py b/page_flow.py index 1291f76..264b191 100644 --- a/page_flow.py +++ b/page_flow.py @@ -38,6 +38,7 @@ from typing import Callable, List, Optional from product_parser import (DEFAULT_ITEMS_COUNT, DEFAULT_ROUTE, # noqa: F401 + detect_bot_challenge, MAX_FROM, api_page_url, detail_api_url, detect_bot_challenge, detect_page_state, pages_available_for, route_of, @@ -491,7 +492,7 @@ def plan_page_urls(args, page_one_url: str, "answers HTTP 200 with page 1 again — so %d page(s) were " "asked for and 1 will be fetched. Planning a page 2 would " "re-collect page 1 and report a complete multi-page run. " - "Use --route api (the default) for more than 100 rows.", + "Use --route api (the default) for more than 50 rows.", args.pages) return [] @@ -531,6 +532,53 @@ def plan_page_urls(args, page_one_url: str, +class SitemapBlocked(RuntimeError): + """The sitemap fetch was refused (403/429 or a challenge page).""" + + +def raise_if_sitemap_blocked(status, text) -> None: + """Called with each sitemap response; raises SitemapBlocked on a refusal. + + Without it a 403 and a Cloudflare page both read as "no elements", + and the run ended `enumeration_empty` ("the sitemap is empty"), which + sends a person to check the sitemap when the exit is what was refused. + """ + if status in (403, 429): + raise SitemapBlocked("HTTP %s" % status) + marker = detect_bot_challenge(text) + if marker: + raise SitemapBlocked("challenge page (%s)" % marker) + + +def enumerate_offers_resilient(fetch_source, args, pool, relaunch): + """`enumerate_offers`, with the block handling the listing loop has. + + Returns (slugs, blocked, exits_tried). A refused sitemap is retried from + the next exit of a pool (a fresh browser: `relaunch()`), or once more + without one, bounded by the same knobs as the page loop. + """ + has_pool = bool(pool and len(pool) > 1) + retries = 0 if not RETRY_ON_BLOCKED else ( + getattr(args, "proxy_block_retries", 0) if has_pool + else BLOCK_RETRIES_WITHOUT_POOL) + for attempt in range(retries + 1): + try: + return enumerate_offers(fetch_source, args), False, attempt + 1 + except SitemapBlocked as exc: + if attempt >= retries: + log.error("The sitemap was refused (%s) after %d attempt(s)." + "%s", exc, attempt + 1, + " Every exit tried was refused." if has_pool else "") + return [], True, attempt + 1 + log.warning("The sitemap was refused (%s) — retrying%s (%d/%d).", + exc, " from another exit" if has_pool else "", + attempt + 1, retries) + if has_pool: + pool.advance("sitemap refused") + relaunch() + return [], True, retries + 1 + + def enumerate_offers(fetch_source, args) -> List[str]: """Every offer slug this run could fetch, in a stable order. @@ -558,6 +606,8 @@ def enumerate_offers(fetch_source, args) -> List[str]: read += 1 try: xml = fetch_source(url) + except SitemapBlocked: + raise except Exception as exc: log.warning("Sitemap %s did not load (%s); continuing with " "what has been read.", url, mask_secrets(str(exc))[:120]) @@ -582,3 +632,19 @@ def enumerate_offers(fetch_source, args) -> List[str]: return slugs + + + +def sleep_or_interrupt(seconds: float) -> bool: + """Sleep; False when Ctrl+C arrived instead. + + The page loops call this between pages, so that an interrupt there ends + the run as `interrupted` with the pages already fetched written out, + instead of unwinding past `finish_run` and losing them. + """ + import time + try: + time.sleep(seconds) + except KeyboardInterrupt: + return False + return True diff --git a/playwright_scraper.py b/playwright_scraper.py index 61cfde8..ea1f09e 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -70,7 +70,7 @@ python3 playwright_scraper.py --mode facets python3 playwright_scraper.py --mode offer --pages 50 --concurrency 4 python3 playwright_scraper.py --mode offer --url "https://justjoin.it/job-offer/goodylabs-senior-devops-engineer-k-m-x--lodz-devops" - python3 playwright_scraper.py --route ssr # robots.txt-clean, 100 rows + python3 playwright_scraper.py --route ssr # robots.txt-clean, ~50 rows This site answered every one of those from a bare datacentre IP with no credentials on 2026-09-18. `--proxy` and `--cdp-endpoint` work and are not @@ -79,7 +79,7 @@ Note on robots.txt: justjoin.it allows `/job-offers/` and `/job-offer/`, advertises eight sitemaps of its own, and carries `Disallow: /api/` — which is where the offers endpoint lives. `--route ssr` reads the rendered -listing page instead (100 rows, one page). The README states this plainly +listing page instead (about 50 rows, one page). The README states this plainly rather than leaving it to be found. """ @@ -99,7 +99,9 @@ from captcha_solver import (detect_recaptcha_v3, detect_recaptcha_in_page, reconcile_detections, solve_recaptcha, + solve_attempts, CaptchaUnsolvable, INJECT_TOKEN_JS, + SUBMIT_RECAPTCHA_FORM_JS, RECAPTCHA_DISCOVERY_JS, detect_turnstile, wait_for_turnstile, TURNSTILE_INTERCEPT_JS, TURNSTILE_INJECT_JS, turnstile_task_for) @@ -132,7 +134,7 @@ import page_flow from page_flow import MIN_CARD_MATCHES from proxy_pool import (from_args as proxy_pool_from_args, to_playwright, mask, - ROTATE_MODES, ProxyError, ProxyPool) + ROTATE_MODES, ProxyError, ProxyPool, verify_credentials) import env_config logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") @@ -170,6 +172,9 @@ class PageOutcome: products: List = field(default_factory=list) blocked_by: Optional[str] = None load_failed: bool = False + # Chromium's own name for a proxy fault when that, not a timeout, ended + # the load (ERR_PROXY_CONNECTION_FAILED, ...). Lets the sidecar say so. + proxy_failed: Optional[str] = None # The page_flow state this page came back as ("content", "blocked", # "challenge", "empty", "unknown"). Carried so the caller can tell an # EMPTY page — a /p/ hub, a no-match query, or one page past the @@ -333,6 +338,15 @@ def _fetch_text(session, url: str, timeout_ms: int = 60000) -> str: return _snapshot(session.page, url) or "" +def _fetch_sitemap(session, url: str, timeout_ms: int = 60000) -> str: + """`_fetch_page_source` for the sitemap, which raises on a refusal.""" + response = session.page.goto(url, wait_until="domcontentloaded", + timeout=timeout_ms) + text = session.page.content() or "" + page_flow.raise_if_sitemap_blocked( + response.status if response is not None else None, text) + return text + def _fetch_page_source(session, url: str, timeout_ms: int = 60000) -> str: """Navigate and return the serialised document. @@ -634,6 +648,35 @@ def _content_when_settled(page, attempts: int = 4, pause_ms: int = 700): return None +def _markup_view(session, url: str, state: str, body: str): + """(state, body) re-read from the document's MARKUP when that can tell more. + + On the endpoint route the body is `innerText`, which keeps neither the + `` nor the `<script>` tags a challenge page is recognised by, and + which was also what a refusal saved to `*_debug.html` (41 bytes of text + for a page the message says to inspect as HTML). So for a state the + policy cannot use, or one it already counts as blocked, look at the + serialised document: a challenge marker there turns `unknown` into + `blocked` (exit 3, not 4/5), and a blocked page is saved with its markup. + """ + if not _is_endpoint(url): + return state, body + if not (state in ("unknown", "parse_error") + or page_flow.counts_as_blocked(state)): + return state, body + try: + markup = session.page.content() or "" + except (PWError, PWTimeout): + return state, body + if not markup: + return state, body + if state != "blocked" and detect_bot_challenge(markup): + state = "blocked" + if page_flow.counts_as_blocked(state): + body = markup + return state, body + + def _is_endpoint(url: str) -> bool: """Whether this address answers with raw JSON rather than a document. @@ -865,7 +908,16 @@ def handle_captcha_if_present(page, args) -> bool: "is worth ruling out if a paid token is refused.", challenge.solved_user_agent[:60] + "…") else: - page.evaluate(INJECT_TOKEN_JS, token) + called = page.evaluate(INJECT_TOKEN_JS, token) + submitted = False if called else bool( + page.evaluate(SUBMIT_RECAPTCHA_FORM_JS)) + # No reload: it discards the token, which cost money to get. + logger.info("Token injected%s; the page is left as it is, because a " + "reload would drop the token.", + " and handed to the page's callback" if called + else (" and its form submitted" if submitted else "")) + page.wait_for_timeout(1500) + return True logger.info("Token injected. Reloading page to continue.") page.wait_for_timeout(1500) page.reload(wait_until="domcontentloaded", timeout=60000) @@ -993,8 +1045,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Counted, because it can BUY. See _solve_budget. if _solve_budget(args, solves_bought): - solves_bought += 1 - if handle_captcha_if_present(session.page, args): + attempts_before = solve_attempts() + solved = handle_captcha_if_present(session.page, args) + solves_bought += solve_attempts() - attempts_before + if solved: # A solve navigated the page. Give the destination a moment # before judging what came back. session.page.wait_for_timeout(1000) @@ -1047,6 +1101,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome state = _classify(session.page, body, http_status, args.mode, args.route) + state, body = _markup_view(session, url, state, body) + # The paid path is reached for any state page_flow.STATE_POLICY marks # solvable — on this site that is `blocked`; there is no separate # `challenge` state (see page_flow's docstring). Bounded by @@ -1055,8 +1111,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # paying for one the API will reject. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): - solves_bought += 1 - if handle_captcha_if_present(session.page, args): + attempts_before = solve_attempts() + solved = handle_captcha_if_present(session.page, args) + solves_bought += solve_attempts() - attempts_before + if solved: session.page.wait_for_timeout(1000) body = _fetch_body(session, args, url) or body state = _classify(session.page, body, http_status, args.mode, @@ -1132,6 +1190,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.error("Gave up loading %s after %d attempt(s).", url, args.retries) outcome.load_failed = True + outcome.proxy_failed = exit_failed outcome.blocked_by = None return outcome @@ -1506,6 +1565,8 @@ def scrape(args) -> int: "remote browser has its own exit, and layering a second " "proxy on top would contradict it.") pool = None + if pool: + verify_credentials(pool.current) concurrency = max(1, args.concurrency) if concurrency > 1: @@ -1559,8 +1620,17 @@ def scrape(args) -> int: # are, so the order is the other way round. offer_slugs: List[str] = [] if args.mode == "offer" and not is_single_offer_url(args): - offer_slugs = page_flow.enumerate_offers( - lambda u: _fetch_page_source(session, u), args) + offer_slugs, sitemap_blocked, _tried = ( + page_flow.enumerate_offers_resilient( + lambda u: _fetch_sitemap(session, u), args, pool, + session.relaunch)) + if sitemap_blocked: + return finish_run( + [], args.out, args.format, args.allow_empty, + blocked=True, stop_reason="blocked_sitemap", + pages_requested=args.pages, pages_completed=0, + mode=args.mode, source=SOURCE_DEFAULT, + start_url=args.url or "", final_url="") total_enumerated = len(offer_slugs) if not offer_slugs: # Nothing to fetch, so nothing is written — `save` @@ -1614,7 +1684,8 @@ def scrape(args) -> int: first_verdict = page_flow.queue_decision(first.state, None, independent) if not first.ok: - stop_reason = ("page_load_timeout" if first.load_failed + stop_reason = (("proxy_unreachable" if first.proxy_failed + else "page_load_timeout") if first.load_failed else f"blocked_{first.blocked_by}") blocked = first.blocked_by is not None elif first_verdict: @@ -1633,9 +1704,15 @@ def scrape(args) -> int: if len(planned) + 1 < args.pages: # Capped by the site's own arithmetic — a complete # answer rather than an early stop. - stop_reason = "page_cap_reached" + stop_reason = ("queue_exhausted" if args.mode == "offer" + else "page_cap_reached") + # A queue of offers is addressable by construction: every + # entry has its own URL and the list was fixed before the + # first fetch. The check below asks whether a LISTING's page N + # has an address, which is the wrong question for it. if (args.pages > 1 and concurrency > 1 + and not independent and not page_flow.pagination_is_addressable( first.final_url or target, args.route)): logger.warning("--concurrency %d requested, but this " @@ -1660,7 +1737,8 @@ def scrape(args) -> int: failed = [o for o in rest if not o.ok] if failed: worst = min(failed, key=lambda o: o.page_num) - stop_reason = ("page_load_timeout" if worst.load_failed + stop_reason = (("proxy_unreachable" if worst.proxy_failed + else "page_load_timeout") if worst.load_failed else f"blocked_{worst.blocked_by}") blocked = any(o.blocked_by for o in rest) elif exhausted: @@ -1681,10 +1759,17 @@ def scrape(args) -> int: pool.advance(f"per-page rotation, page {page_num}") session.relaunch() - outcome = _fetch_one_page(session, args, pool, page_num, url) + try: + outcome = _fetch_one_page(session, args, pool, page_num, url) + except KeyboardInterrupt: + logger.warning("Interrupted during page %d — writing the %d " + "page(s) already fetched.", page_num, len(outcomes)) + stop_reason = "interrupted" + break outcomes.append(outcome) if not outcome.ok: - stop_reason = ("page_load_timeout" if outcome.load_failed + stop_reason = (("proxy_unreachable" if outcome.proxy_failed + else "page_load_timeout") if outcome.load_failed else f"blocked_{outcome.blocked_by}") blocked = outcome.blocked_by is not None break @@ -1720,7 +1805,11 @@ def scrape(args) -> int: if page_num - 1 < len(planned): url = planned[page_num - 1] - time.sleep(args.delay) + if not page_flow.sleep_or_interrupt(args.delay): + logger.warning("Interrupted — writing the %d page(s) already " + "fetched.", len(outcomes) + 1) + stop_reason = "interrupted" + break finally: if session is not None: session.close() @@ -1920,7 +2009,7 @@ def parse_args(): help="Which route --mode listings reads. api (default): " "justjoin.it's offers endpoint — every page, every " "filter, up to 1,000 rows a request. ssr: the " - "rendered listing page, ONE page of 100 rows with a " + "rendered listing page, ONE page of about 50 rows with a " "poorer schema (no category, no languages, no skill " "levels), and the only route justjoin.it's " "robots.txt permits — it carries Disallow: /api/. " @@ -2080,8 +2169,8 @@ def parse_args(): p.add_argument("--fp-tags", default="Windows", help="ONE OS-family tag for the fingerprint filter: " "Windows, Microsoft Windows or Android. NOT a list — " - "Chrome, Desktop and Mobile are each rejected by the " - "API with 400, and no combination is accepted. Use " + "Chrome, Desktop and Mobile were each accepted by the " + "API on 2026-11-12; combinations are not tried. Use " "--fp-country to narrow further. (default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to " @@ -2176,7 +2265,7 @@ def parse_args(): "listing serves ONE page and answers ?page=N with " "page 1 again, so planning more would re-collect page " "1 and report a complete multi-page run. Use --route " - "api (the default) for more than 100 rows. The run " + "api (the default) for more than 50 rows. The run " "status will say single_page_route.", args.pages) args.pages = 1 @@ -2215,6 +2304,11 @@ def parse_args(): "than better cover.") try: sys.exit(scrape(args)) + except KeyboardInterrupt: + # Reached only when nothing was written yet: once a page has been + # fetched the loops end the run as `interrupted` and write it. + logger.error("Interrupted before any page was fetched; nothing written.") + sys.exit(130) except ProxyError as e: # Bad usage, not a crash: a typo in a proxy list would otherwise # surface as a connection failure on page 1 with nothing naming it. diff --git a/product_parser.py b/product_parser.py index 7126f8c..53fda14 100644 --- a/product_parser.py +++ b/product_parser.py @@ -141,6 +141,7 @@ import html as _html import json +import os import re import urllib.parse from dataclasses import dataclass, field @@ -194,7 +195,7 @@ # `--route` applies to `--mode listings` only. # # api the endpoint. Every page, every filter, offset pagination. -# ssr the rendered listing page's flight payload. ONE page of 100 rows +# ssr the rendered listing page's flight payload. ONE page of about 50 rows # with a poorer schema — and the only route `robots.txt` permits # (see ROBOTS_NOTE). Kept because a reader who wants to stay inside # robots.txt should not have to fork this repo to do it. @@ -207,10 +208,10 @@ # prefix. It is the site's own front end calling its own origin rather than # a crawler enumerating URLs, and it is stated plainly in the README rather # than left for a reader to find — with `--route ssr` there for anyone who -# would rather take the 100 rows robots.txt invites. +# would rather take the 50 rows robots.txt invites. ROBOTS_NOTE = ("justjoin.it's robots.txt disallows /api/, which is where " "the offers endpoint lives. --route ssr reads the rendered " - "listing page instead (100 rows, one page).") + "listing page instead (about 50 rows, one page).") # --------------------------------------------------------------------------- # The endpoint's own vocabulary @@ -268,15 +269,16 @@ WORKING_TIMES = ("full_time", "part_time", "internship", "freelance", "b2b_contract") -# The site's own 25 category keys, from `/offers/categories/count` on -# 2026-09-18. Kept as a hint for `--category`'s help text and for a +# The site's own category keys from `/offers/categories/count` on +# 2026-09-18, minus `html`, which `--mode facets` no longer returned on +# 2026-11-12 (24 keys). It is a hint that can go stale, never a gate). Kept as a hint for `--category`'s help text and for a # suggestion when a user misspells one; NOT as a gate, because the site can # add a category and refusing an unknown one would be this repo deciding # what justjoin.it sells. CATEGORY_KEYS = ("data", "java", "analytics", "pm", "devops", "erp", "architecture", "testing", "ai", "security", "other", "python", "net", "admin", "javascript", "support", "mobile", - "c", "ux", "php", "go", "game", "scala", "ruby", "html") + "c", "ux", "php", "go", "game", "scala", "ruby") # `itemsCount=2000` is HTTP 400; 1000 is served. MAX_ITEMS_COUNT = 1000 @@ -759,11 +761,60 @@ def filters_from_listing_url(url: str) -> Filters: if len(segments) > 1: category = segments[1].lower() keyword = None + experience: List[str] = [] + employment: List[str] = [] for key, value in urllib.parse.parse_qsl(query): if key in ("keyword", "keywords") and value: keyword = value + elif key == "experience-level": + experience += [v for v in value.split(",") if v] + elif key == "employment-type": + employment += [v for v in value.split(",") if v] return Filters(city=city, category=category, remote=remote, - keyword=keyword) + keyword=keyword, experience=experience or None, + employment=employment or None) + + +# Query parameters of a browsable listing URL that `filters_from_listing_url` +# turns into endpoint filters. Anything else on such a URL (the site writes +# `remote=yes`, `working-time=...` and others when a filter is ticked) used to +# be dropped without a word, so the run looked filtered and was not. +LISTING_URL_PARAMS = ("keyword", "keywords", "experience-level", + "employment-type") + + +def listing_url_problem(url: str) -> Optional[str]: + """Why a browsable listing URL cannot be honoured, or None.""" + if not url or route_of(url) != "listing": + return None + _, _, _, query = _split_url(url) + unknown = [] + bad = [] + for key, value in urllib.parse.parse_qsl(query): + if key not in LISTING_URL_PARAMS: + if key not in unknown: + unknown.append(key) + elif key == "experience-level": + bad += [v for v in value.split(",") + if v and v not in EXPERIENCE_LEVELS] + elif key == "employment-type": + bad += [v for v in value.split(",") + if v and v not in EMPLOYMENT_TYPES] + if unknown: + return ("--url carries the query parameter%s %s, which this scraper " + "does not translate into an endpoint filter, so the run would " + "quietly not be narrowed by %s. Supported on a listing URL: " + "%s. Use the matching flags (--remote, --working-time, " + "--with-salary, ...) instead." + % ("" if len(unknown) == 1 else "s", ", ".join(unknown), + "it" if len(unknown) == 1 else "them", + ", ".join(LISTING_URL_PARAMS))) + if bad: + return ("--url names %s, which is not a value the endpoint filters " + "on. Valid experience levels: %s; employment types: %s." + % (", ".join(bad), ", ".join(EXPERIENCE_LEVELS), + ", ".join(EMPLOYMENT_TYPES))) + return None def page_url(url: str, page: int) -> Optional[str]: @@ -2074,9 +2125,9 @@ def filters_from_args(args) -> Filters: city_radius=args.city_radius, remote=("remote" if args.remote else from_url.remote), keyword=args.keyword or from_url.keyword, - experience=args.experience or None, + experience=args.experience or from_url.experience, language=args.language or None, - employment=args.employment or None, + employment=args.employment or from_url.employment, working_time=args.working_time or None, with_salary=bool(args.with_salary), ) @@ -2144,6 +2195,42 @@ def args_problem(args) -> Optional[str]: "site — a 'nothing found' for a fetch that never happened. " "Use 1 for a single attempt.") + category = getattr(args, "category", None) + if category and category.lower() not in CATEGORY_KEYS: + import difflib + close = difflib.get_close_matches(category.lower(), CATEGORY_KEYS, n=2) + # A warning, not a refusal: the site can add a category, and this + # list is a dated snapshot. But a misspelling and an empty category + # both end as "0 offers", and only one of them is the site's answer. + log.warning("--category %r is not one of the 24 keys seen on " + "2026-11-12%s. If the run finds 0 offers, check the " + "spelling (--mode facets lists the current keys).", + category, " — did you mean %s?" % " or ".join(close) + if close else "") + + pages = getattr(args, "pages", None) + if pages is not None and pages < 1: + return ("--pages must be at least 1 (it is how many pages to fetch); " + "%d used to be accepted and fetched one page anyway." + % pages) + + problem = listing_url_problem(getattr(args, "url", None)) + if problem: + return problem + + slugs_file = getattr(args, "slugs_file", None) + if slugs_file and not os.path.isfile(slugs_file): + return "--slugs-file %s does not exist." % slugs_file + + out = getattr(args, "out", None) + if out: + out_dir = os.path.dirname(os.path.abspath(out)) + if not os.path.isdir(out_dir): + return ("--out %s: the directory %s does not exist. Checked " + "before the first request, because a run that finds out " + "at the write has already fetched everything." + % (out, out_dir)) + used = [name for name, value in ( ("--category", getattr(args, "category", None)), ("--city", getattr(args, "city", None)), @@ -2173,6 +2260,13 @@ def args_problem(args) -> Optional[str]: unsupported = [name for name in used if name not in ("--category", "--city", "--keyword", "--remote")] + url_filters = (filters_from_listing_url(args.url) + if getattr(args, "url", None) + and route_of(args.url) == "listing" else Filters()) + if url_filters.experience: + unsupported.append("--url experience-level=") + if url_filters.employment: + unsupported.append("--url employment-type=") if getattr(args, "city_radius", 30) not in (None, 30): unsupported.append("--city-radius") if getattr(args, "sort", DEFAULT_SORT) != DEFAULT_SORT: diff --git a/proxy_pool.py b/proxy_pool.py index 5b9b666..d5e65f5 100644 --- a/proxy_pool.py +++ b/proxy_pool.py @@ -25,8 +25,11 @@ from __future__ import annotations +import base64 import logging import random +import socket +import ssl from typing import List, Optional from urllib.parse import urlparse @@ -65,7 +68,18 @@ def parse_proxy_line(line: str, source: str = "<arg>") -> Optional[str]: # strings hold a password, and an error message is a log: a CI run # printed a live proxy login and password into its own public log this # way, from a traceback nobody expected to carry a credential. - shown = mask(line) + # `mask()` takes a bare URL and answers "?://?" for anything without a + # scheme, which is exactly what a mistyped entry is. So a line with no + # scheme is shown with its userinfo cut, or not at all when it looks like + # the host:port:login:password form. + if "://" in line: + shown = mask(line) + elif "@" in line: + shown = "***@" + line.rsplit("@", 1)[1] + elif line.count(":") >= 3: + shown = "<redacted>" + else: + shown = line parsed = urlparse(line) if parsed.scheme not in _SUPPORTED_SCHEMES: raise ProxyError( @@ -309,3 +323,56 @@ def from_args(args) -> Optional[ProxyPool]: return ProxyPool([entry], rotate="per-run") return None + + + +def verify_credentials(url: Optional[str], timeout: float = 10.0) -> None: + """Raise ProxyError when the proxy answers 407 to a CONNECT. + + A wrong login or password makes the proxy answer at once, but Chromium + reports it as a page that never loads: a run waited two minutes (60 s for + the board count, 60 s for page 1) and ended `page_load_timeout`, naming + neither the proxy nor the credentials. The commonest mistake of someone + filling in `.env`. One CONNECT to the site's host costs a few hundred + bytes and says so in seconds. + + Only a 407 is decisive. Any other outcome (an unreachable proxy, a + non-HTTP scheme, a timeout, a 200) returns silently: the engines already + handle those, and a preflight that guessed would add failures of its own. + """ + if not url: + return + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + return + try: + port = parsed.port or (443 if parsed.scheme == "https" else 80) + except ValueError: + return + headers = ["CONNECT justjoin.it:443 HTTP/1.1", "Host: justjoin.it:443"] + if parsed.username or parsed.password: + token = base64.b64encode(("%s:%s" % (parsed.username or "", + parsed.password or "")) + .encode("utf-8")).decode("ascii") + headers.append("Proxy-Authorization: Basic " + token) + request = ("\r\n".join(headers) + "\r\n\r\n").encode("ascii") + try: + sock = socket.create_connection((parsed.hostname, port), + timeout=timeout) + try: + if parsed.scheme == "https": + sock = ssl.create_default_context().wrap_socket( + sock, server_hostname=parsed.hostname) + sock.settimeout(timeout) + sock.sendall(request) + status = sock.recv(256).split(b"\r\n", 1)[0].decode("latin-1") + finally: + sock.close() + except (OSError, ValueError): + return + if " 407" in status[:16]: + raise ProxyError( + "%s answered 407 Proxy Authentication Required: the login or " + "password in the proxy URL was refused (host and port are " + "reachable). Check the credentials in --proxy / the proxy file / " + "JUSTJOIN_PROXY." % mask(url)) diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index 4d6d323..501bb8a 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -30,6 +30,7 @@ import concurrent.futures import logging import re +import platform import sys import threading import time @@ -48,10 +49,13 @@ # `pip install pyppeteer`) without anything failing, because nothing ever # imported it. from pyppeteer import launch, connect +from pyppeteer.errors import BrowserError from captcha_solver import (detect_recaptcha_v3, detect_recaptcha_in_page, reconcile_detections, solve_recaptcha, + solve_attempts, CaptchaUnsolvable, INJECT_TOKEN_JS, + SUBMIT_RECAPTCHA_FORM_JS, RECAPTCHA_DISCOVERY_JS, detect_turnstile, wait_for_turnstile, TURNSTILE_INTERCEPT_JS, TURNSTILE_INJECT_JS, turnstile_task_for) @@ -74,7 +78,7 @@ import page_flow from page_flow import MIN_CARD_MATCHES from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, - ProxyError, split_credentials) + ProxyError, split_credentials, verify_credentials) import env_config logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") @@ -225,6 +229,9 @@ class PageOutcome: products: List = field(default_factory=list) blocked_by: Optional[str] = None load_failed: bool = False + # Chromium's own name for a proxy fault when that, not a timeout, ended + # the load (ERR_PROXY_CONNECTION_FAILED, ...). Lets the sidecar say so. + proxy_failed: Optional[str] = None state: Optional[str] = None # What this response said about itself. # @@ -275,6 +282,50 @@ def _mask_credentials(text: str) -> str: return _CREDENTIALS_IN_URL_RE.sub(r"\1***:***@", text or "") +async def _install_proxy_auth(page, username: str, password: str) -> None: + """Answer the proxy's 407 over the CDP `Fetch` domain. + + Replaces `page.authenticate()`: pyppeteer 2.0.0 implements that with + `Network.setRequestInterception`, which current Chromium no longer has + ("'Network.setRequestInterception' wasn't found"), so a proxy with a + login crashed the browser before the first request (verified against + Chromium 154 and reported against 157). + + `Fetch.enable` pauses every request, so each one is continued; the + credentials are offered once per request and then cancelled, so a + refused login ends the load instead of looping. + """ + client = page._client + answered = set() + + async def send(method, params): + try: + await client.send(method, params) + except Exception: # noqa: BLE001 - the page may already be gone + pass + + def on_paused(event): + asyncio.ensure_future(send("Fetch.continueRequest", + {"requestId": event["requestId"]})) + + def on_auth(event): + request_id = event["requestId"] + from_proxy = (event.get("authChallenge") or {}).get("source") == "Proxy" + if from_proxy and request_id not in answered: + answered.add(request_id) + response = {"response": "ProvideCredentials", + "username": username, "password": password} + else: + response = {"response": "CancelAuth" if from_proxy else "Default"} + asyncio.ensure_future(send("Fetch.continueWithAuth", { + "requestId": request_id, "authChallengeResponse": response})) + + client.on("Fetch.requestPaused", on_paused) + client.on("Fetch.authRequired", on_auth) + await client.send("Fetch.enable", {"handleAuthRequests": True, + "patterns": [{"urlPattern": "*"}]}) + + def _chrome_ua(version: str) -> str: """A desktop-Chrome UA naming the browser's OWN real version. @@ -353,7 +404,7 @@ def open(self): credentials = None if self.pool: exit_url = self.pool.current - # Credentials go through page.authenticate(), never onto the + # Credentials go through CDP (_install_proxy_auth), never onto the # command line: --proxy-server= becomes part of the browser's # argv, readable by anything that can run `ps`. scrubbed, credentials = split_credentials(exit_url) @@ -378,8 +429,8 @@ def open(self): if self.args.fingerprint: self._apply_fingerprint() if credentials: - self.bridge.run(self.page.authenticate( - {"username": credentials[0], "password": credentials[1]})) + self.bridge.run(_install_proxy_auth( + self.page, credentials[0], credentials[1])) return self def _apply_fingerprint(self): @@ -394,15 +445,31 @@ def _apply_fingerprint(self): identity, and stacking a second is worse than none) — that branch returns before this is called. """ - from fingerprint_client import get_fingerprint, playwright_init_script + from fingerprint_client import (cdp_emulation, get_fingerprint, + fingerprint_user_agent, + playwright_init_script) fp = get_fingerprint(self.args.twocaptcha_key, tags=self.args.fp_tags, country=self.args.fp_country) - ua = (fp.get("userAgent") or {}).get("value") + ua = fingerprint_user_agent(fp) + emulation = cdp_emulation(fp) try: if ua: self.bridge.run(self.page.setUserAgent(ua)) - self.bridge.run( - self.page.evaluateOnNewDocument(playwright_init_script(fp))) + # NOT `page.evaluateOnNewDocument(script)`: pyppeteer treats its + # argument as a function and wraps it as `(script)()`, and the + # script is already a complete `(() => {...})();`, so the result + # was a syntax error that silently ran nothing. + self.bridge.run(self.page._client.send( + "Page.addScriptToEvaluateOnNewDocument", + {"source": playwright_init_script(fp)})) + client = self.page._client + if "metrics" in emulation: + self.bridge.run(client.send( + "Emulation.setDeviceMetricsOverride", emulation["metrics"])) + if "timezone" in emulation: + self.bridge.run(client.send( + "Emulation.setTimezoneOverride", + {"timezoneId": emulation["timezone"]})) logger.info("Using 2captcha fingerprint %s (%s)", fp.get("id"), fp.get("country")) except Exception as e: # noqa: BLE001 — a fingerprint is not the run @@ -487,6 +554,35 @@ def _content(session) -> Optional[str]: return _driver(session)["content"]() +def _markup_view(session, url: str, state: str, body: str): + """(state, body) re-read from the document's MARKUP when that can tell more. + + On the endpoint route the body is `innerText`, which keeps neither the + `<title>` nor the `<script>` tags a challenge page is recognised by, and + which was also what a refusal saved to `*_debug.html` (41 bytes of text + for a page the message says to inspect as HTML). So for a state the + policy cannot use, or one it already counts as blocked, look at the + serialised document: a challenge marker there turns `unknown` into + `blocked` (exit 3, not 4/5), and a blocked page is saved with its markup. + """ + if not _is_endpoint(url): + return state, body + if not (state in ("unknown", "parse_error") + or page_flow.counts_as_blocked(state)): + return state, body + try: + markup = session.bridge.run(session.page.content()) or "" + except Exception: + return state, body + if not markup: + return state, body + if state != "blocked" and detect_bot_challenge(markup): + state = "blocked" + if page_flow.counts_as_blocked(state): + body = markup + return state, body + + def _is_endpoint(url: str) -> bool: """Whether this address answers with raw JSON rather than a document. @@ -515,6 +611,15 @@ def _snapshot(session, url: str): return _driver(session)["content"]() +def _fetch_sitemap(session, url: str, timeout_ms: int = 60000) -> str: + """`_fetch_page_source` for the sitemap, which raises on a refusal.""" + response = session.bridge.run(session.page.goto( + url, {"waitUntil": "domcontentloaded", "timeout": timeout_ms})) + text = session.bridge.run(session.page.content()) or "" + page_flow.raise_if_sitemap_blocked( + getattr(response, "status", None), text) + return text + def _fetch_page_source(session, url: str, timeout_ms: int = 60000) -> str: """Navigate and return the serialised document. Used for enumeration. @@ -600,10 +705,15 @@ def handle_captcha_if_present(session, args) -> bool: except Exception as e: # noqa: BLE001 logger.error("Solving the challenge failed (%s).", e) return False - bridge.run(page.evaluate(INJECT_TOKEN_JS, token)) - logger.info("Token injected. Reloading page to continue.") + called = bridge.run(page.evaluate(INJECT_TOKEN_JS, token)) + submitted = False if called else bool( + bridge.run(page.evaluate(SUBMIT_RECAPTCHA_FORM_JS))) + # No reload: it discards the token, which cost money to get. + logger.info("Token injected%s; the page is left as it is, because a " + "reload would drop the token.", + " and handed to the page's callback" if called + else (" and its form submitted" if submitted else "")) time.sleep(1.5) - bridge.run(page.reload({"waitUntil": "domcontentloaded", "timeout": 60000})) return True @@ -719,8 +829,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Counted, because it can BUY. See _solve_budget. if _solve_budget(args, solves_bought): - solves_bought += 1 - if handle_captcha_if_present(session, args): + attempts_before = solve_attempts() + solved = handle_captcha_if_present(session, args) + solves_bought += solve_attempts() - attempts_before + if solved: time.sleep(1) elif solves_bought: @@ -753,6 +865,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome state = page_flow.classify(html, http_status, page.url, args.mode, args.route) + state, body = _markup_view(session, url, state, body) + # The paid path is reached for any state page_flow.STATE_POLICY marks # solvable — on this site that is `blocked`; there is no separate # `challenge` state (see page_flow's docstring). Bounded by @@ -761,8 +875,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # paying for one the API will reject. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): - solves_bought += 1 - if handle_captcha_if_present(session, args): + attempts_before = solve_attempts() + solved = handle_captcha_if_present(session, args) + solves_bought += solve_attempts() - attempts_before + if solved: time.sleep(1) html = _content(session) or html state = page_flow.classify(html, http_status, page.url, args.mode, @@ -842,6 +958,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.error("Gave up loading %s after %d attempt(s).", url, args.retries) outcome.load_failed = True + outcome.proxy_failed = exit_failed outcome.blocked_by = None return outcome @@ -1050,6 +1167,8 @@ def scrape(args) -> int: "remote browser has its own exit, and layering a second " "proxy on top would contradict it.") pool = None + if pool: + verify_credentials(pool.current) if args.concurrency > 1: logger.warning("--concurrency is ignored in this engine: parallel page " "fetching is implemented in playwright_scraper.py, " @@ -1068,8 +1187,17 @@ def scrape(args) -> int: # content decides how many pages there are, so the order reverses. offer_slugs = [] if args.mode == "offer" and not is_single_offer_url(args): - offer_slugs = page_flow.enumerate_offers( - lambda u: _fetch_page_source(session, u), args) + offer_slugs, sitemap_blocked, _tried = ( + page_flow.enumerate_offers_resilient( + lambda u: _fetch_sitemap(session, u), args, pool, + session.relaunch)) + if sitemap_blocked: + return finish_run( + [], args.out, args.format, args.allow_empty, + blocked=True, stop_reason="blocked_sitemap", + pages_requested=args.pages, pages_completed=0, + mode=args.mode, source=SOURCE_DEFAULT, + start_url=args.url or "", final_url="") if not offer_slugs: return finish_run( [], args.out, args.format, args.allow_empty, @@ -1119,7 +1247,8 @@ def scrape(args) -> int: independent = page_flow.pages_are_independent(args.mode, offer_slugs) first_verdict = page_flow.queue_decision(first.state, None, independent) if not first.ok: - stop_reason = ("page_load_timeout" if first.load_failed + stop_reason = (("proxy_unreachable" if first.proxy_failed + else "page_load_timeout") if first.load_failed else f"blocked_{first.blocked_by}") blocked = first.blocked_by is not None elif first_verdict: @@ -1141,7 +1270,8 @@ def scrape(args) -> int: planned = page_flow.plan_page_urls(args, page_one, first.pages_available, offer_slugs=offer_slugs) if len(planned) + 1 < args.pages: - stop_reason = "page_cap_reached" + stop_reason = ("queue_exhausted" if args.mode == "offer" + else "page_cap_reached") for index, url in enumerate(planned): page_num = index + 2 @@ -1149,10 +1279,17 @@ def scrape(args) -> int: pool.advance(f"per-page rotation, page {page_num}") session.relaunch() - outcome = _fetch_one_page(session, args, pool, page_num, url) + try: + outcome = _fetch_one_page(session, args, pool, page_num, url) + except KeyboardInterrupt: + logger.warning("Interrupted during page %d — writing the %d " + "page(s) already fetched.", page_num, len(outcomes)) + stop_reason = "interrupted" + break outcomes.append(outcome) if not outcome.ok: - stop_reason = ("page_load_timeout" if outcome.load_failed + stop_reason = (("proxy_unreachable" if outcome.proxy_failed + else "page_load_timeout") if outcome.load_failed else f"blocked_{outcome.blocked_by}") blocked = outcome.blocked_by is not None break @@ -1175,7 +1312,11 @@ def scrape(args) -> int: break if index + 1 < len(planned): - time.sleep(args.delay) + if not page_flow.sleep_or_interrupt(args.delay): + logger.warning("Interrupted — writing the %d page(s) already " + "fetched.", len(outcomes) + 1) + stop_reason = "interrupted" + break finally: if session is not None: session.close() @@ -1365,7 +1506,7 @@ def parse_args(): help="Which route --mode listings reads. api (default): " "justjoin.it's offers endpoint — every page, every " "filter, up to 1,000 rows a request. ssr: the " - "rendered listing page, ONE page of 100 rows with a " + "rendered listing page, ONE page of about 50 rows with a " "poorer schema (no category, no languages, no skill " "levels), and the only route justjoin.it's " "robots.txt permits — it carries Disallow: /api/. " @@ -1471,7 +1612,7 @@ def parse_args(): p.add_argument("--out", default="justjoin_jobs", help="Output file prefix") p.add_argument("--proxy", default=None, help="Proxy URL, e.g. http://ACCOUNT:PASSWORD@HOST:9999. " - "Credentials are sent over CDP (page.authenticate), " + "Credentials are sent over CDP (Fetch.authRequired), " "never on the browser's command line.") p.add_argument("--proxy-file", default=None, help="File with one proxy URL per line to rotate across. " @@ -1514,8 +1655,9 @@ def parse_args(): p.add_argument("--fp-tags", default="Windows", help="ONE OS-family tag for the fingerprint filter: " "Windows, Microsoft Windows or Android. NOT a list — " - "Chrome, Desktop and Mobile are each rejected by the " - "API with 400. (default: Windows)") + "Chrome, Desktop and Mobile were each accepted by the " + "API on 2026-11-12; combinations are not tried. " + "(default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to " "your proxy's exit country — a US fingerprint on a " @@ -1580,7 +1722,7 @@ def parse_args(): "listing serves ONE page and answers ?page=N with " "page 1 again, so planning more would re-collect page " "1 and report a complete multi-page run. Use --route " - "api (the default) for more than 100 rows. The run " + "api (the default) for more than 50 rows. The run " "status will say single_page_route.", args.pages) args.pages = 1 @@ -1609,6 +1751,22 @@ def parse_args(): args = parse_args() try: sys.exit(scrape(args)) + except BrowserError as e: + # pyppeteer's own download is an x86_64 Chromium; on Apple Silicon it + # cannot start and the failure is a bare "Browser closed unexpectedly" + # with a traceback. Say what to do about it. + logger.error("Chromium did not start (%s).%s Pass --chromium-path " + "with an installed Chrome or Chromium.", e, + " pyppeteer's bundled Chromium is x86_64 and does not " + "run on Apple Silicon." if platform.machine().lower() + in ("arm64", "aarch64") and sys.platform == "darwin" + else "") + sys.exit(1) + except KeyboardInterrupt: + # Reached only when nothing was written yet: once a page has been + # fetched the loops end the run as `interrupted` and write it. + logger.error("Interrupted before any page was fetched; nothing written.") + sys.exit(130) except ProxyError as e: logger.error("%s", e) sys.exit(2) diff --git a/sample_output.csv b/sample_output.csv index 50f1cda..66bdbc1 100644 --- a/sample_output.csv +++ b/sample_output.csv @@ -1,10 +1,8 @@ source,scraped_at,url,sku,title,slug,company_name,company_logo_url,company_url,company_size,category,experience_level,workplace_type,working_time,salary_min,salary_max,salary_currency,salary_unit,salary_is_gross,salary_contract_type,salary_monthly_min,salary_monthly_max,salary_options,city,street,country_code,latitude,longitude,locations,location_count,required_skills,required_skill_levels,nice_to_have_skills,languages,language_levels,office_days,remote_days,remote_interview,open_to_hire_ukrainians,is_super_offer,is_active,apply_method,apply_url,published_at,last_published_at,expires_at,description,data_source,page,position -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/port-blue-sky-sp-z-o-o-sp-k--senior-software-engineer---python-react-data-warszawa-python,3be7c75b-f182-4dd5-a956-16b505f93fa1,Senior Software Engineer – Python / React / Data,port-blue-sky-sp-z-o-o-sp-k--senior-software-engineer---python-react-data-warszawa-python,PORT BLUE SKY Sp. z o.o. sp.k.,https://imgproxy.justjoinit.tech/OeuSuuCp2rJICtPt10dWVrEYrgRPo_mYWIkJ7BElWFU/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/images.justjoin.it/justjoinit/company-logos/1780035184_019e725d66817d0fbe061b0d2f119419.png,,,python,senior,remote,b2b_contract,200.0,240.0,PLN,hour,False,b2b,33600.0,40320.0,,Warszawa,Rondo ONZ 1,,52.23,21.0,Warszawa,1,Python | React | Docker | SQL | AWS | PostreSQL | Pandas | GitHub | Snowflake,5 | 5 | 5 | 5 | 5 | 5 | 5 | 5 | 5,,,,,,False,False,False,,form,,2026-09-18T11:19:11.846Z,2026-09-18T11:19:11.846Z,2026-10-18T11:19:11.846Z,,api,1,1 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/jit-team-mlops-data-engineer-warszawa-python,a8fdc280-9a73-4177-b03e-f070c1f54c06,MLOps / Data Engineer,jit-team-mlops-data-engineer-warszawa-python,Jit Team,https://imgproxy.justjoinit.tech/wWSYQ3PfIRATt5r6FpNQ1tx2EsfW1STfzPhAXi3RVQg/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/00ba86031f7afdcdef6c386aea90b95428e30224.png,,,python,mid,hybrid,freelance,25200.0,28560.0,PLN,month,False,b2b,25200.0,28560.0,,Warszawa,-,,52.23,21.01,Warszawa,1,Machine Learning | Azure | DevOps | Git | CI/CD | Python | Data Science,3 | 3 | 3 | 3 | 3 | 3 | 3,,,,,,False,False,False,,external,https://jit.traffit.com/public/form/a/f898cf4220ff1c069a6ced36bd0b2e7b4834516e63773d3d?source=justjoin.it,2026-09-18T11:00:14.740Z,2026-09-14T11:03:25.646Z,2026-10-14T11:03:25.646Z,,api,1,2 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/pko-bp-finat-python-developer-warszawa-python-70c10ca4,6287119b-2416-489c-a713-804787985063,Python Developer,pko-bp-finat-python-developer-warszawa-python-70c10ca4,PKO BP Finat,https://imgproxy.justjoinit.tech/C34hSPBYfpu0Ne7h2wrWjocxJm599fWx3xXmxb8bP38/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/9974fc4b8d460b9beff677978e200e6fc56dc894.png,,,python,mid,hybrid,b2b_contract,,,PLN,month,False,b2b,,,,Warszawa,Chmielna 89,,52.23,20.99,Warszawa,1,ETL | ELT | API | BL | SQL | fastapi | Git | Cl/CD,3 | 3 | 3 | 3 | 3 | 3 | 3 | 3,,,,,,False,False,False,,external,https://form.erecruiter.pl/form/10049cb8ecad47e5b18b130c665ebf2d,2026-09-18T11:00:13.119Z,2026-08-24T11:07:17.781Z,2026-11-22T11:50:25.667Z,,api,1,3 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/altimi-mid-fullstack-developer-python-react--wroclaw-python,5423295b-898e-4c4f-bd74-4441f2da65b9,Mid Fullstack Developer (Python + React),altimi-mid-fullstack-developer-python-react--wroclaw-python,ALTIMI,https://imgproxy.justjoinit.tech/5vHmwO7ebHWyrlvquTlQu73qpEo97GaqONKbL-TUrLU/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/public.justjoin.com/justjoinit/hiring-company-logos/01a019a1-95bd-7752-9eaf-cb43793ce113.png,,,python,mid,remote,b2b_contract,100.0,130.0,PLN,hour,False,b2b,16800.0,21840.0,,Wrocław,Na Ostatnim Groszu 3,,51.13,16.98,Wrocław | Warszawa | Kraków | Poznań | Gdańsk,5,Python | TypeScript | React,3 | 3 | 3,,pl,C2,,,False,False,False,,external,https://altimi.traffit.com/public/form/a/323c942b3fd293ac393c60885978bf2b49513962,2026-09-18T11:00:14.185Z,2026-08-19T10:46:58.488Z,2026-11-17T11:41:40.729Z,,api,1,4 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/grafton-recruitment-python-developer-warszawa-python,2ede5f8a-bb8a-47c4-bd8c-db74c38537af,Python Developer,grafton-recruitment-python-developer-warszawa-python,Grafton Recruitment,https://imgproxy.justjoinit.tech/UnArtSkvFtg_oV3Y9LSorwvjOG39sG6GBvvl1zzFiKU/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/fee24d8180b0795f6de05451520612cc62219c0d.jpg,,,python,senior,hybrid,full_time,25000.0,55000.0,PLN,month,True,permanent,25000.0,55000.0,,Warszawa,Śródmieście,,52.23,21.01,Warszawa,1,Python | SQL | Kubernetes | AWS | Kafka,5 | 5 | 3 | 1 | 3,,,,3,2,False,False,False,,form,,2026-09-18T09:00:12.515Z,2026-09-14T08:19:40.597Z,2026-10-14T08:19:40.597Z,,api,1,5 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/dmtech-polska-senior-backend-developer-python-z-j-angielskim-lub-niemieckim---customer-service-management-b2b--wroclaw-python-cd75eb4b,86970e05-a558-43fd-b7ea-5e6f5dd11edc,Senior Backend Developer (Python) z j. angielskim lub niemieckim – Customer Service Management (B2B),dmtech-polska-senior-backend-developer-python-z-j-angielskim-lub-niemieckim---customer-service-management-b2b--wroclaw-python-cd75eb4b,dmTECH Polska,https://imgproxy.justjoinit.tech/rWaZUcEDNlx8Mk_PTHgE9ua1Pmn8tOJgeSDsLxf1GzY/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/1ffcf67553276c2e2c881399c96a396b328ddb14.png,,,python,senior,hybrid,b2b_contract,100.0,180.0,PLN,hour,False,b2b,16800.0,30240.0,,Wrocław,-,,51.11,17.04,Wrocław,1,Langchain | Python | fastapi | Langgraph | RestAPI | React,4 | 5 | 4 | 3 | 4 | 3,,pl | en,C1 | B2,,,True,False,True,,external,https://system.erecruiter.pl/FormTemplates/RecruitmentForm.aspx?WebID=114cf15c5be64edb89078fc9096e3b69,2026-09-18T11:00:15.918Z,2026-09-10T11:11:05.959Z,2026-09-30T21:59:59.999Z,,api,1,6 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/scalo-python-developer-warszawa-python-e84a43dc,6727f25e-3081-4171-a887-794ee0272c4a,Python Developer,scalo-python-developer-warszawa-python-e84a43dc,SCALO,https://imgproxy.justjoinit.tech/KZ7Ye9DAc8P985WzJ6b84Miw7mTiB1s42lbQU4CpWTE/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/8ba5b243db4213dae53629b886aa1a8c06975325.png,,,python,mid,hybrid,full_time,16000.0,19200.0,PLN,month,False,b2b,16000.0,19200.0,,Warszawa,Centrum,,52.24,21.01,Warszawa,1,API | Django | Flask | SQL | CI/CD | Git,3 | 3 | 3 | 3 | 3 | 3,,,,,,False,False,True,,external,https://scalo.traffit.com/public/form/a/1928124147e71bab61fcf01e94c8026f7866336b65673d3d?source=justjoin.it,2026-09-18T11:00:14.617Z,2026-08-25T10:50:31.418Z,2026-09-24T21:59:59.999Z,,api,1,7 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/ntiative-integration-engineer-full-stack--warszawa-python,1f347f80-5860-44d9-b839-ec3591c398d3,Integration Engineer (Full-Stack),ntiative-integration-engineer-full-stack--warszawa-python,Ntiative,https://imgproxy.justjoinit.tech/LpB6sXCgKrPFRppmBCzBt4E71JZ6RrtUNwglSnrc6nE/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/7d4d906a54942904d419b4fd60bc6a3ffec25759.jpeg,,,python,mid,remote,full_time,,,PLN,month,False,b2b,,,,Warszawa,Marszałkowska,,52.23,21.01,Warszawa,1,REST API | API integration | Node.js | TypeScript | PostgreSQL | oauth | React,5 | 5 | 5 | 5 | 5 | 5 | 2,,en,C1,,,False,False,True,,external,https://app.spott.io/careers/ntiative/71b26fcf-5967-4ece-9b7a-2dc6eba13d63,2026-09-18T10:09:42.476Z,2026-09-18T10:09:42.476Z,2026-12-17T22:59:59.999Z,,api,1,8 -justjoin.it,2026-09-18T11:56:56Z,https://justjoin.it/job-offer/dcg-python-developer-warszawa-python-07704999,cee1582b-843e-4c5b-a222-5502d169f7b1,Python Developer,dcg-python-developer-warszawa-python-07704999,DCG,https://imgproxy.justjoinit.tech/PdCyca2QkSkirNSRiEMg1FF20u5AkO_Synp-ecBVvfg/h:200/w:200/plain/https://public.rocketjobs.pl/companies/logos/original/45ec3be407eb26204aba7a12de3d5f3b1b4a6cc8.jpg,,,python,senior,hybrid,full_time,140.0,155.0,PLN,hour,False,b2b,23520.0,26040.0,,Warszawa,Aleje Jerozolimskie,,52.22,20.96,Warszawa,1,Python | Docker | RabbitMQ/Google Pub/Sub | MongoDB | fastapi | PyTorch/TensorFlow/Transformers | Apache Airflow | Google Cloud Platform (GCP) | LangChain/LangGraph (LLM) | spaCy/Stanza,4 | 4 | 3 | 3 | 3 | 3 | 3 | 3 | 3 | 3,,pl,C2,2,3,False,False,False,,external,"https://dcg.pl/aplikuj,234,pl&offer=6848",2026-09-18T11:00:14.771Z,2026-07-30T10:27:16.636Z,2026-10-28T11:22:07.113Z,,api,1,9 +justjoin.it,2026-11-16T12:59:09Z,https://justjoin.it/job-offer/infinite-services-dwh-snowflake-developer-warszawa-data,9cfbd806-eba8-4938-8ce3-48ff38f9e33a,DWH/ Snowflake Developer,infinite-services-dwh-snowflake-developer-warszawa-data,Infinite Services,https://imgproxy.justjoinit.tech/pP97ruKMREFhG8iIrQFDFfKSqduSiusWZHCQ6WPTIwA/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/c1e593041a4819976776fbf0628f0b8d639319de.jpg,,,data,mid,remote,b2b_contract,,,PLN,month,False,b2b,,,,Warszawa,Prosta 32,,52.23,20.99,Warszawa,1,DWH | Data | Snowflake | SQL | Data modeling | ETL/ELT,4 | 4 | 4 | 3 | 3 | 3,,,,,,False,False,False,,external,https://infiniteservices.traffit.com/public/form/a/37f43537fb97e63334ac037e49cfddf049516c77,2026-11-17T08:25:06.644Z,2026-11-05T10:31:11.889Z,2026-12-05T10:31:11.889Z,,api,1,1 +justjoin.it,2026-11-16T12:59:09Z,https://justjoin.it/job-offer/autopay-s-a--senior-infrastructure-manager-sopot-pm,834bb795-729c-4857-9c11-74c8cdd00b65,Senior Infrastructure Manager,autopay-s-a--senior-infrastructure-manager-sopot-pm,Autopay S.A.,https://imgproxy.justjoinit.tech/tWP_vaFByQ-DAgJBCJoaoc0dSEl7NMRT2zu3mzxJtHA/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/3bc2bda81872dd137398bccf9f939e5406f2e763.png,,,pm,senior,hybrid,full_time,,,PLN,month,True,permanent,,,,Sopot,Powstańców Warszawy 6,,54.44,18.57,Sopot,1,Management | People Management | Leadership,3 | 3 | 3,,,,,,False,False,False,,external,https://autopay.traffit.com/public/form/a/44a078bab12251cc81946dfdf59700a96b667551?source=justjoin.it,2026-11-17T08:05:11.722Z,2026-11-04T13:29:38.232Z,2026-12-04T13:29:38.232Z,,api,1,2 +justjoin.it,2026-11-16T12:59:09Z,https://justjoin.it/job-offer/miele-global-services-hr-systems-consultant---integrations-poznan-architecture,cf85e00f-9ef5-4398-bd45-61e7e75c8268,HR Systems Consultant – Integrations,miele-global-services-hr-systems-consultant---integrations-poznan-architecture,Miele Global Services,https://imgproxy.justjoinit.tech/WEvdw-GE1WWazXqqGmeVe65t0otxsHWcwtJIbrTXO7s/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/images.justjoin.it/justjoinit/company-logos/aef2fb98-4ed9-4f66-916c-153f941f4c77%24.png,,,architecture,mid,remote,full_time,11500.0,14500.0,PLN,month,True,permanent,11500.0,14500.0,,Poznań,Pastelowa 6,,52.41,16.87,Poznań,1,HRIS | Workday | Workday Integration Cloud | System Integration | Cloud HCM,3 | 1 | 3 | 3 | 3,,en,B2,,,False,False,False,,external,https://miele.wd3.myworkdayjobs.com/miele-jobs/job/Poznan/HR-Systems-Consultant---Integrations_R4751-1,2026-11-17T08:00:11.234Z,2026-10-20T07:12:24.907Z,2026-11-19T07:12:24.907Z,,api,1,3 +justjoin.it,2026-11-16T12:59:09Z,https://justjoin.it/job-offer/miele-global-services-hr-systems-consultant---workday-platform-configuration-poznan-admin,1abc662e-4103-4b7c-9c2d-6194a360eb18,HR Systems Consultant – Workday Platform & Configuration,miele-global-services-hr-systems-consultant---workday-platform-configuration-poznan-admin,Miele Global Services,https://imgproxy.justjoinit.tech/WEvdw-GE1WWazXqqGmeVe65t0otxsHWcwtJIbrTXO7s/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/images.justjoin.it/justjoinit/company-logos/aef2fb98-4ed9-4f66-916c-153f941f4c77%24.png,,,admin,mid,remote,full_time,11500.0,14500.0,PLN,month,True,permanent,11500.0,14500.0,,Poznań,Pastelowa 6,,52.41,16.87,Poznań,1,HRIS | Workday | Cloud HCM | Business Process Configuration | workday configuration,3 | 1 | 3 | 3 | 1,,en,B2,,,False,False,False,,external,https://miele.wd3.myworkdayjobs.com/miele-jobs/job/Poznan/HR-Systems-Consultant---Workday-Platform---Configuration_R4754,2026-11-17T08:00:11.198Z,2026-10-20T07:13:44.037Z,2026-11-19T07:13:44.037Z,,api,1,4 +justjoin.it,2026-11-16T12:59:09Z,https://justjoin.it/job-offer/proekspert-ux-designer-warszawa-ux,d2f9bea8-d41a-4204-8ca4-b539b445aacf,UX Designer,proekspert-ux-designer-warszawa-ux,PROEKSPERT,https://imgproxy.justjoinit.tech/m3FOlC6S5daxSVAvESZk7_uut8rFCOmxhXS-yix9kGw/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/public.justjoin.com/justjoinit/hiring-company-logos/01a1a0d5-565c-759c-b5ea-9247e76bc7b5.jpg,,,ux,senior,hybrid,full_time,115.0,140.0,PLN,hour,False,b2b,19320.0,23520.0,,Warszawa,Rondo Ignacego Daszyńskiego 2c,,52.23,20.98,Warszawa,1,UX/UI | Figma | Design Systems | Prototyping | WCAG | Information Architecture,5 | 5 | 4 | 4 | 4 | 4,,,,2,3,False,False,False,,external,https://forms.clickup.com/90151851573/f/2kyqg5hn-675/HNXUTBQN6HPNA6PBQP,2026-11-17T07:15:08.396Z,2026-11-03T10:12:21.189Z,2026-12-03T10:12:21.189Z,,api,1,8 +justjoin.it,2026-11-16T12:59:09Z,https://justjoin.it/job-offer/alten-polska-embedded-software-developer-android-automotive-c--krakow-c,2ff640c2-f17c-4374-a76b-ced27eb53e40,"Embedded Software Developer (Android Automotive, C++)",alten-polska-embedded-software-developer-android-automotive-c--krakow-c,ALTEN Polska,https://imgproxy.justjoinit.tech/fzczffju2ZnHChSLmUmzWst6DghxxcJrJsBTgqtNJG0/h:200/w:200/plain/https://public.rocketjobs.pl/companies/logos/original/b8c3e3a044e44aa0951d32149d17f826555d4f9f.png,,,c,mid,hybrid,full_time,,,PLN,month,True,mandate_contract,,,mandate_contract undisclosed PLN/month gross | b2b undisclosed PLN/month net,Kraków,-,,50.06,19.94,Kraków,1,Embedded | C++ | Android | Linux | CI/CD,4 | 4 | 3 | 3 | 3,,fr | es | en,C1 | C1 | B1,2,3,False,False,False,,external,https://form.erecruiter.pl/form/60a139774e704ca7b5ec4726d0906edc,2026-11-16T12:45:06.381Z,2026-10-12T12:40:26.568Z,2026-12-31T22:59:59.999Z,,api,1,39 +justjoin.it,2026-11-16T12:59:09Z,https://justjoin.it/job-offer/dataart-senior-python-developer-with-typescript-music-rights-platform-wroclaw-python,cd397773-dd00-46a1-87a4-cea6258df561,"Senior Python Developer with TypeScript, Music Rights Platform",dataart-senior-python-developer-with-typescript-music-rights-platform-wroclaw-python,DataArt,https://imgproxy.justjoinit.tech/_zUZ-tqR7v4zhtyzlQ1uoS3DfJ-fNRaKiXPHgKPb7vE/h:200/w:200/plain/https://public.justjoin.it/offers/company_logos/original/9abf1d2b2081f547efa8e9722d8527fb79bc7041.png,,,python,senior,remote,full_time,18000.0,21000.0,PLN,month,False,b2b,18000.0,21000.0,b2b 18000-21000 PLN/month net | permanent 15000-17500 PLN/month gross,Wrocław,plac Grunwaldzki 23,,51.11,17.06,Wrocław | Warszawa | Lublin | Kraków | Łódź,5,Python | TypeScript | SQL | REST API | AWS | CI/CD,4 | 4 | 4 | 3 | 4 | 4,,en,B2,,,False,False,True,,form,,2026-11-16T12:45:06.030Z,2026-11-12T12:41:37.048Z,2027-02-10T12:36:19.937Z,,api,1,45 diff --git a/sample_output.json b/sample_output.json index 62bbff8..664d696 100644 --- a/sample_output.json +++ b/sample_output.json @@ -1,125 +1,49 @@ [ { "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/port-blue-sky-sp-z-o-o-sp-k--senior-software-engineer---python-react-data-warszawa-python", - "sku": "3be7c75b-f182-4dd5-a956-16b505f93fa1", - "title": "Senior Software Engineer – Python / React / Data", - "slug": "port-blue-sky-sp-z-o-o-sp-k--senior-software-engineer---python-react-data-warszawa-python", - "company_name": "PORT BLUE SKY Sp. z o.o. sp.k.", - "company_logo_url": "https://imgproxy.justjoinit.tech/OeuSuuCp2rJICtPt10dWVrEYrgRPo_mYWIkJ7BElWFU/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/images.justjoin.it/justjoinit/company-logos/1780035184_019e725d66817d0fbe061b0d2f119419.png", + "scraped_at": "2026-11-16T12:59:09Z", + "url": "https://justjoin.it/job-offer/infinite-services-dwh-snowflake-developer-warszawa-data", + "sku": "9cfbd806-eba8-4938-8ce3-48ff38f9e33a", + "title": "DWH/ Snowflake Developer", + "slug": "infinite-services-dwh-snowflake-developer-warszawa-data", + "company_name": "Infinite Services", + "company_logo_url": "https://imgproxy.justjoinit.tech/pP97ruKMREFhG8iIrQFDFfKSqduSiusWZHCQ6WPTIwA/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/c1e593041a4819976776fbf0628f0b8d639319de.jpg", "company_url": null, "company_size": null, - "category": "python", - "experience_level": "senior", + "category": "data", + "experience_level": "mid", "workplace_type": "remote", "working_time": "b2b_contract", - "salary_min": 200.0, - "salary_max": 240.0, - "salary_currency": "PLN", - "salary_unit": "hour", - "salary_is_gross": false, - "salary_contract_type": "b2b", - "salary_monthly_min": 33600.0, - "salary_monthly_max": 40320.0, - "salary_options": null, - "city": "Warszawa", - "street": "Rondo ONZ 1", - "country_code": null, - "latitude": 52.23, - "longitude": 21.0, - "locations": [ - "Warszawa" - ], - "location_count": 1, - "required_skills": [ - "Python", - "React", - "Docker", - "SQL", - "AWS", - "PostreSQL", - "Pandas", - "GitHub", - "Snowflake" - ], - "required_skill_levels": [ - 5, - 5, - 5, - 5, - 5, - 5, - 5, - 5, - 5 - ], - "nice_to_have_skills": null, - "languages": null, - "language_levels": null, - "office_days": null, - "remote_days": null, - "remote_interview": false, - "open_to_hire_ukrainians": false, - "is_super_offer": false, - "is_active": null, - "apply_method": "form", - "apply_url": null, - "published_at": "2026-09-18T11:19:11.846Z", - "last_published_at": "2026-09-18T11:19:11.846Z", - "expires_at": "2026-10-18T11:19:11.846Z", - "description": null, - "data_source": "api", - "page": 1, - "position": 1 - }, - { - "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/jit-team-mlops-data-engineer-warszawa-python", - "sku": "a8fdc280-9a73-4177-b03e-f070c1f54c06", - "title": "MLOps / Data Engineer", - "slug": "jit-team-mlops-data-engineer-warszawa-python", - "company_name": "Jit Team", - "company_logo_url": "https://imgproxy.justjoinit.tech/wWSYQ3PfIRATt5r6FpNQ1tx2EsfW1STfzPhAXi3RVQg/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/00ba86031f7afdcdef6c386aea90b95428e30224.png", - "company_url": null, - "company_size": null, - "category": "python", - "experience_level": "mid", - "workplace_type": "hybrid", - "working_time": "freelance", - "salary_min": 25200.0, - "salary_max": 28560.0, + "salary_min": null, + "salary_max": null, "salary_currency": "PLN", "salary_unit": "month", "salary_is_gross": false, "salary_contract_type": "b2b", - "salary_monthly_min": 25200.0, - "salary_monthly_max": 28560.0, + "salary_monthly_min": null, + "salary_monthly_max": null, "salary_options": null, "city": "Warszawa", - "street": "-", + "street": "Prosta 32", "country_code": null, "latitude": 52.23, - "longitude": 21.01, + "longitude": 20.99, "locations": [ "Warszawa" ], "location_count": 1, "required_skills": [ - "Machine Learning", - "Azure", - "DevOps", - "Git", - "CI/CD", - "Python", - "Data Science" + "DWH", + "Data", + "Snowflake", + "SQL", + "Data modeling", + "ETL/ELT" ], "required_skill_levels": [ - 3, - 3, - 3, - 3, + 4, + 4, + 4, 3, 3, 3 @@ -134,64 +58,54 @@ "is_super_offer": false, "is_active": null, "apply_method": "external", - "apply_url": "https://jit.traffit.com/public/form/a/f898cf4220ff1c069a6ced36bd0b2e7b4834516e63773d3d?source=justjoin.it", - "published_at": "2026-09-18T11:00:14.740Z", - "last_published_at": "2026-09-14T11:03:25.646Z", - "expires_at": "2026-10-14T11:03:25.646Z", + "apply_url": "https://infiniteservices.traffit.com/public/form/a/37f43537fb97e63334ac037e49cfddf049516c77", + "published_at": "2026-11-17T08:25:06.644Z", + "last_published_at": "2026-11-05T10:31:11.889Z", + "expires_at": "2026-12-05T10:31:11.889Z", "description": null, "data_source": "api", "page": 1, - "position": 2 + "position": 1 }, { "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/pko-bp-finat-python-developer-warszawa-python-70c10ca4", - "sku": "6287119b-2416-489c-a713-804787985063", - "title": "Python Developer", - "slug": "pko-bp-finat-python-developer-warszawa-python-70c10ca4", - "company_name": "PKO BP Finat", - "company_logo_url": "https://imgproxy.justjoinit.tech/C34hSPBYfpu0Ne7h2wrWjocxJm599fWx3xXmxb8bP38/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/9974fc4b8d460b9beff677978e200e6fc56dc894.png", + "scraped_at": "2026-11-16T12:59:09Z", + "url": "https://justjoin.it/job-offer/autopay-s-a--senior-infrastructure-manager-sopot-pm", + "sku": "834bb795-729c-4857-9c11-74c8cdd00b65", + "title": "Senior Infrastructure Manager", + "slug": "autopay-s-a--senior-infrastructure-manager-sopot-pm", + "company_name": "Autopay S.A.", + "company_logo_url": "https://imgproxy.justjoinit.tech/tWP_vaFByQ-DAgJBCJoaoc0dSEl7NMRT2zu3mzxJtHA/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/3bc2bda81872dd137398bccf9f939e5406f2e763.png", "company_url": null, "company_size": null, - "category": "python", - "experience_level": "mid", + "category": "pm", + "experience_level": "senior", "workplace_type": "hybrid", - "working_time": "b2b_contract", + "working_time": "full_time", "salary_min": null, "salary_max": null, "salary_currency": "PLN", "salary_unit": "month", - "salary_is_gross": false, - "salary_contract_type": "b2b", + "salary_is_gross": true, + "salary_contract_type": "permanent", "salary_monthly_min": null, "salary_monthly_max": null, "salary_options": null, - "city": "Warszawa", - "street": "Chmielna 89", + "city": "Sopot", + "street": "Powstańców Warszawy 6", "country_code": null, - "latitude": 52.23, - "longitude": 20.99, + "latitude": 54.44, + "longitude": 18.57, "locations": [ - "Warszawa" + "Sopot" ], "location_count": 1, "required_skills": [ - "ETL", - "ELT", - "API", - "BL", - "SQL", - "fastapi", - "Git", - "Cl/CD" + "Management", + "People Management", + "Leadership" ], "required_skill_levels": [ - 3, - 3, - 3, - 3, - 3, 3, 3, 3 @@ -206,68 +120,68 @@ "is_super_offer": false, "is_active": null, "apply_method": "external", - "apply_url": "https://form.erecruiter.pl/form/10049cb8ecad47e5b18b130c665ebf2d", - "published_at": "2026-09-18T11:00:13.119Z", - "last_published_at": "2026-08-24T11:07:17.781Z", - "expires_at": "2026-11-22T11:50:25.667Z", + "apply_url": "https://autopay.traffit.com/public/form/a/44a078bab12251cc81946dfdf59700a96b667551?source=justjoin.it", + "published_at": "2026-11-17T08:05:11.722Z", + "last_published_at": "2026-11-04T13:29:38.232Z", + "expires_at": "2026-12-04T13:29:38.232Z", "description": null, "data_source": "api", "page": 1, - "position": 3 + "position": 2 }, { "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/altimi-mid-fullstack-developer-python-react--wroclaw-python", - "sku": "5423295b-898e-4c4f-bd74-4441f2da65b9", - "title": "Mid Fullstack Developer (Python + React)", - "slug": "altimi-mid-fullstack-developer-python-react--wroclaw-python", - "company_name": "ALTIMI", - "company_logo_url": "https://imgproxy.justjoinit.tech/5vHmwO7ebHWyrlvquTlQu73qpEo97GaqONKbL-TUrLU/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/public.justjoin.com/justjoinit/hiring-company-logos/01a019a1-95bd-7752-9eaf-cb43793ce113.png", + "scraped_at": "2026-11-16T12:59:09Z", + "url": "https://justjoin.it/job-offer/miele-global-services-hr-systems-consultant---integrations-poznan-architecture", + "sku": "cf85e00f-9ef5-4398-bd45-61e7e75c8268", + "title": "HR Systems Consultant – Integrations", + "slug": "miele-global-services-hr-systems-consultant---integrations-poznan-architecture", + "company_name": "Miele Global Services", + "company_logo_url": "https://imgproxy.justjoinit.tech/WEvdw-GE1WWazXqqGmeVe65t0otxsHWcwtJIbrTXO7s/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/images.justjoin.it/justjoinit/company-logos/aef2fb98-4ed9-4f66-916c-153f941f4c77%24.png", "company_url": null, "company_size": null, - "category": "python", + "category": "architecture", "experience_level": "mid", "workplace_type": "remote", - "working_time": "b2b_contract", - "salary_min": 100.0, - "salary_max": 130.0, + "working_time": "full_time", + "salary_min": 11500.0, + "salary_max": 14500.0, "salary_currency": "PLN", - "salary_unit": "hour", - "salary_is_gross": false, - "salary_contract_type": "b2b", - "salary_monthly_min": 16800.0, - "salary_monthly_max": 21840.0, + "salary_unit": "month", + "salary_is_gross": true, + "salary_contract_type": "permanent", + "salary_monthly_min": 11500.0, + "salary_monthly_max": 14500.0, "salary_options": null, - "city": "Wrocław", - "street": "Na Ostatnim Groszu 3", + "city": "Poznań", + "street": "Pastelowa 6", "country_code": null, - "latitude": 51.13, - "longitude": 16.98, + "latitude": 52.41, + "longitude": 16.87, "locations": [ - "Wrocław", - "Warszawa", - "Kraków", - "Poznań", - "Gdańsk" + "Poznań" ], - "location_count": 5, + "location_count": 1, "required_skills": [ - "Python", - "TypeScript", - "React" + "HRIS", + "Workday", + "Workday Integration Cloud", + "System Integration", + "Cloud HCM" ], "required_skill_levels": [ + 3, + 1, 3, 3, 3 ], "nice_to_have_skills": null, "languages": [ - "pl" + "en" ], "language_levels": [ - "C2" + "B2" ], "office_days": null, "remote_days": null, @@ -276,375 +190,307 @@ "is_super_offer": false, "is_active": null, "apply_method": "external", - "apply_url": "https://altimi.traffit.com/public/form/a/323c942b3fd293ac393c60885978bf2b49513962", - "published_at": "2026-09-18T11:00:14.185Z", - "last_published_at": "2026-08-19T10:46:58.488Z", - "expires_at": "2026-11-17T11:41:40.729Z", + "apply_url": "https://miele.wd3.myworkdayjobs.com/miele-jobs/job/Poznan/HR-Systems-Consultant---Integrations_R4751-1", + "published_at": "2026-11-17T08:00:11.234Z", + "last_published_at": "2026-10-20T07:12:24.907Z", + "expires_at": "2026-11-19T07:12:24.907Z", "description": null, "data_source": "api", "page": 1, - "position": 4 + "position": 3 }, { "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/grafton-recruitment-python-developer-warszawa-python", - "sku": "2ede5f8a-bb8a-47c4-bd8c-db74c38537af", - "title": "Python Developer", - "slug": "grafton-recruitment-python-developer-warszawa-python", - "company_name": "Grafton Recruitment", - "company_logo_url": "https://imgproxy.justjoinit.tech/UnArtSkvFtg_oV3Y9LSorwvjOG39sG6GBvvl1zzFiKU/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/fee24d8180b0795f6de05451520612cc62219c0d.jpg", + "scraped_at": "2026-11-16T12:59:09Z", + "url": "https://justjoin.it/job-offer/miele-global-services-hr-systems-consultant---workday-platform-configuration-poznan-admin", + "sku": "1abc662e-4103-4b7c-9c2d-6194a360eb18", + "title": "HR Systems Consultant – Workday Platform & Configuration", + "slug": "miele-global-services-hr-systems-consultant---workday-platform-configuration-poznan-admin", + "company_name": "Miele Global Services", + "company_logo_url": "https://imgproxy.justjoinit.tech/WEvdw-GE1WWazXqqGmeVe65t0otxsHWcwtJIbrTXO7s/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/images.justjoin.it/justjoinit/company-logos/aef2fb98-4ed9-4f66-916c-153f941f4c77%24.png", "company_url": null, "company_size": null, - "category": "python", - "experience_level": "senior", - "workplace_type": "hybrid", + "category": "admin", + "experience_level": "mid", + "workplace_type": "remote", "working_time": "full_time", - "salary_min": 25000.0, - "salary_max": 55000.0, + "salary_min": 11500.0, + "salary_max": 14500.0, "salary_currency": "PLN", "salary_unit": "month", "salary_is_gross": true, "salary_contract_type": "permanent", - "salary_monthly_min": 25000.0, - "salary_monthly_max": 55000.0, + "salary_monthly_min": 11500.0, + "salary_monthly_max": 14500.0, "salary_options": null, - "city": "Warszawa", - "street": "Śródmieście", + "city": "Poznań", + "street": "Pastelowa 6", "country_code": null, - "latitude": 52.23, - "longitude": 21.01, + "latitude": 52.41, + "longitude": 16.87, "locations": [ - "Warszawa" + "Poznań" ], "location_count": 1, "required_skills": [ - "Python", - "SQL", - "Kubernetes", - "AWS", - "Kafka" + "HRIS", + "Workday", + "Cloud HCM", + "Business Process Configuration", + "workday configuration" ], "required_skill_levels": [ - 5, - 5, 3, 1, - 3 - ], - "nice_to_have_skills": null, - "languages": null, - "language_levels": null, - "office_days": 3, - "remote_days": 2, - "remote_interview": false, - "open_to_hire_ukrainians": false, - "is_super_offer": false, - "is_active": null, - "apply_method": "form", - "apply_url": null, - "published_at": "2026-09-18T09:00:12.515Z", - "last_published_at": "2026-09-14T08:19:40.597Z", - "expires_at": "2026-10-14T08:19:40.597Z", - "description": null, - "data_source": "api", - "page": 1, - "position": 5 - }, - { - "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/dmtech-polska-senior-backend-developer-python-z-j-angielskim-lub-niemieckim---customer-service-management-b2b--wroclaw-python-cd75eb4b", - "sku": "86970e05-a558-43fd-b7ea-5e6f5dd11edc", - "title": "Senior Backend Developer (Python) z j. angielskim lub niemieckim – Customer Service Management (B2B)", - "slug": "dmtech-polska-senior-backend-developer-python-z-j-angielskim-lub-niemieckim---customer-service-management-b2b--wroclaw-python-cd75eb4b", - "company_name": "dmTECH Polska", - "company_logo_url": "https://imgproxy.justjoinit.tech/rWaZUcEDNlx8Mk_PTHgE9ua1Pmn8tOJgeSDsLxf1GzY/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/1ffcf67553276c2e2c881399c96a396b328ddb14.png", - "company_url": null, - "company_size": null, - "category": "python", - "experience_level": "senior", - "workplace_type": "hybrid", - "working_time": "b2b_contract", - "salary_min": 100.0, - "salary_max": 180.0, - "salary_currency": "PLN", - "salary_unit": "hour", - "salary_is_gross": false, - "salary_contract_type": "b2b", - "salary_monthly_min": 16800.0, - "salary_monthly_max": 30240.0, - "salary_options": null, - "city": "Wrocław", - "street": "-", - "country_code": null, - "latitude": 51.11, - "longitude": 17.04, - "locations": [ - "Wrocław" - ], - "location_count": 1, - "required_skills": [ - "Langchain", - "Python", - "fastapi", - "Langgraph", - "RestAPI", - "React" - ], - "required_skill_levels": [ - 4, - 5, - 4, 3, - 4, - 3 + 3, + 1 ], "nice_to_have_skills": null, "languages": [ - "pl", "en" ], "language_levels": [ - "C1", "B2" ], "office_days": null, "remote_days": null, - "remote_interview": true, + "remote_interview": false, "open_to_hire_ukrainians": false, - "is_super_offer": true, + "is_super_offer": false, "is_active": null, "apply_method": "external", - "apply_url": "https://system.erecruiter.pl/FormTemplates/RecruitmentForm.aspx?WebID=114cf15c5be64edb89078fc9096e3b69", - "published_at": "2026-09-18T11:00:15.918Z", - "last_published_at": "2026-09-10T11:11:05.959Z", - "expires_at": "2026-09-30T21:59:59.999Z", + "apply_url": "https://miele.wd3.myworkdayjobs.com/miele-jobs/job/Poznan/HR-Systems-Consultant---Workday-Platform---Configuration_R4754", + "published_at": "2026-11-17T08:00:11.198Z", + "last_published_at": "2026-10-20T07:13:44.037Z", + "expires_at": "2026-11-19T07:13:44.037Z", "description": null, "data_source": "api", "page": 1, - "position": 6 + "position": 4 }, { "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/scalo-python-developer-warszawa-python-e84a43dc", - "sku": "6727f25e-3081-4171-a887-794ee0272c4a", - "title": "Python Developer", - "slug": "scalo-python-developer-warszawa-python-e84a43dc", - "company_name": "SCALO", - "company_logo_url": "https://imgproxy.justjoinit.tech/KZ7Ye9DAc8P985WzJ6b84Miw7mTiB1s42lbQU4CpWTE/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/8ba5b243db4213dae53629b886aa1a8c06975325.png", + "scraped_at": "2026-11-16T12:59:09Z", + "url": "https://justjoin.it/job-offer/proekspert-ux-designer-warszawa-ux", + "sku": "d2f9bea8-d41a-4204-8ca4-b539b445aacf", + "title": "UX Designer", + "slug": "proekspert-ux-designer-warszawa-ux", + "company_name": "PROEKSPERT", + "company_logo_url": "https://imgproxy.justjoinit.tech/m3FOlC6S5daxSVAvESZk7_uut8rFCOmxhXS-yix9kGw/h:200/w:200/plain/https://s3.eu-west-1.amazonaws.com/public.justjoin.com/justjoinit/hiring-company-logos/01a1a0d5-565c-759c-b5ea-9247e76bc7b5.jpg", "company_url": null, "company_size": null, - "category": "python", - "experience_level": "mid", + "category": "ux", + "experience_level": "senior", "workplace_type": "hybrid", "working_time": "full_time", - "salary_min": 16000.0, - "salary_max": 19200.0, + "salary_min": 115.0, + "salary_max": 140.0, "salary_currency": "PLN", - "salary_unit": "month", + "salary_unit": "hour", "salary_is_gross": false, "salary_contract_type": "b2b", - "salary_monthly_min": 16000.0, - "salary_monthly_max": 19200.0, + "salary_monthly_min": 19320.0, + "salary_monthly_max": 23520.0, "salary_options": null, "city": "Warszawa", - "street": "Centrum", + "street": "Rondo Ignacego Daszyńskiego 2c", "country_code": null, - "latitude": 52.24, - "longitude": 21.01, + "latitude": 52.23, + "longitude": 20.98, "locations": [ "Warszawa" ], "location_count": 1, "required_skills": [ - "API", - "Django", - "Flask", - "SQL", - "CI/CD", - "Git" + "UX/UI", + "Figma", + "Design Systems", + "Prototyping", + "WCAG", + "Information Architecture" ], "required_skill_levels": [ - 3, - 3, - 3, - 3, - 3, - 3 + 5, + 5, + 4, + 4, + 4, + 4 ], "nice_to_have_skills": null, "languages": null, "language_levels": null, - "office_days": null, - "remote_days": null, + "office_days": 2, + "remote_days": 3, "remote_interview": false, "open_to_hire_ukrainians": false, - "is_super_offer": true, + "is_super_offer": false, "is_active": null, "apply_method": "external", - "apply_url": "https://scalo.traffit.com/public/form/a/1928124147e71bab61fcf01e94c8026f7866336b65673d3d?source=justjoin.it", - "published_at": "2026-09-18T11:00:14.617Z", - "last_published_at": "2026-08-25T10:50:31.418Z", - "expires_at": "2026-09-24T21:59:59.999Z", + "apply_url": "https://forms.clickup.com/90151851573/f/2kyqg5hn-675/HNXUTBQN6HPNA6PBQP", + "published_at": "2026-11-17T07:15:08.396Z", + "last_published_at": "2026-11-03T10:12:21.189Z", + "expires_at": "2026-12-03T10:12:21.189Z", "description": null, "data_source": "api", "page": 1, - "position": 7 + "position": 8 }, { "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/ntiative-integration-engineer-full-stack--warszawa-python", - "sku": "1f347f80-5860-44d9-b839-ec3591c398d3", - "title": "Integration Engineer (Full-Stack)", - "slug": "ntiative-integration-engineer-full-stack--warszawa-python", - "company_name": "Ntiative", - "company_logo_url": "https://imgproxy.justjoinit.tech/LpB6sXCgKrPFRppmBCzBt4E71JZ6RrtUNwglSnrc6nE/h:200/w:200/plain/https://public.justjoin.it/companies/logos/original/7d4d906a54942904d419b4fd60bc6a3ffec25759.jpeg", + "scraped_at": "2026-11-16T12:59:09Z", + "url": "https://justjoin.it/job-offer/alten-polska-embedded-software-developer-android-automotive-c--krakow-c", + "sku": "2ff640c2-f17c-4374-a76b-ced27eb53e40", + "title": "Embedded Software Developer (Android Automotive, C++)", + "slug": "alten-polska-embedded-software-developer-android-automotive-c--krakow-c", + "company_name": "ALTEN Polska", + "company_logo_url": "https://imgproxy.justjoinit.tech/fzczffju2ZnHChSLmUmzWst6DghxxcJrJsBTgqtNJG0/h:200/w:200/plain/https://public.rocketjobs.pl/companies/logos/original/b8c3e3a044e44aa0951d32149d17f826555d4f9f.png", "company_url": null, "company_size": null, - "category": "python", + "category": "c", "experience_level": "mid", - "workplace_type": "remote", + "workplace_type": "hybrid", "working_time": "full_time", "salary_min": null, "salary_max": null, "salary_currency": "PLN", "salary_unit": "month", - "salary_is_gross": false, - "salary_contract_type": "b2b", + "salary_is_gross": true, + "salary_contract_type": "mandate_contract", "salary_monthly_min": null, "salary_monthly_max": null, - "salary_options": null, - "city": "Warszawa", - "street": "Marszałkowska", + "salary_options": [ + "mandate_contract undisclosed PLN/month gross", + "b2b undisclosed PLN/month net" + ], + "city": "Kraków", + "street": "-", "country_code": null, - "latitude": 52.23, - "longitude": 21.01, + "latitude": 50.06, + "longitude": 19.94, "locations": [ - "Warszawa" + "Kraków" ], "location_count": 1, "required_skills": [ - "REST API", - "API integration", - "Node.js", - "TypeScript", - "PostgreSQL", - "oauth", - "React" + "Embedded", + "C++", + "Android", + "Linux", + "CI/CD" ], "required_skill_levels": [ - 5, - 5, - 5, - 5, - 5, - 5, - 2 + 4, + 4, + 3, + 3, + 3 ], "nice_to_have_skills": null, "languages": [ + "fr", + "es", "en" ], "language_levels": [ - "C1" + "C1", + "C1", + "B1" ], - "office_days": null, - "remote_days": null, + "office_days": 2, + "remote_days": 3, "remote_interview": false, "open_to_hire_ukrainians": false, - "is_super_offer": true, + "is_super_offer": false, "is_active": null, "apply_method": "external", - "apply_url": "https://app.spott.io/careers/ntiative/71b26fcf-5967-4ece-9b7a-2dc6eba13d63", - "published_at": "2026-09-18T10:09:42.476Z", - "last_published_at": "2026-09-18T10:09:42.476Z", - "expires_at": "2026-12-17T22:59:59.999Z", + "apply_url": "https://form.erecruiter.pl/form/60a139774e704ca7b5ec4726d0906edc", + "published_at": "2026-11-16T12:45:06.381Z", + "last_published_at": "2026-10-12T12:40:26.568Z", + "expires_at": "2026-12-31T22:59:59.999Z", "description": null, "data_source": "api", "page": 1, - "position": 8 + "position": 39 }, { "source": "justjoin.it", - "scraped_at": "2026-09-18T11:56:56Z", - "url": "https://justjoin.it/job-offer/dcg-python-developer-warszawa-python-07704999", - "sku": "cee1582b-843e-4c5b-a222-5502d169f7b1", - "title": "Python Developer", - "slug": "dcg-python-developer-warszawa-python-07704999", - "company_name": "DCG", - "company_logo_url": "https://imgproxy.justjoinit.tech/PdCyca2QkSkirNSRiEMg1FF20u5AkO_Synp-ecBVvfg/h:200/w:200/plain/https://public.rocketjobs.pl/companies/logos/original/45ec3be407eb26204aba7a12de3d5f3b1b4a6cc8.jpg", + "scraped_at": "2026-11-16T12:59:09Z", + "url": "https://justjoin.it/job-offer/dataart-senior-python-developer-with-typescript-music-rights-platform-wroclaw-python", + "sku": "cd397773-dd00-46a1-87a4-cea6258df561", + "title": "Senior Python Developer with TypeScript, Music Rights Platform", + "slug": "dataart-senior-python-developer-with-typescript-music-rights-platform-wroclaw-python", + "company_name": "DataArt", + "company_logo_url": "https://imgproxy.justjoinit.tech/_zUZ-tqR7v4zhtyzlQ1uoS3DfJ-fNRaKiXPHgKPb7vE/h:200/w:200/plain/https://public.justjoin.it/offers/company_logos/original/9abf1d2b2081f547efa8e9722d8527fb79bc7041.png", "company_url": null, "company_size": null, "category": "python", "experience_level": "senior", - "workplace_type": "hybrid", + "workplace_type": "remote", "working_time": "full_time", - "salary_min": 140.0, - "salary_max": 155.0, + "salary_min": 18000.0, + "salary_max": 21000.0, "salary_currency": "PLN", - "salary_unit": "hour", + "salary_unit": "month", "salary_is_gross": false, "salary_contract_type": "b2b", - "salary_monthly_min": 23520.0, - "salary_monthly_max": 26040.0, - "salary_options": null, - "city": "Warszawa", - "street": "Aleje Jerozolimskie", + "salary_monthly_min": 18000.0, + "salary_monthly_max": 21000.0, + "salary_options": [ + "b2b 18000-21000 PLN/month net", + "permanent 15000-17500 PLN/month gross" + ], + "city": "Wrocław", + "street": "plac Grunwaldzki 23", "country_code": null, - "latitude": 52.22, - "longitude": 20.96, + "latitude": 51.11, + "longitude": 17.06, "locations": [ - "Warszawa" + "Wrocław", + "Warszawa", + "Lublin", + "Kraków", + "Łódź" ], - "location_count": 1, + "location_count": 5, "required_skills": [ "Python", - "Docker", - "RabbitMQ/Google Pub/Sub", - "MongoDB", - "fastapi", - "PyTorch/TensorFlow/Transformers", - "Apache Airflow", - "Google Cloud Platform (GCP)", - "LangChain/LangGraph (LLM)", - "spaCy/Stanza" + "TypeScript", + "SQL", + "REST API", + "AWS", + "CI/CD" ], "required_skill_levels": [ + 4, 4, 4, 3, - 3, - 3, - 3, - 3, - 3, - 3, - 3 + 4, + 4 ], "nice_to_have_skills": null, "languages": [ - "pl" + "en" ], "language_levels": [ - "C2" + "B2" ], - "office_days": 2, - "remote_days": 3, + "office_days": null, + "remote_days": null, "remote_interview": false, "open_to_hire_ukrainians": false, - "is_super_offer": false, + "is_super_offer": true, "is_active": null, - "apply_method": "external", - "apply_url": "https://dcg.pl/aplikuj,234,pl&offer=6848", - "published_at": "2026-09-18T11:00:14.771Z", - "last_published_at": "2026-07-30T10:27:16.636Z", - "expires_at": "2026-10-28T11:22:07.113Z", + "apply_method": "form", + "apply_url": null, + "published_at": "2026-11-16T12:45:06.030Z", + "last_published_at": "2026-11-12T12:41:37.048Z", + "expires_at": "2027-02-10T12:36:19.937Z", "description": null, "data_source": "api", "page": 1, - "position": 9 + "position": 45 } ] \ No newline at end of file diff --git a/scraper_api_client.py b/scraper_api_client.py index a57e749..d5433b9 100644 --- a/scraper_api_client.py +++ b/scraper_api_client.py @@ -58,7 +58,9 @@ from product_parser import (BOT_CHALLENGE_MARKERS, MODES, DEFAULT_MODE, detect_bot_challenge, detect_page_state, parse_page, parse_detail_response, - ROUTES, DEFAULT_ROUTE, missing_facet_groups) + ROUTES, DEFAULT_ROUTE, missing_facet_groups, + route_of, is_supported_url, slug_from_url, + detail_api_url, facets_api_url) from output_writer import SOURCE_DEFAULT, finish_run import env_config import page_flow @@ -319,11 +321,12 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: logger.error( "The Scraper API delivered, but what it rendered is %r — upstream " "HTTP %s, %d bytes, saved to %s. That is NOT an empty result: " - "nothing here says the board has no jobs on it. This site's own " - "endpoint (/api/candidate-api/offers) is a few tens of KB and " - "renders reliably; its listing PAGE is 1.8 MB and timed out at " - "the default 60s. Point --url at the endpoint, raise --timeout, " - "or use playwright_scraper.py, which fetches it directly.", + "nothing here says the board has no jobs on it. Check the saved " + "file for what came back. If it is a timeout or an empty body, " + "the endpoint (/api/candidate-api/offers) is a few tens of KB " + "and renders more reliably than the 1.8 MB listing page; raise " + "--timeout or use playwright_scraper.py, which fetches it " + "directly.", state, upstream_status, len(html), dump) return EXIT_API_ERROR @@ -361,7 +364,8 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: # was complete, which is exactly the evidence `diff_runs.py` needs and # used to wave through when it was absent. stop_reason = "single_page_route" - extra = {"route": "scraper_api", "mode": args.mode} + extra = {"route": ("scraper_api" if getattr(args, "route", DEFAULT_ROUTE) == "api" + else args.route), "mode": args.mode} if args.mode == "facets": # One request, so the category counts (their own endpoint) are never # part of this body: the run is partial by construction, and says so. @@ -383,15 +387,15 @@ def _run_once(args, attempt: int = 1, attempts: int = 1) -> int: def parse_args(): p = argparse.ArgumentParser( - description="justjoin.it scraper — 2captcha Scraper API edition (no local " - "browser). Built for --mode profile, which is one " - "server-rendered page and a good fit. NOT the way to " - "browser needed). What decides success is the EXIT: " - "ordinary HTTPS request with no key and no proxy, so " - "paying for a rendered page there buys nothing. " - "--cdp-url is REQUIRED — the Scraper API's own exits are " - "datacenter addresses and justjoin.it refuses them — it does " - "not, as measured.") + description="justjoin.it scraper — 2captcha Scraper API edition (no " + "local browser). Reads the offers endpoint " + "(/api/candidate-api/offers..., one offer, or the facets " + "count) or a rendered listing page (/job-offers/...). " + "justjoin.it serves every route to a bare datacentre " + "address with no account (measured 2026-09-18), so this " + "client buys someone else's infrastructure and an exit " + "country (--cdp-url), not access. What decides success is " + "the EXIT CODE.") # NOT required: prefer the TWOCAPTCHA_KEY env var. A key passed on the # command line is visible to anyone who can run `ps`, and it lands in # shell history and in any log that echoes the command line. @@ -399,18 +403,21 @@ def parse_args(): help="2captcha.com API key (sent as a Bearer token). " "Defaults to $TWOCAPTCHA_KEY, which is the safer way to pass it.") p.add_argument("--url", default=None, - help="A justjoin.it URL: a listing page " - "(/job-offers/...) or one offer " - "(/job-offer/{slug}). " - "/jobs, or /jobs/{id}-{slug} with --mode job. Used if " - "endpoint answers that for free. Required, unless " - "JUSTJOIN_URL is set in the environment or .env.") + help="A justjoin.it URL: an endpoint " + "(/api/candidate-api/offers?..., " + "/api/candidate-api/offers/{slug}, " + "/api/candidate-api/offers/facets/count) or a " + "listing page (/job-offers/..., read as the rendered " + "page, ~50 rows). An offer PAGE (/job-offer/{slug}) " + "is refused: use its endpoint. --mode facets " + "defaults to the facets endpoint. Required otherwise, " + "unless JUSTJOIN_URL is set in the environment or " + ".env.") p.add_argument("--mode", choices=list(MODES), default=DEFAULT_MODE, - help="Default profile, unlike the browser engines, " - "because a profile is the only page kind this path " - "reads that the free endpoint cannot.") - p.add_argument("--category", default=None, help="Label to tag output rows with. Defaults to the category segment of the URL, so the column is never empty just because the flag was omitted.") + help="listings (default), offer (one offer, by its " + "endpoint URL) or facets (the filter taxonomy).") + p.add_argument("--category", default=None, help="Label for the run in the .meta.json sidecar. It does not add a column to the rows.") p.add_argument("--format", choices=["json", "csv", "both"], default="both") p.add_argument("--out", default="justjoin_jobs_scraperapi", help="Output file prefix") @@ -425,7 +432,7 @@ def parse_args(): "on protected sites — a DOM/load wait is satisfied instantly by " "the challenge page itself.") wait.add_argument("--wait-element", default=None, - help="Wait until this CSS selector is visible, e.g. 'a[href*=\"-item-\"]'") + help="Wait until this CSS selector is visible, e.g. 'a[href^=\"/job-offer/\"]'") wait.add_argument("--wait-state", choices=["load", "domcontentloaded"], default=None, help="Wait for a page load state instead of specific content") p.add_argument("--allow-empty", action="store_true", @@ -447,12 +454,45 @@ def parse_args(): "JUSTJOIN_CDP_ENDPOINT": "cdp_url", "JUSTJOIN_URL": "url", }) + if not args.url and args.mode == "facets": + args.url = facets_api_url() if not args.url: p.error("no --url given, and JUSTJOIN_URL is not set in the environment " "or in .env.") + problem = _args_problem(args) + if problem: + # Before any request: every attempt below is a billed task. + p.error(problem) return args +def _args_problem(args) -> Optional[str]: + """Why this request cannot be honoured, or None. Sets `args.route`.""" + ok, reason = is_supported_url(args.url) + if not ok: + return reason + if not 1 <= args.timeout <= MAX_API_TIMEOUT: + return "--timeout must be 1-%d seconds (got %d)." % ( + MAX_API_TIMEOUT, args.timeout) + if args.retries < 0: + return "--retries cannot be negative." + route = route_of(args.url) + args.route = DEFAULT_ROUTE + if route == "offer": + slug = slug_from_url(args.url) + hint = detail_api_url(slug) if slug else None + return ("%s is an offer PAGE, and this client has no parser for it " + "(only the endpoint's JSON). Use %s with --mode offer." + % (args.url, hint or "/api/candidate-api/offers/{slug}")) + if route == "listing": + if args.mode != "listings": + return ("%s is a listing page, which --mode %s does not read. " + "Use --mode listings, or the endpoint URL." + % (args.url, args.mode)) + args.route = "ssr" + return None + + if __name__ == "__main__": try: sys.exit(main()) diff --git a/selenium_scraper.py b/selenium_scraper.py index 5db63c8..1e09e93 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -43,7 +43,9 @@ from captcha_solver import (detect_recaptcha_v3, detect_recaptcha_in_page, reconcile_detections, solve_recaptcha, + solve_attempts, CaptchaUnsolvable, INJECT_TOKEN_JS, + SUBMIT_RECAPTCHA_FORM_JS, RECAPTCHA_DISCOVERY_JS, detect_turnstile, wait_for_turnstile, TURNSTILE_INTERCEPT_JS, TURNSTILE_INJECT_JS, turnstile_task_for) @@ -66,7 +68,7 @@ import page_flow from page_flow import MIN_CARD_MATCHES from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, - ProxyError, split_credentials) + ProxyError, split_credentials, verify_credentials) import env_config logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") @@ -125,6 +127,9 @@ class PageOutcome: products: List = field(default_factory=list) blocked_by: Optional[str] = None load_failed: bool = False + # Chromium's own name for a proxy fault when that, not a timeout, ended + # the load (ERR_PROXY_CONNECTION_FAILED, ...). Lets the sidecar say so. + proxy_failed: Optional[str] = None state: Optional[str] = None # What this response said about itself. # @@ -292,15 +297,25 @@ def _apply_timeouts(self): self.driver.set_script_timeout(SCRIPT_TIMEOUT) def _apply_fingerprint(self): - from fingerprint_client import get_fingerprint, playwright_init_script + from fingerprint_client import (cdp_emulation, get_fingerprint, + fingerprint_user_agent, + playwright_init_script) fp = get_fingerprint(self.args.twocaptcha_key, tags=self.args.fp_tags, country=self.args.fp_country) - ua = (fp.get("userAgent") or {}).get("value") + ua = fingerprint_user_agent(fp) script = playwright_init_script(fp) + emulation = cdp_emulation(fp) try: if ua: self.driver.execute_cdp_cmd("Network.setUserAgentOverride", {"userAgent": ua}) + if "metrics" in emulation: + self.driver.execute_cdp_cmd( + "Emulation.setDeviceMetricsOverride", emulation["metrics"]) + if "timezone" in emulation: + self.driver.execute_cdp_cmd( + "Emulation.setTimezoneOverride", + {"timezoneId": emulation["timezone"]}) # The same patch script the Playwright engine installs on its # context. Shared deliberately: two engines applying different # halves of one fingerprint would be a contradiction of exactly @@ -375,6 +390,35 @@ def current_url(): "current_url": current_url} +def _markup_view(session, url: str, state: str, body: str): + """(state, body) re-read from the document's MARKUP when that can tell more. + + On the endpoint route the body is `innerText`, which keeps neither the + `<title>` nor the `<script>` tags a challenge page is recognised by, and + which was also what a refusal saved to `*_debug.html` (41 bytes of text + for a page the message says to inspect as HTML). So for a state the + policy cannot use, or one it already counts as blocked, look at the + serialised document: a challenge marker there turns `unknown` into + `blocked` (exit 3, not 4/5), and a blocked page is saved with its markup. + """ + if not _is_endpoint(url): + return state, body + if not (state in ("unknown", "parse_error") + or page_flow.counts_as_blocked(state)): + return state, body + try: + markup = session.driver.page_source or "" + except WebDriverException: + return state, body + if not markup: + return state, body + if state != "blocked" and detect_bot_challenge(markup): + state = "blocked" + if page_flow.counts_as_blocked(state): + body = markup + return state, body + + def _is_endpoint(url: str) -> bool: """Whether this address answers with raw JSON rather than a document. @@ -408,6 +452,15 @@ def _snapshot(session, url: str): return _driver(session)["content"]() +def _fetch_sitemap(session, url: str, timeout_ms: int = 60000) -> str: + """`_fetch_page_source` for the sitemap, which raises on a refusal. + + Selenium exposes no HTTP status, so only a challenge page is recognised. + """ + text = _fetch_page_source(session, url, timeout_ms) + page_flow.raise_if_sitemap_blocked(None, text) + return text + def _fetch_page_source(session, url: str, timeout_ms: int = 60000) -> str: """Navigate and return the serialised document. Used for enumeration. @@ -500,13 +553,19 @@ def handle_captcha_if_present(session, args) -> bool: logger.error("Solving the challenge failed (%s).", e) return False try: - driver.execute_script(f"return ({INJECT_TOKEN_JS})(arguments[0]);", token) + called = driver.execute_script( + f"return ({INJECT_TOKEN_JS})(arguments[0]);", token) + submitted = False if called else bool(driver.execute_script( + f"return ({SUBMIT_RECAPTCHA_FORM_JS})();")) except WebDriverException as e: logger.error("Could not inject the token (%s).", e) return False - logger.info("Token injected. Reloading page to continue.") + # No reload: it discards the token, which cost money to get. + logger.info("Token injected%s; the page is left as it is, because a " + "reload would drop the token.", + " and handed to the page's callback" if called + else (" and its form submitted" if submitted else "")) time.sleep(1.5) - driver.refresh() return True @@ -610,8 +669,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # Counted, because it can BUY. See _solve_budget. if _solve_budget(args, solves_bought): - solves_bought += 1 - if handle_captcha_if_present(session, args): + attempts_before = solve_attempts() + solved = handle_captcha_if_present(session, args) + solves_bought += solve_attempts() - attempts_before + if solved: time.sleep(1) elif solves_bought: @@ -645,6 +706,8 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome state = page_flow.classify(html, http_status, d["current_url"](), args.mode, args.route) + state, body = _markup_view(session, url, state, body) + # The paid path is reached for any state page_flow.STATE_POLICY marks # solvable — on this site that is `blocked`; there is no separate # `challenge` state (see page_flow's docstring). Bounded by @@ -653,8 +716,10 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome # paying for one the API will reject. if (page_flow.should_solve(state) and _solve_budget(args, solves_bought)): - solves_bought += 1 - if handle_captcha_if_present(session, args): + attempts_before = solve_attempts() + solved = handle_captcha_if_present(session, args) + solves_bought += solve_attempts() - attempts_before + if solved: time.sleep(1) html = d["content"]() or html state = page_flow.classify(html, http_status, d["current_url"](), @@ -727,6 +792,7 @@ def _fetch_one_page(session, args, pool, page_num: int, url: str) -> PageOutcome logger.error("Gave up loading %s after %d attempt(s).", url, args.retries) outcome.load_failed = True + outcome.proxy_failed = exit_failed outcome.blocked_by = None return outcome @@ -935,6 +1001,8 @@ def scrape(args) -> int: "remote browser has its own exit, and layering a second " "proxy on top would contradict it.") pool = None + if pool: + verify_credentials(pool.current) if args.concurrency > 1: logger.warning("--concurrency is ignored in this engine: parallel page " "fetching is implemented in playwright_scraper.py, " @@ -952,8 +1020,17 @@ def scrape(args) -> int: # content decides how many pages there are, so the order reverses. offer_slugs = [] if args.mode == "offer" and not is_single_offer_url(args): - offer_slugs = page_flow.enumerate_offers( - lambda u: _fetch_page_source(session, u), args) + offer_slugs, sitemap_blocked, _tried = ( + page_flow.enumerate_offers_resilient( + lambda u: _fetch_sitemap(session, u), args, pool, + session.relaunch)) + if sitemap_blocked: + return finish_run( + [], args.out, args.format, args.allow_empty, + blocked=True, stop_reason="blocked_sitemap", + pages_requested=args.pages, pages_completed=0, + mode=args.mode, source=SOURCE_DEFAULT, + start_url=args.url or "", final_url="") if not offer_slugs: return finish_run( [], args.out, args.format, args.allow_empty, @@ -1003,7 +1080,8 @@ def scrape(args) -> int: independent = page_flow.pages_are_independent(args.mode, offer_slugs) first_verdict = page_flow.queue_decision(first.state, None, independent) if not first.ok: - stop_reason = ("page_load_timeout" if first.load_failed + stop_reason = (("proxy_unreachable" if first.proxy_failed + else "page_load_timeout") if first.load_failed else f"blocked_{first.blocked_by}") blocked = first.blocked_by is not None elif first_verdict: @@ -1025,7 +1103,8 @@ def scrape(args) -> int: planned = page_flow.plan_page_urls(args, page_one, first.pages_available, offer_slugs=offer_slugs) if len(planned) + 1 < args.pages: - stop_reason = "page_cap_reached" + stop_reason = ("queue_exhausted" if args.mode == "offer" + else "page_cap_reached") for index, url in enumerate(planned): page_num = index + 2 @@ -1033,10 +1112,17 @@ def scrape(args) -> int: pool.advance(f"per-page rotation, page {page_num}") session.relaunch() - outcome = _fetch_one_page(session, args, pool, page_num, url) + try: + outcome = _fetch_one_page(session, args, pool, page_num, url) + except KeyboardInterrupt: + logger.warning("Interrupted during page %d — writing the %d " + "page(s) already fetched.", page_num, len(outcomes)) + stop_reason = "interrupted" + break outcomes.append(outcome) if not outcome.ok: - stop_reason = ("page_load_timeout" if outcome.load_failed + stop_reason = (("proxy_unreachable" if outcome.proxy_failed + else "page_load_timeout") if outcome.load_failed else f"blocked_{outcome.blocked_by}") blocked = outcome.blocked_by is not None break @@ -1059,7 +1145,11 @@ def scrape(args) -> int: break if index + 1 < len(planned): - time.sleep(args.delay) + if not page_flow.sleep_or_interrupt(args.delay): + logger.warning("Interrupted — writing the %d page(s) already " + "fetched.", len(outcomes) + 1) + stop_reason = "interrupted" + break finally: if session is not None: session.close() @@ -1249,7 +1339,7 @@ def parse_args(): help="Which route --mode listings reads. api (default): " "justjoin.it's offers endpoint — every page, every " "filter, up to 1,000 rows a request. ssr: the " - "rendered listing page, ONE page of 100 rows with a " + "rendered listing page, ONE page of about 50 rows with a " "poorer schema (no category, no languages, no skill " "levels), and the only route justjoin.it's " "robots.txt permits — it carries Disallow: /api/. " @@ -1385,8 +1475,8 @@ def parse_args(): p.add_argument("--fp-tags", default="Windows", help="ONE OS-family tag for the fingerprint filter: " "Windows, Microsoft Windows or Android. NOT a list — " - "Chrome, Desktop and Mobile are each rejected by the " - "API with 400, and no combination is accepted. Use " + "Chrome, Desktop and Mobile were each accepted by the " + "API on 2026-11-12; combinations are not tried. Use " "--fp-country to narrow further. (default: Windows)") p.add_argument("--fp-country", default=None, help="Fingerprint country, ISO 3166-1 alpha-2. Match it to " @@ -1459,7 +1549,7 @@ def parse_args(): "listing serves ONE page and answers ?page=N with " "page 1 again, so planning more would re-collect page " "1 and report a complete multi-page run. Use --route " - "api (the default) for more than 100 rows. The run " + "api (the default) for more than 50 rows. The run " "status will say single_page_route.", args.pages) args.pages = 1 @@ -1494,6 +1584,11 @@ def parse_args(): "remote browser supplies its own.") try: sys.exit(scrape(args)) + except KeyboardInterrupt: + # Reached only when nothing was written yet: once a page has been + # fetched the loops end the run as `interrupted` and write it. + logger.error("Interrupted before any page was fetched; nothing written.") + sys.exit(130) except ProxyError as e: logger.error("%s", e) sys.exit(2) diff --git a/smoke_test.py b/smoke_test.py index 558977d..ee3aa90 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -2255,8 +2255,10 @@ def check_category_is_a_filter_and_not_the_run_label(): equal("and is read back out", pp.category_from_url(url), "devops") check("the site's own keys are offered in the help text", "devops" in pp.CATEGORY_KEYS and "python" in pp.CATEGORY_KEYS) - equal("25 of them were published when this was measured", - len(pp.CATEGORY_KEYS), 25) + equal("24 are known: 25 were published on 2026-09-18 and `html` was gone " + "on 2026-11-12", len(pp.CATEGORY_KEYS), 24) + check("a key the site dropped is not offered", + "html" not in pp.CATEGORY_KEYS) def check_the_ordering_vocabulary_is_the_request_not_the_payload(): @@ -3601,7 +3603,7 @@ def check_every_solve_is_counted_against_the_budget(): if not os.path.exists(path): continue source = open(path, encoding="utf-8").read() - calls = source.count("if handle_captcha_if_present(") + calls = source.count("= handle_captcha_if_present(") check("%s calls the solver from two places, as designed" % module, calls == 2, "found %d call site(s)" % calls) # Both must sit under a budget test. Counting the guard is the cheap @@ -3610,15 +3612,43 @@ def check_every_solve_is_counted_against_the_budget(): check("%s gates BOTH solve call sites on the budget" % module, guards == calls, "%d budget guard(s) for %d call site(s)" % (guards, calls)) - check("%s increments the counter beside each guard" % module, - source.count("solves_bought += 1") == calls, - "%d increment(s) for %d call site(s)" - % (source.count("solves_bought += 1"), calls)) + # The counter moves by purchases actually STARTED, read off the + # solver, not by "a check was made": a call that finds nothing to + # solve buys nothing and must not spend the page's one allowance. + bumps = source.count("solves_bought += solve_attempts() - attempts_before") + check("%s counts purchases beside each guard" % module, + bumps == calls, + "%d purchase count(s) for %d call site(s)" % (bumps, calls)) + check("%s no longer counts a call that bought nothing" % module, + "solves_bought += 1" not in source) # And the constant must still be the one thing that decides it. import page_flow equal("at most one purchase per page", page_flow.SOLVES_PER_PAGE, 1) +def check_solve_attempts_counts_only_started_purchases(): + """`solve_attempts()` moves when a paid task is created and not when a + solve is refused before one (no key, Turnstile on the v1 API).""" + import captcha_solver as cs + before = cs.solve_attempts() + ch = cs.CaptchaChallenge(kind="recaptcha_v2", sitekey="k", + page_url="https://justjoin.it/") + try: + cs.solve_recaptcha(ch, None) + except RuntimeError: + pass + equal("a refusal for want of a key buys nothing", + cs.solve_attempts(), before) + real = cs._solve_with_2captcha_v2 + cs._solve_with_2captcha_v2 = lambda *a, **k: "token" + try: + cs.solve_recaptcha(ch, "key") + finally: + cs._solve_with_2captcha_v2 = real + equal("a task handed to the solver is counted once", + cs.solve_attempts(), before + 1) + + def check_a_dead_proxy_is_reported_as_a_proxy_failure(): """A proxy failure is not a timeout, and the two want opposite responses — another try at the same exit versus a different one. @@ -4115,6 +4145,572 @@ def emit(self, record): +# --------------------------------------------------------------------------- +# Audit of 2026-11-16 (manual-test report on v0.1.4, 67be0b3) +# --------------------------------------------------------------------------- +# +# Each check below fails on the code as it stood at that commit. + +def _argv_exit(engine, argv): + """The exit code `engine.parse_args` ends with for `argv`, or None.""" + saved = sys.argv[:] + sys.argv = [engine.__name__ + ".py"] + argv + try: + with __import__("contextlib").redirect_stderr(io.StringIO()): + engine.parse_args() + return None + except SystemExit as e: + return e.code + finally: + sys.argv = saved + + +def check_audit_arguments_are_refused_before_any_request(): + ran = 0 + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + ran += 1 + cases = { + "--pages 0": ["--pages", "0"], + "--pages -1": ["--pages", "-1"], + "--out into a directory that does not exist": + ["--out", "/nonexistent-dir-xyz/out"], + "a listing URL with a filter this scraper cannot apply": + ["--url", "https://justjoin.it/job-offers/all-locations/" + "python?remote=yes"], + "an experience-level the endpoint does not know": + ["--url", "https://justjoin.it/job-offers/all-locations/" + "python?experience-level=wizard"], + "--slugs-file that is not there": + ["--mode", "offer", "--slugs-file", "/nonexistent-xyz.txt"], + } + for label, argv in cases.items(): + equal("%s: %s is exit 2" % (module, label), + _argv_exit(engine, argv), 2) + equal("%s: a supported listing URL still parses" % module, + _argv_exit(engine, [ + "--url", "https://justjoin.it/job-offers/all-locations/" + "python?experience-level=senior"]), None) + if ran == 0: + skip("audit arguments", "no engine library installed here") + + +def check_audit_listing_url_filters_reach_the_endpoint(): + f = pp.filters_from_listing_url( + "https://justjoin.it/job-offers/all-locations/python" + "?experience-level=senior,mid&employment-type=b2b&keyword=rust") + equal("experience-level", list(f.experience), ["senior", "mid"]) + equal("employment-type", list(f.employment), ["b2b"]) + equal("the endpoint parameter", f.as_params()["experienceLevels"], + ["senior", "mid"]) + equal("an explicit flag beats the URL", list(pp.filters_from_args( + types_ns(url="https://justjoin.it/job-offers/all-locations/python" + "?experience-level=senior", experience=["junior"]) + ).experience), ["junior"]) + check("a parameter nothing translates is named", + "remote" in (pp.listing_url_problem( + "https://justjoin.it/job-offers/all-locations?remote=yes") + or "")) + + +def types_ns(**kw): + base = dict(url=None, category=None, city=None, city_radius=None, + remote=False, keyword=None, experience=None, language=None, + employment=None, working_time=None, with_salary=False) + base.update(kw) + return _types.SimpleNamespace(**base) + + +def check_audit_csv_dash_and_allow_empty_status(): + import output_writer as ow + equal("a lone dash is not a formula", ow._csv_cell("-"), ("-", False)) + equal("-abc still is", ow._csv_cell("-abc"), ("'-abc", True)) + equal("=1+1 still is", ow._csv_cell("=1+1"), ("'=1+1", True)) + tmp = tempfile.mkdtemp(prefix="jj_ae_") + out = os.path.join(tmp, "ae") + with __import__("contextlib").redirect_stdout(io.StringIO()): + code = ow.finish_run([], out, "json", True, blocked=False, + stop_reason="completed", pages_requested=1, + pages_completed=1, start_url="u", final_url="u") + meta = json.load(open(out + ".meta.json", encoding="utf-8")) + equal("--allow-empty: the exit stays 4", code, 4) + equal("--allow-empty: a finished empty run is not `failed`", + meta["status"], "complete") + with __import__("contextlib").redirect_stdout(io.StringIO()): + ow.finish_run([], out + "2", "json", True, blocked=False, + stop_reason="page_load_timeout", pages_requested=1, + pages_completed=0, start_url="u", final_url="u") + meta = json.load(open(out + "2.meta.json", encoding="utf-8")) + equal("...but an unfinished one still is", meta["status"], "failed") + + +def check_audit_an_empty_flag_beats_the_environment(): + import env_config + saved = os.environ.get("JUSTJOIN_PROXY") + os.environ["JUSTJOIN_PROXY"] = "http://u:p@host.example:1" + try: + a = _types.SimpleNamespace(proxy="") + env_config.apply(a, keys={"JUSTJOIN_PROXY": "proxy"}, quiet=True) + equal("--proxy \"\" stays empty", a.proxy, "") + b = _types.SimpleNamespace(proxy=None) + env_config.apply(b, keys={"JUSTJOIN_PROXY": "proxy"}, quiet=True) + equal("an unset flag still takes the variable", b.proxy, + "http://u:p@host.example:1") + finally: + if saved is None: + os.environ.pop("JUSTJOIN_PROXY", None) + else: + os.environ["JUSTJOIN_PROXY"] = saved + + +def check_audit_ctrl_c_keeps_the_pages_already_fetched(): + ran = 0 + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + ran += 1 + calls = [] + + def outcome_for(url, calls=calls): + calls.append(url) + if len(calls) == 3: + raise KeyboardInterrupt + return "content", ["p%d" % len(calls)] + code, fetched, meta = _drive_scrape( + engine, ["--category", "python", "--pages", "5", + "--per-page", "1"], outcome_for) + equal("%s: Ctrl+C on page 3 is exit 6" % module, code, 6) + check("%s: a sidecar was written" % module, meta is not None) + if meta is not None: + equal("%s: stop_reason" % module, meta["stop_reason"], + "interrupted") + equal("%s: status" % module, meta["status"], "partial") + equal("%s: the two fetched pages are kept" % module, + meta["products"], 2) + if ran == 0: + skip("ctrl-c", "no engine library installed here") + + +def check_audit_a_finished_offer_queue_is_not_a_page_cap(): + ran = 0 + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + ran += 1 + path = _offer_queue_file(["a", "b", "c"]) + try: + code, fetched, meta = _drive_scrape( + engine, ["--mode", "offer", "--slugs-file", path, + "--pages", "10"], + lambda url: ("content", [url.rsplit("/", 1)[-1]])) + finally: + os.unlink(path) + equal("%s: exit 0" % module, code, 0) + equal("%s: the queue ended, the cap did not" % module, + (meta or {}).get("stop_reason"), "queue_exhausted") + equal("%s: and the run is complete" % module, + (meta or {}).get("status"), "complete") + if ran == 0: + skip("queue exhausted", "no engine library installed here") + + +def check_audit_diff_runs_refuses_what_it_cannot_compare(): + import diff_runs + tmp = tempfile.mkdtemp(prefix="jj_diff_") + + def write(name, route, share=100, mode="listings"): + base = os.path.join(tmp, name) + json.dump([], open(base + ".json", "w")) + json.dump({"status": "complete", "mode": mode, "route": route, + "sort": "published", "order": "descending", + "filters": {}, "share_of_query_pct": share}, + open(base + ".meta.json", "w")) + return base + ".json" + api = write("api", "api") + ssr = write("ssr", "ssr") + sapi = write("sapi", "scraper_api") + part = write("part", "api", share=57.4) + ns = lambda o, n: _types.SimpleNamespace(old=o, new=n, force=False) + buf = io.StringIO() + with __import__("contextlib").redirect_stdout(buf): + refused = diff_runs._check_comparable(ns(api, ssr)) + check("api against ssr is refused", refused is False) + check("...and the refusal names the routes", "routes" in buf.getvalue()) + with __import__("contextlib").redirect_stdout(io.StringIO()): + check("api against scraper_api stays comparable", + diff_runs._check_comparable(ns(api, sapi)) is True) + buf = io.StringIO() + with __import__("contextlib").redirect_stdout(buf): + diff_runs._check_comparable(ns(part, part)) + check("a partial-share listing run is flagged as a slice", + "slice" in buf.getvalue(), buf.getvalue()[:200]) + saved = sys.argv[:] + sys.argv = ["diff_runs.py", "--old", os.path.join(tmp, "nofile.json"), + "--new", api] + buf = io.StringIO() + try: + with __import__("contextlib").redirect_stdout(buf): + code = diff_runs.main() + finally: + sys.argv = saved + equal("a missing file is exit 2", code, 2) + check("...and says so rather than blaming the sidecar", + "no such file" in buf.getvalue().lower(), buf.getvalue()) + + +def check_audit_fingerprint_reaches_every_engine(): + import fingerprint_client as fc + fp = {"userAgent": {"userAgent": "UA/154"}, + "navigator": {"platform": "Win32", "hardwareConcurrency": 16, + "deviceMemory": 32}, + "webgl": {"glParameters": {"37445": "Google Inc. (NVIDIA)", + "37446": "ANGLE (NVIDIA RTX)"}}, + "screen": {"width": 1920, "height": 1080, "outerWidth": 1920, + "outerHeight": 960, "deviceScaleFactor": 1.25}, + "intl": {"timeZone": "Europe/Warsaw"}} + script = fc.playwright_init_script(fp) + check("WebGL vendor is read from glParameters", + "Google Inc. (NVIDIA)" in script and "ANGLE (NVIDIA RTX)" in script) + emu = fc.cdp_emulation(fp) + equal("the timezone goes over CDP", emu.get("timezone"), "Europe/Warsaw") + equal("the screen goes over CDP", emu["metrics"]["screenWidth"], 1920) + for module in ("selenium_scraper", "puppeteer_scraper"): + path = os.path.join(HERE, module + ".py") + source = open(path, encoding="utf-8").read() + check("%s takes the user agent from fingerprint_user_agent" % module, + "fingerprint_user_agent(fp)" in source + and '(fp.get("userAgent") or {}).get("value")' not in source) + source = open(os.path.join(HERE, "puppeteer_scraper.py"), + encoding="utf-8").read() + check("puppeteer installs the init script over CDP, not through " + "evaluateOnNewDocument (which wraps it into a syntax error)", + "self.page.evaluateOnNewDocument(" not in source + and "Page.addScriptToEvaluateOnNewDocument" in source) + + +def check_audit_scraper_api_client_refuses_before_paying(): + import scraper_api_client as sac + ns = lambda **kw: _types.SimpleNamespace(**{**dict( + url="https://justjoin.it/api/candidate-api/offers", timeout=60, + retries=1, mode="listings"), **kw}) + check("a foreign host", sac._args_problem( + ns(url="https://example.com/")) is not None) + check("--timeout 0", sac._args_problem(ns(timeout=0)) is not None) + check("--timeout 121", sac._args_problem(ns(timeout=121)) is not None) + problem = sac._args_problem(ns( + url="https://justjoin.it/job-offer/some-offer-gdynia-python", + mode="offer")) + check("an offer page names its endpoint", + problem and "/api/candidate-api/offers/some-offer-gdynia-python" + in problem, problem) + a = ns(url="https://justjoin.it/job-offers/all-locations/python") + equal("a listing page is read as the rendered page", + (sac._args_problem(a), a.route), (None, "ssr")) + check("a listing page with --mode offer is refused", + sac._args_problem(ns(url=a.url, mode="offer")) is not None) + + +def check_audit_a_mistyped_proxy_is_shown_not_masked_away(): + import proxy_pool + try: + proxy_pool.parse_proxy_line("notaproxy", "--proxy") + except proxy_pool.ProxyError as e: + check("the entry is quoted", "notaproxy" in str(e), str(e)) + check("not as ?://?", "?://?" not in str(e), str(e)) + try: + proxy_pool.parse_proxy_line("login:secret@host.example:8080", "x") + except proxy_pool.ProxyError as e: + check("a credential without a scheme is not echoed", + "secret" not in str(e), str(e)) + + +def check_audit_dockerfile_installs_libatomic_first(): + text = open(os.path.join(HERE, "Dockerfile"), encoding="utf-8").read() + check("libatomic1 is installed", "libatomic1" in text) + check("...before pip install", + text.index("RUN apt-get") < text.index("RUN pip install")) + + +def check_audit_offer_queue_runs_concurrently(): + engine = _import_engine("playwright_scraper") + if engine is None: + return + import logging + records = [] + + class Catch(logging.Handler): + def emit(self, record): + records.append(record.getMessage()) + handler = Catch() + engine.logger.addHandler(handler) + path = _offer_queue_file(["a", "b", "c", "d", "e"]) + try: + code, fetched, meta = _drive_scrape( + engine, ["--mode", "offer", "--slugs-file", path, "--pages", "5", + "--concurrency", "3"], + lambda url: ("content", [url.rsplit("/", 1)[-1]])) + finally: + os.unlink(path) + engine.logger.removeHandler(handler) + check("it is not told its pages cannot be addressed", + not any("cannot be addressed" in m for m in records), records) + equal("every offer fetched", len(fetched), 5) + equal("exit 0", code, 0) + equal("all five written", (meta or {}).get("products"), 5) + + +def _proxy_with_login(scheme, user, password, hostport): + # Assembled at run time so no credential-shaped URL sits in the source. + return scheme + "://" + user + ":" + password + "@" + hostport + + +def check_audit_a_refused_proxy_login_is_named(): + """A proxy that answers 407 to CONNECT is reported as such, quickly.""" + import socket + import threading + import proxy_pool + + def serve(reply): + srv = socket.socket() + srv.bind(("127.0.0.1", 0)) + srv.listen(1) + + def run(): + try: + conn, _ = srv.accept() + conn.recv(1024) + conn.sendall(reply) + conn.close() + finally: + srv.close() + threading.Thread(target=run, daemon=True).start() + return srv.getsockname()[1] + port = serve(b"HTTP/1.1 407 Proxy Authentication Required\r\n\r\n") + try: + proxy_pool.verify_credentials( + _proxy_with_login("http", "u", "WRONGPW", "127.0.0.1:%d" % port), timeout=5) + check("a 407 raises", False) + except proxy_pool.ProxyError as e: + text = str(e) + check("the 407 is named", "407" in text, text) + check("the password is not in the message", "WRONGPW" not in text, + text) + port = serve(b"HTTP/1.1 200 Connection established\r\n\r\n") + proxy_pool.verify_credentials( + _proxy_with_login("http", "u", "p", "127.0.0.1:%d" % port), timeout=5) + check("a 200 passes", True) + proxy_pool.verify_credentials( + _proxy_with_login("http", "u", "p", "127.0.0.1:1"), timeout=2) + check("an unreachable proxy is left to the engine", True) + proxy_pool.verify_credentials( + _proxy_with_login("socks5", "u", "p", "127.0.0.1:1")) + check("a non-HTTP scheme is not probed", True) + for module in ENGINES: + source = open(os.path.join(HERE, module + ".py"), + encoding="utf-8").read() + check("%s checks the proxy login before launching" % module, + "verify_credentials(pool.current)" in source) + + +def check_audit_a_misspelt_category_is_flagged(): + import logging + records = [] + + class Catch(logging.Handler): + def emit(self, record): + records.append(record.getMessage()) + handler = Catch() + pp.log.addHandler(handler) + try: + args = _types.SimpleNamespace(mode="listings", category="pythonx", + retries=1, pages=1, url=None) + pp.args_problem(args) + args.category = "python" + pp.args_problem(args) + finally: + pp.log.removeHandler(handler) + check("pythonx is flagged with the near miss", + any("pythonx" in m and "python" in m for m in records), records) + equal("python is not flagged", len(records), 1) + + +def check_audit_pyppeteer_launch_failure_gets_a_hint(): + engine = _import_engine("puppeteer_scraper") + if engine is None: + return + source = open(os.path.join(HERE, "puppeteer_scraper.py"), + encoding="utf-8").read() + check("a BrowserError is caught at the entry point", + "except BrowserError as e:" in source) + check("and the message names --chromium-path", + "Pass --chromium-path" in source) + + +def check_audit_a_refused_sitemap_rotates_the_exit_and_ends_as_blocked(): + import proxy_pool + ns = _types.SimpleNamespace(slugs_file=None, proxy_block_retries=2) + + def run(blocked_urls): + pool = proxy_pool.ProxyPool(["http://127.0.0.1:1", "http://127.0.0.1:2", + "http://127.0.0.1:3"], rotate="per-run") + relaunches = [] + + def fetch(url): + if url.endswith(".xml") and pool.current in blocked_urls: + raise page_flow.SitemapBlocked("HTTP 403") + return SITEMAP_INDEX_XML if "active-jobs.xml" in url \ + else SITEMAP_PART_XML + out = page_flow.enumerate_offers_resilient( + fetch, ns, pool, lambda: relaunches.append(pool.current)) + return out, relaunches + (slugs, blocked, tried), relaunches = run({"http://127.0.0.1:1"}) + check("one refused exit of three: the slugs arrive", bool(slugs)) + equal("...from the second exit", (blocked, tried, len(relaunches)), + (False, 2, 1)) + (slugs, blocked, tried), relaunches = run( + {"http://127.0.0.1:1", "http://127.0.0.1:2", "http://127.0.0.1:3"}) + equal("every exit refused: blocked, nothing enumerated", + (slugs, blocked, tried), ([], True, 3)) + # The classifier itself. + for status, text, want in ((403, "", True), (429, "", True), + (200, "<title>Just a moment...", True), + (200, SITEMAP_PART_XML, False)): + try: + page_flow.raise_if_sitemap_blocked(status, text) + got = False + except page_flow.SitemapBlocked: + got = True + equal("status %s %r is %s" % (status, text[:20], + "refused" if want else "served"), + got, want) + # And through each engine's own loop: exit 3, not `enumeration_empty`. + ran = 0 + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + ran += 1 + real = engine._fetch_sitemap + + def refuse(session, url, timeout_ms=60000): + raise page_flow.SitemapBlocked("HTTP 403") + engine._fetch_sitemap = refuse + try: + code, fetched, meta = _drive_scrape( + engine, ["--mode", "offer", "--pages", "2"], + lambda url: ("content", ["x"])) + finally: + engine._fetch_sitemap = real + equal("%s: a refused sitemap is exit 3" % module, code, 3) + check("%s: and, with no rows, writes no sidecar over a good run" + % module, meta is None) + if ran == 0: + skip("blocked sitemap", "no engine library installed here") + + +def check_audit_a_challenge_page_on_the_endpoint_route_is_blocked(): + """The endpoint route reads `innerText`, which has no title and no + scripts. A challenge answered with HTTP 200 and no marker in its text was + `unknown` (exit 4/5), and a real refusal was saved as 41 bytes of text.""" + html = ("Just a moment..." + "Performing security verification") + check("the fixture is a recognised challenge", + pp.detect_bot_challenge(html) is not None) + api_url = pp.api_url(items=100) + ran = 0 + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + ran += 1 + if module == "selenium_scraper": + session = _types.SimpleNamespace( + driver=_types.SimpleNamespace(page_source=html)) + elif module == "puppeteer_scraper": + session = _types.SimpleNamespace( + bridge=_types.SimpleNamespace(run=lambda x: x), + page=_types.SimpleNamespace(content=lambda: html)) + else: + session = _types.SimpleNamespace( + page=_types.SimpleNamespace(content=lambda: html)) + text = "Performing security verification" + state, body = engine._markup_view(session, api_url, "unknown", text) + equal("%s: unknown + a marker in the markup is blocked" % module, + state, "blocked") + equal("%s: and the markup is what gets saved" % module, body, html) + state, body = engine._markup_view(session, api_url, "content", text) + equal("%s: a served page is left alone" % module, + (state, body), ("content", text)) + state, body = engine._markup_view( + session, "https://justjoin.it/job-offers/all-locations", + "unknown", text) + equal("%s: a rendered listing already has its markup" % module, + (state, body), ("unknown", text)) + if ran == 0: + skip("markup view", "no engine library installed here") + + +def check_audit_a_dead_proxy_is_named_in_the_outcome(): + engine = _import_engine("playwright_scraper") + if engine is None: + return + import proxy_pool + + class Page: + url = "about:blank" + + def goto(self, *a, **k): + raise engine.PWError("net::ERR_PROXY_CONNECTION_FAILED at x") + session = _types.SimpleNamespace(page=Page(), pool=None) + args = _types.SimpleNamespace( + retries=1, retry_delay=0, proxy_block_retries=0, pages=1, + mode="listings", route="api", out="unused", dump_html=None) + pool = proxy_pool.ProxyPool(["http://127.0.0.1:1"], rotate="per-run") + outcome = engine._fetch_one_page(session, args, pool, 1, + pp.api_url(items=100)) + check("the load failed", outcome.load_failed) + equal("and the proxy is what is named", outcome.proxy_failed, + "ERR_PROXY_CONNECTION_FAILED") + for module in ENGINES: + source = open(os.path.join(HERE, module + ".py"), + encoding="utf-8").read() + check("%s reports proxy_unreachable rather than a timeout" % module, + '"proxy_unreachable"' in source + and "outcome.proxy_failed = exit_failed" in source) + + +def check_audit_a_paid_recaptcha_token_is_not_discarded_by_a_reload(): + import captcha_solver as cs + check("the injector reports how many callbacks it called", + "return called;" in cs.INJECT_TOKEN_JS) + check("a form-submit helper exists", "requestSubmit" in + cs.SUBMIT_RECAPTCHA_FORM_JS) + for module in ENGINES: + engine = _import_engine(module) + if engine is None: + continue + src = inspect.getsource(engine.handle_captcha_if_present) + check("%s submits the form instead of reloading" % module, + "SUBMIT_RECAPTCHA_FORM_JS" in src, module) + if module != "playwright_scraper": + # Playwright keeps one reload, for Turnstile, whose cookie + # survives it; the other two engines have no Turnstile branch. + check("%s has no reload after the token is in" % module, + "reload(" not in src and "refresh(" not in src, module) + source = open(os.path.join(HERE, "puppeteer_scraper.py"), + encoding="utf-8").read() + check("puppeteer answers the proxy's 407 over Fetch, not " + "page.authenticate (needs a CDP method current Chromium lacks)", + "self.page.authenticate(" not in source + and "Fetch.authRequired" in source + and "_install_proxy_auth(" in source) + + CHECKS = [v for k, v in sorted(globals().items()) if k.startswith("check_")]