From 4e2693e68c588159268eff3fa632b3a35fcf6f7b Mon Sep 17 00:00:00 2001 From: Battleplus <3559424769@qq.com> Date: Thu, 20 Aug 2026 22:49:34 +0800 Subject: [PATCH 1/2] fix: clean up Playwright process on browser startup failure Resolves #2155 When Playwright starts successfully but Chromium launch fails (e.g. missing browser binary, sandbox issues), the Playwright driver subprocess was leaked because no cleanup ran on the exception path. Each failed attempt left one live `node cli.js run-driver` child process, growing the process count linearly with retries. Changes: - browser_manager.py: Extract browser launch into `_launch_browser()`, wrap call in `start()` with try/except that stops Playwright on failure - async_webcrawler.py: Wrap `crawler_strategy.__aenter__()` in `start()` with try/except that calls `__aexit__()` on failure Both layers ensure partial resources are cleaned up before the exception propagates to the caller. --- crawl4ai/async_webcrawler.py | 10 +++++++++- crawl4ai/browser_manager.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 8216d19bc..89640ac1f 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -180,7 +180,15 @@ async def start(self): Returns: AsyncWebCrawler: The initialized crawler instance """ - await self.crawler_strategy.__aenter__() + try: + await self.crawler_strategy.__aenter__() + except Exception: + # Ensure partial resources are cleaned up on failure + try: + await self.crawler_strategy.__aexit__(None, None, None) + except Exception: + pass + raise self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT") self.ready = True return self diff --git a/crawl4ai/browser_manager.py b/crawl4ai/browser_manager.py index d7b8009c8..fe7f10853 100644 --- a/crawl4ai/browser_manager.py +++ b/crawl4ai/browser_manager.py @@ -808,6 +808,21 @@ async def start(self): # Initialize playwright self.playwright = await async_playwright().start() + try: + await self._launch_browser() + except Exception: + # If browser launch fails, ensure Playwright process is cleaned up + # to avoid leaking the "node cli.js run-driver" subprocess. + if self.playwright is not None and not self._using_cached_cdp: + try: + await self.playwright.stop() + except Exception: + pass + self.playwright = None + raise + + async def _launch_browser(self): + """Launch or connect to the browser after Playwright is initialized.""" # ── Persistent context via Playwright's native API ────────────── # When use_persistent_context is set and we're not connecting to an # external CDP endpoint, use launch_persistent_context() instead of From 15a14c9bfa55b3f6c2a7f5c866c573edd3aafd16 Mon Sep 17 00:00:00 2001 From: Battleplus <121445871+Battleplus@users.noreply.github.com> Date: Sat, 22 Aug 2026 19:04:32 +0800 Subject: [PATCH 2/2] test: cover browser startup rollback Signed-off-by: Battleplus --- crawl4ai/async_webcrawler.py | 2514 +++++----- crawl4ai/browser_manager.py | 4222 ++++++++--------- tests/browser/test_startup_failure_cleanup.py | 52 + 3 files changed, 3420 insertions(+), 3368 deletions(-) create mode 100644 tests/browser/test_startup_failure_cleanup.py diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 89640ac1f..8ee4fbe8f 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -1,1257 +1,1257 @@ -from .__version__ import __version__ as crawl4ai_version -import os -import re -import sys -import time -from pathlib import Path -from typing import Optional, List -import json -import asyncio - -# from contextlib import nullcontext, asynccontextmanager -from contextlib import asynccontextmanager -from .models import ( - CrawlResult, - MarkdownGenerationResult, - DispatchResult, - ScrapingResult, - CrawlResultContainer, - RunManyReturn -) -from .async_database import async_db_manager -from .chunking_strategy import * # noqa: F403 -from .chunking_strategy import IdentityChunking -from .content_filter_strategy import * # noqa: F403 -from .extraction_strategy import * # noqa: F403 -from .extraction_strategy import NoExtractionStrategy -from .async_crawler_strategy import ( - AsyncCrawlerStrategy, - AsyncPlaywrightCrawlerStrategy, - AsyncCrawlResponse, -) -from .cache_context import CacheMode, CacheContext -from .markdown_generation_strategy import ( - DefaultMarkdownGenerator, - MarkdownGenerationStrategy, -) -from .deep_crawling import DeepCrawlDecorator -from .async_logger import AsyncLogger, AsyncLoggerBase -from .async_configs import BrowserConfig, CrawlerRunConfig, ProxyConfig, SeedingConfig, DomainMapperConfig -from .async_dispatcher import * # noqa: F403 -from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter -from .async_url_seeder import AsyncUrlSeeder -from .domain_mapper import DomainMapper - -from .utils import ( - sanitize_input_encode, - InvalidCSSSelectorError, - fast_format_html, - get_error_context, - RobotsParser, - preprocess_html_for_schema, - compute_head_fingerprint, -) -from .cache_validator import CacheValidator, CacheValidationResult -from .antibot_detector import is_blocked - - -class AsyncWebCrawler: - """ - Asynchronous web crawler with flexible caching capabilities. - - There are two ways to use the crawler: - - 1. Using context manager (recommended for simple cases): - ```python - async with AsyncWebCrawler() as crawler: - result = await crawler.arun(url="https://example.com") - ``` - - 2. Using explicit lifecycle management (recommended for long-running applications): - ```python - crawler = AsyncWebCrawler() - await crawler.start() - - # Use the crawler multiple times - result1 = await crawler.arun(url="https://example.com") - result2 = await crawler.arun(url="https://another.com") - - await crawler.close() - ``` - - Attributes: - browser_config (BrowserConfig): Configuration object for browser settings. - crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages. - logger (AsyncLogger): Logger instance for recording events and errors. - crawl4ai_folder (str): Directory for storing cache. - base_directory (str): Base directory for storing cache. - ready (bool): Whether the crawler is ready for use. - - Methods: - start(): Start the crawler explicitly without using context manager. - close(): Close the crawler explicitly without using context manager. - arun(): Run the crawler for a single source: URL (web, local file, or raw HTML). - awarmup(): Perform warmup sequence. - arun_many(): Run the crawler for multiple sources. - aprocess_html(): Process HTML content. - - Typical Usage: - async with AsyncWebCrawler() as crawler: - result = await crawler.arun(url="https://example.com") - print(result.markdown) - - Using configuration: - browser_config = BrowserConfig(browser_type="chromium", headless=True) - async with AsyncWebCrawler(config=browser_config) as crawler: - crawler_config = CrawlerRunConfig( - cache_mode=CacheMode.BYPASS - ) - result = await crawler.arun(url="https://example.com", config=crawler_config) - print(result.markdown) - """ - - _domain_last_hit = {} - - def __init__( - self, - crawler_strategy: AsyncCrawlerStrategy = None, - config: BrowserConfig = None, - base_directory: str = str( - os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())), - thread_safe: bool = False, - logger: AsyncLoggerBase = None, - **kwargs, - ): - """ - Initialize the AsyncWebCrawler. - - Args: - crawler_strategy: Strategy for crawling web pages. Default AsyncPlaywrightCrawlerStrategy - config: Configuration object for browser settings. Default BrowserConfig() - base_directory: Base directory for storing cache - thread_safe: Whether to use thread-safe operations - **kwargs: Additional arguments for backwards compatibility - """ - # Handle browser configuration - browser_config = config or BrowserConfig() - - self.browser_config = browser_config - - # Initialize logger first since other components may need it - self.logger = logger or AsyncLogger( - log_file=os.path.join(base_directory, ".crawl4ai", "crawler.log"), - verbose=self.browser_config.verbose, - tag_width=10, - ) - - # Initialize crawler strategy - params = {k: v for k, v in kwargs.items() if k in [ - "browser_config", "logger"]} - self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy( - browser_config=browser_config, - logger=self.logger, - **params, # Pass remaining kwargs for backwards compatibility - ) - - # Thread safety setup - self._lock = asyncio.Lock() if thread_safe else None - - # Initialize directories - self.crawl4ai_folder = os.path.join(base_directory, ".crawl4ai") - os.makedirs(self.crawl4ai_folder, exist_ok=True) - os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True) - - # Initialize robots parser - self.robots_parser = RobotsParser() - - self.ready = False - - # Decorate arun method with deep crawling capabilities - self._deep_handler = DeepCrawlDecorator(self) - self.arun = self._deep_handler(self.arun) - - self.url_seeder: Optional[AsyncUrlSeeder] = None - self._domain_mapper: Optional[DomainMapper] = None - - async def start(self): - """ - Start the crawler explicitly without using context manager. - This is equivalent to using 'async with' but gives more control over the lifecycle. - Returns: - AsyncWebCrawler: The initialized crawler instance - """ - try: - await self.crawler_strategy.__aenter__() - except Exception: - # Ensure partial resources are cleaned up on failure - try: - await self.crawler_strategy.__aexit__(None, None, None) - except Exception: - pass - raise - self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT") - self.ready = True - return self - - async def close(self): - """ - Close the crawler explicitly without using context manager. - This should be called when you're done with the crawler if you used start(). - - This method will: - 1. Clean up browser resources - 2. Close any open pages and contexts - """ - await self.crawler_strategy.__aexit__(None, None, None) - - async def __aenter__(self): - return await self.start() - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close() - - @asynccontextmanager - async def nullcontext(self): - """异步空上下文管理器""" - yield - - async def arun( - self, - url: str, - config: CrawlerRunConfig = None, - **kwargs, - ) -> CrawlResultContainer: - """ - Runs the crawler for a single source: URL (web, local file, or raw HTML). - - Migration Guide: - Old way (deprecated): - result = await crawler.arun( - url="https://example.com", - word_count_threshold=200, - screenshot=True, - ... - ) - - New way (recommended): - config = CrawlerRunConfig( - word_count_threshold=200, - screenshot=True, - ... - ) - result = await crawler.arun(url="https://example.com", config=config) - - Args: - url: The URL to crawl (http://, https://, file://, or raw:) - config: Configuration object controlling crawl behavior - [other parameters maintained for backwards compatibility] - - Returns: - CrawlResultContainer: A single-result container that proxies - attribute access to the underlying CrawlResult for backwards - compatibility (e.g. result.markdown, result.html). - """ - # Auto-start if not ready - if not self.ready: - await self.start() - - config = config or CrawlerRunConfig() - if not isinstance(url, str) or not url: - raise ValueError( - "Invalid URL, make sure the URL is a non-empty string") - - async with self._lock or self.nullcontext(): - try: - self.logger.verbose = config.verbose - - # Default to ENABLED if no cache mode specified - if config.cache_mode is None: - config.cache_mode = CacheMode.ENABLED - - # Create cache context - cache_context = CacheContext(url, config.cache_mode, False) - - # Initialize processing variables - async_response: AsyncCrawlResponse = None - cached_result: CrawlResult = None - screenshot_data = None - pdf_data = None - extracted_content = None - start_time = time.perf_counter() - - # Try to get cached result if appropriate - if cache_context.should_read(): - cached_result = await async_db_manager.aget_cached_url(url) - - # Smart Cache: Validate cache freshness if enabled - if cached_result and config.check_cache_freshness: - cache_metadata = await async_db_manager.aget_cache_metadata(url) - if cache_metadata: - async with CacheValidator(timeout=config.cache_validation_timeout) as validator: - validation = await validator.validate( - url=url, - stored_etag=cache_metadata.get("etag"), - stored_last_modified=cache_metadata.get("last_modified"), - stored_head_fingerprint=cache_metadata.get("head_fingerprint"), - ) - - if validation.status == CacheValidationResult.FRESH: - cached_result.cache_status = "hit_validated" - self.logger.info( - message="Cache validated: {reason}", - tag="CACHE", - params={"reason": validation.reason} - ) - # Update metadata if we got new values - if validation.new_etag or validation.new_last_modified: - await async_db_manager.aupdate_cache_metadata( - url=url, - etag=validation.new_etag, - last_modified=validation.new_last_modified, - head_fingerprint=validation.new_head_fingerprint, - ) - elif validation.status == CacheValidationResult.ERROR: - cached_result.cache_status = "hit_fallback" - self.logger.warning( - message="Cache validation failed, using cached: {reason}", - tag="CACHE", - params={"reason": validation.reason} - ) - else: - # STALE or UNKNOWN - force recrawl - self.logger.info( - message="Cache stale: {reason}", - tag="CACHE", - params={"reason": validation.reason} - ) - cached_result = None - elif cached_result: - cached_result.cache_status = "hit" - - if cached_result: - html = sanitize_input_encode(cached_result.html) - extracted_content = sanitize_input_encode( - cached_result.extracted_content or "" - ) - extracted_content = ( - None - if not extracted_content or extracted_content == "[]" - else extracted_content - ) - # If screenshot is requested but its not in cache, then set cache_result to None - screenshot_data = cached_result.screenshot - pdf_data = cached_result.pdf - # if config.screenshot and not screenshot or config.pdf and not pdf: - if config.screenshot and not screenshot_data: - cached_result = None - - if config.pdf and not pdf_data: - cached_result = None - - self.logger.url_status( - url=cache_context.display_url, - success=bool(html), - timing=time.perf_counter() - start_time, - tag="FETCH", - ) - - # Update proxy configuration from rotation strategy if available - if config and config.proxy_rotation_strategy: - # Handle sticky sessions - use same proxy for all requests with same session_id - if config.proxy_session_id: - next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_proxy_for_session( - config.proxy_session_id, - ttl=config.proxy_session_ttl - ) - if next_proxy: - self.logger.info( - message="Using sticky proxy session: {session_id} -> {proxy}", - tag="PROXY", - params={ - "session_id": config.proxy_session_id, - "proxy": next_proxy.server - } - ) - config.proxy_config = next_proxy - else: - # Existing behavior: rotate on each request - next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_next_proxy() - if next_proxy: - self.logger.info( - message="Switch proxy: {proxy}", - tag="PROXY", - params={"proxy": next_proxy.server} - ) - config.proxy_config = next_proxy - - # Fetch fresh content if needed - if not cached_result or not html: - from urllib.parse import urlparse - - # Check robots.txt if enabled (once, before any attempts) - if config and config.check_robots_txt: - if not await self.robots_parser.can_fetch( - url, self.browser_config.user_agent - ): - return CrawlResult( - url=url, - html="", - success=False, - status_code=403, - error_message="Access denied by robots.txt", - response_headers={ - "X-Robots-Status": "Blocked by robots.txt" - }, - ) - - # --- Anti-bot retry setup --- - # raw: URLs contain caller-provided HTML (e.g. from cache), - # not content fetched from a web server. Anti-bot detection, - # proxy retries, and fallback fetching are meaningless here. - _is_raw_url = url.startswith("raw:") or url.startswith("raw://") - - _max_attempts = 1 + getattr(config, "max_retries", 0) - _proxy_list = config._get_proxy_list() - _original_proxy_config = config.proxy_config - _block_reason = "" - _done = False - crawl_result = None - _crawl_stats = { - "attempts": 0, - "retries": 0, - "proxies_used": [], - "fallback_fetch_used": False, - "resolved_by": None, - } - - for _attempt in range(_max_attempts): - if _done: - break - - if _attempt > 0: - _crawl_stats["retries"] = _attempt - self.logger.warning( - message="Anti-bot retry {attempt}/{max_retries} for {url} — {reason}", - tag="ANTIBOT", - params={ - "attempt": _attempt, - "max_retries": config.max_retries, - "url": url[:80], - "reason": _block_reason, - }, - ) - - for _p_idx, _proxy in enumerate(_proxy_list): - if _p_idx > 0 or _attempt > 0: - self.logger.info( - message="Trying proxy {idx}/{total}: {proxy}", - tag="ANTIBOT", - params={ - "idx": _p_idx + 1, - "total": len(_proxy_list), - "proxy": _proxy.server if _proxy else "direct", - }, - ) - - # Set the active proxy for this attempt - config.proxy_config = _proxy - _crawl_stats["attempts"] += 1 - - try: - t1 = time.perf_counter() - - if config.user_agent: - self.crawler_strategy.update_user_agent( - config.user_agent) - - async_response = await self.crawler_strategy.crawl( - url, config=config) - - html = sanitize_input_encode(async_response.html) - screenshot_data = async_response.screenshot - pdf_data = async_response.pdf_data - js_execution_result = async_response.js_execution_result - - self.logger.url_status( - url=cache_context.display_url, - success=bool(html), - timing=time.perf_counter() - t1, - tag="FETCH", - ) - - crawl_result = await self.aprocess_html( - url=url, html=html, - extracted_content=extracted_content, - config=config, - screenshot_data=screenshot_data, - pdf_data=pdf_data, - verbose=config.verbose, - is_raw_html=True if url.startswith("raw:") else False, - redirected_url=async_response.redirected_url, - original_scheme=urlparse(url).scheme, - **kwargs, - ) - - crawl_result.status_code = async_response.status_code - is_raw_url = url.startswith("raw:") or url.startswith("raw://") - crawl_result.redirected_url = async_response.redirected_url or (None if is_raw_url else url) - crawl_result.redirected_status_code = async_response.redirected_status_code - crawl_result.response_headers = async_response.response_headers - crawl_result.downloaded_files = async_response.downloaded_files - crawl_result.js_execution_result = js_execution_result - crawl_result.mhtml = async_response.mhtml_data - crawl_result.ssl_certificate = async_response.ssl_certificate - crawl_result.network_requests = async_response.network_requests - crawl_result.console_messages = async_response.console_messages - # Success when html is non-empty OR a binary - # download was retrieved (PDFs, archives etc. - # have empty html by design — file content is - # in downloaded_files). - crawl_result.success = bool(html) or bool(async_response.downloaded_files) - crawl_result.session_id = getattr(config, "session_id", None) - crawl_result.cache_status = "miss" - - # Check if blocked (skip for raw: URLs — - # caller-provided content, anti-bot N/A) - if _is_raw_url: - _blocked = False - _block_reason = "" - else: - _blocked, _block_reason = is_blocked( - async_response.status_code, html) - - _crawl_stats["proxies_used"].append({ - "proxy": _proxy.server if _proxy else None, - "status_code": async_response.status_code, - "blocked": _blocked, - "reason": _block_reason if _blocked else "", - }) - - if not _blocked: - _crawl_stats["resolved_by"] = "proxy" if _proxy else "direct" - _done = True - break # Success — exit proxy loop - - except Exception as _crawl_err: - _crawl_stats["proxies_used"].append({ - "proxy": _proxy.server if _proxy else None, - "status_code": None, - "blocked": True, - "reason": str(_crawl_err), - }) - self.logger.error_status( - url=url, - error=f"Proxy {_proxy.server if _proxy else 'direct'} failed: {_crawl_err}", - tag="ANTIBOT", - ) - _block_reason = str(_crawl_err) - # If this is the only proxy and only attempt, re-raise - # so the caller gets the real error (not a silent swallow). - # But if there are more proxies or retries to try, continue. - if len(_proxy_list) <= 1 and _max_attempts <= 1: - raise - - # Restore original proxy_config - config.proxy_config = _original_proxy_config - - # --- Fallback fetch function (last resort after all retries+proxies exhausted) --- - # Invoke fallback when: (a) crawl_result exists but is blocked, OR - # (b) crawl_result is None because all proxies threw exceptions (browser crash, timeout). - # Skip for raw: URLs — fallback expects a real URL, not raw HTML content. - _fallback_fn = getattr(config, "fallback_fetch_function", None) - if _fallback_fn and not _done and not _is_raw_url: - _needs_fallback = ( - crawl_result is None # All proxies threw exceptions - or is_blocked(crawl_result.status_code, crawl_result.html or "")[0] - ) - if _needs_fallback: - self.logger.warning( - message="All retries exhausted, invoking fallback_fetch_function for {url}", - tag="ANTIBOT", - params={"url": url[:80]}, - ) - _crawl_stats["fallback_fetch_used"] = True - try: - _fallback_html = await _fallback_fn(url) - if _fallback_html: - _sanitized_html = sanitize_input_encode(_fallback_html) - try: - crawl_result = await self.aprocess_html( - url=url, - html=_sanitized_html, - extracted_content=extracted_content, - config=config, - screenshot_data=None, - pdf_data=None, - verbose=config.verbose, - is_raw_html=True, - redirected_url=url, - original_scheme=urlparse(url).scheme, - **kwargs, - ) - except Exception as _proc_err: - # aprocess_html may fail if browser is dead (e.g., - # consent popup removal needs Page.evaluate). - # Fall back to a minimal result with raw HTML. - self.logger.warning( - message="Fallback HTML processing failed ({err}), using raw HTML", - tag="ANTIBOT", - params={"err": str(_proc_err)[:100]}, - ) - crawl_result = CrawlResult( - url=url, - html=_sanitized_html, - success=True, - status_code=200, - ) - crawl_result.success = True - crawl_result.status_code = 200 - crawl_result.session_id = getattr(config, "session_id", None) - crawl_result.cache_status = "miss" - _crawl_stats["resolved_by"] = "fallback_fetch" - except Exception as _fallback_err: - self.logger.error_status( - url=url, - error=f"Fallback fetch failed: {_fallback_err}", - tag="ANTIBOT", - ) - - # --- Mark blocked results as failed --- - # Skip re-check ONLY when fallback SUCCEEDED — the fallback result - # is authoritative and real pages may contain anti-bot script markers - # (e.g. PerimeterX JS on Walmart) that trigger false positives. - # When fallback was attempted but FAILED, we must still re-check - # because the result is from a blocked proxy attempt. - # Also skip for raw: URLs — caller-provided content, anti-bot N/A. - # Also skip for binary downloads (PDFs, archives, etc.) — content - # was delivered via downloaded_files, html is empty by design, - # and is_blocked() would misread "0 bytes html" as a block. - if crawl_result: - _fallback_succeeded = _crawl_stats.get("resolved_by") == "fallback_fetch" - # Skip the block check for binary downloads (PDFs, archives, - # etc.) — content was delivered via downloaded_files, html is - # empty by design, and is_blocked() would misread "0 bytes - # html" as a block. - _has_download = bool(getattr(crawl_result, "downloaded_files", None)) - if not _fallback_succeeded and not _is_raw_url and not _has_download: - _blocked, _block_reason = is_blocked( - crawl_result.status_code, crawl_result.html or "") - if _blocked: - crawl_result.success = False - crawl_result.error_message = f"Blocked by anti-bot protection: {_block_reason}" - crawl_result.crawl_stats = _crawl_stats - else: - # All proxies threw exceptions and fallback either wasn't - # configured or also failed. Build a minimal result so the - # caller gets crawl_stats instead of None. - crawl_result = CrawlResult( - url=url, - html="", - success=False, - status_code=None, - error_message=f"All proxies failed: {_block_reason}" if _block_reason else "All proxies failed", - ) - crawl_result.crawl_stats = _crawl_stats - - # Compute head fingerprint for cache validation - if crawl_result and crawl_result.html: - head_end = crawl_result.html.lower().find('') - if head_end != -1: - head_html = crawl_result.html[:head_end + 7] - crawl_result.head_fingerprint = compute_head_fingerprint(head_html) - - # Log failure reason before COMPLETE so users can see why it failed. - if crawl_result and not crawl_result.success and crawl_result.error_message: - self.logger.error_status( - url=cache_context.display_url, - error=crawl_result.error_message, - tag="ERROR", - ) - - self.logger.url_status( - url=cache_context.display_url, - success=crawl_result.success if crawl_result else False, - timing=time.perf_counter() - start_time, - tag="COMPLETE", - ) - - # Update cache if appropriate - if cache_context.should_write() and not bool(cached_result): - await async_db_manager.acache_url(crawl_result) - - return CrawlResultContainer(crawl_result) - - else: - self.logger.url_status( - url=cache_context.display_url, - success=True, - timing=time.perf_counter() - start_time, - tag="COMPLETE" - ) - # Same binary-download awareness as the live-fetch path - # — a cached PDF/archive should replay as success. - cached_result.success = bool(html) or bool(getattr(cached_result, "downloaded_files", None)) - cached_result.session_id = getattr( - config, "session_id", None) - # For raw: URLs, don't fall back to the raw HTML string as redirected_url - is_raw_url = url.startswith("raw:") or url.startswith("raw://") - cached_result.redirected_url = cached_result.redirected_url or (None if is_raw_url else url) - return CrawlResultContainer(cached_result) - - except Exception as e: - error_context = get_error_context(sys.exc_info()) - - error_message = ( - f"Unexpected error in _crawl_web at line {error_context['line_no']} " - f"in {error_context['function']} ({error_context['filename']}):\n" - f"Error: {str(e)}\n\n" - f"Code context:\n{error_context['code_context']}" - ) - - self.logger.error_status( - url=url, - error=error_message, - tag="ERROR", - ) - - return CrawlResultContainer( - CrawlResult( - url=url, html="", success=False, error_message=error_message - ) - ) - - async def aprocess_html( - self, - url: str, - html: str, - extracted_content: str, - config: CrawlerRunConfig, - screenshot_data: str, - pdf_data: str, - verbose: bool, - **kwargs, - ) -> CrawlResult: - """ - Process HTML content using the provided configuration. - - Args: - url: The URL being processed - html: Raw HTML content - extracted_content: Previously extracted content (if any) - config: Configuration object controlling processing behavior - screenshot_data: Screenshot data (if any) - pdf_data: PDF data (if any) - verbose: Whether to enable verbose logging - **kwargs: Additional parameters for backwards compatibility - - Returns: - CrawlResult: Processed result containing extracted and formatted content - """ - # === PREFETCH MODE SHORT-CIRCUIT === - if getattr(config, 'prefetch', False): - from .utils import quick_extract_links - - # Use base_url from config (for raw: URLs), redirected_url, or original url - effective_url = getattr(config, 'base_url', None) or kwargs.get('redirected_url') or url - links = quick_extract_links(html, effective_url) - - return CrawlResult( - url=url, - html=html, - success=True, - links=links, - status_code=kwargs.get('status_code'), - response_headers=kwargs.get('response_headers'), - redirected_url=kwargs.get('redirected_url'), - ssl_certificate=kwargs.get('ssl_certificate'), - # All other fields default to None - ) - # === END PREFETCH SHORT-CIRCUIT === - - cleaned_html = "" - try: - _url = url if not kwargs.get("is_raw_html", False) else "Raw HTML" - t1 = time.perf_counter() - - # Get scraping strategy and ensure it has a logger - scraping_strategy = config.scraping_strategy - if not scraping_strategy.logger: - scraping_strategy.logger = self.logger - - # Process HTML content - params = config.__dict__.copy() - params.pop("url", None) - # add keys from kwargs to params that doesn't exist in params - params.update({k: v for k, v in kwargs.items() - if k not in params.keys()}) - - ################################ - # Scraping Strategy Execution # - ################################ - result: ScrapingResult = scraping_strategy.scrap( - url, html, **params) - - if result is None: - raise ValueError( - f"Process HTML, Failed to extract content from the website: {url}" - ) - - except InvalidCSSSelectorError as e: - raise ValueError(str(e)) - except Exception as e: - raise ValueError( - f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}" - ) - - # Extract results - handle both dict and ScrapingResult - if isinstance(result, dict): - cleaned_html = sanitize_input_encode( - result.get("cleaned_html", "")) - media = result.get("media", {}) - tables = media.pop("tables", []) if isinstance(media, dict) else [] - links = result.get("links", {}) - metadata = result.get("metadata", {}) - else: - cleaned_html = sanitize_input_encode(result.cleaned_html) - # media = result.media.model_dump() - # tables = media.pop("tables", []) - # links = result.links.model_dump() - media = result.media.model_dump() if hasattr(result.media, 'model_dump') else result.media - tables = media.pop("tables", []) if isinstance(media, dict) else [] - links = result.links.model_dump() if hasattr(result.links, 'model_dump') else result.links - metadata = result.metadata - - fit_html = preprocess_html_for_schema(html_content=html, text_threshold= 500, max_size= 300_000) - - ################################ - # Generate Markdown # - ################################ - markdown_generator: Optional[MarkdownGenerationStrategy] = ( - config.markdown_generator or DefaultMarkdownGenerator() - ) - - # --- SELECT HTML SOURCE BASED ON CONTENT_SOURCE --- - # Get the desired source from the generator config, default to 'cleaned_html' - selected_html_source = getattr(markdown_generator, 'content_source', 'cleaned_html') - - # Define the source selection logic using dict dispatch - html_source_selector = { - "raw_html": lambda: html, # The original raw HTML - "cleaned_html": lambda: cleaned_html, # The HTML after scraping strategy - "fit_html": lambda: fit_html, # The HTML after preprocessing for schema - } - - markdown_input_html = cleaned_html # Default to cleaned_html - - try: - # Get the appropriate lambda function, default to returning cleaned_html if key not found - source_lambda = html_source_selector.get(selected_html_source, lambda: cleaned_html) - # Execute the lambda to get the selected HTML - markdown_input_html = source_lambda() - - # Log which source is being used (optional, but helpful for debugging) - # if self.logger and verbose: - # actual_source_used = selected_html_source if selected_html_source in html_source_selector else 'cleaned_html (default)' - # self.logger.debug(f"Using '{actual_source_used}' as source for Markdown generation for {url}", tag="MARKDOWN_SRC") - - except Exception as e: - # Handle potential errors, especially from preprocess_html_for_schema - if self.logger: - self.logger.warning( - f"Error getting/processing '{selected_html_source}' for markdown source: {e}. Falling back to cleaned_html.", - tag="MARKDOWN_SRC" - ) - # Ensure markdown_input_html is still the default cleaned_html in case of error - markdown_input_html = cleaned_html - # --- END: HTML SOURCE SELECTION --- - - # Uncomment if by default we want to use PruningContentFilter - # if not config.content_filter and not markdown_generator.content_filter: - # markdown_generator.content_filter = PruningContentFilter() - - # Extract from raw HTML before it gets stripped by cleaning. - # This ensures relative URLs resolve correctly even with cleaned_html. - base_url = params.get("base_url") or params.get("redirected_url") or url - base_tag_match = re.search(r']*href\s*=\s*["\']([^"\']+)["\']', html, re.IGNORECASE) - if base_tag_match: - base_url = base_tag_match.group(1) - - markdown_result: MarkdownGenerationResult = ( - markdown_generator.generate_markdown( - input_html=markdown_input_html, - base_url=base_url - # html2text_options=kwargs.get('html2text', {}) - ) - ) - - # Log processing completion — reflect actual content outcome - self.logger.url_status( - url=_url, - success=bool(cleaned_html), - timing=int((time.perf_counter() - t1) * 1000) / 1000, - tag="SCRAPE" - ) - # self.logger.info( - # message="{url:.50}... | Time: {timing}s", - # tag="SCRAPE", - # params={"url": _url, "timing": int((time.perf_counter() - t1) * 1000) / 1000}, - # ) - - ################################ - # Structured Content Extraction # - ################################ - if ( - not bool(extracted_content) - and config.extraction_strategy - and not isinstance(config.extraction_strategy, NoExtractionStrategy) - ): - t1 = time.perf_counter() - # Choose content based on input_format - content_format = config.extraction_strategy.input_format - if content_format == "fit_markdown" and not markdown_result.fit_markdown: - - self.logger.url_status( - url=_url, - success=bool(html), - timing=time.perf_counter() - t1, - tag="EXTRACT", - ) - content_format = "markdown" - - content = { - "markdown": markdown_result.raw_markdown, - "html": html, - "fit_html": fit_html, - "cleaned_html": cleaned_html, - "fit_markdown": markdown_result.fit_markdown, - }.get(content_format, markdown_result.raw_markdown) - - # Use IdentityChunking for HTML input, otherwise use provided chunking strategy - chunking = ( - IdentityChunking() - if content_format in ["html", "cleaned_html", "fit_html"] - else config.chunking_strategy - ) - sections = chunking.chunk(content) - # extracted_content = config.extraction_strategy.run(_url, sections) - - # Use async version if available for better parallelism - if hasattr(config.extraction_strategy, 'arun'): - extracted_content = await config.extraction_strategy.arun(_url, sections) - else: - # Fallback to sync version run in thread pool to avoid blocking - extracted_content = await asyncio.to_thread( - config.extraction_strategy.run, url, sections - ) - - extracted_content = json.dumps( - extracted_content, indent=4, default=str, ensure_ascii=False - ) - - # Log extraction completion - self.logger.url_status( - url=_url, - success=bool(html), - timing=time.perf_counter() - t1, - tag="EXTRACT", - ) - - # Apply HTML formatting if requested - if config.prettiify: - cleaned_html = fast_format_html(cleaned_html) - - # Return complete crawl result - return CrawlResult( - url=url, - html=html, - fit_html=fit_html, - cleaned_html=cleaned_html, - markdown=markdown_result, - media=media, - tables=tables, # NEW - links=links, - metadata=metadata, - screenshot=screenshot_data, - pdf=pdf_data, - extracted_content=extracted_content, - success=True, - error_message="", - ) - - async def arun_many( - self, - urls: List[str], - config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None, - dispatcher: Optional[BaseDispatcher] = None, - # Legacy parameters maintained for backwards compatibility - # word_count_threshold=MIN_WORD_THRESHOLD, - # extraction_strategy: ExtractionStrategy = None, - # chunking_strategy: ChunkingStrategy = RegexChunking(), - # content_filter: RelevantContentFilter = None, - # cache_mode: Optional[CacheMode] = None, - # bypass_cache: bool = False, - # css_selector: str = None, - # screenshot: bool = False, - # pdf: bool = False, - # user_agent: str = None, - # verbose=True, - **kwargs, - ) -> RunManyReturn: - """ - Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy. - - Args: - urls: List of URLs to crawl - config: Configuration object(s) controlling crawl behavior. Can be: - - Single CrawlerRunConfig: Used for all URLs - - List[CrawlerRunConfig]: Configs with url_matcher for URL-specific settings - dispatcher: The dispatcher strategy instance to use. Defaults to MemoryAdaptiveDispatcher - [other parameters maintained for backwards compatibility] - - Returns: - Union[List[CrawlResult], AsyncGenerator[CrawlResult, None]]: - Either a list of all results or an async generator yielding results - - Examples: - - # Batch processing (default) - results = await crawler.arun_many( - urls=["https://example1.com", "https://example2.com"], - config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS) - ) - for result in results: - print(f"Processed {result.url}: {len(result.markdown)} chars") - - # Streaming results - async for result in await crawler.arun_many( - urls=["https://example1.com", "https://example2.com"], - config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS, stream=True), - ): - print(f"Processed {result.url}: {len(result.markdown)} chars") - """ - config = config or CrawlerRunConfig() - - # When deep_crawl_strategy is set, bypass the dispatcher and call - # arun() directly for each URL. The DeepCrawlDecorator on arun() - # will invoke the strategy and return List[CrawlResult]. The - # dispatcher cannot handle that return type (it expects a single - # CrawlResult), so we must handle it here. - primary_cfg = config[0] if isinstance(config, list) else config - if getattr(primary_cfg, "deep_crawl_strategy", None): - if primary_cfg.stream: - async def _deep_crawl_stream(): - for url in urls: - result = await self.arun(url, config=primary_cfg) - if isinstance(result, list): - for r in result: - yield r - else: - async for r in result: - yield r - return _deep_crawl_stream() - else: - all_results = [] - for url in urls: - result = await self.arun(url, config=primary_cfg) - if isinstance(result, list): - all_results.extend(result) - else: - all_results.append(result) - return all_results - - if dispatcher is None: - primary_cfg = config[0] if isinstance(config, list) else config - mean_delay = getattr(primary_cfg, "mean_delay", 0.1) - max_range = getattr(primary_cfg, "max_range", 0.3) - max_session_permit = max(1, int(getattr(primary_cfg, "semaphore_count", 10) or 10)) - dispatcher = MemoryAdaptiveDispatcher( - max_session_permit=max_session_permit, - rate_limiter=RateLimiter( - base_delay=(mean_delay, mean_delay + max_range), - max_delay=60.0, - max_retries=3, - ), - ) - - def transform_result(task_result): - return ( - setattr( - task_result.result, - "dispatch_result", - DispatchResult( - task_id=task_result.task_id, - memory_usage=task_result.memory_usage, - peak_memory=task_result.peak_memory, - start_time=task_result.start_time, - end_time=task_result.end_time, - error_message=task_result.error_message, - ), - ) - or task_result.result - ) - - # Handle stream setting - use first config's stream setting if config is a list - if isinstance(config, list): - stream = config[0].stream if config else False - primary_config = config[0] if config else None - else: - stream = config.stream - primary_config = config - - # Helper to release sticky session if auto_release is enabled - async def maybe_release_session(): - if (primary_config and - primary_config.proxy_session_id and - primary_config.proxy_session_auto_release and - primary_config.proxy_rotation_strategy): - await primary_config.proxy_rotation_strategy.release_session( - primary_config.proxy_session_id - ) - self.logger.info( - message="Auto-released proxy session: {session_id}", - tag="PROXY", - params={"session_id": primary_config.proxy_session_id} - ) - - if stream: - async def result_transformer(): - try: - async for task_result in dispatcher.run_urls_stream( - crawler=self, urls=urls, config=config - ): - yield transform_result(task_result) - finally: - # Auto-release session after streaming completes - await maybe_release_session() - - return result_transformer() - else: - try: - _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config) - return [transform_result(res) for res in _results] - finally: - # Auto-release session after batch completes - await maybe_release_session() - - async def aseed_urls( - self, - domain_or_domains: Union[str, List[str]], - config: Optional[SeedingConfig] = None, - **kwargs - ) -> Union[List[str], Dict[str, List[Union[str, Dict[str, Any]]]]]: - """ - Discovers, filters, and optionally validates URLs for a given domain(s) - using sitemaps and Common Crawl archives. - - Args: - domain_or_domains: A single domain string (e.g., "iana.org") or a list of domains. - config: A SeedingConfig object to control the seeding process. - Parameters passed directly via kwargs will override those in 'config'. - **kwargs: Additional parameters (e.g., `source`, `live_check`, `extract_head`, - `pattern`, `concurrency`, `hits_per_sec`, `force_refresh`, `verbose`) - that will be used to construct or update the SeedingConfig. - - Returns: - If `extract_head` is False: - - For a single domain: `List[str]` of discovered URLs. - - For multiple domains: `Dict[str, List[str]]` mapping each domain to its URLs. - If `extract_head` is True: - - For a single domain: `List[Dict[str, Any]]` where each dict contains 'url' - and 'head_data' (parsed metadata). - - For multiple domains: `Dict[str, List[Dict[str, Any]]]` mapping each domain - to a list of URL data dictionaries. - - Raises: - ValueError: If `domain_or_domains` is not a string or a list of strings. - Exception: Any underlying exceptions from AsyncUrlSeeder or network operations. - - Example: - >>> # Discover URLs from sitemap with live check for 'example.com' - >>> result = await crawler.aseed_urls("example.com", source="sitemap", live_check=True, hits_per_sec=10) - - >>> # Discover URLs from Common Crawl, extract head data for 'example.com' and 'python.org' - >>> multi_domain_result = await crawler.aseed_urls( - >>> ["example.com", "python.org"], - >>> source="cc", extract_head=True, concurrency=200, hits_per_sec=50 - >>> ) - """ - # Initialize AsyncUrlSeeder here if it hasn't been already - if not self.url_seeder: - # Pass the crawler's base_directory for seeder's cache management - # Pass the crawler's logger for consistent logging - self.url_seeder = AsyncUrlSeeder( - base_directory=self.crawl4ai_folder, - logger=self.logger - ) - - # Merge config object with direct kwargs, giving kwargs precedence - seeding_config = config.clone(**kwargs) if config else SeedingConfig.from_kwargs(kwargs) - - # Ensure base_directory is set for the seeder's cache - seeding_config.base_directory = seeding_config.base_directory or self.crawl4ai_folder - # Ensure the seeder uses the crawler's logger (if not already set) - if not self.url_seeder.logger: - self.url_seeder.logger = self.logger - - # Pass verbose setting if explicitly provided in SeedingConfig or kwargs - if seeding_config.verbose is not None: - self.url_seeder.logger.verbose = seeding_config.verbose - else: # Default to crawler's verbose setting - self.url_seeder.logger.verbose = self.logger.verbose - - - if isinstance(domain_or_domains, str): - self.logger.info( - message="Starting URL seeding for domain: {domain}", - tag="SEED", - params={"domain": domain_or_domains} - ) - return await self.url_seeder.urls( - domain_or_domains, - seeding_config - ) - elif isinstance(domain_or_domains, (list, tuple)): - self.logger.info( - message="Starting URL seeding for {count} domains", - tag="SEED", - params={"count": len(domain_or_domains)} - ) - # AsyncUrlSeeder.many_urls directly accepts a list of domains and individual params. - return await self.url_seeder.many_urls( - domain_or_domains, - seeding_config - ) - else: - raise ValueError("`domain_or_domains` must be a string or a list of strings.") - - async def amap_domain( - self, - domain: str, - config: Optional[DomainMapperConfig] = None, - **kwargs, - ) -> List[Dict[str, Any]]: - """ - Discover all URLs under a domain without deep crawling. - - Uses DomainMapper to combine sitemap, Common Crawl, Wayback Machine, - certificate transparency, path probing, robots.txt mining, feed discovery, - and homepage link extraction. - - Args: - domain: Domain to map (e.g., "example.com") - config: DomainMapperConfig object. kwargs override config fields. - - Returns: - List of discovered URL dicts with metadata. - """ - if not self._domain_mapper: - self._domain_mapper = DomainMapper( - logger=self.logger, - base_directory=self.crawl4ai_folder, - ) - - mapper_config = config.clone(**kwargs) if config and kwargs else ( - config or DomainMapperConfig(**kwargs) if kwargs else DomainMapperConfig() - ) - - return await self._domain_mapper.scan(domain, mapper_config) \ No newline at end of file +from .__version__ import __version__ as crawl4ai_version +import os +import re +import sys +import time +from pathlib import Path +from typing import Optional, List +import json +import asyncio + +# from contextlib import nullcontext, asynccontextmanager +from contextlib import asynccontextmanager +from .models import ( + CrawlResult, + MarkdownGenerationResult, + DispatchResult, + ScrapingResult, + CrawlResultContainer, + RunManyReturn +) +from .async_database import async_db_manager +from .chunking_strategy import * # noqa: F403 +from .chunking_strategy import IdentityChunking +from .content_filter_strategy import * # noqa: F403 +from .extraction_strategy import * # noqa: F403 +from .extraction_strategy import NoExtractionStrategy +from .async_crawler_strategy import ( + AsyncCrawlerStrategy, + AsyncPlaywrightCrawlerStrategy, + AsyncCrawlResponse, +) +from .cache_context import CacheMode, CacheContext +from .markdown_generation_strategy import ( + DefaultMarkdownGenerator, + MarkdownGenerationStrategy, +) +from .deep_crawling import DeepCrawlDecorator +from .async_logger import AsyncLogger, AsyncLoggerBase +from .async_configs import BrowserConfig, CrawlerRunConfig, ProxyConfig, SeedingConfig, DomainMapperConfig +from .async_dispatcher import * # noqa: F403 +from .async_dispatcher import BaseDispatcher, MemoryAdaptiveDispatcher, RateLimiter +from .async_url_seeder import AsyncUrlSeeder +from .domain_mapper import DomainMapper + +from .utils import ( + sanitize_input_encode, + InvalidCSSSelectorError, + fast_format_html, + get_error_context, + RobotsParser, + preprocess_html_for_schema, + compute_head_fingerprint, +) +from .cache_validator import CacheValidator, CacheValidationResult +from .antibot_detector import is_blocked + + +class AsyncWebCrawler: + """ + Asynchronous web crawler with flexible caching capabilities. + + There are two ways to use the crawler: + + 1. Using context manager (recommended for simple cases): + ```python + async with AsyncWebCrawler() as crawler: + result = await crawler.arun(url="https://example.com") + ``` + + 2. Using explicit lifecycle management (recommended for long-running applications): + ```python + crawler = AsyncWebCrawler() + await crawler.start() + + # Use the crawler multiple times + result1 = await crawler.arun(url="https://example.com") + result2 = await crawler.arun(url="https://another.com") + + await crawler.close() + ``` + + Attributes: + browser_config (BrowserConfig): Configuration object for browser settings. + crawler_strategy (AsyncCrawlerStrategy): Strategy for crawling web pages. + logger (AsyncLogger): Logger instance for recording events and errors. + crawl4ai_folder (str): Directory for storing cache. + base_directory (str): Base directory for storing cache. + ready (bool): Whether the crawler is ready for use. + + Methods: + start(): Start the crawler explicitly without using context manager. + close(): Close the crawler explicitly without using context manager. + arun(): Run the crawler for a single source: URL (web, local file, or raw HTML). + awarmup(): Perform warmup sequence. + arun_many(): Run the crawler for multiple sources. + aprocess_html(): Process HTML content. + + Typical Usage: + async with AsyncWebCrawler() as crawler: + result = await crawler.arun(url="https://example.com") + print(result.markdown) + + Using configuration: + browser_config = BrowserConfig(browser_type="chromium", headless=True) + async with AsyncWebCrawler(config=browser_config) as crawler: + crawler_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS + ) + result = await crawler.arun(url="https://example.com", config=crawler_config) + print(result.markdown) + """ + + _domain_last_hit = {} + + def __init__( + self, + crawler_strategy: AsyncCrawlerStrategy = None, + config: BrowserConfig = None, + base_directory: str = str( + os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())), + thread_safe: bool = False, + logger: AsyncLoggerBase = None, + **kwargs, + ): + """ + Initialize the AsyncWebCrawler. + + Args: + crawler_strategy: Strategy for crawling web pages. Default AsyncPlaywrightCrawlerStrategy + config: Configuration object for browser settings. Default BrowserConfig() + base_directory: Base directory for storing cache + thread_safe: Whether to use thread-safe operations + **kwargs: Additional arguments for backwards compatibility + """ + # Handle browser configuration + browser_config = config or BrowserConfig() + + self.browser_config = browser_config + + # Initialize logger first since other components may need it + self.logger = logger or AsyncLogger( + log_file=os.path.join(base_directory, ".crawl4ai", "crawler.log"), + verbose=self.browser_config.verbose, + tag_width=10, + ) + + # Initialize crawler strategy + params = {k: v for k, v in kwargs.items() if k in [ + "browser_config", "logger"]} + self.crawler_strategy = crawler_strategy or AsyncPlaywrightCrawlerStrategy( + browser_config=browser_config, + logger=self.logger, + **params, # Pass remaining kwargs for backwards compatibility + ) + + # Thread safety setup + self._lock = asyncio.Lock() if thread_safe else None + + # Initialize directories + self.crawl4ai_folder = os.path.join(base_directory, ".crawl4ai") + os.makedirs(self.crawl4ai_folder, exist_ok=True) + os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True) + + # Initialize robots parser + self.robots_parser = RobotsParser() + + self.ready = False + + # Decorate arun method with deep crawling capabilities + self._deep_handler = DeepCrawlDecorator(self) + self.arun = self._deep_handler(self.arun) + + self.url_seeder: Optional[AsyncUrlSeeder] = None + self._domain_mapper: Optional[DomainMapper] = None + + async def start(self): + """ + Start the crawler explicitly without using context manager. + This is equivalent to using 'async with' but gives more control over the lifecycle. + Returns: + AsyncWebCrawler: The initialized crawler instance + """ + try: + await self.crawler_strategy.__aenter__() + except BaseException: + # Ensure partial resources are cleaned up on failure + try: + await self.crawler_strategy.__aexit__(None, None, None) + except Exception: + pass + raise + self.logger.info(f"Crawl4AI {crawl4ai_version}", tag="INIT") + self.ready = True + return self + + async def close(self): + """ + Close the crawler explicitly without using context manager. + This should be called when you're done with the crawler if you used start(). + + This method will: + 1. Clean up browser resources + 2. Close any open pages and contexts + """ + await self.crawler_strategy.__aexit__(None, None, None) + + async def __aenter__(self): + return await self.start() + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + @asynccontextmanager + async def nullcontext(self): + """异步空上下文管理器""" + yield + + async def arun( + self, + url: str, + config: CrawlerRunConfig = None, + **kwargs, + ) -> CrawlResultContainer: + """ + Runs the crawler for a single source: URL (web, local file, or raw HTML). + + Migration Guide: + Old way (deprecated): + result = await crawler.arun( + url="https://example.com", + word_count_threshold=200, + screenshot=True, + ... + ) + + New way (recommended): + config = CrawlerRunConfig( + word_count_threshold=200, + screenshot=True, + ... + ) + result = await crawler.arun(url="https://example.com", config=config) + + Args: + url: The URL to crawl (http://, https://, file://, or raw:) + config: Configuration object controlling crawl behavior + [other parameters maintained for backwards compatibility] + + Returns: + CrawlResultContainer: A single-result container that proxies + attribute access to the underlying CrawlResult for backwards + compatibility (e.g. result.markdown, result.html). + """ + # Auto-start if not ready + if not self.ready: + await self.start() + + config = config or CrawlerRunConfig() + if not isinstance(url, str) or not url: + raise ValueError( + "Invalid URL, make sure the URL is a non-empty string") + + async with self._lock or self.nullcontext(): + try: + self.logger.verbose = config.verbose + + # Default to ENABLED if no cache mode specified + if config.cache_mode is None: + config.cache_mode = CacheMode.ENABLED + + # Create cache context + cache_context = CacheContext(url, config.cache_mode, False) + + # Initialize processing variables + async_response: AsyncCrawlResponse = None + cached_result: CrawlResult = None + screenshot_data = None + pdf_data = None + extracted_content = None + start_time = time.perf_counter() + + # Try to get cached result if appropriate + if cache_context.should_read(): + cached_result = await async_db_manager.aget_cached_url(url) + + # Smart Cache: Validate cache freshness if enabled + if cached_result and config.check_cache_freshness: + cache_metadata = await async_db_manager.aget_cache_metadata(url) + if cache_metadata: + async with CacheValidator(timeout=config.cache_validation_timeout) as validator: + validation = await validator.validate( + url=url, + stored_etag=cache_metadata.get("etag"), + stored_last_modified=cache_metadata.get("last_modified"), + stored_head_fingerprint=cache_metadata.get("head_fingerprint"), + ) + + if validation.status == CacheValidationResult.FRESH: + cached_result.cache_status = "hit_validated" + self.logger.info( + message="Cache validated: {reason}", + tag="CACHE", + params={"reason": validation.reason} + ) + # Update metadata if we got new values + if validation.new_etag or validation.new_last_modified: + await async_db_manager.aupdate_cache_metadata( + url=url, + etag=validation.new_etag, + last_modified=validation.new_last_modified, + head_fingerprint=validation.new_head_fingerprint, + ) + elif validation.status == CacheValidationResult.ERROR: + cached_result.cache_status = "hit_fallback" + self.logger.warning( + message="Cache validation failed, using cached: {reason}", + tag="CACHE", + params={"reason": validation.reason} + ) + else: + # STALE or UNKNOWN - force recrawl + self.logger.info( + message="Cache stale: {reason}", + tag="CACHE", + params={"reason": validation.reason} + ) + cached_result = None + elif cached_result: + cached_result.cache_status = "hit" + + if cached_result: + html = sanitize_input_encode(cached_result.html) + extracted_content = sanitize_input_encode( + cached_result.extracted_content or "" + ) + extracted_content = ( + None + if not extracted_content or extracted_content == "[]" + else extracted_content + ) + # If screenshot is requested but its not in cache, then set cache_result to None + screenshot_data = cached_result.screenshot + pdf_data = cached_result.pdf + # if config.screenshot and not screenshot or config.pdf and not pdf: + if config.screenshot and not screenshot_data: + cached_result = None + + if config.pdf and not pdf_data: + cached_result = None + + self.logger.url_status( + url=cache_context.display_url, + success=bool(html), + timing=time.perf_counter() - start_time, + tag="FETCH", + ) + + # Update proxy configuration from rotation strategy if available + if config and config.proxy_rotation_strategy: + # Handle sticky sessions - use same proxy for all requests with same session_id + if config.proxy_session_id: + next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_proxy_for_session( + config.proxy_session_id, + ttl=config.proxy_session_ttl + ) + if next_proxy: + self.logger.info( + message="Using sticky proxy session: {session_id} -> {proxy}", + tag="PROXY", + params={ + "session_id": config.proxy_session_id, + "proxy": next_proxy.server + } + ) + config.proxy_config = next_proxy + else: + # Existing behavior: rotate on each request + next_proxy: ProxyConfig = await config.proxy_rotation_strategy.get_next_proxy() + if next_proxy: + self.logger.info( + message="Switch proxy: {proxy}", + tag="PROXY", + params={"proxy": next_proxy.server} + ) + config.proxy_config = next_proxy + + # Fetch fresh content if needed + if not cached_result or not html: + from urllib.parse import urlparse + + # Check robots.txt if enabled (once, before any attempts) + if config and config.check_robots_txt: + if not await self.robots_parser.can_fetch( + url, self.browser_config.user_agent + ): + return CrawlResult( + url=url, + html="", + success=False, + status_code=403, + error_message="Access denied by robots.txt", + response_headers={ + "X-Robots-Status": "Blocked by robots.txt" + }, + ) + + # --- Anti-bot retry setup --- + # raw: URLs contain caller-provided HTML (e.g. from cache), + # not content fetched from a web server. Anti-bot detection, + # proxy retries, and fallback fetching are meaningless here. + _is_raw_url = url.startswith("raw:") or url.startswith("raw://") + + _max_attempts = 1 + getattr(config, "max_retries", 0) + _proxy_list = config._get_proxy_list() + _original_proxy_config = config.proxy_config + _block_reason = "" + _done = False + crawl_result = None + _crawl_stats = { + "attempts": 0, + "retries": 0, + "proxies_used": [], + "fallback_fetch_used": False, + "resolved_by": None, + } + + for _attempt in range(_max_attempts): + if _done: + break + + if _attempt > 0: + _crawl_stats["retries"] = _attempt + self.logger.warning( + message="Anti-bot retry {attempt}/{max_retries} for {url} — {reason}", + tag="ANTIBOT", + params={ + "attempt": _attempt, + "max_retries": config.max_retries, + "url": url[:80], + "reason": _block_reason, + }, + ) + + for _p_idx, _proxy in enumerate(_proxy_list): + if _p_idx > 0 or _attempt > 0: + self.logger.info( + message="Trying proxy {idx}/{total}: {proxy}", + tag="ANTIBOT", + params={ + "idx": _p_idx + 1, + "total": len(_proxy_list), + "proxy": _proxy.server if _proxy else "direct", + }, + ) + + # Set the active proxy for this attempt + config.proxy_config = _proxy + _crawl_stats["attempts"] += 1 + + try: + t1 = time.perf_counter() + + if config.user_agent: + self.crawler_strategy.update_user_agent( + config.user_agent) + + async_response = await self.crawler_strategy.crawl( + url, config=config) + + html = sanitize_input_encode(async_response.html) + screenshot_data = async_response.screenshot + pdf_data = async_response.pdf_data + js_execution_result = async_response.js_execution_result + + self.logger.url_status( + url=cache_context.display_url, + success=bool(html), + timing=time.perf_counter() - t1, + tag="FETCH", + ) + + crawl_result = await self.aprocess_html( + url=url, html=html, + extracted_content=extracted_content, + config=config, + screenshot_data=screenshot_data, + pdf_data=pdf_data, + verbose=config.verbose, + is_raw_html=True if url.startswith("raw:") else False, + redirected_url=async_response.redirected_url, + original_scheme=urlparse(url).scheme, + **kwargs, + ) + + crawl_result.status_code = async_response.status_code + is_raw_url = url.startswith("raw:") or url.startswith("raw://") + crawl_result.redirected_url = async_response.redirected_url or (None if is_raw_url else url) + crawl_result.redirected_status_code = async_response.redirected_status_code + crawl_result.response_headers = async_response.response_headers + crawl_result.downloaded_files = async_response.downloaded_files + crawl_result.js_execution_result = js_execution_result + crawl_result.mhtml = async_response.mhtml_data + crawl_result.ssl_certificate = async_response.ssl_certificate + crawl_result.network_requests = async_response.network_requests + crawl_result.console_messages = async_response.console_messages + # Success when html is non-empty OR a binary + # download was retrieved (PDFs, archives etc. + # have empty html by design — file content is + # in downloaded_files). + crawl_result.success = bool(html) or bool(async_response.downloaded_files) + crawl_result.session_id = getattr(config, "session_id", None) + crawl_result.cache_status = "miss" + + # Check if blocked (skip for raw: URLs — + # caller-provided content, anti-bot N/A) + if _is_raw_url: + _blocked = False + _block_reason = "" + else: + _blocked, _block_reason = is_blocked( + async_response.status_code, html) + + _crawl_stats["proxies_used"].append({ + "proxy": _proxy.server if _proxy else None, + "status_code": async_response.status_code, + "blocked": _blocked, + "reason": _block_reason if _blocked else "", + }) + + if not _blocked: + _crawl_stats["resolved_by"] = "proxy" if _proxy else "direct" + _done = True + break # Success — exit proxy loop + + except Exception as _crawl_err: + _crawl_stats["proxies_used"].append({ + "proxy": _proxy.server if _proxy else None, + "status_code": None, + "blocked": True, + "reason": str(_crawl_err), + }) + self.logger.error_status( + url=url, + error=f"Proxy {_proxy.server if _proxy else 'direct'} failed: {_crawl_err}", + tag="ANTIBOT", + ) + _block_reason = str(_crawl_err) + # If this is the only proxy and only attempt, re-raise + # so the caller gets the real error (not a silent swallow). + # But if there are more proxies or retries to try, continue. + if len(_proxy_list) <= 1 and _max_attempts <= 1: + raise + + # Restore original proxy_config + config.proxy_config = _original_proxy_config + + # --- Fallback fetch function (last resort after all retries+proxies exhausted) --- + # Invoke fallback when: (a) crawl_result exists but is blocked, OR + # (b) crawl_result is None because all proxies threw exceptions (browser crash, timeout). + # Skip for raw: URLs — fallback expects a real URL, not raw HTML content. + _fallback_fn = getattr(config, "fallback_fetch_function", None) + if _fallback_fn and not _done and not _is_raw_url: + _needs_fallback = ( + crawl_result is None # All proxies threw exceptions + or is_blocked(crawl_result.status_code, crawl_result.html or "")[0] + ) + if _needs_fallback: + self.logger.warning( + message="All retries exhausted, invoking fallback_fetch_function for {url}", + tag="ANTIBOT", + params={"url": url[:80]}, + ) + _crawl_stats["fallback_fetch_used"] = True + try: + _fallback_html = await _fallback_fn(url) + if _fallback_html: + _sanitized_html = sanitize_input_encode(_fallback_html) + try: + crawl_result = await self.aprocess_html( + url=url, + html=_sanitized_html, + extracted_content=extracted_content, + config=config, + screenshot_data=None, + pdf_data=None, + verbose=config.verbose, + is_raw_html=True, + redirected_url=url, + original_scheme=urlparse(url).scheme, + **kwargs, + ) + except Exception as _proc_err: + # aprocess_html may fail if browser is dead (e.g., + # consent popup removal needs Page.evaluate). + # Fall back to a minimal result with raw HTML. + self.logger.warning( + message="Fallback HTML processing failed ({err}), using raw HTML", + tag="ANTIBOT", + params={"err": str(_proc_err)[:100]}, + ) + crawl_result = CrawlResult( + url=url, + html=_sanitized_html, + success=True, + status_code=200, + ) + crawl_result.success = True + crawl_result.status_code = 200 + crawl_result.session_id = getattr(config, "session_id", None) + crawl_result.cache_status = "miss" + _crawl_stats["resolved_by"] = "fallback_fetch" + except Exception as _fallback_err: + self.logger.error_status( + url=url, + error=f"Fallback fetch failed: {_fallback_err}", + tag="ANTIBOT", + ) + + # --- Mark blocked results as failed --- + # Skip re-check ONLY when fallback SUCCEEDED — the fallback result + # is authoritative and real pages may contain anti-bot script markers + # (e.g. PerimeterX JS on Walmart) that trigger false positives. + # When fallback was attempted but FAILED, we must still re-check + # because the result is from a blocked proxy attempt. + # Also skip for raw: URLs — caller-provided content, anti-bot N/A. + # Also skip for binary downloads (PDFs, archives, etc.) — content + # was delivered via downloaded_files, html is empty by design, + # and is_blocked() would misread "0 bytes html" as a block. + if crawl_result: + _fallback_succeeded = _crawl_stats.get("resolved_by") == "fallback_fetch" + # Skip the block check for binary downloads (PDFs, archives, + # etc.) — content was delivered via downloaded_files, html is + # empty by design, and is_blocked() would misread "0 bytes + # html" as a block. + _has_download = bool(getattr(crawl_result, "downloaded_files", None)) + if not _fallback_succeeded and not _is_raw_url and not _has_download: + _blocked, _block_reason = is_blocked( + crawl_result.status_code, crawl_result.html or "") + if _blocked: + crawl_result.success = False + crawl_result.error_message = f"Blocked by anti-bot protection: {_block_reason}" + crawl_result.crawl_stats = _crawl_stats + else: + # All proxies threw exceptions and fallback either wasn't + # configured or also failed. Build a minimal result so the + # caller gets crawl_stats instead of None. + crawl_result = CrawlResult( + url=url, + html="", + success=False, + status_code=None, + error_message=f"All proxies failed: {_block_reason}" if _block_reason else "All proxies failed", + ) + crawl_result.crawl_stats = _crawl_stats + + # Compute head fingerprint for cache validation + if crawl_result and crawl_result.html: + head_end = crawl_result.html.lower().find('') + if head_end != -1: + head_html = crawl_result.html[:head_end + 7] + crawl_result.head_fingerprint = compute_head_fingerprint(head_html) + + # Log failure reason before COMPLETE so users can see why it failed. + if crawl_result and not crawl_result.success and crawl_result.error_message: + self.logger.error_status( + url=cache_context.display_url, + error=crawl_result.error_message, + tag="ERROR", + ) + + self.logger.url_status( + url=cache_context.display_url, + success=crawl_result.success if crawl_result else False, + timing=time.perf_counter() - start_time, + tag="COMPLETE", + ) + + # Update cache if appropriate + if cache_context.should_write() and not bool(cached_result): + await async_db_manager.acache_url(crawl_result) + + return CrawlResultContainer(crawl_result) + + else: + self.logger.url_status( + url=cache_context.display_url, + success=True, + timing=time.perf_counter() - start_time, + tag="COMPLETE" + ) + # Same binary-download awareness as the live-fetch path + # — a cached PDF/archive should replay as success. + cached_result.success = bool(html) or bool(getattr(cached_result, "downloaded_files", None)) + cached_result.session_id = getattr( + config, "session_id", None) + # For raw: URLs, don't fall back to the raw HTML string as redirected_url + is_raw_url = url.startswith("raw:") or url.startswith("raw://") + cached_result.redirected_url = cached_result.redirected_url or (None if is_raw_url else url) + return CrawlResultContainer(cached_result) + + except Exception as e: + error_context = get_error_context(sys.exc_info()) + + error_message = ( + f"Unexpected error in _crawl_web at line {error_context['line_no']} " + f"in {error_context['function']} ({error_context['filename']}):\n" + f"Error: {str(e)}\n\n" + f"Code context:\n{error_context['code_context']}" + ) + + self.logger.error_status( + url=url, + error=error_message, + tag="ERROR", + ) + + return CrawlResultContainer( + CrawlResult( + url=url, html="", success=False, error_message=error_message + ) + ) + + async def aprocess_html( + self, + url: str, + html: str, + extracted_content: str, + config: CrawlerRunConfig, + screenshot_data: str, + pdf_data: str, + verbose: bool, + **kwargs, + ) -> CrawlResult: + """ + Process HTML content using the provided configuration. + + Args: + url: The URL being processed + html: Raw HTML content + extracted_content: Previously extracted content (if any) + config: Configuration object controlling processing behavior + screenshot_data: Screenshot data (if any) + pdf_data: PDF data (if any) + verbose: Whether to enable verbose logging + **kwargs: Additional parameters for backwards compatibility + + Returns: + CrawlResult: Processed result containing extracted and formatted content + """ + # === PREFETCH MODE SHORT-CIRCUIT === + if getattr(config, 'prefetch', False): + from .utils import quick_extract_links + + # Use base_url from config (for raw: URLs), redirected_url, or original url + effective_url = getattr(config, 'base_url', None) or kwargs.get('redirected_url') or url + links = quick_extract_links(html, effective_url) + + return CrawlResult( + url=url, + html=html, + success=True, + links=links, + status_code=kwargs.get('status_code'), + response_headers=kwargs.get('response_headers'), + redirected_url=kwargs.get('redirected_url'), + ssl_certificate=kwargs.get('ssl_certificate'), + # All other fields default to None + ) + # === END PREFETCH SHORT-CIRCUIT === + + cleaned_html = "" + try: + _url = url if not kwargs.get("is_raw_html", False) else "Raw HTML" + t1 = time.perf_counter() + + # Get scraping strategy and ensure it has a logger + scraping_strategy = config.scraping_strategy + if not scraping_strategy.logger: + scraping_strategy.logger = self.logger + + # Process HTML content + params = config.__dict__.copy() + params.pop("url", None) + # add keys from kwargs to params that doesn't exist in params + params.update({k: v for k, v in kwargs.items() + if k not in params.keys()}) + + ################################ + # Scraping Strategy Execution # + ################################ + result: ScrapingResult = scraping_strategy.scrap( + url, html, **params) + + if result is None: + raise ValueError( + f"Process HTML, Failed to extract content from the website: {url}" + ) + + except InvalidCSSSelectorError as e: + raise ValueError(str(e)) + except Exception as e: + raise ValueError( + f"Process HTML, Failed to extract content from the website: {url}, error: {str(e)}" + ) + + # Extract results - handle both dict and ScrapingResult + if isinstance(result, dict): + cleaned_html = sanitize_input_encode( + result.get("cleaned_html", "")) + media = result.get("media", {}) + tables = media.pop("tables", []) if isinstance(media, dict) else [] + links = result.get("links", {}) + metadata = result.get("metadata", {}) + else: + cleaned_html = sanitize_input_encode(result.cleaned_html) + # media = result.media.model_dump() + # tables = media.pop("tables", []) + # links = result.links.model_dump() + media = result.media.model_dump() if hasattr(result.media, 'model_dump') else result.media + tables = media.pop("tables", []) if isinstance(media, dict) else [] + links = result.links.model_dump() if hasattr(result.links, 'model_dump') else result.links + metadata = result.metadata + + fit_html = preprocess_html_for_schema(html_content=html, text_threshold= 500, max_size= 300_000) + + ################################ + # Generate Markdown # + ################################ + markdown_generator: Optional[MarkdownGenerationStrategy] = ( + config.markdown_generator or DefaultMarkdownGenerator() + ) + + # --- SELECT HTML SOURCE BASED ON CONTENT_SOURCE --- + # Get the desired source from the generator config, default to 'cleaned_html' + selected_html_source = getattr(markdown_generator, 'content_source', 'cleaned_html') + + # Define the source selection logic using dict dispatch + html_source_selector = { + "raw_html": lambda: html, # The original raw HTML + "cleaned_html": lambda: cleaned_html, # The HTML after scraping strategy + "fit_html": lambda: fit_html, # The HTML after preprocessing for schema + } + + markdown_input_html = cleaned_html # Default to cleaned_html + + try: + # Get the appropriate lambda function, default to returning cleaned_html if key not found + source_lambda = html_source_selector.get(selected_html_source, lambda: cleaned_html) + # Execute the lambda to get the selected HTML + markdown_input_html = source_lambda() + + # Log which source is being used (optional, but helpful for debugging) + # if self.logger and verbose: + # actual_source_used = selected_html_source if selected_html_source in html_source_selector else 'cleaned_html (default)' + # self.logger.debug(f"Using '{actual_source_used}' as source for Markdown generation for {url}", tag="MARKDOWN_SRC") + + except Exception as e: + # Handle potential errors, especially from preprocess_html_for_schema + if self.logger: + self.logger.warning( + f"Error getting/processing '{selected_html_source}' for markdown source: {e}. Falling back to cleaned_html.", + tag="MARKDOWN_SRC" + ) + # Ensure markdown_input_html is still the default cleaned_html in case of error + markdown_input_html = cleaned_html + # --- END: HTML SOURCE SELECTION --- + + # Uncomment if by default we want to use PruningContentFilter + # if not config.content_filter and not markdown_generator.content_filter: + # markdown_generator.content_filter = PruningContentFilter() + + # Extract from raw HTML before it gets stripped by cleaning. + # This ensures relative URLs resolve correctly even with cleaned_html. + base_url = params.get("base_url") or params.get("redirected_url") or url + base_tag_match = re.search(r']*href\s*=\s*["\']([^"\']+)["\']', html, re.IGNORECASE) + if base_tag_match: + base_url = base_tag_match.group(1) + + markdown_result: MarkdownGenerationResult = ( + markdown_generator.generate_markdown( + input_html=markdown_input_html, + base_url=base_url + # html2text_options=kwargs.get('html2text', {}) + ) + ) + + # Log processing completion — reflect actual content outcome + self.logger.url_status( + url=_url, + success=bool(cleaned_html), + timing=int((time.perf_counter() - t1) * 1000) / 1000, + tag="SCRAPE" + ) + # self.logger.info( + # message="{url:.50}... | Time: {timing}s", + # tag="SCRAPE", + # params={"url": _url, "timing": int((time.perf_counter() - t1) * 1000) / 1000}, + # ) + + ################################ + # Structured Content Extraction # + ################################ + if ( + not bool(extracted_content) + and config.extraction_strategy + and not isinstance(config.extraction_strategy, NoExtractionStrategy) + ): + t1 = time.perf_counter() + # Choose content based on input_format + content_format = config.extraction_strategy.input_format + if content_format == "fit_markdown" and not markdown_result.fit_markdown: + + self.logger.url_status( + url=_url, + success=bool(html), + timing=time.perf_counter() - t1, + tag="EXTRACT", + ) + content_format = "markdown" + + content = { + "markdown": markdown_result.raw_markdown, + "html": html, + "fit_html": fit_html, + "cleaned_html": cleaned_html, + "fit_markdown": markdown_result.fit_markdown, + }.get(content_format, markdown_result.raw_markdown) + + # Use IdentityChunking for HTML input, otherwise use provided chunking strategy + chunking = ( + IdentityChunking() + if content_format in ["html", "cleaned_html", "fit_html"] + else config.chunking_strategy + ) + sections = chunking.chunk(content) + # extracted_content = config.extraction_strategy.run(_url, sections) + + # Use async version if available for better parallelism + if hasattr(config.extraction_strategy, 'arun'): + extracted_content = await config.extraction_strategy.arun(_url, sections) + else: + # Fallback to sync version run in thread pool to avoid blocking + extracted_content = await asyncio.to_thread( + config.extraction_strategy.run, url, sections + ) + + extracted_content = json.dumps( + extracted_content, indent=4, default=str, ensure_ascii=False + ) + + # Log extraction completion + self.logger.url_status( + url=_url, + success=bool(html), + timing=time.perf_counter() - t1, + tag="EXTRACT", + ) + + # Apply HTML formatting if requested + if config.prettiify: + cleaned_html = fast_format_html(cleaned_html) + + # Return complete crawl result + return CrawlResult( + url=url, + html=html, + fit_html=fit_html, + cleaned_html=cleaned_html, + markdown=markdown_result, + media=media, + tables=tables, # NEW + links=links, + metadata=metadata, + screenshot=screenshot_data, + pdf=pdf_data, + extracted_content=extracted_content, + success=True, + error_message="", + ) + + async def arun_many( + self, + urls: List[str], + config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None, + dispatcher: Optional[BaseDispatcher] = None, + # Legacy parameters maintained for backwards compatibility + # word_count_threshold=MIN_WORD_THRESHOLD, + # extraction_strategy: ExtractionStrategy = None, + # chunking_strategy: ChunkingStrategy = RegexChunking(), + # content_filter: RelevantContentFilter = None, + # cache_mode: Optional[CacheMode] = None, + # bypass_cache: bool = False, + # css_selector: str = None, + # screenshot: bool = False, + # pdf: bool = False, + # user_agent: str = None, + # verbose=True, + **kwargs, + ) -> RunManyReturn: + """ + Runs the crawler for multiple URLs concurrently using a configurable dispatcher strategy. + + Args: + urls: List of URLs to crawl + config: Configuration object(s) controlling crawl behavior. Can be: + - Single CrawlerRunConfig: Used for all URLs + - List[CrawlerRunConfig]: Configs with url_matcher for URL-specific settings + dispatcher: The dispatcher strategy instance to use. Defaults to MemoryAdaptiveDispatcher + [other parameters maintained for backwards compatibility] + + Returns: + Union[List[CrawlResult], AsyncGenerator[CrawlResult, None]]: + Either a list of all results or an async generator yielding results + + Examples: + + # Batch processing (default) + results = await crawler.arun_many( + urls=["https://example1.com", "https://example2.com"], + config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS) + ) + for result in results: + print(f"Processed {result.url}: {len(result.markdown)} chars") + + # Streaming results + async for result in await crawler.arun_many( + urls=["https://example1.com", "https://example2.com"], + config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS, stream=True), + ): + print(f"Processed {result.url}: {len(result.markdown)} chars") + """ + config = config or CrawlerRunConfig() + + # When deep_crawl_strategy is set, bypass the dispatcher and call + # arun() directly for each URL. The DeepCrawlDecorator on arun() + # will invoke the strategy and return List[CrawlResult]. The + # dispatcher cannot handle that return type (it expects a single + # CrawlResult), so we must handle it here. + primary_cfg = config[0] if isinstance(config, list) else config + if getattr(primary_cfg, "deep_crawl_strategy", None): + if primary_cfg.stream: + async def _deep_crawl_stream(): + for url in urls: + result = await self.arun(url, config=primary_cfg) + if isinstance(result, list): + for r in result: + yield r + else: + async for r in result: + yield r + return _deep_crawl_stream() + else: + all_results = [] + for url in urls: + result = await self.arun(url, config=primary_cfg) + if isinstance(result, list): + all_results.extend(result) + else: + all_results.append(result) + return all_results + + if dispatcher is None: + primary_cfg = config[0] if isinstance(config, list) else config + mean_delay = getattr(primary_cfg, "mean_delay", 0.1) + max_range = getattr(primary_cfg, "max_range", 0.3) + max_session_permit = max(1, int(getattr(primary_cfg, "semaphore_count", 10) or 10)) + dispatcher = MemoryAdaptiveDispatcher( + max_session_permit=max_session_permit, + rate_limiter=RateLimiter( + base_delay=(mean_delay, mean_delay + max_range), + max_delay=60.0, + max_retries=3, + ), + ) + + def transform_result(task_result): + return ( + setattr( + task_result.result, + "dispatch_result", + DispatchResult( + task_id=task_result.task_id, + memory_usage=task_result.memory_usage, + peak_memory=task_result.peak_memory, + start_time=task_result.start_time, + end_time=task_result.end_time, + error_message=task_result.error_message, + ), + ) + or task_result.result + ) + + # Handle stream setting - use first config's stream setting if config is a list + if isinstance(config, list): + stream = config[0].stream if config else False + primary_config = config[0] if config else None + else: + stream = config.stream + primary_config = config + + # Helper to release sticky session if auto_release is enabled + async def maybe_release_session(): + if (primary_config and + primary_config.proxy_session_id and + primary_config.proxy_session_auto_release and + primary_config.proxy_rotation_strategy): + await primary_config.proxy_rotation_strategy.release_session( + primary_config.proxy_session_id + ) + self.logger.info( + message="Auto-released proxy session: {session_id}", + tag="PROXY", + params={"session_id": primary_config.proxy_session_id} + ) + + if stream: + async def result_transformer(): + try: + async for task_result in dispatcher.run_urls_stream( + crawler=self, urls=urls, config=config + ): + yield transform_result(task_result) + finally: + # Auto-release session after streaming completes + await maybe_release_session() + + return result_transformer() + else: + try: + _results = await dispatcher.run_urls(crawler=self, urls=urls, config=config) + return [transform_result(res) for res in _results] + finally: + # Auto-release session after batch completes + await maybe_release_session() + + async def aseed_urls( + self, + domain_or_domains: Union[str, List[str]], + config: Optional[SeedingConfig] = None, + **kwargs + ) -> Union[List[str], Dict[str, List[Union[str, Dict[str, Any]]]]]: + """ + Discovers, filters, and optionally validates URLs for a given domain(s) + using sitemaps and Common Crawl archives. + + Args: + domain_or_domains: A single domain string (e.g., "iana.org") or a list of domains. + config: A SeedingConfig object to control the seeding process. + Parameters passed directly via kwargs will override those in 'config'. + **kwargs: Additional parameters (e.g., `source`, `live_check`, `extract_head`, + `pattern`, `concurrency`, `hits_per_sec`, `force_refresh`, `verbose`) + that will be used to construct or update the SeedingConfig. + + Returns: + If `extract_head` is False: + - For a single domain: `List[str]` of discovered URLs. + - For multiple domains: `Dict[str, List[str]]` mapping each domain to its URLs. + If `extract_head` is True: + - For a single domain: `List[Dict[str, Any]]` where each dict contains 'url' + and 'head_data' (parsed metadata). + - For multiple domains: `Dict[str, List[Dict[str, Any]]]` mapping each domain + to a list of URL data dictionaries. + + Raises: + ValueError: If `domain_or_domains` is not a string or a list of strings. + Exception: Any underlying exceptions from AsyncUrlSeeder or network operations. + + Example: + >>> # Discover URLs from sitemap with live check for 'example.com' + >>> result = await crawler.aseed_urls("example.com", source="sitemap", live_check=True, hits_per_sec=10) + + >>> # Discover URLs from Common Crawl, extract head data for 'example.com' and 'python.org' + >>> multi_domain_result = await crawler.aseed_urls( + >>> ["example.com", "python.org"], + >>> source="cc", extract_head=True, concurrency=200, hits_per_sec=50 + >>> ) + """ + # Initialize AsyncUrlSeeder here if it hasn't been already + if not self.url_seeder: + # Pass the crawler's base_directory for seeder's cache management + # Pass the crawler's logger for consistent logging + self.url_seeder = AsyncUrlSeeder( + base_directory=self.crawl4ai_folder, + logger=self.logger + ) + + # Merge config object with direct kwargs, giving kwargs precedence + seeding_config = config.clone(**kwargs) if config else SeedingConfig.from_kwargs(kwargs) + + # Ensure base_directory is set for the seeder's cache + seeding_config.base_directory = seeding_config.base_directory or self.crawl4ai_folder + # Ensure the seeder uses the crawler's logger (if not already set) + if not self.url_seeder.logger: + self.url_seeder.logger = self.logger + + # Pass verbose setting if explicitly provided in SeedingConfig or kwargs + if seeding_config.verbose is not None: + self.url_seeder.logger.verbose = seeding_config.verbose + else: # Default to crawler's verbose setting + self.url_seeder.logger.verbose = self.logger.verbose + + + if isinstance(domain_or_domains, str): + self.logger.info( + message="Starting URL seeding for domain: {domain}", + tag="SEED", + params={"domain": domain_or_domains} + ) + return await self.url_seeder.urls( + domain_or_domains, + seeding_config + ) + elif isinstance(domain_or_domains, (list, tuple)): + self.logger.info( + message="Starting URL seeding for {count} domains", + tag="SEED", + params={"count": len(domain_or_domains)} + ) + # AsyncUrlSeeder.many_urls directly accepts a list of domains and individual params. + return await self.url_seeder.many_urls( + domain_or_domains, + seeding_config + ) + else: + raise ValueError("`domain_or_domains` must be a string or a list of strings.") + + async def amap_domain( + self, + domain: str, + config: Optional[DomainMapperConfig] = None, + **kwargs, + ) -> List[Dict[str, Any]]: + """ + Discover all URLs under a domain without deep crawling. + + Uses DomainMapper to combine sitemap, Common Crawl, Wayback Machine, + certificate transparency, path probing, robots.txt mining, feed discovery, + and homepage link extraction. + + Args: + domain: Domain to map (e.g., "example.com") + config: DomainMapperConfig object. kwargs override config fields. + + Returns: + List of discovered URL dicts with metadata. + """ + if not self._domain_mapper: + self._domain_mapper = DomainMapper( + logger=self.logger, + base_directory=self.crawl4ai_folder, + ) + + mapper_config = config.clone(**kwargs) if config and kwargs else ( + config or DomainMapperConfig(**kwargs) if kwargs else DomainMapperConfig() + ) + + return await self._domain_mapper.scan(domain, mapper_config) diff --git a/crawl4ai/browser_manager.py b/crawl4ai/browser_manager.py index fe7f10853..6377452dc 100644 --- a/crawl4ai/browser_manager.py +++ b/crawl4ai/browser_manager.py @@ -1,2111 +1,2111 @@ -import asyncio -import time -from typing import Dict, List, Optional, Tuple -import os -import sys -import shutil -import tempfile -import psutil -import signal -import subprocess -import shlex -from playwright.async_api import BrowserContext -import hashlib -from .js_snippet import load_js_script -from .config import DOWNLOAD_PAGE_TIMEOUT -from .async_configs import BrowserConfig, CrawlerRunConfig -from .utils import get_chromium_path -import warnings - - -BROWSER_DISABLE_OPTIONS = [ - "--disable-background-networking", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-breakpad", - "--disable-client-side-phishing-detection", - "--disable-component-extensions-with-background-pages", - "--disable-default-apps", - "--disable-extensions", - "--disable-features=TranslateUI", - "--disable-hang-monitor", - "--disable-ipc-flooding-protection", - "--disable-popup-blocking", - "--disable-prompt-on-repost", - "--disable-sync", - "--force-color-profile=srgb", - "--metrics-recording-only", - "--no-first-run", - "--password-store=basic", - "--use-mock-keychain", -] - - -class ManagedBrowser: - """ - Manages the browser process and context. This class allows to connect to the browser using CDP protocol. - - Attributes: - browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". - Default: "chromium". - user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a - temporary directory may be used. Default: None. - headless (bool): Whether to run the browser in headless mode (no visible GUI). - Default: True. - browser_process (subprocess.Popen): The process object for the browser. - temp_dir (str): Temporary directory for user data if not provided. - debugging_port (int): Port for debugging the browser. - host (str): Host for debugging the browser. - - Methods: - start(): Starts the browser process and returns the CDP endpoint URL. - _get_browser_path(): Returns the browser executable path based on OS and browser type. - _get_browser_args(): Returns browser-specific command line arguments. - _get_user_data_dir(): Returns the user data directory path. - _cleanup(): Terminates the browser process and removes the temporary directory. - create_profile(): Static method to create a user profile by launching a browser for user interaction. - """ - - @staticmethod - def build_browser_flags(config: BrowserConfig) -> List[str]: - """Common CLI flags for launching Chromium""" - flags = [ - "--no-sandbox", - "--disable-dev-shm-usage", - "--no-first-run", - "--no-default-browser-check", - "--disable-infobars", - "--window-position=0,0", - "--ignore-certificate-errors", - "--ignore-certificate-errors-spki-list", - "--disable-blink-features=AutomationControlled", - "--window-position=400,0", - "--disable-renderer-backgrounding", - "--disable-ipc-flooding-protection", - "--force-color-profile=srgb", - "--mute-audio", - "--disable-background-timer-throttling", - # Memory-saving flags: disable unused Chrome features - "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider", - "--disable-component-update", - "--disable-domain-reliability", - ] - # GPU flags disable WebGL which anti-bot sensors detect as headless. - # Keep WebGL working (via SwiftShader) when stealth mode is active. - if not config.enable_stealth: - flags.extend([ - "--disable-gpu", - "--disable-gpu-compositing", - "--disable-software-rasterizer", - ]) - if config.memory_saving_mode: - flags.extend([ - "--aggressive-cache-discard", - '--js-flags=--max-old-space-size=512', - ]) - if config.light_mode: - flags.extend(BROWSER_DISABLE_OPTIONS) - if config.text_mode: - flags.extend([ - "--blink-settings=imagesEnabled=false", - "--disable-remote-fonts", - "--disable-images", - "--disable-javascript", - "--disable-software-rasterizer", - "--disable-dev-shm-usage", - ]) - # proxy support — only pass server URL, never credentials. - # Chromium's --proxy-server flag silently ignores inline user:pass@. - # Auth credentials are handled at the Playwright context level instead. - if config.proxy: - flags.append(f"--proxy-server={config.proxy}") - elif config.proxy_config: - flags.append(f"--proxy-server={config.proxy_config.server}") - # dedupe - return list(dict.fromkeys(flags)) - - browser_type: str - user_data_dir: str - headless: bool - browser_process: subprocess.Popen - temp_dir: str - debugging_port: int - host: str - - def __init__( - self, - browser_type: str = "chromium", - user_data_dir: Optional[str] = None, - headless: bool = False, - logger=None, - host: str = "localhost", - debugging_port: int = 9222, - cdp_url: Optional[str] = None, - browser_config: Optional[BrowserConfig] = None, - ): - """ - Initialize the ManagedBrowser instance. - - Args: - browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". - Default: "chromium". - user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a - temporary directory may be used. Default: None. - headless (bool): Whether to run the browser in headless mode (no visible GUI). - Default: True. - logger (logging.Logger): Logger instance for logging messages. Default: None. - host (str): Host for debugging the browser. Default: "localhost". - debugging_port (int): Port for debugging the browser. Default: 9222. - cdp_url (str or None): CDP URL to connect to the browser. Default: None. - browser_config (BrowserConfig): Configuration object containing all browser settings. Default: None. - """ - self.browser_type = browser_config.browser_type - self.user_data_dir = browser_config.user_data_dir - self.headless = browser_config.headless - self.browser_process = None - self.temp_dir = None - self.debugging_port = browser_config.debugging_port - self.host = browser_config.host - self.logger = logger - self.shutting_down = False - self.cdp_url = browser_config.cdp_url - self.browser_config = browser_config - - async def start(self) -> str: - """ - Starts the browser process or returns CDP endpoint URL. - If cdp_url is provided, returns it directly. - If user_data_dir is not provided for local browser, creates a temporary directory. - - Returns: - str: CDP endpoint URL - """ - # If CDP URL provided, just return it - if self.cdp_url: - return self.cdp_url - - # Create temp dir if needed - if not self.user_data_dir: - self.temp_dir = tempfile.mkdtemp(prefix="browser-profile-") - self.user_data_dir = self.temp_dir - - # Get browser path and args based on OS and browser type - # browser_path = self._get_browser_path() - args = await self._get_browser_args() - - if self.browser_config.extra_args: - args.extend(self.browser_config.extra_args) - - - # ── make sure no old Chromium instance is owning the same port/profile ── - try: - if sys.platform == "win32": - if psutil is None: - raise RuntimeError("psutil not available, cannot clean old browser") - for p in psutil.process_iter(["pid", "name", "cmdline"]): - cl = " ".join(p.info.get("cmdline") or []) - if ( - f"--remote-debugging-port={self.debugging_port}" in cl - and f"--user-data-dir={self.user_data_dir}" in cl - ): - p.kill() - p.wait(timeout=5) - else: # macOS / Linux - # kill any process listening on the same debugging port - try: - pids = ( - subprocess.check_output( - shlex.split(f"lsof -t -i:{self.debugging_port}"), - stderr=subprocess.DEVNULL, - ) - .decode() - .strip() - .splitlines() - ) - except (FileNotFoundError, subprocess.CalledProcessError): - pids = [] - for pid in pids: - try: - os.kill(int(pid), signal.SIGTERM) - except ProcessLookupError: - pass - - # remove Chromium singleton locks, or new launch exits with - # “Opening in existing browser session.” - for f in ("SingletonLock", "SingletonSocket", "SingletonCookie"): - fp = os.path.join(self.user_data_dir, f) - if os.path.exists(fp): - os.remove(fp) - except Exception as _e: - # non-fatal — we'll try to start anyway, but log what happened - self.logger.warning(f"pre-launch cleanup failed: {_e}", tag="BROWSER") - - - # Start browser process - try: - # Use DETACHED_PROCESS flag on Windows to fully detach the process - # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group - if sys.platform == "win32": - self.browser_process = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP - ) - else: - self.browser_process = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn=os.setpgrp # Start in a new process group - ) - - # If verbose is True print args used to run the process - if self.logger and self.browser_config.verbose: - self.logger.debug( - f"Starting browser with args: {' '.join(args)}", - tag="BROWSER" - ) - - # We'll monitor for a short time to make sure it starts properly, but won't keep monitoring - await asyncio.sleep(0.5) # Give browser time to start - await self._initial_startup_check() - await asyncio.sleep(2) # Give browser time to start - return f"http://{self.host}:{self.debugging_port}" - except Exception as e: - await self.cleanup() - raise Exception(f"Failed to start browser: {e}") - - async def _initial_startup_check(self): - """ - Perform a quick check to make sure the browser started successfully. - This only runs once at startup rather than continuously monitoring. - """ - if not self.browser_process: - return - - # Check that process started without immediate termination - await asyncio.sleep(0.5) - if self.browser_process.poll() is not None: - # Process already terminated - stdout, stderr = b"", b"" - try: - stdout, stderr = self.browser_process.communicate(timeout=0.5) - except subprocess.TimeoutExpired: - pass - - self.logger.error( - message="Browser process terminated during startup | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}", - tag="ERROR", - params={ - "code": self.browser_process.returncode, - "stdout": stdout.decode() if stdout else "", - "stderr": stderr.decode() if stderr else "", - }, - ) - - async def _monitor_browser_process(self): - """ - Monitor the browser process for unexpected termination. - - How it works: - 1. Read stdout and stderr from the browser process. - 2. If the process has terminated, log the error message and terminate the browser. - 3. If the shutting_down flag is set, log the normal termination message. - 4. If any other error occurs, log the error message. - - Note: This method should be called in a separate task to avoid blocking the main event loop. - This is DEPRECATED and should not be used for builtin browsers that need to outlive the Python process. - """ - if self.browser_process: - try: - stdout, stderr = await asyncio.gather( - asyncio.to_thread(self.browser_process.stdout.read), - asyncio.to_thread(self.browser_process.stderr.read), - ) - - # Check shutting_down flag BEFORE logging anything - if self.browser_process.poll() is not None: - if not self.shutting_down: - self.logger.error( - message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}", - tag="ERROR", - params={ - "code": self.browser_process.returncode, - "stdout": stdout.decode(), - "stderr": stderr.decode(), - }, - ) - await self.cleanup() - else: - self.logger.info( - message="Browser process terminated normally | Code: {code}", - tag="INFO", - params={"code": self.browser_process.returncode}, - ) - except Exception as e: - if not self.shutting_down: - self.logger.error( - message="Error monitoring browser process: {error}", - tag="ERROR", - params={"error": str(e)}, - ) - - def _get_browser_path_WIP(self) -> str: - """Returns the browser executable path based on OS and browser type""" - if sys.platform == "darwin": # macOS - paths = { - "chromium": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox", - "webkit": "/Applications/Safari.app/Contents/MacOS/Safari", - } - elif sys.platform == "win32": # Windows - paths = { - "chromium": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", - "firefox": "C:\\Program Files\\Mozilla Firefox\\firefox.exe", - "webkit": None, # WebKit not supported on Windows - } - else: # Linux - paths = { - "chromium": "google-chrome", - "firefox": "firefox", - "webkit": None, # WebKit not supported on Linux - } - - return paths.get(self.browser_type) - - async def _get_browser_path(self) -> str: - browser_path = await get_chromium_path(self.browser_type) - return browser_path - - async def _get_browser_args(self) -> List[str]: - """Returns full CLI args for launching the browser""" - base = [await self._get_browser_path()] - if self.browser_type == "chromium": - flags = [ - f"--remote-debugging-port={self.debugging_port}", - f"--user-data-dir={self.user_data_dir}", - ] - if self.headless: - flags.append("--headless=new") - # Add viewport flag if specified in config - if self.browser_config.viewport_height and self.browser_config.viewport_width: - flags.append(f"--window-size={self.browser_config.viewport_width},{self.browser_config.viewport_height}") - # merge common launch flags - flags.extend(self.build_browser_flags(self.browser_config)) - elif self.browser_type == "firefox": - flags = [ - "--remote-debugging-port", - str(self.debugging_port), - "--profile", - self.user_data_dir, - ] - if self.headless: - flags.append("--headless") - else: - raise NotImplementedError(f"Browser type {self.browser_type} not supported") - return base + flags - - async def cleanup(self): - """Cleanup browser process and temporary directory""" - # Set shutting_down flag BEFORE any termination actions - self.shutting_down = True - - if self.browser_process: - try: - # For builtin browsers that should persist, we should check if it's a detached process - # Only terminate if we have proper control over the process - if not self.browser_process.poll(): - # Process is still running - self.browser_process.terminate() - # Wait for process to end gracefully - for _ in range(10): # 10 attempts, 100ms each - if self.browser_process.poll() is not None: - break - await asyncio.sleep(0.1) - - # Force kill if still running - if self.browser_process.poll() is None: - if sys.platform == "win32": - # On Windows, use taskkill /T to kill the entire process tree - try: - subprocess.run(["taskkill", "/F", "/T", "/PID", str(self.browser_process.pid)]) - except Exception: - self.browser_process.kill() - else: - # On Unix, kill entire process group to reap child processes - try: - os.killpg(os.getpgid(self.browser_process.pid), signal.SIGKILL) - except (ProcessLookupError, OSError): - pass - await asyncio.sleep(0.1) # Brief wait for kill to take effect - - except Exception as e: - self.logger.error( - message="Error terminating browser: {error}", - tag="ERROR", - params={"error": str(e)}, - ) - - if self.temp_dir and os.path.exists(self.temp_dir): - try: - shutil.rmtree(self.temp_dir) - except Exception as e: - self.logger.error( - message="Error removing temporary directory: {error}", - tag="ERROR", - params={"error": str(e)}, - ) - - # These methods have been moved to BrowserProfiler class - @staticmethod - async def create_profile(browser_config=None, profile_name=None, logger=None): - """ - This method has been moved to the BrowserProfiler class. - - Creates a browser profile by launching a browser for interactive user setup - and waits until the user closes it. The profile is stored in a directory that - can be used later with BrowserConfig.user_data_dir. - - Please use BrowserProfiler.create_profile() instead. - - Example: - ```python - from crawl4ai.browser_profiler import BrowserProfiler - - profiler = BrowserProfiler() - profile_path = await profiler.create_profile(profile_name="my-login-profile") - ``` - """ - from .browser_profiler import BrowserProfiler - - # Create a BrowserProfiler instance and delegate to it - profiler = BrowserProfiler(logger=logger) - return await profiler.create_profile(profile_name=profile_name, browser_config=browser_config) - - @staticmethod - def list_profiles(): - """ - This method has been moved to the BrowserProfiler class. - - Lists all available browser profiles in the Crawl4AI profiles directory. - - Please use BrowserProfiler.list_profiles() instead. - - Example: - ```python - from crawl4ai.browser_profiler import BrowserProfiler - - profiler = BrowserProfiler() - profiles = profiler.list_profiles() - ``` - """ - from .browser_profiler import BrowserProfiler - - # Create a BrowserProfiler instance and delegate to it - profiler = BrowserProfiler() - return profiler.list_profiles() - - @staticmethod - def delete_profile(profile_name_or_path): - """ - This method has been moved to the BrowserProfiler class. - - Delete a browser profile by name or path. - - Please use BrowserProfiler.delete_profile() instead. - - Example: - ```python - from crawl4ai.browser_profiler import BrowserProfiler - - profiler = BrowserProfiler() - success = profiler.delete_profile("my-profile") - ``` - """ - from .browser_profiler import BrowserProfiler - - # Create a BrowserProfiler instance and delegate to it - profiler = BrowserProfiler() - return profiler.delete_profile(profile_name_or_path) - - -async def clone_runtime_state( - src: BrowserContext, - dst: BrowserContext, - crawlerRunConfig: CrawlerRunConfig | None = None, - browserConfig: BrowserConfig | None = None, -) -> None: - """ - Bring everything that *can* be changed at runtime from `src` → `dst`. - - 1. Cookies - 2. localStorage (and sessionStorage, same API) - 3. Extra headers, permissions, geolocation if supplied in configs - """ - - # ── 1. cookies ──────────────────────────────────────────────────────────── - cookies = await src.cookies() - if cookies: - await dst.add_cookies(cookies) - - # ── 2. localStorage / sessionStorage ────────────────────────────────────── - state = await src.storage_state() - for origin in state.get("origins", []): - url = origin["origin"] - kvs = origin.get("localStorage", []) - if not kvs: - continue - - page = dst.pages[0] if dst.pages else await dst.new_page() - await page.goto(url, wait_until="domcontentloaded") - for k, v in kvs: - await page.evaluate("(k,v)=>localStorage.setItem(k,v)", k, v) - - # ── 3. runtime-mutable extras from configs ──────────────────────────────── - # headers - if browserConfig and browserConfig.headers: - await dst.set_extra_http_headers(browserConfig.headers) - - # geolocation - if crawlerRunConfig and crawlerRunConfig.geolocation: - await dst.grant_permissions(["geolocation"]) - await dst.set_geolocation( - { - "latitude": crawlerRunConfig.geolocation.latitude, - "longitude": crawlerRunConfig.geolocation.longitude, - "accuracy": crawlerRunConfig.geolocation.accuracy, - } - ) - - return dst - - - -class _CDPConnectionCache: - """ - Class-level cache for Playwright + CDP browser connections. - - When enabled via BrowserConfig(cache_cdp_connection=True), multiple - BrowserManager instances connecting to the same cdp_url will share - a single Playwright subprocess and CDP WebSocket. Reference-counted; - the connection is closed when the last user releases it. - """ - - _cache: Dict[str, Tuple] = {} # cdp_url -> (playwright, browser, ref_count) - _lock: Optional[asyncio.Lock] = None # lazy-init to avoid event loop issues - _lock_loop: Optional[asyncio.AbstractEventLoop] = None - - @classmethod - def _get_lock(cls) -> asyncio.Lock: - loop = asyncio.get_running_loop() - if cls._lock is None or cls._lock_loop is not loop: - cls._lock = asyncio.Lock() - cls._lock_loop = loop - return cls._lock - - @classmethod - async def acquire(cls, cdp_url: str, use_undetected: bool = False): - """Get or create a cached (playwright, browser) for this cdp_url.""" - async with cls._get_lock(): - if cdp_url in cls._cache: - pw, browser, count = cls._cache[cdp_url] - if browser.is_connected(): - cls._cache[cdp_url] = (pw, browser, count + 1) - return pw, browser - # Stale connection — clean up and fall through to create new - try: - await pw.stop() - except Exception: - pass - del cls._cache[cdp_url] - - # Create new connection - if use_undetected: - from patchright.async_api import async_playwright - else: - from playwright.async_api import async_playwright - pw = await async_playwright().start() - browser = await pw.chromium.connect_over_cdp(cdp_url) - cls._cache[cdp_url] = (pw, browser, 1) - return pw, browser - - @classmethod - async def release(cls, cdp_url: str): - """Decrement ref count; close connection when last user releases.""" - async with cls._get_lock(): - if cdp_url not in cls._cache: - return - pw, browser, count = cls._cache[cdp_url] - if count <= 1: - try: - await browser.close() - except Exception: - pass - try: - await pw.stop() - except Exception: - pass - del cls._cache[cdp_url] - else: - cls._cache[cdp_url] = (pw, browser, count - 1) - - @classmethod - async def close_all(cls): - """Force-close all cached connections. Call on application shutdown.""" - async with cls._get_lock(): - for cdp_url in list(cls._cache.keys()): - pw, browser, _ = cls._cache[cdp_url] - try: - await browser.close() - except Exception: - pass - try: - await pw.stop() - except Exception: - pass - cls._cache.clear() - - -class BrowserManager: - """ - Manages the browser instance and context. - - Attributes: - config (BrowserConfig): Configuration object containing all browser settings - logger: Logger instance for recording events and errors - browser (Browser): The browser instance - default_context (BrowserContext): The default browser context - managed_browser (ManagedBrowser): The managed browser instance - playwright (Playwright): The Playwright instance - sessions (dict): Dictionary to store session information - session_ttl (int): Session timeout in seconds - """ - - _playwright_instance = None - - # Class-level tracking of pages in use, keyed by browser endpoint (CDP URL or instance id) - # This ensures multiple BrowserManager instances connecting to the same browser - # share the same page tracking, preventing race conditions. - _global_pages_in_use: dict = {} # endpoint_key -> set of pages - _global_pages_lock: asyncio.Lock = None # Initialized lazily - - @classmethod - def _get_global_lock(cls) -> asyncio.Lock: - """Get or create the global pages lock (lazy initialization for async context).""" - if cls._global_pages_lock is None: - cls._global_pages_lock = asyncio.Lock() - return cls._global_pages_lock - - @classmethod - async def get_playwright(cls, use_undetected: bool = False): - if use_undetected: - from patchright.async_api import async_playwright - else: - from playwright.async_api import async_playwright - cls._playwright_instance = await async_playwright().start() - return cls._playwright_instance - - def __init__(self, browser_config: BrowserConfig, logger=None, use_undetected: bool = False): - """ - Initialize the BrowserManager with a browser configuration. - - Args: - browser_config (BrowserConfig): Configuration object containing all browser settings - logger: Logger instance for recording events and errors - use_undetected (bool): Whether to use undetected browser (Patchright) - """ - self.config: BrowserConfig = browser_config - self.logger = logger - self.use_undetected = use_undetected - - # Browser state - self.browser = None - self.default_context = None - self.managed_browser = None - self.playwright = None - self._using_cached_cdp = False - self._launched_persistent = False # True when using launch_persistent_context - - # Session management - self.sessions = {} - self.session_ttl = 1800 # 30 minutes - - # Keep track of contexts by a "config signature," so each unique config reuses a single context - self.contexts_by_config = {} - self._contexts_lock = asyncio.Lock() - - # Context lifecycle tracking for LRU eviction - self._context_refcounts = {} # sig -> int (active crawls using this context) - self._context_last_used = {} # sig -> float (monotonic timestamp for LRU) - self._page_to_sig = {} # page -> sig (for decrement lookup on release) - self._max_contexts = 20 # LRU eviction threshold - - # Serialize context.new_page() across concurrent tasks to avoid races - # when using a shared persistent context (context.pages may be empty - # for all racers). Prevents 'Target page/context closed' errors. - self._page_lock = asyncio.Lock() - - # Browser endpoint key for global page tracking (set after browser starts) - self._browser_endpoint_key: Optional[str] = None - - # Browser recycling state (version-based approach) - self._pages_served = 0 - self._browser_version = 1 # included in signature, bump to create new browser - self._pending_cleanup = {} # old_sig -> {"browser": browser, "contexts": [...], "done": Event} - self._pending_cleanup_lock = asyncio.Lock() - self._max_pending_browsers = 3 # safety cap — block if too many draining - self._cleanup_slot_available = asyncio.Event() - self._cleanup_slot_available.set() # starts open - - # Stealth adapter for stealth mode - self._stealth_adapter = None - if self.config.enable_stealth and not self.use_undetected: - from .browser_adapter import StealthAdapter - self._stealth_adapter = StealthAdapter() - - # Initialize ManagedBrowser if needed - if self.config.use_managed_browser: - self.managed_browser = ManagedBrowser( - browser_type=self.config.browser_type, - user_data_dir=self.config.user_data_dir, - headless=self.config.headless, - logger=self.logger, - debugging_port=self.config.debugging_port, - cdp_url=self.config.cdp_url, - browser_config=self.config, - ) - - async def start(self): - """ - Start the browser instance and set up the default context. - - How it works: - 1. Check if Playwright is already initialized. - 2. If not, initialize Playwright. - 3. If managed browser is used, start it and connect to the CDP endpoint. - 4. If managed browser is not used, launch the browser and set up the default context. - - Note: This method should be called in a separate task to avoid blocking the main event loop. - """ - if self.playwright is not None: - await self.close() - - # Use cached CDP connection if enabled and cdp_url is set - if self.config.cache_cdp_connection and self.config.cdp_url: - self._using_cached_cdp = True - self.config.use_managed_browser = True - self.playwright, self.browser = await _CDPConnectionCache.acquire( - self.config.cdp_url, self.use_undetected - ) - else: - self._using_cached_cdp = False - if self.use_undetected: - from patchright.async_api import async_playwright - else: - from playwright.async_api import async_playwright - - # Initialize playwright - self.playwright = await async_playwright().start() - - try: - await self._launch_browser() - except Exception: - # If browser launch fails, ensure Playwright process is cleaned up - # to avoid leaking the "node cli.js run-driver" subprocess. - if self.playwright is not None and not self._using_cached_cdp: - try: - await self.playwright.stop() - except Exception: - pass - self.playwright = None - raise - - async def _launch_browser(self): - """Launch or connect to the browser after Playwright is initialized.""" - # ── Persistent context via Playwright's native API ────────────── - # When use_persistent_context is set and we're not connecting to an - # external CDP endpoint, use launch_persistent_context() instead of - # subprocess + CDP. This properly supports proxy authentication - # (server + username + password) which the --proxy-server CLI flag - # cannot handle. - if ( - self.config.use_persistent_context - and not self.config.cdp_url - and not self._using_cached_cdp - ): - # Collect stealth / optimization CLI flags, excluding ones that - # launch_persistent_context handles via keyword arguments. - _skip_prefixes = ( - "--proxy-server", - "--remote-debugging-port", - "--user-data-dir", - "--headless", - "--window-size", - ) - cli_args = [ - flag - for flag in ManagedBrowser.build_browser_flags(self.config) - if not flag.startswith(_skip_prefixes) - ] - if self.config.extra_args: - cli_args.extend(self.config.extra_args) - - launch_kwargs = { - "headless": self.config.headless, - "args": list(dict.fromkeys(cli_args)), # dedupe - "viewport": { - "width": self.config.viewport_width, - "height": self.config.viewport_height, - }, - "user_agent": self.config.user_agent or None, - "ignore_https_errors": self.config.ignore_https_errors, - "accept_downloads": self.config.accept_downloads, - } - - if self.config.proxy_config: - launch_kwargs["proxy"] = { - "server": self.config.proxy_config.server, - "username": self.config.proxy_config.username, - "password": self.config.proxy_config.password, - } - - if self.config.storage_state: - launch_kwargs["storage_state"] = self.config.storage_state - - user_data_dir = self.config.user_data_dir or tempfile.mkdtemp( - prefix="crawl4ai-persistent-" - ) - - self.default_context = ( - await self.playwright.chromium.launch_persistent_context( - user_data_dir, **launch_kwargs - ) - ) - self.browser = None # persistent context has no separate Browser - self._launched_persistent = True - - await self.setup_context(self.default_context) - - # Set the browser endpoint key for global page tracking - self._browser_endpoint_key = self._compute_browser_endpoint_key() - if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: - BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() - return - - if self.config.cdp_url or self.config.use_managed_browser: - self.config.use_managed_browser = True - - if not self._using_cached_cdp: - cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url - - # Add CDP endpoint verification before connecting - if not await self._verify_cdp_ready(cdp_url): - raise Exception(f"CDP endpoint at {cdp_url} is not ready after startup") - - self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url) - - contexts = self.browser.contexts - - # If browser_context_id is provided, we're using a pre-created context - if self.config.browser_context_id: - if self.logger: - self.logger.debug( - f"Using pre-existing browser context: {self.config.browser_context_id}", - tag="BROWSER" - ) - # When connecting to a pre-created context, it should be in contexts - if contexts: - self.default_context = contexts[0] - if self.logger: - self.logger.debug( - f"Found {len(contexts)} existing context(s), using first one", - tag="BROWSER" - ) - else: - # Context was created but not yet visible - wait a bit - await asyncio.sleep(0.2) - contexts = self.browser.contexts - if contexts: - self.default_context = contexts[0] - else: - # Still no contexts - this shouldn't happen with pre-created context - if self.logger: - self.logger.warning( - "Pre-created context not found, creating new one", - tag="BROWSER" - ) - self.default_context = await self.create_browser_context() - elif contexts: - self.default_context = contexts[0] - else: - self.default_context = await self.create_browser_context() - await self.setup_context(self.default_context) - else: - browser_args = self._build_browser_args() - - # Launch appropriate browser type - if self.config.browser_type == "firefox": - self.browser = await self.playwright.firefox.launch(**browser_args) - elif self.config.browser_type == "webkit": - self.browser = await self.playwright.webkit.launch(**browser_args) - else: - self.browser = await self.playwright.chromium.launch(**browser_args) - - self.default_context = self.browser - - # Set the browser endpoint key for global page tracking - self._browser_endpoint_key = self._compute_browser_endpoint_key() - # Initialize global tracking set for this endpoint if needed - if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: - BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() - - def _compute_browser_endpoint_key(self) -> str: - """ - Compute a unique key identifying this browser connection. - - For CDP connections, uses the normalized CDP URL so all BrowserManager - instances connecting to the same browser share page tracking. - For standalone browsers, uses instance id since each is independent. - - Returns: - str: Unique identifier for this browser connection - """ - # For CDP connections, use the CDP URL as the key (normalized) - if self.config.cdp_url: - return self._normalize_cdp_url(self.config.cdp_url) - - # For managed browsers, use the CDP URL/port that was assigned - if self.managed_browser: - # Use debugging port as the key since it uniquely identifies the browser - port = getattr(self.managed_browser, 'debugging_port', None) - host = getattr(self.managed_browser, 'host', 'localhost') - if port: - return f"cdp:http://{host}:{port}" - - # For standalone browsers, use instance id (no sharing needed) - return f"instance:{id(self)}" - - def _normalize_cdp_url(self, cdp_url: str) -> str: - """ - Normalize a CDP URL to a canonical form for consistent tracking. - - Handles various formats: - - http://localhost:9222 - - ws://localhost:9222/devtools/browser/xxx - - http://localhost:9222?browser_id=xxx - - Returns: - str: Normalized CDP key in format "cdp:http://host:port" - """ - from urllib.parse import urlparse - - parsed = urlparse(cdp_url) - host = parsed.hostname or 'localhost' - port = parsed.port or 9222 - - return f"cdp:http://{host}:{port}" - - def _get_pages_in_use(self) -> set: - """Get the set of pages currently in use for this browser.""" - if self._browser_endpoint_key and self._browser_endpoint_key in BrowserManager._global_pages_in_use: - return BrowserManager._global_pages_in_use[self._browser_endpoint_key] - # Fallback: shouldn't happen, but return empty set - return set() - - def _mark_page_in_use(self, page) -> None: - """Mark a page as in use.""" - if self._browser_endpoint_key: - if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: - BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() - BrowserManager._global_pages_in_use[self._browser_endpoint_key].add(page) - - def _release_page_from_use(self, page) -> None: - """Release a page from the in-use tracking.""" - if self._browser_endpoint_key and self._browser_endpoint_key in BrowserManager._global_pages_in_use: - BrowserManager._global_pages_in_use[self._browser_endpoint_key].discard(page) - - async def _verify_cdp_ready(self, cdp_url: str) -> bool: - """Verify CDP endpoint is ready with exponential backoff. - - Supports multiple URL formats: - - HTTP URLs: http://localhost:9222 - - HTTP URLs with query params: http://localhost:9222?browser_id=XXX - - WebSocket URLs: ws://localhost:9222/devtools/browser/XXX - """ - import aiohttp - from urllib.parse import urlparse, urlunparse - - # If WebSocket URL, Playwright handles connection directly - skip HTTP verification - if cdp_url.startswith(('ws://', 'wss://')): - self.logger.debug(f"WebSocket CDP URL provided, skipping HTTP verification", tag="BROWSER") - return True - - # Parse HTTP URL and properly construct /json/version endpoint - parsed = urlparse(cdp_url) - # Build URL with /json/version path, preserving query params - verify_url = urlunparse(( - parsed.scheme, - parsed.netloc, - '/json/version', # Always use this path for verification - '', # params - parsed.query, # preserve query string - '' # fragment - )) - - self.logger.debug(f"Starting CDP verification for {verify_url}", tag="BROWSER") - for attempt in range(5): - try: - async with aiohttp.ClientSession() as session: - async with session.get(verify_url, timeout=aiohttp.ClientTimeout(total=2)) as response: - if response.status == 200: - self.logger.debug(f"CDP endpoint ready after {attempt + 1} attempts", tag="BROWSER") - return True - except Exception as e: - self.logger.debug(f"CDP check attempt {attempt + 1} failed: {e}", tag="BROWSER") - delay = 0.5 * (1.4 ** attempt) - self.logger.debug(f"Waiting {delay:.2f}s before next CDP check...", tag="BROWSER") - await asyncio.sleep(delay) - self.logger.debug(f"CDP verification failed after 5 attempts", tag="BROWSER") - return False - - def _build_browser_args(self) -> dict: - """Build browser launch arguments from config.""" - args = [ - "--disable-gpu", - "--disable-gpu-compositing", - "--disable-software-rasterizer", - "--no-sandbox", - "--disable-dev-shm-usage", - "--no-first-run", - "--no-default-browser-check", - "--disable-infobars", - "--window-position=0,0", - "--ignore-certificate-errors", - "--ignore-certificate-errors-spki-list", - "--disable-blink-features=AutomationControlled", - "--window-position=400,0", - "--disable-renderer-backgrounding", - "--disable-ipc-flooding-protection", - "--force-color-profile=srgb", - "--mute-audio", - "--disable-background-timer-throttling", - # Memory-saving flags: disable unused Chrome features - "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider", - "--disable-component-update", - "--disable-domain-reliability", - # "--single-process", - f"--window-size={self.config.viewport_width},{self.config.viewport_height}", - ] - - if self.config.memory_saving_mode: - args.extend([ - "--aggressive-cache-discard", - '--js-flags=--max-old-space-size=512', - ]) - - if self.config.light_mode: - args.extend(BROWSER_DISABLE_OPTIONS) - - if self.config.text_mode: - args.extend( - [ - "--blink-settings=imagesEnabled=false", - "--disable-remote-fonts", - "--disable-images", - "--disable-javascript", - "--disable-software-rasterizer", - "--disable-dev-shm-usage", - ] - ) - - if self.config.extra_args: - args.extend(self.config.extra_args) - - # Deduplicate args - args = list(dict.fromkeys(args)) - - browser_args = {"headless": self.config.headless, "args": args} - - # On Windows, passing channel='chromium' (the default) causes Playwright - # to look for a system Chrome installation instead of using the bundled - # ms-playwright binary. This makes Chrome exit immediately with code 0, - # resulting in TargetClosedError. Skip the default channel. - if self.config.chrome_channel and self.config.chrome_channel != "chromium": - browser_args["channel"] = self.config.chrome_channel - - if self.config.accept_downloads: - browser_args["downloads_path"] = self.config.downloads_path or os.path.join( - os.getcwd(), "downloads" - ) - os.makedirs(browser_args["downloads_path"], exist_ok=True) - - if self.config.proxy: - warnings.warn( - "BrowserConfig.proxy is deprecated and ignored. Use proxy_config instead.", - DeprecationWarning, - ) - if self.config.proxy_config: - from playwright.async_api import ProxySettings - - proxy_settings = ProxySettings( - server=self.config.proxy_config.server, - username=self.config.proxy_config.username, - password=self.config.proxy_config.password, - ) - browser_args["proxy"] = proxy_settings - - return browser_args - - async def setup_context( - self, - context: BrowserContext, - crawlerRunConfig: CrawlerRunConfig = None, - is_default=False, - ): - """ - Set up a browser context with the configured options. - - How it works: - 1. Set extra HTTP headers if provided. - 2. Add cookies if provided. - 3. Load storage state if provided. - 4. Accept downloads if enabled. - 5. Set default timeouts for navigation and download. - 6. Set user agent if provided. - 7. Set browser hints if provided. - 8. Set proxy if provided. - 9. Set downloads path if provided. - 10. Set storage state if provided. - 11. Set cache if provided. - 12. Set extra HTTP headers if provided. - 13. Add cookies if provided. - 14. Set default timeouts for navigation and download if enabled. - 15. Set user agent if provided. - 16. Set browser hints if provided. - - Args: - context (BrowserContext): The browser context to set up - crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings - is_default (bool): Flag indicating if this is the default context - Returns: - None - """ - if self.config.headers: - await context.set_extra_http_headers(self.config.headers) - - if self.config.cookies: - await context.add_cookies(self.config.cookies) - - if self.config.storage_state: - await context.storage_state(path=None) - - if self.config.accept_downloads: - context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT) - context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT) - if self.config.downloads_path: - context._impl_obj._options["accept_downloads"] = True - context._impl_obj._options[ - "downloads_path" - ] = self.config.downloads_path - - # Handle user agent and browser hints - if self.config.user_agent: - combined_headers = { - "User-Agent": self.config.user_agent, - "sec-ch-ua": self.config.browser_hint, - } - combined_headers.update(self.config.headers) - await context.set_extra_http_headers(combined_headers) - - # Add default cookie (skip for raw:/file:// URLs which are not valid cookie URLs) - cookie_url = None - if crawlerRunConfig and crawlerRunConfig.url: - url = crawlerRunConfig.url - # Only set cookie for http/https URLs - if url.startswith(("http://", "https://")): - cookie_url = url - elif crawlerRunConfig.base_url and crawlerRunConfig.base_url.startswith(("http://", "https://")): - # Use base_url as fallback for raw:/file:// URLs - cookie_url = crawlerRunConfig.base_url - - if cookie_url: - await context.add_cookies( - [ - { - "name": "cookiesEnabled", - "value": "true", - "url": cookie_url, - } - ] - ) - - # Handle navigator overrides - if crawlerRunConfig: - if ( - crawlerRunConfig.override_navigator - or crawlerRunConfig.simulate_user - or crawlerRunConfig.magic - ): - await context.add_init_script(load_js_script("navigator_overrider")) - context._crawl4ai_nav_overrider_injected = True - - # Force-open closed shadow roots when flatten_shadow_dom is enabled - if crawlerRunConfig and crawlerRunConfig.flatten_shadow_dom: - await context.add_init_script(""" - const _origAttachShadow = Element.prototype.attachShadow; - Element.prototype.attachShadow = function(init) { - return _origAttachShadow.call(this, {...init, mode: 'open'}); - }; - """) - context._crawl4ai_shadow_dom_injected = True - - # Apply custom init_scripts from BrowserConfig (for stealth evasions, etc.) - if self.config.init_scripts: - for script in self.config.init_scripts: - await context.add_init_script(script) - - async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None): - """ - Creates and returns a new browser context with configured settings. - Applies text-only mode settings if text_mode is enabled in config. - - Returns: - Context: Browser context object with the specified configurations - """ - if self.browser is None: - if self._launched_persistent: - raise RuntimeError( - "Cannot create new browser contexts when using " - "use_persistent_context=True. Persistent context uses a " - "single shared context." - ) - raise RuntimeError( - "Browser is not available. It may have been closed, crashed, " - "or not yet started. Ensure the browser is running before " - "creating new contexts." - ) - # Base settings - user_agent = self.config.headers.get("User-Agent", self.config.user_agent) - viewport_settings = { - "width": self.config.viewport_width, - "height": self.config.viewport_height, - } - proxy_settings = {"server": self.config.proxy} if self.config.proxy else None - - # CSS extensions (blocked separately via avoid_css flag) - css_extensions = ["css", "less", "scss", "sass"] - - # Static resource extensions (blocked when text_mode is enabled) - static_extensions = [ - # Images - "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff", "psd", - # Fonts - "woff", "woff2", "ttf", "otf", "eot", - # Media - "mp4", "webm", "ogg", "avi", "mov", "wmv", "flv", "m4v", - "mp3", "wav", "aac", "m4a", "opus", "flac", - # Documents - "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", - # Archives - "zip", "rar", "7z", "tar", "gz", - # Scripts and data - "xml", "swf", "wasm", - ] - - # Ad and tracker domain patterns (curated from uBlock/EasyList sources) - ad_tracker_patterns = [ - "**/google-analytics.com/**", - "**/googletagmanager.com/**", - "**/googlesyndication.com/**", - "**/doubleclick.net/**", - "**/adservice.google.com/**", - "**/adsystem.com/**", - "**/adzerk.net/**", - "**/adnxs.com/**", - "**/ads.linkedin.com/**", - "**/facebook.net/**", - "**/analytics.twitter.com/**", - "**/ads-twitter.com/**", - "**/hotjar.com/**", - "**/clarity.ms/**", - "**/scorecardresearch.com/**", - "**/pixel.wp.com/**", - "**/amazon-adsystem.com/**", - "**/mixpanel.com/**", - "**/segment.com/**", - ] - - # Common context settings - context_settings = { - "user_agent": user_agent, - "viewport": viewport_settings, - "proxy": proxy_settings, - "accept_downloads": self.config.accept_downloads, - "storage_state": self.config.storage_state, - "ignore_https_errors": self.config.ignore_https_errors, - "device_scale_factor": self.config.device_scale_factor, - "java_script_enabled": self.config.java_script_enabled, - } - - if crawlerRunConfig: - # Check if there is value for crawlerRunConfig.proxy_config set add that to context - if crawlerRunConfig.proxy_config: - from playwright.async_api import ProxySettings - proxy_settings = ProxySettings( - server=crawlerRunConfig.proxy_config.server, - username=crawlerRunConfig.proxy_config.username, - password=crawlerRunConfig.proxy_config.password, - ) - context_settings["proxy"] = proxy_settings - - if self.config.text_mode: - text_mode_settings = { - "has_touch": False, - "is_mobile": False, - } - # Update context settings with text mode settings - context_settings.update(text_mode_settings) - - # inject locale / tz / geo if user provided them - if crawlerRunConfig: - if crawlerRunConfig.locale: - context_settings["locale"] = crawlerRunConfig.locale - if crawlerRunConfig.timezone_id: - context_settings["timezone_id"] = crawlerRunConfig.timezone_id - if crawlerRunConfig.geolocation: - context_settings["geolocation"] = { - "latitude": crawlerRunConfig.geolocation.latitude, - "longitude": crawlerRunConfig.geolocation.longitude, - "accuracy": crawlerRunConfig.geolocation.accuracy, - } - # ensure geolocation permission - perms = context_settings.get("permissions", []) - perms.append("geolocation") - context_settings["permissions"] = perms - - # Create and return the context with all settings - context = await self.browser.new_context(**context_settings) - - # Build dynamic blocking list based on config flags - to_block = [] - if self.config.avoid_css: - to_block.extend(css_extensions) - if self.config.text_mode: - to_block.extend(static_extensions) - - if to_block: - for ext in to_block: - await context.route(f"**/*.{ext}", lambda route: route.abort()) - - if self.config.avoid_ads: - for pattern in ad_tracker_patterns: - await context.route(pattern, lambda route: route.abort()) - - return context - - def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str: - """ - Hash ONLY the CrawlerRunConfig fields that affect browser context - creation (create_browser_context) or context setup (setup_context). - - Whitelist approach: fields like css_selector, word_count_threshold, - screenshot, verbose, etc. do NOT cause a new context to be created. - """ - import json - - sig_dict = {} - - # Fields that flow into create_browser_context() - pc = crawlerRunConfig.proxy_config - if pc is not None: - sig_dict["proxy_config"] = { - "server": getattr(pc, "server", None), - "username": getattr(pc, "username", None), - "password": getattr(pc, "password", None), - } - else: - sig_dict["proxy_config"] = None - - sig_dict["locale"] = crawlerRunConfig.locale - sig_dict["timezone_id"] = crawlerRunConfig.timezone_id - - geo = crawlerRunConfig.geolocation - if geo is not None: - sig_dict["geolocation"] = { - "latitude": geo.latitude, - "longitude": geo.longitude, - "accuracy": geo.accuracy, - } - else: - sig_dict["geolocation"] = None - - # Fields that flow into setup_context() as init scripts - sig_dict["override_navigator"] = crawlerRunConfig.override_navigator - sig_dict["simulate_user"] = crawlerRunConfig.simulate_user - sig_dict["magic"] = crawlerRunConfig.magic - - # Browser version — bumped on recycle to force new browser instance - sig_dict["_browser_version"] = self._browser_version - - signature_json = json.dumps(sig_dict, sort_keys=True, default=str) - return hashlib.sha256(signature_json.encode("utf-8")).hexdigest() - - def _evict_lru_context_locked(self): - """ - If contexts exceed the limit, find the least-recently-used context - with zero active crawls and remove it from all tracking dicts. - - MUST be called while holding self._contexts_lock. - - Returns the BrowserContext to close (caller closes it OUTSIDE the - lock), or None if no eviction is needed or possible. - """ - if len(self.contexts_by_config) <= self._max_contexts: - return None - - # Sort candidates by last-used timestamp (oldest first) - candidates = sorted( - self._context_last_used.items(), - key=lambda item: item[1], - ) - for evict_sig, _ in candidates: - if self._context_refcounts.get(evict_sig, 0) == 0: - ctx = self.contexts_by_config.pop(evict_sig, None) - self._context_refcounts.pop(evict_sig, None) - self._context_last_used.pop(evict_sig, None) - # Clean up stale page->sig mappings for evicted context - stale_pages = [ - p for p, s in self._page_to_sig.items() if s == evict_sig - ] - for p in stale_pages: - del self._page_to_sig[p] - return ctx - - # All contexts are in active use — cannot evict - return None - - async def _apply_stealth_to_page(self, page): - """Apply stealth to a page if stealth mode is enabled""" - if self._stealth_adapter: - try: - await self._stealth_adapter.apply_stealth(page) - except Exception as e: - if self.logger: - self.logger.warning( - message="Failed to apply stealth to page: {error}", - tag="STEALTH", - params={"error": str(e)} - ) - - async def _get_page_by_target_id(self, context: BrowserContext, target_id: str): - """ - Get an existing page by its CDP target ID. - - This is used when connecting to a pre-created browser context with an existing page. - Playwright may not immediately see targets created via raw CDP commands, so we - use CDP to get all targets and find the matching one. - - Args: - context: The browser context to search in - target_id: The CDP target ID to find - - Returns: - Page object if found, None otherwise - """ - try: - # First check if Playwright already sees the page - for page in context.pages: - # Playwright's internal target ID might match - if hasattr(page, '_impl_obj') and hasattr(page._impl_obj, '_target_id'): - if page._impl_obj._target_id == target_id: - return page - - # If not found, try using CDP to get targets - if hasattr(self.browser, '_impl_obj') and hasattr(self.browser._impl_obj, '_connection'): - cdp_session = await context.new_cdp_session(context.pages[0] if context.pages else None) - if cdp_session: - try: - result = await cdp_session.send("Target.getTargets") - targets = result.get("targetInfos", []) - for target in targets: - if target.get("targetId") == target_id: - # Found the target - if it's a page type, we can use it - if target.get("type") == "page": - # The page exists, let Playwright discover it - await asyncio.sleep(0.1) - # Refresh pages list - if context.pages: - return context.pages[0] - finally: - await cdp_session.detach() - - # Fallback: if there are any pages now, return the first one - if context.pages: - return context.pages[0] - - return None - except Exception as e: - if self.logger: - self.logger.warning( - message="Failed to get page by target ID: {error}", - tag="BROWSER", - params={"error": str(e)} - ) - return None - - async def get_page(self, crawlerRunConfig: CrawlerRunConfig): - """ - Get a page for the given session ID, creating a new one if needed. - - Args: - crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings - - Returns: - (page, context): The Page and its BrowserContext - """ - self._cleanup_expired_sessions() - - # If a session_id is provided and we already have it, reuse that page + context - if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions: - context, page, _ = self.sessions[crawlerRunConfig.session_id] - # Update last-used timestamp - self.sessions[crawlerRunConfig.session_id] = (context, page, time.time()) - return page, context - - # If using a managed browser, just grab the shared default_context - if self.config.use_managed_browser: - # If create_isolated_context is True, create isolated contexts for concurrent crawls - # Uses the same caching mechanism as non-CDP mode: cache context by config signature, - # but always create a new page. This prevents navigation conflicts while allowing - # context reuse for multiple URLs with the same config (e.g., batch/deep crawls). - if self.config.create_isolated_context: - config_signature = self._make_config_signature(crawlerRunConfig) - to_close = None - - async with self._contexts_lock: - if config_signature in self.contexts_by_config: - context = self.contexts_by_config[config_signature] - else: - context = await self.create_browser_context(crawlerRunConfig) - await self.setup_context(context, crawlerRunConfig) - self.contexts_by_config[config_signature] = context - self._context_refcounts[config_signature] = 0 - to_close = self._evict_lru_context_locked() - - # Increment refcount INSIDE lock before releasing - self._context_refcounts[config_signature] = ( - self._context_refcounts.get(config_signature, 0) + 1 - ) - self._context_last_used[config_signature] = time.monotonic() - - # Close evicted context OUTSIDE lock - if to_close is not None: - try: - await to_close.close() - except Exception: - pass - - # Always create a new page for each crawl (isolation for navigation) - try: - page = await context.new_page() - except Exception: - async with self._contexts_lock: - if config_signature in self._context_refcounts: - self._context_refcounts[config_signature] = max( - 0, self._context_refcounts[config_signature] - 1 - ) - raise - await self._apply_stealth_to_page(page) - self._page_to_sig[page] = config_signature - elif self.config.storage_state: - tmp_context = await self.create_browser_context(crawlerRunConfig) - ctx = self.default_context # default context, one window only - ctx = await clone_runtime_state(tmp_context, ctx, crawlerRunConfig, self.config) - # Close the temporary context — only needed as a clone source - try: - await tmp_context.close() - except Exception: - pass - context = ctx # so (page, context) return value is correct - # Avoid concurrent new_page on shared persistent context - # See GH-1198: context.pages can be empty under races - async with self._page_lock: - page = await ctx.new_page() - await self._apply_stealth_to_page(page) - else: - context = self.default_context - - # Handle pre-existing target case (for reconnecting to specific CDP targets) - if self.config.browser_context_id and self.config.target_id: - page = await self._get_page_by_target_id(context, self.config.target_id) - if not page: - async with self._page_lock: - page = await context.new_page() - self._mark_page_in_use(page) - await self._apply_stealth_to_page(page) - else: - # Mark pre-existing target as in use - self._mark_page_in_use(page) - else: - # For CDP connections (external browser), multiple Playwright connections - # create separate browser/context objects. Page reuse across connections - # isn't reliable because each connection sees different page objects. - # Always create new pages for CDP to avoid cross-connection race conditions. - if self.config.cdp_url and not self.config.use_managed_browser: - async with self._page_lock: - page = await context.new_page() - self._mark_page_in_use(page) - await self._apply_stealth_to_page(page) - else: - # For managed browsers (single process), page reuse is safe. - # Use lock to safely check for available pages and track usage. - # This prevents race conditions when multiple crawls run concurrently. - async with BrowserManager._get_global_lock(): - pages = context.pages - pages_in_use = self._get_pages_in_use() - # Find first available page (exists and not currently in use) - available_page = next( - (p for p in pages if p not in pages_in_use), - None - ) - if available_page: - page = available_page - else: - # No available pages - create a new one - page = await context.new_page() - await self._apply_stealth_to_page(page) - # Mark page as in use (global tracking) - self._mark_page_in_use(page) - else: - # Otherwise, check if we have an existing context for this config - config_signature = self._make_config_signature(crawlerRunConfig) - to_close = None - - async with self._contexts_lock: - if config_signature in self.contexts_by_config: - context = self.contexts_by_config[config_signature] - else: - # Create and setup a new context - context = await self.create_browser_context(crawlerRunConfig) - await self.setup_context(context, crawlerRunConfig) - self.contexts_by_config[config_signature] = context - self._context_refcounts[config_signature] = 0 - to_close = self._evict_lru_context_locked() - - # Increment refcount INSIDE lock before releasing - self._context_refcounts[config_signature] = ( - self._context_refcounts.get(config_signature, 0) + 1 - ) - self._context_last_used[config_signature] = time.monotonic() - - # Close evicted context OUTSIDE lock - if to_close is not None: - try: - await to_close.close() - except Exception: - pass - - # Create a new page from the chosen context - try: - page = await context.new_page() - except Exception: - async with self._contexts_lock: - if config_signature in self._context_refcounts: - self._context_refcounts[config_signature] = max( - 0, self._context_refcounts[config_signature] - 1 - ) - raise - await self._apply_stealth_to_page(page) - self._page_to_sig[page] = config_signature - - # If a session_id is specified, store this session so we can reuse later - if crawlerRunConfig.session_id: - self.sessions[crawlerRunConfig.session_id] = (context, page, time.time()) - - self._pages_served += 1 - - # Check if browser recycle threshold is hit — bump version for next requests - # This happens AFTER incrementing counter so concurrent requests see correct count - await self._maybe_bump_browser_version() - - return page, context - - async def kill_session(self, session_id: str): - """ - Kill a browser session and clean up resources. - - Args: - session_id (str): The session ID to kill. - """ - if session_id in self.sessions: - context, page, _ = self.sessions[session_id] - self._release_page_from_use(page) - # Decrement context refcount for the session's page - should_close_context = False - async with self._contexts_lock: - sig = self._page_to_sig.pop(page, None) - if sig is not None and sig in self._context_refcounts: - self._context_refcounts[sig] = max( - 0, self._context_refcounts[sig] - 1 - ) - # Only close the context if no other pages are using it - # (refcount dropped to 0) AND we own the context (not managed) - if not self.config.use_managed_browser: - if self._context_refcounts.get(sig, 0) == 0: - self.contexts_by_config.pop(sig, None) - self._context_refcounts.pop(sig, None) - self._context_last_used.pop(sig, None) - should_close_context = True - await page.close() - if should_close_context: - await context.close() - del self.sessions[session_id] - - def release_page(self, page): - """ - Release a page from the in-use tracking set (global tracking). - Sync variant — does NOT decrement context refcount. - """ - self._release_page_from_use(page) - - async def release_page_with_context(self, page): - """ - Release a page and decrement its context's refcount under the lock. - - Should be called from the async crawl finally block instead of - release_page() so the context lifecycle is properly tracked. - """ - self._release_page_from_use(page) - sig = None - refcount = -1 - async with self._contexts_lock: - sig = self._page_to_sig.pop(page, None) - if sig is not None and sig in self._context_refcounts: - self._context_refcounts[sig] = max( - 0, self._context_refcounts[sig] - 1 - ) - refcount = self._context_refcounts[sig] - - # Check if this signature belongs to an old browser waiting to be cleaned up - if sig is not None and refcount == 0: - await self._maybe_cleanup_old_browser(sig) - - def _should_recycle(self) -> bool: - """Check if page threshold reached for browser recycling.""" - limit = self.config.max_pages_before_recycle - if limit <= 0: - return False - return self._pages_served >= limit - - async def _maybe_bump_browser_version(self): - """Bump browser version if threshold reached, moving old browser to pending cleanup. - - New requests automatically get a new browser (via new signature). - Old browser drains naturally and gets cleaned up when refcount hits 0. - """ - if not self._should_recycle(): - return - - # Safety cap: wait if too many old browsers are draining - while True: - async with self._pending_cleanup_lock: - # Re-check threshold under lock (another request may have bumped already) - if not self._should_recycle(): - return - - # Check safety cap - if len(self._pending_cleanup) >= self._max_pending_browsers: - if self.logger: - self.logger.debug( - message="Waiting for old browser to drain (pending: {count})", - tag="BROWSER", - params={"count": len(self._pending_cleanup)}, - ) - self._cleanup_slot_available.clear() - # Release lock and wait - else: - # We have a slot — do the bump inside this lock hold - old_version = self._browser_version - active_sigs = [] - idle_sigs = [] - async with self._contexts_lock: - for sig in list(self._context_refcounts.keys()): - if self._context_refcounts.get(sig, 0) > 0: - active_sigs.append(sig) - else: - idle_sigs.append(sig) - - if self.logger: - self.logger.info( - message="Bumping browser version {old} -> {new} after {count} pages ({active} active, {idle} idle sigs)", - tag="BROWSER", - params={ - "old": old_version, - "new": old_version + 1, - "count": self._pages_served, - "active": len(active_sigs), - "idle": len(idle_sigs), - }, - ) - - # Only add sigs with active crawls to pending cleanup. - # Sigs with refcount 0 are cleaned up immediately below - # to avoid them being stuck in _pending_cleanup forever - # (no future release would trigger their cleanup). - done_event = asyncio.Event() - for sig in active_sigs: - self._pending_cleanup[sig] = { - "version": old_version, - "done": done_event, - } - - # Bump version — new get_page() calls will create new contexts - self._browser_version += 1 - self._pages_served = 0 - - # Clean up idle sigs immediately (outside pending_cleanup_lock below) - break # exit while loop to do cleanup outside locks - - # Safety cap path: wait for a cleanup slot, then retry. - # Timeout prevents permanent deadlock if stuck entries never drain. - try: - await asyncio.wait_for( - self._cleanup_slot_available.wait(), timeout=30.0 - ) - except asyncio.TimeoutError: - # Force-clean any pending entries that have refcount 0 - # (they're stuck and will never drain naturally) - async with self._pending_cleanup_lock: - stuck_sigs = [ - s for s in list(self._pending_cleanup.keys()) - if self._context_refcounts.get(s, 0) == 0 - ] - for sig in stuck_sigs: - self._pending_cleanup.pop(sig, None) - if stuck_sigs: - if self.logger: - self.logger.warning( - message="Force-cleaned {count} stuck pending entries after timeout", - tag="BROWSER", - params={"count": len(stuck_sigs)}, - ) - # Clean up the stuck contexts - for sig in stuck_sigs: - async with self._contexts_lock: - context = self.contexts_by_config.pop(sig, None) - self._context_refcounts.pop(sig, None) - self._context_last_used.pop(sig, None) - if context is not None: - try: - await context.close() - except Exception: - pass - if len(self._pending_cleanup) < self._max_pending_browsers: - self._cleanup_slot_available.set() - - # Reached via break — clean up idle sigs immediately (outside locks) - for sig in idle_sigs: - async with self._contexts_lock: - context = self.contexts_by_config.pop(sig, None) - self._context_refcounts.pop(sig, None) - self._context_last_used.pop(sig, None) - if context is not None: - try: - await context.close() - except Exception: - pass - if idle_sigs and self.logger: - self.logger.debug( - message="Immediately cleaned up {count} idle contexts from version {version}", - tag="BROWSER", - params={"count": len(idle_sigs), "version": old_version}, - ) - - async def _maybe_cleanup_old_browser(self, sig: str): - """Clean up an old browser's context if its refcount hit 0 and it's pending cleanup.""" - async with self._pending_cleanup_lock: - if sig not in self._pending_cleanup: - return # Not an old browser signature - - cleanup_info = self._pending_cleanup.pop(sig) - old_version = cleanup_info["version"] - - if self.logger: - self.logger.debug( - message="Cleaning up context from browser version {version} (sig: {sig})", - tag="BROWSER", - params={"version": old_version, "sig": sig[:12]}, - ) - - # Remove context from tracking - async with self._contexts_lock: - context = self.contexts_by_config.pop(sig, None) - self._context_refcounts.pop(sig, None) - self._context_last_used.pop(sig, None) - - # Close context outside locks - if context is not None: - try: - await context.close() - except Exception: - pass - - # Check if any signatures from this old version remain - remaining_old = [ - s for s, info in self._pending_cleanup.items() - if info["version"] == old_version - ] - - if not remaining_old: - if self.logger: - self.logger.info( - message="All contexts from browser version {version} cleaned up", - tag="BROWSER", - params={"version": old_version}, - ) - - # Open a cleanup slot if we're below the cap - if len(self._pending_cleanup) < self._max_pending_browsers: - self._cleanup_slot_available.set() - - def _cleanup_expired_sessions(self): - """Clean up expired sessions based on TTL.""" - current_time = time.time() - expired_sessions = [ - sid - for sid, (_, _, last_used) in self.sessions.items() - if current_time - last_used > self.session_ttl - ] - for sid in expired_sessions: - asyncio.create_task(self.kill_session(sid)) - - async def close(self): - """Close all browser resources and clean up.""" - # Cached CDP path: only clean up this instance's sessions/contexts, - # then release the shared connection reference. - if self._using_cached_cdp: - session_ids = list(self.sessions.keys()) - for session_id in session_ids: - await self.kill_session(session_id) - for ctx in list(self.contexts_by_config.values()): - try: - await ctx.close() - except Exception: - pass - self.contexts_by_config.clear() - self._context_refcounts.clear() - self._context_last_used.clear() - self._page_to_sig.clear() - await _CDPConnectionCache.release(self.config.cdp_url) - self.browser = None - self.playwright = None - self._using_cached_cdp = False - return - - if self.config.cdp_url: - # When using external CDP, we don't own the browser process. - # If cdp_cleanup_on_close is True, properly disconnect from the browser - # and clean up Playwright resources. This frees the browser for other clients. - if self.config.cdp_cleanup_on_close: - # First close all sessions (pages) - session_ids = list(self.sessions.keys()) - for session_id in session_ids: - await self.kill_session(session_id) - - # Close all contexts we created - for ctx in list(self.contexts_by_config.values()): - try: - await ctx.close() - except Exception: - pass - self.contexts_by_config.clear() - self._context_refcounts.clear() - self._context_last_used.clear() - self._page_to_sig.clear() - - # Disconnect from browser (doesn't terminate it, just releases connection) - if self.browser: - try: - await self.browser.close() - except Exception as e: - if self.logger: - self.logger.debug( - message="Error disconnecting from CDP browser: {error}", - tag="BROWSER", - params={"error": str(e)} - ) - self.browser = None - # Allow time for CDP connection to fully release before another client connects - if self.config.cdp_close_delay > 0: - await asyncio.sleep(self.config.cdp_close_delay) - - # Stop Playwright instance to prevent memory leaks - if self.playwright: - await self.playwright.stop() - self.playwright = None - return - - # ── Persistent context launched via launch_persistent_context ── - if self._launched_persistent: - session_ids = list(self.sessions.keys()) - for session_id in session_ids: - await self.kill_session(session_id) - for ctx in list(self.contexts_by_config.values()): - try: - await ctx.close() - except Exception: - pass - self.contexts_by_config.clear() - self._context_refcounts.clear() - self._context_last_used.clear() - self._page_to_sig.clear() - - # Closing the persistent context also terminates the browser - if self.default_context: - try: - await self.default_context.close() - except Exception: - pass - self.default_context = None - - if self.playwright: - await self.playwright.stop() - self.playwright = None - self._launched_persistent = False - return - - if self.config.sleep_on_close: - await asyncio.sleep(0.5) - - session_ids = list(self.sessions.keys()) - for session_id in session_ids: - await self.kill_session(session_id) - - # Now close all contexts we created. This reclaims memory from ephemeral contexts. - for ctx in list(self.contexts_by_config.values()): - try: - await ctx.close() - except Exception as e: - self.logger.error( - message="Error closing context: {error}", - tag="ERROR", - params={"error": str(e)} - ) - self.contexts_by_config.clear() - self._context_refcounts.clear() - self._context_last_used.clear() - self._page_to_sig.clear() - - if self.browser: - await self.browser.close() - self.browser = None - - if self.managed_browser: - await asyncio.sleep(0.5) - await self.managed_browser.cleanup() - self.managed_browser = None - - if self.playwright: - await self.playwright.stop() - self.playwright = None +import asyncio +import time +from typing import Dict, List, Optional, Tuple +import os +import sys +import shutil +import tempfile +import psutil +import signal +import subprocess +import shlex +from playwright.async_api import BrowserContext +import hashlib +from .js_snippet import load_js_script +from .config import DOWNLOAD_PAGE_TIMEOUT +from .async_configs import BrowserConfig, CrawlerRunConfig +from .utils import get_chromium_path +import warnings + + +BROWSER_DISABLE_OPTIONS = [ + "--disable-background-networking", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-breakpad", + "--disable-client-side-phishing-detection", + "--disable-component-extensions-with-background-pages", + "--disable-default-apps", + "--disable-extensions", + "--disable-features=TranslateUI", + "--disable-hang-monitor", + "--disable-ipc-flooding-protection", + "--disable-popup-blocking", + "--disable-prompt-on-repost", + "--disable-sync", + "--force-color-profile=srgb", + "--metrics-recording-only", + "--no-first-run", + "--password-store=basic", + "--use-mock-keychain", +] + + +class ManagedBrowser: + """ + Manages the browser process and context. This class allows to connect to the browser using CDP protocol. + + Attributes: + browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". + Default: "chromium". + user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a + temporary directory may be used. Default: None. + headless (bool): Whether to run the browser in headless mode (no visible GUI). + Default: True. + browser_process (subprocess.Popen): The process object for the browser. + temp_dir (str): Temporary directory for user data if not provided. + debugging_port (int): Port for debugging the browser. + host (str): Host for debugging the browser. + + Methods: + start(): Starts the browser process and returns the CDP endpoint URL. + _get_browser_path(): Returns the browser executable path based on OS and browser type. + _get_browser_args(): Returns browser-specific command line arguments. + _get_user_data_dir(): Returns the user data directory path. + _cleanup(): Terminates the browser process and removes the temporary directory. + create_profile(): Static method to create a user profile by launching a browser for user interaction. + """ + + @staticmethod + def build_browser_flags(config: BrowserConfig) -> List[str]: + """Common CLI flags for launching Chromium""" + flags = [ + "--no-sandbox", + "--disable-dev-shm-usage", + "--no-first-run", + "--no-default-browser-check", + "--disable-infobars", + "--window-position=0,0", + "--ignore-certificate-errors", + "--ignore-certificate-errors-spki-list", + "--disable-blink-features=AutomationControlled", + "--window-position=400,0", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--force-color-profile=srgb", + "--mute-audio", + "--disable-background-timer-throttling", + # Memory-saving flags: disable unused Chrome features + "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider", + "--disable-component-update", + "--disable-domain-reliability", + ] + # GPU flags disable WebGL which anti-bot sensors detect as headless. + # Keep WebGL working (via SwiftShader) when stealth mode is active. + if not config.enable_stealth: + flags.extend([ + "--disable-gpu", + "--disable-gpu-compositing", + "--disable-software-rasterizer", + ]) + if config.memory_saving_mode: + flags.extend([ + "--aggressive-cache-discard", + '--js-flags=--max-old-space-size=512', + ]) + if config.light_mode: + flags.extend(BROWSER_DISABLE_OPTIONS) + if config.text_mode: + flags.extend([ + "--blink-settings=imagesEnabled=false", + "--disable-remote-fonts", + "--disable-images", + "--disable-javascript", + "--disable-software-rasterizer", + "--disable-dev-shm-usage", + ]) + # proxy support — only pass server URL, never credentials. + # Chromium's --proxy-server flag silently ignores inline user:pass@. + # Auth credentials are handled at the Playwright context level instead. + if config.proxy: + flags.append(f"--proxy-server={config.proxy}") + elif config.proxy_config: + flags.append(f"--proxy-server={config.proxy_config.server}") + # dedupe + return list(dict.fromkeys(flags)) + + browser_type: str + user_data_dir: str + headless: bool + browser_process: subprocess.Popen + temp_dir: str + debugging_port: int + host: str + + def __init__( + self, + browser_type: str = "chromium", + user_data_dir: Optional[str] = None, + headless: bool = False, + logger=None, + host: str = "localhost", + debugging_port: int = 9222, + cdp_url: Optional[str] = None, + browser_config: Optional[BrowserConfig] = None, + ): + """ + Initialize the ManagedBrowser instance. + + Args: + browser_type (str): The type of browser to launch. Supported values: "chromium", "firefox", "webkit". + Default: "chromium". + user_data_dir (str or None): Path to a user data directory for persistent sessions. If None, a + temporary directory may be used. Default: None. + headless (bool): Whether to run the browser in headless mode (no visible GUI). + Default: True. + logger (logging.Logger): Logger instance for logging messages. Default: None. + host (str): Host for debugging the browser. Default: "localhost". + debugging_port (int): Port for debugging the browser. Default: 9222. + cdp_url (str or None): CDP URL to connect to the browser. Default: None. + browser_config (BrowserConfig): Configuration object containing all browser settings. Default: None. + """ + self.browser_type = browser_config.browser_type + self.user_data_dir = browser_config.user_data_dir + self.headless = browser_config.headless + self.browser_process = None + self.temp_dir = None + self.debugging_port = browser_config.debugging_port + self.host = browser_config.host + self.logger = logger + self.shutting_down = False + self.cdp_url = browser_config.cdp_url + self.browser_config = browser_config + + async def start(self) -> str: + """ + Starts the browser process or returns CDP endpoint URL. + If cdp_url is provided, returns it directly. + If user_data_dir is not provided for local browser, creates a temporary directory. + + Returns: + str: CDP endpoint URL + """ + # If CDP URL provided, just return it + if self.cdp_url: + return self.cdp_url + + # Create temp dir if needed + if not self.user_data_dir: + self.temp_dir = tempfile.mkdtemp(prefix="browser-profile-") + self.user_data_dir = self.temp_dir + + # Get browser path and args based on OS and browser type + # browser_path = self._get_browser_path() + args = await self._get_browser_args() + + if self.browser_config.extra_args: + args.extend(self.browser_config.extra_args) + + + # ── make sure no old Chromium instance is owning the same port/profile ── + try: + if sys.platform == "win32": + if psutil is None: + raise RuntimeError("psutil not available, cannot clean old browser") + for p in psutil.process_iter(["pid", "name", "cmdline"]): + cl = " ".join(p.info.get("cmdline") or []) + if ( + f"--remote-debugging-port={self.debugging_port}" in cl + and f"--user-data-dir={self.user_data_dir}" in cl + ): + p.kill() + p.wait(timeout=5) + else: # macOS / Linux + # kill any process listening on the same debugging port + try: + pids = ( + subprocess.check_output( + shlex.split(f"lsof -t -i:{self.debugging_port}"), + stderr=subprocess.DEVNULL, + ) + .decode() + .strip() + .splitlines() + ) + except (FileNotFoundError, subprocess.CalledProcessError): + pids = [] + for pid in pids: + try: + os.kill(int(pid), signal.SIGTERM) + except ProcessLookupError: + pass + + # remove Chromium singleton locks, or new launch exits with + # “Opening in existing browser session.” + for f in ("SingletonLock", "SingletonSocket", "SingletonCookie"): + fp = os.path.join(self.user_data_dir, f) + if os.path.exists(fp): + os.remove(fp) + except Exception as _e: + # non-fatal — we'll try to start anyway, but log what happened + self.logger.warning(f"pre-launch cleanup failed: {_e}", tag="BROWSER") + + + # Start browser process + try: + # Use DETACHED_PROCESS flag on Windows to fully detach the process + # On Unix, we'll use preexec_fn=os.setpgrp to start the process in a new process group + if sys.platform == "win32": + self.browser_process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=subprocess.DETACHED_PROCESS | subprocess.CREATE_NEW_PROCESS_GROUP + ) + else: + self.browser_process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setpgrp # Start in a new process group + ) + + # If verbose is True print args used to run the process + if self.logger and self.browser_config.verbose: + self.logger.debug( + f"Starting browser with args: {' '.join(args)}", + tag="BROWSER" + ) + + # We'll monitor for a short time to make sure it starts properly, but won't keep monitoring + await asyncio.sleep(0.5) # Give browser time to start + await self._initial_startup_check() + await asyncio.sleep(2) # Give browser time to start + return f"http://{self.host}:{self.debugging_port}" + except Exception as e: + await self.cleanup() + raise Exception(f"Failed to start browser: {e}") + + async def _initial_startup_check(self): + """ + Perform a quick check to make sure the browser started successfully. + This only runs once at startup rather than continuously monitoring. + """ + if not self.browser_process: + return + + # Check that process started without immediate termination + await asyncio.sleep(0.5) + if self.browser_process.poll() is not None: + # Process already terminated + stdout, stderr = b"", b"" + try: + stdout, stderr = self.browser_process.communicate(timeout=0.5) + except subprocess.TimeoutExpired: + pass + + self.logger.error( + message="Browser process terminated during startup | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}", + tag="ERROR", + params={ + "code": self.browser_process.returncode, + "stdout": stdout.decode() if stdout else "", + "stderr": stderr.decode() if stderr else "", + }, + ) + + async def _monitor_browser_process(self): + """ + Monitor the browser process for unexpected termination. + + How it works: + 1. Read stdout and stderr from the browser process. + 2. If the process has terminated, log the error message and terminate the browser. + 3. If the shutting_down flag is set, log the normal termination message. + 4. If any other error occurs, log the error message. + + Note: This method should be called in a separate task to avoid blocking the main event loop. + This is DEPRECATED and should not be used for builtin browsers that need to outlive the Python process. + """ + if self.browser_process: + try: + stdout, stderr = await asyncio.gather( + asyncio.to_thread(self.browser_process.stdout.read), + asyncio.to_thread(self.browser_process.stderr.read), + ) + + # Check shutting_down flag BEFORE logging anything + if self.browser_process.poll() is not None: + if not self.shutting_down: + self.logger.error( + message="Browser process terminated unexpectedly | Code: {code} | STDOUT: {stdout} | STDERR: {stderr}", + tag="ERROR", + params={ + "code": self.browser_process.returncode, + "stdout": stdout.decode(), + "stderr": stderr.decode(), + }, + ) + await self.cleanup() + else: + self.logger.info( + message="Browser process terminated normally | Code: {code}", + tag="INFO", + params={"code": self.browser_process.returncode}, + ) + except Exception as e: + if not self.shutting_down: + self.logger.error( + message="Error monitoring browser process: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + def _get_browser_path_WIP(self) -> str: + """Returns the browser executable path based on OS and browser type""" + if sys.platform == "darwin": # macOS + paths = { + "chromium": "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "firefox": "/Applications/Firefox.app/Contents/MacOS/firefox", + "webkit": "/Applications/Safari.app/Contents/MacOS/Safari", + } + elif sys.platform == "win32": # Windows + paths = { + "chromium": "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe", + "firefox": "C:\\Program Files\\Mozilla Firefox\\firefox.exe", + "webkit": None, # WebKit not supported on Windows + } + else: # Linux + paths = { + "chromium": "google-chrome", + "firefox": "firefox", + "webkit": None, # WebKit not supported on Linux + } + + return paths.get(self.browser_type) + + async def _get_browser_path(self) -> str: + browser_path = await get_chromium_path(self.browser_type) + return browser_path + + async def _get_browser_args(self) -> List[str]: + """Returns full CLI args for launching the browser""" + base = [await self._get_browser_path()] + if self.browser_type == "chromium": + flags = [ + f"--remote-debugging-port={self.debugging_port}", + f"--user-data-dir={self.user_data_dir}", + ] + if self.headless: + flags.append("--headless=new") + # Add viewport flag if specified in config + if self.browser_config.viewport_height and self.browser_config.viewport_width: + flags.append(f"--window-size={self.browser_config.viewport_width},{self.browser_config.viewport_height}") + # merge common launch flags + flags.extend(self.build_browser_flags(self.browser_config)) + elif self.browser_type == "firefox": + flags = [ + "--remote-debugging-port", + str(self.debugging_port), + "--profile", + self.user_data_dir, + ] + if self.headless: + flags.append("--headless") + else: + raise NotImplementedError(f"Browser type {self.browser_type} not supported") + return base + flags + + async def cleanup(self): + """Cleanup browser process and temporary directory""" + # Set shutting_down flag BEFORE any termination actions + self.shutting_down = True + + if self.browser_process: + try: + # For builtin browsers that should persist, we should check if it's a detached process + # Only terminate if we have proper control over the process + if not self.browser_process.poll(): + # Process is still running + self.browser_process.terminate() + # Wait for process to end gracefully + for _ in range(10): # 10 attempts, 100ms each + if self.browser_process.poll() is not None: + break + await asyncio.sleep(0.1) + + # Force kill if still running + if self.browser_process.poll() is None: + if sys.platform == "win32": + # On Windows, use taskkill /T to kill the entire process tree + try: + subprocess.run(["taskkill", "/F", "/T", "/PID", str(self.browser_process.pid)]) + except Exception: + self.browser_process.kill() + else: + # On Unix, kill entire process group to reap child processes + try: + os.killpg(os.getpgid(self.browser_process.pid), signal.SIGKILL) + except (ProcessLookupError, OSError): + pass + await asyncio.sleep(0.1) # Brief wait for kill to take effect + + except Exception as e: + self.logger.error( + message="Error terminating browser: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + if self.temp_dir and os.path.exists(self.temp_dir): + try: + shutil.rmtree(self.temp_dir) + except Exception as e: + self.logger.error( + message="Error removing temporary directory: {error}", + tag="ERROR", + params={"error": str(e)}, + ) + + # These methods have been moved to BrowserProfiler class + @staticmethod + async def create_profile(browser_config=None, profile_name=None, logger=None): + """ + This method has been moved to the BrowserProfiler class. + + Creates a browser profile by launching a browser for interactive user setup + and waits until the user closes it. The profile is stored in a directory that + can be used later with BrowserConfig.user_data_dir. + + Please use BrowserProfiler.create_profile() instead. + + Example: + ```python + from crawl4ai.browser_profiler import BrowserProfiler + + profiler = BrowserProfiler() + profile_path = await profiler.create_profile(profile_name="my-login-profile") + ``` + """ + from .browser_profiler import BrowserProfiler + + # Create a BrowserProfiler instance and delegate to it + profiler = BrowserProfiler(logger=logger) + return await profiler.create_profile(profile_name=profile_name, browser_config=browser_config) + + @staticmethod + def list_profiles(): + """ + This method has been moved to the BrowserProfiler class. + + Lists all available browser profiles in the Crawl4AI profiles directory. + + Please use BrowserProfiler.list_profiles() instead. + + Example: + ```python + from crawl4ai.browser_profiler import BrowserProfiler + + profiler = BrowserProfiler() + profiles = profiler.list_profiles() + ``` + """ + from .browser_profiler import BrowserProfiler + + # Create a BrowserProfiler instance and delegate to it + profiler = BrowserProfiler() + return profiler.list_profiles() + + @staticmethod + def delete_profile(profile_name_or_path): + """ + This method has been moved to the BrowserProfiler class. + + Delete a browser profile by name or path. + + Please use BrowserProfiler.delete_profile() instead. + + Example: + ```python + from crawl4ai.browser_profiler import BrowserProfiler + + profiler = BrowserProfiler() + success = profiler.delete_profile("my-profile") + ``` + """ + from .browser_profiler import BrowserProfiler + + # Create a BrowserProfiler instance and delegate to it + profiler = BrowserProfiler() + return profiler.delete_profile(profile_name_or_path) + + +async def clone_runtime_state( + src: BrowserContext, + dst: BrowserContext, + crawlerRunConfig: CrawlerRunConfig | None = None, + browserConfig: BrowserConfig | None = None, +) -> None: + """ + Bring everything that *can* be changed at runtime from `src` → `dst`. + + 1. Cookies + 2. localStorage (and sessionStorage, same API) + 3. Extra headers, permissions, geolocation if supplied in configs + """ + + # ── 1. cookies ──────────────────────────────────────────────────────────── + cookies = await src.cookies() + if cookies: + await dst.add_cookies(cookies) + + # ── 2. localStorage / sessionStorage ────────────────────────────────────── + state = await src.storage_state() + for origin in state.get("origins", []): + url = origin["origin"] + kvs = origin.get("localStorage", []) + if not kvs: + continue + + page = dst.pages[0] if dst.pages else await dst.new_page() + await page.goto(url, wait_until="domcontentloaded") + for k, v in kvs: + await page.evaluate("(k,v)=>localStorage.setItem(k,v)", k, v) + + # ── 3. runtime-mutable extras from configs ──────────────────────────────── + # headers + if browserConfig and browserConfig.headers: + await dst.set_extra_http_headers(browserConfig.headers) + + # geolocation + if crawlerRunConfig and crawlerRunConfig.geolocation: + await dst.grant_permissions(["geolocation"]) + await dst.set_geolocation( + { + "latitude": crawlerRunConfig.geolocation.latitude, + "longitude": crawlerRunConfig.geolocation.longitude, + "accuracy": crawlerRunConfig.geolocation.accuracy, + } + ) + + return dst + + + +class _CDPConnectionCache: + """ + Class-level cache for Playwright + CDP browser connections. + + When enabled via BrowserConfig(cache_cdp_connection=True), multiple + BrowserManager instances connecting to the same cdp_url will share + a single Playwright subprocess and CDP WebSocket. Reference-counted; + the connection is closed when the last user releases it. + """ + + _cache: Dict[str, Tuple] = {} # cdp_url -> (playwright, browser, ref_count) + _lock: Optional[asyncio.Lock] = None # lazy-init to avoid event loop issues + _lock_loop: Optional[asyncio.AbstractEventLoop] = None + + @classmethod + def _get_lock(cls) -> asyncio.Lock: + loop = asyncio.get_running_loop() + if cls._lock is None or cls._lock_loop is not loop: + cls._lock = asyncio.Lock() + cls._lock_loop = loop + return cls._lock + + @classmethod + async def acquire(cls, cdp_url: str, use_undetected: bool = False): + """Get or create a cached (playwright, browser) for this cdp_url.""" + async with cls._get_lock(): + if cdp_url in cls._cache: + pw, browser, count = cls._cache[cdp_url] + if browser.is_connected(): + cls._cache[cdp_url] = (pw, browser, count + 1) + return pw, browser + # Stale connection — clean up and fall through to create new + try: + await pw.stop() + except Exception: + pass + del cls._cache[cdp_url] + + # Create new connection + if use_undetected: + from patchright.async_api import async_playwright + else: + from playwright.async_api import async_playwright + pw = await async_playwright().start() + browser = await pw.chromium.connect_over_cdp(cdp_url) + cls._cache[cdp_url] = (pw, browser, 1) + return pw, browser + + @classmethod + async def release(cls, cdp_url: str): + """Decrement ref count; close connection when last user releases.""" + async with cls._get_lock(): + if cdp_url not in cls._cache: + return + pw, browser, count = cls._cache[cdp_url] + if count <= 1: + try: + await browser.close() + except Exception: + pass + try: + await pw.stop() + except Exception: + pass + del cls._cache[cdp_url] + else: + cls._cache[cdp_url] = (pw, browser, count - 1) + + @classmethod + async def close_all(cls): + """Force-close all cached connections. Call on application shutdown.""" + async with cls._get_lock(): + for cdp_url in list(cls._cache.keys()): + pw, browser, _ = cls._cache[cdp_url] + try: + await browser.close() + except Exception: + pass + try: + await pw.stop() + except Exception: + pass + cls._cache.clear() + + +class BrowserManager: + """ + Manages the browser instance and context. + + Attributes: + config (BrowserConfig): Configuration object containing all browser settings + logger: Logger instance for recording events and errors + browser (Browser): The browser instance + default_context (BrowserContext): The default browser context + managed_browser (ManagedBrowser): The managed browser instance + playwright (Playwright): The Playwright instance + sessions (dict): Dictionary to store session information + session_ttl (int): Session timeout in seconds + """ + + _playwright_instance = None + + # Class-level tracking of pages in use, keyed by browser endpoint (CDP URL or instance id) + # This ensures multiple BrowserManager instances connecting to the same browser + # share the same page tracking, preventing race conditions. + _global_pages_in_use: dict = {} # endpoint_key -> set of pages + _global_pages_lock: asyncio.Lock = None # Initialized lazily + + @classmethod + def _get_global_lock(cls) -> asyncio.Lock: + """Get or create the global pages lock (lazy initialization for async context).""" + if cls._global_pages_lock is None: + cls._global_pages_lock = asyncio.Lock() + return cls._global_pages_lock + + @classmethod + async def get_playwright(cls, use_undetected: bool = False): + if use_undetected: + from patchright.async_api import async_playwright + else: + from playwright.async_api import async_playwright + cls._playwright_instance = await async_playwright().start() + return cls._playwright_instance + + def __init__(self, browser_config: BrowserConfig, logger=None, use_undetected: bool = False): + """ + Initialize the BrowserManager with a browser configuration. + + Args: + browser_config (BrowserConfig): Configuration object containing all browser settings + logger: Logger instance for recording events and errors + use_undetected (bool): Whether to use undetected browser (Patchright) + """ + self.config: BrowserConfig = browser_config + self.logger = logger + self.use_undetected = use_undetected + + # Browser state + self.browser = None + self.default_context = None + self.managed_browser = None + self.playwright = None + self._using_cached_cdp = False + self._launched_persistent = False # True when using launch_persistent_context + + # Session management + self.sessions = {} + self.session_ttl = 1800 # 30 minutes + + # Keep track of contexts by a "config signature," so each unique config reuses a single context + self.contexts_by_config = {} + self._contexts_lock = asyncio.Lock() + + # Context lifecycle tracking for LRU eviction + self._context_refcounts = {} # sig -> int (active crawls using this context) + self._context_last_used = {} # sig -> float (monotonic timestamp for LRU) + self._page_to_sig = {} # page -> sig (for decrement lookup on release) + self._max_contexts = 20 # LRU eviction threshold + + # Serialize context.new_page() across concurrent tasks to avoid races + # when using a shared persistent context (context.pages may be empty + # for all racers). Prevents 'Target page/context closed' errors. + self._page_lock = asyncio.Lock() + + # Browser endpoint key for global page tracking (set after browser starts) + self._browser_endpoint_key: Optional[str] = None + + # Browser recycling state (version-based approach) + self._pages_served = 0 + self._browser_version = 1 # included in signature, bump to create new browser + self._pending_cleanup = {} # old_sig -> {"browser": browser, "contexts": [...], "done": Event} + self._pending_cleanup_lock = asyncio.Lock() + self._max_pending_browsers = 3 # safety cap — block if too many draining + self._cleanup_slot_available = asyncio.Event() + self._cleanup_slot_available.set() # starts open + + # Stealth adapter for stealth mode + self._stealth_adapter = None + if self.config.enable_stealth and not self.use_undetected: + from .browser_adapter import StealthAdapter + self._stealth_adapter = StealthAdapter() + + # Initialize ManagedBrowser if needed + if self.config.use_managed_browser: + self.managed_browser = ManagedBrowser( + browser_type=self.config.browser_type, + user_data_dir=self.config.user_data_dir, + headless=self.config.headless, + logger=self.logger, + debugging_port=self.config.debugging_port, + cdp_url=self.config.cdp_url, + browser_config=self.config, + ) + + async def start(self): + """ + Start the browser instance and set up the default context. + + How it works: + 1. Check if Playwright is already initialized. + 2. If not, initialize Playwright. + 3. If managed browser is used, start it and connect to the CDP endpoint. + 4. If managed browser is not used, launch the browser and set up the default context. + + Note: This method should be called in a separate task to avoid blocking the main event loop. + """ + if self.playwright is not None: + await self.close() + + # Use cached CDP connection if enabled and cdp_url is set + if self.config.cache_cdp_connection and self.config.cdp_url: + self._using_cached_cdp = True + self.config.use_managed_browser = True + self.playwright, self.browser = await _CDPConnectionCache.acquire( + self.config.cdp_url, self.use_undetected + ) + else: + self._using_cached_cdp = False + if self.use_undetected: + from patchright.async_api import async_playwright + else: + from playwright.async_api import async_playwright + + # Initialize playwright + self.playwright = await async_playwright().start() + + try: + await self._launch_browser() + except BaseException: + # If browser launch fails, ensure Playwright process is cleaned up + # to avoid leaking the "node cli.js run-driver" subprocess. + if self.playwright is not None and not self._using_cached_cdp: + try: + await self.playwright.stop() + except Exception: + pass + self.playwright = None + raise + + async def _launch_browser(self): + """Launch or connect to the browser after Playwright is initialized.""" + # ── Persistent context via Playwright's native API ────────────── + # When use_persistent_context is set and we're not connecting to an + # external CDP endpoint, use launch_persistent_context() instead of + # subprocess + CDP. This properly supports proxy authentication + # (server + username + password) which the --proxy-server CLI flag + # cannot handle. + if ( + self.config.use_persistent_context + and not self.config.cdp_url + and not self._using_cached_cdp + ): + # Collect stealth / optimization CLI flags, excluding ones that + # launch_persistent_context handles via keyword arguments. + _skip_prefixes = ( + "--proxy-server", + "--remote-debugging-port", + "--user-data-dir", + "--headless", + "--window-size", + ) + cli_args = [ + flag + for flag in ManagedBrowser.build_browser_flags(self.config) + if not flag.startswith(_skip_prefixes) + ] + if self.config.extra_args: + cli_args.extend(self.config.extra_args) + + launch_kwargs = { + "headless": self.config.headless, + "args": list(dict.fromkeys(cli_args)), # dedupe + "viewport": { + "width": self.config.viewport_width, + "height": self.config.viewport_height, + }, + "user_agent": self.config.user_agent or None, + "ignore_https_errors": self.config.ignore_https_errors, + "accept_downloads": self.config.accept_downloads, + } + + if self.config.proxy_config: + launch_kwargs["proxy"] = { + "server": self.config.proxy_config.server, + "username": self.config.proxy_config.username, + "password": self.config.proxy_config.password, + } + + if self.config.storage_state: + launch_kwargs["storage_state"] = self.config.storage_state + + user_data_dir = self.config.user_data_dir or tempfile.mkdtemp( + prefix="crawl4ai-persistent-" + ) + + self.default_context = ( + await self.playwright.chromium.launch_persistent_context( + user_data_dir, **launch_kwargs + ) + ) + self.browser = None # persistent context has no separate Browser + self._launched_persistent = True + + await self.setup_context(self.default_context) + + # Set the browser endpoint key for global page tracking + self._browser_endpoint_key = self._compute_browser_endpoint_key() + if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() + return + + if self.config.cdp_url or self.config.use_managed_browser: + self.config.use_managed_browser = True + + if not self._using_cached_cdp: + cdp_url = await self.managed_browser.start() if not self.config.cdp_url else self.config.cdp_url + + # Add CDP endpoint verification before connecting + if not await self._verify_cdp_ready(cdp_url): + raise Exception(f"CDP endpoint at {cdp_url} is not ready after startup") + + self.browser = await self.playwright.chromium.connect_over_cdp(cdp_url) + + contexts = self.browser.contexts + + # If browser_context_id is provided, we're using a pre-created context + if self.config.browser_context_id: + if self.logger: + self.logger.debug( + f"Using pre-existing browser context: {self.config.browser_context_id}", + tag="BROWSER" + ) + # When connecting to a pre-created context, it should be in contexts + if contexts: + self.default_context = contexts[0] + if self.logger: + self.logger.debug( + f"Found {len(contexts)} existing context(s), using first one", + tag="BROWSER" + ) + else: + # Context was created but not yet visible - wait a bit + await asyncio.sleep(0.2) + contexts = self.browser.contexts + if contexts: + self.default_context = contexts[0] + else: + # Still no contexts - this shouldn't happen with pre-created context + if self.logger: + self.logger.warning( + "Pre-created context not found, creating new one", + tag="BROWSER" + ) + self.default_context = await self.create_browser_context() + elif contexts: + self.default_context = contexts[0] + else: + self.default_context = await self.create_browser_context() + await self.setup_context(self.default_context) + else: + browser_args = self._build_browser_args() + + # Launch appropriate browser type + if self.config.browser_type == "firefox": + self.browser = await self.playwright.firefox.launch(**browser_args) + elif self.config.browser_type == "webkit": + self.browser = await self.playwright.webkit.launch(**browser_args) + else: + self.browser = await self.playwright.chromium.launch(**browser_args) + + self.default_context = self.browser + + # Set the browser endpoint key for global page tracking + self._browser_endpoint_key = self._compute_browser_endpoint_key() + # Initialize global tracking set for this endpoint if needed + if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() + + def _compute_browser_endpoint_key(self) -> str: + """ + Compute a unique key identifying this browser connection. + + For CDP connections, uses the normalized CDP URL so all BrowserManager + instances connecting to the same browser share page tracking. + For standalone browsers, uses instance id since each is independent. + + Returns: + str: Unique identifier for this browser connection + """ + # For CDP connections, use the CDP URL as the key (normalized) + if self.config.cdp_url: + return self._normalize_cdp_url(self.config.cdp_url) + + # For managed browsers, use the CDP URL/port that was assigned + if self.managed_browser: + # Use debugging port as the key since it uniquely identifies the browser + port = getattr(self.managed_browser, 'debugging_port', None) + host = getattr(self.managed_browser, 'host', 'localhost') + if port: + return f"cdp:http://{host}:{port}" + + # For standalone browsers, use instance id (no sharing needed) + return f"instance:{id(self)}" + + def _normalize_cdp_url(self, cdp_url: str) -> str: + """ + Normalize a CDP URL to a canonical form for consistent tracking. + + Handles various formats: + - http://localhost:9222 + - ws://localhost:9222/devtools/browser/xxx + - http://localhost:9222?browser_id=xxx + + Returns: + str: Normalized CDP key in format "cdp:http://host:port" + """ + from urllib.parse import urlparse + + parsed = urlparse(cdp_url) + host = parsed.hostname or 'localhost' + port = parsed.port or 9222 + + return f"cdp:http://{host}:{port}" + + def _get_pages_in_use(self) -> set: + """Get the set of pages currently in use for this browser.""" + if self._browser_endpoint_key and self._browser_endpoint_key in BrowserManager._global_pages_in_use: + return BrowserManager._global_pages_in_use[self._browser_endpoint_key] + # Fallback: shouldn't happen, but return empty set + return set() + + def _mark_page_in_use(self, page) -> None: + """Mark a page as in use.""" + if self._browser_endpoint_key: + if self._browser_endpoint_key not in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key] = set() + BrowserManager._global_pages_in_use[self._browser_endpoint_key].add(page) + + def _release_page_from_use(self, page) -> None: + """Release a page from the in-use tracking.""" + if self._browser_endpoint_key and self._browser_endpoint_key in BrowserManager._global_pages_in_use: + BrowserManager._global_pages_in_use[self._browser_endpoint_key].discard(page) + + async def _verify_cdp_ready(self, cdp_url: str) -> bool: + """Verify CDP endpoint is ready with exponential backoff. + + Supports multiple URL formats: + - HTTP URLs: http://localhost:9222 + - HTTP URLs with query params: http://localhost:9222?browser_id=XXX + - WebSocket URLs: ws://localhost:9222/devtools/browser/XXX + """ + import aiohttp + from urllib.parse import urlparse, urlunparse + + # If WebSocket URL, Playwright handles connection directly - skip HTTP verification + if cdp_url.startswith(('ws://', 'wss://')): + self.logger.debug(f"WebSocket CDP URL provided, skipping HTTP verification", tag="BROWSER") + return True + + # Parse HTTP URL and properly construct /json/version endpoint + parsed = urlparse(cdp_url) + # Build URL with /json/version path, preserving query params + verify_url = urlunparse(( + parsed.scheme, + parsed.netloc, + '/json/version', # Always use this path for verification + '', # params + parsed.query, # preserve query string + '' # fragment + )) + + self.logger.debug(f"Starting CDP verification for {verify_url}", tag="BROWSER") + for attempt in range(5): + try: + async with aiohttp.ClientSession() as session: + async with session.get(verify_url, timeout=aiohttp.ClientTimeout(total=2)) as response: + if response.status == 200: + self.logger.debug(f"CDP endpoint ready after {attempt + 1} attempts", tag="BROWSER") + return True + except Exception as e: + self.logger.debug(f"CDP check attempt {attempt + 1} failed: {e}", tag="BROWSER") + delay = 0.5 * (1.4 ** attempt) + self.logger.debug(f"Waiting {delay:.2f}s before next CDP check...", tag="BROWSER") + await asyncio.sleep(delay) + self.logger.debug(f"CDP verification failed after 5 attempts", tag="BROWSER") + return False + + def _build_browser_args(self) -> dict: + """Build browser launch arguments from config.""" + args = [ + "--disable-gpu", + "--disable-gpu-compositing", + "--disable-software-rasterizer", + "--no-sandbox", + "--disable-dev-shm-usage", + "--no-first-run", + "--no-default-browser-check", + "--disable-infobars", + "--window-position=0,0", + "--ignore-certificate-errors", + "--ignore-certificate-errors-spki-list", + "--disable-blink-features=AutomationControlled", + "--window-position=400,0", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--force-color-profile=srgb", + "--mute-audio", + "--disable-background-timer-throttling", + # Memory-saving flags: disable unused Chrome features + "--disable-features=OptimizationHints,MediaRouter,DialMediaRouteProvider", + "--disable-component-update", + "--disable-domain-reliability", + # "--single-process", + f"--window-size={self.config.viewport_width},{self.config.viewport_height}", + ] + + if self.config.memory_saving_mode: + args.extend([ + "--aggressive-cache-discard", + '--js-flags=--max-old-space-size=512', + ]) + + if self.config.light_mode: + args.extend(BROWSER_DISABLE_OPTIONS) + + if self.config.text_mode: + args.extend( + [ + "--blink-settings=imagesEnabled=false", + "--disable-remote-fonts", + "--disable-images", + "--disable-javascript", + "--disable-software-rasterizer", + "--disable-dev-shm-usage", + ] + ) + + if self.config.extra_args: + args.extend(self.config.extra_args) + + # Deduplicate args + args = list(dict.fromkeys(args)) + + browser_args = {"headless": self.config.headless, "args": args} + + # On Windows, passing channel='chromium' (the default) causes Playwright + # to look for a system Chrome installation instead of using the bundled + # ms-playwright binary. This makes Chrome exit immediately with code 0, + # resulting in TargetClosedError. Skip the default channel. + if self.config.chrome_channel and self.config.chrome_channel != "chromium": + browser_args["channel"] = self.config.chrome_channel + + if self.config.accept_downloads: + browser_args["downloads_path"] = self.config.downloads_path or os.path.join( + os.getcwd(), "downloads" + ) + os.makedirs(browser_args["downloads_path"], exist_ok=True) + + if self.config.proxy: + warnings.warn( + "BrowserConfig.proxy is deprecated and ignored. Use proxy_config instead.", + DeprecationWarning, + ) + if self.config.proxy_config: + from playwright.async_api import ProxySettings + + proxy_settings = ProxySettings( + server=self.config.proxy_config.server, + username=self.config.proxy_config.username, + password=self.config.proxy_config.password, + ) + browser_args["proxy"] = proxy_settings + + return browser_args + + async def setup_context( + self, + context: BrowserContext, + crawlerRunConfig: CrawlerRunConfig = None, + is_default=False, + ): + """ + Set up a browser context with the configured options. + + How it works: + 1. Set extra HTTP headers if provided. + 2. Add cookies if provided. + 3. Load storage state if provided. + 4. Accept downloads if enabled. + 5. Set default timeouts for navigation and download. + 6. Set user agent if provided. + 7. Set browser hints if provided. + 8. Set proxy if provided. + 9. Set downloads path if provided. + 10. Set storage state if provided. + 11. Set cache if provided. + 12. Set extra HTTP headers if provided. + 13. Add cookies if provided. + 14. Set default timeouts for navigation and download if enabled. + 15. Set user agent if provided. + 16. Set browser hints if provided. + + Args: + context (BrowserContext): The browser context to set up + crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings + is_default (bool): Flag indicating if this is the default context + Returns: + None + """ + if self.config.headers: + await context.set_extra_http_headers(self.config.headers) + + if self.config.cookies: + await context.add_cookies(self.config.cookies) + + if self.config.storage_state: + await context.storage_state(path=None) + + if self.config.accept_downloads: + context.set_default_timeout(DOWNLOAD_PAGE_TIMEOUT) + context.set_default_navigation_timeout(DOWNLOAD_PAGE_TIMEOUT) + if self.config.downloads_path: + context._impl_obj._options["accept_downloads"] = True + context._impl_obj._options[ + "downloads_path" + ] = self.config.downloads_path + + # Handle user agent and browser hints + if self.config.user_agent: + combined_headers = { + "User-Agent": self.config.user_agent, + "sec-ch-ua": self.config.browser_hint, + } + combined_headers.update(self.config.headers) + await context.set_extra_http_headers(combined_headers) + + # Add default cookie (skip for raw:/file:// URLs which are not valid cookie URLs) + cookie_url = None + if crawlerRunConfig and crawlerRunConfig.url: + url = crawlerRunConfig.url + # Only set cookie for http/https URLs + if url.startswith(("http://", "https://")): + cookie_url = url + elif crawlerRunConfig.base_url and crawlerRunConfig.base_url.startswith(("http://", "https://")): + # Use base_url as fallback for raw:/file:// URLs + cookie_url = crawlerRunConfig.base_url + + if cookie_url: + await context.add_cookies( + [ + { + "name": "cookiesEnabled", + "value": "true", + "url": cookie_url, + } + ] + ) + + # Handle navigator overrides + if crawlerRunConfig: + if ( + crawlerRunConfig.override_navigator + or crawlerRunConfig.simulate_user + or crawlerRunConfig.magic + ): + await context.add_init_script(load_js_script("navigator_overrider")) + context._crawl4ai_nav_overrider_injected = True + + # Force-open closed shadow roots when flatten_shadow_dom is enabled + if crawlerRunConfig and crawlerRunConfig.flatten_shadow_dom: + await context.add_init_script(""" + const _origAttachShadow = Element.prototype.attachShadow; + Element.prototype.attachShadow = function(init) { + return _origAttachShadow.call(this, {...init, mode: 'open'}); + }; + """) + context._crawl4ai_shadow_dom_injected = True + + # Apply custom init_scripts from BrowserConfig (for stealth evasions, etc.) + if self.config.init_scripts: + for script in self.config.init_scripts: + await context.add_init_script(script) + + async def create_browser_context(self, crawlerRunConfig: CrawlerRunConfig = None): + """ + Creates and returns a new browser context with configured settings. + Applies text-only mode settings if text_mode is enabled in config. + + Returns: + Context: Browser context object with the specified configurations + """ + if self.browser is None: + if self._launched_persistent: + raise RuntimeError( + "Cannot create new browser contexts when using " + "use_persistent_context=True. Persistent context uses a " + "single shared context." + ) + raise RuntimeError( + "Browser is not available. It may have been closed, crashed, " + "or not yet started. Ensure the browser is running before " + "creating new contexts." + ) + # Base settings + user_agent = self.config.headers.get("User-Agent", self.config.user_agent) + viewport_settings = { + "width": self.config.viewport_width, + "height": self.config.viewport_height, + } + proxy_settings = {"server": self.config.proxy} if self.config.proxy else None + + # CSS extensions (blocked separately via avoid_css flag) + css_extensions = ["css", "less", "scss", "sass"] + + # Static resource extensions (blocked when text_mode is enabled) + static_extensions = [ + # Images + "jpg", "jpeg", "png", "gif", "webp", "svg", "ico", "bmp", "tiff", "psd", + # Fonts + "woff", "woff2", "ttf", "otf", "eot", + # Media + "mp4", "webm", "ogg", "avi", "mov", "wmv", "flv", "m4v", + "mp3", "wav", "aac", "m4a", "opus", "flac", + # Documents + "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", + # Archives + "zip", "rar", "7z", "tar", "gz", + # Scripts and data + "xml", "swf", "wasm", + ] + + # Ad and tracker domain patterns (curated from uBlock/EasyList sources) + ad_tracker_patterns = [ + "**/google-analytics.com/**", + "**/googletagmanager.com/**", + "**/googlesyndication.com/**", + "**/doubleclick.net/**", + "**/adservice.google.com/**", + "**/adsystem.com/**", + "**/adzerk.net/**", + "**/adnxs.com/**", + "**/ads.linkedin.com/**", + "**/facebook.net/**", + "**/analytics.twitter.com/**", + "**/ads-twitter.com/**", + "**/hotjar.com/**", + "**/clarity.ms/**", + "**/scorecardresearch.com/**", + "**/pixel.wp.com/**", + "**/amazon-adsystem.com/**", + "**/mixpanel.com/**", + "**/segment.com/**", + ] + + # Common context settings + context_settings = { + "user_agent": user_agent, + "viewport": viewport_settings, + "proxy": proxy_settings, + "accept_downloads": self.config.accept_downloads, + "storage_state": self.config.storage_state, + "ignore_https_errors": self.config.ignore_https_errors, + "device_scale_factor": self.config.device_scale_factor, + "java_script_enabled": self.config.java_script_enabled, + } + + if crawlerRunConfig: + # Check if there is value for crawlerRunConfig.proxy_config set add that to context + if crawlerRunConfig.proxy_config: + from playwright.async_api import ProxySettings + proxy_settings = ProxySettings( + server=crawlerRunConfig.proxy_config.server, + username=crawlerRunConfig.proxy_config.username, + password=crawlerRunConfig.proxy_config.password, + ) + context_settings["proxy"] = proxy_settings + + if self.config.text_mode: + text_mode_settings = { + "has_touch": False, + "is_mobile": False, + } + # Update context settings with text mode settings + context_settings.update(text_mode_settings) + + # inject locale / tz / geo if user provided them + if crawlerRunConfig: + if crawlerRunConfig.locale: + context_settings["locale"] = crawlerRunConfig.locale + if crawlerRunConfig.timezone_id: + context_settings["timezone_id"] = crawlerRunConfig.timezone_id + if crawlerRunConfig.geolocation: + context_settings["geolocation"] = { + "latitude": crawlerRunConfig.geolocation.latitude, + "longitude": crawlerRunConfig.geolocation.longitude, + "accuracy": crawlerRunConfig.geolocation.accuracy, + } + # ensure geolocation permission + perms = context_settings.get("permissions", []) + perms.append("geolocation") + context_settings["permissions"] = perms + + # Create and return the context with all settings + context = await self.browser.new_context(**context_settings) + + # Build dynamic blocking list based on config flags + to_block = [] + if self.config.avoid_css: + to_block.extend(css_extensions) + if self.config.text_mode: + to_block.extend(static_extensions) + + if to_block: + for ext in to_block: + await context.route(f"**/*.{ext}", lambda route: route.abort()) + + if self.config.avoid_ads: + for pattern in ad_tracker_patterns: + await context.route(pattern, lambda route: route.abort()) + + return context + + def _make_config_signature(self, crawlerRunConfig: CrawlerRunConfig) -> str: + """ + Hash ONLY the CrawlerRunConfig fields that affect browser context + creation (create_browser_context) or context setup (setup_context). + + Whitelist approach: fields like css_selector, word_count_threshold, + screenshot, verbose, etc. do NOT cause a new context to be created. + """ + import json + + sig_dict = {} + + # Fields that flow into create_browser_context() + pc = crawlerRunConfig.proxy_config + if pc is not None: + sig_dict["proxy_config"] = { + "server": getattr(pc, "server", None), + "username": getattr(pc, "username", None), + "password": getattr(pc, "password", None), + } + else: + sig_dict["proxy_config"] = None + + sig_dict["locale"] = crawlerRunConfig.locale + sig_dict["timezone_id"] = crawlerRunConfig.timezone_id + + geo = crawlerRunConfig.geolocation + if geo is not None: + sig_dict["geolocation"] = { + "latitude": geo.latitude, + "longitude": geo.longitude, + "accuracy": geo.accuracy, + } + else: + sig_dict["geolocation"] = None + + # Fields that flow into setup_context() as init scripts + sig_dict["override_navigator"] = crawlerRunConfig.override_navigator + sig_dict["simulate_user"] = crawlerRunConfig.simulate_user + sig_dict["magic"] = crawlerRunConfig.magic + + # Browser version — bumped on recycle to force new browser instance + sig_dict["_browser_version"] = self._browser_version + + signature_json = json.dumps(sig_dict, sort_keys=True, default=str) + return hashlib.sha256(signature_json.encode("utf-8")).hexdigest() + + def _evict_lru_context_locked(self): + """ + If contexts exceed the limit, find the least-recently-used context + with zero active crawls and remove it from all tracking dicts. + + MUST be called while holding self._contexts_lock. + + Returns the BrowserContext to close (caller closes it OUTSIDE the + lock), or None if no eviction is needed or possible. + """ + if len(self.contexts_by_config) <= self._max_contexts: + return None + + # Sort candidates by last-used timestamp (oldest first) + candidates = sorted( + self._context_last_used.items(), + key=lambda item: item[1], + ) + for evict_sig, _ in candidates: + if self._context_refcounts.get(evict_sig, 0) == 0: + ctx = self.contexts_by_config.pop(evict_sig, None) + self._context_refcounts.pop(evict_sig, None) + self._context_last_used.pop(evict_sig, None) + # Clean up stale page->sig mappings for evicted context + stale_pages = [ + p for p, s in self._page_to_sig.items() if s == evict_sig + ] + for p in stale_pages: + del self._page_to_sig[p] + return ctx + + # All contexts are in active use — cannot evict + return None + + async def _apply_stealth_to_page(self, page): + """Apply stealth to a page if stealth mode is enabled""" + if self._stealth_adapter: + try: + await self._stealth_adapter.apply_stealth(page) + except Exception as e: + if self.logger: + self.logger.warning( + message="Failed to apply stealth to page: {error}", + tag="STEALTH", + params={"error": str(e)} + ) + + async def _get_page_by_target_id(self, context: BrowserContext, target_id: str): + """ + Get an existing page by its CDP target ID. + + This is used when connecting to a pre-created browser context with an existing page. + Playwright may not immediately see targets created via raw CDP commands, so we + use CDP to get all targets and find the matching one. + + Args: + context: The browser context to search in + target_id: The CDP target ID to find + + Returns: + Page object if found, None otherwise + """ + try: + # First check if Playwright already sees the page + for page in context.pages: + # Playwright's internal target ID might match + if hasattr(page, '_impl_obj') and hasattr(page._impl_obj, '_target_id'): + if page._impl_obj._target_id == target_id: + return page + + # If not found, try using CDP to get targets + if hasattr(self.browser, '_impl_obj') and hasattr(self.browser._impl_obj, '_connection'): + cdp_session = await context.new_cdp_session(context.pages[0] if context.pages else None) + if cdp_session: + try: + result = await cdp_session.send("Target.getTargets") + targets = result.get("targetInfos", []) + for target in targets: + if target.get("targetId") == target_id: + # Found the target - if it's a page type, we can use it + if target.get("type") == "page": + # The page exists, let Playwright discover it + await asyncio.sleep(0.1) + # Refresh pages list + if context.pages: + return context.pages[0] + finally: + await cdp_session.detach() + + # Fallback: if there are any pages now, return the first one + if context.pages: + return context.pages[0] + + return None + except Exception as e: + if self.logger: + self.logger.warning( + message="Failed to get page by target ID: {error}", + tag="BROWSER", + params={"error": str(e)} + ) + return None + + async def get_page(self, crawlerRunConfig: CrawlerRunConfig): + """ + Get a page for the given session ID, creating a new one if needed. + + Args: + crawlerRunConfig (CrawlerRunConfig): Configuration object containing all browser settings + + Returns: + (page, context): The Page and its BrowserContext + """ + self._cleanup_expired_sessions() + + # If a session_id is provided and we already have it, reuse that page + context + if crawlerRunConfig.session_id and crawlerRunConfig.session_id in self.sessions: + context, page, _ = self.sessions[crawlerRunConfig.session_id] + # Update last-used timestamp + self.sessions[crawlerRunConfig.session_id] = (context, page, time.time()) + return page, context + + # If using a managed browser, just grab the shared default_context + if self.config.use_managed_browser: + # If create_isolated_context is True, create isolated contexts for concurrent crawls + # Uses the same caching mechanism as non-CDP mode: cache context by config signature, + # but always create a new page. This prevents navigation conflicts while allowing + # context reuse for multiple URLs with the same config (e.g., batch/deep crawls). + if self.config.create_isolated_context: + config_signature = self._make_config_signature(crawlerRunConfig) + to_close = None + + async with self._contexts_lock: + if config_signature in self.contexts_by_config: + context = self.contexts_by_config[config_signature] + else: + context = await self.create_browser_context(crawlerRunConfig) + await self.setup_context(context, crawlerRunConfig) + self.contexts_by_config[config_signature] = context + self._context_refcounts[config_signature] = 0 + to_close = self._evict_lru_context_locked() + + # Increment refcount INSIDE lock before releasing + self._context_refcounts[config_signature] = ( + self._context_refcounts.get(config_signature, 0) + 1 + ) + self._context_last_used[config_signature] = time.monotonic() + + # Close evicted context OUTSIDE lock + if to_close is not None: + try: + await to_close.close() + except Exception: + pass + + # Always create a new page for each crawl (isolation for navigation) + try: + page = await context.new_page() + except Exception: + async with self._contexts_lock: + if config_signature in self._context_refcounts: + self._context_refcounts[config_signature] = max( + 0, self._context_refcounts[config_signature] - 1 + ) + raise + await self._apply_stealth_to_page(page) + self._page_to_sig[page] = config_signature + elif self.config.storage_state: + tmp_context = await self.create_browser_context(crawlerRunConfig) + ctx = self.default_context # default context, one window only + ctx = await clone_runtime_state(tmp_context, ctx, crawlerRunConfig, self.config) + # Close the temporary context — only needed as a clone source + try: + await tmp_context.close() + except Exception: + pass + context = ctx # so (page, context) return value is correct + # Avoid concurrent new_page on shared persistent context + # See GH-1198: context.pages can be empty under races + async with self._page_lock: + page = await ctx.new_page() + await self._apply_stealth_to_page(page) + else: + context = self.default_context + + # Handle pre-existing target case (for reconnecting to specific CDP targets) + if self.config.browser_context_id and self.config.target_id: + page = await self._get_page_by_target_id(context, self.config.target_id) + if not page: + async with self._page_lock: + page = await context.new_page() + self._mark_page_in_use(page) + await self._apply_stealth_to_page(page) + else: + # Mark pre-existing target as in use + self._mark_page_in_use(page) + else: + # For CDP connections (external browser), multiple Playwright connections + # create separate browser/context objects. Page reuse across connections + # isn't reliable because each connection sees different page objects. + # Always create new pages for CDP to avoid cross-connection race conditions. + if self.config.cdp_url and not self.config.use_managed_browser: + async with self._page_lock: + page = await context.new_page() + self._mark_page_in_use(page) + await self._apply_stealth_to_page(page) + else: + # For managed browsers (single process), page reuse is safe. + # Use lock to safely check for available pages and track usage. + # This prevents race conditions when multiple crawls run concurrently. + async with BrowserManager._get_global_lock(): + pages = context.pages + pages_in_use = self._get_pages_in_use() + # Find first available page (exists and not currently in use) + available_page = next( + (p for p in pages if p not in pages_in_use), + None + ) + if available_page: + page = available_page + else: + # No available pages - create a new one + page = await context.new_page() + await self._apply_stealth_to_page(page) + # Mark page as in use (global tracking) + self._mark_page_in_use(page) + else: + # Otherwise, check if we have an existing context for this config + config_signature = self._make_config_signature(crawlerRunConfig) + to_close = None + + async with self._contexts_lock: + if config_signature in self.contexts_by_config: + context = self.contexts_by_config[config_signature] + else: + # Create and setup a new context + context = await self.create_browser_context(crawlerRunConfig) + await self.setup_context(context, crawlerRunConfig) + self.contexts_by_config[config_signature] = context + self._context_refcounts[config_signature] = 0 + to_close = self._evict_lru_context_locked() + + # Increment refcount INSIDE lock before releasing + self._context_refcounts[config_signature] = ( + self._context_refcounts.get(config_signature, 0) + 1 + ) + self._context_last_used[config_signature] = time.monotonic() + + # Close evicted context OUTSIDE lock + if to_close is not None: + try: + await to_close.close() + except Exception: + pass + + # Create a new page from the chosen context + try: + page = await context.new_page() + except Exception: + async with self._contexts_lock: + if config_signature in self._context_refcounts: + self._context_refcounts[config_signature] = max( + 0, self._context_refcounts[config_signature] - 1 + ) + raise + await self._apply_stealth_to_page(page) + self._page_to_sig[page] = config_signature + + # If a session_id is specified, store this session so we can reuse later + if crawlerRunConfig.session_id: + self.sessions[crawlerRunConfig.session_id] = (context, page, time.time()) + + self._pages_served += 1 + + # Check if browser recycle threshold is hit — bump version for next requests + # This happens AFTER incrementing counter so concurrent requests see correct count + await self._maybe_bump_browser_version() + + return page, context + + async def kill_session(self, session_id: str): + """ + Kill a browser session and clean up resources. + + Args: + session_id (str): The session ID to kill. + """ + if session_id in self.sessions: + context, page, _ = self.sessions[session_id] + self._release_page_from_use(page) + # Decrement context refcount for the session's page + should_close_context = False + async with self._contexts_lock: + sig = self._page_to_sig.pop(page, None) + if sig is not None and sig in self._context_refcounts: + self._context_refcounts[sig] = max( + 0, self._context_refcounts[sig] - 1 + ) + # Only close the context if no other pages are using it + # (refcount dropped to 0) AND we own the context (not managed) + if not self.config.use_managed_browser: + if self._context_refcounts.get(sig, 0) == 0: + self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + should_close_context = True + await page.close() + if should_close_context: + await context.close() + del self.sessions[session_id] + + def release_page(self, page): + """ + Release a page from the in-use tracking set (global tracking). + Sync variant — does NOT decrement context refcount. + """ + self._release_page_from_use(page) + + async def release_page_with_context(self, page): + """ + Release a page and decrement its context's refcount under the lock. + + Should be called from the async crawl finally block instead of + release_page() so the context lifecycle is properly tracked. + """ + self._release_page_from_use(page) + sig = None + refcount = -1 + async with self._contexts_lock: + sig = self._page_to_sig.pop(page, None) + if sig is not None and sig in self._context_refcounts: + self._context_refcounts[sig] = max( + 0, self._context_refcounts[sig] - 1 + ) + refcount = self._context_refcounts[sig] + + # Check if this signature belongs to an old browser waiting to be cleaned up + if sig is not None and refcount == 0: + await self._maybe_cleanup_old_browser(sig) + + def _should_recycle(self) -> bool: + """Check if page threshold reached for browser recycling.""" + limit = self.config.max_pages_before_recycle + if limit <= 0: + return False + return self._pages_served >= limit + + async def _maybe_bump_browser_version(self): + """Bump browser version if threshold reached, moving old browser to pending cleanup. + + New requests automatically get a new browser (via new signature). + Old browser drains naturally and gets cleaned up when refcount hits 0. + """ + if not self._should_recycle(): + return + + # Safety cap: wait if too many old browsers are draining + while True: + async with self._pending_cleanup_lock: + # Re-check threshold under lock (another request may have bumped already) + if not self._should_recycle(): + return + + # Check safety cap + if len(self._pending_cleanup) >= self._max_pending_browsers: + if self.logger: + self.logger.debug( + message="Waiting for old browser to drain (pending: {count})", + tag="BROWSER", + params={"count": len(self._pending_cleanup)}, + ) + self._cleanup_slot_available.clear() + # Release lock and wait + else: + # We have a slot — do the bump inside this lock hold + old_version = self._browser_version + active_sigs = [] + idle_sigs = [] + async with self._contexts_lock: + for sig in list(self._context_refcounts.keys()): + if self._context_refcounts.get(sig, 0) > 0: + active_sigs.append(sig) + else: + idle_sigs.append(sig) + + if self.logger: + self.logger.info( + message="Bumping browser version {old} -> {new} after {count} pages ({active} active, {idle} idle sigs)", + tag="BROWSER", + params={ + "old": old_version, + "new": old_version + 1, + "count": self._pages_served, + "active": len(active_sigs), + "idle": len(idle_sigs), + }, + ) + + # Only add sigs with active crawls to pending cleanup. + # Sigs with refcount 0 are cleaned up immediately below + # to avoid them being stuck in _pending_cleanup forever + # (no future release would trigger their cleanup). + done_event = asyncio.Event() + for sig in active_sigs: + self._pending_cleanup[sig] = { + "version": old_version, + "done": done_event, + } + + # Bump version — new get_page() calls will create new contexts + self._browser_version += 1 + self._pages_served = 0 + + # Clean up idle sigs immediately (outside pending_cleanup_lock below) + break # exit while loop to do cleanup outside locks + + # Safety cap path: wait for a cleanup slot, then retry. + # Timeout prevents permanent deadlock if stuck entries never drain. + try: + await asyncio.wait_for( + self._cleanup_slot_available.wait(), timeout=30.0 + ) + except asyncio.TimeoutError: + # Force-clean any pending entries that have refcount 0 + # (they're stuck and will never drain naturally) + async with self._pending_cleanup_lock: + stuck_sigs = [ + s for s in list(self._pending_cleanup.keys()) + if self._context_refcounts.get(s, 0) == 0 + ] + for sig in stuck_sigs: + self._pending_cleanup.pop(sig, None) + if stuck_sigs: + if self.logger: + self.logger.warning( + message="Force-cleaned {count} stuck pending entries after timeout", + tag="BROWSER", + params={"count": len(stuck_sigs)}, + ) + # Clean up the stuck contexts + for sig in stuck_sigs: + async with self._contexts_lock: + context = self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + if context is not None: + try: + await context.close() + except Exception: + pass + if len(self._pending_cleanup) < self._max_pending_browsers: + self._cleanup_slot_available.set() + + # Reached via break — clean up idle sigs immediately (outside locks) + for sig in idle_sigs: + async with self._contexts_lock: + context = self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + if context is not None: + try: + await context.close() + except Exception: + pass + if idle_sigs and self.logger: + self.logger.debug( + message="Immediately cleaned up {count} idle contexts from version {version}", + tag="BROWSER", + params={"count": len(idle_sigs), "version": old_version}, + ) + + async def _maybe_cleanup_old_browser(self, sig: str): + """Clean up an old browser's context if its refcount hit 0 and it's pending cleanup.""" + async with self._pending_cleanup_lock: + if sig not in self._pending_cleanup: + return # Not an old browser signature + + cleanup_info = self._pending_cleanup.pop(sig) + old_version = cleanup_info["version"] + + if self.logger: + self.logger.debug( + message="Cleaning up context from browser version {version} (sig: {sig})", + tag="BROWSER", + params={"version": old_version, "sig": sig[:12]}, + ) + + # Remove context from tracking + async with self._contexts_lock: + context = self.contexts_by_config.pop(sig, None) + self._context_refcounts.pop(sig, None) + self._context_last_used.pop(sig, None) + + # Close context outside locks + if context is not None: + try: + await context.close() + except Exception: + pass + + # Check if any signatures from this old version remain + remaining_old = [ + s for s, info in self._pending_cleanup.items() + if info["version"] == old_version + ] + + if not remaining_old: + if self.logger: + self.logger.info( + message="All contexts from browser version {version} cleaned up", + tag="BROWSER", + params={"version": old_version}, + ) + + # Open a cleanup slot if we're below the cap + if len(self._pending_cleanup) < self._max_pending_browsers: + self._cleanup_slot_available.set() + + def _cleanup_expired_sessions(self): + """Clean up expired sessions based on TTL.""" + current_time = time.time() + expired_sessions = [ + sid + for sid, (_, _, last_used) in self.sessions.items() + if current_time - last_used > self.session_ttl + ] + for sid in expired_sessions: + asyncio.create_task(self.kill_session(sid)) + + async def close(self): + """Close all browser resources and clean up.""" + # Cached CDP path: only clean up this instance's sessions/contexts, + # then release the shared connection reference. + if self._using_cached_cdp: + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception: + pass + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + await _CDPConnectionCache.release(self.config.cdp_url) + self.browser = None + self.playwright = None + self._using_cached_cdp = False + return + + if self.config.cdp_url: + # When using external CDP, we don't own the browser process. + # If cdp_cleanup_on_close is True, properly disconnect from the browser + # and clean up Playwright resources. This frees the browser for other clients. + if self.config.cdp_cleanup_on_close: + # First close all sessions (pages) + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + + # Close all contexts we created + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception: + pass + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + + # Disconnect from browser (doesn't terminate it, just releases connection) + if self.browser: + try: + await self.browser.close() + except Exception as e: + if self.logger: + self.logger.debug( + message="Error disconnecting from CDP browser: {error}", + tag="BROWSER", + params={"error": str(e)} + ) + self.browser = None + # Allow time for CDP connection to fully release before another client connects + if self.config.cdp_close_delay > 0: + await asyncio.sleep(self.config.cdp_close_delay) + + # Stop Playwright instance to prevent memory leaks + if self.playwright: + await self.playwright.stop() + self.playwright = None + return + + # ── Persistent context launched via launch_persistent_context ── + if self._launched_persistent: + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception: + pass + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + + # Closing the persistent context also terminates the browser + if self.default_context: + try: + await self.default_context.close() + except Exception: + pass + self.default_context = None + + if self.playwright: + await self.playwright.stop() + self.playwright = None + self._launched_persistent = False + return + + if self.config.sleep_on_close: + await asyncio.sleep(0.5) + + session_ids = list(self.sessions.keys()) + for session_id in session_ids: + await self.kill_session(session_id) + + # Now close all contexts we created. This reclaims memory from ephemeral contexts. + for ctx in list(self.contexts_by_config.values()): + try: + await ctx.close() + except Exception as e: + self.logger.error( + message="Error closing context: {error}", + tag="ERROR", + params={"error": str(e)} + ) + self.contexts_by_config.clear() + self._context_refcounts.clear() + self._context_last_used.clear() + self._page_to_sig.clear() + + if self.browser: + await self.browser.close() + self.browser = None + + if self.managed_browser: + await asyncio.sleep(0.5) + await self.managed_browser.cleanup() + self.managed_browser = None + + if self.playwright: + await self.playwright.stop() + self.playwright = None diff --git a/tests/browser/test_startup_failure_cleanup.py b/tests/browser/test_startup_failure_cleanup.py new file mode 100644 index 000000000..0d86f5e5d --- /dev/null +++ b/tests/browser/test_startup_failure_cleanup.py @@ -0,0 +1,52 @@ +import asyncio +from unittest.mock import AsyncMock + +import pytest + +from crawl4ai import AsyncWebCrawler, BrowserConfig +from crawl4ai.browser_manager import BrowserManager + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [RuntimeError("launch failed"), pytest.param(None, id="cancelled")]) +async def test_browser_manager_stops_playwright_after_failed_launch(monkeypatch, failure): + if failure is None: + failure = asyncio.CancelledError() + + playwright = type("FakePlaywright", (), {})() + playwright.stop = AsyncMock() + starter = type("FakeStarter", (), {})() + starter.start = AsyncMock(return_value=playwright) + + monkeypatch.setattr("playwright.async_api.async_playwright", lambda: starter) + + manager = BrowserManager(BrowserConfig(), logger=None) + manager._launch_browser = AsyncMock(side_effect=failure) + + with pytest.raises(type(failure)): + await manager.start() + + playwright.stop.assert_awaited_once() + assert manager.playwright is None + + +@pytest.mark.asyncio +async def test_async_webcrawler_rolls_back_cancelled_strategy_start(tmp_path): + class CancelledStrategy: + def __init__(self): + self.exit = AsyncMock() + + async def __aenter__(self): + raise asyncio.CancelledError() + + async def __aexit__(self, exc_type, exc_value, traceback): + await self.exit(exc_type, exc_value, traceback) + + strategy = CancelledStrategy() + crawler = AsyncWebCrawler(crawler_strategy=strategy, base_directory=str(tmp_path)) + + with pytest.raises(asyncio.CancelledError): + await crawler.start() + + strategy.exit.assert_awaited_once_with(None, None, None) + assert crawler.ready is False