From e77bb73bd12330dd2d3bc4a2084c3f8c5eab0b0e Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Mon, 24 Aug 2026 15:53:54 -0700 Subject: [PATCH 1/8] fix: harden LibGen result parsing --- .../src/pullbox_provider_libgen/metadata.py | 184 ++++++++++++++++-- .../src/pullbox_provider_libgen/parser.py | 69 +++++-- .../src/pullbox_provider_libgen/service.py | 7 +- .../libgen/search-results-compact-v1.html | 41 ++++ tests/unit/test_libgen_metadata.py | 83 ++++++++ tests/unit/test_libgen_parser.py | 34 ++++ tests/unit/test_libgen_service.py | 39 +++- 7 files changed, 425 insertions(+), 32 deletions(-) create mode 100644 tests/fixtures/libgen/search-results-compact-v1.html diff --git a/providers/libgen/src/pullbox_provider_libgen/metadata.py b/providers/libgen/src/pullbox_provider_libgen/metadata.py index 6ca55cc..31321b9 100644 --- a/providers/libgen/src/pullbox_provider_libgen/metadata.py +++ b/providers/libgen/src/pullbox_provider_libgen/metadata.py @@ -4,7 +4,7 @@ import json import re -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from typing import Any from urllib.parse import urlencode, urlsplit, urlunsplit @@ -77,18 +77,7 @@ def __init__( async def enrich(self, discovered: DiscoveredRecord) -> Candidate | None: origin = _source_origin(discovered.source_reference) - file_key = ":".join( - ( - "file", - origin, - discovered.md5, - str(discovered.file_id or ""), - str(discovered.edition_id or ""), - str(discovered.size_bytes or ""), - str(discovered.size_tolerance_bytes or ""), - discovered.extension or "", - ) - ) + file_key = _file_cache_key(origin, discovered) file_lookup = self._cache.get(file_key) if file_lookup.hit: if not isinstance(file_lookup.value, FileMetadata): @@ -105,7 +94,7 @@ async def enrich(self, discovered: DiscoveredRecord) -> Candidate | None: edition_metadata: EditionMetadata | None = None if file_metadata.edition_id is not None: - edition_key = f"edition:{origin}:{file_metadata.edition_id}:{file_metadata.file_id}" + edition_key = _edition_cache_key(origin, file_metadata) edition_lookup = self._cache.get(edition_key) if edition_lookup.hit: if isinstance(edition_lookup.value, EditionMetadata): @@ -126,6 +115,125 @@ async def enrich(self, discovered: DiscoveredRecord) -> Candidate | None: self._cache.set(edition_key, edition_metadata) return build_candidate(discovered, file_metadata, edition_metadata) + async def enrich_many( + self, + discovered_records: Sequence[DiscoveredRecord], + ) -> list[Candidate]: + """Enrich bounded discovery rows with batched keyed API requests.""" + if not discovered_records: + return [] + if len(discovered_records) == 1: + candidate = await self.enrich(discovered_records[0]) + return [candidate] if candidate is not None else [] + + origins = {_source_origin(record.source_reference) for record in discovered_records} + if len(origins) != 1: + raise LibGenMetadataError("LibGen discovery records use conflicting origins.") + origin = origins.pop() + file_metadata_by_md5: dict[str, FileMetadata | None] = {} + pending: list[DiscoveredRecord] = [] + individual: list[DiscoveredRecord] = [] + + for discovered in discovered_records: + file_key = _file_cache_key(origin, discovered) + cached = self._cache.get(file_key) + if cached.hit: + file_metadata_by_md5[discovered.md5] = ( + cached.value if isinstance(cached.value, FileMetadata) else None + ) + elif discovered.file_id is None: + individual.append(discovered) + else: + pending.append(discovered) + + if pending: + payload = await self._fetcher( + _file_metadata_ids_url( + origin, + [record.file_id for record in pending if record.file_id is not None], + ) + ) + for discovered in pending: + file_key = _file_cache_key(origin, discovered) + try: + file_metadata = parse_file_metadata(payload, expected=discovered) + except LibGenMetadataError: + self._cache.set(file_key, None) + file_metadata_by_md5[discovered.md5] = None + else: + self._cache.set(file_key, file_metadata) + file_metadata_by_md5[discovered.md5] = file_metadata + + edition_metadata_by_file_id: dict[int, EditionMetadata | None] = {} + pending_editions: list[FileMetadata] = [] + for cached_file in file_metadata_by_md5.values(): + if cached_file is None or cached_file.edition_id is None: + continue + edition_key = _edition_cache_key(origin, cached_file) + cached = self._cache.get(edition_key) + if cached.hit: + edition_metadata_by_file_id[cached_file.file_id] = ( + cached.value if isinstance(cached.value, EditionMetadata) else None + ) + else: + pending_editions.append(cached_file) + + if pending_editions: + payload = await self._fetcher( + _edition_metadata_ids_url( + origin, + [ + file_metadata.edition_id + for file_metadata in pending_editions + if file_metadata.edition_id is not None + ], + ) + ) + for file_metadata in pending_editions: + edition_id = file_metadata.edition_id + if edition_id is None: + continue + edition_key = _edition_cache_key(origin, file_metadata) + try: + edition_metadata = parse_edition_metadata( + payload, + expected_edition_id=edition_id, + expected_file_id=file_metadata.file_id, + ) + except LibGenMetadataError: + self._cache.set(edition_key, None) + edition_metadata_by_file_id[file_metadata.file_id] = None + else: + self._cache.set(edition_key, edition_metadata) + edition_metadata_by_file_id[file_metadata.file_id] = edition_metadata + + candidates_by_md5: dict[str, Candidate] = {} + for discovered in discovered_records: + candidate_file = file_metadata_by_md5.get(discovered.md5) + if candidate_file is None: + continue + edition_metadata = ( + edition_metadata_by_file_id.get(candidate_file.file_id) + if candidate_file.edition_id is not None + else None + ) + candidates_by_md5[discovered.md5] = build_candidate( + discovered, + candidate_file, + edition_metadata, + ) + + for discovered in individual: + candidate = await self.enrich(discovered) + if candidate is not None: + candidates_by_md5[discovered.md5] = candidate + + return [ + candidates_by_md5[discovered.md5] + for discovered in discovered_records + if discovered.md5 in candidates_by_md5 + ] + def parse_file_metadata(payload: str | bytes, *, expected: DiscoveredRecord) -> FileMetadata: return _parse_file_metadata( @@ -452,6 +560,17 @@ def _file_metadata_url(origin: str, md5: str) -> str: return f"{origin}/json.php?{urlencode(params)}" +def _file_metadata_ids_url(origin: str, file_ids: Sequence[int]) -> str: + identifiers = _bounded_identifiers(file_ids) + params = { + "object": "f", + "ids": ",".join(str(identifier) for identifier in identifiers), + "topic": "c", + "fields": _FILE_FIELDS, + } + return f"{origin}/json.php?{urlencode(params)}" + + def _edition_metadata_url(origin: str, edition_id: int) -> str: params = { "object": "e", @@ -460,3 +579,40 @@ def _edition_metadata_url(origin: str, edition_id: int) -> str: "fields": _EDITION_FIELDS, } return f"{origin}/json.php?{urlencode(params)}" + + +def _edition_metadata_ids_url(origin: str, edition_ids: Sequence[int]) -> str: + identifiers = _bounded_identifiers(edition_ids) + params = { + "object": "e", + "ids": ",".join(str(identifier) for identifier in identifiers), + "topic": "c", + "fields": _EDITION_FIELDS, + } + return f"{origin}/json.php?{urlencode(params)}" + + +def _bounded_identifiers(identifiers: Sequence[int]) -> tuple[int, ...]: + unique = tuple(dict.fromkeys(identifiers)) + if not unique or len(unique) > 100 or any(identifier <= 0 for identifier in unique): + raise LibGenMetadataError("LibGen metadata identifier batch is invalid.") + return unique + + +def _file_cache_key(origin: str, discovered: DiscoveredRecord) -> str: + return ":".join( + ( + "file", + origin, + discovered.md5, + str(discovered.file_id or ""), + str(discovered.edition_id or ""), + str(discovered.size_bytes or ""), + str(discovered.size_tolerance_bytes or ""), + discovered.extension or "", + ) + ) + + +def _edition_cache_key(origin: str, file_metadata: FileMetadata) -> str: + return f"edition:{origin}:{file_metadata.edition_id}:{file_metadata.file_id}" diff --git a/providers/libgen/src/pullbox_provider_libgen/parser.py b/providers/libgen/src/pullbox_provider_libgen/parser.py index 6b8a74f..2c77da1 100644 --- a/providers/libgen/src/pullbox_provider_libgen/parser.py +++ b/providers/libgen/src/pullbox_provider_libgen/parser.py @@ -50,7 +50,9 @@ class _Link: @dataclass(slots=True) class _Cell: tag: str + colspan: int = 1 parts: list[str] = field(default_factory=list) + primary_parts: list[str] = field(default_factory=list) secondary_parts: list[str] = field(default_factory=list) links: list[_Link] = field(default_factory=list) @@ -62,6 +64,10 @@ def text(self) -> str: def secondary_text(self) -> str: return _normalize_text(" ".join(self.secondary_parts)) + @property + def primary_text(self) -> str: + return _normalize_text(" ".join(self.primary_parts)) + @dataclass(slots=True) class _Row: @@ -93,7 +99,7 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None if tag == "tr" and self._row is None: self._row = _Row() elif tag in {"td", "th"} and self._row is not None and self._cell is None: - self._cell = _Cell(tag=tag) + self._cell = _Cell(tag=tag, colspan=_cell_colspan(values.get("colspan"))) elif tag == "a" and self._cell is not None and self._link is None: self._link = (values.get("href", "").strip(), []) elif tag == "font" and self._cell is not None: @@ -131,6 +137,8 @@ def handle_data(self, data: str) -> None: self._cell.parts.append(normalized) if self._secondary_depth: self._cell.secondary_parts.append(normalized) + else: + self._cell.primary_parts.append(normalized) if self._link is not None: self._link[1].append(normalized) @@ -148,9 +156,7 @@ def parse_search_html(html: str, *, source_origin: str) -> list[DiscoveredRecord raise LibGenLayoutError("LibGen search layout is no longer recognized.") data_rows = parser.rows[1:] - if data_rows and not any( - len(row.cells) == 9 and all(cell.tag == "td" for cell in row.cells) for row in data_rows - ): + if data_rows and not any(_is_supported_data_row(row) for row in data_rows): raise LibGenLayoutError("LibGen search result row layout is no longer recognized.") records: list[DiscoveredRecord] = [] @@ -168,17 +174,35 @@ def parse_search_html(html: str, *, source_origin: str) -> list[DiscoveredRecord def _parse_record(row: _Row, *, origin: str) -> DiscoveredRecord | None: - if len(row.cells) != 9 or any(cell.tag != "td" for cell in row.cells): + if not _is_supported_data_row(row): return None - identity, author, publisher, year, language, pages, size, extension, mirrors = row.cells + if len(row.cells) == 9: + identity, author, publisher, year, language, pages, size, extension, mirrors = row.cells + display_title = _display_title(identity.links) + raw_title = identity.secondary_text or display_title + author_text = _optional_text(author.text) + publisher_text = _optional_text(publisher.text) + year_value = _optional_year(year.text) + language_text = _optional_text(language.text) + edition_id = _linked_id(identity.links, path="edition.php") + source_series_id = _linked_id(identity.links, path="series.php") + else: + identity, pages, size, extension, mirrors = row.cells + display_title = identity.primary_text + raw_title = display_title + author_text = None + publisher_text = None + year_value = None + language_text = None + edition_id = None + source_series_id = None + get_url, md5 = _get_reference(mirrors.links, origin=origin) if get_url is None or md5 is None: return None file_url, file_id = _id_reference(size.links, origin=origin, path="/file.php") source_reference = file_url or get_url - display_title = _display_title(identity.links) - raw_title = identity.secondary_text or display_title if not display_title or not raw_title: return None if len(display_title) > _MAX_TITLE_CHARS or len(raw_title) > _MAX_TITLE_CHARS: @@ -191,12 +215,12 @@ def _parse_record(row: _Row, *, origin: str) -> DiscoveredRecord | None: display_title=display_title, raw_title=raw_title, file_id=file_id, - edition_id=_linked_id(identity.links, path="edition.php"), - source_series_id=_linked_id(identity.links, path="series.php"), - author=_optional_text(author.text), - publisher=_optional_text(publisher.text), - year=_optional_year(year.text), - language=_optional_text(language.text), + edition_id=edition_id, + source_series_id=source_series_id, + author=author_text, + publisher=publisher_text, + year=year_value, + language=language_text, pages=_optional_integer(pages.text), size_bytes=size_bytes, size_tolerance_bytes=size_tolerance_bytes, @@ -204,6 +228,23 @@ def _parse_record(row: _Row, *, origin: str) -> DiscoveredRecord | None: ) +def _is_supported_data_row(row: _Row) -> bool: + if any(cell.tag != "td" for cell in row.cells): + return False + colspans = [cell.colspan for cell in row.cells] + return colspans == [1] * 9 or colspans == [5, 1, 1, 1, 1] + + +def _cell_colspan(raw_value: str | None) -> int: + if raw_value is None or raw_value == "": + return 1 + normalized = raw_value.strip() + if _POSITIVE_INTEGER.fullmatch(normalized) is None: + return 0 + value = int(normalized) + return value if value <= 9 else 0 + + def _is_header_row(row: _Row) -> bool: if not row.cells or any(cell.tag != "th" for cell in row.cells): return False diff --git a/providers/libgen/src/pullbox_provider_libgen/service.py b/providers/libgen/src/pullbox_provider_libgen/service.py index a435ea4..ea4bade 100644 --- a/providers/libgen/src/pullbox_provider_libgen/service.py +++ b/providers/libgen/src/pullbox_provider_libgen/service.py @@ -429,13 +429,14 @@ async def fetch_metadata(url: str) -> str: seen: set[str] = set() for query in _build_queries(intent): html = await session.fetch_text(_search_url(origin, query)) + discoveries = [] for discovered in parse_search_html(html, source_origin=origin): if discovered.md5 in seen: continue seen.add(discovered.md5) - candidate = await enricher.enrich(discovered) - if candidate is not None: - candidates.append(candidate) + discoveries.append(discovered) + for candidate in await enricher.enrich_many(discoveries): + candidates.append(candidate) if len(candidates) >= limit: result = tuple(candidates[:limit]) self._search_cache.set(cache_key, result) diff --git a/tests/fixtures/libgen/search-results-compact-v1.html b/tests/fixtures/libgen/search-results-compact-v1.html new file mode 100644 index 0000000..afc4697 --- /dev/null +++ b/tests/fixtures/libgen/search-results-compact-v1.html @@ -0,0 +1,41 @@ + + + +
+ + + + + + + + + + + + + + + + + + + +
ID / Time / Title / SeriesAuthor(s)PublisherYearLanguagePagesSizeExt.Mirrors
+
+
+ + Clockwork Harbor 003 (2024) +
sample\2024.04.03 + +
1600x2400px 72dpi +
+
+ 		2418 MBcbz + Libgen + + External + +
+ + diff --git a/tests/unit/test_libgen_metadata.py b/tests/unit/test_libgen_metadata.py index 6d27f83..43e50a8 100644 --- a/tests/unit/test_libgen_metadata.py +++ b/tests/unit/test_libgen_metadata.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json +from dataclasses import replace from pathlib import Path import pytest @@ -179,6 +181,87 @@ async def fetcher(url: str) -> str: assert "ids=910" in calls[1] +async def test_metadata_enricher_batches_multiple_keyed_records() -> None: + calls: list[str] = [] + file_records = json.loads(_fixture("file-v1.json")) | json.loads( + _fixture("file-sparse-v1.json") + ) + file_records["1202"]["editions"] = {"50002": {"e_id": "911"}} + edition_records = json.loads(_fixture("edition-v1.json")) + edition_records["911"] = { + "title": "Clockwork Harbor Deluxe Collection", + "series_name": "Clockwork Harbor", + "publisher": "Example Press", + "year": "2025", + "issue_number": "2", + "issue_volume": "2", + "visible": "1", + "files": {"70002": {"f_id": "1202"}}, + } + + async def fetcher(url: str) -> str: + calls.append(url) + return json.dumps(edition_records if "object=e" in url else file_records) + + enricher = LibGenMetadataEnricher(fetcher=fetcher) + candidates = await enricher.enrich_many(_records()) + cached = await enricher.enrich_many(_records()) + + assert cached == candidates + assert [candidate.provider_candidate_id for candidate in candidates] == [ + "libgen:0123456789abcdef0123456789abcdef", + "libgen:fedcba9876543210fedcba9876543210", + ] + assert len(calls) == 2 + assert "object=f" in calls[0] + assert "ids=1201%2C1202" in calls[0] + assert "object=e" in calls[1] + assert "ids=910%2C911" in calls[1] + + +async def test_metadata_enricher_rejects_only_the_invalid_record_in_a_batch() -> None: + calls: list[str] = [] + file_records = json.loads(_fixture("file-v1.json")) + + async def fetcher(url: str) -> str: + calls.append(url) + return _fixture("edition-v1.json") if "object=e" in url else json.dumps(file_records) + + enricher = LibGenMetadataEnricher(fetcher=fetcher) + candidates = await enricher.enrich_many(_records()) + cached = await enricher.enrich_many(_records()) + + assert cached == candidates + assert [candidate.provider_candidate_id for candidate in candidates] == [ + "libgen:0123456789abcdef0123456789abcdef" + ] + assert len(calls) == 2 + + +async def test_metadata_enricher_validates_shared_edition_for_each_file_once() -> None: + calls: list[str] = [] + file_records = json.loads(_fixture("file-v1.json")) | json.loads( + _fixture("file-sparse-v1.json") + ) + file_records["1202"]["editions"] = {"50002": {"e_id": "910"}} + edition_records = json.loads(_fixture("edition-v1.json")) + edition_records["910"]["files"]["70002"] = {"f_id": "1202"} + + async def fetcher(url: str) -> str: + calls.append(url) + return json.dumps(edition_records if "object=e" in url else file_records) + + enricher = LibGenMetadataEnricher(fetcher=fetcher) + records = _records() + records[1] = replace(records[1], edition_id=910) + candidates = await enricher.enrich_many(records) + + assert len(candidates) == 2 + assert await enricher.enrich_many(records) == candidates + assert len(calls) == 2 + assert "ids=910" in calls[1] + + async def test_metadata_enricher_negatively_caches_invalid_source_records() -> None: calls = 0 diff --git a/tests/unit/test_libgen_parser.py b/tests/unit/test_libgen_parser.py index 23115fa..27ef044 100644 --- a/tests/unit/test_libgen_parser.py +++ b/tests/unit/test_libgen_parser.py @@ -50,6 +50,40 @@ def test_parse_search_html_extracts_issue_and_collection_evidence() -> None: assert collection.pages is None +def test_parse_search_html_extracts_compact_file_row() -> None: + records = parse_search_html( + _fixture("search-results-compact-v1.html"), + source_origin="https://libgen.gl", + ) + + assert len(records) == 1 + record = records[0] + assert record.md5 == "0123456789abcdef0123456789abcdef" + assert record.source_reference == "https://libgen.gl/file.php?id=1201" + assert record.display_title == "Clockwork Harbor 003 (2024)" + assert record.raw_title == "Clockwork Harbor 003 (2024)" + assert record.file_id == 1201 + assert record.edition_id is None + assert record.source_series_id is None + assert record.author is None + assert record.publisher is None + assert record.year is None + assert record.language is None + assert record.pages == 24 + assert record.size_bytes == 18 * 1024 * 1024 + assert record.extension == "cbz" + + +def test_parse_search_html_rejects_unrecognized_compact_row_shape() -> None: + html = _fixture("search-results-compact-v1.html").replace( + 'colspan="5"', + 'colspan="4"', + ) + + with pytest.raises(LibGenLayoutError, match="layout"): + parse_search_html(html, source_origin="https://libgen.gl") + + def test_parse_search_html_isolates_malformed_rows_and_external_links() -> None: records = parse_search_html( _fixture("search-results-v1.html"), diff --git a/tests/unit/test_libgen_service.py b/tests/unit/test_libgen_service.py index 55f07ce..a156dc8 100644 --- a/tests/unit/test_libgen_service.py +++ b/tests/unit/test_libgen_service.py @@ -183,6 +183,10 @@ async def fetch_text(self, url: str, *, max_bytes: int = 2 * 1024 * 1024) -> str if "/index.php" in url: return self.search_html if "object=f" in url: + if "ids=1201%2C1202" in url: + first = json.loads(_fixture("file-v1.json")) + second = json.loads(_fixture("file-sparse-v1.json")) + return json.dumps(first | second) return ( _fixture("file-v1.json") if "0123456789abcdef0123456789abcdef" in url @@ -278,7 +282,13 @@ async def test_service_search_discovers_enriches_deduplicates_and_caches() -> No assert factory.profiles == [profile] assert factory.sessions[0].closed is True assert sum("/index.php" in url for url in factory.sessions[0].urls) == 2 - assert sum("object=f" in url for url in factory.sessions[0].urls) == 2 + file_urls = [url for url in factory.sessions[0].urls if "object=f" in url] + assert len(file_urls) == 1 + assert "ids=1201%2C1202" in file_urls[0] + assert "md5=" not in file_urls[0] + edition_urls = [url for url in factory.sessions[0].urls if "object=e" in url] + assert len(edition_urls) == 1 + assert "ids=910" in edition_urls[0] assert all( factory.sessions[0].max_bytes_by_url[url] == 512 * 1024 for url in factory.sessions[0].urls @@ -286,6 +296,33 @@ async def test_service_search_discovers_enriches_deduplicates_and_caches() -> No ) +async def test_service_search_enriches_compact_file_row() -> None: + factory = _SessionFactory( + search_by_origin={ + "https://libgen.gl": _fixture("search-results-compact-v1.html"), + "https://libgen.li": _fixture("search-results-compact-v1.html"), + } + ) + service = LibGenProviderService( + session_factory=factory, + origin_resolver=_public_resolver, + ) + + candidates = await service.search( + _search_intent(), + provider_config={"source_url": "https://libgen.gl"}, + limit=10, + ) + + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate.provider_candidate_id == "libgen:0123456789abcdef0123456789abcdef" + assert candidate.display_title == "Clockwork Harbor: Signal Fires #3" + assert candidate.parsed.series_title == "Clockwork Harbor" + assert candidate.parsed.issue_numbers == ["3"] + assert candidate.parsed.format == "cbz" + + async def test_zero_results_are_not_misclassified_as_failover() -> None: factory = _SessionFactory( search_by_origin={"https://libgen.gl": _fixture("search-zero-v1.html")} From 9b2a7954e019e4951e796f9e563b1c76ca333c42 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Mon, 24 Aug 2026 15:53:59 -0700 Subject: [PATCH 2/8] fix: search GetComics by issue release year --- .../src/pullbox_provider_getcomics/service.py | 96 +++++++-- tests/unit/test_getcomics_service.py | 186 +++++++++++++++++- 2 files changed, 252 insertions(+), 30 deletions(-) diff --git a/providers/getcomics/src/pullbox_provider_getcomics/service.py b/providers/getcomics/src/pullbox_provider_getcomics/service.py index a18d85c..6428c6a 100644 --- a/providers/getcomics/src/pullbox_provider_getcomics/service.py +++ b/providers/getcomics/src/pullbox_provider_getcomics/service.py @@ -64,21 +64,25 @@ async def search( seen_candidate_ids=seen_candidate_ids, resolver_profile=resolver_profile, ) - if len(candidates) >= limit and ( - is_collection_intent(intent.issue_type) - or _has_requested_issue_coverage(candidates, intent) + if not is_collection_intent(intent.issue_type) and _has_requested_issue_coverage( + candidates, intent ): + return _prioritize_requested_issue_coverage(candidates, intent)[:limit] + if len(candidates) >= limit and is_collection_intent(intent.issue_type): return candidates[:limit] - fallback = _standard_issue_fallback_query(intent) - if fallback is not None and not _has_requested_issue_coverage(candidates, intent): + fallbacks = _standard_issue_fallback_queries(intent) + if fallbacks and not _has_requested_issue_coverage(candidates, intent): fallback_candidates: list[Candidate] = [] - await self._append_search_candidates( - fallback, - candidates=fallback_candidates, - seen_candidate_ids=seen_candidate_ids, - resolver_profile=resolver_profile, - ) + for fallback in fallbacks: + await self._append_search_candidates( + fallback, + candidates=fallback_candidates, + seen_candidate_ids=seen_candidate_ids, + resolver_profile=resolver_profile, + ) + if _has_requested_issue_coverage(fallback_candidates, intent): + break # Exact queries can return unrelated releases first. Prioritize only # fallback packs that explicitly cover the requested issue; broad # fallback noise must not displace a targeted exact-search result. @@ -174,9 +178,12 @@ def _build_query(intent: SearchIntent) -> str: def _build_queries(intent: SearchIntent) -> list[str]: + if not is_collection_intent(intent.issue_type): + return _standard_issue_exact_queries(intent) + title_fragment = collection_title_fragment(intent.issue_title) explicit_title_volume = collection_title_number(intent.issue_title) - if not is_collection_intent(intent.issue_type) or title_fragment is None: + if title_fragment is None: queries = [_build_query(intent)] else: queries = [f"{intent.series_title} {title_fragment}"[:700]] @@ -215,14 +222,44 @@ def _build_queries(intent: SearchIntent) -> list[str]: return queries[:5] -def _standard_issue_fallback_query(intent: SearchIntent) -> str | None: - """Return one bounded range-pack fallback after an exact issue search misses.""" +def _standard_issue_exact_queries(intent: SearchIntent) -> list[str]: + """Return release-aware exact queries without losing legacy year behavior.""" + if not intent.issue_number: + return [_build_query(intent)] + + base = f"{intent.series_title} {intent.issue_number}" + queries: list[str] = [] + preferred_year = intent.release_year + if preferred_year is None and intent.series_year is None: + preferred_year = intent.year + if preferred_year is not None: + queries.append(f"{base} {preferred_year}"[:700]) + queries.append(base[:700]) + + compatibility_year = intent.series_year or intent.year + if compatibility_year is not None: + queries.append(f"{base} {compatibility_year}"[:700]) + return list(dict.fromkeys(queries))[:3] + + +def _standard_issue_fallback_queries(intent: SearchIntent) -> list[str]: + """Return bounded release-year and series-year range-pack fallbacks.""" if is_collection_intent(intent.issue_type) or not intent.issue_number: - return None - parts = [intent.series_title] - if intent.year: - parts.append(str(intent.year)) - return " ".join(parts)[:700] + return [] + + years: list[int] = [] + preferred_year = intent.release_year + if preferred_year is None and intent.series_year is None: + preferred_year = intent.year + if preferred_year is not None: + years.append(preferred_year) + compatibility_year = intent.series_year or intent.year + if compatibility_year is not None: + years.append(compatibility_year) + unique_years = list(dict.fromkeys(years)) + if not unique_years: + return [intent.series_title[:700]] + return [f"{intent.series_title} {year}"[:700] for year in unique_years[:2]] def _has_requested_issue_coverage( @@ -235,6 +272,20 @@ def _has_requested_issue_coverage( ) +def _prioritize_requested_issue_coverage( + candidates: list[Candidate], + intent: SearchIntent, +) -> list[Candidate]: + """Keep a later exact query from being displaced by earlier search noise.""" + covering = [ + candidate for candidate in candidates if _candidate_covers_intent(candidate, intent) + ] + noncovering = [ + candidate for candidate in candidates if not _candidate_covers_intent(candidate, intent) + ] + return [*covering, *noncovering] + + def _candidate_covers_intent(candidate: Candidate, intent: SearchIntent) -> bool: """Return whether a candidate covers this issue for the requested series.""" if intent.issue_number is None: @@ -243,10 +294,13 @@ def _candidate_covers_intent(candidate: Candidate, intent: SearchIntent) -> bool return False if _normalized_series_title(candidate.parsed.series_title) not in _intent_series_titles(intent): return False + expected_years = { + year for year in (intent.release_year, intent.series_year, intent.year) if year is not None + } return not ( - intent.year is not None + expected_years and candidate.parsed.year is not None - and candidate.parsed.year != intent.year + and candidate.parsed.year not in expected_years ) diff --git a/tests/unit/test_getcomics_service.py b/tests/unit/test_getcomics_service.py index f6d8ca1..458459b 100644 --- a/tests/unit/test_getcomics_service.py +++ b/tests/unit/test_getcomics_service.py @@ -2,6 +2,7 @@ import asyncio from pathlib import Path +from urllib.parse import parse_qs, urlsplit import pytest from pullbox_provider_contract.errors import ProtocolError @@ -61,6 +62,78 @@ async def test_service_builds_bounded_search_and_resolves_stateless_candidate() assert pages.urls[1] == candidates[0].source_reference +async def test_standard_issue_search_prefers_release_year_over_series_year() -> None: + queries: list[str] = [] + + async def pages(url: str, **_kwargs: object) -> str: + query = parse_qs(urlsplit(url).query)["s"][0] + queries.append(query) + if query != "Absolute Batman 22 2026": + return '

Search Result

' + return """ + +

Search Result

+ + + """ + + candidates = await GetComicsProviderService(page_fetcher=pages).search( + SearchIntent( + series_title="Absolute Batman", + normalized_title="absolute batman", + issue_number="22", + issue_type="issue", + series_year=2024, + release_year=2026, + year=2024, + ), + limit=20, + ) + + assert [candidate.display_title for candidate in candidates] == ["Absolute Batman #22 (2026)"] + assert queries == ["Absolute Batman 22 2026"] + + +async def test_standard_issue_search_retries_without_year_before_series_year() -> None: + queries: list[str] = [] + + async def pages(url: str, **_kwargs: object) -> str: + query = parse_qs(urlsplit(url).query)["s"][0] + queries.append(query) + if query != "Absolute Batman 22": + return '

Search Result

' + return """ + +

Search Result

+ + + """ + + candidates = await GetComicsProviderService(page_fetcher=pages).search( + SearchIntent( + series_title="Absolute Batman", + normalized_title="absolute batman", + issue_number="22", + issue_type="issue", + series_year=2024, + release_year=2026, + year=2024, + ), + limit=20, + ) + + assert [candidate.display_title for candidate in candidates] == ["Absolute Batman #22 (2026)"] + assert queries == ["Absolute Batman 22 2026", "Absolute Batman 22"] + + async def test_service_retries_collection_search_without_synthetic_issue_number() -> None: urls: list[str] = [] @@ -211,7 +284,8 @@ async def test_service_falls_back_to_series_year_for_standard_issue_in_contiguou async def pages(url: str, **_kwargs: object) -> str: urls.append(url) - if "Example+Heroes+7+2026" in url: + query = parse_qs(urlsplit(url).query)["s"][0] + if query != "Example Heroes 2026": return '

Search Result

' return """ @@ -238,9 +312,97 @@ async def pages(url: str, **_kwargs: object) -> str: assert [candidate.parsed.issue_numbers for candidate in candidates] == [ ["5", "6", "7", "8", "9", "10"] ] - assert len(urls) == 2 + assert len(urls) == 3 assert "Example+Heroes+7+2026" in urls[0] - assert "Example+Heroes+2026" in urls[1] + assert "Example+Heroes+7" in urls[1] + assert "Example+Heroes+2026" in urls[2] + + +async def test_standard_issue_pack_fallback_prefers_release_year() -> None: + queries: list[str] = [] + + async def pages(url: str, **_kwargs: object) -> str: + query = parse_qs(urlsplit(url).query)["s"][0] + queries.append(query) + if query != "Example Heroes 2026": + return '

Search Result

' + return """ + +

Search Result

+ + + """ + + candidates = await GetComicsProviderService(page_fetcher=pages).search( + SearchIntent( + series_title="Example Heroes", + normalized_title="example heroes", + issue_number="7", + issue_type="issue", + series_year=2024, + release_year=2026, + year=2024, + ), + limit=20, + ) + + assert [candidate.parsed.issue_numbers for candidate in candidates] == [ + ["5", "6", "7", "8", "9", "10"] + ] + assert queries == [ + "Example Heroes 7 2026", + "Example Heroes 7", + "Example Heroes 7 2024", + "Example Heroes 2026", + ] + + +async def test_standard_issue_pack_fallback_retains_series_year_compatibility() -> None: + queries: list[str] = [] + + async def pages(url: str, **_kwargs: object) -> str: + query = parse_qs(urlsplit(url).query)["s"][0] + queries.append(query) + if query != "Example Heroes 2024": + return '

Search Result

' + return """ + +

Search Result

+ + + """ + + candidates = await GetComicsProviderService(page_fetcher=pages).search( + SearchIntent( + series_title="Example Heroes", + normalized_title="example heroes", + issue_number="7", + issue_type="issue", + series_year=2024, + release_year=2026, + year=2024, + ), + limit=20, + ) + + assert [candidate.parsed.issue_numbers for candidate in candidates] == [ + ["5", "6", "7", "8", "9", "10"] + ] + assert queries == [ + "Example Heroes 7 2026", + "Example Heroes 7", + "Example Heroes 7 2024", + "Example Heroes 2026", + "Example Heroes 2024", + ] async def test_standard_issue_fallback_prioritizes_covering_pack_over_noncovering_results() -> None: @@ -248,7 +410,8 @@ async def test_standard_issue_fallback_prioritizes_covering_pack_over_noncoverin async def pages(url: str, **_kwargs: object) -> str: urls.append(url) - if "Example+Heroes+7+2026" in url: + query = parse_qs(urlsplit(url).query)["s"][0] + if query == "Example Heroes 7 2026": return """

Search Result

@@ -258,7 +421,8 @@ async def pages(url: str, **_kwargs: object) -> str:

""" - return """ + if query == "Example Heroes 2026": + return """

Search Result

""" + return '

Search Result

' service = GetComicsProviderService(page_fetcher=pages) candidates = await service.search( @@ -288,7 +453,7 @@ async def pages(url: str, **_kwargs: object) -> str: assert [candidate.parsed.issue_numbers for candidate in candidates] == [ ["5", "6", "7", "8", "9", "10"] ] - assert len(urls) == 2 + assert len(urls) == 3 async def test_standard_issue_fallback_ignores_unrelated_series_with_same_issue_number() -> None: @@ -296,7 +461,8 @@ async def test_standard_issue_fallback_ignores_unrelated_series_with_same_issue_ async def pages(url: str, **_kwargs: object) -> str: urls.append(url) - if "Example+Heroes+7+2026" in url: + query = parse_qs(urlsplit(url).query)["s"][0] + if query == "Example Heroes 7 2026": return """

Search Result

@@ -306,7 +472,8 @@ async def pages(url: str, **_kwargs: object) -> str:

""" - return """ + if query == "Example Heroes 2026": + return """

Search Result

 """ + return '

Search Result

' service = GetComicsProviderService(page_fetcher=pages) candidates = await service.search( @@ -332,7 +500,7 @@ async def pages(url: str, **_kwargs: object) -> str: ["5", "6", "7", "8", "9", "10"], ["7"], ] - assert len(urls) == 2 + assert len(urls) == 3 async def test_service_rejects_forged_candidate_identifier() -> None: From 9900aa94f57f8de1cac86a283f253b5226c0a364 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Mon, 24 Aug 2026 18:37:35 -0700 Subject: [PATCH 3/8] fix: preserve LibGen coverage without edition metadata --- .../pullbox_provider_contract/comic_parser.py | 4 +- .../src/pullbox_provider_libgen/service.py | 44 ++++++++++++++++--- .../libgen/file-standard-no-edition-v1.json | 13 ++++++ tests/unit/test_comic_parser.py | 9 ++++ tests/unit/test_libgen_service.py | 18 ++++++++ 5 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/libgen/file-standard-no-edition-v1.json diff --git a/packages/provider_contract/src/pullbox_provider_contract/comic_parser.py b/packages/provider_contract/src/pullbox_provider_contract/comic_parser.py index 1641648..e04c7a2 100644 --- a/packages/provider_contract/src/pullbox_provider_contract/comic_parser.py +++ b/packages/provider_contract/src/pullbox_provider_contract/comic_parser.py @@ -18,7 +18,9 @@ rf"(?P\d+(?:\.\d+)?){_TRAILING_RELEASE_LABELS}\s*$" ) _HASH_ISSUE = re.compile(rf"\s+#\s*(?P\d+(?:\.\d+)?[A-Za-z]?){_TRAILING_RELEASE_LABELS}\s*$") -_TRAILING_ISSUE = re.compile(r"\s+(?P\d{1,5}(?:\.\d+)?[A-Za-z]?)\s*$") +_TRAILING_ISSUE = re.compile( + rf"\s+(?P\d{{1,5}}(?:\.\d+)?[A-Za-z]?){_TRAILING_RELEASE_LABELS}\s*$" +) _SCENE_NO_ISSUE = re.compile( r"^(?:(?P[a-z][a-z0-9]{1,15})-(?P[A-Z].+?)|(?P.+?))" r"[._\s]+(?i:No)\.?[._\s]*(?P\d{1,5}(?:\.\d+)?[A-Za-z]?)\s*$" diff --git a/providers/libgen/src/pullbox_provider_libgen/service.py b/providers/libgen/src/pullbox_provider_libgen/service.py index ea4bade..c107082 100644 --- a/providers/libgen/src/pullbox_provider_libgen/service.py +++ b/providers/libgen/src/pullbox_provider_libgen/service.py @@ -12,6 +12,7 @@ from urllib.parse import urlencode, urlsplit, urlunsplit import structlog +from pullbox_provider_contract.comic_parser import parse_comic_title from pullbox_provider_contract.models import ( Artifact, ArtifactCoverage, @@ -471,14 +472,10 @@ async def _resolve_origin( expected_file_id=file_metadata.file_id, ) destination = await session.resolve_redirect(f"{origin}/get.php?md5={md5}") - issue_numbers = [edition.issue_number] if edition and edition.issue_number else [] + coverage = _resolved_coverage(file_metadata, edition) return Artifact( artifact_id=f"libgen-direct:{md5}", - coverage=ArtifactCoverage( - issue_numbers=issue_numbers, - volume=edition.issue_volume if edition else None, - description=(edition.series_name or edition.title) if edition else None, - ), + coverage=coverage, route=ArtifactRoute.DIRECT_ARTIFACT, format=file_metadata.extension, edition=edition.edition_type if edition else None, @@ -495,6 +492,41 @@ async def _resolve_origin( ) +def _resolved_coverage( + file_metadata: FileMetadata, + edition: EditionMetadata | None, +) -> ArtifactCoverage: + """Build coverage from edition metadata or the MD5-bound locator filename.""" + locator = file_metadata.locator_filename + locator_evidence = parse_comic_title(locator) if locator else None + issue_numbers = ( + [edition.issue_number] + if edition is not None and edition.issue_number + else list(locator_evidence.issue_numbers) + if locator_evidence is not None + else [] + ) + volume = ( + edition.issue_volume + if edition is not None and edition.issue_volume + else locator_evidence.volume + if locator_evidence is not None + else None + ) + description = ( + edition.series_name or edition.title + if edition is not None + else locator_evidence.series_title + if locator_evidence is not None + else None + ) + return ArtifactCoverage( + issue_numbers=issue_numbers, + volume=volume, + description=description, + ) + + def _candidate_md5(provider_candidate_id: str) -> str: match = _CANDIDATE_ID.fullmatch(provider_candidate_id) if match is None: diff --git a/tests/fixtures/libgen/file-standard-no-edition-v1.json b/tests/fixtures/libgen/file-standard-no-edition-v1.json new file mode 100644 index 0000000..7ea50df --- /dev/null +++ b/tests/fixtures/libgen/file-standard-no-edition-v1.json @@ -0,0 +1,13 @@ +{ + "1301": { + "md5": "11111111111111111111111111111111", + "pages": "24", + "visible": "1", + "filesize": "21815394", + "extension": "cbz", + "locator": "c/absolute-batman/Absolute Batman 022 (2026) (Digital) (Shan-Empire).cbz", + "broken": "0", + "comics_id": "3046196", + "editions": {} + } +} diff --git a/tests/unit/test_comic_parser.py b/tests/unit/test_comic_parser.py index dc304ab..50c5333 100644 --- a/tests/unit/test_comic_parser.py +++ b/tests/unit/test_comic_parser.py @@ -27,3 +27,12 @@ def test_explicit_issue_before_release_group_keeps_issue_coverage() -> None: assert evidence.series_title == "War Wolf" assert evidence.issue_numbers == ("2",) + + +def test_unprefixed_issue_before_release_labels_keeps_issue_coverage() -> None: + evidence = parse_comic_title("Absolute Batman 022 (2026) (Digital) (Shan-Empire).cbz") + + assert evidence.series_title == "Absolute Batman" + assert evidence.issue_numbers == ("22",) + assert evidence.year == 2026 + assert evidence.format == "cbz" diff --git a/tests/unit/test_libgen_service.py b/tests/unit/test_libgen_service.py index a156dc8..c544757 100644 --- a/tests/unit/test_libgen_service.py +++ b/tests/unit/test_libgen_service.py @@ -187,6 +187,8 @@ async def fetch_text(self, url: str, *, max_bytes: int = 2 * 1024 * 1024) -> str first = json.loads(_fixture("file-v1.json")) second = json.loads(_fixture("file-sparse-v1.json")) return json.dumps(first | second) + if "11111111111111111111111111111111" in url: + return _fixture("file-standard-no-edition-v1.json") return ( _fixture("file-v1.json") if "0123456789abcdef0123456789abcdef" in url @@ -388,6 +390,22 @@ async def test_resolve_revalidates_by_md5_and_returns_generic_https_artifact() - assert factory.sessions[0].closed is True +async def test_resolve_uses_md5_bound_locator_coverage_without_an_edition() -> None: + factory = _SessionFactory() + service = LibGenProviderService(session_factory=factory, origin_resolver=_public_resolver) + + artifacts = await service.resolve( + "libgen:11111111111111111111111111111111", + provider_config={"source_url": "https://libgen.gl"}, + ) + + assert len(artifacts) == 1 + artifact = artifacts[0] + assert artifact.coverage.issue_numbers == ["22"] + assert artifact.coverage.volume is None + assert artifact.coverage.description == "Absolute Batman" + + @pytest.mark.parametrize( "candidate_id", [ From dd9bbab519ea7b83b9604854c9df7fa3e034068a Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Mon, 24 Aug 2026 18:55:21 -0700 Subject: [PATCH 4/8] fix: fall back to broad LibGen issue discovery --- .../src/pullbox_provider_libgen/service.py | 26 ++++++-- tests/unit/test_libgen_service.py | 59 +++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/providers/libgen/src/pullbox_provider_libgen/service.py b/providers/libgen/src/pullbox_provider_libgen/service.py index c107082..765cce1 100644 --- a/providers/libgen/src/pullbox_provider_libgen/service.py +++ b/providers/libgen/src/pullbox_provider_libgen/service.py @@ -191,6 +191,10 @@ def _build_queries(intent: SearchIntent) -> list[str]: return queries +def _title_fallback_query(intent: SearchIntent) -> str: + return " ".join(intent.series_title.split())[:500].rstrip() + + def _search_url(origin: str, query: str) -> str: params = urlencode( [ @@ -428,7 +432,8 @@ async def fetch_metadata(url: str) -> str: ) candidates: list[Candidate] = [] seen: set[str] = set() - for query in _build_queries(intent): + + async def collect_query(query: str) -> bool: html = await session.fetch_text(_search_url(origin, query)) discoveries = [] for discovered in parse_search_html(html, source_origin=origin): @@ -439,9 +444,22 @@ async def fetch_metadata(url: str) -> str: for candidate in await enricher.enrich_many(discoveries): candidates.append(candidate) if len(candidates) >= limit: - result = tuple(candidates[:limit]) - self._search_cache.set(cache_key, result) - return list(result) + return True + return False + + exact_queries = _build_queries(intent) + for query in exact_queries: + if await collect_query(query): + result = tuple(candidates[:limit]) + self._search_cache.set(cache_key, result) + return list(result) + + fallback_query = _title_fallback_query(intent) + if not candidates and fallback_query and fallback_query not in exact_queries: + if await collect_query(fallback_query): + result = tuple(candidates[:limit]) + self._search_cache.set(cache_key, result) + return list(result) result = tuple(candidates[:limit]) self._search_cache.set(cache_key, result or None) return list(result) diff --git a/tests/unit/test_libgen_service.py b/tests/unit/test_libgen_service.py index c544757..4432d08 100644 --- a/tests/unit/test_libgen_service.py +++ b/tests/unit/test_libgen_service.py @@ -164,11 +164,13 @@ def __init__( origin: str, *, search_html: str | None = None, + search_by_query: dict[str, str] | None = None, fail: BaseException | None = None, redirect_fail: BaseException | None = None, ) -> None: self.origin = origin self.search_html = search_html or _fixture("search-results-v1.html") + self.search_by_query = search_by_query or {} self.fail = fail self.redirect_fail = redirect_fail self.urls: list[str] = [] @@ -181,6 +183,9 @@ async def fetch_text(self, url: str, *, max_bytes: int = 2 * 1024 * 1024) -> str if self.fail is not None: raise self.fail if "/index.php" in url: + query = parse_qs(urlsplit(url).query).get("req", [""])[0] + if query in self.search_by_query: + return self.search_by_query[query] return self.search_html if "object=f" in url: if "ids=1201%2C1202" in url: @@ -215,10 +220,12 @@ def __init__( self, *, search_by_origin: dict[str, str] | None = None, + search_by_query: dict[str, str] | None = None, failure_by_origin: dict[str, BaseException] | None = None, redirect_failure_by_origin: dict[str, BaseException] | None = None, ) -> None: self.search_by_origin = search_by_origin or {} + self.search_by_query = search_by_query or {} self.failure_by_origin = failure_by_origin or {} self.redirect_failure_by_origin = redirect_failure_by_origin or {} self.sessions: list[_SourceSession] = [] @@ -232,6 +239,7 @@ def __call__( session = _SourceSession( origin, search_html=self.search_by_origin.get(origin), + search_by_query=self.search_by_query, fail=self.failure_by_origin.get(origin), redirect_fail=self.redirect_failure_by_origin.get(origin), ) @@ -342,6 +350,57 @@ async def test_zero_results_are_not_misclassified_as_failover() -> None: assert [session.origin for session in factory.sessions] == ["https://libgen.gl"] +async def test_search_uses_title_only_fallback_after_exact_queries_return_no_candidates() -> None: + factory = _SessionFactory( + search_by_query={ + "Clockwork Harbor 3 2024": _fixture("search-zero-v1.html"), + "Clockwork Harbor 3": _fixture("search-zero-v1.html"), + "Clockwork Harbor": _fixture("search-results-v1.html"), + } + ) + service = LibGenProviderService(session_factory=factory, origin_resolver=_public_resolver) + + candidates = await service.search( + _search_intent(), + provider_config={"source_url": "https://libgen.gl"}, + limit=10, + ) + + assert [candidate.provider_candidate_id for candidate in candidates] == [ + "libgen:0123456789abcdef0123456789abcdef", + "libgen:fedcba9876543210fedcba9876543210", + ] + search_queries = [ + parse_qs(urlsplit(url).query)["req"][0] + for url in factory.sessions[0].urls + if "/index.php" in url + ] + assert search_queries == [ + "Clockwork Harbor 3 2024", + "Clockwork Harbor 3", + "Clockwork Harbor", + ] + + +async def test_search_skips_title_only_fallback_when_exact_queries_find_candidates() -> None: + factory = _SessionFactory() + service = LibGenProviderService(session_factory=factory, origin_resolver=_public_resolver) + + candidates = await service.search( + _search_intent(), + provider_config={"source_url": "https://libgen.gl"}, + limit=10, + ) + + assert len(candidates) == 2 + search_queries = [ + parse_qs(urlsplit(url).query)["req"][0] + for url in factory.sessions[0].urls + if "/index.php" in url + ] + assert "Clockwork Harbor" not in search_queries + + async def test_search_fails_over_once_for_temporary_source_failure() -> None: factory = _SessionFactory( search_by_origin={"https://libgen.li": _fixture("search-zero-v1.html")}, From 265d1c84225d61d4c6b725fb86d7d57f91594b26 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Mon, 24 Aug 2026 19:09:33 -0700 Subject: [PATCH 5/8] fix: prioritize canonical LibGen issue queries --- .../src/pullbox_provider_libgen/service.py | 15 +++++++++-- tests/unit/test_libgen_service.py | 27 ++++++++++++++++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/providers/libgen/src/pullbox_provider_libgen/service.py b/providers/libgen/src/pullbox_provider_libgen/service.py index 765cce1..d7dcd28 100644 --- a/providers/libgen/src/pullbox_provider_libgen/service.py +++ b/providers/libgen/src/pullbox_provider_libgen/service.py @@ -171,6 +171,9 @@ def _build_queries(intent: SearchIntent) -> list[str]: elif intent.alternate_titles: variants.append((intent.alternate_titles[0], "Vol", issue_or_volume, year)) else: + padded_issue = _zero_padded_issue_number(intent.issue_number) + if padded_issue and padded_issue != intent.issue_number: + variants.append((intent.series_title, padded_issue, year)) variants.extend( ( (intent.series_title, intent.issue_number, year), @@ -191,6 +194,12 @@ def _build_queries(intent: SearchIntent) -> list[str]: return queries +def _zero_padded_issue_number(issue_number: str | None) -> str | None: + if not issue_number or not issue_number.isascii() or not issue_number.isdigit(): + return None + return issue_number.zfill(3) + + def _title_fallback_query(intent: SearchIntent) -> str: return " ".join(intent.series_title.split())[:500].rstrip() @@ -449,14 +458,16 @@ async def collect_query(query: str) -> bool: exact_queries = _build_queries(intent) for query in exact_queries: - if await collect_query(query): + await collect_query(query) + if candidates: result = tuple(candidates[:limit]) self._search_cache.set(cache_key, result) return list(result) fallback_query = _title_fallback_query(intent) if not candidates and fallback_query and fallback_query not in exact_queries: - if await collect_query(fallback_query): + await collect_query(fallback_query) + if candidates: result = tuple(candidates[:limit]) self._search_cache.set(cache_key, result) return list(result) diff --git a/tests/unit/test_libgen_service.py b/tests/unit/test_libgen_service.py index 4432d08..157fec0 100644 --- a/tests/unit/test_libgen_service.py +++ b/tests/unit/test_libgen_service.py @@ -90,7 +90,7 @@ async def test_validate_source_origin_rejects_unresolved_or_non_public_dns(resol await validate_source_origin("https://libgen.example", resolver=resolver) -def test_build_queries_is_bounded_deterministic_and_uses_one_alternate() -> None: +def test_build_queries_is_bounded_deterministic_and_zero_pads_whole_issues() -> None: intent = SearchIntent( series_title="Clockwork Harbor", normalized_title="clockwork harbor", @@ -100,9 +100,23 @@ def test_build_queries_is_bounded_deterministic_and_uses_one_alternate() -> None ) assert _build_queries(intent) == [ + "Clockwork Harbor 003 2024", "Clockwork Harbor 3 2024", "Clockwork Harbor 3", - "The Clockwork Harbor 3 2024", + ] + + +def test_build_queries_does_not_pad_non_integer_issue_numbers() -> None: + intent = SearchIntent( + series_title="Clockwork Harbor", + normalized_title="clockwork harbor", + issue_number="3.1", + year=2024, + ) + + assert _build_queries(intent) == [ + "Clockwork Harbor 3.1 2024", + "Clockwork Harbor 3.1", ] @@ -291,7 +305,12 @@ async def test_service_search_discovers_enriches_deduplicates_and_caches() -> No assert len(factory.sessions) == 1 assert factory.profiles == [profile] assert factory.sessions[0].closed is True - assert sum("/index.php" in url for url in factory.sessions[0].urls) == 2 + search_queries = [ + parse_qs(urlsplit(url).query)["req"][0] + for url in factory.sessions[0].urls + if "/index.php" in url + ] + assert search_queries == ["Clockwork Harbor 003 2024"] file_urls = [url for url in factory.sessions[0].urls if "object=f" in url] assert len(file_urls) == 1 assert "ids=1201%2C1202" in file_urls[0] @@ -353,6 +372,7 @@ async def test_zero_results_are_not_misclassified_as_failover() -> None: async def test_search_uses_title_only_fallback_after_exact_queries_return_no_candidates() -> None: factory = _SessionFactory( search_by_query={ + "Clockwork Harbor 003 2024": _fixture("search-zero-v1.html"), "Clockwork Harbor 3 2024": _fixture("search-zero-v1.html"), "Clockwork Harbor 3": _fixture("search-zero-v1.html"), "Clockwork Harbor": _fixture("search-results-v1.html"), @@ -376,6 +396,7 @@ async def test_search_uses_title_only_fallback_after_exact_queries_return_no_can if "/index.php" in url ] assert search_queries == [ + "Clockwork Harbor 003 2024", "Clockwork Harbor 3 2024", "Clockwork Harbor 3", "Clockwork Harbor", From 697ee8378c6af528e7732ce4caaf3b07d92d80a8 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Mon, 24 Aug 2026 19:14:02 -0700 Subject: [PATCH 6/8] perf: defer LibGen edition enrichment until resolve --- .../src/pullbox_provider_libgen/metadata.py | 23 +++++++++++++++---- .../src/pullbox_provider_libgen/service.py | 5 +++- tests/unit/test_libgen_service.py | 7 +++--- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/providers/libgen/src/pullbox_provider_libgen/metadata.py b/providers/libgen/src/pullbox_provider_libgen/metadata.py index 31321b9..f06da11 100644 --- a/providers/libgen/src/pullbox_provider_libgen/metadata.py +++ b/providers/libgen/src/pullbox_provider_libgen/metadata.py @@ -75,7 +75,12 @@ def __init__( negative_ttl_seconds=2 * 60, ) - async def enrich(self, discovered: DiscoveredRecord) -> Candidate | None: + async def enrich( + self, + discovered: DiscoveredRecord, + *, + include_edition: bool = True, + ) -> Candidate | None: origin = _source_origin(discovered.source_reference) file_key = _file_cache_key(origin, discovered) file_lookup = self._cache.get(file_key) @@ -93,7 +98,7 @@ async def enrich(self, discovered: DiscoveredRecord) -> Candidate | None: self._cache.set(file_key, file_metadata) edition_metadata: EditionMetadata | None = None - if file_metadata.edition_id is not None: + if include_edition and file_metadata.edition_id is not None: edition_key = _edition_cache_key(origin, file_metadata) edition_lookup = self._cache.get(edition_key) if edition_lookup.hit: @@ -118,12 +123,17 @@ async def enrich(self, discovered: DiscoveredRecord) -> Candidate | None: async def enrich_many( self, discovered_records: Sequence[DiscoveredRecord], + *, + include_editions: bool = True, ) -> list[Candidate]: """Enrich bounded discovery rows with batched keyed API requests.""" if not discovered_records: return [] if len(discovered_records) == 1: - candidate = await self.enrich(discovered_records[0]) + candidate = await self.enrich( + discovered_records[0], + include_edition=include_editions, + ) return [candidate] if candidate is not None else [] origins = {_source_origin(record.source_reference) for record in discovered_records} @@ -167,6 +177,8 @@ async def enrich_many( edition_metadata_by_file_id: dict[int, EditionMetadata | None] = {} pending_editions: list[FileMetadata] = [] for cached_file in file_metadata_by_md5.values(): + if not include_editions: + continue if cached_file is None or cached_file.edition_id is None: continue edition_key = _edition_cache_key(origin, cached_file) @@ -224,7 +236,10 @@ async def enrich_many( ) for discovered in individual: - candidate = await self.enrich(discovered) + candidate = await self.enrich( + discovered, + include_edition=include_editions, + ) if candidate is not None: candidates_by_md5[discovered.md5] = candidate diff --git a/providers/libgen/src/pullbox_provider_libgen/service.py b/providers/libgen/src/pullbox_provider_libgen/service.py index d7dcd28..eecb85a 100644 --- a/providers/libgen/src/pullbox_provider_libgen/service.py +++ b/providers/libgen/src/pullbox_provider_libgen/service.py @@ -450,7 +450,10 @@ async def collect_query(query: str) -> bool: continue seen.add(discovered.md5) discoveries.append(discovered) - for candidate in await enricher.enrich_many(discoveries): + for candidate in await enricher.enrich_many( + discoveries, + include_editions=False, + ): candidates.append(candidate) if len(candidates) >= limit: return True diff --git a/tests/unit/test_libgen_service.py b/tests/unit/test_libgen_service.py index 157fec0..c348b1d 100644 --- a/tests/unit/test_libgen_service.py +++ b/tests/unit/test_libgen_service.py @@ -316,8 +316,8 @@ async def test_service_search_discovers_enriches_deduplicates_and_caches() -> No assert "ids=1201%2C1202" in file_urls[0] assert "md5=" not in file_urls[0] edition_urls = [url for url in factory.sessions[0].urls if "object=e" in url] - assert len(edition_urls) == 1 - assert "ids=910" in edition_urls[0] + assert edition_urls == [] + assert all(candidate.provider_confidence == 0.75 for candidate in first) assert all( factory.sessions[0].max_bytes_by_url[url] == 512 * 1024 for url in factory.sessions[0].urls @@ -346,7 +346,8 @@ async def test_service_search_enriches_compact_file_row() -> None: assert len(candidates) == 1 candidate = candidates[0] assert candidate.provider_candidate_id == "libgen:0123456789abcdef0123456789abcdef" - assert candidate.display_title == "Clockwork Harbor: Signal Fires #3" + assert candidate.display_title == "Clockwork Harbor 003 (2024)" + assert candidate.parsed.issue_numbers == ["3"] assert candidate.parsed.series_title == "Clockwork Harbor" assert candidate.parsed.issue_numbers == ["3"] assert candidate.parsed.format == "cbz" From a30196e79d0310037075cf1c8cfd47208340589c Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Mon, 24 Aug 2026 19:19:57 -0700 Subject: [PATCH 7/8] perf: validate LibGen metadata during resolve --- .../src/pullbox_provider_libgen/metadata.py | 52 ++++++++++++------- .../src/pullbox_provider_libgen/service.py | 25 ++------- tests/unit/test_libgen_service.py | 7 ++- 3 files changed, 39 insertions(+), 45 deletions(-) diff --git a/providers/libgen/src/pullbox_provider_libgen/metadata.py b/providers/libgen/src/pullbox_provider_libgen/metadata.py index f06da11..d5e64ba 100644 --- a/providers/libgen/src/pullbox_provider_libgen/metadata.py +++ b/providers/libgen/src/pullbox_provider_libgen/metadata.py @@ -75,12 +75,7 @@ def __init__( negative_ttl_seconds=2 * 60, ) - async def enrich( - self, - discovered: DiscoveredRecord, - *, - include_edition: bool = True, - ) -> Candidate | None: + async def enrich(self, discovered: DiscoveredRecord) -> Candidate | None: origin = _source_origin(discovered.source_reference) file_key = _file_cache_key(origin, discovered) file_lookup = self._cache.get(file_key) @@ -98,7 +93,7 @@ async def enrich( self._cache.set(file_key, file_metadata) edition_metadata: EditionMetadata | None = None - if include_edition and file_metadata.edition_id is not None: + if file_metadata.edition_id is not None: edition_key = _edition_cache_key(origin, file_metadata) edition_lookup = self._cache.get(edition_key) if edition_lookup.hit: @@ -123,17 +118,12 @@ async def enrich( async def enrich_many( self, discovered_records: Sequence[DiscoveredRecord], - *, - include_editions: bool = True, ) -> list[Candidate]: """Enrich bounded discovery rows with batched keyed API requests.""" if not discovered_records: return [] if len(discovered_records) == 1: - candidate = await self.enrich( - discovered_records[0], - include_edition=include_editions, - ) + candidate = await self.enrich(discovered_records[0]) return [candidate] if candidate is not None else [] origins = {_source_origin(record.source_reference) for record in discovered_records} @@ -177,8 +167,6 @@ async def enrich_many( edition_metadata_by_file_id: dict[int, EditionMetadata | None] = {} pending_editions: list[FileMetadata] = [] for cached_file in file_metadata_by_md5.values(): - if not include_editions: - continue if cached_file is None or cached_file.edition_id is None: continue edition_key = _edition_cache_key(origin, cached_file) @@ -236,10 +224,7 @@ async def enrich_many( ) for discovered in individual: - candidate = await self.enrich( - discovered, - include_edition=include_editions, - ) + candidate = await self.enrich(discovered) if candidate is not None: candidates_by_md5[discovered.md5] = candidate @@ -441,6 +426,35 @@ def build_candidate( ) +def build_discovered_candidate(discovered: DiscoveredRecord) -> Candidate: + """Build a search candidate; resolve performs authoritative metadata checks.""" + evidence = parse_comic_title(discovered.display_title) + return Candidate( + provider_candidate_id=f"libgen:{discovered.md5}", + content_fingerprint=f"md5:{discovered.md5}", + source_reference=discovered.source_reference, + display_title=discovered.display_title, + raw_title=discovered.raw_title, + parsed=ParsedCandidate( + series_title=evidence.series_title, + issue_numbers=list(evidence.issue_numbers), + volume=evidence.volume, + year=discovered.year or evidence.year, + publisher=discovered.publisher, + language=_language_code(discovered.language), + format=discovered.extension, + ), + provider_confidence=0.65, + provenance={ + "layout": "libgen-search-v1", + "source_kind": "catalog_discovery", + "file_id": discovered.file_id, + "edition_id": discovered.edition_id, + "comics_id": discovered.source_series_id, + }, + ) + + def _candidate_display_title( discovered: DiscoveredRecord, edition: EditionMetadata | None, diff --git a/providers/libgen/src/pullbox_provider_libgen/service.py b/providers/libgen/src/pullbox_provider_libgen/service.py index eecb85a..3090179 100644 --- a/providers/libgen/src/pullbox_provider_libgen/service.py +++ b/providers/libgen/src/pullbox_provider_libgen/service.py @@ -31,10 +31,10 @@ from pullbox_provider_libgen.metadata import ( EditionMetadata, FileMetadata, - LibGenMetadataEnricher, LibGenMetadataError, _edition_metadata_url, _file_metadata_url, + build_discovered_candidate, parse_edition_metadata, parse_file_metadata_by_md5, ) @@ -235,14 +235,6 @@ def __init__( ttl_seconds=10 * 60, negative_ttl_seconds=2 * 60, ) - self._metadata_cache: BoundedTTLCache[ - str, - FileMetadata | EditionMetadata, - ] = BoundedTTLCache( - max_entries=2_048, - ttl_seconds=60 * 60, - negative_ttl_seconds=2 * 60, - ) async def source_health(self) -> dict[str, ProviderStatus]: health: dict[str, ProviderStatus] = {} @@ -431,14 +423,6 @@ async def _search_origin( session = self._session_factory(origin, resolver_profile) try: - - async def fetch_metadata(url: str) -> str: - return await session.fetch_text(url, max_bytes=_MAX_METADATA_BYTES) - - enricher = LibGenMetadataEnricher( - fetcher=fetch_metadata, - cache=self._metadata_cache, - ) candidates: list[Candidate] = [] seen: set[str] = set() @@ -450,11 +434,8 @@ async def collect_query(query: str) -> bool: continue seen.add(discovered.md5) discoveries.append(discovered) - for candidate in await enricher.enrich_many( - discoveries, - include_editions=False, - ): - candidates.append(candidate) + for discovered in discoveries: + candidates.append(build_discovered_candidate(discovered)) if len(candidates) >= limit: return True return False diff --git a/tests/unit/test_libgen_service.py b/tests/unit/test_libgen_service.py index c348b1d..a59ad33 100644 --- a/tests/unit/test_libgen_service.py +++ b/tests/unit/test_libgen_service.py @@ -312,12 +312,11 @@ async def test_service_search_discovers_enriches_deduplicates_and_caches() -> No ] assert search_queries == ["Clockwork Harbor 003 2024"] file_urls = [url for url in factory.sessions[0].urls if "object=f" in url] - assert len(file_urls) == 1 - assert "ids=1201%2C1202" in file_urls[0] - assert "md5=" not in file_urls[0] + assert file_urls == [] edition_urls = [url for url in factory.sessions[0].urls if "object=e" in url] assert edition_urls == [] - assert all(candidate.provider_confidence == 0.75 for candidate in first) + assert all(candidate.provider_confidence == 0.65 for candidate in first) + assert all(candidate.provenance["source_kind"] == "catalog_discovery" for candidate in first) assert all( factory.sessions[0].max_bytes_by_url[url] == 512 * 1024 for url in factory.sessions[0].urls From b57e04a826f6dd3e39d30242fb10d2d2265888c0 Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Wed, 26 Aug 2026 17:38:54 -0700 Subject: [PATCH 8/8] fix(security): update provider runtime base --- .../container-vulnerability-baseline.json | 42 ++----------------- docker/Dockerfile.annas-archive | 15 +++++-- docker/Dockerfile.getcomics | 15 +++++-- docker/Dockerfile.libgen | 15 +++++-- docker/Dockerfile.provider-smoke | 11 ++++- docker/Dockerfile.synthetic | 17 ++++++-- docker/compose.providers-test.yml | 8 ++-- docker/compose.synthetic-test.yml | 4 +- tests/unit/test_container_contract.py | 19 ++++++++- tests/unit/test_vulnerability_baseline.py | 15 +++++-- 10 files changed, 95 insertions(+), 66 deletions(-) diff --git a/.github/security/container-vulnerability-baseline.json b/.github/security/container-vulnerability-baseline.json index ea1a45b..935f1c1 100644 --- a/.github/security/container-vulnerability-baseline.json +++ b/.github/security/container-vulnerability-baseline.json @@ -1,8 +1,8 @@ { "schema_version": 1, - "reviewed_on": "2026-08-23", - "expires_on": "2026-11-01", - "base_image": "python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6", + "reviewed_on": "2026-08-26", + "expires_on": "2026-11-26", + "base_image": "python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83", "images": { "synthetic": "python-3.14-slim", "getcomics": "python-3.14-slim", @@ -89,24 +89,6 @@ "severity": "High", "rationale": "Inherited from the pinned Python 3.14 Debian base; no supported-line fix is currently available." }, - { - "id": "CVE-2026-11940", - "package": "python", - "severity": "High", - "rationale": "Inherited from Python 3.14; the available fix is outside the supported Python line." - }, - { - "id": "CVE-2026-11972", - "package": "python", - "severity": "High", - "rationale": "Inherited from Python 3.14; the available fix is outside the supported Python line." - }, - { - "id": "CVE-2026-15308", - "package": "python", - "severity": "High", - "rationale": "Inherited from Python 3.14; the available fix is outside the supported Python line." - }, { "id": "CVE-2026-41992", "package": "gzip", @@ -190,24 +172,6 @@ "package": "perl-base", "severity": "High", "rationale": "Inherited from the pinned Python 3.14 Debian base; no supported-line fix is currently available." - }, - { - "id": "CVE-2026-14456", - "package": "libssl3t64", - "severity": "High", - "rationale": "Inherited from the pinned Python 3.14 Debian base. The affected QUIC listener code is not enabled or used by these HTTP-only provider images, and no supported-line upstream fix is currently available." - }, - { - "id": "CVE-2026-14456", - "package": "openssl", - "severity": "High", - "rationale": "Inherited from the pinned Python 3.14 Debian base. The affected QUIC listener code is not enabled or used by these HTTP-only provider images, and no supported-line upstream fix is currently available." - }, - { - "id": "CVE-2026-14456", - "package": "openssl-provider-legacy", - "severity": "High", - "rationale": "Inherited from the pinned Python 3.14 Debian base. The affected QUIC listener code is not enabled or used by these HTTP-only provider images, and no supported-line upstream fix is currently available." } ] } diff --git a/docker/Dockerfile.annas-archive b/docker/Dockerfile.annas-archive index d2ac781..26474bc 100644 --- a/docker/Dockerfile.annas-archive +++ b/docker/Dockerfile.annas-archive @@ -1,6 +1,15 @@ -ARG PYTHON_BASE=python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 +ARG PYTHON_BASE=python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 -FROM ${PYTHON_BASE} AS build +FROM ${PYTHON_BASE} AS security-patched + +RUN apt-get update && \ + apt-get install --yes --no-install-recommends \ + libssl3t64=3.5.7-1~deb13u2 \ + openssl=3.5.7-1~deb13u2 \ + openssl-provider-legacy=3.5.7-1~deb13u2 && \ + rm -rf /var/lib/apt/lists/* + +FROM security-patched AS build ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 @@ -13,7 +22,7 @@ COPY providers/annas_archive providers/annas_archive RUN python -m pip wheel --wheel-dir /wheels ./packages/provider_contract && \ python -m pip wheel --find-links /wheels --wheel-dir /wheels ./providers/annas_archive -FROM ${PYTHON_BASE} AS runtime +FROM security-patched AS runtime ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 \ diff --git a/docker/Dockerfile.getcomics b/docker/Dockerfile.getcomics index 5b9092d..f0161f9 100644 --- a/docker/Dockerfile.getcomics +++ b/docker/Dockerfile.getcomics @@ -1,6 +1,15 @@ -ARG PYTHON_BASE=python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 +ARG PYTHON_BASE=python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 -FROM ${PYTHON_BASE} AS build +FROM ${PYTHON_BASE} AS security-patched + +RUN apt-get update && \ + apt-get install --yes --no-install-recommends \ + libssl3t64=3.5.7-1~deb13u2 \ + openssl=3.5.7-1~deb13u2 \ + openssl-provider-legacy=3.5.7-1~deb13u2 && \ + rm -rf /var/lib/apt/lists/* + +FROM security-patched AS build ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 @@ -13,7 +22,7 @@ COPY providers/getcomics providers/getcomics RUN python -m pip wheel --wheel-dir /wheels ./packages/provider_contract && \ python -m pip wheel --find-links /wheels --wheel-dir /wheels ./providers/getcomics -FROM ${PYTHON_BASE} AS runtime +FROM security-patched AS runtime ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 \ diff --git a/docker/Dockerfile.libgen b/docker/Dockerfile.libgen index e8f98d5..2b9c86c 100644 --- a/docker/Dockerfile.libgen +++ b/docker/Dockerfile.libgen @@ -1,6 +1,15 @@ -ARG PYTHON_BASE=python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 +ARG PYTHON_BASE=python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 -FROM ${PYTHON_BASE} AS build +FROM ${PYTHON_BASE} AS security-patched + +RUN apt-get update && \ + apt-get install --yes --no-install-recommends \ + libssl3t64=3.5.7-1~deb13u2 \ + openssl=3.5.7-1~deb13u2 \ + openssl-provider-legacy=3.5.7-1~deb13u2 && \ + rm -rf /var/lib/apt/lists/* + +FROM security-patched AS build ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 @@ -13,7 +22,7 @@ COPY providers/libgen providers/libgen RUN python -m pip wheel --wheel-dir /wheels ./packages/provider_contract && \ python -m pip wheel --find-links /wheels --wheel-dir /wheels ./providers/libgen -FROM ${PYTHON_BASE} AS runtime +FROM security-patched AS runtime ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 \ diff --git a/docker/Dockerfile.provider-smoke b/docker/Dockerfile.provider-smoke index bfa9283..deb37e1 100644 --- a/docker/Dockerfile.provider-smoke +++ b/docker/Dockerfile.provider-smoke @@ -1,5 +1,12 @@ -ARG PYTHON_BASE=python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 -FROM ${PYTHON_BASE} +ARG PYTHON_BASE=python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 +FROM ${PYTHON_BASE} AS runtime + +RUN apt-get update && \ + apt-get install --yes --no-install-recommends \ + libssl3t64=3.5.7-1~deb13u2 \ + openssl=3.5.7-1~deb13u2 \ + openssl-provider-legacy=3.5.7-1~deb13u2 && \ + rm -rf /var/lib/apt/lists/* ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 diff --git a/docker/Dockerfile.synthetic b/docker/Dockerfile.synthetic index 95464a2..3916f00 100644 --- a/docker/Dockerfile.synthetic +++ b/docker/Dockerfile.synthetic @@ -1,6 +1,15 @@ -ARG PYTHON_BASE=python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 +ARG PYTHON_BASE=python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 -FROM ${PYTHON_BASE} AS build +FROM ${PYTHON_BASE} AS security-patched + +RUN apt-get update && \ + apt-get install --yes --no-install-recommends \ + libssl3t64=3.5.7-1~deb13u2 \ + openssl=3.5.7-1~deb13u2 \ + openssl-provider-legacy=3.5.7-1~deb13u2 && \ + rm -rf /var/lib/apt/lists/* + +FROM security-patched AS build ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 @@ -13,7 +22,7 @@ COPY providers/synthetic providers/synthetic RUN python -m pip wheel --wheel-dir /wheels . -FROM ${PYTHON_BASE} AS runtime +FROM security-patched AS runtime ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 \ @@ -32,7 +41,7 @@ HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ ENTRYPOINT ["python", "-m", "uvicorn"] CMD ["pullbox_provider_synthetic.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8780", "--no-access-log"] -FROM ${PYTHON_BASE} AS conformance +FROM security-patched AS conformance ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ PIP_NO_CACHE_DIR=1 \ diff --git a/docker/compose.providers-test.yml b/docker/compose.providers-test.yml index 549b544..b82309b 100644 --- a/docker/compose.providers-test.yml +++ b/docker/compose.providers-test.yml @@ -4,7 +4,7 @@ services: context: .. dockerfile: docker/Dockerfile.getcomics args: - PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 + PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 environment: PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-test-provider-token-with-sufficient-entropy} read_only: true @@ -23,7 +23,7 @@ services: context: .. dockerfile: docker/Dockerfile.annas-archive args: - PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 + PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 environment: PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-test-provider-token-with-sufficient-entropy} read_only: true @@ -42,7 +42,7 @@ services: context: .. dockerfile: docker/Dockerfile.libgen args: - PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 + PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 environment: PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-test-provider-token-with-sufficient-entropy} read_only: true @@ -61,7 +61,7 @@ services: context: .. dockerfile: docker/Dockerfile.provider-smoke args: - PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6 + PYTHON_BASE: public.ecr.aws/docker/library/python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83 depends_on: getcomics: condition: service_healthy diff --git a/docker/compose.synthetic-test.yml b/docker/compose.synthetic-test.yml index bd2cbc2..4195c72 100644 --- a/docker/compose.synthetic-test.yml +++ b/docker/compose.synthetic-test.yml @@ -5,7 +5,7 @@ services: dockerfile: docker/Dockerfile.synthetic target: runtime args: - PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6} + PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83} image: pullbox-provider-synthetic:dd0-test environment: PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-synthetic-conformance-token-00000000} @@ -26,7 +26,7 @@ services: dockerfile: docker/Dockerfile.synthetic target: conformance args: - PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6} + PYTHON_BASE: ${PYTHON_BASE:-mirror.gcr.io/library/python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83} environment: PULLBOX_PROVIDER_BASE_URL: http://synthetic:8780 PULLBOX_PROVIDER_TOKEN: ${PULLBOX_PROVIDER_TOKEN:-synthetic-conformance-token-00000000} diff --git a/tests/unit/test_container_contract.py b/tests/unit/test_container_contract.py index 39ff1e7..c9bfbaf 100644 --- a/tests/unit/test_container_contract.py +++ b/tests/unit/test_container_contract.py @@ -17,13 +17,18 @@ } SOURCE_COMPOSE_FILE = ROOT / "docker" / "compose.providers-test.yml" SOURCE_SMOKE_SCRIPT = ROOT / "docker" / "provider_smoke.py" +ALL_RUNTIME_DOCKERFILES = { + DOCKERFILE, + ROOT / "docker" / "Dockerfile.provider-smoke", + *(path for path, _source_path in SOURCE_DOCKERFILES.values()), +} def test_runtime_uses_pinned_python_314_and_non_root_identity() -> None: dockerfile = DOCKERFILE.read_text(encoding="utf-8") assert ( - "python:3.14-slim@sha256:cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6" + "python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83" in dockerfile ) assert "AS build" in dockerfile @@ -63,7 +68,7 @@ def test_source_provider_images_are_independent_hardened_python_314_services() - assert ( "python:3.14-slim@sha256:" - "cea0e6040540fb2b965b6e7fb5ffa00871e632eef63719f0ea54bca189ce14a6" in dockerfile + "83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83" in dockerfile ) assert "USER 65532:65532" in dockerfile assert "EXPOSE 8780" in dockerfile @@ -75,6 +80,16 @@ def test_source_provider_images_are_independent_hardened_python_314_services() - assert all(source not in dockerfile for source in other_source_paths) +def test_runtime_images_install_pinned_openssl_security_update() -> None: + for path in ALL_RUNTIME_DOCKERFILES: + dockerfile = path.read_text(encoding="utf-8") + + assert "libssl3t64=3.5.7-1~deb13u2" in dockerfile + assert "openssl=3.5.7-1~deb13u2" in dockerfile + assert "openssl-provider-legacy=3.5.7-1~deb13u2" in dockerfile + assert "rm -rf /var/lib/apt/lists/*" in dockerfile + + def test_source_provider_healthchecks_test_process_liveness_without_upstream_work() -> None: for path, _source_path in SOURCE_DOCKERFILES.values(): dockerfile = path.read_text(encoding="utf-8") diff --git a/tests/unit/test_vulnerability_baseline.py b/tests/unit/test_vulnerability_baseline.py index 71430c9..a5be8e3 100644 --- a/tests/unit/test_vulnerability_baseline.py +++ b/tests/unit/test_vulnerability_baseline.py @@ -187,7 +187,10 @@ def test_repository_baseline_is_current_and_documents_every_exception() -> None: baseline = json.loads(REPOSITORY_BASELINE.read_text(encoding="utf-8")) assert baseline["schema_version"] == 1 - assert baseline["reviewed_on"] == "2026-08-23" + assert baseline["reviewed_on"] == "2026-08-26" + assert baseline["base_image"] == ( + "python:3.14-slim@sha256:83ff1d245a3d57d04152252d3ef9cb361494d0b3395abd65a5ebe91c401c8e83" + ) assert date.fromisoformat(baseline["expires_on"]) >= date.today() assert set(baseline["images"]) == { "synthetic", @@ -202,7 +205,7 @@ def test_repository_baseline_is_current_and_documents_every_exception() -> None: assert all(finding["severity"] in {"High", "Critical"} for finding in findings) -def test_repository_baseline_tracks_the_reviewed_openssl_findings() -> None: +def test_repository_baseline_does_not_retain_fixed_python_or_openssl_findings() -> None: baseline = json.loads(REPOSITORY_BASELINE.read_text(encoding="utf-8")) findings = { (finding["id"], finding["package"], finding["severity"]) @@ -210,11 +213,15 @@ def test_repository_baseline_tracks_the_reviewed_openssl_findings() -> None: for finding in profile } - assert { + fixed_findings = { + ("CVE-2026-11940", "python", "High"), + ("CVE-2026-11972", "python", "High"), + ("CVE-2026-15308", "python", "High"), ("CVE-2026-14456", "libssl3t64", "High"), ("CVE-2026-14456", "openssl", "High"), ("CVE-2026-14456", "openssl-provider-legacy", "High"), - } <= findings + } + assert fixed_findings.isdisjoint(findings) def test_sarif_filter_removes_reviewed_findings_and_keeps_new_findings(