diff --git a/extract-core/extract_core/__init__.py b/extract-core/extract_core/__init__.py index ce1a15a..537f69b 100644 --- a/extract-core/extract_core/__init__.py +++ b/extract-core/extract_core/__init__.py @@ -3,7 +3,7 @@ from icij_common.pydantic_utils import make_enum_discriminator, tagged_union from pydantic import Discriminator -from .configs import BasePipelineConfig, PipelineType +from .configs import BasePipelineConfig, PipelineType, ResultBufferConfig from .objects import ( BaseModel, ConversionOutput, @@ -21,9 +21,24 @@ from .pipeline import Pipeline try: - from .docling_ import DoclingFormatOption, DoclingPipelineConfig + from .docling_ import ( + BatchConcurrencySettings, + DoclingFormatOption, + DoclingPipelineConfig, + DoclingSettings, + ) except ModuleNotFoundError: - DoclingPipelineConfig, DoclingFormatOption = None, None + ( + BatchConcurrencySettings, + DoclingFormatOption, + DoclingPipelineConfig, + DoclingSettings, + ) = ( + None, + None, + None, + None, + ) try: from .marker_ import MarkerPipelineConfig @@ -66,4 +81,7 @@ "Result", "Status", "SupportedExt", + "ResultBufferConfig", + "DoclingSettings", + "BatchConcurrencySettings", ] diff --git a/extract-core/extract_core/configs.py b/extract-core/extract_core/configs.py index f5c249e..682d79b 100644 --- a/extract-core/extract_core/configs.py +++ b/extract-core/extract_core/configs.py @@ -1,12 +1,13 @@ from abc import ABC, abstractmethod from enum import StrEnum +from pathlib import Path from typing import ClassVar from icij_common.pydantic_utils import icij_config, merge_configs, no_enum_values_config from icij_common.registrable import RegistrableConfig -from pydantic import Field +from pydantic import ByteSize, Field -from .objects import Device, SupportedExt +from .objects import BaseModel, Device, SupportedExt class PipelineType(StrEnum): @@ -27,3 +28,8 @@ class BasePipelineConfig(RegistrableConfig, ABC): @classmethod @abstractmethod def supported_exts(cls) -> set[SupportedExt]: ... + + +class ResultBufferConfig(BaseModel): + max_size: ByteSize = "500MiB" + root: Path | None = None diff --git a/extract-core/extract_core/docling_.py b/extract-core/extract_core/docling_.py index 212b1ab..c535087 100644 --- a/extract-core/extract_core/docling_.py +++ b/extract-core/extract_core/docling_.py @@ -1,6 +1,6 @@ import importlib from functools import cache -from typing import Annotated, Any, ClassVar, TypeVar, get_type_hints +from typing import Annotated, Any, ClassVar, get_type_hints from docling.datamodel.backend_options import BackendOptions, BaseBackendOptions from docling.datamodel.base_models import ( @@ -21,7 +21,9 @@ ThreadedPdfPipelineOptions, ) from docling.datamodel.settings import ( - BatchConcurrencySettings, + BatchConcurrencySettings as DoclingBatchConcurrencySettings, +) +from docling.datamodel.settings import ( DebugSettings, InferenceSettings, ) @@ -40,7 +42,7 @@ ) from pydantic_core.core_schema import SerializerFunctionWrapHandler -from .configs import BasePipelineConfig, PipelineType +from .configs import BasePipelineConfig, PipelineType, ResultBufferConfig from .objects import BaseModel, Device, SupportedExt from .utils import all_subclasses @@ -69,10 +71,7 @@ def _validate_pipeline_opts(v: PipelineOptions) -> PipelineOptions: return v -T = TypeVar("T") - - -def _find_subcls(cls: type[T], name: str) -> type[T]: +def _find_subcls[T](cls: type[T], name: str) -> type[T]: # Check if the class available for c in all_subclasses(cls): if c.__name__ == name: @@ -264,6 +263,13 @@ def _default_format_opts() -> dict[InputFormat, DoclingFormatOption]: } +class BatchConcurrencySettings(DoclingBatchConcurrencySettings): + # process up to 16 pages in || on GPU + page_batch_size: int = 16 + # call convert_all with at most page_batch_size * page_batch_size + max_page_batches: int = 2 + + class DoclingSettings(BaseModel): perf: BatchConcurrencySettings = Field(default_factory=BatchConcurrencySettings) debug: DebugSettings = Field(default_factory=DebugSettings) @@ -276,7 +282,9 @@ class DoclingPipelineConfig(BasePipelineConfig): format_options: dict[InputFormat, DoclingFormatOption] = Field( default_factory=_default_format_opts ) + settings: DoclingSettings = Field(default_factory=DoclingSettings) + result_buffer: ResultBufferConfig = Field(default_factory=ResultBufferConfig) @classmethod @cache diff --git a/extract-core/extract_core/objects.py b/extract-core/extract_core/objects.py index eac8f67..d5eb9d9 100644 --- a/extract-core/extract_core/objects.py +++ b/extract-core/extract_core/objects.py @@ -8,17 +8,16 @@ from functools import cache from io import BytesIO from pathlib import Path -from typing import Annotated, Any, NoReturn, Self +from typing import Any, NoReturn, Self from docling.datamodel.accelerator_options import AcceleratorDevice from icij_common.pydantic_utils import ( icij_config, merge_configs, no_enum_values_config, - safe_copy, ) -from pydantic import AfterValidator, Field, TypeAdapter from pydantic import BaseModel as _BaseModel +from pydantic import Field, TypeAdapter logger = logging.getLogger(__name__) base_config = merge_configs(icij_config(), no_enum_values_config()) @@ -120,8 +119,8 @@ def to_marker(self) -> str: class Status(StrEnum): FAILURE = "failure" - SUCCESS = "success" PARTIAL_SUCCESS = "partial_success" + SUCCESS = "success" @classmethod def from_docling(cls, v: Any) -> Self: @@ -139,6 +138,26 @@ def from_docling(cls, v: Any) -> Self: def allows_conversion(self) -> bool: return self is Status.SUCCESS or self is Status.PARTIAL_SUCCESS + def __add__(self, other: "Status") -> "Status": + if not isinstance(other, Status): + msg = ( + f"can't add {other} of type {other.__class__.__name__} " + f"to {self.__class__.__name__}" + ) + raise TypeError(msg) + statuses = sorted((self, other), key=lambda x: x.value) + match statuses: + case (Status.FAILURE, Status.FAILURE): + return Status.FAILURE + case (Status.FAILURE, Status.SUCCESS): + return Status.PARTIAL_SUCCESS + case (_, Status.PARTIAL_SUCCESS) | (Status.PARTIAL_SUCCESS, _): + return Status.PARTIAL_SUCCESS + case (Status.SUCCESS, Status.SUCCESS): + return Status.SUCCESS + case _: + raise ValueError(f"unexpected value {statuses}") + class Error(BaseModel): id: str @@ -179,30 +198,24 @@ def _id_title(title: str) -> str: class InputDoc(BaseModel): ext: SupportedExt path: Path - content: bytes | None = None + n_pages: int @classmethod - def from_path(cls, path: str | Path) -> Self: + def from_path(cls, path: str | Path, n_pages: int) -> Self: if isinstance(path, str): path = Path(path) ext = SupportedExt(path.suffix) - return cls(path=path, ext=ext) + return cls(path=path, ext=ext, n_pages=n_pages) def to_docling(self): # noqa: ANN201 from docling_core.types.io import DocumentStream # noqa: PLC0415 - if self.content is not None: - return DocumentStream(name=str(self.path), stream=BytesIO(self.content)) - if not self.path.suffix: return DocumentStream( name=str(self.path), stream=BytesIO(self.path.read_bytes()) ) return self.path - def without_content(self) -> Self: - return safe_copy(self, update={"content": None}) - Ranges = list[tuple[int, int]] @@ -223,6 +236,7 @@ def from_pages_bytes_sizes(cls, sizes: Sequence[int]) -> Self: class ConversionOutput(BaseModel): path: Path pages: Pages = Field(default_factory=Pages) + confidence: float | None class MarkdownDoc(ConversionOutput): @@ -235,12 +249,6 @@ def _valid_conversion_statuses(cls) -> set: return {ConversionStatus.SUCCESS, ConversionStatus.PARTIAL_SUCCESS} -def _input_should_not_have_content(value: InputDoc) -> InputDoc: - if value.content is not None: - raise ValueError(f"response input can't have content, but got {value}") - return value - - class _BaseResult(BaseModel, ABC): input: InputDoc status: Status @@ -248,7 +256,7 @@ class _BaseResult(BaseModel, ABC): class ResponseResult(_BaseResult): - input: Annotated[InputDoc, AfterValidator(func=_input_should_not_have_content)] + input: InputDoc output_path: Path @@ -258,7 +266,7 @@ class Result(_BaseResult): def to_response(self) -> ResponseResult: return ResponseResult( - input=self.input.without_content(), + input=self.input, status=self.status, errors=self.errors, output_path=self.output.path, diff --git a/extract-core/extract_core/pipeline.py b/extract-core/extract_core/pipeline.py index 0887d9a..d985668 100644 --- a/extract-core/extract_core/pipeline.py +++ b/extract-core/extract_core/pipeline.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator, Iterable +from collections.abc import AsyncIterable, Iterable from pathlib import Path -from typing import Generic, Self, TypeVar +from typing import Self from icij_common.registrable import RegistrableFromConfig @@ -9,10 +9,8 @@ from .objects import InputDoc, OutputFormat, Result -C = TypeVar("C", bound="BasePipelineConfig") - -class Pipeline(RegistrableFromConfig, Generic[C], ABC): +class Pipeline[C: BasePipelineConfig](RegistrableFromConfig, ABC): def __init__(self, config: C): self._config = config self._device = self._config.device @@ -20,7 +18,7 @@ def __init__(self, config: C): @abstractmethod async def extract_content( self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path - ) -> AsyncGenerator[Result, None]: ... + ) -> AsyncIterable[Result]: ... @classmethod def _from_config(cls, config: C) -> Self: diff --git a/extract-core/extract_core/utils.py b/extract-core/extract_core/utils.py index 3710f8f..fbe57e6 100644 --- a/extract-core/extract_core/utils.py +++ b/extract-core/extract_core/utils.py @@ -1,9 +1,4 @@ -from typing import TypeVar - -T = TypeVar("T") - - -def all_subclasses(cls: type[T]) -> set[type[T]]: +def all_subclasses[T](cls: type[T]) -> set[type[T]]: return set(cls.__subclasses__()).union( [s for c in cls.__subclasses__() for s in all_subclasses(c)] ) diff --git a/extract-core/tests/test_objects.py b/extract-core/tests/test_objects.py index a2b6bcb..7774e84 100644 --- a/extract-core/tests/test_objects.py +++ b/extract-core/tests/test_objects.py @@ -1,3 +1,4 @@ +import pytest from docling.datamodel.accelerator_options import AcceleratorDevice, AcceleratorOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.pipeline_options import ( @@ -6,7 +7,7 @@ ) from docling.document_converter import PdfFormatOption from extract_core import DoclingPipelineConfig, PipelineConfig -from extract_core.objects import Device +from extract_core.objects import Device, Status from pydantic import TypeAdapter @@ -43,3 +44,24 @@ def test_docling_pipeline_config() -> None: ) ) assert pdf_pipeline_options.model_dump() == expected_options.model_dump() + + +@pytest.mark.parametrize( + ("left", "right", "expected_status"), + [ + (Status.FAILURE, Status.FAILURE, Status.FAILURE), + (Status.FAILURE, Status.PARTIAL_SUCCESS, Status.PARTIAL_SUCCESS), + (Status.FAILURE, Status.SUCCESS, Status.PARTIAL_SUCCESS), + (Status.PARTIAL_SUCCESS, Status.FAILURE, Status.PARTIAL_SUCCESS), + (Status.PARTIAL_SUCCESS, Status.PARTIAL_SUCCESS, Status.PARTIAL_SUCCESS), + (Status.PARTIAL_SUCCESS, Status.SUCCESS, Status.PARTIAL_SUCCESS), + (Status.SUCCESS, Status.FAILURE, Status.PARTIAL_SUCCESS), + (Status.SUCCESS, Status.PARTIAL_SUCCESS, Status.PARTIAL_SUCCESS), + (Status.SUCCESS, Status.SUCCESS, Status.SUCCESS), + ], +) +def test_add_statuses(left: Status, right: Status, expected_status: Status) -> None: + # When + status = left + right + # Then + assert status == expected_status diff --git a/extract-python/extract_python/docling_.py b/extract-python/extract_python/docling_.py index a3565e8..8c7e47d 100644 --- a/extract-python/extract_python/docling_.py +++ b/extract-python/extract_python/docling_.py @@ -1,21 +1,27 @@ import asyncio import json import logging +import operator import shutil import tempfile -from collections.abc import AsyncGenerator, Iterable, Iterator -from functools import partial +from collections.abc import AsyncIterable, Iterable +from contextlib import AbstractContextManager +from functools import partial, reduce from pathlib import Path from typing import Any, Self -from docling.datamodel.document import ConversionResult +from docling.datamodel.document import ConversionAssets from docling.datamodel.pipeline_options import PipelineOptions -from docling.datamodel.settings import scoped +from docling.datamodel.settings import ( + DEFAULT_PAGE_RANGE, + AppSettings, + scoped, +) from docling.document_converter import DocumentConverter, FormatOption +from docling_core.types import DoclingDocument # TODO: this is long to load improve it from docling_core.types.doc import ImageRefMode -from docling_core.types.io import DocumentStream from extract_core import ( BaseModel, DoclingFormatOption, @@ -34,7 +40,14 @@ from pydantic_core.core_schema import SerializerFunctionWrapHandler from .constants import ARTIFACTS, DEFAULT_MD_PAGE_SEP -from .utils import chdir, map_and_preserve, path_to_artifacts_dirname, write_pages +from .utils import ( + Range, + ResultBuffer, + batch_per_pages, + chdir, + path_to_artifacts_dirname, + write_pages, +) logger = logging.getLogger(__name__) @@ -62,61 +75,112 @@ def __init__(self, config: DoclingPipelineConfig): async def extract_content( self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path - ) -> AsyncGenerator[Result, None]: + ) -> AsyncIterable[Result]: settings = self._config.settings logger.info("starting extraction with settings: %s", settings) - with scoped( - perf=settings.perf, debug=settings.debug, inference=settings.inference - ): - docs, path_or_streams = map_and_preserve(_to_docling, docs) - outputs = self._converter.convert_all( - path_or_streams, raises_on_error=False + with self._scoped_settings, self._result_buffer as buffer: + max_page_batches = settings.perf.max_page_batches + page_batch_size = settings.perf.page_batch_size + batches = batch_per_pages( + docs, page_batch_size, max_page_batches=max_page_batches ) + for batch in batches: + page_range = list({pages.page_range for pages in batch}) + if len(page_range) > 1: + msg = "convert_all only accept 1 page range for all docs" + raise ValueError(msg) + page_range = page_range[0] + page_range = _docling_range(page_range) + docling_docs = (pages.doc.to_docling() for pages in batch) + outputs = self._converter.convert_all( + docling_docs, raises_on_error=False, page_range=page_range + ) + processed = iter(batch) + sentinel = object() + while True: + res = await asyncio.to_thread(next, outputs, sentinel) + if res is sentinel: + break + pages = next(processed) + buffer.add(pages, res) + if buffer.is_complete(pages.doc_idx): + doc_pages = buffer.pop_complete(pages.doc_idx) + yield _to_result( + doc_pages, pages.doc, output_format, output_path=output_path + ) - sentinel = object() - while True: - res = await asyncio.to_thread(next, outputs, sentinel) - if res is sentinel: - return - doc = next(docs) - yield _to_result(res, doc, output_format, output_path=output_path) + @property + def _scoped_settings(self) -> AbstractContextManager[AppSettings]: + settings = self._config.settings + docling_settings = scoped( + perf=settings.perf, debug=settings.debug, inference=settings.inference + ) + return docling_settings + @property + def _result_buffer(self) -> ResultBuffer: + buffer = ResultBuffer( + max_size_bytes=self._config.result_buffer.max_size, + root=self._config.result_buffer.root, + save_fn=_save_conversion_result, + load_fn=_load_conversion_result, + ) + return buffer -def _to_docling(docs: Iterable[InputDoc]) -> Iterator["Path | DocumentStream"]: - for d in docs: - yield d.to_docling() + +def _save_conversion_result(res: ConversionAssets, path: Path) -> None: + return res.save(filename=path) + + +def _load_conversion_result(path: Path) -> ConversionAssets: + return ConversionAssets.load(path) def _to_result( - res: ConversionResult, - input_document: InputDoc, + buffer: list[ConversionAssets], + input_doc: InputDoc, output_format: OutputFormat, output_path: Path, **kwargs, ) -> Result: + import numpy as np # noqa: PLC0415 + + if not buffer: + raise ValueError("empty buffer") + merged = DoclingDocument.concatenate([res.document for res in buffer]) output_path.mkdir(parents=True, exist_ok=True) + status = reduce(operator.iadd, (Status.from_docling(d.status) for d in buffer)) + # TODO: implement confidence weight + confidence = np.mean([res.confidence.mean_score for res in buffer]) output = None - status = Status.from_docling(res.status) if status.allows_conversion: match output_format: case OutputFormat.MARKDOWN: - output = _to_markdown_doc(res, output_path, **kwargs) + output = _to_markdown_doc( + merged, + input_path=input_doc.path, + output_path=output_path, + confidence=confidence, + **kwargs, + ) case _: raise NotImplementedError(f"unsupported output format {output_format}") - errors = [Error.from_docling(e) for e in res.errors] - input_doc = input_document.without_content() + errors = [Error.from_docling(e) for res in buffer for e in res.errors] return Result(input=input_doc, status=status, errors=errors, output=output) def _to_markdown_doc( - res: ConversionResult, + doc: DoclingDocument, + input_path: Path, + *, output_path: Path, page_sep: str = DEFAULT_MD_PAGE_SEP, + confidence: float, **kwargs, ) -> MarkdownDoc: # TODO: Should we add a hash to avoid collision between files with same names # nested in the tree structured - md_dir_name = path_to_artifacts_dirname(res.input.file) + md_dir_name = path_to_artifacts_dirname(input_path) md_dir = output_path / md_dir_name if md_dir.exists(): raise FileExistsError(f"directory {md_dir} already exists") @@ -129,21 +193,21 @@ def _to_markdown_doc( with chdir(tmp_dir): # We do a chdir to bypass a Docling bug which only allows to maintain # relative image ref when saving the markdown to a relative path - pages = _docling_pages_it(res, current_page_path, **kwargs) + pages = _docling_pages_it(doc, current_page_path, **kwargs) with md_path.open("wb") as f: pages = write_pages(pages, page_sep, f) # Clean up the tmp page file before move everything to the end destination current_page_path.unlink(missing_ok=True) shutil.move(tmp_dir, md_dir) - return MarkdownDoc(path=Path(md_dir_name), pages=pages) + return MarkdownDoc(path=Path(md_dir_name), pages=pages, confidence=confidence) def _docling_pages_it( - res: ConversionResult, output_path: Path, **kwargs + doc: DoclingDocument, output_path: Path, **kwargs ) -> Iterable[str]: - n_pages = len(res.pages) + n_pages = len(doc.pages) for page_i in range(n_pages): - res.document.save_as_markdown( + doc.save_as_markdown( output_path, page_no=page_i + 1, image_mode=ImageRefMode.REFERENCED, @@ -201,3 +265,9 @@ def _serialize_pipeline_opts( serialized["table_structure_options"] = dict() serialized["table_structure_options"]["kind"] = table_structure_opts.kind return serialized + + +def _docling_range(rng: Range | None) -> tuple[int, int]: + if rng is None: + return DEFAULT_PAGE_RANGE + return (rng[0] + 1, rng[1] + 1) diff --git a/extract-python/extract_python/marker_.py b/extract-python/extract_python/marker_.py index 162faa0..73a05df 100644 --- a/extract-python/extract_python/marker_.py +++ b/extract-python/extract_python/marker_.py @@ -1,6 +1,6 @@ import asyncio import gc -from collections.abc import AsyncGenerator, Iterable +from collections.abc import AsyncIterable, Iterable from copy import deepcopy from pathlib import Path from typing import TYPE_CHECKING @@ -30,7 +30,7 @@ class MarkerPipeline(Pipeline): async def extract_content( self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path - ) -> AsyncGenerator[Result, None]: + ) -> AsyncIterable[Result]: from marker.config.parser import ConfigParser # noqa: PLC0415 from marker.converters.pdf import PdfConverter # noqa: PLC0415 from marker.models import create_model_dict # noqa: PLC0415 @@ -67,8 +67,7 @@ async def _process_doc( ) case _: raise NotImplementedError(f"unsupported output format {output_format}") - input_doc = doc.without_content() - return Result(input=input_doc, status=Status.SUCCESS, output=output) + return Result(input=doc, status=Status.SUCCESS, output=output) def _to_markdown_doc( @@ -96,4 +95,4 @@ def _to_markdown_doc( md_path = md_path.with_suffix(OutputFormat.MARKDOWN.value) with md_path.open("wb") as f: pages = write_pages(pages, page_sep, f) - return MarkdownDoc(path=Path(md_dir_name), pages=pages) + return MarkdownDoc(path=Path(md_dir_name), pages=pages, confidence=None) diff --git a/extract-python/extract_python/miner_u.py b/extract-python/extract_python/miner_u.py index 399253c..5a83c71 100644 --- a/extract-python/extract_python/miner_u.py +++ b/extract-python/extract_python/miner_u.py @@ -1,7 +1,7 @@ import json import os import shutil -from collections.abc import AsyncGenerator, Callable, Iterable +from collections.abc import AsyncIterable, Callable, Iterable from functools import partial from pathlib import Path from tempfile import TemporaryDirectory @@ -34,7 +34,7 @@ def __init__(self, config: MinerUPipelineConfig): async def extract_content( self, docs: Iterable[InputDoc], output_format: OutputFormat, output_path: Path - ) -> AsyncGenerator[Result, None]: + ) -> AsyncIterable[Result]: from mineru.cli.common import aio_do_parse # noqa: PLC0415 with reset_env(): @@ -126,15 +126,13 @@ def _process_doc( raise NotImplementedError(f"unsupported output format {output_format}") middle_json_path = res_path / f"{doc.path.name}_middle.json" middle_json = json.loads(middle_json_path.read_text()) - pdf_info = middle_json["pdf_info"] shutil.move(res_path / "images", artifacts_dir) - output = dump_content_fn(pdf_info) - input_doc = doc.without_content() - return Result(input=input_doc, status=Status.SUCCESS, output=output) + output = dump_content_fn(middle_json) + return Result(input=doc, status=Status.SUCCESS, output=output) def _dump_md_content( - pdf_info: list[dict], + middle_json: dict, *, md_make_fn: MDMakeFunction, page_sep: str = DEFAULT_MD_PAGE_SEP, @@ -145,11 +143,72 @@ def _dump_md_content( ) -> ConversionOutput: from mineru.utils.enum_class import MakeMode # noqa: PLC0415 + pdf_info = middle_json["pdf_info"] if md_make_mode is None: md_make_mode = MakeMode.MM_MD pages = (md_make_fn([p], md_make_mode, str(im_dir)) for p in pdf_info) with md_path.open("wb") as f: pages = write_pages(pages, page_sep, f) output_path = md_path.parent.relative_to(output_path) - output = ConversionOutput(path=output_path, pages=pages) + confidence = _mineru_confidence(pdf_info) + output = ConversionOutput(path=output_path, pages=pages, confidence=confidence) return output + + +def _mineru_confidence(pdf_info: list[dict]) -> float: + if not pdf_info: + return 1.0 + block_conf = _mineru_block_confidence(pdf_info) + line_config = _mineru_line_confidence(pdf_info) + return (block_conf + line_config) / 2.0 + + +def _mineru_block_confidence(pdf_info: list[dict]) -> float: + import numpy as np # noqa: PLC0415 + + scores = [] + for info in pdf_info: + for block in info["para_blocks"]: + score = block.get("score") + if score is not None: + scores.append(score) + if scores: + return np.average(scores) + return 1.0 + + +def _mineru_line_confidence(pdf_info: list[dict]) -> float: + import numpy as np # noqa: PLC0415 + + scores = [] + lengths = [] + for info in pdf_info: + for block in info["para_blocks"]: + for line in block.get("lines", []): + for span in line["spans"]: + score = span.get("score") + if score is not None: + scores.append(score) + lengths.append(len(span["content"])) + if scores: + return np.average(scores, weights=lengths) + return 1.0 + + +def _parse_block(block: dict) -> tuple[list[float], list[float]]: + if "lines" in block: + scores = [] + lengths = [] + for line in block.get("lines", []): + for span in line["spans"]: + score = span.get("score") + if score is not None: + scores.append(score) + lengths.append(len(span["content"])) + return scores, lengths + if "blocs" in block: + scores, lengths = (_parse_block(b) for b in block["blocs"]) + scores = sum(*scores, start=[]) + lengths = sum(*lengths, start=[]) + return scores, lengths + raise NotImplementedError(f"unsupported block: {block}") diff --git a/extract-python/extract_python/utils.py b/extract-python/extract_python/utils.py index 85902a4..536fff9 100644 --- a/extract-python/extract_python/utils.py +++ b/extract-python/extract_python/utils.py @@ -1,21 +1,30 @@ +import gc +import itertools +import logging import os +import shutil +import uuid +from collections import defaultdict, deque from collections.abc import Callable, Generator, Iterable, Iterator from contextlib import contextmanager from copy import copy +from dataclasses import dataclass from functools import wraps from itertools import tee from pathlib import Path, PurePath -from typing import BinaryIO, Protocol, TypeVar +from tempfile import TemporaryDirectory +from types import TracebackType +from typing import BinaryIO, Protocol, Self from extract_core import Error, InputDoc, Pages, Result, Status +from pympler import asizeof -R = TypeVar("R") -In = TypeVar("In") +logger = logging.getLogger(__name__) -def map_and_preserve( - fn: Callable[[Iterable[In]], Iterator[R]], inputs: Iterable[In] -) -> tuple[Iterable[In], Iterator[R]]: +def map_and_preserve[I, R]( + fn: Callable[[Iterable[I]], Iterator[R]], inputs: Iterable[I] +) -> tuple[Iterable[I], Iterator[R]]: save_inputs, function_inputs = tee(inputs) outputs = iter(fn(function_inputs)) return save_inputs, outputs @@ -44,10 +53,7 @@ def wrapped(doc: InputDoc, *args, **kwargs) -> Result: except recoverable_errors as e: error = Error.from_exception(e) return Result( - input=doc.without_content(), - status=Status.FAILURE, - errors=[error], - output=None, + input=doc, status=Status.FAILURE, errors=[error], output=None ) return wrapped @@ -56,7 +62,7 @@ def wrapped(doc: InputDoc, *args, **kwargs) -> Result: @contextmanager -def chdir(path: Path) -> Generator[None, None, None]: +def chdir(path: Path) -> Generator[None]: cwd = Path.cwd() try: os.chdir(path) @@ -66,7 +72,7 @@ def chdir(path: Path) -> Generator[None, None, None]: @contextmanager -def reset_env() -> Generator[None, None, None]: +def reset_env() -> Generator[None]: old_env = copy(dict(os.environ)) try: yield @@ -86,3 +92,208 @@ def write_pages(pages: Iterable[str], page_sep: str, out: BinaryIO) -> Pages: if content: pages_byte_sizes.append(out.write(content.encode())) return Pages.from_pages_bytes_sizes(pages_byte_sizes) + + +Range = tuple[int, int] + + +@dataclass(frozen=True) +class ProcessedPages: + doc: InputDoc + doc_idx: int + page_range: Range | None = None + + @property + def page_length(self) -> int: + if self.page_range is None: + return self.doc.n_pages + return self.page_range[1] - self.page_range[0] + + +class ResultBuffer[R]: + def __init__( + self, + max_size_bytes: int, + save_fn: Callable[[R, Path], None], + *, + load_fn: Callable[[Path], R], + root: Path | None = None, + ): + self._max_bytes = max_size_bytes + self._save_fn = save_fn + self._load_fn = load_fn + self._tmp_dir = None + if root is None: + self._tmp_dir = TemporaryDirectory() + root = Path(self._tmp_dir.name) + self._root = root + self.__fs_buffer_path = None + self._mem_buffer: dict[int, list[R | Path]] = defaultdict(list) + self._missing_pages: dict[int, int] = dict() + self._current_size: int = 0 + + def __enter__(self) -> Self: + if self._tmp_dir is not None: + self._tmp_dir.__enter__() + self.__fs_buffer_path = self._root / uuid.uuid4().hex + self.__fs_buffer_path.mkdir() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + if self._mem_buffer or self._missing_pages: + logger.warning("closing an non empty buffer") + if self._tmp_dir is not None: + self._tmp_dir.__exit__(exc_type, exc_val, exc_tb) + if self._fs_buffer_path.exists(): + shutil.rmtree(self._fs_buffer_path) + self._mem_buffer = dict() + self._missing_pages = dict() + + @property + def _fs_buffer_path(self) -> Path: + if not self.__fs_buffer_path: + msg = ( + f"inconsistent state, {ResultBuffer.__class__.__name__} is a context" + f" manager, call __enter__ before using it" + ) + raise ValueError(msg) + return self.__fs_buffer_path + + def add(self, processed: ProcessedPages, result: R) -> None: + size = asizeof.asizeof(result) + if self._current_size + size > self._max_bytes: + path = self._page_path(processed.doc_idx) + self._save_fn(result, path) + result = path + else: + self._current_size += size + self._mem_buffer[processed.doc_idx].append(result) + if processed.doc_idx not in self._missing_pages: + self._missing_pages[processed.doc_idx] = processed.doc.n_pages + self._missing_pages[processed.doc_idx] -= processed.page_length + + def is_complete(self, doc: int) -> bool: + return self._missing_pages[doc] == 0 + + def pop_complete(self, doc: int) -> list[R]: + if not self.is_complete(doc): + raise ValueError(f"{doc} is incomplete") + pages = self._mem_buffer.pop(doc) + self._missing_pages.pop(doc) + for i, page in enumerate(pages): + if isinstance(page, Path): + page = self._load_fn(page) # noqa: PLW2901 + else: + self._current_size -= asizeof.asizeof(page) + pages[i] = page + return pages + + def _page_path(self, doc_id: int) -> Path: + pages = self._mem_buffer[doc_id] + return self._fs_buffer_path / f"doc-{doc_id}-pages-{len(pages)}" + + def __len__(self) -> int: + return len(self._mem_buffer) + + +# The converter process page_batch_size in parallel (GPU sees page_batch_size batches). +# +# The batching tradeoff is: avoid calling convert_all to many times vs. releasing the +# GIL often enough. +# +# Calling convert_all to many times on the same doc results in overhead. Each time we +# call the function, we create a doc processing backend + reload the doc. +# +# On the other hand we have to use a reasonable max_page_batches otherwise we process +# all the stream in a single call and take the risk to lock the GIL for too long. Some +# docling ops are sadly not async (numpy or torch inference are, but document loading +# and conversion aren't, so the asyncio.to_thread is not helping) +def batch_per_pages( + docs: Iterable[InputDoc], + page_batch_size: int, + *, + max_page_batches: int, + chunk_size: int = 1000, +) -> Iterable[tuple[ProcessedPages]]: + # convert_all only accept to process docs on the exact same page_range + # + # We collect by chunk to avoid collecting too many inputs, input docs are + # lightweight anyway so memory impact should stay limited + # + # Additionally, results can be output unordered and partial results are buffered + # it's OK to process doc pages unordered. + # TODO: if it's not OK to sort because inputs is l + max_pages = page_batch_size * max_page_batches + docs = itertools.batched(docs, chunk_size, strict=False) + for chunk in docs: + short_docs = [d for d in chunk if d.n_pages <= max_pages] + long_docs = [d for d in chunk if d.n_pages > max_pages] + del chunk + gc.collect() + # Bin fill for docs smaller than max_pages + offset = yield from _bin_fill(short_docs, max_pages=max_pages) + # otherwise we just yield chunks of max_pages except the last chunk which is + # grouped by page_range + yield from _by_page_ranges(long_docs, max_pages=max_pages, offset=offset) + + +def _bin_fill( + docs: Iterable[InputDoc], max_pages: int, offset: int = 0 +) -> Generator[tuple[ProcessedPages], None, int]: + bins = defaultdict(deque) + doc_idx = offset + for doc in docs: + if doc.n_pages > max_pages: + msg = f"expected docs to have <= {max_pages} pages" + raise ValueError(msg) + pages = ProcessedPages(doc=doc, doc_idx=doc_idx) + doc_idx += 1 + available_space = (s for s in sorted(bins.keys()) if doc.n_pages <= s) + available_space = next(available_space, max_pages) + selected = bins[available_space] + selected = selected.pop() if selected else [] + selected.append(pages) + available_space -= doc.n_pages + if available_space == 0: + yield tuple(selected) + continue + bins[available_space].append(selected) + for range_bins in bins.values(): + for b in range_bins: + yield tuple(b) + return doc_idx + + +def _by_page_ranges( + docs: Iterable[InputDoc], max_pages: int, offset: int = 0 +) -> Generator[tuple[ProcessedPages], None, int]: + by_range = defaultdict(list) + doc_idx = offset + for doc in docs: + if doc.n_pages < max_pages: + msg = f"expected docs to have >= {max_pages} pages" + raise ValueError(msg) + + for i in range(0, doc.n_pages, max_pages): + start = i + end = min(start + max_pages, doc.n_pages) + rng = (start, end) + rng_size = end - start + alone_in_batch = rng_size == max_pages + pages = ProcessedPages(doc=doc, doc_idx=doc_idx, page_range=rng) + if alone_in_batch: + yield (pages,) + continue + by_range[rng].append(pages) + is_complete = len(by_range[rng]) == (max_pages // rng_size) + if is_complete: + yield tuple(by_range.pop(rng)) + doc_idx += 1 + for v in by_range.values(): + yield tuple(v) + return doc_idx diff --git a/extract-python/pyproject.toml b/extract-python/pyproject.toml index 234f8f9..6b2406a 100644 --- a/extract-python/pyproject.toml +++ b/extract-python/pyproject.toml @@ -8,8 +8,9 @@ authors = [ readme = "README.md" requires-python = ">=3.13,<3.15" dependencies = [ - "icij-common~=0.8.2", "extract-core~=0.7.0", + "icij-common~=0.8.2", + "pympler~=1.1", ] [project.optional-dependencies] diff --git a/extract-python/tests/conftest.py b/extract-python/tests/conftest.py index 39ddfa0..b5bf7f8 100644 --- a/extract-python/tests/conftest.py +++ b/extract-python/tests/conftest.py @@ -18,9 +18,9 @@ def device() -> Device: @pytest.fixture(scope="session") def docs() -> list[InputDoc]: - doc_paths = ("scanned.pdf", "computer_generated.pdf") - doc_paths = (TEST_DATA_DIR / p for p in doc_paths) - docs = [InputDoc.from_path(p) for p in doc_paths] + docs = (("scanned.pdf", 1), ("computer_generated.pdf", 3)) + docs = ((TEST_DATA_DIR / path, n_pages) for path, n_pages in docs) + docs = [InputDoc.from_path(path, n_pages=n_pages) for path, n_pages in docs] return docs diff --git a/extract-python/tests/test_docling.py b/extract-python/tests/test_docling.py index b233276..1420144 100644 --- a/extract-python/tests/test_docling.py +++ b/extract-python/tests/test_docling.py @@ -2,15 +2,19 @@ from typing import cast import pytest +from _pytest.legacypath import TempdirFactory from docling.datamodel.accelerator_options import AcceleratorDevice, AcceleratorOptions from docling.datamodel.base_models import InputFormat from docling.datamodel.pipeline_options import VlmConvertOptions, VlmPipelineOptions from extract_core import ( + BatchConcurrencySettings, DoclingFormatOption, DoclingPipelineConfig, + DoclingSettings, InputDoc, OutputFormat, Pipeline, + ResultBufferConfig, Status, ) from extract_core.objects import Device @@ -20,9 +24,17 @@ @pytest.fixture(scope="session") -def config(device: Device) -> DoclingPipelineConfig: +def config(device: Device, tmpdir_factory: TempdirFactory) -> DoclingPipelineConfig: # TODO: for testing add a lightweight configuration - config = DoclingPipelineConfig(device=device) + fs_buffer_root = Path(tmpdir_factory.mktemp("fs_buffer_root")) + settings = DoclingSettings( + perf=BatchConcurrencySettings(page_batch_size=2, max_page_batches=1) + ) + config = DoclingPipelineConfig( + device=device, + settings=settings, + result_buffer=ResultBufferConfig(root=fs_buffer_root, max_size="0MB"), + ) return config @@ -42,9 +54,9 @@ async def test_docling_pdf_to_markdown( res = [r async for r in pipeline.extract_content(docs, output_format, output_path)] # Then assert all(r.status == Status.SUCCESS for r in res) - expected_output_paths = ["scanned_pdf", "computer_generated_pdf"] + expected_output_paths = ["computer_generated_pdf", "scanned_pdf"] expected_output_paths = [Path(p) for p in expected_output_paths] - output_paths = [r.output.path for r in res] + output_paths = sorted(r.output.path for r in res) assert output_paths == expected_output_paths for p in expected_output_paths: assert (output_path / p).exists() @@ -53,10 +65,10 @@ async def test_docling_pdf_to_markdown( assert any((output_path / p).glob("artifacts/*.png")) assert all(r.output.pages.byte_ranges for r in res) assert not any(r.errors for r in res) - input_path = [r.input.path for r in res] + input_path = sorted(r.input.path for r in res) expected_input_path = [ - TEST_DATA_DIR / "scanned.pdf", TEST_DATA_DIR / "computer_generated.pdf", + TEST_DATA_DIR / "scanned.pdf", ] assert input_path == expected_input_path diff --git a/extract-python/tests/test_utils.py b/extract-python/tests/test_utils.py index 87e0ab2..b1ba1d0 100644 --- a/extract-python/tests/test_utils.py +++ b/extract-python/tests/test_utils.py @@ -1,7 +1,19 @@ +import json +import sys from io import BytesIO +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock import pytest -from extract_python.utils import write_pages +from extract_core import InputDoc, SupportedExt +from extract_python.utils import ( + ProcessedPages, + ResultBuffer, + batch_per_pages, + write_pages, +) +from pympler import asizeof def _read_page(doc: BytesIO, start: int, *, end: int) -> str: @@ -90,3 +102,302 @@ def test_write_pages( start, end = byte_range page = _read_page(output, start, end=end) assert page == expected_content + + +class TestResultBuffer(ResultBuffer): + @property + def current_size(self) -> int: + return self._current_size + + @property + def root(self) -> Path: + return self._root + + +def _json_save(obj: Any, path: Path) -> None: + path.write_text(json.dumps(obj)) + + +def _json_load(path: Path) -> Any: + return json.loads(path.read_text()) + + +def test_result_buffer_in_memory() -> None: + # Given + max_size_bytes = sys.maxsize + save_fn = MagicMock() + load_fn = MagicMock() + buffer = TestResultBuffer( + max_size_bytes=max_size_bytes, save_fn=save_fn, load_fn=load_fn + ) + doc_idx = 0 + first = ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), + doc_idx=doc_idx, + page_range=(0, 1), + ) + last = ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), + doc_idx=doc_idx, + page_range=(1, 2), + ) + # When + with buffer: + buffer.add(first, 0) + buffer.add(last, -1) + # Then + assert buffer.current_size == 64 + assert buffer.is_complete(doc_idx) + all_res = buffer.pop_complete(doc_idx) + assert all_res == [0, -1] + assert buffer.current_size == 0 + + +def test_result_buffer_offload_on_fs() -> None: + # Given + first_res = 1 + max_size_bytes = asizeof.asizeof(first_res) + 1 + + save_fn = MagicMock(side_effect=_json_save) + load_fn = MagicMock(side_effect=_json_load) + buffer = TestResultBuffer( + max_size_bytes=max_size_bytes, save_fn=save_fn, load_fn=load_fn + ) + doc_idx = 0 + first = ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), + doc_idx=doc_idx, + page_range=(0, 1), + ) + last = ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), + doc_idx=doc_idx, + page_range=(1, 2), + ) + # When + with buffer: + buffer.add(first, 0) + save_fn.assert_not_called() + buffer.add(last, -1) + save_fn.assert_called_once() + # Then + assert buffer.current_size == 32 + assert buffer.is_complete(doc_idx) + all_res = buffer.pop_complete(doc_idx) + assert all_res == [0, -1] + assert buffer.current_size == 0 + assert not buffer.root.exists() + + +def test_result_buffer_offload_on_fs_should_preserve_root(tmpdir: Path) -> None: + # Given + root = Path(tmpdir) + max_size_bytes = 0 + buffer = TestResultBuffer( + max_size_bytes=max_size_bytes, save_fn=_json_save, load_fn=_json_load, root=root + ) + # When + with buffer: + pass + assert buffer.root.exists() + + +def test_result_buffer_raise_for_inconsistent_state() -> None: + # Given + max_size_bytes = 0 + pages = ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path(""), n_pages=2), + doc_idx=0, + page_range=(0, 1), + ) + buffer = TestResultBuffer( + max_size_bytes=max_size_bytes, save_fn=_json_save, load_fn=_json_load + ) + # When/Then + expected = ( + "inconsistent state, type is a context manager, call __enter__ before using it" + ) + with pytest.raises(ValueError, match=expected): + buffer.add(pages, 0) + + +def test_batch_per_pages_should_yield_short_docs_first() -> None: + # Given + page_batch_size = 3 + max_page_batches = 2 + docs = [ + InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=8), + InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=6), + ] + # When + batches = list( + batch_per_pages(docs, page_batch_size, max_page_batches=max_page_batches) + ) + # Then + expected_batches = [ + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=6), doc_idx=0 + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=8), + doc_idx=1, + page_range=(0, 6), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=8), + doc_idx=1, + page_range=(6, 8), + ), + ), + ] + assert batches == expected_batches + + +def test_batch_per_pages_should_bin_short_docs() -> None: + # Given + page_batch_size = 3 + max_page_batches = 2 + docs = [ + InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=3), + InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=4), + InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=5), + InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=2), + InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=1), + ] + # When + batches = list( + batch_per_pages(docs, page_batch_size, max_page_batches=max_page_batches) + ) + # Then + expected_batches = [ + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=4), doc_idx=1 + ), + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=2), doc_idx=3 + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=5), doc_idx=2 + ), + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=1), + doc_idx=4, + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=3), doc_idx=0 + ), + ), + ] + assert batches == expected_batches + + +def test_batch_per_pages_should_group_long_docs_by_page_ranges() -> None: + # Given + page_batch_size = 4 + max_page_batches = 2 + docs = [ + InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=12), + InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=11), + InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=12), + InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=11), + InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=9), + InputDoc(ext=SupportedExt.PDF, path=Path("5"), n_pages=10), + ] + # When + batches = list( + batch_per_pages(docs, page_batch_size, max_page_batches=max_page_batches) + ) + # Then + expected_batches = [ + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=12), + doc_idx=0, + page_range=(0, 8), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=11), + doc_idx=1, + page_range=(0, 8), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=12), + doc_idx=2, + page_range=(0, 8), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("0"), n_pages=12), + doc_idx=0, + page_range=(8, 12), + ), + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("2"), n_pages=12), + doc_idx=2, + page_range=(8, 12), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=11), + doc_idx=3, + page_range=(0, 8), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("1"), n_pages=11), + doc_idx=1, + page_range=(8, 11), + ), + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("3"), n_pages=11), + doc_idx=3, + page_range=(8, 11), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=9), + doc_idx=4, + page_range=(0, 8), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("5"), n_pages=10), + doc_idx=5, + page_range=(0, 8), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("4"), n_pages=9), + doc_idx=4, + page_range=(8, 9), + ), + ), + ( + ProcessedPages( + doc=InputDoc(ext=SupportedExt.PDF, path=Path("5"), n_pages=10), + doc_idx=5, + page_range=(8, 10), + ), + ), + ] + assert batches == expected_batches diff --git a/extract-python/uv.lock b/extract-python/uv.lock index a22157a..2208028 100644 --- a/extract-python/uv.lock +++ b/extract-python/uv.lock @@ -919,6 +919,7 @@ source = { editable = "." } dependencies = [ { name = "extract-core" }, { name = "icij-common" }, + { name = "pympler" }, ] [package.optional-dependencies] @@ -965,6 +966,7 @@ requires-dist = [ { name = "mineru", extras = ["pipeline", "vlm"], marker = "extra == 'mineru'", specifier = "~=3.2" }, { name = "notebook", marker = "extra == 'benches'", specifier = ">=7.4.5" }, { name = "pydantic-extra-types", extras = ["pycountry"], marker = "extra == 'mineru'", specifier = "~=2.11" }, + { name = "pympler", specifier = "~=1.1" }, { name = "pypdfium2", marker = "extra == 'benches'", specifier = ">=4.30.0" }, { name = "python-pptx", marker = "extra == 'mineru'", specifier = "~=1.0" }, { name = "six", marker = "extra == 'mineru'", specifier = "~=1.17" }, @@ -2600,9 +2602,9 @@ name = "ocrmac" version = "1.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, - { name = "pillow", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, - { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, + { name = "click", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, + { name = "pillow", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, + { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5e/07/3e15ab404f75875c5e48c47163300eb90b7409044d8711fc3aaf52503f2e/ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9", size = 1454317, upload-time = "2026-01-08T16:44:26.412Z" } wheels = [ @@ -3291,6 +3293,18 @@ version = "2.10" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/5d/ab/34ec41718af73c00119d0351b7a2531d2ebddb51833a36448fc7b862be60/pylatexenc-2.10.tar.gz", hash = "sha256:3dd8fd84eb46dc30bee1e23eaab8d8fb5a7f507347b23e5f38ad9675c84f40d3", size = 162597, upload-time = "2021-04-06T07:56:07.854Z" } +[[package]] +name = "pympler" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/37/c384631908029676d8e7213dd956bb686af303a80db7afbc9be36bc49495/pympler-1.1.tar.gz", hash = "sha256:1eaa867cb8992c218430f1708fdaccda53df064144d1c5656b1e6f1ee6000424", size = 179954, upload-time = "2024-06-28T19:56:06.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/4f/a6a2e2b202d7fd97eadfe90979845b8706676b41cbd3b42ba75adf329d1f/Pympler-1.1-py3-none-any.whl", hash = "sha256:5b223d6027d0619584116a0cbc28e8d2e378f7a79c1e5e024f9ff3b673c58506", size = 165766, upload-time = "2024-06-28T19:56:05.087Z" }, +] + [[package]] name = "pyobjc-core" version = "12.2.1" @@ -3308,7 +3322,7 @@ name = "pyobjc-framework-cocoa" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/51/34/fbe38a204643aa4e1b91391cdce07a34da565a69171ebcad08de7438a556/pyobjc_framework_cocoa-12.2.1.tar.gz", hash = "sha256:b94b37fe5730e5ae1fb0052912cd174e6ec329b0bfba4a012ae5db1014b5864b", size = 3125751, upload-time = "2026-06-19T16:20:05.159Z" } wheels = [ @@ -3323,8 +3337,8 @@ name = "pyobjc-framework-coreml" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/1e/7d2db3e4468eb04cc92264be83113d86eea4f96302742437de695a445d6d/pyobjc_framework_coreml-12.2.1.tar.gz", hash = "sha256:ef3c2b6a160891b44173235603d10174929656b9c206d6f2f443fe2aa903c2cb", size = 49272, upload-time = "2026-06-19T16:20:18.459Z" } wheels = [ @@ -3339,8 +3353,8 @@ name = "pyobjc-framework-quartz" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3b/f6/2a8b84dbf1fe7c04dd96ea73d991678d4e09a909f51971ecc51629bb2ab4/pyobjc_framework_quartz-12.2.1.tar.gz", hash = "sha256:b3b8b6f71e66147f8ff9e6213864cc8527e3a0b1ee90835b93ce221f4802d9b0", size = 3215521, upload-time = "2026-06-19T16:21:30.199Z" } wheels = [ @@ -3355,10 +3369,10 @@ name = "pyobjc-framework-vision" version = "12.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, - { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, - { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin' or extra == 'extra-14-extract-python-marker' or extra != 'extra-14-extract-python-mineru'" }, + { name = "pyobjc-core", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, + { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, + { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, + { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin' or (extra == 'extra-14-extract-python-marker' and extra == 'extra-14-extract-python-mineru')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/7a/1fdffff1b6bf124b260a2169869f4b71a08b9f6603698f7dec990d5ae5f3/pyobjc_framework_vision-12.2.1.tar.gz", hash = "sha256:debfd59dd7d962a6053bf733370148c11a9ec44091b517a0966f48d81c305879", size = 72683, upload-time = "2026-06-19T16:22:01.102Z" } wheels = [ diff --git a/qa/ruff.toml b/qa/ruff.toml index d019826..b754c8d 100644 --- a/qa/ruff.toml +++ b/qa/ruff.toml @@ -1,4 +1,4 @@ -target-version = "py311" +target-version = "py313" [lint] select = [ "A",