diff --git a/.github/workflows/pdf-regressions.yml b/.github/workflows/pdf-regressions.yml index 20d4ab2..548bcf0 100644 --- a/.github/workflows/pdf-regressions.yml +++ b/.github/workflows/pdf-regressions.yml @@ -7,6 +7,10 @@ on: paths: - "pdf_metadata.py" - "tag-ocr-pdf.py" + - "ocr-scanned-pdf.sh" + - "compress-and-tag-ocr-pdf.sh" + - "setup-python-env.sh" + - "tests/test_ocr_shell_startup.py" - "tests/test_pdf_metadata.py" - "tests/test_tag_existing_text.py" - ".github/workflows/pdf-regressions.yml" @@ -21,5 +25,7 @@ jobs: python-version: "3.12" - name: Install regression dependencies run: python -m pip install pikepdf pdfminer.six + - name: Run OCR shell startup regression + run: bash -n ocr-scanned-pdf.sh && bash -n compress-and-tag-ocr-pdf.sh && bash -n setup-python-env.sh && python -m unittest discover -s tests -p 'test_ocr_shell_startup.py' -v - name: Run metadata and tagged PDF tests run: python -m unittest discover -s tests -p 'test_pdf_metadata.py' -v && python -m unittest discover -s tests -p 'test_tag_existing_text.py' -v diff --git a/.gitignore b/.gitignore index 7a22182..f7bcc1f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__/ .google-books-browser/ *-pages/ +.venv/ diff --git a/OCR.md b/OCR.md index b15e77e..4cd75d9 100644 --- a/OCR.md +++ b/OCR.md @@ -3,6 +3,25 @@ These tools create reproducible searchable, structured, and losslessly compressed derivatives of scanned PDFs without replacing the source. +## Python environment + +Install the Enhancer's isolated Python dependencies once: + +```sh +./setup-python-env.sh +``` + +This creates `.venv/` beside the Enhancer scripts with `pikepdf` and Pillow. +The OCR and compression scripts prefer `.venv/bin/python` for tagging, local AI +review, and metadata helpers. Existing standalone installations without this +environment can still use Python with the required packages, including the +interpreter bundled with OCRmyPDF. An incomplete project venv produces an error +instead of silently using an unrelated global interpreter. + +The browser viewer's native-host installer also initializes this environment +when an Enhancer checkout supplies `setup-python-env.sh`. OCRmyPDF and qpdf +remain separate system executables, typically installed using Homebrew. + ## Create searchable OCR Run the OCR, paragraph grouping, table-block classification, and heading diff --git a/compress-and-tag-ocr-pdf.sh b/compress-and-tag-ocr-pdf.sh old mode 100755 new mode 100644 index a3343f3..3bf679d --- a/compress-and-tag-ocr-pdf.sh +++ b/compress-and-tag-ocr-pdf.sh @@ -53,23 +53,36 @@ for command_name in ocrmypdf qpdf; do fi done -tagger_python=python3 -if ! "$tagger_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then - ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)") - tagger_python=${ocrmypdf_shebang#\#!} -fi - -if [[ ! -x "$tagger_python" ]] || \ - ! "$tagger_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then - printf 'Unable to find the Python environment used by OCRmyPDF.\n' >&2 - exit 1 +venv_python="$script_dir/.venv/bin/python" +if [[ -e "$venv_python" || -L "$venv_python" ]]; then + if [[ ! -x "$venv_python" ]] || ! "$venv_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then + printf 'PDF Enhancer virtual environment needs repair: %s\n' "$venv_python" >&2 + printf 'Run %s/setup-python-env.sh to install its dependencies.\n' "$script_dir" >&2 + exit 1 + fi + tagger_python=("$venv_python") +else + tagger_python=(python3) + if ! "${tagger_python[@]}" -c 'import pikepdf, PIL' >/dev/null 2>&1; then + ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)") + if [[ "$ocrmypdf_shebang" == '#!'* ]]; then + read -r -a tagger_python <<< "${ocrmypdf_shebang:2}" + else + tagger_python=() + fi + fi + if (( ${#tagger_python[@]} == 0 )) || \ + ! "${tagger_python[@]}" -c 'import pikepdf, PIL' >/dev/null 2>&1; then + printf 'No Python with pikepdf/Pillow found; run %s/setup-python-env.sh.\n' "$script_dir" >&2 + exit 1 + fi fi work_dir=$(mktemp -d "${TMPDIR:-/tmp}/compress-tag-pdf.XXXXXX") trap 'rm -rf -- "$work_dir"' EXIT compressed="$work_dir/compressed.pdf" -"$tagger_python" "$script_dir/compress-pdf-images.py" \ +"${tagger_python[@]}" "$script_dir/compress-pdf-images.py" \ "$input" "$compressed" \ --report "$compression_report" \ --reported-output "$output" @@ -88,7 +101,7 @@ if [[ -n "$review" ]]; then fi tagger_args+=(--review "$review") fi -"$tagger_python" "$script_dir/tag-ocr-pdf.py" "${tagger_args[@]}" +"${tagger_python[@]}" "$script_dir/tag-ocr-pdf.py" "${tagger_args[@]}" qpdf --check "$output" input_pages=$(qpdf --show-npages "$input") diff --git a/ocr-scanned-pdf.sh b/ocr-scanned-pdf.sh old mode 100755 new mode 100644 index f95be3a..6935345 --- a/ocr-scanned-pdf.sh +++ b/ocr-scanned-pdf.sh @@ -109,18 +109,6 @@ if [[ ! -f "$tagger" ]]; then exit 1 fi -tagger_python=python3 -if ! "$tagger_python" -c 'import pikepdf' >/dev/null 2>&1; then - ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)") - tagger_python=${ocrmypdf_shebang#\#!} -fi - -if [[ ! -x "$tagger_python" ]] || \ - ! "$tagger_python" -c 'import pikepdf' >/dev/null 2>&1; then - printf 'Unable to find the Python environment used by OCRmyPDF.\n' >&2 - exit 1 -fi - for command_name in ocrmypdf qpdf; do if ! command -v "$command_name" >/dev/null 2>&1; then printf 'Required command not found: %s\n' "$command_name" >&2 @@ -129,6 +117,34 @@ for command_name in ocrmypdf qpdf; do fi done +# Prefer the Enhancer's own isolated Python dependencies when installed. +# Keep the prior interpreter discovery for existing standalone installations. +venv_python="$script_dir/.venv/bin/python" +if [[ -e "$venv_python" || -L "$venv_python" ]]; then + if [[ ! -x "$venv_python" ]] || ! "$venv_python" -c 'import pikepdf' >/dev/null 2>&1; then + printf 'PDF Enhancer virtual environment needs repair: %s\n' "$venv_python" >&2 + printf 'Run %s/setup-python-env.sh to install its dependencies.\n' "$script_dir" >&2 + exit 1 + fi + tagger_python=("$venv_python") +else + # With no project venv, support preexisting Python and OCRmyPDF environments. + tagger_python=(python3) + if ! "${tagger_python[@]}" -c 'import pikepdf' >/dev/null 2>&1; then + ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)") + if [[ "$ocrmypdf_shebang" == '#!'* ]]; then + read -r -a tagger_python <<< "${ocrmypdf_shebang:2}" + else + tagger_python=() + fi + fi + if (( ${#tagger_python[@]} == 0 )) || \ + ! "${tagger_python[@]}" -c 'import pikepdf' >/dev/null 2>&1; then + printf 'No Python with pikepdf found; run %s/setup-python-env.sh.\n' "$script_dir" >&2 + exit 1 + fi +fi + input_pages=$(qpdf --warning-exit-0 --show-npages "$input") printf 'PDF_MAGIC_PROGRESS ocr %s\n' "$input_pages" @@ -157,7 +173,7 @@ if "$ai_review"; then initial_report="$tagging_dir/initial.tagging.json" fi -"$tagger_python" "$tagger" \ +"${tagger_python[@]}" "$tagger" \ "$output" \ "$tagged_output" \ --report "$initial_report" \ @@ -166,11 +182,11 @@ fi if "$ai_review"; then printf 'PDF_MAGIC_PROGRESS review %s\n' "$input_pages" - "$tagger_python" "$script_dir/ollama-review-ocr.py" \ + "${tagger_python[@]}" "$script_dir/ollama-review-ocr.py" \ "$tagged_output" "$initial_report" "$review_report" reviewed_output="$tagging_dir/reviewed.pdf" printf 'PDF_MAGIC_PROGRESS finalizing\n' - "$tagger_python" "$tagger" \ + "${tagger_python[@]}" "$tagger" \ "$output" "$reviewed_output" \ --review "$review_report" \ --report "$tagging_report" \ @@ -181,7 +197,7 @@ fi # OCRmyPDF's sidecar omits skipped pages. Build it from the tagging report so # preserved embedded text remains searchable in the exported plain text too. -"$tagger_python" - "$tagging_report" "$sidecar" <<'PYTEXT' +"${tagger_python[@]}" - "$tagging_report" "$sidecar" <<'PYTEXT' import json from pathlib import Path import sys diff --git a/setup-python-env.sh b/setup-python-env.sh new file mode 100755 index 0000000..b0867f0 --- /dev/null +++ b/setup-python-env.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install only Enhancer's Python dependencies; Homebrew manages the OCR binaries. +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +venv_python="$script_dir/.venv/bin/python" +if [[ ! -x "$venv_python" ]]; then + bootstrap_python="${PDF_MAGIC_BOOTSTRAP_PYTHON:-python3}" + if ! command -v "$bootstrap_python" >/dev/null 2>&1; then + printf 'Python 3 is required to create the Enhancer virtual environment.\n' >&2 + exit 1 + fi + "$bootstrap_python" -m venv "$script_dir/.venv" +fi +if ! "$venv_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then + "$venv_python" -m pip install --disable-pip-version-check pikepdf Pillow +fi +"$venv_python" -c 'import pikepdf, PIL' +printf 'PDF Enhancer Python environment ready: %s\n' "$venv_python" diff --git a/tests/test_ocr_shell_startup.py b/tests/test_ocr_shell_startup.py new file mode 100644 index 0000000..10db270 --- /dev/null +++ b/tests/test_ocr_shell_startup.py @@ -0,0 +1,119 @@ +"""Regression tests for the shell OCR interpreter setup.""" + +import os +from pathlib import Path +import shlex +import subprocess +import sys +import tempfile +import unittest + + +SCRIPT = Path(__file__).resolve().parents[1] / "ocr-scanned-pdf.sh" + + +class OcrShellStartupTests(unittest.TestCase): + def test_python3_on_path_with_pikepdf_can_start_ocr(self): + """A bare executable name is valid even when ./python3 does not exist.""" + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + commands = directory / "bin" + commands.mkdir() + pdf = directory / "source.pdf" + pdf.write_bytes(b"%PDF-1.7\n") + python3 = commands / "python3" + python3.write_text( + "#!/bin/bash\n" + 'if [[ "${1:-}" == "-c" && "${2:-}" == "import pikepdf" ]]; then exit 0; fi\n' + f'exec {shlex.quote(sys.executable)} "$@"\n' + ) + python3.chmod(0o755) + qpdf = commands / "qpdf" + qpdf.write_text("#!/bin/bash\necho 1\n") + qpdf.chmod(0o755) + ocrmypdf = commands / "ocrmypdf" + ocrmypdf.write_text( + "#!/bin/bash\n" + 'echo "REACHED_OCR_COMMAND" >&2\n' + "exit 47\n" + ) + ocrmypdf.chmod(0o755) + environment = {**os.environ, "PATH": str(commands) + os.pathsep + os.environ["PATH"]} + result = subprocess.run( + ["/bin/bash", str(SCRIPT), str(pdf), str(directory / "output.pdf"), "--force-ocr"], + env=environment, capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 47, result.stderr) + self.assertIn("REACHED_OCR_COMMAND", result.stderr) + self.assertNotIn("Unable to find the Python environment", result.stderr) + + + def test_project_venv_wins_over_python_on_path(self): + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + script = directory / "ocr-scanned-pdf.sh" + script.write_bytes(SCRIPT.read_bytes()) + (directory / "tag-ocr-pdf.py").write_text("# placeholder\n") + venv = directory / ".venv" / "bin" + venv.mkdir(parents=True) + venv_python = venv / "python" + venv_python.write_text( + "#!/bin/bash\n" + 'if [[ "${1:-}" == "-c" ]]; then touch "$PDF_MAGIC_TEST_VENV_MARKER"; exit 0; fi\n' + "exit 83\n" + ) + venv_python.chmod(0o755) + commands = directory / "bin" + commands.mkdir() + for name, code in { + "qpdf": "#!/bin/bash\necho 1\n", + "ocrmypdf": '#!/bin/bash\necho "REACHED_OCR" >&2\nexit 47\n', + "python3": '#!/bin/bash\ntouch "$PDF_MAGIC_TEST_SYSTEM_MARKER"\nexit 0\n', + }.items(): + executable = commands / name + executable.write_text(code) + executable.chmod(0o755) + pdf = directory / "scan.pdf" + pdf.write_bytes(b"%PDF-1.7\n") + env = { + **os.environ, "PATH": str(commands) + os.pathsep + os.environ["PATH"], + "PDF_MAGIC_TEST_VENV_MARKER": str(directory / "venv-selected"), + "PDF_MAGIC_TEST_SYSTEM_MARKER": str(directory / "system-selected"), + } + result = subprocess.run( + ["/bin/bash", str(script), str(pdf), str(directory / "enhanced.pdf")], + env=env, capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 47, result.stderr) + self.assertTrue((directory / "venv-selected").exists()) + self.assertFalse((directory / "system-selected").exists()) + + def test_incomplete_project_venv_fails_without_falling_back_to_system_python(self): + with tempfile.TemporaryDirectory() as folder: + directory = Path(folder) + script = directory / "ocr-scanned-pdf.sh" + script.write_bytes(SCRIPT.read_bytes()) + (directory / "tag-ocr-pdf.py").write_text("# placeholder\n") + venv = directory / ".venv" / "bin" + venv.mkdir(parents=True) + executable = venv / "python" + executable.write_text("#!/bin/bash\nexit 1\n") + executable.chmod(0o755) + commands = directory / "bin" + commands.mkdir() + for name in ("qpdf", "ocrmypdf", "python3"): + shim = commands / name + shim.write_text("#!/bin/bash\nexit 0\n") + shim.chmod(0o755) + pdf = directory / "scan.pdf" + pdf.write_bytes(b"%PDF-1.7\n") + env = {**os.environ, "PATH": str(commands) + os.pathsep + os.environ["PATH"]} + result = subprocess.run( + ["/bin/bash", str(script), str(pdf)], + env=env, capture_output=True, text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("setup-python-env.sh", result.stderr) + +if __name__ == "__main__": + unittest.main()