🐛 Prefer Enhancer virtualenv and fix OCR interpreter selection

.github/workflows/pdf-regressions.yml

@@ -7,6 +7,10 @@ on:
     paths:
       - "pdf_metadata.py"
       - "tag-ocr-pdf.py"
+      - "ocr-scanned-pdf.sh"
+      - "compress-and-tag-ocr-pdf.sh"
+      - "setup-python-env.sh"
+      - "tests/test_ocr_shell_startup.py"
       - "tests/test_pdf_metadata.py"
       - "tests/test_tag_existing_text.py"
       - ".github/workflows/pdf-regressions.yml"
@@ -21,5 +25,7 @@ jobs:
           python-version: "3.12"
       - name: Install regression dependencies
         run: python -m pip install pikepdf pdfminer.six
+      - name: Run OCR shell startup regression
+        run: bash -n ocr-scanned-pdf.sh && bash -n compress-and-tag-ocr-pdf.sh && bash -n setup-python-env.sh && python -m unittest discover -s tests -p 'test_ocr_shell_startup.py' -v
       - name: Run metadata and tagged PDF tests
         run: python -m unittest discover -s tests -p 'test_pdf_metadata.py' -v && python -m unittest discover -s tests -p 'test_tag_existing_text.py' -v

.gitignore

@@ -2,3 +2,4 @@
 __pycache__/
 .google-books-browser/
 *-pages/
+.venv/

OCR.md

@@ -3,6 +3,25 @@
 These tools create reproducible searchable, structured, and losslessly
 compressed derivatives of scanned PDFs without replacing the source.
 
+## Python environment
+
+Install the Enhancer's isolated Python dependencies once:
+
+```sh
+./setup-python-env.sh
+```
+
+This creates `.venv/` beside the Enhancer scripts with `pikepdf` and Pillow.
+The OCR and compression scripts prefer `.venv/bin/python` for tagging, local AI
+review, and metadata helpers. Existing standalone installations without this
+environment can still use Python with the required packages, including the
+interpreter bundled with OCRmyPDF. An incomplete project venv produces an error
+instead of silently using an unrelated global interpreter.
+
+The browser viewer's native-host installer also initializes this environment
+when an Enhancer checkout supplies `setup-python-env.sh`. OCRmyPDF and qpdf
+remain separate system executables, typically installed using Homebrew.
+
 ## Create searchable OCR
 
 Run the OCR, paragraph grouping, table-block classification, and heading

compress-and-tag-ocr-pdf.sh

@@ -53,23 +53,36 @@ for command_name in ocrmypdf qpdf; do
     fi
 done
 
-tagger_python=python3
-if ! "$tagger_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then
-    ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)")
-    tagger_python=${ocrmypdf_shebang#\#!}
-fi
-
-if [[ ! -x "$tagger_python" ]] || \
-    ! "$tagger_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then
-    printf 'Unable to find the Python environment used by OCRmyPDF.\n' >&2
-    exit 1
+venv_python="$script_dir/.venv/bin/python"
+if [[ -e "$venv_python" || -L "$venv_python" ]]; then
+    if [[ ! -x "$venv_python" ]] || ! "$venv_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then
+        printf 'PDF Enhancer virtual environment needs repair: %s\n' "$venv_python" >&2
+        printf 'Run %s/setup-python-env.sh to install its dependencies.\n' "$script_dir" >&2
+        exit 1
+    fi
+    tagger_python=("$venv_python")
+else
+    tagger_python=(python3)
+    if ! "${tagger_python[@]}" -c 'import pikepdf, PIL' >/dev/null 2>&1; then
+        ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)")
+        if [[ "$ocrmypdf_shebang" == '#!'* ]]; then
+            read -r -a tagger_python <<< "${ocrmypdf_shebang:2}"
+        else
+            tagger_python=()
+        fi
+    fi
+    if (( ${#tagger_python[@]} == 0 )) || \
+        ! "${tagger_python[@]}" -c 'import pikepdf, PIL' >/dev/null 2>&1; then
+        printf 'No Python with pikepdf/Pillow found; run %s/setup-python-env.sh.\n' "$script_dir" >&2
+        exit 1
+    fi
 fi
 
 work_dir=$(mktemp -d "${TMPDIR:-/tmp}/compress-tag-pdf.XXXXXX")
 trap 'rm -rf -- "$work_dir"' EXIT
 compressed="$work_dir/compressed.pdf"
 
-"$tagger_python" "$script_dir/compress-pdf-images.py" \
+"${tagger_python[@]}" "$script_dir/compress-pdf-images.py" \
     "$input" "$compressed" \
     --report "$compression_report" \
     --reported-output "$output"
@@ -88,7 +101,7 @@ if [[ -n "$review" ]]; then
     fi
     tagger_args+=(--review "$review")
 fi
-"$tagger_python" "$script_dir/tag-ocr-pdf.py" "${tagger_args[@]}"
+"${tagger_python[@]}" "$script_dir/tag-ocr-pdf.py" "${tagger_args[@]}"
 
 qpdf --check "$output"
 input_pages=$(qpdf --show-npages "$input")

ocr-scanned-pdf.sh

@@ -109,18 +109,6 @@ if [[ ! -f "$tagger" ]]; then
     exit 1
 fi
 
-tagger_python=python3
-if ! "$tagger_python" -c 'import pikepdf' >/dev/null 2>&1; then
-    ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)")
-    tagger_python=${ocrmypdf_shebang#\#!}
-fi
-
-if [[ ! -x "$tagger_python" ]] || \
-    ! "$tagger_python" -c 'import pikepdf' >/dev/null 2>&1; then
-    printf 'Unable to find the Python environment used by OCRmyPDF.\n' >&2
-    exit 1
-fi
-
 for command_name in ocrmypdf qpdf; do
     if ! command -v "$command_name" >/dev/null 2>&1; then
         printf 'Required command not found: %s\n' "$command_name" >&2
@@ -129,6 +117,34 @@ for command_name in ocrmypdf qpdf; do
     fi
 done
 
+# Prefer the Enhancer's own isolated Python dependencies when installed.
+# Keep the prior interpreter discovery for existing standalone installations.
+venv_python="$script_dir/.venv/bin/python"
+if [[ -e "$venv_python" || -L "$venv_python" ]]; then
+    if [[ ! -x "$venv_python" ]] || ! "$venv_python" -c 'import pikepdf' >/dev/null 2>&1; then
+        printf 'PDF Enhancer virtual environment needs repair: %s\n' "$venv_python" >&2
+        printf 'Run %s/setup-python-env.sh to install its dependencies.\n' "$script_dir" >&2
+        exit 1
+    fi
+    tagger_python=("$venv_python")
+else
+    # With no project venv, support preexisting Python and OCRmyPDF environments.
+    tagger_python=(python3)
+    if ! "${tagger_python[@]}" -c 'import pikepdf' >/dev/null 2>&1; then
+        ocrmypdf_shebang=$(head -n 1 "$(command -v ocrmypdf)")
+        if [[ "$ocrmypdf_shebang" == '#!'* ]]; then
+            read -r -a tagger_python <<< "${ocrmypdf_shebang:2}"
+        else
+            tagger_python=()
+        fi
+    fi
+    if (( ${#tagger_python[@]} == 0 )) || \
+        ! "${tagger_python[@]}" -c 'import pikepdf' >/dev/null 2>&1; then
+        printf 'No Python with pikepdf found; run %s/setup-python-env.sh.\n' "$script_dir" >&2
+        exit 1
+    fi
+fi
+
 input_pages=$(qpdf --warning-exit-0 --show-npages "$input")
 printf 'PDF_MAGIC_PROGRESS ocr %s\n' "$input_pages"
 
@@ -157,7 +173,7 @@ if "$ai_review"; then
     initial_report="$tagging_dir/initial.tagging.json"
 fi
 
-"$tagger_python" "$tagger" \
+"${tagger_python[@]}" "$tagger" \
     "$output" \
     "$tagged_output" \
     --report "$initial_report" \
@@ -166,11 +182,11 @@ fi
 
 if "$ai_review"; then
     printf 'PDF_MAGIC_PROGRESS review %s\n' "$input_pages"
-    "$tagger_python" "$script_dir/ollama-review-ocr.py" \
+    "${tagger_python[@]}" "$script_dir/ollama-review-ocr.py" \
         "$tagged_output" "$initial_report" "$review_report"
     reviewed_output="$tagging_dir/reviewed.pdf"
     printf 'PDF_MAGIC_PROGRESS finalizing\n'
-    "$tagger_python" "$tagger" \
+    "${tagger_python[@]}" "$tagger" \
         "$output" "$reviewed_output" \
         --review "$review_report" \
         --report "$tagging_report" \
@@ -181,7 +197,7 @@ fi
 
 # OCRmyPDF's sidecar omits skipped pages. Build it from the tagging report so
 # preserved embedded text remains searchable in the exported plain text too.
-"$tagger_python" - "$tagging_report" "$sidecar" <<'PYTEXT'
+"${tagger_python[@]}" - "$tagging_report" "$sidecar" <<'PYTEXT'
 import json
 from pathlib import Path
 import sys

setup-python-env.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Install only Enhancer's Python dependencies; Homebrew manages the OCR binaries.
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+venv_python="$script_dir/.venv/bin/python"
+if [[ ! -x "$venv_python" ]]; then
+    bootstrap_python="${PDF_MAGIC_BOOTSTRAP_PYTHON:-python3}"
+    if ! command -v "$bootstrap_python" >/dev/null 2>&1; then
+        printf 'Python 3 is required to create the Enhancer virtual environment.\n' >&2
+        exit 1
+    fi
+    "$bootstrap_python" -m venv "$script_dir/.venv"
+fi
+if ! "$venv_python" -c 'import pikepdf, PIL' >/dev/null 2>&1; then
+    "$venv_python" -m pip install --disable-pip-version-check pikepdf Pillow
+fi
+"$venv_python" -c 'import pikepdf, PIL'
+printf 'PDF Enhancer Python environment ready: %s\n' "$venv_python"

tests/test_ocr_shell_startup.py

@@ -0,0 +1,119 @@
+"""Regression tests for the shell OCR interpreter setup."""
+
+import os
+from pathlib import Path
+import shlex
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+SCRIPT = Path(__file__).resolve().parents[1] / "ocr-scanned-pdf.sh"
+
+
+class OcrShellStartupTests(unittest.TestCase):
+    def test_python3_on_path_with_pikepdf_can_start_ocr(self):
+        """A bare executable name is valid even when ./python3 does not exist."""
+        with tempfile.TemporaryDirectory() as folder:
+            directory = Path(folder)
+            commands = directory / "bin"
+            commands.mkdir()
+            pdf = directory / "source.pdf"
+            pdf.write_bytes(b"%PDF-1.7\n")
+            python3 = commands / "python3"
+            python3.write_text(
+                "#!/bin/bash\n"
+                'if [[ "${1:-}" == "-c" && "${2:-}" == "import pikepdf" ]]; then exit 0; fi\n'
+                f'exec {shlex.quote(sys.executable)} "$@"\n'
+            )
+            python3.chmod(0o755)
+            qpdf = commands / "qpdf"
+            qpdf.write_text("#!/bin/bash\necho 1\n")
+            qpdf.chmod(0o755)
+            ocrmypdf = commands / "ocrmypdf"
+            ocrmypdf.write_text(
+                "#!/bin/bash\n"
+                'echo "REACHED_OCR_COMMAND" >&2\n'
+                "exit 47\n"
+            )
+            ocrmypdf.chmod(0o755)
+            environment = {**os.environ, "PATH": str(commands) + os.pathsep + os.environ["PATH"]}
+            result = subprocess.run(
+                ["/bin/bash", str(SCRIPT), str(pdf), str(directory / "output.pdf"), "--force-ocr"],
+                env=environment, capture_output=True, text=True,
+            )
+            self.assertEqual(result.returncode, 47, result.stderr)
+            self.assertIn("REACHED_OCR_COMMAND", result.stderr)
+            self.assertNotIn("Unable to find the Python environment", result.stderr)
+
+
+    def test_project_venv_wins_over_python_on_path(self):
+        with tempfile.TemporaryDirectory() as folder:
+            directory = Path(folder)
+            script = directory / "ocr-scanned-pdf.sh"
+            script.write_bytes(SCRIPT.read_bytes())
+            (directory / "tag-ocr-pdf.py").write_text("# placeholder\n")
+            venv = directory / ".venv" / "bin"
+            venv.mkdir(parents=True)
+            venv_python = venv / "python"
+            venv_python.write_text(
+                "#!/bin/bash\n"
+                'if [[ "${1:-}" == "-c" ]]; then touch "$PDF_MAGIC_TEST_VENV_MARKER"; exit 0; fi\n'
+                "exit 83\n"
+            )
+            venv_python.chmod(0o755)
+            commands = directory / "bin"
+            commands.mkdir()
+            for name, code in {
+                "qpdf": "#!/bin/bash\necho 1\n",
+                "ocrmypdf": '#!/bin/bash\necho "REACHED_OCR" >&2\nexit 47\n',
+                "python3": '#!/bin/bash\ntouch "$PDF_MAGIC_TEST_SYSTEM_MARKER"\nexit 0\n',
+            }.items():
+                executable = commands / name
+                executable.write_text(code)
+                executable.chmod(0o755)
+            pdf = directory / "scan.pdf"
+            pdf.write_bytes(b"%PDF-1.7\n")
+            env = {
+                **os.environ, "PATH": str(commands) + os.pathsep + os.environ["PATH"],
+                "PDF_MAGIC_TEST_VENV_MARKER": str(directory / "venv-selected"),
+                "PDF_MAGIC_TEST_SYSTEM_MARKER": str(directory / "system-selected"),
+            }
+            result = subprocess.run(
+                ["/bin/bash", str(script), str(pdf), str(directory / "enhanced.pdf")],
+                env=env, capture_output=True, text=True,
+            )
+            self.assertEqual(result.returncode, 47, result.stderr)
+            self.assertTrue((directory / "venv-selected").exists())
+            self.assertFalse((directory / "system-selected").exists())
+
+    def test_incomplete_project_venv_fails_without_falling_back_to_system_python(self):
+        with tempfile.TemporaryDirectory() as folder:
+            directory = Path(folder)
+            script = directory / "ocr-scanned-pdf.sh"
+            script.write_bytes(SCRIPT.read_bytes())
+            (directory / "tag-ocr-pdf.py").write_text("# placeholder\n")
+            venv = directory / ".venv" / "bin"
+            venv.mkdir(parents=True)
+            executable = venv / "python"
+            executable.write_text("#!/bin/bash\nexit 1\n")
+            executable.chmod(0o755)
+            commands = directory / "bin"
+            commands.mkdir()
+            for name in ("qpdf", "ocrmypdf", "python3"):
+                shim = commands / name
+                shim.write_text("#!/bin/bash\nexit 0\n")
+                shim.chmod(0o755)
+            pdf = directory / "scan.pdf"
+            pdf.write_bytes(b"%PDF-1.7\n")
+            env = {**os.environ, "PATH": str(commands) + os.pathsep + os.environ["PATH"]}
+            result = subprocess.run(
+                ["/bin/bash", str(script), str(pdf)],
+                env=env, capture_output=True, text=True,
+            )
+            self.assertNotEqual(result.returncode, 0)
+            self.assertIn("setup-python-env.sh", result.stderr)
+
+if __name__ == "__main__":
+    unittest.main()