diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 873b544..a401449 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: ๐Ÿ› Bug Report description: Report a bug or issue you've encountered title: "[BUG] " -labels: ["bug"] +labels: ["type:bug"] assignees: [] --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d930795..e96fdb2 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: โœจ Feature Request description: Suggest an idea for NullifyPDF title: "[FEATURE] " -labels: ["enhancement"] +labels: ["type:feature"] assignees: [] --- diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000..ddf030a --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,28 @@ +lang:python: + - changed-files: + - any-glob-to-any-file: "**/*.py" + +type:docs: + - changed-files: + - any-glob-to-any-file: + - "**/*.md" + - "docs/**" + +type:ci: + - changed-files: + - any-glob-to-any-file: + - ".github/workflows/**" + - ".github/dependabot.yml" + +scope:core: + - changed-files: + - any-glob-to-any-file: + - "NullifyPDF.py" + - "privacy_core.py" + - "build_local.py" + - "setup_env.py" + +scope:deps: + - changed-files: + - any-glob-to-any-file: + - "requirements.txt" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2be4fd9..7faf4f3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -27,8 +27,7 @@ Describe what testing you've done: - [ ] Tested on macOS - [ ] Tested on Linux - [ ] All tests pass locally: `pytest tests/ -v` -- [ ] Build succeeds locally: `python build_local.py --lite` -- [ ] Full OCR build checked when packaging/OCR changes: `python build_local.py --full` +- [ ] Build succeeds locally: `python build_local.py` **Manual Testing Steps:** 1. Load PDF: [describe what PDF you used] @@ -71,8 +70,7 @@ Any additional context that reviewers should know? Before marking as ready, please ensure: - [ ] Tests pass: `pytest tests/ -v` -- [ ] Build passes: `python build_local.py --lite` -- [ ] Full OCR build passes when packaging/OCR changes: `python build_local.py --full` +- [ ] Build passes: `python build_local.py` - [ ] Code is clean and follows style guide - [ ] All checkboxes above are complete - [ ] You've tested your changes locally diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml index f24e355..f5a4a8b 100644 --- a/.github/workflows/beta-release.yml +++ b/.github/workflows/beta-release.yml @@ -22,7 +22,6 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-22.04, macos-latest] - variant: [lite, full] steps: - name: Checkout Code @@ -58,20 +57,18 @@ jobs: echo "NULLIFYPDF_BETA_SUFFIX=${TAG#*-}" >> "$GITHUB_ENV" - name: Build - env: - NULLIFYPDF_BUILD_VARIANT: ${{ matrix.variant }} run: python build_local.py - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: beta-build-${{ runner.os }}-${{ matrix.variant }} + name: beta-build-${{ runner.os }} retention-days: 5 path: | - dist/NullifyPDF_v*_Windows_*.exe - dist/NullifyPDF_v*_macOS_*.zip - dist/NullifyPDF_v*_Ubuntu_*.deb - dist/NullifyPDF_v*_Fedora_*.rpm + dist/NullifyPDF_v*_Windows.exe + dist/NullifyPDF_v*_macOS.zip + dist/NullifyPDF_v*_Ubuntu.deb + dist/NullifyPDF_v*_Fedora.rpm create_beta_release: name: Create GitHub Beta Release @@ -86,15 +83,49 @@ jobs: pattern: beta-build-* merge-multiple: true - - name: Release - uses: softprops/action-gh-release@v3 - with: - prerelease: true - files: | - all-builds/NullifyPDF_v*_Windows_*.exe - all-builds/NullifyPDF_v*_macOS_*.zip - all-builds/NullifyPDF_v*_Ubuntu_*.deb - all-builds/NullifyPDF_v*_Fedora_*.rpm - body: "NullifyPDF beta release (build from develop). Includes Lite and Full OCR variants. Do not use in production. If validation succeeds, this build will be promoted to stable without rebuilding." + - name: Create release and upload assets sequentially + shell: bash env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TAG="${GITHUB_REF#refs/tags/}" + REPO="${{ github.repository }}" + NOTES="NullifyPDF beta release (build from develop). Do not use in production. If validation succeeds, this build will be promoted to stable without rebuilding." + + if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then + echo "Release $TAG already exists, uploading/replacing assets." + else + gh release create "$TAG" --repo "$REPO" --title "$TAG" --prerelease --notes "$NOTES" + fi + + # Uploaded one at a time (not via softprops/action-gh-release's + # concurrent batch) because parallel uploads of these large + # binaries (100-300MB each) were timing out with an opaque + # "Error saving asset" and no per-file retry. + shopt -s nullglob + files=( + all-builds/NullifyPDF_v*_Windows.exe + all-builds/NullifyPDF_v*_macOS.zip + all-builds/NullifyPDF_v*_Ubuntu.deb + all-builds/NullifyPDF_v*_Fedora.rpm + ) + + if [ ${#files[@]} -eq 0 ]; then + echo "No release assets found to upload" >&2 + exit 1 + fi + + for f in "${files[@]}"; do + echo "Uploading $f ..." + attempt=1 + until gh release upload "$TAG" "$f" --repo "$REPO" --clobber; do + if [ "$attempt" -ge 3 ]; then + echo "Failed to upload $f after $attempt attempts" >&2 + exit 1 + fi + attempt=$((attempt + 1)) + echo "Retrying upload of $f (attempt $attempt)..." + sleep 10 + done + done diff --git a/.github/workflows/guard-main-source.yml b/.github/workflows/guard-main-source.yml new file mode 100644 index 0000000..baa8353 --- /dev/null +++ b/.github/workflows/guard-main-source.yml @@ -0,0 +1,33 @@ +name: Guard Main Source Branch + +on: + pull_request: + types: [opened, reopened, synchronize, edited] + branches: + - main + +permissions: + contents: read + +jobs: + check-source-branch: + name: Only allow PRs into main from develop + runs-on: ubuntu-latest + steps: + - name: Verify head branch is develop + run: | + HEAD_REPO="${{ github.event.pull_request.head.repo.full_name }}" + BASE_REPO="${{ github.repository }}" + HEAD_REF="${{ github.event.pull_request.head.ref }}" + + if [ "$HEAD_REPO" != "$BASE_REPO" ]; then + echo "::error::PRs into main must come from this repository's develop branch, not a fork ($HEAD_REPO)." + exit 1 + fi + + if [ "$HEAD_REF" != "develop" ]; then + echo "::error::PRs into main are only allowed from the develop branch (got '$HEAD_REF'). Merge into develop first, then open the release PR from develop." + exit 1 + fi + + echo "[OK] PR into main originates from develop." diff --git a/.github/workflows/project-automation.yml b/.github/workflows/project-automation.yml index 0534276..88afcf6 100644 --- a/.github/workflows/project-automation.yml +++ b/.github/workflows/project-automation.yml @@ -4,7 +4,7 @@ on: issues: types: [opened, reopened] pull_request: - types: [opened, closed, reopened] + types: [opened, closed, reopened, synchronize] permissions: issues: write @@ -22,6 +22,8 @@ jobs: steps: - name: "Assign and Add to Project" + env: + PR_TITLE: ${{ github.event.pull_request.title }} run: | REPO="${{ github.repository }}" TOKEN="${{ secrets.GH_PROJECT_TOKEN }}" @@ -82,6 +84,28 @@ jobs: "https://api.github.com/repos/$REPO/issues/$PR_NUM" > /dev/null echo " Assigned" + # Etichetta in base al prefisso conventional-commit del titolo + TYPE_LABEL="" + case "$PR_TITLE" in + feat:*|feat\(*) TYPE_LABEL="type:feature" ;; + fix:*|fix\(*) TYPE_LABEL="type:bug" ;; + docs:*|docs\(*) TYPE_LABEL="type:docs" ;; + chore:*|chore\(*) TYPE_LABEL="type:chore" ;; + ci:*|ci\(*) TYPE_LABEL="type:ci" ;; + refactor:*|refactor\(*) TYPE_LABEL="type:chore" ;; + perf:*|perf\(*) TYPE_LABEL="type:chore" ;; + test:*|test\(*) TYPE_LABEL="type:chore" ;; + esac + if [ -n "$TYPE_LABEL" ]; then + echo " - Applying label $TYPE_LABEL..." + curl -s -X POST \ + -H "Authorization: token $TOKEN" \ + -H "Accept: application/vnd.github+json" \ + -d "{\"labels\":[\"$TYPE_LABEL\"]}" \ + "https://api.github.com/repos/$REPO/issues/$PR_NUM/labels" > /dev/null + echo " Labeled" + fi + # Aggiungi al Project (solo se opened o reopened) if [ "$PR_ACTION" = "opened" ] || [ "$PR_ACTION" = "reopened" ]; then echo " - Adding to Project..." @@ -125,6 +149,14 @@ jobs: fi fi + - name: "Label by Changed Files" + if: github.event_name == 'pull_request' + uses: actions/labeler@v7 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/labeler.yml + sync-labels: false + - name: "Workflow Summary" if: always() run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a737452..4342a16 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,28 @@ jobs: name: Promote latest beta artifacts to stable runs-on: ubuntu-latest steps: + - name: Checkout Code + uses: actions/checkout@v7 + + - name: Verify CHANGELOG has an entry for this version + if: startsWith(github.ref, 'refs/tags/') + run: | + TAG="${GITHUB_REF#refs/tags/}" + VERSION="${TAG#v}" + + if ! grep -qE "^## \[$VERSION\] - [0-9]{4}-[0-9]{2}-[0-9]{2}" CHANGELOG.md; then + echo "::error::CHANGELOG.md has no '## [$VERSION] - YYYY-MM-DD' heading. On develop, move the [Unreleased] content under this heading before tagging the stable release." + exit 1 + fi + + UNRELEASED_BODY=$(awk '/^## \[Unreleased\]/{flag=1;next}/^## \[/{flag=0}flag' CHANGELOG.md | sed '/^[[:space:]]*$/d') + if [ -n "$UNRELEASED_BODY" ]; then + echo "::error::CHANGELOG.md still has content under [Unreleased] that hasn't been moved to [$VERSION]. Update it on develop first." + exit 1 + fi + + echo "[OK] CHANGELOG.md has a clean [$VERSION] entry and an empty [Unreleased] section." + - name: Resolve target beta release id: beta env: @@ -50,31 +72,71 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p stable-assets + # Patterns are deliberately tolerant of a trailing "_Full"/"_Lite" + # suffix (no underscore pinned before the extension) so a beta + # built before the lite/full split was removed can still be + # promoted to stable. gh release download "${{ steps.beta.outputs.tag }}" --repo "${{ github.repository }}" --dir stable-assets \ - --pattern "NullifyPDF_v*_Windows_*.exe" \ - --pattern "NullifyPDF_v*_macOS_*.zip" \ - --pattern "NullifyPDF_v*_Ubuntu_*.deb" \ - --pattern "NullifyPDF_v*_Fedora_*.rpm" + --pattern "NullifyPDF_v*_Windows*.exe" \ + --pattern "NullifyPDF_v*_macOS*.zip" \ + --pattern "NullifyPDF_v*_Ubuntu*.deb" \ + --pattern "NullifyPDF_v*_Fedora*.rpm" - name: Strip beta suffix from filenames run: | cd stable-assets + # Drop any leftover "_Lite" assets from betas built before the + # lite/full split was removed: only the full variant is promoted. + rm -f *_Lite.* *_Lite for f in *; do - new="$(echo "$f" | sed -E 's/-beta\.[0-9]+//')" + # Also drops the now-redundant "_Full" variant tag. + new="$(echo "$f" | sed -E 's/-beta\.[0-9]+//; s/_Full//')" if [ "$f" != "$new" ]; then mv "$f" "$new" fi done ls -la - - name: Create Stable Release - uses: softprops/action-gh-release@v3 - with: - files: stable-assets/* - prerelease: false - body: | - NullifyPDF stable release. - Promoted from beta `${{ steps.beta.outputs.tag }}` (already verified build, no rebuild). - Includes Lite and Full OCR variants for Windows, macOS, and Linux. + - name: Create stable release and upload assets sequentially + shell: bash env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TAG="${GITHUB_REF#refs/tags/}" + REPO="${{ github.repository }}" + NOTES="NullifyPDF stable release. + Promoted from beta \`${{ steps.beta.outputs.tag }}\` (already verified build, no rebuild). + Available for Windows, macOS, and Linux." + + if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then + echo "Release $TAG already exists, uploading/replacing assets." + else + gh release create "$TAG" --repo "$REPO" --title "$TAG" --notes "$NOTES" + fi + + # Uploaded one at a time, not via softprops/action-gh-release's + # concurrent batch: parallel uploads of these large binaries + # (100-300MB each) were timing out with an opaque "Error saving + # asset" and no per-file retry (see beta-release.yml). + shopt -s nullglob + files=(stable-assets/*) + + if [ ${#files[@]} -eq 0 ]; then + echo "No release assets found to upload" >&2 + exit 1 + fi + + for f in "${files[@]}"; do + echo "Uploading $f ..." + attempt=1 + until gh release upload "$TAG" "$f" --repo "$REPO" --clobber; do + if [ "$attempt" -ge 3 ]; then + echo "Failed to upload $f after $attempt attempts" >&2 + exit 1 + fi + attempt=$((attempt + 1)) + echo "Retrying upload of $f (attempt $attempt)..." + sleep 10 + done + done diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index cf7995b..cb0aed2 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -22,7 +22,6 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-22.04, macos-latest] - variant: [lite, full] steps: - name: Checkout Code @@ -51,17 +50,15 @@ jobs: run: mypy NullifyPDF.py --ignore-missing-imports --no-error-summary 2>&1 | head -50 || true - name: Build - env: - NULLIFYPDF_BUILD_VARIANT: ${{ matrix.variant }} run: python build_local.py - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: test-build-${{ runner.os }}-${{ matrix.variant }} + name: test-build-${{ runner.os }} path: | - dist/NullifyPDF_v*_Windows_*.exe - dist/NullifyPDF_v*_macOS_*.zip - dist/NullifyPDF_v*_Ubuntu_*.deb - dist/NullifyPDF_v*_Fedora_*.rpm + dist/NullifyPDF_v*_Windows.exe + dist/NullifyPDF_v*_macOS.zip + dist/NullifyPDF_v*_Ubuntu.deb + dist/NullifyPDF_v*_Fedora.rpm retention-days: 3 diff --git a/.gitignore b/.gitignore index 7b0cf14..419d75d 100644 --- a/.gitignore +++ b/.gitignore @@ -74,7 +74,7 @@ desktop.ini *.nullifypdf.tmp *.nullifypdf-map -# OCR data is downloaded for Full builds, not stored in git. +# OCR data is downloaded at build time, not stored in git. ocr/tessdata/*.traineddata # Packaging temp directories diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 81b74be..0bb4b33 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # ๐Ÿ—๏ธ Architecture โ€” NullifyPDF -This document explains how NullifyPDF is structured today: application layers, privacy export flow, OCR support, release variants, and the main security boundaries that matter during maintenance. +This document explains how NullifyPDF is structured today: application layers, privacy export flow, OCR support, release packaging, and the main security boundaries that matter during maintenance. > [!TIP] > Read this after the [README](./README.md) if you want the technical view of how detection, redaction, pseudonymization, and packaging work together. @@ -14,7 +14,7 @@ NullifyPDF is a local-first desktop application for preparing PDFs before they a It supports two privacy export modes: - **Irreversible anonymization** โ€” selected content is redacted and removed from the exported PDF -- **Reversible pseudonymization** โ€” selected content is replaced with stable placeholders, while original values are stored in a separate encrypted restore map +- **Reversible pseudonymization** โ€” selected content is replaced with stable placeholders, while original values are stored in a separate encrypted restore map. The same restore map can later be used to reconstruct the original values back into a pseudonymized PDF. > [!NOTE] > Reversible mode is pseudonymization, not true anonymization, because the encrypted restore map can re-identify the original values. @@ -25,12 +25,12 @@ It supports two privacy export modes: | ๐Ÿ“„ File | โœจ Responsibility | | ------ | ----------------- | -| `NullifyPDF.py` | Main PySide6 UI, PDF rendering, AI scan orchestration, redaction, export, OCR integration | -| `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads | +| `NullifyPDF.py` | Main PySide6 UI, PDF rendering, AI scan orchestration, redaction, export, reconstruction, OCR integration | +| `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads, restore-map parsing | | `PDF_Checker.py` | Heuristic post-export inspection helper | -| `build_local.py` | PyInstaller build script with Lite/Full release variants | -| `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files for Full builds | -| `tests/` | Unit and smoke tests for validation, privacy primitives, OCR config, and build variants | +| `build_local.py` | PyInstaller build script | +| `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files bundled into every build | +| `tests/` | Unit and smoke tests for validation, privacy primitives, OCR config, and build configuration | --- @@ -212,7 +212,7 @@ Runtime behavior: 4. OCR text is used for Presidio detection. 5. OCR search rectangles are used to place redactions on scanned pages. -The Full release bundles EN/IT OCR data. The Lite release requires local Tesseract `tessdata` when OCR is enabled. +Every release bundles EN/IT OCR data. --- @@ -232,22 +232,16 @@ This separation prevents UI freezes and reduces concurrent document-mutation ris --- -## ๐Ÿ“ฆ Build Variants +## ๐Ÿ“ฆ Build -### ๐Ÿชถ Lite - -Lite builds do not bundle OCR language data. They are smaller and still support OCR if the user has Tesseract `tessdata` configured locally. - -### ๐Ÿงฐ Full - -Full builds bundle: +Every build bundles OCR language data: ```text ocr/tessdata/eng.traineddata ocr/tessdata/ita.traineddata ``` -If the files are missing, `build_local.py --full` automatically downloads them from `tesseract-ocr/tessdata_fast` through `scripts/download_ocr_data.py`. +If the files are missing, `build_local.py` automatically downloads them from `tesseract-ocr/tessdata_fast` through `scripts/download_ocr_data.py`. The `.traineddata` files are ignored by git and are not stored in the repository. @@ -255,26 +249,22 @@ The `.traineddata` files are ignored by git and are not stored in the repository ## ๐Ÿš€ CI/CD -GitHub Actions build both variants on each supported OS: +GitHub Actions build one artifact per supported OS: -- Windows Lite / Full -- macOS Lite / Full -- Ubuntu Lite / Full -- Fedora Lite / Full +- Windows +- macOS +- Ubuntu +- Fedora Beta releases build and upload all artifacts. Stable releases promote the latest verified beta artifacts without recompilation. Expected artifact names include: ```text -NullifyPDF_vX.Y.Z_Windows_Lite.exe -NullifyPDF_vX.Y.Z_Windows_Full.exe -NullifyPDF_vX.Y.Z_macOS_Lite.zip -NullifyPDF_vX.Y.Z_macOS_Full.zip -NullifyPDF_vX.Y.Z_Ubuntu_Lite.deb -NullifyPDF_vX.Y.Z_Ubuntu_Full.deb -NullifyPDF_vX.Y.Z_Fedora_Lite.rpm -NullifyPDF_vX.Y.Z_Fedora_Full.rpm +NullifyPDF_vX.Y.Z_Windows.exe +NullifyPDF_vX.Y.Z_macOS.zip +NullifyPDF_vX.Y.Z_Ubuntu.deb +NullifyPDF_vX.Y.Z_Fedora.rpm ``` --- @@ -288,8 +278,9 @@ Current tests cover: - OCR configuration helpers - privacy placeholder registry - encrypted restore-map round trip -- build variant selection -- Full build OCR data inclusion behavior +- reconstruction of a pseudonymized PDF from its restore map +- build configuration +- OCR data inclusion behavior Run: @@ -313,5 +304,5 @@ pytest tests/ -v --- -*Last updated: 2026-07-23* +*Last updated: 2026-09-18* *[Back to README](./README.md) | [Development โ†’](./DEVELOPMENT.md)* diff --git a/CHANGELOG.md b/CHANGELOG.md index b710897..41db79d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.2.0] - 2026-09-18 + +Validated through the 2.2.0-beta.1/beta.2/beta.3/beta.4 prereleases before this stable release, promoted from `v2.2.0-beta.4` without rebuilding; only the final stable version gets its own heading here. + +### โœจ Added + +- **Reconstruct pseudonymized PDFs**: New "Ricostruisci PDF" action reopens a previously pseudonymized PDF together with its encrypted `.nullifypdf-map` and password, and writes out a new PDF with the original values restored in place of the placeholders. The reconstructed PDF is refused if its SHA-256 hash does not match the `output_sha256` recorded in the restore map, to prevent applying a mapping to the wrong document. Because a placeholder's on-page box is usually smaller than the original text, restored text may be visually clipped or overlap โ€” this restores the original *content*, not the original layout. +- **Confirmation prompts for blocklist/allowlist changes**: manually drawing a redaction over recognized text now asks whether to add it to the permanent blocklist (redacted in every future document) instead of silently persisting it. Removing an existing redaction now offers a 3-way choice: exclude it only from this document, add it to the permanent whitelist (never redacted again), or cancel. +- **Italian-specific PII recognition**: the AI engine now also detects `IT_FISCAL_CODE` (Codice Fiscale), `IT_VAT_CODE` (Partita IVA), `IT_DRIVER_LICENSE`, `IT_IDENTITY_CARD`, and `IT_PASSPORT` โ€” all built into the pinned `presidio-analyzer` version but previously excluded from the requested entity list. +- **Italian phone number formats**: Presidio's default phone-number recognizer only validates against `US/UK/DE/FR/IL/IN/CA/BR` regional formats, so Italian mobile/landline numbers without an explicit `+39` prefix (e.g. `340 1234567`, `02 1234567`) were never recognized. Added `IT` to the supported regions. +- **Visible scanned-PDF warning**: opening a PDF with no extractable text (typical of scanned documents) now shows a popup confirming this and reminding to enable OCR before AI scanning, in addition to the existing log message. Manual selection already works on scanned pages regardless (see the crash fix below). + +### ๐Ÿ› Fixed + +- **Reconstruction silently restored nothing on scanned PDFs**: "Ricostruisci PDF" located placeholders by searching the exported PDF for their text (`search_for("DATA_001")`). When a redaction box is narrow and tall โ€” the usual shape when drawing over a short field on a scan โ€” `apply_redactions()` wraps the placeholder across lines and breaks it *mid-token* (`DATA_` + `001`). The page still reads `DATA_001`, but the string is no longer findable, so reconstruction matched nothing, reported success and wrote an output identical to its input, with no error shown anywhere. Restore maps are now version 2 and record each placeholder's page and redaction box, so reconstruction addresses those boxes directly instead of matching rendered text. Version 1 maps are still accepted and fall back to text search, now backed by a character-level scan (spanning text-block boundaries, not just line boundaries within one block โ€” PyMuPDF 1.28.2 can wrap a placeholder across separate blocks) that also finds wrapped placeholders. The export side additionally picks a font size that fits the box, so placeholders are no longer broken mid-token in the first place. +- **Reconstruction could silently destroy the value it was restoring**: `apply_redactions()` drops its replacement text entirely when the text cannot be laid out inside the box, which happens whenever an original value is longer than the placeholder's tight bounding box and cannot be wrapped โ€” precisely the case for space-free high-sensitivity values such as fiscal codes, IBANs, card and document numbers. The old count was incremented when a redaction was *requested*, so the app reported those values as restored while writing nothing. The restored count is now read back off the saved file, values are placed into their recorded original box at a font size chosen to fit, and an incomplete reconstruction raises a warning dialog instead of looking like a success. +- **Reconstruction missed repeated placeholder occurrences**: `PlaceholderRegistry` reuses the same placeholder (e.g. `PERSON_001`) for a repeated value but only records the page of its *first* occurrence. "Ricostruisci PDF" trusted that recorded page, so a later occurrence of the same value on another page (or elsewhere on the same page) was never found and stayed unreplaced. Reconstruction now addresses the redaction box recorded for each occurrence directly. +- **Manual redaction selection was silently broken**: drawing a rectangle on the PDF view to manually mark text/areas for redaction raised an uncaught `TypeError` on every mouse press (`QGraphicsView.mapToScene()` does not accept the `QPointF` that `event.position()` returns in this PySide6 version), so only AI-assisted detection actually redacted anything. Fixed by converting to `QPoint` before mapping. +- **Exporting a redacted scanned PDF produced a blank page**: `apply_redactions()` was called with `images=PDF_REDACT_IMAGE_REMOVE`, which deletes an image object *entirely* if it intersects *any* redaction rectangle โ€” on a scanned document (the whole page is one image), redacting even a single word wiped out the entire page, leaving only the black bar/placeholder visible on an otherwise blank sheet. Switched to `PDF_REDACT_IMAGE_PIXELS`, which blanks only the pixels under each redaction rectangle and leaves the rest of the image intact (verified this also still fully blanks whole-image redactions from "Oscura Immagini", including under page rotation, with no visible fringe). + +### ๐Ÿ”ง Changed + +- **BREAKING: Removed the Lite/Full build split**: every build and release artifact now bundles the EN/IT OCR data unconditionally (previously only "Full" builds did). The `--lite`/`--full` flags and `NULLIFYPDF_BUILD_VARIANT` environment variable are gone from `build_local.py`. Release artifact filenames no longer carry a `_Lite`/`_Full` suffix (e.g. `NullifyPDF_vX.Y.Z_Windows.exe` instead of `NullifyPDF_vX.Y.Z_Windows_Full.exe`). Users who were downloading the Lite asset should switch to the single artifact for their OS. +- Renamed the **"Esporta Privacy"** button to **"Esporta PDF"** (it exports the redacted PDF regardless of the chosen privacy mode, so the old label was misleading). + +### ๐Ÿ”’ Security + +- **cryptography 50.0.0**: Updated from 49.0.0, resolving [CVE-2026-69247](https://github.com/pyca/cryptography/security/advisories/GHSA-g6cj-pr64-35w5) (high severity), a Bleichenbacher-style timing/error oracle in PKCS7 decryption. Not directly exercised by `privacy_core.py` (which only uses `Fernet`/`hashes`/`PBKDF2HMAC`), but updated as a precaution since it closes an open Dependabot security alert. Followed up by a 50.0.1 patch that only rebuilds the wheels against OpenSSL 4.0.2 (no further security-relevant change). + +### ๐Ÿ”ง Internals + +- **PySide6 6.11.2**, **PyMuPDF 1.28.2** (still ships `cp310-abi3` wheels, so the Python 3.13 compatibility story is unaffected), **spaCy 3.8.16**, **pyinstaller 6.22.3**: routine minor/patch dependency updates. +- **actions/labeler v5 -> v7**: CI-only bump; the `changed-files`/`any-glob-to-any-file` config schema in `.github/labeler.yml` is unchanged across these versions. +- Dropped a stale `requirements.txt` comment on the `spacy` line that no longer matched the pinned version. +- **mypy cleanup (18 -> 0 errors)**: Scoped all flattened PySide6/Qt6 enum aliases used in `NullifyPDF.py` (e.g. `Qt.LeftButton` -> `Qt.MouseButton.LeftButton`, `Qt.AlignCenter` -> `Qt.AlignmentFlag.AlignCenter`, `QImage.Format_RGB888` -> `QImage.Format.Format_RGB888`) โ€” no behavior change, PySide6 accepts both forms at runtime. Renamed `NullifyPDF.render()` to `render_page()` to remove an accidental signature clash with `QWidget.render()` (all 7 call sites were internal, no external references). Used `getattr(sys, "_MEIPASS")` for the PyInstaller-injected attribute instead of direct access. Added a `DetectionDict` `TypedDict` for the AI-detection dict built during OCR-assisted scanning, replacing an ambiguous untyped dict literal. + ## [2.1.0] - 2026-07-29 Focused on advanced privacy export, full OCR support for scanned PDFs, and new Lite/Full release variants. Validated through the 2.1.0-beta.1/beta.2/beta.3 prereleases before this stable release, promoted from `v2.1.0-beta.3` without rebuilding; only the final stable version gets its own heading here. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96f2888..2decd7e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ source .venv/bin/activate # or .venv\Scripts\Activate.ps1 on Windows pytest tests/ -v # Build to verify compilation -python build_local.py --lite +python build_local.py ``` ### 5๏ธโƒฃ Commit with Clear Messages @@ -89,7 +89,7 @@ Then open Pull Request on GitHub with description from template. โŒ **Major Architecture Changes** โ€” Discuss first via issue โŒ **New Heavy Dependencies** โ€” Discuss first; keep the app local and maintainable -โœ… **OCR Improvements** โ€” In scope when they stay local/offline and preserve the Lite/Full build split +โœ… **OCR Improvements** โ€” In scope when they stay local/offline โŒ **Cloud Features** โ€” Must remain 100% local > [!TIP] @@ -219,7 +219,7 @@ thread when scanning large PDFs concurrently." ### Before Opening PR 1. โœ… Run tests: `pytest tests/ -v` -2. โœ… Build locally: `python build_local.py --lite` and, when OCR packaging changes, `python build_local.py --full` +2. โœ… Build locally: `python build_local.py` 3. โœ… Update CHANGELOG.md with your changes 4. โœ… Self-review your code (would you understand this in 6 months?) 5. โœ… Check links in documentation @@ -378,5 +378,5 @@ Every contribution โ€” code, docs, bug reports, ideas โ€” helps make NullifyPDF --- -*Last updated: 2026-07-23* +*Last updated: 2026-09-18* *โ† [Back to README](./README.md) | [Architecture โ†’](./ARCHITECTURE.md)* diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f3271e6..0436a62 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -108,13 +108,14 @@ NullifyPDF/ |-- privacy_core.py # Privacy modes and encrypted restore maps |-- PDF_Checker.py # Heuristic verification utility |-- setup_env.py # Environment setup script -|-- build_local.py # Lite/Full PyInstaller build script +|-- build_local.py # PyInstaller build script |-- requirements.txt # Python dependencies |-- scripts/ -| `-- download_ocr_data.py # EN/IT OCR data downloader for Full builds +| `-- download_ocr_data.py # EN/IT OCR data downloader |-- tests/ | |-- test_validation.py | |-- test_privacy_core.py +| |-- test_reconstruct.py | `-- test_build_config.py |-- images/ | `-- NullifyPDF.png @@ -133,11 +134,11 @@ NullifyPDF/ | Package | Version | Purpose | | --------------------- | ------- | ---------------------------- | -| **PySide6** | 6.11.1 | GUI framework (Qt6 bindings) | -| **PyMuPDF** | 1.28.0 | PDF manipulation and OCR bridge | +| **PySide6** | 6.11.2 | GUI framework (Qt6 bindings) | +| **PyMuPDF** | 1.28.2 | PDF manipulation and OCR bridge | | **presidio-analyzer** | 2.2.364 | PII detection | -| **spaCy** | 3.8.14 | NLP for entity recognition | -| **cryptography** | 49.0.0 | Encrypted restore maps | +| **spaCy** | 3.8.16 | NLP for entity recognition | +| **cryptography** | 50.0.1 | Encrypted restore maps | | **pytest** | 9.1.1 | Testing framework | ### Language Models (Auto-Downloaded) @@ -179,6 +180,7 @@ class NullifyPDF: def cmd_auto_ai(self) # Start AI/OCR scan def apply_ai_to_page(self, i, data) # Receive AI results def cmd_export(self) # Export privacy PDF + def cmd_reconstruct(self) # Reconstruct pseudonymized PDF from restore map def user_draw_rect(self, rect) # Draw manual redaction ``` @@ -226,7 +228,7 @@ Opens `htmlcov/index.html` in browser. - โœ… **PDFListManager** โ€” File I/O, persistence - โœ… **OCR Config** โ€” Tesseract language selection and tessdata discovery - โœ… **Privacy Core** โ€” Placeholder mapping and encrypted restore maps -- โœ… **Build Config** โ€” Lite/Full build variant behavior +- โœ… **Build Config** โ€” OCR data bundling behavior - โœ… **Resource Paths** โ€” PyInstaller compatibility --- @@ -236,27 +238,27 @@ Opens `htmlcov/index.html` in browser. ### Quick Build ```bash -python build_local.py --lite -python build_local.py --full +python build_local.py ``` -**Output:** `dist/NullifyPDF_vX.Y.Z_Windows_Lite.exe` or `dist/NullifyPDF_vX.Y.Z_Windows_Full.exe` (on Windows) +**Output:** `dist/NullifyPDF_vX.Y.Z_Windows.exe` (on Windows) ### What It Does 1. Cleans `build/` and `dist/` directories 2. Detects your OS (Windows/macOS/Linux) 3. Reads version from `NullifyPDF.py` (`__version__`) -4. Compiles with PyInstaller -5. Renames with version and variant: `NullifyPDF_v{VERSION}_{OS}_{Lite|Full}.exe` +4. Bundles EN/IT OCR data (downloaded automatically if missing) +5. Compiles with PyInstaller +6. Renames with version: `NullifyPDF_v{VERSION}_{OS}.exe` ### Distribution Artifacts | OS | Output | | ----------- | --------------------------------- | -| **Windows** | Lite/Full `.exe` executables | -| **macOS** | Lite/Full `.app` bundle ZIPs | -| **Linux** | Lite/Full binary + `.deb` + `.rpm` packages | +| **Windows** | `.exe` executable | +| **macOS** | `.app` bundle ZIP | +| **Linux** | Portable binary + `.deb` + `.rpm` packages | ### Troubleshooting Build Issues @@ -279,7 +281,7 @@ sys.setrecursionlimit(5000) Build succeeds but executable won't run 1. Check antivirus isn't blocking -2. Run in debug mode: `NullifyPDF_vX.Y.Z_Windows_Lite.exe` or `NullifyPDF_vX.Y.Z_Windows_Full.exe` from PowerShell +2. Run in debug mode: `NullifyPDF_vX.Y.Z_Windows.exe` from PowerShell 3. Check `.stdout` file if created 4. Report on GitHub @@ -305,7 +307,7 @@ git checkout -b feature/my-feature # 4. Test pytest tests/ -v -python build_local.py --lite +python build_local.py # 5. Commit with clear message git commit -m "feat(ai): add IBAN detection" @@ -437,7 +439,7 @@ isort NullifyPDF.py | `NullifyPDF.py` | Main app, GUI, OCR, export logic | | `privacy_core.py` | Placeholder and restore-map logic | | `setup_env.py` | Environment setup | -| `build_local.py` | PyInstaller Lite/Full build | +| `build_local.py` | PyInstaller build | | `PDF_Checker.py` | Post-processing utility | | `requirements.txt` | Dependencies | | `tests/` | Unit and smoke tests | @@ -450,6 +452,7 @@ isort NullifyPDF.py | Auto Redact | `NullifyPDF.py` | `cmd_auto_ai()` | | AI Processing | `NullifyPDF.py` | `AIWorker.run_scan()` | | Export | `NullifyPDF.py` | `cmd_export()` | +| Reconstruct | `NullifyPDF.py` | `cmd_reconstruct()` | | Blocklist/Allowlist | `NullifyPDF.py` | `PDFListManager` | --- @@ -504,7 +507,7 @@ Never hardcode API keys or passwords. 2. Edit code following code standards 3. Add tests: `pytest tests/test_my_feature.py` 4. Run full test suite: `pytest tests/ -v` -5. Build locally: `python build_local.py --lite` +5. Build locally: `python build_local.py` 6. Commit and push ### Q: How do I test on different OS? @@ -560,5 +563,5 @@ Ready to contribute? See [CONTRIBUTING.md](./CONTRIBUTING.md) for: --- -*Last updated: 2026-07-29* +*Last updated: 2026-09-18* *โ† [Troubleshooting](./TROUBLESHOOTING.md) | [Back to README โ†’](./README.md)* diff --git a/NullifyPDF.py b/NullifyPDF.py index b845c34..6eecefa 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -13,7 +13,8 @@ import html import importlib.util import json -from typing import Optional, List, Set, Dict, Any, Tuple +import math +from typing import Optional, List, Set, Dict, Any, Tuple, TypedDict, NotRequired import fitz from PySide6.QtWidgets import ( QApplication, @@ -50,13 +51,17 @@ from PySide6.QtCore import Qt, QThread, QObject, Signal, Slot, QRectF, QPointF, QMutex, QMutexLocker from privacy_core import ( + PlaceholderEntry, PlaceholderRegistry, PrivacyMode, build_restore_payload, + decrypt_restore_payload, encrypt_restore_payload, + parse_restore_entries, + sort_entries_longest_placeholder_first, ) -__version__ = "2.1.0" +__version__ = "2.2.0" __version_prerelease__ = "" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" @@ -65,6 +70,13 @@ ) +class DetectionDict(TypedDict): + text: str + entity_type: str + rects: List[Tuple[float, float, float, float]] + source: NotRequired[str] + + def setup_logging() -> logging.Logger: """Configure file-based logging. @@ -111,7 +123,7 @@ def resource_path(relative_path: str) -> str: str: Absolute path to resource. """ try: - base_path = sys._MEIPASS + base_path = getattr(sys, "_MEIPASS") except Exception: base_path = os.path.dirname(os.path.abspath(__file__)) return os.path.join(base_path, relative_path) @@ -126,6 +138,243 @@ def sha256_file(path: str) -> str: return digest.hexdigest() +class RestoreMapMismatch(ValueError): + """Raised when a restore map's recorded hash doesn't match the PDF.""" + + +REDACTION_FONTNAME = "helv" +REDACTION_MAX_FONTSIZE = 8.0 +REDACTION_MIN_FONTSIZE = 3.0 + + +def fit_redaction_fontsize( + text: str, + rect: "fitz.Rect", + max_size: float = REDACTION_MAX_FONTSIZE, + min_size: float = REDACTION_MIN_FONTSIZE, +) -> float: + """Return the largest font size that lays `text` out inside `rect` intact. + + `apply_redactions()` lays its replacement text out as a text *box*: when + the text is wider than the box it wraps, and MuPDF will happily break in + the middle of a token, turning ``DATA_001`` into ``DATA_`` + ``001``. The + result is still perfectly legible on screen but is no longer findable by + `page.search_for("DATA_001")`, which silently breaks reconstruction. If + the text cannot be wrapped into the box at all, MuPDF drops it entirely. + + Picking a size that fits up front avoids both outcomes. Wrapping + *between words* is fine and stays legible, so the size only has to make + the longest unbreakable token fit the box width; the box must then be + tall enough for the resulting number of wrapped lines. + + Args: + text: The replacement string to be stamped into the box. + rect: The redaction box. + max_size: Preferred (largest) font size. + min_size: Floor; returned even if the text still does not fit. + + Returns: + float: Font size to pass to `add_redact_annot(..., fontsize=...)`. + """ + tokens = text.split() or [text] + usable_width = max(1.0, rect.width - 2.0) + size = max_size + while size > min_size: + line_height = size * 1.35 + widest_token = max( + fitz.get_text_length(token, fontname=REDACTION_FONTNAME, fontsize=size) + for token in tokens + ) + total_width = fitz.get_text_length( + text, fontname=REDACTION_FONTNAME, fontsize=size + ) + lines_needed = max(1, math.ceil(total_width / usable_width)) + if widest_token <= usable_width and lines_needed * line_height <= rect.height: + return size + size -= 0.25 + return min_size + + +def _placeholder_rects_by_chars(page: Any, placeholder: str) -> List["fitz.Rect"]: + """Locate a placeholder that `search_for()` cannot find, character by character. + + Fallback for version 1 restore maps (which carry no coordinates) whose + placeholder was wrapped by `apply_redactions()`. Depending on the + PyMuPDF version, a wrap can land its pieces on separate *lines* of the + same text block, or in entirely separate *blocks* (observed with + PyMuPDF 1.28.2: a narrow box wraps "DATA_001" into a "DATA_" block and a + separate "001" block). Concatenating only within each block therefore + still misses it, so this walks every character on the page, in reading + order, ignoring block/line boundaries entirely. + """ + rects: List["fitz.Rect"] = [] + try: + raw = page.get_text("rawdict") + except Exception: + return rects + chars: List[Any] = [] + for block in raw.get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + chars.extend(span.get("chars", [])) + if not chars: + return rects + text = "".join(ch.get("c", "") for ch in chars) + start = text.find(placeholder) + while start != -1: + matched = chars[start:start + len(placeholder)] + box = fitz.Rect(matched[0]["bbox"]) + for ch in matched[1:]: + box |= fitz.Rect(ch["bbox"]) + rects.append(box) + start = text.find(placeholder, start + len(placeholder)) + return rects + + +def _restore_targets(page: Any, entry: PlaceholderEntry) -> List["fitz.Rect"]: + """Return the boxes on `page` that should get `entry.original` back. + + Prefers the coordinates recorded in the restore map (version 2+), which + are the boxes the original value was taken from, so the value fits back + into them. Falls back to text search only for version 1 maps. + """ + stored = [ + fitz.Rect(occurrence.rect) + for occurrence in entry.occurrences + if occurrence.page == page.number + ] + if stored: + return stored + hits = list(page.search_for(entry.placeholder)) + if hits: + return hits + return _placeholder_rects_by_chars(page, entry.placeholder) + + +def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int: + """Rebuild a pseudonymized PDF's original values from a decrypted restore map. + + Replaces every placeholder occurrence (e.g. ``PERSON_001``) with the + original value, using the same add_redact_annot()-then-apply_redactions() + mechanism used to create the placeholders during export. Does not restore + the original layout: a placeholder's redaction box is sized for the + placeholder text, so a longer original value may be visually clipped or + crowd the box. + + Uses the redaction boxes recorded in the restore map (version 2+) and + only falls back to searching for the placeholder text with version 1 + maps, which carry no coordinates. + + Args: + in_path: Path to the pseudonymized PDF to reconstruct from. + out_path: Path to write the reconstructed PDF to. + payload: Decrypted restore-map payload (see `privacy_core.build_restore_payload`). + + Returns: + int: Number of placeholder occurrences whose original value was + **verified present** in the saved output. The count is taken + from the written file, not from the number of redactions + requested, so a value that `apply_redactions()` silently + dropped is never reported as restored. + + Raises: + ValueError: If the payload is not a recognized restore map. + RestoreMapMismatch: If `output_sha256` in the payload does not match + the SHA-256 of `in_path`. + """ + expected_sha256 = payload.get("output_sha256") + if expected_sha256 and sha256_file(in_path) != expected_sha256: + raise RestoreMapMismatch( + "La mappa di ripristino non corrisponde a questo PDF " + "(hash SHA-256 diverso)." + ) + + entries = sort_entries_longest_placeholder_first(parse_restore_entries(payload)) + # (page index, box, original value) for every restoration we attempted, + # so the saved output can be checked against it below. + attempted: List[Tuple[int, fitz.Rect, str]] = [] + doc = fitz.open(in_path) + try: + for page in doc: + claimed: List[fitz.Rect] = [] + for entry in entries: + for rect in _restore_targets(page, entry): + # Entries are sorted longest-placeholder-first, so a + # shorter placeholder must not re-redact a box a longer + # one already claimed: searching for PERSON_001 also hits + # the first ten characters of a rendered PERSON_0010, and + # that hit sits *inside* the longer one's box. + # Containment, not intersection: two separate fields the + # user boxed side by side may legitimately touch, and + # skipping the second would drop a value. + if any(taken.contains(rect) for taken in claimed): + continue + claimed.append(fitz.Rect(rect)) + page.add_redact_annot( + rect, + text=entry.original, + fill=(1, 1, 1), + text_color=(0, 0, 0), + align=1, + fontsize=fit_redaction_fontsize(entry.original, rect), + ) + attempted.append((page.number, fitz.Rect(rect), entry.original)) + page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True) + doc.save(out_path, garbage=4, deflate=True, clean=True) + finally: + doc.close() + + return _verify_restored(out_path, attempted) + + +def expected_restore_count(payload: Dict[str, Any]) -> int: + """Return how many placeholder occurrences a restore map should put back. + + Counts occurrences, not entries: one entry can carry several boxes (the + same value redacted on more than one page), and `reconstruct_pdf` + returns restored *occurrences*. Comparing its result against the entry + count would hide a partial restore -- which is exactly the repeated-value + case reconstruction has already gone wrong on once. + """ + return sum( + max(1, len(entry.occurrences)) for entry in parse_restore_entries(payload) + ) + + +def _verify_restored( + out_path: str, attempted: List[Tuple[int, "fitz.Rect", str]] +) -> int: + """Count how many attempted restorations actually landed in the output. + + `apply_redactions()` drops its replacement text silently when the text + cannot be laid out inside the box, so counting the redactions we asked + for would over-report: the feature would claim to have restored values + it had in fact destroyed. Read the answer back off the written file. + """ + logger = logging.getLogger("nullifypdf") + restored = 0 + doc = fitz.open(out_path) + try: + for page_number, rect, original in attempted: + probe = fitz.Rect(rect) + (-2, -2, 2, 2) + # Whitespace-insensitive: a long value legitimately wraps onto + # several lines inside its box. What matters is that its + # characters are back on the page, not how they are broken up. + found = "".join(doc[page_number].get_text("text", clip=probe).split()) + if "".join(original.split()) in found: + restored += 1 + else: + logger.warning( + "Reconstruction: value not restored on page %s at %s " + "(it did not fit its redaction box).", + page_number + 1, + tuple(round(v, 1) for v in rect), + ) + finally: + doc.close() + return restored + + def find_tessdata_dir() -> Optional[str]: """Find a local Tesseract tessdata directory for PyMuPDF OCR.""" candidates = [] @@ -367,8 +616,9 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, ocr_language = ocr_language_for_choice(choice, tessdata_dir) if not self.analyzer or sorted(self.loaded_langs) != sorted(target_langs): self.log_sig.emit(f"Inizializzazione AI ({choice})...") - from presidio_analyzer import AnalyzerEngine + from presidio_analyzer import AnalyzerEngine, RecognizerRegistry from presidio_analyzer.nlp_engine import NlpEngineProvider + from presidio_analyzer.predefined_recognizers import PhoneRecognizer models = ( [{"lang_code": "en", "model_name": "en_core_web_md"}] @@ -380,8 +630,32 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, provider = NlpEngineProvider( nlp_configuration={"nlp_engine_name": "spacy", "models": models} ) + nlp_engine = provider.create_engine() + + registry = RecognizerRegistry(supported_languages=target_langs) + registry.load_predefined_recognizers( + languages=target_langs, nlp_engine=nlp_engine + ) + # Presidio's default PhoneRecognizer regions + # (US/UK/DE/FR/IL/IN/CA/BR) omit Italy, so Italian numbers + # without an explicit "+39" prefix (e.g. mobile "340 1234567" + # or landline "02 1234567") are never matched. Replace it + # with an IT-aware instance for each loaded language. + phone_regions = PhoneRecognizer.DEFAULT_SUPPORTED_REGIONS + ("IT",) + registry.recognizers = [ + r for r in registry.recognizers + if not isinstance(r, PhoneRecognizer) + ] + for lang in target_langs: + registry.recognizers.append( + PhoneRecognizer( + supported_language=lang, supported_regions=phone_regions + ) + ) + self.analyzer = AnalyzerEngine( - nlp_engine=provider.create_engine(), + nlp_engine=nlp_engine, + registry=registry, supported_languages=target_langs, ) self.loaded_langs = target_langs @@ -394,6 +668,13 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, "IBAN_CODE", "CREDIT_CARD", "CRYPTO", + # Italian national-ID formats (auto-loaded by Presidio only + # for language "it", harmless to always request). + "IT_FISCAL_CODE", + "IT_VAT_CODE", + "IT_DRIVER_LICENSE", + "IT_IDENTITY_CARD", + "IT_PASSPORT", ] # Determine page count under mutex (cheap, but doc may be replaced @@ -455,7 +736,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, f_reg.search(clean) or clean_pattern.search(a_str) for a_str, f_reg in compiled_allowlist ): - detection = { + detection: DetectionDict = { "text": m, "entity_type": entity_type, "rects": [], @@ -523,16 +804,15 @@ def __init__(self, scene: QGraphicsScene, parent: Optional[Any] = None) -> None: parent: Parent widget. """ super().__init__(scene, parent) - self.setRenderHints(QPainter.Antialiasing | QPainter.SmoothPixmapTransform) + self.setRenderHints(QPainter.RenderHint.Antialiasing | QPainter.RenderHint.SmoothPixmapTransform) self.start_pos: Optional[QPointF] = None self.temp_rect: Optional[QGraphicsRectItem] = None def mousePressEvent(self, event: Any) -> None: """Handle mouse press: start selection rectangle on left-click.""" - if event.button() == Qt.LeftButton: - sp = self.mapToScene(event.position()) + if event.button() == Qt.MouseButton.LeftButton: + sp = self.mapToScene(event.position().toPoint()) self.start_pos = sp - self.point_clicked.emit(sp) self.temp_rect = QGraphicsRectItem(QRectF(sp, sp)) self.temp_rect.setPen(QPen(QColor("#ef4444"), 2)) self.scene().addItem(self.temp_rect) @@ -541,23 +821,30 @@ def mousePressEvent(self, event: Any) -> None: def mouseMoveEvent(self, event: Any) -> None: """Update in-progress selection rectangle while dragging.""" if self.start_pos and self.temp_rect: - ep = self.mapToScene(event.position()) + ep = self.mapToScene(event.position().toPoint()) self.temp_rect.setRect(QRectF(self.start_pos, ep).normalized()) super().mouseMoveEvent(event) def mouseReleaseEvent(self, event: Any) -> None: - """Finalize selection rectangle and emit rect_drawn if large enough.""" - if event.button() == Qt.LeftButton and self.temp_rect: + """Finalize the gesture: a small movement is a click (point_clicked), + a larger one is a drawn selection (rect_drawn). Deferring both to + release, based on the final swept distance, keeps them mutually + exclusive so a drag never also fires a click mid-gesture. + """ + if event.button() == Qt.MouseButton.LeftButton and self.temp_rect: rect = self.temp_rect.rect() + start = self.start_pos self.scene().removeItem(self.temp_rect) self.temp_rect = None self.start_pos = None if rect.width() > 5 and rect.height() > 5: self.rect_drawn.emit(rect) + elif start is not None: + self.point_clicked.emit(start) super().mouseReleaseEvent(event) def wheelEvent(self, event: Any) -> None: - if event.modifiers() == Qt.ControlModifier: + if event.modifiers() == Qt.KeyboardModifier.ControlModifier: self.zoom_req.emit(1 if event.angleDelta().y() > 0 else -1) event.accept() else: @@ -744,10 +1031,13 @@ def build_ui(self) -> None: btn_clear.clicked.connect(self.cmd_clear) s_lay.addWidget(btn_clear) s_lay.addSpacing(20) - btn_export = QPushButton("Esporta Privacy") + btn_export = QPushButton("Esporta PDF") btn_export.setStyleSheet("border-color: #0ea5e9; color: #0ea5e9;") btn_export.clicked.connect(self.cmd_export) s_lay.addWidget(btn_export) + btn_reconstruct = QPushButton("Ricostruisci PDF") + btn_reconstruct.clicked.connect(self.cmd_reconstruct) + s_lay.addWidget(btn_reconstruct) s_lay.addStretch() btn_about = QPushButton("Info") btn_about.clicked.connect(self.cmd_about) @@ -777,7 +1067,7 @@ def build_ui(self) -> None: btn_prev.clicked.connect(lambda: self.move_page(-1)) self.le_page = QLineEdit("0") self.le_page.setFixedWidth(40) - self.le_page.setAlignment(Qt.AlignCenter) + self.le_page.setAlignment(Qt.AlignmentFlag.AlignCenter) self.le_page.returnPressed.connect(self.jump_page) self.lbl_tot = QLabel("/ 0") btn_next = QPushButton(">") @@ -915,8 +1205,17 @@ def _warn_if_scanned_pdf(self) -> None: f"nelle pagine {preview}{suffix}. " "Serve OCR prima della rilevazione automatica dei dati personali." ) + QMessageBox.information( + self, + "PDF scansionato rilevato", + f"Questo documento sembra scansionato (nessun testo trovato " + f"nelle pagine {preview}{suffix}).\n\n" + 'Puoi comunque selezionare manualmente le aree da censurare. ' + 'Per il riconoscimento automatico dei dati personali, attiva ' + 'prima "OCR PDF scansionati" e poi avvia "Auto Redact (AI)".', + ) - def render(self) -> None: + def render_page(self) -> None: """Render current PDF page to display.""" if not self.doc: return @@ -929,7 +1228,7 @@ def render(self) -> None: # QImage does NOT copy `pix.samples`; if `pix` is garbage-collected before # the image is used the buffer is freed (UB / crash). Force a deep copy. img = QImage( - pix.samples, pix.width, pix.height, pix.stride, QImage.Format_RGB888 + pix.samples, pix.width, pix.height, pix.stride, QImage.Format.Format_RGB888 ).copy() self.scene.clear() self.scene.addPixmap(QPixmap.fromImage(img)) @@ -945,7 +1244,7 @@ def adjust_zoom(self, d: int) -> None: """ self.scale = max(0.5, min(4.0, self.scale + (0.25 * d))) self.lbl_zoom.setText(f"{int(self.scale * 100)}%") - self.render() + self.render_page() def move_page(self, d: int) -> None: """Move to adjacent page. @@ -955,7 +1254,7 @@ def move_page(self, d: int) -> None: """ if self.doc and 0 <= self.page_num + d < len(self.doc): self.page_num += d - self.render() + self.render_page() def jump_page(self) -> None: """Jump to the page number entered in the page selector.""" @@ -966,7 +1265,7 @@ def jump_page(self) -> None: n = int(self.le_page.text()) - 1 if 0 <= n < len(self.doc): self.page_num = n - self.render() + self.render_page() else: self.write_log(f"Avviso: Pagina {n + 1} non esiste (intervallo: 1-{len(self.doc)})") except ValueError: @@ -989,6 +1288,7 @@ def user_draw_rect(self, qrect: QRectF) -> None: qrect.right() / self.scale, qrect.bottom() / self.scale, ) + cl = "" with QMutexLocker(self.doc_mutex): if not self.doc: return @@ -1010,12 +1310,26 @@ def user_draw_rect(self, qrect: QRectF) -> None: entity_type=self._infer_entity_type(txt), ) cl = " ".join(txt.split()).lower() - if len(cl) > 2: - self.allowlist.discard(cl) - self.blocklist.add(cl) - self.list_manager.save_blocklist(self.blocklist) - self.list_manager.save_allowlist(self.allowlist) - self.render() + self.render_page() + # The redaction on this page is already applied regardless of the + # answer below; this only decides whether the word is also + # persisted to blocklist.txt so it gets auto-redacted in every + # future document. + if len(cl) > 2 and cl not in self.blocklist: + answer = QMessageBox.question( + self, + "Aggiungi alla blacklist", + f'Vuoi aggiungere "{cl}" alla blacklist?\n\n' + "Verrร  censurata automaticamente in ogni documento PDF " + "che aprirai in futuro.", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if answer == QMessageBox.StandardButton.Yes: + self.allowlist.discard(cl) + self.blocklist.add(cl) + self.list_manager.save_blocklist(self.blocklist) + self.list_manager.save_allowlist(self.allowlist) def user_click_pt(self, qpt: QPointF) -> None: """Handle user click on redaction to delete it. @@ -1026,7 +1340,7 @@ def user_click_pt(self, qpt: QPointF) -> None: if not self.doc: return pt = fitz.Point(qpt.x() / self.scale, qpt.y() / self.scale) - has_ans = False + cl = "" with QMutexLocker(self.doc_mutex): if not self.doc: return @@ -1036,19 +1350,54 @@ def user_click_pt(self, qpt: QPointF) -> None: for a in (p.annots() or []) if a.type[0] == fitz.PDF_ANNOT_REDACT and a.rect.contains(pt) ] - if ans: - has_ans = True - txt = p.get_text("text", clip=ans[0].rect) - for a in ans: - p.delete_annot(a) - cl = " ".join(txt.split()).lower() - if len(cl) > 2: - self.blocklist.discard(cl) - self.allowlist.add(cl) - self.list_manager.save_blocklist(self.blocklist) - self.list_manager.save_allowlist(self.allowlist) - if has_ans: - self.render() + if not ans: + return + txt = p.get_text("text", clip=ans[0].rect) + cl = " ".join(txt.split()).lower() + + add_to_whitelist = False + if len(cl) > 2: + box = QMessageBox(self) + box.setWindowTitle("Rimuovi censura") + box.setText( + f'Rimuovere la censura su "{cl}"?\n\n' + "Scegli se escluderla solo da questo documento oppure " + "aggiungerla alla whitelist permanente (non verrร  mai piรน " + "censurata in nessun documento)." + ) + btn_doc_only = box.addButton( + "Solo questo documento", QMessageBox.ButtonRole.AcceptRole + ) + btn_whitelist = box.addButton( + "Whitelist permanente", QMessageBox.ButtonRole.YesRole + ) + btn_cancel = box.addButton( + "Annulla", QMessageBox.ButtonRole.RejectRole + ) + box.setDefaultButton(btn_doc_only) + box.exec() + clicked = box.clickedButton() + if clicked == btn_cancel: + return + add_to_whitelist = clicked == btn_whitelist + + with QMutexLocker(self.doc_mutex): + if not self.doc: + return + p = self.doc[self.page_num] + ans = [ + a + for a in (p.annots() or []) + if a.type[0] == fitz.PDF_ANNOT_REDACT and a.rect.contains(pt) + ] + for a in ans: + p.delete_annot(a) + if add_to_whitelist and len(cl) > 2: + self.blocklist.discard(cl) + self.allowlist.add(cl) + self.list_manager.save_blocklist(self.blocklist) + self.list_manager.save_allowlist(self.allowlist) + self.render_page() def cmd_clear(self) -> None: """Clear all redactions on current page.""" @@ -1066,7 +1415,7 @@ def cmd_clear(self) -> None: ] for a in to_delete: p.delete_annot(a) - self.render() + self.render_page() self.write_log(f"Censure rimosse su pagina {self.page_num+1}") def cmd_dict(self) -> None: @@ -1111,30 +1460,30 @@ def cmd_about(self) -> None: d.setWindowTitle("Info") d.setFixedSize(340, 440) lay = QVBoxLayout(d) - lay.setAlignment(Qt.AlignCenter) + lay.setAlignment(Qt.AlignmentFlag.AlignCenter) ip = resource_path(os.path.join("images", "NullifyPDF_icon.png")) if os.path.exists(ip): lbl_icon = QLabel() lbl_icon.setPixmap( QPixmap(ip).scaled( - 100, 100, Qt.KeepAspectRatio, Qt.SmoothTransformation + 100, 100, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation ) ) - lbl_icon.setAlignment(Qt.AlignCenter) + lbl_icon.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(lbl_icon) lay.addSpacing(10) lbl_title = QLabel("NullifyPDF") lbl_title.setStyleSheet("font-size: 24px; font-weight: bold;") - lbl_title.setAlignment(Qt.AlignCenter) + lbl_title.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(lbl_title) lbl_ver = QLabel(f"v{APP_VERSION} AI Privacy Beta") lbl_ver.setStyleSheet("color: #0ea5e9; font-weight: bold;") - lbl_ver.setAlignment(Qt.AlignCenter) + lbl_ver.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(lbl_ver) desc = QLabel( "\nAnonimizzazione PDF Offline.\n\nSviluppato da: Graziano Mariella\nLicenza MIT" ) - desc.setAlignment(Qt.AlignCenter) + desc.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(desc) lay.addSpacing(20) btn = QPushButton("Chiudi") @@ -1213,13 +1562,20 @@ def _select_privacy_export(self) -> Optional[PrivacyMode]: def _infer_entity_type(self, value: str) -> str: """Infer a coarse placeholder type for manually marked text.""" compact = " ".join(value.split()) + compact_nospace = compact.replace(" ", "") if re.search(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", compact, re.I): return "EMAIL_ADDRESS" - if re.search(r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b", compact.replace(" ", ""), re.I): + if re.search(r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b", compact_nospace, re.I): return "IBAN_CODE" + if re.fullmatch( + r"[A-Z]{6}\d{2}[A-EHLMPR-T]\d{2}[A-Z]\d{3}[A-Z]", compact_nospace, re.I + ): + return "IT_FISCAL_CODE" + if re.fullmatch(r"\d{11}", compact_nospace): + return "IT_VAT_CODE" if re.search(r"\+?\d[\d\s()./-]{6,}\d", compact): return "PHONE_NUMBER" - if re.search(r"\b\d{13,19}\b", compact.replace(" ", "")): + if re.search(r"\b\d{13,19}\b", compact_nospace): return "CREDIT_CARD" return "DATA" @@ -1295,6 +1651,7 @@ def _prepare_pseudonymized_page( clean_original, entity_type, page=page.number, + rect=(rect.x0, rect.y0, rect.x1, rect.y1), ) page.add_redact_annot( rect, @@ -1302,7 +1659,10 @@ def _prepare_pseudonymized_page( fill=(1, 1, 1), text_color=(0, 0, 0), align=1, - fontsize=8, + # Must fit on one line: a wrapped placeholder is still + # readable but no longer findable by search_for(), which + # is the only handle a version 1 restore map has. + fontsize=fit_redaction_fontsize(placeholder, rect), ) else: page.add_redact_annot( @@ -1311,7 +1671,7 @@ def _prepare_pseudonymized_page( fill=(1, 1, 1), text_color=(0, 0, 0), align=1, - fontsize=8, + fontsize=fit_redaction_fontsize("[IMAGE_REMOVED]", rect), ) @Slot(int, object) @@ -1393,7 +1753,7 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: def ai_finished(self) -> None: """Handle AI scan completion.""" self.btn_ai.setEnabled(True) - self.render() + self.render_page() def cmd_export(self) -> None: """Export current PDF in irreversible or reversible privacy mode. @@ -1441,7 +1801,7 @@ def cmd_export(self) -> None: self, "Password mappa", "Password per cifrare la mappa di ripristino (minimo 12 caratteri):", - QLineEdit.Password, + QLineEdit.EchoMode.Password, ) if not ok: return @@ -1501,8 +1861,17 @@ def cmd_export(self) -> None: page.delete_link(lnk) except (RuntimeError, AttributeError, KeyError) as e: self.logger.debug(f"Could not delete link: {e}") + # PDF_REDACT_IMAGE_REMOVE deletes the ENTIRE image object if it + # intersects ANY redaction rect, even a small one. On a scanned + # PDF (the whole page is one image), redacting a single word + # wiped out the entire page, leaving a blank sheet with only + # the placeholder text visible. PDF_REDACT_IMAGE_PIXELS blanks + # out only the pixels under each redaction rect and leaves the + # rest of the image intact (verified this also fully blanks + # whole-image redactions from "Oscura Immagini", including + # under page rotation, with no visible fringe). page.apply_redactions( - images=fitz.PDF_REDACT_IMAGE_REMOVE, graphics=True + images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True ) try: # Materialize first: mutating during iteration of @@ -1554,6 +1923,87 @@ def cmd_export(self) -> None: except OSError as e: self.logger.debug(f"Could not remove temp file {tmp_path}: {e}") + def cmd_reconstruct(self) -> None: + """Reconstruct a previously pseudonymized PDF using its restore map. + + Operates on files chosen by the user, independent of whatever + document (if any) is currently loaded in the viewer. + """ + if importlib.util.find_spec("cryptography") is None: + QMessageBox.critical( + self, + "Dipendenza mancante", + "La ricostruzione richiede la libreria 'cryptography'.", + ) + return + + pdf_path, _ = QFileDialog.getOpenFileName( + self, "Seleziona il PDF pseudonimizzato", "", "PDF (*.pdf)" + ) + if not pdf_path: + return + map_path, _ = QFileDialog.getOpenFileName( + self, + "Seleziona la mappa di ripristino", + "", + "NullifyPDF map (*.nullifypdf-map)", + ) + if not map_path: + return + map_password, ok = QInputDialog.getText( + self, + "Password mappa", + "Password per decifrare la mappa di ripristino:", + QLineEdit.EchoMode.Password, + ) + if not ok or not map_password: + return + out_path, _ = QFileDialog.getSaveFileName( + self, + "Salva PDF ricostruito", + f"{os.path.splitext(pdf_path)[0]}_ricostruito.pdf", + "PDF (*.pdf)", + ) + if not out_path: + return + + from cryptography.fernet import InvalidToken + + self.write_log("Ricostruzione PDF in corso...") + try: + with open(map_path, "rb") as fh: + encrypted = fh.read() + payload = decrypt_restore_payload(encrypted, map_password) + expected = expected_restore_count(payload) + restored_count = reconstruct_pdf(pdf_path, out_path, payload) + self.write_log( + f"RICOSTRUITO. {restored_count} segnaposto ripristinati." + ) + # A restore that silently puts nothing back used to look like a + # success: same message, no dialog, output identical to input. + if restored_count < expected: + QMessageBox.warning( + self, + "Ricostruzione incompleta", + f"Ripristinati solo {restored_count} valori su {expected} " + "attesi.\n\nI valori mancanti non sono stati riscritti nel " + "PDF: il file originale e la mappa restano l'unica copia. " + "Controlla il log per i dettagli.", + ) + except RestoreMapMismatch as e: + self.write_log(f"ERRORE ricostruzione: {e}") + QMessageBox.critical(self, "Mappa non corrispondente", str(e)) + except InvalidToken: + self.write_log("ERRORE ricostruzione: password errata o mappa corrotta.") + QMessageBox.critical( + self, + "Errore decifrazione", + "Password errata o file della mappa corrotto.", + ) + except Exception as e: + self.logger.error(f"Reconstruction failed: {traceback.format_exc()}") + self.write_log(f"ERRORE ricostruzione: {type(e).__name__}: {str(e)}") + if __name__ == "__main__": if platform.system() == "Linux": diff --git a/OCR_SETUP.md b/OCR_SETUP.md index b8d1190..35a0755 100644 --- a/OCR_SETUP.md +++ b/OCR_SETUP.md @@ -21,24 +21,21 @@ Install or download the language data you need: - `eng.traineddata` for English OCR. - `ita.traineddata` for Italian OCR. -For release builds, NullifyPDF uses the `Full` variant to bundle these two -files automatically. Developers can prepare a local Full build with: +Release builds always bundle these two files automatically. Developers can +prepare a local build with: ```bash -python build_local.py --full +python build_local.py ``` -If the OCR files are missing, the Full build downloads them automatically from +If the OCR files are missing, the build downloads them automatically from `tesseract-ocr/tessdata_fast` before running PyInstaller. -Use `python build_local.py --lite` for a smaller build without bundled OCR -data. - If OCR is enabled in the app but `tessdata` cannot be found, NullifyPDF stops before scanning and asks you to configure Tesseract instead of silently processing scanned pages without OCR. --- -*Last updated: 2026-07-23* +*Last updated: 2026-09-18* *[Back to README](./README.md) | [Architecture โ†’](./ARCHITECTURE.md)* diff --git a/README.md b/README.md index d5f7eff..ea5cdde 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi - ๐Ÿ—„๏ธ **Fluid UI & Thread-Safe** โ€” PySide6 modern dark-mode interface with zero UI freezing. Text extraction in worker thread with QMutex serialization - ๐Ÿ“– **Persistent Dictionaries** โ€” Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with O(1) fast-path matching - ๐Ÿ›ก๏ธ **Privacy Export Modes** โ€” Choose irreversible anonymization or reversible pseudonymization with a separately encrypted restore map -- ๐Ÿ”Ž **OCR Support** โ€” Full builds bundle EN/IT Tesseract language data for scanned PDFs +- ๐Ÿ”Ž **OCR Support** โ€” Bundled EN/IT Tesseract language data for scanned PDFs - ๐Ÿ–ผ๏ธ **Blindfold Mode** โ€” One-click image/logo censoring with professional placeholder: `[ IMAGE REMOVED ]` - ๐Ÿ“ฆ **Native Cross-Platform** โ€” Automated build scripts generate Windows `.exe`, macOS ZIP bundles, and Linux `.deb`/`.rpm` packages - ๐ŸŽฏ **Drag & Drop Support** โ€” Native file drag-and-drop on main window @@ -50,7 +50,6 @@ To keep NullifyPDF lightweight, 100% offline, and secure, be aware of these tech | โŒ Limitation | ๐Ÿ’ก Workaround | | ---------------------------------- | ------------------------------------------------------------------------------------------------------- | -| **OCR availability** | Full builds include EN/IT OCR data. Lite builds require local Tesseract tessdata for scanned PDFs. | | **Handwritten Text** | NLP models cannot analyze non-digitized handwriting. | | **Password-Protected PDFs** | Encrypted documents are blocked at load. Decrypt before importing. | | **Digital Signatures Invalidated** | Privacy export changes the PDF; cryptographic signatures (PAdES, notarized) become invalid. | @@ -82,12 +81,10 @@ To keep NullifyPDF lightweight, 100% offline, and secure, be aware of these tech Download the latest pre-compiled executable from [Releases](https://github.com/overwrite00/NullifyPDF/releases): -- **Windows Lite:** `NullifyPDF_vX.Y.Z_Windows_Lite.exe` -- **Windows Full OCR:** `NullifyPDF_vX.Y.Z_Windows_Full.exe` -- **macOS Lite:** `NullifyPDF_vX.Y.Z_macOS_Lite.zip` -- **macOS Full OCR:** `NullifyPDF_vX.Y.Z_macOS_Full.zip` -- **Ubuntu Lite/Full:** `NullifyPDF_vX.Y.Z_Ubuntu_Lite.deb` or `NullifyPDF_vX.Y.Z_Ubuntu_Full.deb` -- **Fedora Lite/Full:** `NullifyPDF_vX.Y.Z_Fedora_Lite.rpm` or `NullifyPDF_vX.Y.Z_Fedora_Full.rpm` +- **Windows:** `NullifyPDF_vX.Y.Z_Windows.exe` +- **macOS:** `NullifyPDF_vX.Y.Z_macOS.zip` +- **Ubuntu:** `NullifyPDF_vX.Y.Z_Ubuntu.deb` +- **Fedora:** `NullifyPDF_vX.Y.Z_Fedora.rpm` No installation needed on Windows/macOS - just run or unzip. Linux users can install the `.deb` or `.rpm` package. @@ -169,15 +166,15 @@ python setup_env.py Compiles standalone executable with PyInstaller. ```bash -python build_local.py --lite -python build_local.py --full +python build_local.py ``` **Features:** - Cleans temporary directories - Auto-detects your OS - Reads version dynamically from code -- Generates named executables such as `NullifyPDF_vX.Y.Z_Windows_Lite.exe` or `NullifyPDF_vX.Y.Z_Windows_Full.exe` +- Generates named executables such as `NullifyPDF_vX.Y.Z_Windows.exe` +- Bundles EN/IT OCR data (downloaded automatically if missing) **Linux bonus:** On Ubuntu/Fedora, automatically builds `.deb` and `.rpm` packages in `dist/` @@ -200,7 +197,7 @@ pytest tests/ -v - Resource path resolution - OCR configuration helpers - Privacy placeholder and encrypted restore-map primitives -- Build variant configuration +- Build configuration @@ -214,7 +211,7 @@ pytest tests/ -v | [CONTRIBUTING.md](./CONTRIBUTING.md) | How to contribute code & report issues | | [ARCHITECTURE.md](./ARCHITECTURE.md) | System design & technical overview | | [DEVELOPMENT.md](./DEVELOPMENT.md) | Local dev setup, testing & builds | -| [OCR_SETUP.md](./OCR_SETUP.md) | OCR data setup and Lite/Full build notes | +| [OCR_SETUP.md](./OCR_SETUP.md) | OCR data setup and build notes | | [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) | Common issues & solutions | | [CHANGELOG.md](./CHANGELOG.md) | Release history & updates | @@ -263,5 +260,5 @@ Want to help improve NullifyPDF? See [CONTRIBUTING.md](./CONTRIBUTING.md) for gu --- -*Last updated: 2026-07-23* +*Last updated: 2026-09-18* *[User Guide โ†’](./USER_GUIDE.md)* diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 0dcf8aa..1a114be 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -58,7 +58,7 @@ If log isn't updating: ## ๐Ÿ”ด Export Fails or Crashes ### Symptom -Clicking "Esporta Privacy" causes error or app crash. +Clicking "Esporta PDF" causes error or app crash. ### Root Cause - Insufficient disk space @@ -183,7 +183,7 @@ If you can't copy โ†’ it's an image โ†’ enable **OCR PDF scansionati** For scanned PDFs: 1. Enable **"OCR PDF scansionati"** in the sidebar -2. Make sure you use a Full build or have local Tesseract `tessdata` +2. Make sure OCR data is present (bundled by default) or you have local Tesseract `tessdata` 3. Run **"Auto Redact (AI)"** again Use image redaction only when you intentionally want to remove whole images or @@ -215,8 +215,8 @@ AI redacted text that shouldn't be redacted (false positive). **Option 1 โ€” Remove and Allowlist** 1. **Click once** on the black redaction box -2. It disappears -3. Word is added to allowlist (won't be detected again) +2. Choose **"Whitelist permanente"** in the dialog โ€” the box disappears and the word is added to the allowlist (won't be detected again in any document) +3. Choose **"Solo questo documento"** instead if the false positive is specific to this one PDF (nothing is saved to the allowlist) **Option 2 โ€” Manual Redaction** @@ -461,5 +461,5 @@ The more details, the faster we can fix it! ๐Ÿš€ --- -*Last updated: 2026-07-23* +*Last updated: 2026-09-18* *โ† [User Guide](./USER_GUIDE.md) | [Development โ†’](./DEVELOPMENT.md)* diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 2966bd9..2613ef7 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -27,7 +27,7 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel 3. The document appears in the center. Scroll with mouse or use arrow buttons (top right) > [!NOTE] -> Supported format: unencrypted PDF. Digital text is scanned directly; scanned pages require OCR. +> Supported format: unencrypted PDF. Digital text is scanned directly; scanned pages require OCR. If NullifyPDF detects a page with no extractable text, a popup confirms it and reminds you to enable **OCR PDF scansionati** before AI scanning โ€” manual selection (Step 4) works on scanned pages either way. --- @@ -53,9 +53,10 @@ Click the **"Auto Redact (AI)"** button. NullifyPDF will automatically find and - ๐Ÿง‘ **Names & Surnames** - ๐Ÿ™๏ธ **Cities & Addresses** - ๐Ÿ“ง **Email Addresses** -- ๐Ÿ“ฑ **Phone Numbers** +- ๐Ÿ“ฑ **Phone Numbers** (including Italian mobile/landline formats) - ๐Ÿ’ณ **IBANs & Credit Card Numbers** - โ‚ฟ **Cryptocurrency Addresses** +- ๐Ÿ‡ฎ๐Ÿ‡น **Italian national IDs**: Codice Fiscale, Partita IVA, carta d'identitร , patente, passaporto **What to expect:** - Black boxes appear over detected data @@ -75,14 +76,16 @@ No AI is 100% accurate. You can fix mistakes: ### Add a Redaction **Draw a box** over text you want to hide: 1. Click and drag mouse over sensitive text -2. A black box appears -3. Redaction is scheduled (not destructive yet) +2. A black box appears (redaction is scheduled, not destructive yet) +3. If the covered text is recognized, you're asked whether to add it to the **Blocklist** โ€” say yes only if it should be redacted in *every* PDF you open in the future, not just this one ### Remove a Redaction **Click once** on a black box to remove it: 1. Single click on the redaction box -2. Box disappears -3. Word is added to **Allowlist** (AI will ignore it next time) +2. Choose what happens next: + - **Solo questo documento**: the box disappears, nothing is saved to the lists (this document only) + - **Whitelist permanente**: the box disappears and the word is added to the **Allowlist** (AI will ignore it in every future document) + - **Annulla**: keeps the redaction in place ### Zoom In/Out Make small text larger: @@ -103,7 +106,7 @@ Remove logos, signatures, or scanned photos: 3. All images replaced with gray placeholder: `[ IMAGE REMOVED ]` > [!NOTE] -> For scanned documents, enable **OCR PDF scansionati** before running the AI scan. Full builds include EN/IT OCR data; Lite builds require local Tesseract tessdata. +> For scanned documents, enable **OCR PDF scansionati** before running the AI scan. Every build includes EN/IT OCR data. --- @@ -111,7 +114,7 @@ Remove logos, signatures, or scanned photos: When satisfied with redactions: -1. Click **"Esporta Privacy"** +1. Click **"Esporta PDF"** 2. Choose the export mode: - **Anonimizzazione irreversibile** removes selected personal data permanently - **Pseudonimizzazione reversibile** replaces selected personal data with placeholders and creates a separate encrypted restore map @@ -123,6 +126,23 @@ When satisfied with redactions: --- +## Step 7๏ธโƒฃ โ€” Reconstruct a Pseudonymized PDF + +If you exported a PDF with **Pseudonimizzazione reversibile**, you can later restore the original values back into it: + +1. Click **"Ricostruisci PDF"** +2. Select the pseudonymized PDF (the one containing placeholders like `PERSON_001`) +3. Select its matching `.nullifypdf-map` restore map file +4. Enter the map's password +5. Choose where to save the reconstructed PDF + +NullifyPDF checks the SHA-256 hash recorded in the restore map against the selected PDF and refuses to proceed if they don't match, to avoid applying the wrong mapping to a document. + +> [!NOTE] +> Reconstruction restores the original *content*, not the original *layout*: since placeholders like `PERSON_001` are usually shorter than the original values, restored text may be clipped or visually crowd its redaction box. + +--- + ## ๐Ÿ”ง Dictionary Management Your redaction preferences are saved automatically: @@ -238,7 +258,7 @@ python3.13 NullifyPDF.py ## โ“ Common Questions ### Q: Why doesn't AI detect text in my scanned PDF? -**A:** Enable **OCR PDF scansionati** before running the AI scan. Full builds include EN/IT OCR data; Lite builds need local Tesseract `tessdata`. +**A:** Enable **OCR PDF scansionati** before running the AI scan. Every build includes EN/IT OCR data. ### Q: Can I password-protect the exported PDF? **A:** Not built-in. Use a PDF editor after export for password protection. @@ -247,10 +267,10 @@ python3.13 NullifyPDF.py **A:** Export applies PDF redactions and removes selected metadata. Review the exported file before sharing, especially for high-risk documents. ### Q: Can I undo changes after export? -**A:** Irreversible anonymization cannot be undone from the exported PDF. Pseudonymization can be restored only with the encrypted restore map and its password. +**A:** Irreversible anonymization cannot be undone from the exported PDF. Pseudonymization can be restored using **"Ricostruisci PDF"** together with the encrypted restore map and its password (see [Step 7](#step-7%EF%B8%8F%E2%83%A3--reconstruct-a-pseudonymized-pdf) below). ### Q: Does NullifyPDF send data to the cloud? -**A:** **No.** PDF processing is local. Full builds may download OCR data during build time, not while processing your PDFs. +**A:** **No.** PDF processing is local. OCR data may be downloaded during build time, not while processing your PDFs. --- @@ -263,5 +283,5 @@ python3.13 NullifyPDF.py --- -*Last updated: 2026-07-23* +*Last updated: 2026-09-18* *โ† [Back to README](./README.md) | [Troubleshooting โ†’](./TROUBLESHOOTING.md)* diff --git a/build_local.py b/build_local.py index 4e7d49f..4cf7ddb 100644 --- a/build_local.py +++ b/build_local.py @@ -4,14 +4,12 @@ import subprocess import platform import re -import argparse import pathlib from typing import List, Optional, Tuple from scripts.download_ocr_data import download_ocr_data -VALID_BUILD_VARIANTS = {"lite", "full"} OCR_TESSDATA_FILES = ("eng.traineddata", "ita.traineddata") @@ -75,35 +73,8 @@ def ensure_icon(sys_os: str) -> Optional[str]: return os.path.join(base_dir, "NullifyPDF_icon.png").replace("\\", "/") -def normalize_build_variant(value: Optional[str]) -> str: - """Return a supported build variant name.""" - variant = (value or os.environ.get("NULLIFYPDF_BUILD_VARIANT") or "lite").lower() - if variant not in VALID_BUILD_VARIANTS: - allowed = ", ".join(sorted(VALID_BUILD_VARIANTS)) - raise ValueError(f"Build variant non valida: {variant}. Valori: {allowed}") - return variant - - -def parse_args() -> argparse.Namespace: - """Parse build command-line options.""" - parser = argparse.ArgumentParser(description="Build NullifyPDF with PyInstaller") - group = parser.add_mutually_exclusive_group() - group.add_argument("--lite", action="store_true", help="Build without bundled OCR") - group.add_argument("--full", action="store_true", help="Build with bundled OCR data") - return parser.parse_args() - - -def variant_from_args(args: argparse.Namespace) -> str: - """Resolve build variant from CLI flags or environment.""" - if args.lite: - return "lite" - if args.full: - return "full" - return normalize_build_variant(None) - - def ensure_ocr_data(download_func=download_ocr_data) -> None: - """Ensure EN/IT OCR data is available for Full builds.""" + """Ensure EN/IT OCR data is available for the build.""" tessdata_dir = os.path.join("ocr", "tessdata") missing = [ name for name in OCR_TESSDATA_FILES @@ -114,7 +85,7 @@ def ensure_ocr_data(download_func=download_ocr_data) -> None: missing_list = ", ".join(missing) print( - "[INFO] Build Full richiesto: mancano dati OCR " + "[INFO] Mancano dati OCR " f"({missing_list}). Download automatico da tesseract-ocr/tessdata_fast..." ) download_func(pathlib.Path(tessdata_dir)) @@ -125,37 +96,34 @@ def ensure_ocr_data(download_func=download_ocr_data) -> None: if still_missing: raise FileNotFoundError( "Download OCR incompleto. Mancano ancora: " - f"{', '.join(still_missing)}. Usa --lite oppure controlla la rete." + f"{', '.join(still_missing)}. Controlla la connessione di rete." ) -def pyinstaller_datas( - build_variant: str, download_missing_ocr: bool = True -) -> List[Tuple[str, str]]: +def pyinstaller_datas(download_missing_ocr: bool = True) -> List[Tuple[str, str]]: """Return data files/directories to include in the PyInstaller bundle.""" datas: List[Tuple[str, str]] = [] if os.path.exists("images"): datas.append(("images", "images")) - if build_variant == "full": - tessdata_dir = os.path.join("ocr", "tessdata") - if download_missing_ocr: - ensure_ocr_data() - elif any( - not os.path.exists(os.path.join(tessdata_dir, name)) - for name in OCR_TESSDATA_FILES - ): - raise FileNotFoundError( - "Build Full richiesto, ma mancano file OCR. " - "Usa download_missing_ocr=True oppure --lite." - ) - for name in OCR_TESSDATA_FILES: - source = os.path.join(tessdata_dir, name).replace("\\", "/") - datas.append((source, "ocr/tessdata")) + tessdata_dir = os.path.join("ocr", "tessdata") + if download_missing_ocr: + ensure_ocr_data() + elif any( + not os.path.exists(os.path.join(tessdata_dir, name)) + for name in OCR_TESSDATA_FILES + ): + raise FileNotFoundError( + "Mancano file OCR. Usa download_missing_ocr=True oppure esegui " + "scripts/download_ocr_data.py." + ) + for name in OCR_TESSDATA_FILES: + source = os.path.join(tessdata_dir, name).replace("\\", "/") + datas.append((source, "ocr/tessdata")) return datas def build_rpm( - version: str, file_version: str, executable_name: str, variant_label: str + version: str, file_version: str, executable_name: str ) -> None: """Build RPM package for Fedora/RHEL. @@ -232,7 +200,7 @@ def build_rpm( if file.endswith(".rpm"): shutil.move( os.path.join(root, file), - f"dist/NullifyPDF_v{file_version}_Fedora_{variant_label}.rpm", + f"dist/NullifyPDF_v{file_version}_Fedora.rpm", ) print("[OK] RPM creato con successo.") except Exception as e: @@ -242,7 +210,7 @@ def build_rpm( def build_deb( - version: str, file_version: str, executable_name: str, variant_label: str + version: str, file_version: str, executable_name: str ) -> None: """Build DEB package for Ubuntu/Debian. @@ -297,7 +265,7 @@ def build_deb( "dpkg-deb", "--build", pkg_dir, - f"dist/NullifyPDF_v{file_version}_Ubuntu_{variant_label}.deb", + f"dist/NullifyPDF_v{file_version}_Ubuntu.deb", ], check=True, stdout=subprocess.DEVNULL, @@ -309,7 +277,7 @@ def build_deb( shutil.rmtree(pkg_dir, ignore_errors=True) -def build_app(build_variant: Optional[str] = None) -> None: +def build_app() -> None: """Build NullifyPDF application for current OS using PyInstaller. Automatically generates platform-specific executables: @@ -318,8 +286,6 @@ def build_app(build_variant: Optional[str] = None) -> None: - Linux: portable binary + .deb + .rpm packages """ print("--- Avvio Compilazione NullifyPDF (PySide6) ---") - build_variant = normalize_build_variant(build_variant) - variant_label = build_variant.capitalize() version, code_prerelease = get_version_info() beta_suffix = os.environ.get("NULLIFYPDF_BETA_SUFFIX", "").strip() file_version = get_file_version(version, code_prerelease, beta_suffix) @@ -334,13 +300,12 @@ def build_app(build_variant: Optional[str] = None) -> None: if sys_os == "Windows" else ("macOS", "") if sys_os == "Darwin" else ("Linux_Portable", "") ) - final_name = f"NullifyPDF_v{file_version}_{os_name}_{variant_label}{ext}" + final_name = f"NullifyPDF_v{file_version}_{os_name}{ext}" icon_path = ensure_icon(sys_os) # Use repr() to safely embed the path as a Python literal in the spec file. # Manual single-quote wrapping is unsafe for paths containing quotes/backslashes. icon_str = repr(icon_path) if icon_path else "None" - datas_literal = repr(pyinstaller_datas(build_variant)) - print(f"[INFO] Variante build: {variant_label}") + datas_literal = repr(pyinstaller_datas()) if sys_os == "Darwin": spec_content = f"""# -*- mode: python ; coding: utf-8 -*- @@ -386,7 +351,7 @@ def build_app(build_variant: Optional[str] = None) -> None: print(f"[OK] Compilazione completata: dist/{final_name}") elif sys_os == "Darwin": print("[INFO] Compressione App Bundle per macOS in formato ZIP...") - zip_filename = f"NullifyPDF_v{file_version}_macOS_{variant_label}.zip" + zip_filename = f"NullifyPDF_v{file_version}_macOS.zip" subprocess.run( ["zip", "-r", "-y", zip_filename, "NullifyPDF.app"], cwd="dist", @@ -399,9 +364,9 @@ def build_app(build_variant: Optional[str] = None) -> None: os.rename("dist/NullifyPDF", f"dist/{final_name}") print(f"[OK] Eseguibile portatile pronto: dist/{final_name}") if shutil.which("rpmbuild"): - build_rpm(version, file_version, final_name, variant_label) + build_rpm(version, file_version, final_name) if shutil.which("dpkg-deb"): - build_deb(version, file_version, final_name, variant_label) + build_deb(version, file_version, final_name) except subprocess.CalledProcessError as e: print(f"\n[ERROR] ERRORE CRITICO: Compilazione fallita (exit {e.returncode}).") @@ -409,4 +374,4 @@ def build_app(build_variant: Optional[str] = None) -> None: if __name__ == "__main__": - build_app(variant_from_args(parse_args())) + build_app() diff --git a/privacy_core.py b/privacy_core.py index afa5e8b..3319ee8 100644 --- a/privacy_core.py +++ b/privacy_core.py @@ -12,7 +12,7 @@ import re from dataclasses import asdict, dataclass from enum import Enum -from typing import Dict, Iterable, List, Optional +from typing import Dict, Iterable, List, Optional, Sequence, Tuple class PrivacyMode(str, Enum): @@ -22,6 +22,22 @@ class PrivacyMode(str, Enum): PSEUDONYMIZE = "pseudonymize" +@dataclass(frozen=True) +class PlaceholderOccurrence: + """Where one placeholder was stamped into the exported PDF. + + Reconstruction uses these coordinates instead of searching the output + for the placeholder string. Text search is unreliable: MuPDF may wrap + the placeholder across lines to fit a narrow redaction box, which makes + the rendered string unfindable even though it is perfectly legible. + The rect is also the box the *original* value came from, so it is wide + enough to take that value back at a readable font size. + """ + + page: int + rect: Tuple[float, float, float, float] + + @dataclass(frozen=True) class PlaceholderEntry: """One reversible placeholder mapping entry.""" @@ -30,6 +46,7 @@ class PlaceholderEntry: original: str entity_type: str page: int + occurrences: Tuple[PlaceholderOccurrence, ...] = () class PlaceholderRegistry: @@ -38,7 +55,9 @@ class PlaceholderRegistry: def __init__(self) -> None: self._counters: Dict[str, int] = {} self._by_value: Dict[tuple[str, str], str] = {} - self._entries: List[PlaceholderEntry] = [] + self._order: List[str] = [] + self._meta: Dict[str, Tuple[str, str, int]] = {} + self._occurrences: Dict[str, List[PlaceholderOccurrence]] = {} @staticmethod def normalize_entity_type(entity_type: Optional[str]) -> str: @@ -47,30 +66,68 @@ def normalize_entity_type(entity_type: Optional[str]) -> str: return value or "DATA" def placeholder_for( - self, original: str, entity_type: Optional[str] = None, page: int = 0 + self, + original: str, + entity_type: Optional[str] = None, + page: int = 0, + rect: Optional[Sequence[float]] = None, ) -> str: + """Return the stable placeholder for `original`, recording this hit. + + Args: + original: The sensitive value being replaced. + entity_type: Detected entity type, normalized to a placeholder prefix. + page: 0-based page index of *this* occurrence. + rect: Redaction box (x0, y0, x1, y1) of *this* occurrence, if known. + + Every call records an occurrence, not just the first one: the same + value may appear on several pages and reconstruction needs all of + their coordinates. + """ clean_original = " ".join((original or "").split()) clean_type = self.normalize_entity_type(entity_type) key = (clean_type, clean_original.casefold()) - if key in self._by_value: - return self._by_value[key] - - next_index = self._counters.get(clean_type, 0) + 1 - self._counters[clean_type] = next_index - placeholder = f"{clean_type}_{next_index:03d}" - self._by_value[key] = placeholder - self._entries.append( - PlaceholderEntry( - placeholder=placeholder, - original=clean_original, - entity_type=clean_type, - page=max(0, int(page)), + page_index = max(0, int(page)) + + placeholder = self._by_value.get(key) + if placeholder is None: + next_index = self._counters.get(clean_type, 0) + 1 + self._counters[clean_type] = next_index + placeholder = f"{clean_type}_{next_index:03d}" + self._by_value[key] = placeholder + self._order.append(placeholder) + self._meta[placeholder] = (clean_original, clean_type, page_index) + self._occurrences[placeholder] = [] + + if rect is not None: + x0, y0, x1, y1 = (float(v) for v in tuple(rect)[:4]) + self._occurrences[placeholder].append( + PlaceholderOccurrence(page=page_index, rect=(x0, y0, x1, y1)) ) - ) return placeholder def entries(self) -> List[PlaceholderEntry]: - return list(self._entries) + entries: List[PlaceholderEntry] = [] + for placeholder in self._order: + clean_original, clean_type, first_page = self._meta[placeholder] + entries.append( + PlaceholderEntry( + placeholder=placeholder, + original=clean_original, + entity_type=clean_type, + page=first_page, + occurrences=tuple(self._occurrences[placeholder]), + ) + ) + return entries + + +RESTORE_MAP_FORMAT = "NullifyPDF restore map" +RESTORE_MAP_VERSION = 2 +# Version 1 maps carry no `occurrences`; they are still readable, and +# reconstruction falls back to searching the PDF for the placeholder text. +# Refusing them would permanently strand the PII of every map already issued. +SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2}) def build_restore_payload( @@ -82,16 +139,84 @@ def build_restore_payload( ) -> Dict[str, object]: """Build the JSON-serializable restore-map payload.""" + raw_entries: List[Dict[str, object]] = [] + for entry in entries: + raw = asdict(entry) + # asdict() keeps tuples as tuples; JSON round-trips them to lists, so + # normalize here and keep the payload byte-comparable with itself. + raw["occurrences"] = [ + {"page": occurrence.page, "rect": list(occurrence.rect)} + for occurrence in entry.occurrences + ] + raw_entries.append(raw) + return { - "format": "NullifyPDF restore map", - "version": 1, + "format": RESTORE_MAP_FORMAT, + "version": RESTORE_MAP_VERSION, "source_name": os.path.basename(source_name), "source_sha256": source_sha256, "output_sha256": output_sha256, - "entries": [asdict(entry) for entry in entries], + "entries": raw_entries, } +def parse_restore_entries(payload: Dict[str, object]) -> List[PlaceholderEntry]: + """Validate a decrypted restore-map payload and return its entries. + + Raises: + ValueError: If the payload is not a recognized restore map. + """ + if ( + not isinstance(payload, dict) + or payload.get("format") != RESTORE_MAP_FORMAT + or payload.get("version") not in SUPPORTED_RESTORE_MAP_VERSIONS + ): + raise ValueError("Formato mappa di ripristino non riconosciuto.") + + entries: List[PlaceholderEntry] = [] + raw_entries = payload.get("entries") or [] + assert isinstance(raw_entries, list) + for raw in raw_entries: + occurrences: List[PlaceholderOccurrence] = [] + for raw_occ in raw.get("occurrences") or []: + coords = [float(v) for v in raw_occ["rect"]] + if len(coords) != 4: + raise ValueError("Mappa di ripristino: rettangolo non valido.") + occurrences.append( + PlaceholderOccurrence( + page=int(raw_occ["page"]), + rect=(coords[0], coords[1], coords[2], coords[3]), + ) + ) + entries.append( + PlaceholderEntry( + placeholder=str(raw["placeholder"]), + original=str(raw["original"]), + entity_type=str(raw["entity_type"]), + page=int(raw["page"]), + occurrences=tuple(occurrences), + ) + ) + return entries + + +def sort_entries_longest_placeholder_first( + entries: Iterable[PlaceholderEntry], +) -> List[PlaceholderEntry]: + """Sort restore entries with the longest placeholder text first. + + Longest-first ordering matters because placeholder search is a substring + match: once a type exceeds 9 instances (``PERSON_0010``), a shorter + placeholder like ``PERSON_001`` would otherwise match its first 10 + characters and restore the wrong value. + + Only relevant to the version 1 fallback path: version 2 maps carry each + occurrence's coordinates, so reconstruction addresses boxes directly + instead of matching placeholder text. + """ + return sorted(entries, key=lambda e: len(e.placeholder), reverse=True) + + def encrypt_restore_payload(payload: Dict[str, object], password: str) -> bytes: """Encrypt and authenticate a restore-map payload with a password.""" diff --git a/requirements.txt b/requirements.txt index 76276df..2137d8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,15 +3,15 @@ # Update script handles version detection automatically # UI Framework (compatible with Python 3.13) -PySide6==6.11.1 +PySide6==6.11.2 # PDF Processing (abi3 wheel, compatible with Python 3.13) -PyMuPDF==1.28.0 +PyMuPDF==1.28.2 # NLP & Security Analysis presidio-analyzer==2.2.364 -spacy==3.8.14 # presidio-analyzer 2.2.364 excludes spacy==3.8.14 only for python_version>=3.14; safe on our Python 3.13 target -cryptography==49.0.0 +spacy==3.8.16 +cryptography==50.0.1 # spaCy language models (pinned to exact versions for reproducibility) # IMPORTANT: model minor version MUST match the spaCy core minor version # (spaCy 3.8.x <-> models 3.8.x). The models declare spacy>=3.8.0,<3.9.0. @@ -20,7 +20,7 @@ en-core-web-md @ https://github.com/explosion/spacy-models/releases/download/en_ it-core-news-md @ https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.8.0/it_core_news_md-3.8.0-py3-none-any.whl # Build Tool (compatible with Python 3.13) -pyinstaller==6.21.0 +pyinstaller==6.22.3 # Development Tools isort==5.13.2 diff --git a/scripts/download_ocr_data.py b/scripts/download_ocr_data.py index 14a3ade..9a12e6e 100644 --- a/scripts/download_ocr_data.py +++ b/scripts/download_ocr_data.py @@ -1,4 +1,4 @@ -"""Download bundled OCR language data for NullifyPDF Full builds.""" +"""Download bundled OCR language data for NullifyPDF builds.""" from __future__ import annotations diff --git a/tests/test_build_config.py b/tests/test_build_config.py index 37e45cc..1df6682 100644 --- a/tests/test_build_config.py +++ b/tests/test_build_config.py @@ -1,4 +1,4 @@ -"""Tests for build variant configuration.""" +"""Tests for build configuration.""" import pathlib @@ -8,36 +8,10 @@ ensure_ocr_data, get_file_version, get_version_info, - normalize_build_variant, pyinstaller_datas, - variant_from_args, ) -class Args: - def __init__(self, *, lite=False, full=False): - self.lite = lite - self.full = full - - -def test_normalize_build_variant_defaults_to_lite(monkeypatch): - monkeypatch.delenv("NULLIFYPDF_BUILD_VARIANT", raising=False) - - assert normalize_build_variant(None) == "lite" - - -def test_variant_from_args_prefers_cli_over_environment(monkeypatch): - monkeypatch.setenv("NULLIFYPDF_BUILD_VARIANT", "full") - - assert variant_from_args(Args(lite=True)) == "lite" - assert variant_from_args(Args(full=True)) == "full" - - -def test_invalid_build_variant_is_rejected(): - with pytest.raises(ValueError): - normalize_build_variant("huge") - - def test_get_version_info_reads_base_and_prerelease(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) (tmp_path / "NullifyPDF.py").write_text( @@ -54,22 +28,14 @@ def test_get_file_version_prefers_env_suffix_without_duplication(): assert get_file_version("2.1.0", "", "beta.2") == "2.1.0-beta.2" -def test_lite_build_does_not_require_ocr_data(tmp_path, monkeypatch): - monkeypatch.chdir(tmp_path) - - assert pyinstaller_datas("lite") == [] - - -def test_full_build_can_reject_missing_ocr_data_when_download_disabled( - tmp_path, monkeypatch -): +def test_build_rejects_missing_ocr_data_when_download_disabled(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) with pytest.raises(FileNotFoundError): - pyinstaller_datas("full", download_missing_ocr=False) + pyinstaller_datas(download_missing_ocr=False) -def test_full_build_downloads_missing_ocr_data(tmp_path, monkeypatch): +def test_build_downloads_missing_ocr_data(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) def fake_download(output_dir): @@ -83,14 +49,14 @@ def fake_download(output_dir): assert (pathlib.Path("ocr") / "tessdata" / "ita.traineddata").exists() -def test_full_build_includes_ocr_data(tmp_path, monkeypatch): +def test_build_includes_ocr_data(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) tessdata = pathlib.Path("ocr") / "tessdata" tessdata.mkdir(parents=True) (tessdata / "eng.traineddata").write_bytes(b"eng") (tessdata / "ita.traineddata").write_bytes(b"ita") - datas = pyinstaller_datas("full") + datas = pyinstaller_datas() assert ("ocr/tessdata/eng.traineddata", "ocr/tessdata") in datas assert ("ocr/tessdata/ita.traineddata", "ocr/tessdata") in datas diff --git a/tests/test_privacy_core.py b/tests/test_privacy_core.py index 97e2097..b92e4f4 100644 --- a/tests/test_privacy_core.py +++ b/tests/test_privacy_core.py @@ -3,10 +3,13 @@ import pytest from privacy_core import ( + PlaceholderEntry, PlaceholderRegistry, build_restore_payload, decrypt_restore_payload, encrypt_restore_payload, + parse_restore_entries, + sort_entries_longest_placeholder_first, ) @@ -51,3 +54,46 @@ def test_encrypted_restore_map_round_trip(): def test_restore_map_requires_strong_enough_password(): with pytest.raises(ValueError): encrypt_restore_payload({}, "short") + + +def test_parse_restore_entries_round_trip(): + registry = PlaceholderRegistry() + registry.placeholder_for("Mario Rossi", "PERSON", page=0) + payload = build_restore_payload( + source_name="input.pdf", + source_sha256="a" * 64, + output_sha256="b" * 64, + entries=registry.entries(), + ) + + entries = parse_restore_entries(payload) + + assert entries == [ + PlaceholderEntry( + placeholder="PERSON_001", + original="Mario Rossi", + entity_type="PERSON", + page=0, + ) + ] + + +def test_parse_restore_entries_rejects_unknown_format(): + with pytest.raises(ValueError): + parse_restore_entries({"format": "something else", "version": 1}) + + +def test_sort_entries_longest_placeholder_first(): + entries = [ + PlaceholderEntry("PERSON_001", "Ann", "PERSON", page=0), + PlaceholderEntry("PERSON_0010", "Bob", "PERSON", page=0), + PlaceholderEntry("EMAIL_ADDRESS_001", "a@b.com", "EMAIL_ADDRESS", page=1), + ] + + sorted_entries = sort_entries_longest_placeholder_first(entries) + + assert [e.placeholder for e in sorted_entries] == [ + "EMAIL_ADDRESS_001", + "PERSON_0010", + "PERSON_001", + ] diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py new file mode 100644 index 0000000..f772586 --- /dev/null +++ b/tests/test_reconstruct.py @@ -0,0 +1,376 @@ +"""End-to-end test for reconstructing a pseudonymized PDF.""" + +import pathlib +import sys + +import fitz +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) + +from NullifyPDF import RestoreMapMismatch, reconstruct_pdf +from privacy_core import ( + PlaceholderRegistry, + build_restore_payload, + decrypt_restore_payload, + encrypt_restore_payload, +) + + +def _make_pseudonymized_pdf(path: pathlib.Path) -> PlaceholderRegistry: + """Write a one-page PDF with "John Doe" replaced by its placeholder.""" + doc = fitz.open() + page = doc.new_page() + page.insert_text((72, 72), "John Doe lives in Rome.") + + registry = PlaceholderRegistry() + placeholder = registry.placeholder_for("John Doe", "PERSON", page=0) + rect = page.search_for("John Doe")[0] + page.add_redact_annot( + rect, text=placeholder, fill=(1, 1, 1), text_color=(0, 0, 0), align=1, fontsize=8 + ) + page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True) + doc.save(str(path)) + doc.close() + return registry + + +def test_reconstruct_pdf_restores_original_value(tmp_path): + pseudonymized_path = tmp_path / "pseudonymized.pdf" + registry = _make_pseudonymized_pdf(pseudonymized_path) + + from NullifyPDF import sha256_file + + payload = build_restore_payload( + source_name="original.pdf", + source_sha256="a" * 64, + output_sha256=sha256_file(str(pseudonymized_path)), + entries=registry.entries(), + ) + encrypted = encrypt_restore_payload(payload, "correct horse battery staple") + decrypted = decrypt_restore_payload(encrypted, "correct horse battery staple") + + out_path = tmp_path / "reconstructed.pdf" + restored_count = reconstruct_pdf(str(pseudonymized_path), str(out_path), decrypted) + + assert restored_count == 1 + reconstructed_doc = fitz.open(str(out_path)) + text = reconstructed_doc[0].get_text("text") + reconstructed_doc.close() + assert "John Doe" in text + assert "PERSON_001" not in text + + +def test_reconstruct_pdf_restores_repeated_value_on_later_page(tmp_path): + """Regression test: PlaceholderRegistry only records the page of a + value's *first* occurrence, so an entry's `page` field cannot be trusted + to find every occurrence of its placeholder. + """ + doc = fitz.open() + doc.new_page().insert_text((72, 72), "Mario Rossi lives in Rome.") + doc.new_page().insert_text((72, 72), "Contact: Mario Rossi again.") + + registry = PlaceholderRegistry() + placeholder = registry.placeholder_for("Mario Rossi", "PERSON", page=0) + assert placeholder == "PERSON_001" + + for page_index in (0, 1): + page = doc[page_index] + for rect in page.search_for("Mario Rossi"): + page.add_redact_annot( + rect, text=placeholder, fill=(1, 1, 1), text_color=(0, 0, 0), + align=1, fontsize=8, + ) + page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True) + + pseudonymized_path = tmp_path / "pseudonymized.pdf" + doc.save(str(pseudonymized_path)) + doc.close() + + from NullifyPDF import sha256_file + + payload = build_restore_payload( + source_name="original.pdf", + source_sha256="a" * 64, + output_sha256=sha256_file(str(pseudonymized_path)), + entries=registry.entries(), + ) + + out_path = tmp_path / "reconstructed.pdf" + restored_count = reconstruct_pdf(str(pseudonymized_path), str(out_path), payload) + + assert restored_count == 2 + reconstructed_doc = fitz.open(str(out_path)) + text = "\n".join(p.get_text("text") for p in reconstructed_doc) + reconstructed_doc.close() + assert text.count("Mario Rossi") == 2 + assert "PERSON_001" not in text + + +def test_reconstruct_pdf_rejects_hash_mismatch(tmp_path): + pseudonymized_path = tmp_path / "pseudonymized.pdf" + registry = _make_pseudonymized_pdf(pseudonymized_path) + + payload = build_restore_payload( + source_name="original.pdf", + source_sha256="a" * 64, + output_sha256="0" * 64, + entries=registry.entries(), + ) + + out_path = tmp_path / "reconstructed.pdf" + with pytest.raises(RestoreMapMismatch): + reconstruct_pdf(str(pseudonymized_path), str(out_path), payload) + + +def _scanned_page_pdf(path: pathlib.Path) -> None: + """Write a scanner-like PDF: one full-page raster image, no text layer.""" + doc = fitz.open() + page = doc.new_page(width=595, height=842) + scan = fitz.open() + scan_page = scan.new_page(width=595, height=842) + scan_page.insert_text((80, 120), "Cliente: Mario Rossi", fontsize=14) + image = scan_page.get_pixmap(dpi=150).tobytes("jpeg") + scan.close() + page.insert_image(page.rect, stream=image) + doc.save(str(path), garbage=4, deflate=True, clean=True) + doc.close() + + +def test_reconstruct_restores_placeholder_in_narrow_box(tmp_path): + """Regression test for the real-world scanned-PDF failure. + + A redaction box that is narrow and tall (the shape you get drawing over + a short field on a scan) makes `apply_redactions()` wrap the placeholder + across lines: the page still *shows* ``DATA_001``, but + `page.search_for("DATA_001")` finds nothing. Reconstruction used to key + off that search alone, so it restored nothing, reported success and + wrote an output identical to its input -- with no error anywhere. + """ + from NullifyPDF import fit_redaction_fontsize, sha256_file + + scanned = tmp_path / "scan.pdf" + _scanned_page_pdf(scanned) + + narrow_box = fitz.Rect(80, 108, 110, 148) # 30 x 40: narrow and tall + original = "Mario Rossi" + + registry = PlaceholderRegistry() + doc = fitz.open(str(scanned)) + page = doc[0] + placeholder = registry.placeholder_for( + original, + "DATA", + page=page.number, + rect=(narrow_box.x0, narrow_box.y0, narrow_box.x1, narrow_box.y1), + ) + # Deliberately stamp the placeholder at the *unfitted* size the buggy + # export used, so the pseudonymized PDF reproduces the wrapped, + # unsearchable placeholder the user actually has on disk. + page.add_redact_annot( + narrow_box, text=placeholder, fill=(1, 1, 1), text_color=(0, 0, 0), + align=1, fontsize=8, + ) + page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True) + pseudonymized = tmp_path / "pseudonymized.pdf" + doc.save(str(pseudonymized), garbage=4, deflate=True, clean=True) + doc.close() + + # Precondition: the placeholder is on the page but is NOT searchable. + check = fitz.open(str(pseudonymized)) + assert placeholder not in check[0].get_text("text") + assert check[0].search_for(placeholder) == [] + check.close() + + payload = build_restore_payload( + source_name="scan.pdf", + source_sha256="a" * 64, + output_sha256=sha256_file(str(pseudonymized)), + entries=registry.entries(), + ) + out_path = tmp_path / "reconstructed.pdf" + restored_count = reconstruct_pdf(str(pseudonymized), str(out_path), payload) + + assert restored_count == 1 + result = fitz.open(str(out_path)) + text = result[0].get_text("text") + result.close() + # The value is back. It wraps at its space inside the narrow box, which + # is legible and correct -- what matters is that no token is broken. + assert "".join(original.split()) in "".join(text.split()) + assert "Mario" in text and "Rossi" in text + + # And the fitted size stops the export side creating such a box at all. + assert fit_redaction_fontsize(placeholder, narrow_box) < 8.0 + + +def test_reconstruct_v1_map_finds_wrapped_placeholder(tmp_path): + """Version 1 maps carry no coordinates, so the char-level fallback must + find a placeholder that `search_for()` cannot.""" + from NullifyPDF import sha256_file + + scanned = tmp_path / "scan.pdf" + _scanned_page_pdf(scanned) + narrow_box = fitz.Rect(80, 108, 110, 148) + + doc = fitz.open(str(scanned)) + page = doc[0] + page.add_redact_annot( + narrow_box, text="DATA_001", fill=(1, 1, 1), text_color=(0, 0, 0), + align=1, fontsize=8, + ) + page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True) + pseudonymized = tmp_path / "pseudonymized.pdf" + doc.save(str(pseudonymized), garbage=4, deflate=True, clean=True) + doc.close() + + # A map exactly as version 1 wrote them: no `occurrences` key at all. + payload = { + "format": "NullifyPDF restore map", + "version": 1, + "source_name": "scan.pdf", + "source_sha256": "a" * 64, + "output_sha256": sha256_file(str(pseudonymized)), + "entries": [ + { + "placeholder": "DATA_001", + "original": "Mario Rossi", + "entity_type": "DATA", + "page": 0, + } + ], + } + out_path = tmp_path / "reconstructed.pdf" + assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 1 + result = fitz.open(str(out_path)) + text = result[0].get_text("text") + result.close() + assert "MarioRossi" in "".join(text.split()) + + +def test_reconstruct_does_not_report_values_it_failed_to_write(tmp_path): + """An original too long for its box is dropped by `apply_redactions()`. + That must never be counted as a restored value.""" + from NullifyPDF import sha256_file + + doc = fitz.open() + page = doc.new_page(width=595, height=842) + page.insert_text((100, 300), "DATA_001", fontsize=8) + pseudonymized = tmp_path / "pseudonymized.pdf" + doc.save(str(pseudonymized), garbage=4, deflate=True, clean=True) + doc.close() + + payload = { + "format": "NullifyPDF restore map", + "version": 1, + "source_name": "x.pdf", + "source_sha256": "a" * 64, + "output_sha256": sha256_file(str(pseudonymized)), + "entries": [ + { + "placeholder": "DATA_001", + "original": "RSSMRA80C12H501Z-UNBREAKABLE-TOKEN-FAR-TOO-LONG", + "entity_type": "DATA", + "page": 0, + } + ], + } + out_path = tmp_path / "reconstructed.pdf" + # The value cannot fit the tight box; the honest answer is zero. + assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 0 + + +def test_expected_restore_count_counts_occurrences_not_entries(tmp_path): + """A repeated value is one entry but several boxes. The completeness + check must compare against the boxes, or a half-restored repeated value + looks like a full success.""" + from NullifyPDF import expected_restore_count + + registry = PlaceholderRegistry() + registry.placeholder_for("Mario Rossi", "PERSON", page=0, rect=(10, 10, 80, 25)) + registry.placeholder_for("Mario Rossi", "PERSON", page=1, rect=(10, 10, 80, 25)) + registry.placeholder_for("Anna Bianchi", "PERSON", page=0, rect=(10, 40, 80, 55)) + entries = registry.entries() + + payload = build_restore_payload( + source_name="x.pdf", source_sha256="a" * 64, + output_sha256=None, entries=entries, + ) + assert len(entries) == 2 # two distinct values... + assert expected_restore_count(payload) == 3 # ...but three boxes to restore + + # Version 1 maps carry no boxes; each entry still counts for one. + legacy = { + "format": "NullifyPDF restore map", "version": 1, + "source_name": "x.pdf", "source_sha256": "a" * 64, "output_sha256": None, + "entries": [ + {"placeholder": "DATA_001", "original": "a", "entity_type": "DATA", "page": 0}, + {"placeholder": "DATA_002", "original": "b", "entity_type": "DATA", "page": 0}, + ], + } + assert expected_restore_count(legacy) == 2 + + +def test_reconstruct_restores_adjacent_touching_boxes(tmp_path): + """Two different values the user boxed side by side may have touching + redaction boxes. Both must come back -- an overlap guard must not treat + the second as a duplicate of the first.""" + from NullifyPDF import sha256_file + + doc = fitz.open() + page = doc.new_page(width=595, height=842) + page.insert_text((100, 300), "Placeholder area", fontsize=8) + pseudonymized = tmp_path / "pseudonymized.pdf" + doc.save(str(pseudonymized), garbage=4, deflate=True, clean=True) + doc.close() + + registry = PlaceholderRegistry() + # Deliberately touching along x=160, as two adjacent drawn boxes would be. + registry.placeholder_for("Anna", "DATA", page=0, rect=(100, 290, 160, 310)) + registry.placeholder_for("Bruno", "DATA", page=0, rect=(160, 290, 220, 310)) + + payload = build_restore_payload( + source_name="x.pdf", source_sha256="a" * 64, + output_sha256=sha256_file(str(pseudonymized)), entries=registry.entries(), + ) + out_path = tmp_path / "reconstructed.pdf" + assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 2 + + result = fitz.open(str(out_path)) + text = result[0].get_text("text") + result.close() + assert "Anna" in text and "Bruno" in text + + +def test_reconstruct_uses_stored_boxes_across_pages(tmp_path): + """A repeated value records one occurrence per page; all must be restored + from their own recorded box.""" + from NullifyPDF import sha256_file + + doc = fitz.open() + for _ in range(2): + doc.new_page(width=595, height=842).insert_text((100, 300), "x", fontsize=8) + pseudonymized = tmp_path / "pseudonymized.pdf" + doc.save(str(pseudonymized), garbage=4, deflate=True, clean=True) + doc.close() + + registry = PlaceholderRegistry() + for page_index in (0, 1): + placeholder = registry.placeholder_for( + "Mario Rossi", "PERSON", page=page_index, rect=(100, 290, 200, 310) + ) + assert placeholder == "PERSON_001" + assert len(registry.entries()) == 1 + assert len(registry.entries()[0].occurrences) == 2 + + payload = build_restore_payload( + source_name="x.pdf", source_sha256="a" * 64, + output_sha256=sha256_file(str(pseudonymized)), entries=registry.entries(), + ) + out_path = tmp_path / "reconstructed.pdf" + assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 2 + + result = fitz.open(str(out_path)) + text = "\n".join(p.get_text("text") for p in result) + result.close() + assert text.count("Mario Rossi") == 2