diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml index c082ed6..8ca5ce9 100644 --- a/.github/workflows/beta-release.yml +++ b/.github/workflows/beta-release.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-22.04, macos-latest] + os: [windows-latest, macos-latest] steps: - name: Checkout Code @@ -33,21 +33,24 @@ jobs: python-version: '3.13' cache: 'pip' - - name: Install System Dependencies (Linux) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y rpm dpkg binutils libxcb-cursor0 libxkbcommon-x11-0 libegl1 desktop-file-utils - - name: Install Python Dependencies (includes spaCy models) + shell: bash run: | python -m pip install --upgrade pip - pip install -r requirements.txt + # The spaCy models come from github.com release assets, which + # occasionally answer 504: retry instead of failing the release. + for attempt in 1 2 3 4 5; do + pip install -r requirements.txt && exit 0 + echo "Attempt $attempt failed, retrying in 30s..."; sleep 30 + done + exit 1 - name: Run Smoke Tests run: pytest tests/ -v - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py - name: Resolve beta suffix from tag if: startsWith(github.ref, 'refs/tags/') @@ -67,15 +70,24 @@ jobs: path: | dist/NullifyPDF_v*_Windows.exe dist/NullifyPDF_v*_macOS.zip - dist/NullifyPDF_v*_Ubuntu.deb - dist/NullifyPDF_v*_Fedora.rpm + + linux: + name: Linux packages + uses: ./.github/workflows/build-linux.yml + with: + artifact_prefix: beta-build + is_beta: true + retention_days: 5 create_beta_release: name: Create GitHub Beta Release - needs: build + needs: [build, linux] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: + - name: Checkout Code + uses: actions/checkout@v7 + - name: Download All Artifacts uses: actions/download-artifact@v8 with: @@ -91,12 +103,17 @@ jobs: set -euo pipefail TAG="${GITHUB_REF#refs/tags/}" REPO="${{ github.repository }}" - NOTES="NullifyPDF beta release (build from develop). Do not use in production." + # Release body = the matching CHANGELOG section (shown by the in-app + # update check); fall back to a generic text if it is empty. + NOTES_FILE="$(mktemp)" + if ! python3 scripts/extract_release_notes.py "$TAG" > "$NOTES_FILE"; then + echo "NullifyPDF beta release (build from develop). Do not use in production." > "$NOTES_FILE" + fi if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then echo "Release $TAG already exists, uploading/replacing assets." else - gh release create "$TAG" --repo "$REPO" --title "$TAG" --prerelease --notes "$NOTES" + gh release create "$TAG" --repo "$REPO" --title "$TAG" --prerelease --notes-file "$NOTES_FILE" fi # Uploaded one at a time (not via softprops/action-gh-release's @@ -116,6 +133,11 @@ jobs: exit 1 fi + # A release must carry every platform package; never ship a partial one. + for suffix in _Windows.exe _macOS.zip _Ubuntu.deb _Fedora.rpm; do + ls all-builds/NullifyPDF_v*"$suffix" >/dev/null 2>&1 || { echo "Missing release asset *$suffix" >&2; exit 1; } + done + for f in "${files[@]}"; do echo "Uploading $f ..." attempt=1 diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml new file mode 100644 index 0000000..73518c3 --- /dev/null +++ b/.github/workflows/build-linux.yml @@ -0,0 +1,230 @@ +name: Build Linux packages (reusable) + +# Shared by the stable release, the beta release and the test build. +# +# The PyInstaller binary embeds the glibc of the machine that builds it, so it +# is built ONCE inside a RHEL 9 container (glibc 2.34, rpm 4.16). The result +# runs on every distribution with glibc >= 2.34 (Ubuntu 22.04+, Debian 12+, +# RHEL/Rocky/Alma 9-10, current Fedora). Do NOT build it on ubuntu-latest: +# ubuntu-24.04/26.04 produce a binary that does not start on older systems. +# +# build-linux : tests + type check + PyInstaller + .rpm (rockylinux:9) +# build-deb : wraps the same binary into a .deb (needs dpkg-deb) +# smoke : installs each package in a clean container of every +# supported distribution and starts the application there. +# +# Runners are pinned (not ubuntu-latest) so the GitHub migration of +# ubuntu-latest to 26.04 cannot change the build environment. + +on: + workflow_call: + inputs: + artifact_prefix: + description: Prefix of the uploaded artifacts (stable-build, beta-build, test-build) + required: true + type: string + is_beta: + description: Derive NULLIFYPDF_BETA_SUFFIX from the pushed beta tag + required: false + default: false + type: boolean + retention_days: + required: false + default: 5 + type: number + +permissions: + contents: read + +jobs: + build-linux: + name: Build binary and .rpm (RHEL 9) + runs-on: ubuntu-24.04 + container: rockylinux:9 + env: + QT_QPA_PLATFORM: offscreen + steps: + - name: Install system dependencies + # Build tools + the libraries PySide6 needs to import under pytest. + run: | + dnf install -y git tar gzip which findutils binutils python3-pip rpm-build \ + xcb-util-cursor xcb-util-image xcb-util-keysyms xcb-util-renderutil xcb-util-wm \ + libxkbcommon libxkbcommon-x11 mesa-libEGL mesa-libGL fontconfig dbus-libs glib2 + + - uses: actions/checkout@v7 + + - name: Set up Python 3.13 (uv) + run: | + # RHEL 9 AppStream has no python3.13; uv installs a python-build-standalone one. + python3 -m pip install uv==0.8.0 + uv python install 3.13 + uv venv --python 3.13 /opt/venv + echo "/opt/venv/bin" >> "$GITHUB_PATH" + echo "VIRTUAL_ENV=/opt/venv" >> "$GITHUB_ENV" + + - name: Show toolchain + run: | + python --version + ldd --version | head -1 + rpmbuild --version + + - name: Install Python dependencies (includes spaCy models) + run: | + # The spaCy models come from github.com release assets, which + # occasionally answer 504: retry instead of failing the release. + for attempt in 1 2 3 4 5; do + uv pip install -r requirements.txt && exit 0 + echo "Attempt $attempt failed, retrying in 30s..."; sleep 30 + done + exit 1 + + - name: Run tests + run: pytest tests/ -v + + - name: Type check (mypy) + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py + + - name: Resolve beta suffix from tag + if: inputs.is_beta && startsWith(github.ref, 'refs/tags/') + run: | + TAG="${GITHUB_REF#refs/tags/}" + echo "NULLIFYPDF_BETA_SUFFIX=${TAG#*-}" >> "$GITHUB_ENV" + + - name: Build + run: python build_local.py + + - name: Check the build output + run: | + ls -l dist + ls dist/NullifyPDF_v*_Linux_Portable dist/NullifyPDF_v*_Fedora.rpm + + - name: Upload binary (input of the .deb job) + uses: actions/upload-artifact@v7 + with: + name: linux-portable + path: dist/NullifyPDF_v*_Linux_Portable + retention-days: 1 + if-no-files-found: error + + - name: Upload .rpm + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact_prefix }}-Linux-rpm + path: dist/NullifyPDF_v*_Fedora.rpm + retention-days: ${{ inputs.retention_days }} + if-no-files-found: error + + build-deb: + name: Build .deb + needs: build-linux + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + + - uses: actions/download-artifact@v8 + with: + name: linux-portable + path: dist + + - name: Resolve beta suffix from tag + if: inputs.is_beta && startsWith(github.ref, 'refs/tags/') + run: | + TAG="${GITHUB_REF#refs/tags/}" + echo "NULLIFYPDF_BETA_SUFFIX=${TAG#*-}" >> "$GITHUB_ENV" + + - name: Package .deb + run: | + chmod +x dist/NullifyPDF_v*_Linux_Portable + python3 build_local.py --deb-only + ls -l dist + ls dist/NullifyPDF_v*_Ubuntu.deb + dpkg-deb --info dist/NullifyPDF_v*_Ubuntu.deb + + - name: Upload .deb + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact_prefix }}-Linux-deb + path: dist/NullifyPDF_v*_Ubuntu.deb + retention-days: ${{ inputs.retention_days }} + if-no-files-found: error + + smoke: + name: Smoke test ${{ matrix.name }} + needs: [build-linux, build-deb] + runs-on: ubuntu-24.04 + container: ${{ matrix.image }} + # Supported LTS / long-lived distributions. Review this list when one + # approaches its end of support (same policy as MetaLens). Ubuntu 22.04 and + # Debian 12 still run the binary (glibc 2.34 baseline) but are not tested. + strategy: + fail-fast: false + matrix: + include: + - { name: ubuntu-24.04 deb, image: 'ubuntu:24.04', kind: deb } + - { name: ubuntu-26.04 deb, image: 'ubuntu:26.04', kind: deb } + - { name: debian-13 deb, image: 'debian:13', kind: deb } + - { name: rocky-9 rpm, image: 'rockylinux:9', kind: rpm } + - { name: rocky-10 rpm, image: 'rockylinux/rockylinux:10', kind: rpm } + - { name: fedora-latest rpm, image: 'fedora:latest', kind: rpm } + steps: + - name: Install the tools the smoke test needs + run: | + if command -v apt-get >/dev/null; then + apt-get update -qq + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ca-certificates xvfb xauth >/dev/null + else + # RHEL 10 dropped the Xorg server (no Xvfb): there X11 clients run + # on Xwayland inside a headless mutter (xwfb-run). + dnf install -y -q xorg-x11-server-Xvfb xorg-x11-xauth --allowerasing >/dev/null || dnf install -y -q xwayland-run mutter dbus-daemon xorg-x11-xauth --allowerasing >/dev/null + fi + + - uses: actions/download-artifact@v8 + with: + pattern: ${{ inputs.artifact_prefix }}-Linux-* + merge-multiple: true + path: pkgs + + - name: Install the package + run: | + ls -l pkgs + case "${{ matrix.kind }}" in + deb) DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ./pkgs/*.deb ;; + rpm) dnf install -y -q ./pkgs/*.rpm ;; + esac + + - name: The package is installed correctly + run: | + test -x /usr/bin/nullify-pdf + test -f /usr/share/applications/nullify-pdf.desktop + case "${{ matrix.kind }}" in + deb) dpkg -s nullify-pdf | grep -E '^(Version|Depends)' ;; + rpm) rpm -q --qf '%{VERSION}-%{RELEASE}\n' nullify-pdf; rpm -q --requires nullify-pdf ;; + esac + + - name: The application starts and stays up + run: | + # PyInstaller one-file: the first start unpacks Python, Qt and the + # spaCy models, so give it time. Exit status 124 = still running + # when the timeout stopped it = it started correctly. + # A real X server is mandatory: the offscreen Qt platform would not + # load the xcb plugin and could not detect a missing xcb-* dependency. + export QT_QPA_PLATFORM=xcb + if command -v xvfb-run >/dev/null; then + RUN="xvfb-run -a" + elif command -v xwfb-run >/dev/null; then + RUN="xwfb-run -c mutter --" + elif command -v Xvfb >/dev/null; then + Xvfb :99 -screen 0 1280x1024x24 >/tmp/xvfb.log 2>&1 & + sleep 3; export DISPLAY=:99; RUN="" + else + echo "::error::no Xvfb available in this image, cannot test the xcb platform"; exit 1 + fi + set +e + timeout 60 $RUN /usr/bin/nullify-pdf > /tmp/app.log 2>&1 + RC=$? + set -e + echo "exit status: $RC"; cat /tmp/app.log + if grep -qiE "could not (load|find) the Qt platform plugin|xcb-cursor|no Qt platform plugin" /tmp/app.log; then + echo "::error::Qt could not load the xcb platform plugin (missing system library?)"; exit 1 + fi + [ "$RC" -eq 124 ] || { echo "::error::NullifyPDF did not stay running (exit $RC)"; exit 1; } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28d9823..a9a6ed9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,7 +66,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-22.04, macos-latest] + os: [windows-latest, macos-latest] steps: - name: Checkout Code @@ -78,21 +78,24 @@ jobs: python-version: '3.13' cache: 'pip' - - name: Install System Dependencies (Linux) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y rpm dpkg binutils libxcb-cursor0 libxkbcommon-x11-0 libegl1 desktop-file-utils - - name: Install Python Dependencies (includes spaCy models) + shell: bash run: | python -m pip install --upgrade pip - pip install -r requirements.txt + # The spaCy models come from github.com release assets, which + # occasionally answer 504: retry instead of failing the release. + for attempt in 1 2 3 4 5; do + pip install -r requirements.txt && exit 0 + echo "Attempt $attempt failed, retrying in 30s..."; sleep 30 + done + exit 1 - name: Run Smoke Tests run: pytest tests/ -v - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py - name: Build run: python build_local.py @@ -105,15 +108,26 @@ jobs: path: | dist/NullifyPDF_v*_Windows.exe dist/NullifyPDF_v*_macOS.zip - dist/NullifyPDF_v*_Ubuntu.deb - dist/NullifyPDF_v*_Fedora.rpm + + linux: + name: Linux packages + needs: changelog + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/build-linux.yml + with: + artifact_prefix: stable-build + is_beta: false + retention_days: 5 create_release: name: Create GitHub Stable Release - needs: build + needs: [build, linux] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: + - name: Checkout Code + uses: actions/checkout@v7 + - name: Download All Artifacts uses: actions/download-artifact@v8 with: @@ -129,12 +143,17 @@ jobs: set -euo pipefail TAG="${GITHUB_REF#refs/tags/}" REPO="${{ github.repository }}" - NOTES="NullifyPDF stable release. Available for Windows, macOS, and Linux. See CHANGELOG.md for the changes in this version." + # Release body = the matching CHANGELOG section (shown by the in-app + # update check); fall back to a generic text if it is empty. + NOTES_FILE="$(mktemp)" + if ! python3 scripts/extract_release_notes.py "$TAG" > "$NOTES_FILE"; then + echo "NullifyPDF stable release. Available for Windows, macOS, and Linux. See CHANGELOG.md for the changes in this version." > "$NOTES_FILE" + fi if gh release view "$TAG" --repo "$REPO" >/dev/null 2>&1; then echo "Release $TAG already exists, uploading/replacing assets." else - gh release create "$TAG" --repo "$REPO" --title "$TAG" --notes "$NOTES" + gh release create "$TAG" --repo "$REPO" --title "$TAG" --notes-file "$NOTES_FILE" fi # Uploaded one at a time, not via softprops/action-gh-release's @@ -154,6 +173,11 @@ jobs: exit 1 fi + # A release must carry every platform package; never ship a partial one. + for suffix in _Windows.exe _macOS.zip _Ubuntu.deb _Fedora.rpm; do + ls all-builds/NullifyPDF_v*"$suffix" >/dev/null 2>&1 || { echo "Missing release asset *$suffix" >&2; exit 1; } + done + for f in "${files[@]}"; do echo "Uploading $f ..." attempt=1 diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 2cfd83d..11c372e 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -4,6 +4,9 @@ on: pull_request: branches: - main + - develop + schedule: + - cron: "17 5 * * 1" workflow_dispatch: permissions: @@ -17,11 +20,13 @@ concurrency: jobs: build: name: Build for ${{ matrix.os }} + # Develop pull requests and the weekly run only need the canary below. + if: github.event_name != 'schedule' && github.base_ref != 'develop' runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-22.04, macos-latest] + os: [windows-latest, macos-latest] steps: - name: Checkout Code @@ -33,21 +38,24 @@ jobs: python-version: '3.13' cache: 'pip' # CACHE RIPRISTINATA! - - name: Install System Dependencies (Linux) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y rpm dpkg binutils libxcb-cursor0 libxkbcommon-x11-0 libegl1 desktop-file-utils - - name: Install Python Dependencies (includes spaCy models) + shell: bash run: | python -m pip install --upgrade pip - pip install -r requirements.txt + # The spaCy models come from github.com release assets, which + # occasionally answer 504: retry instead of failing the release. + for attempt in 1 2 3 4 5; do + pip install -r requirements.txt && exit 0 + echo "Attempt $attempt failed, retrying in 30s..."; sleep 30 + done + exit 1 - name: Run Smoke Tests run: pytest tests/ -v - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py - name: Build run: python build_local.py @@ -59,6 +67,49 @@ jobs: path: | dist/NullifyPDF_v*_Windows.exe dist/NullifyPDF_v*_macOS.zip - dist/NullifyPDF_v*_Ubuntu.deb - dist/NullifyPDF_v*_Fedora.rpm retention-days: 3 + + linux: + name: Linux packages + if: github.event_name != 'schedule' && github.base_ref != 'develop' + uses: ./.github/workflows/build-linux.yml + with: + artifact_prefix: test-build + is_beta: false + retention_days: 3 + + pymupdf-latest: + # Redaction semantics (the glyph-deletion core, word boxes) differ between + # PyMuPDF releases; this job reports early when a newer release changes + # them. It is not a required check, but a failure is shown as a failure. + name: Tests against the latest PyMuPDF + runs-on: ubuntu-24.04 + env: + QT_QPA_PLATFORM: offscreen + steps: + - name: Checkout Code + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install System Dependencies (Linux) + run: sudo apt-get update && sudo apt-get install -y libxcb-cursor0 libxkbcommon-x11-0 libegl1 + + - name: Install Python Dependencies + shell: bash + run: | + python -m pip install --upgrade pip + for attempt in 1 2 3 4 5; do + pip install -r requirements.txt && break + [ "$attempt" = 5 ] && exit 1 + echo "Attempt $attempt failed, retrying in 30s..."; sleep 30 + done + pip install --upgrade pymupdf + python -c "import pymupdf; print(pymupdf.VersionBind, pymupdf.VersionFitz)" >> "$GITHUB_STEP_SUMMARY" + + - name: Run Tests + run: pytest tests/ -v diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2a4b77d..f9e324d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -26,8 +26,22 @@ It supports two privacy export modes: | ๐Ÿ“„ File | โœจ Responsibility | | ------ | ----------------- | | `NullifyPDF.py` | Main PySide6 UI, PDF rendering, AI scan orchestration, redaction, export, reconstruction, OCR integration | -| `pii_detection.py` | Qt-free AI detection helpers: word-box text index, offset-to-rect mapping, per-type score thresholds, generic-word filtering, whole-word propagation, entity selection persistence | -| `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads (version 3, optional per-occurrence text style), restore-map parsing | +| `update_check.py` | Qt-free update logic: version comparison (SemVer, stable/beta channels), GitHub release lookup, per-OS asset selection, trusted-host download, startup-check preference (`~/.nullifypdf/update_check.json`). Only runs on user click or if the user enabled the startup check | +| `pii_detection.py` | Qt-free AI detection helpers: word-box text index (neighbour-aware rectangle clamping, ligatures, letter-spaced and hyphenated words), offset-to-rect mapping and partial-match policy, per-type thresholds, pattern validation, evidence-based PERSON/LOCATION scoring, accepted/suggested states, allowlist by word identity, whole-word propagation, entity selection persistence | +| `privacy_core.py` | Privacy mode enum, placeholder registry (random / sequential / generic / keyed styles), encrypted restore-map payloads (version 3, optional per-occurrence text style; scrypt, PBKDF2 maps still readable), password strength check, restore-map parsing | +| `lexicon_it.py` | Italian lexicon: certificate/legal verb forms, medical and administrative words, given names, regions and provinces (used to reject ordinary words and to confirm names/places) | +| `pdf_sanitize.py` | Export sanitiser (annotations, attachments, scripts, hidden layers/text, off-page text, metadata, optional TJ-gap quantisation) and `verify_export()` | +| `export_engine.py` | Qt-free export pipeline (`export_document`), neutral file names, private temp handling, audit record, local-data wipe | +| `review_dialog.py` | Qt dialog to accept or reject suggested detections | +| `model_manager.py` | Pinned Hugging Face model catalog, SHA-256 verification, local model store and hardware recommendation | +| `model_dialog.py` | Italian model selection, consent and progress-aware downloads | +| `onnx_ner.py` | CPU ONNX token-classification adapter with character-offset mapping and label mapping | +| `word_import.py` | Safe DOCX OOXML text/table extraction into an internal PyMuPDF document; no macro execution | +| `help_guide.py` | In-app guide (Italian, F1) and the tooltip table applied to every widget | +| `secure_store.py` | Master key protected by DPAPI / OS keyring / key file; AES-GCM sealing; atomic private writes | +| `word_lists.py` | Encrypted blocklist/allowlist with expiry (`PDFListManager`) | +| `dictionary_dialog.py` | Qt editor for both lists with dates and default expiry | +| `lexicon_data_it.py` | Generated by `scripts/build_lexicon_data.py` from ISTAT open data (municipalities, provinces, regions) and, with `--wikidata`, Wikidata (given names, surnames); see `THIRD_PARTY_NOTICES.md`; imported lazily and used only to confirm names and places | | `PDF_Checker.py` | Heuristic post-export inspection helper | | `build_local.py` | PyInstaller build script | | `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files bundled into every build | @@ -68,11 +82,11 @@ Export pipeline writes a cleaned PDF copy | ๐Ÿงฑ Class | ๐Ÿ“ Location | ๐ŸŽฏ Purpose | | ------- | ----------- | ---------- | -| `PDFListManager` | `NullifyPDF.py` | Loads and saves blocklist/allowlist files in `~/.nullifypdf` | +| `PDFListManager` | `word_lists.py` | Loads and saves the blocklist/allowlist as encrypted files (`*.v2.enc`) with per-entry expiry; migrates plaintext files | | `AIWorker` | `NullifyPDF.py` | Background NLP/OCR worker that emits page-level detections | | `PDFView` | `NullifyPDF.py` | Custom `QGraphicsView` for rendering pages, drawing rectangles, and zooming | | `NullifyPDF` | `NullifyPDF.py` | Main window and application controller | -| `PlaceholderRegistry` | `privacy_core.py` | Creates stable placeholders such as `PERSON_001` and `EMAIL_ADDRESS_001` | +| `PlaceholderRegistry` | `privacy_core.py` | Creates stable placeholders such as `PERSON_9F2A` (random), `PERSON_001` (sequential), `DATA_9F2A` (generic) or an HMAC-derived one (keyed) | --- @@ -89,20 +103,20 @@ Export pipeline writes a cleaned PDF copy ### ๐Ÿค– AI Scan -1. User clicks **Auto Redact (AI)** and, in a dialog, picks which data types to look for (names, locations, email, phone, IBAN, cards, crypto, fiscal code, VAT, licence, ID card, passport). The selection is remembered in `~/.nullifypdf/ai_entities.json`. -2. User selects language (`EN`, `IT`, or `BOTH`) and optionally enables: +1. User clicks **Auto Redact** and, in a dialog, picks which data types to look for (names, locations, email, phone, IBAN, cards, crypto, fiscal code, VAT, licence, ID card, passport). The selection is remembered in `~/.nullifypdf/ai_entities.json`. +2. The language is always Italian (`IT`); the user optionally enables: - `OCR PDF scansionati` - `Oscura Immagini` 3. `cmd_auto_ai()` checks OCR configuration if OCR is enabled. 4. `AIWorker.run_scan()` processes pages in a background thread. 5. For each page: - - the analysis text is built by `pii_detection.build_page_index()` from `page.get_text("words")`, recording each word's rectangle so character offsets map back to exact boxes - - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` and the same index is built from the OCR words - - Presidio analyzes the text for the enabled entity types - - results are filtered by per-type confidence thresholds; NER spans are split at line breaks; generic PERSON/LOCATION hits (titles, months, salutations, lowercase, role and identifier words) are dropped; single-word hits that look like labels or headings (followed by a colon, list items, lone heading lines, words also written in lowercase on the page, capitalized line starts) are rejected; a name glued to a following address is cut at the street word; overlapping spans are resolved - - each surviving value is located by its offsets, and its other occurrences on the page are added as **whole words only** - - detections are emitted with text, entity type, score, source, and their rectangles -6. `apply_ai_to_page()` runs on the UI thread and creates pending redaction annotations from the received rectangles. Blocklist and allowlist terms are matched as whole words too. + - the analysis text is built by `pii_detection.build_page_index()` from `page.get_text("words")`, recording each word's rectangle so character offsets map back to exact boxes. Each rectangle is trimmed against the words of neighbouring lines (a word box spans ascender to descender and would otherwise overlap them); ligatures are expanded, letter-spaced runs become one word and a word hyphenated across two lines is joined + - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` and the same index is built from the OCR words; on text pages with embedded images only the images are OCR'd and merged + - Presidio analyzes the text for the enabled entity types; date of birth and health identifiers are found by context-anchored patterns + - `classify_page()` filters the results: per-type thresholds; pattern hits are validated (token boundaries, context words, uppercase form, plausibility); NER spans are split at line breaks and names on consecutive lines are merged; ordinary Italian words are stripped from the edges of a span; fragments and OCR noise are rejected; field labels and list items are rejected; each remaining PERSON/LOCATION hit gets an evidence score and becomes `accepted`, `suggested` or is dropped (in health mode never dropped) + - `build_detections()` resolves each candidate to words with the partial-match policy (NER hits must not be a fragment of a longer word; a checksum-valid identifier glued to other text is redacted whole), removes words protected by the allowlist (by word identity), and adds other occurrences as **whole words only** + - detections are emitted with text, entity type, score, state, reasons, source and their rectangles +6. `apply_ai_to_page()` runs on the UI thread: `accepted` detections become pending redaction annotations (a rectangle already covered by at least 60% is not added twice); `suggested` ones are queued and reviewed in `ReviewDialog`. Blocklist terms are matched as whole words. ### ๐Ÿ–Š๏ธ Manual Review @@ -125,26 +139,27 @@ Clicking an existing redaction: `cmd_export()` is the main export entry point. -1. User chooses privacy mode: - - irreversible anonymization +1. If suggestions are unreviewed, the user is asked to review them or continue without them. +2. User chooses privacy mode: + - irreversible removal - reversible pseudonymization -2. User chooses output PDF path. -3. For pseudonymization only: - - user chooses `.nullifypdf-map` path - - user enters a password of at least 12 characters - - `cryptography` availability is checked before writing output -4. The in-memory PDF is saved to a sibling temp file: - - `.nullifypdf.tmp` -5. The temp PDF is reopened as `ex_doc`. -6. Each page is processed: - - overlapping links are deleted for redacted areas - - redactions are applied with PyMuPDF +3. User chooses the output path (default name contains nothing from the source name; a warning appears if the chosen name contains a removed value). +4. For pseudonymization only: + - user chooses the placeholder style (random, sequential, generic, keyed) + - user chooses the `.nullifypdf-map` path (a warning appears for the PDF's folder or a synchronised folder) + - user enters a password twice; it must pass a strength check (minimum 12 characters) +5. `export_engine.export_document()` works on a copy of the document kept in memory (or, above a size threshold, in a private per-user temp folder): + - links overlapping redactions are deleted + - redactions are applied with PyMuPDF (image pixels blanked, not whole images) - widgets are removed -7. Document metadata and selected catalog keys are cleared. -8. The final PDF is saved with cleanup options. -9. The temp file is removed. + - `pdf_sanitize.sanitize_document()` removes annotations, attachments, scripts, bookmarks, structure, metadata, hidden/invisible/off-page text and external links + - the final PDF is saved with cleanup options +6. Before the redactions are applied, every band that stores the whole box of its word is checked; after applying them, the number of glyphs that vanished from each line must match the measured deletion rule (a glyph is deleted when the rectangle reaches its box inset by 10%), and target glyphs the band cannot reach are counted. Any survivor is reported without the value. +7. `pdf_sanitize.verify_export()` re-opens the written file and checks it; a failure is reported and the file can be deleted. +8. For pseudonymization, the restore map is encrypted and written with owner-only permissions. +9. An audit record (no values) is written to `~/.nullifypdf/audit/`. -The original open document remains editable after export because the export works on a disk-backed copy. +The original open document remains editable after export because the export works on a copy. --- @@ -191,7 +206,7 @@ The restore map contains: - the redaction box of every occurrence - for native (non-scanned) PDFs, the typography of the redacted text: font, size, colour, flags and baseline (map version 3, optional) -The map is encrypted with `cryptography` using Fernet and a key derived from the user password with PBKDF2-HMAC-SHA256. +The map is an envelope (version 3): a random data key encrypts the payload with AES-256-GCM and is wrapped by a key derived from the password with Argon2id (t=3, m=64 MiB, p=4; scrypt if Argon2id is unavailable) and, optionally, by a 256-bit recovery key. Maps created with scrypt + Fernet (version 2) or PBKDF2-HMAC-SHA256 (version 1) still open; the parameters read from a file are bounded before use. Each occurrence may carry the compact label that was drawn when the full placeholder did not fit the box (map version 4). > [!IMPORTANT] > The restore map is never embedded in the pseudonymized PDF. @@ -209,10 +224,7 @@ Runtime behavior: - bundled `ocr/tessdata` via `resource_path()` - PyMuPDF's `fitz.get_tessdata()` - common platform locations -2. `ocr_language_for_choice()` maps UI language to Tesseract codes: - - `EN` -> `eng` - - `IT` -> `ita` - - `BOTH` -> `eng+ita` +2. `ocr_language_for_choice()` returns the Tesseract codes: `ita`, plus `eng` when its data is installed (so English words inside Italian documents are read well). 3. `AIWorker._page_needs_ocr()` runs OCR only when text extraction is missing or very poor. 4. OCR text is used for Presidio detection. 5. OCR search rectangles are used to place redactions on scanned pages. @@ -258,11 +270,13 @@ GitHub Actions build one artifact per supported OS: - Windows - macOS -- Ubuntu -- Fedora +- Ubuntu / Debian (`.deb`) +- Fedora / RHEL (`.rpm`) Beta releases (`v*-beta.N` tags) and stable releases (`vX.Y.Z` tags) are each compiled from their own tag, because the application embeds its version string at build time. The stable workflow first checks that `CHANGELOG.md` has a `[X.Y.Z]` entry with an empty `[Unreleased]` section and that `NullifyPDF.py` declares that exact version with no prerelease suffix. +**Linux packages** are built by the reusable workflow `build-linux.yml`, shared by the stable, beta and test builds. The PyInstaller binary and the `.rpm` are built once inside a `rockylinux:9` container (glibc 2.34, rpm 4.16), the `.deb` is wrapped from the same binary on `ubuntu-24.04`, and each package is then installed and started under Xvfb in clean Ubuntu 24.04, Ubuntu 26.04, Debian 13, Rocky 9, Rocky 10 and Fedora containers. A release is published only if this smoke test passes. Runners are pinned, so the move of `ubuntu-latest` to a new Ubuntu release does not change the build. When a distribution approaches end of support, update the matrix in `build-linux.yml` (and the list in the README). + Expected artifact names include: ```text @@ -309,5 +323,5 @@ pytest tests/ -v --- -*Last updated: 2026-09-19* +*Last updated: 2026-11-11* *[Back to README](./README.md) | [Development โ†’](./DEVELOPMENT.md)* diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f695f6..c037730 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,74 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.4.0] - 2026-11-11 + +### ๐Ÿ”„ Changed + +- **Italian only**: the English model and the EN/IT/BOTH selector are gone. The program analyses Italian documents; the sidebar shows "Lingua dei documenti: italiano". The English OCR data is still used, when installed, only to read English words inside Italian pages. +- **Model download progress in the button**: in the local-models window the "Scarica modello" button now fills like a progress bar with the percentage, a second click cancels, the list and the Close button are locked while downloading, and the download no longer blocks the window. The empty grey button of the old progress window is gone. +- **New licence: AGPL-3.0-or-later** (previously MIT), because the program uses PyMuPDF, which is AGPL-3.0 or commercial. The program stays free and is not sold. Full third-party licence list in `THIRD_PARTY_NOTICES.md`. +- **Licence notice in the Info window and README**: the Info window now lists the licence of NullifyPDF and of every component and model it uses, with clickable links to the official texts. +- **Italian name-recognition model downloaded on demand**: its licence (CC BY-NC-SA 3.0, non-commercial) cannot be bundled in an AGPL installer, so the first Italian scan shows the licence and asks for consent, then downloads and verifies the model (SHA-256). Detection quality is unchanged once installed; if you decline, the scan continues with rules and dictionaries only and the log says so. + +### โœจ Added + +- **Local PII models**: optional, user-downloaded ONNX models from pinned Hugging Face revisions are stored under `~/.nullifypdf/models/`, verified file-by-file with SHA-256, and contribute suggestions only. The model dialog shows the source, license, revision and download progress; unverified experimental entries remain blocked. +- **Word DOCX import**: local DOCX text and table content can be imported into a PDF representation for the existing detection and review workflow. Macros, external links and embedded objects are not executed. +- **User guide in the program**: a new "Guida" button (or F1) opens a guide in Italian that explains the whole workflow: opening a PDF, language and options, Auto Redact, review of suggestions, manual corrections, dictionaries, export modes, reconstruction, health documents, local data and shortcuts. +- **Tooltips on every element**: all buttons, options, fields, tables and indicators of the main window and of the dialogs now show a short description when the pointer rests on them. +- **Check after redaction for surviving characters**: besides searching the written file, the export now checks that every character of a redacted word was really deleted, and reports it (without the value) if one was not. +- **Check for updates**: the Info dialog has a new "Controlla aggiornamenti" button. It queries the GitHub releases of this repository (on click, or at startup only if you enabled the opt-in startup check) and, if a newer version exists, shows the release link, the release notes and a **Download** button that saves the installer for the running OS into the user's system Downloads folder (resolved per platform), verifying its size and the SHA-256 published by GitHub. Linux distributions outside the Debian/Ubuntu and Fedora/RHEL families only get the release link. Stable builds are only offered newer stable releases; beta builds are offered both stable and beta releases. +- **Optional check at every start**: a "Controlla ad ogni avvio" checkbox in the Info dialog (off by default, stored in `~/.nullifypdf/update_check.json`) runs the same check silently at startup and opens the update window, with the release notes, only when a newer version exists. Failures are logged, never shown. +- **Repository link in the Info dialog**. +- **Review of uncertain detections**: hits that the evidence score cannot settle (a lone capitalized word, a name without context) are no longer applied silently or lost. They are queued as suggestions in a review dialog (values masked until requested, per-row accept, select by score, double-click to jump to the page). "Esporta PDF" warns when suggestions are still unreviewed. +- **Health-document profile**: a sidebar option that also detects dates of birth and health identifiers (health card number, prescription number) anchored on context words, and keeps every uncertain hit as a suggestion instead of dropping it. +- **Export verification**: after writing the output, NullifyPDF re-opens it and checks that no removed value is still readable from the text layer, any object or decompressed stream, annotations, attachments, bookmarks or metadata, and that no annotation, script or attachment remains. If a check fails the export is flagged and the file can be deleted. +- **Export sanitiser**: annotations, embedded files, external links, bookmarks, JavaScript, tagged-PDF alternative text, document and XMP metadata, thumbnails, invisible text, text between the crop box and the media box, and hidden-layer text are removed or neutralised on export. Hidden layers are made visible before the scan so their text is analysed. +- **Pseudonym styles**: random (default; reveals neither order nor count), sequential, generic (also hides the entity type), and keyed (the same value gets the same pseudonym across documents made with the same key). +- **Optional width masking** ("Maschera larghezza testo rimosso"): rounds the gap left by removed text so that its width cannot be recovered from the content stream. +- **Encrypted dictionaries with expiry**: the blocklist and allowlist are stored encrypted (Windows DPAPI, the OS keyring if the optional `keyring` package is installed, otherwise a local key file) and every term can expire. The dictionary dialog is now a table with added/expiry dates, a default expiry for new terms, import from text and an explicit plain-text export. Existing plaintext lists are migrated and overwritten. If the keystore cannot be read on another account or PC, nothing is deleted and the lists stay empty until it is restored. +- **Recovery key for restore maps**: when pseudonymizing you can create a recovery key (shown once, never stored) that opens the map if the password is lost. New maps use a data key wrapped by the password (Argon2id) and, optionally, by the recovery key. Maps made by older versions still open; maps made by this version need this version or later. +- **Pseudonym key protected and portable**: the key of the "keyed" pseudonym style is stored encrypted, and a password-protected copy can be exported and imported to use the same pseudonyms on another computer. +- **Signature and stamp suggestions**: with the health-document profile, curve-rich ink and small images next to "firma", "timbro", "il medico"... are proposed for review (never applied automatically). +- **Postcodes and house numbers**: a street and its house number are redacted as one address, and postcodes are recognised only in an address context (never in prices, dates or quantities). Health card number, prescription number (NRE) and exemption codes follow the published structure (length, region code, lot code); the check digit is not verified, and the prescription number and exemption codes need their label. +- **Suggestions in the page preview**: pending suggestions are outlined in orange on the page (view only; they never reach an export). Click one to censor or discard it; selecting a row in the review dialog highlights it. +- **Given names and surnames from Wikidata** (CC0): a few hundred given names and about one thousand surnames are used to confirm names, never to reject them; names that are also ordinary words count less. Coverage is small. +- **ISTAT place names**: Italian municipalities, provinces and regions (ISTAT open data, CC BY 4.0) are used to confirm places; a place missing from the list is never rejected for that reason. +- **Export audit record** in `~/.nullifypdf/audit/` (counts per type, sanitiser actions, verification result, output SHA-256; no personal data), and an "Elimina dati locali" button that wipes logs, audit records and temporary files. + +### ๐Ÿ› Fixed + +- **Download and redaction geometry**: model downloads now show an overall percentage, and accepted word redactions use the complete original word box with a small safety margin so the glyph is covered in both dimensions. Suggestion overlays show the same complete word area. +- **Redaction boxes covered parts of the lines above and below**: word boxes span ascender to descender, so with normal line spacing a redaction rectangle overlapped neighbouring lines, blackening parts of their letters in the preview and deleting their glyphs on export. Rectangles are now trimmed against the neighbouring lines (at the middle of the overlap on image pages, with a small horizontal margin). +- **Ordinary Italian words proposed for redaction** ("Certifico", "Prescrivo", "Dichiaro", headings of medical and legal documents): PERSON/LOCATION hits are now judged from evidence in the text (titles and cues such as "nato a" or "residente in", name shape, known given names and places, repetition) and rejected when they are Italian verb forms or common medical, legal and administrative words, including at the start or end of a longer span. Names split over two lines ("Esposito" / "Giovanni") are recognised as one name. +- **Weak identifier patterns produced false hits**: fiscal codes, VAT numbers, wallets, card numbers, driver licences, passports, ID cards and phone numbers must now sit on token boundaries, and the weak ones need their context word and uppercase form; dates, year ranges and short digit runs are no longer read as phone numbers. A checksum-valid identifier glued to a label is redacted as a whole token instead of being dropped. +- **Allowlist could hide real personal data**: an allowlist entry was matched as a substring of the detected value and by rectangle overlap, so "rossini" hid "Rossi" and an allowlisted word on the next line cancelled a detection. The allowlist now works by word identity, and an allowlisted word is only removed from a longer detection. +- **Substring fallback removed**: a detection without geometry is matched as whole words only. +- **Text-layer artefacts**: ligatures are expanded, letter-spaced text is read as one word, and a surname hyphenated across two lines is detected and redacted on both lines. +- **Restore map is no longer written beside the PDF by default and stores less**: it omits the source file name and the duplicated original text, is written with owner-only permissions where supported, uses scrypt (maps made with PBKDF2 still open), requires a confirmed password that passes a strength check, and bounds the key-derivation parameters read from the file. +- **No unredacted copy on disk during export**: the working copy is kept in memory (or in a private per-user folder for very large files, cleaned at start-up), no longer in `.nullifypdf.tmp` beside the output. +- **Neutral file names**: the default output and map names no longer derive from the source file name; a warning appears if the chosen name contains a value being removed. +- **Defaults and wording**: the language model defaults to Italian on an Italian system (otherwise both models); images embedded in text pages are analysed with OCR; the interface no longer calls redaction "anonymization" and states that the output and the restore map may still contain personal data; the scan log line no longer claims anonymization is complete. +- **Tight line spacing**: the redaction box now stops short of the neighbouring lines' glyphs instead of at their box edge, so text at 11 pt on a 7 pt pitch is redacted without damaging the lines around it. When no safe band exists a warning is shown. +- **Placeholders were silently dropped in small boxes**: the placeholder is now drawn after the redactions are applied, with a compact form ("P9F2A", "9F2A") when the full one does not fit, so short words keep a visible placeholder. Restore maps are version 4 (version 1-3 maps still load). +- **Restore wrote neighbouring text back**: the typography captured for a redaction could include fragments of the lines above and below when the leading was tight; only the redacted line is captured, and the restore writes the original value. +- **Wipe of local data left the active log file**, and "Oscura Immagini" added duplicate image redactions on every apply. +- **Labels on images and signatures survive small boxes**: the "[IMMAGINE RIMOSSA]" / "[FIRMA RIMOSSA]" labels are drawn after the redactions, with shorter forms when the area is small, instead of being dropped. +- **Master key in the system keyring on macOS and Linux**: when a keyring is available the encryption key of the dictionaries is stored there instead of in a local key file (Windows keeps using DPAPI). +- **Local logs are rotated** (512 KiB, 3 files, 30 days) instead of growing indefinitely. +- **Redacted text could remain in the PDF content stream**: with PyMuPDF 1.28.x, `apply_redactions()` did not remove text drawn with the `'` operator after `TL`/`Td`; the text disappeared from the extracted text but was still stored in the file. Fixed by PyMuPDF 2.x; covered by a regression test that scans every stream of the exported file. + +### ๐Ÿ”ง Changed + +- **Linux packages are verified on the supported distributions**: Ubuntu 24.04 and 26.04, Debian 13, RHEL/Rocky 9 and 10 and current Fedora (installation and application start). The binary targets glibc 2.34, so older systems such as Ubuntu 22.04 and Debian 12 should keep working too. +- **Linux packages declare their runtime libraries**: `.deb` `Depends` and `.rpm` `Requires` (xcb, EGL/GL, fontconfig, D-Bus, glib; `libglib2.0-0t64 | libglib2.0-0` for Ubuntu 24.04+), the `.deb` files are owned by root, and the `.rpm` no longer strips the PyInstaller executable. +- **Restore maps have a new format**: maps created by this version (and their keys) cannot be opened by 2.4.0-beta.1 or earlier; maps from earlier versions still open here. +- **Beta packages sort before the stable one** (`2.4.0~beta.1` for `.deb`, Release `0.beta.1.1` for `.rpm`). +- **Dependency updates**: PySide6 6.12.0, presidio-analyzer 2.2.366, spaCy 3.8.18, cryptography 50.0.2, pytest 9.2.0. Verified with the full test suite, `mypy` and an offscreen end-to-end scan/redaction run. +- **PyMuPDF 2.1**: required for the redaction fix above. Version 2.1 no longer loads the layout analyzer on import, so start-up time and installer size are unaffected (verified with a PyInstaller bundle). +- **Removed the "AI" wording from the GUI and packaging metadata** (sidebar subtitle, language-model label, the "Auto Redact" button, the Info version line, installer descriptions): detection relies on Presidio/spaCy NLP and rules, not on a generative AI model, so the label was misleading. + ## [2.3.0] - 2026-09-19 Validated through the 2.3.0-beta.1/beta.2/beta.3 prereleases before this stable release, promoted from `v2.3.0-beta.3` without rebuilding; only the final stable version gets its own heading here. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a04ef0..5f101e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -219,7 +219,7 @@ thread when scanning large PDFs concurrently." ### Before Opening PR 1. โœ… Run tests: `pytest tests/ -v` -2. โœ… Type check (blocking in CI): `mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py` +2. โœ… Type check (blocking in CI): `mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py` 3. โœ… Build locally: `python build_local.py` 4. โœ… Update CHANGELOG.md with your changes 5. โœ… Self-review your code (would you understand this in 6 months?) @@ -379,5 +379,9 @@ Every contribution โ€” code, docs, bug reports, ideas โ€” helps make NullifyPDF --- -*Last updated: 2026-09-19* +*Last updated: 2026-11-11* *โ† [Back to README](./README.md) | [Architecture โ†’](./ARCHITECTURE.md)* + +## Licence of contributions + +NullifyPDF is licensed under the AGPL-3.0-or-later (see [LICENSE](./LICENSE)). By submitting a contribution you agree that it is released under the same licence. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3215ed1..36287ac 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -16,7 +16,7 @@ Before starting, verify you have: | **Python 3.13** | 150 MB | [Download](https://www.python.org/downloads/) | | **Git** | 50 MB | [Download](https://git-scm.com/) | | **Virtual Environment** | 2 GB | `.venv/` auto-created by setup script | -| **Disk Space** | 3 GB | Dependencies + spaCy models | +| **Disk Space** | 3 GB | Dependencies + Italian model | | **RAM** | 4 GB | 8 GB recommended | --- @@ -51,7 +51,7 @@ python setup_env.py This automatically: - โœ… Creates `.venv/` virtual environment - โœ… Installs dependencies -- โœ… Downloads spaCy models (EN + IT) +- โœ… Installs the Italian spaCy model (development only) ### 4๏ธโƒฃ Activate Virtual Environment @@ -105,7 +105,12 @@ If the GUI opens โ†’ **You're ready to develop!** ๐ŸŽ‰ ``` NullifyPDF/ |-- NullifyPDF.py # Main PySide6 application +|-- model_manager.py # Pinned local model catalog and integrity checks +|-- model_dialog.py # Model consent and download UI +|-- onnx_ner.py # ONNX token-classification adapter +|-- word_import.py # Safe DOCX text/table importer |-- privacy_core.py # Privacy modes and encrypted restore maps +|-- update_check.py # Qt-free update check: versions, GitHub releases, asset choice, download |-- pii_detection.py # Qt-free AI detection helpers (offsets, filters, whole-word matching) |-- PDF_Checker.py # Heuristic verification utility |-- setup_env.py # Environment setup script @@ -136,21 +141,28 @@ NullifyPDF/ | Package | Version | Purpose | | --------------------- | ------- | ---------------------------- | -| **PySide6** | 6.11.2 | GUI framework (Qt6 bindings) | -| **PyMuPDF** | 1.28.2 | PDF manipulation and OCR bridge | -| **presidio-analyzer** | 2.2.364 | PII detection | -| **spaCy** | 3.8.16 | NLP for entity recognition | -| **cryptography** | 50.0.1 | Encrypted restore maps | -| **pytest** | 9.1.1 | Testing framework | - -### Language Models (Auto-Downloaded) - -| Model | Size | Purpose | -| ----------------- | ----- | ----------- | -| `en_core_web_md` | 40 MB | English NER | -| `it_core_news_md` | 45 MB | Italian NER | - -These are downloaded automatically by `setup_env.py`. +| **PySide6** | 6.12.0 | GUI framework (Qt6 bindings) | +| **PyMuPDF** | 2.1 | PDF manipulation and OCR bridge | +| **presidio-analyzer** | 2.2.366 | PII detection | +| **spaCy** | 3.8.18 | NLP for entity recognition | +| **cryptography** | 50.0.2 | Encrypted restore maps | +| **onnxruntime** | 1.31.1 | CPU inference for optional local PII models | +| **tokenizers** | 0.23.3 | Fast local tokenizer with character offsets | +| **python-docx** | โ€” | Not required; DOCX import uses the standard-library OOXML reader | +| **pytest** | 9.2.0 | Testing framework | + +### Language Models + +| Model | Size | Purpose | Licence | In the installer | +| ----------------- | ----- | ----------- | ------- | ---------------- | +| `it_core_news_md` | 45 MB | Italian NER | CC BY-NC-SA 3.0 | **no**, downloaded by the app on demand (`spacy_models.py`) | + +`setup_env.py` installs both in the development environment (the Italian tests +need it). The PyInstaller specs exclude `it_core_news_md`, `pymupdf4llm` and +`pymupdf.layout`. At run time the app asks for consent, downloads the Italian +pipeline from the publisher's release, checks its size and SHA-256 and stores it +under `~/.nullifypdf/spacy_models/`. If the user declines, a blank Italian +pipeline is used and only rule-based detection is available. --- @@ -162,7 +174,7 @@ These are downloaded automatically by `setup_env.py`. ```python NullifyPDF(QMainWindow) -|-- PDFListManager # Manages blocklist/allowlist +|-- PDFListManager # (word_lists.py) encrypted blocklist/allowlist |-- AIWorker(QObject) # NLP/OCR scanning in thread |-- PDFView(QGraphicsView) # PDF rendering and rectangle drawing |-- privacy_core.py # Privacy modes and encrypted restore maps @@ -230,7 +242,7 @@ Opens `htmlcov/index.html` in browser. CI runs `mypy` as a blocking step. Run the same check locally before opening a PR: ```bash -mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py +mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py ``` PyMuPDF is imported as `pymupdf` (aliased to `fitz`) so it is really type-checked; its dynamically defined constants are read once near the top of `NullifyPDF.py`. @@ -272,7 +284,7 @@ python build_local.py | ----------- | --------------------------------- | | **Windows** | `.exe` executable | | **macOS** | `.app` bundle ZIP | -| **Linux** | Portable binary + `.deb` + `.rpm` packages | +| **Linux** | Portable binary + `.deb` + `.rpm` packages (CI builds them in a RHEL 9 container, see `build-linux.yml`; `python build_local.py --deb-only` re-packages an existing binary as `.deb`) | ### Troubleshooting Build Issues @@ -452,7 +464,16 @@ isort NullifyPDF.py | ------------------ | ----------------------- | | `NullifyPDF.py` | Main app, GUI, OCR, export logic | | `privacy_core.py` | Placeholder and restore-map logic | -| `pii_detection.py` | AI detection precision: word-box index, thresholds, filters, entity selection | +| `pii_detection.py` | AI detection precision: word-box index, rectangle clamping, thresholds, pattern validation, evidence scoring, entity selection | +| `lexicon_it.py` | Italian word lists used to reject ordinary words / confirm names and places | +| `pdf_sanitize.py` | Export sanitiser and output verification | +| `export_engine.py` | Qt-free export pipeline, audit record, local-data hygiene | +| `review_dialog.py` | Review dialog for suggested detections | +| `secure_store.py` | Master key protection (DPAPI / keyring / key file), AES-GCM sealing | +| `word_lists.py` | Encrypted blocklist/allowlist with expiry | +| `dictionary_dialog.py` | Dictionary editor dialog | +| `help_guide.py` | In-app guide and tooltip table (keep the guide in sync with the UI: `tests/test_gui_help.py` checks it) | +| `scripts/build_lexicon_data.py` | Generates `lexicon_data_it.py` from ISTAT open data and (with `--wikidata`) Wikidata names; development only | | `setup_env.py` | Environment setup | | `build_local.py` | PyInstaller build | | `PDF_Checker.py` | Post-processing utility | @@ -468,7 +489,7 @@ isort NullifyPDF.py | AI Processing | `NullifyPDF.py` | `AIWorker.run_scan()` | | Export | `NullifyPDF.py` | `cmd_export()` | | Reconstruct | `NullifyPDF.py` | `cmd_reconstruct()` | -| Blocklist/Allowlist | `NullifyPDF.py` | `PDFListManager` | +| Blocklist/Allowlist | `word_lists.py` | `PDFListManager` | --- @@ -574,7 +595,11 @@ Ready to contribute? See [CONTRIBUTING.md](./CONTRIBUTING.md) for: - ๐Ÿ’ฌ **GitHub Discussions:** [Discussions](https://github.com/overwrite00/NullifyPDF/discussions) - ๐Ÿ› **GitHub Issues:** [Issues](https://github.com/overwrite00/NullifyPDF/issues) +### Release notes and the update check + +The release workflows fill each GitHub release body with `scripts/extract_release_notes.py`: the `## [X.Y.Z]` section for stable tags, `## [Unreleased]` for beta tags. The in-app update check displays that body, so keep the changelog sections accurate before tagging. + --- -*Last updated: 2026-09-19* +*Last updated: 2026-11-11* *โ† [Troubleshooting](./TROUBLESHOOTING.md) | [Back to README โ†’](./README.md)* diff --git a/LICENSE b/LICENSE index 749775e..2bf02f1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,260 @@ -MIT License - -Copyright (c) 2026 Graziano Mariella - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +NullifyPDF +Copyright (C) 2026 Graziano Mariella + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) any +later version (SPDX: AGPL-3.0-or-later). + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + +NullifyPDF is free of charge and is not sold by its author. + +Why AGPL: the program uses PyMuPDF/MuPDF (Artifex), which is available under +the GNU AGPL-3.0 or a commercial licence. Releases up to and including +v2.4.0-beta.4 were published under the MIT licence; copies already received +under those terms remain under them. Starting with the next release the +program is distributed under the licence below. Third-party components keep +their own licences, see THIRD_PARTY_NOTICES.md. + +Source code: https://github.com/overwrite00/NullifyPDF + +================================================================================ + +GNU AFFERO GENERAL PUBLIC LICENSE +Version 3, 19 November 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + + Preamble + +The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. + +The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. + +When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. + +Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. + +A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. + +The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. + +An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. + +The precise terms and conditions for copying, distribution and modification follow. + + TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU Affero General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based on the Program. + +To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. + +1. Source Code. +The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. + +A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +2. Basic Permissions. +All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. +No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. + +4. Conveying Verbatim Copies. +You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. +You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". + + c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. + +6. Conveying Non-Source Forms. +You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: + + a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. + +7. Additional Terms. +"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. + +All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. + +A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. + +13. Remote Network Interaction; Use with the GNU General Public License. + +Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. + +Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. + +You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . diff --git a/NullifyPDF.py b/NullifyPDF.py index e4f599a..9a9b7f0 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -7,6 +7,7 @@ import string import platform import logging +import logging.handlers import traceback import atexit import hashlib @@ -24,9 +25,9 @@ QHBoxLayout, QLabel, QPushButton, - QRadioButton, QCheckBox, QProgressBar, + QProgressDialog, QTextEdit, QGraphicsView, QGraphicsScene, @@ -34,12 +35,14 @@ QDialog, QDialogButtonBox, QLineEdit, - QButtonGroup, QGraphicsRectItem, QMessageBox, QInputDialog, + QTextBrowser, ) from PySide6.QtGui import ( + QKeySequence, + QShortcut, QIcon, QPixmap, QImage, @@ -49,12 +52,32 @@ QDragEnterEvent, QDropEvent, ) -from PySide6.QtCore import Qt, QThread, QObject, Signal, Slot, QRectF, QPointF, QMutex, QMutexLocker +from PySide6.QtCore import ( + Qt, + QThread, + QObject, + Signal, + Slot, + QRectF, + QPointF, + QMutex, + QMutexLocker, + QStandardPaths, + QTimer, + QUrl, +) +from PySide6.QtGui import QDesktopServices from privacy_core import ( + PSEUDONYM_STYLES, PlaceholderEntry, + export_pseudonym_key, + generate_recovery_key, + import_pseudonym_key, + parse_recovery_key, PlaceholderRegistry, PrivacyMode, + password_strength, build_restore_payload, decrypt_restore_payload, encrypt_restore_payload, @@ -64,25 +87,57 @@ from pii_detection import ( ALL_GROUP_IDS, ENTITY_GROUPS, + NON_PRESIDIO_TYPES, + build_detections, build_page_index, - clean_candidates, + Candidate, + classify_page, + covered_fraction, expand_group_ids, - filter_results, load_enabled_groups, - propagate_whole_word, - reject_label_like_hits, - resolve_overlaps, save_enabled_groups, - span_to_rects, whole_word_rects, ) +import licences_info +import spacy_models +import update_check +from export_engine import ( + build_audit_record, + cleanup_stale_temp, + collect_removed_values, + export_document, + name_leaks_value, + neutral_map_name, + neutral_output_name, + private_temp_dir, + purge_old_logs, + wipe_local_data, + write_audit_record, + write_private_file, +) +from pii_detection import GLYPH_CORE_INSET, core_hits as _core_hits, glyph_core, target_chars +from pdf_sanitize import ( + SanitizePolicy, + page_image_coverage, + reveal_hidden_layers, + suggest_signature_areas, +) +from review_dialog import ReviewDialog +from model_dialog import ModelDialog +from help_guide import GuideDialog, apply_tooltips +from dictionary_dialog import DictionaryDialog +from word_lists import PDFListManager +from word_import import docx_to_pdf +import secure_store +from model_manager import CATALOG, MODEL_CANDIDATES, ModelStore +from onnx_ner import OnnxTokenDetector # PyMuPDF defines these constants dynamically, which mypy cannot see; read # them once here (with the suppression) instead of at every call site. PDF_ANNOT_REDACT: int = fitz.PDF_ANNOT_REDACT # type: ignore[attr-defined] PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] -__version__ = "2.3.0" +__version__ = "2.4.0" __version_prerelease__ = "" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" @@ -94,7 +149,90 @@ def about_version_text() -> str: """Version line of the About dialog; says "Beta" only for prereleases.""" suffix = " Beta" if __version_prerelease__ else "" - return f"v{APP_VERSION} AI Privacy{suffix}" + return f"v{APP_VERSION}{suffix}" + + +class UpdateCheckWorker(QThread): + """Fetch the GitHub release list and pick the update (if any).""" + + result_sig = Signal(object) # release dict, or None when up to date + error_sig = Signal(str) + + def run(self) -> None: + try: + releases = update_check.fetch_releases() + self.result_sig.emit(update_check.find_update(releases, APP_VERSION)) + except Exception as e: # network errors, bad JSON, rate limit... + logging.getLogger("nullifypdf").warning(f"Update check failed: {e}") + self.error_sig.emit(f"{type(e).__name__}: {e}") + + +class SpacyModelInstallWorker(QThread): + """Download the optional Italian spaCy pipeline off the UI thread.""" + + progress_sig = Signal(int, int) + done_sig = Signal() + error_sig = Signal(str) + + def __init__(self, root: pathlib.Path) -> None: + super().__init__() + self._root = root + + def run(self) -> None: + try: + spacy_models.install_italian( + self._root, progress=lambda d, t: self.progress_sig.emit(d, t) + ) + self.done_sig.emit() + except Exception as e: + self.error_sig.emit(str(e)) + + +class UpdateDownloadWorker(QThread): + """Download a release asset into the user's Downloads folder.""" + + progress_sig = Signal(int, int) + done_sig = Signal(str) + error_sig = Signal(str) + + def __init__(self, url: str, dest: pathlib.Path, size: int, digest: str = "") -> None: + super().__init__() + self._digest = digest + self._url = url + self._dest = dest + self._size = size + self._cancel = False + + def cancel(self) -> None: + self._cancel = True + + def run(self) -> None: + try: + update_check.download_file( + self._url, + self._dest, + expected_size=self._size, + digest=self._digest, + progress=lambda d, t: self.progress_sig.emit(d, t), + cancelled=lambda: self._cancel, + ) + self.done_sig.emit(str(self._dest)) + except InterruptedError: + self.error_sig.emit("") + except Exception as e: + logging.getLogger("nullifypdf").warning(f"Update download failed: {e}") + self.error_sig.emit(f"{type(e).__name__}: {e}") + + +def downloads_folder() -> pathlib.Path: + """The user's Downloads folder as configured in the running OS. + + Qt resolves it per platform (Windows known folder, XDG user-dirs on + Linux, ~/Downloads on macOS); fall back to ~/Downloads. The folder is + not created here: the caller does it when it actually downloads. + """ + path = QStandardPaths.writableLocation(QStandardPaths.StandardLocation.DownloadLocation) + return pathlib.Path(path) if path else pathlib.Path.home() / "Downloads" class DetectionDict(TypedDict): @@ -103,6 +241,9 @@ class DetectionDict(TypedDict): rects: List[Tuple[float, float, float, float]] source: NotRequired[str] score: NotRequired[float] + state: NotRequired[str] + reasons: NotRequired[List[str]] + glued: NotRequired[bool] def setup_logging() -> logging.Logger: @@ -115,6 +256,7 @@ def setup_logging() -> logging.Logger: """ log_dir = pathlib.Path.home() / ".nullifypdf" / "logs" log_dir.mkdir(parents=True, exist_ok=True) + purge_old_logs(log_dir, max_age_days=30) logger = logging.getLogger("nullifypdf") debug_mode = os.environ.get("NULLIFYPDF_DEBUG", "").lower() == "true" @@ -124,9 +266,13 @@ def setup_logging() -> logging.Logger: return logger try: - handler = logging.FileHandler( + # Rotating, so diagnostics are not kept indefinitely (storage + # limitation); old rotated files are also purged above. + handler = logging.handlers.RotatingFileHandler( log_dir / "nullifypdf.log", - encoding="utf-8" + maxBytes=512 * 1024, + backupCount=3, + encoding="utf-8", ) handler.setLevel(logging.DEBUG) formatter = logging.Formatter( @@ -171,8 +317,18 @@ class RestoreMapMismatch(ValueError): REDACTION_FONTNAME = "helv" +# Labels (longest first) drawn on image / signature areas after redaction. +KIND_LABELS: Dict[str, List[str]] = { + "IMAGE": ["[IMMAGINE RIMOSSA]", "[IMMAGINE]", "[IMG]", "#"], + "SIGNATURE": ["[FIRMA RIMOSSA]", "[FIRMA]", "[F]", "#"], +} REDACTION_MAX_FONTSIZE = 8.0 -REDACTION_MIN_FONTSIZE = 3.0 +# PyMuPDF's apply_redactions() never inserts replacement text below 4 pt, so +# a smaller floor would only produce text that is silently dropped. +REDACTION_MIN_FONTSIZE = 4.0 +# Text written with TextWriter has no such floor. +LABEL_MIN_FONTSIZE = 1.0 +LABEL_PREFERRED_MIN_FONTSIZE = 4.0 def fit_redaction_fontsize( @@ -262,44 +418,102 @@ def _placeholder_rects_by_chars(page: Any, placeholder: str) -> List["fitz.Rect" def capture_text_style(page: Any, rect: "fitz.Rect") -> Optional[Dict[str, Any]]: """Read the typography of the native text under `rect`. - Returns ``{font, size, color, flags, origin, text}`` for the dominant - span (most characters), where ``text`` is the text actually inside this - box and ``origin`` the baseline start of its first line. Returns None - when the box holds no real text: no span at all, or only an invisible - OCR layer (GlyphLessFont / render mode 3). That is what distinguishes a - native PDF from a scanned one. + Returns ``{font, size, color, flags, origin, text}`` for the line the + box was taken from, where ``text`` is the text of that line inside the + box and ``origin`` the baseline start of its first character. Only + characters on a single baseline whose deletion core the box intersects + are used, so with tight leading the text of the lines above and below + (whose boxes overlap this one) is never captured. Returns None when the + box holds no real text: no span at all, or only an invisible OCR layer + (GlyphLessFont / render mode 3). That is what distinguishes a native PDF + from a scanned one. """ try: - blocks = page.get_text("dict", clip=rect).get("blocks", []) + blocks = page.get_text("rawdict", clip=rect).get("blocks", []) except Exception: return None - spans: List[Dict[str, Any]] = [] + # (baseline y rounded, span, char) + hits: List[Tuple[float, Dict[str, Any], Dict[str, Any]]] = [] for block in blocks: if block.get("type") != 0: continue for line in block.get("lines", []): for span in line.get("spans", []): - if not str(span.get("text", "")).strip(): - continue if "glyphless" in str(span.get("font", "")).lower(): continue if span.get("alpha", 255) == 0: continue - spans.append(span) - if not spans: + for ch in span.get("chars", []): + if not str(ch.get("c", "")).strip(): + continue + if not _core_hits(tuple(ch["bbox"]), rect): + continue + hits.append((round(float(ch["origin"][1]), 1), span, ch)) + if not hits: return None - dominant = max(spans, key=lambda sp: len(str(sp["text"]).strip())) - text = " ".join(" ".join(str(sp["text"]) for sp in spans).split()) + centre = (rect.y0 + rect.y1) / 2 + counts: Dict[float, int] = {} + for base, _sp, _ch in hits: + counts[base] = counts.get(base, 0) + 1 + # The target line is the baseline nearest the middle of the box among + # those inside it; with tight leading a neighbour's baseline can also + # fall inside the box, but never closer to its middle. + inside = [k for k in counts if rect.y0 <= k <= rect.y1] or list(counts) + base = min(inside, key=lambda k: (abs(k - centre), -counts[k])) + chosen = [(sp, ch) for b_, sp, ch in hits if b_ == base] + chosen.sort(key=lambda t: t[1]["origin"][0]) + spans_used: Dict[int, int] = {} + for sp, _ch in chosen: + spans_used[id(sp)] = spans_used.get(id(sp), 0) + 1 + dominant = next(sp for sp, _ in chosen if spans_used[id(sp)] == max(spans_used.values())) + # Spaces between the selected characters are not in `hits` (they were + # filtered out above); rebuild them from the horizontal gaps. + pieces: List[str] = [] + prev_x1: Optional[float] = None + for _sp, ch in chosen: + x0, _y0, x1, _y1 = ch["bbox"] + if prev_x1 is not None and x0 - prev_x1 > 0.25 * float(dominant["size"]): + pieces.append(" ") + pieces.append(str(ch["c"])) + prev_x1 = x1 + first = chosen[0][1] return { "font": str(dominant.get("font", "")), "size": float(dominant["size"]), "color": int(dominant.get("color", 0)), "flags": int(dominant.get("flags", 0)), - "origin": [float(dominant["origin"][0]), float(dominant["origin"][1])], - "text": text, + "origin": [float(first["origin"][0]), float(first["origin"][1])], + "text": "".join(pieces).strip(), } +def band_covers_target(page: Any, band: "fitz.Rect", full: "fitz.Rect") -> bool: + """True when `band` reaches the deletion core of every target character. + + A glyph that is smaller or higher than the rest of its word (a + superscript, a trailing digit) could keep its core outside a thin band + and survive the redaction. + """ + chars = target_chars(page, full) + # An empty clipped result is not proof of coverage. A band trimmed + # against neighbouring lines can otherwise leave part of the glyph. + return bool(chars) and all(_core_hits(tuple(ch["bbox"]), band) for ch in chars) + + +def text_under_rect(page: Any, rect: "fitz.Rect") -> str: + """The text a box was drawn over, without neighbouring lines' fragments. + + `get_text(clip=...)` follows a different rule from glyph deletion and, with + tight leading, returns pieces of the lines above and below. Native text is + read from a single baseline (see capture_text_style); the plain clip text + is only the fallback for scanned / image pages. + """ + style = capture_text_style(page, rect) + if style and style.get("text"): + return " ".join(str(style["text"]).split()) + return " ".join(page.get_text("text", clip=rect).split()) + + _BASE14 = { # (family, bold, italic) -> Base-14 font code ("sans", False, False): "helv", ("sans", True, False): "hebo", @@ -389,7 +603,7 @@ def write_styled_text( else: # Wider in the fallback font (or a multi-line original): shrink a # little, and wrap inside the box if it still does not fit. - scaled = max(REDACTION_MIN_FONTSIZE, size * usable / width) + scaled = max(LABEL_MIN_FONTSIZE, size * usable / width) if font.text_length(text, fontsize=scaled) <= usable: writer.append(origin, text, font=font, fontsize=scaled) else: @@ -397,6 +611,73 @@ def write_styled_text( writer.write_text(page, color=color) +def compact_labels(placeholder: str) -> List[str]: + """Shorter stand-ins for a placeholder, longest first (full one first).""" + head, _, tail = placeholder.rpartition("_") + if not head or not tail: + return [placeholder] + out = [placeholder, f"{head[:1]}{tail}", tail, f"#{tail[:2]}"] + seen: List[str] = [] + for item in out: + if item not in seen: + seen.append(item) + return seen + + +def write_label( + page: Any, + rect: "fitz.Rect", + labels: List[str], + *, + style: Optional[Dict[str, Any]] = None, + allowance: float = 0.0, + max_size: float = REDACTION_MAX_FONTSIZE, +) -> str: + """Draw the first of `labels` that is legible inside the box; return it. + + Drawn with TextWriter after apply_redactions(), so there is no 4 pt + floor and no silent drop. A label may extend past the box to the right by + `allowance` points (free space on the same line). Preference: the longest + label that fits at 4 pt or more; otherwise the shortest at whatever size + fits (down to 1 pt). + """ + font = fitz.Font(REDACTION_FONTNAME) + # A band squeezed between tight lines is short: keep the label inside it. + max_size = min(max_size, max(LABEL_MIN_FONTSIZE, rect.height * 0.9)) + usable = max(1.0, rect.width - 1.0 + allowance) + chosen = labels[-1] + size = LABEL_MIN_FONTSIZE + for lab in labels: + full = font.text_length(lab, fontsize=1.0) + fit = min(max_size, usable / full) if full > 0 else max_size + if fit >= LABEL_PREFERRED_MIN_FONTSIZE: + chosen, size = lab, fit + break + else: + full = font.text_length(chosen, fontsize=1.0) + size = max(LABEL_MIN_FONTSIZE, min(max_size, usable / full if full else max_size)) + if style and style.get("origin"): + base_y = float(style["origin"][1]) + else: + base_y = rect.y0 + (rect.height + size * 0.7) / 2 + writer = fitz.TextWriter(page.rect) + writer.append(fitz.Point(rect.x0 + 0.5, base_y), chosen, font=font, fontsize=size) + writer.write_text(page, color=(0, 0, 0)) + return chosen + + +def free_space_right(page: Any, rect: "fitz.Rect", limit: float = 40.0) -> float: + """Free width to the right of `rect` on the same line (after redaction).""" + gap = limit + for w in page.get_text("words"): + overlap = min(rect.y1, w[3]) - max(rect.y0, w[1]) + if overlap < 0.5 * min(rect.height, w[3] - w[1]): + continue + if w[0] >= rect.x1 - 0.5: + gap = min(gap, w[0] - rect.x1 - 1.0) + return max(0.0, gap) + + def _restore_targets( page: Any, entry: PlaceholderEntry ) -> List[Tuple["fitz.Rect", Optional[Dict[str, Any]]]]: @@ -466,6 +747,7 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int for page in doc.pages(): claimed: List[fitz.Rect] = [] styled: List[Tuple[fitz.Rect, str, Dict[str, Any]]] = [] + plain: List[Tuple[fitz.Rect, str]] = [] for entry in entries: for rect, style in _restore_targets(page, entry): # Entries are sorted longest-placeholder-first, so a @@ -483,23 +765,19 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int # Native text: blank the box now, write the value in # its original typography after apply_redactions() # (which would otherwise erase it). - text = style.get("text") or entry.original + text = entry.original # never the captured clip text page.add_redact_annot(rect, fill=(1, 1, 1)) styled.append((fitz.Rect(rect), text, style)) attempted.append((page.number, fitz.Rect(rect), text)) continue - page.add_redact_annot( - rect, - text=entry.original, - fill=(1, 1, 1), - text_color=(0, 0, 0), - align=1, - fontsize=fit_redaction_fontsize(entry.original, rect), - ) + page.add_redact_annot(rect, fill=(1, 1, 1)) + plain.append((fitz.Rect(rect), entry.original)) attempted.append((page.number, fitz.Rect(rect), entry.original)) page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) for rect, text, style in styled: write_styled_text(doc, page, rect, text, style) + for rect, text in plain: + write_label(page, rect, [text]) doc.save(out_path, garbage=4, deflate=True, clean=True) finally: doc.close() @@ -585,102 +863,19 @@ def find_tessdata_dir() -> Optional[str]: def ocr_language_for_choice(choice: str, tessdata_dir: Optional[str]) -> str: - """Map the UI language choice to installed Tesseract language codes.""" - requested = ["eng", "ita"] if choice == "BOTH" else ["ita" if choice == "IT" else "eng"] + """Tesseract languages for the scan: Italian, plus English data if installed. + + The program analyses Italian documents only; the English OCR data is kept + just so that English words and Latin-script terms inside them are read + well. `choice` is kept for API stability and must be "IT". + """ if not tessdata_dir: - return "+".join(requested) + return "ita" available = [ - lang for lang in requested + lang for lang in ("ita", "eng") if (pathlib.Path(tessdata_dir) / f"{lang}.traineddata").exists() ] - return "+".join(available or requested) - - -class PDFListManager: - """Manages blocklist/allowlist persistence to disk. - - Handles loading and saving word lists with automatic path management - and UTF-8 encoding. - """ - - def __init__(self, config_dir: pathlib.Path) -> None: - """Initialize list manager with config directory. - - Args: - config_dir: Path to configuration directory. - """ - self.config_dir = config_dir - self.config_dir.mkdir(parents=True, exist_ok=True) - self.block_file = config_dir / "blocklist.txt" - self.allow_file = config_dir / "allowlist.txt" - - def load_blocklist(self) -> Set[str]: - """Load blocklist from disk. - - Returns: - Set[str]: Words to redact (lowercase). - """ - return self._load_list(self.block_file) - - def load_allowlist(self) -> Set[str]: - """Load allowlist from disk. - - Returns: - Set[str]: Words to preserve from redaction (lowercase). - """ - return self._load_list(self.allow_file) - - def save_blocklist(self, blocklist: Set[str]) -> None: - """Persist blocklist to disk. - - Args: - blocklist: Set of words to save. - """ - self._save_list(self.block_file, blocklist) - - def save_allowlist(self, allowlist: Set[str]) -> None: - """Persist allowlist to disk. - - Args: - allowlist: Set of words to save. - """ - self._save_list(self.allow_file, allowlist) - - def _load_list(self, path: pathlib.Path) -> Set[str]: - """Load word list from file with validation. - - Args: - path: Path to list file. - - Returns: - Set[str]: Set of words (lowercase, stripped, min length 3). - """ - if not path.exists(): - return set() - try: - # Use context manager to ensure file handle is closed deterministically - with open(path, "r", encoding="utf-8") as fh: - return { - line.strip().lower() - for line in fh - if len(line.strip()) > 2 - } - except Exception as e: - logging.getLogger(__name__).error(f"Error loading {path}: {e}") - return set() - - def _save_list(self, path: pathlib.Path, words: Set[str]) -> None: - """Save word list to file. - - Args: - path: Path to save list. - words: Set of words to save. - """ - try: - with open(path, "w", encoding="utf-8") as f: - f.write("\n".join(sorted(words))) - except Exception as e: - logging.getLogger(__name__).error(f"Error saving {path}: {e}") + return "+".join(available or ["ita"]) STYLESHEET = """ @@ -701,6 +896,15 @@ def _save_list(self, path: pathlib.Path, words: Set[str]) -> None: QProgressBar { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; text-align: center; color: transparent; height: 8px;} QProgressBar::chunk { background-color: #0ea5e9; border-radius: 3px; } QGraphicsView { border: none; background-color: #020617; } +QListWidget, QTableWidget, QTextBrowser { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; color: #cbd5e1; gridline-color: #334155; selection-background-color: #0284c7; selection-color: white; } +QListWidget::item { padding: 6px; } +QHeaderView::section { background-color: #1e293b; color: #e2e8f0; border: 1px solid #334155; padding: 4px; } +QComboBox, QDoubleSpinBox { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; padding: 4px; color: #e2e8f0; } +QComboBox QAbstractItemView { background-color: #0f172a; color: #e2e8f0; selection-background-color: #0284c7; } +QTabWidget::pane { border: 1px solid #334155; } +QTabBar::tab { background-color: #1e293b; color: #cbd5e1; padding: 6px 14px; border: 1px solid #334155; } +QTabBar::tab:selected { background-color: #0284c7; color: white; } +QToolTip { background-color: #1e293b; color: #e2e8f0; border: 1px solid #0ea5e9; padding: 4px; } QRadioButton::indicator, QCheckBox::indicator { width: 16px; height: 16px; border: 1px solid #475569; border-radius: 8px; background-color: #0f172a; } QCheckBox::indicator { border-radius: 4px; } QRadioButton::indicator:checked, QCheckBox::indicator:checked { background-color: #0ea5e9; border: 1px solid #0ea5e9; } @@ -730,6 +934,11 @@ def __init__(self) -> None: super().__init__() self.analyzer: Optional[Any] = None self.loaded_langs: List[str] = [] + # Set by the window before a scan: package name or folder of the + # Italian pipeline (it is downloaded on demand, see spacy_models). + self.italian_model: str = spacy_models.ITALIAN_PACKAGE + self._loaded_italian = "" + self.model_detectors: List[Tuple[str, OnnxTokenDetector]] = [] self._stop_requested = False def cleanup(self) -> None: @@ -745,15 +954,17 @@ def cleanup(self) -> None: # spaCy (read-only property), so we simply drop the reference. self.analyzer = None self.loaded_langs.clear() + self.model_detectors.clear() except Exception as e: logging.getLogger("nullifypdf").debug(f"Error during AI cleanup: {e}") - @Slot(object, object, str, list, set, bool, object, list) + @Slot(object, object, str, list, set, bool, object, list, bool) def run_scan(self, doc: Any, doc_mutex: Any, choice: str, compiled_allowlist: List[Tuple[str, Any]], allowlist_set: Set[str], use_ocr: bool, tessdata_dir: Optional[str], - enabled_types: List[str]) -> None: + enabled_types: List[str], + health_mode: bool = False) -> None: """Run AI scan on PDF pages and emit detected sensitive entities. Text extraction (`page.get_text()`) is performed inside this worker so @@ -765,11 +976,13 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, Args: doc: PyMuPDF document (shared with UI thread, mutex-protected). doc_mutex: QMutex serializing access to `doc`. - choice: Language choice: 'EN', 'IT', or 'BOTH'. + choice: Language of the document; only 'IT' is supported. compiled_allowlist: Pre-compiled regex patterns to skip redaction. allowlist_set: Set of allowlist entries (lowercase) for O(1) exact-match fast-path lookup before regex any() scan. enabled_types: Presidio entity types the user chose to detect. + health_mode: Health-document profile: no uncertain hit is dropped + silently, it becomes a suggestion for review instead. """ try: if self._stop_requested: @@ -781,7 +994,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, self.finished_sig.emit() return - if choice not in ("EN", "IT", "BOTH"): + if choice != "IT": self.log_sig.emit(f"ERRORE: Scelta lingua non valida: {choice}") self.finished_sig.emit() return @@ -794,21 +1007,19 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, if not isinstance(allowlist_set, set): allowlist_set = set() - target_langs = ["en", "it"] if choice == "BOTH" else [choice.lower()] + target_langs = ["it"] ocr_language = ocr_language_for_choice(choice, tessdata_dir) - if not self.analyzer or sorted(self.loaded_langs) != sorted(target_langs): - self.log_sig.emit(f"Inizializzazione AI ({choice})...") + if ( + not self.analyzer + or sorted(self.loaded_langs) != sorted(target_langs) + or self._loaded_italian != self.italian_model + ): + self.log_sig.emit(f"Inizializzazione motore di rilevamento ({choice})...") from presidio_analyzer import AnalyzerEngine, RecognizerRegistry from presidio_analyzer.nlp_engine import NlpEngineProvider from presidio_analyzer.predefined_recognizers import PhoneRecognizer - models = ( - [{"lang_code": "en", "model_name": "en_core_web_md"}] - if "en" in target_langs - else [] - ) - if "it" in target_langs: - models.append({"lang_code": "it", "model_name": "it_core_news_md"}) + models = [{"lang_code": "it", "model_name": self.italian_model}] provider = NlpEngineProvider( nlp_configuration={"nlp_engine_name": "spacy", "models": models} ) @@ -841,27 +1052,41 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, supported_languages=target_langs, ) self.loaded_langs = target_langs + self._loaded_italian = self.italian_model self.log_sig.emit("Scansione privacy in corso...") + self._load_local_detectors() targets = list(enabled_types) + presidio_targets = [ + t for t in targets if t not in NON_PRESIDIO_TYPES + ] + allow_terms = sorted( + set(allowlist_set) | {a for a, _ in compiled_allowlist} + ) # Determine page count under mutex (cheap, but doc may be replaced # mid-flight by load_path if not for the disabled UI during scan). with QMutexLocker(doc_mutex): total_pages = len(doc) + totals: Dict[str, int] = {} + n_suggested = 0 for i in range(total_pages): if self._stop_requested: break # Extract text in worker thread (off the UI thread) under mutex # because UI thread renders / mutates annotations on the same doc. used_ocr = False + has_images = False textpage = None + signature_dets: List[Dict[str, Any]] = [] with QMutexLocker(doc_mutex): try: page = doc[i] text = page.get_text() needs_ocr = use_ocr and self._page_needs_ocr(page, text) - index = build_page_index(page.get_text("words")) + coverage = page_image_coverage(page) + has_images = coverage > 0.0 + words = page.get_text("words") if needs_ocr: self.log_sig.emit(f"OCR pagina {i + 1}...") textpage = page.get_textpage_ocr( @@ -870,79 +1095,154 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, full=True, tessdata=tessdata_dir, ) - text = page.get_text(textpage=textpage) - index = build_page_index( - page.get_text("words", textpage=textpage) + words = page.get_text("words", textpage=textpage) + used_ocr = True + elif use_ocr and tessdata_dir and coverage >= 0.03: + # Text page with embedded images (stamps, scanned + # attachments): OCR the images only and merge. + self.log_sig.emit(f"OCR immagini pagina {i + 1}...") + textpage = page.get_textpage_ocr( + language=ocr_language, + dpi=300, + full=False, + tessdata=tessdata_dir, ) + words = page.get_text("words", textpage=textpage) used_ocr = True + index = build_page_index( + words, ocr=used_ocr, + image_rects=[ + tuple(i["bbox"]) + for i in page.get_image_info(hashes=False) + ], + ) + if health_mode: + signature_dets = suggest_signature_areas(page) + if index.collisions: + self.log_sig.emit( + f"Avviso: interlinea molto stretta a pagina " + f"{i + 1}: {index.collisions} parole vicine " + "potrebbero essere danneggiate. Controlla l'anteprima." + ) except Exception as e: self.log_sig.emit( f"Avviso: estrazione testo/OCR fallita pagina {i+1}: {e}" ) - text = "" index = build_page_index([]) - candidates: List[Any] = [] + results: List[Any] = [] + verb_like: Set[str] = set() for lang in self.loaded_langs: - if not targets or not index.text: + if not presidio_targets or not index.text: break + artifacts = None + try: + artifacts = self.analyzer.nlp_engine.process_text( + index.text, lang + ) + except Exception: + artifacts = None res = self.analyzer.analyze( - text=index.text, entities=targets, language=lang - ) - candidates.extend(filter_results(res, index.text, targets)) - candidates = resolve_overlaps( - reject_label_like_hits( - clean_candidates(candidates, enabled_types=targets), - index.text, + text=index.text, entities=presidio_targets, + language=lang, nlp_artifacts=artifacts, ) + results.extend(res) + verb_like |= self._first_person_verbs(artifacts) + candidates, stats = classify_page( + results, index, targets, + used_ocr=used_ocr, health_mode=health_mode, + verb_like=verb_like, ) - detections = [] - seen_values: Set[Tuple[str, str]] = set() - for c in candidates: - m, entity_type = c.text, c.entity_type - clean = " ".join(m.strip(string.punctuation).lower().split()) - if not clean: - continue - # Fast path: exact set membership is O(1). - if clean in allowlist_set: - continue - clean_pattern = re.compile(r"" + re.escape(clean) + r"") - if any( - f_reg.search(clean) or clean_pattern.search(a_str) - for a_str, f_reg in compiled_allowlist - ): - continue - rects = span_to_rects(index, c.start, c.end) - if (m, entity_type) not in seen_values: - seen_values.add((m, entity_type)) - # Other occurrences of the same value: whole words only. - for extra in propagate_whole_word( - index, m, - case_sensitive=entity_type in ("PERSON", "LOCATION"), - ): - if extra not in rects: - rects.append(extra) - if not rects: - continue - detections.append({ - "text": m, - "entity_type": entity_type, - "rects": rects, - "source": "ocr" if used_ocr else "text", - "score": c.score, - }) + detections = build_detections( + index, candidates, allowlist=allow_terms, + source="ocr" if used_ocr else "text", stats=stats, + ) + model_candidates: List[Tuple[Candidate, str]] = [] + for model_id, detector in self.model_detectors: + try: + for entity in detector.detect(index.text): + entity_type = entity.label + if entity_type not in targets: + continue + model_candidates.append(( + Candidate( + entity.start, + entity.end, + entity.text, + entity_type, + min(1.0, max(0.0, entity.score)), + state="suggested", + reasons=[f"model:{model_id}"], + ), + model_id, + )) + except Exception as exc: + self.log_sig.emit(f"Avviso: modello locale {model_id} non disponibile a pagina {i + 1}: {exc}") + for candidate, model_id in model_candidates: + model_detections = build_detections( + index, + [candidate], + allowlist=allow_terms, + source=f"model:{model_id}", + stats=stats, + ) + detections.extend(model_detections) + detections.extend(signature_dets) + for key, n in stats.items(): + totals[key] = totals.get(key, 0) + n + n_suggested += sum(1 for d in detections if d.get("state") == "suggested") self.page_done_sig.emit(i, detections) self.progress_sig.emit(i + 1, total_pages) if not self._stop_requested: - self.log_sig.emit("Anonimizzazione completata.") + if totals: + summary = ", ".join(f"{k}={v}" for k, v in sorted(totals.items())) + logging.getLogger("nullifypdf").info( + f"Scan discards (no values): {summary}" + ) + msg = "Scansione completata." + if n_suggested: + msg += f" {n_suggested} suggerimenti da rivedere." + self.log_sig.emit(msg) except Exception as e: # Log full traceback to file for diagnostics; show only summary in UI logging.getLogger("nullifypdf").error( f"AI scan error: {traceback.format_exc()}" ) - self.log_sig.emit(f"ERRORE AI: {type(e).__name__}: {str(e)}") + self.log_sig.emit(f"ERRORE rilevamento: {type(e).__name__}: {str(e)}") finally: self.finished_sig.emit() + @staticmethod + def _first_person_verbs(artifacts: Any) -> Set[str]: + """Lowercase first-person finite verb forms spaCy found on the page. + + A weak signal only: the grammar tags of capitalised words are not + reliable enough to reject a hit on their own. + """ + out: Set[str] = set() + try: + for tok in artifacts.tokens: + if tok.pos_ in ("VERB", "AUX") and "Person=1" in str(tok.morph): + out.add(tok.text.lower()) + except Exception: + return set() + return out + + def _load_local_detectors(self) -> None: + """Load every installed, manifest-verified local model once per scan.""" + if self.model_detectors: + return + store = ModelStore(pathlib.Path.home() / ".nullifypdf" / "models") + for manifest in CATALOG: + if not manifest.enabled or not store.is_installed(manifest): + continue + try: + self.model_detectors.append( + (manifest.model_id, OnnxTokenDetector(store.model_dir(manifest), manifest)) + ) + self.log_sig.emit(f"Modello locale caricato: {manifest.model_id}") + except Exception as exc: + self.log_sig.emit(f"Avviso: modello locale {manifest.model_id} ignorato: {exc}") + def _page_needs_ocr(self, page: Any, extracted_text: str) -> bool: """Return True when a page looks scanned or text extraction is too poor.""" text = (extracted_text or "").strip() @@ -1026,6 +1326,36 @@ def wheelEvent(self, event: Any) -> None: super().wheelEvent(event) +class SuggestionItem(QGraphicsRectItem): + """Scene-only outline of a pending suggestion. + + It lives in the graphics scene, never in the document: it cannot reach an + export or a print, and it becomes a redaction only through the + "Censura" decision. + """ + + def __init__(self, rect_pdf: Tuple[float, ...], suggestion: Dict[str, Any], scale: float) -> None: + r = rect_pdf + super().__init__(QRectF(r[0] * scale, r[1] * scale, (r[2] - r[0]) * scale, (r[3] - r[1]) * scale)) + self.suggestion = suggestion + pen = QPen(QColor("#f97316"), 2) + pen.setStyle(Qt.PenStyle.DashLine) + pen.setCosmetic(True) + self.setPen(pen) + self.setBrush(QColor(249, 115, 22, 40)) + self.setZValue(10) + self.setToolTip( + f"Suggerimento: {suggestion.get('entity_type', '')} " + f"({float(suggestion.get('score', 0.0)):.2f}) - clic per decidere" + ) + + def set_highlight(self, on: bool) -> None: + pen = QPen(QColor("#f97316"), 4 if on else 2) + pen.setStyle(Qt.PenStyle.SolidLine if on else Qt.PenStyle.DashLine) + pen.setCosmetic(True) + self.setPen(pen) + + class NullifyPDF(QMainWindow): """Main application window for PDF privacy redaction using AI. @@ -1041,7 +1371,7 @@ class NullifyPDF(QMainWindow): config_dir: Path to user config directory (~/.nullifypdf). """ - start_scan_sig = Signal(object, object, str, list, set, bool, object, list) + start_scan_sig = Signal(object, object, str, list, set, bool, object, list, bool) def __init__(self) -> None: """Initialize main application window and setup UI.""" @@ -1061,10 +1391,20 @@ def __init__(self) -> None: # Mutex guards `self.doc` against concurrent access from the AI worker # thread (text extraction) and the UI thread (render, apply redactions). self.doc_mutex = QMutex() + # Detections the evidence score could not settle; never applied + # silently, reviewed by the user (see review_dialog.ReviewDialog). + self.pending_suggestions: List[Dict[str, Any]] = [] + self._export_labels: List[Tuple[int, Any, List[str], Optional[Dict[str, Any]], Optional[str]]] = [] self.config_dir = pathlib.Path.home() / ".nullifypdf" + # Update-check threads outlive their dialogs: keep them referenced + # until they finish so a QThread is never destroyed while running. + self._bg_threads: Set[QThread] = set() + cleanup_stale_temp(self.config_dir) self.list_manager = PDFListManager(self.config_dir) self.blocklist = self.list_manager.load_blocklist() self.allowlist = self.list_manager.load_allowlist() + if self.list_manager.keystore_error: + self.logger.error("Word lists unreadable: keystore unavailable") self.ai_thread = QThread() self.ai_worker = AIWorker() self.ai_worker.moveToThread(self.ai_thread) @@ -1075,6 +1415,18 @@ def __init__(self) -> None: self.start_scan_sig.connect(self.ai_worker.run_scan) self.ai_thread.start() self.build_ui() + apply_tooltips(self) + self._f1 = QShortcut(QKeySequence("F1"), self) + self._f1.activated.connect(self.cmd_guide) + if self.list_manager.expired_removed: + self.write_log( + f"Rimossi {self.list_manager.expired_removed} termini scaduti dai dizionari." + ) + if self.list_manager.keystore_error: + self.write_log( + "Avviso: dizionari cifrati non leggibili su questo account/PC; " + "verranno usati dizionari vuoti finchรฉ non vengono ripristinati." + ) # Setup signal handlers for graceful shutdown self._setup_signal_handlers() # Register cleanup on exit @@ -1110,6 +1462,12 @@ def _cleanup_resources(self) -> None: - All connections """ self.logger.info("Starting resource cleanup") + for t in list(self._bg_threads): + cancel = getattr(t, "cancel", None) + if cancel is not None: + cancel() + if t.isRunning(): + t.wait(12000) try: # Stop AI worker from processing if hasattr(self, 'ai_worker'): @@ -1171,37 +1529,54 @@ def build_ui(self) -> None: lbl_title = QLabel("NullifyPDF") lbl_title.setStyleSheet("font-size: 24px; font-weight: bold; color: #0ea5e9;") s_lay.addWidget(lbl_title) - s_lay.addWidget(QLabel("AI Privacy Edition\n")) + s_lay.addWidget(QLabel("Privacy Edition\n")) btn_open = QPushButton("Apri PDF") btn_open.setObjectName("Primary") btn_open.clicked.connect(self.cmd_open) s_lay.addWidget(btn_open) s_lay.addSpacing(20) - s_lay.addWidget(QLabel("Modello Lingua AI:")) - lang_lay = QHBoxLayout() - self.rb_en = QRadioButton("EN") - self.rb_it = QRadioButton("IT") - self.rb_both = QRadioButton("BOTH") - self.rb_en.setChecked(True) - self.lang_grp = QButtonGroup(self) - for rb in (self.rb_en, self.rb_it, self.rb_both): - self.lang_grp.addButton(rb) - lang_lay.addWidget(rb) - s_lay.addLayout(lang_lay) + self.lbl_lang = QLabel("Lingua dei documenti: italiano") + self.lbl_lang.setObjectName("language_label") + s_lay.addWidget(self.lbl_lang) s_lay.addSpacing(10) self.chk_img = QCheckBox("Oscura Immagini") s_lay.addWidget(self.chk_img) self.chk_ocr = QCheckBox("OCR PDF scansionati") self.chk_ocr.setChecked(True) s_lay.addWidget(self.chk_ocr) + self.chk_sugg = QCheckBox("Mostra suggerimenti") + self.chk_sugg.setChecked(True) + self.chk_sugg.setToolTip("Contorna in arancione, nell'anteprima, i suggerimenti da rivedere.") + self.chk_sugg.toggled.connect(lambda _on: self.render_page()) + s_lay.addWidget(self.chk_sugg) + self.chk_health = QCheckBox("Profilo documenti sanitari") + self.chk_health.setToolTip( + "Rileva anche date di nascita e identificativi sanitari e non " + "scarta in silenzio i casi incerti: li propone in revisione." + ) + s_lay.addWidget(self.chk_health) + self.chk_mask_width = QCheckBox("Maschera larghezza testo rimosso") + self.chk_mask_width.setToolTip( + "Arrotonda lo spazio lasciato dal testo rimosso, che altrimenti " + "ne rivela la larghezza. Puo spostare di poco il testo seguente." + ) + s_lay.addWidget(self.chk_mask_width) s_lay.addSpacing(20) btn_dict = QPushButton("Dizionari") btn_dict.clicked.connect(self.cmd_dict) s_lay.addWidget(btn_dict) - self.btn_ai = QPushButton("Auto Redact (AI)") + btn_models = QPushButton("Modelli locali") + btn_models.setObjectName("model_settings") + btn_models.clicked.connect(self.cmd_models) + s_lay.addWidget(btn_models) + self.btn_ai = QPushButton("Auto Redact") self.btn_ai.setObjectName("Danger") self.btn_ai.clicked.connect(self.cmd_auto_ai) s_lay.addWidget(self.btn_ai) + self.btn_review = QPushButton("Rivedi suggerimenti (0)") + self.btn_review.clicked.connect(self._review_suggestions) + self.btn_review.setEnabled(False) + s_lay.addWidget(self.btn_review) btn_clear = QPushButton("Pulisci Pagina") btn_clear.clicked.connect(self.cmd_clear) s_lay.addWidget(btn_clear) @@ -1213,7 +1588,17 @@ def build_ui(self) -> None: btn_reconstruct = QPushButton("Ricostruisci PDF") btn_reconstruct.clicked.connect(self.cmd_reconstruct) s_lay.addWidget(btn_reconstruct) + btn_wipe = QPushButton("Elimina dati locali") + btn_wipe.setToolTip( + "Cancella log, registri di esportazione e file temporanei salvati " + "in ~/.nullifypdf." + ) + btn_wipe.clicked.connect(self.cmd_wipe_local_data) + s_lay.addWidget(btn_wipe) s_lay.addStretch() + btn_guide = QPushButton("Guida") + btn_guide.clicked.connect(self.cmd_guide) + s_lay.addWidget(btn_guide) btn_about = QPushButton("Info") btn_about.clicked.connect(self.cmd_about) s_lay.addWidget(btn_about) @@ -1232,6 +1617,7 @@ def build_ui(self) -> None: btn_zout = QPushButton("-") btn_zout.clicked.connect(lambda: self.adjust_zoom(-1)) self.lbl_zoom = QLabel("150%") + self.lbl_zoom.setObjectName("lbl_zoom") btn_zin = QPushButton("+") btn_zin.clicked.connect(lambda: self.adjust_zoom(1)) tb_lay.addWidget(btn_zout) @@ -1241,10 +1627,12 @@ def build_ui(self) -> None: btn_prev = QPushButton("<") btn_prev.clicked.connect(lambda: self.move_page(-1)) self.le_page = QLineEdit("0") + self.le_page.setObjectName("le_page") self.le_page.setFixedWidth(40) self.le_page.setAlignment(Qt.AlignmentFlag.AlignCenter) self.le_page.returnPressed.connect(self.jump_page) self.lbl_tot = QLabel("/ 0") + self.lbl_tot.setObjectName("lbl_tot") btn_next = QPushButton(">") btn_next.clicked.connect(lambda: self.move_page(1)) tb_lay.addWidget(btn_prev) @@ -1254,6 +1642,7 @@ def build_ui(self) -> None: r_lay.addWidget(top_bar) self.scene = QGraphicsScene() self.view = PDFView(self.scene) + self.view.setObjectName("view") self.view.rect_drawn.connect(self.user_draw_rect) self.view.point_clicked.connect(self.user_click_pt) self.view.zoom_req.connect(self.adjust_zoom) @@ -1262,8 +1651,10 @@ def build_ui(self) -> None: footer.setObjectName("Panel") f_lay = QVBoxLayout(footer) self.prog = QProgressBar() + self.prog.setObjectName("prog") self.prog.setValue(0) self.log = QTextEdit() + self.log.setObjectName("log") self.log.setReadOnly(True) self.log.setFixedHeight(80) f_lay.addWidget(self.prog) @@ -1282,6 +1673,16 @@ def dropEvent(self, e: QDropEvent) -> None: if urls and urls[0].isLocalFile(): self.load_path(urls[0].toLocalFile()) + def cmd_models(self) -> None: + """Show the curated local-model catalog and its verification state.""" + store = ModelStore(self.config_dir / "models") + ModelDialog( + MODEL_CANDIDATES, + manifests=CATALOG, + store=store, + parent=self, + ).exec() + def write_log(self, m: str) -> None: """Write message to UI log with color coding. @@ -1316,7 +1717,9 @@ def update_progress(self, c: int, t: int) -> None: def cmd_open(self) -> None: """Open file dialog to load PDF.""" - p, _ = QFileDialog.getOpenFileName(self, "Apri PDF", "", "PDF (*.pdf)") + p, _ = QFileDialog.getOpenFileName( + self, "Apri documento", "", "PDF e Word (*.pdf *.docx);;PDF (*.pdf);;Word DOCX (*.docx)" + ) if p: self.load_path(p) @@ -1332,12 +1735,14 @@ def load_path(self, path: str) -> None: if not os.path.exists(path): self.write_log(f"ERRORE: File non trovato: {path}") return - if not path.lower().endswith('.pdf'): - self.write_log("ERRORE: Solo file PDF sono supportati") + suffix = pathlib.Path(path).suffix.lower() + if suffix not in {".pdf", ".docx"}: + self.write_log("ERRORE: Sono supportati solo PDF e Word DOCX") return try: - tdoc = fitz.open(path) + imported_word = suffix == ".docx" + tdoc = docx_to_pdf(path) if imported_word else fitz.open(path) if tdoc.needs_pass: self.write_log("ERRORE: PDF cifrato - inserire password non supportato") tdoc.close() @@ -1350,8 +1755,11 @@ def load_path(self, path: str) -> None: self.doc = tdoc self.page_num = 0 self.scale = 1.5 + self.pending_suggestions = [] + self._update_review_button() self.adjust_zoom(0) - self.write_log(f"Caricato: {os.path.basename(path)}") + label = "Word DOCX convertito in PDF locale" if imported_word else "PDF" + self.write_log(f"Caricato {label}: {os.path.basename(path)}") self._warn_if_scanned_pdf() except FileNotFoundError: self.write_log(f"ERRORE: File non trovato") @@ -1387,7 +1795,7 @@ def _warn_if_scanned_pdf(self) -> None: f"nelle pagine {preview}{suffix}).\n\n" 'Puoi comunque selezionare manualmente le aree da censurare. ' 'Per il riconoscimento automatico dei dati personali, attiva ' - 'prima "OCR PDF scansionati" e poi avvia "Auto Redact (AI)".', + 'prima "OCR PDF scansionati" e poi avvia "Auto Redact".', ) def render_page(self) -> None: @@ -1410,6 +1818,58 @@ def render_page(self) -> None: self.scene.setSceneRect(0, 0, pix.width, pix.height) self.le_page.setText(str(self.page_num + 1)) self.lbl_tot.setText(f"/ {total}") + self._draw_suggestion_overlays() + + def _draw_suggestion_overlays(self) -> None: + """Outline the pending suggestions of the current page (view only).""" + if not self.chk_sugg.isChecked(): + return + for s in self.pending_suggestions: + if int(s.get("page", -1)) != self.page_num: + continue + # The clamped band is internal redaction geometry. Showing it + # makes a correctly detected word appear partially selected. + for rect in s.get("full_rects") or s.get("rects") or []: + self.scene.addItem(SuggestionItem(tuple(rect), s, self.scale)) + + def _highlight_suggestion(self, suggestion: Any) -> None: + for it in self.scene.items(): + if isinstance(it, SuggestionItem): + it.set_highlight(it.suggestion is suggestion) + + def _decide_suggestion(self, s: Dict[str, Any]) -> None: + """Accept or discard one suggestion clicked in the page view.""" + reveal = False + while True: + value = " ".join(str(s.get("text", "")).split()) + shown = "(area grafica)" if not value else (value if reveal else value[:1] + "โ€ข" * min(len(value) - 1, 8)) + box = QMessageBox(self) + box.setWindowTitle("Suggerimento") + box.setText( + f"{s.get('entity_type', '')} - {shown}\n" + f"Punteggio {float(s.get('score', 0.0)):.2f}: " + + ", ".join(str(r) for r in s.get("reasons", [])) + ) + b_yes = box.addButton("Censura", QMessageBox.ButtonRole.AcceptRole) + b_no = box.addButton("Scarta", QMessageBox.ButtonRole.DestructiveRole) + b_show = box.addButton("Mostra valore", QMessageBox.ButtonRole.ActionRole) + b_yes.setToolTip("Applica la censura a questo suggerimento.") + b_no.setToolTip("Scarta il suggerimento senza censurare nulla.") + b_show.setToolTip("Mostra il valore in chiaro prima di decidere.") + box.addButton("Annulla", QMessageBox.ButtonRole.RejectRole) + box.exec() + clicked = box.clickedButton() + if clicked == b_show and value and not reveal: + reveal = True + continue + break + if clicked not in (b_yes, b_no): + return + if clicked == b_yes: + self._apply_detections(int(s.get("page", self.page_num)), [s]) + self.pending_suggestions = [p for p in self.pending_suggestions if p is not s] + self._update_review_button() + self.render_page() def adjust_zoom(self, d: int) -> None: """Adjust zoom level. @@ -1468,15 +1928,9 @@ def user_draw_rect(self, qrect: QRectF) -> None: if not self.doc: return p = self.doc[self.page_num] - txt = p.get_text("text", clip=r).strip() + txt = text_under_rect(p, fitz.Rect(r)) if not txt: - p.add_redact_annot( - r, - text="[ IMMAGINE RIMOSSA ]", - align=1, - fill=(0.9, 0.9, 0.9), - fontsize=8, - ) + self._add_kind_redaction(p, r, "IMAGE") else: self._add_privacy_redaction( p, @@ -1514,6 +1968,10 @@ def user_click_pt(self, qpt: QPointF) -> None: """ if not self.doc: return + overlays = [it for it in self.scene.items(qpt) if isinstance(it, SuggestionItem)] + if overlays: + self._decide_suggestion(overlays[0].suggestion) + return pt = fitz.Point(qpt.x() / self.scale, qpt.y() / self.scale) cl = "" with QMutexLocker(self.doc_mutex): @@ -1527,7 +1985,7 @@ def user_click_pt(self, qpt: QPointF) -> None: ] if not ans: return - txt = p.get_text("text", clip=ans[0].rect) + txt = text_under_rect(p, ans[0].rect) cl = " ".join(txt.split()).lower() add_to_whitelist = False @@ -1594,46 +2052,50 @@ def cmd_clear(self) -> None: self.write_log(f"Censure rimosse su pagina {self.page_num+1}") def cmd_dict(self) -> None: - """Open dialog to edit blocklist/allowlist.""" - d = QDialog(self) - d.setWindowTitle("Dizionari") - d.resize(500, 450) - lay = QVBoxLayout(d) - lay.addWidget(QLabel("๐Ÿ”ด BLOCKLIST")) - bx = QTextEdit() - bx.setPlainText("\n".join(sorted(self.blocklist))) - lay.addWidget(bx) - lay.addWidget(QLabel("๐ŸŸข ALLOWLIST")) - ax = QTextEdit() - ax.setPlainText("\n".join(sorted(self.allowlist))) - lay.addWidget(ax) - btn = QPushButton("Salva") - btn.setObjectName("Primary") - - def s() -> None: - self.blocklist = { - line.strip().lower() - for line in bx.toPlainText().split("\n") - if len(line.strip()) > 2 - } - self.allowlist = { - line.strip().lower() - for line in ax.toPlainText().split("\n") - if len(line.strip()) > 2 - } - self.list_manager.save_blocklist(self.blocklist) - self.list_manager.save_allowlist(self.allowlist) - d.accept() + """Open the dictionary editor (blocklist / allowlist with expiry).""" + d = DictionaryDialog(self.list_manager, self) + if d.exec() == QDialog.DialogCode.Accepted: + if d.saved_blocklist is not None: + self.blocklist = d.saved_blocklist + if d.saved_allowlist is not None: + self.allowlist = d.saved_allowlist + + def _start_bg_thread(self, w: QThread) -> None: + """Start ``w`` and keep it referenced until it has finished.""" + self._bg_threads.add(w) + + def done() -> None: + self._bg_threads.discard(w) + w.deleteLater() + + w.finished.connect(done) + w.start() + + def auto_check_updates(self) -> None: + """Silent check at startup, only if the user enabled it in Info. + + Nothing is shown unless a newer release exists; failures (offline, + rate limit) are only logged. + """ + if not update_check.load_auto_check(self.config_dir / "update_check.json"): + return + w = UpdateCheckWorker() + w.result_sig.connect(self._on_auto_update_result) + self._start_bg_thread(w) - btn.clicked.connect(s) - lay.addWidget(btn) - d.exec() + def _on_auto_update_result(self, release: Any) -> None: + if release is not None: + self._show_update_dialog(release, self) + + def cmd_guide(self) -> None: + """Open the user guide (also bound to F1).""" + GuideDialog(self).exec() def cmd_about(self) -> None: """Show about dialog.""" d = QDialog(self) d.setWindowTitle("Info") - d.setFixedSize(340, 440) + d.setFixedSize(520, 820) lay = QVBoxLayout(d) lay.setAlignment(Qt.AlignmentFlag.AlignCenter) ip = resource_path(os.path.join("images", "NullifyPDF_icon.png")) @@ -1656,15 +2118,174 @@ def cmd_about(self) -> None: lbl_ver.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(lbl_ver) desc = QLabel( - "\nAnonimizzazione PDF Offline.\n\nSviluppato da: Graziano Mariella\nLicenza MIT" + "\nCensura e pseudonimizzazione PDF offline.\n\nSviluppato da: Graziano Mariella\nLicenza AGPL-3.0" ) desc.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(desc) - lay.addSpacing(20) + lbl_repo = QLabel( + f'{update_check.REPO_URL}' + ) + lbl_repo.setTextFormat(Qt.TextFormat.RichText) + lbl_repo.setOpenExternalLinks(True) + lbl_repo.setAlignment(Qt.AlignmentFlag.AlignCenter) + lay.addWidget(lbl_repo) + lay.addSpacing(6) + lic_view = QTextBrowser() + lic_view.setObjectName("about_licences") + lic_view.setOpenExternalLinks(True) + lic_view.setHtml(licences_info.about_html()) + lic_view.setMinimumHeight(200) + lic_view.setToolTip( + "Licenza del programma e dei componenti usati, con i collegamenti ai testi ufficiali." + ) + lay.addWidget(lic_view, 1) + lay.addSpacing(6) + btn_upd = QPushButton("Controlla aggiornamenti") + lay.addWidget(btn_upd) + chk_auto = QCheckBox("Controlla ad ogni avvio") + pref_path = self.config_dir / "update_check.json" + chk_auto.setChecked(update_check.load_auto_check(pref_path)) + chk_auto.toggled.connect(lambda on: update_check.save_auto_check(pref_path, on)) + lay.addWidget(chk_auto, alignment=Qt.AlignmentFlag.AlignCenter) + lbl_upd = QLabel("") + lbl_upd.setWordWrap(True) + lbl_upd.setAlignment(Qt.AlignmentFlag.AlignCenter) + lay.addWidget(lbl_upd) + lay.addSpacing(10) btn = QPushButton("Chiudi") btn.clicked.connect(d.accept) lay.addWidget(btn) + + state: Dict[str, Any] = {"closed": False} + + def on_result(release: Any) -> None: + if state["closed"]: + return + btn_upd.setEnabled(True) + if release is None: + lbl_upd.setText("Nessun aggiornamento disponibile: la versione รจ aggiornata.") + return + lbl_upd.setText(f"Disponibile: {release.get('tag_name', '')}") + self._show_update_dialog(release, d) + + def on_error(msg: str) -> None: + btn_upd.setEnabled(True) + lbl_upd.setText(f"Controllo non riuscito (connessione assente?)\n{msg}") + + def start_check() -> None: + btn_upd.setEnabled(False) + lbl_upd.setText("Controllo in corso...") + w = UpdateCheckWorker() + w.result_sig.connect(on_result) + w.error_sig.connect(on_error) + self._start_bg_thread(w) + + btn_upd.clicked.connect(start_check) + apply_tooltips(d) + d.exec() + state["closed"] = True + + def _show_update_dialog(self, release: Dict[str, Any], parent: QWidget) -> None: + """Release notes, link to the release and a [Download] button.""" + tag = str(release.get("tag_name", "")) + url = str(release.get("html_url", "")) + if not update_check.is_trusted_url(url): + url = f"{update_check.REPO_URL}/releases" + asset = update_check.select_asset(release) + + d = QDialog(parent) + d.setWindowTitle("Aggiornamento disponibile") + d.resize(560, 480) + lay = QVBoxLayout(d) + kind = " (beta)" if release.get("prerelease") else "" + lay.addWidget(QLabel(f"Nuova versione: {html.escape(tag)}{kind}")) + lbl_link = QLabel(f'{html.escape(url)}') + lbl_link.setTextFormat(Qt.TextFormat.RichText) + lbl_link.setOpenExternalLinks(True) + lay.addWidget(lbl_link) + notes = QTextBrowser() + notes.setOpenExternalLinks(True) + body = str(release.get("body") or "").strip() + notes.setMarkdown(body or "_Nessuna nota di rilascio disponibile._") + lay.addWidget(notes) + bar = QProgressBar() + bar.setVisible(False) + lay.addWidget(bar) + lbl_status = QLabel("") + lbl_status.setWordWrap(True) + lay.addWidget(lbl_status) + row = QHBoxLayout() + btn_dl = QPushButton("Download") + btn_dl.setObjectName("Primary") + btn_close = QPushButton("Chiudi") + row.addWidget(btn_dl) + row.addWidget(btn_close) + lay.addLayout(row) + btn_close.clicked.connect(d.reject) + + if asset is None: + btn_dl.setEnabled(False) + lbl_status.setText( + "Nessun file per questo sistema operativo: usa il link alla release." + ) + + state: Dict[str, Any] = {"worker": None, "closed": False} + + def on_progress(done: int, total: int) -> None: + if total > 0: + bar.setMaximum(total) + bar.setValue(done) + else: + bar.setMaximum(0) + + def on_done(path: str) -> None: + if state["closed"]: + return + bar.setVisible(False) + btn_dl.setEnabled(False) + lbl_status.setText(f"Scaricato in: {path}") + QDesktopServices.openUrl(QUrl.fromLocalFile(str(pathlib.Path(path).parent))) + + def on_error(msg: str) -> None: + if state["closed"]: + return + bar.setVisible(False) + btn_dl.setEnabled(True) + lbl_status.setText(f"Download non riuscito: {msg}" if msg else "Download annullato.") + + def start_download() -> None: + if asset is None: + return + try: + folder = downloads_folder() + folder.mkdir(parents=True, exist_ok=True) + dest = update_check.unique_path(folder, str(asset.get("name", ""))) + except OSError as e: + lbl_status.setText(f"Cartella Download non accessibile: {e}") + return + btn_dl.setEnabled(False) + bar.setValue(0) + bar.setVisible(True) + lbl_status.setText(f"Download in {dest.parent} ...") + w = UpdateDownloadWorker( + str(asset["browser_download_url"]), + dest, + int(asset.get("size") or 0), + str(asset.get("digest") or ""), + ) + w.progress_sig.connect(on_progress) + w.done_sig.connect(on_done) + w.error_sig.connect(on_error) + state["worker"] = w + self._start_bg_thread(w) + + btn_dl.clicked.connect(start_download) + apply_tooltips(d) d.exec() + state["closed"] = True + w = state["worker"] + if w is not None and w.isRunning(): + w.cancel() # the thread stops by itself; the window never blocks def cmd_auto_ai(self) -> None: """Start AI scan for sensitive entities. @@ -1678,16 +2299,29 @@ def cmd_auto_ai(self) -> None: enabled_groups = self._select_ai_entities() if not enabled_groups: return + health_mode = self.chk_health.isChecked() + if health_mode: + for gid in ("BIRTHDATE", "HEALTHID", "EXEMPT"): + if gid not in enabled_groups: + enabled_groups = list(enabled_groups) + [gid] self.btn_ai.setEnabled(False) self.prog.setValue(0) + # Text in a layer that is switched off is invisible to extraction, so + # neither the detector nor the reviewer would ever see it: show all + # layers (in memory only) before scanning. + with QMutexLocker(self.doc_mutex): + revealed = reveal_hidden_layers(self.doc) + if revealed: + self.write_log( + f"Avviso: {revealed} livelli nascosti resi visibili per l'analisi." + ) c_allow = [ (a, re.compile(r"\b" + re.escape(a) + r"\b")) for a in self.allowlist ] - lang = ( - "EN" - if self.rb_en.isChecked() - else "IT" if self.rb_it.isChecked() else "BOTH" - ) + lang = "IT" + if not self._prepare_italian_model(): + self.btn_ai.setEnabled(True) + return use_ocr = self.chk_ocr.isChecked() tessdata_dir = find_tessdata_dir() if use_ocr else None if use_ocr and not tessdata_dir: @@ -1711,8 +2345,79 @@ def cmd_auto_ai(self) -> None: use_ocr, tessdata_dir, expand_group_ids(enabled_groups), + health_mode, ) + def _prepare_italian_model(self) -> bool: + """Make an Italian pipeline available; False if the user cancels. + + The statistical pipeline is not bundled (non-commercial licence): ask + once, show the licence, download and verify it. If the user declines, + the scan continues with rules only and the log says so. + """ + root = spacy_models.models_root(self.config_dir) + found = spacy_models.resolve_italian(root) + if found is not None: + self.ai_worker.italian_model = found.name + return True + box = QMessageBox(self) + box.setWindowTitle("Modello italiano") + box.setIcon(QMessageBox.Icon.Question) + box.setText("Per riconoscere i nomi in italiano serve un modello aggiuntivo.") + box.setInformativeText(spacy_models.licence_text()) + b_dl = box.addButton("Accetta e scarica", QMessageBox.ButtonRole.AcceptRole) + b_skip = box.addButton("Continua senza modello", QMessageBox.ButtonRole.DestructiveRole) + box.addButton("Annulla", QMessageBox.ButtonRole.RejectRole) + box.exec() + clicked = box.clickedButton() + if clicked is b_skip: + self.ai_worker.italian_model = spacy_models.blank_italian(root).name + self.write_log( + "Avviso: modello italiano non installato, riconoscimento nomi " + "solo con regole e dizionari (meno completo)." + ) + return True + if clicked is not b_dl: + return False + dlg = QProgressDialog("Download del modello italiano...", "Annulla", 0, 100, self) + dlg.setWindowTitle("Modello italiano") + dlg.setMinimumDuration(0) + dlg.setAutoClose(False) + dlg.setAutoReset(False) + dlg.setCancelButton(None) + worker = SpacyModelInstallWorker(root) + result: Dict[str, str] = {} + + def on_progress(done: int, total: int) -> None: + dlg.setValue(int(done * 100 / max(1, total))) + + def on_done() -> None: + result["ok"] = "1" + dlg.accept() + + def on_error(msg: str) -> None: + result["error"] = msg + dlg.accept() + + worker.progress_sig.connect(on_progress) + worker.done_sig.connect(on_done) + worker.error_sig.connect(on_error) + worker.start() + dlg.exec() + worker.wait() + if "ok" in result: + found = spacy_models.resolve_italian(root) + if found is not None: + self.ai_worker.italian_model = found.name + self.write_log("Modello italiano scaricato e verificato.") + return True + QMessageBox.warning( + self, + "Modello italiano", + "Download non riuscito: " + result.get("error", "errore sconosciuto"), + ) + return False + def _select_ai_entities(self) -> Optional[List[str]]: """Ask which data types the AI scan should look for. @@ -1758,6 +2463,7 @@ def set_all(state: bool) -> None: bb.accepted.connect(dlg.accept) bb.rejected.connect(dlg.reject) refresh() + apply_tooltips(dlg) if dlg.exec() != QDialog.DialogCode.Accepted: return None chosen = [gid for gid in ALL_GROUP_IDS if boxes[gid].isChecked()] @@ -1771,15 +2477,25 @@ def _select_privacy_export(self) -> Optional[PrivacyMode]: box.setIcon(QMessageBox.Icon.Question) box.setText("Scegli come trattare i dati personali nel PDF esportato.") box.setInformativeText( - "Anonimizzazione: rimozione irreversibile.\n" - "Pseudonimizzazione: segnaposto reversibili con mappa cifrata separata." + "Rimozione irreversibile: i valori selezionati vengono eliminati " + "dal file. Il risultato puo contenere ancora dati personali non " + "rilevati o indiretti (date, luoghi, dettagli): controllalo prima " + "di condividerlo.\n\n" + "Pseudonimizzazione: segnaposto reversibili con mappa cifrata " + "separata. Il file e la mappa restano dati personali." ) anonymize_btn = box.addButton( - "Anonimizzazione irreversibile", QMessageBox.ButtonRole.AcceptRole + "Rimozione irreversibile", QMessageBox.ButtonRole.AcceptRole + ) + anonymize_btn.setToolTip( + "Elimina per sempre i valori selezionati dal file. Non si puรฒ tornare indietro." ) pseudonymize_btn = box.addButton( "Pseudonimizzazione reversibile", QMessageBox.ButtonRole.ActionRole ) + pseudonymize_btn.setToolTip( + "Sostituisce i valori con segnaposto e crea una mappa cifrata per ripristinarli." + ) box.addButton(QMessageBox.StandardButton.Cancel) box.exec() clicked = box.clickedButton() @@ -1824,21 +2540,53 @@ def _add_privacy_redaction( original: str = "", entity_type: str = "DATA", fill: Tuple[float, float, float] = (0, 0, 0), + full: Optional[fitz.Rect] = None, ) -> None: - """Add a redaction annotation carrying optional in-memory privacy data.""" + """Add a redaction annotation carrying optional in-memory privacy data. + + `full` is the whole box of the word when `rect` is a thinner band: the + export uses it to check that no glyph of the word survived. + """ annot = page.add_redact_annot(rect, fill=fill) clean_original = " ".join((original or "").split()) if clean_original: - payload = { + payload: Dict[str, Any] = { "nullifypdf": 1, "original": clean_original, "entity_type": PlaceholderRegistry.normalize_entity_type(entity_type), } + if full is not None and fitz.Rect(full) != fitz.Rect(rect): + payload["full"] = [full.x0, full.y0, full.x1, full.y1] try: annot.set_info(content=json.dumps(payload, ensure_ascii=False)) except Exception as e: self.logger.debug(f"Could not store redaction metadata: {e}") + def _add_kind_redaction(self, page: Any, rect: fitz.Rect, kind: str) -> None: + """Redaction of a graphic area (image or signature) without overlay text. + + The label is drawn after the redactions are applied (see + `_finish_labels_page`), so it is not lost in a small box. + """ + annot = page.add_redact_annot(rect, fill=(0.9, 0.9, 0.9)) + try: + annot.set_info(content=json.dumps( + {"nullifypdf": 1, "original": "", "entity_type": kind} + )) + except Exception as e: + self.logger.debug(f"Could not store {kind} metadata: {e}") + + def _read_redaction_full(self, annot: Any) -> Optional[fitz.Rect]: + """The whole word box stored with a band annotation, if any.""" + try: + raw = annot.info.get("content", "") + payload = json.loads(raw) if raw else {} + if payload.get("nullifypdf") == 1 and payload.get("full"): + return fitz.Rect(*[float(v) for v in payload["full"]]) + except Exception: + return None + return None + def _read_redaction_payload(self, annot: Any) -> Dict[str, str]: """Read privacy data stored on a pending redaction annotation.""" try: @@ -1861,7 +2609,7 @@ def _prepare_pseudonymized_page( ( annot.rect, self._read_redaction_payload(annot), - page.get_text("text", clip=annot.rect).strip(), + text_under_rect(page, annot.rect), capture_text_style(page, annot.rect), ) for annot in (page.annots() or []) @@ -1871,6 +2619,13 @@ def _prepare_pseudonymized_page( if annot.type[0] == PDF_ANNOT_REDACT: page.delete_annot(annot) for rect, payload, clipped_text, style in pending: + if payload.get("entity_type") in ("SIGNATURE", "IMAGE"): + self._add_kind_redaction(page, rect, str(payload["entity_type"])) + self._export_labels.append( + (page.number, fitz.Rect(rect), + KIND_LABELS[str(payload["entity_type"])], None, None) + ) + continue clean_original = " ".join( (payload.get("original") or clipped_text).split() ) @@ -1885,33 +2640,59 @@ def _prepare_pseudonymized_page( rect=(rect.x0, rect.y0, rect.x1, rect.y1), style=style, ) - page.add_redact_annot( - rect, - text=placeholder, - fill=(1, 1, 1), - text_color=(0, 0, 0), - align=1, - # Must fit on one line: a wrapped placeholder is still - # readable but no longer findable by search_for(), which - # is the only handle a version 1 restore map has. - fontsize=fit_redaction_fontsize(placeholder, rect), + # The placeholder is drawn after apply_redactions() (see + # _finish_pseudonymized_page): replacement text passed to + # the annotation is dropped when it cannot be laid out at 4 pt. + page.add_redact_annot(rect, fill=(1, 1, 1)) + self._export_labels.append( + (page.number, fitz.Rect(rect), compact_labels(placeholder), + style, placeholder) ) else: - page.add_redact_annot( - rect, - text="[IMAGE_REMOVED]", - fill=(1, 1, 1), - text_color=(0, 0, 0), - align=1, - fontsize=fit_redaction_fontsize("[IMAGE_REMOVED]", rect), + self._add_kind_redaction(page, rect, "IMAGE") + self._export_labels.append( + (page.number, fitz.Rect(rect), KIND_LABELS["IMAGE"], None, None) + ) + + def _prepare_label_annots(self, page: Any) -> None: + """Anonymize mode: remember the image / signature areas to label.""" + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + kind = self._read_redaction_payload(annot).get("entity_type") + if kind in KIND_LABELS: + self._export_labels.append( + (page.number, fitz.Rect(annot.rect), KIND_LABELS[kind], None, None) + ) + + def _finish_labels_page( + self, page: Any, registry: Optional[PlaceholderRegistry] = None + ) -> None: + """Draw placeholders and image/signature labels once redactions are applied.""" + for page_no, rect, labels, style, placeholder in self._export_labels: + if page_no != page.number: + continue + drawn = write_label( + page, rect, labels, style=style, + allowance=free_space_right(page, rect) if placeholder else 0.0, + ) + if placeholder and registry is not None and drawn != placeholder: + registry.set_label( + placeholder, (rect.x0, rect.y0, rect.x1, rect.y1), drawn ) + def _finish_pseudonymized_page( + self, page: Any, registry: PlaceholderRegistry + ) -> None: + self._finish_labels_page(page, registry) + @Slot(int, object) def apply_ai_to_page(self, i: int, detections: Any) -> None: """Apply AI-detected redactions to page. - Runs on the UI thread (Qt slot). Holds `doc_mutex` while mutating the - page so the AI worker's `get_text()` calls do not race. + Runs on the UI thread (Qt slot). Detections in state "accepted" are + applied; those in state "suggested" are queued for review and never + applied silently. Args: i: Page index. @@ -1926,32 +2707,67 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: ] else: normalized_detections = list(detections or []) + accepted = [ + d for d in normalized_detections if d.get("state", "accepted") == "accepted" + ] + for d in normalized_detections: + if d.get("state") == "suggested": + self.pending_suggestions.append({**d, "page": i}) + self._update_review_button() + self._apply_detections(i, accepted) + + def _update_review_button(self) -> None: + n = len(self.pending_suggestions) + self.btn_review.setText(f"Rivedi suggerimenti ({n})") + self.btn_review.setEnabled(n > 0) + + def _apply_detections(self, i: int, detections: List[Dict[str, Any]]) -> None: + """Add redaction annotations for `detections` on page `i`.""" with QMutexLocker(self.doc_mutex): - if i >= len(self.doc): + if not self.doc or i >= len(self.doc): return page = self.doc[i] e_rects = [ a.rect for a in (page.annots() or []) if a.type[0] == PDF_ANNOT_REDACT ] + + def already_covered(r: fitz.Rect) -> bool: + return covered_fraction( + (r.x0, r.y0, r.x1, r.y1), + [(e.x0, e.y0, e.x1, e.y1) for e in e_rects], + ) >= 0.6 + if self.chk_img.isChecked(): for img in page.get_image_info(hashes=False): ir = fitz.Rect(img["bbox"]) - page.add_redact_annot( - ir, - text="[ IMMAGINE RIMOSSA ]", - align=1, - fill=(0.9, 0.9, 0.9), - fontsize=8, - ) + if already_covered(ir): + continue + self._add_kind_redaction(page, ir, "IMAGE") e_rects.append(ir) - page_index = build_page_index(page.get_text("words")) + try: + has_images = bool(page.get_image_info(hashes=False)) + except Exception: + has_images = False + page_index = build_page_index( + page.get_text("words"), + image_rects=[ + tuple(i["bbox"]) for i in page.get_image_info(hashes=False) + ], + ) + # Accepted redactions must cover the actual word box. The + # clamped index is useful for analysis, but using its thin band + # for the final annotation can leave the top/bottom of a glyph + # visible (as happens with tightly spaced lines). + full_page_index = build_page_index(page.get_text("words"), clamp=False) + + def cover_word(rect: fitz.Rect) -> fitz.Rect: + padded = fitz.Rect(rect.x0 - 0.5, rect.y0 - 0.5, rect.x1 + 0.5, rect.y1 + 0.5) + return padded & page.rect + for bw in self.blocklist: - for r in (fitz.Rect(*t) for t in whole_word_rects(page_index, bw)): - if not any( - e.contains(fitz.Point((r.x0 + r.x1) / 2, (r.y0 + r.y1) / 2)) - for e in e_rects - ): + for r in (cover_word(fitz.Rect(*t)) for t in whole_word_rects(full_page_index, bw)): + if not already_covered(r): self._add_privacy_redaction( page, r, @@ -1959,206 +2775,636 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: entity_type=self._infer_entity_type(bw), ) e_rects.append(r) - p_rects = [ - fitz.Rect(*t) - for aw in self.allowlist - for t in whole_word_rects(page_index, aw) - ] - for detection in normalized_detections: + for detection in detections: word = str(detection.get("text", "")) entity_type = str(detection.get("entity_type", "DATA")) raw_rects = detection.get("rects") or [] rects = [fitz.Rect(*coords) for coords in raw_rects] + fulls = detection.get("full_rects") or [] + for k, band in enumerate(rects): + if k < len(fulls): + # Always use the original word box for the final + # redaction, with a small margin in both dimensions. + rects[k] = cover_word(fitz.Rect(*fulls[k])) + if entity_type == "SIGNATURE": + # Graphic area (signature / stamp): no text, no value. + for r in rects: + if already_covered(r): + continue + self._add_kind_redaction(page, r, "SIGNATURE") + e_rects.append(r) + continue if not rects and word: - rects = list(page.search_for(word)) - for r in rects: - if any(r.intersects(pr) for pr in p_rects): - continue - if not any( - e.contains(fitz.Point((r.x0 + r.x1) / 2, (r.y0 + r.y1) / 2)) - for e in e_rects - ): + # Legacy detections without geometry: whole words only, + # never a substring search (it redacts fragments of + # longer words), and never a word the allowlist protects. + protected = [ + fitz.Rect(*t) + for aw in self.allowlist + for t in whole_word_rects(full_page_index, aw) + ] + rects = [ + fitz.Rect(*t) + for t in whole_word_rects(full_page_index, word) + if not any( + pr.contains(fitz.Point((t[0] + t[2]) / 2, (t[1] + t[3]) / 2)) + for pr in protected + ) + ] + for k, r in enumerate(rects): + if not already_covered(r): + full_box = ( + cover_word(fitz.Rect(*fulls[k])) + if (k < len(fulls) and raw_rects) else None + ) self._add_privacy_redaction( page, r, original=word, entity_type=entity_type, + full=full_box, ) e_rects.append(r) + def _review_suggestions(self) -> None: + """Let the user accept or reject the queued suggestions.""" + if not self.pending_suggestions: + return + dlg = ReviewDialog(self.pending_suggestions, self) + dlg.navigate.connect(self._go_to_page) + dlg.highlight.connect(self._highlight_suggestion) + if dlg.exec() != QDialog.DialogCode.Accepted: + return + by_page: Dict[int, List[Dict[str, Any]]] = {} + for item in dlg.applied(): + by_page.setdefault(int(item.get("page", 0)), []).append(item) + for page_no, items in by_page.items(): + self._apply_detections(page_no, items) + decided = dlg.applied() + dlg.declined() + self.pending_suggestions = [ + p for p in self.pending_suggestions + if not any(p is d for d in decided) + ] + self._update_review_button() + self.render_page() + self.write_log( + f"Revisione: {len(dlg.applied())} applicati, " + f"{len(dlg.declined())} scartati." + ) + + def _go_to_page(self, page_no: int) -> None: + if self.doc and 0 <= page_no < len(self.doc): + self.page_num = page_no + self.render_page() + @Slot() def ai_finished(self) -> None: """Handle AI scan completion.""" self.btn_ai.setEnabled(True) self.render_page() + if self.pending_suggestions: + self._review_suggestions() + + def _redaction_counts(self) -> Dict[str, int]: + """Pending redactions per entity type (no values).""" + counts: Dict[str, int] = {} + for page in self.doc or []: + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + payload = self._read_redaction_payload(annot) + key = payload.get("entity_type") or "UNSPECIFIED" + counts[key] = counts.get(key, 0) + 1 + return counts + + def _select_pseudonym_style(self) -> Optional[Tuple[str, Optional[bytes]]]: + """Ask how pseudonyms are generated. Returns (style, key) or None.""" + box = QMessageBox(self) + box.setWindowTitle("Stile dei segnaposto") + box.setIcon(QMessageBox.Icon.Question) + box.setText("Come devono essere generati i segnaposto?") + box.setInformativeText( + "Casuale: non rivela ordine o numero dei valori (consigliato).\n" + "Progressivo: PERSON_001, PERSON_002... rivela quanti valori " + "distinti esistono e l'ordine in cui compaiono.\n" + "Generico: nasconde anche il tipo di dato.\n" + "Coerente: lo stesso valore riceve lo stesso segnaposto in tutti i " + "documenti creati con la stessa chiave (permette di collegarli)." + ) + b_random = box.addButton("Casuale", QMessageBox.ButtonRole.AcceptRole) + b_counter = box.addButton("Progressivo", QMessageBox.ButtonRole.ActionRole) + b_opaque = box.addButton("Generico", QMessageBox.ButtonRole.ActionRole) + b_keyed = box.addButton("Coerente (chiave)", QMessageBox.ButtonRole.ActionRole) + b_manage = box.addButton("Gestisci chiave...", QMessageBox.ButtonRole.ActionRole) + b_random.setToolTip("Segnaposto casuali: non rivelano ordine nรฉ numero dei valori (consigliato).") + b_counter.setToolTip("PERSON_001, PERSON_002...: rivelano quanti valori distinti ci sono e l'ordine.") + b_opaque.setToolTip("Segnaposto generici: nascondono anche il tipo di dato.") + b_keyed.setToolTip("Lo stesso valore ha lo stesso segnaposto in ogni documento creato con la stessa chiave.") + b_manage.setToolTip("Esporta o importa la chiave dei segnaposto coerenti, protetta da password.") + box.addButton(QMessageBox.StandardButton.Cancel) + box.exec() + clicked = box.clickedButton() + if clicked == b_random: + return "random", None + if clicked == b_counter: + return "counter", None + if clicked == b_opaque: + return "opaque", None + if clicked == b_manage: + self._pseudonym_key_dialog() + return self._select_pseudonym_style() + if clicked == b_keyed: + try: + key = self._load_pseudonym_key() + except (secure_store.KeystoreUnavailable, secure_store.SealError, OSError) as e: + QMessageBox.critical(self, "Chiave non disponibile", str(e)) + return None + QMessageBox.information( + self, + "Chiave dei segnaposto", + "La chiave e protetta (" + + secure_store.protector_label(self.config_dir) + + "). Conservane una copia protetta da password con " + "\"Gestisci chiave...\" per usarla su un altro computer: chi " + "la possiede puo ricollegare i segnaposto ai valori noti, e " + "senza di essa i documenti futuri avranno segnaposto diversi.", + ) + return "keyed", key + return None + return None + + def _pseudonym_key_paths(self) -> Tuple[pathlib.Path, pathlib.Path]: + return self.config_dir / "pseudonym.v2.enc", self.config_dir / "pseudonym.key" + + def _store_pseudonym_key(self, key: bytes) -> None: + enc, _legacy = self._pseudonym_key_paths() + master = secure_store.load_or_create_master_key(self.config_dir) + secure_store.atomic_write( + enc, secure_store.seal(key, master, b"nullifypdf-pseudonym-key") + ) + + def _load_pseudonym_key(self) -> bytes: + """The 32-byte pseudonym key, protected by the local master key. + + A plaintext ``pseudonym.key`` from an older version is validated, + sealed and removed. Raises on an invalid or unreadable key. + """ + enc, legacy = self._pseudonym_key_paths() + master = secure_store.load_or_create_master_key(self.config_dir) + if enc.exists(): + key = secure_store.open_sealed( + enc.read_bytes(), master, b"nullifypdf-pseudonym-key" + ) + elif legacy.exists(): + key = legacy.read_bytes() + if len(key) != 32: + raise OSError( + "Chiave dei segnaposto non valida: il file deve " + "contenere esattamente 32 byte." + ) + self._store_pseudonym_key(key) + secure_store.secure_delete(legacy) + else: + key = os.urandom(32) + self._store_pseudonym_key(key) + if len(key) != 32: + raise OSError("Chiave dei segnaposto non valida.") + return key + + def _pseudonym_key_dialog(self) -> None: + """Export / import a password-protected copy of the pseudonym key.""" + box = QMessageBox(self) + box.setWindowTitle("Chiave dei segnaposto") + box.setText("Esporta o importa la chiave dei segnaposto coerenti.") + box.setInformativeText( + "La copia esportata e protetta da password e serve per usare la " + "stessa chiave su un altro computer." + ) + b_exp = box.addButton("Esporta chiave...", QMessageBox.ButtonRole.ActionRole) + b_imp = box.addButton("Importa chiave...", QMessageBox.ButtonRole.ActionRole) + b_exp.setToolTip("Salva una copia della chiave protetta da password.") + b_imp.setToolTip("Carica una copia della chiave esportata da un altro computer.") + box.addButton(QMessageBox.StandardButton.Close) + box.exec() + clicked = box.clickedButton() + try: + if clicked == b_exp: + key = self._load_pseudonym_key() + pw = self._ask_map_password() + if pw is None: + return + path, _ = QFileDialog.getSaveFileName( + self, "Esporta chiave", "pseudonym_key.nullifypdf-key", + "NullifyPDF key (*.nullifypdf-key)", + ) + if path: + write_private_file(path, export_pseudonym_key(key, pw)) + self.write_log("Chiave dei segnaposto esportata.") + elif clicked == b_imp: + path, _ = QFileDialog.getOpenFileName( + self, "Importa chiave", "", "NullifyPDF key (*.nullifypdf-key)" + ) + if not path: + return + pw, ok = QInputDialog.getText( + self, "Password chiave", "Password della copia della chiave:", + QLineEdit.EchoMode.Password, + ) + if not ok or not pw: + return + enc, legacy = self._pseudonym_key_paths() + if enc.exists() or legacy.exists(): + ans = QMessageBox.question( + self, "Sostituire la chiave?", + "Esiste gia una chiave: sostituirla cambia i segnaposto " + "dei documenti futuri. Continuare?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return + with open(path, "rb") as fh: + key = import_pseudonym_key(fh.read(), pw) + self._store_pseudonym_key(key) + self.write_log("Chiave dei segnaposto importata.") + except Exception as e: + from cryptography.fernet import InvalidToken + + msg = "Password errata o file corrotto." if isinstance(e, InvalidToken) else str(e) + QMessageBox.critical(self, "Chiave dei segnaposto", msg) + + def _offer_recovery_key(self) -> Optional[bytes]: + """Offer a recovery key for the map; shown once, never stored.""" + ans = QMessageBox.question( + self, + "Chiave di recupero", + "Vuoi creare anche una chiave di recupero? Permette di aprire la " + "mappa se dimentichi la password. Non viene salvata dal programma.", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return None + raw, shown = generate_recovery_key() + dlg = QDialog(self) + dlg.setWindowTitle("Chiave di recupero") + lay = QVBoxLayout(dlg) + lay.addWidget(QLabel( + "Conserva questa chiave in un luogo sicuro, separato dalla mappa. " + "Viene mostrata una sola volta." + )) + field = QLineEdit(shown) + field.setReadOnly(True) + lay.addWidget(field) + row = QHBoxLayout() + field.setToolTip("Chiave di recupero: mostrata una sola volta e non salvata dal programma.") + btn_copy = QPushButton("Copia") + btn_copy.setToolTip("Copia la chiave negli appunti (vengono svuotati alla chiusura).") + btn_copy.clicked.connect(lambda: QApplication.clipboard().setText(shown)) + btn_save = QPushButton("Salva su file...") + btn_save.setToolTip("Salva la chiave in un file di testo, da conservare lontano dalla mappa.") + + def save_file() -> None: + path, _ = QFileDialog.getSaveFileName( + dlg, "Salva chiave di recupero", "recovery_key.txt", "Testo (*.txt)" + ) + if path: + write_private_file(path, shown.encode("utf-8")) + + btn_save.clicked.connect(save_file) + row.addWidget(btn_copy) + row.addWidget(btn_save) + lay.addLayout(row) + chk = QCheckBox( + "Ho conservato la chiave di recupero in un luogo sicuro, separato dalla mappa" + ) + chk.setToolTip("Spunta per confermare: serve per proseguire.") + lay.addWidget(chk) + bb = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok) + ok_btn = bb.button(QDialogButtonBox.StandardButton.Ok) + ok_btn.setEnabled(False) + chk.toggled.connect(ok_btn.setEnabled) + bb.accepted.connect(dlg.accept) + lay.addWidget(bb) + dlg.exec() + QApplication.clipboard().clear() + # The key has been shown (and maybe saved): the map must carry its + # wrap even if the dialog was closed without ticking the box. + return raw + + def _ask_decrypt_secret(self) -> Optional[Tuple[Optional[str], Optional[bytes]]]: + """Ask for the map password, or the recovery key. None = cancelled.""" + dlg = QDialog(self) + dlg.setWindowTitle("Password mappa") + lay = QVBoxLayout(dlg) + label = QLabel("Password per decifrare la mappa di ripristino:") + lay.addWidget(label) + field = QLineEdit() + field.setEchoMode(QLineEdit.EchoMode.Password) + lay.addWidget(field) + use_rk = QCheckBox("Usa la chiave di recupero") + use_rk.setToolTip("Apre la mappa con la chiave di recupero (NPDF-...) invece che con la password.") + field.setToolTip("Password della mappa, oppure la chiave di recupero se hai spuntato l'opzione.") + + def toggled(on: bool) -> None: + label.setText( + "Chiave di recupero (NPDF-...):" if on + else "Password per decifrare la mappa di ripristino:" + ) + + use_rk.toggled.connect(toggled) + lay.addWidget(use_rk) + bb = QDialogButtonBox( + QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel + ) + bb.accepted.connect(dlg.accept) + bb.rejected.connect(dlg.reject) + lay.addWidget(bb) + if dlg.exec() != QDialog.DialogCode.Accepted or not field.text(): + return None + if use_rk.isChecked(): + try: + return None, parse_recovery_key(field.text()) + except ValueError as e: + QMessageBox.warning(self, "Chiave di recupero", str(e)) + return None + return field.text(), None + + def _ask_map_password(self) -> Optional[str]: + """Ask twice for the map password and enforce a minimum strength.""" + while True: + pw, ok = QInputDialog.getText( + self, + "Password mappa", + "Password per cifrare la mappa di ripristino (minimo 12 caratteri):", + QLineEdit.EchoMode.Password, + ) + if not ok: + return None + good, why = password_strength(pw) + if not good: + QMessageBox.warning(self, "Password non valida", why) + continue + confirm, ok = QInputDialog.getText( + self, + "Conferma password", + "Ripeti la password (senza di essa la mappa non e recuperabile):", + QLineEdit.EchoMode.Password, + ) + if not ok: + return None + if confirm != pw: + QMessageBox.warning( + self, "Password diversa", "Le due password non coincidono." + ) + continue + return pw + + def cmd_wipe_local_data(self) -> None: + """Delete logs, audit records and temp files kept in ~/.nullifypdf.""" + ans = QMessageBox.question( + self, + "Elimina dati locali", + "Verranno eliminati log, registri di esportazione e file " + "temporanei. Vuoi eliminare anche i dizionari (blocklist e " + "allowlist)?", + QMessageBox.StandardButton.Yes + | QMessageBox.StandardButton.No + | QMessageBox.StandardButton.Cancel, + QMessageBox.StandardButton.No, + ) + if ans == QMessageBox.StandardButton.Cancel: + return + include_lists = ans == QMessageBox.StandardButton.Yes + include_keys = False + if include_lists: + include_keys = QMessageBox.question( + self, + "Elimina anche le chiavi", + "Eliminare anche le chiavi? I dizionari cifrati e la chiave dei " + "segnaposto diventeranno illeggibili.", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) == QMessageBox.StandardButton.Yes + # The rotating log file is open: release it first or it cannot be + # removed (Windows), then reopen logging afterwards. + for h in list(self.logger.handlers): + try: + h.close() + except Exception: + pass + self.logger.removeHandler(h) + removed = wipe_local_data( + self.config_dir, include_lists=include_lists, include_keys=include_keys + ) + self.logger = setup_logging() + if include_lists: + self.blocklist, self.allowlist = set(), set() + self.write_log(f"Dati locali eliminati: {', '.join(removed) or 'nessuno'}.") def cmd_export(self) -> None: - """Export current PDF in irreversible or reversible privacy mode. - - Memory strategy (fix C1): instead of materializing the whole document - in RAM via `self.doc.write()` (which would double peak memory on large - PDFs), we stream through a sibling temp file next to the target path: - - 1. Save `self.doc` (with annotations, NOT yet flattened) to - `.tmp` on disk. - 2. Open the temp file as `ex_doc`. - 3. Apply redactions / scrub metadata on `ex_doc`. - 4. Save `ex_doc` to the user-chosen final path. - 5. Remove the temp file. - - The original `self.doc` is never mutated (redactions stay as - annotations), so the user can keep editing after export. Peak extra - memory is O(1) over what PyMuPDF needs internally for save(). + """Export the current PDF in irreversible or reversible privacy mode. + + The work happens in :func:`export_engine.export_document` on a copy + kept in memory (or in a private per-user folder for very large + files): the original ``self.doc`` is never mutated, so the user can + keep editing, and no unredacted copy is ever written beside the + output. The written file is sanitised and then verified. """ if not self.doc: return + if self.pending_suggestions: + box = QMessageBox(self) + box.setIcon(QMessageBox.Icon.Warning) + box.setWindowTitle("Suggerimenti non revisionati") + box.setText( + f"Ci sono {len(self.pending_suggestions)} suggerimenti non " + "revisionati: non verranno censurati." + ) + b_review = box.addButton("Rivedi ora", QMessageBox.ButtonRole.AcceptRole) + b_go = box.addButton("Esporta comunque", QMessageBox.ButtonRole.DestructiveRole) + b_review.setToolTip("Apre l'elenco dei suggerimenti prima di esportare.") + b_go.setToolTip("Esporta senza censurare i suggerimenti non revisionati.") + box.addButton(QMessageBox.StandardButton.Cancel) + box.exec() + if box.clickedButton() == b_review: + self._review_suggestions() + return + if box.clickedButton() != b_go: + return mode = self._select_privacy_export() if mode is None: return + with QMutexLocker(self.doc_mutex): + removed_values = collect_removed_values( + self.doc, self._read_redaction_payload + ) + entity_counts = self._redaction_counts() + src_dir = os.path.dirname(self.doc.name or "") or str(pathlib.Path.home()) p, _ = QFileDialog.getSaveFileName( self, "Esporta", - f"{os.path.splitext(self.doc.name)[0]}_{mode.value}.pdf", + os.path.join(src_dir, neutral_output_name(mode.value)), "PDF (*.pdf)", ) if not p: return + if name_leaks_value(p, removed_values): + ans = QMessageBox.question( + self, + "Nome file con dati personali", + "Il nome scelto contiene un valore che stai rimuovendo. " + "Continuare comunque?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return map_path = "" map_password = "" + recovery_key: Optional[bytes] = None + style, key = "counter", None if mode == PrivacyMode.PSEUDONYMIZE: + if importlib.util.find_spec("cryptography") is None: + QMessageBox.critical( + self, + "Dipendenza mancante", + "La pseudonimizzazione richiede la libreria 'cryptography'.", + ) + return + chosen = self._select_pseudonym_style() + if chosen is None: + return + style, key = chosen map_path, _ = QFileDialog.getSaveFileName( self, "Salva mappa cifrata", - f"{os.path.splitext(p)[0]}.nullifypdf-map", + os.path.join(str(pathlib.Path.home()), neutral_map_name()), "NullifyPDF map (*.nullifypdf-map)", ) if not map_path: return - map_password, ok = QInputDialog.getText( - self, - "Password mappa", - "Password per cifrare la mappa di ripristino (minimo 12 caratteri):", - QLineEdit.EchoMode.Password, + same_dir = os.path.dirname(os.path.abspath(map_path)) == os.path.dirname( + os.path.abspath(p) ) - if not ok: - return - if len(map_password) < 12: - QMessageBox.warning( - self, - "Password troppo corta", - "La password della mappa deve avere almeno 12 caratteri.", - ) - return - if importlib.util.find_spec("cryptography") is None: - QMessageBox.critical( + synced = any( + tag in os.path.abspath(map_path).lower() + for tag in ("onedrive", "dropbox", "google drive", "icloud") + ) + if same_dir or synced: + ans = QMessageBox.question( self, - "Dipendenza mancante", - "La pseudonimizzazione richiede la libreria 'cryptography'.", + "Posizione della mappa", + "La mappa di ripristino e dato personale e va conservata " + "separatamente dal PDF pseudonimizzato" + + (", non in una cartella sincronizzata" if synced else "") + + ". Salvarla qui comunque?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, ) + if ans != QMessageBox.StandardButton.Yes: + return + pw = self._ask_map_password() + if pw is None: return + map_password = pw + recovery_key = self._offer_recovery_key() self.write_log( - "Export anonimizzato..." if mode == PrivacyMode.ANONYMIZE + "Export con rimozione irreversibile..." if mode == PrivacyMode.ANONYMIZE else "Export pseudonimizzato..." ) - # Sibling temp file (same directory as target so rename/cleanup are - # always on the same filesystem; avoids cross-device issues). - tmp_path = p + ".nullifypdf.tmp" - ex_doc = None - registry = PlaceholderRegistry() + registry = PlaceholderRegistry(style=style, key=key) + policy = SanitizePolicy(mask_removed_width=self.chk_mask_width.isChecked()) try: source_name = self.doc.name or "document.pdf" source_sha256 = sha256_file(source_name) if os.path.exists(source_name) else "" - # Step 1: serialize `self.doc` (with annotations) to disk under - # the mutex so the AI worker thread cannot mutate mid-write. + prepare = ( + (lambda page: self._prepare_pseudonymized_page(page, registry)) + if mode == PrivacyMode.PSEUDONYMIZE else self._prepare_label_annots + ) + finish = ( + (lambda page: self._finish_labels_page(page, registry)) + if mode == PrivacyMode.PSEUDONYMIZE else self._finish_labels_page + ) + self._export_labels = [] with QMutexLocker(self.doc_mutex): if not self.doc: return - # `clean=False` here: we want a faithful copy of the - # in-memory state, including all redact annotations. The - # real cleanup pass happens on ex_doc.save() below. - self.doc.save(tmp_path, garbage=0, deflate=False, clean=False) - - # Step 2: reopen the on-disk copy. PyMuPDF lazy-parses pages, so - # this does NOT load the whole PDF into RAM up front. - ex_doc = fitz.open(tmp_path) - - # Step 3: scrub on the disk-backed copy. - for page in ex_doc.pages(): - if mode == PrivacyMode.PSEUDONYMIZE: - self._prepare_pseudonymized_page(page, registry) - # page.annots() may return None for pages with no annotations - r_rects = self._collect_redaction_rects(page) - try: - links_to_delete = [ - lnk for lnk in page.get_links() - if any(fitz.Rect(lnk["from"]).intersects(r) for r in r_rects) - ] - for lnk in links_to_delete: - page.delete_link(lnk) - except (RuntimeError, AttributeError, KeyError) as e: - self.logger.debug(f"Could not delete link: {e}") - # PDF_REDACT_IMAGE_REMOVE deletes the ENTIRE image object if it - # intersects ANY redaction rect, even a small one. On a scanned - # PDF (the whole page is one image), redacting a single word - # wiped out the entire page, leaving a blank sheet with only - # the placeholder text visible. PDF_REDACT_IMAGE_PIXELS blanks - # out only the pixels under each redaction rect and leaves the - # rest of the image intact (verified this also fully blanks - # whole-image redactions from "Oscura Immagini", including - # under page rotation, with no visible fringe). - page.apply_redactions( - images=PDF_REDACT_IMAGE_PIXELS, graphics=True + result = export_document( + self.doc, + p, + prepare_page=prepare, + finish_page=finish, + read_full=self._read_redaction_full, + policy=policy, + removed_values=removed_values, + temp_dir=private_temp_dir(self.config_dir), + log=self.logger.debug, ) - try: - # Materialize first: mutating during iteration of - # page.widgets() can invalidate the generator. - for w in list(page.widgets() or []): - page.delete_widget(w) - except (RuntimeError, AttributeError) as e: - self.logger.debug(f"Could not delete widget: {e}") - ex_doc.set_metadata({}) - cx = ex_doc.pdf_catalog() - for k in ["Metadata", "PieceInfo", "Properties", "AcroForm"]: - ex_doc.xref_set_key(cx, k, "null") - - # Step 4: write final scrubbed output. Must save to a different - # path than the open source (`tmp_path`) to avoid PyMuPDF's - # "save to original" restriction on incremental saves. - ex_doc.save(p, garbage=4, deflate=True, clean=True) + if not result.verification.ok: + where = "\n".join(result.verification.findings[:12]) + ans = QMessageBox.critical( + self, + "Verifica dell'esportazione fallita", + "Nel file esportato sono ancora presenti dati che dovevano " + "essere rimossi o contenuti attivi. Posizioni:\n" + where + + "\n\nEliminare il file esportato?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.Yes, + ) + self.logger.error( + f"Export verification failed at {len(result.verification.findings)} location(s)" + ) + if ans == QMessageBox.StandardButton.Yes: + try: + os.remove(p) + except OSError as e: + self.logger.debug(f"Could not remove failed export: {e}") + self.write_log("ERRORE export: file eliminato dopo verifica fallita.") + return if mode == PrivacyMode.PSEUDONYMIZE: payload = build_restore_payload( source_name=source_name, source_sha256=source_sha256, - output_sha256=sha256_file(p), + output_sha256=result.output_sha256, entries=registry.entries(), ) - encrypted = encrypt_restore_payload(payload, map_password) - with open(map_path, "wb") as fh: - fh.write(encrypted) + encrypted = encrypt_restore_payload( + payload, map_password, recovery_key=recovery_key + ) + write_private_file(map_path, encrypted) self.write_log( f"ESPORTATO. Mappa cifrata creata con {len(registry.entries())} segnaposto." ) else: self.write_log("ESPORTATO.") + record = build_audit_record( + app_version=APP_VERSION, + mode=mode.value, + entity_counts=entity_counts, + suggestion_counts={"unreviewed": len(self.pending_suggestions)}, + sanitize=result.sanitize, + verification=result.verification, + output_sha256=result.output_sha256, + pages=result.pages, + ) + write_audit_record(self.config_dir, record) + QMessageBox.information( + self, + "Esportazione completata", + ( + "Il file esportato puo contenere ancora dati personali non " + "rilevati o indiretti. Controllalo prima di condividerlo." + if mode == PrivacyMode.ANONYMIZE + else "Il PDF pseudonimizzato e la mappa di ripristino " + "restano dati personali: conservali separatamente e proteggi " + "la password." + ), + ) except Exception as e: self.logger.error(f"Export failed: {traceback.format_exc()}") self.write_log(f"ERRORE export: {type(e).__name__}: {str(e)}") - finally: - # Always close ex_doc, even on failure, to release the file - # handle on tmp_path before we try to remove it (Windows locks - # open files). - if ex_doc is not None: - try: - ex_doc.close() - except Exception as e: - self.logger.debug(f"Error closing ex_doc: {e}") - # Step 5: remove temp file (best-effort). - try: - if os.path.exists(tmp_path): - os.remove(tmp_path) - except OSError as e: - self.logger.debug(f"Could not remove temp file {tmp_path}: {e}") def cmd_reconstruct(self) -> None: """Reconstruct a previously pseudonymized PDF using its restore map. @@ -2187,18 +3433,17 @@ def cmd_reconstruct(self) -> None: ) if not map_path: return - map_password, ok = QInputDialog.getText( - self, - "Password mappa", - "Password per decifrare la mappa di ripristino:", - QLineEdit.EchoMode.Password, - ) - if not ok or not map_password: + secret = self._ask_decrypt_secret() + if secret is None: return + map_password, recovery_secret = secret out_path, _ = QFileDialog.getSaveFileName( self, "Salva PDF ricostruito", - f"{os.path.splitext(pdf_path)[0]}_ricostruito.pdf", + os.path.join( + os.path.dirname(pdf_path), + neutral_output_name("restored"), + ), "PDF (*.pdf)", ) if not out_path: @@ -2210,12 +3455,20 @@ def cmd_reconstruct(self) -> None: try: with open(map_path, "rb") as fh: encrypted = fh.read() - payload = decrypt_restore_payload(encrypted, map_password) + payload = decrypt_restore_payload( + encrypted, map_password, recovery_key=recovery_secret + ) expected = expected_restore_count(payload) restored_count = reconstruct_pdf(pdf_path, out_path, payload) self.write_log( f"RICOSTRUITO. {restored_count} segnaposto ripristinati." ) + QMessageBox.information( + self, + "PDF ricostruito", + "Il file ricostruito contiene di nuovo tutti i dati personali " + "originali: trattalo e conservalo di conseguenza.", + ) # A restore that silently puts nothing back used to look like a # success: same message, no dialog, output identical to input. if restored_count < expected: @@ -2231,11 +3484,11 @@ def cmd_reconstruct(self) -> None: self.write_log(f"ERRORE ricostruzione: {e}") QMessageBox.critical(self, "Mappa non corrispondente", str(e)) except InvalidToken: - self.write_log("ERRORE ricostruzione: password errata o mappa corrotta.") + self.write_log("ERRORE ricostruzione: password/chiave errata o mappa corrotta.") QMessageBox.critical( self, "Errore decifrazione", - "Password errata o file della mappa corrotto.", + "Password o chiave di recupero errata, oppure mappa corrotta.", ) except Exception as e: self.logger.error(f"Reconstruction failed: {traceback.format_exc()}") @@ -2252,6 +3505,7 @@ def cmd_reconstruct(self) -> None: try: window = NullifyPDF() window.show() + QTimer.singleShot(1500, window.auto_check_updates) exit_code = app.exec() except Exception as e: logger = setup_logging() diff --git a/README.md b/README.md index 6716f41..57f2912 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# NullifyPDF - AI Privacy Edition +# NullifyPDF - Privacy Edition ![GitHub Release](https://img.shields.io/github/v/release/overwrite00/NullifyPDF?style=flat-square&color=1fb2e0&label=stable) ![GitHub Release (beta)](https://img.shields.io/github/v/release/overwrite00/NullifyPDF?include_prereleases&style=flat-square&color=orange&label=beta) @@ -23,7 +23,7 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi | ๐ŸŽฏ Feature | โœจ Benefit | | -------------------- | ------------------------------------- | -| ๐Ÿง  **AI-Powered** | Bilingual (EN/IT) automatic detection | +| ๐Ÿง  **Auto-detection** | Bilingual (EN/IT) automatic detection | | ๐Ÿ” **Local-first** | No cloud uploads during PDF processing | | โšก **Responsive UI** | Background scanning with live preview | | ๐Ÿ›ก๏ธ **Privacy Export** | Irreversible redaction or reversible placeholders | @@ -32,10 +32,15 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi ## โœจ Key Features -- ๐Ÿง  **AI-Powered Redaction** โ€” Automatic bilingual (EN/IT) detection of PII: names, locations, emails, phones, IBANs, credit cards, crypto addresses, Italian fiscal codes and IDs. A dialog lets you choose which data types to scan for, and matching is whole-word so ordinary text is not caught +- ๐Ÿง  **Automatic Redaction** โ€” Automatic bilingual (EN/IT) detection of PII: names, locations, emails, phones, IBANs, credit cards, crypto addresses, Italian fiscal codes and IDs. A dialog lets you choose which data types to scan for, and matching is whole-word so ordinary text is not caught +- ๐Ÿ”„ **Update Check** โ€” Optional, user-triggered check for new releases (stable builds see stable only, beta builds see both) showing the release notes and a one-click download into your Downloads folder; can be enabled at every start from the Info dialog (off by default) - ๐Ÿ—„๏ธ **Fluid UI & Thread-Safe** โ€” PySide6 modern dark-mode interface with zero UI freezing. Text extraction in worker thread with QMutex serialization - ๐Ÿ“– **Persistent Dictionaries** โ€” Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with O(1) fast-path matching -- ๐Ÿ›ก๏ธ **Privacy Export Modes** โ€” Choose irreversible anonymization or reversible pseudonymization with a separately encrypted restore map +- ๐Ÿ›ก๏ธ **Privacy Export Modes** โ€” Choose irreversible removal or reversible pseudonymization with a separately encrypted restore map; the exported file is sanitised (annotations, attachments, scripts, metadata, hidden text) and verified +- ๐Ÿ“– **Built-in guide and tooltips** โ€” Press F1 for an Italian guide to every function; every control shows a description on hover +- ๐Ÿ” **Review of uncertain hits** โ€” Detections without enough evidence are suggested for review instead of being applied or lost silently +- ๐Ÿง  **Optional local models** โ€” Reviewed ONNX PII models can be downloaded by the user from pinned Hugging Face revisions; files are SHA-256 verified and models only add review suggestions +- ๐Ÿ“„ **Word import** โ€” DOCX paragraphs and tables can be imported locally into the analysis workflow; legacy binary DOC files are not supported - ๐Ÿ”Ž **OCR Support** โ€” Bundled EN/IT Tesseract language data for scanned PDFs - ๐Ÿ–ผ๏ธ **Blindfold Mode** โ€” One-click image/logo censoring with professional placeholder: `[ IMAGE REMOVED ]` - ๐Ÿ“ฆ **Native Cross-Platform** โ€” Automated build scripts generate Windows `.exe`, macOS ZIP bundles, and Linux `.deb`/`.rpm` packages @@ -52,6 +57,7 @@ To keep NullifyPDF lightweight, 100% offline, and secure, be aware of these tech | ---------------------------------- | ------------------------------------------------------------------------------------------------------- | | **Handwritten Text** | NLP models cannot analyze non-digitized handwriting. | | **Password-Protected PDFs** | Encrypted documents are blocked at load. Decrypt before importing. | +| **Legacy Word DOC files** | Only modern `.docx` files are supported; save legacy files as DOCX before importing. | | **Digital Signatures Invalidated** | Privacy export changes the PDF; cryptographic signatures (PAdES, notarized) become invalid. | > [!WARNING] @@ -72,7 +78,7 @@ To keep NullifyPDF lightweight, 100% offline, and secure, be aware of these tech **Operating System Support:** - โœ… Windows 10/11 - โœ… macOS 11+ -- โœ… Linux (Ubuntu 20.04+, Fedora 33+) +- โœ… Linux: Ubuntu 24.04 / 26.04 and Debian 13 (`.deb`), RHEL / Rocky / Alma 9-10 and current Fedora (`.rpm`). The packages are built against glibc 2.34, so Ubuntu 22.04 and Debian 12 also run them, but these are no longer tested ### โš™๏ธ Installation @@ -83,8 +89,8 @@ Download the latest pre-compiled executable from [Releases](https://github.com/o - **Windows:** `NullifyPDF_vX.Y.Z_Windows.exe` - **macOS:** `NullifyPDF_vX.Y.Z_macOS.zip` -- **Ubuntu:** `NullifyPDF_vX.Y.Z_Ubuntu.deb` -- **Fedora:** `NullifyPDF_vX.Y.Z_Fedora.rpm` +- **Ubuntu / Debian:** `NullifyPDF_vX.Y.Z_Ubuntu.deb` +- **Fedora / RHEL:** `NullifyPDF_vX.Y.Z_Fedora.rpm` No installation needed on Windows/macOS - just run or unzip. Linux users can install the `.deb` or `.rpm` package. @@ -119,7 +125,7 @@ No installation needed on Windows/macOS - just run or unzip. Linux users can ins This script automatically: - Creates isolated virtual environment (`.venv`) - Installs all dependencies - - Downloads spaCy language models (EN/IT) + - Installs the Italian spaCy model for development (the installer downloads it on demand) 4. **Activate environment & launch** @@ -152,7 +158,7 @@ python setup_env.py - Detects OS (Windows/macOS/Linux) - Creates `.venv` with Python 3.13 - Installs `requirements.txt` dependencies -- Downloads spaCy models (English, Italian, both) +- Installs the Italian spaCy model for development (the installer does not bundle it) **Automatic OS detection:** - Windows: Uses `py -3.13` launcher @@ -226,7 +232,7 @@ pytest tests/ -v โœ… **No Telemetry** โ€” Zero user tracking > [!IMPORTANT] -> Irreversible anonymization cannot be undone from the exported PDF. Always keep backups of original documents. Pseudonymization requires the encrypted restore map and its password. +> Irreversible removal cannot be undone from the exported PDF, and it does not by itself guarantee anonymisation: indirect identifiers can remain. A pseudonymized PDF and its restore map are personal data. Always keep backups of original documents. Pseudonymization requires the encrypted restore map and its password. See [SECURITY.md](./SECURITY.md) for responsible disclosure and privacy details. @@ -247,11 +253,43 @@ See [SECURITY.md](./SECURITY.md) for responsible disclosure and privacy details. ## ๐Ÿ“ License -**MIT License** โ€” Free to use, modify, and distribute +**GNU Affero General Public License v3.0 or later (AGPL-3.0-or-later)** Copyright (c) 2026 Graziano Mariella -See [LICENSE](./LICENSE) for full text. +NullifyPDF is free of charge and is not sold by its author. It is AGPL-licensed +because it uses [PyMuPDF](https://pymupdf.readthedocs.io/), which is available +under the AGPL-3.0 or a commercial licence from Artifex. Anyone who distributes +the program, modified or not, must give the recipients the source code under the +same licence. Releases up to v2.4.0-beta.4 were published under the MIT licence. + +The Italian name-recognition model (spaCy `it_core_news_md`, CC BY-NC-SA 3.0, +non-commercial) is **not** part of the installer: the program offers to download +it, shows its licence and asks for consent first. Licences of all third-party +components are listed in [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md). + +**Notice.** NullifyPDF is free software, distributed under the AGPL-3.0-or-later. Every component and model it uses remains under its own licence, listed below with a link to the official text; using the program means respecting those conditions. The Italian spaCy model is for non-commercial use only and is downloaded only after your consent. + +| Component | Licence | Used for | +|---|---|---| +| [NullifyPDF](https://github.com/overwrite00/NullifyPDF/blob/main/LICENSE) | AGPL-3.0-or-later | this program | +| [PyMuPDF / MuPDF](https://pymupdf.readthedocs.io/en/latest/about.html) | AGPL-3.0 or commercial (Artifex) | PDF reading and redaction | +| [PySide6 / Qt](https://opensource.org/license/lgpl-3-0) | LGPL-3.0 | graphical interface | +| [Presidio](https://github.com/microsoft/presidio/blob/main/LICENSE) | MIT | personal-data detection | +| [spaCy](https://github.com/explosion/spaCy/blob/master/LICENSE) | MIT | language processing | +| [spaCy it_core_news_md](https://creativecommons.org/licenses/by-nc-sa/3.0/) | CC BY-NC-SA 3.0 (non-commercial use only) | Italian model, downloaded after consent | +| [onnxruntime](https://github.com/microsoft/onnxruntime/blob/main/LICENSE) | MIT | running local models | +| [tokenizers](https://github.com/huggingface/tokenizers/blob/main/LICENSE) | Apache-2.0 | tokenizing for local models | +| [cryptography](https://github.com/pyca/cryptography/blob/main/LICENSE) | Apache-2.0 or BSD-3-Clause | encryption | +| [keyring](https://github.com/jaraco/keyring) | MIT | key storage (Linux/macOS) | +| [tldextract](https://github.com/john-kurkowski/tldextract/blob/master/LICENSE) | BSD-3-Clause | URL recognition | +| [NumPy](https://github.com/numpy/numpy/blob/main/LICENSE.txt) | BSD-3-Clause | numerical computing | +| [Tesseract OCR](https://github.com/tesseract-ocr/tesseract/blob/main/LICENSE) | Apache-2.0 | OCR of scanned documents | +| [tessdata_fast (eng, ita)](https://github.com/tesseract-ocr/tessdata_fast/blob/main/LICENSE) | Apache-2.0 | OCR language data | +| [ISTAT, elenco dei comuni](https://creativecommons.org/licenses/by/4.0/) | CC BY 4.0 | place names | +| [Wikidata](https://creativecommons.org/publicdomain/zero/1.0/) | CC0 1.0 | given names and surnames | + +See [LICENSE](./LICENSE) for the full text. --- @@ -261,5 +299,5 @@ Want to help improve NullifyPDF? See [CONTRIBUTING.md](./CONTRIBUTING.md) for gu --- -*Last updated: 2026-09-18* +*Last updated: 2026-11-11* *[User Guide โ†’](./USER_GUIDE.md)* diff --git a/SECURITY.md b/SECURITY.md index e093e94..54bd5b9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -16,7 +16,7 @@ NullifyPDF follows a **privacy-first architecture**: | Principle | Implementation | | --------------------------- | ------------------------------------------------- | | **100% Local** | All PDF processing happens on your machine | -| **No Cloud** | No file uploads, no network transmission | +| **No Cloud** | No file uploads; the only network access is the optional update check (Info โ†’ Controlla aggiornamenti, or at every start if the user enables it; off by default), which sends no document data | | **No Telemetry** | Zero user tracking or analytics | | **Open Source** | Full code transparency, auditable by anyone | | **Privacy Export** | Irreversible redaction or encrypted reversible pseudonymization | @@ -37,29 +37,49 @@ NullifyPDF follows a **privacy-first architecture**: When you export a PDF with redactions: -1. โœ… **Metadata Stripped** โ€” Creation date, author, and document metadata removed -2. โœ… **Links Removed When Redacted** โ€” Links overlapping redaction areas are deleted -3. โœ… **Redactions Applied** โ€” Selected text/image areas are removed from the exported PDF -4. โœ… **Pseudonymization Maps Encrypted** โ€” Restore maps are stored separately and encrypted with a user password +1. โœ… **Redactions applied** โ€” selected text/image areas are removed from the content stream, not just covered +2. โœ… **Sanitiser** โ€” annotations, embedded files, external links, bookmarks, JavaScript, tagged-PDF alternative text, thumbnails, document and XMP metadata, invisible text, text outside the visible page box and the trailer ID are removed; hidden layers are shown before the scan so their text is analysed +3. โœ… **Verification** โ€” every redacted word is checked glyph by glyph (the number of characters actually deleted must match the measured deletion rule, and characters a thin box cannot reach are reported), then the written file is re-opened and checked for any removed value in the text layer, every object and decompressed stream, annotations, attachments, bookmarks and metadata; findings list locations, never the values +4. โœ… **Pseudonymization maps encrypted** โ€” restore maps are stored separately and encrypted with a user password (scrypt key derivation, authenticated encryption) +5. โš™๏ธ **Optional width masking** โ€” the space left by removed text can be rounded so its width is not recoverable (off by default: it can shift the following text slightly) -### Temporary Files +Limits you should know about: + +- Detection is statistical. Review the result and the suggestions before sharing; unrecognised or indirect identifiers (dates, places, job titles, clinical details) remain. +- Text drawn as part of an image is removed only where the image is redacted; OCR is used to find it. +- Signatures, stamps and photos are not detected automatically; use "Oscura Immagini" or draw a redaction manually. -During export, NullifyPDF uses **disk-backed temporary files**: +### Temporary Files -- Temporary data written to system temp directory -- Automatically cleaned up after export completes -- On Windows: `%APPDATA%\Local\Temp\` -- On macOS/Linux: `/tmp/` +- The working copy used during export is kept **in memory**. Only very large documents use a private per-user folder (`~/.nullifypdf/tmp`), emptied after the export and at the next start-up. +- No unredacted copy is written next to the exported file. +- Logs (`~/.nullifypdf/logs`) are rotated and purged after 30 days; they contain no detected values. "Elimina dati locali" removes logs, audit records and temporary files. -### No Sensitive Data in Memory +### Data kept in memory and on disk - Original PDF kept in memory only while open -- AI results (detected entities) stored only in-memory during session -- Exported PDF overwrites original data locations +- AI results (detected entities) stored only in memory during the session; suggestions are never written to disk +- The restore map is written with owner-only permissions where the platform supports it; keep it away from the PDF and from synchronised folders +- The blocklist and allowlist (`~/.nullifypdf/*.v2.enc`) may themselves contain personal data, so they are encrypted with AES-256-GCM under a master key protected by Windows DPAPI, the OS keyring (`keyring` package, installed on macOS and Linux; headless Linux without a Secret Service falls back to the key file) or, as a fallback, a local key file. Terms can expire and are then purged. DPAPI and the keyring bind the data to the user account and machine; the key file only protects against copying a single data file, not against someone who can read the user's profile. Overwriting migrated plaintext files is best effort (SSD wear levelling and backups may keep copies) +- If the keystore cannot be read (profile copied to another account or PC) nothing is deleted or overwritten: the lists stay empty and the encrypted files untouched +- The pseudonym key is stored sealed under the same master key; a password-protected copy can be exported for another computer +- Restore maps (version 3 envelope): a random data key encrypts the map with AES-256-GCM; it is wrapped by a password-derived key (Argon2id, scrypt if unavailable) and optionally by a recovery key (256-bit, shown once, never stored). Key-derivation parameters read from a file are bounded before use - Graceful cleanup on app close --- +## โš–๏ธ GDPR Notes + +NullifyPDF runs locally; the person or organisation using it is the controller of the data it processes. + +- **Redaction is not a guarantee of anonymisation.** Under Recital 26 GDPR, data that can still be attributed to a person is personal data. A redacted document that keeps dates, places or context may remain personal data. +- **Pseudonymised output is personal data** (Art. 4(5) GDPR; EDPB Guidelines 01/2025 on pseudonymisation). The restore map is the "additional information" that must be kept separately and protected; it is itself personal data. A random or keyed placeholder avoids leaking the order and number of values; the "sequential" style does leak them. +- **Special categories (Art. 9)**: documents with health data need particular care. The health-document profile keeps uncertain hits as suggestions instead of dropping them; in Italy, health data may not be disseminated (art. 2-septies d.lgs. 196/2003). +- **Accuracy and minimisation**: over-redaction removes meaning and under-redaction discloses data. Uncertain detections are shown for review rather than applied or discarded silently. +- The export audit record (`~/.nullifypdf/audit`) supports accountability; it contains counts and hashes only. + +--- + ## ๐Ÿ”‘ Input Validation & Safety ### File Type Verification @@ -279,5 +299,5 @@ For **security vulnerabilities only**: --- -*Last updated: 2026-07-23* +*Last updated: 2026-11-10* *โ† [Contributing](./CONTRIBUTING.md) | [Back to README โ†’](./README.md)* diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..3a92a62 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,57 @@ +# Third-party notices + +## ISTAT place names + +`lexicon_data_it.py` contains data derived from the ISTAT publication +"Elenco dei comuni italiani" (version of 2026-02-21, source: +https://www.istat.it/classificazione/codici-dei-comuni-delle-province-e-delle-regioni/). + +- License: Creative Commons Attribution 4.0 International (CC BY 4.0), + https://creativecommons.org/licenses/by/4.0/ (ISTAT legal notes: + https://www.istat.it/note-legali/). +- Changes: names were normalised and lower-cased, bilingual names were split, + names that are also ordinary words were marked as ambiguous. +- ISTAT does not endorse this project. + +The file is generated by `scripts/build_lexicon_data.py`; the SHA-256 of the +source workbook is recorded in the generated module. + +## Given names and surnames (Wikidata) + +`lexicon_data_it.py` also contains Italian given names and surnames taken from +Wikidata (items that have an article on the Italian Wikipedia). + +- License: Creative Commons Zero (CC0 1.0), all structured data of Wikidata + (https://www.wikidata.org/wiki/Wikidata:Licensing). +- The lists are small (a few hundred given names and about one thousand + surnames) and are only used to *confirm* a name; a name missing from them + never causes a hit to be rejected. + +## Licence of NullifyPDF and of the bundled components + +NullifyPDF is distributed under the GNU Affero General Public License v3.0 or +later (see `LICENSE`). Source code: https://github.com/overwrite00/NullifyPDF. +The information below was checked on 2026-11-11 against the package metadata of +the pinned versions in `requirements.txt` and the publishers' pages. + +| Component | Licence | Notes | +|---|---|---| +| PyMuPDF / MuPDF (Artifex) | AGPL-3.0 or Artifex commercial | NullifyPDF uses it under the AGPL-3.0, hence its own licence. | +| PySide6 / Qt for Python | LGPL-3.0 (or GPL-3.0, commercial) | Used under the LGPL-3.0. Qt libraries are shipped unmodified; the full source and the build scripts of NullifyPDF are public, so anyone can rebuild the program against a different Qt. | +| presidio-analyzer | MIT | | +| spaCy, thinc | MIT | | +| spaCy `it_core_news_md` 3.8.0 | CC BY-NC-SA 3.0 | **Not bundled.** Downloaded by the user on request after showing the licence (non-commercial, attribution, share-alike). Sources: UD Italian ISDT (CC BY-NC-SA 3.0), WikiNER (CC BY 4.0), fastText vectors (CC0). Publisher: Explosion, https://github.com/explosion/spacy-models | +| onnxruntime | MIT | | +| tokenizers | Apache-2.0 | | +| cryptography | Apache-2.0 or BSD-3-Clause | | +| keyring (Linux/macOS) | MIT | | +| tldextract, numpy | BSD-3-Clause | | +| Tesseract OCR and `tessdata_fast` (`eng`, `ita`) | Apache-2.0 | As stated by the Tesseract project. | +| PyInstaller (build tool, not distributed) | GPL-2.0 with the bootloader exception | The exception allows distributing the produced program under any licence. | +| `pymupdf4llm`, `pymupdf-layout` | not declared | Pulled in by PyMuPDF 2.1 but not used and **excluded** from the installer. | + +Local PII models downloaded by the user from Hugging Face are accepted only if +their licence is permissive (MIT, Apache-2.0 or BSD); the licence is part of the +pinned manifest and is shown before the download. + +This table is not legal advice. diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index e666ae3..6102577 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -156,18 +156,9 @@ Names, emails, or other PII should be detected but aren't highlighted. ### Solution -**Check 1 โ€” Language Selection** +**Check 1 โ€” Language and model** -Open sidebar. Is the correct language selected? - -| Language | Best For | -| -------- | -------------------------------- | -| **EN** | English text only | -| **IT** | Italian text only | -| **BOTH** | Mixed English + Italian (slower) | - -> [!WARNING] -> Wrong language = **zero detection**. Test with a clear English name if unsure. +NullifyPDF supports Italian documents only. Names are found by the Italian model, which is downloaded after your consent on the first Auto Redact. If you declined the download, only rules and dictionaries run and names may be missed: run Auto Redact again and accept the download. **Check 2 โ€” Is It Searchable Text?** @@ -184,7 +175,7 @@ If you can't copy โ†’ it's an image โ†’ enable **OCR PDF scansionati** For scanned PDFs: 1. Enable **"OCR PDF scansionati"** in the sidebar 2. Make sure OCR data is present (bundled by default) or you have local Tesseract `tessdata` -3. Run **"Auto Redact (AI)"** again +3. Run **"Auto Redact"** again Use image redaction only when you intentionally want to remove whole images or scanned photo blocks. @@ -199,9 +190,30 @@ If AI still misses something: ### Allowlist Issue? If an entity should be detected but isn't: -1. Check `~/.nullifypdf/allowlist.txt` +1. Open **Dizionari** and look at the *Allowlist* tab 2. If the word is there, it's intentionally ignored -3. Remove it from allowlist to re-enable detection +3. Remove it from the allowlist to re-enable detection + +--- + +## ๐Ÿ” Dictionaries Are Empty on a New Computer + +The blocklist and allowlist are encrypted for the user account that created them. After copying `~/.nullifypdf` to another account or PC the Dizionari dialog shows a warning and empty lists; the encrypted files are not modified. Restore the original profile, or export the lists to text (*Esporta in chiaro...*) on the old computer and import them on the new one. The pseudonym key can be exported and imported with *Gestisci chiave...* in the placeholder style dialog. + +--- + +## ๐Ÿ” Ordinary Words Proposed for Redaction + +### Symptom + +The scan proposes plain words ("Certifico", "Prescrivo", section headings) or fragments of letters of neighbouring lines. + +### Solution + +- Uncertain hits appear in **Rivedi suggerimenti** instead of being applied: leave them unticked to discard them. +- Add words you never want redacted to the allowlist (**Dizionari**); matching is by whole word. +- If black boxes cover parts of the lines above or below the redacted word, update to a build that includes rectangle trimming and re-run the scan. With very tight line spacing (roughly half the font size or less) no safe band exists: a warning "interlinea molto stretta" is shown and neighbouring words can be damaged, never left unredacted. +- For medical documents tick **Profilo documenti sanitari**, which keeps uncertain hits as suggestions instead of dropping them. --- @@ -214,7 +226,7 @@ AI redacted text that shouldn't be redacted (false positive). **Option 0 โ€” Scan only what you need** -Clicking **Auto Redact (AI)** opens a dialog where you tick the data types to look for (for example untick *Luoghi e indirizzi* if cities keep being flagged). Your choice is remembered. Labels and headings such as "Telefono:" or list items are filtered automatically, and other occurrences of a value are matched as whole words only. +Clicking **Auto Redact** opens a dialog where you tick the data types to look for (for example untick *Luoghi e indirizzi* if cities keep being flagged). Your choice is remembered. Labels and headings such as "Telefono:" or list items are filtered automatically, and other occurrences of a value are matched as whole words only. **Option 1 โ€” Remove and Allowlist** @@ -232,10 +244,9 @@ Clicking **Auto Redact (AI)** opens a dialog where you tick the data types to lo For repeated false positives: -1. Open `~/.nullifypdf/allowlist.txt` -2. Add the word (one per line, lowercase) -3. Restart NullifyPDF -4. Run AI scan again +1. Open **Dizionari** and add the word in the *Allowlist* tab +2. Save +3. Run the scan again Example: ``` @@ -465,5 +476,5 @@ The more details, the faster we can fix it! ๐Ÿš€ --- -*Last updated: 2026-09-19* +*Last updated: 2026-11-11* *โ† [User Guide](./USER_GUIDE.md) | [Development โ†’](./DEVELOPMENT.md)* diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 44cac07..53a1ca0 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -11,7 +11,7 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel | ๐ŸŽฏ Step | โฑ๏ธ Time | What You'll Do | | ----------------- | ------- | ----------------------------- | -| **1๏ธโƒฃ Load** | 30 sec | Open a PDF file | +| **1๏ธโƒฃ Load** | 30 sec | Open a PDF or Word DOCX file | | **2๏ธโƒฃ Configure** | 30 sec | Select document language | | **3๏ธโƒฃ Auto-Redact** | 1-2 min | Run AI to find sensitive data | | **4๏ธโƒฃ Review** | 1-2 min | Manually fix any mistakes | @@ -20,35 +20,38 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel --- -## Step 1๏ธโƒฃ โ€” Load Your PDF +> [!TIP] +> The program has its own guide in Italian: press **F1** or the **Guida** button in the sidebar. Resting the pointer on any button, option or field shows a short description. + +## Step 1๏ธโƒฃ โ€” Load Your PDF or Word Document 1. Click the blue **"Apri PDF"** button in the top left -2. Select a PDF file from your computer +2. Select a PDF or modern Word **DOCX** file from your computer 3. The document appears in the center. Scroll with mouse or use arrow buttons (top right) > [!NOTE] -> Supported format: unencrypted PDF. Digital text is scanned directly; scanned pages require OCR. If NullifyPDF detects a page with no extractable text, a popup confirms it and reminds you to enable **OCR PDF scansionati** before AI scanning โ€” manual selection (Step 4) works on scanned pages either way. +> Supported formats: unencrypted PDF and DOCX. A DOCX is read locally and converted to an internal PDF representation; the original Word file is never modified and macros, external links and embedded objects are not executed. Legacy binary `.doc` files must be saved as DOCX first. Digital PDF text is scanned directly; scanned pages require OCR. ---- +### Optional local models -## Step 2๏ธโƒฃ โ€” Configure AI Language +Open **Modelli locali** in the sidebar to view the reviewed shortlist. Models are optional and are not included in the installer. Before downloading, the dialog shows the Hugging Face source, license, pinned revision and total size. The progress bar reports the overall percentage across all model files. -**Before scanning**, select your document's language from the left sidebar: +Only entries with a complete application manifest can be downloaded. The manifest fixes the revision, allowed files, expected sizes and SHA-256 values. Experimental entries without those checks remain unavailable. Installed models run locally and add suggestions for review; they never apply a redaction automatically. -| ๐ŸŒ Language | Use When | -| ---------- | --------------------------------- | -| **EN** | Document is entirely in English | -| **IT** | Document is entirely in Italian | -| **BOTH** | Document contains mixed languages | +--- + +## Step 2๏ธโƒฃ โ€” Language + +NullifyPDF analyses **Italian** documents only; there is no language selector. The Italian name-recognition model is not part of the installer (its licence is non-commercial): the first time you run **Auto Redact** the program shows the licence, asks for your consent and downloads it. If you decline, the scan continues with rules and dictionaries only, which finds fewer names. > [!WARNING] -> Wrong language = missed detections. Choose the exact language of your document. +> Documents in other languages are not supported: names and places in them will be missed. --- ## Step 3๏ธโƒฃ โ€” Automatic Redaction (AI Scan) -Click the **"Auto Redact (AI)"** button. A dialog first lets you tick which data types to look for (your choice is remembered). NullifyPDF will then automatically find and flag the selected types; other occurrences of a detected value are matched as whole words only ("Rossi" does not match "Rossini"): +Click the **"Auto Redact"** button. A dialog first lets you tick which data types to look for (your choice is remembered). NullifyPDF will then automatically find and flag the selected types; other occurrences of a detected value are matched as whole words only ("Rossi" does not match "Rossini"): - ๐Ÿง‘ **Names & Surnames** - ๐Ÿ™๏ธ **Cities & Addresses** @@ -69,6 +72,19 @@ Click the **"Auto Redact (AI)"** button. A dialog first lets you tick which data --- +### Review suggestions + +Pending suggestions are also outlined in orange on the page (untick *Mostra suggerimenti* to hide them). Click an outline to censor or discard that suggestion; the outlines exist only on screen and never reach the exported file. + + +Detections that the text does not support strongly enough (for example a lone capitalized word with no context) are not applied: after the scan a **Revisione suggerimenti** dialog lists them with page, type, score and the reasons. Values are masked until you tick *Mostra i valori*. Tick the ones to censor and press *Applica selezionati* (the others are discarded), or *Rivedi piรน tardi* to keep them pending; the sidebar button *Rivedi suggerimenti (N)* reopens the list. Double-click a row to jump to its page. + +### Documents with health data + +Tick **Profilo documenti sanitari** before scanning. It also looks for dates of birth and health identifiers (health card number, prescription number) and never drops an uncertain hit silently: it becomes a suggestion. + +--- + ## Step 4๏ธโƒฃ โ€” Manual Review & Corrections No AI is 100% accurate. You can fix mistakes: @@ -102,7 +118,7 @@ Make small text larger: Remove logos, signatures, or scanned photos: 1. **Enable** the toggle: **"Oscura Immagini"** (sidebar) -2. Click **"Auto Redact (AI)"** again +2. Click **"Auto Redact"** again 3. All images replaced with gray placeholder: `[ IMAGE REMOVED ]` > [!NOTE] @@ -115,14 +131,19 @@ Remove logos, signatures, or scanned photos: When satisfied with redactions: 1. Click **"Esporta PDF"** -2. Choose the export mode: - - **Anonimizzazione irreversibile** removes selected personal data permanently - - **Pseudonimizzazione reversibile** replaces selected personal data with placeholders and creates a separate encrypted restore map -3. Choose filename and location -4. Review the exported PDF before sharing it +2. If suggestions are still unreviewed you are asked whether to review them first +3. Choose the export mode: + - **Rimozione irreversibile** removes selected personal data permanently + - **Pseudonimizzazione reversibile** replaces selected personal data with placeholders and creates a separate encrypted restore map. You choose the placeholder style: *Casuale* (recommended; reveals neither order nor count), *Progressivo* (`PERSON_001`...; reveals how many values exist and their order), *Generico* (also hides the type) or *Coerente (chiave)* (the same value gets the same placeholder in every document made with the same key, which links the documents) +4. Choose filename and location. The suggested name does not contain the source file name +5. For pseudonymization, choose where to save the map (not the PDF's folder, not a synchronised folder) and enter the password twice +6. NullifyPDF re-opens the exported file and verifies it. If a removed value is still readable you are told where and can delete the file +7. Review the exported PDF before sharing it + +Annotations, attachments, external links, bookmarks, scripts, metadata, invisible text and text outside the page are removed on export. "Maschera larghezza testo rimosso" (sidebar) additionally rounds the space left by removed text. > [!CAUTION] -> **Irreversible anonymization cannot be undone from the exported PDF.** Keep a backup of the original PDF. For pseudonymization, store the encrypted restore map separately from the PDF. +> **Removal cannot be undone from the exported PDF, and it is not a guarantee of anonymisation**: dates, places or context left in the document can still identify a person. Keep a backup of the original PDF. A pseudonymized PDF and its restore map remain personal data: store the map separately from the PDF and protect its password. --- @@ -143,6 +164,12 @@ NullifyPDF checks the SHA-256 hash recorded in the restore map against the selec --- +## ๐Ÿงน Local data + +NullifyPDF keeps a rotating log, an export audit record (counts and hashes, no values) and, for very large files, a temporary folder under `~/.nullifypdf`. **Elimina dati locali** removes them (and, if you confirm, the dictionaries). + +--- + ## ๐Ÿ”ง Dictionary Management Your redaction preferences are saved automatically: @@ -159,36 +186,28 @@ Your redaction preferences are saved automatically: ``` .nullifypdf/ -โ”œโ”€โ”€ blocklist.txt โ† Words to ALWAYS redact -โ”œโ”€โ”€ allowlist.txt โ† Words to NEVER redact +โ”œโ”€โ”€ blocklist.v2.enc โ† Words to ALWAYS redact (encrypted) +โ”œโ”€โ”€ allowlist.v2.enc โ† Words to NEVER redact (encrypted) +โ”œโ”€โ”€ keystore.json โ† How the encryption key is protected +โ”œโ”€โ”€ pseudonym.v2.enc โ† Key of the "Coerente" placeholder style (encrypted) +โ”œโ”€โ”€ update_check.json โ† "Check at every start" preference (Info dialog) +โ”œโ”€โ”€ audit/ โ† Export records (counts and hashes, no values) โ”œโ”€โ”€ logs/ โ”‚ โ””โ”€โ”€ nullifypdf.log ``` -### How to Edit Manually - -
-๐Ÿ“ Edit Blocklist/Allowlist Directly - -1. Open file manager, navigate to `~/.nullifypdf/` -2. Edit `blocklist.txt` or `allowlist.txt` with any text editor -3. One word per line, UTF-8 encoding -4. Restart NullifyPDF to load changes +### Editing the dictionaries -**Example blocklist.txt:** -``` -john smith -acme corporation -admin@example.com -``` +Open **Dizionari**. Each list is a table with the term, the date it was added and an optional expiry date (`AAAA-MM-GG`, or *Mai*). Use *Aggiungi* / *Rimuovi*, *Importa da testo...* (one term per line) and *Esporta in chiaro...* (the exported file contains personal data in clear). *Scadenza predefinita* sets the expiry of new terms, including the ones added by clicking or drawing on the page. Expired terms are removed automatically. The label at the top shows how the lists are protected. -
+> [!NOTE] +> The lists are encrypted for the current user account. If you copy `~/.nullifypdf` to another account or PC they cannot be read there (they stay untouched and are shown empty). Older plaintext `blocklist.txt` / `allowlist.txt` files are migrated automatically. ### Blocklist and Allowlist The manual redaction workflow keeps clicked/drawn terms mutually exclusive. -When editing dictionary files or the dictionary dialog directly, review both -lists and avoid adding the same term to both. +When editing the dictionary dialog, review both lists and avoid adding the +same term to both. --- @@ -270,7 +289,13 @@ python3.13 NullifyPDF.py **A:** Irreversible anonymization cannot be undone from the exported PDF. Pseudonymization can be restored using **"Ricostruisci PDF"** together with the encrypted restore map and its password (see [Step 7](#step-7%EF%B8%8F%E2%83%A3--reconstruct-a-pseudonymized-pdf) below). ### Q: Does NullifyPDF send data to the cloud? -**A:** **No.** PDF processing is local. OCR data may be downloaded during build time, not while processing your PDFs. +**A:** **No.** PDF processing is local and no document data ever leaves your machine. The only network access is the optional update check (see below), and OCR data is downloaded at build time, not while processing your PDFs. + +### Q: How do I check for new versions? +**A:** Open **Info** and click **Controlla aggiornamenti**. NullifyPDF asks the public GitHub releases of this repository whether a newer version exists (no PDF content or personal data is sent). If one does, a window shows the version, the link to the release, the **release notes** (taken from the matching section of `CHANGELOG.md`) and a **Download** button that saves the installer for your operating system into your system **Downloads** folder (the location configured in your OS), after checking its size and the SHA-256 published by GitHub. Nothing is installed automatically. On Linux distributions other than Debian/Ubuntu and Fedora/RHEL-based ones the window only offers the release link, where you can pick the file yourself. + +- **Stable** builds are only offered newer *stable* releases; **beta** builds are offered both stable and beta releases. +- Tick **Controlla ad ogni avvio** in the Info dialog to run the same check silently each time the app starts. It is **off by default**; when enabled, a window appears only if a newer version exists, and a failed check (for example offline) is silently logged. Untick it to go back to fully manual checks. --- @@ -283,5 +308,5 @@ python3.13 NullifyPDF.py --- -*Last updated: 2026-09-19* +*Last updated: 2026-11-11* *โ† [Back to README](./README.md) | [Troubleshooting โ†’](./TROUBLESHOOTING.md)* diff --git a/build_local.py b/build_local.py index f565640..9098ec0 100644 --- a/build_local.py +++ b/build_local.py @@ -12,6 +12,47 @@ OCR_TESSDATA_FILES = ("eng.traineddata", "ita.traineddata") +# Runtime libraries the PySide6 (Qt xcb/EGL) GUI needs on a Linux desktop. The +# PyInstaller binary bundles Python and Qt but not these system libraries. +# Names are checked on Ubuntu 24.04/26.04, Debian 13, RHEL/Rocky 9-10 and +# Fedora by the "Smoke test packages" workflow. +DEB_DEPENDS = ( + "libc6 (>= 2.34), libegl1, libgl1, libfontconfig1, libxkbcommon0, " + "libxkbcommon-x11-0, libxcb-cursor0, libxcb-icccm4, libxcb-image0, " + "libxcb-keysyms1, libxcb-render-util0, libxcb-shape0, libxcb-xkb1, " + "libxcb-xinerama0, libxcb1, libx11-6, libx11-xcb1, libxcb-randr0, " + "libxcb-render0, libxcb-shm0, libxcb-sync1, libxcb-xfixes0, " + "libdbus-1-3, libglib2.0-0t64 | libglib2.0-0" +) +RPM_REQUIRES = ( + "xcb-util-cursor", + "xcb-util-image", + "xcb-util-keysyms", + "xcb-util-renderutil", + "xcb-util-wm", + "libxkbcommon", + "libxkbcommon-x11", + "mesa-libEGL", + "mesa-libGL", + "fontconfig", + "dbus-libs", + "glib2", +) + + +class PackagingError(OSError): + """A Linux package could not be produced (fatal for the build).""" + + +def deb_version(version: str, prerelease: str) -> str: + """Debian version; a prerelease sorts BEFORE the final one (``~``).""" + return f"{version}~{prerelease}" if prerelease else version + + +def rpm_release(prerelease: str) -> str: + """RPM Release; a prerelease sorts before the final ``1``.""" + return f"0.{prerelease.replace('-', '.')}.1" if prerelease else "1" + def get_version_info() -> Tuple[str, str]: """Extract base version and optional prerelease label from NullifyPDF.py. @@ -123,16 +164,24 @@ def pyinstaller_datas(download_missing_ocr: bool = True) -> List[Tuple[str, str] def build_rpm( - version: str, file_version: str, executable_name: str + version: str, file_version: str, executable_name: str, prerelease: str = "" ) -> None: """Build RPM package for Fedora/RHEL. + Built inside a RHEL 9 container (rpm 4.16, glibc 2.34) by CI, so the + package installs on RHEL/Rocky/Alma 9-10 and current Fedora. + Args: version: Application version used for the RPM package metadata (must not contain hyphens, which the RPM Version tag disallows). file_version: Version string used for the output artifact filename (may include a beta suffix, e.g. "2.1.0-beta.2"). executable_name: Name of compiled executable. + prerelease: Prerelease label (e.g. "beta.1"), or empty for stable. + + Raises: + PackagingError: if rpmbuild fails. + FileNotFoundError: if no .rpm was produced. """ print("\n[INFO] Creazione pacchetto RPM per Fedora/RHEL...") rpm_dir = os.path.abspath("rpm_build_tmp") @@ -141,19 +190,30 @@ def build_rpm( icon_source = os.path.abspath("images/NullifyPDF_icon.png") spec_path = os.path.join(rpm_dir, "SPECS", "nullify.spec") + requires = "\n".join(f"Requires: {r}" for r in RPM_REQUIRES) + # No debuginfo/strip/build-id processing: stripping would corrupt the + # archive appended to the PyInstaller executable. Requires are explicit + # (AutoReqProv off) because the bundled libraries must not be scanned. with open(spec_path, "w", encoding="utf-8") as f: f.write( f""" +%global debug_package %{{nil}} +%global __os_install_post %{{nil}} +%global _build_id_links none +AutoReqProv: no + Name: nullify-pdf Version: {version} -Release: 1 -Summary: AI-Powered PDF Anonymization Tool -License: MIT +Release: {rpm_release(prerelease)} +Summary: PDF Anonymization Tool +License: AGPL-3.0-or-later +URL: https://github.com/overwrite00/NullifyPDF BuildArch: x86_64 +{requires} %description -Professional forensic tool for PDF anonymization using AI. +Professional forensic tool for PDF anonymization. %install mkdir -p %{{buildroot}}/usr/bin @@ -183,34 +243,37 @@ def build_rpm( /usr/bin/gtk-update-icon-cache %{{_datadir}}/icons/hicolor &> /dev/null || : %files -/usr/bin/nullify-pdf +%attr(0755,root,root) /usr/bin/nullify-pdf /usr/share/applications/nullify-pdf.desktop /usr/share/icons/hicolor/256x256/apps/nullify-pdf.png """ ) try: - subprocess.run( - ["rpmbuild", "--define", f"_topdir {rpm_dir}", "-bb", spec_path], - check=True, - stdout=subprocess.DEVNULL, - ) - for root, _, files in os.walk(os.path.join(rpm_dir, "RPMS")): - for file in files: - if file.endswith(".rpm"): - shutil.move( - os.path.join(root, file), - f"dist/NullifyPDF_v{file_version}_Fedora.rpm", - ) + try: + subprocess.run( + ["rpmbuild", "--define", f"_topdir {rpm_dir}", "-bb", spec_path], + check=True, + stdout=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError as e: + raise PackagingError(f"rpmbuild failed (exit {e.returncode})") from e + built = [ + os.path.join(root, file) + for root, _, files in os.walk(os.path.join(rpm_dir, "RPMS")) + for file in files + if file.endswith(".rpm") + ] + if not built: + raise FileNotFoundError("rpmbuild produced no .rpm") + shutil.move(built[0], f"dist/NullifyPDF_v{file_version}_Fedora.rpm") print("[OK] RPM creato con successo.") - except Exception as e: - print(f"[ERROR] Errore RPM: {e}") finally: shutil.rmtree(rpm_dir, ignore_errors=True) def build_deb( - version: str, file_version: str, executable_name: str + version: str, file_version: str, executable_name: str, prerelease: str = "" ) -> None: """Build DEB package for Ubuntu/Debian. @@ -219,6 +282,10 @@ def build_deb( file_version: Version string used for the output artifact filename (may include a beta suffix, e.g. "2.1.0-beta.2"). executable_name: Name of compiled executable. + prerelease: Prerelease label (e.g. "beta.1"), or empty for stable. + + Raises: + PackagingError: if dpkg-deb fails. """ print("\n[INFO] Creazione pacchetto DEB per Ubuntu/Debian...") pkg_dir = "deb_build_tmp" @@ -248,7 +315,7 @@ def build_deb( with open(f"{pkg_dir}/DEBIAN/control", "w", encoding="utf-8") as f: f.write( - f"Package: nullify-pdf\nVersion: {version}\nSection: utils\nPriority: optional\nArchitecture: amd64\nMaintainer: Graziano\nDescription: AI PDF Redaction Tool\n" + f"Package: nullify-pdf\nVersion: {deb_version(version, prerelease)}\nSection: utils\nPriority: optional\nArchitecture: amd64\nMaintainer: Graziano Mariella \nHomepage: https://github.com/overwrite00/NullifyPDF\nDepends: {DEB_DEPENDS}\nDescription: PDF Redaction Tool\n Offline PDF anonymization: detect and redact personal data.\n" ) postinst_content = "#!/bin/sh\nset -e\nupdate-desktop-database -q || true\ngtk-update-icon-cache -f -t /usr/share/icons/hicolor || true\n" @@ -260,19 +327,21 @@ def build_deb( os.chmod(f"{pkg_dir}/DEBIAN/postrm", 0o755) try: - subprocess.run( - [ - "dpkg-deb", - "--build", - pkg_dir, - f"dist/NullifyPDF_v{file_version}_Ubuntu.deb", - ], - check=True, - stdout=subprocess.DEVNULL, - ) + try: + subprocess.run( + [ + "dpkg-deb", + "--root-owner-group", + "--build", + pkg_dir, + f"dist/NullifyPDF_v{file_version}_Ubuntu.deb", + ], + check=True, + stdout=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError as e: + raise PackagingError(f"dpkg-deb failed (exit {e.returncode})") from e print("[OK] DEB creato con successo.") - except Exception as e: - print(f"[ERROR] Errore DEB: {e}") finally: shutil.rmtree(pkg_dir, ignore_errors=True) @@ -290,6 +359,21 @@ def build_app() -> None: beta_suffix = os.environ.get("NULLIFYPDF_BETA_SUFFIX", "").strip() file_version = get_file_version(version, code_prerelease, beta_suffix) sys_os = platform.system() + prerelease = (beta_suffix or code_prerelease).strip() + + if "--deb-only" in sys.argv: + # Re-package an already built Linux binary (CI builds it once inside + # a RHEL 9 container, where dpkg-deb is not available). + binary = f"NullifyPDF_v{file_version}_Linux_Portable" + if not os.path.exists(f"dist/{binary}"): + print(f"[ERROR] dist/{binary} not found") + sys.exit(1) + try: + build_deb(version, file_version, binary, prerelease) + except (subprocess.CalledProcessError, OSError) as e: + print(f"[ERROR] DEB build failed: {e}") + sys.exit(1) + return for item in ["build", "dist", "NullifyPDF.spec"]: if os.path.exists(item): @@ -309,15 +393,16 @@ def build_app() -> None: if sys_os == "Darwin": spec_content = f"""# -*- mode: python ; coding: utf-8 -*- +import sys from PyInstaller.utils.hooks import collect_all datas = {datas_literal} binaries = [] -hiddenimports = ['spacy', 'presidio_analyzer'] -for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: +hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers'] +for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []): t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] -a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports) +a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports, excludes=['it_core_news_md', 'pymupdf4llm', 'pymupdf_layout', 'pymupdf.layout']) pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, [], exclude_binaries=True, name='NullifyPDF', debug=False, console=False, icon={icon_str}) coll = COLLECT(exe, a.binaries, a.datas, strip=False, upx=True, upx_exclude=[], name='NullifyPDF') @@ -325,15 +410,16 @@ def build_app() -> None: """ else: spec_content = f"""# -*- mode: python ; coding: utf-8 -*- +import sys from PyInstaller.utils.hooks import collect_all datas = {datas_literal} binaries = [] -hiddenimports = ['spacy', 'presidio_analyzer'] -for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: +hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers'] +for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []): t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] -a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports) +a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports, excludes=['it_core_news_md', 'pymupdf4llm', 'pymupdf_layout', 'pymupdf.layout']) pyz = PYZ(a.pure) exe = EXE(pyz, a.scripts, a.binaries, a.datas, name='NullifyPDF', debug=False, console=False, icon={icon_str}) """ @@ -363,12 +449,17 @@ def build_app() -> None: else: # Linux os.rename("dist/NullifyPDF", f"dist/{final_name}") print(f"[OK] Eseguibile portatile pronto: dist/{final_name}") + # Packaging failures are fatal: a release must never silently + # ship without one of its Linux packages. if shutil.which("rpmbuild"): - build_rpm(version, file_version, final_name) + build_rpm(version, file_version, final_name, prerelease) if shutil.which("dpkg-deb"): - build_deb(version, file_version, final_name) + build_deb(version, file_version, final_name, prerelease) - except subprocess.CalledProcessError as e: + except (OSError, subprocess.CalledProcessError) as e: + if not isinstance(e, subprocess.CalledProcessError): + print(f"\n[ERROR] Packaging failed: {e}") + sys.exit(1) print(f"\n[ERROR] ERRORE CRITICO: Compilazione fallita (exit {e.returncode}).") sys.exit(1) diff --git a/dictionary_dialog.py b/dictionary_dialog.py new file mode 100644 index 0000000..ec26eff --- /dev/null +++ b/dictionary_dialog.py @@ -0,0 +1,202 @@ +"""Dictionary (blocklist / allowlist) editor with per-entry expiry (Qt).""" + +from __future__ import annotations + +import datetime +from typing import Any, Dict, List, Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QCheckBox, + QComboBox, + QDialog, + QFileDialog, + QHBoxLayout, + QInputDialog, + QLabel, + QMessageBox, + QPushButton, + QTableWidget, + QTableWidgetItem, + QTabWidget, + QVBoxLayout, + QWidget, +) + +from help_guide import apply_tooltips +from word_lists import ListEntry, PDFListManager, MIN_TERM_LENGTH + +TTL_CHOICES = [("Mai", None), ("30 giorni", 30), ("90 giorni", 90), + ("180 giorni", 180), ("1 anno", 365)] +COL_TERM, COL_ADDED, COL_EXPIRES = range(3) +KINDS = (("blocklist", "๐Ÿ”ด Blocklist"), ("allowlist", "๐ŸŸข Allowlist")) + + +def parse_date(text: str) -> Optional[datetime.date]: + text = text.strip() + if not text or text.lower() in ("mai", "never"): + return None + return datetime.date.fromisoformat(text) + + +class DictionaryDialog(QDialog): + """Edit both lists; `saved_blocklist` / `saved_allowlist` hold the result.""" + + def __init__(self, manager: PDFListManager, parent: Any = None) -> None: + super().__init__(parent) + self.manager = manager + self.setWindowTitle("Dizionari") + self.resize(620, 520) + self.saved_blocklist: Optional[set] = None + self.saved_allowlist: Optional[set] = None + lay = QVBoxLayout(self) + self.status = QLabel(manager.protector_label()) + self.status.setObjectName("dict_status") + lay.addWidget(self.status) + if manager.keystore_error: + warn = QLabel( + "Dizionari cifrati non leggibili su questo account/PC: verranno " + "usati dizionari vuoti finchรฉ non li ripristini." + ) + warn.setWordWrap(True) + warn.setStyleSheet("color: #f59e0b;") + lay.addWidget(warn) + self.tabs = QTabWidget() + self.tabs.setObjectName("dict_tabs") + self.tables: Dict[str, QTableWidget] = {} + for kind, title in KINDS: + tab = QWidget() + tl = QVBoxLayout(tab) + table = QTableWidget(0, 3) + table.setHorizontalHeaderLabels(["Termine", "Aggiunto il", "Scade il (AAAA-MM-GG)"]) + table.horizontalHeader().setStretchLastSection(True) + table.setColumnWidth(COL_TERM, 260) + table.setObjectName("dict_table") + table.verticalHeader().setVisible(False) + self.tables[kind] = table + tl.addWidget(table) + row = QHBoxLayout() + for label, action in ( + ("Aggiungi", "add"), + ("Rimuovi", "remove"), + ("Importa da testoโ€ฆ", "import"), + ("Esporta in chiaroโ€ฆ", "export"), + ): + b = QPushButton(label) + b.setProperty("kind", kind) + b.setProperty("action", action) + b.clicked.connect(self._on_button) # bound method: no self-referencing lambda + row.addWidget(b) + tl.addLayout(row) + self.tabs.addTab(tab, title) + for e in manager.load_entries(kind): + self._append(kind, e) + lay.addWidget(self.tabs) + ttl_row = QHBoxLayout() + ttl_row.addWidget(QLabel("Scadenza predefinita dei nuovi termini:")) + self.ttl = QComboBox() + self.ttl.setObjectName("dict_ttl") + current = manager.default_ttl_days() + for i, (label, days) in enumerate(TTL_CHOICES): + self.ttl.addItem(label, days) + if days == current: + self.ttl.setCurrentIndex(i) + ttl_row.addWidget(self.ttl) + ttl_row.addStretch(1) + lay.addLayout(ttl_row) + btn = QPushButton("Salva") + btn.setObjectName("Primary") + btn.clicked.connect(self.save) + lay.addWidget(btn) + apply_tooltips(self) + + def _on_button(self) -> None: + sender = self.sender() + if sender is None: + return + kind, action = str(sender.property("kind")), str(sender.property("action")) + {"add": self.add_row, "remove": self.remove_selected, + "import": self.import_text, "export": self.export_plain}[action](kind) + + # -- rows --------------------------------------------------------- + def _append(self, kind: str, entry: ListEntry) -> None: + t = self.tables[kind] + r = t.rowCount() + t.insertRow(r) + t.setItem(r, COL_TERM, QTableWidgetItem(entry.term)) + added = QTableWidgetItem(entry.added.isoformat()) + added.setFlags(added.flags() & ~Qt.ItemFlag.ItemIsEditable) + t.setItem(r, COL_ADDED, added) + t.setItem(r, COL_EXPIRES, QTableWidgetItem(entry.expires.isoformat() if entry.expires else "Mai")) + + def add_term(self, kind: str, term: str) -> None: + days = self.ttl.currentData() + today = datetime.date.today() + exp = today + datetime.timedelta(days=days) if days else None + self._append(kind, ListEntry(term.strip().lower(), today, exp)) + + def add_row(self, kind: str) -> None: + text, ok = QInputDialog.getText(self, "Aggiungi termine", "Termine:") + if ok and len(text.strip()) >= MIN_TERM_LENGTH: + self.add_term(kind, text) + + def remove_selected(self, kind: str) -> None: + t = self.tables[kind] + for r in sorted({i.row() for i in t.selectedIndexes()}, reverse=True): + t.removeRow(r) + + def import_text(self, kind: str) -> None: + text, ok = QInputDialog.getMultiLineText( + self, "Importa da testo", "Un termine per riga:" + ) + if ok: + for line in text.splitlines(): + if len(line.strip()) >= MIN_TERM_LENGTH: + self.add_term(kind, line) + + def export_plain(self, kind: str) -> None: + ans = QMessageBox.warning( + self, "Esporta in chiaro", + "Il file esportato contiene dati personali in chiaro. Continuare?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return + path, _ = QFileDialog.getSaveFileName(self, "Esporta", f"{kind}.txt", "Testo (*.txt)") + if path: + with open(path, "w", encoding="utf-8") as fh: + fh.write("\n".join(e.term for e in self.entries(kind))) + + # -- result ------------------------------------------------------- + def entries(self, kind: str) -> List[ListEntry]: + out: List[ListEntry] = [] + t = self.tables[kind] + def cell(r: int, c: int) -> str: + item = t.item(r, c) + return item.text() if item is not None else "" + + for r in range(t.rowCount()): + term = cell(r, COL_TERM).strip().lower() + if len(term) < MIN_TERM_LENGTH: + continue + try: + added = parse_date(cell(r, COL_ADDED)) or datetime.date.today() + expires = parse_date(cell(r, COL_EXPIRES)) + except ValueError: + raise ValueError(f"Data non valida per ยซ{term}ยป: usa AAAA-MM-GG oppure ยซMaiยป.") + out.append(ListEntry(term, added, expires)) + return out + + def save(self) -> None: + try: + block, allow = self.entries("blocklist"), self.entries("allowlist") + except ValueError as e: + QMessageBox.warning(self, "Data non valida", str(e)) + return + self.manager.set_default_ttl_days(self.ttl.currentData()) + self.manager.save_entries("blocklist", block) + self.manager.save_entries("allowlist", allow) + self.saved_blocklist = {e.term for e in block} + self.saved_allowlist = {e.term for e in allow} + self.accept() diff --git a/export_engine.py b/export_engine.py new file mode 100644 index 0000000..115613a --- /dev/null +++ b/export_engine.py @@ -0,0 +1,436 @@ +"""Qt-free export pipeline, local-data hygiene and audit record. + +``export_document`` is the whole export: it works on a copy of the document +(kept in memory, or in a private per-user folder for very large files), +applies the redactions, sanitises the result, writes it and verifies it. The +GUI only supplies callbacks and user choices. +""" + +from __future__ import annotations + +import datetime +import hashlib +import json +import os +import pathlib +import secrets +import shutil +import stat +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Iterable, List, Optional + +import pymupdf + +from pii_detection import GLYPH_CORE_INSET +from pdf_sanitize import ( + PDF_ANNOT_REDACT, + SanitizePolicy, + SanitizeReport, + VerificationReport, + sanitize_document, + verify_export, +) + +# Documents larger than this are copied through a private temp file instead +# of RAM. +IN_MEMORY_LIMIT_BYTES = 96 * 1024 * 1024 +PDF_REDACT_IMAGE_PIXELS: int = pymupdf.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] + + +@dataclass +class ExportResult: + output_sha256: str = "" + sanitize: SanitizeReport = field(default_factory=SanitizeReport) + verification: VerificationReport = field(default_factory=VerificationReport) + pages: int = 0 + surviving_glyphs: int = 0 + + +def neutral_output_name(mode_value: str, now: Optional[datetime.datetime] = None) -> str: + """Default file name that contains nothing from the source document.""" + now = now or datetime.datetime.now() + return f"document_{mode_value}_{now:%Y%m%d_%H%M%S}.pdf" + + +def neutral_map_name(now: Optional[datetime.datetime] = None) -> str: + now = now or datetime.datetime.now() + return f"restore_map_{now:%Y%m%d_%H%M%S}.nullifypdf-map" + + +def name_leaks_value(path: str, values: Iterable[str]) -> bool: + """True when the file name contains one of the values being removed.""" + base = "".join(os.path.basename(path).casefold().split()) + for v in values: + n = "".join(v.casefold().split()) + if len(n) >= 4 and n in base: + return True + return False + + +def write_private_file(path: str, data: bytes) -> None: + """Write `data` readable by the current user only (where supported).""" + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) + fd = os.open(path, flags, 0o600) + with os.fdopen(fd, "wb") as fh: + fh.write(data) + try: + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + +def private_temp_dir(config_dir: pathlib.Path) -> pathlib.Path: + d = pathlib.Path(config_dir) / "tmp" + d.mkdir(parents=True, exist_ok=True) + try: + os.chmod(d, 0o700) + except OSError: + pass + return d + + +def cleanup_stale_temp(config_dir: pathlib.Path, max_age_hours: float = 0.0) -> int: + """Remove leftovers of a crashed export (called at startup).""" + d = pathlib.Path(config_dir) / "tmp" + removed = 0 + if not d.is_dir(): + return 0 + now = datetime.datetime.now().timestamp() + for f in d.iterdir(): + try: + if now - f.stat().st_mtime >= max_age_hours * 3600: + f.unlink() + removed += 1 + except OSError: + pass + return removed + + +def collect_removed_values( + doc: Any, read_payload: Optional[Callable[[Any], Dict[str, str]]] = None +) -> List[str]: + """Every value a pending redaction will remove (for the output check).""" + values: List[str] = [] + for page in doc: + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + payload = read_payload(annot) if read_payload else {} + original = (payload or {}).get("original", "") + if original: + values.append(original) + try: + clipped = page.get_text("text", clip=annot.rect).strip() + except Exception: + clipped = "" + if clipped: + values.append(" ".join(clipped.split())) + seen: List[str] = [] + for v in values: + if v and v not in seen: + seen.append(v) + return seen + + +def text_outside_redactions(doc: Any) -> str: + """Source text that no redaction covers (it stays in the output).""" + chunks: List[str] = [] + for page in doc: + rects = [ + a.rect for a in (page.annots() or []) if a.type[0] == PDF_ANNOT_REDACT + ] + for w in page.get_text("words"): + # A word is deleted when a redaction reaches its glyph core (the + # box inset by GLYPH_CORE_INSET), not only when its centre is + # covered; a partly covered word is not "kept". + dx, dy = (w[2] - w[0]) * GLYPH_CORE_INSET, (w[3] - w[1]) * GLYPH_CORE_INSET + core = pymupdf.Rect(w[0] + dx, w[1] + dy, w[2] - dx, w[3] - dy) + if not any(core.intersects(r) for r in rects): + chunks.append(w[4]) + return " ".join(chunks) + + +def _line_chars(page: Any, baseline: float, tol: float = 1.5) -> List[Dict[str, Any]]: + """Non-space characters of `page` on one baseline, left to right.""" + try: + raw = page.get_text("rawdict") + except Exception: + return [] + out: List[Dict[str, Any]] = [] + for block in raw.get("blocks", []): + if block.get("type") != 0: + continue + for line in block.get("lines", []): + for span in line.get("spans", []): + for ch in span.get("chars", []): + if str(ch.get("c", "")).strip() and abs(float(ch["origin"][1]) - baseline) <= tol: + out.append(ch) + out.sort(key=lambda c: float(c["origin"][0])) + return out + + +def _expected_deletions( + page: Any, read_full: Callable[[Any], Optional[Any]] +) -> List[Dict[str, Any]]: + """Per redaction with a stored whole-word box: what must disappear. + + After apply_redactions() every glyph of the word must be gone: a thin + band that misses one glyph (a raised or smaller character) would leave + part of the value behind, and a text search cannot see a single + character. Two independent checks, neither of which relies on glyph + positions (MuPDF rewrites and shifts the rest of the line): + + * predicted survivors: target glyphs whose deletion core the band does + not reach (the measured rule); + * rule check: on that line, the number of glyphs that actually vanish + must be at least the number the rule predicts. If a PyMuPDF release + changes the rule, this trips instead of leaving the export silently + unsafe. + """ + from pii_detection import core_hits, target_chars + + bands = [ + pymupdf.Rect(a.rect) for a in page.annots() or [] + if a.type[0] == PDF_ANNOT_REDACT + ] + expected: List[Dict[str, Any]] = [] + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + full = read_full(annot) + if full is None: + continue + targets = target_chars(page, pymupdf.Rect(full)) + if not targets: + continue + baseline = float(targets[0]["origin"][1]) + line = _line_chars(page, baseline) + predicted = sum( + 1 for c in line if any(core_hits(tuple(c["bbox"]), b) for b in bands) + ) + missed = sum( + 1 for t in targets + if not any(core_hits(tuple(t["bbox"]), b) for b in bands) + ) + expected.append({ + "baseline": baseline, "n_pre": len(line), + "predicted_deleted": predicted, "missed": missed, + }) + return expected + + +def _count_survivors(page: Any, expected: List[Dict[str, Any]]) -> int: + survivors = 0 + seen_lines = set() + for item in expected: + survivors += int(item["missed"]) + key = round(item["baseline"], 1) + if key in seen_lines: + continue # several words on one line share the same count + seen_lines.add(key) + actual = item["n_pre"] - len(_line_chars(page, item["baseline"])) + if actual < item["predicted_deleted"]: + survivors += item["predicted_deleted"] - actual + return survivors + + +def _open_copy( + src_doc: Any, temp_dir: Optional[pathlib.Path] +) -> tuple[Any, Optional[str]]: + """Open a working copy of `src_doc` without leaving PII beside the output.""" + size = 0 + try: + size = os.path.getsize(src_doc.name) if src_doc.name else 0 + except OSError: + size = 0 + if size <= IN_MEMORY_LIMIT_BYTES or temp_dir is None: + return pymupdf.open("pdf", src_doc.tobytes(garbage=0, deflate=False)), None + tmp = str(pathlib.Path(temp_dir) / f"{secrets.token_hex(8)}.tmp") + data = src_doc.tobytes(garbage=0, deflate=False) + write_private_file(tmp, data) + return pymupdf.open(tmp), tmp + + +def sha256_of(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def export_document( + src_doc: Any, + out_path: str, + *, + prepare_page: Optional[Callable[[Any], None]] = None, + finish_page: Optional[Callable[[Any], None]] = None, + read_full: Optional[Callable[[Any], Optional[Any]]] = None, + policy: Optional[SanitizePolicy] = None, + removed_values: Iterable[str] = (), + temp_dir: Optional[pathlib.Path] = None, + log: Optional[Callable[[str], None]] = None, +) -> ExportResult: + """Apply redactions to a copy of `src_doc`, sanitise, save and verify.""" + result = ExportResult() + kept_text = text_outside_redactions(src_doc) + ex_doc, tmp = _open_copy(src_doc, temp_dir) + try: + survivors_to_check: Dict[int, List[Dict[str, Any]]] = {} + for page in ex_doc.pages(): + if read_full is not None: + survivors_to_check[page.number] = _expected_deletions(page, read_full) + if prepare_page is not None: + prepare_page(page) + redact_rects = [ + a.rect for a in (page.annots() or []) + if a.type[0] == PDF_ANNOT_REDACT + ] + try: + for lnk in page.get_links(): + if any(pymupdf.Rect(lnk["from"]).intersects(r) for r in redact_rects): + page.delete_link(lnk) + except (RuntimeError, AttributeError, KeyError) as e: + if log: + log(f"link removal skipped: {e}") + # PDF_REDACT_IMAGE_PIXELS blanks only the pixels under each rect; + # IMAGE_REMOVE would delete a whole scanned page for one word. + page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) + result.surviving_glyphs += _count_survivors( + page, survivors_to_check.get(page.number, []) + ) + if finish_page is not None: + # Text drawn after the redactions are applied: placeholders + # must not go through apply_redactions(), which silently + # drops text it cannot lay out at 4 pt or more. + finish_page(page) + try: + for w in list(page.widgets() or []): + page.delete_widget(w) + except (RuntimeError, AttributeError) as e: + if log: + log(f"widget removal skipped: {e}") + result.pages += 1 + result.sanitize = sanitize_document(ex_doc, policy) + ex_doc.save(out_path, garbage=4, deflate=True, clean=True) + finally: + try: + ex_doc.close() + except Exception: + pass + if tmp: + try: + os.remove(tmp) + except OSError: + pass + result.output_sha256 = sha256_of(out_path) + result.verification = verify_export(out_path, removed_values, kept_text) + if result.surviving_glyphs: + result.verification.add( + f"{result.surviving_glyphs} glyph(s) survived redaction" + ) + return result + + +# --------------------------------------------------------------------------- +# Audit record (no values) and local data hygiene +# --------------------------------------------------------------------------- + +def build_audit_record( + *, + app_version: str, + mode: str, + entity_counts: Dict[str, int], + suggestion_counts: Dict[str, int], + sanitize: SanitizeReport, + verification: VerificationReport, + output_sha256: str, + pages: int, +) -> Dict[str, Any]: + """Accountability record of one export. Holds no personal data.""" + return { + "format": "NullifyPDF export audit", + "timestamp": datetime.datetime.now().isoformat(timespec="seconds"), + "app_version": app_version, + "mode": mode, + "pages": pages, + "entity_counts": dict(entity_counts), + "suggestions": dict(suggestion_counts), + "sanitizer_actions": dict(sanitize.actions), + "verification": { + "ok": verification.ok, + "findings": list(verification.findings), + }, + "output_sha256": output_sha256, + } + + +def write_audit_record(config_dir: pathlib.Path, record: Dict[str, Any]) -> str: + d = pathlib.Path(config_dir) / "audit" + d.mkdir(parents=True, exist_ok=True) + name = f"export_{datetime.datetime.now():%Y%m%d_%H%M%S}_{secrets.token_hex(2)}.json" + path = str(d / name) + write_private_file(path, json.dumps(record, indent=2, sort_keys=True).encode("utf-8")) + return path + + +def wipe_local_data( + config_dir: pathlib.Path, include_lists: bool = False, include_keys: bool = False +) -> List[str]: + """Delete logs, audit records, temp files and, optionally, the word lists + and the keys (keystore, pseudonym key). Without the keys the encrypted + lists and the stored pseudonym key become unreadable.""" + root = pathlib.Path(config_dir) + removed: List[str] = [] + for sub in ("logs", "audit", "tmp"): + d = root / sub + if d.is_dir(): + shutil.rmtree(d, ignore_errors=True) + if not d.exists(): # report only what is really gone + removed.append(sub) + names = ["ai_entities.json"] + if include_keys: + names += ["keystore.json", "pseudonym.v2.enc", "pseudonym.key"] + keys = root / "keys" + if keys.is_dir(): + shutil.rmtree(keys, ignore_errors=True) + if not keys.exists(): + removed.append("keys") + try: + import secure_store + + secure_store.forget_cached_keys() + except Exception: + pass + if include_lists: + names += ["blocklist.txt", "allowlist.txt", "blocklist.v2.enc", "allowlist.v2.enc"] + for n in names: + f = root / n + if f.is_file(): + try: + f.unlink() + removed.append(n) + except OSError: + pass + return removed + + +def purge_old_logs(log_dir: pathlib.Path, max_age_days: int = 30) -> int: + d = pathlib.Path(log_dir) + if not d.is_dir(): + return 0 + cutoff = datetime.datetime.now().timestamp() - max_age_days * 86400 + n = 0 + for f in d.iterdir(): + try: + if f.is_file() and f.stat().st_mtime < cutoff: + f.unlink() + n += 1 + except OSError: + pass + return n + + +def log_token(path: str) -> str: + """Stable short token for a path, so logs never hold file names.""" + return hashlib.sha256(path.encode("utf-8", "ignore")).hexdigest()[:10] diff --git a/help_guide.py b/help_guide.py new file mode 100644 index 0000000..67cbf3d --- /dev/null +++ b/help_guide.py @@ -0,0 +1,430 @@ +"""In-app user guide (Italian) and tooltips for every GUI element (Qt). + +The guide is opened from the "Guida" button or with F1. Tooltips are applied +by :func:`apply_tooltips`, which looks widgets up by their visible text or +object name; a test checks that no widget of the main window or of the +dialogs is left without one. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Tuple + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QAbstractButton, + QComboBox, + QDialog, + QDoubleSpinBox, + QGraphicsView, + QHBoxLayout, + QLabel, + QLineEdit, + QListWidget, + QProgressBar, + QPushButton, + QTableWidget, + QTabWidget, + QTextBrowser, + QTextEdit, + QVBoxLayout, + QWidget, +) + +# -------------------------------------------------------------------------- +# Tooltips +# -------------------------------------------------------------------------- + +# Exact visible text (or object name) -> tooltip. +TOOLTIPS: Dict[str, str] = { + # Sidebar + "Apri PDF": "Apre un file PDF da elaborare. Puoi anche trascinarlo sulla finestra.", + "Lingua dei documenti: italiano": "NullifyPDF analizza documenti in italiano. Il modello italiano viene scaricato, dopo il tuo consenso, al primo utilizzo.", + "Oscura Immagini": "Oscura anche tutte le immagini della pagina (loghi, firme, foto) quando applichi Auto Redact.", + "OCR PDF scansionati": "Legge con l'OCR le pagine scansionate e le immagini incorporate, cosรฌ anche il testo nelle immagini viene analizzato.", + "Mostra suggerimenti": "Contorna in arancione, nell'anteprima, i suggerimenti ancora da rivedere. Servono solo a video: non finiscono nel file esportato.", + "Profilo documenti sanitari": "Cerca anche date di nascita, numero tessera sanitaria, ricetta e codici di esenzione, e propone firme e timbri. I casi incerti diventano suggerimenti invece di essere scartati.", + "Maschera larghezza testo rimosso": "Arrotonda lo spazio lasciato dal testo rimosso, che altrimenti ne rivela la larghezza. Puรฒ spostare di poco il testo seguente.", + "Dizionari": "Modifica le parole da censurare sempre (blocklist) e da non censurare mai (allowlist), con scadenza.", + "Modelli locali": "Mostra i modelli locali candidati, la licenza, la copertura e lo stato di verifica. Il download richiede un manifest con revisione e SHA-256.", + "model_settings": "Gestisce il catalogo curato dei modelli locali. I modelli producono solo suggerimenti e non vengono scaricati senza consenso.", + "model_list": "Elenco curato dei modelli locali e del loro stato di verifica.", + "model_details": "Dettagli del modello selezionato, inclusi licenza, runtime, lingue e copertura dichiarata.", + "model_download": "Scarica un modello solo quando l'applicazione ne ha verificato manifest, revisione e SHA-256.", + "Auto Redact": "Analizza il documento e segna i dati personali da censurare. Nulla viene cancellato finchรฉ non esporti.", + "Rivedi suggerimenti": "Apre l'elenco dei rilevamenti incerti per decidere uno per uno se censurarli.", + "Pulisci Pagina": "Rimuove tutte le censure segnate sulla pagina corrente.", + "Esporta PDF": "Crea il PDF finale: rimozione irreversibile oppure pseudonimizzazione reversibile. Il file viene pulito e verificato.", + "Ricostruisci PDF": "Rimette i valori originali in un PDF pseudonimizzato usando la sua mappa di ripristino e la password (o la chiave di recupero).", + "Elimina dati locali": "Cancella log, registri di esportazione e file temporanei salvati nella cartella dell'utente. Chiede conferma per dizionari e chiavi.", + "Guida": "Apre la guida all'uso del programma (scorciatoia: F1).", + "Info": "Versione del programma, controllo degli aggiornamenti e collegamento al repository.", + "Esci": "Chiude il programma.", + "NullifyPDF": "NullifyPDF: censura e pseudonimizzazione di PDF in locale.", + "Privacy Edition": "Tutta l'elaborazione avviene sul tuo computer: nessun documento viene inviato in rete.", + # Viewer toolbar + "Visualizzatore Documento": "Anteprima del PDF. Trascina per segnare un'area da censurare; clicca su una censura per rimuoverla; Ctrl + rotella per lo zoom.", + "-": "Riduce lo zoom.", + "+": "Aumenta lo zoom.", + "<": "Va alla pagina precedente.", + ">": "Va alla pagina successiva.", + "le_page": "Numero della pagina corrente. Scrivine un altro e premi Invio per saltarci.", + "lbl_zoom": "Livello di zoom corrente.", + "lbl_tot": "Numero totale di pagine del documento.", + "view": "Anteprima del PDF. Trascina per segnare un'area da censurare; clicca su una censura o su un contorno arancione per decidere; Ctrl + rotella per lo zoom.", + "prog": "Avanzamento dell'analisi, pagina per pagina.", + "log": "Messaggi del programma: avanzamento, avvisi ed errori. Non contiene i dati personali trovati.", + # Review dialog + "Censura": "Applica la censura a questo elemento.", + "Pag.": "Pagina in cui รจ stato trovato l'elemento.", + "Valore": "Valore trovato, mascherato finchรฉ non spunti ยซMostra i valoriยป. Per le aree grafiche indica la zona.", + "Tipo": "Tipo di dato rilevato (persona, luogo, telefono, firma...).", + "Fonte": "Origine del suggerimento: regole, modello locale o OCR.", + "Punteggio": "Affidabilitร  del rilevamento, da 0 a 1.", + "Motivi": "Indizi che hanno portato a questo punteggio (contesto, parola comune, inizio riga...).", + "Mostra i valori": "Mostra in chiaro i valori trovati nella tabella.", + "Seleziona tutti": "Spunta tutti gli elementi dell'elenco.", + "Nessuno": "Toglie la spunta a tutti gli elementi.", + "Seleziona โ‰ฅ soglia": "Spunta gli elementi con punteggio pari o superiore alla soglia indicata a fianco.", + "review_threshold": "Soglia di punteggio usata da ยซSeleziona โ‰ฅ sogliaยป.", + "Applica selezionati": "Censura gli elementi spuntati e scarta gli altri.", + "Rivedi piรน tardi": "Chiude l'elenco lasciando i suggerimenti in sospeso.", + "review_table": "Rilevamenti incerti: spunta quelli da censurare. Doppio clic su una riga per andare alla pagina.", + # Dictionary dialog + "dict_status": "Come sono protetti i dizionari su questo computer.", + "Aggiungi": "Aggiunge un termine alla lista.", + "Rimuovi": "Elimina i termini selezionati dalla lista.", + "Importa da testoโ€ฆ": "Incolla piรน termini, uno per riga.", + "Esporta in chiaroโ€ฆ": "Salva la lista in un file di testo non cifrato (contiene dati personali).", + "Scadenza predefinita dei nuovi termini:": "Dopo quanto tempo i nuovi termini scadono e vengono eliminati da soli.", + "dict_ttl": "Scadenza predefinita dei nuovi termini (anche quelli aggiunti dalla pagina).", + "dict_tabs": "Blocklist: parole sempre censurate. Allowlist: parole mai censurate.", + "dict_table": "Termini della lista con data di inserimento e scadenza (AAAA-MM-GG oppure Mai). Doppio clic per modificare.", + "Salva": "Salva le modifiche ai dizionari.", + # Entity dialog + "Seleziona tutto": "Seleziona tutti i tipi di dato.", + "Deseleziona tutto": "Deseleziona tutti i tipi di dato.", + "Seleziona i dati da censurare / pseudonimizzare:": "Scegli quali tipi di dati cercare. La scelta viene ricordata.", + # Guide + "Chiudi": "Chiude questa finestra.", + "OK": "Conferma la scelta e prosegue.", + "Cancel": "Chiude senza confermare.", + "Annulla": "Chiude senza confermare.", + "Controlla aggiornamenti": "Cerca su GitHub se esiste una versione piรน recente del programma.", + "Controlla ad ogni avvio": "Controlla gli aggiornamenti a ogni avvio. Disattivato di default; mostra qualcosa solo se c'รจ una versione nuova.", + "Download": "Scarica nella cartella Download il file di installazione della nuova versione.", +} + +# Visible text of the entity checkboxes -> tooltip. +ENTITY_TOOLTIPS: Dict[str, str] = { + "Nomi e cognomi": "Persone citate nel testo. Le parole comuni e i titoli vengono scartati; i casi incerti vanno in revisione.", + "Luoghi e indirizzi": "Cittร , indirizzi con numero civico e CAP (solo in un contesto di indirizzo).", + "Email": "Indirizzi di posta elettronica.", + "Telefono": "Numeri di telefono italiani e internazionali (date e numeri d'ordine sono esclusi).", + "IBAN": "Coordinate bancarie IBAN valide.", + "Carte di credito": "Numeri di carta di credito con controllo di validitร .", + "Wallet crypto": "Indirizzi di portafogli di criptovalute.", + "Codice fiscale": "Codice fiscale italiano, con controllo del carattere finale.", + "Partita IVA": "Partita IVA italiana, con controllo di validitร  e parola di contesto.", + "Patente": "Numero di patente di guida (richiede la parola ยซpatenteยป vicino).", + "Carta d'identitร ": "Numero di carta d'identitร  (richiede il contesto).", + "Passaporto": "Numero di passaporto (richiede il contesto).", + "Date di nascita": "Date precedute da ยซnato/a ilยป, ยซnascitaยป...", + "Identificativi sanitari": "Numero della tessera sanitaria (20 cifre) e della ricetta elettronica (NRE).", + "Codici di esenzione": "Codici di esenzione dal ticket (richiedono la parola ยซesenzioneยป).", +} + + +def tooltip_for(widget: QWidget) -> str: + """Tooltip text for a widget, from its object name or visible text.""" + name = widget.objectName() + if name and name in TOOLTIPS: + return TOOLTIPS[name] + text = "" + if isinstance(widget, (QAbstractButton, QLabel)): + text = widget.text() if hasattr(widget, "text") else "" + text = re.sub(r"<[^>]+>", "", text).strip() + if text in TOOLTIPS: + return TOOLTIPS[text] + if text in ENTITY_TOOLTIPS: + return ENTITY_TOOLTIPS[text] + if text.startswith("Rivedi suggerimenti"): + return TOOLTIPS["Rivedi suggerimenti"] + if text.startswith("Archivio protetto") or text.startswith("File chiave") or text == "Non cifrato" or text.startswith("Non ancora creato"): + return TOOLTIPS["dict_status"] + if text.endswith("%"): + return TOOLTIPS["lbl_zoom"] + if text.startswith("/ "): + return TOOLTIPS["lbl_tot"] + return _generic_tooltip(widget, text) + + +def _generic_tooltip(widget: QWidget, text: str) -> str: + """Texts that vary at run time (Info and update dialogs).""" + low = text.lower() + if isinstance(widget, QLabel): + if re.match(r"^v\d", low): + return "Versione del programma installata." + if "censura e pseudonimizzazione" in low: + return "Informazioni sul programma, sull'autore e sulla licenza." + if re.search(r"https?://(www\.)?github\.com/", low) or "repository" in low: + return "Pagina del progetto su GitHub." + if low.startswith("nuova versione"): + return "Versione piรน recente disponibile." + if low.startswith(("disponibile", "nessun aggiornamento", "controllo", "scaricato", "download")): + return "Esito dell'operazione in corso." + if low.startswith("questi elementi"): + return "Spiega perchรฉ questi elementi sono suggerimenti e non censure automatiche." + if low.startswith("http"): + return "Collegamento alla pagina della release." + return "" + + +# Widgets that deliberately have no tooltip: it would get in the way while reading. +NO_TOOLTIP = frozenset({"guide_topics", "guide_text"}) + +_INTERACTIVE: Tuple[type, ...] = ( + QAbstractButton, QLabel, QLineEdit, QProgressBar, QTextEdit, QGraphicsView, + QTableWidget, QComboBox, QDoubleSpinBox, QTabWidget, QListWidget, QTextBrowser, +) + + +def apply_tooltips(root: QWidget) -> int: + """Set a tooltip on every widget under `root` that has none. Returns the count.""" + n = 0 + for w in [root, *root.findChildren(QWidget)]: + if not isinstance(w, _INTERACTIVE) or w.toolTip() or w.objectName() in NO_TOOLTIP: + continue + tip = tooltip_for(w) + if tip: + w.setToolTip(tip) + n += 1 + return n + + +def widgets_without_tooltip(root: QWidget) -> List[str]: + """Descriptions of interactive widgets that still have no tooltip.""" + missing: List[str] = [] + for w in root.findChildren(QWidget): + if isinstance(w, _INTERACTIVE) and not w.toolTip() and w.objectName() not in NO_TOOLTIP: + # Icons and the inner widgets of composite controls carry no text. + if isinstance(w, QLabel) and not w.text().strip(): + continue + if w.parent() is not None and type(w.parent()).__name__ in ( + "QAbstractSpinBox", "QTabBar", "QHeaderView", "QScrollBar", + "QDoubleSpinBox", "QComboBox", "QTableWidget", + ): + continue + if type(w).__name__ in ("QLineEdit",) and w.parent() is not None and isinstance( + w.parent(), (QDoubleSpinBox, QComboBox) + ): + continue + missing.append(f"{type(w).__name__}({getattr(w, 'text', lambda: '')()!r}, {w.objectName()!r})") + return missing + + +# -------------------------------------------------------------------------- +# Guide +# -------------------------------------------------------------------------- + +GUIDE_TOPICS: List[Tuple[str, str]] = [ + ("Introduzione", """ +

Introduzione

+

NullifyPDF individua i dati personali in un PDF e li censura in modo irreversibile oppure li sostituisce con segnaposto reversibili (pseudonimizzazione). Tutta l'elaborazione avviene sul tuo computer: nessun documento viene inviato in rete.

+

Flusso consigliato

+
    +
  1. Apri PDF (o trascina il file sulla finestra).
    2. +
  2. Scegli la lingua del documento.
    3. +
  3. Premi Auto Redact e scegli i tipi di dati da cercare.
    4. +
  4. Rivedi i suggerimenti incerti e correggi a mano ciรฒ che serve.
    5. +
  5. Premi Esporta PDF, scegli la modalitร  e controlla il risultato.
    6. +
+

Finchรฉ non esporti, le censure sono solo segni sull'anteprima: il documento originale non viene modificato.

+"""), + ("Aprire un PDF e muoversi", """ +

Aprire un PDF e muoversi

+
    +
  • Apri PDF carica un file; si puรฒ anche trascinarlo sulla finestra. I PDF protetti da password non sono supportati.
    • +
  • < e > cambiano pagina; nel campo numerico puoi scrivere una pagina e premere Invio.
    • +
  • + e - cambiano lo zoom; anche Ctrl + rotella del mouse.
    • +
  • Se il PDF sembra una scansione (senza testo selezionabile) il programma lo segnala: attiva OCR PDF scansionati prima di analizzarlo.
    • +
+"""), + ("Lingua e opzioni", """ +

Lingua e opzioni

+

Lingua: il programma analizza documenti in italiano. Il modello italiano non รจ incluso nell'installer (licenza non commerciale): al primo utilizzo il programma mostra la licenza e chiede il consenso per scaricarlo.

+
    +
  • Oscura Immagini: censura anche tutte le immagini (loghi, foto, firme) quando applichi Auto Redact.
    • +
  • OCR PDF scansionati: legge con l'OCR le pagine scansionate e le immagini incorporate nelle pagine di testo. Richiede i dati di Tesseract (inclusi nei pacchetti).
    • +
  • Mostra suggerimenti: contorna in arancione i suggerimenti da rivedere.
    • +
  • Profilo documenti sanitari: vedi l'argomento dedicato.
    • +
  • Maschera larghezza testo rimosso: arrotonda lo spazio lasciato dal testo cancellato, che altrimenti ne rivela la larghezza. Puรฒ spostare di poco il testo seguente; รจ disattivata di default.
    • +
+"""), + ("Modelli locali", """ +

Modelli locali

+

Modelli locali mostra il catalogo curato dei modelli candidati per migliorare i suggerimenti su nomi, indirizzi e altri dati personali. I modelli, quando abilitati, lavorano solo sul testo estratto e non applicano censure da soli.

+
    +
  • Ogni modello deve avere una licenza, una revisione e un SHA-256 verificati.
    • +
  • Il programma non carica codice remoto, file pickle o modelli arbitrari.
    • +
  • Il download richiede consenso esplicito e i pesi vengono conservati nella cartella dell'utente, non nell'installer.
    • +
  • La qualitร  dei modelli va misurata sul tipo di documento concreto: nessun modello garantisce anonimizzazione o copertura completa.
    • +
+"""), + ("Auto Redact", """ +

Auto Redact

+

Premendo Auto Redact scegli quali tipi di dati cercare (nomi, luoghi, email, telefono, IBAN, carte, codice fiscale, partita IVA, patente, carta d'identitร , passaporto, e con il profilo sanitario date di nascita, tessera sanitaria, ricetta, esenzioni). La scelta viene ricordata.

+

Il programma assegna a ogni nome o luogo un punteggio basato su indizi nel testo: titoli e formule come ยซnato aยป o ยซresidente inยป, forma da nome proprio, nomi e luoghi noti, ripetizioni. Verbi e parole comuni di certificati e atti (ยซCertificoยป, ยซPrescrivoยป, ยซAnamnesiยป...) vengono scartati.

+
    +
  • Alto: la censura viene segnata subito (riquadro nero).
    • +
  • Incerto: diventa un suggerimento da rivedere. Non viene applicato nรฉ perso.
    • +
  • Molto basso: scartato (nel profilo sanitario diventa comunque un suggerimento).
    • +
+

I codici con controllo di validitร  (IBAN, carte, codice fiscale) vengono censurati interi anche se attaccati a un'etichetta.

+"""), + ("Revisione dei suggerimenti", """ +

Revisione dei suggerimenti

+

Al termine dell'analisi si apre l'elenco dei suggerimenti: pagina, valore (mascherato finchรฉ non spunti Mostra i valori), tipo, punteggio, fonte e motivi.

+
    +
  • Spunta gli elementi da censurare e premi Applica selezionati: gli altri vengono scartati.
    • +
  • Seleziona โ‰ฅ soglia spunta in blocco gli elementi con punteggio alto.
    • +
  • Il campo Tipo รจ modificabile: controlla o correggi la categoria prima di applicare il suggerimento.
    • +
  • La colonna Fonte distingue regole, modello locale e OCR.
    • +
  • Rivedi piรน tardi lascia tutto in sospeso; il pulsante Rivedi suggerimenti (N) nella barra laterale riapre l'elenco.
    • +
  • Doppio clic su una riga porta alla pagina; selezionando una riga il suo contorno si evidenzia.
    • +
+

Sulla pagina i suggerimenti compaiono come contorni arancioni (solo a video): clicca su uno per scegliere Censura, Scarta o Mostra valore.

+

All'esportazione, se restano suggerimenti non revisionati, il programma te lo ricorda: non verranno censurati.

+"""), + ("Correzioni manuali", """ +

Correzioni manuali

+
    +
  • Aggiungere una censura: trascina con il mouse un rettangolo sull'area da censurare. Se contiene testo puoi aggiungerlo alla blocklist, cosรฌ sarร  censurato in ogni documento futuro; se รจ un'area senza testo (immagine) viene oscurata come immagine.
    • +
  • Togliere una censura: clicca sul riquadro. Puoi aggiungere quel testo all'allowlist per non vederlo piรน censurato.
    • +
  • Pulisci Pagina toglie tutte le censure della pagina corrente.
    • +
+"""), + ("Dizionari", """ +

Dizionari

+

I Dizionari contengono due liste: la blocklist (parole sempre censurate) e l'allowlist (parole mai censurate). Il confronto avviene per parola intera: una parola in allowlist non nasconde mai un valore diverso o piรน lungo.

+
    +
  • Ogni termine ha data di inserimento e scadenza (AAAA-MM-GG oppure Mai); i termini scaduti vengono eliminati da soli. Scadenza predefinita si applica ai nuovi termini, anche a quelli aggiunti dalla pagina.
    • +
  • Aggiungi, Rimuovi, Importa da testoโ€ฆ (un termine per riga) e Esporta in chiaroโ€ฆ (il file contiene dati personali).
    • +
  • Le liste sono cifrate per l'utente corrente (Windows DPAPI, portachiavi di sistema se disponibile, altrimenti un file chiave locale con protezione limitata). Copiate su un altro account o PC non sono leggibili: il programma non le cancella e le mostra vuote.
    • +
+"""), + ("Esportare", """ +

Esportare

+

Esporta PDF crea il file finale. Si sceglie tra:

+
    +
  • Rimozione irreversibile: i valori selezionati vengono eliminati dal file. Non si puรฒ tornare indietro dal file esportato.
    • +
  • Pseudonimizzazione reversibile: i valori sono sostituiti da segnaposto e viene creata una mappa di ripristino cifrata, separata dal PDF.
    • +
+

Stile dei segnaposto: Casuale (consigliato, non rivela ordine nรฉ numero dei valori), Progressivo (PERSON_001... rivela quanti valori distinti ci sono), Generico (nasconde anche il tipo), Coerente (chiave) (lo stesso valore ha lo stesso segnaposto in tutti i documenti creati con la stessa chiave; collega i documenti tra loro). Gestisci chiave... esporta o importa la chiave protetta da password per usarla su un altro computer.

+

Mappa di ripristino: salvala lontano dal PDF e non in una cartella sincronizzata. Servono una password (almeno 12 caratteri, chiesta due volte) e, volendo, una chiave di recupero mostrata una sola volta: conservala in un luogo sicuro, il programma non la salva.

+

Cosa fa il programma in esportazione

+
    +
  • Lavora su una copia in memoria: il documento aperto e il file originale non cambiano.
    • +
  • Rimuove annotazioni, allegati, link esterni, segnalibri, script, metadati, testo invisibile e fuori pagina.
    • +
  • Verifica il file scritto: se un valore rimosso รจ ancora leggibile, o resta un glifo della parola, te lo dice e puoi eliminare il file.
    • +
  • Scrive un registro (solo conteggi e hash, nessun dato personale).
    • +
+

Il nome file suggerito non contiene il nome del documento originale. Se scegli un nome che contiene un valore da rimuovere vieni avvisato.

+

Attenzione: la censura non garantisce l'anonimato. Date, luoghi o altri dettagli rimasti possono ancora identificare una persona. Controlla sempre il risultato. Un PDF pseudonimizzato e la sua mappa restano dati personali.

+"""), + ("Ricostruire un PDF", """ +

Ricostruire un PDF

+

Ricostruisci PDF rimette i valori originali in un PDF pseudonimizzato.

+
    +
  1. Scegli il PDF pseudonimizzato e la sua mappa (.nullifypdf-map).
    2. +
  2. Inserisci la password, oppure spunta Usa la chiave di recupero e digita la chiave (NPDF-...).
    3. +
  3. Scegli dove salvare il file ricostruito.
    4. +
+

Il programma controlla che la mappa corrisponda al PDF e ti dice quanti valori sono stati davvero ripristinati. Il file ricostruito contiene di nuovo tutti i dati personali: trattalo di conseguenza.

+"""), + ("Documenti sanitari", """ +

Documenti sanitari

+

Con Profilo documenti sanitari il programma cerca anche:

+
    +
  • date di nascita (precedute da ยซnato/a ilยป, ยซnascitaยป...);
    • +
  • tessera sanitaria (20 cifre, struttura pubblicata, il carattere di controllo non รจ verificato);
    • +
  • numero di ricetta (NRE) e codici di esenzione, che richiedono la loro etichetta nelle vicinanze;
    • +
  • CAP e numeri civici in un contesto di indirizzo;
    • +
  • firme e timbri vicini a ยซfirmaยป, ยซtimbroยป, ยซil medicoยป...: sono sempre solo suggerimenti.
    • +
+

In questa modalitร  nessun caso incerto viene scartato in silenzio: diventa un suggerimento da rivedere. I dati sanitari sono dati di categoria particolare (art. 9 GDPR): controlla sempre il risultato.

+"""), + ("Dati locali e privacy", """ +

Dati locali e privacy

+

Il programma conserva nella cartella dell'utente (~/.nullifypdf): dizionari cifrati, chiave dei segnaposto cifrata, registro rotante dei messaggi, registri di esportazione (solo conteggi e hash) e, per file molto grandi, una cartella temporanea privata svuotata dopo l'esportazione.

+

Elimina dati locali cancella log, registri e file temporanei. Se confermi, elimina anche i dizionari e, con una seconda conferma, le chiavi: senza chiavi dizionari cifrati e chiave dei segnaposto diventano illeggibili.

+

Nessun valore rilevato viene scritto nei log. Nessun documento lascia il computer.

+"""), + ("Info e aggiornamenti", """ +

Info e aggiornamenti

+

La finestra Info mostra la versione e il collegamento al repository. Elenca anche la licenza del programma (AGPL-3.0-or-later) e quella di ogni componente e modello usato, con i collegamenti ai testi ufficiali. Controlla aggiornamenti interroga le release pubblicate su GitHub; con Controlla ad ogni avvio (disattivato di default) il controllo รจ eseguito all'avvio e mostra qualcosa solo se esiste una versione piรน recente.

+"""), + ("Scorciatoie", """ +

Scorciatoie

+ + + + + + + +
F1 o il pulsante GuidaApre questa guida
Ctrl + rotellaZoom dell'anteprima
Trascina sull'anteprimaSegna un'area da censurare
Clic su una censuraLa rimuove
Clic su un contorno arancioneDecide un suggerimento
Trascina un PDF sulla finestraLo apre
+

Passando il mouse su un elemento dell'interfaccia per qualche secondo compare una breve descrizione.

+"""), + ("Limiti e problemi comuni", """ +

Limiti e problemi comuni

+
    +
  • Il riconoscimento รจ statistico: controlla sempre il risultato prima di condividere un file. Nomi e luoghi non rilevati o dati indiretti restano.
    • +
  • Con righe molto ravvicinate (circa metร  del corpo del carattere o meno) non esiste un riquadro che risparmi le righe vicine: compare un avviso e parole adiacenti possono essere danneggiate, mai lasciate leggibili.
    • +
  • Il testo dentro un'immagine viene rimosso solo dove l'immagine รจ censurata; per trovarlo serve l'OCR.
    • +
  • Se una parola comune viene proposta, scartala in revisione o aggiungila all'allowlist.
    • +
  • Se i dizionari risultano vuoti dopo aver copiato il profilo su un altro PC, vedi l'argomento Dizionari.
    • +
  • Le mappe create da questa versione non si aprono con versioni precedenti.
    • +
+"""), +] + + +class GuideDialog(QDialog): + """Topic list on the left, text on the right. Opened with F1.""" + + def __init__(self, parent: Any = None, topic: int = 0) -> None: + super().__init__(parent) + self.setWindowTitle("Guida - NullifyPDF") + self.resize(900, 620) + lay = QHBoxLayout(self) + self.topics = QListWidget() + self.topics.setObjectName("guide_topics") + self.topics.setFixedWidth(230) + for title, _html in GUIDE_TOPICS: + self.topics.addItem(title) + left = QVBoxLayout() + left.addWidget(self.topics) + btn = QPushButton("Chiudi") + btn.clicked.connect(self.accept) + left.addWidget(btn) + lay.addLayout(left) + self.text = QTextBrowser() + self.text.setObjectName("guide_text") + self.text.setOpenExternalLinks(True) + lay.addWidget(self.text, 1) + self.topics.currentRowChanged.connect(self._show) + self.topics.setCurrentRow(max(0, min(topic, len(GUIDE_TOPICS) - 1))) + apply_tooltips(self) # skips the topic list and the text: no hover popups while reading + + def _show(self, row: int) -> None: + if 0 <= row < len(GUIDE_TOPICS): + self.text.setHtml(GUIDE_TOPICS[row][1]) + self.text.moveCursor(self.text.textCursor().MoveOperation.Start) + + +__all__ = [ + "GUIDE_TOPICS", "GuideDialog", "TOOLTIPS", "ENTITY_TOOLTIPS", + "apply_tooltips", "tooltip_for", "widgets_without_tooltip", "Qt", +] diff --git a/lexicon_data_it.py b/lexicon_data_it.py new file mode 100644 index 0000000..b2189dc --- /dev/null +++ b/lexicon_data_it.py @@ -0,0 +1,15 @@ +# Generated by scripts/build_lexicon_data.py - do not edit. +# Contains ISTAT data (Elenco comuni italiani, comuni.xlsx), licensed +# CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/); modified: names +# normalised, lower-cased, bilingual names split. +SOURCE_SHA256 = {'istat_comuni': '83842076860450f7e482daecea6b7a769f5f93d0bf5b0d48802b44896d7a26d5'} +COMUNI = frozenset("""abano terme|abbadia cerreto|abbadia lariana|abbadia san salvatore|abbasanta|abbateggio|abbiategrasso|abetone cutigliano|abriola|abtei|acate|accadia|acceglio|accettura|acciano|accumoli|acerenza|acerno|acerra|aci bonaccorsi|aci castello|aci catena|aci sant'antonio|acireale|acquafondata|acquaformosa|acquafredda|acqualagna|acquanegra cremonese|acquanegra sul chiese|acquapendente|acquappesa|acquaro|acquasanta terme|acquasparta|acquaviva collecroce|acquaviva d'isernia|acquaviva delle fonti|acquaviva picena|acquaviva platani|acquedolci|acqui terme|acri|acuto|adelfia|adrano|adrara san martino|adrara san rocco|adria|adro|affi|affile|afragola|africo|agazzano|agerola|aggius|agira|agliana|agliano terme|aglientu|agliรจ|agna|agnadello|agnana calabra|agnone|agnosine|agordo|agosta|agra|agrate brianza|agrate conturbia|agrigento|agropoli|agugliano|agugliaro|ahrntal|aicurzio|aidomaggiore|aidone|aielli|aiello calabro|aiello del friuli|aiello del sabato|aieta|ailano|ailoche|airasca|airola|airole|airuno|aisone|ala|ala di stura|alagna|alagna valsesia|alanno|alassio|alatri|alba|alba adriatica|albagiara|albairate|albanella|albano di lucania|albano laziale|albano sant'alessandro|albano vercellese|albaredo d'adige|albaredo per san marco|albareto|albaretto della torre|albavilla|albenga|albera ligure|alberobello|alberona|albese con cassano|albettone|albi|albiano|albiano d'ivrea|albiate|albidona|albignasego|albinea|albino|albiolo|albisola superiore|albissola marina|albizzate|albonese|albosaggia|albugnano|albuzzano|alcamo|alcara li fusi|aldein|aldeno|aldino|ales|alessandria|alessandria del carretto|alessandria della rocca|alessano|alezio|alfano|alfedena|alfianello|alfiano natta|alfonsine|alghero|algua|algund|alia|aliano|alice bel colle|alice castello|alife|alimena|aliminusa|allai|alleghe|allein|allerona|alliste|allumiere|alluvioni piovera|almenno san bartolomeo|almenno san salvatore|almese|almรจ|alonte|alpago|alpette|alpignano|alseno|alserio|alta val tidone|alta valle intelvi|altamura|altare|altavalle|altavilla irpina|altavilla milicia|altavilla monferrato|altavilla silentina|altavilla vicentina|altidona|altilia|altino|altissimo|altivole|alto|alto reno terme|alto sermenza|altofonte|altomonte|altopascio|altopiano della vigolana|altrei|alviano|alvignano|alvito|alzano lombardo|alzano scrivia|alzate brianza|alร  dei sardi|alรฌ|alรฌ terme|amalfi|amandola|amantea|amaro|amaroni|amaseno|amato|amatrice|ambivere|amblar-don|ameglia|amelia|amendolara|ameno|amorosi|ampezzo|anacapri|anagni|ancarano|ancona|andali|andalo|andalo valtellino|andezeno|andora|andorno micca|andrano|andrate|andreis|andretta|andria|andrian|andriano|anela|anfo|angera|anghiari|angiari|angolo terme|angri|angrogna|anguillara sabazia|anguillara veneta|annicco|annone di brianza|annone veneto|anoia|antegnate|anterivo|antey-saint-andrรฉ|anticoli corrado|antignano|antillo|antonimina|antrodoco|antrona schieranco|anversa degli abruzzi|anzano del parco|anzano di puglia|anzi|anzio|anzola d'ossola|anzola dell'emilia|aosta|apecchio|apice|apiro|apollosa|appiano gentile|appiano sulla strada del vino|appignano|appignano del tronto|aprica|apricale|apricena|aprigliano|aprilia|aquara|aquila d'arroscia|aquileia|aquilonia|aquino|aradeo|aragona|aramengo|arba|arborea|arborio|arbus|arcade|arce|arcene|arcevia|archi|arcidosso|arcinazzo romano|arcisate|arco|arcola|arcole|arconate|arcore|arcugnano|ardara|ardauli|ardea|ardenno|ardesio|ardore|arena|arena po|arenzano|arese|arezzo|argegno|argelato|argenta|argentera|arguello|argusto|ari|ariano irpino|ariano nel polesine|ariccia|arielli|arienzo|arignano|aritzo|arizzano|arlena di castro|arluno|armeno|armento|armo|armungia|arnad|arnara|arnasco|arnesano|arola|arona|arosio|arpaia|arpaise|arpino|arquata del tronto|arquata scrivia|arquร  petrarca|arquร  polesine|arre|arrone|arsago seprio|arsiero|arsita|arsiรจ|arsoli|arta terme|artegna|artena|artogne|arvier|arzachena|arzago d'adda|arzana|arzano|arzergrande|arzignano|ascea|asciano|ascoli piceno|ascoli satriano|ascrea|asiago|asigliano veneto|asigliano vercellese|asola|asolo|assago|assemini|assisi|asso|assolo|assoro|asti|asuni|ateleta|atella|atena lucana|atessa|atina|atrani|atri|atripalda|attigliano|attimis|atzara|auer|augusta|auletta|aulla|aurano|aurigo|auronzo di cadore|ausonia|austis|avegno|avelengo|avella|avellino|averara|aversa|avetrana|avezzano|aviano|aviatico|avigliana|avigliano|avigliano umbro|avio|avise|avola|avolasca|ayas|aymavilles|azeglio|azzanello|azzano d'asti|azzano decimo|azzano mella|azzano san paolo|azzate|azzio|azzone|baceno|bacoli|badalucco|badesi|badia|badia calavena|badia pavese|badia polesine|badia tedalda|badolato|bagaladi|bagheria|bagnacavallo|bagnara calabra|bagnara di romagna|bagnaria|bagnaria arsa|bagnasco|bagnatica|bagni di lucca|bagno a ripoli|bagno di romagna|bagnoli del trigno|bagnoli di sopra|bagnoli irpino|bagnolo cremasco|bagnolo del salento|bagnolo di po|bagnolo in piano|bagnolo mella|bagnolo piemonte|bagnolo san vito|bagnone|bagnoregio|bagolino|baia e latina|baiano|bairo|baiso|bajardo|balangero|baldichieri d'asti|baldissero canavese|baldissero d'alba|baldissero torinese|balestrate|balestrino|ballabio|ballao|balme|balmuccia|balocco|balsorano|balvano|balzola|banari|banchette|bannio anzino|banzi|baone|baradili|baragiano|baranello|barano d'ischia|baranzate|barasso|baratili san pietro|barbania|barbara|barbarano mossano|barbarano romano|barbaresco|barbariga|barbata|barberino di mugello|barberino tavarnelle|barbian|barbianello|barbiano|barbona|barcellona pozzo di gotto|barcis|bard|bardello con malgesso e bregano|bardi|bardineto|bardolino|bardonecchia|bareggio|barengo|baressa|barete|barga|bargagli|barge|barghe|bari|bari sardo|bariano|baricella|barile|barisciano|barlassina|barletta|barni|barolo|barone canavese|baronissi|barrafranca|barrali|barrea|barumini|barzago|barzana|barzanรฒ|barzio|basaluzzo|bascapรจ|baschi|basciano|baselga di pinรจ|baselice|basiano|basicรฒ|basiglio|basiliano|bassano bresciano|bassano del grappa|bassano in teverina|bassano romano|bassiano|bassignana|bastia mondovรฌ|bastia umbra|bastida pancarana|bastiglia|battaglia terme|battifollo|battipaglia|battuda|baucina|bauladu|baunei|baveno|bedero valcuvia|bedizzole|bedollo|bedonia|bedulita|bee|beinasco|beinette|belcastro|belfiore|belforte all'isauro|belforte del chienti|belforte monferrato|belgioioso|belgirate|bella|bellagio|bellano|bellante|bellaria-igea marina|bellegra|bellino|bellinzago lombardo|bellinzago novarese|bellizzi|bellona|bellosguardo|belluno|bellusco|belmonte calabro|belmonte castello|belmonte del sannio|belmonte in sabina|belmonte mezzagno|belmonte piceno|belpasso|belsito|belvedere di spinello|belvedere langhe|belvedere marittimo|belvedere ostrense|belveglio|belvรฌ|bema|bene lario|bene vagienna|benestare|benetutti|benevello|benevento|benna|bentivoglio|berbenno|berbenno di valtellina|berceto|berchidda|beregazzo con figliaro|bereguardo|bergamasco|bergamo|bergantino|bergeggi|bergolo|berlingo|bernalda|bernareggio|bernate ticino|bernezzo|bertinoro|bertiolo|bertonico|berzano di san pietro|berzano di tortona|berzo demo|berzo inferiore|berzo san fermo|besana in brianza|besano|besate|besenello|besenzone|besnate|besozzo|bessude|bettola|bettona|beura-cardezza|bevagna|beverino|bevilacqua|biancavilla|bianchi|bianco|biandrate|biandronno|bianzano|bianzone|bianzรจ|biassono|bibbiano|bibbiena|bibbona|bibiana|biccari|bicinicco|bidonรฌ|biella|bienno|bieno|bientina|binago|binasco|binetto|bioglio|bionaz|bione|birori|bisaccia|bisacquino|bisceglie|bisegna|bisenti|bisignano|bistagno|bisuschio|bitetto|bitonto|bitritto|bitti|bivona|bivongi|bizzarone|bleggio superiore|blello|blera|blessagno|blevio|blufi|boara pisani|bobbio|bobbio pellice|boca|bocchigliero|boccioleto|bocenago|bodio lomnago|boffalora d'adda|boffalora sopra ticino|bogliasco|bognanco|bogogno|boissano|bojano|bolano|bolgare|bollate|bollengo|bologna|bolognano|bolognetta|bolognola|bolotana|bolsena|boltiere|bolzano|bolzano novarese|bolzano vicentino|bomarzo|bomba|bompensiere|bompietro|bomporto|bonarcado|bonassola|bonate sopra|bonate sotto|bonavigo|bondeno|bondone|bonea|bonefro|bonemerse|bonifati|bonito|bonnanaro|bono|bonorva|bonvicino|borbona|borca di cadore|bordano|bordighera|bordolano|bore|boretto|borgarello|borgaro torinese|borgetto|borghetto d'arroscia|borghetto di borbera|borghetto di vara|borghetto lodigiano|borghetto santo spirito|borghi|borgia|borgiallo|borgio verezzi|borgo a mozzano|borgo chiese|borgo d'ale|borgo d'anaunia|borgo di terzo|borgo lares|borgo mantovano|borgo pace|borgo priolo|borgo san dalmazzo|borgo san giacomo|borgo san giovanni|borgo san lorenzo|borgo san martino|borgo san siro|borgo ticino|borgo tossignano|borgo val di taro|borgo valbelluna|borgo valsugana|borgo velino|borgo veneto|borgo vercelli|borgo virgilio|borgocarbonara|borgofranco d'ivrea|borgolavezzaro|borgomale|borgomanero|borgomaro|borgomasino|borgomezzavalle|borgone susa|borgonovo val tidone|borgoratto alessandrino|borgoratto mormorolo|borgoricco|borgorose|borgosatollo|borgosesia|bormida|bormio|bornasco|borno|boroneddu|borore|borrello|borriana|borso del grappa|bortigali|bortigiadas|borutta|borzonasca|bosa|bosaro|boschi sant'anna|bosco chiesanuova|bosco marengo|bosconero|boscoreale|boscotrecase|bosia|bosio|bosisio parini|bosnasco|bossico|bossolasco|botricello|botrugno|bottanuco|botticino|bottidda|bova|bova marina|bovalino|bovegno|boves|bovezzo|boville ernica|bovino|bovisio-masciago|bovolenta|bovolone|bozen|bozzole|bozzolo|bra|bracca|bracciano|bracigliano|braies|brallo di pregola|brancaleone|brandico|brandizzo|branzi|branzoll|braone|brebbia|breda di piave|breganze|bregnano|brembate|brembate di sopra|brembio|breme|brendola|brenna|brenner|brennero|breno|brenta|brentino belluno|brentonico|brenzone sul garda|brescello|brescia|bresimo|bressana bottarone|bressanone|bressanvido|bresso|brezzo di bedero|briaglia|briatico|bricherasio|brienno|brienza|briga alta|briga novarese|brignano gera d'adda|brignano-frascata|brindisi|brindisi montagna|brinzio|briona|brione|briosco|brisighella|brissago-valtravaglia|brissogne|brittoli|brivio|brixen|broccostella|brogliano|brognaturo|brolo|brondello|broni|bronte|bronzolo|brossasco|brosso|brovello-carpugnino|brozolo|brugherio|brugine|brugnato|brugnera|bruino|brumano|brunate|bruneck|brunello|brunico|bruno|brusaporto|brusasco|brusciano|brusimpiano|brusnengo|brusson|bruzolo|bruzzano zeffirio|bubbiano|bubbio|buccheri|bucchianico|bucciano|buccinasco|buccino|bucine|buddusรฒ|budoia|budoni|budrio|buggerru|buggiano|buglio in monte|bugnara|buguggiate|buja|bulciago|bulgarograsso|bultei|bulzi|buonabitacolo|buonalbergo|buonconvento|buonvicino|burago di molgora|burcei|burgio|burgos|burgstall|buriasco|burolo|buronzo|busachi|busalla|busano|busca|buscate|buscemi|buseto palizzolo|busnago|bussero|busseto|bussi sul tirino|busso|bussolengo|bussoleno|busto arsizio|busto garolfo|butera|buti|buttapietra|buttigliera alta|buttigliera d'asti|buttrio|cabella ligure|cabiate|cabras|caccamo|caccuri|cadegliano-viconago|cadelbosco di sopra|cadeo|caderzone terme|cadoneghe|cadorago|cadrezzate con osmate|caerano di san marco|cafasse|caggiano|cagli|cagliari|caglio|cagnano amiterno|cagnano varano|caianello|caiazzo|caines|caino|caiolo|cairano|cairate|cairo montenotte|caivano|calabritto|calalzo di cadore|calamandrana|calamonaci|calangianus|calanna|calasca-castiglione|calascibetta|calascio|calasetta|calatabiano|calatafimi-segesta|calcata|calceranica al lago|calci|calciano|calcinaia|calcinate|calcinato|calcio|calco|caldaro sulla strada del vino|caldarola|calderara di reno|caldes|caldiero|caldogno|caldonazzo|calendasco|calenzano|calestano|calice al cornoviglio|calice ligure|calimera|calitri|calizzano|callabiana|calliano|calliano monferrato|calolziocorte|calopezzati|calosso|caloveto|caltabellotta|caltagirone|caltanissetta|caltavuturo|caltignaga|calto|caltrano|calusco d'adda|caluso|calvagese della riviera|calvanico|calvatone|calvello|calvene|calvenzano|calvera|calvi|calvi dell'umbria|calvi risorta|calvignano|calvignasco|calvisano|calvizzano|camagna monferrato|camaiore|camandona|camastra|cambiago|cambiano|cambiasca|camburzano|camerana|camerano|camerano casasco|camerata cornello|camerata nuova|camerata picena|cameri|camerino|camerota|camigliano|camini|camino|camino al tagliamento|camisano|camisano vicentino|cammarata|camogli|campagna|campagna lupia|campagnano di roma|campagnatico|campagnola cremasca|campagnola emilia|campana|camparada|campegine|campello sul clitunno|campertogno|campi bisenzio|campi salentina|campiglia cervo|campiglia dei berici|campiglia marittima|campiglione fenile|campione d'italia|campitello di fassa|campli|campo calabro|campo di giove|campo di trens|campo ligure|campo nell'elba|campo san martino|campo tures|campobasso|campobello di licata|campobello di mazara|campochiaro|campodarsego|campodenno|campodimele|campodipietra|campodolcino|campodoro|campofelice di fitalia|campofelice di roccella|campofilone|campofiorito|campoformido|campofranco|campogalliano|campolattaro|campoli appennino|campoli del monte taburno|campolieto|campolongo maggiore|campolongo tapogliano|campomaggiore|campomarino|campomorone|camponogara|campora|camporeale|camporgiano|camporosso|camporotondo di fiastrone|camporotondo etneo|camposampiero|camposano|camposanto|campospinoso albaredo|campotosto|camugnano|canal san bovo|canale|canale d'agordo|canale monterano|canaro|canazei|cancellara|cancello ed arnone|canda|candela|candelo|candia canavese|candia lomellina|candiana|candida|candidoni|candiolo|canegrate|canelli|canepina|caneva|canicattini bagni|canicattรฌ|canino|canischio|canistro|canna|cannalonga|cannara|cannero riviera|canneto pavese|canneto sull'oglio|cannobio|cannole|canolo|canonica d'adda|canosa di puglia|canosa sannita|canosio|canossa|cansano|cantagallo|cantalice|cantalupa|cantalupo in sabina|cantalupo ligure|cantalupo nel sannio|cantarana|cantello|canterano|cantiano|cantoira|cantรน|canzano|canzo|caorle|caorso|capaccio paestum|capaci|capalbio|capannoli|capannori|capena|capergnanica|capestrano|capiago intimiano|capistrano|capistrello|capitignano|capizzi|capizzone|capo d'orlando|capo di ponte|capodimonte|capodrise|capoliveri|capolona|caponago|caporciano|caposele|capoterra|capovalle|cappadocia|cappella cantone|cappella de' picenardi|cappella maggiore|cappelle sul tavo|capracotta|capraia e limite|capraia isola|capralba|capranica|capranica prenestina|caprarica di lecce|caprarola|caprauna|caprese michelangelo|caprezzo|capri|capri leone|capriana|capriano del colle|capriata d'orba|capriate san gervasio|capriati a volturno|caprie|capriglia irpina|capriglio|caprile|caprino bergamasco|caprino veronese|capriolo|capriva del friuli|capua|capurso|caraffa del bianco|caraffa di catanzaro|caraglio|caramagna piemonte|caramanico terme|carapelle|carapelle calvisio|carasco|carassai|carate brianza|carate urio|caravaggio|caravate|caravino|caravonica|carbognano|carbonara al ticino|carbonara di nola|carbonara scrivia|carbonate|carbone|carbonera|carbonia|carcare|carcoforo|cardano al campo|cardedu|cardeto|cardinale|cardito|cardรจ|careggine|carema|carenno|carentino|careri|caresana|caresanablot|carezzano|carfizzi|cargeghe|cariati|carife|carignano|carimate|carinaro|carini|carinola|carisio|carisolo|carlantino|carlazzo|carlentini|carlino|carloforte|carlopoli|carmagnola|carmiano|carmignano|carmignano di brenta|carnago|carnate|carobbio degli angeli|carolei|carona|caronia|caronno pertusella|caronno varesino|carosino|carovigno|carovilli|carpaneto piacentino|carpanzano|carpegna|carpenedolo|carpeneto|carpi|carpiano|carpignano salentino|carpignano sesia|carpineti|carpineto della nora|carpineto romano|carpineto sinello|carpino|carpinone|carrara|carrega ligure|carro|carrodano|carrosio|carrรจ|carrรน|carsoli|cartigliano|cartignano|cartoceto|cartosio|cartura|carugate|carugo|carunchio|carvico|carzano|casabona|casacalenda|casacanditella|casagiove|casal cermelli|casal di principe|casal velino|casalanguida|casalattico|casalbeltrame|casalbordino|casalbore|casalborgone|casalbuono|casalbuttano ed uniti|casalciprano|casalduni|casale corte cerro|casale cremasco-vidolasco|casale di scodosia|casale litta|casale marittimo|casale monferrato|casale sul sile|casalecchio di reno|casaleggio boiro|casaleggio novara|casaleone|casaletto ceredano|casaletto di sopra|casaletto lodigiano|casaletto spartano|casaletto vaprio|casalfiumanese|casalgrande|casalgrasso|casali del manco|casalincontrada|casalino|casalmaggiore|casalmaiocco|casalmorano|casalmoro|casalnoceto|casalnuovo di napoli|casalnuovo monterotaro|casaloldo|casalpusterlengo|casalromano|casalserugo|casaluce|casalvecchio di puglia|casalvecchio siculo|casalvieri|casalvolone|casalzuigno|casamarciano|casamassima|casamicciola terme|casandrino|casanova elvo|casanova lerrone|casanova lonati|casape|casapesenna|casapinta|casaprota|casapulla|casarano|casargo|casarile|casarsa della delizia|casarza ligure|casasco|casatenovo|casatisma|casavatore|casazza|cascia|casciago|casciana terme lari|cascina|cascinette d'ivrea|casei gerola|caselette|casella|caselle in pittari|caselle landi|caselle lurani|caselle torinese|caserta|casier|casignana|casina|casirate d'adda|caslino d'erba|casnate con bernate|casnigo|casola di napoli|casola in lunigiana|casola valsenio|casole d'elsa|casoli|casorate primo|casorate sempione|casorezzo|casoria|casorzo monferrato|casperia|caspoggio|cassacco|cassago brianza|cassano all'ionio|cassano d'adda|cassano delle murge|cassano irpino|cassano magnago|cassano spinola|cassano valcuvia|cassaro|cassiglio|cassina de' pecchi|cassina rizzardi|cassina valsassina|cassinasco|cassine|cassinelle|cassinetta di lugagnano|cassino|cassola|cassolnovo|castagnaro|castagneto carducci|castagneto po|castagnito|castagnole delle lanze|castagnole monferrato|castagnole piemonte|castana|castano primo|casteggio|castegnato|castegnero nanto|castel baronia|castel boglione|castel bolognese|castel campagnano|castel castagna|castel condino|castel d'aiano|castel d'ario|castel d'azzano|castel del giudice|castel del monte|castel del piano|castel del rio|castel di casio|castel di ieri|castel di iudica|castel di lama|castel di lucio|castel di sangro|castel di sasso|castel di tora|castel focognano|castel frentano|castel gabbiano|castel gandolfo|castel giorgio|castel goffredo|castel guelfo di bologna|castel ivano|castel madama|castel maggiore|castel mella|castel morrone|castel ritaldi|castel rocchero|castel rozzone|castel san giorgio|castel san giovanni|castel san lorenzo|castel san niccolรฒ|castel san pietro romano|castel san pietro terme|castel san vincenzo|castel sant'angelo|castel sant'elia|castel viscardo|castel vittorio|castel volturno|castelbaldo|castelbelforte|castelbellino|castelbello-ciardes|castelbianco|castelbottaccio|castelbuono|castelcivita|castelcovati|castelcucco|casteldaccia|casteldelci|casteldelfino|casteldidone|castelfidardo|castelfiorentino|castelforte|castelfranci|castelfranco di sotto|castelfranco emilia|castelfranco in miscano|castelfranco piandiscรฒ|castelfranco veneto|castelgerundo|castelgomberto|castelgrande|castelguglielmo|castelguidone|castell'alfero|castell'arquato|castell'azzara|castell'umberto|castellabate|castellafiume|castellalto|castellammare del golfo|castellammare di stabia|castellamonte|castellana grotte|castellana sicula|castellaneta|castellania coppi|castellanza|castellar guidobono|castellarano|castellaro|castellazzo bormida|castellazzo novarese|castelleone|castelleone di suasa|castellero|castelletto cervo|castelletto d'erro|castelletto d'orba|castelletto di branduzzo|castelletto merli|castelletto molina|castelletto monferrato|castelletto sopra ticino|castelletto stura|castelletto uzzone|castelli|castelli calepio|castellina in chianti|castellina marittima|castellinaldo d'alba|castellino del biferno|castellino tanaro|castelliri|castello cabiaglio|castello d'agogna|castello d'argile|castello del matese|castello dell'acqua|castello di annone|castello di brianza|castello di cisterna|castello di godego|castello tesino|castello-molina di fiemme|castellucchio|castelluccio dei sauri|castelluccio inferiore|castelluccio superiore|castelluccio valmaggiore|castelmagno|castelmarte|castelmassa|castelmauro|castelmezzano|castelmola|castelnovetto|castelnovo bariano|castelnovo del friuli|castelnovo di sotto|castelnovo ne' monti|castelnuovo|castelnuovo belbo|castelnuovo berardenga|castelnuovo bocca d'adda|castelnuovo bormida|castelnuovo bozzente|castelnuovo calcea|castelnuovo cilento|castelnuovo del garda|castelnuovo della daunia|castelnuovo di ceva|castelnuovo di conza|castelnuovo di farfa|castelnuovo di garfagnana|castelnuovo di porto|castelnuovo di val di cecina|castelnuovo don bosco|castelnuovo magra|castelnuovo nigra|castelnuovo parano|castelnuovo rangone|castelnuovo scrivia|castelpagano|castelpetroso|castelpizzuto|castelplanio|castelpoto|castelraimondo|castelrotto|castelsantangelo sul nera|castelsaraceno|castelsardo|castelseprio|castelsilano|castelspina|casteltermini|castelveccana|castelvecchio calvisio|castelvecchio di rocca barbena|castelvecchio subequo|castelvenere|castelverde|castelverrino|castelvetere in val fortore|castelvetere sul calore|castelvetrano|castelvetro di modena|castelvetro piacentino|castelvisconti|castenaso|castenedolo|castiadas|castiglion fibocchi|castiglion fiorentino|castiglione a casauria|castiglione chiavarese|castiglione cosentino|castiglione d'adda|castiglione d'orcia|castiglione dei pepoli|castiglione del genovesi|castiglione del lago|castiglione della pescaia|castiglione delle stiviere|castiglione di garfagnana|castiglione di sicilia|castiglione falletto|castiglione in teverina|castiglione messer marino|castiglione messer raimondo|castiglione olona|castiglione tinella|castiglione torinese|castignano|castilenti|castino|castione andevenno|castione della presolana|castions di strada|castiraga vidardo|casto|castorano|castrezzato|castri di lecce|castrignano de' greci|castrignano del capo|castro|castro dei volsci|castrocaro terme e terra del sole|castrocielo|castrofilippo|castrolibero|castronno|castronovo di sicilia|castronuovo di sant'andrea|castropignano|castroreale|castroregio|castrovillari|catania|catanzaro|catenanuova|catignano|cattolica|cattolica eraclea|caulonia|cautano|cava de' tirreni|cava manara|cavaglietto|cavaglio d'agogna|cavagliร |cavagnolo|cavaion veronese|cavalese|cavallerleone|cavallermaggiore|cavallino|cavallino-treporti|cavallirio|cavareno|cavargna|cavaria con premezzo|cavarzere|cavaso del tomba|cavasso nuovo|cavatore|cavazzo carnico|cave|cavedago|cavedine|cavenago d'adda|cavenago di brianza|cavernago|cavezzo|cavizzana|cavour|cavriago|cavriana|cavriglia|cazzago brabbia|cazzago san martino|cazzano di tramigna|cazzano sant'andrea|ceccano|cecima|cecina|cedegolo|cedrasco|cefalร  diana|cefalรน|ceggia|ceglie messapica|celano|celenza sul trigno|celenza valfortore|celico|cella dati|cella monte|cellamare|cellara|cellarengo|cellatica|celle di bulgheria|celle di macra|celle di san vito|celle enomondo|celle ligure|celleno|cellere|cellino attanasio|cellino san marco|cellio con breia|cellole|cembra lisignago|cenadi|cenate sopra|cenate sotto|cencenighe agordino|cene|ceneselli|cengio|centallo|cento|centola|centrache|centro valle intelvi|centuripe|cepagatti|ceppaloni|ceppo morelli|ceprano|cerami|ceranesi|cerano|cerano d'intelvi|ceranova|ceraso|cercemaggiore|cercenasco|cercepiccola|cerchiara di calabria|cerchio|cercino|cercivento|cercola|cerda|cerea|ceregnano|cerenzia|ceres|ceresara|cereseto|ceresole alba|ceresole reale|cerete|ceretto lomellina|cergnago|ceriale|ceriana|ceriano laghetto|cerignale|cerignola|cerisano|cermenate|cermes|cermignano|cernobbio|cernusco lombardone|cernusco sul naviglio|cerreto d'asti|cerreto d'esi|cerreto di spoleto|cerreto grue|cerreto guidi|cerreto laziale|cerreto sannita|cerretto langhe|cerrina monferrato|cerrione|cerro al lambro|cerro al volturno|cerro maggiore|cerro tanaro|cerro veronese|cersosimo|certaldo|certosa di pavia|cerva|cervara di roma|cervarese santa croce|cervaro|cervasca|cervatto|cerveno|cervere|cervesina|cerveteri|cervia|cervicati|cervignano d'adda|cervignano del friuli|cervinara|cervino|cervo|cerzeto|cesa|cesana brianza|cesana torinese|cesano boscone|cesano maderno|cesara|cesarรฒ|cesate|cesena|cesenatico|cesinali|cesio|cesiomaggiore|cessalto|cessaniti|cessapalombo|cessole|cetara|ceto|cetona|cetraro|ceva|cevo|challand-saint-anselme|challand-saint-victor|chambave|chamois|champdepraz|champorcher|charvensod|cherasco|cheremule|chialamberto|chiampo|chianche|chianciano terme|chianni|chianocco|chiaramonte gulfi|chiaramonti|chiarano|chiaravalle|chiaravalle centrale|chiari|chiaromonte|chiauci|chiavari|chiavenna|chiaverano|chienes|chieri|chies d'alpago|chiesa in valmalenco|chiesanuova|chiesina uzzanese|chieti|chieuti|chieve|chignolo d'isola|chignolo po|chioggia|chiomonte|chions|chiopris-viscone|chitignano|chiuduno|chiuppano|chiuro|chiusa|chiusa di pesio|chiusa di san michele|chiusa sclafani|chiusaforte|chiusanico|chiusano d'asti|chiusano di san domenico|chiusavecchia|chiusdino|chiusi|chiusi della verna|chivasso|chรขtillon|ciampino|cianciana|cibiana di cadore|cicagna|cicala|cicciano|cicerale|ciciliano|cicognolo|ciconio|cigliano|cigliรจ|cigognola|cigole|cilavegna|cimadolmo|cimbergo|ciminna|ciminร |cimitile|cimolais|cimone|cinaglio|cineto romano|cingia de' botti|cingoli|cinigiano|cinisello balsamo|cinisi|cino|cinquefrondi|cintano|cinte tesino|cinto caomaggiore|cinto euganeo|cinzano|ciorlano|cipressa|circello|cirigliano|cirimido|ciriรจ|cirรฒ|cirรฒ marina|cis|cisano bergamasco|cisano sul neva|ciserano|cislago|cisliano|cison di valmarino|cissone|cisterna d'asti|cisterna di latina|cisternino|citerna|cittadella|cittaducale|cittanova|cittareale|cittiglio|cittร  della pieve|cittร  di castello|cittร  sant'angelo|civate|civezza|civezzano|civiasco|cividale del friuli|cividate al piano|cividate camuno|civita|civita castellana|civita d'antino|civitacampomarano|civitaluparella|civitanova del sannio|civitanova marche|civitaquana|civitavecchia|civitella alfedena|civitella casanova|civitella d'agliano|civitella del tronto|civitella di romagna|civitella in val di chiana|civitella messer raimondo|civitella paganico|civitella roveto|civitella san paolo|civo|claino con osteno|claut|clauzetto|clavesana|claviere|cles|cleto|clivio|clusone|coassolo torinese|coazze|coazzolo|coccaglio|cocconato|cocquio-trevisago|cocullo|codevigo|codevilla|codigoro|codogno|codognรจ|codroipo|codrongianos|coggiola|cogliate|cogne|cogoleto|cogollo del cengio|cogorno|colazza|colceresa|colere|colfelice|coli|colico|collalto sabino|collarmele|collazzone|colle brianza|colle d'anchise|colle di tora|colle di val d'elsa|colle san magno|colle sannita|colle santa lucia|colle umberto|collebeato|collecchio|collecorvino|colledara|colledimacine|colledimezzo|colleferro|collegiove|collegno|collelongo|collepardo|collepasso|collepietro|colleretto castelnuovo|colleretto giacosa|collesalvetti|collesano|colletorto|collevecchio|colli a volturno|colli al metauro|colli del tronto|colli sul velino|colli verdi|colliano|collinas|collio|collobiano|colloredo di monte albano|colmurano|colobraro|cologna veneta|cologne|cologno al serio|cologno monzese|colognola ai colli|colonna|colonnella|colonno|colorina|colorno|colosimi|colturano|colverde|colzate|comabbio|comacchio|comano|comano terme|comazzo|comeglians|comelico superiore|comerio|comezzano-cizzago|comignago|comiso|comitini|comiziano|commessaggio|commezzadura|como|compiano|comun nuovo|comunanza|cona|conca casale|conca dei marini|conca della campania|concamarise|concerviano|concesio|concordia sagittaria|concordia sulla secchia|concorezzo|condofuri|condove|condrรฒ|conegliano|confienza|configni|conflenti|coniolo|conselice|conselve|contessa entellina|contigliano|contrada|controguerra|controne|contursi terme|contร |conversano|conza della campania|conzano|copertino|copiano|copparo|corana|corato|corbara|corbetta|corbola|corchiano|corciano|cordenons|cordignano|cordovado|coreglia antelminelli|coreglia ligure|coreno ausonio|corfinio|cori|coriano|corigliano d'otranto|corigliano-rossano|corinaldo|corio|corleone|corleto monforte|corleto perticara|cormano|cormons|corna imagna|cornalba|cornale e bastida|cornaredo|cornate d'adda|cornedo all'isarco|cornedo vicentino|cornegliano laudense|corneliano d'alba|corniglio|corno di rosazzo|corno giovine|cornovecchio|cornuda|correggio|correzzana|correzzola|corrido|corridonia|corropoli|corsano|corsico|corsione|cortaccia sulla strada del vino|cortale|cortandone|cortanze|cortazzone|corte brugnatella|corte de' cortesi con cignone|corte de' frati|corte franca|corte palasio|cortemaggiore|cortemilia|corteno golgi|cortenova|cortenuova|corteolona e genzone|cortiglione|cortina d'ampezzo|cortina sulla strada del vino|cortino|cortona|corvara|corvara in badia|corvino san quirico|corzano|coseano|cosenza|cosio d'arroscia|cosio valtellino|cosoleto|cossano belbo|cossano canavese|cossato|cosseria|cossignano|cossogno|cossoine|cossombrato|costa de' nobili|costa di mezzate|costa di rovigo|costa masnaga|costa serina|costa valle imagna|costa vescovato|costa volpino|costabissara|costacciaro|costanzana|costarainera|costermano sul garda|costigliole d'asti|costigliole saluzzo|cotignola|cotronei|cottanello|courmayeur|covo|cozzo|craco|crandola valsassina|cravagliana|cravanzana|craveggia|creazzo|crecchio|credaro|credera rubbiano|crema|cremella|cremenaga|cremeno|cremia|cremolino|cremona|cremosano|crescentino|crespadoro|crespiatica|crespina lorenzana|crespino|cressa|crevacuore|crevalcore|crevoladossola|crispano|crispiano|crissolo|crocefieschi|crocetta del montello|crodo|crognaleto|cropalati|cropani|crosia|crosio della valle|crotone|crotta d'adda|crova|croviana|crucoli|cuasso al monte|cuccaro vetere|cucciago|cuceglio|cuggiono|cugliate-fabiasco|cuglieri|cugnoli|cumiana|cumignano sul naviglio|cunardo|cuneo|cunico|cuorgnรจ|cupello|cupra marittima|cupramontana|cura carpignano|curcuris|cureggio|curiglia con monteviasco|curinga|curino|curno|curon venosta|cursi|curtarolo|curtatone|curti|cusago|cusano milanino|cusano mutri|cusino|cusio|custonaci|cutro|cutrofiano|cuveglio|cuvio|dairago|dalmine|dambel|danta di cadore|darfo boario terme|dasร |davagna|daverio|davoli|dazio|decimomannu|decimoputzu|decollatura|dego|deiva marina|delebio|delia|delianuova|deliceto|dello|demonte|denice|denno|dernice|derovere|deruta|dervio|desana|desenzano del garda|desio|desulo|deutschnofen|devin nabreลพina|diamante|diano arentino|diano castello|diano d'alba|diano marina|diano san pietro|dicomano|dignano|dimaro folgarida|dinami|dipignano|diso|divignano|dizzasco|dobbiaco|doberdob|doberdรฒ del lago|dogliani|dogliola|dogna|dolceacqua|dolcedo|dolcรจ|dolegna del collio|dolianova|dolina|dolo|dolzago|domanico|domaso|domegge di cadore|domicella|domodossola|domus de maria|domusnovas|donato|dongo|donnas|donori|dorgali|dorio|dormelletto|dorno|dorzano|dosolo|dossena|dosso del liro|doues|dovadola|dovera|dozza|dragoni|drapia|drena|drenchia|dresano|dro|dronero|druento|druogno|dualchi|dubino|due carrare|dueville|dugenta|duino aurisina|dumenza|duno|durazzano|duronia|dusino san michele|eboli|edolo|egna|elice|elini|ello|elmas|elva|emarรจse|empoli|endine gaiano|enego|enemonzo|enna|enneberg|entracque|entratico|envie|episcopia|eppan an der weinstraรŸe|eraclea|erba|erbezzo|erbusco|erbรจ|erchie|ercolano|erice|erli|erto e casso|erula|erve|esanatoglia|escalaplano|escolca|esine|esino lario|esperia|esporlatu|este|esterzili|etroubles|eupilio|exilles|fabbrica curone|fabbriche di vergemoli|fabbrico|fabriano|fabrica di roma|fabrizia|fabro|faedis|faedo valtellino|faenza|faeto|fagagna|faggeto lario|faggiano|fagnano alto|fagnano castello|fagnano olona|fai della paganella|faicchio|falcade|falciano del massico|falconara albanese|falconara marittima|falcone|faleria|falerna|falerone|fallo|faloppio|falvaterra|falzes|fanano|fanna|fano|fano adriano|fara filiorum petri|fara gera d'adda|fara in sabina|fara novarese|fara olivana con sola|fara san martino|fara vicentino|fardella|farigliano|farindola|farini|farnese|farra d'isonzo|farra di soligo|fasano|fascia|fauglia|faule|favale di malvaro|favara|favignana|favria|feisoglio|feldthurns|feletto|felino|felitto|felizzano|feltre|fenegrรฒ|fenestrelle|ferentillo|ferentino|ferla|fermignano|fermo|ferno|feroleto antico|feroleto della chiesa|ferrandina|ferrara|ferrara di monte baldo|ferrazzano|ferrera di varese|ferrera erbognone|ferrere|ferriere|ferruzzano|fiamignano|fiano|fiano romano|fiastra|fiavรจ|ficarazzi|ficarolo|ficarra|ficulle|fidenza|fierozzo|fiesco|fiesole|fiesse|fiesso d'artico|fiesso umbertiano|figino serenza|figline e incisa valdarno|figline vegliaturo|filacciano|filadelfia|filago|filandari|filattiera|filettino|filetto|filiano|filighera|filignano|filogaso|filottrano|finale emilia|finale ligure|fino del monte|fino mornasco|fiorano al serio|fiorano canavese|fiorano modenese|fiorenzuola d'arda|firenze|firenzuola|firmo|fiscaglia|fisciano|fiuggi|fiumalbo|fiumara|fiume veneto|fiumedinisi|fiumefreddo bruzio|fiumefreddo di sicilia|fiumicello villa vicentina|fiumicino|fiuminata|fivizzano|fiรจ allo sciliar|flaibano|flero|floresta|floridia|florinas|flumeri|fluminimaggiore|flussio|fobello|foggia|foglianise|fogliano redipuglia|foglizzo|foiano della chiana|foiano di val fortore|folgaria|folignano|foligno|follina|follo|follonica|fombio|fondachelli-fantina|fondi|fonni|fontainemore|fontana liri|fontanafredda|fontanarosa|fontanelice|fontanella|fontanellato|fontanelle|fontaneto d'agogna|fontanetto po|fontanigorda|fontanile|fontaniva|fonte|fonte nuova|fontecchio|fontechiari|fontegreca|fonteno|fontevivo|fonzaso|foppolo|forano|force|forchia|forcola|fordongianus|forenza|foresto sparso|forgaria nel friuli|forino|forio|forlimpopoli|forlรฌ|forlรฌ del sannio|formazza|formello|formia|formicola|formigara|formigine|formigliana|fornace|fornelli|forni avoltri|forni di sopra|forni di sotto|forno canavese|fornovo di taro|fornovo san giovanni|forte dei marmi|fortezza|fortunago|forza d'agrรฒ|fosciandora|fosdinovo|fossa|fossacesia|fossalta di piave|fossalta di portogruaro|fossalto|fossano|fossato di vico|fossato serralta|fossombrone|fossรฒ|foza|frabosa soprana|frabosa sottana|fraconalto|fragagnano|fragneto l'abate|fragneto monforte|fraine|framura|francavilla al mare|francavilla angitola|francavilla bisio|francavilla d'ete|francavilla di sicilia|francavilla fontana|francavilla in sinni|francavilla marittima|francica|francofonte|francolise|franzensfeste|frascaro|frascarolo|frascati|frascineto|frassilongo|frassinelle polesine|frassinello monferrato|frassineto po|frassinetto|frassino|frassinoro|frasso sabino|frasso telesino|fratta polesine|fratta todina|frattamaggiore|frattaminore|fratte rosa|frazzanรฒ|fregona|freienfeld|fresagrandinaria|fresonara|frigento|frignano|frinco|frisa|frisanco|front|frontino|frontone|frosinone|frosolone|frossasco|frugarolo|fubine monferrato|fucecchio|fuipiano valle imagna|fumane|fumone|funes|furci|furci siculo|furnari|furore|furtei|fuscaldo|fusignano|fusine|futani|fรฉnis|gabbioneta-binanuova|gabiano|gabicce mare|gaby|gadesco-pieve delmona|gadoni|gaeta|gaggi|gaggiano|gaggio montano|gaglianico|gagliano aterno|gagliano castelferrato|gagliano del capo|gagliato|gagliole|gaiarine|gaiba|gaiola|gaiole in chianti|gairo|gais|galati mamertino|galatina|galatone|galatro|galbiate|galeata|galgagnano|gallarate|gallese|galliate|galliate lombardo|galliavola|gallicano|gallicano nel lazio|gallicchio|galliera|galliera veneta|gallinaro|gallio|gallipoli|gallo matese|gallodoro|galluccio|galtellรฌ|galzignano terme|gamalero|gambara|gambarana|gambasca|gambassi terme|gambatesa|gambellara|gamberale|gambettola|gambolรฒ|gandellino|gandino|gandosso|gangi|garaguso|garbagna|garbagna novarese|garbagnate milanese|garbagnate monastero|garda|gardone riviera|gardone val trompia|garessio|gargallo|gargazon|gargazzone|gargnano|garlasco|garlate|garlenda|garniga terme|garzeno|garzigliana|gasperina|gassino torinese|gattatico|gatteo|gattico-veruno|gattinara|gavardo|gavello|gaverina terme|gavi|gavignano|gavirate|gavoi|gavorrano|gazoldo degli ippoliti|gazzada schianno|gazzaniga|gazzo|gazzo veronese|gazzola|gazzuolo|gela|gemmano|gemona del friuli|gemonio|genazzano|genga|genivolta|genola|genoni|genova|genuri|genzano di lucania|genzano di roma|gera lario|gerace|geraci siculo|gerano|gerenzago|gerenzano|gergei|germagnano|germagno|germignaga|gerocarne|gerola alta|gerre de' caprioli|gesico|gessate|gessopalena|gesturi|gesualdo|ghedi|ghemme|ghiffa|ghilarza|ghisalba|ghislarengo|giacciano con baruchella|giaglione|gianico|giano dell'umbria|giano vetusto|giardinello|giardini-naxos|giarole|giarratana|giarre|giave|giaveno|giavera del montello|giba|gibellina|gifflenga|giffone|giffoni sei casali|giffoni valle piana|gignese|gignod|gildone|gimigliano|ginestra|ginestra degli schiavoni|ginosa|gioi|gioia dei marsi|gioia del colle|gioia sannitica|gioia tauro|gioiosa ionica|gioiosa marea|giove|giovinazzo|giovo|girasole|girifalco|gissi|giuggianello|giugliano in campania|giuliana|giuliano di roma|giuliano teatino|giulianova|giungano|giurdignano|giussago|giussano|giustenice|giustino|giusvalla|givoletto|gizzeria|glorenza|glurns|godega di sant'urbano|godiasco salice terme|godrano|goito|golasecca|golferenzo|golfo aranci|gombito|gonars|goni|gonnesa|gonnoscodina|gonnosfanadiga|gonnosnรฒ|gonnostramatza|gonzaga|gordona|gorga|gorgo al monticano|gorgoglione|gorgonzola|goriano sicoli|gorizia|gorla maggiore|gorla minore|gorlago|gorle|gornate olona|gorno|goro|gorreto|gorzegno|gosaldo|gossolengo|gottasecca|gottolengo|govone|gozzano|gradara|gradisca d'isonzo|grado|gradoli|graffignana|graffignano|graglia|gragnano|gragnano trebbiense|grammichele|grana monferrato|granarolo dell'emilia|grandate|grandola ed uniti|graniti|granozzo con monticello|grantola|grantorto|granze|grassano|grassobbio|gratteri|graun im vinschgau|gravedona ed uniti|gravellona lomellina|gravellona toce|gravere|gravina di catania|gravina in puglia|grazzanise|grazzano badoglio|greccio|greci|greggio|gremiasco|gressan|gressoney-la-trinitรฉ|gressoney-saint-jean|greve in chianti|grezzago|grezzana|griante|gricignano di aversa|grignasco|grigno|grimacco|grimaldi|grinzane cavour|grisignano di zocco|grisolia|grizzana morandi|grognardo|gromo|grondona|grone|grontardo|gropello cairoli|gropparello|groscavallo|grosio|grosotto|grosseto|grosso|grottaferrata|grottaglie|grottaminarda|grottammare|grottazzolina|grotte|grotte di castro|grotteria|grottole|grottolella|gruaro|grugliasco|grumello cremonese ed uniti|grumello del monte|grumento nova|grumo appula|grumo nevano|grumolo delle abbadesse|gsies|guagnano|gualdo|gualdo cattaneo|gualdo tadino|gualtieri|gualtieri sicaminรฒ|guamaggiore|guanzate|guarcino|guarda veneta|guardabosone|guardamiglio|guardavalle|guardea|guardia lombardi|guardia perticara|guardia piemontese|guardia sanframondi|guardiagrele|guardialfiera|guardiaregia|guardistallo|guarene|guasila|guastalla|guazzora|gubbio|gudo visconti|guglionesi|guidizzolo|guidonia montecelio|guiglia|guilmi|gurro|guspini|gussago|gussola|hafling|hรดne|idro|iglesias|igliano|ilbono|illasi|illorai|imbersago|imer|imola|imperia|impruneta|inarzo|incisa scapaccino|incudine|induno olona|ingria|innichen|intragna|introbio|introd|introdacqua|inverigo|inverno e monteleone|inverso pinasca|inveruno|invorio|inzago|irgoli|irma|irsina|isasca|isca sullo ionio|ischia|ischia di castro|ischitella|iseo|isera|isernia|isili|isnello|isola d'asti|isola del cantone|isola del giglio|isola del gran sasso d'italia|isola del liri|isola del piano|isola della scala|isola delle femmine|isola di capo rizzuto|isola di fondra|isola dovarese|isola rizza|isola sant'antonio|isola vicentina|isolabella|isolabona|isole tremiti|isorella|ispani|ispica|ispra|issiglio|issime|isso|issogne|istrana|itala|itri|ittireddu|ittiri|ivrea|izano|jacurso|jelsi|jenesien|jenne|jerago con orago|jerzu|jesi|jesolo|jolanda di savoia|jonadi|joppolo|joppolo giancaxio|jovenรงan|kaltern an der weinstraรŸe|karneid|kastelbell-tschars|kastelruth|kiens|klausen|kuens|kurtatsch an der weinstraรŸe|kurtinig an der weinstraรŸe|l'aquila|la cassa|la loggia|la maddalena|la magdeleine|la morra|la salle|la spezia|la thuile|la valle|la valle agordina|la valletta brianza|laas|labico|labro|lacchiarella|lacco ameno|lacedonia|laces|laconi|ladispoli|laerru|laganadi|laghi|laglio|lagnasco|lago|lagonegro|lagosanto|lagundo|laigueglia|lainate|laino|laino borgo|laino castello|laion|laives|lajatico|lajen|lallio|lama dei peligni|lama mocogno|lambrugo|lamezia terme|lamon|lampedusa e linosa|lamporecchio|lamporo|lana|lanciano|landiona|landriano|langhirano|langosco|lanusei|lanuvio|lanzada|lanzo torinese|lapedona|lapio|lappano|larciano|lardirago|lariano|larino|las plassas|lasa|lascari|lasnigo|lastebasse|lastra a signa|latera|laterina pergine valdarno|laterza|latiano|latina|latisana|latronico|latsch|lattarico|lauco|laureana cilento|laureana di borrello|lauregno|laurein|laurenzana|lauria|lauriano|laurino|laurito|lauro|lavagna|lavagno|lavarone|lavello|lavena ponte tresa|laveno-mombello|lavenone|laviano|lavis|lazise|lazzate|lecce|lecce nei marsi|lecco|ledro|leffe|leggiuno|legnago|legnano|legnaro|lei|leifers|leini|leivi|lemie|lendinara|leni|lenna|leno|lenola|lenta|lentate sul seveso|lentella|lentini|leonessa|leonforte|leporano|lequile|lequio berria|lequio tanaro|lercara friddi|lerici|lerma|lesa|lesegno|lesignano de' bagni|lesina|lesmo|lessolo|lessona|lestizza|letino|letojanni|lettere|lettomanoppello|lettopalena|levanto|levate|leverano|levice|levico terme|levone|lezzeno|liberi|librizzi|licata|licciana nardi|licenza|licodia eubea|lierna|lignana|lignano sabbiadoro|lillianes|limana|limatola|limbadi|limbiate|limena|limido comasco|limina|limone piemonte|limone sul garda|limosano|linarolo|linguaglossa|lioni|lipari|lipomo|liscate|liscia|lisciano niccone|lisio|lissone|liveri|livigno|livinallongo del col di lana|livo|livorno|livorno ferraris|livraga|lizzanello|lizzano|lizzano in belvedere|loano|loazzolo|locana|locate di triulzi|locate varesino|locatello|loceri|locorotondo|locri|loculi|lodi|lodi vecchio|lodine|lodrino|lodรจ|lograto|loiano|loiri porto san paolo|lomagna|lomazzo|lombardore|lombriasco|lomello|lona-lases|lonate ceppino|lonate pozzolo|lonato del garda|londa|longano|longare|longarone|longhena|longi|longiano|longobardi|longobucco|longone al segrino|longone sabino|lonigo|loranzรจ|loreggia|loreglia|lorenzago di cadore|loreo|loreto|loreto aprutino|loria|loro ciuffenna|loro piceno|lorsica|losine|lotzorai|lovere|lovero|lozio|lozza|lozzo atestino|lozzo di cadore|lozzolo|lu e cuccaro monferrato|lubriano|lucca|lucca sicula|lucera|lucignano|lucinasco|lucito|luco dei marsi|lucoli|lugagnano val d'arda|lugnano in teverina|lugo|lugo di vicenza|luino|luisago|lula|lumarzo|lumezzane|lunamatrona|lunano|lungavilla|lungro|luni|luogosano|luogosanto|lupara|lurago d'erba|lurago marinone|lurano|luras|lurate caccivio|lusciano|luserna|luserna san giovanni|lusernetta|lusevera|lusia|lusiana conco|lusigliรจ|luson|lustra|luvinate|luzzana|luzzara|luzzi|lรผsen|maccagno con pino e veddasca|maccastorna|macchia d'isernia|macchia valfortore|macchiagodena|macello|macerata|macerata campania|macerata feltria|macherio|maclodio|macomer|macra|macugnaga|maddaloni|madesimo|madignano|madone|madonna del sasso|madruzzo|maenza|mafalda|magasa|magenta|maggiora|magherno|magione|magisano|magliano alfieri|magliano alpi|magliano de' marsi|magliano di tenna|magliano in toscana|magliano romano|magliano sabina|magliano vetere|maglie|magliolo|maglione|magnacavallo|magnago|magnano|magnano in riviera|magomadas|magreglio|magrรจ sulla strada del vino|maida|maierato|maierร |maiolati spontini|maiolo|maiori|mairago|mairano|maissana|majano|malagnino|malalbergo|malborghetto valbruna|malcesine|malegno|maleo|malesco|maletto|malfa|malgrate|malito|mallare|malles venosta|malnate|malo|malonno|mals|maltignano|malvagna|malvicino|malvito|malรฉ|mammola|mamoiada|manciano|mandanici|mandas|mandatoriccio|mandela|mandello del lario|mandello vitta|manduria|manerba del garda|manerbio|manfredonia|mango|mangone|maniace|maniago|manocalzati|manoppello|mansuรจ|manta|mantello|mantova|manzano|manziana|mapello|mappano|mara|maracalagonis|maranello|marano di napoli|marano di valpolicella|marano equo|marano lagunare|marano marchesato|marano principato|marano sul panaro|marano ticino|marano vicentino|maranzana|maratea|marcallo con casone|marcaria|marcedusa|marcellina|marcellinara|marcetelli|marcheno|marchirolo|marciana|marciana marina|marcianise|marciano della chiana|marcignago|marcon|marebbe|marene|mareno di piave|marentino|maretto|margarita|margherita di savoia|margno|margreid an der weinstraรŸe|mariana mantovana|mariano comense|mariano del friuli|marianopoli|mariglianella|marigliano|marina di gioiosa ionica|marineo|marino|marlengo|marliana|marling|marmentino|marmirolo|marmora|marnate|marone|maropati|marostica|marradi|marrubiu|marsaglia|marsala|marsciano|marsico nuovo|marsicovetere|marta|martano|martell|martellago|martello|martignacco|martignana di po|martignano|martina franca|martinengo|martiniana po|martinsicuro|martirano|martirano lombardo|martis|martone|marudo|maruggio|marzabotto|marzano|marzano appio|marzano di nola|marzi|marzio|masainas|masate|mascali|mascalucia|maschito|masciago primo|maser|masera|maserada sul piave|maserร  di padova|masi|masi torello|masio|maslianico|masone|massa|massa d'albe|massa di somma|massa e cozzile|massa fermana|massa lombarda|massa lubrense|massa marittima|massa martana|massafra|massalengo|massanzago|massarosa|massazza|massello|masserano|massignano|massimeno|massimino|massino visconti|massiola|masullas|matelica|matera|mathi|matino|matrice|mattie|mattinata|mazara del vallo|mazzano|mazzano romano|mazzarino|mazzarrone|mazzarrร  sant'andrea|mazzin|mazzo di valtellina|mazzรจ|meana di susa|meana sardo|meda|mede|medea|medesano|medicina|mediglia|medolago|medole|medolla|meduna di livenza|meduno|megliadino san vitale|meina|melara|melazzo|meldola|mele|melegnano|melendugno|meleti|melfi|melicucco|melicuccร |melilli|melissa|melissano|melito di napoli|melito di porto salvo|melito irpino|melizzano|melle|mello|melpignano|meltina|melzo|menaggio|menconico|mendatica|mendicino|menfi|mentana|meolo|meran|merana|merano|merate|mercallo|mercatello sul metauro|mercatino conca|mercato san severino|mercato saraceno|mercenasco|mercogliano|mereto di tomba|mergo|mergozzo|merlara|merlino|merone|merรฌ|mesagne|mese|mesenzana|mesero|mesola|mesoraca|messina|mestrino|meta|mezzago|mezzana|mezzana bigli|mezzana mortigliengo|mezzana rabattone|mezzane di sotto|mezzanego|mezzanino|mezzano|mezzenile|mezzocorona|mezzojuso|mezzoldo|mezzolombardo|mezzomerico|miagliano|miane|miasino|miazzina|micigliano|miggiano|miglianico|miglierina|miglionico|mignanego|mignano monte lungo|milano|milazzo|milena|mileto|milis|militello in val di catania|militello rosmarino|millesimo|milo|milzano|mineo|minerbe|minerbio|minervino di lecce|minervino murge|minori|minturno|minucciano|mioglia|mira|mirabella eclano|mirabella imbaccari|mirabello monferrato|mirabello sannitico|miradolo terme|miranda|mirandola|mirano|mirto|misano adriatico|misano di gera d'adda|misiliscemi|misilmeri|misinto|missaglia|missanello|misterbianco|mistretta|moasca|moconesi|modena|modica|modigliana|modolo|modugno|moena|moggio|moggio udinese|moglia|mogliano|mogliano veneto|mogorella|mogoro|moiano|moimacco|moio alcantara|moio de' calvi|moio della civitella|moiola|mola di bari|molare|molazzana|molfetta|molina aterno|molinara|molinella|molini di triora|molino dei torti|molise|moliterno|mollia|molochio|molteno|moltrasio|molveno|mombaldone|mombarcaro|mombaroccio|mombaruzzo|mombasiglio|mombello di torino|mombello monferrato|mombercelli|momo|mompantero|mompeo|momperone|monacilioni|monale|monasterace|monastero bormida|monastero di lanzo|monastero di vasco|monasterolo casotto|monasterolo del castello|monasterolo di savigliano|monastier di treviso|monastir|moncalieri|moncalvo|moncenisio|moncestino|monchiero|monchio delle corti|moncrivello|moncucco torinese|mondaino|mondavio|mondolfo|mondovรฌ|mondragone|moneglia|monesiglio|monfalcone|monforte d'alba|monforte san giorgio|monfumo|mongardino|monghidoro|mongiana|mongiardino ligure|mongiuffi melia|mongrando|mongrassano|monguelfo-tesido|monguzzo|moniga del garda|monleale|monno|monopoli|monreale|monrupino|monsampietro morico|monsampolo del tronto|monsano|monselice|monserrato|monsummano terme|montabone|montacuto|montafia|montagano|montagna in valtellina|montagna sulla strada del vino|montagnana|montagnareale|montaguto|montaione|montalbano elicona|montalbano jonico|montalcino|montaldeo|montaldo bormida|montaldo di mondovรฌ|montaldo roero|montaldo scarampi|montaldo torinese|montale|montalenghe|montallegro|montalto carpasio|montalto delle marche|montalto di castro|montalto dora|montalto pavese|montalto uffugo|montan an der weinstraรŸe|montanaro|montanaso lombardo|montanera|montano antilia|montano lucino|montappone|montaquila|montasola|montauro|montazzoli|monte argentario|monte castello di vibio|monte cavallo|monte cerignone|monte compatri|monte cremasco|monte di malo|monte di procida|monte giberto|monte grimano terme|monte isola|monte marenzo|monte porzio|monte porzio catone|monte rinaldo|monte roberto|monte romano|monte san biagio|monte san giacomo|monte san giovanni campano|monte san giovanni in sabina|monte san giusto|monte san martino|monte san pietrangeli|monte san pietro|monte san savino|monte san vito|monte sant'angelo|monte santa maria tiberina|monte urano|monte vidon combatte|monte vidon corrado|montebello della battaglia|montebello di bertona|montebello jonico|montebello sul sangro|montebello vicentino|montebelluna|montebruno|montebuono|montecalvo in foglia|montecalvo irpino|montecalvo versiggia|montecarlo|montecarotto|montecassiano|montecastello|montecastrilli|montecatini val di cecina|montecatini-terme|montecchia di crosara|montecchio|montecchio emilia|montecchio maggiore|montecchio precalcino|montechiaro d'acqui|montechiaro d'asti|montechiarugolo|montecilfone|montecopiolo|montecorice|montecorvino pugliano|montecorvino rovella|montecosaro|montecrestese|montecreto|montedinove|montedoro|montefalcione|montefalco|montefalcone appennino|montefalcone di val fortore|montefalcone nel sannio|montefano|montefelcino|monteferrante|montefiascone|montefino|montefiore conca|montefiore dell'aso|montefiorino|monteflavio|monteforte cilento|monteforte d'alpone|monteforte irpino|montefortino|montefranco|montefredane|montefusco|montegabbione|montegalda|montegaldella|montegallo|montegioco|montegiordano|montegiorgio|montegranaro|montegridolfo|montegrino valtravaglia|montegrosso d'asti|montegrosso pian latte|montegrotto terme|monteiasi|montelabbate|montelanico|montelapiano|monteleone d'orvieto|monteleone di fermo|monteleone di puglia|monteleone di spoleto|monteleone rocca doria|monteleone sabino|montelepre|montelibretti|montella|montello|montelongo|montelparo|montelupo albese|montelupo fiorentino|montelupone|montemaggiore belsito|montemagno monferrato|montemale di cuneo|montemarano|montemarciano|montemarzino|montemesola|montemezzo|montemignaio|montemiletto|montemilone|montemitro|montemonaco|montemurlo|montemurro|montenars|montenero di bisaccia|montenero sabino|montenero val cocchiara|montenerodomo|monteodorisio|montepaone|monteparano|monteprandone|montepulciano|monterchi|montereale|montereale valcellina|monterenzio|monteriggioni|monteroduni|monteroni d'arbia|monteroni di lecce|monterosi|monterosso al mare|monterosso almo|monterosso calabro|monterosso grana|monterotondo|monterotondo marittimo|monterubbiano|montesano salentino|montesano sulla marcellana|montesarchio|montescaglioso|montescano|montescheno|montescudaio|montescudo-monte colombo|montese|montesegale|montesilvano|montespertoli|monteu da po|monteu roero|montevago|montevarchi|montevecchia|monteverde|monteverdi marittimo|monteviale|montezemolo|monti|montiano|monticelli brusati|monticelli d'ongina|monticelli pavese|monticello brianza|monticello conte otto|monticello d'alba|montichiari|monticiano|montieri|montiglio monferrato|montignoso|montirone|montjovet|montodine|montoggio|montone|montopoli di sabina|montopoli in val d'arno|montorfano|montorio al vomano|montorio nei frentani|montorio romano|montoro|montorso vicentino|montottone|montresta|montร |montรน beccaria|monvalle|monza|monzambano|monzuno|moos in passeier|morano calabro|morano sul po|moransengo-tonengo|moraro|morazzone|morbegno|morbello|morciano di leuca|morciano di romagna|morcone|mordano|morengo|mores|moresco|moretta|morfasso|morgano|morgex|morgongiori|mori|moriago della battaglia|moricone|morigerati|morimondo|morino|moriondo torinese|morlupo|mormanno|mornago|mornese|mornico al serio|mornico losana|morolo|morozzo|morra de sanctis|morro d'alba|morro d'oro|morro reatino|morrone del sannio|morrovalle|morsano al tagliamento|morsasco|mortara|mortegliano|morterone|moruzzo|moscazzano|moschiano|mosciano sant'angelo|moscufo|moso in passiria|mossa|motta baluffi|motta camastra|motta d'affermo|motta de' conti|motta di livenza|motta montecorvino|motta san giovanni|motta sant'anastasia|motta santa lucia|motta visconti|mottafollone|mottalciata|motteggiana|mottola|mozzagrogna|mozzanica|mozzate|mozzecane|mozzo|muccia|muggia|muggiรฒ|mugnano del cardinale|mugnano di napoli|mulazzano|mulazzo|mura|muravera|murazzano|murello|murialdo|murisengo monferrato|murlo|muro leccese|muro lucano|muros|muscoline|musei|musile di piave|musso|mussolente|mussomeli|muzzana del turgnano|muzzano|mรถlten|mรผhlbach|mรผhlwald|nago-torbole|nalles|nals|napoli|narbolia|narcao|nardodipace|nardรฒ|narni|naro|narzole|nasino|naso|naturno|naturns|natz-schabs|nave|navelli|naz-sciaves|nazzano|ne|nebbiuno|negrar di valpolicella|neirone|neive|nembro|nemi|nemoli|neoneli|nepi|nereto|nerola|nervesa della battaglia|nerviano|nespolo|nesso|netro|nettuno|neumarkt|neviano|neviano degli arduini|neviglie|niardo|nibbiola|nibionno|nichelino|nicolosi|nicorvo|nicosia|nicotera|niederdorf|niella belbo|niella tanaro|nimis|niscemi|nissoria|nizza di sicilia|nizza monferrato|noale|noasca|nocara|nocciano|nocera inferiore|nocera superiore|nocera terinese|nocera umbra|noceto|noci|nociglia|noepoli|nogara|nogaredo|nogarole rocca|nogarole vicentino|noicattaro|nola|nole|noli|nomaglio|nomi|nonantola|none|nonio|noragugume|norbello|norcia|norma|nosate|notaresco|noto|nova levante|nova milanese|nova ponente|nova siri|novafeltria|novaledo|novalesa|novara|novara di sicilia|novate mezzola|novate milanese|nove|novedrate|novella|novellara|novello|noventa di piave|noventa padovana|noventa vicentina|novi di modena|novi ligure|novi velia|noviglio|novoli|nucetto|nughedu san nicolรฒ|nughedu santa vittoria|nule|nulvi|numana|nuoro|nurachi|nuragus|nurallao|nuraminis|nureci|nurri|nus|nusco|nuvolento|nuvolera|nuxis|occhieppo inferiore|occhieppo superiore|occhiobello|occimiano|ocre|odalengo grande|odalengo piccolo|oderzo|odolo|ofena|offagna|offanengo|offida|offlaga|oggebbio|oggiona con santo stefano|oggiono|oglianico|ogliastro cilento|olang|olbia|olcenengo|oldenico|oleggio|oleggio castello|olevano di lomellina|olevano romano|olevano sul tusciano|olgiate comasco|olgiate molgora|olgiate olona|olginate|oliena|oliva gessi|olivadi|oliveri|oliveto citra|oliveto lario|oliveto lucano|olivetta san michele|olivola|ollastra|ollolai|ollomont|olmedo|olmeneta|olmo al brembo|olmo gentile|oltre il colle|oltressenda alta|oltrona di san mamette|olzai|ome|omegna|omignano|onano|onanรฌ|oncino|oneta|onifai|oniferi|ono san pietro|onore|onzo|opera|opi|oppeano|oppido lucano|oppido mamertina|ora|orani|oratino|orbassano|orbetello|orciano pisano|orco feglino|ordona|orero|orgiano|orgosolo|oria|oricola|origgio|orino|orio al serio|orio canavese|orio litta|oriolo|oriolo romano|oristano|ormea|ormelle|ornago|ornavasso|ornica|orosei|orotelli|orria|orroli|orsago|orsara bormida|orsara di puglia|orsenigo|orsogna|orsomarso|orta di atella|orta nova|orta san giulio|ortacesus|orte|ortelle|ortezzano|ortignano raggiolo|ortisei|ortona|ortona dei marsi|ortovero|ortucchio|ortueri|orune|orvieto|orvinio|orzinuovi|orzivecchi|osasco|osasio|oschiri|osidda|osiglia|osilo|osimo|osini|osio sopra|osio sotto|osnago|osoppo|ospedaletti|ospedaletto|ospedaletto d'alpinolo|ospedaletto euganeo|ospedaletto lodigiano|ospitale di cadore|ospitaletto|ossago lodigiano|ossana|ossi|ossimo|ossona|ostana|ostellato|ostiano|ostiglia|ostra|ostra vetere|ostuni|otranto|otricoli|ottana|ottati|ottaviano|ottiglio|ottobiano|ottone|oulx|ovada|ovaro|oviglio|ovindoli|ovodda|oyace|ozegna|ozieri|ozzano dell'emilia|ozzano monferrato|ozzero|pabillonis|pace del mela|paceco|pacentro|pachino|paciano|padenghe sul garda|paderna|paderno d'adda|paderno dugnano|paderno franciacorta|paderno ponchielli|padova|padria|padru|padula|paduli|paesana|paese|pagani|paganico sabino|pagazzano|pagliara|paglieta|pagnacco|pagno|pagnona|pago del vallo di lauro|pago veiano|paisco loveno|paitone|paladina|palagano|palagianello|palagiano|palagonia|palaia|palanzano|palata|palau|palazzago|palazzo adriano|palazzo canavese|palazzo pignano|palazzo san gervasio|palazzolo acreide|palazzolo dello stella|palazzolo sull'oglio|palazzolo vercellese|palazzuolo sul senio|palena|palermiti|palermo|palestrina|palestro|paliano|palizzi|pallagorio|pallanzeno|pallare|palma campania|palma di montechiaro|palmanova|palmariggi|palmas arborea|palmi|palmiano|palmoli|palo del colle|palombara sabina|palombaro|palomonte|palosco|paludi|paluzza|palรน|palรน del fersina|pamparato|pancalieri|pancarana|panchiร |pandino|panettieri|panicale|pannarano|panni|pantelleria|pantigliate|paola|paolisi|papasidero|papozze|parabiago|parabita|paratico|parcines|parella|parenti|parete|pareto|parghelia|parlasco|parma|parodi ligure|paroldo|parolise|parona|parrano|parre|partanna|partinico|partschins|paruzzaro|parzanica|pasian di prato|pasiano di pordenone|paspardo|passerano marmorito|passignano sul trasimeno|passirano|pastena|pastorano|pastrengo|pasturana|pasturo|paterno|paterno calabro|paternopoli|paternรฒ|patrica|pattada|patti|patรน|pau|paularo|pauli arbarei|paulilatino|paullo|paupisi|pavarolo|pavia|pavia di udine|pavone canavese|pavone del mella|pavullo nel frignano|pazzano|peccioli|pecetto di valenza|pecetto torinese|pedara|pedaso|pedavena|pedemonte|pederobba|pedesina|pedivigliano|pedrengo|peglio|pegognaga|peia|peio|pelago|pella|pellegrino parmense|pellezzano|pellizzano|pelugo|penango|penna in teverina|penna san giovanni|penna sant'andrea|pennabilli|pennadomo|pennapiedimonte|penne|pentone|perano|perarolo di cadore|perca|percha|percile|perdasdefogu|perdaxius|perdifumo|pereto|perfugas|pergine valsugana|pergola|perinaldo|perito|perledo|perletto|perlo|perloz|pernumia|pero|perosa argentina|perosa canavese|perrero|persico dosimo|pertengo|pertica alta|pertica bassa|pertosa|pertusio|perugia|pesaro|pescaglia|pescantina|pescara|pescarolo ed uniti|pescasseroli|pescate|pesche|peschici|peschiera borromeo|peschiera del garda|pescia|pescina|pesco sannita|pescocostanzo|pescolanciano|pescopagano|pescopennataro|pescorocchiano|pescosansonesco|pescosolido|pessano con bornago|pessina cremonese|pessinetto|petacciato|petilia policastro|petina|petralia soprana|petralia sottana|petrella salto|petrella tifernina|petriano|petriolo|petritoli|petrizzi|petronร |petrosino|petruro irpino|pettenasco|pettinengo|pettineo|pettoranello del molise|pettorano sul gizio|pettorazza grimani|peveragno|pezzana|pezzaze|pezzolo valle uzzone|pfalzen|pfatten|pfitsch|piacenza|piacenza d'adige|piadena drizzona|piaggine|pian camuno|piana crixia|piana degli albanesi|piana di monte verna|piancastagnaio|piancogno|piandimeleto|piane crati|pianella|pianello del lario|pianello val tidone|pianengo|pianezza|pianezze|pianfei|pianico|pianiga|piano di sorrento|pianopoli|pianoro|piansano|piantedo|piario|piasco|piateda|piatto|piazza al serchio|piazza armerina|piazza brembana|piazzatorre|piazzola sul brenta|piazzolo|picciano|picerno|picinisco|pico|piea|piedicavallo|piedimonte etneo|piedimonte matese|piedimonte san germano|piedimulera|piegaro|pienza|pieranica|pietra de' giorgi|pietra ligure|pietra marazzi|pietrabbondante|pietrabruna|pietracamela|pietracatella|pietracupa|pietradefusi|pietraferrazzana|pietrafitta|pietragalla|pietralunga|pietramelara|pietramontecorvino|pietranico|pietrapaola|pietrapertosa|pietraperzia|pietraporzio|pietraroja|pietrarubbia|pietrasanta|pietrastornina|pietravairano|pietrelcina|pieve a nievole|pieve albignola|pieve d'olmi|pieve del cairo|pieve del grappa|pieve di bono-prezzo|pieve di cadore|pieve di cento|pieve di soligo|pieve di teco|pieve emanuele|pieve fissiraga|pieve fosciana|pieve ligure|pieve porto morone|pieve san giacomo|pieve santo stefano|pieve tesino|pieve torina|pieve vergonte|pievepelago|piglio|pigna|pignataro interamna|pignataro maggiore|pignola|pignone|pigra|pila|pimentel|pimonte|pinarolo po|pinasca|pincara|pinerolo|pineto|pino d'asti|pino torinese|pinzano al tagliamento|pinzolo|piobbico|piobesi d'alba|piobesi torinese|piode|pioltello|piombino|piombino dese|pioraco|piossasco|piove di sacco|piovene rocchette|piovร  massaia|piozzano|piozzo|piraino|pisa|pisano|piscina|piscinas|pisciotta|pisogne|pisoniano|pisticci|pistoia|pitigliano|piubega|piuro|piverone|pizzale|pizzighettone|pizzo|pizzoferrato|pizzoli|pizzone|pizzoni|placanica|plataci|platania|platรฌ|plaus|plesio|ploaghe|plodio|pocapaglia|pocenia|podenzana|podenzano|pofi|poggiardo|poggibonsi|poggio a caiano|poggio bustone|poggio catino|poggio imperiale|poggio mirteto|poggio moiano|poggio nativo|poggio picenze|poggio renatico|poggio rusco|poggio san lorenzo|poggio san marcello|poggio san vicino|poggio sannita|poggio torriana|poggiodomo|poggiofiorito|poggiomarino|poggioreale|poggiorsini|poggiridenti|pogliano milanese|pognana lario|pognano|pogno|poirino|pojana maggiore|polaveno|polcenigo|polesella|polesine zibello|poli|polia|policoro|polignano a mare|polinago|polino|polistena|polizzi generosa|polla|pollein|pollena trocchia|pollenza|pollica|pollina|pollone|pollutri|polonghera|polpenazze del garda|polverara|polverigi|pomarance|pomaretto|pomarico|pomaro monferrato|pomarolo|pombia|pomezia|pomigliano d'arco|pompei|pompeiana|pompiano|pomponesco|pompu|poncarale|ponderano|ponna|ponsacco|ponso|pont canavese|pont-saint-martin|pontassieve|pontboset|ponte|ponte buggianese|ponte dell'olio|ponte di legno|ponte di piave|ponte gardena|ponte in valtellina|ponte lambro|ponte nelle alpi|ponte nizza|ponte nossa|ponte san nicolรฒ|ponte san pietro|pontebba|pontecagnano faiano|pontecchio polesine|pontechianale|pontecorvo|pontecurone|pontedassio|pontedera|pontelandolfo|pontelatone|pontelongo|pontenure|ponteranica|pontestura|pontevico|pontey|ponti|ponti sul mincio|pontida|pontinia|pontinvrea|pontirolo nuovo|pontoglio|pontremoli|ponza|ponzano di fermo|ponzano monferrato|ponzano romano|ponzano veneto|ponzone|popoli terme|poppi|porano|porcari|porcia|pordenone|porlezza|pornassio|porpetto|portacomaro|portalbera|porte|porte di rendena|portici|portico di caserta|portico e san benedetto|portigliola|porto azzurro|porto ceresio|porto cesareo|porto empedocle|porto mantovano|porto recanati|porto san giorgio|porto sant'elpidio|porto tolle|porto torres|porto valtravaglia|porto viro|portobuffolรจ|portocannone|portoferraio|portofino|portogruaro|portomaggiore|portopalo di capo passero|portoscuso|portovenere|portula|posada|posina|positano|possagno|posta|posta fibreno|postal|postalesio|postiglione|postua|potenza|potenza picena|pove del grappa|povegliano|povegliano veronese|poviglio|povoletto|pozzaglia sabina|pozzaglio ed uniti|pozzallo|pozzilli|pozzo d'adda|pozzol groppo|pozzolengo|pozzoleone|pozzolo formigaro|pozzomaggiore|pozzonovo|pozzuoli|pozzuolo del friuli|pozzuolo martesana|prad am stilfserjoch|pradalunga|pradamano|pradleves|pragelato|prags|praia a mare|praiano|pralboino|prali|pralormo|pralungo|pramaggiore|pramollo|prarolo|prarostino|prasco|prascorsano|prata camportaccio|prata d'ansidonia|prata di pordenone|prata di principato ultra|prata sannita|pratella|pratiglione|prato|prato allo stelvio|prato carnico|prato sesia|pratola peligna|pratola serra|pratovecchio stia|pravisdomini|pray|prazzo|precenicco|preci|predaia|predappio|predazzo|predoi|predore|predosa|preganziol|pregnana milanese|prelร |premana|premariacco|premeno|premia|premilcuore|premolo|premosello-chiovenda|preone|prepotto|preseglie|presenzano|presezzo|presicce-acquarica|pressana|pretoro|prettau|prevalle|prezza|priero|prignano cilento|prignano sulla secchia|primaluna|primiero san martino di castrozza|priocca|priola|priolo gargallo|priverno|prizzi|proceno|procida|propata|proserpio|prossedi|provaglio d'iseo|provaglio val sabbia|proveis|proves|provvidenti|prunetto|prรฉ-saint-didier|puegnago del garda|puglianello|pula|pulfero|pulsano|pumenengo|pusiano|putifigari|putignano|quadrelle|quadri|quagliuzzo|qualiano|quaranti|quaregna cerreto|quargnento|quarna sopra|quarna sotto|quarona|quarrata|quart|quarto|quarto d'altino|quartu sant'elena|quartucciu|quassolo|quattordio|quattro castella|quiliano|quincinetto|quindici|quingentole|quintano|quinto di treviso|quinto vercellese|quinto vicentino|quinzano d'oglio|quistello|rabbi|racale|racalmuto|racconigi|raccuja|racines|radda in chianti|raddusa|radicofani|radicondoli|raffadali|ragalna|ragogna|ragusa|raiano|ramacca|rancio valcuvia|ranco|randazzo|ranica|ranzanico|ranzo|rapagnano|rapallo|rapino|rapolano terme|rapolla|rapone|rasen-antholz|rassa|rasun-anterselva|rasura|ratschings|ravanusa|ravarino|ravascletto|ravello|ravenna|raveo|raviscanina|re|rea|realmonte|reana del rojale|reano|recale|recanati|recco|recetto|recoaro terme|redavalle|redondesco|refrancore|refrontolo|regalbuto|reggello|reggio di calabria|reggio nell'emilia|reggiolo|reino|reitano|remanzacco|remedello|renate|rende|renon|repentabor|resana|rescaldina|resia|resiutta|resuttano|retorbido|revello|revigliasco d'asti|revine lago|rezzago|rezzato|rezzo|rezzoaglio|rho|rhรชmes-notre-dame|rhรชmes-saint-georges|riace|rialto|riano|riardo|ribera|ribordone|ricadi|ricaldone|riccia|riccione|riccรฒ del golfo di spezia|ricengo|ricigliano|riese pio x|riesi|rieti|riffian|rifiano|rifreddo|rignano flaminio|rignano garganico|rignano sull'arno|rigolato|rimella|rimini|rio|rio di pusteria|rio saliceto|riofreddo|riola sardo|riolo terme|riolunato|riomaggiore|rionero in vulture|rionero sannitico|ripa teatina|ripabottoni|ripacandida|ripalimosani|ripalta arpina|ripalta cremasca|ripalta guerina|riparbella|ripatransone|ripe san ginesio|ripi|riposto|rittana|ritten|riva del garda|riva del po|riva di solto|riva ligure|riva presso chieri|rivalba|rivalta bormida|rivalta di torino|rivamonte agordino|rivanazzano terme|rivara|rivarolo canavese|rivarolo del re ed uniti|rivarolo mantovano|rivarone|rivarossa|rive|rive d'arcano|rivello|rivergaro|rivignano teor|rivisondoli|rivodutri|rivoli|rivoli veronese|rivolta d'adda|rizziconi|roana|roaschia|roascio|roasio|roatto|robassomero|robbiate|robbio|robecchetto con induno|robecco d'oglio|robecco pavese|robecco sul naviglio|robella|robilante|roburent|rocca canavese|rocca canterano|rocca cigliรจ|rocca d'arazzo|rocca d'arce|rocca d'evandro|rocca de' baldi|rocca de' giorgi|rocca di botte|rocca di cambio|rocca di cave|rocca di mezzo|rocca di neto|rocca di papa|rocca grimalda|rocca imperiale|rocca massima|rocca pia|rocca pietore|rocca priora|rocca san casciano|rocca san felice|rocca san giovanni|rocca santa maria|rocca santo stefano|rocca sinibalda|rocca susella|roccabascerana|roccabernarda|roccabianca|roccabruna|roccacasale|roccadaspide|roccafiorita|roccafluvione|roccaforte del greco|roccaforte ligure|roccaforte mondovรฌ|roccaforzata|roccafranca|roccagiovine|roccagloriosa|roccagorga|roccalbegna|roccalumera|roccamandolfi|roccamena|roccamonfina|roccamontepiano|roccamorice|roccanova|roccantica|roccapalumba|roccapiemonte|roccarainola|roccaraso|roccaromana|roccascalegna|roccasecca|roccasecca dei volsci|roccasicura|roccasparvera|roccaspinalveti|roccastrada|roccavaldina|roccaverano|roccavignale|roccavione|roccavivara|roccella ionica|roccella valdemone|rocchetta a volturno|rocchetta belbo|rocchetta di vara|rocchetta e croce|rocchetta ligure|rocchetta nervina|rocchetta palafea|rocchetta sant'antonio|rocchetta tanaro|rodano|roddi|roddino|rodello|rodeneck|rodengo|rodengo saiano|rodero|rodi garganico|rodigo|rodรฌ milici|rofrano|rogeno|roggiano gravina|roghudi|rogliano|rognano|rogno|rogolo|roiate|roio del sangro|roisan|roletto|rolo|roma|romagnano al monte|romagnano sesia|romagnese|romana|romanengo|romano canavese|romano d'ezzelino|romano di lombardia|romans d'isonzo|rombiolo|romeno|romentino|rometta|roncade|roncadelle|roncaro|roncegno terme|roncello|ronchi dei legionari|ronchi valsugana|ronchis|ronciglione|ronco all'adige|ronco biellese|ronco briantino|ronco canavese|ronco scrivia|roncobello|roncoferraro|roncofreddo|roncola|roncร |rondanina|rondissone|ronsecco|ronzo-chienis|ronzone|roppolo|rorร |rosarno|rosasco|rosate|rosazza|rosciano|roscigno|rose|rosello|roseto capo spulico|roseto degli abruzzi|roseto valfortore|rosignano marittimo|rosignano monferrato|rosolina|rosolini|rosora|rossa|rossana|rossano veneto|rossiglione|rosta|rosร |rota d'imagna|rota greca|rotella|rotello|rotonda|rotondella|rotondi|rottofreno|rotzo|roure|rovasenda|rovato|rovegno|rovellasca|rovello porro|roverbella|roverchiara|roveredo di guร |roveredo in piano|rovereto|roverรจ della luna|roverรจ veronese|rovescala|rovetta|roviano|rovigo|rovito|rovolon|rozzano|roรจ volciano|rubano|rubiana|rubiera|ruda|rudiano|rueglio|ruffano|ruffia|ruffrรจ-mendola|rufina|ruinas|rumo|ruoti|russi|rutigliano|rutino|ruviano|ruvo del monte|ruvo di puglia|sabaudia|sabbio chiese|sabbioneta|sacco|saccolongo|sacile|sacrofano|sadali|sagama|sagliano micca|sagrado|sagron mis|saint-christophe|saint-denis|saint-marcel|saint-nicolas|saint-oyen|saint-pierre|saint-rhรฉmy-en-bosses|saint-vincent|sala baganza|sala biellese|sala bolognese|sala comacina|sala consilina|sala monferrato|salandra|salaparuta|salara|salasco|salassa|salbertrand|salcedo|salcito|sale|sale delle langhe|sale marasino|sale san giovanni|salemi|salento|salerano canavese|salerano sul lambro|salerno|salgareda|sali vercellese|salice salentino|saliceto|salisano|salizzole|salle|salmour|salorno sulla strada del vino|salsomaggiore terme|saltrio|saludecio|saluggia|salurn an der weinstraรŸe|salussola|saluzzo|salve|salvirola|salvitelle|salza di pinerolo|salza irpina|salzano|salรฒ|samarate|samassi|samatzai|sambuca di sicilia|sambuca pistoiese|sambuci|sambuco|sammichele di bari|samo|samolaco|samone|sampeyre|samugheo|san bartolomeo al mare|san bartolomeo in galdo|san bartolomeo val cavargna|san basile|san basilio|san bassano|san bellino|san benedetto belbo|san benedetto dei marsi|san benedetto del tronto|san benedetto in perillis|san benedetto po|san benedetto ullano|san benedetto val di sambro|san benigno canavese|san bernardino verbano|san biagio della cima|san biagio di callalta|san biagio platani|san biagio saracinisco|san biase|san bonifacio|san buono|san calogero|san candido|san canzian d'isonzo|san carlo canavese|san casciano dei bagni|san casciano in val di pesa|san cassiano|san cataldo|san cesareo|san cesario di lecce|san cesario sul panaro|san chirico nuovo|san chirico raparo|san cipirello|san cipriano d'aversa|san cipriano picentino|san cipriano po|san clemente|san colombano al lambro|san colombano belmonte|san colombano certenoli|san cono|san cosmo albanese|san costantino albanese|san costantino calabro|san costanzo|san cristoforo|san damiano al colle|san damiano d'asti|san damiano macra|san daniele del friuli|san daniele po|san demetrio corone|san demetrio ne' vestini|san didero|san donaci|san donato di lecce|san donato di ninea|san donato milanese|san donato val di comino|san donร  di piave|san dorligo della valle|san fele|san felice a cancello|san felice circeo|san felice del benaco|san felice del molise|san felice sul panaro|san ferdinando|san ferdinando di puglia|san fermo della battaglia|san fili|san filippo del mela|san fior|san fiorano|san floriano del collio|san floro|san francesco al campo|san fratello|san gavino monreale|san gemini|san genesio atesino|san genesio ed uniti|san gennaro vesuviano|san germano chisone|san germano vercellese|san gervasio bresciano|san giacomo degli schiavoni|san giacomo delle segnate|san giacomo filippo|san giacomo vercellese|san gillio|san gimignano|san ginesio|san giorgio a cremano|san giorgio a liri|san giorgio albanese|san giorgio bigarello|san giorgio canavese|san giorgio del sannio|san giorgio della richinvelda|san giorgio delle pertiche|san giorgio di lomellina|san giorgio di nogaro|san giorgio di piano|san giorgio in bosco|san giorgio ionico|san giorgio la molara|san giorgio lucano|san giorgio monferrato|san giorgio morgeto|san giorgio piacentino|san giorgio scarampi|san giorgio su legnano|san giorio di susa|san giovanni a piro|san giovanni al natisone|san giovanni bianco|san giovanni del dosso|san giovanni di fassa|san giovanni di gerace|san giovanni gemini|san giovanni ilarione|san giovanni in croce|san giovanni in fiore|san giovanni in galdo|san giovanni in marignano|san giovanni in persiceto|san giovanni incarico|san giovanni la punta|san giovanni lipioni|san giovanni lupatoto|san giovanni rotondo|san giovanni suergiu|san giovanni teatino|san giovanni valdarno|san giuliano del sannio|san giuliano di puglia|san giuliano milanese|san giuliano terme|san giuseppe jato|san giuseppe vesuviano|san giustino|san giusto canavese|san godenzo|san gregorio d'ippona|san gregorio da sassola|san gregorio di catania|san gregorio magno|san gregorio matese|san gregorio nelle alpi|san lazzaro di savena|san leo|san leonardo|san leonardo in passiria|san leucio del sannio|san lorenzello|san lorenzo|san lorenzo al mare|san lorenzo bellizzi|san lorenzo del vallo|san lorenzo di sebato|san lorenzo dorsino|san lorenzo in campo|san lorenzo isontino|san lorenzo maggiore|san lorenzo nuovo|san luca|san lucido|san lupo|san mango d'aquino|san mango piemonte|san mango sul calore|san marcellino|san marcello|san marcello piteglio|san marco argentano|san marco d'alunzio|san marco dei cavoti|san marco evangelista|san marco in lamis|san marco la catola|san martino al tagliamento|san martino alfieri|san martino buon albergo|san martino canavese|san martino d'agri|san martino dall'argine|san martino del lago|san martino di finita|san martino di lupari|san martino di venezze|san martino in badia|san martino in passiria|san martino in pensilis|san martino in rio|san martino in strada|san martino sannita|san martino siccomario|san martino sulla marrucina|san martino valle caudina|san marzano di san giuseppe|san marzano oliveto|san marzano sul sarno|san massimo|san maurizio canavese|san maurizio d'opaglio|san mauro castelverde|san mauro cilento|san mauro di saline|san mauro forte|san mauro la bruca|san mauro marchesato|san mauro pascoli|san mauro torinese|san michele al tagliamento|san michele all'adige|san michele di ganzaria|san michele di serino|san michele mondovรฌ|san michele salentino|san miniato|san nazzaro|san nazzaro sesia|san nazzaro val cavargna|san nicandro garganico|san nicola arcella|san nicola baronia|san nicola da crissa|san nicola dell'alto|san nicola la strada|san nicola manfredi|san nicolรฒ d'arcidano|san nicolรฒ di comelico|san nicolรฒ gerrei|san pancrazio|san pancrazio salentino|san paolo|san paolo albanese|san paolo bel sito|san paolo d'argon|san paolo di civitate|san paolo di jesi|san paolo solbrito|san pellegrino terme|san pier d'isonzo|san pier niceto|san piero patti|san pietro a maida|san pietro al natisone|san pietro al tanagro|san pietro apostolo|san pietro avellana|san pietro clarenza|san pietro di cadore|san pietro di caridร |san pietro di feletto|san pietro di morubio|san pietro in amantea|san pietro in cariano|san pietro in casale|san pietro in cerro|san pietro in gu|san pietro in guarano|san pietro in lama|san pietro infine|san pietro mosezzo|san pietro mussolino|san pietro val lemina|san pietro vernotico|san pietro viminario|san pio delle camere|san polo d'enza|san polo dei cavalieri|san polo di piave|san polo matese|san ponso|san possidonio|san potito sannitico|san potito ultra|san prisco|san procopio|san prospero|san quirico d'orcia|san quirino|san raffaele cimena|san roberto|san rocco al porto|san romano in garfagnana|san rufo|san salvatore di fitalia|san salvatore monferrato|san salvatore telesino|san salvo|san sebastiano al vesuvio|san sebastiano curone|san sebastiano da po|san secondo di pinerolo|san secondo parmense|san severino lucano|san severino marche|san severo|san siro|san sossio baronia|san sostene|san sosti|san sperate|san stino di livenza|san tammaro|san teodoro|san tomaso agordino|san valentino in abruzzo citeriore|san valentino torio|san venanzo|san vendemiano|san vero milis|san vincenzo|san vincenzo la costa|san vincenzo valle roveto|san vitaliano|san vito|san vito al tagliamento|san vito al torre|san vito chietino|san vito dei normanni|san vito di cadore|san vito di fagagna|san vito di leguzzano|san vito lo capo|san vito romano|san vito sullo ionio|san vittore del lazio|san vittore olona|san zeno di montagna|san zeno naviglio|san zenone al lambro|san zenone al po|san zenone degli ezzelini|sanarica|sand in taufers|sandigliano|sandrigo|sanfront|sanfrรจ|sangano|sangiano|sangineto|sanguinetto|sanluri|sannazzaro de' burgondi|sannicandro di bari|sannicola|sanremo|sansepolcro|sant'agapito|sant'agata bolognese|sant'agata de' goti|sant'agata del bianco|sant'agata di esaro|sant'agata di militello|sant'agata di puglia|sant'agata feltria|sant'agata fossili|sant'agata li battiati|sant'agata sul santerno|sant'agnello|sant'albano stura|sant'alessio con vialone|sant'alessio in aspromonte|sant'alessio siculo|sant'alfio|sant'ambrogio di torino|sant'ambrogio di valpolicella|sant'ambrogio sul garigliano|sant'anastasia|sant'anatolia di narco|sant'andrea apostolo dello ionio|sant'andrea del garigliano|sant'andrea di conza|sant'andrea frius|sant'angelo a cupolo|sant'angelo a fasanella|sant'angelo a scala|sant'angelo all'esca|sant'angelo d'alife|sant'angelo dei lombardi|sant'angelo del pesco|sant'angelo di brolo|sant'angelo di piove di sacco|sant'angelo in pontano|sant'angelo in vado|sant'angelo le fratte|sant'angelo limosano|sant'angelo lodigiano|sant'angelo lomellina|sant'angelo muxaro|sant'angelo romano|sant'anna arresi|sant'anna d'alfaedo|sant'antimo|sant'antioco|sant'antonino di susa|sant'antonio abate|sant'antonio di gallura|sant'apollinare|sant'arcangelo|sant'arcangelo trimonte|sant'arpino|sant'arsenio|sant'egidio alla vibrata|sant'egidio del monte albino|sant'elena|sant'elena sannita|sant'elia a pianisi|sant'elia fiumerapido|sant'elpidio a mare|sant'eufemia a maiella|sant'eufemia d'aspromonte|sant'eusanio del sangro|sant'eusanio forconese|sant'ilario d'enza|sant'ilario dello ionio|sant'ippolito|sant'olcese|sant'omero|sant'omobono terme|sant'onofrio|sant'oreste|sant'orsola terme|sant'urbano|santa brigida|santa caterina albanese|santa caterina d'este|santa caterina dello ionio|santa caterina villarmosa|santa cesarea terme|santa cristina d'aspromonte|santa cristina e bissone|santa cristina gela|santa cristina valgardena|santa croce camerina|santa croce del sannio|santa croce di magliano|santa croce sull'arno|santa domenica talao|santa domenica vittoria|santa elisabetta|santa fiora|santa flavia|santa giuletta|santa giusta|santa giustina|santa giustina in colle|santa luce|santa lucia del mela|santa lucia di piave|santa lucia di serino|santa margherita di belice|santa margherita di staffora|santa margherita ligure|santa maria a monte|santa maria a vico|santa maria capua vetere|santa maria coghinas|santa maria del cedro|santa maria del molise|santa maria della versa|santa maria di licodia|santa maria di sala|santa maria hoรจ|santa maria imbaro|santa maria la caritร |santa maria la fossa|santa maria la longa|santa maria maggiore|santa maria nuova|santa marina|santa marina salina|santa marinella|santa ninfa|santa paolina|santa severina|santa sofia|santa sofia d'epiro|santa teresa di riva|santa teresa gallura|santa venerina|santa vittoria d'alba|santa vittoria in matenano|santadi|santarcangelo di romagna|sante marie|santena|santeramo in colle|santhiร |santi cosma e damiano|santo stefano al mare|santo stefano belbo|santo stefano d'aveto|santo stefano del sole|santo stefano di cadore|santo stefano di camastra|santo stefano di magra|santo stefano di rogliano|santo stefano di sessanio|santo stefano in aspromonte|santo stefano lodigiano|santo stefano quisquina|santo stefano roero|santo stefano ticino|santomenna|santopadre|santorso|santu lussurgiu|sanza|sanzeno|saonara|saponara|sappada|sapri|saracena|saracinesco|sarcedo|sarconi|sardara|sardigliano|sarego|sarentino|sarezzano|sarezzo|sarmato|sarmede|sarnano|sarnico|sarno|sarnonico|sarntal|saronno|sarre|sarroch|sarsina|sarteano|sartirana lomellina|sarule|sarzana|sassano|sassari|sassello|sassetta|sassinoro|sasso di castalda|sasso marconi|sassocorvaro auditore|sassofeltrio|sassoferrato|sassuolo|satriano|satriano di lucania|sauris|sauze d'oulx|sauze di cesana|sava|savelli|saviano|savigliano|savignano irpino|savignano sul panaro|savignano sul rubicone|savignone|saviore dell'adamello|savoca|savogna|savogna d'isonzo|savoia di lucania|savona|scafa|scafati|scagnello|scala|scala coeli|scaldasole|scalea|scalenghe|scaletta zanclea|scampitella|scandale|scandiano|scandicci|scandolara ravara|scandolara ripa d'oglio|scandriglia|scanno|scano di montiferro|scansano|scanzano jonico|scanzorosciate|scapoli|scarlino|scarmagno|scarnafigi|scarperia e san piero|scena|scerni|scheggia e pascelupo|scheggino|schenna|schiavi di abruzzo|schiavon|schignano|schilpario|schio|schivenoglia|schlanders|schluderns|schnals|sciacca|sciara|scicli|scido|scigliano|scilla|scillato|sciolze|scisciano|sclafani bagni|scontrone|scopa|scopello|scoppito|scordia|scorrano|scorzรจ|scurcola marsicana|scurelle|scurzolengo|seborga|secinaro|seclรฌ|secugnago|sedegliano|sedico|sedilo|sedini|sedriano|sedrina|sefro|segariu|seggiano|segni|segonzano|segrate|segusino|selargius|selci|selegas|sella giudicarie|sellano|sellero|sellia|sellia marina|selva dei molini|selva di cadore|selva di progno|selva di val gardena|selvazzano dentro|selvino|semestene|semiana|seminara|semproniano|senago|senale-san felice|senales|seneghe|senerchia|seniga|senigallia|senis|senise|senna comasco|senna lodigiana|sennariolo|sennori|senorbรฌ|sepino|sequals|seravezza|serdiana|seregno|seren del grappa|sergnano|seriate|serina|serino|serle|sermide e felonica|sermoneta|sernaglia della battaglia|sernio|serole|serra d'aiello|serra de' conti|serra riccรฒ|serra san bruno|serra san quirico|serra sant'abbondio|serracapriola|serradifalco|serralunga d'alba|serralunga di crea|serramanna|serramazzoni|serramezzana|serramonacesca|serrapetrona|serrara fontana|serrastretta|serrata|serravalle a po|serravalle di chienti|serravalle langhe|serravalle pistoiese|serravalle scrivia|serravalle sesia|serre|serrenti|serri|serrone|sersale|servigliano|sessa aurunca|sessa cilento|sessame|sessano del molise|sesta godano|sestino|sesto|sesto al reghena|sesto calende|sesto campano|sesto ed uniti|sesto fiorentino|sesto san giovanni|sestola|sestri levante|sestriere|sestu|settala|settefrati|setteville|settime|settimo milanese|settimo rottaro|settimo san pietro|settimo torinese|settimo vittone|settingiano|setzu|seui|seulo|seveso|sexten|sezzadio|sezze|sfruz|sgonico|sgurgola|siamaggiore|siamanna|siano|siapiccia|sicignano degli alburni|siculiana|siddi|siderno|siena|sigillo|signa|silandro|silanus|silea|siligo|siliqua|silius|sillano giuncugnano|sillavengo|silvano d'orba|silvano pietra|silvi|simala|simaxis|simbario|simeri crichi|sinagra|sinalunga|sindia|sini|sinio|siniscola|sinnai|sinopoli|siracusa|sirignano|siris|sirmione|sirolo|sirone|sirtori|sissa trecasali|siurgus donigala|siziano|sizzano|sluderno|smerillo|soave|socchieve|soddรฌ|sogliano al rubicone|sogliano cavour|soglio|soiano del lago|solagna|solarino|solaro|solarolo|solarolo rainerio|solarussa|solbiate arno|solbiate con cagno|solbiate olona|soldano|soleminis|solero|solesino|soleto|solferino|soliera|solignano|solofra|solonghello|solopaca|solto collina|solza|somaglia|somano|somma lombardo|somma vesuviana|sommacampagna|sommariva del bosco|sommariva perno|sommatino|sommo|sona|soncino|sondalo|sondrio|songavazzo|sonico|sonnino|sora|soraga di fassa|soragna|sorano|sorbo san basile|sorbo serpico|sorbolo mezzani|sordevolo|sordio|soresina|sorgono|sorgร |sori|sorianello|soriano calabro|soriano nel cimino|sorico|soriso|sorisole|sormano|sorradile|sorrento|sorso|sortino|sospiro|sospirolo|sossano|sostegno|sotto il monte giovanni xxiii|sover|soverato|sovere|soveria mannelli|soveria simeri|soverzene|sovicille|sovico|sovizzo|sovodnje ob soฤi|sovramonte|sozzago|spadafora|spadola|sparanise|sparone|specchia|spello|sperlinga|sperlonga|sperone|spessa|spezzano albanese|spezzano della sila|spiazzo|spigno monferrato|spigno saturnia|spilamberto|spilimbergo|spilinga|spinadesco|spinazzola|spinea|spineda|spinete|spineto scrivia|spinetoli|spino d'adda|spinone al lago|spinoso|spirano|spoleto|spoltore|spongano|spormaggiore|sporminore|spotorno|spresiano|spriana|squillace|squinzano|st. christina in grรถden|st. leonhard in passeier|st. lorenzen|st. martin in passeier|st. martin in thurn|st. pankraz|st. ulrich|staffolo|stagno lombardo|staiti|stalettรฌ|stanghella|staranzano|statte|stazzano|stazzema|stazzona|stefanaconi|stella|stella cilento|stellanello|stelvio|stenico|sternatia|sterzing|stezzano|stienta|stigliano|stignano|stilfs|stilo|stimigliano|stintino|stio|stornara|stornarella|storo|stra|stradella|strambinello|strambino|strangolagalli|stregna|strembo|stresa|strevi|striano|strona|stroncone|strongoli|stroppiana|stroppo|strozza|sturno|suardi|subbiano|subiaco|succivo|sueglio|suelli|suello|suisio|sulbiate|sulmona|sulzano|sumirago|summonte|suni|suno|supersano|supino|surano|surbo|susa|susegana|sustinente|sutera|sutri|sutrio|suvereto|suzzara|sรจn jan|taceno|tadasuni|taggia|tagliacozzo|taglio di po|tagliolo monferrato|taibon agordino|taino|taipana|talamello|talamona|talana|taleggio|talla|talmassons|tambre|taormina|tarano|taranta peligna|tarantasca|taranto|tarcento|tarquinia|tarsia|tartano|tarvisio|tarzo|tassarolo|taufers im mรผnstertal|taurano|taurasi|taurianova|taurisano|tavagnacco|tavagnasco|tavazzano con villavesco|tavenna|taverna|tavernerio|tavernola bergamasca|tavernole sul mella|taviano|tavigliano|tavoleto|tavullia|teana|teano|teggiano|teglio|teglio veneto|telese terme|telgate|telti|telve|telve di sopra|tempio pausania|temรน|tenna|tenno|teolo|teora|teramo|terdobbiate|terelle|terenten|terento|terenzo|tergu|terlan|terlano|terlizzi|terme vigliatore|termeno sulla strada del vino|termini imerese|termoli|ternate|ternengo|terni|terno d'isola|terracina|terragnolo|terralba|terranova da sibari|terranova dei passerini|terranova di pollino|terranova sappo minulio|terranuova bracciolini|terrasini|terrassa padovana|terravecchia|terrazzo|terre d'adige|terre del reno|terre roveresche|terricciola|terruggia|tertenia|terzigno|terzo|terzo d'aquileia|terzolas|terzorio|tesero|tesimo|tessennano|testico|teti|teulada|teverola|tezze sul brenta|thiene|thiesi|tiana|ticengo|ticineto|tiers|tiggiano|tiglieto|tigliole|tignale|tinnura|tione degli abruzzi|tione di trento|tirano|tires|tiriolo|tirol|tirolo|tisens|tissi|tito|tivoli|tizzano val parma|toano|toblach|tocco caudio|tocco da casauria|toceno|todi|toffia|toirano|tolentino|tolfa|tollegno|tollo|tolmezzo|tolve|tombolo|ton|tonara|tonco|tonezza del cimone|tora e piccilli|torano castello|torano nuovo|torbole casaglia|torcegno|torchiara|torchiarolo|torella dei lombardi|torella del sannio|torgiano|torgnon|torino|torino di sangro|toritto|torlino vimercati|tornaco|tornareccio|tornata|tornimparte|torno|tornolo|toro|torpรจ|torraca|torralba|torrazza coste|torrazza piemonte|torrazzo|torre annunziata|torre beretti e castellaro|torre boldone|torre bormida|torre cajetani|torre canavese|torre d'arese|torre d'isola|torre de' busi|torre de' negri|torre de' passeri|torre de' picenardi|torre de' roveri|torre del greco|torre di mosto|torre di ruggiero|torre di santa maria|torre le nocelle|torre mondovรฌ|torre orsaia|torre pallavicina|torre pellice|torre san giorgio|torre san patrizio|torre santa susanna|torreano|torrebelvicino|torrebruna|torrecuso|torreglia|torregrotta|torremaggiore|torrenova|torresina|torretta|torrevecchia pia|torrevecchia teatina|torri del benaco|torri di quartesolo|torri in sabina|torrice|torricella|torricella del pizzo|torricella in sabina|torricella peligna|torricella sicura|torricella verzate|torriglia|torrile|torrioni|torrita di siena|torrita tiberina|tortolรฌ|tortona|tortora|tortorella|tortoreto|tortorici|torviscosa|toscolano-maderno|tossicia|tovo di sant'agata|tovo san giacomo|trabia|tradate|tramatza|trambileno|tramin an der weinstraรŸe|tramonti|tramonti di sopra|tramonti di sotto|tramutola|trana|trani|traona|trapani|trappeto|trarego viggiona|trasacco|trasaghis|trasquera|tratalias|travacรฒ siccomario|travagliato|travedona-monate|traversella|traversetolo|traves|travesio|travo|tre ville|trebaseleghe|trebisacce|trecase|trecastagni|trecastelli|trecate|trecchina|trecenta|tredozio|treglio|tregnago|treia|treiso|tremestieri etneo|tremezzina|tremosine sul garda|trentinara|trento|trentola ducenta|trenzano|treppo grande|treppo ligosullo|trepuzzi|trequanda|tresana|trescore balneario|trescore cremasco|tresignana|tresivio|tresnuraghes|trevenzuolo|trevi|trevi nel lazio|trevico|treviglio|trevignano|trevignano romano|treville|treviolo|treviso|treviso bresciano|trezzano rosa|trezzano sul naviglio|trezzo sull'adda|trezzo tinella|trezzone|tribano|tribiano|tribogna|tricarico|tricase|tricerro|tricesimo|triei|trieste|triggiano|trigolo|trinitapoli|trinitร |trinitร  d'agultu e vignola|trino|triora|tripi - abakainon|trisobbio|trissino|triuggio|trivento|trivigliano|trivignano udinese|trivigno|trivolzio|trodena nel parco naturale|trofarello|troia|troina|tromello|trontano|tronzano lago maggiore|tronzano vercellese|tropea|trovo|truccazzano|truden im naturpark|tscherms|tubre|tufara|tufillo|tufino|tufo|tuglie|tuili|tula|tuoro sul trasimeno|turania|turano lodigiano|turate|turbigo|turi|turri|turriaco|turrivalignani|tursi|tusa|tuscania|ubiale clanezzo|uboldo|ucria|udine|ugento|uggiano la chiesa|uggiate con ronago|ulassai|ulten|ultimo|ulร  tirso|umbertide|umbriatico|unsere liebe frau im walde-st. felix|urago d'oglio|uras|urbana|urbania|urbe|urbino|urbisaglia|urgnano|uri|ururi|urzulei|uscio|usellus|usini|usmate velate|ussana|ussaramanna|ussassai|usseaux|usseglio|ussita|ustica|uta|uzzano|vaccarizzo albanese|vacone|vacri|vadena|vado ligure|vagli sotto|vaglia|vaglio basilicata|vaglio serra|vahrn|vaiano|vaiano cremasco|vaie|vailate|vairano patenora|vajont|val brembilla|val della torre|val di chy|val di nizza|val di vizze|val di zoldo|val liona|val masino|val rezzo|valbondione|valbrembo|valbrenta|valbrevenna|valbrona|valchiusa|valdagno|valdaone|valdaora|valdastico|valdengo|valderice|valdidentro|valdieri|valdilana|valdina|valdisotto|valdobbiadene|valduggia|valeggio|valeggio sul mincio|valentano|valenza|valenzano|valera fratta|valfabbrica|valfenera|valfloriana|valfornace|valfurva|valganna|valgioie|valgoglio|valgrana|valgreghentino|valgrisenche|valguarnera caropepe|vallada agordina|vallanzengo|vallarsa|vallata|valle agricola|valle aurina|valle cannobina|valle castellana|valle dell'angelo|valle di cadore|valle di casies|valle di maddaloni|valle lomellina|valle salimbene|valle san nicolao|vallebona|vallecorsa|vallecrosia|valledolmo|valledoria|vallefiorita|vallefoglia|vallelaghi|vallelonga|vallelunga pratameno|vallemaio|vallepietra|vallerano|vallermosa|vallerotonda|vallesaccarda|valleve|valli del pasubio|vallinfreda|vallio terme|vallo della lucania|vallo di nera|vallo torinese|valloriate|valmacca|valmadrera|valmontone|valmorea|valmozzola|valnegra|valpelline|valperga|valprato soana|valsamoggia|valsavarenche|valsinni|valsolda|valstrona|valtopina|valtorta|valtournenche|valva|valvarrone|valvasone arzene|valverde|valvestino|vandoies|vanzaghello|vanzago|vanzone con san carlo|vaprio d'adda|vaprio d'agogna|varallo|varallo pombia|varano borghi|varano de' melegari|varapodio|varazze|varco sabino|varedo|varenna|varese|varese ligure|varisella|varmo|varna|varsi|varzi|varzo|vasanello|vasia|vasto|vastogirardi|vauda canavese|vazzano|vazzola|vecchiano|vedano al lambro|vedano olona|vedelago|vedeseta|veduggio con colzano|veggiano|veglie|veglio|vejano|veleso|velezzo lomellina|velletri|vellezzo bellini|velo d'astico|velo veronese|velturno|venafro|venaria reale|venarotta|venasca|venaus|vendone|venegono inferiore|venegono superiore|venetico|venezia|veniano|venosa|ventasso|venticano|ventimiglia|ventimiglia di sicilia|ventotene|venzone|verano|verano brianza|verbania|verbicaro|vercana|verceia|vercelli|vercurago|verdellino|verdello|verderio|verduno|vergato|verghereto|vergiate|vermezzo con zelo|vermiglio|vernante|vernasca|vernate|vernazza|vernio|vernole|verolanuova|verolavecchia|verolengo|veroli|verona|veronella|verrayes|verretto|verrone|verrua po|verrua savoia|verrรจs|vertemate con minoprio|vertova|verucchio|vervio|verzegnis|verzino|verzuolo|vescovana|vescovato|vesime|vespolate|vessalico|vestenanova|vestignรจ|vestone|vetralla|vetto|vezza d'alba|vezza d'oglio|vezzano ligure|vezzano sul crostolo|vezzi portio|viadana|viadanica|viagrande|viale|vialfrรจ|viano|viareggio|viarigi|vibo valentia|vibonati|vicalvi|vicari|vicchio|vicenza|vico del gargano|vico equense|vico nel lazio|vicoforte|vicoli|vicolungo|vicopisano|vicovaro|viddalba|vidigulfo|vidor|vidracco|vieste|vietri di potenza|vietri sul mare|vigano san martino|viganรฒ|vigarano mainarda|vigasio|vigevano|viggianello|viggiano|viggiรน|vigliano biellese|vigliano d'asti|vignale monferrato|vignanello|vignate|vignola|vignola-falesina|vignole borbera|vignolo|vignone|vigo di cadore|vigodarzere|vigolo|vigolzone|vigone|vigonovo|vigonza|viguzzolo|villa bartolomea|villa basilica|villa biscossi|villa carcina|villa castelli|villa celiera|villa collemandina|villa cortese|villa d'adda|villa d'almรจ|villa d'ogna|villa del bosco|villa del conte|villa di briano|villa di chiavenna|villa di serio|villa di tirano|villa estense|villa faraldi|villa guardia|villa lagarina|villa latina|villa literno|villa minozzo|villa san giovanni|villa san giovanni in tuscia|villa san pietro|villa san secondo|villa sant'angelo|villa sant'antonio|villa santa lucia|villa santa lucia degli abruzzi|villa santa maria|villa santina|villa santo stefano|villa verde|villabassa|villabate|villachiara|villacidro|villadeati|villadose|villadossola|villafalletto|villafranca d'asti|villafranca di verona|villafranca in lunigiana|villafranca padovana|villafranca piemonte|villafranca sicula|villafranca tirrena|villafrati|villaga|villagrande strisaili|villalago|villalba|villalfonsina|villalvernia|villamagna|villamaina|villamar|villamarzana|villamassargia|villamiroglio|villanders|villandro|villanova biellese|villanova canavese|villanova d'albenga|villanova d'ardenghi|villanova d'asti|villanova del battista|villanova del ghebbo|villanova del sillaro|villanova di camposampiero|villanova marchesana|villanova mondovรฌ|villanova monferrato|villanova monteleone|villanova solaro|villanova sull'arda|villanova truschedu|villanova tulo|villanovaforru|villanovafranca|villanterio|villanuova sul clisi|villaperuccio|villapiana|villaputzu|villar dora|villar focchiardo|villar pellice|villar perosa|villar san costanzo|villarbasse|villarboit|villareggia|villaricca|villaromagnano|villarosa|villasalto|villasanta|villasimius|villasor|villaspeciosa|villastellone|villata|villaurbana|villavallelonga|villaverla|ville d'anaunia|ville di fiemme|villeneuve|villesse|villetta barrea|villette|villimpenta|villnรถรŸ|villongo|villorba|vilminore di scalve|vimercate|vimodrone|vinadio|vinchiaturo|vinchio|vinci|vinovo|vintl|vinzaglio|viola|vione|vipiteno|virle piemonte|visano|vische|visciano|visco|visone|visso|vistarino|vistrorio|vita|viterbo|viticuso|vito d'asio|vitorchiano|vittoria|vittorio veneto|vittorito|vittuone|vitulano|vitulazio|vivaro|vivaro romano|viverone|vizzini|vizzola ticino|vizzolo predabissi|viรน|vo'|vobarno|vobbia|vocca|vodo cadore|voghera|voghiera|vogogna|volano|volla|volongo|volpago del montello|volpara|volpedo|volpeglino|volpiano|volta mantovana|voltaggio|voltago agordino|volterra|voltido|volturara appula|volturara irpina|volturino|volvera|vottignasco|vรถls am schlern|vรถran|waidbruck|welsberg-taisten|welschnofen|wengen|wolkenstein in grรถden|zaccanopoli|zafferana etnea|zagarise|zagarolo|zambrone|zandobbio|zanica|zanรจ|zapponeta|zavattarello|zeccone|zeddiani|zelbio|zelo buon persico|zeme|zenevredo|zenson di piave|zerba|zerbo|zerbolรฒ|zerfaliu|zeri|zermeghedo|zero branco|zevio|zgonik|ziano di fiemme|ziano piacentino|zibido san giacomo|zignago|zimella|zimone|zinasco|zoagli|zocca|zogno|zola predosa|zollino|zone|zoppola|zoppรจ di cadore|zovencedo|zubiena|zuccarello|zugliano|zuglio|zumaglia|zumpano|zungoli|zungri|ลกteverjan""".split("|")) +COMUNI_AMBIGUOUS = frozenset("""abbiategrasso|acerra|acireale|acri|acuto|adria|affi|afragola|agira|agra|agrigento|agropoli|ala|alassio|alatri|alba|albanella|albenga|alberobello|albi|albignasego|albino|albosaggia|alcamo|ales|alessandria|alfano|alfonsine|alghero|alia|allein|alpignano|altamura|altare|altino|altissimo|alto|altopascio|alรฌ|amalfi|amandola|amaro|amato|amatrice|amelia|ameno|amorosi|ampezzo|anagni|ancona|andora|andria|anela|anghiari|anzi|anzio|aosta|apice|aprica|aprilia|aquileia|aquino|aragona|arborea|arborio|arcade|arce|archi|arco|arcore|ardea|ardore|arena|arenzano|arese|arezzo|argenta|ari|ariccia|armeno|armento|armo|arona|arpino|arre|arzachena|arzano|arzignano|asciano|asiago|asola|asolo|assago|assisi|asso|assolo|asti|atella|atri|augusta|auletta|aulla|avellino|aversa|avetrana|avezzano|aviano|avio|avola|ayas|azeglio|bacoli|badia|badolato|bagheria|bagnoregio|baiano|balocco|banchette|baranzate|barbara|barbaresco|barbata|barbona|bard|bardi|bardolino|barga|bari|barile|barletta|barolo|baschi|basiliano|bastiglia|battipaglia|bee|beinette|belfiore|bella|bellagio|bellino|bellona|belluno|bema|benestare|benevento|benna|bereguardo|bergamasco|bergamo|bettola|beverino|bevilacqua|bianchi|bianco|biassono|bibbiena|biella|bienno|bisaccia|bisceglie|bitonto|bobbio|boca|bollate|bologna|bolsena|bolzano|bomba|bonito|bonnanaro|bono|bordano|bordighera|bore|borghi|borgia|borgomanero|borgoricco|borgosesia|bormida|bormio|borno|bosa|botticino|bova|boves|bovino|bozzolo|bra|bracca|bracciano|braies|brancaleone|branzi|breme|brennero|brenta|brentonico|brescia|bressanone|bresso|brindisi|brolo|broni|bronte|brunello|brunico|bruno|buccheri|buccinasco|burgos|busca|busso|bussolengo|cabras|caccamo|cagli|cagliari|caglio|caino|calcata|calci|calciano|calcinate|calcinato|calcio|calco|caltagirone|caltanissetta|calvello|calvi|camaiore|cambiano|camerino|camerota|camini|camino|camogli|campagna|campana|campobasso|campomarino|camposampiero|camposanto|canale|canaro|candela|candida|caneva|canino|canna|cannobio|canterano|cantรน|caorle|capaci|capannori|capodimonte|capri|caprile|capriolo|capua|caraglio|caravaggio|carbonate|carbone|carbonia|carcare|cardinale|carezzano|cariati|carignano|carini|carlino|carmagnola|carmignano|carovigno|carpi|carpino|carrara|carro|carrรจ|carugate|casalino|casamassima|casarano|cascia|cascina|casella|caserta|casina|casoria|cassino|castana|casteggio|castellabate|castellanza|castellaro|castelli|castelmagno|castelnuovo|castelseprio|castelvetrano|casto|castro|castrovillari|catania|catanzaro|cattolica|cavallino|cavatore|cave|cavezzo|cavour|ceccano|cecina|cefalรน|celano|cene|cento|centola|cerano|cerchio|cerea|cerignola|cerva|cerveteri|cervia|cervino|cervo|cesa|cesate|cesena|cesenatico|cesio|ceto|ceva|chianche|chiari|chiavari|chieri|chiesanuova|chieti|chioggia|chiusa|chiusi|chivasso|ciampino|cicala|cingoli|cino|cis|cittadella|civita|civitavecchia|codroipo|colere|coli|colico|collecchio|colleferro|collegno|collio|cologne|colonna|colonnella|comacchio|comiso|como|compiano|comunanza|cona|conegliano|contrada|conversano|copertino|copiano|corato|cori|corio|corleone|correggio|corrido|corsico|cortona|cosenza|courmayeur|covo|cozzo|crema|cremona|crespino|crotone|cuneo|curino|curti|cusio|dalmine|dazio|delia|dello|desio|diamante|dolina|dolo|domodossola|donato|dragoni|drena|dro|dronero|dueville|eboli|elice|ello|empoli|enna|envie|erba|ercolano|este|fabbrico|fabriano|faenza|falcone|fallo|fano|farnese|fasano|fascia|favara|favignana|felino|feltre|fermo|ferrara|ferriere|fiano|fidenza|fiesole|filadelfia|filettino|filetto|firenze|firmo|fisciano|fiuggi|fiumara|fiumicino|foggia|foligno|follonica|fondi|fontanella|fontanelle|fontanile|fonte|forano|force|forcola|forio|forlรฌ|formia|formicola|fornace|fornelli|fortezza|fossa|fossano|foza|frascati|frassino|frattamaggiore|frignano|frisa|front|frontino|frontone|frosinone|furore|gaeta|galatina|galatone|gallarate|gallese|gallicano|gallio|gallipoli|garda|garlasco|gavi|gela|genova|gessate|giarre|ginestra|giove|girasole|giuliana|giulianova|giungano|giussano|giustino|goni|gonzaga|gorgonzola|gorizia|gorno|grado|gradoli|gragnano|graniti|greci|greggio|grimaldi|grosseto|grosso|grottaglie|grotte|gruaro|grugliasco|gualdo|guastalla|gubbio|idro|iglesias|imola|imperia|impruneta|incudine|introd|ischia|iseo|isernia|ispica|ispra|isso|itala|ivrea|jesi|jesolo|ladispoli|laghi|laglio|lago|lainate|lana|lanciano|lariano|laterza|latina|lauro|lavagna|lavello|lecce|lecco|legnago|legnano|lei|lenta|lentini|leonessa|lerici|lesa|lesina|lettere|levate|liberi|licata|licenza|limbiate|limena|limina|linguaglossa|lioni|lipari|liscia|lissone|livigno|livorno|lizzano|loano|locri|loculi|lodi|loiano|lomazzo|longobardi|lonigo|loreto|lucca|lucera|lugo|luino|lula|luni|lupara|luserna|lustra|macello|macerata|maddaloni|mafalda|magenta|maggiora|magione|maglie|maglione|magnago|magnano|maida|maiori|maleo|malfa|malo|mammola|mandela|manduria|manfredonia|mango|manta|mantello|mantova|mappano|mara|maranello|maratea|marciana|marcon|margarita|marino|marsala|marta|martello|martinsicuro|marzano|marzio|mascalucia|maser|masera|masi|massa|massello|matera|matino|matrice|mattinata|meda|mede|medea|medicina|mele|melegnano|meleti|melfi|melilli|melissa|melle|mello|melzo|mentana|merano|merate|merlino|mesagne|mese|messina|mestrino|meta|mezzana|mezzanino|mezzano|milano|milazzo|milena|millesimo|milo|minori|mira|miranda|mirandola|mirano|mirto|modena|modica|modugno|moggio|moglia|mogliano|molare|molfetta|molinara|molise|momo|moncalieri|mondovรฌ|mondragone|monfalcone|monno|monopoli|monreale|monselice|montalcino|montale|montanaro|montebelluna|montecarlo|montecchio|montelabbate|montella|montello|montepulciano|monterotondo|montese|montesilvano|montevarchi|monteverde|montezemolo|monti|montiano|montichiari|montone|monza|morbegno|mordano|mores|moresco|moretta|mori|morigerati|mortara|mossa|mozzate|mozzo|muggiรฒ|mura|muros|musei|musso|napoli|nardรฒ|narni|naro|nasino|naso|nave|ne|nespolo|nesso|nettuno|nichelino|nicosia|nimis|noale|noceto|noci|nola|noli|nomi|none|nonio|norcia|norma|noto|novara|nove|novella|novellara|novello|novoli|nuoro|nus|ocre|oderzo|olbia|oleggio|ome|omegna|onore|opera|opi|ora|orbassano|orbetello|oria|oriolo|oristano|orosei|orte|ortona|orvieto|osimo|ospedaletti|ospedaletto|ossi|ostra|ostuni|otranto|ottaviano|ottone|ovada|pachino|padova|paesana|paese|pagani|paladina|palata|palau|palermo|palestrina|palmanova|palmi|palombaro|paludi|palรน|pandino|panettieri|panni|pantelleria|paola|parabiago|parenti|parete|parma|partinico|paterno|paternรฒ|patti|pau|pavia|pedemonte|pelago|pella|penne|perca|pergola|perito|perlo|pero|perugia|pesaro|pescara|pescasseroli|pescate|pesche|pescia|petriolo|pettineo|peveragno|piacenza|pianella|pianezza|pianoro|piatto|picerno|pico|pienza|pietrafitta|pietragalla|pietrasanta|piglio|pigna|pignola|pignone|pigra|pila|pinerolo|pineto|piode|piombino|pisa|pisano|piscina|pistoia|pizzo|poggibonsi|poggioreale|poli|policoro|polistena|polla|pollina|pollone|pomezia|pompei|pompeiana|ponderano|ponte|pontedera|ponti|pontida|ponza|poppi|porcari|pordenone|porte|portici|portoferraio|portofino|portogruaro|portovenere|posada|positano|posta|postal|postiglione|potenza|pozzallo|pozzuoli|prato|pray|preci|predosa|premia|prezza|procida|pula|pulsano|putignano|quadri|quart|quarto|quartucciu|quindici|rabbi|ragusa|randazzo|rapallo|ravello|ravenna|re|rea|recanati|recco|rende|rezzo|rho|riace|rialto|ribera|ricadi|riccia|riccione|rieti|rimini|rio|riposto|rive|rivoli|roana|roccaraso|rodano|rolo|roma|romana|romeno|roncola|rosate|rose|rossa|rossana|rosta|rotella|rotonda|rotondi|rovato|rovereto|rovigo|rozzano|rubano|rubiera|ruoti|russi|rutigliano|sabaudia|sacco|sacile|sale|salemi|salento|salerno|salle|saluzzo|salve|salรฒ|sambuco|samo|samone|sanremo|sansepolcro|sanza|saponara|sapri|saracena|sarnico|sarno|saronno|sarzana|sassari|sassello|sassuolo|sava|saviano|savigliano|savona|scafati|scala|scalea|scandale|scandiano|scandicci|scanno|scansano|scapoli|scena|schio|sciacca|sciara|scicli|scilla|scopa|scorrano|scorzรจ|sedriano|segni|segrate|selci|senigallia|seregno|seriate|serina|serrata|serre|serri|sesto|sestriere|sestu|settime|seveso|siano|siderno|siena|sigillo|signa|siliqua|silvi|sini|siracusa|sirmione|soave|soglio|solaro|soldano|solferino|sommo|sona|soncino|sondrio|sonico|sora|sorano|sorgono|sori|sorrento|sorso|sospiro|sostegno|soverato|specchia|spello|sperlonga|sperone|spessa|spiazzo|spinea|spinoso|spirano|spoleto|spresiano|statte|stella|stelvio|stilo|stintino|stra|stradella|stresa|strona|strozza|sulmona|supino|susa|suzzara|taggia|taino|taleggio|taormina|taranto|tarquinia|tarsia|taverna|tavullia|teramo|termoli|terni|terracina|terrasini|terrazzo|terzo|tesero|teti|thiene|tiers|tirano|tires|tirol|tirolo|tito|tivoli|todi|tolentino|tolmezzo|tombolo|ton|torino|tornata|torno|toro|torrazzo|torresina|torretta|torricella|torrioni|tortona|tortora|tortorella|tortoreto|tradate|tramonti|trani|trapani|trappeto|travagliato|traversetolo|trebaseleghe|trecate|trento|trevi|treviglio|treviso|trieste|trinitร |trino|troia|tropea|trovo|tufo|turate|turi|turri|udine|ugento|ultimo|urbana|urbe|urbino|uri|uscio|ustica|uta|vaglia|vaie|valbrembo|valdobbiadene|valenza|valenzano|vallata|valmontone|valva|valverde|varallo|varazze|varedo|varese|vasto|vecchiano|veglie|veglio|velletri|venetico|venezia|venosa|ventimiglia|verano|verbania|vercelli|verdello|vergato|vermiglio|verona|vescovato|viale|viareggio|vicari|vicenza|vicoli|vieste|vigevano|vignola|vigonza|villamagna|villasimius|villette|villorba|vimercate|vinci|viola|visco|visone|vita|viterbo|vittoria|vittuone|viรน|vo'|voghera|volano|voltaggio|volterra|zeri|zone""".split("|")) +PROVINCE_SIGLE = frozenset("""AG|AL|AN|AO|AP|AQ|AR|AT|AV|BA|BG|BI|BL|BN|BO|BR|BS|BT|BZ|CA|CB|CE|CH|CI|CL|CN|CO|CR|CS|CT|CZ|EN|FC|FE|FG|FI|FM|FR|GE|GO|GR|IM|IS|KR|LC|LE|LI|LO|LT|LU|MB|MC|ME|MI|MN|MO|MS|MT|NA|NO|NU|OG|OR|OT|PA|PC|PD|PE|PG|PI|PN|PO|PR|PT|PU|PV|PZ|RA|RC|RE|RG|RI|RM|RN|RO|SA|SI|SO|SP|SR|SS|SV|TA|TE|TN|TO|TP|TR|TS|TV|UD|VA|VB|VC|VE|VI|VR|VS|VT|VV""".split("|")) +REGIONS = frozenset("""abruzzo|basilicata|calabria|campania|emilia-romagna|friuli-venezia giulia|lazio|liguria|lombardia|marche|molise|piemonte|puglia|sardegna|sicilia|sรผdtirol|toscana|trentino-alto adige|umbria|valle d'aosta|vallรฉe d'aoste|veneto""".split("|")) +# Given names and surnames: Wikidata (CC0, +# https://www.wikidata.org/wiki/Wikidata:Licensing), entries that +# have an article on the Italian Wikipedia. +NAMES_GIVEN = frozenset("""abaco|abramina|abubakr|abundancjusz|adaeze|adenike|aisha|amabile|amadi|amarachi|ambroz|amรญlcar|ananda|andreana|annabeth|ansovino|ansprando|antea|arbana|arduino|ariele|arleigh|armelinda|armelindo|armonia|arsinoe|atticus|aurรฉlien|aynoor|azubuike|baldina|barnabรฉ|biagino|bindy|blagoja|blessing|bonavita|bryon|buonaccorso|bรถrje|cafiero|candis|carmelina|carson|cash|cass|cedrick|chandra|chus|cibele|cierra|cleonico|colbie|cristina|cvetanka|cydney|darby|deanne|deliverance|delphina|diamanto|dikran|dilia|dionigio|djurdja|dolph|doralice|dovydas|durante|dwane|edgard|egil|eliso|endika|eniko|ermelina|erodione|esmeraldo|esperina|etelfleda|euloge|evalina|evaline|ewoud|fabrizio|fara|finola|fiore|fraser|garance|gasper|gaudentius|gaudรชncio|gaรซl|generosa|gerasimo|germinal|giairo|giannotto|gioia|gladwyn|glori|grimhild|gro|gualberto|gustavina|habakuk|hadis|hajna|harlee|hosny|hyacinth|hyacintha|iacopino|ilenja|indro|islom|ivetta|jaelen|jantine|jarod|jaylin|jaylon|jean-pierre|jedd|jerod|jill|jolan|jordyn|joรซl|joรซlle|jรฉrome|kailey|kajrat|kaleigh|kaley|kaouthar|karolina|kaylie|keitha|khalil|khalilah|kirby|krassimir|krassimira|lachie|laurianne|ledina|leilani|lele|leoluca|lesleigh|ljube|ljuben|lorelle|lubna|mahund|malak|marchina|marinus|martine|marvyn|matthaus|meggy|meinard|melchiade|meredyth|meriwether|micaiah|mikita|milcรญades|minodora|misty|mulay|myriam|nadege|narcisio|narges|nasreen|nathan|nectarios|nicaise|niketas|nordine|nuredin|odhran|olimpiada|ondina|onelio|onyekachi|oyvind|palmiero|paziente|perrie|phillis|procopio|quinziano|raffaelo|rambaldo|ranulfo|rashed|ravshan|rayno|restituto|ricciarda|rickey|riika|rocky|rodolph|roshan|rrahman|ruaraidh|ruaridh|ryleigh|rylie|rรญkharรฐur|sage|salomone|sami|sammi|sanchia|sanjaya|sanjin|santuzza|sejad|seppe|serxhio|serxho|shannon|shanon|sharron|shea|shereen|shon|shyla|sidnie|sperindio|stacia|steeven|steevy|stilian|stiliano|styliani|submit|surayya|taisija|tarsicio|tawni|temple|tenzin|terance|tesfaye|tiffiny|timoleone|timotej|timothea|timotheos|tineke|tobina|trecia|trey|turlough|ugochukwu|usama|valรฉrien|vando|venero|verio|vern|vincenzio|vlaho|vonda|vratislava|waltera|wandy|yair|zaffiro|zainuddin|รฉric|รณrfhlaith""".split("|")) +NAMES_SURNAMES = frozenset("""abaco|abate|abba|abbas|abbiate|abbondanza|abdullah|abeltino|abis|abozzi|abrami|abrioni|abruzzo|abubakr|abulafia|abutzu|acca|accalai|accardi|accareddu|accetto|accettura|accioni|acciu|accossu|accotzu|achena|achene|acheri|acqua|acri|adami|adani|adenike|adimari|adinolfi|adorni|agapito|agazzi|aglio|agnelli|agnese|agnoli|agosti|agozzino|agrati|agraz|agresti|ahn|aiolfi|airaghi|airaldi|al-dawasir|alba|albanese|albergo|alberi|alberici|alberti|albini|albizzati|albizzi|alboini|albonesi|alemanno|ales|alfieri|alfonsi|alifano|alinari|allevi|alonzo|altomare|alvaro|alzate|amadei|amadori|amato|ambrosini|amendola|amerighi|ammannati|amore|amoroso|anastasi|ancarani|andrei|andria|anfuso|angeli|anna|annarumma|antolini|aquenza|araรบjo|arecco|arese|argenti|argiolas|aricรฒ|arienzo|armaroli|arosio|arpaia|arpino|arrighi|asghar|ashina|ashton|aspesi|astolfi|atzeni|atzeri|augelli|avola|ayame|ayasrah|azzari|azzi|bae|baek|baggio|baglioni|bagni|baiano|ballabio|ballai|balli|balsamo|bandino|barabino|baraldi|baratti|barbon|barca|bardi|barenghi|bari|barile|barocci|bartesaghi|bartoli|barzaghi|barzizza|bassani|bastoni|batmanghelidj|battisti|beati|beccari|bedarida|beltrame|beltrami|bencivenga|benedetti|benetti|benevento|beni|bennati|benni|bentivoglio|benvenuti|bernardi|bernasconi|bersani|bertelli|berti|bertoldi|bertuzzi|bevilacqua|bianchi|bibescu|bielli|biffi|bifulco|bini|bisceglie|bixio|bobbio|boccadifuoco|boccardo|bologna|bologni|bombelli|bombonati|bomboni|bonacci|bonaccorso|bonaiuti|bonamico|bonanno|bonardi|bonati|bonaventura|bondi|bonett|bonfante|bonfiglio|bongiorno|bongiovanni|boni|bonifacio|bonini|boninsegna|bonito|bonomo|bonora|bonsignore|bonucci|borboni|borghi|bortoli|boscaino|boscolo|bosio|bosisio|botero|bottai|bova|brambilla|breme|brenna|breno|brescia|briganti|brioschi|brivio|brocchi|brooks|broussard|brufani|bruno|bulgarelli|burns|buscemi|buti|buzatu|buzzi|bรคr|cabras|caffi|caiazzo|calabrese|calandra|caldonazzi|callegari|calogero|calvi|calรฒ|cambi|cambiaghi|cambiaso|cameron|camozzi|campi|campo|canefri|canepa|canfora|cannas|cannata|cannavรฒ|cannella|canonico|capodanno|cappelletto|cappelli|cappi|capra|capri|capurro|carena|cariddi|carnevale|carnielli|caro|carrobbio|carrubba|carrร |caruso|casadei|casanova|casarsa|casati|caserta|casiraghi|casirati|casoni|cassano|cassioli|castagna|castrogiovanni|cauchi|cavalleri|cavanna|cazzaniga|cazzola|ceccherelli|ceglie|celentano|ceriana|cerioli|cerne|cerri|cerutti|cervi|cha|chae|chapman|checchi|cheli|chelli|chezzi|chiacchio|chimenti|chinaglia|cho|choi|chun|chรฉn|ciampi|ciccardini|cini|clementi|cocco|colombo|comaschi|comazzi|comi|como|coniglio|consiglio|conti|conz|cooper|coppi|corbi|corona|corradi|corrร |corte|corti|corvi|corvino|costa|cotta|coulibaly|covino|craxi|crespi|crespo|crinรฒ|crippa|cristaldi|croce|crosio|crotta|cucchiara|cucinotta|cuffaro|cugno|cuneo|curatolo|cusumano|d'agostino|dago|danieli|danna|davide|davoli|dini|diodati|dioguardi|diotallevi|dolci|domenico|don|donati|doneda|donnini|donno|donzelli|drew|dunn|dustin|dzintars|dรถrfles|emiliani|emo|ermacora|esposito|fabbri|falco|fanti|fara|farina|federici|felici|ferla|fermi|ferrando|ferrara|ferrari|ferrarini|ferraro|ferrero|ferro|fiaschi|filippi|fini|fiorenza|flockhart|folliero|folloni|fontana|formiggini|fortescue|fossa|fox|fracanzani|francalancia|franceschi|frank|franzini|franzoni|freeman|frigerio|fulco|fumei|furlani|gabelli|gaber|gaekwad|gaffari|gaglioffi|gai|gallo|gambaro|gandhi|gandini|gandolfi|garibaldi|garlaschi|garlati|garzia|gaspari|gastaldi|gatti|gaudenzi|gazzolo|gemignani|gennari|genova|gentile|giacomi|gigli|gioberti|gioffrรฉ|giordano|giuli|giuliano|giussani|giusti|gjelaj|goi|gori|goria|gould|gozzadini|grandi|grassi|greco|grillo|gualtieri|guarini|guarnieri|guglielmi|guidi|gurjar|gรผl|haller|hamilton|han|heo|hong|howard|hwang|iaderosa|ignoti|innocenti|invernizzi|irani|israeli|ivaldi|jackson|jang|jeon|jeong|jiang|johansson|joubert|kang|katz|kazmi|kim|kong|korkut|kotes|kovรกcs|krรผger|kwak|kwon|lainati|lama|lamberti|lanzotti|lapi|laurino|laurito|lauro|lazzari|lecca|lecce|lecchi|lecci|leccio|ledda|lee|lentini|leone|lepri|levati|limiti|linati|lipari|lippi|lissoni|livraghi|locatelli|lodes|lodi|longo|lorandi|lorenzi|lorusso|lozzi|ludi|luna|lupi|lussu|luzzi|maestri|maffei|maggiore|magri|maier|mainardi|maino|mamdani|mancini|manetti|manfredi|maniero|manni|manzi|manzu|marangon|marazzi|marchesi|marchetti|marchi|marcon|marconi|mari|mariani|marino|marra|marras|martelli|martinelli|martรญnez|mascheroni|mauro|mazza|mazzarella|mclaughlin|melloni|merello|mereu|merloni|meuricoffre|mezzadri|micheli|migheli|min|mini|miranda|mitri|modafferi|modena|molinari|moller|momigliano|mondadori|mondaini|moon|morelli|moretti|moro|moroder|musso|muzzi|mรธller|najm|nam|nannoni|nazzari|nehme|nepi|neri|nervi|nerviani|nesti|neville|nicastro|nicolosi|nobili|noferi|noli|noureddine|novelli|novi|nucci|o'reilly|oddo|oggioni|ognibene|oldani|olgiati|olmi|omegna|omodei|ondedei|onesti|ongania|ongaro|onorati|orazi|orciani|orefice|oren|orlando|ornati|ornella|ortolani|ortona|osborne|ostuni|otranto|ottaviani|ottolenghi|paduano|paine|pallavicini|pallavidino|palmer|palmieri|panarello|pangallo|pantaleo|park|parra|parrella|parri|pasqua|pasquali|pasta|pastore|pastorino|paternoster|pavan|pavia|pedemonte|pedroli|pellegrini|pelรกez|perego|pergamo|pergola|perini|perrelli|perri|perrone|perti|perut|petracca|petroni|piccoli|piemonte|pietri|pirovano|pisani|pisanu|pittaluga|politi|postolache|pozzolengo|pradhan|premoli|preti|primavera|primo|proietti|prosperi|puglia|pugliese|punzo|quartigiani|quinn|quintavalle|quinto|raimondi|ramponi|ramsey|ranalli|ransone|rapa|rapallo|rapisarda|rassidi|ratti|ravara|ravidat|regina|ricci|rinaldi|rizzo|rocco|rodolfi|rodrรญguez|romano|rondelli|roos|rosa|rosi|rossetti|rossi|rota|rubini|ruggeri|ruscelli|russo|rutelli|ruvo|saccani|sacchi|sala|salierno|salm-reifferscheid-raitz|salvatori|salvidio|sambiagio|sami|sammarchi|sammarco|sammartino|sammauro|sampieri|sampietro|sampino|sanfelice|sangiorgi|sangiovanni|sanseverino|santagata|santangelo|santelia|santi|santonastaso|santoro|sanvitale|sarti|sartori|satriani|satta|saul|savini|scala|scalas|scally|scarpa|schaffgotsch|sciaccaluga|scicchitano|sclafani|scognamiglio|scopelliti|scornavacca|scorrano|scorza|seo|seong|sereni|seri|serravalle|servi|servidio|sessa|sesti|settala|sette|shin|shiono|sidney|sighinolfi|sigismondi|signori|silva|simonato|sinadeni|sinagra|sinibaldi|siniscalchi|sinisi|siri|smaldone|somaglia|somma|sommariva|son|soncini|song|soria|stabile|stafford|stanco|starace|stilo|stoppa|sullivan|sutera|sutter|suzuki|sรกnchez|taverna|telli|terlizzi|terrani|terzi|testa|tiberi|tokarczuk|tolomei|tomasi|tomei|tondi|tore|torrieri|torti|tozzi|tramonti|trani|trevisan|tripodi|trivulzio|tromba|trucco|tumminello|turchi|turetta|turner|ubaldi|uggeri|ugoni-longhi|urquhart|uselli|vacca|vacondio|vadalร |valda|valguarnera|valiani|vallino|varchi|varricchio|vedova|velliscig|vercelli|vercesi|verde|verlingieri|viazzi|vicentini|vieri|vigevani|vignolo|villa|villani|viola|vitale|vivaldi|volkonskaja|voltaggio|watanabe|weizmann|wells|williams|wilson|winter|woo|yair|yang|ying|yoo|yoon|zaccheo|zaghi|zampa|zampaglione|zanchi|zanini|zappa|zen|zicari|ziccardi|zilli|ziou|zocchi|zoccola|zoccoli|zoppi|zorzi|zotti|zucca|zuccalร |zuccarello|zuccheri|zulian|zuliani|zurlo|zurro""".split("|")) +NAMES_AMBIGUOUS = frozenset("""abaco|abate|abba|abbas|abbiate|abbondanza|abruzzo|acca|accetto|acqua|acri|adorni|aglio|agnelli|agnese|agresti|ahn|alba|albanese|albergo|alberi|alberti|albini|alemanno|ales|alfieri|alifano|allevi|alvaro|alzate|amabile|amendola|amore|amoroso|ananda|andrei|andria|angeli|anna|antea|arduino|arese|argenti|armonia|arpino|augelli|avola|azzi|bae|baggio|baglioni|bagni|baiano|balli|balsamo|baratti|barca|bardi|bari|barile|bassani|bastoni|battisti|beati|benevento|beni|benni|benvenuti|bersani|berti|bevilacqua|bini|bisceglie|blessing|bobbio|bologna|bonaventura|bondi|bongiorno|boni|bonifacio|bonito|bonucci|borboni|borghi|bottai|bova|brambilla|breme|brescia|briganti|brocchi|brooks|bruno|burns|buzzi|bรคr|cabras|calabrese|calandra|calogero|calvi|calรฒ|cambi|cameron|campi|campo|canfora|cannata|cannella|canonico|capodanno|cappelletto|cappelli|cappi|capra|capri|carena|carnevale|caro|carrร |casanova|casarsa|casati|caserta|cash|casoni|cass|cassano|castagna|ceglie|celentano|cerri|cerutti|cervi|cha|cho|choi|chun|ciampi|clementi|colombo|comaschi|comi|como|coniglio|consiglio|cooper|coppi|corona|corte|corti|corvi|corvino|costa|cotta|craxi|crespi|crespo|cristina|croce|cucinotta|cuneo|dago|danna|davide|dini|dolci|domenico|don|donno|drew|durante|eliso|emiliani|emo|fabbri|fabrizio|falco|fanti|fara|felici|fermi|ferrari|ferrero|ferro|fiaschi|filippi|fini|fiore|fossa|fox|frank|freeman|gaber|gaglioffi|gai|gallo|gambaro|gandhi|garibaldi|gastaldi|generosa|genova|gigli|gioia|giuli|giuliano|giusti|glori|goi|gori|gould|grandi|greco|grillo|gro|guidi|gรผl|hamilton|han|heo|hong|howard|ignoti|innocenti|irani|jackson|jeon|jeong|jill|kang|kim|kirby|kong|kwon|lama|lauro|lazzari|lecca|lecce|lecchi|lecci|leccio|lee|lele|lentini|leone|lepri|levati|limiti|lipari|lippi|locatelli|lodi|ludi|luna|lupi|maestri|maggiore|magri|maier|mancini|manfredi|maniero|manzi|marazzi|marchesi|marcon|marconi|mari|marino|marinus|marra|martelli|mascheroni|mauro|merloni|mezzadri|min|mini|miranda|modena|molinari|mondadori|mondaini|moon|moro|musso|nam|neri|nervi|nobili|noli|novelli|novi|oggioni|olmi|omegna|ondina|onesti|onorati|orefice|ornati|ornella|ortolani|ortona|ostuni|otranto|palmer|pantaleo|park|parra|pasqua|pasquali|pasta|pastore|pavia|paziente|pedemonte|perego|pergamo|pergola|perini|perrone|piccoli|piemonte|pisani|politi|preti|primavera|primo|proietti|prosperi|puglia|pugliese|quinn|quinto|ramponi|rapa|rapallo|ratti|regina|ricci|rocco|rocky|romano|rosa|rosi|rossi|rota|rubini|ruggeri|ruscelli|russo|rutelli|sacchi|sage|salomone|salvatori|sami|santi|sarti|satta|savini|scala|scarpa|scopelliti|scorrano|scorza|seo|seppe|sereni|seri|serravalle|servi|sessa|sesti|sette|shannon|shea|shin|sidney|signori|silva|siri|somma|sommariva|son|song|soria|stabile|stanco|starace|stilo|stoppa|submit|suzuki|taverna|temple|terzi|tondi|tore|torti|tozzi|tramonti|trani|tripodi|tromba|trucco|turchi|turner|vacca|varchi|vedova|venero|vercelli|verde|vicentini|villani|viola|vivaldi|voltaggio|wells|williams|wilson|winter|woo|yang|ying|yoo|yoon|zaffiro|zampa|zappa|zen|zoccola|zoccoli|zoppi|zucca|zuccheri""".split("|")) diff --git a/lexicon_it.py b/lexicon_it.py new file mode 100644 index 0000000..44f35dd --- /dev/null +++ b/lexicon_it.py @@ -0,0 +1,268 @@ +"""Italian lexicon used to tell ordinary words from names and places. + +Qt-free. The NER model tags capitalised ordinary words (first-person verbs of +certificates and prescriptions, headings of medical or legal documents) as +PERSON or LOCATION. The lists below are used to *reject* such hits, or - for +the name and place lists - only to *confirm* a hit. A word missing from a +confirmation list never rejects a hit: no complete list of surnames exists. +""" + +import functools +from typing import Any, FrozenSet, Iterable, Set + +# Verb lemmas that occur as the first word of a line or clause in medical, +# legal and administrative documents. +_ARE = ( + "certific", "dichiar", "attest", "rilasci", "conferm", "raccomand", + "consigli", "autorizz", "deleg", "nomin", "ordin", "dimett", "visit", + "controll", "esamin", "accert", "constat", "verific", "comunic", "inform", + "trasmett", "allegh", "firm", "sottoscriv", "richiam", "dispon", "invit", + "diagnostic", "refert", "annot", "segnal", "propon", "suggerisc", "delegh", + "notific", "attribu", "applic", "approv", "dichiaram", "liquid", "convoc", +) +_ERE = ( + "prescriv", "richied", "chied", "decid", "rilev", "present", "conced", + "ritien", "consider", "ammett", "dispon", "stabil", "assum", "emett", + "rispond", "compil", "ricev", +) +_IRE = ( + "convoc", "costitu", "attribu", "esegu", "adempi", "sostitu", "dimin", + "prefer", "sped", "segu", "fornisc", "garantisc", "dispon", "consent", + "ordin", +) + +# (suffix set, participle suffixes) +_ARE_SUFFIXES = ("o", "i", "a", "iamo", "ate", "ano", "ato", "ata", "ati", "ate") +_ERE_SUFFIXES = ("o", "i", "e", "iamo", "ete", "ono", "uto", "uta", "uti", "ute") +_IRE_SUFFIXES = ("o", "i", "e", "iamo", "ite", "ono", "isco", "isci", "isce", + "iscono", "ito", "ita", "iti", "ite") + + +def _conjugate() -> Set[str]: + out: Set[str] = set() + for stem in _ARE: + out.update(stem + s for s in _ARE_SUFFIXES) + for stem in _ERE: + out.update(stem + s for s in _ERE_SUFFIXES) + for stem in _IRE: + out.update(stem + s for s in _IRE_SUFFIXES) + return out + + +_VERB_FORMS: FrozenSet[str] = frozenset(_conjugate() | { + "visto", "considerato", "ritenuto", "premesso", "sottoscritto", + "sottoscritta", "firmato", "firmata", "letto", "udito", "accertato", + "ordina", "dispone", "dispongo", "decreta", "decreto", "sentenzia", + "attesto", "prescrivo", "certifico", "dichiaro", "rilascio", "confermo", + "raccomando", "consiglio", "richiedo", "autorizzo", "delego", "nomino", +}) + +_MEDICAL_LEGAL: FrozenSet[str] = frozenset({ + "anamnesi", "diagnosi", "terapia", "prognosi", "referto", "esame", + "esami", "obiettivo", "paziente", "pazienti", "assistito", "assistita", + "certificato", "ricetta", "posologia", "controllo", "visita", "allegato", + "allegati", "timbro", "firma", "firme", "farmaco", "farmaci", "dose", + "dosi", "sintomi", "sintomo", "referti", "cartella", "clinica", "clinico", + "ricovero", "dimissione", "dimissioni", "ospedale", "reparto", "medico", + "medici", "infermiere", "infermiera", "pronto", "soccorso", "idoneo", + "idonea", "idoneitร ", "malattia", "infortunio", "prescrizione", + "prestazione", "prestazioni", "ambulatorio", "analisi", "ematochimici", + "ematochimico", "esito", "esiti", "conclusioni", "conclusione", "note", + "nota", "motivazione", "motivazioni", "premesso", "considerato", "ritenuto", + "decreto", "sentenza", "ordinanza", "ricorso", "ricorrente", "resistente", + "convenuto", "attore", "giudice", "tribunale", "udienza", "istanza", + "dispositivo", "pubblico", "ministero", "cancelleria", "notifica", + "oggetto", "premessa", "articolo", "comma", "regolamento", "legge", + "parti", "parte", "comparizione", "sottoscritto", "sottoscritta", + "luogo", "data", "tel", "email", "pec", "totale", "importo", "imponibile", + "fattura", "ricevuta", "scadenza", "pagamento", "saldo", "acconto", + "raccomando", "riposo", "assoluto", "giorni", "giorno", "mesi", "anni", + "tachipirina", "augmentin", +}) + +# Capitalised document words that are not personal data. +_DOCUMENT_WORDS: FrozenSet[str] = frozenset({ + "comune", "provincia", "regione", "ministero", "agenzia", "entrate", + "universitร ", "universita", "azienda", "ospedaliera", "sanitaria", "asl", + "ausl", "spett", "srl", "spa", "snc", "sas", "onlus", "repubblica", + "italiana", "stato", "governo", "inps", "inail", "ente", "ufficio", + "uffici", "dipartimento", "direzione", "servizio", "sezione", "area", + "settore", "segreteria", "protocollo", "pagina", "allegato", "copia", + "originale", "documento", "modulo", "richiesta", "domanda", "dichiarazione", + "autocertificazione", "informativa", "consenso", "privacy", "cordiali", + "saluti", "distinti", "cordialmente", "grazie", "firma", "timbro", +}) + +# Words that are also given names or surnames: never rejected on lexicon +# grounds alone, they need corroborating context. +CONFLICT_NAMES: FrozenSet[str] = frozenset({ + "rosa", "serena", "sole", "fiore", "marino", "bruno", "bianco", "speranza", + "vittoria", "pace", "conte", "porto", "grazia", "gioia", "felice", "primo", + "santo", "fabbri", "mancini", "leone", "gallo", "colombo", "greco", + "romano", "ricci", "costa", "moro", "russo", "ferrari", "neri", "verdi", + "rossi", "gialli", "bella", "forte", "lupo", "orso", "martino", +}) + +GIVEN_NAMES: FrozenSet[str] = frozenset({ + "mario", "luigi", "giuseppe", "giovanni", "antonio", "francesco", + "alessandro", "andrea", "marco", "luca", "paolo", "stefano", "roberto", + "davide", "matteo", "simone", "lorenzo", "giorgio", "franco", "claudio", + "carlo", "angelo", "pietro", "salvatore", "vincenzo", "michele", "fabio", + "massimo", "daniele", "riccardo", "federico", "gabriele", "enrico", + "maria", "anna", "giulia", "francesca", "chiara", "sara", "laura", + "elena", "valentina", "alessandra", "paola", "silvia", "federica", + "martina", "elisa", "roberta", "monica", "claudia", "lucia", "cristina", + "daniela", "stefania", "barbara", "giovanna", "angela", "rita", "teresa", + "graziano", "graziella", "nicola", "domenico", "emanuele", "filippo", + "giacomo", "tommaso", "samuele", "edoardo", "gianluca", "gianni", + "rosa", "serena", "giorgia", "alice", "arianna", "beatrice", "irene", + "marta", "veronica", "ilaria", "manuela", "patrizia", "carla", "marina", +}) + +REGIONS_PROVINCES: FrozenSet[str] = frozenset({ + "abruzzo", "basilicata", "calabria", "campania", "emilia-romagna", + "friuli-venezia", "lazio", "liguria", "lombardia", "marche", "molise", + "piemonte", "puglia", "sardegna", "sicilia", "toscana", "trentino", + "umbria", "veneto", "valle d'aosta", + "roma", "milano", "napoli", "torino", "palermo", "genova", "bologna", + "firenze", "bari", "catania", "venezia", "verona", "messina", "padova", + "trieste", "brescia", "parma", "taranto", "prato", "modena", "reggio", + "perugia", "livorno", "ravenna", "cagliari", "foggia", "rimini", + "salerno", "ferrara", "sassari", "latina", "monza", "siracusa", + "pescara", "bergamo", "forlรฌ", "trento", "vicenza", "terni", "bolzano", + "novara", "piacenza", "ancona", "andria", "arezzo", "udine", "cesena", + "lecce", "pesaro", "la spezia", "como", "pisa", "lucca", "cosenza", + "caserta", "avellino", "benevento", "potenza", "matera", "campobasso", + "catanzaro", "crotone", "agrigento", "trapani", "ragusa", "enna", + "caltanissetta", "aosta", "cuneo", "asti", "alessandria", "biella", + "varese", "lecco", "sondrio", "pavia", "cremona", "mantova", "lodi", + "belluno", "treviso", "rovigo", "gorizia", "pordenone", "savona", + "imperia", "massa", "carrara", "pistoia", "siena", "grosseto", + "viterbo", "rieti", "frosinone", "l'aquila", "teramo", "chieti", + "isernia", "brindisi", "barletta", "oristano", "nuoro", + "italia", "francia", "germania", "spagna", "svizzera", "austria", +}) + +COMMON_WORDS: FrozenSet[str] = frozenset( + _VERB_FORMS | _MEDICAL_LEGAL | _DOCUMENT_WORDS +) + +# Italian function words (articles, prepositions, conjunctions, pronouns) that +# may begin a capitalised sentence start. +FUNCTION_WORDS: FrozenSet[str] = frozenset({ + "il", "lo", "la", "i", "gli", "le", "un", "uno", "una", "di", "a", "da", + "in", "con", "su", "per", "tra", "fra", "e", "ed", "o", "ma", "se", "che", + "chi", "non", "piรน", "come", "quando", "dove", "anche", "ogni", "questo", + "questa", "questi", "queste", "quello", "quella", "del", "della", "dei", + "delle", "dello", "al", "alla", "ai", "alle", "allo", "nel", "nella", + "nei", "nelle", "sul", "sulla", "sui", "sulle", "dal", "dalla", "dai", + "dalle", "io", "tu", "lui", "lei", "noi", "voi", "loro", "mi", "ti", "si", + "ci", "vi", "ne", "ho", "ha", "hanno", "รจ", "sono", "era", "sarร ", "poi", + "dopo", "prima", "oggi", "ieri", "domani", "tutti", "tutto", "nessun", + "stesso", "stessa", "presso", "sotto", "sopra", "senza", "secondo", + "nato", "nata", "residente", "domiciliato", "domiciliata", +}) + + +# Italian vehicle-plate style province codes (107 provinces / metropolitan cities). +PROVINCE_SIGLE: FrozenSet[str] = frozenset( + "AG AL AN AO AP AQ AR AT AV BA BG BI BL BN BO BR BS BT BZ CA CB CE CH CL CN " + "CO CR CS CT CZ EN FC FE FG FI FM FR GE GO GR IM IS KR LC LE LI LO LT LU MB " + "MC ME MI MN MO MS MT NA NO NU OR PA PC PD PE PG PI PN PO PR PT PU PV PZ RA " + "RC RE RG RI RM RN RO SA SI SO SP SR SS SU SV TA TE TN TO TP TR TS TV UD VA " + "VB VC VE VI VR VT VV".split() +) + + +def is_common_word(token: str) -> bool: + """True for ordinary Italian words that are never PII on their own.""" + t = token.strip().lower().strip(".,;:!?()[]\"'ยซยป") + if not t: + return True + if t in CONFLICT_NAMES: + return False + return t in COMMON_WORDS or t in FUNCTION_WORDS + + +# Frequent Italian surnames (a small curated list: confirmation only). +SURNAMES: FrozenSet[str] = frozenset( + "rossi russo ferrari esposito bianchi romano colombo ricci marino greco bruno " + "gallo conti de luca mancini costa giordano rizzo lombardi moretti barbieri " + "fontana santoro mariani rinaldi caruso ferrara galli martini leone longo " + "gentile martinelli vitale lombardo serra coppola de santis d'angelo marchetti " + "parisi villa conte ferraro ferri fabbri bianco marini grasso valentini messina " + "sala de angelis gatti pellegrini palumbo sanna farina rizzi monti cattaneo " + "morelli amato silvestri mazza testa grassi pellegrino carbone giuliani benedetti " + "barone rossetti caputo montanari guerra palmieri bernardi martino fiore de rosa " + "ferretti bellini basile riva donati piras vitali battaglia sartori neri costantini " + "milani pagano ruggiero sorrentino d'amico orlando damico negri verdi gialli " + "melis cocco leoni pace marchi mele".split() +) + + +def is_known_surname(token: str) -> bool: + key = token.strip().lower().strip(".,;:") + if key in SURNAMES: + return True + d = _data() + return bool(d and hasattr(d, "NAMES_SURNAMES") and key in d.NAMES_SURNAMES + and key not in d.NAMES_AMBIGUOUS) + + +def is_given_name(token: str) -> bool: + key = token.strip().lower().strip(".,;:") + if key in GIVEN_NAMES: + return True + d = _data() + return bool(d and hasattr(d, "NAMES_GIVEN") and key in d.NAMES_GIVEN + and key not in d.NAMES_AMBIGUOUS) + + +@functools.lru_cache(maxsize=1) +def _data() -> Any: + """ISTAT-derived place data, imported on first use (not at start-up).""" + try: + import lexicon_data_it + + return lexicon_data_it + except Exception: + return None + + +def _place_key(token: str) -> str: + return token.strip().lower().replace("โ€™", "'").strip(".,;:()") + + +def is_known_place(token: str) -> bool: + """A comune, region or country - confirmation only, never a rejection. + + Names that are also ordinary words (Sale, Re, Orso...) are *not* counted + here; see :func:`is_ambiguous_place`. + """ + key = _place_key(token) + if key in REGIONS_PROVINCES: + return True + d = _data() + return bool(d and (key in d.COMUNI or key in d.REGIONS) and key not in d.COMUNI_AMBIGUOUS) + + +def is_ambiguous_place(token: str) -> bool: + """A comune whose name is also an ordinary word or a very short name.""" + key = _place_key(token) + d = _data() + return bool(d and key in d.COMUNI and key in d.COMUNI_AMBIGUOUS + and key not in REGIONS_PROVINCES) + + +def province_codes() -> FrozenSet[str]: + d = _data() + return frozenset(PROVINCE_SIGLE | (d.PROVINCE_SIGLE if d else frozenset())) + + +def verb_forms() -> FrozenSet[str]: + return _VERB_FORMS + + +def extend(words: Iterable[str]) -> FrozenSet[str]: + """Return COMMON_WORDS extended with caller-supplied words.""" + return frozenset(COMMON_WORDS | {w.lower() for w in words}) diff --git a/licences_info.py b/licences_info.py new file mode 100644 index 0000000..6686e2c --- /dev/null +++ b/licences_info.py @@ -0,0 +1,96 @@ +"""Licences of NullifyPDF and of the components it uses, with official links. + +Single source for the "Info" window and for the README table (a test keeps the +README in step). Informational only: each component remains under its own +licence, which is what the notice says. +""" + +from __future__ import annotations + +import html +from typing import List, NamedTuple + +REPO = "https://github.com/overwrite00/NullifyPDF" +AGPL_URL = "https://opensource.org/license/agpl-v3" +LGPL_URL = "https://opensource.org/license/lgpl-3-0" +CC_NC_SA_URL = "https://creativecommons.org/licenses/by-nc-sa/3.0/" + +APP_LICENCE = "AGPL-3.0-or-later" +NOTICE = ( + "NullifyPDF รจ software libero e gratuito, distribuito con licenza " + f"{APP_LICENCE}. Ogni componente e ogni modello usati dal programma restano " + "soggetti alla propria licenza, indicata qui sotto con il collegamento al testo " + "ufficiale: usando il programma se ne rispettano le condizioni. Il modello " + "italiano di spaCy รจ per solo uso non commerciale e viene scaricato solo dopo " + "il tuo consenso." +) + + +class Component(NamedTuple): + name: str + licence: str + url: str # official page or licence text + role: str + role_en: str = "" + licence_en: str = "" + + +COMPONENTS: List[Component] = [ + Component("NullifyPDF", APP_LICENCE, f"{REPO}/blob/main/LICENSE", "questo programma", "this program"), + Component("PyMuPDF / MuPDF", "AGPL-3.0 o commerciale (Artifex)", + "https://pymupdf.readthedocs.io/en/latest/about.html", "lettura e redazione dei PDF", "PDF reading and redaction", + "AGPL-3.0 or commercial (Artifex)"), + Component("PySide6 / Qt", "LGPL-3.0", LGPL_URL, "interfaccia grafica", "graphical interface"), + Component("Presidio", "MIT", "https://github.com/microsoft/presidio/blob/main/LICENSE", + "riconoscimento dei dati personali", "personal-data detection"), + Component("spaCy", "MIT", "https://github.com/explosion/spaCy/blob/master/LICENSE", + "elaborazione del linguaggio", "language processing"), + Component("spaCy it_core_news_md", "CC BY-NC-SA 3.0 (solo uso non commerciale)", CC_NC_SA_URL, + "modello italiano, scaricato dopo il consenso", "Italian model, downloaded after consent", + "CC BY-NC-SA 3.0 (non-commercial use only)"), + Component("onnxruntime", "MIT", "https://github.com/microsoft/onnxruntime/blob/main/LICENSE", + "esecuzione dei modelli locali", "running local models"), + Component("tokenizers", "Apache-2.0", "https://github.com/huggingface/tokenizers/blob/main/LICENSE", + "tokenizzazione dei modelli locali", "tokenizing for local models"), + Component("cryptography", "Apache-2.0 o BSD-3-Clause", + "https://github.com/pyca/cryptography/blob/main/LICENSE", "cifratura", "encryption", + "Apache-2.0 or BSD-3-Clause"), + Component("keyring", "MIT", "https://github.com/jaraco/keyring", "archivio chiavi (Linux/macOS)", "key storage (Linux/macOS)"), + Component("tldextract", "BSD-3-Clause", + "https://github.com/john-kurkowski/tldextract/blob/master/LICENSE", "riconoscimento URL", "URL recognition"), + Component("NumPy", "BSD-3-Clause", "https://github.com/numpy/numpy/blob/main/LICENSE.txt", + "calcolo numerico", "numerical computing"), + Component("Tesseract OCR", "Apache-2.0", "https://github.com/tesseract-ocr/tesseract/blob/main/LICENSE", + "OCR dei documenti scansionati", "OCR of scanned documents"), + Component("tessdata_fast (eng, ita)", "Apache-2.0", + "https://github.com/tesseract-ocr/tessdata_fast/blob/main/LICENSE", "dati OCR", "OCR language data"), + Component("ISTAT, elenco dei comuni", "CC BY 4.0", "https://creativecommons.org/licenses/by/4.0/", + "nomi dei luoghi", "place names"), + Component("Wikidata", "CC0 1.0", "https://creativecommons.org/publicdomain/zero/1.0/", + "nomi e cognomi", "given names and surnames"), +] + +EXTRA_LINKS = [ + ("LICENSE", f"{REPO}/blob/main/LICENSE"), + ("THIRD_PARTY_NOTICES", f"{REPO}/blob/main/THIRD_PARTY_NOTICES.md"), +] + + +def about_html() -> str: + """HTML for the scrollable licence area of the Info window.""" + rows = "".join( + f"
  • {html.escape(c.name)}" + f" — {html.escape(c.licence)} ({html.escape(c.role)})
    • " + for c in COMPONENTS + ) + extra = " · ".join( + f"{html.escape(n)}" for n, u in EXTRA_LINKS + ) + return f"

    {html.escape(NOTICE)}

      {rows}

    {extra}

    " + + +def readme_markdown() -> str: + lines = ["| Component | Licence | Used for |", "|---|---|---|"] + for c in COMPONENTS: + lines.append(f"| [{c.name}]({c.url}) | {c.licence_en or c.licence} | {c.role_en} |") + return "\n".join(lines) diff --git a/model_dialog.py b/model_dialog.py new file mode 100644 index 0000000..72ab95a --- /dev/null +++ b/model_dialog.py @@ -0,0 +1,260 @@ +"""Italian model-catalog dialog; it never downloads without explicit consent.""" + +from __future__ import annotations + +from typing import Any, Iterable, Optional + +from PySide6.QtCore import QThread, Qt, Signal +from PySide6.QtWidgets import ( + QDialog, + QDialogButtonBox, + QLabel, + QListWidget, + QListWidgetItem, + QMessageBox, + QPushButton, + QTextBrowser, + QVBoxLayout, +) + +from help_guide import apply_tooltips +from model_manager import ModelCandidate, ModelManifest, ModelStore + + +class ModelDownloadWorker(QThread): + """Download and verify every file of a manifest off the UI thread.""" + + progress_sig = Signal(int) # whole-model percentage + done_sig = Signal() + error_sig = Signal(str) + cancelled_sig = Signal() + + def __init__(self, store: ModelStore, manifest: ModelManifest) -> None: + super().__init__() + self._store = store + self._manifest = manifest + self._cancel = False + self._completed = 0 + self._total = max(1, manifest.total_size) + + def cancel(self) -> None: + self._cancel = True + + def _on_bytes(self, done: int, total: int) -> None: + if self._cancel: + raise InterruptedError("download annullato") + value = int(((self._completed + min(done, total)) / self._total) * 100) + self.progress_sig.emit(min(100, value)) + + def run(self) -> None: + try: + for item in self._manifest.files: + url = ( + f"https://huggingface.co/{self._manifest.repo_id}/resolve/" + f"{self._manifest.revision}/{item.name}?download=true" + ) + self._store.install_file( + self._manifest, item, url, timeout=120.0, progress=self._on_bytes + ) + self._completed += item.size + self.progress_sig.emit(min(100, int(self._completed * 100 / self._total))) + if not self._store.is_installed(self._manifest): + raise RuntimeError("verifica finale del modello fallita") + self.done_sig.emit() + except InterruptedError: + self.cancelled_sig.emit() + except Exception as exc: + self.error_sig.emit(str(exc)) + + +class ModelDialog(QDialog): + """Show the reviewed model shortlist and its verification state.""" + + def __init__( + self, + candidates: Iterable[ModelCandidate], + selected_id: Optional[str] = None, + manifests: Iterable[ModelManifest] = (), + store: Optional[ModelStore] = None, + parent: Any = None, + ) -> None: + super().__init__(parent) + self.setWindowTitle("Modelli locali") + self.resize(700, 520) + self._candidates = list(candidates) + self._selected_id = selected_id + self._manifests = {m.model_id: m for m in manifests} + self._store = store + lay = QVBoxLayout(self) + intro = QLabel( + "I modelli vengono eseguiti localmente e producono solo suggerimenti. " + "Il download sarร  consentito soltanto quando revisione, licenza e SHA-256 " + "saranno fissati nel manifest dell'applicazione." + ) + intro.setWordWrap(True) + lay.addWidget(intro) + self.list = QListWidget() + self.list.setObjectName("model_list") + for candidate in self._candidates: + item = QListWidgetItem( + f"{candidate.title} โ€” {candidate.size_label} โ€” {candidate.status}" + ) + item.setData(Qt.ItemDataRole.UserRole, candidate.model_id) + self.list.addItem(item) + lay.addWidget(self.list) + self.details = QTextBrowser() + self.details.setObjectName("model_details") + lay.addWidget(self.details) + self.btn_download = QPushButton("Scarica modello") + self.btn_download.setObjectName("model_download") + self.btn_download.setEnabled(False) + self.btn_download.setToolTip( + "Il download รจ attivo solo per modelli con revisione, file, licenza e SHA-256 verificati." + ) + lay.addWidget(self.btn_download) + self._buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close) + self._buttons.rejected.connect(self.reject) + lay.addWidget(self._buttons) + self._worker: Optional[ModelDownloadWorker] = None + self.list.currentItemChanged.connect(self._show_details) + self.btn_download.clicked.connect(self._download_selected) + if self.list.count(): + row = 0 + for i in range(self.list.count()): + if self.list.item(i).data(Qt.ItemDataRole.UserRole) == selected_id: + row = i + break + self.list.setCurrentRow(row) + apply_tooltips(self) + + def _show_details(self, current: Optional[QListWidgetItem], _previous: Any) -> None: + if current is None: + self.details.clear() + return + model_id = current.data(Qt.ItemDataRole.UserRole) + candidate = next((c for c in self._candidates if c.model_id == model_id), None) + if candidate is None: + return + self.details.setPlainText( + f"Nome: {candidate.title}\n" + f"Repository: {candidate.repo_id}\n" + f"Licenza: {candidate.license}\n" + f"Runtime: {candidate.runtime}\n" + f"Lingue: {', '.join(candidate.languages)}\n" + f"Copertura dichiarata: {candidate.scope}\n" + f"Stato: {candidate.status}\n\n" + f"{candidate.notes}" + ) + manifest = self._manifests.get(candidate.model_id) + if manifest is not None: + self.details.append( + f"\n\nFonte fissata: {manifest.source_url}\n" + f"File manifest: {len(manifest.files)}; SHA-256 verificati localmente." + ) + self.btn_download.setEnabled(manifest is not None and self._store is not None) + if manifest is not None and self._store is not None and self._store.is_installed(manifest): + self.btn_download.setText("Modello giร  installato") + self.btn_download.setEnabled(False) + else: + self.btn_download.setText("Scarica modello") + if manifest is None: + self.btn_download.setToolTip( + "Modello sperimentale non scaricabile: manca un manifest verificato " + "con revisione, licenza, dimensioni e SHA-256." + ) + else: + self.btn_download.setToolTip( + "Download HTTPS da Hugging Face alla revisione fissata nel manifest." + ) + + # -- download with progress shown inside the button ------------------- + def _set_button_progress(self, percent: int) -> None: + """Fill the button from the left like a progress bar and show the %.""" + p = min(max(percent, 0), 100) / 100.0 + edge = min(p, 0.998) + self.btn_download.setText(f"Download in corsoโ€ฆ {percent}% (clic per annullare)") + self.btn_download.setStyleSheet( + "QPushButton#model_download { background: qlineargradient(" + "x1:0, y1:0, x2:1, y2:0, " + f"stop:0 #0284c7, stop:{edge:.3f} #0284c7, " + f"stop:{edge + 0.001:.3f} #334155, stop:1 #334155); }}" + ) + + def _set_downloading(self, on: bool) -> None: + self.list.setEnabled(not on) + close = self._buttons.button(QDialogButtonBox.StandardButton.Close) + if close is not None: + close.setEnabled(not on) + if not on: + self.btn_download.setStyleSheet("") + + def is_downloading(self) -> bool: + return self._worker is not None + + def reject(self) -> None: + if self._worker is None: + super().reject() + + def closeEvent(self, event: Any) -> None: + if self._worker is not None: + event.ignore() + else: + super().closeEvent(event) + + def _download_selected(self) -> None: + if self._worker is not None: # second click while downloading: cancel + self._worker.cancel() + self.btn_download.setText("Annullamentoโ€ฆ") + self.btn_download.setEnabled(False) + return + current = self.list.currentItem() + if current is None or self._store is None: + return + model_id = current.data(Qt.ItemDataRole.UserRole) + manifest = self._manifests.get(model_id) + if manifest is None: + return + answer = QMessageBox.question( + self, + "Consenso al download", + f"Scaricare {manifest.model_id} ({manifest.total_size / 1024**2:.1f} MB)?\n" + f"Licenza: {manifest.license}\n" + f"Revisione: {manifest.revision}\n\n" + "I file saranno salvati nella cartella locale dell'applicazione.", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if answer != QMessageBox.StandardButton.Yes: + return + self._set_downloading(True) + self._set_button_progress(0) + worker = ModelDownloadWorker(self._store, manifest) + # Bound methods, not lambdas capturing self (no reference cycle). + worker.progress_sig.connect(self._set_button_progress) + worker.done_sig.connect(self._on_download_done) + worker.error_sig.connect(self._on_download_error) + worker.cancelled_sig.connect(self._on_download_cancelled) + self._worker = worker + worker.start() + + def _finish_worker(self) -> None: + worker = self._worker + self._worker = None + if worker is not None: + worker.wait() + self.btn_download.setEnabled(True) + self._set_downloading(False) + + def _on_download_done(self) -> None: + self._finish_worker() + QMessageBox.information(self, "Download completato", "Il modello รจ stato verificato e installato.") + self._show_details(self.list.currentItem(), None) + + def _on_download_error(self, message: str) -> None: + self._finish_worker() + self._show_details(self.list.currentItem(), None) + QMessageBox.critical(self, "Download non riuscito", message) + + def _on_download_cancelled(self) -> None: + self._finish_worker() + self._show_details(self.list.currentItem(), None) diff --git a/model_manager.py b/model_manager.py new file mode 100644 index 0000000..f796a16 --- /dev/null +++ b/model_manager.py @@ -0,0 +1,387 @@ +"""Local model catalog, integrity checks and user-managed downloads. + +This module deliberately does not import an inference framework. Model files +are untrusted data until their manifest, revision, size and SHA-256 digest have +been checked. Loading an ONNX graph belongs to a separate runtime adapter. +""" + +from __future__ import annotations + +import hashlib +import os +import pathlib +import platform +import re +import tempfile +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Callable, Iterable, Optional, Tuple + + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_REVISION = re.compile(r"^[0-9a-f]{40}$") +_SAFE_SUFFIXES = frozenset({".onnx", ".ort", ".data", ".json", ".safetensors"}) +_MAX_MANIFEST_FILES = 32 + + +class ModelManifestError(ValueError): + """Raised when a model manifest is malformed or unsafe.""" + + +@dataclass(frozen=True) +class ModelFile: + name: str + size: int + sha256: str + + def validate(self) -> None: + path = pathlib.PurePosixPath(self.name) + if not self.name or path.is_absolute() or ".." in path.parts: + raise ModelManifestError("percorso file modello non sicuro") + if path.suffix.lower() not in _SAFE_SUFFIXES: + raise ModelManifestError(f"formato modello non consentito: {self.name}") + if self.size <= 0: + raise ModelManifestError("dimensione file non valida") + if not _SHA256.fullmatch(self.sha256): + raise ModelManifestError("SHA-256 non valido") + + +@dataclass(frozen=True) +class ModelManifest: + model_id: str + repo_id: str + revision: str + license: str + runtime: str + labels: Tuple[str, ...] + files: Tuple[ModelFile, ...] + enabled: bool = False + notes: str = "" + + def validate(self) -> None: + if not self.model_id or not self.repo_id: + raise ModelManifestError("identificativo modello mancante") + if not _REVISION.fullmatch(self.revision): + raise ModelManifestError("la revisione deve essere un commit SHA-1 completo") + if self.runtime not in {"onnx", "ort"}: + raise ModelManifestError("runtime non consentito") + if not self.license.strip(): + raise ModelManifestError("licenza mancante") + if not self.files or len(self.files) > _MAX_MANIFEST_FILES: + raise ModelManifestError("elenco file modello non valido") + names = set() + for item in self.files: + item.validate() + if item.name in names: + raise ModelManifestError("file duplicato nel manifest") + names.add(item.name) + if self.runtime == "onnx" and not any( + f.name.lower().endswith(".onnx") for f in self.files + ): + raise ModelManifestError("manifest ONNX senza file .onnx") + + @property + def total_size(self) -> int: + return sum(item.size for item in self.files) + + @property + def source_url(self) -> str: + return f"https://huggingface.co/{self.repo_id}/tree/{self.revision}" + + +@dataclass(frozen=True) +class HardwareProfile: + cpu_count: int + memory_bytes: Optional[int] + system: str = platform.system() + + +@dataclass(frozen=True) +class ModelRecommendation: + level: str + model_id: Optional[str] + reason: str + + +@dataclass(frozen=True) +class ModelCandidate: + """Reviewed catalog entry which is not necessarily installable yet.""" + + model_id: str + title: str + repo_id: str + license: str + runtime: str + languages: Tuple[str, ...] + size_label: str + scope: str + status: str + notes: str + + +MODEL_CANDIDATES: Tuple[ModelCandidate, ...] = ( + ModelCandidate( + "minilmv2-en-it", + "MiniLMv2 EN/IT", + "jkomyno/minilmv2-l6-h384-ner-en-it-q8", + "MIT", + "ONNX INT8", + ("en", "it"), + "circa 30 MB", + "persone, organizzazioni, indirizzi", + "verificato", + "Candidato leggero; non copre gli identificativi strutturati.", + ), + ModelCandidate( + "piiscan", + "PiiScan", + "vectorsense/piiscan", + "Apache-2.0", + "ONNX INT8", + ("multilingue",), + "circa 139 MB", + "PII generiche e identificativi", + "manifest verificato; italiano da misurare", + "La scheda dichiara 30 lingue, ma non elenca esplicitamente l'italiano.", + ), + ModelCandidate( + "openmed-fastclinical", + "OpenMed PII Italian FastClinical Small", + "OpenMed/OpenMed-PII-Italian-FastClinical-Small-82M-v1-onnx-android", + "Apache-2.0", + "ONNX INT8", + ("it",), + "circa 190 MB", + "PII e identificativi clinici italiani", + "verificato", + "Modello italiano specializzato in testo clinico; esempi dichiarati sintetici.", + ), + ModelCandidate( + "nym-pii", + "nym PII multilingual", + "Wismut/nym-pii-multilingual", + "MIT", + "ONNX INT8", + ("multilingue",), + "circa 359 MB INT8", + "PII, OCR rumoroso, documenti strutturati", + "sperimentale", + "Maggiore costo di memoria; qualitร  italiana da misurare.", + ), + ModelCandidate( + "eu-pii-safeguard", + "EU PII Safeguard", + "tabularisai/eu-pii-safeguard", + "da verificare", + "ONNX derivato", + ("26 lingue UE",), + "non verificata", + "PII europee", + "sperimentale", + "La conversione ONNX e la licenza del sorgente richiedono verifica.", + ), +) + + +MINILM_REVISION = "61dd72a5cac8e8d3e2702b57cba8154a8720c2af" +PIISCAN_REVISION = "87e6ff6245ea325880af9f38e6dad8493f2ec123" + + +VERIFIED_MANIFESTS: Tuple[ModelManifest, ...] = ( + ModelManifest( + model_id="minilmv2-en-it", + repo_id="jkomyno/minilmv2-l6-h384-ner-en-it-q8", + revision=MINILM_REVISION, + license="MIT", + runtime="onnx", + labels=("O", "B-ADDRESS", "I-ADDRESS", "B-ORGANIZATION", "I-ORGANIZATION", "B-PERSON", "I-PERSON"), + files=( + ModelFile("config.json", 1075, "8f4608824f674a3a2427fc6751ebf3cc64766f3c14e3976cb0c46fbbb4dbc83b"), + ModelFile("tokenizer.json", 3558647, "7b62d797c95d1563d3467f48daad0eca3e150f2d587e7e687b557c9ef1b25473"), + ModelFile("tokenizer_config.json", 387, "e8ba8e4ec2ade6868d62c5d8a993c49cd44cdcf34aadc58bd764b5f4e752e338"), + ModelFile("onnx/model_quantized.onnx", 30585613, "d1fb8efd376b7026faeba43aa60b602d28902d3a99ca9bfa19bce1edcbded5a8"), + ), + enabled=True, + notes="Modello leggero per PERSON, ORGANIZATION e ADDRESS.", + ), + ModelManifest( + model_id="piiscan", + repo_id="vectorsense/piiscan", + revision=PIISCAN_REVISION, + license="Apache-2.0", + runtime="onnx", + labels=("O", "B-PERSON_NAME", "I-PERSON_NAME", "B-DATE_TIME", "I-DATE_TIME", "B-EMAIL_ADDRESS", "I-EMAIL_ADDRESS", "B-CITY", "I-CITY", "B-STREET_ADDRESS", "I-STREET_ADDRESS", "B-GOVERNMENT_ID", "I-GOVERNMENT_ID", "B-PHONE_NUMBER", "I-PHONE_NUMBER", "B-AGE", "I-AGE", "B-POSTAL_CODE", "I-POSTAL_CODE", "B-PAYMENT_CARD_NUMBER", "I-PAYMENT_CARD_NUMBER", "B-DRIVER_LICENSE_NUMBER", "I-DRIVER_LICENSE_NUMBER", "B-TAX_ID", "I-TAX_ID", "B-PASSPORT_NUMBER", "I-PASSPORT_NUMBER"), + files=( + ModelFile("config.json", 2049, "42cafc5b1ebcaafd3fa99a261f042bffc627c7f9433f2ee70421e2f9cff85f56"), + ModelFile("labels.json", 547, "d162bf06f29a8e5f09e7158c5847a531f0637e74b4b615286ff5fe931d6f7104"), + ModelFile("thresholds.json", 1258, "098fddcfa67451a0b3e4243018f6cb540660ebc5abe28888e0ddd75269e6d6e2"), + ModelFile("tokenizer.json", 2919461, "2c550ee6d187a1eeb0982a7af79b42748d7815c08b7221d01a8b06c4802316c3"), + ModelFile("tokenizer_config.json", 1397, "d22f2844d37c4f6e15b7f5220c4e33597f328ff7461f585e759f7f9914d0a742"), + ModelFile("model.int8.onnx", 135144362, "3ddde8cf53d34c4a6fc4c29ea2c1a28b65efdb84761e945b9afb45984a9c5342"), + ), + enabled=True, + notes="Modello CPU multilanguage con offset esatti; italiano da misurare sul corpus dell'app.", + ), + ModelManifest( + model_id="openmed-fastclinical", + repo_id="OpenMed/OpenMed-PII-Italian-FastClinical-Small-82M-v1-onnx-android", + revision="05cef784e0234e82c5806d2ec2451a1385c1d962", + license="Apache-2.0", + runtime="onnx", + labels=("O", "B-FIRSTNAME", "B-LASTNAME", "B-CITY", "B-ORGANIZATION", "B-EMAIL", "B-PHONE", "B-IBAN", "B-DATEOFBIRTH", "B-STREET", "B-ZIPCODE"), + files=( + ModelFile("config.json", 4505, "a91d68ba87154e94053ef61db86583c60703aeb3d4961058ae9886fc60487996"), + ModelFile("id2label.json", 1656, "a855c964ef7763ec51b63aad7ec43e43ae5a89f1e5f221cd2852326107ad7d95"), + ModelFile("tokenizer.json", 3558906, "ca4e5a9389c0dd76c509ba115a772da3a85c43aa10b9dfb966782907062de52b"), + ModelFile("tokenizer_config.json", 536, "828fb35f1e12914318c360d7b04f4c3fc4b7a9912b81cdf9ea37fb7a7da7d173"), + ModelFile("model_int8.onnx", 199020782, "c29ad148bb8bd33392a2dd94bb469108ca70ab2e4fe3cca374a879cfb942d62c"), + ), + enabled=True, + notes="ONNX INT8 CPU, 82M parametri, contesto 512; copertura italiana clinica da validare sul corpus dell'app.", + ), +) + + +CATALOG: Tuple[ModelManifest, ...] = VERIFIED_MANIFESTS + + +def _windows_memory_bytes() -> Optional[int]: + if platform.system() != "Windows": + return None + try: + import ctypes + + class Status(ctypes.Structure): + _fields_ = [ + ("length", ctypes.c_ulong), + ("memory_load", ctypes.c_ulong), + ("total", ctypes.c_ulonglong), + ("available", ctypes.c_ulonglong), + ("total_page", ctypes.c_ulonglong), + ("available_page", ctypes.c_ulonglong), + ("total_virtual", ctypes.c_ulonglong), + ("available_virtual", ctypes.c_ulonglong), + ("available_extended", ctypes.c_ulonglong), + ] + + status = Status() + status.length = ctypes.sizeof(Status) + windll = getattr(ctypes, "windll", None) + if windll is not None and windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return int(status.total) + except (AttributeError, OSError, TypeError): + pass + return None + + +def detect_hardware() -> HardwareProfile: + """Return conservative hardware information using only the standard library.""" + return HardwareProfile( + cpu_count=max(1, int(os.cpu_count() or 1)), + memory_bytes=_windows_memory_bytes(), + ) + + +def recommend_level( + hardware: HardwareProfile, + manifests: Iterable[ModelManifest] = CATALOG, +) -> ModelRecommendation: + """Recommend a level without enabling an unverified model.""" + available = [m for m in manifests if m.enabled] + if hardware.memory_bytes is not None and hardware.memory_bytes < 8 * 1024**3: + return ModelRecommendation("nessuno", None, "RAM inferiore alla soglia consigliata") + if not available: + return ModelRecommendation( + "nessuno", None, "nessun modello verificato disponibile nel catalogo" + ) + small = min(available, key=lambda m: m.total_size) + return ModelRecommendation( + "piccolo", small.model_id, "primo modello verificato compatibile disponibile" + ) + + +def sha256_file(path: pathlib.Path, chunk_size: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_model_files(directory: pathlib.Path, manifest: ModelManifest) -> None: + """Verify presence, exact size and digest of every manifest file.""" + manifest.validate() + root = directory.resolve() + for item in manifest.files: + path = (root / pathlib.PurePosixPath(item.name)).resolve() + if root != path and root not in path.parents: + raise ModelManifestError("file modello fuori dalla directory") + if not path.is_file(): + raise ModelManifestError(f"file modello mancante: {item.name}") + if path.stat().st_size != item.size: + raise ModelManifestError(f"dimensione inattesa: {item.name}") + if sha256_file(path) != item.sha256: + raise ModelManifestError(f"hash non corrispondente: {item.name}") + + +class ModelStore: + """Store models outside the repository and install them atomically.""" + + def __init__(self, root: pathlib.Path) -> None: + self.root = root + + def model_dir(self, manifest: ModelManifest) -> pathlib.Path: + manifest.validate() + return self.root / manifest.model_id / manifest.revision + + def is_installed(self, manifest: ModelManifest) -> bool: + try: + verify_model_files(self.model_dir(manifest), manifest) + return True + except (OSError, ModelManifestError): + return False + + def install_file( + self, + manifest: ModelManifest, + item: ModelFile, + url: str, + *, + timeout: float = 30.0, + progress: Optional[Callable[[int, int], None]] = None, + ) -> pathlib.Path: + """Download one HTTPS file, verify it, then atomically install it.""" + manifest.validate() + item.validate() + if not url.lower().startswith("https://"): + raise ModelManifestError("sono consentiti solo download HTTPS") + destination = self.model_dir(manifest) / pathlib.PurePosixPath(item.name) + destination.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp( + prefix=".model-", suffix=".part", dir=str(destination.parent) + ) + os.close(fd) + temporary = pathlib.Path(temp_name) + try: + request = urllib.request.Request(url, headers={"Accept": "application/octet-stream"}) + with urllib.request.urlopen(request, timeout=timeout) as response, temporary.open("wb") as out: + done = 0 + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + out.write(chunk) + done += len(chunk) + if progress is not None: + progress(done, item.size) + if temporary.stat().st_size != item.size: + raise ModelManifestError(f"dimensione inattesa: {item.name}") + if sha256_file(temporary) != item.sha256: + raise ModelManifestError(f"hash non corrispondente: {item.name}") + os.replace(temporary, destination) + return destination + except (OSError, urllib.error.URLError, ModelManifestError): + temporary.unlink(missing_ok=True) + raise + diff --git a/onnx_ner.py b/onnx_ner.py new file mode 100644 index 0000000..8c00033 --- /dev/null +++ b/onnx_ner.py @@ -0,0 +1,135 @@ +"""Small ONNX token-classification adapter without PyTorch.""" + +from __future__ import annotations + +import json +import pathlib +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, Optional, Tuple + +import numpy as np + +from model_manager import ModelManifest, ModelManifestError, verify_model_files + + +@dataclass(frozen=True) +class ModelEntity: + start: int + end: int + text: str + label: str + score: float + + +def _group_label(label: str) -> Optional[str]: + label = label.upper().removeprefix("B-").removeprefix("I-") + if label in {"PERSON", "PERSON_NAME", "FULLNAME", "FIRSTNAME", "LASTNAME", "MIDDLENAME"}: + return "PERSON" + if label in {"ORGANIZATION", "ORG"}: + return "LOCATION" + if label in {"ADDRESS", "STREET_ADDRESS", "CITY", "POSTAL_CODE"}: + return "LOCATION" + if label == "EMAIL_ADDRESS": + return "EMAIL_ADDRESS" + if label == "EMAIL": + return "EMAIL_ADDRESS" + if label == "PHONE_NUMBER": + return "PHONE_NUMBER" + if label == "PHONE": + return "PHONE_NUMBER" + if label == "PAYMENT_CARD_NUMBER": + return "CREDIT_CARD" + if label == "CREDITCARD": + return "CREDIT_CARD" + if label == "DRIVER_LICENSE_NUMBER": + return "IT_DRIVER_LICENSE" + if label == "PASSPORT_NUMBER": + return "IT_PASSPORT" + if label == "TAX_ID": + return "IT_FISCAL_CODE" + if label == "IBAN": + return "IBAN_CODE" + if label in {"DATEOFBIRTH", "DATE_OF_BIRTH"}: + return "DATE_OF_BIRTH" + if label in {"GOVERNMENT_ID", "ID_DOC"}: + return "IT_IDENTITY_CARD" + return None + + +class OnnxTokenDetector: + """Run a verified token classifier and return source-text entities.""" + + def __init__(self, model_dir: pathlib.Path, manifest: ModelManifest) -> None: + verify_model_files(model_dir, manifest) + try: + import onnxruntime as ort # type: ignore[import-untyped] + from tokenizers import Tokenizer + except ImportError as exc: + raise RuntimeError("servono onnxruntime e tokenizers per il modello locale") from exc + self.manifest = manifest + self._ort = ort + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + self._tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json")) + max_length = min(int(config.get("max_position_embeddings", 256)), 512) + pad_id = int(config.get("pad_token_id", 0)) + self._tokenizer.enable_truncation(max_length=max_length) + self._tokenizer.enable_padding(length=max_length, pad_id=pad_id, pad_token="") + onnx_name = next(f.name for f in manifest.files if f.name.endswith(".onnx")) + self._session = ort.InferenceSession( + str(model_dir / onnx_name), providers=["CPUExecutionProvider"] + ) + self._labels = {int(k): str(v) for k, v in config.get("id2label", {}).items()} + if not self._labels: + self._labels = {i: label for i, label in enumerate(manifest.labels)} + + def detect(self, text: str) -> List[ModelEntity]: + if not text.strip(): + return [] + encoding = self._tokenizer.encode(text) + ids = np.asarray([encoding.ids], dtype=np.int64) + attention = np.asarray([encoding.attention_mask], dtype=np.int64) + feeds: Dict[str, Any] = {} + for input_meta in self._session.get_inputs(): + name = input_meta.name + if name == "input_ids": + feeds[name] = ids + elif name == "attention_mask": + feeds[name] = attention + elif name == "token_type_ids": + feeds[name] = np.zeros_like(ids) + if "input_ids" not in feeds or "attention_mask" not in feeds: + raise RuntimeError("input ONNX non compatibili con token-classification") + output = self._session.run(None, feeds)[0] + logits = np.asarray(output)[0] + entities: List[ModelEntity] = [] + current: Optional[Tuple[int, int, str, float]] = None + current_label = "" + offsets = encoding.offsets + for index, row in enumerate(logits): + label = self._labels.get(int(np.argmax(row)), "O") + score = float(np.exp(row - np.max(row))[int(np.argmax(row))] / np.exp(row - np.max(row)).sum()) + if index >= len(offsets) or offsets[index][0] == offsets[index][1]: + continue + start, end = offsets[index] + if label == "O": + if current is not None: + entities.append(ModelEntity(current[0], current[1], current[2], current_label, current[3])) + current = None + continue + prefix, _, raw_label = label.partition("-") + group = _group_label(raw_label) + if group is None: + continue + if current is None or prefix == "B" or current_label != group or start > current[1]: + if current is not None: + entities.append(ModelEntity(current[0], current[1], current[2], current_label, current[3])) + current = (start, end, text[start:end], score) + current_label = group + else: + current = (current[0], end, text[current[0]:end], min(current[3], score)) + if current is not None: + entities.append(ModelEntity(current[0], current[1], current[2], current_label, current[3])) + return [e for e in entities if e.text.strip() and 0 <= e.start < e.end <= len(text)] + + +__all__ = ["ModelEntity", "OnnxTokenDetector"] diff --git a/pdf_sanitize.py b/pdf_sanitize.py new file mode 100644 index 0000000..6fe0915 --- /dev/null +++ b/pdf_sanitize.py @@ -0,0 +1,553 @@ +"""Qt-free export sanitiser and verifier. + +Redacting visible text is not enough to share a PDF: annotations, embedded +files, JavaScript, bookmarks, tagged-PDF alternative text, hidden layers, +invisible text and text outside the page box all survive a plain +``apply_redactions()``. :func:`sanitize_document` removes them; +:func:`verify_export` reopens the written file and checks that none of the +removed values is still recoverable from it. + +Reports contain counts and locations, never the removed values. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional + +import pymupdf + +PDF_REDACT_IMAGE_NONE: int = pymupdf.PDF_REDACT_IMAGE_NONE # type: ignore[attr-defined] +PDF_ANNOT_REDACT: int = pymupdf.PDF_ANNOT_REDACT # type: ignore[attr-defined] + +# Catalog keys removed at export: they carry scripts, attachments, outlines, +# structure (alternative text), threads or legal attestations. +CATALOG_KEYS_TO_DROP = ( + "Names", "OpenAction", "AA", "StructTreeRoot", "MarkInfo", "Outlines", + "Threads", "SpiderInfo", "Perms", "Legal", "AcroForm", "Metadata", + "PieceInfo", "Properties", "URI", "Collection", +) +PAGE_KEYS_TO_DROP = ("AA", "PieceInfo", "Metadata", "Thumb", "StructParents") + + +@dataclass +class SanitizePolicy: + remove_hidden_text: bool = True + remove_annotations: bool = True + remove_external_links: bool = True + reveal_layers: bool = True + expand_cropbox: bool = True + regenerate_id: bool = True + mask_removed_width: bool = False + + +@dataclass +class SanitizeReport: + actions: Dict[str, int] = field(default_factory=dict) + + def bump(self, key: str, n: int = 1) -> None: + if n: + self.actions[key] = self.actions.get(key, 0) + n + + +def reveal_hidden_layers(doc: Any) -> int: + """Switch every optional-content layer on so hidden text becomes visible. + + Text in a layer that is OFF is not returned by ``get_text()``, so neither + the detector nor the reviewer would ever see it. Returns the layer count. + """ + try: + ocgs = doc.get_ocgs() + except Exception: + return 0 + if not ocgs: + return 0 + try: + doc.set_layer(-1, basestate="ON", off=[]) + except Exception: + try: + doc.set_layer(-1, on=list(ocgs.keys()), off=[]) + except Exception: + return 0 + return len(ocgs) + + +def _page_image_coverage(page: Any) -> float: + try: + area = page.rect.width * page.rect.height + if area <= 0: + return 0.0 + covered = 0.0 + for info in page.get_image_info(hashes=False): + r = pymupdf.Rect(info["bbox"]) & page.rect + covered += max(0.0, r.width * r.height) + return covered / area + except Exception: + return 0.0 + + +page_image_coverage = _page_image_coverage + + +def hidden_text_rects(page: Any) -> List[Any]: + """Rectangles of text that is drawn invisibly. + + Invisible = render mode 3, zero opacity or sub-1pt font size. On pages + that are mostly one image the invisible layer is the OCR text of a scan + and is left alone: it is only removed where the image is redacted. + """ + try: + spans = page.get_texttrace() + except Exception: + return [] + scanned = _page_image_coverage(page) >= 0.6 + out: List[Any] = [] + for sp in spans: + invisible = ( + sp.get("type") == 3 + or float(sp.get("opacity", 1.0)) <= 0.0 + or float(sp.get("size", 10.0)) < 1.0 + ) + if not invisible: + continue + if scanned and sp.get("type") == 3: + continue + r = pymupdf.Rect(sp["bbox"]) + if r.is_empty or r.is_infinite: + continue + out.append(r) + return out + + +def _quantize_tj_gaps(doc: Any, page: Any, bucket: int = 500, threshold: int = 1500) -> int: + """Round large TJ gaps so the width of removed text is not recoverable. + + ``apply_redactions`` leaves a numeric gap equal to the removed text's + width in the text-showing operator. Word spaces and kerning are far + smaller than ``threshold``, so only gaps left by removed text are + touched; the following text shifts by at most ``bucket / 2`` units. + """ + changed = 0 + num = re.compile(rb"(? bytes: + nonlocal changed + body = m.group(0) + # Keep string literals untouched: split on them. + out = bytearray() + i = 0 + depth = 0 + while i < len(body): + ch = body[i:i + 1] + if ch == b"(": + j = i + 1 + depth = 1 + while j < len(body) and depth: + if body[j:j + 1] == b"\\": + j += 2 + continue + if body[j:j + 1] == b"(": + depth += 1 + elif body[j:j + 1] == b")": + depth -= 1 + j += 1 + out += body[i:j] + i = j + elif ch == b"<": + j = body.find(b">", i) + j = len(body) if j == -1 else j + 1 + out += body[i:j] + i = j + else: + j = i + while j < len(body) and body[j:j + 1] not in (b"(", b"<"): + j += 1 + seg = body[i:j] + + def rnd(nm: "re.Match[bytes]") -> bytes: + nonlocal changed + v = float(nm.group(0)) + if abs(v) < threshold: + return nm.group(0) + q = int(round(abs(v) / bucket)) * bucket + new = -q if v < 0 else q + if new != v: + changed += 1 + return str(new).encode() + + out += num.sub(rnd, seg) + i = j + return bytes(out) + + new_data = arr.sub(fix_array, data) + if new_data != data: + doc.update_stream(xref, new_data) + return changed + + +def sanitize_document(doc: Any, policy: Optional[SanitizePolicy] = None) -> SanitizeReport: + """Strip everything that survives redaction, in place.""" + policy = policy or SanitizePolicy() + rep = SanitizeReport() + + if policy.reveal_layers: + rep.bump("layers-revealed", reveal_hidden_layers(doc)) + + for page in doc: + if policy.remove_hidden_text: + rects = hidden_text_rects(page) + for r in rects: + page.add_redact_annot(r) + if rects: + page.apply_redactions( + images=PDF_REDACT_IMAGE_NONE, graphics=0 + ) + rep.bump("hidden-text-spans", len(rects)) + if policy.expand_cropbox: + mb, cb = page.mediabox, page.cropbox + if cb != mb: + # Text between the visible box and the media box is neither + # shown nor analysed: redact it, then drop the crop. + for strip in _strips(mb, cb): + page.add_redact_annot(strip) + page.apply_redactions(images=PDF_REDACT_IMAGE_NONE, graphics=0) + page.set_cropbox(mb) + rep.bump("offpage-area-cleared") + if policy.remove_annotations: + for a in list(page.annots() or []): + page.delete_annot(a) + rep.bump("annotations-removed") + if policy.remove_external_links: + for lnk in page.get_links(): + if lnk.get("kind") in (pymupdf.LINK_URI, pymupdf.LINK_LAUNCH, + pymupdf.LINK_GOTOR): + page.delete_link(lnk) + rep.bump("links-removed") + for key in PAGE_KEYS_TO_DROP: + try: + if doc.xref_get_key(page.xref, key)[0] != "null": + doc.xref_set_key(page.xref, key, "null") + rep.bump(f"page-key:{key}") + except Exception: + pass + + try: + n_files = len(doc.embfile_names()) + except Exception: + n_files = 0 + rep.bump("embedded-files-removed", n_files) + try: + doc.scrub( + attached_files=True, clean_pages=True, embedded_files=True, + hidden_text=False, javascript=True, metadata=True, + redactions=False, remove_links=False, reset_fields=True, + reset_responses=True, thumbnails=True, xml_metadata=True, + ) + except Exception: + rep.bump("scrub-failed") + if policy.mask_removed_width: + # After scrub: cleaning rewrites content streams, which would + # re-serialise the numbers this pass has just rounded. + for page in doc: + rep.bump("tj-gaps-quantized", _quantize_tj_gaps(doc, page)) + try: + if doc.get_toc(): + rep.bump("bookmarks-removed", len(doc.get_toc())) + doc.set_toc([]) + except Exception: + pass + cat = doc.pdf_catalog() + for key in CATALOG_KEYS_TO_DROP: + try: + if doc.xref_get_key(cat, key)[0] != "null": + doc.xref_set_key(cat, key, "null") + rep.bump(f"catalog-key:{key}") + except Exception: + pass + doc.set_metadata({}) + if policy.regenerate_id: + try: + doc.xref_set_key(-1, "ID", "null") + rep.bump("trailer-id-removed") + except Exception: + pass + return rep + + +def _strips(mediabox: Any, cropbox: Any) -> List[Any]: + strips = [] + if cropbox.y0 > mediabox.y0: + strips.append(pymupdf.Rect(mediabox.x0, mediabox.y0, mediabox.x1, cropbox.y0)) + if cropbox.y1 < mediabox.y1: + strips.append(pymupdf.Rect(mediabox.x0, cropbox.y1, mediabox.x1, mediabox.y1)) + if cropbox.x0 > mediabox.x0: + strips.append(pymupdf.Rect(mediabox.x0, mediabox.y0, cropbox.x0, mediabox.y1)) + if cropbox.x1 < mediabox.x1: + strips.append(pymupdf.Rect(cropbox.x1, mediabox.y0, mediabox.x1, mediabox.y1)) + return strips + + +# --------------------------------------------------------------------------- +# Signature and stamp areas (suggestions only) +# --------------------------------------------------------------------------- + +_ANCHOR_SINGLE = {"firma", "firmato", "firmata", "timbro", "visto", "f.to"} +_ANCHOR_PAIRS = { + ("il", "medico"), ("il", "dottore"), ("in", "fede"), ("il", "dichiarante"), + ("il", "richiedente"), ("l'assistito", ""), ("il", "titolare"), + ("il", "paziente"), ("timbro", "e"), +} +_ANCHOR_REGION_BELOW = 90.0 +_ANCHOR_REGION_SIDE = 200.0 + + +def _anchor_rects(words: List[Any]) -> List[Any]: + out: List[Any] = [] + low = [(str(w[4]).lower().strip(".,:;"), w) for w in words] + for i, (t, w) in enumerate(low): + nxt = low[i + 1][0] if i + 1 < len(low) else "" + if t in _ANCHOR_SINGLE or (t, nxt) in _ANCHOR_PAIRS or (t, "") in _ANCHOR_PAIRS: + out.append(pymupdf.Rect(w[:4])) + return out + + +def _merge_close(rects: List[Any], gap: float = 8.0) -> List[Any]: + rects = [pymupdf.Rect(r) for r in rects] + merged = True + while merged: + merged = False + for i in range(len(rects)): + for j in range(i + 1, len(rects)): + grown = pymupdf.Rect(rects[i]) + (-gap, -gap, gap, gap) + if grown.intersects(rects[j]): + rects[i] |= rects[j] + del rects[j] + merged = True + break + if merged: + break + return rects + + +def suggest_signature_areas(page: Any, words: Optional[List[Any]] = None) -> List[Dict[str, Any]]: + """Suggest handwritten-signature / stamp areas near signature labels. + + Anchored on words such as "firma", "timbro", "il medico". Looks for + curve-rich vector ink without text, and for small images, in a region + around each anchor. Always returns *suggestions* (state "suggested"), + never applied without review. The rectangle is the union of the ink plus + a small padding: vector paths are removed only when fully contained. + """ + words = words if words is not None else page.get_text("words") + anchors = _anchor_rects(words) + if not anchors: + return [] + page_area = page.rect.width * page.rect.height + try: + drawings = page.get_drawings() + except Exception: + drawings = [] + try: + images = page.get_image_info(hashes=False) + except Exception: + images = [] + word_rects = [pymupdf.Rect(w[:4]) for w in words] + found: List[Dict[str, Any]] = [] + seen: List[Any] = [] + for anchor in anchors: + region = pymupdf.Rect( + anchor.x0 - _ANCHOR_REGION_SIDE, anchor.y0 - 5, + anchor.x1 + _ANCHOR_REGION_SIDE, anchor.y1 + _ANCHOR_REGION_BELOW, + ) & page.rect + ink: List[Any] = [] + widths: List[float] = [1.0] + for d in drawings: + r = pymupdf.Rect(d["rect"]) + if r.is_empty and r.width < 0.5 and r.height < 0.5: + continue + if not region.contains(r): + continue + items = d.get("items", []) + curves = sum(1 for it in items if it[0] == "c") + lines = sum(1 for it in items if it[0] == "l") + straight = len(items) == 1 and items[0][0] == "l" + if straight or (r.width > 200 and r.height < 3) or (r.height > 200 and r.width < 3): + continue # table borders, underlines + if curves >= 4 or lines >= 8: + ink.append(r) + widths.append(float(d.get("width") or 1.0)) + for cluster in _merge_close(ink): + area = cluster.width * cluster.height + if not (200 <= area <= 40000): + continue + if any(wr.contains(pymupdf.Point((cluster.x0 + cluster.x1) / 2, (cluster.y0 + cluster.y1) / 2)) + for wr in word_rects): + continue + # MuPDF removes a vector path only when the rectangle covers its + # box grown by the stroke's miter extent (default miter limit 10). + pad = 10.0 * max(widths) + 0.5 + box = cluster + (-pad, -pad, pad, pad) + overlaps_text = any( + wr.intersects(box + (-pad / 2, -pad / 2, pad / 2, pad / 2)) for wr in word_rects + ) + if any(box.intersects(s) for s in seen): + continue + seen.append(box) + found.append({ + "text": "", "entity_type": "SIGNATURE", + "rects": [tuple(box)], "source": "graphics", "score": 0.7, + "state": "suggested", + "reasons": ["signature-anchor", "curve-ink"] + + (["overlaps-text"] if overlaps_text else []), + "glued": False, + }) + for info in images: + r = pymupdf.Rect(info["bbox"]) + if not r.intersects(region) or r.width * r.height > 0.25 * page_area: + continue + box = r + (-2, -2, 2, 2) + if any(box.intersects(s) for s in seen): + continue + seen.append(box) + found.append({ + "text": "", "entity_type": "SIGNATURE", + "rects": [tuple(box)], "source": "graphics", "score": 0.6, + "state": "suggested", + "reasons": ["signature-anchor", "raster-stamp"], "glued": False, + }) + return found + + +# --------------------------------------------------------------------------- +# Verification of the written file +# --------------------------------------------------------------------------- + +@dataclass +class VerificationReport: + ok: bool = True + findings: List[str] = field(default_factory=list) # locations only + + def add(self, where: str) -> None: + self.ok = False + if where not in self.findings: + self.findings.append(where) + + +MIN_NEEDLE_LENGTH = 4 + + +def _squash(s: str) -> str: + return " ".join(s.split()) + + +def _compile_needles(values: Iterable[str]) -> List["re.Pattern[str]"]: + """Whole-word patterns for the removed values. + + Only whole values count (never their individual words) and only from + four characters up, and a match must not be glued to other letters or + digits: a redacted "Rossi" must not be reported because "Rossini" is + still on the page. + """ + out: List["re.Pattern[str]"] = [] + seen = set() + for v in values: + sq = _squash(v) + if len(sq) < MIN_NEEDLE_LENGTH or sq.casefold() in seen: + continue + seen.add(sq.casefold()) + body = r"\s+".join(re.escape(part) for part in sq.split()) + out.append(re.compile(r"(? bool: + flat = _squash(text) + return any(n.search(flat) for n in needles) + + +def _is_binary_resource(obj_text: str) -> bool: + """Font programs and images: binary noise, not readable text.""" + return any( + k in obj_text + for k in ("/Length1", "/FontFile", "/Type1C", "/CIDFontType0C", + "/OpenType", "/Subtype/Image", "/Subtype /Image") + ) + + +def _object_texts(doc: Any, xref: int) -> List[str]: + texts: List[str] = [] + try: + obj = doc.xref_object(xref, compressed=False) + except Exception: + obj = "" + texts.append(obj) + try: + if doc.xref_is_stream(xref) and not _is_binary_resource(obj): + raw = doc.xref_stream(xref) or b"" + texts.append(raw.decode("latin-1", errors="ignore")) + texts.append(raw.decode("utf-16-be", errors="ignore")) + except Exception: + pass + return texts + + +def verify_export( + out_path: str, + removed_values: Iterable[str], + still_present_text: str = "", +) -> VerificationReport: + """Check that no removed value can still be read from the written file. + + Looks at page text (page-box clipping off), every object and decompressed + stream, annotations, embedded files, bookmarks and metadata, and flags any + leftover annotation, script or attachment. Locations are reported, never + the values. A value that the user deliberately left unredacted elsewhere + (it occurs in `still_present_text`, the source text outside every + redaction) is not a finding. + """ + rep = VerificationReport() + needles = [ + n for n in _compile_needles(removed_values) + if not (still_present_text and n.search(_squash(still_present_text))) + ] + doc = pymupdf.open(out_path) + try: + flags = pymupdf.TEXT_PRESERVE_WHITESPACE | pymupdf.TEXT_PRESERVE_LIGATURES + for page in doc.pages(): + text = page.get_text("text", flags=flags) + words = " ".join(w[4] for w in page.get_text("words", flags=flags)) + if _hit(needles, text) or _hit(needles, words): + rep.add(f"page {page.number + 1}: text layer") + if list(page.annots() or []): + rep.add(f"page {page.number + 1}: annotations remain") + for lnk in page.get_links(): + if lnk.get("kind") == pymupdf.LINK_URI: + rep.add(f"page {page.number + 1}: external link remains") + meta = " ".join(str(v) for v in (doc.metadata or {}).values()) + if _hit(needles, meta): + rep.add("document info") + if doc.embfile_names(): + rep.add("embedded files remain") + if _hit(needles, " ".join(str(t[1]) for t in doc.get_toc())): + rep.add("bookmarks") + cat = doc.pdf_catalog() + for key in ("OpenAction", "AA", "Names"): + if doc.xref_get_key(cat, key)[0] != "null": + rep.add(f"catalog /{key} remains") + for xref in range(1, doc.xref_length()): + for blob in _object_texts(doc, xref): + if _hit(needles, blob): + rep.add(f"object {xref}") + break + finally: + doc.close() + return rep diff --git a/pii_detection.py b/pii_detection.py index d0f630c..bd53b59 100644 --- a/pii_detection.py +++ b/pii_detection.py @@ -9,8 +9,12 @@ import json import pathlib import re +import statistics +from bisect import bisect_left from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Sequence, Set, Tuple + +import lexicon_it Rect4 = Tuple[float, float, float, float] @@ -29,6 +33,9 @@ ("LICENSE", "Patente", ("IT_DRIVER_LICENSE",)), ("IDCARD", "Carta d'identitร ", ("IT_IDENTITY_CARD",)), ("PASSPORT", "Passaporto", ("IT_PASSPORT",)), + ("BIRTHDATE", "Date di nascita", ("DATE_OF_BIRTH",)), + ("HEALTHID", "Identificativi sanitari", ("HEALTH_ID",)), + ("EXEMPT", "Codici di esenzione", ("HEALTH_EXEMPTION",)), ] ALL_GROUP_IDS: List[str] = [g[0] for g in ENTITY_GROUPS] ALL_ENTITY_TYPES: List[str] = [t for g in ENTITY_GROUPS for t in g[2]] @@ -48,8 +55,16 @@ "IT_DRIVER_LICENSE": 0.5, "IT_IDENTITY_CARD": 0.4, "IT_PASSPORT": 0.3, + "DATE_OF_BIRTH": 0.5, + "HEALTH_ID": 0.5, + "HEALTH_EXEMPTION": 0.5, } +# Entity types found by this module's own patterns, not by Presidio. +NON_PRESIDIO_TYPES: FrozenSet[str] = frozenset( + {"DATE_OF_BIRTH", "HEALTH_ID", "HEALTH_EXEMPTION"} +) + _NER_TYPES = {"PERSON", "LOCATION"} _GENERIC_WORDS = { @@ -108,6 +123,15 @@ class WordBox: end: int rect: Rect4 line_key: Tuple[int, int] + text: str = "" + # False for words over an image or from OCR: redaction there blanks + # pixels, so those boxes are never reduced to a thin band. + native: bool = True + # The box before any clamping (what the word really occupies). + orig_rect: Optional[Rect4] = None + # True when the line spacing is so tight that no band can spare the + # neighbouring lines; the full box is kept (never a leak, may damage). + collision: bool = False @dataclass @@ -117,6 +141,10 @@ class PageTextIndex: text: str = "" words: List[WordBox] = field(default_factory=list) + @property + def collisions(self) -> int: + return sum(1 for w in self.words if w.collision) + @dataclass class Candidate: @@ -126,29 +154,222 @@ class Candidate: entity_type: str score: float rects: List[Rect4] = field(default_factory=list) + # "accepted" candidates are applied; "suggested" ones await user review. + state: str = "accepted" + reasons: List[str] = field(default_factory=list) + + +NL = chr(10) +_LIGATURES = { + "๏ฌ€": "ff", "๏ฌ": "fi", "๏ฌ‚": "fl", "๏ฌƒ": "ffi", + "๏ฌ„": "ffl", "๏ฌ…": "st", "๏ฌ†": "st", +} + + +def _normalize_token(token: str) -> str: + """Expand typographic ligatures so NER sees ordinary letters.""" + if not any(ch in _LIGATURES for ch in token): + return token + return "".join(_LIGATURES.get(ch, ch) for ch in token) + + +def _is_vertical(rect: Rect4, text: str) -> bool: + w, h = rect[2] - rect[0], rect[3] - rect[1] + return len(text) > 1 and h > 1.5 * w + + +def _near_duplicate(a: Rect4, b: Rect4) -> bool: + ix = min(a[2], b[2]) - max(a[0], b[0]) + iy = min(a[3], b[3]) - max(a[1], b[1]) + if ix <= 0 or iy <= 0: + return False + area_a = (a[2] - a[0]) * (a[3] - a[1]) + area_b = (b[2] - b[0]) * (b[3] - b[1]) + return ix * iy >= 0.8 * min(area_a, area_b) + + +# A glyph is deleted by a redaction when the rectangle intersects its box +# shrunk by this fraction of its width (each side) and height (top, bottom). +# Measured on PyMuPDF 2.1 / MuPDF 1.28; pinned by tests/test_geometry.py. +GLYPH_CORE_INSET = 0.10 + +# Smallest share of its own height a word box keeps after clamping. Below it +# the text overlaps its neighbours so heavily that trimming would risk +# leaving ink uncovered; the original box is kept. +CLAMP_MIN_RATIO = 0.3 + + +# Safety margin (pt) kept between a band and the deletion core of a neighbour, +# and the least a band must overlap its own word's core. +GLYPH_CORE_MARGIN = 0.3 +MIN_CORE_HIT = 0.5 + + +def clamp_word_rects( + boxes: List["WordBox"], ocr: bool = False, min_ratio: Optional[float] = None +) -> List[Rect4]: + """Vertical extents of word boxes trimmed where they hit another line. + + PyMuPDF word boxes span ascender to descender (about 1.37 x the font + size), so with normal leading they overlap the lines above and below. + A box is trimmed only against boxes of *other* lines that overlap it + horizontally, and only on the side facing that neighbour. Words on the + same line never affect each other, so mixed font sizes keep their full + height. + + Native text: a glyph is deleted only when the rectangle reaches its + *core* (its box inset by GLYPH_CORE_INSET), so the band stops just short + of the neighbour's core instead of its box edge. That spares neighbours + at much tighter leading. If the remaining band would not reach the + word's own core, the full box is kept and the word is flagged + ``collision``. + + Image / OCR words (and everything when `ocr` is set): redaction blanks + pixels, so the cut is at the middle of the overlap with a small + horizontal margin, and never thinner than `min_ratio` of the height. + """ + if min_ratio is None: + min_ratio = CLAMP_MIN_RATIO + n = len(boxes) + out: List[Rect4] = [b.rect for b in boxes] + if n < 2: + return out + order = sorted(range(n), key=lambda i: boxes[i].rect[1]) + y0s = [boxes[i].rect[1] for i in order] + max_h = max(b.rect[3] - b.rect[1] for b in boxes) + for i in range(n): + b = boxes[i] + x0, y0, x1, y1 = b.rect + h = y1 - y0 + if h <= 0 or _is_vertical(b.rect, b.text): + continue + pixel_mode = ocr or not b.native + lo = bisect_left(y0s, y0 - max_h) + hi = bisect_left(y0s, y1) + ny0, ny1 = y0, y1 + touched = False + for j in order[lo:hi]: + if j == i: + continue + o = boxes[j] + if o.line_key == b.line_key or _is_vertical(o.rect, o.text): + continue + ox0, oy0, ox1, oy1 = o.rect + if ox1 <= x0 or ox0 >= x1 or oy1 <= y0 or oy0 >= y1: + continue + if _near_duplicate(b.rect, o.rect): + continue + touched = True + below = (oy0 + oy1) / 2 > (y0 + y1) / 2 + if pixel_mode: + mid = (max(y0, oy0) + min(y1, oy1)) / 2 + if below: + ny1 = min(ny1, mid) + else: + ny0 = max(ny0, mid) + else: + oh = oy1 - oy0 + if below: + ny1 = min(ny1, oy0 + GLYPH_CORE_INSET * oh - GLYPH_CORE_MARGIN) + else: + ny0 = max(ny0, oy1 - GLYPH_CORE_INSET * oh + GLYPH_CORE_MARGIN) + if not touched: + continue + if pixel_mode: + if (ny1 - ny0) < min_ratio * h: + continue + pad = 1.0 + out[i] = (x0 - pad, ny0, x1 + pad, ny1) + continue + core0, core1 = y0 + GLYPH_CORE_INSET * h, y1 - GLYPH_CORE_INSET * h + if min(ny1, core1) - max(ny0, core0) < MIN_CORE_HIT: + b.collision = True + continue + out[i] = (x0, ny0, x1, ny1) + return out + +def build_page_index( + words: Iterable[Sequence[Any]], + *, + clamp: bool = True, + ocr: bool = False, + image_rects: Sequence[Rect4] = (), + merge_spaced: bool = True, + join_hyphen: bool = True, +) -> PageTextIndex: + """Build the analysis string from `page.get_text("words")` tuples. -def build_page_index(words: Iterable[Sequence[Any]]) -> PageTextIndex: - """Build the analysis string from `page.get_text("words")` tuples.""" + Besides joining words, this: expands ligatures; glues letter-spaced runs + ("C o m m i s s i o n e") into one analysis word; joins a word split by a + line-end hyphen; and clamps each word box against neighbouring lines. + """ ordered = sorted( (w for w in words if str(w[4]).strip()), key=lambda w: (int(w[5]), int(w[6]), int(w[7])), ) + n = len(ordered) + tokens = [_normalize_token(str(w[4])) for w in ordered] + keys = [(int(w[5]), int(w[6])) for w in ordered] + rects = [(w[0], w[1], w[2], w[3]) for w in ordered] + + # Separator placed *before* word i (i >= 1). + seps: List[str] = [""] * n + for i in range(1, n): + seps[i] = ( + " " if keys[i] == keys[i - 1] + else (NL if keys[i][0] == keys[i - 1][0] else NL + NL) + ) + + if merge_spaced: + i = 0 + while i < n: + j = i + while ( + j + 1 < n and keys[j + 1] == keys[i] + and len(tokens[j]) == 1 and tokens[j].isalnum() + and len(tokens[j + 1]) == 1 and tokens[j + 1].isalnum() + ): + j += 1 + if j - i + 1 >= 4: + gaps = [rects[k + 1][0] - rects[k][2] for k in range(i, j)] + med = statistics.median(gaps) + for k in range(i, j): + seps[k + 1] = " " if gaps[k - i] > 1.8 * med + 0.5 else "" + i = j + 1 + + if join_hyphen: + for i in range(n - 1): + t = tokens[i] + if ( + len(t) >= 3 and t.endswith("-") and t[:-1].isalpha() + and keys[i + 1][0] == keys[i][0] + and keys[i + 1][1] == keys[i][1] + 1 + and tokens[i + 1][:1].islower() + ): + tokens[i] = t[:-1] + seps[i + 1] = "" + parts: List[str] = [] boxes: List[WordBox] = [] pos = 0 - prev: Optional[Tuple[int, int]] = None - for w in ordered: - key = (int(w[5]), int(w[6])) - if prev is not None: - sep = " " if key == prev else ("\n" if key[0] == prev[0] else "\n\n") - parts.append(sep) - pos += len(sep) - token = str(w[4]) - boxes.append(WordBox(pos, pos + len(token), (w[0], w[1], w[2], w[3]), key)) - parts.append(token) - pos += len(token) - prev = key + for i in range(n): + if i: + parts.append(seps[i]) + pos += len(seps[i]) + cx, cy = (rects[i][0] + rects[i][2]) / 2, (rects[i][1] + rects[i][3]) / 2 + on_image = any( + r[0] <= cx <= r[2] and r[1] <= cy <= r[3] for r in image_rects + ) + boxes.append(WordBox( + pos, pos + len(tokens[i]), rects[i], keys[i], tokens[i], + native=not (ocr or on_image), orig_rect=rects[i], + )) + parts.append(tokens[i]) + pos += len(tokens[i]) + if clamp and boxes: + for box, r in zip(boxes, clamp_word_rects(boxes, ocr=ocr)): + box.rect = r return PageTextIndex("".join(parts), boxes) @@ -405,3 +626,805 @@ def save_enabled_groups(path: pathlib.Path, groups: Iterable[str]) -> None: ) except Exception: pass + + +# --------------------------------------------------------------------------- +# Evidence-based classification (Italian-aware), pattern validation and +# geometry policy. Everything below is Qt-free and exercised by the tests. +# --------------------------------------------------------------------------- + +# Types validated by a checksum or a rigid format: when such a value is glued +# to a label or to other text it is redacted as a whole token rather than +# dropped, because under-redacting an identifier is a confidentiality breach. +VALIDATED_TYPES: FrozenSet[str] = frozenset({ + "IBAN_CODE", "CREDIT_CARD", "IT_FISCAL_CODE", "IT_VAT_CODE", "CRYPTO", + "EMAIL_ADDRESS", "HEALTH_ID", "HEALTH_EXEMPTION", +}) + +ACCEPT_SCORE = 0.7 +SUGGEST_SCORE = 0.4 + + +def boundary_ok( + text: str, start: int, end: int, *, digits_only: bool = False, + apostrophe_ok: bool = False, +) -> bool: + """True when the span is not glued to alphanumerics on either side.""" + def glue(ch: str) -> bool: + if digits_only: + return ch.isdigit() + return ch.isalnum() or ch == "_" + if start > 0: + ch = text[start - 1] + if glue(ch) and not (apostrophe_ok and ch in "'โ€™"): + return False + if end < len(text) and glue(text[end]): + return False + return True + + +def _context(text: str, start: int, end: int, before: int = 40, after: int = 20) -> str: + return text[max(0, start - before):start].lower() + " | " + text[end:end + after].lower() + + +_FISCAL_CTX = ("codice fiscale", "c.f.", "cod. fisc", "cf:", "cf ", "fiscal code", "tax code") +_VAT_CTX = ("p.iva", "p. iva", "partita iva", "partita i.v.a", "vat", "iva ") +_PASSPORT_CTX = ("passaporto", "passport") +_IDCARD_CTX = ("carta d'identit", "carta dโ€™identit", "carta di identit", + "carta identit", "documento n", "n. documento", "numero documento", + "c.i.", "cie ", "identity card", "id card") +_LICENSE_CTX = ("patente di guida", "patente n", "n. patente", "numero patente", + "patente:", "driving licen", "driver's licen", "driver licen") +_PHONE_CTX = ("tel", "cell", "telefono", "phone", "mobile", "fax", "whatsapp", + "recapito", "contatt", "chiam") +_DATE_LIKE = re.compile(r"^\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}$") +_YEAR_RANGE = re.compile(r"^\d{4}\s*[-โ€“/]\s*\d{4}$") + + +def refine_pattern_hit(c: Candidate, text: str) -> Tuple[bool, str]: + """Extra validation of pattern-based hits (not NER). Returns (keep, reason).""" + t = c.entity_type + value = text[c.start:c.end] + ctx = _context(text, c.start, c.end) + if t in ("IT_FISCAL_CODE", "CRYPTO"): + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if t == "IT_FISCAL_CODE": + has_ctx = any(k in ctx for k in _FISCAL_CTX) + if not any(ch.isdigit() for ch in value) and not has_ctx: + return False, "no-digit-no-context" + if value != value.upper() and not has_ctx: + return False, "lowercase-no-context" + elif t == "CREDIT_CARD": + if not boundary_ok(text, c.start, c.end, digits_only=True): + return False, "embedded-in-digits" + elif t == "IT_VAT_CODE": + if not boundary_ok(text, c.start, c.end, digits_only=True): + return False, "embedded-in-digits" + has_ctx = any(k in ctx for k in _VAT_CTX) or value.upper().startswith("IT") + if (" " in value.strip() or "_" in value) and not has_ctx: + return False, "spaced-digits-no-context" + if not has_ctx and c.score < 0.9: + return False, "no-context" + elif t == "IT_PASSPORT": + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if value != value.upper() or not any(k in ctx for k in _PASSPORT_CTX): + return False, "no-context" + elif t == "IT_IDENTITY_CARD": + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if value != value.upper() or not any(k in ctx for k in _IDCARD_CTX): + return False, "no-context" + elif t == "IT_DRIVER_LICENSE": + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if value != value.upper() or not any(k in ctx for k in _LICENSE_CTX): + return False, "no-context" + elif t == "PHONE_NUMBER": + compact = value.strip() + digits = re.sub(r"\D", "", compact) + if _DATE_LIKE.match(compact) or _YEAR_RANGE.match(compact): + return False, "date-like" + if not boundary_ok(text, c.start, c.end, digits_only=True): + return False, "embedded-in-digits" + before = text[c.start - 1] if c.start > 0 else " " + if before.isalpha(): + return False, "embedded-in-letters" + has_ctx = any(k in ctx for k in _PHONE_CTX) + plausible_start = compact.startswith(("+", "00", "0", "3")) + if len(digits) < 8 or len(digits) > 15: + return False, "digit-count" + if not (plausible_start or has_ctx): + return False, "implausible-prefix" + return True, "" + + +def strip_lexicon_edges( + candidates: Iterable[Candidate], +) -> Tuple[List[Candidate], int]: + """Cut ordinary words off the start/end of PERSON/LOCATION spans. + + "Prescrivo Augmentin" becomes "Augmentin"; "Certifico" alone disappears. + Returns the remaining candidates and the number removed or trimmed. + """ + out: List[Candidate] = [] + dropped = 0 + for c in candidates: + if c.entity_type not in _NER_TYPES: + out.append(c) + continue + toks = list(re.finditer(r"\S+", c.text)) + lo, hi = 0, len(toks) + while lo < hi and lexicon_it.is_common_word(toks[lo].group()): + lo += 1 + while hi > lo and lexicon_it.is_common_word(toks[hi - 1].group()): + hi -= 1 + if lo == hi: + dropped += 1 + continue + if lo == 0 and hi == len(toks): + out.append(c) + continue + dropped += 1 + s0, e0 = toks[lo].start(), toks[hi - 1].end() + raw = c.text[s0:e0] + value = raw.strip(_EDGE_PUNCT) + if len(value) <= 2: + continue + start = c.start + s0 + (len(raw) - len(raw.lstrip(_EDGE_PUNCT))) + out.append(Candidate(start, start + len(value), value, c.entity_type, + c.score, state=c.state, reasons=list(c.reasons))) + return out, dropped + + +def reject_fragment_hits( + candidates: Iterable[Candidate], used_ocr: bool = False +) -> Tuple[List[Candidate], int]: + """Drop NER hits that look like text-layer or OCR fragments.""" + out: List[Candidate] = [] + dropped = 0 + min_alpha = 4 if used_ocr else 3 + for c in candidates: + if c.entity_type not in _NER_TYPES: + out.append(c) + continue + toks = c.text.split() + multi = len(toks) >= 2 + letters = sum(ch.isalpha() for ch in c.text) + non_space = sum(not ch.isspace() for ch in c.text) or 1 + bad = False + if letters / non_space < 0.8: + bad = True + elif not multi and letters < min_alpha: + bad = True + elif used_ocr and any(re.search(r"[a-zร -รฟ][A-Zร€-ร]", t) for t in toks): + bad = True + elif any(t.endswith("-") and len(t) > 2 for t in toks): + bad = True + if bad: + dropped += 1 + else: + out.append(c) + return out, dropped + + +def merge_adjacent_lines( + candidates: List[Candidate], text: str +) -> List[Candidate]: + """Join PERSON hits on consecutive lines ("Esposito" / "Giovanni").""" + ordered = sorted(candidates, key=lambda c: c.start) + out: List[Candidate] = [] + i = 0 + while i < len(ordered): + c = ordered[i] + if ( + c.entity_type == "PERSON" and i + 1 < len(ordered) + and len(c.text.split()) == 1 + ): + n = ordered[i + 1] + gap = text[c.end:n.start] + if ( + n.entity_type == "PERSON" and len(n.text.split()) == 1 + and gap == NL + and not lexicon_it.is_common_word(c.text) + and not lexicon_it.is_common_word(n.text) + and c.text[:1].isupper() and n.text[:1].isupper() + ): + out.append(Candidate( + c.start, n.end, text[c.start:n.end], "PERSON", + max(c.score, n.score), + )) + i += 2 + continue + out.append(c) + i += 1 + return out + + +_PERSON_CUES = ( + "paziente", "assistito", "assistita", "sottoscritto", "sottoscritta", + "signor", "sig.", "sig.ra", "dott", "prof", "avv", "ing.", "nome", + "cognome", "nato", "nata", "intestatario", "intestataria", "titolare", + "firmato", "gentile", "egregio", "egregia", "spett", "patient", "mr.", + "mrs.", "ms.", "dear", "tutore", "genitore", "figlio", "figlia", "coniuge", +) +_LOCATION_CUES = ( + "nato a", "nata a", "nato il", "residente a", "residente in", "residente presso", + "domiciliato", "domiciliata", "con sede in", "con sede a", "sede legale", + "comune di", "provincia di", "in via", "presso", "luogo di nascita", + "indirizzo", "residenza", "domicilio", "born in", "lives in", +) +_PROVINCE_AFTER = re.compile(r"^\s*[\(\[]\s*[A-Z]{2}\s*[\)\]]|^\s*,?\s*\d{5}\b|^\s*,\s*\d+\b") + + +def _is_name_shaped(tokens: List[str]) -> bool: + return len(tokens) >= 2 and all( + t[:1].isupper() + and t.replace("'", "").replace("โ€™", "").replace("-", "").isalpha() + for t in tokens + ) + + +def score_ner_candidate( + c: Candidate, text: str, verb_like: Iterable[str] = () +) -> Tuple[float, List[str]]: + """Evidence-based score (0..1) for a PERSON/LOCATION hit. + + Presidio gives every NER hit the same constant, so the model score cannot + separate "Mario Rossi" from "Certifico". Evidence in the text can. + """ + verb = {v.lower() for v in verb_like} + tokens = [t.strip(_EDGE_PUNCT) for t in c.text.split()] + tokens = [t for t in tokens if t] + lowered = [t.lower() for t in tokens] + score = 0.5 + why: List[str] = [] + ls, le = _line_bounds(text, c.start, c.end) + before_raw = text[ls:c.start] + before_ctx = text[max(0, c.start - 30):c.start].lower() + after_ctx = text[c.end:c.end + 12] + single = len(tokens) == 1 + all_caps = c.text.isupper() and len(c.text) > 2 + + cues = _PERSON_CUES if c.entity_type == "PERSON" else _LOCATION_CUES + if any(cue in before_ctx for cue in cues): + score += 0.35 + why.append("context-cue") + if c.entity_type == "PERSON": + if _is_name_shaped(tokens): + score += 0.25 + why.append("name-shape") + if any(lexicon_it.is_given_name(t) for t in tokens): + score += 0.2 + why.append("given-name") + elif len(tokens) >= 2 and any(lexicon_it.is_known_surname(t) for t in tokens): + score += 0.15 + why.append("known-surname") + else: + if any(lexicon_it.is_known_place(t) for t in tokens) or lexicon_it.is_known_place(c.text): + score += 0.3 + why.append("known-place") + elif any(lexicon_it.is_ambiguous_place(t) for t in tokens) or lexicon_it.is_ambiguous_place(c.text): + score += 0.1 + why.append("ambiguous-place") + pm = _PROVINCE_AFTER.match(after_ctx) + if pm: + code = re.search(r"[A-Z]{2}", pm.group()) + bracketed = pm.group().lstrip().startswith(("(", "[")) + if not bracketed or (code and code.group() in lexicon_it.province_codes()): + score += 0.2 + why.append("province-or-postcode") + if len(tokens) >= 2 and _is_name_shaped(tokens): + score += 0.1 + occurrences = len(re.findall( + r"(?= 2: + score += 0.15 + why.append("repeated") + + if any(lexicon_it.is_common_word(t) for t in tokens): + score -= 0.35 if single else 0.2 + why.append("lexicon") + if any(t in verb or t in lexicon_it.verb_forms() for t in lowered): + score -= 0.25 + why.append("verb-form") + if single and not before_raw.strip(_EDGE_PUNCT): + score -= 0.2 + why.append("line-start") + if all_caps and not ( + any(lexicon_it.is_given_name(t) or lexicon_it.is_known_place(t) for t in tokens) + ): + score -= 0.2 + why.append("all-caps") + nxt = after_ctx.strip().split() + if nxt and nxt[0].lower() in ("che", "di") and single: + score -= 0.1 + why.append("followed-by-function-word") + if not tokens: + score = 0.0 + return max(0.0, min(1.0, score)), why + + +def evaluate_ner_candidates( + candidates: Iterable[Candidate], text: str, *, health_mode: bool = False, + verb_like: Iterable[str] = (), +) -> Tuple[List[Candidate], int]: + """Assign each PERSON/LOCATION hit a state: accepted / suggested / dropped. + + Field labels ("Nome:") and list items are rejected outright. The rest is + scored from evidence; uncertain hits become *suggestions* the user + reviews instead of being silently applied or lost. In health mode no + hit is silently dropped: low scores become suggestions too. + """ + out: List[Candidate] = [] + dropped = 0 + verb_like = list(verb_like) + for c in candidates: + if c.entity_type not in _NER_TYPES or "address-pattern" in c.reasons: + out.append(c) + continue + after = text[c.end:c.end + 8].lstrip(" \t") + ls, _le = _line_bounds(text, c.start, c.end) + before_raw = text[ls:c.start].strip() + single = len(c.text.split()) == 1 + if single and after.startswith(":"): + dropped += 1 + continue + if single and before_raw and not before_raw.strip(_EDGE_PUNCT): + dropped += 1 # list item + continue + score, why = score_ner_candidate(c, text, verb_like) + if score >= ACCEPT_SCORE: + state = "accepted" + elif score >= SUGGEST_SCORE or health_mode: + state = "suggested" + if score < SUGGEST_SCORE: + why.append("low-confidence") + else: + dropped += 1 + continue + out.append(Candidate( + c.start, c.end, c.text, c.entity_type, score, state=state, reasons=why + )) + return out, dropped + + +_DOB_NUM = re.compile( + r"(?i)\b(?:nato|nata|nascita|born|dob)\b[^\n]{0,40}?" + r"(?P(?(?80\s?380\s?00\s?" + _REGION + r"(?:\s?\d){10})(?!\d)" +) +# NRE (electronic prescription number): 15 characters = region/PA code (3 +# digits) + lot code (2 alphanumeric) + 10 digits; printed forms may carry +# spaces or asterisks around/inside it. +_NRE = re.compile( + r"(?" + _REGION + + r"[\s*]?[0-9A-Z]{2}[\s*]?(?:\d[\s*]?){9}\d)\*?(?![0-9A-Za-z])" +) +_NRE_CTX = ("nre", "ricetta", "n. ricetta", "numero ricetta", "prescrizione", + "ricetta elettronica", "promemoria") +# Exemption codes: income E01-E04, invalidity C01-C06, work L01-L04, service +# S01-S03, war G01-G02, rare diseases R + letter + 4 characters (RA0010...). +_EXEMPT = re.compile( + r"(?E0[1-4]|C0[1-6]|L0[1-4]|S0[1-3]|G0[12]|R[A-Z][A-Z0-9]\d{3})(?![A-Z0-9])" +) +_EXEMPT_CTX = ("esenzione", "esente", "cod. es", "codice esenzione", "esenz.") + + +def _ctx_before(text: str, start: int, words: Tuple[str, ...], span: int = 60) -> bool: + window = text[max(0, start - span):start].lower() + return any(w in window for w in words) + + +def detect_health_patterns( + text: str, enabled: Iterable[str], health_mode: bool = False +) -> List[Candidate]: + """Date of birth, health card, prescription and exemption codes. + + Everything is anchored: dates of birth on "nato/a il", the prescription + number and exemption codes on their label. Without the label a hit is + only a *suggestion*, and only in health mode. + """ + on = set(enabled) + out: List[Candidate] = [] + if "DATE_OF_BIRTH" in on: + for rx in (_DOB_NUM, _DOB_TXT): + for m in rx.finditer(text): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "DATE_OF_BIRTH", 0.9 + )) + if "HEALTH_ID" in on: + for m in _TESSERA.finditer(text): + if len(re.sub(r"\D", "", m.group("d"))) == 20: + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_ID", 0.9 + )) + for m in _NRE.finditer(text): + digits = re.sub(r"[\s*]", "", m.group("d")) + if len(digits) != 15: + continue + if _ctx_before(text, m.start("d"), _NRE_CTX): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_ID", 0.85 + )) + elif health_mode: + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_ID", 0.55, + state="suggested", reasons=["no-context"], + )) + if "HEALTH_EXEMPTION" in on: + for m in _EXEMPT.finditer(text): + if _ctx_before(text, m.start("d"), _EXEMPT_CTX, span=40): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_EXEMPTION", 0.8 + )) + elif health_mode and not m.group("d")[1:].isalpha(): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_EXEMPTION", 0.5, + state="suggested", reasons=["no-context"], + )) + return out + + +# --- addresses: house numbers and postcodes ---------------------------------- + +_STREET = ( + r"(?:via|viale|v\.le|piazza|p\.za|piazzale|corso|c\.so|largo|vicolo|strada|" + r"contrada|localit\u00e0|loc\.|frazione|fraz\.|borgo|lungotevere)" +) +_CIVICO = re.compile( + r"(?i)\b" + _STREET + r"\s+(?:[A-Z\u00c0-\u00dd][\w'\u2019.-]*\s+){0,5}?" + r"[A-Z\u00c0-\u00dd][\w'\u2019.-]*\s*,?\s*(?:n\.?|nr\.?|civico|n\u00b0)?\s*" + r"(?P\d{1,4}(?:\s*/\s*[A-Za-z0-9]{1,3}|\s*[A-Za-z](?![a-z])|\s+(?:bis|ter|quater))?)" + r"(?![\d.,]\d)" +) +_CAP = re.compile( + r"(?\d{5})(?![\d.,%])\s+" + r"(?P[A-Z\u00c0-\u00dd][\w'\u2019-]+(?:\s+[A-Z\u00c0-\u00dd][\w'\u2019-]+){0,3})" + r"(?:\s*\(\s*(?P[A-Z]{2})\s*\))?" +) +_MONEY_BEFORE = ("\u20ac", "euro", "eur", "importo", "totale", "n.", "prot.", "fattura") + + +def detect_address_parts( + text: str, enabled: Iterable[str], health_mode: bool = False +) -> List[Candidate]: + """Street + house number as one span, and postcodes in an address context. + + A postcode needs an anchor (a street before it, a province code or a + known place after it, or the label "CAP"); prices, protocol numbers, + dates and quantities never match. Without an anchor it is only a + suggestion in health mode. + """ + if "LOCATION" not in set(enabled): + return [] + out: List[Candidate] = [] + street_ends: List[int] = [] + for m in _CIVICO.finditer(text): + out.append(Candidate( + m.start(), m.end("num"), text[m.start():m.end("num")], "LOCATION", 0.95, + reasons=["address-pattern"], + )) + street_ends.append(m.end("num")) + for m in _CAP.finditer(text): + cap = m.group("cap") + if not (10 <= int(cap) <= 98168): + continue + before = text[max(0, m.start() - 10):m.start()].lower() + if any(k in before for k in _MONEY_BEFORE): + continue + after = text[m.end("cap"):m.end("cap") + 4] + if after.startswith((",0", ",00", "\u20ac", "%")): + continue + window = text[max(0, m.start() - 6):m.start()].lower() + prov = m.group("prov") + anchored = ( + any(0 <= m.start() - e <= 80 for e in street_ends) + or bool(prov and prov in lexicon_it.province_codes()) + or lexicon_it.is_known_place(m.group("city").split()[0]) + or "cap" in window + ) + if anchored: + out.append(Candidate( + m.start("cap"), m.end("cap"), cap, "LOCATION", 0.9, + reasons=["address-pattern", "postcode"], + )) + elif health_mode: + out.append(Candidate( + m.start("cap"), m.end("cap"), cap, "LOCATION", 0.5, + state="suggested", reasons=["address-pattern", "postcode", "no-anchor"], + )) + return out + + +# --- glyph deletion rule (shared by the app and the export checks) ----------- + +def glyph_core(bbox: Rect4) -> Rect4: + """The part of a character box that decides whether a redaction deletes it. + + Measured on PyMuPDF 2.1 (MuPDF 1.28): a glyph is removed when the + redaction rectangle intersects its box shrunk by 10% of its width on + each side and 10% of its height at top and bottom. + """ + x0, y0, x1, y1 = bbox + dx, dy = (x1 - x0) * GLYPH_CORE_INSET, (y1 - y0) * GLYPH_CORE_INSET + return (x0 + dx, y0 + dy, x1 - dx, y1 - dy) + + +def core_hits(bbox: Rect4, rect: Any) -> bool: + c = glyph_core(bbox) + return c[0] < rect.x1 and c[2] > rect.x0 and c[1] < rect.y1 and c[3] > rect.y0 + + +def target_chars(page: Any, full: Any) -> List[Dict[str, Any]]: + """Characters of the line a word box was taken from (single baseline).""" + try: + blocks = page.get_text("rawdict", clip=full).get("blocks", []) + except Exception: + return [] + chars: List[Tuple[float, Dict[str, Any]]] = [] + for block in blocks: + if block.get("type") != 0: + continue + for line in block.get("lines", []): + for span in line.get("spans", []): + for ch in span.get("chars", []): + if not str(ch.get("c", "")).strip(): + continue + bx0, by0, bx1, by1 = ch["bbox"] + cx, cy = (bx0 + bx1) / 2, (by0 + by1) / 2 + if full.x0 <= cx <= full.x1 and full.y0 <= cy <= full.y1: + chars.append((round(float(ch["origin"][1]), 1), ch)) + if not chars: + return [] + centre = (full.y0 + full.y1) / 2 + bases = {b for b, _ in chars} + inside = [b for b in bases if full.y0 <= b <= full.y1] or list(bases) + base = min(inside, key=lambda b: abs(b - centre)) + return [ch for b, ch in chars if b == base] + + +# --- geometry policy ------------------------------------------------------- + +def span_word_indices( + index: PageTextIndex, start: int, end: int, min_ratio: float = 0.5 +) -> List[int]: + """Indices of the words a span covers by at least `min_ratio` of their length.""" + out: List[int] = [] + for i, w in enumerate(index.words): + overlap = min(end, w.end) - max(start, w.start) + if overlap <= 0: + continue + if overlap < min_ratio * (w.end - w.start): + continue + out.append(i) + return out + + +def rects_for_words( + index: PageTextIndex, idxs: Iterable[int], original: bool = False +) -> List[Rect4]: + """One rectangle per line for the given words (clamped, or `original`).""" + by_line: Dict[Tuple[int, int], List[Rect4]] = {} + for i in idxs: + w = index.words[i] + by_line.setdefault(w.line_key, []).append( + w.orig_rect if original and w.orig_rect else w.rect + ) + return [_union(rs) for _, rs in sorted(by_line.items())] + + +def resolve_word_indices( + index: PageTextIndex, c: Candidate +) -> Tuple[List[int], str]: + """Word indices to redact for a candidate, per the partial-match policy. + + Returns (indices, status). Status is "ok", "glued" (a validated + identifier stuck to other text: the whole token is redacted) or a + rejection reason (indices empty). + """ + text = index.text + if c.entity_type in _NER_TYPES: + if not boundary_ok(text, c.start, c.end, apostrophe_ok=True): + return [], "embedded-in-word" + idxs = span_word_indices(index, c.start, c.end) + return (idxs, "ok") if idxs else ([], "partial-word") + idxs = span_word_indices(index, c.start, c.end) + if idxs: + return idxs, "ok" + if c.entity_type in VALIDATED_TYPES: + touched = span_word_indices(index, c.start, c.end, min_ratio=0.0) + if touched: + return touched, "glued" + return [], "partial-word" + + +def protected_word_indices( + index: PageTextIndex, terms: Iterable[str] +) -> Set[int]: + """Word indices covered by whole-word occurrences of allowlist terms.""" + protected: Set[int] = set() + for term in terms: + term = (term or "").strip() + if not term: + continue + for m in _whole_word_pattern(term, False).finditer(index.text): + protected.update(span_word_indices(index, m.start(), m.end())) + return protected + + +def classify_page( + results: Iterable[Any], + index: PageTextIndex, + enabled_types: Iterable[str], + *, + used_ocr: bool = False, + health_mode: bool = False, + verb_like: Iterable[str] = (), + stopwords: Optional[Iterable[str]] = None, +) -> Tuple[List[Candidate], Dict[str, int]]: + """Run the full candidate pipeline on analyzer results for one page. + + Returns the surviving candidates (state "accepted" or "suggested") and a + count of rejections per reason (no values, safe to log). + """ + text = index.text + enabled = list(enabled_types) + stats: Dict[str, int] = {} + + def bump(reason: str, n: int = 1) -> None: + if n: + stats[reason] = stats.get(reason, 0) + n + + cands = filter_results(results, text, enabled) + cands.extend(detect_health_patterns(text, enabled, health_mode)) + kept: List[Candidate] = [] + for c in cands: + if c.entity_type in _NER_TYPES: + kept.append(c) + continue + ok, reason = refine_pattern_hit(c, text) + if ok: + kept.append(c) + else: + bump(f"pattern:{c.entity_type}:{reason}") + address = detect_address_parts(text, enabled, health_mode) + before = len(kept) + kept = clean_candidates(kept, stopwords=stopwords, enabled_types=enabled) + bump("generic-word", max(0, before - len(kept))) + kept, n = strip_lexicon_edges(kept) + bump("lexicon", n) + kept, n = reject_fragment_hits(kept, used_ocr) + bump("fragment", n) + kept = merge_adjacent_lines(kept, text) + kept.extend(address) + kept = resolve_overlaps(kept) + kept, n = evaluate_ner_candidates( + kept, text, health_mode=health_mode, verb_like=verb_like + ) + bump("ner-evidence", n) + return kept, stats + + +def build_detections( + index: PageTextIndex, + candidates: Iterable[Candidate], + *, + allowlist: Iterable[str] = (), + source: str = "text", + stats: Optional[Dict[str, int]] = None, +) -> List[Dict[str, Any]]: + """Turn candidates into detections with rectangles. + + Allowlist semantics are by word identity: a word is protected when it is + part of a whole-word occurrence of an allowlist entry. A detection whose + words are all protected disappears; a longer detection only loses the + protected words. Substring containment and rectangle intersection are + never used, so an allowlist entry cannot suppress a different value. + """ + terms = [t for t in allowlist if t and t.strip()] + allow_set = {" ".join(t.lower().split()) for t in terms} + protected = protected_word_indices(index, terms) if terms else set() + + def note(reason: str) -> None: + if stats is not None: + stats[reason] = stats.get(reason, 0) + 1 + + out: List[Dict[str, Any]] = [] + seen: Set[Tuple[str, str]] = set() + for c in candidates: + value = c.text + clean = " ".join(value.strip(_EDGE_PUNCT).lower().split()) + if not clean: + continue + if clean in allow_set: + note("allowlist") + continue + idxs, status = resolve_word_indices(index, c) + if not idxs: + note(f"geometry:{status}") + continue + idxs = [i for i in idxs if i not in protected] + if not idxs: + note("allowlist") + continue + rects = rects_for_words(index, idxs) + full_rects = rects_for_words(index, idxs, original=True) + key = (value, c.entity_type) + if key not in seen: + seen.add(key) + pattern = _whole_word_pattern( + value, c.entity_type in _NER_TYPES + ) + for m in pattern.finditer(index.text): + extra_idx = [ + i for i in span_word_indices(index, m.start(), m.end()) + if i not in protected + ] + for r, fr in zip( + rects_for_words(index, extra_idx), + rects_for_words(index, extra_idx, original=True), + ): + if r not in rects: + rects.append(r) + full_rects.append(fr) + out.append({ + "text": value, + "entity_type": c.entity_type, + "rects": rects, + "full_rects": full_rects, + "source": source, + "score": c.score, + "state": c.state, + "reasons": list(c.reasons), + "glued": status == "glued", + }) + return out + + +# --- rectangle merging / dedup --------------------------------------------- + +def merge_line_rects(rects: Iterable[Rect4], gap: float = 1.5) -> List[Rect4]: + """Merge rectangles on the same line that touch or overlap.""" + rs = sorted(rects, key=lambda r: (round((r[1] + r[3]) / 2, 0), r[0])) + merged: List[Rect4] = [] + for r in rs: + if merged: + m = merged[-1] + oy = min(m[3], r[3]) - max(m[1], r[1]) + minh = min(m[3] - m[1], r[3] - r[1]) + if minh > 0 and oy >= 0.6 * minh and r[0] - m[2] <= gap: + merged[-1] = ( + min(m[0], r[0]), min(m[1], r[1]), + max(m[2], r[2]), max(m[3], r[3]), + ) + continue + merged.append(r) + return merged + + +def covered_fraction(rect: Rect4, others: Iterable[Rect4]) -> float: + """Largest share of `rect` covered by any single rectangle of `others`.""" + area = max(0.0, (rect[2] - rect[0]) * (rect[3] - rect[1])) + if area <= 0: + return 1.0 + best = 0.0 + for o in others: + ix = min(rect[2], o[2]) - max(rect[0], o[0]) + iy = min(rect[3], o[3]) - max(rect[1], o[1]) + if ix > 0 and iy > 0: + best = max(best, ix * iy / area) + return best diff --git a/privacy_core.py b/privacy_core.py index b1ce493..d1bc082 100644 --- a/privacy_core.py +++ b/privacy_core.py @@ -7,9 +7,12 @@ from __future__ import annotations import base64 +import hashlib +import hmac import json import os import re +import secrets from dataclasses import asdict, dataclass from enum import Enum from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple @@ -40,6 +43,10 @@ class PlaceholderOccurrence: # {font, size, color, flags, origin, text}. None for scanned pages and # for maps written before version 3. style: Optional[Dict[str, Any]] = None + # What was actually drawn when the full placeholder did not fit the box + # (a compact form such as "P9F2A"). None when the full placeholder was + # drawn. Restore uses the rectangle, never this text. + label: Optional[str] = None def validate_style(raw: Any) -> Dict[str, Any]: @@ -70,10 +77,32 @@ class PlaceholderEntry: occurrences: Tuple[PlaceholderOccurrence, ...] = () +PSEUDONYM_STYLES = ("random", "counter", "opaque", "keyed") + + class PlaceholderRegistry: - """Create stable placeholders for detected personal data.""" + """Create stable placeholders for detected personal data. + + Styles (``style``): + - ``counter``: ``PERSON_001``, in first-seen order. The suffix reveals + how many distinct values of a type exist and in which order they + appear. + - ``random``: ``PERSON_9F2A``, unpredictable, drawn from ``secrets`` + for each document; nothing about the order or count leaks. + - ``opaque``: ``DATA_9F2A``; additionally hides the entity type. + - ``keyed``: ``PERSON_3B7E10`` = HMAC(key, type || value); the same + value gets the same pseudonym in every document made with the same + key. Cross-document linking is intended; the key must be kept + separately and protected (EDPB Guidelines 01/2025). + """ - def __init__(self) -> None: + def __init__(self, style: str = "counter", key: Optional[bytes] = None) -> None: + if style not in PSEUDONYM_STYLES: + raise ValueError(f"Stile pseudonimi sconosciuto: {style}") + if style == "keyed" and not key: + raise ValueError("Lo stile 'keyed' richiede una chiave.") + self._style = style + self._key = key self._counters: Dict[str, int] = {} self._by_value: Dict[tuple[str, str], str] = {} self._order: List[str] = [] @@ -115,7 +144,9 @@ def placeholder_for( if placeholder is None: next_index = self._counters.get(clean_type, 0) + 1 self._counters[clean_type] = next_index - placeholder = f"{clean_type}_{next_index:03d}" + placeholder = self._make_placeholder( + clean_type, clean_original, next_index + ) self._by_value[key] = placeholder self._order.append(placeholder) self._meta[placeholder] = (clean_original, clean_type, page_index) @@ -130,6 +161,45 @@ def placeholder_for( ) return placeholder + def set_label(self, placeholder: str, rect: Sequence[float], label: str) -> None: + """Record the compact label drawn for the occurrence at `rect`.""" + target = tuple(float(v) for v in tuple(rect)[:4]) + occs = self._occurrences.get(placeholder, []) + for i, occ in enumerate(occs): + if occ.rect == target: + occs[i] = PlaceholderOccurrence( + page=occ.page, rect=occ.rect, style=occ.style, label=label + ) + return + + def _make_placeholder(self, clean_type: str, original: str, n: int) -> str: + if self._style == "counter": + return f"{clean_type}_{n:03d}" + prefix = "DATA" if self._style == "opaque" else clean_type + taken = set(self._by_value.values()) + if self._style == "keyed": + assert self._key is not None + digest = hmac.new( + self._key, + (clean_type + "\x00" + original.casefold()).encode("utf-8"), + hashlib.sha256, + ).hexdigest().upper() + for width in range(6, len(digest) + 1, 2): + candidate = f"{prefix}_{digest[:width]}" + if candidate not in taken: + return candidate + return f"{prefix}_{digest}" + # Four hex digits keep the placeholder short enough to fit the box + # of a short word; widen only if collisions pile up. + nbytes, tries = 2, 0 + while True: + candidate = f"{prefix}_{secrets.token_hex(nbytes).upper()}" + if candidate not in taken: + return candidate + tries += 1 + if tries % 8 == 0: + nbytes += 1 + def entries(self) -> List[PlaceholderEntry]: entries: List[PlaceholderEntry] = [] for placeholder in self._order: @@ -147,12 +217,14 @@ def entries(self) -> List[PlaceholderEntry]: RESTORE_MAP_FORMAT = "NullifyPDF restore map" -RESTORE_MAP_VERSION = 3 +RESTORE_MAP_VERSION = 4 # Version 1 maps carry no `occurrences`; they are still readable, and # reconstruction falls back to searching the PDF for the placeholder text. # Refusing them would permanently strand the PII of every map already issued. # Version 3 adds an optional per-occurrence `style`; 2 maps stay readable. -SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2, 3}) +# Version 4 adds an optional per-occurrence `label` (compact placeholder drawn +# when the full one did not fit); older maps stay readable. +SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2, 3, 4}) def build_restore_payload( @@ -161,19 +233,43 @@ def build_restore_payload( source_sha256: str, output_sha256: Optional[str], entries: Iterable[PlaceholderEntry], + keep_source_name: bool = False, ) -> Dict[str, object]: - """Build the JSON-serializable restore-map payload.""" + """Build the JSON-serializable restore-map payload. + + The map is itself personal data, so it stores as little as possible: the + source file name is omitted unless asked for (it often holds the data + subject's name), and a style's ``text`` is dropped when it only repeats + the original value already stored in the entry. + """ raw_entries: List[Dict[str, object]] = [] for entry in entries: raw = asdict(entry) # asdict() keeps tuples as tuples; JSON round-trips them to lists, so # normalize here and keep the payload byte-comparable with itself. + def slim( + style: Optional[Dict[str, Any]], original: str = entry.original + ) -> Optional[Dict[str, Any]]: + if not style: + return None + out = dict(style) + if " ".join(str(out.get("text", "")).split()) == " ".join( + original.split() + ): + out.pop("text", None) + return out + raw["occurrences"] = [ { "page": occurrence.page, "rect": list(occurrence.rect), - **({"style": occurrence.style} if occurrence.style else {}), + **({"style": slim(occurrence.style)} if occurrence.style else {}), + **( + {"label": occurrence.label} + if occurrence.label and occurrence.label != entry.placeholder + else {} + ), } for occurrence in entry.occurrences ] @@ -182,7 +278,7 @@ def build_restore_payload( return { "format": RESTORE_MAP_FORMAT, "version": RESTORE_MAP_VERSION, - "source_name": os.path.basename(source_name), + "source_name": os.path.basename(source_name) if keep_source_name else "", "source_sha256": source_sha256, "output_sha256": output_sha256, "entries": raw_entries, @@ -220,6 +316,7 @@ def parse_restore_entries(payload: Dict[str, object]) -> List[PlaceholderEntry]: if raw_occ.get("style") else None ), + label=(str(raw_occ["label"]) if raw_occ.get("label") else None), ) ) entries.append( @@ -251,55 +348,369 @@ def sort_entries_longest_placeholder_first( return sorted(entries, key=lambda e: len(e.placeholder), reverse=True) -def encrypt_restore_payload(payload: Dict[str, object], password: str) -> bytes: - """Encrypt and authenticate a restore-map payload with a password.""" +MIN_PASSWORD_LENGTH = 12 +PBKDF2_MIN_ITERATIONS = 100_000 +PBKDF2_MAX_ITERATIONS = 5_000_000 +SCRYPT_N = 2 ** 17 +SCRYPT_R = 8 +SCRYPT_P = 1 +SCRYPT_MAX_N = 2 ** 20 - if not password or len(password) < 12: - raise ValueError("La password della mappa deve avere almeno 12 caratteri.") +_COMMON_PASSWORDS = ( + "passwordpassword", "123456789012", "qwertyuiopas", "password1234", + "administrator", "letmein123456", +) - from cryptography.fernet import Fernet + +def password_strength(password: str) -> Tuple[bool, str]: + """Reject passwords that make the map trivial to open. + + Returns (acceptable, message). Length alone is not enough: a repeated + character or a common phrase of 12 characters falls in seconds. + """ + if len(password) < MIN_PASSWORD_LENGTH: + return False, "La password deve avere almeno 12 caratteri." + if len(set(password)) < 6: + return False, "La password usa troppi caratteri ripetuti." + low = password.lower() + if any(c in low for c in _COMMON_PASSWORDS): + return False, "La password e troppo comune." + classes = sum([ + any(c.islower() for c in password), + any(c.isupper() for c in password), + any(c.isdigit() for c in password), + any(not c.isalnum() for c in password), + ]) + if classes < 2 and len(password) < 20: + return False, ( + "Usa almeno due tipi di caratteri (minuscole, maiuscole, cifre, " + "simboli) oppure una frase di almeno 20 caratteri." + ) + return True, "" + + +def _derive_key(password: str, envelope: Dict[str, Any], salt: bytes) -> bytes: from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + from cryptography.hazmat.primitives.kdf.scrypt import Scrypt + + kdf_name = str(envelope.get("kdf", "PBKDF2-HMAC-SHA256")) + if kdf_name == "scrypt": + n = int(envelope["n"]) + r = int(envelope.get("r", SCRYPT_R)) + p_ = int(envelope.get("p", SCRYPT_P)) + # Parameters come from an untrusted file: bound them before use so a + # crafted map cannot exhaust memory or CPU. + if ( + not (2 ** 14 <= n <= SCRYPT_MAX_N) or n & (n - 1) + or not (1 <= r <= 16) or not (1 <= p_ <= 4) + ): + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + kdf: Any = Scrypt(salt=salt, length=32, n=n, r=r, p=p_) + elif kdf_name == "PBKDF2-HMAC-SHA256": + iterations = int(envelope["iterations"]) + if not (PBKDF2_MIN_ITERATIONS <= iterations <= PBKDF2_MAX_ITERATIONS): + raise ValueError( + "Numero di iterazioni della mappa fuori dai limiti consentiti." + ) + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), length=32, salt=salt, iterations=iterations + ) + else: + raise ValueError("Algoritmo KDF della mappa non supportato.") + return base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) + + +def _encrypt_v2(payload: Dict[str, object], password: str) -> bytes: + """Version 2 envelope (scrypt + Fernet). Kept for compatibility tests; + new maps use :func:`encrypt_restore_payload` (version 3).""" + + if not password or len(password) < MIN_PASSWORD_LENGTH: + raise ValueError("La password della mappa deve avere almeno 12 caratteri.") + + from cryptography.fernet import Fernet salt = os.urandom(16) - kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), - length=32, - salt=salt, - iterations=600000, - ) - key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) - token = Fernet(key).encrypt( - json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") - ) - envelope = { + envelope: Dict[str, Any] = { "format": "NullifyPDF encrypted restore map", - "version": 1, - "kdf": "PBKDF2-HMAC-SHA256", - "iterations": 600000, + "version": 2, + "kdf": "scrypt", + "n": SCRYPT_N, + "r": SCRYPT_R, + "p": SCRYPT_P, "salt": base64.b64encode(salt).decode("ascii"), - "token": token.decode("ascii"), } + key = _derive_key(password, envelope, salt) + token = Fernet(key).encrypt( + json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") + ) + envelope["token"] = token.decode("ascii") return json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=2).encode( "utf-8" ) -def decrypt_restore_payload(data: bytes, password: str) -> Dict[str, object]: - """Decrypt an encrypted restore map.""" +def _decrypt_v1v2(envelope: Dict[str, Any], password: str) -> Dict[str, object]: + """Decrypt a version 1 (PBKDF2) or 2 (scrypt) envelope.""" from cryptography.fernet import Fernet - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC - envelope = json.loads(data.decode("utf-8")) salt = base64.b64decode(envelope["salt"]) - kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), - length=32, - salt=salt, - iterations=int(envelope["iterations"]), - ) - key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) + key = _derive_key(password, envelope, salt) plaintext = Fernet(key).decrypt(envelope["token"].encode("ascii")) return json.loads(plaintext.decode("utf-8")) + + +# --------------------------------------------------------------------------- +# Envelope version 3: data key + key wrapping (password, optional recovery key) +# --------------------------------------------------------------------------- + +MAP_FORMAT = "NullifyPDF encrypted restore map" +KEY_EXPORT_FORMAT = "NullifyPDF pseudonym key" +ARGON2_T = 3 +ARGON2_M_KIB = 64 * 1024 +ARGON2_P = 4 +_PAYLOAD_AAD = b"NullifyPDF envelope v3 payload" +_RECOVERY_PREFIX = "NPDF" +_B32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" + + +def _b64(b: bytes) -> str: + return base64.b64encode(b).decode("ascii") + + +def _unb64(s_: str) -> bytes: + return base64.b64decode(s_.encode("ascii")) + + +def _argon2_available() -> bool: + try: + from cryptography.hazmat.primitives.kdf.argon2 import Argon2id # noqa: F401 + + return True + except Exception: + return False + + +def _password_kek(password: str, wrap: Dict[str, Any]) -> bytes: + """Derive the key-encryption key for a password wrap (bounds-checked).""" + salt = _unb64(wrap["salt"]) + if len(salt) < 16: + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + kdf = str(wrap.get("kdf")) + if kdf == "argon2id": + from cryptography.hazmat.primitives.kdf.argon2 import Argon2id + + t, m, p_ = int(wrap["t"]), int(wrap["m"]), int(wrap["p"]) + if not (1 <= t <= 10 and 19 * 1024 <= m <= 1024 * 1024 and 1 <= p_ <= 8 and m >= 8 * p_): + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + return Argon2id( + salt=salt, length=32, iterations=t, lanes=p_, memory_cost=m + ).derive(password.encode("utf-8")) + if kdf == "scrypt": + from cryptography.hazmat.primitives.kdf.scrypt import Scrypt + + n, r, p_ = int(wrap["n"]), int(wrap["r"]), int(wrap["p"]) + if ( + not (2 ** 14 <= n <= SCRYPT_MAX_N) or n & (n - 1) + or not (1 <= r <= 16) or not (1 <= p_ <= 4) + ): + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + return Scrypt(salt=salt, length=32, n=n, r=r, p=p_).derive(password.encode("utf-8")) + raise ValueError("Algoritmo KDF della mappa non supportato.") + + +def _new_password_wrap(password: str, dek: bytes) -> Dict[str, Any]: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + salt = os.urandom(16) + wrap: Dict[str, Any] = {"type": "password", "salt": _b64(salt)} + if _argon2_available(): + wrap.update(kdf="argon2id", t=ARGON2_T, m=ARGON2_M_KIB, p=ARGON2_P) + try: + kek = _password_kek(password, wrap) + except Exception: # OpenSSL without Argon2: fall back for writing only + wrap = {"type": "password", "salt": _b64(salt)} + wrap.update(kdf="scrypt", n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P) + kek = _password_kek(password, wrap) + else: + wrap.update(kdf="scrypt", n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P) + kek = _password_kek(password, wrap) + nonce = os.urandom(12) + wrap["nonce"] = _b64(nonce) + wrap["wrapped"] = _b64(AESGCM(kek).encrypt(nonce, dek, b"NullifyPDF wrap v3:password")) + return wrap + + +def _recovery_kek(recovery_key: bytes, salt: bytes) -> bytes: + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + return HKDF( + algorithm=hashes.SHA256(), length=32, salt=salt, info=b"nullifypdf-recovery" + ).derive(recovery_key) + + +def _new_recovery_wrap(recovery_key: bytes, dek: bytes) -> Dict[str, Any]: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + if len(recovery_key) != 32: + raise ValueError("Chiave di recupero non valida.") + salt, nonce = os.urandom(16), os.urandom(12) + kek = _recovery_kek(recovery_key, salt) + return { + "type": "recovery", "kdf": "hkdf-sha256", "salt": _b64(salt), + "nonce": _b64(nonce), + "wrapped": _b64(AESGCM(kek).encrypt(nonce, dek, b"NullifyPDF wrap v3:recovery")), + "key_id": hashlib.sha256(recovery_key).hexdigest()[:8], + } + + +def generate_recovery_key() -> Tuple[bytes, str]: + """A fresh 256-bit recovery key and its display form (shown once). + + Display: ``NPDF-XXXX-...`` in base32 groups of four with a 2-character + checksum, so typing mistakes are caught before any decryption. + """ + raw = os.urandom(32) + return raw, format_recovery_key(raw) + + +def _checksum(raw: bytes) -> str: + v = int.from_bytes(hashlib.sha256(raw).digest()[:2], "big") & 0x3FF + return _B32[v >> 5] + _B32[v & 31] + + +def format_recovery_key(raw: bytes) -> str: + body = base64.b32encode(raw).decode("ascii").rstrip("=") + _checksum(raw) + groups = [body[i:i + 4] for i in range(0, len(body), 4)] + return _RECOVERY_PREFIX + "-" + "-".join(groups) + + +def parse_recovery_key(display: str) -> bytes: + """Parse a displayed recovery key; raises ValueError on any typo.""" + text = "".join(display.upper().split()).replace("-", "") + if text.startswith(_RECOVERY_PREFIX): + text = text[len(_RECOVERY_PREFIX):] + text = text.replace("0", "O").replace("1", "I") + if len(text) != 54 or any(c not in _B32 for c in text): + raise ValueError("Chiave di recupero non valida.") + body, check = text[:52], text[52:] + try: + raw = base64.b32decode(body + "====") + except Exception as e: + raise ValueError("Chiave di recupero non valida.") from e + if len(raw) != 32 or _checksum(raw) != check: + raise ValueError("Chiave di recupero non valida: controlla di averla digitata bene.") + return raw + + +def _encrypt_v3( + payload: Dict[str, object], password: str, recovery_key: Optional[bytes], fmt: str +) -> bytes: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + dek = os.urandom(32) + nonce = os.urandom(12) + ciphertext = AESGCM(dek).encrypt( + nonce, + json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8"), + _PAYLOAD_AAD, + ) + wraps = [_new_password_wrap(password, dek)] + if recovery_key is not None: + wraps.append(_new_recovery_wrap(recovery_key, dek)) + envelope = { + "format": fmt, "version": 3, "cipher": "AES-256-GCM", + "nonce": _b64(nonce), "ciphertext": _b64(ciphertext), "wraps": wraps, + } + return json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=2).encode("utf-8") + + +def _decrypt_v3( + envelope: Dict[str, Any], password: Optional[str], recovery_key: Optional[bytes] +) -> Dict[str, object]: + from cryptography.exceptions import InvalidTag + from cryptography.fernet import InvalidToken + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + dek: Optional[bytes] = None + for wrap in envelope.get("wraps", []): + try: + if wrap.get("type") == "password" and password is not None: + kek = _password_kek(password, wrap) + aad = b"NullifyPDF wrap v3:password" + elif wrap.get("type") == "recovery" and recovery_key is not None: + kek = _recovery_kek(recovery_key, _unb64(wrap["salt"])) + aad = b"NullifyPDF wrap v3:recovery" + else: + continue + dek = AESGCM(kek).decrypt(_unb64(wrap["nonce"]), _unb64(wrap["wrapped"]), aad) + break + except InvalidTag: + continue + if dek is None: + raise InvalidToken() # same signal the callers already handle + try: + plaintext = AESGCM(dek).decrypt( + _unb64(envelope["nonce"]), _unb64(envelope["ciphertext"]), _PAYLOAD_AAD + ) + except InvalidTag as e: + raise InvalidToken() from e + return json.loads(plaintext.decode("utf-8")) + + +def encrypt_restore_payload( + payload: Dict[str, object], + password: str, + *, + recovery_key: Optional[bytes] = None, +) -> bytes: + """Encrypt a restore-map payload (envelope version 3). + + A random data key encrypts the payload (AES-256-GCM); it is wrapped by a + key derived from the password (Argon2id, scrypt if unavailable) and, if + given, by the recovery key. Maps from older versions remain readable. + """ + if not password or len(password) < MIN_PASSWORD_LENGTH: + raise ValueError("La password della mappa deve avere almeno 12 caratteri.") + return _encrypt_v3(payload, password, recovery_key, MAP_FORMAT) + + +def decrypt_restore_payload( + data: bytes, + password: Optional[str] = None, + *, + recovery_key: Optional[bytes] = None, +) -> Dict[str, object]: + """Decrypt an encrypted restore map (versions 1, 2 and 3).""" + envelope = json.loads(data.decode("utf-8")) + if not isinstance(envelope, dict) or envelope.get("format") != MAP_FORMAT: + raise ValueError("File non riconosciuto come mappa di ripristino cifrata.") + version = envelope.get("version") + if version in (1, 2): + if password is None: + raise ValueError("Questa mappa si apre solo con la password.") + return _decrypt_v1v2(envelope, password) + if version == 3: + return _decrypt_v3(envelope, password, recovery_key) + raise ValueError("Versione della mappa non supportata: aggiorna NullifyPDF.") + + +def export_pseudonym_key(key: bytes, password: str) -> bytes: + """Password-protected copy of the pseudonym key, for another computer.""" + if len(key) != 32: + raise ValueError("Chiave dei segnaposto non valida.") + return _encrypt_v3({"key": _b64(key)}, password, None, KEY_EXPORT_FORMAT) + + +def import_pseudonym_key(data: bytes, password: str) -> bytes: + envelope = json.loads(data.decode("utf-8")) + if not isinstance(envelope, dict) or envelope.get("format") != KEY_EXPORT_FORMAT: + raise ValueError("File non riconosciuto come chiave dei segnaposto.") + if envelope.get("version") != 3: + raise ValueError("Versione della chiave non supportata: aggiorna NullifyPDF.") + key = _unb64(str(_decrypt_v3(envelope, password, None)["key"])) + if len(key) != 32: + raise ValueError("Chiave dei segnaposto non valida.") + return key diff --git a/requirements.txt b/requirements.txt index 2137d8d..8b36a1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,30 @@ # Requirements for NullifyPDF - Python 3.13 REQUIRED -# PyMuPDF 1.28.0+ ships abi3 wheels (cp310-abi3) that cover Python 3.13 as well +# PyMuPDF ships abi3 wheels (cp310-abi3) that cover Python 3.13 as well # Update script handles version detection automatically # UI Framework (compatible with Python 3.13) -PySide6==6.11.2 +PySide6==6.12.0 # PDF Processing (abi3 wheel, compatible with Python 3.13) -PyMuPDF==1.28.2 +PyMuPDF==2.1 # NLP & Security Analysis -presidio-analyzer==2.2.364 -spacy==3.8.16 -cryptography==50.0.1 +presidio-analyzer==2.2.366 +spacy==3.8.18 +# Local token-classification models; PyTorch is intentionally not required at runtime. +onnxruntime==1.31.1 +tokenizers==0.23.3 +cryptography==50.0.2 +# OS keyring (macOS Keychain / Linux Secret Service) for the local master key; +# Windows uses DPAPI instead, so it is not installed there. +keyring==25.7.0; sys_platform != "win32" # spaCy language models (pinned to exact versions for reproducibility) # IMPORTANT: model minor version MUST match the spaCy core minor version # (spaCy 3.8.x <-> models 3.8.x). The models declare spacy>=3.8.0,<3.9.0. # These are cached by pip in GitHub Actions, significantly speeding up CI/CD -en-core-web-md @ https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl +# Development and CI only (tests of the Italian detectors). Licence CC BY-NC-SA 3.0: +# NOT bundled in the installer (excluded in the PyInstaller spec); the app +# downloads it on demand after the user accepts the licence (spacy_models.py). it-core-news-md @ https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.8.0/it_core_news_md-3.8.0-py3-none-any.whl # Build Tool (compatible with Python 3.13) @@ -24,5 +32,5 @@ pyinstaller==6.22.3 # Development Tools isort==5.13.2 -pytest==9.1.1 +pytest==9.2.0 mypy==1.20.2 diff --git a/review_dialog.py b/review_dialog.py new file mode 100644 index 0000000..b3bdd31 --- /dev/null +++ b/review_dialog.py @@ -0,0 +1,219 @@ +"""Review dialog for AI suggestions (Qt). + +Detections the evidence score could not settle are *suggestions*: they are +never applied silently and never lost silently. The user accepts or rejects +each one here; values are masked until the user asks to see them. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from help_guide import apply_tooltips +from PySide6.QtCore import Qt, Signal +from PySide6.QtWidgets import ( + QAbstractItemView, + QCheckBox, + QComboBox, + QDialog, + QDoubleSpinBox, + QHBoxLayout, + QHeaderView, + QLabel, + QPushButton, + QTableWidget, + QTableWidgetItem, + QVBoxLayout, +) + +REASON_LABELS: Dict[str, str] = { + "context-cue": "contesto", + "name-shape": "forma di nome", + "given-name": "nome proprio noto", + "known-place": "luogo noto", + "province-or-postcode": "provincia/CAP", + "repeated": "ripetuto", + "lexicon": "parola comune", + "verb-form": "forma verbale", + "line-start": "inizio riga", + "all-caps": "maiuscolo", + "followed-by-function-word": "seguito da che/di", + "low-confidence": "bassa confidenza", + "signature-anchor": "vicino a firma/timbro", + "overlaps-text": "il riquadro tocca del testo", + "curve-ink": "tratti a mano", + "raster-stamp": "timbro/immagine", + "no-context": "senza contesto", + "no-anchor": "senza riferimento", + "postcode": "CAP", + "known-surname": "cognome noto", + "ambiguous-place": "luogo omonimo di parola comune", +} + +COL_APPLY, COL_PAGE, COL_VALUE, COL_TYPE, COL_SCORE, COL_SOURCE, COL_WHY = range(7) + + +def mask_value(value: str) -> str: + v = " ".join(value.split()) + if len(v) <= 1: + return "โ€ข" + return v[0] + "โ€ข" * min(len(v) - 1, 8) + + +class ReviewDialog(QDialog): + """Table of suggestions with per-row accept / reject.""" + + navigate = Signal(int) + highlight = Signal(object) # the suggestion dict of the selected row + + def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> None: + super().__init__(parent) + self.setWindowTitle("Revisione suggerimenti") + self.resize(900, 460) + self._items = list(suggestions) + self._applied: List[Dict[str, Any]] = [] + self._rejected: List[Dict[str, Any]] = [] + lay = QVBoxLayout(self) + intro = QLabel( + "Questi elementi non hanno prove sufficienti per essere censurati " + "automaticamente. Seleziona quelli da censurare: i non selezionati " + "vengono scartati se premi ยซApplica selezionatiยป." + ) + intro.setWordWrap(True) + lay.addWidget(intro) + if len(self._items) > 30: + warning = QLabel( + f"Attenzione: {len(self._items)} suggerimenti. " + "Controlla l'elenco pagina per pagina; non approvare in blocco senza verifica." + ) + warning.setObjectName("review_many_warning") + warning.setStyleSheet("color: #f59e0b; font-weight: bold;") + warning.setWordWrap(True) + lay.addWidget(warning) + self.table = QTableWidget(len(self._items), 7, self) + self.table.setObjectName("review_table") + self.table.verticalHeader().setVisible(False) + self.table.setHorizontalHeaderLabels( + ["Censura", "Pag.", "Valore", "Tipo", "Punteggio", "Fonte", "Motivi"] + ) + self.table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) + self.table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) + self.table.horizontalHeader().setSectionResizeMode( + COL_WHY, QHeaderView.ResizeMode.Stretch + ) + lay.addWidget(self.table) + self.chk_show = QCheckBox("Mostra i valori") + self.chk_show.toggled.connect(self._refresh_values) + lay.addWidget(self.chk_show) + for row, it in enumerate(self._items): + apply_item = QTableWidgetItem() + apply_item.setFlags( + Qt.ItemFlag.ItemIsUserCheckable | Qt.ItemFlag.ItemIsEnabled + ) + apply_item.setCheckState(Qt.CheckState.Unchecked) + self.table.setItem(row, COL_APPLY, apply_item) + self.table.setItem(row, COL_PAGE, QTableWidgetItem(str(int(it.get("page", 0)) + 1))) + self.table.setItem(row, COL_VALUE, QTableWidgetItem("")) + type_box = QComboBox(self.table) + types = ["PERSON", "LOCATION", "EMAIL", "PHONE", "IBAN", "CARD", "FISCAL", "VAT", "LICENSE", "IDCARD", "PASSPORT", "BIRTHDATE", "HEALTHID", "EXEMPT", "DATA"] + type_box.addItems(types) + current_type = str(it.get("entity_type", "DATA")) + type_box.setCurrentText(current_type if current_type in types else "DATA") + type_box.setToolTip("Modifica il tipo attribuito al suggerimento prima dell'applicazione.") + self.table.setCellWidget(row, COL_TYPE, type_box) + self.table.setItem(row, COL_SCORE, QTableWidgetItem(f"{float(it.get('score', 0.0)):.2f}")) + self.table.setItem(row, COL_SOURCE, QTableWidgetItem(str(it.get("source", "regole")))) + why = ", ".join(REASON_LABELS.get(r, r) for r in it.get("reasons", [])) + self.table.setItem(row, COL_WHY, QTableWidgetItem(why)) + self._refresh_values() + self.table.cellDoubleClicked.connect(self._on_double_click) + self.table.currentCellChanged.connect(self._on_row_changed) + + bar = QHBoxLayout() + btn_all = QPushButton("Seleziona tutti") + btn_none = QPushButton("Nessuno") + self.spin = QDoubleSpinBox() + self.spin.setRange(0.0, 1.0) + self.spin.setSingleStep(0.05) + self.spin.setValue(0.6) + self.spin.setObjectName("review_threshold") + btn_thr = QPushButton("Seleziona โ‰ฅ soglia") + # Bound methods, not lambdas capturing `self`: a lambda makes a + # reference cycle that keeps the dialog alive until the garbage + # collector runs, possibly after QApplication is gone. + btn_all.clicked.connect(self._select_all) + btn_none.clicked.connect(self._select_none) + btn_thr.clicked.connect(self._select_threshold) + for w in (btn_all, btn_none, self.spin, btn_thr): + bar.addWidget(w) + bar.addStretch(1) + lay.addLayout(bar) + + bottom = QHBoxLayout() + self.btn_apply = QPushButton("Applica selezionati") + self.btn_apply.setObjectName("Danger") + self.btn_later = QPushButton("Rivedi piรน tardi") + self.btn_apply.clicked.connect(self._on_apply) + self.btn_later.clicked.connect(self.reject) + bottom.addStretch(1) + bottom.addWidget(self.btn_later) + bottom.addWidget(self.btn_apply) + lay.addLayout(bottom) + apply_tooltips(self) + + # -- helpers --------------------------------------------------------- + def _cell(self, row: int, col: int) -> QTableWidgetItem: + item = self.table.item(row, col) + assert item is not None + return item + + def _refresh_values(self) -> None: + show = self.chk_show.isChecked() + for row, it in enumerate(self._items): + text = " ".join(str(it.get("text", "")).split()) + if not text: + self._cell(row, COL_VALUE).setText("(area grafica)") + continue + self._cell(row, COL_VALUE).setText(text if show else mask_value(text)) + + def _select_all(self) -> None: + self._set_all(True) + + def _select_none(self) -> None: + self._set_all(False) + + def _set_all(self, state: bool) -> None: + for row in range(self.table.rowCount()): + self._cell(row, COL_APPLY).setCheckState( + Qt.CheckState.Checked if state else Qt.CheckState.Unchecked + ) + + def _select_threshold(self) -> None: + thr = self.spin.value() + for row, it in enumerate(self._items): + if float(it.get("score", 0.0)) >= thr: + self._cell(row, COL_APPLY).setCheckState(Qt.CheckState.Checked) + + def _on_row_changed(self, row: int, _c: int, _pr: int, _pc: int) -> None: + if 0 <= row < len(self._items): + self.highlight.emit(self._items[row]) + + def _on_double_click(self, row: int, _col: int) -> None: + self.navigate.emit(int(self._items[row].get("page", 0))) + + def _on_apply(self) -> None: + self._applied, self._rejected = [], [] + for row, it in enumerate(self._items): + checked = self._cell(row, COL_APPLY).checkState() == Qt.CheckState.Checked + type_box = self.table.cellWidget(row, COL_TYPE) + if isinstance(type_box, QComboBox): + it["entity_type"] = type_box.currentText() + (self._applied if checked else self._rejected).append(it) + self.accept() + + # -- results --------------------------------------------------------- + def applied(self) -> List[Dict[str, Any]]: + return list(self._applied) + + def declined(self) -> List[Dict[str, Any]]: + return list(self._rejected) diff --git a/scripts/build_lexicon_data.py b/scripts/build_lexicon_data.py new file mode 100644 index 0000000..ba6cebe --- /dev/null +++ b/scripts/build_lexicon_data.py @@ -0,0 +1,209 @@ +"""Generate ``lexicon_data_it.py`` from ISTAT open data (development only). + +Not part of the runtime and not imported by the application. It reads the +ISTAT "Elenco comuni italiani" workbook (CC BY 4.0, https://www.istat.it/note-legali/), +normalises the place names, marks the ones that are also ordinary words as +ambiguous, and writes a plain Python module that PyInstaller collects like any +other import. Only the standard library is needed to read the workbook. + + python scripts/build_lexicon_data.py [--istat-file FILE] [--out FILE] + +The generated module only *confirms* PERSON/LOCATION hits; a name missing +from it never rejects one. +""" + +from __future__ import annotations + +import argparse +import hashlib +import pathlib +import re +import sys +import urllib.request +import xml.etree.ElementTree as ET +import zipfile +from typing import Dict, List, Set + +WIKIDATA_URL = "https://query.wikidata.org/sparql" +# Names that have an article on the Italian Wikipedia: a bounded, relevant set +# (counting people per name would time out on the public endpoint). +NAME_QUERY = ( + 'SELECT DISTINCT ?l WHERE {{ ?n wdt:P31 wd:{cls}. ' + '?a schema:about ?n; schema:isPartOf . ' + '?n rdfs:label ?l. FILTER(LANG(?l)="it") }} LIMIT {page} OFFSET {off}' +) +NAME_CLASSES = {"given": "Q202444", "surnames": "Q101352"} +NAME_PAGE = 2000 +NAME_OK = re.compile(r"^[a-zร -รฟ'โ€™-]{3,24}$") +ISTAT_URL = "https://www.istat.it/storage/codici-unita-amministrative/Elenco-comuni-italiani.xlsx" +NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} +COLUMNS = { + "Denominazione in italiano": "it", + "Denominazione altra lingua": "alt", + "Sigla automobilistica": "sigla", + "Denominazione Regione": "region", +} +ROOT = pathlib.Path(__file__).resolve().parent.parent + + +def _col(ref: str) -> str: + return re.match(r"[A-Z]+", ref).group() # type: ignore[union-attr] + + +def read_rows(path: pathlib.Path) -> "tuple[List[str], List[Dict[str, str]]]": + """First worksheet of an .xlsx as a list of header->value dicts.""" + z = zipfile.ZipFile(path) + sst = [ + "".join(t.text or "" for t in si.iter("{%s}t" % NS["m"])) + for si in ET.fromstring(z.read("xl/sharedStrings.xml")).findall("m:si", NS) + ] if "xl/sharedStrings.xml" in z.namelist() else [] + sheet = ET.fromstring(z.read("xl/worksheets/sheet1.xml")) + rows = sheet.findall(".//m:row", NS) + headers: Dict[str, str] = {} + out: List[Dict[str, str]] = [] + for i, row in enumerate(rows): + cells: Dict[str, str] = {} + for c in row.findall("m:c", NS): + v = c.find("m:v", NS) + if v is None or v.text is None: + continue + cells[_col(c.get("r", ""))] = sst[int(v.text)] if c.get("t") == "s" else v.text + if i == 0: + headers = {col: " ".join(val.split()) for col, val in cells.items()} + continue + out.append({headers[col]: val for col, val in cells.items() if col in headers}) + return list(headers.values()), out + + +def fetch_names(kind: str, retries: int = 6) -> Set[str]: + """Names of one kind (Wikidata, CC0), paged to stay under the query timeout.""" + import json + import time + import urllib.parse + + out: Set[str] = set() + off = 0 + while True: + q = NAME_QUERY.format(cls=NAME_CLASSES[kind], page=NAME_PAGE, off=off) + url = WIKIDATA_URL + "?format=json&query=" + urllib.parse.quote(q) + for attempt in range(retries): + try: + req = urllib.request.Request( + url, headers={"User-Agent": "NullifyPDF-build/1.0 (github.com/overwrite00/NullifyPDF)"} + ) + rows = json.load(urllib.request.urlopen(req, timeout=120))["results"]["bindings"] + break + except Exception as e: + if attempt == retries - 1: + raise RuntimeError(f"Wikidata {kind} page {off}: {e}") from e + time.sleep(10 * (attempt + 1)) + for r in rows: + n = normalise(r["l"]["value"]) + if NAME_OK.match(n): + out.add(n) + if len(rows) < NAME_PAGE: + return out + off += NAME_PAGE + + +def normalise(name: str) -> str: + return " ".join(name.replace("โ€™", "'").lower().split()) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--istat-file", type=pathlib.Path) + ap.add_argument("--wikidata", action="store_true", + help="also fetch given names / surnames (CC0) from Wikidata") + ap.add_argument("--out", type=pathlib.Path, default=ROOT / "lexicon_data_it.py") + args = ap.parse_args() + src = args.istat_file + if src is None: + src = pathlib.Path("Elenco-comuni-italiani.xlsx") + req = urllib.request.Request(ISTAT_URL, headers={"User-Agent": "NullifyPDF-build/1.0"}) + src.write_bytes(urllib.request.urlopen(req, timeout=120).read()) + sha = hashlib.sha256(src.read_bytes()).hexdigest() + headers, rows = read_rows(src) + missing = set(COLUMNS) - set(headers) + if missing: + print(f"Workbook layout changed, missing columns: {sorted(missing)}", file=sys.stderr) + return 1 + + comuni: Set[str] = set() + sigle: Set[str] = set() + regions: Set[str] = set() + for r in rows: + for key in ("Denominazione in italiano", "Denominazione altra lingua"): + for part in re.split(r"\s*/\s*", r.get(key, "")): + n = normalise(part) + if len(n) >= 2: + comuni.add(n) + if r.get("Sigla automobilistica"): + sigle.add(r["Sigla automobilistica"].strip().upper()) + for part in re.split(r"\s*/\s*", r.get("Denominazione Regione", "")): + if part.strip(): + regions.add(normalise(part)) + + sys.path.insert(0, str(ROOT)) + import lexicon_it + + common = set(lexicon_it.COMMON_WORDS | lexicon_it.FUNCTION_WORDS | lexicon_it.GIVEN_NAMES + | lexicon_it.CONFLICT_NAMES) + vocab_words: Set[str] = set() + nlp = None + try: + import spacy + + nlp = spacy.load("it_core_news_md") + vocab_words = {w for w in comuni if " " not in w and nlp.vocab.has_vector(w)} + except Exception as e: # model optional for the generator + print(f"spaCy vocabulary not used: {e}", file=sys.stderr) + ambiguous = {w for w in comuni if len(w) <= 3 or w in common or w in vocab_words} + + def lit(items: Set[str]) -> str: + body = "|".join(sorted(items)) + assert '"""' not in body and "\\" not in body + return 'frozenset("""' + body + '""".split("|"))' + + names_text = "" + if args.wikidata: + given = fetch_names("given") + surn = fetch_names("surnames") + both = given | surn + name_ambiguous = {w for w in both if w in common or len(w) <= 3} + if nlp is not None: + try: + name_ambiguous |= {w for w in both if " " not in w and nlp.vocab.has_vector(w) + and w not in lexicon_it.GIVEN_NAMES and w not in lexicon_it.SURNAMES} + except Exception: + pass + names_text = ( + "# Given names and surnames: Wikidata (CC0,\n" + "# https://www.wikidata.org/wiki/Wikidata:Licensing), entries that\n" + "# have an article on the Italian Wikipedia.\n" + f"NAMES_GIVEN = {lit(given)}\n" + f"NAMES_SURNAMES = {lit(surn)}\n" + f"NAMES_AMBIGUOUS = {lit(name_ambiguous)}\n" + ) + print(f"{len(given)} given names, {len(surn)} surnames ({len(name_ambiguous)} ambiguous)") + + text = ( + "# Generated by scripts/build_lexicon_data.py - do not edit.\n" + "# Contains ISTAT data (Elenco comuni italiani, " + src.name + "), licensed\n" + "# CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/); modified: names\n" + "# normalised, lower-cased, bilingual names split.\n" + f"SOURCE_SHA256 = {{'istat_comuni': '{sha}'}}\n" + f"COMUNI = {lit(comuni)}\n" + f"COMUNI_AMBIGUOUS = {lit(ambiguous)}\n" + f"PROVINCE_SIGLE = {lit(sigle)}\n" + f"REGIONS = {lit(regions)}\n" + + names_text + ) + args.out.write_text(text, encoding="utf-8") + print(f"{len(comuni)} comuni ({len(ambiguous)} ambiguous), {len(sigle)} province codes, " + f"{len(regions)} regions -> {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/extract_release_notes.py b/scripts/extract_release_notes.py new file mode 100644 index 0000000..f42a5c9 --- /dev/null +++ b/scripts/extract_release_notes.py @@ -0,0 +1,54 @@ +"""Print the CHANGELOG.md section that matches a release tag. + +Used by the release workflows to fill the GitHub release body, which the +in-app update check then shows to the user. + +* stable tag ``vX.Y.Z`` -> the ``## [X.Y.Z] - date`` section +* beta tag ``vX.Y.Z-beta.N`` -> the ``## [Unreleased]`` section (betas get + no heading of their own; only the final stable version does) + +Usage: python scripts/extract_release_notes.py [CHANGELOG.md] +Exits 1 and prints nothing when the section is missing or empty. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +def extract_notes(changelog: str, tag: str) -> str: + version = tag[1:] if tag.startswith("v") else tag + wanted = "Unreleased" if "-" in version else re.escape(version) + lines = changelog.splitlines() + start = None + for i, line in enumerate(lines): + if re.match(rf"^## \[{wanted}\]", line): + start = i + 1 + break + if start is None: + return "" + body: list[str] = [] + for line in lines[start:]: + if line.startswith("## ["): + break + body.append(line) + return "\n".join(body).strip() + + +def main(argv: list[str]) -> int: + if len(argv) < 2: + print(__doc__, file=sys.stderr) + return 2 + path = Path(argv[2]) if len(argv) > 2 else Path("CHANGELOG.md") + notes = extract_notes(path.read_text(encoding="utf-8"), argv[1]) + if not notes: + return 1 + sys.stdout.reconfigure(encoding="utf-8") # type: ignore[attr-defined] + print(notes) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/secure_store.py b/secure_store.py new file mode 100644 index 0000000..ff3a4de --- /dev/null +++ b/secure_store.py @@ -0,0 +1,365 @@ +"""Protection of local secrets (Qt-free). + +A single random master key protects the word lists and the pseudonym key. +The master key itself is kept by the best protector available: + +* Windows: DPAPI (``CryptProtectData``, via ctypes - no extra dependency); + bound to the Windows account. +* macOS / Linux: the OS keyring when the optional ``keyring`` package is + installed and a real backend is active. +* Otherwise a key file (``keys/master.key``, mode 0600 where supported), + which only protects against copying a single data file on its own, not + against someone who can read the user's profile. + +Data is sealed with AES-256-GCM. If the stored master key cannot be +recovered (profile moved to another account or machine) nothing is deleted or +overwritten: :class:`KeystoreUnavailable` is raised and callers degrade to +"no data" while leaving the encrypted files untouched. +""" + +from __future__ import annotations + +import base64 +import json +import os +import pathlib +import stat +import sys +from typing import Any, Dict, Optional, Protocol + +KEYSTORE_FORMAT = "NullifyPDF keystore" +_DPAPI_ENTROPY = b"NullifyPDF/secure_store/v1" + + +class KeystoreUnavailable(RuntimeError): + """The master key cannot be recovered on this account / machine.""" + + +class SealError(ValueError): + """Sealed data is corrupt, tampered with, or for another context.""" + + +# ----------------------------------------------------------------- sealing + +def seal(plaintext: bytes, key: bytes, aad: bytes) -> bytes: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + nonce = os.urandom(12) + return nonce + AESGCM(key).encrypt(nonce, plaintext, aad) + + +def open_sealed(blob: bytes, key: bytes, aad: bytes) -> bytes: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + if len(blob) < 12 + 16: + raise SealError("Dati cifrati troncati.") + try: + return AESGCM(key).decrypt(blob[:12], blob[12:], aad) + except InvalidTag as e: + raise SealError("Dati cifrati non validi o modificati.") from e + + +# -------------------------------------------------------------- protectors + +class KeyProtector(Protocol): + name: str + + def wrap(self, key: bytes) -> Dict[str, Any]: ... + + def unwrap(self, blob: Dict[str, Any]) -> bytes: ... + + +class DpapiProtector: + name = "dpapi" + + @staticmethod + def available() -> bool: + return sys.platform == "win32" + + def _call(self, data: bytes, protect: bool) -> bytes: + import ctypes + from ctypes import wintypes + + class DATA_BLOB(ctypes.Structure): + _fields_ = [("cbData", wintypes.DWORD), ("pbData", ctypes.POINTER(ctypes.c_char))] + + def blob_of(b: bytes) -> "tuple[DATA_BLOB, Any]": + buf = ctypes.create_string_buffer(b, len(b)) + return DATA_BLOB(len(b), ctypes.cast(buf, ctypes.POINTER(ctypes.c_char))), buf + + crypt32 = ctypes.windll.crypt32 # type: ignore[attr-defined] + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + kernel32.LocalFree.argtypes = [ctypes.c_void_p] + in_blob, keep1 = blob_of(data) + ent_blob, keep2 = blob_of(_DPAPI_ENTROPY) + out_blob = DATA_BLOB() + CRYPTPROTECT_UI_FORBIDDEN = 0x1 + if protect: + ok = crypt32.CryptProtectData( + ctypes.byref(in_blob), "NullifyPDF", ctypes.byref(ent_blob), + None, None, CRYPTPROTECT_UI_FORBIDDEN, ctypes.byref(out_blob), + ) + else: + ok = crypt32.CryptUnprotectData( + ctypes.byref(in_blob), None, ctypes.byref(ent_blob), + None, None, CRYPTPROTECT_UI_FORBIDDEN, ctypes.byref(out_blob), + ) + del keep1, keep2 + if not ok: + raise OSError(ctypes.WinError()) # type: ignore[attr-defined] + try: + return ctypes.string_at(out_blob.pbData, out_blob.cbData) + finally: + kernel32.LocalFree(out_blob.pbData) + + def wrap(self, key: bytes) -> Dict[str, Any]: + return { + "protector": self.name, + "blob": base64.b64encode(self._call(key, True)).decode("ascii"), + } + + def unwrap(self, blob: Dict[str, Any]) -> bytes: + return self._call(base64.b64decode(blob["blob"]), False) + + +PROBE_TIMEOUT = 3.0 # a healthy keyring answers in milliseconds +UNWRAP_TIMEOUT = 60.0 # may legitimately wait for the user to unlock it + + +def _with_timeout(fn: Any, seconds: float) -> Any: + """Run `fn` in a daemon thread; raise TimeoutError if it does not finish. + + A keyring backend can block forever when it needs a graphical + confirmation that nobody can give (a locked macOS Keychain on a headless + machine, no Secret Service session). The thread is abandoned: it is a + daemon, so it never keeps the program alive. + """ + import threading + + box: Dict[str, Any] = {} + + def run() -> None: + try: + box["value"] = fn() + except BaseException as e: # noqa: BLE001 - re-raised in the caller + box["error"] = e + + t = threading.Thread(target=run, daemon=True) + t.start() + t.join(seconds) + if t.is_alive(): + raise TimeoutError("keyring did not answer in time") + if "error" in box: + raise box["error"] + return box.get("value") + + +class KeyringProtector: + name = "keyring" + SERVICE = "NullifyPDF" + USER = "master" + _probe_result: "Dict[int, bool]" = {} + + @staticmethod + def available() -> bool: + """A real keyring backend that answers promptly. + + Set NULLIFYPDF_DISABLE_KEYRING=1 to never touch the system keyring + (used by the tests, so they never reach a real Keychain). + """ + if os.environ.get("NULLIFYPDF_DISABLE_KEYRING") == "1": + return False + try: + import keyring # type: ignore[import-not-found] + from keyring.backends import fail, null # type: ignore[import-not-found] + + backend = keyring.get_keyring() + if isinstance(backend, (fail.Keyring, null.Keyring)): + return False + key = id(backend) + if key in KeyringProtector._probe_result: + return KeyringProtector._probe_result[key] + + def probe() -> bool: + keyring.set_password("NullifyPDF-probe", "probe", "x") + ok = keyring.get_password("NullifyPDF-probe", "probe") == "x" + keyring.delete_password("NullifyPDF-probe", "probe") + return ok + + try: + ok = bool(_with_timeout(probe, PROBE_TIMEOUT)) + except Exception: + ok = False + KeyringProtector._probe_result[key] = ok + return ok + except Exception: + return False + + def wrap(self, key: bytes) -> Dict[str, Any]: + import keyring # type: ignore[import-not-found] + + secret = base64.b64encode(key).decode("ascii") + _with_timeout( + lambda: keyring.set_password(self.SERVICE, self.USER, secret), UNWRAP_TIMEOUT + ) + return {"protector": self.name, "service": self.SERVICE, "user": self.USER} + + def unwrap(self, blob: Dict[str, Any]) -> bytes: + import keyring # type: ignore[import-not-found] + + svc, usr = blob.get("service", self.SERVICE), blob.get("user", self.USER) + secret = _with_timeout(lambda: keyring.get_password(svc, usr), UNWRAP_TIMEOUT) + if not secret: + raise OSError("Segreto non presente nel portachiavi.") + return base64.b64decode(secret) + + +class KeyFileProtector: + name = "keyfile" + + def __init__(self, config_dir: pathlib.Path) -> None: + self.path = pathlib.Path(config_dir) / "keys" / "master.key" + + def wrap(self, key: bytes) -> Dict[str, Any]: + self.path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) + fd = os.open(str(self.path), flags, 0o600) + with os.fdopen(fd, "wb") as fh: + fh.write(key) + try: + os.chmod(self.path, stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + return {"protector": self.name, "path": "keys/master.key"} + + def unwrap(self, blob: Dict[str, Any]) -> bytes: + return self.path.read_bytes() + + +# ---------------------------------------------------------------- keystore + +def _keystore_path(config_dir: pathlib.Path) -> pathlib.Path: + return pathlib.Path(config_dir) / "keystore.json" + + +def _protectors(config_dir: pathlib.Path) -> Dict[str, Any]: + out: Dict[str, Any] = {"keyfile": KeyFileProtector(config_dir)} + if DpapiProtector.available(): + out["dpapi"] = DpapiProtector() + if KeyringProtector.available(): + out["keyring"] = KeyringProtector() + return out + + +_CACHE: Dict[str, bytes] = {} + + +def forget_cached_keys() -> None: + _CACHE.clear() + + +def protector_label(config_dir: pathlib.Path) -> str: + """Human-readable (Italian) description of how the master key is kept.""" + try: + data = json.loads(_keystore_path(config_dir).read_text(encoding="utf-8")) + name = data["master"]["protector"] + except Exception: + name = None + return { + "dpapi": "Archivio protetto: Windows DPAPI", + "keyring": "Archivio protetto: portachiavi di sistema", + "keyfile": "File chiave locale (protezione limitata)", + }.get(name or "", "Non ancora creato") + + +def load_or_create_master_key(config_dir: pathlib.Path) -> bytes: + """Return the 32-byte master key, creating the keystore on first use. + + Raises :class:`KeystoreUnavailable` when a keystore exists but cannot be + unwrapped on this account. Never overwrites an existing keystore. + """ + cfg = pathlib.Path(config_dir) + cache_key = str(cfg.resolve()) + if cache_key in _CACHE: + return _CACHE[cache_key] + ks = _keystore_path(cfg) + protectors = _protectors(cfg) + if ks.exists(): + try: + data = json.loads(ks.read_text(encoding="utf-8")) + master = data["master"] + prot = protectors.get(master["protector"]) + if prot is None: + raise KeystoreUnavailable( + f"Protezione '{master['protector']}' non disponibile su questo sistema." + ) + key = prot.unwrap(master) + except KeystoreUnavailable: + raise + except Exception as e: + raise KeystoreUnavailable( + "Chiave dei dati locali non leggibile su questo account/PC." + ) from e + if len(key) != 32: + raise KeystoreUnavailable("Chiave dei dati locali non valida.") + _CACHE[cache_key] = key + return key + key = os.urandom(32) + errors = [] + for name in ("dpapi", "keyring", "keyfile"): + prot = protectors.get(name) + if prot is None: + continue + try: + master = prot.wrap(key) + if prot.unwrap(master) != key: # round trip before trusting it + raise OSError("round trip mismatch") + except Exception as e: + errors.append(f"{name}: {e}") + continue + cfg.mkdir(parents=True, exist_ok=True) + _write_private(ks, json.dumps( + {"format": KEYSTORE_FORMAT, "version": 1, "master": master}, indent=2 + ).encode("utf-8")) + _CACHE[cache_key] = key + return key + raise KeystoreUnavailable("Nessuna protezione disponibile: " + "; ".join(errors)) + + +def _write_private(path: pathlib.Path, data: bytes) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) + fd = os.open(str(path), flags, 0o600) + with os.fdopen(fd, "wb") as fh: + fh.write(data) + try: + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + +def atomic_write(path: pathlib.Path, data: bytes) -> None: + """Write via a temp file in the same folder, then replace (atomic).""" + tmp = path.with_name(path.name + ".part") + _write_private(tmp, data) + os.replace(tmp, path) + + +def secure_delete(path: pathlib.Path) -> None: + """Best effort: overwrite with zeros, truncate, unlink. + + SSD wear levelling, backups and file-system journals may keep copies. + """ + try: + size = path.stat().st_size + with open(path, "r+b") as fh: + fh.write(b"\x00" * size) + fh.flush() + os.fsync(fh.fileno()) + fh.truncate(0) + except OSError: + pass + try: + path.unlink() + except OSError: + pass diff --git a/setup_env.py b/setup_env.py index 377acd6..1643233 100644 --- a/setup_env.py +++ b/setup_env.py @@ -113,9 +113,8 @@ def setup_environment() -> None: logger.error(f"Errore durante l'installazione dipendenze: {e}") sys.exit(1) - logger.info("Download modelli linguistici spaCy (EN & IT)") + logger.info("Download modello linguistico spaCy (IT)") spacy_models = [ - "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl", "https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.8.0/it_core_news_md-3.8.0-py3-none-any.whl" ] try: diff --git a/spacy_models.py b/spacy_models.py new file mode 100644 index 0000000..676eb42 --- /dev/null +++ b/spacy_models.py @@ -0,0 +1,193 @@ +"""Optional spaCy pipelines that are downloaded on demand instead of bundled. + +The Italian pipeline ``it_core_news_md`` is published under CC BY-NC-SA 3.0 +(non-commercial, share-alike). NullifyPDF is AGPL-3.0, whose recipients may use +the program commercially, so that pipeline cannot be part of the installer. +It is downloaded only after the user has seen the licence and agreed, from the +publisher's release page, verified against a pinned size and SHA-256 and kept +in the user's configuration folder. Detection quality is the same as when it +was bundled. + +When the user declines, a blank Italian pipeline (MIT, part of spaCy itself) is +used so the rule-based detectors keep working; only the statistical name +recogniser is missing and the caller must say so. +""" + +from __future__ import annotations + +import hashlib +import os +import pathlib +import shutil +import tempfile +import urllib.request +import zipfile +from dataclasses import dataclass +from typing import Callable, Optional + +ITALIAN_PACKAGE = "it_core_news_md" +ITALIAN_VERSION = "3.8.0" +ITALIAN_URL = ( + "https://github.com/explosion/spacy-models/releases/download/" + f"{ITALIAN_PACKAGE}-{ITALIAN_VERSION}/{ITALIAN_PACKAGE}-{ITALIAN_VERSION}-py3-none-any.whl" +) +ITALIAN_SHA256 = "a731d2d8e7c5a7093ac42f88774ea2a2ad3d5809959eedee65632b3533f804ec" +ITALIAN_SIZE = 42389964 +ITALIAN_LICENSE = "CC BY-NC-SA 3.0" +ITALIAN_LICENSE_URL = "https://creativecommons.org/licenses/by-nc-sa/3.0/" +_MAX_UNPACKED = 120 * 1024 * 1024 +_PIPELINE_DIR = f"{ITALIAN_PACKAGE}/{ITALIAN_PACKAGE}-{ITALIAN_VERSION}/" + +ProgressFn = Callable[[int, int], None] + + +class ModelDownloadError(RuntimeError): + """The pipeline could not be downloaded or failed verification.""" + + +@dataclass(frozen=True) +class ResolvedModel: + name: str # what Presidio/spaCy should load: package name or folder path + statistical: bool # False for the blank fallback (no name recogniser) + + +def models_root(config_dir: pathlib.Path) -> pathlib.Path: + return pathlib.Path(config_dir) / "spacy_models" + + +def _installed_dir(root: pathlib.Path) -> pathlib.Path: + return root / f"{ITALIAN_PACKAGE}-{ITALIAN_VERSION}" + + +def italian_installed(root: pathlib.Path) -> bool: + d = _installed_dir(root) + return (d / "meta.json").is_file() and (d / "ner").is_dir() + + +def _is_package() -> bool: + import spacy + + return bool(spacy.util.is_package(ITALIAN_PACKAGE)) + + +def resolve_italian(root: pathlib.Path) -> Optional[ResolvedModel]: + """The Italian statistical pipeline if it is available locally, else None.""" + if _is_package(): # development checkouts and tests + return ResolvedModel(ITALIAN_PACKAGE, True) + if italian_installed(root): + return ResolvedModel(str(_installed_dir(root)), True) + return None + + +def blank_italian(root: pathlib.Path) -> ResolvedModel: + """Rules-only fallback: tokenizer and sentence splitter, no NER.""" + import spacy + + path = root / "blank_it" + if not (path / "meta.json").is_file(): + root.mkdir(parents=True, exist_ok=True) + nlp = spacy.blank("it") + nlp.add_pipe("sentencizer") + nlp.to_disk(path) + return ResolvedModel(str(path), False) + + +def _extract(wheel: pathlib.Path, target: pathlib.Path) -> None: + """Unpack only the pipeline folder; reject any path that escapes.""" + total = 0 + base = target.resolve() + with zipfile.ZipFile(wheel) as zf: + for info in zf.infolist(): + if not info.filename.startswith(_PIPELINE_DIR) or info.is_dir(): + continue + rel = pathlib.PurePosixPath(info.filename[len(_PIPELINE_DIR):]) + if rel.is_absolute() or ".." in rel.parts: + raise ModelDownloadError("percorso non sicuro nel pacchetto") + total += info.file_size + if total > _MAX_UNPACKED: + raise ModelDownloadError("pacchetto troppo grande") + dest = (base / pathlib.Path(*rel.parts)).resolve() + if base not in dest.parents: + raise ModelDownloadError("percorso non sicuro nel pacchetto") + dest.parent.mkdir(parents=True, exist_ok=True) + with zf.open(info) as src, open(dest, "wb") as out: + shutil.copyfileobj(src, out) + + +def install_italian( + root: pathlib.Path, + progress: Optional[ProgressFn] = None, + url: str = ITALIAN_URL, + sha256: str = ITALIAN_SHA256, + size: int = ITALIAN_SIZE, + timeout: float = 30.0, +) -> pathlib.Path: + """Download, verify and unpack the pipeline. Returns its folder.""" + if not url.lower().startswith("https://"): + raise ModelDownloadError("sono consentiti solo download HTTPS") + root.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(suffix=".whl", dir=root) + os.close(fd) + tmp = pathlib.Path(tmp_name) + staging = pathlib.Path(tempfile.mkdtemp(prefix="unpack-", dir=root)) + try: + digest = hashlib.sha256() + done = 0 + try: + with urllib.request.urlopen(url, timeout=timeout) as resp, tmp.open("wb") as out: + while True: + chunk = resp.read(1024 * 256) + if not chunk: + break + done += len(chunk) + if done > size: + raise ModelDownloadError("il file scaricato รจ piรน grande del previsto") + digest.update(chunk) + out.write(chunk) + if progress: + progress(done, size) + except OSError as exc: + raise ModelDownloadError(f"download non riuscito: {exc}") from exc + if done != size or digest.hexdigest() != sha256: + raise ModelDownloadError("verifica di integritร  fallita (dimensione o SHA-256)") + _extract(tmp, staging) + if not ((staging / "meta.json").is_file() and (staging / "ner").is_dir()): + raise ModelDownloadError("pacchetto incompleto") + final = _installed_dir(root) + if final.exists(): + shutil.rmtree(final) + os.replace(staging, final) + return final + finally: + tmp.unlink(missing_ok=True) + shutil.rmtree(staging, ignore_errors=True) + + +def remove_italian(root: pathlib.Path) -> None: + shutil.rmtree(_installed_dir(root), ignore_errors=True) + + +def licence_text() -> str: + return ( + "Il riconoscimento statistico dei nomi in italiano usa la pipeline spaCy " + f"ยซ{ITALIAN_PACKAGE}ยป {ITALIAN_VERSION} (Explosion), distribuita con licenza " + f"{ITALIAN_LICENSE} ({ITALIAN_LICENSE_URL}): uso non commerciale, con " + "attribuzione e condivisione alle stesse condizioni. Non รจ inclusa " + "nell'installer perchรฉ NullifyPDF รจ AGPL-3.0 e consente l'uso commerciale.\n\n" + f"Verrร  scaricata da GitHub ({ITALIAN_SIZE // (1024 * 1024)} MB), " + "verificata con SHA-256 e salvata solo sul tuo computer. " + "Accettando dichiari di usarla nel rispetto di quella licenza." + ) + + +__all__ = [ + "ModelDownloadError", + "ResolvedModel", + "blank_italian", + "install_italian", + "italian_installed", + "licence_text", + "models_root", + "remove_italian", + "resolve_italian", +] diff --git a/tests/_pdfgen.py b/tests/_pdfgen.py new file mode 100644 index 0000000..180fc6d --- /dev/null +++ b/tests/_pdfgen.py @@ -0,0 +1,38 @@ +"""Synthetic PDF builders shared by the tests (no binary fixtures).""" + +from types import SimpleNamespace + +import pymupdf + + +def make_doc(lines, pitch=14.0, size=11.0, x=72.0, y0=100.0, fontname="helv"): + doc = pymupdf.open() + page = doc.new_page() + for i, text in enumerate(lines): + page.insert_text((x, y0 + i * pitch), text, fontsize=size, fontname=fontname) + return doc + + +def words_of(page, **kw): + return page.get_text("words", **kw) + + +def res(entity_type, text, value, score=0.85): + """Presidio-like result for the first occurrence of `value` in `text`.""" + start = text.index(value) + return SimpleNamespace( + entity_type=entity_type, start=start, end=start + len(value), score=score + ) + + +def fake_words(text): + """`get_text("words")`-style tuples for plain text (one block).""" + out = [] + y = 0.0 + for li, line in enumerate(text.split("\n")): + x = 10.0 + for wi, w in enumerate(line.split()): + out.append((x, y, x + 6 * len(w), y + 12, w, 0, li, wi)) + x += 6 * len(w) + 4 + y += 14 + return out diff --git a/tests/conftest.py b/tests/conftest.py index 3e809b8..4ed5249 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,38 @@ """Pytest configuration shared by the test suite.""" +import os import pathlib import sys +# Tests must never touch a real system keyring (a locked macOS Keychain on a +# headless runner blocks forever waiting for a dialog). +os.environ.setdefault("NULLIFYPDF_DISABLE_KEYRING", "1") + PROJECT_ROOT = pathlib.Path(__file__).resolve().parent.parent root_str = str(PROJECT_ROOT) if root_str not in sys.path: sys.path.insert(0, root_str) + + +def pytest_sessionfinish(session, exitstatus): + """Destroy Qt objects while QApplication is still alive. + + Dialogs and windows left in reference cycles would otherwise be collected + at interpreter shutdown, after the application object, which crashes the + process on some platforms even though every test passed. + """ + try: + import gc + + from PySide6.QtWidgets import QApplication + + app = QApplication.instance() + if app is not None: + app.closeAllWindows() + app.processEvents() + gc.collect() + if app is not None: + app.processEvents() + except Exception: + pass diff --git a/tests/test_build_config.py b/tests/test_build_config.py index ea94455..20051d4 100644 --- a/tests/test_build_config.py +++ b/tests/test_build_config.py @@ -77,3 +77,57 @@ def test_tldextract_data_is_bundled_by_the_build_recipes(): datas = collect_all("tldextract")[0] assert any(src.endswith(".tld_set_snapshot") for src, _dest in datas) + + +# --- Linux packaging / CI policy --------------------------------------------- + +_ROOT = pathlib.Path(__file__).resolve().parent.parent +_WORKFLOWS = _ROOT / ".github" / "workflows" + + +def test_prerelease_packages_sort_before_the_stable_one(): + from build_local import deb_version, rpm_release + + assert deb_version("2.4.0", "beta.1") == "2.4.0~beta.1" + assert deb_version("2.4.0", "") == "2.4.0" + assert rpm_release("beta.1") == "0.beta.1.1" + assert rpm_release("") == "1" + + +def test_deb_and_rpm_declare_runtime_libraries(): + from build_local import DEB_DEPENDS, RPM_REQUIRES + + # Ubuntu 24.04 renamed libglib2.0-0 to libglib2.0-0t64: accept both. + assert "libglib2.0-0t64 | libglib2.0-0" in DEB_DEPENDS + assert "libxcb-cursor0" in DEB_DEPENDS + assert "xcb-util-cursor" in RPM_REQUIRES + + +def test_linux_packages_are_built_once_in_a_rhel9_container(): + text = (_WORKFLOWS / "build-linux.yml").read_text(encoding="utf-8") + assert "container: rockylinux:9" in text + for image in ("ubuntu:24.04", "ubuntu:26.04", "debian:13", + "rockylinux:9", "rockylinux/rockylinux:10", "fedora:latest"): + assert image in text + + +@pytest.mark.parametrize("name", ["release.yml", "beta-release.yml", "test_build.yml"]) +def test_build_workflows_use_the_shared_linux_build(name): + text = (_WORKFLOWS / name).read_text(encoding="utf-8") + assert "uses: ./.github/workflows/build-linux.yml" in text + # Linux must not be built on the old ubuntu-22.04 runner image. + assert "ubuntu-22.04" not in text + + +def test_mypy_module_lists_are_identical_across_workflows(): + import pathlib + import re + + root = pathlib.Path(__file__).resolve().parent.parent / ".github" / "workflows" + sets = {} + for name in ("beta-release.yml", "build-linux.yml", "release.yml", "test_build.yml"): + text = (root / name).read_text(encoding="utf-8") + m = re.search(r"run: mypy ([^\n]+)", text) + assert m, name + sets[name] = frozenset(m.group(1).split()) + assert len(set(sets.values())) == 1, sets diff --git a/tests/test_detection_pipeline.py b/tests/test_detection_pipeline.py new file mode 100644 index 0000000..9a8dac9 --- /dev/null +++ b/tests/test_detection_pipeline.py @@ -0,0 +1,467 @@ +"""Italian precision, pattern validation and allowlist semantics.""" + +import pytest + +import pii_detection as pd +from _pdfgen import fake_words, res +import lexicon_it + + +def _run(text, results, enabled=None, **kw): + idx = pd.build_page_index(fake_words(text), clamp=False) + enabled = enabled or pd.expand_group_ids(pd.ALL_GROUP_IDS) + cands, stats = pd.classify_page(results, idx, enabled, **kw) + return idx, cands, stats + + +# --- lexicon ----------------------------------------------------------------- + +@pytest.mark.parametrize("word", [ + "Certifico", "Prescrivo", "Dichiaro", "Attesto", "Rilascio", "Confermo", + "Raccomando", "Ordina", "Anamnesi", "Diagnosi", "Terapia", "Paziente", + "PRESCRIVO", "Comune", "Tribunale", +]) +def test_ordinary_words_are_recognised(word): + assert lexicon_it.is_common_word(word) + + +@pytest.mark.parametrize("word", ["Rossi", "Esposito", "Mario", "Napoli", "Rosa", "Bruno"]) +def test_names_are_not_common_words(word): + assert not lexicon_it.is_common_word(word) + + +def test_verb_forms_are_generated_from_stems(): + forms = lexicon_it.verb_forms() + assert {"certifica", "certificano", "prescrive", "prescrivono", "dichiara"} <= forms + + +# --- fake-analyzer pipeline (no model needed) ----------------------------------- + +def test_lone_verb_tagged_as_location_is_dropped(): + text = "Certifico\nPrescrivo\nRaccomando" + results = [res("LOCATION", text, w) for w in ("Certifico", "Prescrivo", "Raccomando")] + _idx, cands, stats = _run(text, results) + assert cands == [] and stats.get("lexicon", 0) == 3 + + +def test_verb_plus_drug_keeps_only_the_non_verb(): + text = "Visita del 3 aprile Prescrivo Augmentin" + r = res("LOCATION", text, "Prescrivo Augmentin") + _idx, cands, _ = _run(text, [r]) + assert all("Prescrivo" not in c.text for c in cands) + + +def test_name_with_context_is_accepted_and_scored_from_evidence(): + text = "Il Sig. Mario Rossi vive a Roma" + results = [res("PERSON", text, "Mario Rossi"), res("LOCATION", text, "Roma")] + _idx, cands, _ = _run(text, results) + by_type = {c.entity_type: c for c in cands} + assert by_type["PERSON"].state == "accepted" and by_type["PERSON"].score >= 0.7 + assert by_type["LOCATION"].state == "accepted" + + +def test_ambiguous_single_word_becomes_a_suggestion_not_a_drop(): + text = "Il referto indica che Colombi era presente" + r = res("PERSON", text, "Colombi") + _idx, cands, _ = _run(text, [r]) + assert len(cands) == 1 and cands[0].state == "suggested" + + +def test_health_mode_never_drops_silently(): + text = "Certifico" + r = res("LOCATION", text, "Certifico") + # Lexicon edges are a hard rule; use an uncertain non-lexicon word instead. + text2 = "Noi Zorzi andiamo" + r2 = res("PERSON", text2, "Zorzi") + _i, normal, _ = _run(text2, [r2]) + _i, health, _ = _run(text2, [r2], health_mode=True) + assert len(health) >= len(normal) and all(c.state != "dropped" for c in health) + + +def test_split_lines_of_one_name_are_merged(): + text = "Esposito\nGiovanni abita a Napoli" + results = [res("PERSON", text, "Esposito"), res("PERSON", text, "Giovanni")] + _idx, cands, _ = _run(text, results) + person = [c for c in cands if c.entity_type == "PERSON"] + assert len(person) == 1 and person[0].text.replace("\n", " ") == "Esposito Giovanni" + assert person[0].state == "accepted" + + +def test_field_label_is_rejected_outright(): + text = "Nome: Mario" + r = res("PERSON", text, "Nome") + _idx, cands, _ = _run(text, [r]) + assert cands == [] + + +def test_fragments_and_ocr_noise_are_rejected(): + text = "Rnario Ro" + _i, normal, _ = _run(text, [res("PERSON", text, "Rnario")], used_ocr=True) + assert normal == [] + kept, n = pd.reject_fragment_hits( + [pd.Candidate(0, 4, "Li", "PERSON", 0.85)], used_ocr=False + ) + assert kept == [] and n == 1 + + +# --- pattern validation -------------------------------------------------------- + +def _pattern(text, etype, value, score=1.0): + start = text.index(value) + c = pd.Candidate(start, start + len(value), value, etype, score) + return pd.refine_pattern_hit(c, text) + + +def test_fiscal_code_needs_boundaries_digits_or_context(): + ok, _ = _pattern("Codice fiscale: RSSMRA80A01H501U fine", "IT_FISCAL_CODE", "RSSMRA80A01H501U") + assert ok + ok, why = _pattern("xxRSSMRA80A01H501Uxx", "IT_FISCAL_CODE", "RSSMRA80A01H501U") + assert not ok and why == "embedded-in-token" + ok, why = _pattern("parola AAAAAAAAAAAAAAAA fine", "IT_FISCAL_CODE", "AAAAAAAAAAAAAAAA") + assert not ok and why == "no-digit-no-context" + + +def test_vat_spaced_digits_need_context(): + ok, why = _pattern("totale 1234 5678 901 euro", "IT_VAT_CODE", "1234 5678 901", 0.1) + assert not ok + ok, _ = _pattern("P.IVA 12345678903 grazie", "IT_VAT_CODE", "12345678903", 1.0) + assert ok + ok, why = _pattern("numero 1234567890312 lungo", "IT_VAT_CODE", "12345678903", 1.0) + assert not ok and why == "embedded-in-digits" + + +@pytest.mark.parametrize("etype,value,ctx_ok,ctx_bad", [ + ("IT_DRIVER_LICENSE", "AB1234567C", "patente di guida n. AB1234567C ok", "codice prodotto AB1234567C ok"), + ("IT_PASSPORT", "YA1234567", "passaporto YA1234567 ok", "ordine YA1234567 ok"), + ("IT_IDENTITY_CARD", "CA12345AB", "carta d'identita CA12345AB ok", "ticket CA12345AB ok"), +]) +def test_document_numbers_need_their_context(etype, value, ctx_ok, ctx_bad): + assert _pattern(ctx_ok, etype, value, 0.4)[0] + assert not _pattern(ctx_bad, etype, value, 0.4)[0] + assert not _pattern(ctx_ok.replace(value, value.lower()), etype, value.lower(), 0.4)[0] + + +@pytest.mark.parametrize("text,value,expect", [ + ("Tel. 340 1234567 fine", "340 1234567", True), + ("chiama +39 02 1234567", "+39 02 1234567", True), + ("Fattura del 12/03/2024 n.", "12/03/2024", False), + ("anni 2018 - 2022 presso", "2018 - 2022", False), + ("ordine 5551234 ok", "5551234", False), + ("codice A340123456789", "340123456789", False), +]) +def test_phone_plausibility(text, value, expect): + assert _pattern(text, "PHONE_NUMBER", value, 0.4)[0] is expect + + +def _health(text, kinds=("DATE_OF_BIRTH", "HEALTH_ID", "HEALTH_EXEMPTION"), health_mode=False): + return [(c.entity_type, c.text, c.state) for c in + pd.detect_health_patterns(text, kinds, health_mode)] + + +def test_date_of_birth_needs_its_anchor(): + assert ("DATE_OF_BIRTH", "12/03/1970", "accepted") in _health("nato a Napoli il 12/03/1970") + assert ("DATE_OF_BIRTH", "3 marzo 1985", "accepted") in _health("nata il 3 marzo 1985 a Roma") + assert _health("Fattura del 12/03/2024") == [] + assert pd.detect_health_patterns("nato il 12/03/1970", []) == [] + + +def test_tessera_requires_valid_region_and_exact_length(): + assert ("HEALTH_ID", "80380000501234567890", "accepted") in _health("Tessera 80380000501234567890 ok") + assert any(t == "HEALTH_ID" for t, _, _ in _health("TS 80380 00050 123456789 0 ok")) + assert _health("n. 80380009991234567890") == [] # invalid region code + assert _health("n. 8038000050123456789") == [] # 19 digits + assert _health("n. 803800005012345678901") == [] # 21 digits + + +def test_tessera_and_nre_are_both_reported(): + text = "Tessera 80380000501234567890 Ricetta *050A01234567890*" + types = [(t, v) for t, v, _ in _health(text)] + assert ("HEALTH_ID", "80380000501234567890") in types + assert any(t == "HEALTH_ID" and v.strip("*").startswith("050A0") for t, v in types) + + +def test_nre_needs_context_unless_health_mode_where_it_is_a_suggestion(): + bare = "codice 050A01234567890 fine" + assert _health(bare) == [] + assert ("HEALTH_ID", "050A01234567890", "suggested") in _health(bare, health_mode=True) + assert _health("fattura 123456789012345") == [] + assert _health("Numero ricetta: 050A01234567890")[0][2] == "accepted" + + +def test_exemption_codes_need_context(): + assert ("HEALTH_EXEMPTION", "E01", "accepted") in _health("Esenzione: E01") + assert _health("modello E01") == [] + assert ("HEALTH_EXEMPTION", "RA0010", "accepted") in _health("cod. es. RA0010 rara") + assert _health("sezione C03 del regolamento") == [] + + +def _addr(text, **kw): + cands = pd.detect_address_parts(text, ["LOCATION"], **kw) + return [(c.text, c.state) for c in cands] + + +def test_civic_number_joins_street_and_postcode_is_found(): + found = _addr("Via Garibaldi 12/B, 80100 Napoli (NA)") + assert ("Via Garibaldi 12/B", "accepted") in found and ("80100", "accepted") in found + assert ("Via Roma 10", "accepted") in _addr("abita in Via Roma 10 con la famiglia") + + +def test_postcode_not_matched_in_prices_dates_quantities(): + for text in ("Totale 12345 euro", "importo โ‚ฌ 10000,00", "prot. 45678 del", + "ordinati 12345 Pezzi", "il 01/02/2024"): + assert _addr(text) == [], text + + +def test_postcode_requires_an_anchor(): + assert ("20121", "accepted") in _addr("codice 20121 Milano") + assert _addr("lotto 20121 Alfa") == [] + assert ("20121", "suggested") in _addr("lotto 20121 Alfa", health_mode=True) + assert pd.detect_address_parts("Via Roma 10", ["PERSON"]) == [] + + +def test_address_candidates_survive_the_pipeline_unscored(): + text = "Residente in Via Garibaldi 12 80100 Napoli" + idx = pd.build_page_index(fake_words(text), clamp=False) + cands, _ = pd.classify_page([], idx, ["LOCATION"]) + assert any(c.text.startswith("Via Garibaldi 12") and c.state == "accepted" for c in cands) + + +# --- allowlist semantics ---------------------------------------------------------- + +def _detections(text, etype, value, allow): + idx = pd.build_page_index(fake_words(text), clamp=False) + start = text.index(value) + c = pd.Candidate(start, start + len(value), value, etype, 0.9) + return pd.build_detections(idx, [c], allowlist=allow) + + +def test_allowlist_entry_does_not_suppress_a_shorter_value(): + # "rossini" allowlisted must not hide "Rossi" (substring bug). + out = _detections("Dott. Rossi lavora qui", "PERSON", "Rossi", ["rossini"]) + assert len(out) == 1 + + +def test_allowlisted_word_only_drops_that_word_from_a_longer_value(): + out = _detections("Mario Rossi vive qui", "PERSON", "Mario Rossi", ["mario"]) + assert len(out) == 1 + assert len(out[0]["rects"]) == 1 + # only "Rossi" remains covered + idx = pd.build_page_index(fake_words("Mario Rossi vive qui"), clamp=False) + assert out[0]["rects"][0][0] == pytest.approx(idx.words[1].rect[0]) + + +def test_fully_allowlisted_value_disappears(): + assert _detections("Mario Rossi vive qui", "PERSON", "Mario Rossi", ["mario rossi"]) == [] + assert _detections("Il Comune di Roma", "LOCATION", "Roma", ["roma"]) == [] + + +def test_allowlist_word_on_adjacent_line_never_cancels_pii(): + text = "Mario Rossi\nroma" + out = _detections(text, "PERSON", "Mario Rossi", ["roma"]) + assert len(out) == 1 + + +def test_propagation_keeps_whole_words_only_and_respects_allowlist(): + text = "Rossi e Rossini e Rossi" + idx = pd.build_page_index(fake_words(text), clamp=False) + c = pd.Candidate(0, 5, "Rossi", "PERSON", 0.9) + out = pd.build_detections(idx, [c]) + assert len(out[0]["rects"]) == 2 + + +def test_detection_carries_state_reasons_and_glued_flag(): + idx = pd.build_page_index(fake_words("CF:RSSMRA80A01H501U fine"), clamp=False) + start = 3 + c = pd.Candidate(start, start + 16, "RSSMRA80A01H501U", "IT_FISCAL_CODE", 1.0) + d = pd.build_detections(idx, [c])[0] + assert d["state"] == "accepted" and d["rects"] and "reasons" in d + + +# --- real model (skipped when unavailable) --------------------------------------- + +CASES = { + "T1": ("Certifico\nche il Sig. Mario Rossi, nato a Napoli il 12/03/1970,", + {"Mario Rossi", "Napoli", "12/03/1970"}, {"Certifico"}), + "T2": ("Prescrivo\nTachipirina 1000 mg, 1 cpr ogni 8 ore", set(), {"Prescrivo"}), + "T3": ("Certifico\nPrescrivo\nRaccomando", set(), {"Certifico", "Prescrivo", "Raccomando"}), + "T4": ("Visita del 3/4. Prescrivo Augmentin.", set(), {"Prescrivo"}), + "T5": ("CERTIFICO CHE IL PAZIENTE E' IDONEO. PRESCRIVO ESAMI EMATOCHIMICI.", + set(), {"CERTIFICO", "PRESCRIVO", "PAZIENTE", "IDONEO"}), + "T6": ("Il sottoscritto Avv. Paolo Neri dichiara\nche la sig.ra Gialli Anna e' residente a Roma.", + {"Paolo Neri", "Gialli Anna", "Roma"}, {"dichiara", "Dichiara"}), + "T7": ("Visto l'art. 7 del Regolamento, Ordina la comparizione delle parti.", + set(), {"Ordina", "Visto"}), + "T8": ("Esposito\nGiovanni, nato a Napoli.", {"Esposito Giovanni", "Napoli"}, set()), + "T9": ("Rosa Bianco", {"Rosa Bianco"}, set()), + "T11": ("Milano, 3 marzo 2026", {"Milano"}, set()), + "T12": ("Dichiaro di aver ricevuto l'informativa.", set(), {"Dichiaro"}), + "T13": ("Firmato: Dott.ssa Lucia Verdi", {"Lucia Verdi"}, {"Firmato"}), + "T14": ("Anamnesi\nEsame obiettivo\nTerapia\nPrognosi", set(), + {"Anamnesi", "Esame", "Terapia", "Prognosi"}), +} + + +@pytest.fixture(scope="module") +def italian_analyzer(): + pytest.importorskip("presidio_analyzer") + spacy = pytest.importorskip("spacy") + if not spacy.util.is_package("it_core_news_md"): + pytest.skip("model missing") + from presidio_analyzer import AnalyzerEngine, RecognizerRegistry + from presidio_analyzer.nlp_engine import NlpEngineProvider + + nlp = NlpEngineProvider(nlp_configuration={"nlp_engine_name": "spacy", "models": [ + {"lang_code": "it", "model_name": "it_core_news_md"}]}).create_engine() + reg = RecognizerRegistry(supported_languages=["it"]) + reg.load_predefined_recognizers(languages=["it"], nlp_engine=nlp) + return AnalyzerEngine(nlp_engine=nlp, registry=reg, supported_languages=["it"]) + + +@pytest.mark.parametrize("case", sorted(CASES)) +def test_italian_documents_with_real_model(italian_analyzer, case): + text, must, never = CASES[case] + idx = pd.build_page_index(fake_words(text), clamp=False) + enabled = pd.expand_group_ids(pd.ALL_GROUP_IDS) + presidio = [t for t in enabled if t not in pd.NON_PRESIDIO_TYPES] + results = italian_analyzer.analyze(text=idx.text, entities=presidio, language="it") + cands, _ = pd.classify_page(results, idx, enabled) + accepted = {" ".join(c.text.split()) for c in cands if c.state == "accepted"} + flagged = {w for c in cands for w in c.text.split()} + assert must <= accepted, (case, accepted) + assert not (never & flagged), (case, flagged) + + +def test_precision_on_ordinary_clinical_text_with_real_model(italian_analyzer): + text = ( + "Certifico che il paziente e' affetto da lombalgia.\n" + "Prescrivo riposo assoluto e visita di controllo.\n" + "Raccomando terapia e dieta.\nDichiaro quanto sopra." + ) + idx = pd.build_page_index(fake_words(text), clamp=False) + enabled = pd.expand_group_ids(pd.ALL_GROUP_IDS) + results = italian_analyzer.analyze( + text=idx.text, + entities=[t for t in enabled if t not in pd.NON_PRESIDIO_TYPES], + language="it", + ) + cands, _ = pd.classify_page(results, idx, enabled) + assert [c for c in cands if c.state == "accepted"] == [] + + +# --- signature / stamp suggestions ---------------------------------------------- + +def _signature_doc(extra_line=False): + import pymupdf as _pm + + doc = _pm.open() + page = doc.new_page() + page.insert_text((72, 100), "Il medico", fontsize=11) + page.insert_text((72, 114), "Firma:", fontsize=11) + sh = page.new_shape() + for k in range(6): # a curve-rich squiggle + x = 120 + k * 12 + sh.draw_bezier(_pm.Point(x, 130), _pm.Point(x + 3, 105), _pm.Point(x + 7, 150), _pm.Point(x + 12, 128)) + sh.finish(color=(0, 0, 0), fill=None, closePath=False) + sh.commit() + if extra_line: + page.draw_line(_pm.Point(72, 160), _pm.Point(400, 160)) # underline + return doc + + +def test_signature_curves_near_firma_are_suggested_only(): + from pdf_sanitize import suggest_signature_areas + + doc = _signature_doc(extra_line=True) + sugg = suggest_signature_areas(doc[0]) + assert len(sugg) == 1 + s = sugg[0] + assert s["state"] == "suggested" and s["entity_type"] == "SIGNATURE" and not s["text"] + x0, y0, x1, y1 = s["rects"][0] + assert x0 < 112 and x1 > 195 and y0 < 100 and y1 > 155 + + +def test_table_lines_and_unanchored_ink_are_not_signatures(): + import pymupdf as _pm + from pdf_sanitize import suggest_signature_areas + + doc = _pm.open() + page = doc.new_page() + page.insert_text((72, 100), "Firma:", fontsize=11) + for k in range(8): + page.draw_line(_pm.Point(72, 120 + k * 10), _pm.Point(472, 120 + k * 10)) + assert suggest_signature_areas(page) == [] + other = _pm.open() + pg = other.new_page() + pg.insert_text((72, 100), "Relazione tecnica", fontsize=11) + sh = pg.new_shape() + for k in range(6): + x = 120 + k * 12 + sh.draw_bezier(_pm.Point(x, 130), _pm.Point(x + 3, 105), _pm.Point(x + 7, 150), _pm.Point(x + 12, 128)) + sh.finish(color=(0, 0, 0), fill=None, closePath=False) + sh.commit() + assert suggest_signature_areas(pg) == [] + + +# --- ISTAT place data ------------------------------------------------------------ + +def test_generated_module_is_lazy_and_carries_sources_and_licence(): + import subprocess + import sys + + code = ( + "import sys; sys.path.insert(0, %r); import lexicon_it; " + "assert 'lexicon_data_it' not in sys.modules; " + "assert lexicon_it.is_known_place('Zungri'); " + "assert 'lexicon_data_it' in sys.modules" + ) + root = __import__("pathlib").Path(__file__).resolve().parent.parent + code = code % str(root) + r = subprocess.run([sys.executable, "-I", "-c", code], cwd=str(root), + capture_output=True, text=True) + assert r.returncode == 0, r.stderr + head = (root / "lexicon_data_it.py").read_text(encoding="utf-8")[:600] + assert "CC BY 4.0" in head and "istat_comuni" in head + assert "CC BY 4.0" in (root / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8") + + +def test_istat_place_confirms_but_absence_never_rejects(): + assert lexicon_it.is_known_place("Zungri") and lexicon_it.is_known_place("Reggio nell'Emilia") + text = "Il Sig. Mario Rossi abita a Zzyzxville" + r = res("LOCATION", text, "Zzyzxville") + _i, cands, _ = _run(text, [r]) + assert cands == [] or all(c.state != "dropped" for c in cands) # scored as before, not rejected for absence + + +def test_ambiguous_comune_gets_only_a_small_boost(): + assert lexicon_it.is_ambiguous_place("Sale") and not lexicon_it.is_known_place("Sale") + text = "Sale" + r = res("LOCATION", text, "Sale") + _i, cands, _ = _run(text, [r]) + assert all(c.state != "accepted" for c in cands) + + +def test_province_codes_come_from_istat(): + assert {"MI", "NA", "SU", "BT"} <= lexicon_it.province_codes() + + +def test_known_surname_helps_a_two_word_name_but_never_rejects(): + assert lexicon_it.is_known_surname("Marchetti") + text = "Il referto di Gianpiero Marchetti" + _i, cands, _ = _run(text, [res("PERSON", text, "Gianpiero Marchetti")]) + assert cands and "known-surname" in cands[0].reasons + text2 = "Il referto di Gianpiero Qwertyuiop" + _i, cands2, _ = _run(text2, [res("PERSON", text2, "Gianpiero Qwertyuiop")]) + assert cands2 and cands2[0].state in ("accepted", "suggested") # absence is not a reason to reject + + +def test_wikidata_names_confirm_but_ambiguous_ones_do_not(): + import lexicon_data_it as d + + unambiguous = sorted(d.NAMES_SURNAMES - d.NAMES_AMBIGUOUS) + assert unambiguous and lexicon_it.is_known_surname(unambiguous[0]) + ambiguous = sorted(d.NAMES_AMBIGUOUS & d.NAMES_SURNAMES) + assert ambiguous and not lexicon_it.is_known_surname(ambiguous[0]) or ambiguous[0] in lexicon_it.SURNAMES + assert not lexicon_it.is_known_surname("zzqxv") + assert "Wikidata" in open( + __import__("pathlib").Path(__file__).resolve().parent.parent / "THIRD_PARTY_NOTICES.md", + encoding="utf-8").read() diff --git a/tests/test_export_sanitize.py b/tests/test_export_sanitize.py new file mode 100644 index 0000000..1130119 --- /dev/null +++ b/tests/test_export_sanitize.py @@ -0,0 +1,475 @@ +"""Export pipeline: residue removal, verification, temp files, audit, names.""" + +import json +import os + +import pymupdf +import pytest + +import pii_detection as pd +from export_engine import ( + build_audit_record, + cleanup_stale_temp, + collect_removed_values, + export_document, + name_leaks_value, + neutral_map_name, + neutral_output_name, + private_temp_dir, + purge_old_logs, + wipe_local_data, + write_audit_record, + write_private_file, +) +from pdf_sanitize import ( + SanitizePolicy, + reveal_hidden_layers, + sanitize_document, + verify_export, +) + +SECRET = "Mario Rossi" + + +def _loaded_pdf(path): + """A PDF carrying personal data in every place a plain redaction misses.""" + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Paziente: Mario Rossi nato a Napoli", fontsize=11) + page.insert_text((72, 140), "Referto generico senza dati", fontsize=11) + note = page.add_text_annot((300, 300), "Nota per Mario Rossi") + note.update() + doc.embfile_add("allegato.txt", b"Mario Rossi codice 123", filename="allegato.txt") + doc.set_toc([[1, "Cartella di Mario Rossi", 1]]) + page.insert_link({ + "kind": pymupdf.LINK_URI, "from": pymupdf.Rect(72, 300, 200, 320), + "uri": "mailto:mario.rossi@example.com", + }) + cat = doc.pdf_catalog() + doc.xref_set_key( + cat, "OpenAction", "<>" + ) + # invisible text (render mode 3) + xref = page.get_contents()[0] + doc.update_stream( + xref, + doc.xref_stream(xref) + + b"\nq BT /helv 11 Tf 3 Tr 72 400 Td (INVISIBLETEXT) Tj ET Q\n", + ) + # hidden layer + ocg = doc.add_ocg("Nascosto", on=False) + page.insert_text((72, 450), "HIDDENLAYER", fontsize=11, oc=ocg) + doc.set_metadata({"title": "Cartella Mario Rossi", "author": "Mario Rossi"}) + doc.save(path) + doc.close() + + +def _redact_words(doc, words): + for page in doc: + idx = pd.build_page_index(page.get_text("words")) + for w in words: + for r in pd.whole_word_rects(idx, w): + page.add_redact_annot(pymupdf.Rect(*r)) + + +def test_export_removes_everything_that_survives_plain_redaction(tmp_path): + src = tmp_path / "src.pdf" + out = tmp_path / "out.pdf" + _loaded_pdf(str(src)) + doc = pymupdf.open(str(src)) + assert reveal_hidden_layers(doc) == 1 + _redact_words(doc, ["Mario", "Rossi", "HIDDENLAYER"]) + removed = [SECRET, "INVISIBLETEXT", "HIDDENLAYER", "allegato", "mario.rossi@example.com"] + result = export_document(doc, str(out), removed_values=removed) + doc.close() + assert result.verification.ok, result.verification.findings + + exported = pymupdf.open(str(out)) + page = exported[0] + text = page.get_text() + assert "Mario" not in text and "Rossi" not in text + assert "INVISIBLETEXT" not in text and "HIDDENLAYER" not in text + assert "Referto generico senza dati" in text # collateral damage check + assert not list(page.annots() or []) + assert exported.embfile_names() == [] + assert exported.get_toc() == [] + assert not any(l.get("kind") == pymupdf.LINK_URI for l in page.get_links()) + cat = exported.pdf_catalog() + for key in ("OpenAction", "AA", "Names", "StructTreeRoot", "Metadata"): + assert exported.xref_get_key(cat, key)[0] == "null" + assert not any(v for v in (exported.metadata or {}).values() if v and "Rossi" in str(v)) + exported.close() + raw = out.read_bytes() + assert b"Mario Rossi" not in raw and b"INVISIBLETEXT" not in raw + + +def test_no_unredacted_copy_is_left_beside_the_output(tmp_path): + src = tmp_path / "src.pdf" + out_dir = tmp_path / "outdir" + out_dir.mkdir() + _loaded_pdf(str(src)) + doc = pymupdf.open(str(src)) + _redact_words(doc, ["Mario", "Rossi"]) + export_document(doc, str(out_dir / "o.pdf"), removed_values=[SECRET]) + doc.close() + assert [p.name for p in out_dir.iterdir()] == ["o.pdf"] + + +def test_large_document_uses_private_temp_and_cleans_up(tmp_path, monkeypatch): + import export_engine + + monkeypatch.setattr(export_engine, "IN_MEMORY_LIMIT_BYTES", 0) + src = tmp_path / "src.pdf" + _loaded_pdf(str(src)) + doc = pymupdf.open(str(src)) + _redact_words(doc, ["Mario", "Rossi"]) + tmp_dir = private_temp_dir(tmp_path / "cfg") + export_document(doc, str(tmp_path / "o.pdf"), removed_values=[SECRET], temp_dir=tmp_dir) + doc.close() + assert list(tmp_dir.iterdir()) == [] + + +def test_verify_export_flags_injected_leaks(tmp_path): + leaky = tmp_path / "leaky.pdf" + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "contiene Mario Rossi in chiaro", fontsize=11) + page.add_text_annot((200, 200), "anche qui Mario Rossi") + doc.embfile_add("a.txt", b"x") + doc.save(str(leaky)) + doc.close() + rep = verify_export(str(leaky), [SECRET]) + assert not rep.ok + joined = " ".join(rep.findings) + assert "text layer" in joined and "annotations" in joined and "embedded" in joined + assert SECRET not in joined # findings never repeat the value + + +def test_verify_export_passes_a_clean_file(tmp_path): + clean = tmp_path / "clean.pdf" + doc = pymupdf.open() + doc.new_page().insert_text((72, 100), "nessun dato", fontsize=11) + doc.save(str(clean)) + doc.close() + assert verify_export(str(clean), [SECRET, "ab"]).ok + + +def test_scanned_page_keeps_its_invisible_ocr_layer(): + doc = pymupdf.open() + page = doc.new_page() + pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 100, 100), False) + pix.set_rect(pix.irect, (255, 255, 255)) + page.insert_image(page.rect, pixmap=pix) + xref = page.get_contents()[0] + doc.update_stream( + xref, doc.xref_stream(xref) + b"\nBT /helv 11 Tf 3 Tr 72 400 Td (OCRTEXT) Tj ET\n" + ) + rep = sanitize_document(doc, SanitizePolicy()) + assert "hidden-text-spans" not in rep.actions + assert "OCRTEXT" in doc[0].get_text() + + +def test_offpage_text_is_cleared_and_cropbox_expanded(): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "visibile", fontsize=11) + page.insert_text((72, 700), "FUORIPAGINA", fontsize=11) + page.set_cropbox(pymupdf.Rect(0, 0, 595, 400)) + rep = sanitize_document(doc) + assert "offpage-area-cleared" in rep.actions + assert doc[0].cropbox == doc[0].mediabox + assert "FUORIPAGINA" not in doc[0].get_text() + assert "visibile" in doc[0].get_text() + + +def test_tj_gap_quantization_hides_removed_width(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Paziente: Mario Rossi nato a Napoli", fontsize=11) + idx = pd.build_page_index(page.get_text("words")) + for w in ("Mario", "Rossi"): + for r in pd.whole_word_rects(idx, w): + page.add_redact_annot(pymupdf.Rect(*r)) + page.apply_redactions() + + def gaps(): + found = [] + for xr in page.get_contents(): + data = doc.xref_stream(xr).decode("latin-1") + import re + for m in re.finditer(r"\)\s*(-?\d+(?:\.\d+)?)\s*\(", data): + found.append(float(m.group(1))) + return [g for g in found if abs(g) >= 1500] + + before = gaps() + assert before, "expected a redaction gap in the content stream" + sanitize_document(doc, SanitizePolicy(mask_removed_width=True)) + after = gaps() + assert after and all(abs(round(g)) % 500 == 0 for g in after) + assert before != after + + +def test_collect_removed_values_reads_payload_and_text(): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Mario Rossi", fontsize=11) + page.add_redact_annot(pymupdf.Rect(60, 85, 160, 110)) + vals = collect_removed_values(doc, lambda a: {"original": "Mario Rossi"}) + assert "Mario Rossi" in vals and "Mario" not in vals + + +def test_neutral_names_and_leak_detection(): + assert "Rossi" not in neutral_output_name("pseudonymize") + assert neutral_map_name().endswith(".nullifypdf-map") + assert name_leaks_value("C:/x/Referto_Rossi_Mario.pdf", ["Mario Rossi", "Rossi Mario"]) is False or True + assert name_leaks_value("C:/x/Referto_MarioRossi.pdf", ["Mario Rossi"]) + assert not name_leaks_value("C:/x/document_20260101.pdf", ["Mario Rossi"]) + + +def test_audit_record_has_no_values_and_is_private(tmp_path): + doc = pymupdf.open() + doc.new_page() + from pdf_sanitize import SanitizeReport, VerificationReport + + rec = build_audit_record( + app_version="x", mode="anonymize", entity_counts={"PERSON": 3}, + suggestion_counts={"unreviewed": 0}, sanitize=SanitizeReport({"a": 1}), + verification=VerificationReport(), output_sha256="abc", pages=1, + ) + path = write_audit_record(tmp_path, rec) + data = json.loads(open(path, encoding="utf-8").read()) + assert data["entity_counts"] == {"PERSON": 3} and data["verification"]["ok"] + assert "Rossi" not in json.dumps(data) + + +def test_wipe_stale_temp_and_log_purge(tmp_path): + cfg = tmp_path / "cfg" + (cfg / "logs").mkdir(parents=True) + (cfg / "logs" / "nullifypdf.log").write_text("x") + (cfg / "audit").mkdir() + (cfg / "tmp").mkdir() + (cfg / "tmp" / "leftover.tmp").write_text("x") + (cfg / "blocklist.txt").write_text("rossi") + assert cleanup_stale_temp(cfg) == 1 + removed = wipe_local_data(cfg) + assert {"logs", "audit"} <= set(removed) + assert (cfg / "blocklist.txt").exists() + wipe_local_data(cfg, include_lists=True) + assert not (cfg / "blocklist.txt").exists() + old = tmp_path / "logs" + old.mkdir() + f = old / "old.log" + f.write_text("x") + os.utime(f, (1, 1)) + assert purge_old_logs(old, 30) == 1 + + +def test_write_private_file_roundtrip(tmp_path): + p = tmp_path / "m.bin" + write_private_file(str(p), b"data") + assert p.read_bytes() == b"data" + + +# --- verification must not cry wolf on ordinary documents -------------------- + +def _export_with_rects(doc, rect_words_first_only, tmp_path, values=None): + page = doc[0] + idx = pd.build_page_index(page.get_text("words")) + for w in rect_words_first_only: + page.add_redact_annot(pymupdf.Rect(*pd.whole_word_rects(idx, w)[0])) + out = tmp_path / "o.pdf" + if values is None: + values = collect_removed_values(doc) + return export_document(doc, str(out), removed_values=values) + + +def test_redacting_rossi_is_not_flagged_because_rossini_remains(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Il Sig. Rossi vive a Roma", fontsize=11) + page.insert_text((72, 130), "Il collega Rossini abita a Romagna", fontsize=11) + result = _export_with_rects(doc, ["Rossi", "Roma"], tmp_path, + values=["Rossi", "Roma"]) + assert result.verification.ok, result.verification.findings + + +def test_drawn_rectangle_does_not_make_common_words_needles(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "abita in Via Roma 10 con la famiglia", fontsize=11) + page.insert_text((72, 130), "via di fuga e nato con la camicia", fontsize=11) + words = page.get_text("words") + box = pymupdf.Rect( + min(w[0] for w in words if w[4] in ("Via", "Roma", "10") and w[1] < 110), + words[0][1], + max(w[2] for w in words if w[4] == "10"), + words[0][3], + ) + page.add_redact_annot(box) + values = collect_removed_values(doc) + assert all(len(v) >= 4 for v in values) and "via" not in [v.lower() for v in values] + out = tmp_path / "o.pdf" + result = export_document(doc, str(out), removed_values=values) + assert result.verification.ok, result.verification.findings + exported = pymupdf.open(str(out)) + assert "Roma" not in exported[0].get_text() and "via di fuga" in exported[0].get_text() + exported.close() + + +def test_deliberately_unredacted_occurrence_is_not_a_failure(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Mario Rossi (primo)", fontsize=11) + page.insert_text((72, 140), "Mario Rossi (secondo, lasciato visibile)", fontsize=11) + idx = pd.build_page_index(page.get_text("words")) + for w in ("Mario", "Rossi"): + page.add_redact_annot(pymupdf.Rect(*pd.whole_word_rects(idx, w)[0])) + out = tmp_path / "o.pdf" + result = export_document(doc, str(out), removed_values=["Mario Rossi"]) + assert result.verification.ok, result.verification.findings + + +def test_a_real_leak_is_still_caught(tmp_path): + out = tmp_path / "o.pdf" + doc = pymupdf.open() + doc.new_page().insert_text((72, 100), "Mario Rossi", fontsize=11) + doc.save(str(out)) + doc.close() + # Nothing marks "Mario Rossi" as intentionally kept: it is a leak. + assert not verify_export(str(out), ["Mario Rossi"]).ok + # The same text counts as deliberately kept when it sits outside redactions. + assert verify_export(str(out), ["Mario Rossi"], "Mario Rossi").ok + + +def test_font_programs_and_images_are_not_searched(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "testo qualunque", fontsize=11, fontname="Times-Roman") + out = tmp_path / "o.pdf" + doc.save(str(out)) + # a 4-letter needle that may appear by chance in binary font data + assert verify_export(str(out), ["qwrz"]).ok + + +def test_wipe_reports_only_what_is_really_gone(tmp_path): + import logging + import logging.handlers + import sys + + cfg = tmp_path / "cfg" + (cfg / "logs").mkdir(parents=True) + handler = logging.handlers.RotatingFileHandler(str(cfg / "logs" / "x.log"), encoding="utf-8") + log = logging.getLogger("wipe-test") + log.addHandler(handler) + log.error("line") + try: + removed = wipe_local_data(cfg) + if sys.platform == "win32": + assert "logs" not in removed # still open: honest report + finally: + handler.close() + log.removeHandler(handler) + assert "logs" in wipe_local_data(cfg) or not (cfg / "logs").exists() + + +def test_text_outside_redactions_uses_the_deletion_core(): + from export_engine import text_outside_redactions + + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Alessandro resta", fontsize=11) + x0, y0, x1, y1 = page.get_text("words")[0][:4] + # covers only the left quarter of the word: its glyphs are still deleted + page.add_redact_annot(pymupdf.Rect(x0 - 2, y0, x0 + (x1 - x0) * 0.25, y1)) + kept = text_outside_redactions(doc) + assert "Alessandro" not in kept and "resta" in kept + + +# --- post-apply glyph survival check ---------------------------------------------- + +def _doc_with_word_and_band(band_fraction): + import json as _json + + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Riga ABCDEFGH fine", fontsize=11) + x0, y0, x1, y1 = [w for w in page.get_text("words") if w[4] == "ABCDEFGH"][0][:4] + band = pymupdf.Rect(x0, y0, x0 + (x1 - x0) * band_fraction, y1) + annot = page.add_redact_annot(band) + annot.set_info(content=_json.dumps({ + "nullifypdf": 1, "original": "ABCDEFGH", "entity_type": "PERSON", + "full": [x0, y0, x1, y1], + })) + return doc + + +def _read_full(annot): + import json as _json + + try: + d = _json.loads(annot.info.get("content", "")) + return pymupdf.Rect(*d["full"]) if d.get("full") else None + except Exception: + return None + + +def test_a_band_that_misses_glyphs_is_reported(tmp_path): + doc = _doc_with_word_and_band(0.5) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs >= 3 + assert not result.verification.ok + assert any("survived redaction" in f for f in result.verification.findings) + assert "ABCD" not in " ".join(result.verification.findings) # never repeats the value + + +def test_a_band_covering_the_word_reports_nothing(tmp_path): + doc = _doc_with_word_and_band(1.0) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full, + removed_values=["ABCDEFGH"]) + assert result.surviving_glyphs == 0 and result.verification.ok, result.verification.findings + + +def test_survivor_check_is_inactive_without_a_stored_full_box(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "ABCDEFGH", fontsize=11) + page.add_redact_annot(pymupdf.Rect(72, 90, 80, 110)) # manual box, no `full` + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs == 0 + + +def test_survivor_alignment_survives_mupdf_moving_the_remaining_text(tmp_path): + """The rest of the line is rewritten and shifted; positions are not reliable.""" + doc = _doc_with_word_and_band(0.5) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs == 4 # E, F, G, H + + +def test_repeated_words_on_the_line_are_not_false_survivors(tmp_path): + import json as _json + + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Maria Mario Maria", fontsize=11) + x0, y0, x1, y1 = [w for w in page.get_text("words") if w[4] == "Mario"][0][:4] + annot = page.add_redact_annot(pymupdf.Rect(x0, y0, x1, y1)) + annot.set_info(content=_json.dumps({ + "nullifypdf": 1, "original": "Mario", "entity_type": "PERSON", + "full": [x0 - 1, y0, x1 + 1, y1], + })) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs == 0 + + +def test_rule_deviation_is_reported(tmp_path, monkeypatch): + """If a PyMuPDF release stops deleting what the rule predicts, the export says so.""" + import export_engine as ee + + doc = _doc_with_word_and_band(1.0) + real = ee.PDF_REDACT_IMAGE_PIXELS + orig_apply = pymupdf.Page.apply_redactions + monkeypatch.setattr(pymupdf.Page, "apply_redactions", lambda self, *a, **k: 0) # deletes nothing + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + monkeypatch.setattr(pymupdf.Page, "apply_redactions", orig_apply) + assert real is not None + assert result.surviving_glyphs >= 8 and not result.verification.ok diff --git a/tests/test_geometry.py b/tests/test_geometry.py new file mode 100644 index 0000000..1eed05f --- /dev/null +++ b/tests/test_geometry.py @@ -0,0 +1,251 @@ +"""Redaction geometry: no neighbour damage, no leak, correct partial policy.""" + +import pymupdf +import pytest + +import pii_detection as pd +from _pdfgen import fake_words, make_doc, res +from export_engine import export_document + +LINES = ["Commissione tributaria", "Mario Rossi gggg yyyy", "Programmazione generale"] + + +def _redact_and_export(doc, targets, tmp_path, clamp=True): + page = doc[0] + idx = pd.build_page_index(page.get_text("words"), clamp=clamp) + for t in targets: + for r in pd.whole_word_rects(idx, t): + page.add_redact_annot(pymupdf.Rect(*r)) + out = tmp_path / "out.pdf" + result = export_document(doc, str(out), removed_values=targets) + exported = pymupdf.open(str(out)) + text = exported[0].get_text() + exported.close() + return result, text + + +@pytest.mark.parametrize("pitch", [7, 8, 9, 10, 11, 12, 13.2, 16]) +def test_clamped_rects_keep_neighbour_lines_intact(pitch, tmp_path): + doc = make_doc(LINES, pitch=pitch) + result, text = _redact_and_export(doc, ["Mario", "Rossi", "gggg", "yyyy"], tmp_path) + assert "Commissione tributaria" in text + assert "Programmazione generale" in text + for secret in ("Mario", "Rossi", "gggg", "yyyy"): + assert secret not in text + assert result.verification.ok, result.verification.findings + + +def test_unclamped_boxes_damage_tight_neighbours(tmp_path): + """Documents the defect the clamp fixes (11 pt text, 11 pt pitch).""" + doc = make_doc(LINES, pitch=11) + _r, text = _redact_and_export(doc, ["Mario", "Rossi", "gggg", "yyyy"], tmp_path, clamp=False) + assert not ("Commissione tributaria" in text and "Programmazione generale" in text) + + +def test_clamp_only_trims_against_other_lines(): + words = [ + (0, 0, 40, 15, "Alpha", 0, 0, 0), + (45, 0, 60, 15, "Beta", 0, 0, 1), + (0, 11, 40, 26, "Gamma", 0, 1, 0), + ] + idx = pd.build_page_index(words) + alpha, beta, gamma = idx.words + assert alpha.rect[3] == pytest.approx(11 + 0.1 * 15 - 0.3) # stops short of the neighbour core + assert beta.rect == (45, 0, 60, 15) # no horizontal overlap: untouched + assert gamma.rect[1] == pytest.approx(15 - 0.1 * 15 + 0.3) + assert alpha.rect[0] == 0 and alpha.rect[2] == 40 + + +def test_clamp_keeps_original_when_text_collides_heavily(): + words = [ + (0, 0, 40, 15, "Alpha", 0, 0, 0), + (0, 2, 40, 17, "Beta", 0, 1, 0), + ] + idx = pd.build_page_index(words) + assert idx.words[0].rect == (0, 0, 40, 15) + assert idx.words[1].rect == (0, 2, 40, 17) + + +def test_clamp_ocr_mode_cuts_at_middle_of_overlap_and_pads(): + words = [ + (10, 0, 50, 15, "Alpha", 0, 0, 0), + (10, 11, 50, 26, "Gamma", 0, 1, 0), + ] + idx = pd.build_page_index(words, ocr=True) + assert idx.words[0].rect[3] == pytest.approx(13.0) + assert idx.words[0].rect[0] == pytest.approx(9.0) + assert idx.words[0].rect[2] == pytest.approx(51.0) + + +def test_mixed_sizes_on_one_line_keep_full_height(): + words = [ + (0, 0, 40, 15, "Big", 0, 0, 0), + (42, 0, 50, 8, "2", 0, 0, 1), + (0, 40, 40, 55, "Below", 0, 1, 0), + ] + idx = pd.build_page_index(words) + assert [w.rect for w in idx.words] == [(0, 0, 40, 15), (42, 0, 50, 8), (0, 40, 40, 55)] + + +def test_vertical_text_is_not_clamped(): + words = [ + (0, 0, 8, 60, "Verticale", 0, 0, 0), + (0, 55, 8, 115, "Altra", 0, 1, 0), + ] + idx = pd.build_page_index(words) + assert idx.words[0].rect == (0, 0, 8, 60) + + +def test_letter_spaced_run_becomes_one_analysis_word(): + letters = "Commissione" + words = [ + (10 + 9 * i, 0, 16 + 9 * i, 12, ch, 0, 0, i) for i, ch in enumerate(letters) + ] + idx = pd.build_page_index(words) + assert idx.text == "Commissione" + assert len(idx.words) == len(letters) + + +def test_hyphenated_name_is_joined_across_lines(): + words = [ + (10, 0, 40, 12, "Ros-", 0, 0, 0), + (10, 14, 50, 26, "sini", 0, 1, 0), + ] + idx = pd.build_page_index(words, clamp=False) + assert idx.text == "Rossini" + rects = pd.rects_for_words(idx, pd.span_word_indices(idx, 0, 7)) + assert len(rects) == 2 # one rectangle per line + + +def test_ligatures_are_expanded_for_analysis(): + idx = pd.build_page_index([(0, 0, 50, 12, "Uf๏ฌcio", 0, 0, 0)]) + assert idx.text == "Ufficio" + + +def test_partial_policy_ner_rejects_fragment_of_longer_word(): + text = "Il collega Rossini abita qui" + idx = pd.build_page_index(fake_words(text), clamp=False) + c = pd.Candidate(text.index("Ross"), text.index("Ross") + 4, "Ross", "PERSON", 0.85) + assert pd.resolve_word_indices(idx, c) == ([], "embedded-in-word") + + +def test_partial_policy_validated_identifier_glued_to_label_is_redacted(): + text = "CF:RSSMRA80A01H501U fine" + idx = pd.build_page_index(fake_words(text), clamp=False) + start = text.index("RSSMRA") + c = pd.Candidate(start, start + 16, text[start:start + 16], "IT_FISCAL_CODE", 1.0) + idxs, status = pd.resolve_word_indices(idx, c) + assert idxs and status in ("ok", "glued") + + +def test_partial_policy_short_overlap_of_validated_type_redacts_whole_token(): + text = "IBANIT60X0542811101000000123456 fine" + idx = pd.build_page_index(fake_words(text), clamp=False) + start = text.index("IT60") + c = pd.Candidate(start, start + 5, text[start:start + 5], "IBAN_CODE", 1.0) + idxs, status = pd.resolve_word_indices(idx, c) + assert idxs and status == "glued" + + +def test_merge_line_rects_and_covered_fraction(): + merged = pd.merge_line_rects([(0, 0, 10, 10), (10.5, 0, 20, 10), (0, 30, 10, 40)]) + assert merged == [(0, 0, 20, 10), (0, 30, 10, 40)] + assert pd.covered_fraction((0, 0, 10, 10), [(0, 0, 10, 5)]) == pytest.approx(0.5) + assert pd.covered_fraction((0, 0, 10, 10), [(20, 20, 30, 30)]) == 0.0 + + +def test_fake_result_helper_roundtrip(): + text = "Mario Rossi" + r = res("PERSON", text, "Rossi") + assert text[r.start:r.end] == "Rossi" + + +# --- the deletion rule the core-band clamp relies on (canary for new releases) ---- + +@pytest.mark.parametrize("fontname,size", [("helv", 11), ("helv", 20), ("tiro", 11), ("cour", 11)]) +def test_glyph_deletion_core_is_ten_percent_inset(fontname, size): + """MuPDF deletes a glyph only when the rectangle reaches its core box. + + If a PyMuPDF release changes this, the core-band clamp is unsafe: this + test (and the CI job against the latest PyMuPDF) must fail first. + """ + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((100, 300), "H", fontsize=size, fontname=fontname) + ch = page.get_text("rawdict")["blocks"][0]["lines"][0]["spans"][0]["chars"][0] + x0, y0, x1, y1 = ch["bbox"] + h = y1 - y0 + top_core = y0 + 0.10 * h + bottom_core = y1 - 0.10 * h + + def deleted(band): + d2 = pymupdf.open("pdf", doc.tobytes()) + p2 = d2[0] + p2.add_redact_annot(pymupdf.Rect(*band)) + p2.apply_redactions() + return "H" not in p2.get_text() + + eps = 0.15 + # just short of the core from above / below: kept; just inside: deleted + assert not deleted((x0 - 5, y0 - 5, x1 + 5, top_core - eps)) + assert deleted((x0 - 5, y0 - 5, x1 + 5, top_core + eps)) + assert not deleted((x0 - 5, bottom_core + eps, x1 + 5, y1 + 5)) + assert deleted((x0 - 5, bottom_core - eps, x1 + 5, y1 + 5)) + + +def test_core_band_mixed_sizes_spares_neighbours(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Intestazione piccola", fontsize=9) + page.insert_text((72, 108), "Paziente Alessandro Marchetti", fontsize=11) + page.insert_text((72, 118), "Nota finale minuscola", fontsize=8) + result, text = _redact_and_export(doc, ["Alessandro", "Marchetti"], tmp_path) + assert "Intestazione piccola" in text and "Nota finale minuscola" in text + assert "Alessandro" not in text and "Marchetti" not in text + + +def test_empty_clipped_target_never_approves_thin_band(): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Mario Rossi", fontsize=11) + idx = pd.build_page_index(page.get_text("words"), clamp=False) + full = pymupdf.Rect(*pd.whole_word_rects(idx, "Mario")[0]) + thin = pymupdf.Rect(full.x0, full.y0, full.x1, full.y0 + 0.01) + from NullifyPDF import band_covers_target + assert not band_covers_target(page, thin, full) + + +def test_no_safe_band_flags_collision_and_still_removes_target(tmp_path): + doc = make_doc(LINES, pitch=5) + page = doc[0] + idx = pd.build_page_index(page.get_text("words")) + assert idx.collisions > 0 + result, text = _redact_and_export(doc, ["Mario", "Rossi"], tmp_path) + assert "Mario" not in text and "Rossi" not in text + + +def test_words_over_an_image_never_get_a_thin_band(): + words = [ + (10, 0, 50, 15, "Alpha", 0, 0, 0), + (10, 11, 50, 26, "Gamma", 0, 1, 0), + ] + native = pd.build_page_index(words) + over_image = pd.build_page_index(words, image_rects=[(0, 0, 100, 40)]) + assert native.words[0].native and not over_image.words[0].native + assert over_image.words[0].rect[3] == pytest.approx(13.0) # mid-overlap + assert over_image.words[0].rect[0] == pytest.approx(9.0) # horizontal pad + + +@pytest.mark.parametrize("pitch", [9, 10, 12]) +def test_style_capture_ignores_neighbour_lines(pitch): + from NullifyPDF import capture_text_style + + doc = make_doc( + ["Commissione tributaria provinciale", "Paziente Alessandro nato", "Programmazione generale dei servizi"], + pitch=pitch, + ) + page = doc[0] + idx = pd.build_page_index(page.get_text("words"), clamp=False) + for rect in pd.whole_word_rects(idx, "Alessandro"): # raw, unclamped box + style = capture_text_style(page, pymupdf.Rect(*rect)) + assert style is not None and style["text"] == "Alessandro" diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py new file mode 100644 index 0000000..3dfeaf8 --- /dev/null +++ b/tests/test_gui_flow.py @@ -0,0 +1,980 @@ +"""Offscreen functional tests of the review / export / reconstruct flows. + +mypy and unit tests do not exercise Qt slots and dialogs, so these drive the +real window with Qt's offscreen platform and stubbed modal dialogs. HOME is +isolated so the user's real ~/.nullifypdf is never touched. +""" + +import atexit +import gc +import json +import os + +import pymupdf +import pytest + +pytest.importorskip("PySide6") +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import ( # noqa: E402 + QApplication, QDialog, QFileDialog, QInputDialog, QMessageBox, +) + +import NullifyPDF as app # noqa: E402 +from privacy_core import ( # noqa: E402 + PrivacyMode, decrypt_restore_payload, +) +from review_dialog import ReviewDialog, mask_value # noqa: E402 + + +@pytest.fixture() +def window(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("HOME", str(home)) + qapp = QApplication.instance() or QApplication([]) + w = app.NullifyPDF() + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Paziente: Alessandro Marchetti nato a Napoli", fontsize=11) + page.insert_text((72, 130), "Zorzi era presente alla visita", fontsize=11) + doc.save(str(tmp_path / "Referto_Mario_Rossi.pdf")) + doc.close() + w.doc = pymupdf.open(str(tmp_path / "Referto_Mario_Rossi.pdf")) + yield w + try: + w._cleanup_resources() + except Exception: + pass + # Every window registers an atexit cleanup; left registered, they run on + # already destroyed Qt objects at interpreter exit and crash the process + # (seen as a segmentation fault after the last test on Linux). + atexit.unregister(w._cleanup_on_exit) + w.deleteLater() + qapp.processEvents() + gc.collect() + + +def _rect_of(w, word): + page = w.doc[0] + idx = app.build_page_index(page.get_text("words")) + return list(app.whole_word_rects(idx, word)[0]) + + +def _detection(w, word, etype, state, score=0.9): + return {"text": word, "entity_type": etype, "rects": [_rect_of(w, word)], + "state": state, "score": score, "reasons": ["context-cue"]} + + +def _redacts(w): + return [a for a in w.doc[0].annots() if a.type[0] == app.PDF_ANNOT_REDACT] + + +def test_accepted_applies_and_suggested_waits_for_review(window): + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Zorzi", "PERSON", "suggested", 0.5), + ]) + assert len(_redacts(w)) == 1 + assert len(w.pending_suggestions) == 1 + assert w.btn_review.text() == "Rivedi suggerimenti (1)" and w.btn_review.isEnabled() + + +def test_review_dialog_applies_only_selected(window, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + + def fake_exec(self): + self.table.item(0, 0).setCheckState(app.Qt.CheckState.Checked) + self._on_apply() + return QDialog.DialogCode.Accepted + + monkeypatch.setattr(ReviewDialog, "exec", fake_exec) + w._review_suggestions() + assert w.pending_suggestions == [] + assert len(_redacts(w)) == 1 + + +def test_review_dialog_reject_discards_and_later_keeps_pending(window, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + monkeypatch.setattr( + ReviewDialog, "exec", + lambda self: (self._on_apply(), QDialog.DialogCode.Accepted)[1], + ) + w._review_suggestions() + assert w.pending_suggestions == [] and _redacts(w) == [] + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + monkeypatch.setattr(ReviewDialog, "exec", lambda self: QDialog.DialogCode.Rejected) + w._review_suggestions() + assert len(w.pending_suggestions) == 1 + + +def test_review_dialog_masks_values_until_asked(window): + dlg = ReviewDialog([{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, + "page": 0, "reasons": ["lexicon"]}]) + assert dlg.table.item(0, 2).text() == mask_value("Zorzi") != "Zorzi" + dlg.chk_show.setChecked(True) + assert dlg.table.item(0, 2).text() == "Zorzi" + dlg.spin.setValue(0.4) + dlg._select_threshold() + assert dlg.table.item(0, 0).checkState() == app.Qt.CheckState.Checked + got = [] + dlg.navigate.connect(got.append) + dlg._on_double_click(0, 0) + assert got == [0] + + +def test_overlapping_detections_are_not_applied_twice(window): + w = window + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + assert len(_redacts(w)) == 1 + + +def test_legacy_detection_without_rects_never_redacts_fragments(window): + w = window + w.doc.close() + d = pymupdf.open() + d.new_page().insert_text((72, 100), "Marchettini e Marchetti", fontsize=11) + w.doc = d + w.apply_ai_to_page(0, {"Ross"}) + assert _redacts(w) == [] + w.apply_ai_to_page(0, {"Marchetti"}) + assert len(_redacts(w)) == 1 + + +def _stub_dialogs(monkeypatch, files, answer=QMessageBox.StandardButton.Yes): + seq = list(files) + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (seq.pop(0), ""))) + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: answer)) + monkeypatch.setattr(QMessageBox, "information", staticmethod(lambda *a, **k: None)) + monkeypatch.setattr(QMessageBox, "warning", staticmethod(lambda *a, **k: None)) + monkeypatch.setattr(QMessageBox, "critical", staticmethod(lambda *a, **k: QMessageBox.StandardButton.Yes)) + + +def test_anonymize_export_end_to_end(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out = tmp_path / "exported.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + w.cmd_export() + assert out.exists() + exported = pymupdf.open(str(out)) + text = exported[0].get_text() + exported.close() + assert "Alessandro" not in text and "Marchetti" not in text and "Napoli" in text + assert sorted(p.name for p in tmp_path.iterdir() if p.suffix == ".tmp") == [] + audit = list((w.config_dir / "audit").glob("*.json")) + assert len(audit) == 1 + record = json.loads(audit[0].read_text(encoding="utf-8")) + assert record["entity_counts"] == {"PERSON": 2} and record["verification"]["ok"] + assert "Marchetti" not in audit[0].read_text(encoding="utf-8") + assert w.doc is not None and len(_redacts(w)) == 2 # original untouched + + +def test_export_warns_about_unreviewed_suggestions(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + asked = [] + + class Box: + class _B: # minimal stand-in for a QMessageBox button + pass + + def fake_exec(self): + asked.append(self.text()) + + monkeypatch.setattr(QMessageBox, "exec", fake_exec) + # Clicking nothing = Cancel: export must stop before any file dialog. + monkeypatch.setattr(QMessageBox, "clickedButton", lambda self: None) + monkeypatch.setattr( + QFileDialog, "getSaveFileName", + staticmethod(lambda *a, **k: pytest.fail("export continued despite pending suggestions")), + ) + w.cmd_export() + assert asked and "non revisionati" in asked[0] + + +def test_pseudonymize_export_with_random_style_roundtrips(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out = tmp_path / "pseudo.pdf" + mapf = tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + assert out.exists() and mapf.exists() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Alessandro" not in text and "PERSON_" in text and "PERSON_001" not in text + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + assert payload["source_name"] == "" + assert {e["original"] for e in payload["entries"]} == {"Alessandro", "Marchetti"} + restored = tmp_path / "restored.pdf" + n = app.reconstruct_pdf(str(out), str(restored), payload) + assert n == 2 + back = pymupdf.open(str(restored)) + assert "Alessandro" in back[0].get_text() + back.close() + + +def test_export_stops_when_verification_fails_and_user_deletes(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + out = tmp_path / "bad.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + # A value that survives redaction (it is in the visible text): force a + # failing verification by claiming "Napoli" was removed. + monkeypatch.setattr(app, "collect_removed_values", lambda *a, **k: ["Napoli"]) + import export_engine + monkeypatch.setattr(export_engine, "text_outside_redactions", lambda d: "") + w.cmd_export() + assert not out.exists() + assert not list((w.config_dir / "audit").glob("*.json")) + + +def test_wipe_local_data(window, monkeypatch): + w = window + (w.config_dir / "audit").mkdir(parents=True, exist_ok=True) + (w.config_dir / "audit" / "x.json").write_text("{}") + monkeypatch.setattr( + QMessageBox, "question", + staticmethod(lambda *a, **k: QMessageBox.StandardButton.No), + ) + w.cmd_wipe_local_data() + assert not (w.config_dir / "audit").exists() + + +def test_only_italian_is_offered(window): + assert not hasattr(window, "rb_en") and not hasattr(window, "rb_both") + assert "italiano" in window.lbl_lang.text() + + +def test_worker_scan_end_to_end_with_real_italian_model(tmp_path, monkeypatch): + """Drives AIWorker.run_scan itself: wiring of artifacts, classify, build.""" + spacy = pytest.importorskip("spacy") + pytest.importorskip("presidio_analyzer") + if not spacy.util.is_package("it_core_news_md"): + pytest.skip("model missing") + from PySide6.QtCore import QMutex + + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.setenv("HOME", str(tmp_path)) + QApplication.instance() or QApplication([]) + doc = pymupdf.open() + page = doc.new_page() + lines = [ + "Certifico", + "che il Sig. Mario Rossi, nato a Napoli il 12/03/1970,", + "e' affetto da lombalgia.", + "Prescrivo", + "riposo assoluto.", + "Codice fiscale: RSSMRA70C12F839Y", + ] + for i, t in enumerate(lines): + page.insert_text((72, 100 + 16 * i), t, fontsize=11) + worker = app.AIWorker() + done, logs = [], [] + worker.page_done_sig.connect(lambda i, d: done.append((i, d))) + worker.log_sig.connect(logs.append) + enabled = app.expand_group_ids(app.ALL_GROUP_IDS) + worker.run_scan(doc, QMutex(), "IT", [], set(), False, None, enabled, True) + assert len(done) == 1, logs + dets = done[0][1] + texts = {d["text"] for d in dets} + assert "Mario Rossi" in texts and "Napoli" in texts and "12/03/1970" in texts + assert not texts & {"Certifico", "Prescrivo"} + assert all("state" in d and d["rects"] for d in dets) + assert any(l.startswith("Scansione completata") for l in logs) + worker.cleanup() + + +def test_worker_ocr_of_embedded_images_keeps_native_text(tmp_path, monkeypatch): + """A text page with an image: OCR adds the image text, native text stays.""" + spacy = pytest.importorskip("spacy") + pytest.importorskip("presidio_analyzer") + if not spacy.util.is_package("it_core_news_md"): + pytest.skip("model missing") + tessdata = app.find_tessdata_dir() + if not tessdata: + pytest.skip("tessdata missing") + from PySide6.QtCore import QMutex + + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.setenv("HOME", str(tmp_path)) + QApplication.instance() or QApplication([]) + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Il Sig. Mario Rossi abita a Napoli.", fontsize=12) + src = pymupdf.open() + sp = src.new_page(width=300, height=60) + sp.insert_text((10, 40), "Dott. Giuseppe Verdi", fontsize=22) + page.insert_image(pymupdf.Rect(72, 200, 372, 260), pixmap=sp.get_pixmap(dpi=150)) + worker = app.AIWorker() + done, logs = [], [] + worker.page_done_sig.connect(lambda i, d: done.append(d)) + worker.log_sig.connect(logs.append) + enabled = app.expand_group_ids(app.ALL_GROUP_IDS) + worker.run_scan(doc, QMutex(), "IT", [], set(), True, tessdata, enabled, False) + texts = {" ".join(d["text"].split()) for d in done[0]} + assert "Mario Rossi" in texts, (texts, logs) + assert any("Giuseppe Verdi" in t for t in texts), (texts, logs) + assert any("OCR immagini" in l for l in logs) + worker.cleanup() + + +def test_roundtrip_with_trimmed_rectangles_on_tight_leading(window, tmp_path, monkeypatch): + """11 pt text on a 12 pt pitch: rectangles are trimmed, restore still works.""" + w = window + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + for i, t in enumerate([ + "Commissione tributaria provinciale", + "Paziente Alessandro Marchetti nato oggi", + "Programmazione generale dei servizi", + ]): + pg.insert_text((72, 100 + 12 * i), t, fontsize=11) + w.doc = d + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out = tmp_path / "t.pdf" + mapf = tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Commissione tributaria provinciale" in text + assert "Programmazione generale dei servizi" in text + assert "Alessandro" not in text + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + restored = tmp_path / "r.pdf" + assert app.reconstruct_pdf(str(out), str(restored), payload) == 2 + back = pymupdf.open(str(restored)) + t2 = back[0].get_text() + back.close() + assert "Alessandro" in t2 and "Marchetti" in t2 + assert "Commissione tributaria provinciale" in t2 + + +def test_image_redaction_is_not_duplicated_by_later_applies(window, tmp_path): + w = window + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + pg.insert_text((72, 100), "Alessandro Marchetti", fontsize=11) + pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 40, 40), False) + pix.set_rect(pix.irect, (10, 10, 10)) + pg.insert_image(pymupdf.Rect(300, 300, 340, 340), pixmap=pix) + w.doc = d + w.chk_img.setChecked(True) + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + w.apply_ai_to_page(0, [_detection(w, "Marchetti", "PERSON", "accepted")]) + assert len(_redacts(w)) == 3 # two words + one image, not two images + + +def test_invalid_pseudonym_key_file_is_refused(window, monkeypatch): + w = window + (w.config_dir).mkdir(parents=True, exist_ok=True) + (w.config_dir / "pseudonym.key").write_bytes(b"short") + shown = [] + monkeypatch.setattr(QMessageBox, "exec", lambda self: 0) + monkeypatch.setattr( + QMessageBox, "clickedButton", lambda self: next( + b for b in self.buttons() if b.text() == "Coerente (chiave)" + ), + ) + monkeypatch.setattr( + QMessageBox, "critical", staticmethod(lambda *a, **k: shown.append(a[2])) + ) + assert w._select_pseudonym_style() is None + assert shown and "32 byte" in shown[0] + + +def test_signature_suggestion_is_never_applied_without_review_and_export_removes_ink(window, tmp_path, monkeypatch): + from pdf_sanitize import suggest_signature_areas + + w = window + w.doc.close() + import test_detection_pipeline as tdp + + w.doc = tdp._signature_doc() + sug = suggest_signature_areas(w.doc[0]) + assert len(sug) == 1 + w.apply_ai_to_page(0, sug) + assert _redacts(w) == [] and len(w.pending_suggestions) == 1 # not applied + monkeypatch.setattr( + ReviewDialog, "exec", + lambda self: (self.table.item(0, 0).setCheckState(app.Qt.CheckState.Checked), + self._on_apply(), QDialog.DialogCode.Accepted)[2], + ) + w._review_suggestions() + assert len(_redacts(w)) == 1 + out = tmp_path / "sig.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + w.cmd_export() + exported = pymupdf.open(str(out)) + region = pymupdf.Rect(*sug[0]["rects"][0]) + inside = [ + d for d in exported[0].get_drawings() + if region.contains(d["rect"]) and any(it[0] == "c" for it in d["items"]) + ] # the grey fill rectangle itself is expected + exported.close() + assert inside == [] + + +def test_dictionary_dialog_sets_expiry_and_persists(window, monkeypatch): + import datetime + from dictionary_dialog import DictionaryDialog + + w = window + dlg = DictionaryDialog(w.list_manager) + dlg.ttl.setCurrentIndex(1) # 30 giorni + dlg.add_term("blocklist", "Mario Rossi") + dlg.add_term("allowlist", "comune") + dlg.save() + assert dlg.saved_blocklist == {"mario rossi"} and dlg.saved_allowlist == {"comune"} + entries = w.list_manager.load_entries("blocklist") + assert entries[0].expires == datetime.date.today() + datetime.timedelta(days=30) + assert not (w.config_dir / "blocklist.txt").exists() + assert b"rossi" not in (w.config_dir / "blocklist.v2.enc").read_bytes() + + +def test_dictionary_dialog_rejects_bad_dates_and_keeps_data(window, monkeypatch): + from dictionary_dialog import DictionaryDialog + + w = window + w.list_manager.save_blocklist({"mario rossi"}) + dlg = DictionaryDialog(w.list_manager) + dlg.tables["blocklist"].item(0, 2).setText("domani") + warned = [] + monkeypatch.setattr(QMessageBox, "warning", staticmethod(lambda *a, **k: warned.append(a[2]))) + dlg.save() + assert warned and dlg.saved_blocklist is None + assert w.list_manager.load_blocklist() == {"mario rossi"} + + +def test_cmd_dict_updates_in_memory_lists(window, monkeypatch): + import dictionary_dialog as dd + + w = window + + def fake_exec(self): + self.add_term("blocklist", "nuovo termine") + self.save() + return QDialog.DialogCode.Accepted + + monkeypatch.setattr(dd.DictionaryDialog, "exec", fake_exec) + w.cmd_dict() + assert w.blocklist == {"nuovo termine"} + + +# --- protected pseudonym key, recovery key ------------------------------------------ + +def test_plaintext_pseudonym_key_is_validated_sealed_and_removed(window): + w = window + w.config_dir.mkdir(parents=True, exist_ok=True) + legacy = w.config_dir / "pseudonym.key" + legacy.write_bytes(b"k" * 32) + key = w._load_pseudonym_key() + assert key == b"k" * 32 + assert not legacy.exists() + enc = (w.config_dir / "pseudonym.v2.enc").read_bytes() + assert b"k" * 32 not in enc + assert w._load_pseudonym_key() == key + + +def test_new_pseudonym_key_is_created_sealed(window): + w = window + key = w._load_pseudonym_key() + assert len(key) == 32 and w._load_pseudonym_key() == key + assert key not in (w.config_dir / "pseudonym.v2.enc").read_bytes() + assert not (w.config_dir / "pseudonym.key").exists() + + +def test_pseudonym_key_export_and_import_on_another_profile(window, tmp_path, monkeypatch): + w = window + key = w._load_pseudonym_key() + out = tmp_path / "k.nullifypdf-key" + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(out), ""))) + monkeypatch.setattr(QMessageBox, "exec", lambda self: 0) + monkeypatch.setattr( + QMessageBox, "clickedButton", + lambda self: next(b for b in self.buttons() if b.text() == "Esporta chiave..."), + ) + w._pseudonym_key_dialog() + assert out.exists() and key not in out.read_bytes() + # simulate a new machine: wipe the stored key, then import + (w.config_dir / "pseudonym.v2.enc").unlink() + monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod(lambda *a, **k: (str(out), ""))) + monkeypatch.setattr(QInputDialog, "getText", staticmethod(lambda *a, **k: ("una frase molto lunga 2026", True))) + monkeypatch.setattr( + QMessageBox, "clickedButton", + lambda self: next(b for b in self.buttons() if b.text() == "Importa chiave..."), + ) + w._pseudonym_key_dialog() + assert w._load_pseudonym_key() == key + + +def test_export_with_recovery_key_and_reconstruct_by_recovery(window, tmp_path, monkeypatch): + from privacy_core import generate_recovery_key, parse_recovery_key + + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out, mapf = tmp_path / "p.pdf", tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + raw, shown = generate_recovery_key() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: raw) + w.cmd_export() + blob = mapf.read_bytes() + assert raw not in blob and shown.encode() not in blob + for f in tmp_path.rglob("*"): + if f.is_file(): + assert shown.encode() not in f.read_bytes() + env = json.loads(blob) + assert [x["type"] for x in env["wraps"]] == ["password", "recovery"] + # reconstruct through the recovery key only + restored = tmp_path / "r.pdf" + monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod( + lambda *a, **k: (str(out), "") if "pseudonimizzato" in a[1] else (str(mapf), ""))) + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(restored), ""))) + monkeypatch.setattr(w, "_ask_decrypt_secret", lambda: (None, parse_recovery_key(shown))) + w.cmd_reconstruct() + back = pymupdf.open(str(restored)) + assert "Alessandro" in back[0].get_text() + back.close() + + +def test_wrong_secret_gives_one_clear_message(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + out, mapf = tmp_path / "p.pdf", tmp_path / "m.nullifypdf-map" + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + shown = [] + monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod( + lambda *a, **k: (str(out), "") if "pseudonimizzato" in a[1] else (str(mapf), ""))) + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(tmp_path / "x.pdf"), ""))) + monkeypatch.setattr(w, "_ask_decrypt_secret", lambda: ("password completamente sbagliata", None)) + monkeypatch.setattr(QMessageBox, "critical", staticmethod(lambda *a, **k: shown.append(a[2]))) + w.cmd_reconstruct() + assert shown and "recupero" in shown[0] + + +def test_wipe_keys_makes_encrypted_lists_unreadable_only_with_confirmation(window, monkeypatch): + w = window + w.list_manager.save_blocklist({"mario rossi"}) + answers = iter([QMessageBox.StandardButton.Yes, QMessageBox.StandardButton.Yes]) + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: next(answers))) + w.cmd_wipe_local_data() + assert not (w.config_dir / "keystore.json").exists() + assert not (w.config_dir / "blocklist.v2.enc").exists() + + +# --- preview overlay of pending suggestions (real mouse events) ------------------------ + +def _click_scene_point(w, scene_pt): + from PySide6.QtCore import QEvent, QPointF, Qt + from PySide6.QtGui import QMouseEvent + + vp = w.view.mapFromScene(scene_pt) + gp = QPointF(vp) + glob = QPointF(w.view.viewport().mapToGlobal(vp)) + for typ, fn, btns in ( + (QEvent.Type.MouseButtonPress, w.view.mousePressEvent, Qt.MouseButton.LeftButton), + (QEvent.Type.MouseButtonRelease, w.view.mouseReleaseEvent, Qt.MouseButton.NoButton), + ): + fn(QMouseEvent(typ, gp, glob, Qt.MouseButton.LeftButton, btns, Qt.KeyboardModifier.NoModifier)) + QApplication.processEvents() + # the view defers a lone click slightly (to tell it from a double click) + import time + deadline = time.time() + 1.0 + while time.time() < deadline: + QApplication.processEvents() + time.sleep(0.02) + + +def _overlays(w): + return [i for i in w.scene.items() if isinstance(i, app.SuggestionItem)] + + +def _pick(monkeypatch, label): + monkeypatch.setattr(QMessageBox, "exec", lambda self: 0) + monkeypatch.setattr( + QMessageBox, "clickedButton", + lambda self: next(b for b in self.buttons() if b.text() == label), + ) + + +def _two_page_doc(w): + w.doc.close() + d = pymupdf.open() + for t in ("Pagina uno Zorzi presente", "Pagina due Colombi presente"): + d.new_page().insert_text((72, 100), t, fontsize=11) + w.doc = d + w.page_num = 0 + w.render_page() + + +def _sugg(w, page, word): + idx = app.build_page_index(w.doc[page].get_text("words")) + return {"text": word, "entity_type": "PERSON", "score": 0.5, "page": page, + "rects": [list(app.whole_word_rects(idx, word)[0])], "reasons": ["lexicon"], + "state": "suggested"} + + +def test_overlay_only_on_the_suggestion_page_and_follows_zoom(window): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi"), _sugg(w, 1, "Colombi")] + w.render_page() + assert len(_overlays(w)) == 1 + r0 = _overlays(w)[0].rect() + w.adjust_zoom(+2) + r1 = _overlays(w)[0].rect() + assert r1.width() == pytest.approx(r0.width() * w.scale / (w.scale - 0.5), rel=1e-6) + w.move_page(1) + assert len(_overlays(w)) == 1 and _overlays(w)[0].suggestion["text"] == "Colombi" + w.chk_sugg.setChecked(False) + assert _overlays(w) == [] + + +def test_overlay_click_accept_creates_one_redaction(window, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w._update_review_button() + w.render_page() + _pick(monkeypatch, "Censura") + _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center()) + assert len(_redacts(w)) == 1 and w.pending_suggestions == [] + assert w.btn_review.text() == "Rivedi suggerimenti (0)" + assert _overlays(w) == [] + + +def test_overlay_click_reject_discards_without_annotation(window, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w.render_page() + _pick(monkeypatch, "Scarta") + _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center()) + assert _redacts(w) == [] and w.pending_suggestions == [] + + +def test_overlay_cancel_keeps_the_suggestion(window, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w.render_page() + _pick(monkeypatch, "Annulla") + _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center()) + assert len(w.pending_suggestions) == 1 and _redacts(w) == [] + + +def test_overlay_never_reaches_the_export(window, tmp_path, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w.render_page() + assert _redacts(w) == [] + pix = w.doc[0].get_pixmap() + from PySide6.QtWidgets import QMessageBox as QMB + + out = tmp_path / "o.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + monkeypatch.setattr(QMB, "exec", lambda self: 0) + monkeypatch.setattr(QMB, "clickedButton", lambda self: next( + b for b in self.buttons() if b.text() == "Esporta comunque")) + w.cmd_export() + exported = pymupdf.open(str(out)) + assert "Zorzi" in exported[0].get_text() # not censored: still only a suggestion + exported.close() + assert pix.width > 0 + + +def test_review_row_selection_highlights_the_overlay(window): + w = window + _two_page_doc(w) + s = _sugg(w, 0, "Zorzi") + w.pending_suggestions = [s] + w.render_page() + dlg = ReviewDialog(w.pending_suggestions) + dlg.highlight.connect(w._highlight_suggestion) + dlg.table.setCurrentCell(0, 2) + assert _overlays(w)[0].pen().width() == 4 + + +# --- review findings: clip text, mixed glyph heights, 9 pt round trip, recovery ----------- + +def _tight_doc(w, pitch=9.0): + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + for i, t in enumerate([ + "Commissione tributaria provinciale", + "Paziente Alessandro Marchetti nato oggi", + "Programmazione generale dei servizi", + ]): + pg.insert_text((72, 100 + pitch * i), t, fontsize=11) + w.doc = d + w.page_num = 0 + w.render_page() + + +@pytest.mark.parametrize("pitch", [9.0, 10.0, 12.0]) +def test_manual_rectangle_stores_only_its_own_line(window, tmp_path, monkeypatch, pitch): + """A box drawn over a word at tight leading must not capture the neighbours.""" + from PySide6.QtCore import QRectF + + w = window + _tight_doc(w, pitch) + idx = app.build_page_index(w.doc[0].get_text("words"), clamp=False) # raw word box + x0, y0, x1, y1 = app.whole_word_rects(idx, "Alessandro")[0] + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: QMessageBox.StandardButton.No)) + w.user_draw_rect(QRectF(x0 * w.scale, y0 * w.scale, (x1 - x0) * w.scale, (y1 - y0) * w.scale)) + annots = _redacts(w) + assert len(annots) == 1 + assert w._read_redaction_payload(annots[0])["original"] == "Alessandro" + out, mapf = tmp_path / "m.pdf", tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + assert {e["original"] for e in payload["entries"]} == {"Alessandro"} + + +def test_pseudonymize_round_trip_at_9pt_keeps_neighbours_and_counts(window, tmp_path, monkeypatch): + w = window + _tight_doc(w, 9.0) + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out, mapf = tmp_path / "t9.pdf", tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Commissione tributaria provinciale" in text + assert "Programmazione generale dei servizi" in text + assert "Alessandro" not in text + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + restored = tmp_path / "r9.pdf" + n = app.reconstruct_pdf(str(out), str(restored), payload) + assert n == app.expected_restore_count(payload) == 2 + back = pymupdf.open(str(restored)) + t2 = back[0].get_text() + back.close() + assert "Alessandro" in t2 and "Marchetti" in t2 + assert t2.count("Commissione tributaria provinciale") == 1 + assert "Programmazione generale dei servizi" in t2 + + +def test_word_with_a_taller_glyph_falls_back_to_the_full_box(window): + """A thin band that misses one glyph of the target must not be used.""" + w = window + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + pg.insert_text((72, 100), "Riga sopra del testo", fontsize=11) + pg.insert_text((72, 109), "Codice AB", fontsize=11) + pg.insert_text((72 + 62, 109 - 5), "123", fontsize=6) # raised, smaller trailing digits + pg.insert_text((72, 118), "Riga sotto del testo", fontsize=11) + w.doc = d + idx = app.build_page_index(d[0].get_text("words")) + fulls = app.whole_word_rects(idx, "AB") + band = fitz_rect = pymupdf.Rect(*fulls[0]) + full = pymupdf.Rect(band.x0, band.y0 - 6, band.x1 + 30, band.y1) + thin = pymupdf.Rect(full.x0, full.y0 + 9, full.x1, full.y0 + 10) # misses the raised digits + assert not app.band_covers_target(d[0], thin, full) + assert app.band_covers_target(d[0], full, full) + + +def test_recovery_wrap_is_kept_even_if_the_dialog_is_closed_without_ticking(window, monkeypatch): + w = window + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: QMessageBox.StandardButton.Yes)) + monkeypatch.setattr(QDialog, "exec", lambda self: QDialog.DialogCode.Rejected) + raw = w._offer_recovery_key() + assert isinstance(raw, bytes) and len(raw) == 32 + + +# --- labels on image / signature areas survive small boxes ------------------------------- + +def _tiny_area_doc(w, height): + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + pg.insert_text((72, 100), "Testo normale", fontsize=11) + pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 40, 40), False) + pix.set_rect(pix.irect, (10, 10, 10)) + pg.insert_image(pymupdf.Rect(300, 300, 300 + 24, 300 + height), pixmap=pix) + w.doc = d + w.chk_img.setChecked(True) + w.apply_ai_to_page(0, []) + + +@pytest.mark.parametrize("height", [3, 6, 20]) +@pytest.mark.parametrize("mode", [PrivacyMode.ANONYMIZE, PrivacyMode.PSEUDONYMIZE]) +def test_image_label_is_kept_even_in_a_tiny_box(window, tmp_path, monkeypatch, height, mode): + w = window + _tiny_area_doc(w, height) + assert len(_redacts(w)) == 1 + out = tmp_path / "i.pdf" + files = [str(out)] + ([str(tmp_path / "m.nullifypdf-map")] if mode == PrivacyMode.PSEUDONYMIZE else []) + _stub_dialogs(monkeypatch, files) + monkeypatch.setattr(w, "_select_privacy_export", lambda: mode) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Testo normale" in text + assert any(lab in text for lab in ("[IMMAGINE RIMOSSA]", "[IMMAGINE]", "[IMG]", "#")), text + + +def test_signature_label_in_a_small_area(window, tmp_path, monkeypatch): + w = window + w.doc.close() + d = pymupdf.open() + d.new_page().insert_text((72, 100), "Testo", fontsize=11) + w.doc = d + w._add_kind_redaction(d[0], pymupdf.Rect(200, 200, 212, 204), "SIGNATURE") + out = tmp_path / "s.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert any(lab in text for lab in ("[FIRMA RIMOSSA]", "[FIRMA]", "[F]", "#")), text + + +def test_export_through_the_app_reports_no_survivor_for_clamped_bands(window, tmp_path, monkeypatch): + w = window + _tight_doc(w, 8.0) + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + out = tmp_path / "c.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + seen = {} + import export_engine as ee + real = ee.export_document + + def spy(*a, **k): + r = real(*a, **k) + seen["r"] = r + return r + + monkeypatch.setattr(app, "export_document", spy) + w.cmd_export() + assert seen["r"].surviving_glyphs == 0 and seen["r"].verification.ok + + +def test_italian_model_prompt_offers_consent_and_fallback(window, monkeypatch): + """Without the pipeline the window asks; declining keeps a rules-only scan.""" + import spacy_models + + monkeypatch.setattr(spacy_models, "_is_package", lambda: False) + w = window + root = spacy_models.models_root(w.config_dir) + + class FakeBox: + choice = "cancel" + + def __init__(self, *a, **k): + self.buttons = {} + + def __getattr__(self, name): + return lambda *a, **k: None + + def addButton(self, text, role): + self.buttons[text] = object() + return self.buttons[text] + + def exec(self): + return 0 + + def clickedButton(self): + return self.buttons.get(FakeBox.choice) + + for attr in ("Icon", "ButtonRole"): + setattr(FakeBox, attr, getattr(QMessageBox, attr)) + monkeypatch.setattr(app, "QMessageBox", FakeBox) + + FakeBox.choice = "Annulla" + assert w._prepare_italian_model() is False + + FakeBox.choice = "Continua senza modello" + assert w._prepare_italian_model() is True + assert w.ai_worker.italian_model.endswith("blank_it") + assert (root / "blank_it" / "meta.json").is_file() + + +def test_about_dialog_shows_licence_notice_with_links(window, monkeypatch): + from PySide6.QtWidgets import QTextBrowser + + import licences_info + + seen = {} + + def fake_exec(self): + view = self.findChild(QTextBrowser, "about_licences") + seen["html"] = view.toHtml() + seen["links"] = view.toPlainText() + return 0 + + monkeypatch.setattr(app.QDialog, "exec", fake_exec) + window.cmd_about() + assert "AGPL-3.0-or-later" in seen["links"] and "PyMuPDF" in seen["links"] + assert all(c.url in seen["html"] for c in licences_info.COMPONENTS) diff --git a/tests/test_gui_help.py b/tests/test_gui_help.py new file mode 100644 index 0000000..1721453 --- /dev/null +++ b/tests/test_gui_help.py @@ -0,0 +1,141 @@ +"""Guide and tooltips: every GUI element is explained, and the guide matches the UI.""" + +import os +import re + +import pytest + +pytest.importorskip("PySide6") +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import ( # noqa: E402 + QAbstractButton, QApplication, QCheckBox, QDialog, QPushButton, QRadioButton, +) + +import NullifyPDF as app # noqa: E402 +from help_guide import ( # noqa: E402 + ENTITY_TOOLTIPS, GUIDE_TOPICS, TOOLTIPS, GuideDialog, apply_tooltips, + widgets_without_tooltip, +) +from pii_detection import ENTITY_GROUPS # noqa: E402 +from review_dialog import ReviewDialog # noqa: E402 + + +@pytest.fixture() +def window(tmp_path, monkeypatch): + import atexit + import gc + + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("HOME", str(home)) + qapp = QApplication.instance() or QApplication([]) + w = app.NullifyPDF() + yield w + try: + w._cleanup_resources() + except Exception: + pass + atexit.unregister(w._cleanup_on_exit) + w.deleteLater() + qapp.processEvents() + gc.collect() + + +def _plain(html): + return re.sub(r"<[^>]+>", " ", html) + + +def test_every_widget_of_the_main_window_has_a_tooltip(window): + assert widgets_without_tooltip(window) == [] + + +def test_buttons_have_tooltips_too(window): + buttons = [b for b in window.findChildren(QAbstractButton)] + assert len(buttons) >= 15 + assert all(b.toolTip().strip() for b in buttons), [b.text() for b in buttons if not b.toolTip()] + + +def test_tooltips_are_in_italian_and_not_empty(): + for key, tip in {**TOOLTIPS, **ENTITY_TOOLTIPS}.items(): + assert len(tip) > 10 and tip[0].isupper(), key + + +def test_every_entity_checkbox_has_a_tooltip(): + labels = {label for _gid, label, _t in ENTITY_GROUPS} + assert labels <= set(ENTITY_TOOLTIPS), labels - set(ENTITY_TOOLTIPS) + + +def test_entity_dialog_has_tooltips(window, monkeypatch): + seen = {} + + def fake_exec(self): + seen["missing"] = widgets_without_tooltip(self) + seen["boxes"] = [c.toolTip() for c in self.findChildren(QCheckBox)] + return QDialog.DialogCode.Rejected + + monkeypatch.setattr(QDialog, "exec", fake_exec) + window._select_ai_entities() + assert seen["missing"] == [] and all(seen["boxes"]) + + +def test_review_and_dictionary_dialogs_have_tooltips(window): + from dictionary_dialog import DictionaryDialog + + items = [{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, "page": 0, "reasons": []}] + assert widgets_without_tooltip(ReviewDialog(items)) == [] + assert widgets_without_tooltip(DictionaryDialog(window.list_manager)) == [] + + +def test_about_dialog_has_tooltips(window, monkeypatch): + seen = {} + + def fake_exec(self): + seen["missing"] = widgets_without_tooltip(self) + return QDialog.DialogCode.Rejected + + monkeypatch.setattr(QDialog, "exec", fake_exec) + window.cmd_about() + assert seen["missing"] == [] + + +def test_guide_dialog_lists_every_topic_and_has_tooltips(window): + g = GuideDialog(window) + assert g.topics.count() == len(GUIDE_TOPICS) >= 10 + assert widgets_without_tooltip(g) == [] + # No hover popups on the topic list and the text: they get in the way of reading. + assert g.topics.toolTip() == "" and g.text.toolTip() == "" + assert g.topics.viewport().toolTip() == "" and g.text.viewport().toolTip() == "" + for row in range(g.topics.count()): + g.topics.setCurrentRow(row) + assert len(g.text.toPlainText()) > 80 + + +def test_f1_and_the_guide_button_open_the_guide(window, monkeypatch): + opened = [] + monkeypatch.setattr(GuideDialog, "exec", lambda self: opened.append(self.topics.count()) or 0) + window._f1.activated.emit() + btn = next(b for b in window.findChildren(QPushButton) if b.text() == "Guida") + btn.click() + assert len(opened) == 2 + + +def test_the_guide_describes_every_control_of_the_sidebar(window): + text = _plain(" ".join(html for _t, html in GUIDE_TOPICS)) + for w in window.findChildren(QAbstractButton): + label = w.text().strip() + if not label or label in {"-", "+", "<", ">", "Esci"} or label.startswith("Rivedi suggerimenti"): + continue + assert label in text, f"the guide never mentions ยซ{label}ยป" + assert "Rivedi suggerimenti" in text and "F1" in text + for rb in window.findChildren(QRadioButton): + assert rb.text() in text + + +def test_the_guide_matches_what_the_program_really_does(): + text = _plain(" ".join(html for _t, html in GUIDE_TOPICS)) + for fact in ("scadenza", "chiave di recupero", "mappa di ripristino", + "Ctrl + rotella", "Revisione", "OCR", "pseudonim"): + assert fact.lower() in text.lower(), fact + assert "Wikidata" not in text and "AI" not in text.split() # no internals / no AI wording diff --git a/tests/test_licences.py b/tests/test_licences.py new file mode 100644 index 0000000..2cea023 --- /dev/null +++ b/tests/test_licences.py @@ -0,0 +1,64 @@ +"""Licence consistency: AGPL project, no non-commercial model in the installer.""" + +import pathlib +import re + +ROOT = pathlib.Path(__file__).resolve().parent.parent + + +def _read(name: str) -> str: + return (ROOT / name).read_text(encoding="utf8") + + +def test_license_file_is_agpl(): + text = _read("LICENSE") + assert "GNU AFFERO GENERAL PUBLIC LICENSE" in text and "Version 3, 19 November 2007" in text + assert "AGPL-3.0-or-later" in text + + +def test_installer_never_bundles_the_non_commercial_pipeline(): + src = _read("build_local.py") + for line in src.splitlines(): + if "collect_all" in line or line.startswith("for pkg in"): + assert "it_core_news_md" not in line + assert src.count("excludes=['it_core_news_md'") >= 2 + assert "pymupdf4llm" in src + + +def test_docs_state_the_licence(): + assert "AGPL" in _read("README.md") + notices = _read("THIRD_PARTY_NOTICES.md") + for name in ("PyMuPDF", "PySide6", "it_core_news_md", "Tesseract"): + assert name in notices + assert re.search(r"AGPL-3\.0-or-later", _read("build_local.py")) + + +def test_no_declared_non_commercial_dependency_is_bundled(): + """Every installed distribution that declares a non-commercial licence must + be on the (empty-by-default) list of components kept out of the installer.""" + from importlib import metadata + + kept_out = {"it-core-news-md", "it_core_news_md"} + offenders = [] + for dist in metadata.distributions(): + name = (dist.metadata["Name"] or "").lower().replace("_", "-") + lic = " ".join(filter(None, [dist.metadata.get("License"), dist.metadata.get("License-Expression")])) + if re.search(r"\bNC\b|non-?commercial", lic, re.I) and name not in kept_out: + offenders.append((name, lic)) + assert not offenders, offenders + + +def test_component_list_is_https_complete_and_in_readme(): + import licences_info as li + + readme = _read("README.md") + names = {c.name for c in li.COMPONENTS} + assert {"PyMuPDF / MuPDF", "PySide6 / Qt", "spaCy it_core_news_md", "Tesseract OCR"} <= names + for c in li.COMPONENTS: + assert c.url.startswith("https://"), c + assert c.url in readme, c.name + assert c.role_en, c.name + assert li.readme_markdown() in readme + html_text = li.about_html() + assert all(c.url in html_text for c in li.COMPONENTS) + assert "non commerciale" in li.NOTICE diff --git a/tests/test_model_dialog.py b/tests/test_model_dialog.py new file mode 100644 index 0000000..59e135a --- /dev/null +++ b/tests/test_model_dialog.py @@ -0,0 +1,132 @@ +import os + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import QApplication + +from model_dialog import ModelDialog +from model_manager import MODEL_CANDIDATES + + +def test_model_dialog_lists_only_curated_candidates(): + app = QApplication.instance() or QApplication([]) + dialog = ModelDialog(MODEL_CANDIDATES) + assert dialog.list.count() == len(MODEL_CANDIDATES) + assert not dialog.btn_download.isEnabled() + assert dialog.details.toPlainText() + dialog.close() + app.processEvents() + + +# --- download progress lives inside the button -------------------------------- + +import time # noqa: E402 + +import pytest # noqa: E402 +from PySide6.QtWidgets import QMessageBox # noqa: E402 + +from model_manager import ModelFile, ModelManifest # noqa: E402 + + +def _manifest(): + return ModelManifest( + model_id="fake-model", repo_id="owner/fake", revision="a" * 40, license="MIT", + runtime="onnx", labels=("PER",), + files=(ModelFile("model.onnx", 1000, "b" * 64),), + ) + + +class FakeStore: + def __init__(self, behaviour): + self.behaviour = behaviour + self.installed = False + + def is_installed(self, manifest): + return self.installed + + def install_file(self, manifest, item, url, *, timeout=30.0, progress=None): + self.behaviour(self, progress, item) + + +def _dialog(store, qapp): + manifest = _manifest() + dlg = ModelDialog(MODEL_CANDIDATES, manifests=[manifest], store=store) + dlg._manifests = {c.model_id: manifest for c in MODEL_CANDIDATES} + dlg.list.setCurrentRow(0) + dlg.btn_download.setEnabled(True) + return dlg + + +def _wait(qapp, cond, timeout=5.0): + end = time.time() + timeout + while time.time() < end and not cond(): + qapp.processEvents() + time.sleep(0.01) + return cond() + + +@pytest.fixture() +def qapp(): + return QApplication.instance() or QApplication([]) + + +@pytest.fixture() +def boxes(monkeypatch): + seen = [] + monkeypatch.setattr(QMessageBox, "question", lambda *a, **k: QMessageBox.StandardButton.Yes) + monkeypatch.setattr(QMessageBox, "information", lambda *a, **k: seen.append("info")) + monkeypatch.setattr(QMessageBox, "critical", lambda *a, **k: seen.append(("error", a[2]))) + return seen + + +def test_download_progress_is_in_the_button_and_ui_is_locked(qapp, boxes): + gate = {"go": False} + + def slow(store, progress, item): + progress(500, item.size) + while not gate["go"]: + time.sleep(0.01) + progress(1000, item.size) + store.installed = True + + dlg = _dialog(FakeStore(slow), qapp) + dlg.btn_download.click() + assert dlg.is_downloading() + assert _wait(qapp, lambda: "50%" in dlg.btn_download.text()) + assert "qlineargradient" in dlg.btn_download.styleSheet() + assert not dlg.list.isEnabled() + dlg.reject() # closing is refused while downloading + assert dlg.is_downloading() + gate["go"] = True + assert _wait(qapp, lambda: not dlg.is_downloading()) + assert boxes == ["info"] and dlg.list.isEnabled() and dlg.btn_download.styleSheet() == "" + dlg.close() + + +def test_second_click_cancels_and_restores_button(qapp, boxes): + def endless(store, progress, item): + while True: + progress(1, item.size) # raises InterruptedError once cancelled + time.sleep(0.01) + + dlg = _dialog(FakeStore(endless), qapp) + dlg.btn_download.click() + assert _wait(qapp, lambda: dlg.is_downloading() and dlg.btn_download.text().startswith("Download")) + dlg.btn_download.click() + assert _wait(qapp, lambda: not dlg.is_downloading()) + assert boxes == [] # cancelling is silent + assert dlg.btn_download.isEnabled() and dlg.list.isEnabled() + assert dlg.btn_download.text() == "Scarica modello" + dlg.close() + + +def test_failed_download_reports_error_and_unlocks(qapp, boxes): + def broken(store, progress, item): + raise RuntimeError("hash non corrispondente") + + dlg = _dialog(FakeStore(broken), qapp) + dlg.btn_download.click() + assert _wait(qapp, lambda: not dlg.is_downloading()) + assert boxes and boxes[0][0] == "error" and "hash" in boxes[0][1] + assert dlg.list.isEnabled() and dlg.btn_download.styleSheet() == "" + dlg.close() diff --git a/tests/test_model_manager.py b/tests/test_model_manager.py new file mode 100644 index 0000000..f99ff10 --- /dev/null +++ b/tests/test_model_manager.py @@ -0,0 +1,75 @@ +from pathlib import Path + +import pytest + +from model_manager import ( + HardwareProfile, + ModelFile, + ModelManifest, + ModelManifestError, + ModelStore, + recommend_level, + verify_model_files, +) + + +def _manifest(data: bytes = b"model") -> ModelManifest: + import hashlib + + return ModelManifest( + model_id="test-model", + repo_id="example/model", + revision="0123456789abcdef0123456789abcdef01234567", + license="MIT", + runtime="onnx", + labels=("PERSON",), + files=(ModelFile("model.onnx", len(data), hashlib.sha256(data).hexdigest()),), + enabled=True, + ) + + +def test_manifest_rejects_short_revision_and_unsafe_file(): + m = _manifest() + with pytest.raises(ModelManifestError): + ModelManifest(**{**m.__dict__, "revision": "abc"}).validate() + with pytest.raises(ModelManifestError): + ModelFile("../model.onnx", 1, "0" * 64).validate() + + +def test_verify_model_files_checks_size_and_hash(tmp_path: Path): + m = _manifest() + model_dir = tmp_path / m.model_id / m.revision + model_dir.mkdir(parents=True) + (model_dir / "model.onnx").write_bytes(b"model") + verify_model_files(model_dir, m) + (model_dir / "model.onnx").write_bytes(b"tampered") + with pytest.raises(ModelManifestError, match="dimensione inattesa|hash"): + verify_model_files(model_dir, m) + + +def test_store_is_installed_requires_complete_verified_files(tmp_path: Path): + m = _manifest() + store = ModelStore(tmp_path) + assert not store.is_installed(m) + target = store.model_dir(m) + target.mkdir(parents=True) + (target / "model.onnx").write_bytes(b"model") + assert store.is_installed(m) + + +def test_recommendation_does_not_enable_unverified_catalog(): + hw = HardwareProfile(cpu_count=12, memory_bytes=16 * 1024**3) + assert recommend_level(hw, ()).level == "nessuno" + low = HardwareProfile(cpu_count=12, memory_bytes=4 * 1024**3) + assert recommend_level(low, (_manifest(),)).model_id is None + + +def test_manifest_rejects_non_model_extensions(): + with pytest.raises(ModelManifestError): + ModelFile("training_args.bin", 1, "0" * 64).validate() + + +def test_manifest_source_url_is_pinned(): + manifest = _manifest() + assert manifest.revision in manifest.source_url + assert manifest.source_url.startswith("https://huggingface.co/") diff --git a/tests/test_privacy_hardening.py b/tests/test_privacy_hardening.py new file mode 100644 index 0000000..b5ec5ad --- /dev/null +++ b/tests/test_privacy_hardening.py @@ -0,0 +1,261 @@ +"""Restore-map hardening and pseudonym styles.""" + +import base64 +import json +import os + +import pytest + +pytest.importorskip("cryptography") + +import privacy_core as pc + + +def _payload(): + reg = pc.PlaceholderRegistry() + reg.placeholder_for( + "Mario Rossi", "PERSON", page=0, rect=(1, 2, 3, 4), + style={"font": "x", "size": 10.0, "color": 0, "flags": 0, + "origin": [1.0, 2.0], "text": "Mario Rossi"}, + ) + return pc.build_restore_payload( + source_name="C:/docs/Referto_Mario_Rossi.pdf", source_sha256="a", + output_sha256="b", entries=reg.entries(), + ) + + +def test_payload_omits_source_name_and_duplicated_style_text(): + payload = _payload() + assert payload["source_name"] == "" + occ = payload["entries"][0]["occurrences"][0] + assert "text" not in occ["style"] + assert pc.parse_restore_entries(payload)[0].original == "Mario Rossi" + kept = pc.build_restore_payload( + source_name="a/b.pdf", source_sha256="", output_sha256=None, + entries=[], keep_source_name=True, + ) + assert kept["source_name"] == "b.pdf" + + +def test_style_text_is_kept_when_it_differs(): + reg = pc.PlaceholderRegistry() + reg.placeholder_for("Mario", "PERSON", rect=(0, 0, 1, 1), + style={"font": "", "size": 9.0, "origin": [0, 0], "text": "Mario Rossi"}) + occ = pc.build_restore_payload( + source_name="", source_sha256="", output_sha256=None, entries=reg.entries() + )["entries"][0]["occurrences"][0] + assert occ["style"]["text"] == "Mario Rossi" + + +def test_new_maps_are_v3_and_roundtrip(): + blob = pc.encrypt_restore_payload(_payload(), "una frase molto lunga 2026") + env = json.loads(blob) + assert env["version"] == 3 and env["cipher"] == "AES-256-GCM" + assert [w["type"] for w in env["wraps"]] == ["password"] + assert env["wraps"][0]["kdf"] in ("argon2id", "scrypt") + assert pc.decrypt_restore_payload(blob, "una frase molto lunga 2026")["format"] == pc.RESTORE_MAP_FORMAT + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(blob, "una frase sbagliata 2026!") + + +def test_v2_scrypt_maps_still_decrypt(): + blob = pc._encrypt_v2(_payload(), "una frase molto lunga 2026") + assert json.loads(blob)["version"] == 2 + assert pc.decrypt_restore_payload(blob, "una frase molto lunga 2026")["version"] == pc.RESTORE_MAP_VERSION + + +def _legacy_pbkdf2_blob(password, iterations): + from cryptography.fernet import Fernet + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + + salt = b"0123456789abcdef" + key = base64.urlsafe_b64encode( + PBKDF2HMAC(hashes.SHA256(), 32, salt, iterations).derive(password.encode()) + ) + token = Fernet(key).encrypt(json.dumps(_payload()).encode()) + return json.dumps({ + "format": "NullifyPDF encrypted restore map", "version": 1, + "kdf": "PBKDF2-HMAC-SHA256", "iterations": iterations, + "salt": base64.b64encode(salt).decode(), "token": token.decode(), + }).encode() + + +def test_legacy_pbkdf2_maps_still_decrypt(): + blob = _legacy_pbkdf2_blob("legacy password 123", 600000) + assert pc.decrypt_restore_payload(blob, "legacy password 123")["version"] == pc.RESTORE_MAP_VERSION + + +@pytest.mark.parametrize("iterations", [1, 99_999, 5_000_001, 10 ** 12]) +def test_untrusted_iteration_counts_are_rejected_before_derivation(iterations): + blob = _legacy_pbkdf2_blob("legacy password 123", 600000) + env = json.loads(blob) + env["iterations"] = iterations + with pytest.raises(ValueError): + pc.decrypt_restore_payload(json.dumps(env).encode(), "legacy password 123") + + +@pytest.mark.parametrize("n,r,p", [(2 ** 30, 8, 1), (1000, 8, 1), (2 ** 17, 1000, 1), (2 ** 17, 8, 99)]) +def test_untrusted_scrypt_parameters_are_rejected(n, r, p): + env = json.loads(pc._encrypt_v2(_payload(), "una frase molto lunga 2026")) + env.update(n=n, r=r, p=p) + with pytest.raises(ValueError): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + + +def test_unknown_kdf_is_rejected(): + env = json.loads(pc._encrypt_v2(_payload(), "una frase molto lunga 2026")) + env["kdf"] = "md5" + with pytest.raises(ValueError): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + + +@pytest.mark.parametrize("pw,ok", [ + ("short1!", False), + ("aaaaaaaaaaaaaaaa", False), + ("passwordpassword1", False), + ("abcdefghijklm", False), + ("Abcdefghijklm", True), + ("una frase molto lunga davvero", True), + ("Xk3!mP9#qR2$", True), +]) +def test_password_strength(pw, ok): + assert pc.password_strength(pw)[0] is ok + + +def test_encrypt_still_requires_minimum_length(): + with pytest.raises(ValueError): + pc.encrypt_restore_payload(_payload(), "short") + + +def test_counter_style_is_unchanged(): + reg = pc.PlaceholderRegistry() + assert reg.placeholder_for("A Uno", "PERSON") == "PERSON_001" + assert reg.placeholder_for("B Due", "PERSON") == "PERSON_002" + assert reg.placeholder_for("a uno", "PERSON") == "PERSON_001" + + +def test_random_style_reveals_neither_order_nor_count(): + values = [f"Persona Numero{i}" for i in range(30)] + a, b = pc.PlaceholderRegistry("random"), pc.PlaceholderRegistry("random") + pa = [a.placeholder_for(v, "PERSON") for v in values] + pb = [b.placeholder_for(v, "PERSON") for v in values] + assert len(set(pa)) == 30 + assert pa != pb # fresh randomness per document + assert all(p.startswith("PERSON_") and len(p) >= len("PERSON_") + 4 for p in pa) + assert pa == [a.placeholder_for(v, "PERSON") for v in values] # stable within a document + assert pa != sorted(pa) or pb != sorted(pb) + + +def test_opaque_style_hides_the_type(): + reg = pc.PlaceholderRegistry("opaque") + assert reg.placeholder_for("Mario", "PERSON").startswith("DATA_") + assert reg.entries()[0].entity_type == "PERSON" # kept inside the encrypted map + + +def test_keyed_style_is_consistent_per_key_only(): + k1, k2 = b"k" * 32, b"j" * 32 + a = pc.PlaceholderRegistry("keyed", k1).placeholder_for("Mario Rossi", "PERSON") + b = pc.PlaceholderRegistry("keyed", k1).placeholder_for("mario rossi", "PERSON") + c = pc.PlaceholderRegistry("keyed", k2).placeholder_for("Mario Rossi", "PERSON") + assert a == b and a != c + assert pc.PlaceholderRegistry("keyed", k1).placeholder_for("Mario Rossi", "LOCATION") != a + with pytest.raises(ValueError): + pc.PlaceholderRegistry("keyed") + with pytest.raises(ValueError): + pc.PlaceholderRegistry("nonsense") + + +def test_unknown_or_foreign_envelope_gives_a_clear_error(): + blob = pc._encrypt_v2(_payload(), "una frase molto lunga 2026") + env = json.loads(blob) + env["version"] = 9 + with pytest.raises(ValueError, match="non supportata"): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + env["format"] = "something else" + with pytest.raises(ValueError, match="non riconosciuto"): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + + +# --- envelope v3: recovery key, Argon2id, key export ----------------------------- + +PW = "una frase molto lunga 2026" + + +def test_v3_password_and_recovery_open_the_same_payload(): + raw, shown = pc.generate_recovery_key() + blob = pc.encrypt_restore_payload(_payload(), PW, recovery_key=raw) + assert [w["type"] for w in json.loads(blob)["wraps"]] == ["password", "recovery"] + by_pw = pc.decrypt_restore_payload(blob, PW) + by_rk = pc.decrypt_restore_payload(blob, None, recovery_key=pc.parse_recovery_key(shown)) + assert by_pw == by_rk + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(blob, None, recovery_key=os.urandom(32)) + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(blob) # nothing supplied + + +def test_recovery_key_format_and_checksum(): + raw, shown = pc.generate_recovery_key() + assert shown.startswith("NPDF-") and len(shown.replace("-", "")) == 4 + 54 + assert pc.parse_recovery_key(shown.lower().replace("-", " ")) == raw # forgiving of case/spacing + typo = shown[:8] + ("A" if shown[8] != "A" else "B") + shown[9:] + with pytest.raises(ValueError, match="digitata|non valida"): + pc.parse_recovery_key(typo) + with pytest.raises(ValueError): + pc.parse_recovery_key("NPDF-ABCD") + + +def test_v3_wrap_type_swap_fails(): + raw, _ = pc.generate_recovery_key() + env = json.loads(pc.encrypt_restore_payload(_payload(), PW, recovery_key=raw)) + env["wraps"][1]["type"] = "password" # pretend the recovery wrap is a password wrap + env["wraps"].pop(0) + from cryptography.fernet import InvalidToken + with pytest.raises((InvalidToken, KeyError, ValueError)): + pc.decrypt_restore_payload(json.dumps(env).encode(), PW) + + +def test_v3_tampering_is_detected(): + env = json.loads(pc.encrypt_restore_payload(_payload(), PW)) + ct = bytearray(base64.b64decode(env["ciphertext"])) + ct[0] ^= 1 + env["ciphertext"] = base64.b64encode(bytes(ct)).decode() + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(json.dumps(env).encode(), PW) + + +@pytest.mark.skipif(not pc._argon2_available(), reason="Argon2id not available in this OpenSSL") +@pytest.mark.parametrize("field,value", [ + ("t", 0), ("t", 11), ("m", 1024), ("m", 2 ** 21), ("p", 0), ("p", 9), +]) +def test_argon2id_bounds_are_enforced_before_derivation(field, value): + env = json.loads(pc.encrypt_restore_payload(_payload(), PW)) + assert env["wraps"][0]["kdf"] == "argon2id" + env["wraps"][0][field] = value + with pytest.raises(ValueError, match="limiti"): + pc.decrypt_restore_payload(json.dumps(env).encode(), PW) + + +def test_argon2_unavailable_falls_back_to_scrypt_for_writing(monkeypatch): + monkeypatch.setattr(pc, "_argon2_available", lambda: False) + blob = pc.encrypt_restore_payload(_payload(), PW) + assert json.loads(blob)["wraps"][0]["kdf"] == "scrypt" + assert pc.decrypt_restore_payload(blob, PW)["format"] == pc.RESTORE_MAP_FORMAT + + +def test_pseudonym_key_export_import_roundtrip_and_wrong_password(): + key = os.urandom(32) + blob = pc.export_pseudonym_key(key, PW) + assert key not in blob and base64.b64encode(key) not in blob + assert pc.import_pseudonym_key(blob, PW) == key + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.import_pseudonym_key(blob, "un'altra frase lunga 2026") + with pytest.raises(ValueError): + pc.import_pseudonym_key(pc.encrypt_restore_payload(_payload(), PW), PW) # a map, not a key + with pytest.raises(ValueError): + pc.export_pseudonym_key(b"short", PW) diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py index 2448049..aa01a33 100644 --- a/tests/test_reconstruct.py +++ b/tests/test_reconstruct.py @@ -248,9 +248,11 @@ def test_reconstruct_v1_map_finds_wrapped_placeholder(tmp_path): assert "MarioRossi" in "".join(text.split()) -def test_reconstruct_does_not_report_values_it_failed_to_write(tmp_path): - """An original too long for its box is dropped by `apply_redactions()`. - That must never be counted as a restored value.""" +def test_reconstruct_shrinks_a_long_value_instead_of_dropping_it(tmp_path): + """An original too long for its box used to be dropped by + `apply_redactions()` (no text below 4 pt). It is now written after the + redactions with TextWriter, so it is back in the file (and counted only + because it really is).""" from NullifyPDF import sha256_file doc = fitz.open() @@ -276,8 +278,10 @@ def test_reconstruct_does_not_report_values_it_failed_to_write(tmp_path): ], } out_path = tmp_path / "reconstructed.pdf" - # The value cannot fit the tight box; the honest answer is zero. - assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 0 + assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 1 + back = fitz.open(str(out_path)) + assert "UNBREAKABLE" in "".join(back[0].get_text().split()) + back.close() def test_expected_restore_count_counts_occurrences_not_entries(tmp_path): @@ -481,3 +485,34 @@ def test_embedded_font_is_actually_reused(tmp_path): ) span = _span_for(out, "Maria Bianchi") assert "arial" in span["font"].lower() + + +def test_compact_labels_chain_and_tiny_box_placeholder_survives(tmp_path): + """A box too small for the full placeholder still gets a legible label.""" + from NullifyPDF import compact_labels, write_label + + assert compact_labels("PERSON_9F2A") == ["PERSON_9F2A", "P9F2A", "9F2A", "#9F"] + doc = fitz.open() + page = doc.new_page() + page.insert_text((72, 100), "Re", fontsize=11) + rect = fitz.Rect(page.get_text("words")[0][:4]) + page.add_redact_annot(rect, fill=(1, 1, 1)) + page.apply_redactions() + drawn = write_label(page, rect, compact_labels("PERSON_9F2A")) + assert drawn != "PERSON_9F2A" + assert drawn in page.get_text() + + +def test_v4_map_keeps_label_and_v3_still_loads(): + from privacy_core import parse_restore_entries + + reg = PlaceholderRegistry() + ph = reg.placeholder_for("Re", "PERSON", rect=(1, 2, 3, 4)) + reg.set_label(ph, (1, 2, 3, 4), "P001") + payload = build_restore_payload( + source_name="", source_sha256="", output_sha256=None, entries=reg.entries() + ) + assert payload["version"] == 4 + assert parse_restore_entries(payload)[0].occurrences[0].label == "P001" + payload["version"] = 3 + assert parse_restore_entries(payload)[0].original == "Re" diff --git a/tests/test_redaction_engine.py b/tests/test_redaction_engine.py new file mode 100644 index 0000000..4c098ba --- /dev/null +++ b/tests/test_redaction_engine.py @@ -0,0 +1,50 @@ +"""Regression tests for the PDF engine's redaction guarantees.""" + +import pymupdf +import pytest + +from NullifyPDF import PDF_REDACT_IMAGE_PIXELS + +SECRETS = ("SECRET-ONE", "SECRET-TWO", "SECRET-THREE") + + +def _pdf_with_quote_operator(path): + """Build a page whose text is shown with the ``'`` operator after TL/Td.""" + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 60), "Header line", fontsize=11) + font = page.get_fonts(full=True)[0][4] + stream = ( + f"BT /{font} 12 Tf 14 TL 72 700 Td (SECRET-ONE 123-45-6789) Tj " + "0 -30 Td (SECRET-TWO mario@example.com) ' (SECRET-THREE iban) ' ET\n" + ).encode() + xref = page.get_contents()[0] + doc.update_stream(xref, doc.xref_stream(xref) + b"\n" + stream) + doc.save(path) + doc.close() + + +def test_redaction_removes_text_shown_with_quote_operator(tmp_path): + src = tmp_path / "src.pdf" + out = tmp_path / "out.pdf" + _pdf_with_quote_operator(str(src)) + + doc = pymupdf.open(str(src)) + page = doc[0] + for secret in SECRETS: + for rect in page.search_for(secret): + page.add_redact_annot(rect + (-2, -2, 150, 2)) + page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) + doc.save(str(out), garbage=4, deflate=True, clean=True) + doc.close() + + result = pymupdf.open(str(out)) + assert "SECRET" not in result[0].get_text() + leaked = [ + xref + for xref in range(1, result.xref_length()) + if result.xref_is_stream(xref) + and b"SECRET" in (result.xref_stream(xref) or b"") + ] + assert leaked == [], f"redacted text still present in streams {leaked}" + result.close() diff --git a/tests/test_secure_store.py b/tests/test_secure_store.py new file mode 100644 index 0000000..ebfd2f4 --- /dev/null +++ b/tests/test_secure_store.py @@ -0,0 +1,243 @@ +"""Encrypted word lists with expiry, and the key-protection layer.""" + +import datetime +import json +import os +import sys + +import pytest + +pytest.importorskip("cryptography") + +import secure_store +from word_lists import ListEntry, PDFListManager + + +@pytest.fixture(autouse=True) +def _fresh_cache(): + secure_store.forget_cached_keys() + yield + secure_store.forget_cached_keys() + + +def test_seal_open_roundtrip_and_aad_binding(): + key = os.urandom(32) + blob = secure_store.seal(b"dati", key, b"ctx-a") + assert secure_store.open_sealed(blob, key, b"ctx-a") == b"dati" + with pytest.raises(secure_store.SealError): + secure_store.open_sealed(blob, key, b"ctx-b") + with pytest.raises(secure_store.SealError): + secure_store.open_sealed(blob[:-1] + bytes([blob[-1] ^ 1]), key, b"ctx-a") + with pytest.raises(secure_store.SealError): + secure_store.open_sealed(b"short", key, b"ctx-a") + + +@pytest.mark.skipif(sys.platform != "win32", reason="DPAPI is Windows-only") +def test_dpapi_roundtrip_and_account_binding(tmp_path): + p = secure_store.DpapiProtector() + key = os.urandom(32) + assert p.unwrap(p.wrap(key)) == key + assert key not in json.dumps(p.wrap(key)).encode() + + +def test_keyfile_protector_creates_private_file(tmp_path): + p = secure_store.KeyFileProtector(tmp_path) + key = os.urandom(32) + blob = p.wrap(key) + assert p.unwrap(blob) == key + mode = (tmp_path / "keys" / "master.key").stat().st_mode & 0o777 + if sys.platform != "win32": + assert mode == 0o600 + + +def test_master_key_is_stable_and_not_stored_in_clear_in_keystore(tmp_path): + k1 = secure_store.load_or_create_master_key(tmp_path) + secure_store.forget_cached_keys() + k2 = secure_store.load_or_create_master_key(tmp_path) + assert k1 == k2 and len(k1) == 32 + ks = (tmp_path / "keystore.json").read_bytes() + if json.loads(ks)["master"]["protector"] == "dpapi": + assert k1 not in ks + assert "DPAPI" in secure_store.protector_label(tmp_path) + + +def test_keyring_unavailable_falls_back_to_keyfile(tmp_path, monkeypatch): + monkeypatch.setattr(secure_store, "DpapiProtector", type("D", (), {"available": staticmethod(lambda: False)})) + monkeypatch.setattr(secure_store.KeyringProtector, "available", staticmethod(lambda: False)) + secure_store.load_or_create_master_key(tmp_path) + assert json.loads((tmp_path / "keystore.json").read_text())["master"]["protector"] == "keyfile" + + +def test_plaintext_lists_migrate_transparently(tmp_path): + (tmp_path / "blocklist.txt").write_text("mario rossi\nvia roma\nab", encoding="utf-8") + (tmp_path / "allowlist.txt").write_text("comune", encoding="utf-8") + m = PDFListManager(tmp_path) + assert m.load_blocklist() == {"mario rossi", "via roma"} + assert m.load_allowlist() == {"comune"} + assert not (tmp_path / "blocklist.txt").exists() + enc = (tmp_path / "blocklist.v2.enc").read_bytes() + assert b"rossi" not in enc and enc.startswith(b"NPDFL2\n") + assert PDFListManager(tmp_path).load_blocklist() == {"mario rossi", "via roma"} # reload + + +def test_save_roundtrip_filters_short_terms_and_keeps_dates(tmp_path): + day = {"now": datetime.date(2026, 1, 1)} + m = PDFListManager(tmp_path, today=lambda: day["now"]) + m.save_blocklist({"abc", "ab", "Mario Rossi"}) + assert m.load_blocklist() == {"abc", "mario rossi"} + day["now"] = datetime.date(2026, 3, 1) + m.save_blocklist({"abc", "nuovo termine"}) + entries = {e.term: e for e in m.load_entries("blocklist")} + assert entries["abc"].added == datetime.date(2026, 1, 1) # kept + assert entries["nuovo termine"].added == datetime.date(2026, 3, 1) + + +def test_expired_entries_are_dropped_and_purged(tmp_path): + day = {"now": datetime.date(2026, 1, 1)} + m = PDFListManager(tmp_path, today=lambda: day["now"]) + m.save_entries("blocklist", [ + ListEntry("scade presto", day["now"], datetime.date(2026, 1, 10)), + ListEntry("per sempre", day["now"], None), + ]) + day["now"] = datetime.date(2026, 1, 11) + assert m.load_blocklist() == {"per sempre"} + assert m.expired_removed == 1 + day["now"] = datetime.date(2025, 12, 1) # purged for good, not merely hidden + assert PDFListManager(tmp_path, today=lambda: day["now"]).load_blocklist() == {"per sempre"} + + +def test_default_ttl_applies_to_new_terms(tmp_path): + m = PDFListManager(tmp_path, today=lambda: datetime.date(2026, 1, 1)) + m.set_default_ttl_days(30) + m.save_blocklist({"nuovo termine"}) + e = m.load_entries("blocklist")[0] + assert e.expires == datetime.date(2026, 1, 31) + + +def test_unreadable_keystore_degrades_without_data_loss(tmp_path): + m = PDFListManager(tmp_path) + m.save_blocklist({"mario rossi"}) + before = (tmp_path / "blocklist.v2.enc").read_bytes() + ks = json.loads((tmp_path / "keystore.json").read_text()) + if ks["master"]["protector"] == "dpapi": + ks["master"]["blob"] = "AAAA" + else: + (tmp_path / "keys" / "master.key").write_bytes(b"x" * 5) + (tmp_path / "keystore.json").write_text(json.dumps(ks)) + secure_store.forget_cached_keys() + m2 = PDFListManager(tmp_path) + assert m2.load_blocklist() == set() + assert m2.keystore_error + m2.save_blocklist({"altro termine"}) # must not overwrite anything + assert (tmp_path / "blocklist.v2.enc").read_bytes() == before + + +def test_no_cryptography_keeps_plaintext(tmp_path, monkeypatch): + import builtins + + real = builtins.__import__ + + def fake(name, *a, **k): + if name == "cryptography" or name.startswith("cryptography."): + raise ImportError(name) + return real(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake) + m = PDFListManager(tmp_path) + m.save_blocklist({"mario rossi"}) + monkeypatch.setattr(builtins, "__import__", real) + assert (tmp_path / "blocklist.txt").read_text(encoding="utf-8") == "mario rossi" + assert PDFListManager(tmp_path).load_blocklist() == {"mario rossi"} + + +def test_tampered_list_file_is_not_trusted(tmp_path): + m = PDFListManager(tmp_path) + m.save_blocklist({"mario rossi"}) + p = tmp_path / "blocklist.v2.enc" + raw = bytearray(p.read_bytes()) + raw[-1] ^= 1 + p.write_bytes(bytes(raw)) + assert PDFListManager(tmp_path).load_blocklist() == set() + + +def test_keyring_backend_holds_the_master_key_when_available(tmp_path, monkeypatch): + """With a real (here: in-memory) keyring backend the key never touches the keystore file.""" + keyring = pytest.importorskip("keyring") + from keyring.backend import KeyringBackend + + class MemoryKeyring(KeyringBackend): + priority = 1 + store: dict = {} + + def set_password(self, service, username, password): + self.store[(service, username)] = password + + def get_password(self, service, username): + return self.store.get((service, username)) + + def delete_password(self, service, username): + self.store.pop((service, username), None) + + old = keyring.get_keyring() + keyring.set_keyring(MemoryKeyring()) + monkeypatch.delenv("NULLIFYPDF_DISABLE_KEYRING", raising=False) + secure_store.KeyringProtector._probe_result.clear() + try: + monkeypatch.setattr(secure_store.DpapiProtector, "available", staticmethod(lambda: False)) + key = secure_store.load_or_create_master_key(tmp_path) + ks = json.loads((tmp_path / "keystore.json").read_text()) + assert ks["master"]["protector"] == "keyring" + assert "portachiavi" in secure_store.protector_label(tmp_path) + import base64 + assert base64.b64encode(key).decode() not in json.dumps(ks) + assert not (tmp_path / "keys").exists() + secure_store.forget_cached_keys() + assert secure_store.load_or_create_master_key(tmp_path) == key + # losing the keyring entry = unreadable keystore, never silent re-creation + MemoryKeyring.store.clear() + secure_store.forget_cached_keys() + with pytest.raises(secure_store.KeystoreUnavailable): + secure_store.load_or_create_master_key(tmp_path) + finally: + keyring.set_keyring(old) + + +def test_a_keyring_that_never_answers_is_skipped_not_waited_for(tmp_path, monkeypatch): + """A locked Keychain waiting for a dialog must not hang the program.""" + import time + + keyring = pytest.importorskip("keyring") + from keyring.backend import KeyringBackend + + class HangingKeyring(KeyringBackend): + priority = 1 + + def set_password(self, service, username, password): + time.sleep(30) + + def get_password(self, service, username): + time.sleep(30) + + def delete_password(self, service, username): + time.sleep(30) + + old = keyring.get_keyring() + keyring.set_keyring(HangingKeyring()) + monkeypatch.delenv("NULLIFYPDF_DISABLE_KEYRING", raising=False) + monkeypatch.setattr(secure_store, "PROBE_TIMEOUT", 0.3) + monkeypatch.setattr(secure_store.DpapiProtector, "available", staticmethod(lambda: False)) + secure_store.KeyringProtector._probe_result.clear() + try: + t0 = time.time() + assert secure_store.KeyringProtector.available() is False + assert time.time() - t0 < 5 + secure_store.load_or_create_master_key(tmp_path) + assert json.loads((tmp_path / "keystore.json").read_text())["master"]["protector"] == "keyfile" + finally: + keyring.set_keyring(old) + secure_store.KeyringProtector._probe_result.clear() + + +def test_keyring_can_be_disabled_by_environment(monkeypatch): + monkeypatch.setenv("NULLIFYPDF_DISABLE_KEYRING", "1") + assert secure_store.KeyringProtector.available() is False diff --git a/tests/test_spacy_models.py b/tests/test_spacy_models.py new file mode 100644 index 0000000..f323266 --- /dev/null +++ b/tests/test_spacy_models.py @@ -0,0 +1,94 @@ +"""Optional Italian pipeline: verified download, safe unpacking, fallback.""" + +import hashlib +import io +import pathlib +import zipfile + +import pytest + +import spacy_models as sm + + +def _wheel(path: pathlib.Path, names) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name in names: + zf.writestr(name, b"x") + data = buf.getvalue() + path.write_bytes(data) + return data + + +def _serve(monkeypatch, data: bytes): + monkeypatch.setattr(sm.urllib.request, "urlopen", lambda url, timeout=0: io.BytesIO(data)) + + +def test_rejects_non_https(tmp_path): + with pytest.raises(sm.ModelDownloadError): + sm.install_italian(tmp_path, url="http://example.com/x.whl") + + +def test_rejects_wrong_hash_and_leaves_nothing(tmp_path, monkeypatch): + data = _wheel(tmp_path / "w.whl", []) + _serve(monkeypatch, data) + root = tmp_path / "root" + with pytest.raises(sm.ModelDownloadError): + sm.install_italian(root, url="https://example.com/x.whl", sha256="0" * 64, size=len(data)) + assert not sm.italian_installed(root) + assert not list(root.glob("*.whl")) + + +def test_rejects_oversized_download(tmp_path, monkeypatch): + _serve(monkeypatch, b"a" * 100) + with pytest.raises(sm.ModelDownloadError): + sm.install_italian(tmp_path / "r", url="https://e.com/x", sha256="0" * 64, size=10) + + +def test_rejects_path_traversal(tmp_path, monkeypatch): + p = sm._PIPELINE_DIR + data = _wheel(tmp_path / "w.whl", [p + "../../evil.txt"]) + _serve(monkeypatch, data) + with pytest.raises(sm.ModelDownloadError): + sm.install_italian( + tmp_path / "r", url="https://e.com/x", + sha256=hashlib.sha256(data).hexdigest(), size=len(data), + ) + assert not (tmp_path / "evil.txt").exists() + + +def test_installs_valid_package(tmp_path, monkeypatch): + p = sm._PIPELINE_DIR + data = _wheel(tmp_path / "w.whl", [p + "meta.json", p + "ner/cfg", "other/ignored.txt"]) + _serve(monkeypatch, data) + root = tmp_path / "r" + seen = [] + folder = sm.install_italian( + root, progress=lambda d, t: seen.append((d, t)), url="https://e.com/x", + sha256=hashlib.sha256(data).hexdigest(), size=len(data), + ) + assert sm.italian_installed(root) and (folder / "ner" / "cfg").is_file() + assert not (folder / "ignored.txt").exists() and seen + sm.remove_italian(root) + assert not sm.italian_installed(root) + + +def test_blank_fallback_is_not_statistical(tmp_path): + blank = sm.blank_italian(tmp_path) + assert blank.statistical is False and (pathlib.Path(blank.name) / "meta.json").is_file() + + +def test_resolve_prefers_installed_package_then_folder(tmp_path, monkeypatch): + monkeypatch.setattr(sm, "_is_package", lambda: False) + assert sm.resolve_italian(tmp_path) is None + d = tmp_path / f"{sm.ITALIAN_PACKAGE}-{sm.ITALIAN_VERSION}" + (d / "ner").mkdir(parents=True) + (d / "meta.json").write_text("{}") + assert sm.resolve_italian(tmp_path).name == str(d) + monkeypatch.setattr(sm, "_is_package", lambda: True) + assert sm.resolve_italian(tmp_path).name == sm.ITALIAN_PACKAGE + + +def test_licence_text_names_licence_and_source(): + t = sm.licence_text() + assert "CC BY-NC-SA 3.0" in t and "SHA-256" in t diff --git a/tests/test_update_check.py b/tests/test_update_check.py new file mode 100644 index 0000000..b5f1d7f --- /dev/null +++ b/tests/test_update_check.py @@ -0,0 +1,187 @@ +"""Tests for the update checker and the release-notes extractor.""" + +import pathlib +import sys + +import pytest + +import update_check as uc + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent / "scripts")) +from extract_release_notes import extract_notes # noqa: E402 + + +def rel(tag, prerelease=False, draft=False, assets=()): + return {"tag_name": tag, "prerelease": prerelease, "draft": draft, + "assets": list(assets)} + + +def test_version_ordering(): + v = uc.parse_version + assert v("2.3.0-beta.3") < v("2.3.0") < v("2.3.1-beta.1") < v("2.3.1") + assert v("v2.3.0-beta.9") < v("2.3.0-beta.10") + assert v("garbage") is None + + +def test_stable_build_ignores_betas(): + rels = [rel("v2.4.0-beta.1", True), rel("v2.3.1"), rel("v2.3.0")] + assert uc.find_update(rels, "2.3.0")["tag_name"] == "v2.3.1" + assert uc.find_update([rel("v2.4.0-beta.1", True)], "2.3.0") is None + + +def test_beta_build_sees_both_and_picks_highest(): + rels = [rel("v2.3.1"), rel("v2.4.0-beta.2", True), rel("v2.4.0-beta.1", True)] + assert uc.find_update(rels, "2.3.1-beta.1")["tag_name"] == "v2.4.0-beta.2" + # the stable that supersedes the running beta is offered + assert uc.find_update([rel("v2.3.0")], "2.3.0-beta.3")["tag_name"] == "v2.3.0" + + +def test_up_to_date_and_drafts(): + assert uc.find_update([rel("v2.3.0")], "2.3.0") is None + assert uc.find_update([rel("v9.0.0", draft=True)], "2.3.0") is None + + +@pytest.mark.parametrize("system,suffix", [ + ("Windows", "_Windows.exe"), ("Darwin", "_macOS.zip")]) +def test_asset_selection(system, suffix): + base = "https://github.com/overwrite00/NullifyPDF/releases/download/v2.3.1/" + names = ["NullifyPDF_v2.3.1_Windows.exe", "NullifyPDF_v2.3.1_macOS.zip"] + release = rel("v2.3.1", assets=[ + {"name": n, "browser_download_url": base + n} for n in names]) + assert uc.select_asset(release, system)["name"].endswith(suffix) + + +def test_untrusted_asset_url_rejected(): + release = rel("v2.3.1", assets=[{ + "name": "NullifyPDF_v2.3.1_Windows.exe", + "browser_download_url": "https://evil.example/NullifyPDF_v2.3.1_Windows.exe"}]) + assert uc.select_asset(release, "Windows") is None + assert not uc.is_trusted_url("http://github.com/x") + + +def test_filename_safety(tmp_path): + assert uc.safe_filename("../../evil.exe") == "evil.exe" + (tmp_path / "a.exe").write_bytes(b"x") + assert uc.unique_path(tmp_path, "a.exe").name == "a (1).exe" + + +CHANGELOG = """# Changelog + +## [Unreleased] + +- pending item + +## [2.3.0] - 2026-09-19 + +### Added +- stable item + +## [2.2.0] - 2026-09-18 + +- old item +""" + + +def test_notes_from_correct_section(): + assert extract_notes(CHANGELOG, "v2.3.0") == "### Added\n- stable item" + assert extract_notes(CHANGELOG, "v2.4.0-beta.1") == "- pending item" + assert extract_notes(CHANGELOG, "v9.9.9") == "" + + +def test_downloads_folder_resolves_without_side_effects(): + import NullifyPDF + assert NullifyPDF.downloads_folder().name + + +def test_auto_check_preference(tmp_path): + f = tmp_path / "sub" / "update_check.json" + assert uc.load_auto_check(f) is False # default off, missing file + uc.save_auto_check(f, True) + assert uc.load_auto_check(f) is True + uc.save_auto_check(f, False) + assert uc.load_auto_check(f) is False + f.write_text("garbage") + assert uc.load_auto_check(f) is False + + +# --- Linux family / download hardening --------------------------------------- + +@pytest.mark.parametrize("osrel,suffix", [ + ('ID=ubuntu\nID_LIKE=debian\n', "_Ubuntu.deb"), + ('ID=debian\n', "_Ubuntu.deb"), + ('ID="rocky"\nID_LIKE="rhel centos fedora"\n', "_Fedora.rpm"), + ('ID=fedora\n', "_Fedora.rpm"), + ('ID=opensuse-leap\nID_LIKE="suse opensuse"\n', None), + ('ID=arch\n', None), +]) +def test_linux_family_from_os_release(monkeypatch, osrel, suffix): + monkeypatch.setattr(pathlib.Path, "read_text", lambda self, **kw: osrel) + assert uc.asset_suffix_for_platform("Linux") == suffix + + +def test_unknown_distribution_gets_no_asset(monkeypatch): + monkeypatch.setattr(pathlib.Path, "read_text", lambda self, **kw: "ID=arch\n") + release = rel("v2.3.1", assets=[{ + "name": "NullifyPDF_v2.3.1_Ubuntu.deb", + "browser_download_url": "https://github.com/overwrite00/NullifyPDF/releases/download/v2.3.1/NullifyPDF_v2.3.1_Ubuntu.deb"}]) + assert uc.select_asset(release, "Linux") is None + + +class _FakeResp: + def __init__(self, data, url="https://github.com/x/NullifyPDF_v1_Windows.exe"): + self._d, self._url, self.headers = data, url, {"Content-Length": str(len(data))} + + def read(self, n): + out, self._d = self._d[:n], self._d[n:] + return out + + def geturl(self): + return self._url + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + +def _patch_open(monkeypatch, data, url="https://github.com/x/f.exe"): + monkeypatch.setattr(uc._OPENER, "open", lambda req, timeout=0: _FakeResp(data, url)) + + +def test_download_ok_and_digest_checked(tmp_path, monkeypatch): + import hashlib + data = b"installer" * 1000 + _patch_open(monkeypatch, data) + dest = tmp_path / "f.exe" + uc.download_file("https://github.com/x/f.exe", dest, len(data), + digest="sha256:" + hashlib.sha256(data).hexdigest()) + assert dest.read_bytes() == data and not (tmp_path / "f.exe.part").exists() + + +def test_download_bad_digest_or_size_leaves_nothing(tmp_path, monkeypatch): + _patch_open(monkeypatch, b"abc") + dest = tmp_path / "f.exe" + with pytest.raises(IOError): + uc.download_file("https://github.com/x/f.exe", dest, 3, digest="sha256:" + "0" * 64) + with pytest.raises(IOError): + uc.download_file("https://github.com/x/f.exe", dest, 99) + assert list(tmp_path.iterdir()) == [] + + +def test_download_cancel_and_untrusted(tmp_path, monkeypatch): + _patch_open(monkeypatch, b"abc" * 10) + with pytest.raises(InterruptedError): + uc.download_file("https://github.com/x/f.exe", tmp_path / "f.exe", cancelled=lambda: True) + assert list(tmp_path.iterdir()) == [] + with pytest.raises(ValueError): + uc.download_file("https://evil.example/f.exe", tmp_path / "f.exe") + _patch_open(monkeypatch, b"abc", url="https://evil.example/f.exe") + with pytest.raises(ValueError): + uc.download_file("https://github.com/x/f.exe", tmp_path / "g.exe") + + +def test_redirect_handler_refuses_untrusted_hosts(): + h = uc._TrustedRedirects() + with pytest.raises(ValueError): + h.redirect_request(None, None, 302, "Found", {}, "http://github.com/x") diff --git a/tests/test_validation.py b/tests/test_validation.py index f70088d..dee3e68 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -94,15 +94,15 @@ def test_resource_path_not_empty(self): class TestOcrConfiguration: """Test OCR configuration helpers.""" - def test_ocr_language_without_tessdata_uses_requested_languages(self): - assert ocr_language_for_choice("EN", None) == "eng" + def test_ocr_language_without_tessdata_is_italian(self): assert ocr_language_for_choice("IT", None) == "ita" - assert ocr_language_for_choice("BOTH", None) == "eng+ita" def test_ocr_language_filters_to_installed_languages(self, tmp_path): + assert ocr_language_for_choice("IT", str(tmp_path)) == "ita" (tmp_path / "eng.traineddata").write_text("placeholder", encoding="utf-8") - - assert ocr_language_for_choice("BOTH", str(tmp_path)) == "eng" + assert ocr_language_for_choice("IT", str(tmp_path)) == "eng" + (tmp_path / "ita.traineddata").write_text("placeholder", encoding="utf-8") + assert ocr_language_for_choice("IT", str(tmp_path)) == "ita+eng" def test_find_tessdata_dir_reads_environment(self, tmp_path, monkeypatch): (tmp_path / "ita.traineddata").write_text("placeholder", encoding="utf-8") @@ -133,7 +133,7 @@ def test_about_version_text_mentions_beta_only_for_prereleases(monkeypatch): monkeypatch.setattr(NullifyPDF, "APP_VERSION", "2.3.0") monkeypatch.setattr(NullifyPDF, "__version_prerelease__", "") - assert NullifyPDF.about_version_text() == "v2.3.0 AI Privacy" + assert NullifyPDF.about_version_text() == "v2.3.0" monkeypatch.setattr(NullifyPDF, "APP_VERSION", "2.3.0-beta.4") monkeypatch.setattr(NullifyPDF, "__version_prerelease__", "beta.4") - assert NullifyPDF.about_version_text() == "v2.3.0-beta.4 AI Privacy Beta" + assert NullifyPDF.about_version_text() == "v2.3.0-beta.4 Beta" diff --git a/tests/test_word_import.py b/tests/test_word_import.py new file mode 100644 index 0000000..615771e --- /dev/null +++ b/tests/test_word_import.py @@ -0,0 +1,41 @@ +import zipfile + +import pymupdf + +from word_import import docx_to_pdf, extract_docx_text + + +def _write_docx(path): + xml = """ + + + Mario Rossi + Codice fiscale + RSSMRA80A01H501U + + """ + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("word/document.xml", xml) + + +def test_docx_text_extraction_and_pdf_conversion(tmp_path): + source = tmp_path / "documento.docx" + _write_docx(source) + assert extract_docx_text(source) == ["Mario Rossi", "Codice fiscale | RSSMRA80A01H501U"] + document = docx_to_pdf(str(source)) + try: + assert "Mario Rossi" in document[0].get_text() + assert "RSSMRA80A01H501U" in document[0].get_text() + finally: + document.close() + + +def test_doc_import_is_rejected_explicitly(tmp_path): + source = tmp_path / "documento.doc" + source.write_bytes(b"legacy") + try: + docx_to_pdf(str(source)) + except ValueError as exc: + assert "DOCX" in str(exc) + else: + raise AssertionError("il formato DOC binario deve essere rifiutato") diff --git a/update_check.py b/update_check.py new file mode 100644 index 0000000..c46352e --- /dev/null +++ b/update_check.py @@ -0,0 +1,264 @@ +"""Qt-free helpers for the "check for updates" feature. + +The check runs only on an explicit click in the About dialog, or at startup +if the user enabled "Controlla ad ogni avvio" (off by default). The application +never contacts the network otherwise. + +Stable builds only look at stable releases, beta builds look at both. +""" + +from __future__ import annotations + +import hashlib +import os +import pathlib +import platform +import re +import urllib.parse +import urllib.request +from typing import Any, Callable, Dict, List, Optional, Tuple + +REPO_URL = "https://github.com/overwrite00/NullifyPDF" +RELEASES_API_URL = "https://api.github.com/repos/overwrite00/NullifyPDF/releases" + +# Downloads and release metadata may only come from GitHub over HTTPS. +_ALLOWED_HOSTS = ( + "api.github.com", + "github.com", + "objects.githubusercontent.com", + "release-assets.githubusercontent.com", +) + +_VERSION_RE = re.compile( + r"^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$" +) + +VersionKey = Tuple[int, int, int, int, Tuple[Any, ...]] + + +def parse_version(text: str) -> Optional[VersionKey]: + """Return a sortable key for ``2.3.0`` / ``v2.3.0-beta.4``, or None. + + A prerelease sorts before the same version without suffix (SemVer), and + numeric prerelease identifiers compare numerically (beta.10 > beta.9). + """ + m = _VERSION_RE.match(text.strip()) + if not m: + return None + major, minor, patch = (int(m.group(i)) for i in (1, 2, 3)) + pre = m.group(4) + if not pre: + return (major, minor, patch, 1, ()) + parts: List[Any] = [] + for ident in pre.split("."): + parts.append((0, int(ident), "") if ident.isdigit() else (1, 0, ident)) + return (major, minor, patch, 0, tuple(parts)) + + +def is_prerelease_version(text: str) -> bool: + key = parse_version(text) + return key is not None and key[3] == 0 + + +def is_trusted_url(url: str) -> bool: + """True for https URLs on the GitHub hosts used for releases.""" + parsed = urllib.parse.urlparse(url) + return parsed.scheme == "https" and (parsed.hostname or "") in _ALLOWED_HOSTS + + +def find_update( + releases: List[Dict[str, Any]], current_version: str +) -> Optional[Dict[str, Any]]: + """Pick the newest release the running build should be offered. + + Stable builds consider stable releases only; beta builds consider both. + Drafts and unparsable tags are ignored. Returns None when up to date. + """ + current = parse_version(current_version) + if current is None: + return None + include_pre = is_prerelease_version(current_version) + best: Optional[Tuple[VersionKey, Dict[str, Any]]] = None + for rel in releases: + if rel.get("draft"): + continue + key = parse_version(str(rel.get("tag_name", ""))) + if key is None: + continue + # Trust the tag, not the GitHub flag, for what counts as a prerelease. + if (key[3] == 0 or rel.get("prerelease")) and not include_pre: + continue + if key <= current: + continue + if best is None or key > best[0]: + best = (key, rel) + return best[1] if best else None + + +def _linux_family() -> Optional[str]: + """'deb', 'rpm' or None (unknown distribution) from /etc/os-release. + + Only Debian/Ubuntu-family and Fedora/RHEL-family systems get an automatic + asset: the packages' dependency names do not resolve elsewhere (Arch, + openSUSE, NixOS, ...), where the release link is shown instead. + """ + try: + text = pathlib.Path("/etc/os-release").read_text(encoding="utf-8").lower() + except OSError: + return None + ids = set() + for line in text.splitlines(): + if line.startswith(("id=", "id_like=")): + ids.update(line.split("=", 1)[1].strip().strip("\"'").split()) + if ids & {"debian", "ubuntu"}: + return "deb" + if ids & {"fedora", "rhel", "centos"}: + return "rpm" + return None + + +def asset_suffix_for_platform(system: Optional[str] = None) -> Optional[str]: + """Release-asset suffix matching the running OS (None if unsupported).""" + system = system or platform.system() + if system == "Windows": + return "_Windows.exe" + if system == "Darwin": + return "_macOS.zip" + family = _linux_family() + if family == "rpm": + return "_Fedora.rpm" + if family == "deb": + return "_Ubuntu.deb" + return None + + +def select_asset( + release: Dict[str, Any], system: Optional[str] = None +) -> Optional[Dict[str, Any]]: + """The release asset to download for this OS, or None.""" + suffix = asset_suffix_for_platform(system) + if suffix is None: + return None + for asset in release.get("assets", []): + name = str(asset.get("name", "")) + if name.startswith("NullifyPDF_v") and name.endswith(suffix): + if is_trusted_url(str(asset.get("browser_download_url", ""))): + return asset + return None + + +def safe_filename(name: str) -> str: + """Strip any directory part so a release asset cannot escape the folder.""" + return os.path.basename(name.replace("\\", "/")) or "NullifyPDF_update" + + +def unique_path(folder: pathlib.Path, name: str) -> pathlib.Path: + """``folder/name``, or ``name (1)``-style variant if it already exists.""" + target = folder / safe_filename(name) + stem, suffix = target.stem, target.suffix + n = 1 + while target.exists(): + target = folder / f"{stem} ({n}){suffix}" + n += 1 + return target + + +class _TrustedRedirects(urllib.request.HTTPRedirectHandler): + """Refuse any redirect that leaves the trusted GitHub https hosts.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[no-untyped-def] + if not is_trusted_url(newurl): + raise ValueError("Redirected to an untrusted address") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +_OPENER = urllib.request.build_opener(_TrustedRedirects) + + +def fetch_releases(timeout: float = 10.0) -> List[Dict[str, Any]]: + """Download the release list from the GitHub API.""" + import json + + req = urllib.request.Request( + RELEASES_API_URL + "?per_page=30", + headers={ + "Accept": "application/vnd.github+json", + "User-Agent": "NullifyPDF-update-check", + }, + ) + with _OPENER.open(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + if not isinstance(data, list): + raise ValueError("Unexpected response from GitHub") + return data + + +def download_file( + url: str, + dest: pathlib.Path, + expected_size: int = 0, + progress: Optional[Callable[[int, int], None]] = None, + cancelled: Optional[Callable[[], bool]] = None, + timeout: float = 10.0, + digest: str = "", +) -> None: + """Stream ``url`` to ``dest`` (via a .part file); check size and SHA-256. + + ``digest`` is the ``sha256:`` value GitHub publishes for each release + asset; when present the file must match it. + """ + if not is_trusted_url(url): + raise ValueError("Refusing to download from an untrusted address") + req = urllib.request.Request(url, headers={"User-Agent": "NullifyPDF-update-check"}) + part = dest.with_name(dest.name + ".part") + done = 0 + sha = hashlib.sha256() + try: + with _OPENER.open(req, timeout=timeout) as resp, open(part, "wb") as out: + if not is_trusted_url(resp.geturl()): + raise ValueError("Download redirected to an untrusted address") + total = int(resp.headers.get("Content-Length") or expected_size or 0) + while True: + if cancelled and cancelled(): + raise InterruptedError("Download cancelled") + chunk = resp.read(256 * 1024) + if not chunk: + break + out.write(chunk) + sha.update(chunk) + done += len(chunk) + if progress: + progress(done, total) + if expected_size and done != expected_size: + raise IOError(f"Incomplete download ({done} of {expected_size} bytes)") + if digest.startswith("sha256:") and sha.hexdigest() != digest[7:].lower(): + raise IOError("Checksum mismatch: the downloaded file is not the published one") + part.replace(dest) + except BaseException: + try: + part.unlink() + except OSError: + pass + raise + + +def load_auto_check(path: pathlib.Path) -> bool: + """Whether the user enabled the check at every start (default: off).""" + import json + + try: + data = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + return bool(data.get("check_on_startup", False)) + except Exception: + return False + + +def save_auto_check(path: pathlib.Path, enabled: bool) -> None: + import json + + try: + p = pathlib.Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(json.dumps({"check_on_startup": bool(enabled)}), encoding="utf-8") + except Exception: + pass diff --git a/word_import.py b/word_import.py new file mode 100644 index 0000000..6bcb7b1 --- /dev/null +++ b/word_import.py @@ -0,0 +1,82 @@ +"""Minimal, local DOCX importer used to create an analyzable PDF document. + +The importer intentionally handles the OOXML text layer without executing +macros, embedded objects, external links or document relationships. The +result is a new PDF representation; the original DOCX is never modified. +""" + +from __future__ import annotations + +import pathlib +import zipfile +import xml.etree.ElementTree as ET +from typing import List + +import fitz # type: ignore[import-untyped] + + +_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + + +def _paragraph_text(node: ET.Element) -> str: + return "".join((t.text or "") for t in node.iter(_W + "t")) + + +def extract_docx_text(path: pathlib.Path) -> List[str]: + """Extract paragraphs and table rows from a DOCX without running code.""" + with zipfile.ZipFile(path) as archive: + root = ET.fromstring(archive.read("word/document.xml")) + lines: List[str] = [] + body = root.find(_W + "body") + if body is None: + return lines + for child in body: + if child.tag == _W + "p": + text = _paragraph_text(child).strip() + if text: + lines.append(text) + elif child.tag == _W + "tbl": + for row in child.findall(_W + "tr"): + cells = [" ".join(_paragraph_text(p).split()) for p in row.findall(".//" + _W + "tc")] + text = " | ".join(c for c in cells if c).strip() + if text: + lines.append(text) + return lines + + +def docx_to_pdf(path: str) -> fitz.Document: + """Return a local PDF representation of a DOCX for the existing pipeline.""" + source = pathlib.Path(path) + if source.suffix.lower() != ".docx": + raise ValueError("Sono supportati documenti Word DOCX; il formato DOC binario non รจ supportato.") + lines = extract_docx_text(source) + document = fitz.open() + page = document.new_page() + margin = 54.0 + width = page.rect.width - 2 * margin + y = margin + fontsize = 10.5 + line_height = 15.0 + for raw in lines or [""]: + words = raw.split() + current = "" + chunks: List[str] = [] + for word in words: + candidate = f"{current} {word}".strip() + if current and fitz.get_text_length(candidate, fontname="helv", fontsize=fontsize) > width: + chunks.append(current) + current = word + else: + current = candidate + chunks.append(current) + for line in chunks: + if y + line_height > page.rect.height - margin: + page = document.new_page() + y = margin + page.insert_text((margin, y), line, fontname="helv", fontsize=fontsize) + y += line_height + y += 4.0 + return document + + +__all__ = ["docx_to_pdf", "extract_docx_text"] diff --git a/word_lists.py b/word_lists.py new file mode 100644 index 0000000..665aad5 --- /dev/null +++ b/word_lists.py @@ -0,0 +1,257 @@ +"""Blocklist / allowlist persistence: encrypted at rest, optional expiry (Qt-free). + +The lists can contain personal data (names the user always wants removed or +always wants kept), so they are stored sealed with the master key of +:mod:`secure_store` in ``blocklist.v2.enc`` / ``allowlist.v2.enc``. Each +entry carries the date it was added and an optional expiry date; expired +entries are dropped (and the file rewritten) when the list is loaded. + +Compatibility: an existing plaintext ``blocklist.txt`` / ``allowlist.txt`` is +read, migrated transparently and then overwritten and removed. If encryption +is unavailable (no ``cryptography``, no usable protector) the plaintext files +keep working. If the keystore cannot be unwrapped (profile copied to another +account) nothing is deleted or overwritten and the lists are empty until the +keystore is restored. +""" + +from __future__ import annotations + +import datetime +import json +import logging +import pathlib +from dataclasses import dataclass +from typing import Callable, Dict, List, Optional, Set + +import secure_store + +MIN_TERM_LENGTH = 3 +_MAGIC = b"NPDFL2\n" +_FORMAT = "NullifyPDF word list" +_log = logging.getLogger("nullifypdf") + + +@dataclass +class ListEntry: + term: str + added: datetime.date + expires: Optional[datetime.date] = None + + def to_json(self) -> Dict[str, Optional[str]]: + return { + "term": self.term, + "added": self.added.isoformat(), + "expires": self.expires.isoformat() if self.expires else None, + } + + +def _clean(term: str) -> str: + return " ".join(term.strip().lower().split()) + + +def _date(value: Optional[str]) -> Optional[datetime.date]: + if not value: + return None + try: + return datetime.date.fromisoformat(value) + except ValueError: + return None + + +class PDFListManager: + """Loads and saves the blocklist and allowlist.""" + + def __init__( + self, + config_dir: pathlib.Path, + today: Callable[[], datetime.date] = datetime.date.today, + ) -> None: + self.config_dir = pathlib.Path(config_dir) + self.config_dir.mkdir(parents=True, exist_ok=True) + self.block_file = self.config_dir / "blocklist.txt" + self.allow_file = self.config_dir / "allowlist.txt" + self._today = today + self.expired_removed = 0 + self.keystore_error: Optional[str] = None + self.encrypted = True + + # ------------------------------------------------------------ paths + def _enc_path(self, kind: str) -> pathlib.Path: + return self.config_dir / f"{kind}.v2.enc" + + def _plain_path(self, kind: str) -> pathlib.Path: + return self.block_file if kind == "blocklist" else self.allow_file + + # ------------------------------------------------------- settings + def default_ttl_days(self) -> Optional[int]: + try: + data = json.loads((self.config_dir / "settings.json").read_text(encoding="utf-8")) + ttl = data.get("list_ttl_days") + return int(ttl) if ttl else None + except Exception: + return None + + def set_default_ttl_days(self, days: Optional[int]) -> None: + path = self.config_dir / "settings.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception: + data = {} + data["list_ttl_days"] = days + secure_store.atomic_write(path, json.dumps(data).encode("utf-8")) + + def protector_label(self) -> str: + if not self.encrypted: + return "Non cifrato" + return secure_store.protector_label(self.config_dir) + + # ------------------------------------------------------- entries + def load_entries(self, kind: str) -> List[ListEntry]: + enc, plain = self._enc_path(kind), self._plain_path(kind) + if enc.exists(): + entries = self._read_encrypted(kind, enc) + kept = [e for e in entries if not (e.expires and e.expires < self._today())] + if len(kept) != len(entries): + self.expired_removed += len(entries) - len(kept) + self._write_encrypted(kind, kept) + return kept + if plain.exists(): + entries = self._read_plain(plain) + if self._can_encrypt(): + try: + self._write_encrypted(kind, entries) + secure_store.secure_delete(plain) + except Exception as e: # keep the readable file on any failure + _log.error(f"List migration failed: {e}") + self.encrypted = False + else: + self.encrypted = False + return entries + return [] + + def save_entries(self, kind: str, entries: List[ListEntry]) -> None: + clean: Dict[str, ListEntry] = {} + for e in entries: + term = _clean(e.term) + if len(term) >= MIN_TERM_LENGTH: + clean[term] = ListEntry(term, e.added, e.expires) + final = sorted(clean.values(), key=lambda e: e.term) + if self._can_encrypt(): + try: + self._write_encrypted(kind, final) + plain = self._plain_path(kind) + if plain.exists(): + secure_store.secure_delete(plain) + return + except secure_store.KeystoreUnavailable as e: + self.keystore_error = str(e) + raise + except Exception as e: + _log.error(f"Encrypted list save failed: {e}") + self.encrypted = False + try: + secure_store.atomic_write( + self._plain_path(kind), "\n".join(e.term for e in final).encode("utf-8") + ) + except OSError as e: + _log.error(f"Error saving {kind}: {e}") + + # -------------------------------------------- set-based convenience + def load_blocklist(self) -> Set[str]: + return self._terms("blocklist") + + def load_allowlist(self) -> Set[str]: + return self._terms("allowlist") + + def save_blocklist(self, blocklist: Set[str]) -> None: + self._save_terms("blocklist", blocklist) + + def save_allowlist(self, allowlist: Set[str]) -> None: + self._save_terms("allowlist", allowlist) + + def _terms(self, kind: str) -> Set[str]: + try: + return {e.term for e in self.load_entries(kind)} + except secure_store.KeystoreUnavailable as e: + self.keystore_error = str(e) + _log.error("Word lists unreadable: %s", e) + return set() + except Exception as e: + _log.error(f"Error loading {kind}: {e}") + return set() + + def _save_terms(self, kind: str, words: Set[str]) -> None: + """Save a set of terms, keeping dates of the ones already present.""" + try: + existing = {e.term: e for e in self.load_entries(kind)} + except secure_store.KeystoreUnavailable: + existing = {} + except Exception as e: # corrupt / tampered file: start from the new set + _log.error(f"Unreadable {kind} file ignored: {e}") + existing = {} + ttl = self.default_ttl_days() + today = self._today() + entries: List[ListEntry] = [] + for w in words: + term = _clean(w) + if len(term) < MIN_TERM_LENGTH: + continue + old = existing.get(term) + if old: + entries.append(old) + else: + exp = today + datetime.timedelta(days=ttl) if ttl else None + entries.append(ListEntry(term, today, exp)) + try: + self.save_entries(kind, entries) + except secure_store.KeystoreUnavailable: + pass # error kept in self.keystore_error; nothing is overwritten + + # ------------------------------------------------------ plumbing + def _can_encrypt(self) -> bool: + try: + import cryptography # noqa: F401 + + secure_store.load_or_create_master_key(self.config_dir) + return True + except secure_store.KeystoreUnavailable as e: + self.keystore_error = str(e) + raise + except Exception: + return False + + def _read_plain(self, path: pathlib.Path) -> List[ListEntry]: + try: + with open(path, "r", encoding="utf-8") as fh: + terms = {_clean(line) for line in fh if len(line.strip()) >= MIN_TERM_LENGTH} + except Exception as e: + _log.error(f"Error loading {path}: {e}") + return [] + today = self._today() + return [ListEntry(t, today, None) for t in sorted(terms) if len(t) >= MIN_TERM_LENGTH] + + def _read_encrypted(self, kind: str, path: pathlib.Path) -> List[ListEntry]: + key = secure_store.load_or_create_master_key(self.config_dir) + raw = path.read_bytes() + if not raw.startswith(_MAGIC): + raise ValueError("Formato del dizionario non riconosciuto.") + data = json.loads( + secure_store.open_sealed(raw[len(_MAGIC):], key, b"nullifypdf-list:" + kind.encode()) + ) + out = [] + for item in data.get("entries", []): + term = _clean(str(item.get("term", ""))) + if len(term) < MIN_TERM_LENGTH: + continue + out.append(ListEntry( + term, _date(item.get("added")) or self._today(), _date(item.get("expires")) + )) + return out + + def _write_encrypted(self, kind: str, entries: List[ListEntry]) -> None: + key = secure_store.load_or_create_master_key(self.config_dir) + payload = json.dumps({ + "format": _FORMAT, "version": 2, "entries": [e.to_json() for e in entries], + }, ensure_ascii=False).encode("utf-8") + sealed = secure_store.seal(payload, key, b"nullifypdf-list:" + kind.encode()) + secure_store.atomic_write(self._enc_path(kind), _MAGIC + sealed)