From b40ee64af17acaeaebdf76ade3cc777251682b75 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:08:42 +0100 Subject: [PATCH 1/8] feat: harden Auto Redact and export for GDPR, fewer false positives Detection - Evidence-based scoring of PERSON/LOCATION hits with an Italian lexicon, accepted/suggested states and a review dialog, preview overlay and health-document profile (birth dates, health card, prescription number, exemption codes, postcodes, house numbers, signature/stamp suggestions). - Validation of weak identifier patterns, allowlist by word identity, ligature/letter-spaced/hyphenated word handling, ISTAT place names. Redaction geometry - Rectangles stop short of neighbouring lines' glyph cores (measured deletion rule, pinned by a test); per-character coverage check; text under a box is read from a single baseline. - Placeholders drawn after the redactions, with compact labels in small boxes (restore map v4). Export and privacy - Export sanitiser and output verification, no unredacted temp copy, neutral file names, audit record, local data wipe, rotating logs. - Encrypted dictionaries with expiry, sealed pseudonym key, restore map envelope v3 (Argon2id, optional recovery key), pseudonym styles. Also bumps the prerelease label to beta.2 and aligns the mypy module list across workflows. --- .github/workflows/beta-release.yml | 2 +- .github/workflows/build-linux.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/test_build.yml | 46 +- ARCHITECTURE.md | 66 +- CHANGELOG.md | 30 + DEVELOPMENT.md | 14 +- NullifyPDF.py | 1518 +++++++++++++++++++++------- README.md | 5 +- SECURITY.md | 46 +- THIRD_PARTY_NOTICES.md | 17 + TROUBLESHOOTING.md | 33 +- USER_GUIDE.md | 67 +- build_local.py | 4 +- dictionary_dialog.py | 185 ++++ export_engine.py | 340 +++++++ lexicon_data_it.py | 9 + lexicon_it.py | 258 +++++ pdf_sanitize.py | 553 ++++++++++ pii_detection.py | 1006 +++++++++++++++++- privacy_core.py | 491 ++++++++- review_dialog.py | 183 ++++ scripts/build_lexicon_data.py | 141 +++ secure_store.py | 304 ++++++ tests/_pdfgen.py | 38 + tests/test_build_config.py | 14 + tests/test_detection_pipeline.py | 454 +++++++++ tests/test_export_sanitize.py | 385 +++++++ tests/test_geometry.py | 240 +++++ tests/test_gui_flow.py | 839 +++++++++++++++ tests/test_privacy_hardening.py | 261 +++++ tests/test_reconstruct.py | 45 +- tests/test_secure_store.py | 160 +++ word_lists.py | 257 +++++ 34 files changed, 7496 insertions(+), 519 deletions(-) create mode 100644 THIRD_PARTY_NOTICES.md create mode 100644 dictionary_dialog.py create mode 100644 export_engine.py create mode 100644 lexicon_data_it.py create mode 100644 lexicon_it.py create mode 100644 pdf_sanitize.py create mode 100644 review_dialog.py create mode 100644 scripts/build_lexicon_data.py create mode 100644 secure_store.py create mode 100644 tests/_pdfgen.py create mode 100644 tests/test_detection_pipeline.py create mode 100644 tests/test_export_sanitize.py create mode 100644 tests/test_geometry.py create mode 100644 tests/test_gui_flow.py create mode 100644 tests/test_privacy_hardening.py create mode 100644 tests/test_secure_store.py create mode 100644 word_lists.py diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml index 6a549bd..6cc48b1 100644 --- a/.github/workflows/beta-release.yml +++ b/.github/workflows/beta-release.yml @@ -50,7 +50,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py - name: Resolve beta suffix from tag if: startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 3adaed0..334f0a8 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -82,7 +82,7 @@ jobs: run: pytest tests/ -v - name: Type check (mypy) - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py - name: Resolve beta suffix from tag if: inputs.is_beta && startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f26461e..7273a52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -95,7 +95,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py - name: Build run: python build_local.py diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index f51816f..df1acb5 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -4,6 +4,9 @@ on: pull_request: branches: - main + - develop + schedule: + - cron: "17 5 * * 1" workflow_dispatch: permissions: @@ -17,6 +20,8 @@ concurrency: jobs: build: name: Build for ${{ matrix.os }} + # Develop pull requests and the weekly run only need the canary below. + if: github.event_name != 'schedule' && github.base_ref != 'develop' runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -50,7 +55,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py - name: Build run: python build_local.py @@ -66,8 +71,47 @@ jobs: linux: name: Linux packages + if: github.event_name != 'schedule' && github.base_ref != 'develop' uses: ./.github/workflows/build-linux.yml with: artifact_prefix: test-build is_beta: false retention_days: 3 + + pymupdf-latest: + # Redaction semantics (the glyph-deletion core, word boxes) differ between + # PyMuPDF releases; this job reports early when a newer release changes + # them. It never fails the workflow: it raises a visible warning. + name: Tests against the latest PyMuPDF + runs-on: ubuntu-24.04 + env: + QT_QPA_PLATFORM: offscreen + steps: + - name: Checkout Code + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install System Dependencies (Linux) + run: sudo apt-get update && sudo apt-get install -y libxcb-cursor0 libxkbcommon-x11-0 libegl1 + + - name: Install Python Dependencies + shell: bash + run: | + python -m pip install --upgrade pip + for attempt in 1 2 3 4 5; do + pip install -r requirements.txt && break + [ "$attempt" = 5 ] && exit 1 + echo "Attempt $attempt failed, retrying in 30s..."; sleep 30 + done + pip install --upgrade pymupdf + python -c "import pymupdf; print(pymupdf.VersionBind, pymupdf.VersionFitz)" >> "$GITHUB_STEP_SUMMARY" + + - name: Run Tests + shell: bash + run: | + pytest tests/ -v -k "geometry or reconstruct or export or gui_flow" || echo "::warning title=PyMuPDF latest::tests failed on $(python -c 'import pymupdf;print(pymupdf.VersionBind)')" diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2c888e4..15d146f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -27,8 +27,16 @@ It supports two privacy export modes: | ------ | ----------------- | | `NullifyPDF.py` | Main PySide6 UI, PDF rendering, AI scan orchestration, redaction, export, reconstruction, OCR integration | | `update_check.py` | Qt-free update logic: version comparison (SemVer, stable/beta channels), GitHub release lookup, per-OS asset selection, trusted-host download, startup-check preference (`~/.nullifypdf/update_check.json`). Only runs on user click or if the user enabled the startup check | -| `pii_detection.py` | Qt-free AI detection helpers: word-box text index, offset-to-rect mapping, per-type score thresholds, generic-word filtering, whole-word propagation, entity selection persistence | -| `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads (version 3, optional per-occurrence text style), restore-map parsing | +| `pii_detection.py` | Qt-free AI detection helpers: word-box text index (neighbour-aware rectangle clamping, ligatures, letter-spaced and hyphenated words), offset-to-rect mapping and partial-match policy, per-type thresholds, pattern validation, evidence-based PERSON/LOCATION scoring, accepted/suggested states, allowlist by word identity, whole-word propagation, entity selection persistence | +| `privacy_core.py` | Privacy mode enum, placeholder registry (random / sequential / generic / keyed styles), encrypted restore-map payloads (version 3, optional per-occurrence text style; scrypt, PBKDF2 maps still readable), password strength check, restore-map parsing | +| `lexicon_it.py` | Italian lexicon: certificate/legal verb forms, medical and administrative words, given names, regions and provinces (used to reject ordinary words and to confirm names/places) | +| `pdf_sanitize.py` | Export sanitiser (annotations, attachments, scripts, hidden layers/text, off-page text, metadata, optional TJ-gap quantisation) and `verify_export()` | +| `export_engine.py` | Qt-free export pipeline (`export_document`), neutral file names, private temp handling, audit record, local-data wipe | +| `review_dialog.py` | Qt dialog to accept or reject suggested detections | +| `secure_store.py` | Master key protected by DPAPI / OS keyring / key file; AES-GCM sealing; atomic private writes | +| `word_lists.py` | Encrypted blocklist/allowlist with expiry (`PDFListManager`) | +| `dictionary_dialog.py` | Qt editor for both lists with dates and default expiry | +| `lexicon_data_it.py` | Generated from ISTAT open data by `scripts/build_lexicon_data.py` (see `THIRD_PARTY_NOTICES.md`); imported lazily | | `PDF_Checker.py` | Heuristic post-export inspection helper | | `build_local.py` | PyInstaller build script | | `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files bundled into every build | @@ -69,11 +77,11 @@ Export pipeline writes a cleaned PDF copy | 🧱 Class | 📍 Location | 🎯 Purpose | | ------- | ----------- | ---------- | -| `PDFListManager` | `NullifyPDF.py` | Loads and saves blocklist/allowlist files in `~/.nullifypdf` | +| `PDFListManager` | `word_lists.py` | Loads and saves the blocklist/allowlist as encrypted files (`*.v2.enc`) with per-entry expiry; migrates plaintext files | | `AIWorker` | `NullifyPDF.py` | Background NLP/OCR worker that emits page-level detections | | `PDFView` | `NullifyPDF.py` | Custom `QGraphicsView` for rendering pages, drawing rectangles, and zooming | | `NullifyPDF` | `NullifyPDF.py` | Main window and application controller | -| `PlaceholderRegistry` | `privacy_core.py` | Creates stable placeholders such as `PERSON_001` and `EMAIL_ADDRESS_001` | +| `PlaceholderRegistry` | `privacy_core.py` | Creates stable placeholders such as `PERSON_9F2A` (random), `PERSON_001` (sequential), `DATA_9F2A` (generic) or an HMAC-derived one (keyed) | --- @@ -97,13 +105,13 @@ Export pipeline writes a cleaned PDF copy 3. `cmd_auto_ai()` checks OCR configuration if OCR is enabled. 4. `AIWorker.run_scan()` processes pages in a background thread. 5. For each page: - - the analysis text is built by `pii_detection.build_page_index()` from `page.get_text("words")`, recording each word's rectangle so character offsets map back to exact boxes - - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` and the same index is built from the OCR words - - Presidio analyzes the text for the enabled entity types - - results are filtered by per-type confidence thresholds; NER spans are split at line breaks; generic PERSON/LOCATION hits (titles, months, salutations, lowercase, role and identifier words) are dropped; single-word hits that look like labels or headings (followed by a colon, list items, lone heading lines, words also written in lowercase on the page, capitalized line starts) are rejected; a name glued to a following address is cut at the street word; overlapping spans are resolved - - each surviving value is located by its offsets, and its other occurrences on the page are added as **whole words only** - - detections are emitted with text, entity type, score, source, and their rectangles -6. `apply_ai_to_page()` runs on the UI thread and creates pending redaction annotations from the received rectangles. Blocklist and allowlist terms are matched as whole words too. + - the analysis text is built by `pii_detection.build_page_index()` from `page.get_text("words")`, recording each word's rectangle so character offsets map back to exact boxes. Each rectangle is trimmed against the words of neighbouring lines (a word box spans ascender to descender and would otherwise overlap them); ligatures are expanded, letter-spaced runs become one word and a word hyphenated across two lines is joined + - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` and the same index is built from the OCR words; on text pages with embedded images only the images are OCR'd and merged + - Presidio analyzes the text for the enabled entity types; date of birth and health identifiers are found by context-anchored patterns + - `classify_page()` filters the results: per-type thresholds; pattern hits are validated (token boundaries, context words, uppercase form, plausibility); NER spans are split at line breaks and names on consecutive lines are merged; ordinary Italian words are stripped from the edges of a span; fragments and OCR noise are rejected; field labels and list items are rejected; each remaining PERSON/LOCATION hit gets an evidence score and becomes `accepted`, `suggested` or is dropped (in health mode never dropped) + - `build_detections()` resolves each candidate to words with the partial-match policy (NER hits must not be a fragment of a longer word; a checksum-valid identifier glued to other text is redacted whole), removes words protected by the allowlist (by word identity), and adds other occurrences as **whole words only** + - detections are emitted with text, entity type, score, state, reasons, source and their rectangles +6. `apply_ai_to_page()` runs on the UI thread: `accepted` detections become pending redaction annotations (a rectangle already covered by at least 60% is not added twice); `suggested` ones are queued and reviewed in `ReviewDialog`. Blocklist terms are matched as whole words. ### 🖊️ Manual Review @@ -126,26 +134,26 @@ Clicking an existing redaction: `cmd_export()` is the main export entry point. -1. User chooses privacy mode: - - irreversible anonymization +1. If suggestions are unreviewed, the user is asked to review them or continue without them. +2. User chooses privacy mode: + - irreversible removal - reversible pseudonymization -2. User chooses output PDF path. -3. For pseudonymization only: - - user chooses `.nullifypdf-map` path - - user enters a password of at least 12 characters - - `cryptography` availability is checked before writing output -4. The in-memory PDF is saved to a sibling temp file: - - `.nullifypdf.tmp` -5. The temp PDF is reopened as `ex_doc`. -6. Each page is processed: - - overlapping links are deleted for redacted areas - - redactions are applied with PyMuPDF +3. User chooses the output path (default name contains nothing from the source name; a warning appears if the chosen name contains a removed value). +4. For pseudonymization only: + - user chooses the placeholder style (random, sequential, generic, keyed) + - user chooses the `.nullifypdf-map` path (a warning appears for the PDF's folder or a synchronised folder) + - user enters a password twice; it must pass a strength check (minimum 12 characters) +5. `export_engine.export_document()` works on a copy of the document kept in memory (or, above a size threshold, in a private per-user temp folder): + - links overlapping redactions are deleted + - redactions are applied with PyMuPDF (image pixels blanked, not whole images) - widgets are removed -7. Document metadata and selected catalog keys are cleared. -8. The final PDF is saved with cleanup options. -9. The temp file is removed. + - `pdf_sanitize.sanitize_document()` removes annotations, attachments, scripts, bookmarks, structure, metadata, hidden/invisible/off-page text and external links + - the final PDF is saved with cleanup options +6. `pdf_sanitize.verify_export()` re-opens the written file and checks it; a failure is reported and the file can be deleted. +7. For pseudonymization, the restore map is encrypted and written with owner-only permissions. +8. An audit record (no values) is written to `~/.nullifypdf/audit/`. -The original open document remains editable after export because the export works on a disk-backed copy. +The original open document remains editable after export because the export works on a copy. --- @@ -192,7 +200,7 @@ The restore map contains: - the redaction box of every occurrence - for native (non-scanned) PDFs, the typography of the redacted text: font, size, colour, flags and baseline (map version 3, optional) -The map is encrypted with `cryptography` using Fernet and a key derived from the user password with PBKDF2-HMAC-SHA256. +The map is an envelope (version 3): a random data key encrypts the payload with AES-256-GCM and is wrapped by a key derived from the password with Argon2id (t=3, m=64 MiB, p=4; scrypt if Argon2id is unavailable) and, optionally, by a 256-bit recovery key. Maps created with scrypt + Fernet (version 2) or PBKDF2-HMAC-SHA256 (version 1) still open; the parameters read from a file are bounded before use. Each occurrence may carry the compact label that was drawn when the full placeholder did not fit the box (map version 4). > [!IMPORTANT] > The restore map is never embedded in the pseudonymized PDF. diff --git a/CHANGELOG.md b/CHANGELOG.md index b945988..7f799b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,15 +12,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Check for updates**: the Info dialog has a new "Controlla aggiornamenti" button. It queries the GitHub releases of this repository (on click, or at startup only if you enabled the opt-in startup check) and, if a newer version exists, shows the release link, the release notes and a **Download** button that saves the installer for the running OS into the user's system Downloads folder (resolved per platform), verifying its size and the SHA-256 published by GitHub. Linux distributions outside the Debian/Ubuntu and Fedora/RHEL families only get the release link. Stable builds are only offered newer stable releases; beta builds are offered both stable and beta releases. - **Optional check at every start**: a "Controlla ad ogni avvio" checkbox in the Info dialog (off by default, stored in `~/.nullifypdf/update_check.json`) runs the same check silently at startup and opens the update window, with the release notes, only when a newer version exists. Failures are logged, never shown. - **Repository link in the Info dialog**. +- **Review of uncertain detections**: hits that the evidence score cannot settle (a lone capitalized word, a name without context) are no longer applied silently or lost. They are queued as suggestions in a review dialog (values masked until requested, per-row accept, select by score, double-click to jump to the page). "Esporta PDF" warns when suggestions are still unreviewed. +- **Health-document profile**: a sidebar option that also detects dates of birth and health identifiers (health card number, prescription number) anchored on context words, and keeps every uncertain hit as a suggestion instead of dropping it. +- **Export verification**: after writing the output, NullifyPDF re-opens it and checks that no removed value is still readable from the text layer, any object or decompressed stream, annotations, attachments, bookmarks or metadata, and that no annotation, script or attachment remains. If a check fails the export is flagged and the file can be deleted. +- **Export sanitiser**: annotations, embedded files, external links, bookmarks, JavaScript, tagged-PDF alternative text, document and XMP metadata, thumbnails, invisible text, text between the crop box and the media box, and hidden-layer text are removed or neutralised on export. Hidden layers are made visible before the scan so their text is analysed. +- **Pseudonym styles**: random (default; reveals neither order nor count), sequential, generic (also hides the entity type), and keyed (the same value gets the same pseudonym across documents made with the same key). +- **Optional width masking** ("Maschera larghezza testo rimosso"): rounds the gap left by removed text so that its width cannot be recovered from the content stream. +- **Encrypted dictionaries with expiry**: the blocklist and allowlist are stored encrypted (Windows DPAPI, the OS keyring if the optional `keyring` package is installed, otherwise a local key file) and every term can expire. The dictionary dialog is now a table with added/expiry dates, a default expiry for new terms, import from text and an explicit plain-text export. Existing plaintext lists are migrated and overwritten. If the keystore cannot be read on another account or PC, nothing is deleted and the lists stay empty until it is restored. +- **Recovery key for restore maps**: when pseudonymizing you can create a recovery key (shown once, never stored) that opens the map if the password is lost. New maps use a data key wrapped by the password (Argon2id) and, optionally, by the recovery key. Maps made by older versions still open; maps made by this version need this version or later. +- **Pseudonym key protected and portable**: the key of the "keyed" pseudonym style is stored encrypted, and a password-protected copy can be exported and imported to use the same pseudonyms on another computer. +- **Signature and stamp suggestions**: with the health-document profile, curve-rich ink and small images next to "firma", "timbro", "il medico"... are proposed for review (never applied automatically). +- **Postcodes and house numbers**: a street and its house number are redacted as one address, and postcodes are recognised only in an address context (never in prices, dates or quantities). Health card number, prescription number (NRE) and exemption codes follow the published structure (length, region code, lot code); the check digit is not verified, and the prescription number and exemption codes need their label. +- **Suggestions in the page preview**: pending suggestions are outlined in orange on the page (view only; they never reach an export). Click one to censor or discard it; selecting a row in the review dialog highlights it. +- **ISTAT place names**: Italian municipalities, provinces and regions (ISTAT open data, CC BY 4.0) are used to confirm places; a place missing from the list is never rejected for that reason. +- **Export audit record** in `~/.nullifypdf/audit/` (counts per type, sanitiser actions, verification result, output SHA-256; no personal data), and an "Elimina dati locali" button that wipes logs, audit records and temporary files. ### 🐛 Fixed +- **Redaction boxes covered parts of the lines above and below**: word boxes span ascender to descender, so with normal line spacing a redaction rectangle overlapped neighbouring lines, blackening parts of their letters in the preview and deleting their glyphs on export. Rectangles are now trimmed against the neighbouring lines (at the middle of the overlap on image pages, with a small horizontal margin). +- **Ordinary Italian words proposed for redaction** ("Certifico", "Prescrivo", "Dichiaro", headings of medical and legal documents): PERSON/LOCATION hits are now judged from evidence in the text (titles and cues such as "nato a" or "residente in", name shape, known given names and places, repetition) and rejected when they are Italian verb forms or common medical, legal and administrative words, including at the start or end of a longer span. Names split over two lines ("Esposito" / "Giovanni") are recognised as one name. +- **Weak identifier patterns produced false hits**: fiscal codes, VAT numbers, wallets, card numbers, driver licences, passports, ID cards and phone numbers must now sit on token boundaries, and the weak ones need their context word and uppercase form; dates, year ranges and short digit runs are no longer read as phone numbers. A checksum-valid identifier glued to a label is redacted as a whole token instead of being dropped. +- **Allowlist could hide real personal data**: an allowlist entry was matched as a substring of the detected value and by rectangle overlap, so "rossini" hid "Rossi" and an allowlisted word on the next line cancelled a detection. The allowlist now works by word identity, and an allowlisted word is only removed from a longer detection. +- **Substring fallback removed**: a detection without geometry is matched as whole words only. +- **Text-layer artefacts**: ligatures are expanded, letter-spaced text is read as one word, and a surname hyphenated across two lines is detected and redacted on both lines. +- **Restore map is no longer written beside the PDF by default and stores less**: it omits the source file name and the duplicated original text, is written with owner-only permissions where supported, uses scrypt (maps made with PBKDF2 still open), requires a confirmed password that passes a strength check, and bounds the key-derivation parameters read from the file. +- **No unredacted copy on disk during export**: the working copy is kept in memory (or in a private per-user folder for very large files, cleaned at start-up), no longer in `.nullifypdf.tmp` beside the output. +- **Neutral file names**: the default output and map names no longer derive from the source file name; a warning appears if the chosen name contains a value being removed. +- **Defaults and wording**: the language model defaults to Italian on an Italian system (otherwise both models); images embedded in text pages are analysed with OCR; the interface no longer calls redaction "anonymization" and states that the output and the restore map may still contain personal data; the scan log line no longer claims anonymization is complete. +- **Tight line spacing**: the redaction box now stops short of the neighbouring lines' glyphs instead of at their box edge, so text at 11 pt on a 7 pt pitch is redacted without damaging the lines around it. When no safe band exists a warning is shown. +- **Placeholders were silently dropped in small boxes**: the placeholder is now drawn after the redactions are applied, with a compact form ("P9F2A", "9F2A") when the full one does not fit, so short words keep a visible placeholder. Restore maps are version 4 (version 1-3 maps still load). +- **Restore wrote neighbouring text back**: the typography captured for a redaction could include fragments of the lines above and below when the leading was tight; only the redacted line is captured, and the restore writes the original value. +- **Wipe of local data left the active log file**, and "Oscura Immagini" added duplicate image redactions on every apply. +- **Local logs are rotated** (512 KiB, 3 files, 30 days) instead of growing indefinitely. - **Redacted text could remain in the PDF content stream**: with PyMuPDF 1.28.x, `apply_redactions()` did not remove text drawn with the `'` operator after `TL`/`Td`; the text disappeared from the extracted text but was still stored in the file. Fixed by PyMuPDF 2.x; covered by a regression test that scans every stream of the exported file. ### 🔧 Changed - **Linux packages are verified on the supported distributions**: Ubuntu 24.04 and 26.04, Debian 13, RHEL/Rocky 9 and 10 and current Fedora (installation and application start). The binary targets glibc 2.34, so older systems such as Ubuntu 22.04 and Debian 12 should keep working too. - **Linux packages declare their runtime libraries**: `.deb` `Depends` and `.rpm` `Requires` (xcb, EGL/GL, fontconfig, D-Bus, glib; `libglib2.0-0t64 | libglib2.0-0` for Ubuntu 24.04+), the `.deb` files are owned by root, and the `.rpm` no longer strips the PyInstaller executable. +- **Restore maps have a new format**: maps created by this version (and their keys) cannot be opened by 2.4.0-beta.1 or earlier; maps from earlier versions still open here. - **Beta packages sort before the stable one** (`2.4.0~beta.1` for `.deb`, Release `0.beta.1.1` for `.rpm`). - **Dependency updates**: PySide6 6.12.0, presidio-analyzer 2.2.366, spaCy 3.8.18, cryptography 50.0.2, pytest 9.2.0. Verified with the full test suite, `mypy` and an offscreen end-to-end scan/redaction run. - **PyMuPDF 2.1**: required for the redaction fix above. Version 2.1 no longer loads the layout analyzer on import, so start-up time and installer size are unaffected (verified with a PyInstaller bundle). diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index a7fa4ad..8a2144d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -163,7 +163,7 @@ These are downloaded automatically by `setup_env.py`. ```python NullifyPDF(QMainWindow) -|-- PDFListManager # Manages blocklist/allowlist +|-- PDFListManager # (word_lists.py) encrypted blocklist/allowlist |-- AIWorker(QObject) # NLP/OCR scanning in thread |-- PDFView(QGraphicsView) # PDF rendering and rectangle drawing |-- privacy_core.py # Privacy modes and encrypted restore maps @@ -453,7 +453,15 @@ isort NullifyPDF.py | ------------------ | ----------------------- | | `NullifyPDF.py` | Main app, GUI, OCR, export logic | | `privacy_core.py` | Placeholder and restore-map logic | -| `pii_detection.py` | AI detection precision: word-box index, thresholds, filters, entity selection | +| `pii_detection.py` | AI detection precision: word-box index, rectangle clamping, thresholds, pattern validation, evidence scoring, entity selection | +| `lexicon_it.py` | Italian word lists used to reject ordinary words / confirm names and places | +| `pdf_sanitize.py` | Export sanitiser and output verification | +| `export_engine.py` | Qt-free export pipeline, audit record, local-data hygiene | +| `review_dialog.py` | Review dialog for suggested detections | +| `secure_store.py` | Master key protection (DPAPI / keyring / key file), AES-GCM sealing | +| `word_lists.py` | Encrypted blocklist/allowlist with expiry | +| `dictionary_dialog.py` | Dictionary editor dialog | +| `scripts/build_lexicon_data.py` | Generates `lexicon_data_it.py` from ISTAT open data (development only) | | `setup_env.py` | Environment setup | | `build_local.py` | PyInstaller build | | `PDF_Checker.py` | Post-processing utility | @@ -469,7 +477,7 @@ isort NullifyPDF.py | AI Processing | `NullifyPDF.py` | `AIWorker.run_scan()` | | Export | `NullifyPDF.py` | `cmd_export()` | | Reconstruct | `NullifyPDF.py` | `cmd_reconstruct()` | -| Blocklist/Allowlist | `NullifyPDF.py` | `PDFListManager` | +| Blocklist/Allowlist | `word_lists.py` | `PDFListManager` | --- diff --git a/NullifyPDF.py b/NullifyPDF.py index beba095..cf1fec0 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -7,6 +7,7 @@ import string import platform import logging +import logging.handlers import traceback import atexit import hashlib @@ -51,6 +52,7 @@ QDropEvent, ) from PySide6.QtCore import ( + QLocale, Qt, QThread, QObject, @@ -67,9 +69,15 @@ from PySide6.QtGui import QDesktopServices from privacy_core import ( + PSEUDONYM_STYLES, PlaceholderEntry, + export_pseudonym_key, + generate_recovery_key, + import_pseudonym_key, + parse_recovery_key, PlaceholderRegistry, PrivacyMode, + password_strength, build_restore_payload, decrypt_restore_payload, encrypt_restore_payload, @@ -79,19 +87,42 @@ from pii_detection import ( ALL_GROUP_IDS, ENTITY_GROUPS, + NON_PRESIDIO_TYPES, + build_detections, build_page_index, - clean_candidates, + classify_page, + covered_fraction, expand_group_ids, - filter_results, load_enabled_groups, - propagate_whole_word, - reject_label_like_hits, - resolve_overlaps, save_enabled_groups, - span_to_rects, whole_word_rects, ) import update_check +from export_engine import ( + build_audit_record, + cleanup_stale_temp, + collect_removed_values, + export_document, + name_leaks_value, + neutral_map_name, + neutral_output_name, + private_temp_dir, + purge_old_logs, + wipe_local_data, + write_audit_record, + write_private_file, +) +from pii_detection import GLYPH_CORE_INSET +from pdf_sanitize import ( + SanitizePolicy, + page_image_coverage, + reveal_hidden_layers, + suggest_signature_areas, +) +from review_dialog import ReviewDialog +from dictionary_dialog import DictionaryDialog +from word_lists import PDFListManager +import secure_store # PyMuPDF defines these constants dynamically, which mypy cannot see; read # them once here (with the suppression) instead of at every call site. @@ -99,7 +130,7 @@ PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] __version__ = "2.4.0" -__version_prerelease__ = "beta.1" +__version_prerelease__ = "beta.2" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" if __version_prerelease__ @@ -181,6 +212,9 @@ class DetectionDict(TypedDict): rects: List[Tuple[float, float, float, float]] source: NotRequired[str] score: NotRequired[float] + state: NotRequired[str] + reasons: NotRequired[List[str]] + glued: NotRequired[bool] def setup_logging() -> logging.Logger: @@ -193,6 +227,7 @@ def setup_logging() -> logging.Logger: """ log_dir = pathlib.Path.home() / ".nullifypdf" / "logs" log_dir.mkdir(parents=True, exist_ok=True) + purge_old_logs(log_dir, max_age_days=30) logger = logging.getLogger("nullifypdf") debug_mode = os.environ.get("NULLIFYPDF_DEBUG", "").lower() == "true" @@ -202,9 +237,13 @@ def setup_logging() -> logging.Logger: return logger try: - handler = logging.FileHandler( + # Rotating, so diagnostics are not kept indefinitely (storage + # limitation); old rotated files are also purged above. + handler = logging.handlers.RotatingFileHandler( log_dir / "nullifypdf.log", - encoding="utf-8" + maxBytes=512 * 1024, + backupCount=3, + encoding="utf-8", ) handler.setLevel(logging.DEBUG) formatter = logging.Formatter( @@ -250,7 +289,12 @@ class RestoreMapMismatch(ValueError): REDACTION_FONTNAME = "helv" REDACTION_MAX_FONTSIZE = 8.0 -REDACTION_MIN_FONTSIZE = 3.0 +# PyMuPDF's apply_redactions() never inserts replacement text below 4 pt, so +# a smaller floor would only produce text that is silently dropped. +REDACTION_MIN_FONTSIZE = 4.0 +# Text written with TextWriter has no such floor. +LABEL_MIN_FONTSIZE = 1.0 +LABEL_PREFERRED_MIN_FONTSIZE = 4.0 def fit_redaction_fontsize( @@ -337,47 +381,148 @@ def _placeholder_rects_by_chars(page: Any, placeholder: str) -> List["fitz.Rect" return rects +def glyph_core(bbox: Tuple[float, float, float, float]) -> Tuple[float, float, float, float]: + """The part of a character box that decides whether a redaction deletes it. + + Measured on PyMuPDF 2.1 (MuPDF 1.28): a glyph is removed when the + redaction rectangle intersects its box shrunk by 10% of its width on + each side and 10% of its height at top and bottom. + """ + x0, y0, x1, y1 = bbox + dx, dy = (x1 - x0) * GLYPH_CORE_INSET, (y1 - y0) * GLYPH_CORE_INSET + return (x0 + dx, y0 + dy, x1 - dx, y1 - dy) + + +def _core_hits(bbox: Tuple[float, float, float, float], rect: "fitz.Rect") -> bool: + c = glyph_core(bbox) + return c[0] < rect.x1 and c[2] > rect.x0 and c[1] < rect.y1 and c[3] > rect.y0 + + def capture_text_style(page: Any, rect: "fitz.Rect") -> Optional[Dict[str, Any]]: """Read the typography of the native text under `rect`. - Returns ``{font, size, color, flags, origin, text}`` for the dominant - span (most characters), where ``text`` is the text actually inside this - box and ``origin`` the baseline start of its first line. Returns None - when the box holds no real text: no span at all, or only an invisible - OCR layer (GlyphLessFont / render mode 3). That is what distinguishes a - native PDF from a scanned one. + Returns ``{font, size, color, flags, origin, text}`` for the line the + box was taken from, where ``text`` is the text of that line inside the + box and ``origin`` the baseline start of its first character. Only + characters on a single baseline whose deletion core the box intersects + are used, so with tight leading the text of the lines above and below + (whose boxes overlap this one) is never captured. Returns None when the + box holds no real text: no span at all, or only an invisible OCR layer + (GlyphLessFont / render mode 3). That is what distinguishes a native PDF + from a scanned one. """ try: - blocks = page.get_text("dict", clip=rect).get("blocks", []) + blocks = page.get_text("rawdict", clip=rect).get("blocks", []) except Exception: return None - spans: List[Dict[str, Any]] = [] + # (baseline y rounded, span, char) + hits: List[Tuple[float, Dict[str, Any], Dict[str, Any]]] = [] for block in blocks: if block.get("type") != 0: continue for line in block.get("lines", []): for span in line.get("spans", []): - if not str(span.get("text", "")).strip(): - continue if "glyphless" in str(span.get("font", "")).lower(): continue if span.get("alpha", 255) == 0: continue - spans.append(span) - if not spans: + for ch in span.get("chars", []): + if not str(ch.get("c", "")).strip(): + continue + if not _core_hits(tuple(ch["bbox"]), rect): + continue + hits.append((round(float(ch["origin"][1]), 1), span, ch)) + if not hits: return None - dominant = max(spans, key=lambda sp: len(str(sp["text"]).strip())) - text = " ".join(" ".join(str(sp["text"]) for sp in spans).split()) + centre = (rect.y0 + rect.y1) / 2 + counts: Dict[float, int] = {} + for base, _sp, _ch in hits: + counts[base] = counts.get(base, 0) + 1 + # The target line is the baseline nearest the middle of the box among + # those inside it; with tight leading a neighbour's baseline can also + # fall inside the box, but never closer to its middle. + inside = [k for k in counts if rect.y0 <= k <= rect.y1] or list(counts) + base = min(inside, key=lambda k: (abs(k - centre), -counts[k])) + chosen = [(sp, ch) for b_, sp, ch in hits if b_ == base] + chosen.sort(key=lambda t: t[1]["origin"][0]) + spans_used: Dict[int, int] = {} + for sp, _ch in chosen: + spans_used[id(sp)] = spans_used.get(id(sp), 0) + 1 + dominant = next(sp for sp, _ in chosen if spans_used[id(sp)] == max(spans_used.values())) + # Spaces between the selected characters are not in `hits` (they were + # filtered out above); rebuild them from the horizontal gaps. + pieces: List[str] = [] + prev_x1: Optional[float] = None + for _sp, ch in chosen: + x0, _y0, x1, _y1 = ch["bbox"] + if prev_x1 is not None and x0 - prev_x1 > 0.25 * float(dominant["size"]): + pieces.append(" ") + pieces.append(str(ch["c"])) + prev_x1 = x1 + first = chosen[0][1] return { "font": str(dominant.get("font", "")), "size": float(dominant["size"]), "color": int(dominant.get("color", 0)), "flags": int(dominant.get("flags", 0)), - "origin": [float(dominant["origin"][0]), float(dominant["origin"][1])], - "text": text, + "origin": [float(first["origin"][0]), float(first["origin"][1])], + "text": "".join(pieces).strip(), } +def target_chars(page: Any, full: "fitz.Rect") -> List[Dict[str, Any]]: + """Characters of the line a word box was taken from (single baseline).""" + try: + blocks = page.get_text("rawdict", clip=full).get("blocks", []) + except Exception: + return [] + chars: List[Tuple[float, Dict[str, Any]]] = [] + for block in blocks: + if block.get("type") != 0: + continue + for line in block.get("lines", []): + for span in line.get("spans", []): + for ch in span.get("chars", []): + if not str(ch.get("c", "")).strip(): + continue + bx0, by0, bx1, by1 = ch["bbox"] + cx, cy = (bx0 + bx1) / 2, (by0 + by1) / 2 + if full.x0 <= cx <= full.x1 and full.y0 <= cy <= full.y1: + chars.append((round(float(ch["origin"][1]), 1), ch)) + if not chars: + return [] + centre = (full.y0 + full.y1) / 2 + bases = {b for b, _ in chars} + inside = [b for b in bases if full.y0 <= b <= full.y1] or list(bases) + base = min(inside, key=lambda b: abs(b - centre)) + return [ch for b, ch in chars if b == base] + + +def band_covers_target(page: Any, band: "fitz.Rect", full: "fitz.Rect") -> bool: + """True when `band` reaches the deletion core of every target character. + + A glyph that is smaller or higher than the rest of its word (a + superscript, a trailing digit) could keep its core outside a thin band + and survive the redaction. + """ + chars = target_chars(page, full) + return all(_core_hits(tuple(ch["bbox"]), band) for ch in chars) + + +def text_under_rect(page: Any, rect: "fitz.Rect") -> str: + """The text a box was drawn over, without neighbouring lines' fragments. + + `get_text(clip=...)` follows a different rule from glyph deletion and, with + tight leading, returns pieces of the lines above and below. Native text is + read from a single baseline (see capture_text_style); the plain clip text + is only the fallback for scanned / image pages. + """ + style = capture_text_style(page, rect) + if style and style.get("text"): + return " ".join(str(style["text"]).split()) + return " ".join(page.get_text("text", clip=rect).split()) + + _BASE14 = { # (family, bold, italic) -> Base-14 font code ("sans", False, False): "helv", ("sans", True, False): "hebo", @@ -467,7 +612,7 @@ def write_styled_text( else: # Wider in the fallback font (or a multi-line original): shrink a # little, and wrap inside the box if it still does not fit. - scaled = max(REDACTION_MIN_FONTSIZE, size * usable / width) + scaled = max(LABEL_MIN_FONTSIZE, size * usable / width) if font.text_length(text, fontsize=scaled) <= usable: writer.append(origin, text, font=font, fontsize=scaled) else: @@ -475,6 +620,73 @@ def write_styled_text( writer.write_text(page, color=color) +def compact_labels(placeholder: str) -> List[str]: + """Shorter stand-ins for a placeholder, longest first (full one first).""" + head, _, tail = placeholder.rpartition("_") + if not head or not tail: + return [placeholder] + out = [placeholder, f"{head[:1]}{tail}", tail, f"#{tail[:2]}"] + seen: List[str] = [] + for item in out: + if item not in seen: + seen.append(item) + return seen + + +def write_label( + page: Any, + rect: "fitz.Rect", + labels: List[str], + *, + style: Optional[Dict[str, Any]] = None, + allowance: float = 0.0, + max_size: float = REDACTION_MAX_FONTSIZE, +) -> str: + """Draw the first of `labels` that is legible inside the box; return it. + + Drawn with TextWriter after apply_redactions(), so there is no 4 pt + floor and no silent drop. A label may extend past the box to the right by + `allowance` points (free space on the same line). Preference: the longest + label that fits at 4 pt or more; otherwise the shortest at whatever size + fits (down to 1 pt). + """ + font = fitz.Font(REDACTION_FONTNAME) + # A band squeezed between tight lines is short: keep the label inside it. + max_size = min(max_size, max(LABEL_MIN_FONTSIZE, rect.height * 0.9)) + usable = max(1.0, rect.width - 1.0 + allowance) + chosen = labels[-1] + size = LABEL_MIN_FONTSIZE + for lab in labels: + full = font.text_length(lab, fontsize=1.0) + fit = min(max_size, usable / full) if full > 0 else max_size + if fit >= LABEL_PREFERRED_MIN_FONTSIZE: + chosen, size = lab, fit + break + else: + full = font.text_length(chosen, fontsize=1.0) + size = max(LABEL_MIN_FONTSIZE, min(max_size, usable / full if full else max_size)) + if style and style.get("origin"): + base_y = float(style["origin"][1]) + else: + base_y = rect.y0 + (rect.height + size * 0.7) / 2 + writer = fitz.TextWriter(page.rect) + writer.append(fitz.Point(rect.x0 + 0.5, base_y), chosen, font=font, fontsize=size) + writer.write_text(page, color=(0, 0, 0)) + return chosen + + +def free_space_right(page: Any, rect: "fitz.Rect", limit: float = 40.0) -> float: + """Free width to the right of `rect` on the same line (after redaction).""" + gap = limit + for w in page.get_text("words"): + overlap = min(rect.y1, w[3]) - max(rect.y0, w[1]) + if overlap < 0.5 * min(rect.height, w[3] - w[1]): + continue + if w[0] >= rect.x1 - 0.5: + gap = min(gap, w[0] - rect.x1 - 1.0) + return max(0.0, gap) + + def _restore_targets( page: Any, entry: PlaceholderEntry ) -> List[Tuple["fitz.Rect", Optional[Dict[str, Any]]]]: @@ -544,6 +756,7 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int for page in doc.pages(): claimed: List[fitz.Rect] = [] styled: List[Tuple[fitz.Rect, str, Dict[str, Any]]] = [] + plain: List[Tuple[fitz.Rect, str]] = [] for entry in entries: for rect, style in _restore_targets(page, entry): # Entries are sorted longest-placeholder-first, so a @@ -561,23 +774,19 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int # Native text: blank the box now, write the value in # its original typography after apply_redactions() # (which would otherwise erase it). - text = style.get("text") or entry.original + text = entry.original # never the captured clip text page.add_redact_annot(rect, fill=(1, 1, 1)) styled.append((fitz.Rect(rect), text, style)) attempted.append((page.number, fitz.Rect(rect), text)) continue - page.add_redact_annot( - rect, - text=entry.original, - fill=(1, 1, 1), - text_color=(0, 0, 0), - align=1, - fontsize=fit_redaction_fontsize(entry.original, rect), - ) + page.add_redact_annot(rect, fill=(1, 1, 1)) + plain.append((fitz.Rect(rect), entry.original)) attempted.append((page.number, fitz.Rect(rect), entry.original)) page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) for rect, text, style in styled: write_styled_text(doc, page, rect, text, style) + for rect, text in plain: + write_label(page, rect, [text]) doc.save(out_path, garbage=4, deflate=True, clean=True) finally: doc.close() @@ -674,93 +883,6 @@ def ocr_language_for_choice(choice: str, tessdata_dir: Optional[str]) -> str: return "+".join(available or requested) -class PDFListManager: - """Manages blocklist/allowlist persistence to disk. - - Handles loading and saving word lists with automatic path management - and UTF-8 encoding. - """ - - def __init__(self, config_dir: pathlib.Path) -> None: - """Initialize list manager with config directory. - - Args: - config_dir: Path to configuration directory. - """ - self.config_dir = config_dir - self.config_dir.mkdir(parents=True, exist_ok=True) - self.block_file = config_dir / "blocklist.txt" - self.allow_file = config_dir / "allowlist.txt" - - def load_blocklist(self) -> Set[str]: - """Load blocklist from disk. - - Returns: - Set[str]: Words to redact (lowercase). - """ - return self._load_list(self.block_file) - - def load_allowlist(self) -> Set[str]: - """Load allowlist from disk. - - Returns: - Set[str]: Words to preserve from redaction (lowercase). - """ - return self._load_list(self.allow_file) - - def save_blocklist(self, blocklist: Set[str]) -> None: - """Persist blocklist to disk. - - Args: - blocklist: Set of words to save. - """ - self._save_list(self.block_file, blocklist) - - def save_allowlist(self, allowlist: Set[str]) -> None: - """Persist allowlist to disk. - - Args: - allowlist: Set of words to save. - """ - self._save_list(self.allow_file, allowlist) - - def _load_list(self, path: pathlib.Path) -> Set[str]: - """Load word list from file with validation. - - Args: - path: Path to list file. - - Returns: - Set[str]: Set of words (lowercase, stripped, min length 3). - """ - if not path.exists(): - return set() - try: - # Use context manager to ensure file handle is closed deterministically - with open(path, "r", encoding="utf-8") as fh: - return { - line.strip().lower() - for line in fh - if len(line.strip()) > 2 - } - except Exception as e: - logging.getLogger(__name__).error(f"Error loading {path}: {e}") - return set() - - def _save_list(self, path: pathlib.Path, words: Set[str]) -> None: - """Save word list to file. - - Args: - path: Path to save list. - words: Set of words to save. - """ - try: - with open(path, "w", encoding="utf-8") as f: - f.write("\n".join(sorted(words))) - except Exception as e: - logging.getLogger(__name__).error(f"Error saving {path}: {e}") - - STYLESHEET = """ QMainWindow, QDialog { background-color: #0f172a; } QWidget { color: #cbd5e1; font-family: 'Segoe UI', 'Roboto', sans-serif; font-size: 13px; } @@ -826,12 +948,13 @@ def cleanup(self) -> None: except Exception as e: logging.getLogger("nullifypdf").debug(f"Error during AI cleanup: {e}") - @Slot(object, object, str, list, set, bool, object, list) + @Slot(object, object, str, list, set, bool, object, list, bool) def run_scan(self, doc: Any, doc_mutex: Any, choice: str, compiled_allowlist: List[Tuple[str, Any]], allowlist_set: Set[str], use_ocr: bool, tessdata_dir: Optional[str], - enabled_types: List[str]) -> None: + enabled_types: List[str], + health_mode: bool = False) -> None: """Run AI scan on PDF pages and emit detected sensitive entities. Text extraction (`page.get_text()`) is performed inside this worker so @@ -848,6 +971,8 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, allowlist_set: Set of allowlist entries (lowercase) for O(1) exact-match fast-path lookup before regex any() scan. enabled_types: Presidio entity types the user chose to detect. + health_mode: Health-document profile: no uncertain hit is dropped + silently, it becomes a suggestion for review instead. """ try: if self._stop_requested: @@ -921,25 +1046,37 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, self.loaded_langs = target_langs self.log_sig.emit("Scansione privacy in corso...") targets = list(enabled_types) + presidio_targets = [ + t for t in targets if t not in NON_PRESIDIO_TYPES + ] + allow_terms = sorted( + set(allowlist_set) | {a for a, _ in compiled_allowlist} + ) # Determine page count under mutex (cheap, but doc may be replaced # mid-flight by load_path if not for the disabled UI during scan). with QMutexLocker(doc_mutex): total_pages = len(doc) + totals: Dict[str, int] = {} + n_suggested = 0 for i in range(total_pages): if self._stop_requested: break # Extract text in worker thread (off the UI thread) under mutex # because UI thread renders / mutates annotations on the same doc. used_ocr = False + has_images = False textpage = None + signature_dets: List[Dict[str, Any]] = [] with QMutexLocker(doc_mutex): try: page = doc[i] text = page.get_text() needs_ocr = use_ocr and self._page_needs_ocr(page, text) - index = build_page_index(page.get_text("words")) + coverage = page_image_coverage(page) + has_images = coverage > 0.0 + words = page.get_text("words") if needs_ocr: self.log_sig.emit(f"OCR pagina {i + 1}...") textpage = page.get_textpage_ocr( @@ -948,70 +1085,83 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, full=True, tessdata=tessdata_dir, ) - text = page.get_text(textpage=textpage) - index = build_page_index( - page.get_text("words", textpage=textpage) + words = page.get_text("words", textpage=textpage) + used_ocr = True + elif use_ocr and tessdata_dir and coverage >= 0.03: + # Text page with embedded images (stamps, scanned + # attachments): OCR the images only and merge. + self.log_sig.emit(f"OCR immagini pagina {i + 1}...") + textpage = page.get_textpage_ocr( + language=ocr_language, + dpi=300, + full=False, + tessdata=tessdata_dir, ) + words = page.get_text("words", textpage=textpage) used_ocr = True + index = build_page_index( + words, ocr=used_ocr, + image_rects=[ + tuple(i["bbox"]) + for i in page.get_image_info(hashes=False) + ], + ) + if health_mode: + signature_dets = suggest_signature_areas(page) + if index.collisions: + self.log_sig.emit( + f"Avviso: interlinea molto stretta a pagina " + f"{i + 1}: {index.collisions} parole vicine " + "potrebbero essere danneggiate. Controlla l'anteprima." + ) except Exception as e: self.log_sig.emit( f"Avviso: estrazione testo/OCR fallita pagina {i+1}: {e}" ) - text = "" index = build_page_index([]) - candidates: List[Any] = [] + results: List[Any] = [] + verb_like: Set[str] = set() for lang in self.loaded_langs: - if not targets or not index.text: + if not presidio_targets or not index.text: break + artifacts = None + try: + artifacts = self.analyzer.nlp_engine.process_text( + index.text, lang + ) + except Exception: + artifacts = None res = self.analyzer.analyze( - text=index.text, entities=targets, language=lang - ) - candidates.extend(filter_results(res, index.text, targets)) - candidates = resolve_overlaps( - reject_label_like_hits( - clean_candidates(candidates, enabled_types=targets), - index.text, + text=index.text, entities=presidio_targets, + language=lang, nlp_artifacts=artifacts, ) + results.extend(res) + verb_like |= self._first_person_verbs(artifacts) + candidates, stats = classify_page( + results, index, targets, + used_ocr=used_ocr, health_mode=health_mode, + verb_like=verb_like, ) - detections = [] - seen_values: Set[Tuple[str, str]] = set() - for c in candidates: - m, entity_type = c.text, c.entity_type - clean = " ".join(m.strip(string.punctuation).lower().split()) - if not clean: - continue - # Fast path: exact set membership is O(1). - if clean in allowlist_set: - continue - clean_pattern = re.compile(r"" + re.escape(clean) + r"") - if any( - f_reg.search(clean) or clean_pattern.search(a_str) - for a_str, f_reg in compiled_allowlist - ): - continue - rects = span_to_rects(index, c.start, c.end) - if (m, entity_type) not in seen_values: - seen_values.add((m, entity_type)) - # Other occurrences of the same value: whole words only. - for extra in propagate_whole_word( - index, m, - case_sensitive=entity_type in ("PERSON", "LOCATION"), - ): - if extra not in rects: - rects.append(extra) - if not rects: - continue - detections.append({ - "text": m, - "entity_type": entity_type, - "rects": rects, - "source": "ocr" if used_ocr else "text", - "score": c.score, - }) + detections = build_detections( + index, candidates, allowlist=allow_terms, + source="ocr" if used_ocr else "text", stats=stats, + ) + detections.extend(signature_dets) + for key, n in stats.items(): + totals[key] = totals.get(key, 0) + n + n_suggested += sum(1 for d in detections if d.get("state") == "suggested") self.page_done_sig.emit(i, detections) self.progress_sig.emit(i + 1, total_pages) if not self._stop_requested: - self.log_sig.emit("Anonimizzazione completata.") + if totals: + summary = ", ".join(f"{k}={v}" for k, v in sorted(totals.items())) + logging.getLogger("nullifypdf").info( + f"Scan discards (no values): {summary}" + ) + msg = "Scansione completata." + if n_suggested: + msg += f" {n_suggested} suggerimenti da rivedere." + self.log_sig.emit(msg) except Exception as e: # Log full traceback to file for diagnostics; show only summary in UI logging.getLogger("nullifypdf").error( @@ -1021,6 +1171,22 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, finally: self.finished_sig.emit() + @staticmethod + def _first_person_verbs(artifacts: Any) -> Set[str]: + """Lowercase first-person finite verb forms spaCy found on the page. + + A weak signal only: the grammar tags of capitalised words are not + reliable enough to reject a hit on their own. + """ + out: Set[str] = set() + try: + for tok in artifacts.tokens: + if tok.pos_ in ("VERB", "AUX") and "Person=1" in str(tok.morph): + out.add(tok.text.lower()) + except Exception: + return set() + return out + def _page_needs_ocr(self, page: Any, extracted_text: str) -> bool: """Return True when a page looks scanned or text extraction is too poor.""" text = (extracted_text or "").strip() @@ -1104,6 +1270,36 @@ def wheelEvent(self, event: Any) -> None: super().wheelEvent(event) +class SuggestionItem(QGraphicsRectItem): + """Scene-only outline of a pending suggestion. + + It lives in the graphics scene, never in the document: it cannot reach an + export or a print, and it becomes a redaction only through the + "Censura" decision. + """ + + def __init__(self, rect_pdf: Tuple[float, ...], suggestion: Dict[str, Any], scale: float) -> None: + r = rect_pdf + super().__init__(QRectF(r[0] * scale, r[1] * scale, (r[2] - r[0]) * scale, (r[3] - r[1]) * scale)) + self.suggestion = suggestion + pen = QPen(QColor("#f97316"), 2) + pen.setStyle(Qt.PenStyle.DashLine) + pen.setCosmetic(True) + self.setPen(pen) + self.setBrush(QColor(249, 115, 22, 40)) + self.setZValue(10) + self.setToolTip( + f"Suggerimento: {suggestion.get('entity_type', '')} " + f"({float(suggestion.get('score', 0.0)):.2f}) - clic per decidere" + ) + + def set_highlight(self, on: bool) -> None: + pen = QPen(QColor("#f97316"), 4 if on else 2) + pen.setStyle(Qt.PenStyle.SolidLine if on else Qt.PenStyle.DashLine) + pen.setCosmetic(True) + self.setPen(pen) + + class NullifyPDF(QMainWindow): """Main application window for PDF privacy redaction using AI. @@ -1119,7 +1315,7 @@ class NullifyPDF(QMainWindow): config_dir: Path to user config directory (~/.nullifypdf). """ - start_scan_sig = Signal(object, object, str, list, set, bool, object, list) + start_scan_sig = Signal(object, object, str, list, set, bool, object, list, bool) def __init__(self) -> None: """Initialize main application window and setup UI.""" @@ -1139,13 +1335,20 @@ def __init__(self) -> None: # Mutex guards `self.doc` against concurrent access from the AI worker # thread (text extraction) and the UI thread (render, apply redactions). self.doc_mutex = QMutex() + # Detections the evidence score could not settle; never applied + # silently, reviewed by the user (see review_dialog.ReviewDialog). + self.pending_suggestions: List[Dict[str, Any]] = [] + self._export_labels: List[Tuple[int, Any, str, Optional[Dict[str, Any]]]] = [] self.config_dir = pathlib.Path.home() / ".nullifypdf" # Update-check threads outlive their dialogs: keep them referenced # until they finish so a QThread is never destroyed while running. self._bg_threads: Set[QThread] = set() + cleanup_stale_temp(self.config_dir) self.list_manager = PDFListManager(self.config_dir) self.blocklist = self.list_manager.load_blocklist() self.allowlist = self.list_manager.load_allowlist() + if self.list_manager.keystore_error: + self.logger.error("Word lists unreadable: keystore unavailable") self.ai_thread = QThread() self.ai_worker = AIWorker() self.ai_worker.moveToThread(self.ai_thread) @@ -1156,6 +1359,15 @@ def __init__(self) -> None: self.start_scan_sig.connect(self.ai_worker.run_scan) self.ai_thread.start() self.build_ui() + if self.list_manager.expired_removed: + self.write_log( + f"Rimossi {self.list_manager.expired_removed} termini scaduti dai dizionari." + ) + if self.list_manager.keystore_error: + self.write_log( + "Avviso: dizionari cifrati non leggibili su questo account/PC; " + "verranno usati dizionari vuoti finché non vengono ripristinati." + ) # Setup signal handlers for graceful shutdown self._setup_signal_handlers() # Register cleanup on exit @@ -1269,7 +1481,12 @@ def build_ui(self) -> None: self.rb_en = QRadioButton("EN") self.rb_it = QRadioButton("IT") self.rb_both = QRadioButton("BOTH") - self.rb_en.setChecked(True) + # Default to the interface language: running the English model on + # Italian text only adds false positives. + if QLocale.system().language() == QLocale.Language.Italian: + self.rb_it.setChecked(True) + else: + self.rb_both.setChecked(True) self.lang_grp = QButtonGroup(self) for rb in (self.rb_en, self.rb_it, self.rb_both): self.lang_grp.addButton(rb) @@ -1281,6 +1498,23 @@ def build_ui(self) -> None: self.chk_ocr = QCheckBox("OCR PDF scansionati") self.chk_ocr.setChecked(True) s_lay.addWidget(self.chk_ocr) + self.chk_sugg = QCheckBox("Mostra suggerimenti") + self.chk_sugg.setChecked(True) + self.chk_sugg.setToolTip("Contorna in arancione, nell'anteprima, i suggerimenti da rivedere.") + self.chk_sugg.toggled.connect(lambda _on: self.render_page()) + s_lay.addWidget(self.chk_sugg) + self.chk_health = QCheckBox("Profilo documenti sanitari") + self.chk_health.setToolTip( + "Rileva anche date di nascita e identificativi sanitari e non " + "scarta in silenzio i casi incerti: li propone in revisione." + ) + s_lay.addWidget(self.chk_health) + self.chk_mask_width = QCheckBox("Maschera larghezza testo rimosso") + self.chk_mask_width.setToolTip( + "Arrotonda lo spazio lasciato dal testo rimosso, che altrimenti " + "ne rivela la larghezza. Puo spostare di poco il testo seguente." + ) + s_lay.addWidget(self.chk_mask_width) s_lay.addSpacing(20) btn_dict = QPushButton("Dizionari") btn_dict.clicked.connect(self.cmd_dict) @@ -1289,6 +1523,10 @@ def build_ui(self) -> None: self.btn_ai.setObjectName("Danger") self.btn_ai.clicked.connect(self.cmd_auto_ai) s_lay.addWidget(self.btn_ai) + self.btn_review = QPushButton("Rivedi suggerimenti (0)") + self.btn_review.clicked.connect(self._review_suggestions) + self.btn_review.setEnabled(False) + s_lay.addWidget(self.btn_review) btn_clear = QPushButton("Pulisci Pagina") btn_clear.clicked.connect(self.cmd_clear) s_lay.addWidget(btn_clear) @@ -1300,6 +1538,13 @@ def build_ui(self) -> None: btn_reconstruct = QPushButton("Ricostruisci PDF") btn_reconstruct.clicked.connect(self.cmd_reconstruct) s_lay.addWidget(btn_reconstruct) + btn_wipe = QPushButton("Elimina dati locali") + btn_wipe.setToolTip( + "Cancella log, registri di esportazione e file temporanei salvati " + "in ~/.nullifypdf." + ) + btn_wipe.clicked.connect(self.cmd_wipe_local_data) + s_lay.addWidget(btn_wipe) s_lay.addStretch() btn_about = QPushButton("Info") btn_about.clicked.connect(self.cmd_about) @@ -1437,6 +1682,8 @@ def load_path(self, path: str) -> None: self.doc = tdoc self.page_num = 0 self.scale = 1.5 + self.pending_suggestions = [] + self._update_review_button() self.adjust_zoom(0) self.write_log(f"Caricato: {os.path.basename(path)}") self._warn_if_scanned_pdf() @@ -1497,6 +1744,53 @@ def render_page(self) -> None: self.scene.setSceneRect(0, 0, pix.width, pix.height) self.le_page.setText(str(self.page_num + 1)) self.lbl_tot.setText(f"/ {total}") + self._draw_suggestion_overlays() + + def _draw_suggestion_overlays(self) -> None: + """Outline the pending suggestions of the current page (view only).""" + if not self.chk_sugg.isChecked(): + return + for s in self.pending_suggestions: + if int(s.get("page", -1)) != self.page_num: + continue + for rect in s.get("rects") or []: + self.scene.addItem(SuggestionItem(tuple(rect), s, self.scale)) + + def _highlight_suggestion(self, suggestion: Any) -> None: + for it in self.scene.items(): + if isinstance(it, SuggestionItem): + it.set_highlight(it.suggestion is suggestion) + + def _decide_suggestion(self, s: Dict[str, Any]) -> None: + """Accept or discard one suggestion clicked in the page view.""" + reveal = False + while True: + value = " ".join(str(s.get("text", "")).split()) + shown = "(area grafica)" if not value else (value if reveal else value[:1] + "•" * min(len(value) - 1, 8)) + box = QMessageBox(self) + box.setWindowTitle("Suggerimento") + box.setText( + f"{s.get('entity_type', '')} - {shown}\n" + f"Punteggio {float(s.get('score', 0.0)):.2f}: " + + ", ".join(str(r) for r in s.get("reasons", [])) + ) + b_yes = box.addButton("Censura", QMessageBox.ButtonRole.AcceptRole) + b_no = box.addButton("Scarta", QMessageBox.ButtonRole.DestructiveRole) + b_show = box.addButton("Mostra valore", QMessageBox.ButtonRole.ActionRole) + box.addButton("Annulla", QMessageBox.ButtonRole.RejectRole) + box.exec() + clicked = box.clickedButton() + if clicked == b_show and value and not reveal: + reveal = True + continue + break + if clicked not in (b_yes, b_no): + return + if clicked == b_yes: + self._apply_detections(int(s.get("page", self.page_num)), [s]) + self.pending_suggestions = [p for p in self.pending_suggestions if p is not s] + self._update_review_button() + self.render_page() def adjust_zoom(self, d: int) -> None: """Adjust zoom level. @@ -1555,7 +1849,7 @@ def user_draw_rect(self, qrect: QRectF) -> None: if not self.doc: return p = self.doc[self.page_num] - txt = p.get_text("text", clip=r).strip() + txt = text_under_rect(p, fitz.Rect(r)) if not txt: p.add_redact_annot( r, @@ -1601,6 +1895,10 @@ def user_click_pt(self, qpt: QPointF) -> None: """ if not self.doc: return + overlays = [it for it in self.scene.items(qpt) if isinstance(it, SuggestionItem)] + if overlays: + self._decide_suggestion(overlays[0].suggestion) + return pt = fitz.Point(qpt.x() / self.scale, qpt.y() / self.scale) cl = "" with QMutexLocker(self.doc_mutex): @@ -1614,7 +1912,7 @@ def user_click_pt(self, qpt: QPointF) -> None: ] if not ans: return - txt = p.get_text("text", clip=ans[0].rect) + txt = text_under_rect(p, ans[0].rect) cl = " ".join(txt.split()).lower() add_to_whitelist = False @@ -1681,40 +1979,13 @@ def cmd_clear(self) -> None: self.write_log(f"Censure rimosse su pagina {self.page_num+1}") def cmd_dict(self) -> None: - """Open dialog to edit blocklist/allowlist.""" - d = QDialog(self) - d.setWindowTitle("Dizionari") - d.resize(500, 450) - lay = QVBoxLayout(d) - lay.addWidget(QLabel("🔴 BLOCKLIST")) - bx = QTextEdit() - bx.setPlainText("\n".join(sorted(self.blocklist))) - lay.addWidget(bx) - lay.addWidget(QLabel("🟢 ALLOWLIST")) - ax = QTextEdit() - ax.setPlainText("\n".join(sorted(self.allowlist))) - lay.addWidget(ax) - btn = QPushButton("Salva") - btn.setObjectName("Primary") - - def s() -> None: - self.blocklist = { - line.strip().lower() - for line in bx.toPlainText().split("\n") - if len(line.strip()) > 2 - } - self.allowlist = { - line.strip().lower() - for line in ax.toPlainText().split("\n") - if len(line.strip()) > 2 - } - self.list_manager.save_blocklist(self.blocklist) - self.list_manager.save_allowlist(self.allowlist) - d.accept() - - btn.clicked.connect(s) - lay.addWidget(btn) - d.exec() + """Open the dictionary editor (blocklist / allowlist with expiry).""" + d = DictionaryDialog(self.list_manager, self) + if d.exec() == QDialog.DialogCode.Accepted: + if d.saved_blocklist is not None: + self.blocklist = d.saved_blocklist + if d.saved_allowlist is not None: + self.allowlist = d.saved_allowlist def _start_bg_thread(self, w: QThread) -> None: """Start ``w`` and keep it referenced until it has finished.""" @@ -1770,7 +2041,7 @@ def cmd_about(self) -> None: lbl_ver.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(lbl_ver) desc = QLabel( - "\nAnonimizzazione PDF Offline.\n\nSviluppato da: Graziano Mariella\nLicenza MIT" + "\nCensura e pseudonimizzazione PDF offline.\n\nSviluppato da: Graziano Mariella\nLicenza MIT" ) desc.setAlignment(Qt.AlignmentFlag.AlignCenter) lay.addWidget(desc) @@ -1939,8 +2210,22 @@ def cmd_auto_ai(self) -> None: enabled_groups = self._select_ai_entities() if not enabled_groups: return + health_mode = self.chk_health.isChecked() + if health_mode: + for gid in ("BIRTHDATE", "HEALTHID", "EXEMPT"): + if gid not in enabled_groups: + enabled_groups = list(enabled_groups) + [gid] self.btn_ai.setEnabled(False) self.prog.setValue(0) + # Text in a layer that is switched off is invisible to extraction, so + # neither the detector nor the reviewer would ever see it: show all + # layers (in memory only) before scanning. + with QMutexLocker(self.doc_mutex): + revealed = reveal_hidden_layers(self.doc) + if revealed: + self.write_log( + f"Avviso: {revealed} livelli nascosti resi visibili per l'analisi." + ) c_allow = [ (a, re.compile(r"\b" + re.escape(a) + r"\b")) for a in self.allowlist ] @@ -1972,6 +2257,7 @@ def cmd_auto_ai(self) -> None: use_ocr, tessdata_dir, expand_group_ids(enabled_groups), + health_mode, ) def _select_ai_entities(self) -> Optional[List[str]]: @@ -2032,11 +2318,15 @@ def _select_privacy_export(self) -> Optional[PrivacyMode]: box.setIcon(QMessageBox.Icon.Question) box.setText("Scegli come trattare i dati personali nel PDF esportato.") box.setInformativeText( - "Anonimizzazione: rimozione irreversibile.\n" - "Pseudonimizzazione: segnaposto reversibili con mappa cifrata separata." + "Rimozione irreversibile: i valori selezionati vengono eliminati " + "dal file. Il risultato puo contenere ancora dati personali non " + "rilevati o indiretti (date, luoghi, dettagli): controllalo prima " + "di condividerlo.\n\n" + "Pseudonimizzazione: segnaposto reversibili con mappa cifrata " + "separata. Il file e la mappa restano dati personali." ) anonymize_btn = box.addButton( - "Anonimizzazione irreversibile", QMessageBox.ButtonRole.AcceptRole + "Rimozione irreversibile", QMessageBox.ButtonRole.AcceptRole ) pseudonymize_btn = box.addButton( "Pseudonimizzazione reversibile", QMessageBox.ButtonRole.ActionRole @@ -2122,7 +2412,7 @@ def _prepare_pseudonymized_page( ( annot.rect, self._read_redaction_payload(annot), - page.get_text("text", clip=annot.rect).strip(), + text_under_rect(page, annot.rect), capture_text_style(page, annot.rect), ) for annot in (page.annots() or []) @@ -2132,6 +2422,16 @@ def _prepare_pseudonymized_page( if annot.type[0] == PDF_ANNOT_REDACT: page.delete_annot(annot) for rect, payload, clipped_text, style in pending: + if payload.get("entity_type") == "SIGNATURE": + page.add_redact_annot( + rect, + text="[FIRMA RIMOSSA]", + fill=(1, 1, 1), + text_color=(0, 0, 0), + align=1, + fontsize=fit_redaction_fontsize("[FIRMA RIMOSSA]", rect), + ) + continue clean_original = " ".join( (payload.get("original") or clipped_text).split() ) @@ -2146,16 +2446,12 @@ def _prepare_pseudonymized_page( rect=(rect.x0, rect.y0, rect.x1, rect.y1), style=style, ) - page.add_redact_annot( - rect, - text=placeholder, - fill=(1, 1, 1), - text_color=(0, 0, 0), - align=1, - # Must fit on one line: a wrapped placeholder is still - # readable but no longer findable by search_for(), which - # is the only handle a version 1 restore map has. - fontsize=fit_redaction_fontsize(placeholder, rect), + # The placeholder is drawn after apply_redactions() (see + # _finish_pseudonymized_page): replacement text passed to + # the annotation is dropped when it cannot be laid out at 4 pt. + page.add_redact_annot(rect, fill=(1, 1, 1)) + self._export_labels.append( + (page.number, fitz.Rect(rect), placeholder, style) ) else: page.add_redact_annot( @@ -2167,12 +2463,32 @@ def _prepare_pseudonymized_page( fontsize=fit_redaction_fontsize("[IMAGE_REMOVED]", rect), ) + def _finish_pseudonymized_page( + self, page: Any, registry: PlaceholderRegistry + ) -> None: + """Draw the placeholders of `page` once its redactions are applied.""" + for page_no, rect, placeholder, style in self._export_labels: + if page_no != page.number: + continue + drawn = write_label( + page, + rect, + compact_labels(placeholder), + style=style, + allowance=free_space_right(page, rect), + ) + if drawn != placeholder: + registry.set_label( + placeholder, (rect.x0, rect.y0, rect.x1, rect.y1), drawn + ) + @Slot(int, object) def apply_ai_to_page(self, i: int, detections: Any) -> None: """Apply AI-detected redactions to page. - Runs on the UI thread (Qt slot). Holds `doc_mutex` while mutating the - page so the AI worker's `get_text()` calls do not race. + Runs on the UI thread (Qt slot). Detections in state "accepted" are + applied; those in state "suggested" are queued for review and never + applied silently. Args: i: Page index. @@ -2187,17 +2503,42 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: ] else: normalized_detections = list(detections or []) + accepted = [ + d for d in normalized_detections if d.get("state", "accepted") == "accepted" + ] + for d in normalized_detections: + if d.get("state") == "suggested": + self.pending_suggestions.append({**d, "page": i}) + self._update_review_button() + self._apply_detections(i, accepted) + + def _update_review_button(self) -> None: + n = len(self.pending_suggestions) + self.btn_review.setText(f"Rivedi suggerimenti ({n})") + self.btn_review.setEnabled(n > 0) + + def _apply_detections(self, i: int, detections: List[Dict[str, Any]]) -> None: + """Add redaction annotations for `detections` on page `i`.""" with QMutexLocker(self.doc_mutex): - if i >= len(self.doc): + if not self.doc or i >= len(self.doc): return page = self.doc[i] e_rects = [ a.rect for a in (page.annots() or []) if a.type[0] == PDF_ANNOT_REDACT ] + + def already_covered(r: fitz.Rect) -> bool: + return covered_fraction( + (r.x0, r.y0, r.x1, r.y1), + [(e.x0, e.y0, e.x1, e.y1) for e in e_rects], + ) >= 0.6 + if self.chk_img.isChecked(): for img in page.get_image_info(hashes=False): ir = fitz.Rect(img["bbox"]) + if already_covered(ir): + continue page.add_redact_annot( ir, text="[ IMMAGINE RIMOSSA ]", @@ -2206,13 +2547,19 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: fontsize=8, ) e_rects.append(ir) - page_index = build_page_index(page.get_text("words")) + try: + has_images = bool(page.get_image_info(hashes=False)) + except Exception: + has_images = False + page_index = build_page_index( + page.get_text("words"), + image_rects=[ + tuple(i["bbox"]) for i in page.get_image_info(hashes=False) + ], + ) for bw in self.blocklist: for r in (fitz.Rect(*t) for t in whole_word_rects(page_index, bw)): - if not any( - e.contains(fitz.Point((r.x0 + r.x1) / 2, (r.y0 + r.y1) / 2)) - for e in e_rects - ): + if not already_covered(r): self._add_privacy_redaction( page, r, @@ -2220,25 +2567,51 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: entity_type=self._infer_entity_type(bw), ) e_rects.append(r) - p_rects = [ - fitz.Rect(*t) - for aw in self.allowlist - for t in whole_word_rects(page_index, aw) - ] - for detection in normalized_detections: + for detection in detections: word = str(detection.get("text", "")) entity_type = str(detection.get("entity_type", "DATA")) raw_rects = detection.get("rects") or [] rects = [fitz.Rect(*coords) for coords in raw_rects] + fulls = detection.get("full_rects") or [] + for k, band in enumerate(rects): + if k < len(fulls): + full = fitz.Rect(*fulls[k]) + if band != full and not band_covers_target(page, band, full): + rects[k] = full # never leave a glyph of the target behind + if entity_type == "SIGNATURE": + # Graphic area (signature / stamp): no text, no value. + for r in rects: + if already_covered(r): + continue + annot = page.add_redact_annot(r, fill=(0.9, 0.9, 0.9)) + try: + annot.set_info(content=json.dumps({ + "nullifypdf": 1, "original": "", + "entity_type": "SIGNATURE", + })) + except Exception as e: + self.logger.debug(f"Could not store signature metadata: {e}") + e_rects.append(r) + continue if not rects and word: - rects = list(page.search_for(word)) + # Legacy detections without geometry: whole words only, + # never a substring search (it redacts fragments of + # longer words), and never a word the allowlist protects. + protected = [ + fitz.Rect(*t) + for aw in self.allowlist + for t in whole_word_rects(page_index, aw) + ] + rects = [ + fitz.Rect(*t) + for t in whole_word_rects(page_index, word) + if not any( + pr.contains(fitz.Point((t[0] + t[2]) / 2, (t[1] + t[3]) / 2)) + for pr in protected + ) + ] for r in rects: - if any(r.intersects(pr) for pr in p_rects): - continue - if not any( - e.contains(fitz.Point((r.x0 + r.x1) / 2, (r.y0 + r.y1) / 2)) - for e in e_rects - ): + if not already_covered(r): self._add_privacy_redaction( page, r, @@ -2247,179 +2620,569 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: ) e_rects.append(r) + def _review_suggestions(self) -> None: + """Let the user accept or reject the queued suggestions.""" + if not self.pending_suggestions: + return + dlg = ReviewDialog(self.pending_suggestions, self) + dlg.navigate.connect(self._go_to_page) + dlg.highlight.connect(self._highlight_suggestion) + if dlg.exec() != QDialog.DialogCode.Accepted: + return + by_page: Dict[int, List[Dict[str, Any]]] = {} + for item in dlg.applied(): + by_page.setdefault(int(item.get("page", 0)), []).append(item) + for page_no, items in by_page.items(): + self._apply_detections(page_no, items) + decided = dlg.applied() + dlg.declined() + self.pending_suggestions = [ + p for p in self.pending_suggestions + if not any(p is d for d in decided) + ] + self._update_review_button() + self.render_page() + self.write_log( + f"Revisione: {len(dlg.applied())} applicati, " + f"{len(dlg.declined())} scartati." + ) + + def _go_to_page(self, page_no: int) -> None: + if self.doc and 0 <= page_no < len(self.doc): + self.page_num = page_no + self.render_page() + @Slot() def ai_finished(self) -> None: """Handle AI scan completion.""" self.btn_ai.setEnabled(True) self.render_page() + if self.pending_suggestions: + self._review_suggestions() + + def _redaction_counts(self) -> Dict[str, int]: + """Pending redactions per entity type (no values).""" + counts: Dict[str, int] = {} + for page in self.doc or []: + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + payload = self._read_redaction_payload(annot) + key = payload.get("entity_type") or "UNSPECIFIED" + counts[key] = counts.get(key, 0) + 1 + return counts + + def _select_pseudonym_style(self) -> Optional[Tuple[str, Optional[bytes]]]: + """Ask how pseudonyms are generated. Returns (style, key) or None.""" + box = QMessageBox(self) + box.setWindowTitle("Stile dei segnaposto") + box.setIcon(QMessageBox.Icon.Question) + box.setText("Come devono essere generati i segnaposto?") + box.setInformativeText( + "Casuale: non rivela ordine o numero dei valori (consigliato).\n" + "Progressivo: PERSON_001, PERSON_002... rivela quanti valori " + "distinti esistono e l'ordine in cui compaiono.\n" + "Generico: nasconde anche il tipo di dato.\n" + "Coerente: lo stesso valore riceve lo stesso segnaposto in tutti i " + "documenti creati con la stessa chiave (permette di collegarli)." + ) + b_random = box.addButton("Casuale", QMessageBox.ButtonRole.AcceptRole) + b_counter = box.addButton("Progressivo", QMessageBox.ButtonRole.ActionRole) + b_opaque = box.addButton("Generico", QMessageBox.ButtonRole.ActionRole) + b_keyed = box.addButton("Coerente (chiave)", QMessageBox.ButtonRole.ActionRole) + b_manage = box.addButton("Gestisci chiave...", QMessageBox.ButtonRole.ActionRole) + box.addButton(QMessageBox.StandardButton.Cancel) + box.exec() + clicked = box.clickedButton() + if clicked == b_random: + return "random", None + if clicked == b_counter: + return "counter", None + if clicked == b_opaque: + return "opaque", None + if clicked == b_manage: + self._pseudonym_key_dialog() + return self._select_pseudonym_style() + if clicked == b_keyed: + try: + key = self._load_pseudonym_key() + except (secure_store.KeystoreUnavailable, secure_store.SealError, OSError) as e: + QMessageBox.critical(self, "Chiave non disponibile", str(e)) + return None + QMessageBox.information( + self, + "Chiave dei segnaposto", + "La chiave e protetta (" + + secure_store.protector_label(self.config_dir) + + "). Conservane una copia protetta da password con " + "\"Gestisci chiave...\" per usarla su un altro computer: chi " + "la possiede puo ricollegare i segnaposto ai valori noti, e " + "senza di essa i documenti futuri avranno segnaposto diversi.", + ) + return "keyed", key + return None + return None + + def _pseudonym_key_paths(self) -> Tuple[pathlib.Path, pathlib.Path]: + return self.config_dir / "pseudonym.v2.enc", self.config_dir / "pseudonym.key" + + def _store_pseudonym_key(self, key: bytes) -> None: + enc, _legacy = self._pseudonym_key_paths() + master = secure_store.load_or_create_master_key(self.config_dir) + secure_store.atomic_write( + enc, secure_store.seal(key, master, b"nullifypdf-pseudonym-key") + ) + + def _load_pseudonym_key(self) -> bytes: + """The 32-byte pseudonym key, protected by the local master key. + + A plaintext ``pseudonym.key`` from an older version is validated, + sealed and removed. Raises on an invalid or unreadable key. + """ + enc, legacy = self._pseudonym_key_paths() + master = secure_store.load_or_create_master_key(self.config_dir) + if enc.exists(): + key = secure_store.open_sealed( + enc.read_bytes(), master, b"nullifypdf-pseudonym-key" + ) + elif legacy.exists(): + key = legacy.read_bytes() + if len(key) != 32: + raise OSError( + "Chiave dei segnaposto non valida: il file deve " + "contenere esattamente 32 byte." + ) + self._store_pseudonym_key(key) + secure_store.secure_delete(legacy) + else: + key = os.urandom(32) + self._store_pseudonym_key(key) + if len(key) != 32: + raise OSError("Chiave dei segnaposto non valida.") + return key + + def _pseudonym_key_dialog(self) -> None: + """Export / import a password-protected copy of the pseudonym key.""" + box = QMessageBox(self) + box.setWindowTitle("Chiave dei segnaposto") + box.setText("Esporta o importa la chiave dei segnaposto coerenti.") + box.setInformativeText( + "La copia esportata e protetta da password e serve per usare la " + "stessa chiave su un altro computer." + ) + b_exp = box.addButton("Esporta chiave...", QMessageBox.ButtonRole.ActionRole) + b_imp = box.addButton("Importa chiave...", QMessageBox.ButtonRole.ActionRole) + box.addButton(QMessageBox.StandardButton.Close) + box.exec() + clicked = box.clickedButton() + try: + if clicked == b_exp: + key = self._load_pseudonym_key() + pw = self._ask_map_password() + if pw is None: + return + path, _ = QFileDialog.getSaveFileName( + self, "Esporta chiave", "pseudonym_key.nullifypdf-key", + "NullifyPDF key (*.nullifypdf-key)", + ) + if path: + write_private_file(path, export_pseudonym_key(key, pw)) + self.write_log("Chiave dei segnaposto esportata.") + elif clicked == b_imp: + path, _ = QFileDialog.getOpenFileName( + self, "Importa chiave", "", "NullifyPDF key (*.nullifypdf-key)" + ) + if not path: + return + pw, ok = QInputDialog.getText( + self, "Password chiave", "Password della copia della chiave:", + QLineEdit.EchoMode.Password, + ) + if not ok or not pw: + return + enc, legacy = self._pseudonym_key_paths() + if enc.exists() or legacy.exists(): + ans = QMessageBox.question( + self, "Sostituire la chiave?", + "Esiste gia una chiave: sostituirla cambia i segnaposto " + "dei documenti futuri. Continuare?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return + with open(path, "rb") as fh: + key = import_pseudonym_key(fh.read(), pw) + self._store_pseudonym_key(key) + self.write_log("Chiave dei segnaposto importata.") + except Exception as e: + from cryptography.fernet import InvalidToken + + msg = "Password errata o file corrotto." if isinstance(e, InvalidToken) else str(e) + QMessageBox.critical(self, "Chiave dei segnaposto", msg) + + def _offer_recovery_key(self) -> Optional[bytes]: + """Offer a recovery key for the map; shown once, never stored.""" + ans = QMessageBox.question( + self, + "Chiave di recupero", + "Vuoi creare anche una chiave di recupero? Permette di aprire la " + "mappa se dimentichi la password. Non viene salvata dal programma.", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return None + raw, shown = generate_recovery_key() + dlg = QDialog(self) + dlg.setWindowTitle("Chiave di recupero") + lay = QVBoxLayout(dlg) + lay.addWidget(QLabel( + "Conserva questa chiave in un luogo sicuro, separato dalla mappa. " + "Viene mostrata una sola volta." + )) + field = QLineEdit(shown) + field.setReadOnly(True) + lay.addWidget(field) + row = QHBoxLayout() + btn_copy = QPushButton("Copia") + btn_copy.clicked.connect(lambda: QApplication.clipboard().setText(shown)) + btn_save = QPushButton("Salva su file...") + + def save_file() -> None: + path, _ = QFileDialog.getSaveFileName( + dlg, "Salva chiave di recupero", "recovery_key.txt", "Testo (*.txt)" + ) + if path: + write_private_file(path, shown.encode("utf-8")) + + btn_save.clicked.connect(save_file) + row.addWidget(btn_copy) + row.addWidget(btn_save) + lay.addLayout(row) + chk = QCheckBox( + "Ho conservato la chiave di recupero in un luogo sicuro, separato dalla mappa" + ) + lay.addWidget(chk) + bb = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok) + ok_btn = bb.button(QDialogButtonBox.StandardButton.Ok) + ok_btn.setEnabled(False) + chk.toggled.connect(ok_btn.setEnabled) + bb.accepted.connect(dlg.accept) + lay.addWidget(bb) + dlg.exec() + QApplication.clipboard().clear() + # The key has been shown (and maybe saved): the map must carry its + # wrap even if the dialog was closed without ticking the box. + return raw + + def _ask_decrypt_secret(self) -> Optional[Tuple[Optional[str], Optional[bytes]]]: + """Ask for the map password, or the recovery key. None = cancelled.""" + dlg = QDialog(self) + dlg.setWindowTitle("Password mappa") + lay = QVBoxLayout(dlg) + label = QLabel("Password per decifrare la mappa di ripristino:") + lay.addWidget(label) + field = QLineEdit() + field.setEchoMode(QLineEdit.EchoMode.Password) + lay.addWidget(field) + use_rk = QCheckBox("Usa la chiave di recupero") + + def toggled(on: bool) -> None: + label.setText( + "Chiave di recupero (NPDF-...):" if on + else "Password per decifrare la mappa di ripristino:" + ) + + use_rk.toggled.connect(toggled) + lay.addWidget(use_rk) + bb = QDialogButtonBox( + QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel + ) + bb.accepted.connect(dlg.accept) + bb.rejected.connect(dlg.reject) + lay.addWidget(bb) + if dlg.exec() != QDialog.DialogCode.Accepted or not field.text(): + return None + if use_rk.isChecked(): + try: + return None, parse_recovery_key(field.text()) + except ValueError as e: + QMessageBox.warning(self, "Chiave di recupero", str(e)) + return None + return field.text(), None + + def _ask_map_password(self) -> Optional[str]: + """Ask twice for the map password and enforce a minimum strength.""" + while True: + pw, ok = QInputDialog.getText( + self, + "Password mappa", + "Password per cifrare la mappa di ripristino (minimo 12 caratteri):", + QLineEdit.EchoMode.Password, + ) + if not ok: + return None + good, why = password_strength(pw) + if not good: + QMessageBox.warning(self, "Password non valida", why) + continue + confirm, ok = QInputDialog.getText( + self, + "Conferma password", + "Ripeti la password (senza di essa la mappa non e recuperabile):", + QLineEdit.EchoMode.Password, + ) + if not ok: + return None + if confirm != pw: + QMessageBox.warning( + self, "Password diversa", "Le due password non coincidono." + ) + continue + return pw + + def cmd_wipe_local_data(self) -> None: + """Delete logs, audit records and temp files kept in ~/.nullifypdf.""" + ans = QMessageBox.question( + self, + "Elimina dati locali", + "Verranno eliminati log, registri di esportazione e file " + "temporanei. Vuoi eliminare anche i dizionari (blocklist e " + "allowlist)?", + QMessageBox.StandardButton.Yes + | QMessageBox.StandardButton.No + | QMessageBox.StandardButton.Cancel, + QMessageBox.StandardButton.No, + ) + if ans == QMessageBox.StandardButton.Cancel: + return + include_lists = ans == QMessageBox.StandardButton.Yes + include_keys = False + if include_lists: + include_keys = QMessageBox.question( + self, + "Elimina anche le chiavi", + "Eliminare anche le chiavi? I dizionari cifrati e la chiave dei " + "segnaposto diventeranno illeggibili.", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) == QMessageBox.StandardButton.Yes + # The rotating log file is open: release it first or it cannot be + # removed (Windows), then reopen logging afterwards. + for h in list(self.logger.handlers): + try: + h.close() + except Exception: + pass + self.logger.removeHandler(h) + removed = wipe_local_data( + self.config_dir, include_lists=include_lists, include_keys=include_keys + ) + self.logger = setup_logging() + if include_lists: + self.blocklist, self.allowlist = set(), set() + self.write_log(f"Dati locali eliminati: {', '.join(removed) or 'nessuno'}.") def cmd_export(self) -> None: - """Export current PDF in irreversible or reversible privacy mode. - - Memory strategy (fix C1): instead of materializing the whole document - in RAM via `self.doc.write()` (which would double peak memory on large - PDFs), we stream through a sibling temp file next to the target path: - - 1. Save `self.doc` (with annotations, NOT yet flattened) to - `.tmp` on disk. - 2. Open the temp file as `ex_doc`. - 3. Apply redactions / scrub metadata on `ex_doc`. - 4. Save `ex_doc` to the user-chosen final path. - 5. Remove the temp file. - - The original `self.doc` is never mutated (redactions stay as - annotations), so the user can keep editing after export. Peak extra - memory is O(1) over what PyMuPDF needs internally for save(). + """Export the current PDF in irreversible or reversible privacy mode. + + The work happens in :func:`export_engine.export_document` on a copy + kept in memory (or in a private per-user folder for very large + files): the original ``self.doc`` is never mutated, so the user can + keep editing, and no unredacted copy is ever written beside the + output. The written file is sanitised and then verified. """ if not self.doc: return + if self.pending_suggestions: + box = QMessageBox(self) + box.setIcon(QMessageBox.Icon.Warning) + box.setWindowTitle("Suggerimenti non revisionati") + box.setText( + f"Ci sono {len(self.pending_suggestions)} suggerimenti non " + "revisionati: non verranno censurati." + ) + b_review = box.addButton("Rivedi ora", QMessageBox.ButtonRole.AcceptRole) + b_go = box.addButton("Esporta comunque", QMessageBox.ButtonRole.DestructiveRole) + box.addButton(QMessageBox.StandardButton.Cancel) + box.exec() + if box.clickedButton() == b_review: + self._review_suggestions() + return + if box.clickedButton() != b_go: + return mode = self._select_privacy_export() if mode is None: return + with QMutexLocker(self.doc_mutex): + removed_values = collect_removed_values( + self.doc, self._read_redaction_payload + ) + entity_counts = self._redaction_counts() + src_dir = os.path.dirname(self.doc.name or "") or str(pathlib.Path.home()) p, _ = QFileDialog.getSaveFileName( self, "Esporta", - f"{os.path.splitext(self.doc.name)[0]}_{mode.value}.pdf", + os.path.join(src_dir, neutral_output_name(mode.value)), "PDF (*.pdf)", ) if not p: return + if name_leaks_value(p, removed_values): + ans = QMessageBox.question( + self, + "Nome file con dati personali", + "Il nome scelto contiene un valore che stai rimuovendo. " + "Continuare comunque?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return map_path = "" map_password = "" + recovery_key: Optional[bytes] = None + style, key = "counter", None if mode == PrivacyMode.PSEUDONYMIZE: + if importlib.util.find_spec("cryptography") is None: + QMessageBox.critical( + self, + "Dipendenza mancante", + "La pseudonimizzazione richiede la libreria 'cryptography'.", + ) + return + chosen = self._select_pseudonym_style() + if chosen is None: + return + style, key = chosen map_path, _ = QFileDialog.getSaveFileName( self, "Salva mappa cifrata", - f"{os.path.splitext(p)[0]}.nullifypdf-map", + os.path.join(str(pathlib.Path.home()), neutral_map_name()), "NullifyPDF map (*.nullifypdf-map)", ) if not map_path: return - map_password, ok = QInputDialog.getText( - self, - "Password mappa", - "Password per cifrare la mappa di ripristino (minimo 12 caratteri):", - QLineEdit.EchoMode.Password, + same_dir = os.path.dirname(os.path.abspath(map_path)) == os.path.dirname( + os.path.abspath(p) ) - if not ok: - return - if len(map_password) < 12: - QMessageBox.warning( + synced = any( + tag in os.path.abspath(map_path).lower() + for tag in ("onedrive", "dropbox", "google drive", "icloud") + ) + if same_dir or synced: + ans = QMessageBox.question( self, - "Password troppo corta", - "La password della mappa deve avere almeno 12 caratteri.", - ) - return - if importlib.util.find_spec("cryptography") is None: - QMessageBox.critical( - self, - "Dipendenza mancante", - "La pseudonimizzazione richiede la libreria 'cryptography'.", + "Posizione della mappa", + "La mappa di ripristino e dato personale e va conservata " + "separatamente dal PDF pseudonimizzato" + + (", non in una cartella sincronizzata" if synced else "") + + ". Salvarla qui comunque?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, ) + if ans != QMessageBox.StandardButton.Yes: + return + pw = self._ask_map_password() + if pw is None: return + map_password = pw + recovery_key = self._offer_recovery_key() self.write_log( - "Export anonimizzato..." if mode == PrivacyMode.ANONYMIZE + "Export con rimozione irreversibile..." if mode == PrivacyMode.ANONYMIZE else "Export pseudonimizzato..." ) - # Sibling temp file (same directory as target so rename/cleanup are - # always on the same filesystem; avoids cross-device issues). - tmp_path = p + ".nullifypdf.tmp" - ex_doc = None - registry = PlaceholderRegistry() + registry = PlaceholderRegistry(style=style, key=key) + policy = SanitizePolicy(mask_removed_width=self.chk_mask_width.isChecked()) try: source_name = self.doc.name or "document.pdf" source_sha256 = sha256_file(source_name) if os.path.exists(source_name) else "" - # Step 1: serialize `self.doc` (with annotations) to disk under - # the mutex so the AI worker thread cannot mutate mid-write. + prepare = ( + (lambda page: self._prepare_pseudonymized_page(page, registry)) + if mode == PrivacyMode.PSEUDONYMIZE else None + ) + finish = ( + (lambda page: self._finish_pseudonymized_page(page, registry)) + if mode == PrivacyMode.PSEUDONYMIZE else None + ) + self._export_labels = [] with QMutexLocker(self.doc_mutex): if not self.doc: return - # `clean=False` here: we want a faithful copy of the - # in-memory state, including all redact annotations. The - # real cleanup pass happens on ex_doc.save() below. - self.doc.save(tmp_path, garbage=0, deflate=False, clean=False) - - # Step 2: reopen the on-disk copy. PyMuPDF lazy-parses pages, so - # this does NOT load the whole PDF into RAM up front. - ex_doc = fitz.open(tmp_path) - - # Step 3: scrub on the disk-backed copy. - for page in ex_doc.pages(): - if mode == PrivacyMode.PSEUDONYMIZE: - self._prepare_pseudonymized_page(page, registry) - # page.annots() may return None for pages with no annotations - r_rects = self._collect_redaction_rects(page) - try: - links_to_delete = [ - lnk for lnk in page.get_links() - if any(fitz.Rect(lnk["from"]).intersects(r) for r in r_rects) - ] - for lnk in links_to_delete: - page.delete_link(lnk) - except (RuntimeError, AttributeError, KeyError) as e: - self.logger.debug(f"Could not delete link: {e}") - # PDF_REDACT_IMAGE_REMOVE deletes the ENTIRE image object if it - # intersects ANY redaction rect, even a small one. On a scanned - # PDF (the whole page is one image), redacting a single word - # wiped out the entire page, leaving a blank sheet with only - # the placeholder text visible. PDF_REDACT_IMAGE_PIXELS blanks - # out only the pixels under each redaction rect and leaves the - # rest of the image intact (verified this also fully blanks - # whole-image redactions from "Oscura Immagini", including - # under page rotation, with no visible fringe). - page.apply_redactions( - images=PDF_REDACT_IMAGE_PIXELS, graphics=True + result = export_document( + self.doc, + p, + prepare_page=prepare, + finish_page=finish, + policy=policy, + removed_values=removed_values, + temp_dir=private_temp_dir(self.config_dir), + log=self.logger.debug, ) - try: - # Materialize first: mutating during iteration of - # page.widgets() can invalidate the generator. - for w in list(page.widgets() or []): - page.delete_widget(w) - except (RuntimeError, AttributeError) as e: - self.logger.debug(f"Could not delete widget: {e}") - ex_doc.set_metadata({}) - cx = ex_doc.pdf_catalog() - for k in ["Metadata", "PieceInfo", "Properties", "AcroForm"]: - ex_doc.xref_set_key(cx, k, "null") - - # Step 4: write final scrubbed output. Must save to a different - # path than the open source (`tmp_path`) to avoid PyMuPDF's - # "save to original" restriction on incremental saves. - ex_doc.save(p, garbage=4, deflate=True, clean=True) + if not result.verification.ok: + where = "\n".join(result.verification.findings[:12]) + ans = QMessageBox.critical( + self, + "Verifica dell'esportazione fallita", + "Nel file esportato sono ancora presenti dati che dovevano " + "essere rimossi o contenuti attivi. Posizioni:\n" + where + + "\n\nEliminare il file esportato?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.Yes, + ) + self.logger.error( + f"Export verification failed at {len(result.verification.findings)} location(s)" + ) + if ans == QMessageBox.StandardButton.Yes: + try: + os.remove(p) + except OSError as e: + self.logger.debug(f"Could not remove failed export: {e}") + self.write_log("ERRORE export: file eliminato dopo verifica fallita.") + return if mode == PrivacyMode.PSEUDONYMIZE: payload = build_restore_payload( source_name=source_name, source_sha256=source_sha256, - output_sha256=sha256_file(p), + output_sha256=result.output_sha256, entries=registry.entries(), ) - encrypted = encrypt_restore_payload(payload, map_password) - with open(map_path, "wb") as fh: - fh.write(encrypted) + encrypted = encrypt_restore_payload( + payload, map_password, recovery_key=recovery_key + ) + write_private_file(map_path, encrypted) self.write_log( f"ESPORTATO. Mappa cifrata creata con {len(registry.entries())} segnaposto." ) else: self.write_log("ESPORTATO.") + record = build_audit_record( + app_version=APP_VERSION, + mode=mode.value, + entity_counts=entity_counts, + suggestion_counts={"unreviewed": len(self.pending_suggestions)}, + sanitize=result.sanitize, + verification=result.verification, + output_sha256=result.output_sha256, + pages=result.pages, + ) + write_audit_record(self.config_dir, record) + QMessageBox.information( + self, + "Esportazione completata", + ( + "Il file esportato puo contenere ancora dati personali non " + "rilevati o indiretti. Controllalo prima di condividerlo." + if mode == PrivacyMode.ANONYMIZE + else "Il PDF pseudonimizzato e la mappa di ripristino " + "restano dati personali: conservali separatamente e proteggi " + "la password." + ), + ) except Exception as e: self.logger.error(f"Export failed: {traceback.format_exc()}") self.write_log(f"ERRORE export: {type(e).__name__}: {str(e)}") - finally: - # Always close ex_doc, even on failure, to release the file - # handle on tmp_path before we try to remove it (Windows locks - # open files). - if ex_doc is not None: - try: - ex_doc.close() - except Exception as e: - self.logger.debug(f"Error closing ex_doc: {e}") - # Step 5: remove temp file (best-effort). - try: - if os.path.exists(tmp_path): - os.remove(tmp_path) - except OSError as e: - self.logger.debug(f"Could not remove temp file {tmp_path}: {e}") def cmd_reconstruct(self) -> None: """Reconstruct a previously pseudonymized PDF using its restore map. @@ -2448,18 +3211,17 @@ def cmd_reconstruct(self) -> None: ) if not map_path: return - map_password, ok = QInputDialog.getText( - self, - "Password mappa", - "Password per decifrare la mappa di ripristino:", - QLineEdit.EchoMode.Password, - ) - if not ok or not map_password: + secret = self._ask_decrypt_secret() + if secret is None: return + map_password, recovery_secret = secret out_path, _ = QFileDialog.getSaveFileName( self, "Salva PDF ricostruito", - f"{os.path.splitext(pdf_path)[0]}_ricostruito.pdf", + os.path.join( + os.path.dirname(pdf_path), + neutral_output_name("restored"), + ), "PDF (*.pdf)", ) if not out_path: @@ -2471,12 +3233,20 @@ def cmd_reconstruct(self) -> None: try: with open(map_path, "rb") as fh: encrypted = fh.read() - payload = decrypt_restore_payload(encrypted, map_password) + payload = decrypt_restore_payload( + encrypted, map_password, recovery_key=recovery_secret + ) expected = expected_restore_count(payload) restored_count = reconstruct_pdf(pdf_path, out_path, payload) self.write_log( f"RICOSTRUITO. {restored_count} segnaposto ripristinati." ) + QMessageBox.information( + self, + "PDF ricostruito", + "Il file ricostruito contiene di nuovo tutti i dati personali " + "originali: trattalo e conservalo di conseguenza.", + ) # A restore that silently puts nothing back used to look like a # success: same message, no dialog, output identical to input. if restored_count < expected: @@ -2492,11 +3262,11 @@ def cmd_reconstruct(self) -> None: self.write_log(f"ERRORE ricostruzione: {e}") QMessageBox.critical(self, "Mappa non corrispondente", str(e)) except InvalidToken: - self.write_log("ERRORE ricostruzione: password errata o mappa corrotta.") + self.write_log("ERRORE ricostruzione: password/chiave errata o mappa corrotta.") QMessageBox.critical( self, "Errore decifrazione", - "Password errata o file della mappa corrotto.", + "Password o chiave di recupero errata, oppure mappa corrotta.", ) except Exception as e: self.logger.error(f"Reconstruction failed: {traceback.format_exc()}") diff --git a/README.md b/README.md index 873aff4..e4af195 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,8 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi - 🔄 **Update Check** — Optional, user-triggered check for new releases (stable builds see stable only, beta builds see both) showing the release notes and a one-click download into your Downloads folder; can be enabled at every start from the Info dialog (off by default) - 🗄️ **Fluid UI & Thread-Safe** — PySide6 modern dark-mode interface with zero UI freezing. Text extraction in worker thread with QMutex serialization - 📖 **Persistent Dictionaries** — Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with O(1) fast-path matching -- 🛡️ **Privacy Export Modes** — Choose irreversible anonymization or reversible pseudonymization with a separately encrypted restore map +- 🛡️ **Privacy Export Modes** — Choose irreversible removal or reversible pseudonymization with a separately encrypted restore map; the exported file is sanitised (annotations, attachments, scripts, metadata, hidden text) and verified +- 🔍 **Review of uncertain hits** — Detections without enough evidence are suggested for review instead of being applied or lost silently - 🔎 **OCR Support** — Bundled EN/IT Tesseract language data for scanned PDFs - 🖼️ **Blindfold Mode** — One-click image/logo censoring with professional placeholder: `[ IMAGE REMOVED ]` - 📦 **Native Cross-Platform** — Automated build scripts generate Windows `.exe`, macOS ZIP bundles, and Linux `.deb`/`.rpm` packages @@ -227,7 +228,7 @@ pytest tests/ -v ✅ **No Telemetry** — Zero user tracking > [!IMPORTANT] -> Irreversible anonymization cannot be undone from the exported PDF. Always keep backups of original documents. Pseudonymization requires the encrypted restore map and its password. +> Irreversible removal cannot be undone from the exported PDF, and it does not by itself guarantee anonymisation: indirect identifiers can remain. A pseudonymized PDF and its restore map are personal data. Always keep backups of original documents. Pseudonymization requires the encrypted restore map and its password. See [SECURITY.md](./SECURITY.md) for responsible disclosure and privacy details. diff --git a/SECURITY.md b/SECURITY.md index 35af844..3ae64aa 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -37,29 +37,49 @@ NullifyPDF follows a **privacy-first architecture**: When you export a PDF with redactions: -1. ✅ **Metadata Stripped** — Creation date, author, and document metadata removed -2. ✅ **Links Removed When Redacted** — Links overlapping redaction areas are deleted -3. ✅ **Redactions Applied** — Selected text/image areas are removed from the exported PDF -4. ✅ **Pseudonymization Maps Encrypted** — Restore maps are stored separately and encrypted with a user password +1. ✅ **Redactions applied** — selected text/image areas are removed from the content stream, not just covered +2. ✅ **Sanitiser** — annotations, embedded files, external links, bookmarks, JavaScript, tagged-PDF alternative text, thumbnails, document and XMP metadata, invisible text, text outside the visible page box and the trailer ID are removed; hidden layers are shown before the scan so their text is analysed +3. ✅ **Verification** — the written file is re-opened and checked for any removed value in the text layer, every object and decompressed stream, annotations, attachments, bookmarks and metadata; findings list locations, never the values +4. ✅ **Pseudonymization maps encrypted** — restore maps are stored separately and encrypted with a user password (scrypt key derivation, authenticated encryption) +5. ⚙️ **Optional width masking** — the space left by removed text can be rounded so its width is not recoverable (off by default: it can shift the following text slightly) -### Temporary Files +Limits you should know about: + +- Detection is statistical. Review the result and the suggestions before sharing; unrecognised or indirect identifiers (dates, places, job titles, clinical details) remain. +- Text drawn as part of an image is removed only where the image is redacted; OCR is used to find it. +- Signatures, stamps and photos are not detected automatically; use "Oscura Immagini" or draw a redaction manually. -During export, NullifyPDF uses **disk-backed temporary files**: +### Temporary Files -- Temporary data written to system temp directory -- Automatically cleaned up after export completes -- On Windows: `%APPDATA%\Local\Temp\` -- On macOS/Linux: `/tmp/` +- The working copy used during export is kept **in memory**. Only very large documents use a private per-user folder (`~/.nullifypdf/tmp`), emptied after the export and at the next start-up. +- No unredacted copy is written next to the exported file. +- Logs (`~/.nullifypdf/logs`) are rotated and purged after 30 days; they contain no detected values. "Elimina dati locali" removes logs, audit records and temporary files. -### No Sensitive Data in Memory +### Data kept in memory and on disk - Original PDF kept in memory only while open -- AI results (detected entities) stored only in-memory during session -- Exported PDF overwrites original data locations +- AI results (detected entities) stored only in memory during the session; suggestions are never written to disk +- The restore map is written with owner-only permissions where the platform supports it; keep it away from the PDF and from synchronised folders +- The blocklist and allowlist (`~/.nullifypdf/*.v2.enc`) may themselves contain personal data, so they are encrypted with AES-256-GCM under a master key protected by Windows DPAPI, the OS keyring (optional `keyring` package) or, as a fallback, a local key file. Terms can expire and are then purged. DPAPI and the keyring bind the data to the user account and machine; the key file only protects against copying a single data file, not against someone who can read the user's profile. Overwriting migrated plaintext files is best effort (SSD wear levelling and backups may keep copies) +- If the keystore cannot be read (profile copied to another account or PC) nothing is deleted or overwritten: the lists stay empty and the encrypted files untouched +- The pseudonym key is stored sealed under the same master key; a password-protected copy can be exported for another computer +- Restore maps (version 3 envelope): a random data key encrypts the map with AES-256-GCM; it is wrapped by a password-derived key (Argon2id, scrypt if unavailable) and optionally by a recovery key (256-bit, shown once, never stored). Key-derivation parameters read from a file are bounded before use - Graceful cleanup on app close --- +## ⚖️ GDPR Notes + +NullifyPDF runs locally; the person or organisation using it is the controller of the data it processes. + +- **Redaction is not a guarantee of anonymisation.** Under Recital 26 GDPR, data that can still be attributed to a person is personal data. A redacted document that keeps dates, places or context may remain personal data. +- **Pseudonymised output is personal data** (Art. 4(5) GDPR; EDPB Guidelines 01/2025 on pseudonymisation). The restore map is the "additional information" that must be kept separately and protected; it is itself personal data. A random or keyed placeholder avoids leaking the order and number of values; the "sequential" style does leak them. +- **Special categories (Art. 9)**: documents with health data need particular care. The health-document profile keeps uncertain hits as suggestions instead of dropping them; in Italy, health data may not be disseminated (art. 2-septies d.lgs. 196/2003). +- **Accuracy and minimisation**: over-redaction removes meaning and under-redaction discloses data. Uncertain detections are shown for review rather than applied or discarded silently. +- The export audit record (`~/.nullifypdf/audit`) supports accountability; it contains counts and hashes only. + +--- + ## 🔑 Input Validation & Safety ### File Type Verification diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..a294112 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,17 @@ +# Third-party notices + +## ISTAT place names + +`lexicon_data_it.py` contains data derived from the ISTAT publication +"Elenco dei comuni italiani" (version of 2026-02-21, source: +https://www.istat.it/classificazione/codici-dei-comuni-delle-province-e-delle-regioni/). + +- License: Creative Commons Attribution 4.0 International (CC BY 4.0), + https://creativecommons.org/licenses/by/4.0/ (ISTAT legal notes: + https://www.istat.it/note-legali/). +- Changes: names were normalised and lower-cased, bilingual names were split, + names that are also ordinary words were marked as ambiguous. +- ISTAT does not endorse this project. + +The file is generated by `scripts/build_lexicon_data.py`; the SHA-256 of the +source workbook is recorded in the generated module. diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 17c9a06..c5088c2 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -199,9 +199,31 @@ If AI still misses something: ### Allowlist Issue? If an entity should be detected but isn't: -1. Check `~/.nullifypdf/allowlist.txt` +1. Open **Dizionari** and look at the *Allowlist* tab 2. If the word is there, it's intentionally ignored -3. Remove it from allowlist to re-enable detection +3. Remove it from the allowlist to re-enable detection + +--- + +## 🔐 Dictionaries Are Empty on a New Computer + +The blocklist and allowlist are encrypted for the user account that created them. After copying `~/.nullifypdf` to another account or PC the Dizionari dialog shows a warning and empty lists; the encrypted files are not modified. Restore the original profile, or export the lists to text (*Esporta in chiaro...*) on the old computer and import them on the new one. The pseudonym key can be exported and imported with *Gestisci chiave...* in the placeholder style dialog. + +--- + +## 🔍 Ordinary Words Proposed for Redaction + +### Symptom + +The scan proposes plain words ("Certifico", "Prescrivo", section headings) or fragments of letters of neighbouring lines. + +### Solution + +- Use the **IT** language model (or **BOTH**) for Italian documents; the English model adds false hits on Italian text. +- Uncertain hits appear in **Rivedi suggerimenti** instead of being applied: leave them unticked to discard them. +- Add words you never want redacted to the allowlist (**Dizionari**); matching is by whole word. +- If black boxes cover parts of the lines above or below the redacted word, update to a build that includes rectangle trimming and re-run the scan. With very tight line spacing (roughly half the font size or less) no safe band exists: a warning "interlinea molto stretta" is shown and neighbouring words can be damaged, never left unredacted. +- For medical documents tick **Profilo documenti sanitari**, which keeps uncertain hits as suggestions instead of dropping them. --- @@ -232,10 +254,9 @@ Clicking **Auto Redact** opens a dialog where you tick the data types to look fo For repeated false positives: -1. Open `~/.nullifypdf/allowlist.txt` -2. Add the word (one per line, lowercase) -3. Restart NullifyPDF -4. Run AI scan again +1. Open **Dizionari** and add the word in the *Allowlist* tab +2. Save +3. Run the scan again Example: ``` diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 030c72d..26d9630 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -69,6 +69,19 @@ Click the **"Auto Redact"** button. A dialog first lets you tick which data type --- +### Review suggestions + +Pending suggestions are also outlined in orange on the page (untick *Mostra suggerimenti* to hide them). Click an outline to censor or discard that suggestion; the outlines exist only on screen and never reach the exported file. + + +Detections that the text does not support strongly enough (for example a lone capitalized word with no context) are not applied: after the scan a **Revisione suggerimenti** dialog lists them with page, type, score and the reasons. Values are masked until you tick *Mostra i valori*. Tick the ones to censor and press *Applica selezionati* (the others are discarded), or *Rivedi più tardi* to keep them pending; the sidebar button *Rivedi suggerimenti (N)* reopens the list. Double-click a row to jump to its page. + +### Documents with health data + +Tick **Profilo documenti sanitari** before scanning. It also looks for dates of birth and health identifiers (health card number, prescription number) and never drops an uncertain hit silently: it becomes a suggestion. + +--- + ## Step 4️⃣ — Manual Review & Corrections No AI is 100% accurate. You can fix mistakes: @@ -115,14 +128,19 @@ Remove logos, signatures, or scanned photos: When satisfied with redactions: 1. Click **"Esporta PDF"** -2. Choose the export mode: - - **Anonimizzazione irreversibile** removes selected personal data permanently - - **Pseudonimizzazione reversibile** replaces selected personal data with placeholders and creates a separate encrypted restore map -3. Choose filename and location -4. Review the exported PDF before sharing it +2. If suggestions are still unreviewed you are asked whether to review them first +3. Choose the export mode: + - **Rimozione irreversibile** removes selected personal data permanently + - **Pseudonimizzazione reversibile** replaces selected personal data with placeholders and creates a separate encrypted restore map. You choose the placeholder style: *Casuale* (recommended; reveals neither order nor count), *Progressivo* (`PERSON_001`...; reveals how many values exist and their order), *Generico* (also hides the type) or *Coerente (chiave)* (the same value gets the same placeholder in every document made with the same key, which links the documents) +4. Choose filename and location. The suggested name does not contain the source file name +5. For pseudonymization, choose where to save the map (not the PDF's folder, not a synchronised folder) and enter the password twice +6. NullifyPDF re-opens the exported file and verifies it. If a removed value is still readable you are told where and can delete the file +7. Review the exported PDF before sharing it + +Annotations, attachments, external links, bookmarks, scripts, metadata, invisible text and text outside the page are removed on export. "Maschera larghezza testo rimosso" (sidebar) additionally rounds the space left by removed text. > [!CAUTION] -> **Irreversible anonymization cannot be undone from the exported PDF.** Keep a backup of the original PDF. For pseudonymization, store the encrypted restore map separately from the PDF. +> **Removal cannot be undone from the exported PDF, and it is not a guarantee of anonymisation**: dates, places or context left in the document can still identify a person. Keep a backup of the original PDF. A pseudonymized PDF and its restore map remain personal data: store the map separately from the PDF and protect its password. --- @@ -143,6 +161,12 @@ NullifyPDF checks the SHA-256 hash recorded in the restore map against the selec --- +## 🧹 Local data + +NullifyPDF keeps a rotating log, an export audit record (counts and hashes, no values) and, for very large files, a temporary folder under `~/.nullifypdf`. **Elimina dati locali** removes them (and, if you confirm, the dictionaries). + +--- + ## 🔧 Dictionary Management Your redaction preferences are saved automatically: @@ -159,37 +183,28 @@ Your redaction preferences are saved automatically: ``` .nullifypdf/ -├── blocklist.txt ← Words to ALWAYS redact -├── allowlist.txt ← Words to NEVER redact +├── blocklist.v2.enc ← Words to ALWAYS redact (encrypted) +├── allowlist.v2.enc ← Words to NEVER redact (encrypted) +├── keystore.json ← How the encryption key is protected +├── pseudonym.v2.enc ← Key of the "Coerente" placeholder style (encrypted) ├── update_check.json ← "Check at every start" preference (Info dialog) +├── audit/ ← Export records (counts and hashes, no values) ├── logs/ │ └── nullifypdf.log ``` -### How to Edit Manually - -
-📝 Edit Blocklist/Allowlist Directly - -1. Open file manager, navigate to `~/.nullifypdf/` -2. Edit `blocklist.txt` or `allowlist.txt` with any text editor -3. One word per line, UTF-8 encoding -4. Restart NullifyPDF to load changes +### Editing the dictionaries -**Example blocklist.txt:** -``` -john smith -acme corporation -admin@example.com -``` +Open **Dizionari**. Each list is a table with the term, the date it was added and an optional expiry date (`AAAA-MM-GG`, or *Mai*). Use *Aggiungi* / *Rimuovi*, *Importa da testo...* (one term per line) and *Esporta in chiaro...* (the exported file contains personal data in clear). *Scadenza predefinita* sets the expiry of new terms, including the ones added by clicking or drawing on the page. Expired terms are removed automatically. The label at the top shows how the lists are protected. -
+> [!NOTE] +> The lists are encrypted for the current user account. If you copy `~/.nullifypdf` to another account or PC they cannot be read there (they stay untouched and are shown empty). Older plaintext `blocklist.txt` / `allowlist.txt` files are migrated automatically. ### Blocklist and Allowlist The manual redaction workflow keeps clicked/drawn terms mutually exclusive. -When editing dictionary files or the dictionary dialog directly, review both -lists and avoid adding the same term to both. +When editing the dictionary dialog, review both lists and avoid adding the +same term to both. --- diff --git a/build_local.py b/build_local.py index ec4c562..773527b 100644 --- a/build_local.py +++ b/build_local.py @@ -396,7 +396,7 @@ def build_app() -> None: from PyInstaller.utils.hooks import collect_all datas = {datas_literal} binaries = [] -hiddenimports = ['spacy', 'presidio_analyzer'] +hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it'] for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] @@ -412,7 +412,7 @@ def build_app() -> None: from PyInstaller.utils.hooks import collect_all datas = {datas_literal} binaries = [] -hiddenimports = ['spacy', 'presidio_analyzer'] +hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it'] for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] diff --git a/dictionary_dialog.py b/dictionary_dialog.py new file mode 100644 index 0000000..ae66454 --- /dev/null +++ b/dictionary_dialog.py @@ -0,0 +1,185 @@ +"""Dictionary (blocklist / allowlist) editor with per-entry expiry (Qt).""" + +from __future__ import annotations + +import datetime +from typing import Any, Dict, List, Optional + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QCheckBox, + QComboBox, + QDialog, + QFileDialog, + QHBoxLayout, + QInputDialog, + QLabel, + QMessageBox, + QPushButton, + QTableWidget, + QTableWidgetItem, + QTabWidget, + QVBoxLayout, + QWidget, +) + +from word_lists import ListEntry, PDFListManager, MIN_TERM_LENGTH + +TTL_CHOICES = [("Mai", None), ("30 giorni", 30), ("90 giorni", 90), + ("180 giorni", 180), ("1 anno", 365)] +COL_TERM, COL_ADDED, COL_EXPIRES = range(3) +KINDS = (("blocklist", "🔴 Blocklist"), ("allowlist", "🟢 Allowlist")) + + +def parse_date(text: str) -> Optional[datetime.date]: + text = text.strip() + if not text or text.lower() in ("mai", "never"): + return None + return datetime.date.fromisoformat(text) + + +class DictionaryDialog(QDialog): + """Edit both lists; `saved_blocklist` / `saved_allowlist` hold the result.""" + + def __init__(self, manager: PDFListManager, parent: Any = None) -> None: + super().__init__(parent) + self.manager = manager + self.setWindowTitle("Dizionari") + self.resize(620, 520) + self.saved_blocklist: Optional[set] = None + self.saved_allowlist: Optional[set] = None + lay = QVBoxLayout(self) + self.status = QLabel(manager.protector_label()) + lay.addWidget(self.status) + if manager.keystore_error: + warn = QLabel( + "Dizionari cifrati non leggibili su questo account/PC: verranno " + "usati dizionari vuoti finché non li ripristini." + ) + warn.setWordWrap(True) + warn.setStyleSheet("color: #f59e0b;") + lay.addWidget(warn) + self.tabs = QTabWidget() + self.tables: Dict[str, QTableWidget] = {} + for kind, title in KINDS: + tab = QWidget() + tl = QVBoxLayout(tab) + table = QTableWidget(0, 3) + table.setHorizontalHeaderLabels(["Termine", "Aggiunto il", "Scade il (AAAA-MM-GG)"]) + table.horizontalHeader().setStretchLastSection(True) + table.setColumnWidth(COL_TERM, 260) + self.tables[kind] = table + tl.addWidget(table) + row = QHBoxLayout() + for label, slot in ( + ("Aggiungi", lambda _=False, k=kind: self.add_row(k)), + ("Rimuovi", lambda _=False, k=kind: self.remove_selected(k)), + ("Importa da testo…", lambda _=False, k=kind: self.import_text(k)), + ("Esporta in chiaro…", lambda _=False, k=kind: self.export_plain(k)), + ): + b = QPushButton(label) + b.clicked.connect(slot) + row.addWidget(b) + tl.addLayout(row) + self.tabs.addTab(tab, title) + for e in manager.load_entries(kind): + self._append(kind, e) + lay.addWidget(self.tabs) + ttl_row = QHBoxLayout() + ttl_row.addWidget(QLabel("Scadenza predefinita dei nuovi termini:")) + self.ttl = QComboBox() + current = manager.default_ttl_days() + for i, (label, days) in enumerate(TTL_CHOICES): + self.ttl.addItem(label, days) + if days == current: + self.ttl.setCurrentIndex(i) + ttl_row.addWidget(self.ttl) + ttl_row.addStretch(1) + lay.addLayout(ttl_row) + btn = QPushButton("Salva") + btn.setObjectName("Primary") + btn.clicked.connect(self.save) + lay.addWidget(btn) + + # -- rows --------------------------------------------------------- + def _append(self, kind: str, entry: ListEntry) -> None: + t = self.tables[kind] + r = t.rowCount() + t.insertRow(r) + t.setItem(r, COL_TERM, QTableWidgetItem(entry.term)) + added = QTableWidgetItem(entry.added.isoformat()) + added.setFlags(added.flags() & ~Qt.ItemFlag.ItemIsEditable) + t.setItem(r, COL_ADDED, added) + t.setItem(r, COL_EXPIRES, QTableWidgetItem(entry.expires.isoformat() if entry.expires else "Mai")) + + def add_term(self, kind: str, term: str) -> None: + days = self.ttl.currentData() + today = datetime.date.today() + exp = today + datetime.timedelta(days=days) if days else None + self._append(kind, ListEntry(term.strip().lower(), today, exp)) + + def add_row(self, kind: str) -> None: + text, ok = QInputDialog.getText(self, "Aggiungi termine", "Termine:") + if ok and len(text.strip()) >= MIN_TERM_LENGTH: + self.add_term(kind, text) + + def remove_selected(self, kind: str) -> None: + t = self.tables[kind] + for r in sorted({i.row() for i in t.selectedIndexes()}, reverse=True): + t.removeRow(r) + + def import_text(self, kind: str) -> None: + text, ok = QInputDialog.getMultiLineText( + self, "Importa da testo", "Un termine per riga:" + ) + if ok: + for line in text.splitlines(): + if len(line.strip()) >= MIN_TERM_LENGTH: + self.add_term(kind, line) + + def export_plain(self, kind: str) -> None: + ans = QMessageBox.warning( + self, "Esporta in chiaro", + "Il file esportato contiene dati personali in chiaro. Continuare?", + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + QMessageBox.StandardButton.No, + ) + if ans != QMessageBox.StandardButton.Yes: + return + path, _ = QFileDialog.getSaveFileName(self, "Esporta", f"{kind}.txt", "Testo (*.txt)") + if path: + with open(path, "w", encoding="utf-8") as fh: + fh.write("\n".join(e.term for e in self.entries(kind))) + + # -- result ------------------------------------------------------- + def entries(self, kind: str) -> List[ListEntry]: + out: List[ListEntry] = [] + t = self.tables[kind] + def cell(r: int, c: int) -> str: + item = t.item(r, c) + return item.text() if item is not None else "" + + for r in range(t.rowCount()): + term = cell(r, COL_TERM).strip().lower() + if len(term) < MIN_TERM_LENGTH: + continue + try: + added = parse_date(cell(r, COL_ADDED)) or datetime.date.today() + expires = parse_date(cell(r, COL_EXPIRES)) + except ValueError: + raise ValueError(f"Data non valida per «{term}»: usa AAAA-MM-GG oppure «Mai».") + out.append(ListEntry(term, added, expires)) + return out + + def save(self) -> None: + try: + block, allow = self.entries("blocklist"), self.entries("allowlist") + except ValueError as e: + QMessageBox.warning(self, "Data non valida", str(e)) + return + self.manager.set_default_ttl_days(self.ttl.currentData()) + self.manager.save_entries("blocklist", block) + self.manager.save_entries("allowlist", allow) + self.saved_blocklist = {e.term for e in block} + self.saved_allowlist = {e.term for e in allow} + self.accept() diff --git a/export_engine.py b/export_engine.py new file mode 100644 index 0000000..629dc01 --- /dev/null +++ b/export_engine.py @@ -0,0 +1,340 @@ +"""Qt-free export pipeline, local-data hygiene and audit record. + +``export_document`` is the whole export: it works on a copy of the document +(kept in memory, or in a private per-user folder for very large files), +applies the redactions, sanitises the result, writes it and verifies it. The +GUI only supplies callbacks and user choices. +""" + +from __future__ import annotations + +import datetime +import hashlib +import json +import os +import pathlib +import secrets +import shutil +import stat +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Iterable, List, Optional + +import pymupdf + +from pii_detection import GLYPH_CORE_INSET +from pdf_sanitize import ( + PDF_ANNOT_REDACT, + SanitizePolicy, + SanitizeReport, + VerificationReport, + sanitize_document, + verify_export, +) + +# Documents larger than this are copied through a private temp file instead +# of RAM. +IN_MEMORY_LIMIT_BYTES = 96 * 1024 * 1024 +PDF_REDACT_IMAGE_PIXELS: int = pymupdf.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] + + +@dataclass +class ExportResult: + output_sha256: str = "" + sanitize: SanitizeReport = field(default_factory=SanitizeReport) + verification: VerificationReport = field(default_factory=VerificationReport) + pages: int = 0 + + +def neutral_output_name(mode_value: str, now: Optional[datetime.datetime] = None) -> str: + """Default file name that contains nothing from the source document.""" + now = now or datetime.datetime.now() + return f"document_{mode_value}_{now:%Y%m%d_%H%M%S}.pdf" + + +def neutral_map_name(now: Optional[datetime.datetime] = None) -> str: + now = now or datetime.datetime.now() + return f"restore_map_{now:%Y%m%d_%H%M%S}.nullifypdf-map" + + +def name_leaks_value(path: str, values: Iterable[str]) -> bool: + """True when the file name contains one of the values being removed.""" + base = "".join(os.path.basename(path).casefold().split()) + for v in values: + n = "".join(v.casefold().split()) + if len(n) >= 4 and n in base: + return True + return False + + +def write_private_file(path: str, data: bytes) -> None: + """Write `data` readable by the current user only (where supported).""" + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) + fd = os.open(path, flags, 0o600) + with os.fdopen(fd, "wb") as fh: + fh.write(data) + try: + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + +def private_temp_dir(config_dir: pathlib.Path) -> pathlib.Path: + d = pathlib.Path(config_dir) / "tmp" + d.mkdir(parents=True, exist_ok=True) + try: + os.chmod(d, 0o700) + except OSError: + pass + return d + + +def cleanup_stale_temp(config_dir: pathlib.Path, max_age_hours: float = 0.0) -> int: + """Remove leftovers of a crashed export (called at startup).""" + d = pathlib.Path(config_dir) / "tmp" + removed = 0 + if not d.is_dir(): + return 0 + now = datetime.datetime.now().timestamp() + for f in d.iterdir(): + try: + if now - f.stat().st_mtime >= max_age_hours * 3600: + f.unlink() + removed += 1 + except OSError: + pass + return removed + + +def collect_removed_values( + doc: Any, read_payload: Optional[Callable[[Any], Dict[str, str]]] = None +) -> List[str]: + """Every value a pending redaction will remove (for the output check).""" + values: List[str] = [] + for page in doc: + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + payload = read_payload(annot) if read_payload else {} + original = (payload or {}).get("original", "") + if original: + values.append(original) + try: + clipped = page.get_text("text", clip=annot.rect).strip() + except Exception: + clipped = "" + if clipped: + values.append(" ".join(clipped.split())) + seen: List[str] = [] + for v in values: + if v and v not in seen: + seen.append(v) + return seen + + +def text_outside_redactions(doc: Any) -> str: + """Source text that no redaction covers (it stays in the output).""" + chunks: List[str] = [] + for page in doc: + rects = [ + a.rect for a in (page.annots() or []) if a.type[0] == PDF_ANNOT_REDACT + ] + for w in page.get_text("words"): + # A word is deleted when a redaction reaches its glyph core (the + # box inset by GLYPH_CORE_INSET), not only when its centre is + # covered; a partly covered word is not "kept". + dx, dy = (w[2] - w[0]) * GLYPH_CORE_INSET, (w[3] - w[1]) * GLYPH_CORE_INSET + core = pymupdf.Rect(w[0] + dx, w[1] + dy, w[2] - dx, w[3] - dy) + if not any(core.intersects(r) for r in rects): + chunks.append(w[4]) + return " ".join(chunks) + + +def _open_copy( + src_doc: Any, temp_dir: Optional[pathlib.Path] +) -> tuple[Any, Optional[str]]: + """Open a working copy of `src_doc` without leaving PII beside the output.""" + size = 0 + try: + size = os.path.getsize(src_doc.name) if src_doc.name else 0 + except OSError: + size = 0 + if size <= IN_MEMORY_LIMIT_BYTES or temp_dir is None: + return pymupdf.open("pdf", src_doc.tobytes(garbage=0, deflate=False)), None + tmp = str(pathlib.Path(temp_dir) / f"{secrets.token_hex(8)}.tmp") + data = src_doc.tobytes(garbage=0, deflate=False) + write_private_file(tmp, data) + return pymupdf.open(tmp), tmp + + +def sha256_of(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def export_document( + src_doc: Any, + out_path: str, + *, + prepare_page: Optional[Callable[[Any], None]] = None, + finish_page: Optional[Callable[[Any], None]] = None, + policy: Optional[SanitizePolicy] = None, + removed_values: Iterable[str] = (), + temp_dir: Optional[pathlib.Path] = None, + log: Optional[Callable[[str], None]] = None, +) -> ExportResult: + """Apply redactions to a copy of `src_doc`, sanitise, save and verify.""" + result = ExportResult() + kept_text = text_outside_redactions(src_doc) + ex_doc, tmp = _open_copy(src_doc, temp_dir) + try: + for page in ex_doc.pages(): + if prepare_page is not None: + prepare_page(page) + redact_rects = [ + a.rect for a in (page.annots() or []) + if a.type[0] == PDF_ANNOT_REDACT + ] + try: + for lnk in page.get_links(): + if any(pymupdf.Rect(lnk["from"]).intersects(r) for r in redact_rects): + page.delete_link(lnk) + except (RuntimeError, AttributeError, KeyError) as e: + if log: + log(f"link removal skipped: {e}") + # PDF_REDACT_IMAGE_PIXELS blanks only the pixels under each rect; + # IMAGE_REMOVE would delete a whole scanned page for one word. + page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) + if finish_page is not None: + # Text drawn after the redactions are applied: placeholders + # must not go through apply_redactions(), which silently + # drops text it cannot lay out at 4 pt or more. + finish_page(page) + try: + for w in list(page.widgets() or []): + page.delete_widget(w) + except (RuntimeError, AttributeError) as e: + if log: + log(f"widget removal skipped: {e}") + result.pages += 1 + result.sanitize = sanitize_document(ex_doc, policy) + ex_doc.save(out_path, garbage=4, deflate=True, clean=True) + finally: + try: + ex_doc.close() + except Exception: + pass + if tmp: + try: + os.remove(tmp) + except OSError: + pass + result.output_sha256 = sha256_of(out_path) + result.verification = verify_export(out_path, removed_values, kept_text) + return result + + +# --------------------------------------------------------------------------- +# Audit record (no values) and local data hygiene +# --------------------------------------------------------------------------- + +def build_audit_record( + *, + app_version: str, + mode: str, + entity_counts: Dict[str, int], + suggestion_counts: Dict[str, int], + sanitize: SanitizeReport, + verification: VerificationReport, + output_sha256: str, + pages: int, +) -> Dict[str, Any]: + """Accountability record of one export. Holds no personal data.""" + return { + "format": "NullifyPDF export audit", + "timestamp": datetime.datetime.now().isoformat(timespec="seconds"), + "app_version": app_version, + "mode": mode, + "pages": pages, + "entity_counts": dict(entity_counts), + "suggestions": dict(suggestion_counts), + "sanitizer_actions": dict(sanitize.actions), + "verification": { + "ok": verification.ok, + "findings": list(verification.findings), + }, + "output_sha256": output_sha256, + } + + +def write_audit_record(config_dir: pathlib.Path, record: Dict[str, Any]) -> str: + d = pathlib.Path(config_dir) / "audit" + d.mkdir(parents=True, exist_ok=True) + name = f"export_{datetime.datetime.now():%Y%m%d_%H%M%S}_{secrets.token_hex(2)}.json" + path = str(d / name) + write_private_file(path, json.dumps(record, indent=2, sort_keys=True).encode("utf-8")) + return path + + +def wipe_local_data( + config_dir: pathlib.Path, include_lists: bool = False, include_keys: bool = False +) -> List[str]: + """Delete logs, audit records, temp files and, optionally, the word lists + and the keys (keystore, pseudonym key). Without the keys the encrypted + lists and the stored pseudonym key become unreadable.""" + root = pathlib.Path(config_dir) + removed: List[str] = [] + for sub in ("logs", "audit", "tmp"): + d = root / sub + if d.is_dir(): + shutil.rmtree(d, ignore_errors=True) + if not d.exists(): # report only what is really gone + removed.append(sub) + names = ["ai_entities.json"] + if include_keys: + names += ["keystore.json", "pseudonym.v2.enc", "pseudonym.key"] + keys = root / "keys" + if keys.is_dir(): + shutil.rmtree(keys, ignore_errors=True) + if not keys.exists(): + removed.append("keys") + try: + import secure_store + + secure_store.forget_cached_keys() + except Exception: + pass + if include_lists: + names += ["blocklist.txt", "allowlist.txt", "blocklist.v2.enc", "allowlist.v2.enc"] + for n in names: + f = root / n + if f.is_file(): + try: + f.unlink() + removed.append(n) + except OSError: + pass + return removed + + +def purge_old_logs(log_dir: pathlib.Path, max_age_days: int = 30) -> int: + d = pathlib.Path(log_dir) + if not d.is_dir(): + return 0 + cutoff = datetime.datetime.now().timestamp() - max_age_days * 86400 + n = 0 + for f in d.iterdir(): + try: + if f.is_file() and f.stat().st_mtime < cutoff: + f.unlink() + n += 1 + except OSError: + pass + return n + + +def log_token(path: str) -> str: + """Stable short token for a path, so logs never hold file names.""" + return hashlib.sha256(path.encode("utf-8", "ignore")).hexdigest()[:10] diff --git a/lexicon_data_it.py b/lexicon_data_it.py new file mode 100644 index 0000000..7c447c3 --- /dev/null +++ b/lexicon_data_it.py @@ -0,0 +1,9 @@ +# Generated by scripts/build_lexicon_data.py - do not edit. +# Contains ISTAT data (Elenco comuni italiani, comuni.xlsx), licensed +# CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/); modified: names +# normalised, lower-cased, bilingual names split. +SOURCE_SHA256 = {'istat_comuni': '83842076860450f7e482daecea6b7a769f5f93d0bf5b0d48802b44896d7a26d5'} +COMUNI = frozenset("""abano terme|abbadia cerreto|abbadia lariana|abbadia san salvatore|abbasanta|abbateggio|abbiategrasso|abetone cutigliano|abriola|abtei|acate|accadia|acceglio|accettura|acciano|accumoli|acerenza|acerno|acerra|aci bonaccorsi|aci castello|aci catena|aci sant'antonio|acireale|acquafondata|acquaformosa|acquafredda|acqualagna|acquanegra cremonese|acquanegra sul chiese|acquapendente|acquappesa|acquaro|acquasanta terme|acquasparta|acquaviva collecroce|acquaviva d'isernia|acquaviva delle fonti|acquaviva picena|acquaviva platani|acquedolci|acqui terme|acri|acuto|adelfia|adrano|adrara san martino|adrara san rocco|adria|adro|affi|affile|afragola|africo|agazzano|agerola|aggius|agira|agliana|agliano terme|aglientu|agliè|agna|agnadello|agnana calabra|agnone|agnosine|agordo|agosta|agra|agrate brianza|agrate conturbia|agrigento|agropoli|agugliano|agugliaro|ahrntal|aicurzio|aidomaggiore|aidone|aielli|aiello calabro|aiello del friuli|aiello del sabato|aieta|ailano|ailoche|airasca|airola|airole|airuno|aisone|ala|ala di stura|alagna|alagna valsesia|alanno|alassio|alatri|alba|alba adriatica|albagiara|albairate|albanella|albano di lucania|albano laziale|albano sant'alessandro|albano vercellese|albaredo d'adige|albaredo per san marco|albareto|albaretto della torre|albavilla|albenga|albera ligure|alberobello|alberona|albese con cassano|albettone|albi|albiano|albiano d'ivrea|albiate|albidona|albignasego|albinea|albino|albiolo|albisola superiore|albissola marina|albizzate|albonese|albosaggia|albugnano|albuzzano|alcamo|alcara li fusi|aldein|aldeno|aldino|ales|alessandria|alessandria del carretto|alessandria della rocca|alessano|alezio|alfano|alfedena|alfianello|alfiano natta|alfonsine|alghero|algua|algund|alia|aliano|alice bel colle|alice castello|alife|alimena|aliminusa|allai|alleghe|allein|allerona|alliste|allumiere|alluvioni piovera|almenno san bartolomeo|almenno san salvatore|almese|almè|alonte|alpago|alpette|alpignano|alseno|alserio|alta val tidone|alta valle intelvi|altamura|altare|altavalle|altavilla irpina|altavilla milicia|altavilla monferrato|altavilla silentina|altavilla vicentina|altidona|altilia|altino|altissimo|altivole|alto|alto reno terme|alto sermenza|altofonte|altomonte|altopascio|altopiano della vigolana|altrei|alviano|alvignano|alvito|alzano lombardo|alzano scrivia|alzate brianza|alà dei sardi|alì|alì terme|amalfi|amandola|amantea|amaro|amaroni|amaseno|amato|amatrice|ambivere|amblar-don|ameglia|amelia|amendolara|ameno|amorosi|ampezzo|anacapri|anagni|ancarano|ancona|andali|andalo|andalo valtellino|andezeno|andora|andorno micca|andrano|andrate|andreis|andretta|andria|andrian|andriano|anela|anfo|angera|anghiari|angiari|angolo terme|angri|angrogna|anguillara sabazia|anguillara veneta|annicco|annone di brianza|annone veneto|anoia|antegnate|anterivo|antey-saint-andré|anticoli corrado|antignano|antillo|antonimina|antrodoco|antrona schieranco|anversa degli abruzzi|anzano del parco|anzano di puglia|anzi|anzio|anzola d'ossola|anzola dell'emilia|aosta|apecchio|apice|apiro|apollosa|appiano gentile|appiano sulla strada del vino|appignano|appignano del tronto|aprica|apricale|apricena|aprigliano|aprilia|aquara|aquila d'arroscia|aquileia|aquilonia|aquino|aradeo|aragona|aramengo|arba|arborea|arborio|arbus|arcade|arce|arcene|arcevia|archi|arcidosso|arcinazzo romano|arcisate|arco|arcola|arcole|arconate|arcore|arcugnano|ardara|ardauli|ardea|ardenno|ardesio|ardore|arena|arena po|arenzano|arese|arezzo|argegno|argelato|argenta|argentera|arguello|argusto|ari|ariano irpino|ariano nel polesine|ariccia|arielli|arienzo|arignano|aritzo|arizzano|arlena di castro|arluno|armeno|armento|armo|armungia|arnad|arnara|arnasco|arnesano|arola|arona|arosio|arpaia|arpaise|arpino|arquata del tronto|arquata scrivia|arquà petrarca|arquà polesine|arre|arrone|arsago seprio|arsiero|arsita|arsiè|arsoli|arta terme|artegna|artena|artogne|arvier|arzachena|arzago d'adda|arzana|arzano|arzergrande|arzignano|ascea|asciano|ascoli piceno|ascoli satriano|ascrea|asiago|asigliano veneto|asigliano vercellese|asola|asolo|assago|assemini|assisi|asso|assolo|assoro|asti|asuni|ateleta|atella|atena lucana|atessa|atina|atrani|atri|atripalda|attigliano|attimis|atzara|auer|augusta|auletta|aulla|aurano|aurigo|auronzo di cadore|ausonia|austis|avegno|avelengo|avella|avellino|averara|aversa|avetrana|avezzano|aviano|aviatico|avigliana|avigliano|avigliano umbro|avio|avise|avola|avolasca|ayas|aymavilles|azeglio|azzanello|azzano d'asti|azzano decimo|azzano mella|azzano san paolo|azzate|azzio|azzone|baceno|bacoli|badalucco|badesi|badia|badia calavena|badia pavese|badia polesine|badia tedalda|badolato|bagaladi|bagheria|bagnacavallo|bagnara calabra|bagnara di romagna|bagnaria|bagnaria arsa|bagnasco|bagnatica|bagni di lucca|bagno a ripoli|bagno di romagna|bagnoli del trigno|bagnoli di sopra|bagnoli irpino|bagnolo cremasco|bagnolo del salento|bagnolo di po|bagnolo in piano|bagnolo mella|bagnolo piemonte|bagnolo san vito|bagnone|bagnoregio|bagolino|baia e latina|baiano|bairo|baiso|bajardo|balangero|baldichieri d'asti|baldissero canavese|baldissero d'alba|baldissero torinese|balestrate|balestrino|ballabio|ballao|balme|balmuccia|balocco|balsorano|balvano|balzola|banari|banchette|bannio anzino|banzi|baone|baradili|baragiano|baranello|barano d'ischia|baranzate|barasso|baratili san pietro|barbania|barbara|barbarano mossano|barbarano romano|barbaresco|barbariga|barbata|barberino di mugello|barberino tavarnelle|barbian|barbianello|barbiano|barbona|barcellona pozzo di gotto|barcis|bard|bardello con malgesso e bregano|bardi|bardineto|bardolino|bardonecchia|bareggio|barengo|baressa|barete|barga|bargagli|barge|barghe|bari|bari sardo|bariano|baricella|barile|barisciano|barlassina|barletta|barni|barolo|barone canavese|baronissi|barrafranca|barrali|barrea|barumini|barzago|barzana|barzanò|barzio|basaluzzo|bascapè|baschi|basciano|baselga di pinè|baselice|basiano|basicò|basiglio|basiliano|bassano bresciano|bassano del grappa|bassano in teverina|bassano romano|bassiano|bassignana|bastia mondovì|bastia umbra|bastida pancarana|bastiglia|battaglia terme|battifollo|battipaglia|battuda|baucina|bauladu|baunei|baveno|bedero valcuvia|bedizzole|bedollo|bedonia|bedulita|bee|beinasco|beinette|belcastro|belfiore|belforte all'isauro|belforte del chienti|belforte monferrato|belgioioso|belgirate|bella|bellagio|bellano|bellante|bellaria-igea marina|bellegra|bellino|bellinzago lombardo|bellinzago novarese|bellizzi|bellona|bellosguardo|belluno|bellusco|belmonte calabro|belmonte castello|belmonte del sannio|belmonte in sabina|belmonte mezzagno|belmonte piceno|belpasso|belsito|belvedere di spinello|belvedere langhe|belvedere marittimo|belvedere ostrense|belveglio|belvì|bema|bene lario|bene vagienna|benestare|benetutti|benevello|benevento|benna|bentivoglio|berbenno|berbenno di valtellina|berceto|berchidda|beregazzo con figliaro|bereguardo|bergamasco|bergamo|bergantino|bergeggi|bergolo|berlingo|bernalda|bernareggio|bernate ticino|bernezzo|bertinoro|bertiolo|bertonico|berzano di san pietro|berzano di tortona|berzo demo|berzo inferiore|berzo san fermo|besana in brianza|besano|besate|besenello|besenzone|besnate|besozzo|bessude|bettola|bettona|beura-cardezza|bevagna|beverino|bevilacqua|biancavilla|bianchi|bianco|biandrate|biandronno|bianzano|bianzone|bianzè|biassono|bibbiano|bibbiena|bibbona|bibiana|biccari|bicinicco|bidonì|biella|bienno|bieno|bientina|binago|binasco|binetto|bioglio|bionaz|bione|birori|bisaccia|bisacquino|bisceglie|bisegna|bisenti|bisignano|bistagno|bisuschio|bitetto|bitonto|bitritto|bitti|bivona|bivongi|bizzarone|bleggio superiore|blello|blera|blessagno|blevio|blufi|boara pisani|bobbio|bobbio pellice|boca|bocchigliero|boccioleto|bocenago|bodio lomnago|boffalora d'adda|boffalora sopra ticino|bogliasco|bognanco|bogogno|boissano|bojano|bolano|bolgare|bollate|bollengo|bologna|bolognano|bolognetta|bolognola|bolotana|bolsena|boltiere|bolzano|bolzano novarese|bolzano vicentino|bomarzo|bomba|bompensiere|bompietro|bomporto|bonarcado|bonassola|bonate sopra|bonate sotto|bonavigo|bondeno|bondone|bonea|bonefro|bonemerse|bonifati|bonito|bonnanaro|bono|bonorva|bonvicino|borbona|borca di cadore|bordano|bordighera|bordolano|bore|boretto|borgarello|borgaro torinese|borgetto|borghetto d'arroscia|borghetto di borbera|borghetto di vara|borghetto lodigiano|borghetto santo spirito|borghi|borgia|borgiallo|borgio verezzi|borgo a mozzano|borgo chiese|borgo d'ale|borgo d'anaunia|borgo di terzo|borgo lares|borgo mantovano|borgo pace|borgo priolo|borgo san dalmazzo|borgo san giacomo|borgo san giovanni|borgo san lorenzo|borgo san martino|borgo san siro|borgo ticino|borgo tossignano|borgo val di taro|borgo valbelluna|borgo valsugana|borgo velino|borgo veneto|borgo vercelli|borgo virgilio|borgocarbonara|borgofranco d'ivrea|borgolavezzaro|borgomale|borgomanero|borgomaro|borgomasino|borgomezzavalle|borgone susa|borgonovo val tidone|borgoratto alessandrino|borgoratto mormorolo|borgoricco|borgorose|borgosatollo|borgosesia|bormida|bormio|bornasco|borno|boroneddu|borore|borrello|borriana|borso del grappa|bortigali|bortigiadas|borutta|borzonasca|bosa|bosaro|boschi sant'anna|bosco chiesanuova|bosco marengo|bosconero|boscoreale|boscotrecase|bosia|bosio|bosisio parini|bosnasco|bossico|bossolasco|botricello|botrugno|bottanuco|botticino|bottidda|bova|bova marina|bovalino|bovegno|boves|bovezzo|boville ernica|bovino|bovisio-masciago|bovolenta|bovolone|bozen|bozzole|bozzolo|bra|bracca|bracciano|bracigliano|braies|brallo di pregola|brancaleone|brandico|brandizzo|branzi|branzoll|braone|brebbia|breda di piave|breganze|bregnano|brembate|brembate di sopra|brembio|breme|brendola|brenna|brenner|brennero|breno|brenta|brentino belluno|brentonico|brenzone sul garda|brescello|brescia|bresimo|bressana bottarone|bressanone|bressanvido|bresso|brezzo di bedero|briaglia|briatico|bricherasio|brienno|brienza|briga alta|briga novarese|brignano gera d'adda|brignano-frascata|brindisi|brindisi montagna|brinzio|briona|brione|briosco|brisighella|brissago-valtravaglia|brissogne|brittoli|brivio|brixen|broccostella|brogliano|brognaturo|brolo|brondello|broni|bronte|bronzolo|brossasco|brosso|brovello-carpugnino|brozolo|brugherio|brugine|brugnato|brugnera|bruino|brumano|brunate|bruneck|brunello|brunico|bruno|brusaporto|brusasco|brusciano|brusimpiano|brusnengo|brusson|bruzolo|bruzzano zeffirio|bubbiano|bubbio|buccheri|bucchianico|bucciano|buccinasco|buccino|bucine|buddusò|budoia|budoni|budrio|buggerru|buggiano|buglio in monte|bugnara|buguggiate|buja|bulciago|bulgarograsso|bultei|bulzi|buonabitacolo|buonalbergo|buonconvento|buonvicino|burago di molgora|burcei|burgio|burgos|burgstall|buriasco|burolo|buronzo|busachi|busalla|busano|busca|buscate|buscemi|buseto palizzolo|busnago|bussero|busseto|bussi sul tirino|busso|bussolengo|bussoleno|busto arsizio|busto garolfo|butera|buti|buttapietra|buttigliera alta|buttigliera d'asti|buttrio|cabella ligure|cabiate|cabras|caccamo|caccuri|cadegliano-viconago|cadelbosco di sopra|cadeo|caderzone terme|cadoneghe|cadorago|cadrezzate con osmate|caerano di san marco|cafasse|caggiano|cagli|cagliari|caglio|cagnano amiterno|cagnano varano|caianello|caiazzo|caines|caino|caiolo|cairano|cairate|cairo montenotte|caivano|calabritto|calalzo di cadore|calamandrana|calamonaci|calangianus|calanna|calasca-castiglione|calascibetta|calascio|calasetta|calatabiano|calatafimi-segesta|calcata|calceranica al lago|calci|calciano|calcinaia|calcinate|calcinato|calcio|calco|caldaro sulla strada del vino|caldarola|calderara di reno|caldes|caldiero|caldogno|caldonazzo|calendasco|calenzano|calestano|calice al cornoviglio|calice ligure|calimera|calitri|calizzano|callabiana|calliano|calliano monferrato|calolziocorte|calopezzati|calosso|caloveto|caltabellotta|caltagirone|caltanissetta|caltavuturo|caltignaga|calto|caltrano|calusco d'adda|caluso|calvagese della riviera|calvanico|calvatone|calvello|calvene|calvenzano|calvera|calvi|calvi dell'umbria|calvi risorta|calvignano|calvignasco|calvisano|calvizzano|camagna monferrato|camaiore|camandona|camastra|cambiago|cambiano|cambiasca|camburzano|camerana|camerano|camerano casasco|camerata cornello|camerata nuova|camerata picena|cameri|camerino|camerota|camigliano|camini|camino|camino al tagliamento|camisano|camisano vicentino|cammarata|camogli|campagna|campagna lupia|campagnano di roma|campagnatico|campagnola cremasca|campagnola emilia|campana|camparada|campegine|campello sul clitunno|campertogno|campi bisenzio|campi salentina|campiglia cervo|campiglia dei berici|campiglia marittima|campiglione fenile|campione d'italia|campitello di fassa|campli|campo calabro|campo di giove|campo di trens|campo ligure|campo nell'elba|campo san martino|campo tures|campobasso|campobello di licata|campobello di mazara|campochiaro|campodarsego|campodenno|campodimele|campodipietra|campodolcino|campodoro|campofelice di fitalia|campofelice di roccella|campofilone|campofiorito|campoformido|campofranco|campogalliano|campolattaro|campoli appennino|campoli del monte taburno|campolieto|campolongo maggiore|campolongo tapogliano|campomaggiore|campomarino|campomorone|camponogara|campora|camporeale|camporgiano|camporosso|camporotondo di fiastrone|camporotondo etneo|camposampiero|camposano|camposanto|campospinoso albaredo|campotosto|camugnano|canal san bovo|canale|canale d'agordo|canale monterano|canaro|canazei|cancellara|cancello ed arnone|canda|candela|candelo|candia canavese|candia lomellina|candiana|candida|candidoni|candiolo|canegrate|canelli|canepina|caneva|canicattini bagni|canicattì|canino|canischio|canistro|canna|cannalonga|cannara|cannero riviera|canneto pavese|canneto sull'oglio|cannobio|cannole|canolo|canonica d'adda|canosa di puglia|canosa sannita|canosio|canossa|cansano|cantagallo|cantalice|cantalupa|cantalupo in sabina|cantalupo ligure|cantalupo nel sannio|cantarana|cantello|canterano|cantiano|cantoira|cantù|canzano|canzo|caorle|caorso|capaccio paestum|capaci|capalbio|capannoli|capannori|capena|capergnanica|capestrano|capiago intimiano|capistrano|capistrello|capitignano|capizzi|capizzone|capo d'orlando|capo di ponte|capodimonte|capodrise|capoliveri|capolona|caponago|caporciano|caposele|capoterra|capovalle|cappadocia|cappella cantone|cappella de' picenardi|cappella maggiore|cappelle sul tavo|capracotta|capraia e limite|capraia isola|capralba|capranica|capranica prenestina|caprarica di lecce|caprarola|caprauna|caprese michelangelo|caprezzo|capri|capri leone|capriana|capriano del colle|capriata d'orba|capriate san gervasio|capriati a volturno|caprie|capriglia irpina|capriglio|caprile|caprino bergamasco|caprino veronese|capriolo|capriva del friuli|capua|capurso|caraffa del bianco|caraffa di catanzaro|caraglio|caramagna piemonte|caramanico terme|carapelle|carapelle calvisio|carasco|carassai|carate brianza|carate urio|caravaggio|caravate|caravino|caravonica|carbognano|carbonara al ticino|carbonara di nola|carbonara scrivia|carbonate|carbone|carbonera|carbonia|carcare|carcoforo|cardano al campo|cardedu|cardeto|cardinale|cardito|cardè|careggine|carema|carenno|carentino|careri|caresana|caresanablot|carezzano|carfizzi|cargeghe|cariati|carife|carignano|carimate|carinaro|carini|carinola|carisio|carisolo|carlantino|carlazzo|carlentini|carlino|carloforte|carlopoli|carmagnola|carmiano|carmignano|carmignano di brenta|carnago|carnate|carobbio degli angeli|carolei|carona|caronia|caronno pertusella|caronno varesino|carosino|carovigno|carovilli|carpaneto piacentino|carpanzano|carpegna|carpenedolo|carpeneto|carpi|carpiano|carpignano salentino|carpignano sesia|carpineti|carpineto della nora|carpineto romano|carpineto sinello|carpino|carpinone|carrara|carrega ligure|carro|carrodano|carrosio|carrè|carrù|carsoli|cartigliano|cartignano|cartoceto|cartosio|cartura|carugate|carugo|carunchio|carvico|carzano|casabona|casacalenda|casacanditella|casagiove|casal cermelli|casal di principe|casal velino|casalanguida|casalattico|casalbeltrame|casalbordino|casalbore|casalborgone|casalbuono|casalbuttano ed uniti|casalciprano|casalduni|casale corte cerro|casale cremasco-vidolasco|casale di scodosia|casale litta|casale marittimo|casale monferrato|casale sul sile|casalecchio di reno|casaleggio boiro|casaleggio novara|casaleone|casaletto ceredano|casaletto di sopra|casaletto lodigiano|casaletto spartano|casaletto vaprio|casalfiumanese|casalgrande|casalgrasso|casali del manco|casalincontrada|casalino|casalmaggiore|casalmaiocco|casalmorano|casalmoro|casalnoceto|casalnuovo di napoli|casalnuovo monterotaro|casaloldo|casalpusterlengo|casalromano|casalserugo|casaluce|casalvecchio di puglia|casalvecchio siculo|casalvieri|casalvolone|casalzuigno|casamarciano|casamassima|casamicciola terme|casandrino|casanova elvo|casanova lerrone|casanova lonati|casape|casapesenna|casapinta|casaprota|casapulla|casarano|casargo|casarile|casarsa della delizia|casarza ligure|casasco|casatenovo|casatisma|casavatore|casazza|cascia|casciago|casciana terme lari|cascina|cascinette d'ivrea|casei gerola|caselette|casella|caselle in pittari|caselle landi|caselle lurani|caselle torinese|caserta|casier|casignana|casina|casirate d'adda|caslino d'erba|casnate con bernate|casnigo|casola di napoli|casola in lunigiana|casola valsenio|casole d'elsa|casoli|casorate primo|casorate sempione|casorezzo|casoria|casorzo monferrato|casperia|caspoggio|cassacco|cassago brianza|cassano all'ionio|cassano d'adda|cassano delle murge|cassano irpino|cassano magnago|cassano spinola|cassano valcuvia|cassaro|cassiglio|cassina de' pecchi|cassina rizzardi|cassina valsassina|cassinasco|cassine|cassinelle|cassinetta di lugagnano|cassino|cassola|cassolnovo|castagnaro|castagneto carducci|castagneto po|castagnito|castagnole delle lanze|castagnole monferrato|castagnole piemonte|castana|castano primo|casteggio|castegnato|castegnero nanto|castel baronia|castel boglione|castel bolognese|castel campagnano|castel castagna|castel condino|castel d'aiano|castel d'ario|castel d'azzano|castel del giudice|castel del monte|castel del piano|castel del rio|castel di casio|castel di ieri|castel di iudica|castel di lama|castel di lucio|castel di sangro|castel di sasso|castel di tora|castel focognano|castel frentano|castel gabbiano|castel gandolfo|castel giorgio|castel goffredo|castel guelfo di bologna|castel ivano|castel madama|castel maggiore|castel mella|castel morrone|castel ritaldi|castel rocchero|castel rozzone|castel san giorgio|castel san giovanni|castel san lorenzo|castel san niccolò|castel san pietro romano|castel san pietro terme|castel san vincenzo|castel sant'angelo|castel sant'elia|castel viscardo|castel vittorio|castel volturno|castelbaldo|castelbelforte|castelbellino|castelbello-ciardes|castelbianco|castelbottaccio|castelbuono|castelcivita|castelcovati|castelcucco|casteldaccia|casteldelci|casteldelfino|casteldidone|castelfidardo|castelfiorentino|castelforte|castelfranci|castelfranco di sotto|castelfranco emilia|castelfranco in miscano|castelfranco piandiscò|castelfranco veneto|castelgerundo|castelgomberto|castelgrande|castelguglielmo|castelguidone|castell'alfero|castell'arquato|castell'azzara|castell'umberto|castellabate|castellafiume|castellalto|castellammare del golfo|castellammare di stabia|castellamonte|castellana grotte|castellana sicula|castellaneta|castellania coppi|castellanza|castellar guidobono|castellarano|castellaro|castellazzo bormida|castellazzo novarese|castelleone|castelleone di suasa|castellero|castelletto cervo|castelletto d'erro|castelletto d'orba|castelletto di branduzzo|castelletto merli|castelletto molina|castelletto monferrato|castelletto sopra ticino|castelletto stura|castelletto uzzone|castelli|castelli calepio|castellina in chianti|castellina marittima|castellinaldo d'alba|castellino del biferno|castellino tanaro|castelliri|castello cabiaglio|castello d'agogna|castello d'argile|castello del matese|castello dell'acqua|castello di annone|castello di brianza|castello di cisterna|castello di godego|castello tesino|castello-molina di fiemme|castellucchio|castelluccio dei sauri|castelluccio inferiore|castelluccio superiore|castelluccio valmaggiore|castelmagno|castelmarte|castelmassa|castelmauro|castelmezzano|castelmola|castelnovetto|castelnovo bariano|castelnovo del friuli|castelnovo di sotto|castelnovo ne' monti|castelnuovo|castelnuovo belbo|castelnuovo berardenga|castelnuovo bocca d'adda|castelnuovo bormida|castelnuovo bozzente|castelnuovo calcea|castelnuovo cilento|castelnuovo del garda|castelnuovo della daunia|castelnuovo di ceva|castelnuovo di conza|castelnuovo di farfa|castelnuovo di garfagnana|castelnuovo di porto|castelnuovo di val di cecina|castelnuovo don bosco|castelnuovo magra|castelnuovo nigra|castelnuovo parano|castelnuovo rangone|castelnuovo scrivia|castelpagano|castelpetroso|castelpizzuto|castelplanio|castelpoto|castelraimondo|castelrotto|castelsantangelo sul nera|castelsaraceno|castelsardo|castelseprio|castelsilano|castelspina|casteltermini|castelveccana|castelvecchio calvisio|castelvecchio di rocca barbena|castelvecchio subequo|castelvenere|castelverde|castelverrino|castelvetere in val fortore|castelvetere sul calore|castelvetrano|castelvetro di modena|castelvetro piacentino|castelvisconti|castenaso|castenedolo|castiadas|castiglion fibocchi|castiglion fiorentino|castiglione a casauria|castiglione chiavarese|castiglione cosentino|castiglione d'adda|castiglione d'orcia|castiglione dei pepoli|castiglione del genovesi|castiglione del lago|castiglione della pescaia|castiglione delle stiviere|castiglione di garfagnana|castiglione di sicilia|castiglione falletto|castiglione in teverina|castiglione messer marino|castiglione messer raimondo|castiglione olona|castiglione tinella|castiglione torinese|castignano|castilenti|castino|castione andevenno|castione della presolana|castions di strada|castiraga vidardo|casto|castorano|castrezzato|castri di lecce|castrignano de' greci|castrignano del capo|castro|castro dei volsci|castrocaro terme e terra del sole|castrocielo|castrofilippo|castrolibero|castronno|castronovo di sicilia|castronuovo di sant'andrea|castropignano|castroreale|castroregio|castrovillari|catania|catanzaro|catenanuova|catignano|cattolica|cattolica eraclea|caulonia|cautano|cava de' tirreni|cava manara|cavaglietto|cavaglio d'agogna|cavaglià|cavagnolo|cavaion veronese|cavalese|cavallerleone|cavallermaggiore|cavallino|cavallino-treporti|cavallirio|cavareno|cavargna|cavaria con premezzo|cavarzere|cavaso del tomba|cavasso nuovo|cavatore|cavazzo carnico|cave|cavedago|cavedine|cavenago d'adda|cavenago di brianza|cavernago|cavezzo|cavizzana|cavour|cavriago|cavriana|cavriglia|cazzago brabbia|cazzago san martino|cazzano di tramigna|cazzano sant'andrea|ceccano|cecima|cecina|cedegolo|cedrasco|cefalà diana|cefalù|ceggia|ceglie messapica|celano|celenza sul trigno|celenza valfortore|celico|cella dati|cella monte|cellamare|cellara|cellarengo|cellatica|celle di bulgheria|celle di macra|celle di san vito|celle enomondo|celle ligure|celleno|cellere|cellino attanasio|cellino san marco|cellio con breia|cellole|cembra lisignago|cenadi|cenate sopra|cenate sotto|cencenighe agordino|cene|ceneselli|cengio|centallo|cento|centola|centrache|centro valle intelvi|centuripe|cepagatti|ceppaloni|ceppo morelli|ceprano|cerami|ceranesi|cerano|cerano d'intelvi|ceranova|ceraso|cercemaggiore|cercenasco|cercepiccola|cerchiara di calabria|cerchio|cercino|cercivento|cercola|cerda|cerea|ceregnano|cerenzia|ceres|ceresara|cereseto|ceresole alba|ceresole reale|cerete|ceretto lomellina|cergnago|ceriale|ceriana|ceriano laghetto|cerignale|cerignola|cerisano|cermenate|cermes|cermignano|cernobbio|cernusco lombardone|cernusco sul naviglio|cerreto d'asti|cerreto d'esi|cerreto di spoleto|cerreto grue|cerreto guidi|cerreto laziale|cerreto sannita|cerretto langhe|cerrina monferrato|cerrione|cerro al lambro|cerro al volturno|cerro maggiore|cerro tanaro|cerro veronese|cersosimo|certaldo|certosa di pavia|cerva|cervara di roma|cervarese santa croce|cervaro|cervasca|cervatto|cerveno|cervere|cervesina|cerveteri|cervia|cervicati|cervignano d'adda|cervignano del friuli|cervinara|cervino|cervo|cerzeto|cesa|cesana brianza|cesana torinese|cesano boscone|cesano maderno|cesara|cesarò|cesate|cesena|cesenatico|cesinali|cesio|cesiomaggiore|cessalto|cessaniti|cessapalombo|cessole|cetara|ceto|cetona|cetraro|ceva|cevo|challand-saint-anselme|challand-saint-victor|chambave|chamois|champdepraz|champorcher|charvensod|cherasco|cheremule|chialamberto|chiampo|chianche|chianciano terme|chianni|chianocco|chiaramonte gulfi|chiaramonti|chiarano|chiaravalle|chiaravalle centrale|chiari|chiaromonte|chiauci|chiavari|chiavenna|chiaverano|chienes|chieri|chies d'alpago|chiesa in valmalenco|chiesanuova|chiesina uzzanese|chieti|chieuti|chieve|chignolo d'isola|chignolo po|chioggia|chiomonte|chions|chiopris-viscone|chitignano|chiuduno|chiuppano|chiuro|chiusa|chiusa di pesio|chiusa di san michele|chiusa sclafani|chiusaforte|chiusanico|chiusano d'asti|chiusano di san domenico|chiusavecchia|chiusdino|chiusi|chiusi della verna|chivasso|châtillon|ciampino|cianciana|cibiana di cadore|cicagna|cicala|cicciano|cicerale|ciciliano|cicognolo|ciconio|cigliano|cigliè|cigognola|cigole|cilavegna|cimadolmo|cimbergo|ciminna|ciminà|cimitile|cimolais|cimone|cinaglio|cineto romano|cingia de' botti|cingoli|cinigiano|cinisello balsamo|cinisi|cino|cinquefrondi|cintano|cinte tesino|cinto caomaggiore|cinto euganeo|cinzano|ciorlano|cipressa|circello|cirigliano|cirimido|ciriè|cirò|cirò marina|cis|cisano bergamasco|cisano sul neva|ciserano|cislago|cisliano|cison di valmarino|cissone|cisterna d'asti|cisterna di latina|cisternino|citerna|cittadella|cittaducale|cittanova|cittareale|cittiglio|città della pieve|città di castello|città sant'angelo|civate|civezza|civezzano|civiasco|cividale del friuli|cividate al piano|cividate camuno|civita|civita castellana|civita d'antino|civitacampomarano|civitaluparella|civitanova del sannio|civitanova marche|civitaquana|civitavecchia|civitella alfedena|civitella casanova|civitella d'agliano|civitella del tronto|civitella di romagna|civitella in val di chiana|civitella messer raimondo|civitella paganico|civitella roveto|civitella san paolo|civo|claino con osteno|claut|clauzetto|clavesana|claviere|cles|cleto|clivio|clusone|coassolo torinese|coazze|coazzolo|coccaglio|cocconato|cocquio-trevisago|cocullo|codevigo|codevilla|codigoro|codogno|codognè|codroipo|codrongianos|coggiola|cogliate|cogne|cogoleto|cogollo del cengio|cogorno|colazza|colceresa|colere|colfelice|coli|colico|collalto sabino|collarmele|collazzone|colle brianza|colle d'anchise|colle di tora|colle di val d'elsa|colle san magno|colle sannita|colle santa lucia|colle umberto|collebeato|collecchio|collecorvino|colledara|colledimacine|colledimezzo|colleferro|collegiove|collegno|collelongo|collepardo|collepasso|collepietro|colleretto castelnuovo|colleretto giacosa|collesalvetti|collesano|colletorto|collevecchio|colli a volturno|colli al metauro|colli del tronto|colli sul velino|colli verdi|colliano|collinas|collio|collobiano|colloredo di monte albano|colmurano|colobraro|cologna veneta|cologne|cologno al serio|cologno monzese|colognola ai colli|colonna|colonnella|colonno|colorina|colorno|colosimi|colturano|colverde|colzate|comabbio|comacchio|comano|comano terme|comazzo|comeglians|comelico superiore|comerio|comezzano-cizzago|comignago|comiso|comitini|comiziano|commessaggio|commezzadura|como|compiano|comun nuovo|comunanza|cona|conca casale|conca dei marini|conca della campania|concamarise|concerviano|concesio|concordia sagittaria|concordia sulla secchia|concorezzo|condofuri|condove|condrò|conegliano|confienza|configni|conflenti|coniolo|conselice|conselve|contessa entellina|contigliano|contrada|controguerra|controne|contursi terme|contà|conversano|conza della campania|conzano|copertino|copiano|copparo|corana|corato|corbara|corbetta|corbola|corchiano|corciano|cordenons|cordignano|cordovado|coreglia antelminelli|coreglia ligure|coreno ausonio|corfinio|cori|coriano|corigliano d'otranto|corigliano-rossano|corinaldo|corio|corleone|corleto monforte|corleto perticara|cormano|cormons|corna imagna|cornalba|cornale e bastida|cornaredo|cornate d'adda|cornedo all'isarco|cornedo vicentino|cornegliano laudense|corneliano d'alba|corniglio|corno di rosazzo|corno giovine|cornovecchio|cornuda|correggio|correzzana|correzzola|corrido|corridonia|corropoli|corsano|corsico|corsione|cortaccia sulla strada del vino|cortale|cortandone|cortanze|cortazzone|corte brugnatella|corte de' cortesi con cignone|corte de' frati|corte franca|corte palasio|cortemaggiore|cortemilia|corteno golgi|cortenova|cortenuova|corteolona e genzone|cortiglione|cortina d'ampezzo|cortina sulla strada del vino|cortino|cortona|corvara|corvara in badia|corvino san quirico|corzano|coseano|cosenza|cosio d'arroscia|cosio valtellino|cosoleto|cossano belbo|cossano canavese|cossato|cosseria|cossignano|cossogno|cossoine|cossombrato|costa de' nobili|costa di mezzate|costa di rovigo|costa masnaga|costa serina|costa valle imagna|costa vescovato|costa volpino|costabissara|costacciaro|costanzana|costarainera|costermano sul garda|costigliole d'asti|costigliole saluzzo|cotignola|cotronei|cottanello|courmayeur|covo|cozzo|craco|crandola valsassina|cravagliana|cravanzana|craveggia|creazzo|crecchio|credaro|credera rubbiano|crema|cremella|cremenaga|cremeno|cremia|cremolino|cremona|cremosano|crescentino|crespadoro|crespiatica|crespina lorenzana|crespino|cressa|crevacuore|crevalcore|crevoladossola|crispano|crispiano|crissolo|crocefieschi|crocetta del montello|crodo|crognaleto|cropalati|cropani|crosia|crosio della valle|crotone|crotta d'adda|crova|croviana|crucoli|cuasso al monte|cuccaro vetere|cucciago|cuceglio|cuggiono|cugliate-fabiasco|cuglieri|cugnoli|cumiana|cumignano sul naviglio|cunardo|cuneo|cunico|cuorgnè|cupello|cupra marittima|cupramontana|cura carpignano|curcuris|cureggio|curiglia con monteviasco|curinga|curino|curno|curon venosta|cursi|curtarolo|curtatone|curti|cusago|cusano milanino|cusano mutri|cusino|cusio|custonaci|cutro|cutrofiano|cuveglio|cuvio|dairago|dalmine|dambel|danta di cadore|darfo boario terme|dasà|davagna|daverio|davoli|dazio|decimomannu|decimoputzu|decollatura|dego|deiva marina|delebio|delia|delianuova|deliceto|dello|demonte|denice|denno|dernice|derovere|deruta|dervio|desana|desenzano del garda|desio|desulo|deutschnofen|devin nabrežina|diamante|diano arentino|diano castello|diano d'alba|diano marina|diano san pietro|dicomano|dignano|dimaro folgarida|dinami|dipignano|diso|divignano|dizzasco|dobbiaco|doberdob|doberdò del lago|dogliani|dogliola|dogna|dolceacqua|dolcedo|dolcè|dolegna del collio|dolianova|dolina|dolo|dolzago|domanico|domaso|domegge di cadore|domicella|domodossola|domus de maria|domusnovas|donato|dongo|donnas|donori|dorgali|dorio|dormelletto|dorno|dorzano|dosolo|dossena|dosso del liro|doues|dovadola|dovera|dozza|dragoni|drapia|drena|drenchia|dresano|dro|dronero|druento|druogno|dualchi|dubino|due carrare|dueville|dugenta|duino aurisina|dumenza|duno|durazzano|duronia|dusino san michele|eboli|edolo|egna|elice|elini|ello|elmas|elva|emarèse|empoli|endine gaiano|enego|enemonzo|enna|enneberg|entracque|entratico|envie|episcopia|eppan an der weinstraße|eraclea|erba|erbezzo|erbusco|erbè|erchie|ercolano|erice|erli|erto e casso|erula|erve|esanatoglia|escalaplano|escolca|esine|esino lario|esperia|esporlatu|este|esterzili|etroubles|eupilio|exilles|fabbrica curone|fabbriche di vergemoli|fabbrico|fabriano|fabrica di roma|fabrizia|fabro|faedis|faedo valtellino|faenza|faeto|fagagna|faggeto lario|faggiano|fagnano alto|fagnano castello|fagnano olona|fai della paganella|faicchio|falcade|falciano del massico|falconara albanese|falconara marittima|falcone|faleria|falerna|falerone|fallo|faloppio|falvaterra|falzes|fanano|fanna|fano|fano adriano|fara filiorum petri|fara gera d'adda|fara in sabina|fara novarese|fara olivana con sola|fara san martino|fara vicentino|fardella|farigliano|farindola|farini|farnese|farra d'isonzo|farra di soligo|fasano|fascia|fauglia|faule|favale di malvaro|favara|favignana|favria|feisoglio|feldthurns|feletto|felino|felitto|felizzano|feltre|fenegrò|fenestrelle|ferentillo|ferentino|ferla|fermignano|fermo|ferno|feroleto antico|feroleto della chiesa|ferrandina|ferrara|ferrara di monte baldo|ferrazzano|ferrera di varese|ferrera erbognone|ferrere|ferriere|ferruzzano|fiamignano|fiano|fiano romano|fiastra|fiavè|ficarazzi|ficarolo|ficarra|ficulle|fidenza|fierozzo|fiesco|fiesole|fiesse|fiesso d'artico|fiesso umbertiano|figino serenza|figline e incisa valdarno|figline vegliaturo|filacciano|filadelfia|filago|filandari|filattiera|filettino|filetto|filiano|filighera|filignano|filogaso|filottrano|finale emilia|finale ligure|fino del monte|fino mornasco|fiorano al serio|fiorano canavese|fiorano modenese|fiorenzuola d'arda|firenze|firenzuola|firmo|fiscaglia|fisciano|fiuggi|fiumalbo|fiumara|fiume veneto|fiumedinisi|fiumefreddo bruzio|fiumefreddo di sicilia|fiumicello villa vicentina|fiumicino|fiuminata|fivizzano|fiè allo sciliar|flaibano|flero|floresta|floridia|florinas|flumeri|fluminimaggiore|flussio|fobello|foggia|foglianise|fogliano redipuglia|foglizzo|foiano della chiana|foiano di val fortore|folgaria|folignano|foligno|follina|follo|follonica|fombio|fondachelli-fantina|fondi|fonni|fontainemore|fontana liri|fontanafredda|fontanarosa|fontanelice|fontanella|fontanellato|fontanelle|fontaneto d'agogna|fontanetto po|fontanigorda|fontanile|fontaniva|fonte|fonte nuova|fontecchio|fontechiari|fontegreca|fonteno|fontevivo|fonzaso|foppolo|forano|force|forchia|forcola|fordongianus|forenza|foresto sparso|forgaria nel friuli|forino|forio|forlimpopoli|forlì|forlì del sannio|formazza|formello|formia|formicola|formigara|formigine|formigliana|fornace|fornelli|forni avoltri|forni di sopra|forni di sotto|forno canavese|fornovo di taro|fornovo san giovanni|forte dei marmi|fortezza|fortunago|forza d'agrò|fosciandora|fosdinovo|fossa|fossacesia|fossalta di piave|fossalta di portogruaro|fossalto|fossano|fossato di vico|fossato serralta|fossombrone|fossò|foza|frabosa soprana|frabosa sottana|fraconalto|fragagnano|fragneto l'abate|fragneto monforte|fraine|framura|francavilla al mare|francavilla angitola|francavilla bisio|francavilla d'ete|francavilla di sicilia|francavilla fontana|francavilla in sinni|francavilla marittima|francica|francofonte|francolise|franzensfeste|frascaro|frascarolo|frascati|frascineto|frassilongo|frassinelle polesine|frassinello monferrato|frassineto po|frassinetto|frassino|frassinoro|frasso sabino|frasso telesino|fratta polesine|fratta todina|frattamaggiore|frattaminore|fratte rosa|frazzanò|fregona|freienfeld|fresagrandinaria|fresonara|frigento|frignano|frinco|frisa|frisanco|front|frontino|frontone|frosinone|frosolone|frossasco|frugarolo|fubine monferrato|fucecchio|fuipiano valle imagna|fumane|fumone|funes|furci|furci siculo|furnari|furore|furtei|fuscaldo|fusignano|fusine|futani|fénis|gabbioneta-binanuova|gabiano|gabicce mare|gaby|gadesco-pieve delmona|gadoni|gaeta|gaggi|gaggiano|gaggio montano|gaglianico|gagliano aterno|gagliano castelferrato|gagliano del capo|gagliato|gagliole|gaiarine|gaiba|gaiola|gaiole in chianti|gairo|gais|galati mamertino|galatina|galatone|galatro|galbiate|galeata|galgagnano|gallarate|gallese|galliate|galliate lombardo|galliavola|gallicano|gallicano nel lazio|gallicchio|galliera|galliera veneta|gallinaro|gallio|gallipoli|gallo matese|gallodoro|galluccio|galtellì|galzignano terme|gamalero|gambara|gambarana|gambasca|gambassi terme|gambatesa|gambellara|gamberale|gambettola|gambolò|gandellino|gandino|gandosso|gangi|garaguso|garbagna|garbagna novarese|garbagnate milanese|garbagnate monastero|garda|gardone riviera|gardone val trompia|garessio|gargallo|gargazon|gargazzone|gargnano|garlasco|garlate|garlenda|garniga terme|garzeno|garzigliana|gasperina|gassino torinese|gattatico|gatteo|gattico-veruno|gattinara|gavardo|gavello|gaverina terme|gavi|gavignano|gavirate|gavoi|gavorrano|gazoldo degli ippoliti|gazzada schianno|gazzaniga|gazzo|gazzo veronese|gazzola|gazzuolo|gela|gemmano|gemona del friuli|gemonio|genazzano|genga|genivolta|genola|genoni|genova|genuri|genzano di lucania|genzano di roma|gera lario|gerace|geraci siculo|gerano|gerenzago|gerenzano|gergei|germagnano|germagno|germignaga|gerocarne|gerola alta|gerre de' caprioli|gesico|gessate|gessopalena|gesturi|gesualdo|ghedi|ghemme|ghiffa|ghilarza|ghisalba|ghislarengo|giacciano con baruchella|giaglione|gianico|giano dell'umbria|giano vetusto|giardinello|giardini-naxos|giarole|giarratana|giarre|giave|giaveno|giavera del montello|giba|gibellina|gifflenga|giffone|giffoni sei casali|giffoni valle piana|gignese|gignod|gildone|gimigliano|ginestra|ginestra degli schiavoni|ginosa|gioi|gioia dei marsi|gioia del colle|gioia sannitica|gioia tauro|gioiosa ionica|gioiosa marea|giove|giovinazzo|giovo|girasole|girifalco|gissi|giuggianello|giugliano in campania|giuliana|giuliano di roma|giuliano teatino|giulianova|giungano|giurdignano|giussago|giussano|giustenice|giustino|giusvalla|givoletto|gizzeria|glorenza|glurns|godega di sant'urbano|godiasco salice terme|godrano|goito|golasecca|golferenzo|golfo aranci|gombito|gonars|goni|gonnesa|gonnoscodina|gonnosfanadiga|gonnosnò|gonnostramatza|gonzaga|gordona|gorga|gorgo al monticano|gorgoglione|gorgonzola|goriano sicoli|gorizia|gorla maggiore|gorla minore|gorlago|gorle|gornate olona|gorno|goro|gorreto|gorzegno|gosaldo|gossolengo|gottasecca|gottolengo|govone|gozzano|gradara|gradisca d'isonzo|grado|gradoli|graffignana|graffignano|graglia|gragnano|gragnano trebbiense|grammichele|grana monferrato|granarolo dell'emilia|grandate|grandola ed uniti|graniti|granozzo con monticello|grantola|grantorto|granze|grassano|grassobbio|gratteri|graun im vinschgau|gravedona ed uniti|gravellona lomellina|gravellona toce|gravere|gravina di catania|gravina in puglia|grazzanise|grazzano badoglio|greccio|greci|greggio|gremiasco|gressan|gressoney-la-trinité|gressoney-saint-jean|greve in chianti|grezzago|grezzana|griante|gricignano di aversa|grignasco|grigno|grimacco|grimaldi|grinzane cavour|grisignano di zocco|grisolia|grizzana morandi|grognardo|gromo|grondona|grone|grontardo|gropello cairoli|gropparello|groscavallo|grosio|grosotto|grosseto|grosso|grottaferrata|grottaglie|grottaminarda|grottammare|grottazzolina|grotte|grotte di castro|grotteria|grottole|grottolella|gruaro|grugliasco|grumello cremonese ed uniti|grumello del monte|grumento nova|grumo appula|grumo nevano|grumolo delle abbadesse|gsies|guagnano|gualdo|gualdo cattaneo|gualdo tadino|gualtieri|gualtieri sicaminò|guamaggiore|guanzate|guarcino|guarda veneta|guardabosone|guardamiglio|guardavalle|guardea|guardia lombardi|guardia perticara|guardia piemontese|guardia sanframondi|guardiagrele|guardialfiera|guardiaregia|guardistallo|guarene|guasila|guastalla|guazzora|gubbio|gudo visconti|guglionesi|guidizzolo|guidonia montecelio|guiglia|guilmi|gurro|guspini|gussago|gussola|hafling|hône|idro|iglesias|igliano|ilbono|illasi|illorai|imbersago|imer|imola|imperia|impruneta|inarzo|incisa scapaccino|incudine|induno olona|ingria|innichen|intragna|introbio|introd|introdacqua|inverigo|inverno e monteleone|inverso pinasca|inveruno|invorio|inzago|irgoli|irma|irsina|isasca|isca sullo ionio|ischia|ischia di castro|ischitella|iseo|isera|isernia|isili|isnello|isola d'asti|isola del cantone|isola del giglio|isola del gran sasso d'italia|isola del liri|isola del piano|isola della scala|isola delle femmine|isola di capo rizzuto|isola di fondra|isola dovarese|isola rizza|isola sant'antonio|isola vicentina|isolabella|isolabona|isole tremiti|isorella|ispani|ispica|ispra|issiglio|issime|isso|issogne|istrana|itala|itri|ittireddu|ittiri|ivrea|izano|jacurso|jelsi|jenesien|jenne|jerago con orago|jerzu|jesi|jesolo|jolanda di savoia|jonadi|joppolo|joppolo giancaxio|jovençan|kaltern an der weinstraße|karneid|kastelbell-tschars|kastelruth|kiens|klausen|kuens|kurtatsch an der weinstraße|kurtinig an der weinstraße|l'aquila|la cassa|la loggia|la maddalena|la magdeleine|la morra|la salle|la spezia|la thuile|la valle|la valle agordina|la valletta brianza|laas|labico|labro|lacchiarella|lacco ameno|lacedonia|laces|laconi|ladispoli|laerru|laganadi|laghi|laglio|lagnasco|lago|lagonegro|lagosanto|lagundo|laigueglia|lainate|laino|laino borgo|laino castello|laion|laives|lajatico|lajen|lallio|lama dei peligni|lama mocogno|lambrugo|lamezia terme|lamon|lampedusa e linosa|lamporecchio|lamporo|lana|lanciano|landiona|landriano|langhirano|langosco|lanusei|lanuvio|lanzada|lanzo torinese|lapedona|lapio|lappano|larciano|lardirago|lariano|larino|las plassas|lasa|lascari|lasnigo|lastebasse|lastra a signa|latera|laterina pergine valdarno|laterza|latiano|latina|latisana|latronico|latsch|lattarico|lauco|laureana cilento|laureana di borrello|lauregno|laurein|laurenzana|lauria|lauriano|laurino|laurito|lauro|lavagna|lavagno|lavarone|lavello|lavena ponte tresa|laveno-mombello|lavenone|laviano|lavis|lazise|lazzate|lecce|lecce nei marsi|lecco|ledro|leffe|leggiuno|legnago|legnano|legnaro|lei|leifers|leini|leivi|lemie|lendinara|leni|lenna|leno|lenola|lenta|lentate sul seveso|lentella|lentini|leonessa|leonforte|leporano|lequile|lequio berria|lequio tanaro|lercara friddi|lerici|lerma|lesa|lesegno|lesignano de' bagni|lesina|lesmo|lessolo|lessona|lestizza|letino|letojanni|lettere|lettomanoppello|lettopalena|levanto|levate|leverano|levice|levico terme|levone|lezzeno|liberi|librizzi|licata|licciana nardi|licenza|licodia eubea|lierna|lignana|lignano sabbiadoro|lillianes|limana|limatola|limbadi|limbiate|limena|limido comasco|limina|limone piemonte|limone sul garda|limosano|linarolo|linguaglossa|lioni|lipari|lipomo|liscate|liscia|lisciano niccone|lisio|lissone|liveri|livigno|livinallongo del col di lana|livo|livorno|livorno ferraris|livraga|lizzanello|lizzano|lizzano in belvedere|loano|loazzolo|locana|locate di triulzi|locate varesino|locatello|loceri|locorotondo|locri|loculi|lodi|lodi vecchio|lodine|lodrino|lodè|lograto|loiano|loiri porto san paolo|lomagna|lomazzo|lombardore|lombriasco|lomello|lona-lases|lonate ceppino|lonate pozzolo|lonato del garda|londa|longano|longare|longarone|longhena|longi|longiano|longobardi|longobucco|longone al segrino|longone sabino|lonigo|loranzè|loreggia|loreglia|lorenzago di cadore|loreo|loreto|loreto aprutino|loria|loro ciuffenna|loro piceno|lorsica|losine|lotzorai|lovere|lovero|lozio|lozza|lozzo atestino|lozzo di cadore|lozzolo|lu e cuccaro monferrato|lubriano|lucca|lucca sicula|lucera|lucignano|lucinasco|lucito|luco dei marsi|lucoli|lugagnano val d'arda|lugnano in teverina|lugo|lugo di vicenza|luino|luisago|lula|lumarzo|lumezzane|lunamatrona|lunano|lungavilla|lungro|luni|luogosano|luogosanto|lupara|lurago d'erba|lurago marinone|lurano|luras|lurate caccivio|lusciano|luserna|luserna san giovanni|lusernetta|lusevera|lusia|lusiana conco|lusigliè|luson|lustra|luvinate|luzzana|luzzara|luzzi|lüsen|maccagno con pino e veddasca|maccastorna|macchia d'isernia|macchia valfortore|macchiagodena|macello|macerata|macerata campania|macerata feltria|macherio|maclodio|macomer|macra|macugnaga|maddaloni|madesimo|madignano|madone|madonna del sasso|madruzzo|maenza|mafalda|magasa|magenta|maggiora|magherno|magione|magisano|magliano alfieri|magliano alpi|magliano de' marsi|magliano di tenna|magliano in toscana|magliano romano|magliano sabina|magliano vetere|maglie|magliolo|maglione|magnacavallo|magnago|magnano|magnano in riviera|magomadas|magreglio|magrè sulla strada del vino|maida|maierato|maierà|maiolati spontini|maiolo|maiori|mairago|mairano|maissana|majano|malagnino|malalbergo|malborghetto valbruna|malcesine|malegno|maleo|malesco|maletto|malfa|malgrate|malito|mallare|malles venosta|malnate|malo|malonno|mals|maltignano|malvagna|malvicino|malvito|malé|mammola|mamoiada|manciano|mandanici|mandas|mandatoriccio|mandela|mandello del lario|mandello vitta|manduria|manerba del garda|manerbio|manfredonia|mango|mangone|maniace|maniago|manocalzati|manoppello|mansuè|manta|mantello|mantova|manzano|manziana|mapello|mappano|mara|maracalagonis|maranello|marano di napoli|marano di valpolicella|marano equo|marano lagunare|marano marchesato|marano principato|marano sul panaro|marano ticino|marano vicentino|maranzana|maratea|marcallo con casone|marcaria|marcedusa|marcellina|marcellinara|marcetelli|marcheno|marchirolo|marciana|marciana marina|marcianise|marciano della chiana|marcignago|marcon|marebbe|marene|mareno di piave|marentino|maretto|margarita|margherita di savoia|margno|margreid an der weinstraße|mariana mantovana|mariano comense|mariano del friuli|marianopoli|mariglianella|marigliano|marina di gioiosa ionica|marineo|marino|marlengo|marliana|marling|marmentino|marmirolo|marmora|marnate|marone|maropati|marostica|marradi|marrubiu|marsaglia|marsala|marsciano|marsico nuovo|marsicovetere|marta|martano|martell|martellago|martello|martignacco|martignana di po|martignano|martina franca|martinengo|martiniana po|martinsicuro|martirano|martirano lombardo|martis|martone|marudo|maruggio|marzabotto|marzano|marzano appio|marzano di nola|marzi|marzio|masainas|masate|mascali|mascalucia|maschito|masciago primo|maser|masera|maserada sul piave|maserà di padova|masi|masi torello|masio|maslianico|masone|massa|massa d'albe|massa di somma|massa e cozzile|massa fermana|massa lombarda|massa lubrense|massa marittima|massa martana|massafra|massalengo|massanzago|massarosa|massazza|massello|masserano|massignano|massimeno|massimino|massino visconti|massiola|masullas|matelica|matera|mathi|matino|matrice|mattie|mattinata|mazara del vallo|mazzano|mazzano romano|mazzarino|mazzarrone|mazzarrà sant'andrea|mazzin|mazzo di valtellina|mazzè|meana di susa|meana sardo|meda|mede|medea|medesano|medicina|mediglia|medolago|medole|medolla|meduna di livenza|meduno|megliadino san vitale|meina|melara|melazzo|meldola|mele|melegnano|melendugno|meleti|melfi|melicucco|melicuccà|melilli|melissa|melissano|melito di napoli|melito di porto salvo|melito irpino|melizzano|melle|mello|melpignano|meltina|melzo|menaggio|menconico|mendatica|mendicino|menfi|mentana|meolo|meran|merana|merano|merate|mercallo|mercatello sul metauro|mercatino conca|mercato san severino|mercato saraceno|mercenasco|mercogliano|mereto di tomba|mergo|mergozzo|merlara|merlino|merone|merì|mesagne|mese|mesenzana|mesero|mesola|mesoraca|messina|mestrino|meta|mezzago|mezzana|mezzana bigli|mezzana mortigliengo|mezzana rabattone|mezzane di sotto|mezzanego|mezzanino|mezzano|mezzenile|mezzocorona|mezzojuso|mezzoldo|mezzolombardo|mezzomerico|miagliano|miane|miasino|miazzina|micigliano|miggiano|miglianico|miglierina|miglionico|mignanego|mignano monte lungo|milano|milazzo|milena|mileto|milis|militello in val di catania|militello rosmarino|millesimo|milo|milzano|mineo|minerbe|minerbio|minervino di lecce|minervino murge|minori|minturno|minucciano|mioglia|mira|mirabella eclano|mirabella imbaccari|mirabello monferrato|mirabello sannitico|miradolo terme|miranda|mirandola|mirano|mirto|misano adriatico|misano di gera d'adda|misiliscemi|misilmeri|misinto|missaglia|missanello|misterbianco|mistretta|moasca|moconesi|modena|modica|modigliana|modolo|modugno|moena|moggio|moggio udinese|moglia|mogliano|mogliano veneto|mogorella|mogoro|moiano|moimacco|moio alcantara|moio de' calvi|moio della civitella|moiola|mola di bari|molare|molazzana|molfetta|molina aterno|molinara|molinella|molini di triora|molino dei torti|molise|moliterno|mollia|molochio|molteno|moltrasio|molveno|mombaldone|mombarcaro|mombaroccio|mombaruzzo|mombasiglio|mombello di torino|mombello monferrato|mombercelli|momo|mompantero|mompeo|momperone|monacilioni|monale|monasterace|monastero bormida|monastero di lanzo|monastero di vasco|monasterolo casotto|monasterolo del castello|monasterolo di savigliano|monastier di treviso|monastir|moncalieri|moncalvo|moncenisio|moncestino|monchiero|monchio delle corti|moncrivello|moncucco torinese|mondaino|mondavio|mondolfo|mondovì|mondragone|moneglia|monesiglio|monfalcone|monforte d'alba|monforte san giorgio|monfumo|mongardino|monghidoro|mongiana|mongiardino ligure|mongiuffi melia|mongrando|mongrassano|monguelfo-tesido|monguzzo|moniga del garda|monleale|monno|monopoli|monreale|monrupino|monsampietro morico|monsampolo del tronto|monsano|monselice|monserrato|monsummano terme|montabone|montacuto|montafia|montagano|montagna in valtellina|montagna sulla strada del vino|montagnana|montagnareale|montaguto|montaione|montalbano elicona|montalbano jonico|montalcino|montaldeo|montaldo bormida|montaldo di mondovì|montaldo roero|montaldo scarampi|montaldo torinese|montale|montalenghe|montallegro|montalto carpasio|montalto delle marche|montalto di castro|montalto dora|montalto pavese|montalto uffugo|montan an der weinstraße|montanaro|montanaso lombardo|montanera|montano antilia|montano lucino|montappone|montaquila|montasola|montauro|montazzoli|monte argentario|monte castello di vibio|monte cavallo|monte cerignone|monte compatri|monte cremasco|monte di malo|monte di procida|monte giberto|monte grimano terme|monte isola|monte marenzo|monte porzio|monte porzio catone|monte rinaldo|monte roberto|monte romano|monte san biagio|monte san giacomo|monte san giovanni campano|monte san giovanni in sabina|monte san giusto|monte san martino|monte san pietrangeli|monte san pietro|monte san savino|monte san vito|monte sant'angelo|monte santa maria tiberina|monte urano|monte vidon combatte|monte vidon corrado|montebello della battaglia|montebello di bertona|montebello jonico|montebello sul sangro|montebello vicentino|montebelluna|montebruno|montebuono|montecalvo in foglia|montecalvo irpino|montecalvo versiggia|montecarlo|montecarotto|montecassiano|montecastello|montecastrilli|montecatini val di cecina|montecatini-terme|montecchia di crosara|montecchio|montecchio emilia|montecchio maggiore|montecchio precalcino|montechiaro d'acqui|montechiaro d'asti|montechiarugolo|montecilfone|montecopiolo|montecorice|montecorvino pugliano|montecorvino rovella|montecosaro|montecrestese|montecreto|montedinove|montedoro|montefalcione|montefalco|montefalcone appennino|montefalcone di val fortore|montefalcone nel sannio|montefano|montefelcino|monteferrante|montefiascone|montefino|montefiore conca|montefiore dell'aso|montefiorino|monteflavio|monteforte cilento|monteforte d'alpone|monteforte irpino|montefortino|montefranco|montefredane|montefusco|montegabbione|montegalda|montegaldella|montegallo|montegioco|montegiordano|montegiorgio|montegranaro|montegridolfo|montegrino valtravaglia|montegrosso d'asti|montegrosso pian latte|montegrotto terme|monteiasi|montelabbate|montelanico|montelapiano|monteleone d'orvieto|monteleone di fermo|monteleone di puglia|monteleone di spoleto|monteleone rocca doria|monteleone sabino|montelepre|montelibretti|montella|montello|montelongo|montelparo|montelupo albese|montelupo fiorentino|montelupone|montemaggiore belsito|montemagno monferrato|montemale di cuneo|montemarano|montemarciano|montemarzino|montemesola|montemezzo|montemignaio|montemiletto|montemilone|montemitro|montemonaco|montemurlo|montemurro|montenars|montenero di bisaccia|montenero sabino|montenero val cocchiara|montenerodomo|monteodorisio|montepaone|monteparano|monteprandone|montepulciano|monterchi|montereale|montereale valcellina|monterenzio|monteriggioni|monteroduni|monteroni d'arbia|monteroni di lecce|monterosi|monterosso al mare|monterosso almo|monterosso calabro|monterosso grana|monterotondo|monterotondo marittimo|monterubbiano|montesano salentino|montesano sulla marcellana|montesarchio|montescaglioso|montescano|montescheno|montescudaio|montescudo-monte colombo|montese|montesegale|montesilvano|montespertoli|monteu da po|monteu roero|montevago|montevarchi|montevecchia|monteverde|monteverdi marittimo|monteviale|montezemolo|monti|montiano|monticelli brusati|monticelli d'ongina|monticelli pavese|monticello brianza|monticello conte otto|monticello d'alba|montichiari|monticiano|montieri|montiglio monferrato|montignoso|montirone|montjovet|montodine|montoggio|montone|montopoli di sabina|montopoli in val d'arno|montorfano|montorio al vomano|montorio nei frentani|montorio romano|montoro|montorso vicentino|montottone|montresta|montà|montù beccaria|monvalle|monza|monzambano|monzuno|moos in passeier|morano calabro|morano sul po|moransengo-tonengo|moraro|morazzone|morbegno|morbello|morciano di leuca|morciano di romagna|morcone|mordano|morengo|mores|moresco|moretta|morfasso|morgano|morgex|morgongiori|mori|moriago della battaglia|moricone|morigerati|morimondo|morino|moriondo torinese|morlupo|mormanno|mornago|mornese|mornico al serio|mornico losana|morolo|morozzo|morra de sanctis|morro d'alba|morro d'oro|morro reatino|morrone del sannio|morrovalle|morsano al tagliamento|morsasco|mortara|mortegliano|morterone|moruzzo|moscazzano|moschiano|mosciano sant'angelo|moscufo|moso in passiria|mossa|motta baluffi|motta camastra|motta d'affermo|motta de' conti|motta di livenza|motta montecorvino|motta san giovanni|motta sant'anastasia|motta santa lucia|motta visconti|mottafollone|mottalciata|motteggiana|mottola|mozzagrogna|mozzanica|mozzate|mozzecane|mozzo|muccia|muggia|muggiò|mugnano del cardinale|mugnano di napoli|mulazzano|mulazzo|mura|muravera|murazzano|murello|murialdo|murisengo monferrato|murlo|muro leccese|muro lucano|muros|muscoline|musei|musile di piave|musso|mussolente|mussomeli|muzzana del turgnano|muzzano|mölten|mühlbach|mühlwald|nago-torbole|nalles|nals|napoli|narbolia|narcao|nardodipace|nardò|narni|naro|narzole|nasino|naso|naturno|naturns|natz-schabs|nave|navelli|naz-sciaves|nazzano|ne|nebbiuno|negrar di valpolicella|neirone|neive|nembro|nemi|nemoli|neoneli|nepi|nereto|nerola|nervesa della battaglia|nerviano|nespolo|nesso|netro|nettuno|neumarkt|neviano|neviano degli arduini|neviglie|niardo|nibbiola|nibionno|nichelino|nicolosi|nicorvo|nicosia|nicotera|niederdorf|niella belbo|niella tanaro|nimis|niscemi|nissoria|nizza di sicilia|nizza monferrato|noale|noasca|nocara|nocciano|nocera inferiore|nocera superiore|nocera terinese|nocera umbra|noceto|noci|nociglia|noepoli|nogara|nogaredo|nogarole rocca|nogarole vicentino|noicattaro|nola|nole|noli|nomaglio|nomi|nonantola|none|nonio|noragugume|norbello|norcia|norma|nosate|notaresco|noto|nova levante|nova milanese|nova ponente|nova siri|novafeltria|novaledo|novalesa|novara|novara di sicilia|novate mezzola|novate milanese|nove|novedrate|novella|novellara|novello|noventa di piave|noventa padovana|noventa vicentina|novi di modena|novi ligure|novi velia|noviglio|novoli|nucetto|nughedu san nicolò|nughedu santa vittoria|nule|nulvi|numana|nuoro|nurachi|nuragus|nurallao|nuraminis|nureci|nurri|nus|nusco|nuvolento|nuvolera|nuxis|occhieppo inferiore|occhieppo superiore|occhiobello|occimiano|ocre|odalengo grande|odalengo piccolo|oderzo|odolo|ofena|offagna|offanengo|offida|offlaga|oggebbio|oggiona con santo stefano|oggiono|oglianico|ogliastro cilento|olang|olbia|olcenengo|oldenico|oleggio|oleggio castello|olevano di lomellina|olevano romano|olevano sul tusciano|olgiate comasco|olgiate molgora|olgiate olona|olginate|oliena|oliva gessi|olivadi|oliveri|oliveto citra|oliveto lario|oliveto lucano|olivetta san michele|olivola|ollastra|ollolai|ollomont|olmedo|olmeneta|olmo al brembo|olmo gentile|oltre il colle|oltressenda alta|oltrona di san mamette|olzai|ome|omegna|omignano|onano|onanì|oncino|oneta|onifai|oniferi|ono san pietro|onore|onzo|opera|opi|oppeano|oppido lucano|oppido mamertina|ora|orani|oratino|orbassano|orbetello|orciano pisano|orco feglino|ordona|orero|orgiano|orgosolo|oria|oricola|origgio|orino|orio al serio|orio canavese|orio litta|oriolo|oriolo romano|oristano|ormea|ormelle|ornago|ornavasso|ornica|orosei|orotelli|orria|orroli|orsago|orsara bormida|orsara di puglia|orsenigo|orsogna|orsomarso|orta di atella|orta nova|orta san giulio|ortacesus|orte|ortelle|ortezzano|ortignano raggiolo|ortisei|ortona|ortona dei marsi|ortovero|ortucchio|ortueri|orune|orvieto|orvinio|orzinuovi|orzivecchi|osasco|osasio|oschiri|osidda|osiglia|osilo|osimo|osini|osio sopra|osio sotto|osnago|osoppo|ospedaletti|ospedaletto|ospedaletto d'alpinolo|ospedaletto euganeo|ospedaletto lodigiano|ospitale di cadore|ospitaletto|ossago lodigiano|ossana|ossi|ossimo|ossona|ostana|ostellato|ostiano|ostiglia|ostra|ostra vetere|ostuni|otranto|otricoli|ottana|ottati|ottaviano|ottiglio|ottobiano|ottone|oulx|ovada|ovaro|oviglio|ovindoli|ovodda|oyace|ozegna|ozieri|ozzano dell'emilia|ozzano monferrato|ozzero|pabillonis|pace del mela|paceco|pacentro|pachino|paciano|padenghe sul garda|paderna|paderno d'adda|paderno dugnano|paderno franciacorta|paderno ponchielli|padova|padria|padru|padula|paduli|paesana|paese|pagani|paganico sabino|pagazzano|pagliara|paglieta|pagnacco|pagno|pagnona|pago del vallo di lauro|pago veiano|paisco loveno|paitone|paladina|palagano|palagianello|palagiano|palagonia|palaia|palanzano|palata|palau|palazzago|palazzo adriano|palazzo canavese|palazzo pignano|palazzo san gervasio|palazzolo acreide|palazzolo dello stella|palazzolo sull'oglio|palazzolo vercellese|palazzuolo sul senio|palena|palermiti|palermo|palestrina|palestro|paliano|palizzi|pallagorio|pallanzeno|pallare|palma campania|palma di montechiaro|palmanova|palmariggi|palmas arborea|palmi|palmiano|palmoli|palo del colle|palombara sabina|palombaro|palomonte|palosco|paludi|paluzza|palù|palù del fersina|pamparato|pancalieri|pancarana|panchià|pandino|panettieri|panicale|pannarano|panni|pantelleria|pantigliate|paola|paolisi|papasidero|papozze|parabiago|parabita|paratico|parcines|parella|parenti|parete|pareto|parghelia|parlasco|parma|parodi ligure|paroldo|parolise|parona|parrano|parre|partanna|partinico|partschins|paruzzaro|parzanica|pasian di prato|pasiano di pordenone|paspardo|passerano marmorito|passignano sul trasimeno|passirano|pastena|pastorano|pastrengo|pasturana|pasturo|paterno|paterno calabro|paternopoli|paternò|patrica|pattada|patti|patù|pau|paularo|pauli arbarei|paulilatino|paullo|paupisi|pavarolo|pavia|pavia di udine|pavone canavese|pavone del mella|pavullo nel frignano|pazzano|peccioli|pecetto di valenza|pecetto torinese|pedara|pedaso|pedavena|pedemonte|pederobba|pedesina|pedivigliano|pedrengo|peglio|pegognaga|peia|peio|pelago|pella|pellegrino parmense|pellezzano|pellizzano|pelugo|penango|penna in teverina|penna san giovanni|penna sant'andrea|pennabilli|pennadomo|pennapiedimonte|penne|pentone|perano|perarolo di cadore|perca|percha|percile|perdasdefogu|perdaxius|perdifumo|pereto|perfugas|pergine valsugana|pergola|perinaldo|perito|perledo|perletto|perlo|perloz|pernumia|pero|perosa argentina|perosa canavese|perrero|persico dosimo|pertengo|pertica alta|pertica bassa|pertosa|pertusio|perugia|pesaro|pescaglia|pescantina|pescara|pescarolo ed uniti|pescasseroli|pescate|pesche|peschici|peschiera borromeo|peschiera del garda|pescia|pescina|pesco sannita|pescocostanzo|pescolanciano|pescopagano|pescopennataro|pescorocchiano|pescosansonesco|pescosolido|pessano con bornago|pessina cremonese|pessinetto|petacciato|petilia policastro|petina|petralia soprana|petralia sottana|petrella salto|petrella tifernina|petriano|petriolo|petritoli|petrizzi|petronà|petrosino|petruro irpino|pettenasco|pettinengo|pettineo|pettoranello del molise|pettorano sul gizio|pettorazza grimani|peveragno|pezzana|pezzaze|pezzolo valle uzzone|pfalzen|pfatten|pfitsch|piacenza|piacenza d'adige|piadena drizzona|piaggine|pian camuno|piana crixia|piana degli albanesi|piana di monte verna|piancastagnaio|piancogno|piandimeleto|piane crati|pianella|pianello del lario|pianello val tidone|pianengo|pianezza|pianezze|pianfei|pianico|pianiga|piano di sorrento|pianopoli|pianoro|piansano|piantedo|piario|piasco|piateda|piatto|piazza al serchio|piazza armerina|piazza brembana|piazzatorre|piazzola sul brenta|piazzolo|picciano|picerno|picinisco|pico|piea|piedicavallo|piedimonte etneo|piedimonte matese|piedimonte san germano|piedimulera|piegaro|pienza|pieranica|pietra de' giorgi|pietra ligure|pietra marazzi|pietrabbondante|pietrabruna|pietracamela|pietracatella|pietracupa|pietradefusi|pietraferrazzana|pietrafitta|pietragalla|pietralunga|pietramelara|pietramontecorvino|pietranico|pietrapaola|pietrapertosa|pietraperzia|pietraporzio|pietraroja|pietrarubbia|pietrasanta|pietrastornina|pietravairano|pietrelcina|pieve a nievole|pieve albignola|pieve d'olmi|pieve del cairo|pieve del grappa|pieve di bono-prezzo|pieve di cadore|pieve di cento|pieve di soligo|pieve di teco|pieve emanuele|pieve fissiraga|pieve fosciana|pieve ligure|pieve porto morone|pieve san giacomo|pieve santo stefano|pieve tesino|pieve torina|pieve vergonte|pievepelago|piglio|pigna|pignataro interamna|pignataro maggiore|pignola|pignone|pigra|pila|pimentel|pimonte|pinarolo po|pinasca|pincara|pinerolo|pineto|pino d'asti|pino torinese|pinzano al tagliamento|pinzolo|piobbico|piobesi d'alba|piobesi torinese|piode|pioltello|piombino|piombino dese|pioraco|piossasco|piove di sacco|piovene rocchette|piovà massaia|piozzano|piozzo|piraino|pisa|pisano|piscina|piscinas|pisciotta|pisogne|pisoniano|pisticci|pistoia|pitigliano|piubega|piuro|piverone|pizzale|pizzighettone|pizzo|pizzoferrato|pizzoli|pizzone|pizzoni|placanica|plataci|platania|platì|plaus|plesio|ploaghe|plodio|pocapaglia|pocenia|podenzana|podenzano|pofi|poggiardo|poggibonsi|poggio a caiano|poggio bustone|poggio catino|poggio imperiale|poggio mirteto|poggio moiano|poggio nativo|poggio picenze|poggio renatico|poggio rusco|poggio san lorenzo|poggio san marcello|poggio san vicino|poggio sannita|poggio torriana|poggiodomo|poggiofiorito|poggiomarino|poggioreale|poggiorsini|poggiridenti|pogliano milanese|pognana lario|pognano|pogno|poirino|pojana maggiore|polaveno|polcenigo|polesella|polesine zibello|poli|polia|policoro|polignano a mare|polinago|polino|polistena|polizzi generosa|polla|pollein|pollena trocchia|pollenza|pollica|pollina|pollone|pollutri|polonghera|polpenazze del garda|polverara|polverigi|pomarance|pomaretto|pomarico|pomaro monferrato|pomarolo|pombia|pomezia|pomigliano d'arco|pompei|pompeiana|pompiano|pomponesco|pompu|poncarale|ponderano|ponna|ponsacco|ponso|pont canavese|pont-saint-martin|pontassieve|pontboset|ponte|ponte buggianese|ponte dell'olio|ponte di legno|ponte di piave|ponte gardena|ponte in valtellina|ponte lambro|ponte nelle alpi|ponte nizza|ponte nossa|ponte san nicolò|ponte san pietro|pontebba|pontecagnano faiano|pontecchio polesine|pontechianale|pontecorvo|pontecurone|pontedassio|pontedera|pontelandolfo|pontelatone|pontelongo|pontenure|ponteranica|pontestura|pontevico|pontey|ponti|ponti sul mincio|pontida|pontinia|pontinvrea|pontirolo nuovo|pontoglio|pontremoli|ponza|ponzano di fermo|ponzano monferrato|ponzano romano|ponzano veneto|ponzone|popoli terme|poppi|porano|porcari|porcia|pordenone|porlezza|pornassio|porpetto|portacomaro|portalbera|porte|porte di rendena|portici|portico di caserta|portico e san benedetto|portigliola|porto azzurro|porto ceresio|porto cesareo|porto empedocle|porto mantovano|porto recanati|porto san giorgio|porto sant'elpidio|porto tolle|porto torres|porto valtravaglia|porto viro|portobuffolè|portocannone|portoferraio|portofino|portogruaro|portomaggiore|portopalo di capo passero|portoscuso|portovenere|portula|posada|posina|positano|possagno|posta|posta fibreno|postal|postalesio|postiglione|postua|potenza|potenza picena|pove del grappa|povegliano|povegliano veronese|poviglio|povoletto|pozzaglia sabina|pozzaglio ed uniti|pozzallo|pozzilli|pozzo d'adda|pozzol groppo|pozzolengo|pozzoleone|pozzolo formigaro|pozzomaggiore|pozzonovo|pozzuoli|pozzuolo del friuli|pozzuolo martesana|prad am stilfserjoch|pradalunga|pradamano|pradleves|pragelato|prags|praia a mare|praiano|pralboino|prali|pralormo|pralungo|pramaggiore|pramollo|prarolo|prarostino|prasco|prascorsano|prata camportaccio|prata d'ansidonia|prata di pordenone|prata di principato ultra|prata sannita|pratella|pratiglione|prato|prato allo stelvio|prato carnico|prato sesia|pratola peligna|pratola serra|pratovecchio stia|pravisdomini|pray|prazzo|precenicco|preci|predaia|predappio|predazzo|predoi|predore|predosa|preganziol|pregnana milanese|prelà|premana|premariacco|premeno|premia|premilcuore|premolo|premosello-chiovenda|preone|prepotto|preseglie|presenzano|presezzo|presicce-acquarica|pressana|pretoro|prettau|prevalle|prezza|priero|prignano cilento|prignano sulla secchia|primaluna|primiero san martino di castrozza|priocca|priola|priolo gargallo|priverno|prizzi|proceno|procida|propata|proserpio|prossedi|provaglio d'iseo|provaglio val sabbia|proveis|proves|provvidenti|prunetto|pré-saint-didier|puegnago del garda|puglianello|pula|pulfero|pulsano|pumenengo|pusiano|putifigari|putignano|quadrelle|quadri|quagliuzzo|qualiano|quaranti|quaregna cerreto|quargnento|quarna sopra|quarna sotto|quarona|quarrata|quart|quarto|quarto d'altino|quartu sant'elena|quartucciu|quassolo|quattordio|quattro castella|quiliano|quincinetto|quindici|quingentole|quintano|quinto di treviso|quinto vercellese|quinto vicentino|quinzano d'oglio|quistello|rabbi|racale|racalmuto|racconigi|raccuja|racines|radda in chianti|raddusa|radicofani|radicondoli|raffadali|ragalna|ragogna|ragusa|raiano|ramacca|rancio valcuvia|ranco|randazzo|ranica|ranzanico|ranzo|rapagnano|rapallo|rapino|rapolano terme|rapolla|rapone|rasen-antholz|rassa|rasun-anterselva|rasura|ratschings|ravanusa|ravarino|ravascletto|ravello|ravenna|raveo|raviscanina|re|rea|realmonte|reana del rojale|reano|recale|recanati|recco|recetto|recoaro terme|redavalle|redondesco|refrancore|refrontolo|regalbuto|reggello|reggio di calabria|reggio nell'emilia|reggiolo|reino|reitano|remanzacco|remedello|renate|rende|renon|repentabor|resana|rescaldina|resia|resiutta|resuttano|retorbido|revello|revigliasco d'asti|revine lago|rezzago|rezzato|rezzo|rezzoaglio|rho|rhêmes-notre-dame|rhêmes-saint-georges|riace|rialto|riano|riardo|ribera|ribordone|ricadi|ricaldone|riccia|riccione|riccò del golfo di spezia|ricengo|ricigliano|riese pio x|riesi|rieti|riffian|rifiano|rifreddo|rignano flaminio|rignano garganico|rignano sull'arno|rigolato|rimella|rimini|rio|rio di pusteria|rio saliceto|riofreddo|riola sardo|riolo terme|riolunato|riomaggiore|rionero in vulture|rionero sannitico|ripa teatina|ripabottoni|ripacandida|ripalimosani|ripalta arpina|ripalta cremasca|ripalta guerina|riparbella|ripatransone|ripe san ginesio|ripi|riposto|rittana|ritten|riva del garda|riva del po|riva di solto|riva ligure|riva presso chieri|rivalba|rivalta bormida|rivalta di torino|rivamonte agordino|rivanazzano terme|rivara|rivarolo canavese|rivarolo del re ed uniti|rivarolo mantovano|rivarone|rivarossa|rive|rive d'arcano|rivello|rivergaro|rivignano teor|rivisondoli|rivodutri|rivoli|rivoli veronese|rivolta d'adda|rizziconi|roana|roaschia|roascio|roasio|roatto|robassomero|robbiate|robbio|robecchetto con induno|robecco d'oglio|robecco pavese|robecco sul naviglio|robella|robilante|roburent|rocca canavese|rocca canterano|rocca cigliè|rocca d'arazzo|rocca d'arce|rocca d'evandro|rocca de' baldi|rocca de' giorgi|rocca di botte|rocca di cambio|rocca di cave|rocca di mezzo|rocca di neto|rocca di papa|rocca grimalda|rocca imperiale|rocca massima|rocca pia|rocca pietore|rocca priora|rocca san casciano|rocca san felice|rocca san giovanni|rocca santa maria|rocca santo stefano|rocca sinibalda|rocca susella|roccabascerana|roccabernarda|roccabianca|roccabruna|roccacasale|roccadaspide|roccafiorita|roccafluvione|roccaforte del greco|roccaforte ligure|roccaforte mondovì|roccaforzata|roccafranca|roccagiovine|roccagloriosa|roccagorga|roccalbegna|roccalumera|roccamandolfi|roccamena|roccamonfina|roccamontepiano|roccamorice|roccanova|roccantica|roccapalumba|roccapiemonte|roccarainola|roccaraso|roccaromana|roccascalegna|roccasecca|roccasecca dei volsci|roccasicura|roccasparvera|roccaspinalveti|roccastrada|roccavaldina|roccaverano|roccavignale|roccavione|roccavivara|roccella ionica|roccella valdemone|rocchetta a volturno|rocchetta belbo|rocchetta di vara|rocchetta e croce|rocchetta ligure|rocchetta nervina|rocchetta palafea|rocchetta sant'antonio|rocchetta tanaro|rodano|roddi|roddino|rodello|rodeneck|rodengo|rodengo saiano|rodero|rodi garganico|rodigo|rodì milici|rofrano|rogeno|roggiano gravina|roghudi|rogliano|rognano|rogno|rogolo|roiate|roio del sangro|roisan|roletto|rolo|roma|romagnano al monte|romagnano sesia|romagnese|romana|romanengo|romano canavese|romano d'ezzelino|romano di lombardia|romans d'isonzo|rombiolo|romeno|romentino|rometta|roncade|roncadelle|roncaro|roncegno terme|roncello|ronchi dei legionari|ronchi valsugana|ronchis|ronciglione|ronco all'adige|ronco biellese|ronco briantino|ronco canavese|ronco scrivia|roncobello|roncoferraro|roncofreddo|roncola|roncà|rondanina|rondissone|ronsecco|ronzo-chienis|ronzone|roppolo|rorà|rosarno|rosasco|rosate|rosazza|rosciano|roscigno|rose|rosello|roseto capo spulico|roseto degli abruzzi|roseto valfortore|rosignano marittimo|rosignano monferrato|rosolina|rosolini|rosora|rossa|rossana|rossano veneto|rossiglione|rosta|rosà|rota d'imagna|rota greca|rotella|rotello|rotonda|rotondella|rotondi|rottofreno|rotzo|roure|rovasenda|rovato|rovegno|rovellasca|rovello porro|roverbella|roverchiara|roveredo di guà|roveredo in piano|rovereto|roverè della luna|roverè veronese|rovescala|rovetta|roviano|rovigo|rovito|rovolon|rozzano|roè volciano|rubano|rubiana|rubiera|ruda|rudiano|rueglio|ruffano|ruffia|ruffrè-mendola|rufina|ruinas|rumo|ruoti|russi|rutigliano|rutino|ruviano|ruvo del monte|ruvo di puglia|sabaudia|sabbio chiese|sabbioneta|sacco|saccolongo|sacile|sacrofano|sadali|sagama|sagliano micca|sagrado|sagron mis|saint-christophe|saint-denis|saint-marcel|saint-nicolas|saint-oyen|saint-pierre|saint-rhémy-en-bosses|saint-vincent|sala baganza|sala biellese|sala bolognese|sala comacina|sala consilina|sala monferrato|salandra|salaparuta|salara|salasco|salassa|salbertrand|salcedo|salcito|sale|sale delle langhe|sale marasino|sale san giovanni|salemi|salento|salerano canavese|salerano sul lambro|salerno|salgareda|sali vercellese|salice salentino|saliceto|salisano|salizzole|salle|salmour|salorno sulla strada del vino|salsomaggiore terme|saltrio|saludecio|saluggia|salurn an der weinstraße|salussola|saluzzo|salve|salvirola|salvitelle|salza di pinerolo|salza irpina|salzano|salò|samarate|samassi|samatzai|sambuca di sicilia|sambuca pistoiese|sambuci|sambuco|sammichele di bari|samo|samolaco|samone|sampeyre|samugheo|san bartolomeo al mare|san bartolomeo in galdo|san bartolomeo val cavargna|san basile|san basilio|san bassano|san bellino|san benedetto belbo|san benedetto dei marsi|san benedetto del tronto|san benedetto in perillis|san benedetto po|san benedetto ullano|san benedetto val di sambro|san benigno canavese|san bernardino verbano|san biagio della cima|san biagio di callalta|san biagio platani|san biagio saracinisco|san biase|san bonifacio|san buono|san calogero|san candido|san canzian d'isonzo|san carlo canavese|san casciano dei bagni|san casciano in val di pesa|san cassiano|san cataldo|san cesareo|san cesario di lecce|san cesario sul panaro|san chirico nuovo|san chirico raparo|san cipirello|san cipriano d'aversa|san cipriano picentino|san cipriano po|san clemente|san colombano al lambro|san colombano belmonte|san colombano certenoli|san cono|san cosmo albanese|san costantino albanese|san costantino calabro|san costanzo|san cristoforo|san damiano al colle|san damiano d'asti|san damiano macra|san daniele del friuli|san daniele po|san demetrio corone|san demetrio ne' vestini|san didero|san donaci|san donato di lecce|san donato di ninea|san donato milanese|san donato val di comino|san donà di piave|san dorligo della valle|san fele|san felice a cancello|san felice circeo|san felice del benaco|san felice del molise|san felice sul panaro|san ferdinando|san ferdinando di puglia|san fermo della battaglia|san fili|san filippo del mela|san fior|san fiorano|san floriano del collio|san floro|san francesco al campo|san fratello|san gavino monreale|san gemini|san genesio atesino|san genesio ed uniti|san gennaro vesuviano|san germano chisone|san germano vercellese|san gervasio bresciano|san giacomo degli schiavoni|san giacomo delle segnate|san giacomo filippo|san giacomo vercellese|san gillio|san gimignano|san ginesio|san giorgio a cremano|san giorgio a liri|san giorgio albanese|san giorgio bigarello|san giorgio canavese|san giorgio del sannio|san giorgio della richinvelda|san giorgio delle pertiche|san giorgio di lomellina|san giorgio di nogaro|san giorgio di piano|san giorgio in bosco|san giorgio ionico|san giorgio la molara|san giorgio lucano|san giorgio monferrato|san giorgio morgeto|san giorgio piacentino|san giorgio scarampi|san giorgio su legnano|san giorio di susa|san giovanni a piro|san giovanni al natisone|san giovanni bianco|san giovanni del dosso|san giovanni di fassa|san giovanni di gerace|san giovanni gemini|san giovanni ilarione|san giovanni in croce|san giovanni in fiore|san giovanni in galdo|san giovanni in marignano|san giovanni in persiceto|san giovanni incarico|san giovanni la punta|san giovanni lipioni|san giovanni lupatoto|san giovanni rotondo|san giovanni suergiu|san giovanni teatino|san giovanni valdarno|san giuliano del sannio|san giuliano di puglia|san giuliano milanese|san giuliano terme|san giuseppe jato|san giuseppe vesuviano|san giustino|san giusto canavese|san godenzo|san gregorio d'ippona|san gregorio da sassola|san gregorio di catania|san gregorio magno|san gregorio matese|san gregorio nelle alpi|san lazzaro di savena|san leo|san leonardo|san leonardo in passiria|san leucio del sannio|san lorenzello|san lorenzo|san lorenzo al mare|san lorenzo bellizzi|san lorenzo del vallo|san lorenzo di sebato|san lorenzo dorsino|san lorenzo in campo|san lorenzo isontino|san lorenzo maggiore|san lorenzo nuovo|san luca|san lucido|san lupo|san mango d'aquino|san mango piemonte|san mango sul calore|san marcellino|san marcello|san marcello piteglio|san marco argentano|san marco d'alunzio|san marco dei cavoti|san marco evangelista|san marco in lamis|san marco la catola|san martino al tagliamento|san martino alfieri|san martino buon albergo|san martino canavese|san martino d'agri|san martino dall'argine|san martino del lago|san martino di finita|san martino di lupari|san martino di venezze|san martino in badia|san martino in passiria|san martino in pensilis|san martino in rio|san martino in strada|san martino sannita|san martino siccomario|san martino sulla marrucina|san martino valle caudina|san marzano di san giuseppe|san marzano oliveto|san marzano sul sarno|san massimo|san maurizio canavese|san maurizio d'opaglio|san mauro castelverde|san mauro cilento|san mauro di saline|san mauro forte|san mauro la bruca|san mauro marchesato|san mauro pascoli|san mauro torinese|san michele al tagliamento|san michele all'adige|san michele di ganzaria|san michele di serino|san michele mondovì|san michele salentino|san miniato|san nazzaro|san nazzaro sesia|san nazzaro val cavargna|san nicandro garganico|san nicola arcella|san nicola baronia|san nicola da crissa|san nicola dell'alto|san nicola la strada|san nicola manfredi|san nicolò d'arcidano|san nicolò di comelico|san nicolò gerrei|san pancrazio|san pancrazio salentino|san paolo|san paolo albanese|san paolo bel sito|san paolo d'argon|san paolo di civitate|san paolo di jesi|san paolo solbrito|san pellegrino terme|san pier d'isonzo|san pier niceto|san piero patti|san pietro a maida|san pietro al natisone|san pietro al tanagro|san pietro apostolo|san pietro avellana|san pietro clarenza|san pietro di cadore|san pietro di caridà|san pietro di feletto|san pietro di morubio|san pietro in amantea|san pietro in cariano|san pietro in casale|san pietro in cerro|san pietro in gu|san pietro in guarano|san pietro in lama|san pietro infine|san pietro mosezzo|san pietro mussolino|san pietro val lemina|san pietro vernotico|san pietro viminario|san pio delle camere|san polo d'enza|san polo dei cavalieri|san polo di piave|san polo matese|san ponso|san possidonio|san potito sannitico|san potito ultra|san prisco|san procopio|san prospero|san quirico d'orcia|san quirino|san raffaele cimena|san roberto|san rocco al porto|san romano in garfagnana|san rufo|san salvatore di fitalia|san salvatore monferrato|san salvatore telesino|san salvo|san sebastiano al vesuvio|san sebastiano curone|san sebastiano da po|san secondo di pinerolo|san secondo parmense|san severino lucano|san severino marche|san severo|san siro|san sossio baronia|san sostene|san sosti|san sperate|san stino di livenza|san tammaro|san teodoro|san tomaso agordino|san valentino in abruzzo citeriore|san valentino torio|san venanzo|san vendemiano|san vero milis|san vincenzo|san vincenzo la costa|san vincenzo valle roveto|san vitaliano|san vito|san vito al tagliamento|san vito al torre|san vito chietino|san vito dei normanni|san vito di cadore|san vito di fagagna|san vito di leguzzano|san vito lo capo|san vito romano|san vito sullo ionio|san vittore del lazio|san vittore olona|san zeno di montagna|san zeno naviglio|san zenone al lambro|san zenone al po|san zenone degli ezzelini|sanarica|sand in taufers|sandigliano|sandrigo|sanfront|sanfrè|sangano|sangiano|sangineto|sanguinetto|sanluri|sannazzaro de' burgondi|sannicandro di bari|sannicola|sanremo|sansepolcro|sant'agapito|sant'agata bolognese|sant'agata de' goti|sant'agata del bianco|sant'agata di esaro|sant'agata di militello|sant'agata di puglia|sant'agata feltria|sant'agata fossili|sant'agata li battiati|sant'agata sul santerno|sant'agnello|sant'albano stura|sant'alessio con vialone|sant'alessio in aspromonte|sant'alessio siculo|sant'alfio|sant'ambrogio di torino|sant'ambrogio di valpolicella|sant'ambrogio sul garigliano|sant'anastasia|sant'anatolia di narco|sant'andrea apostolo dello ionio|sant'andrea del garigliano|sant'andrea di conza|sant'andrea frius|sant'angelo a cupolo|sant'angelo a fasanella|sant'angelo a scala|sant'angelo all'esca|sant'angelo d'alife|sant'angelo dei lombardi|sant'angelo del pesco|sant'angelo di brolo|sant'angelo di piove di sacco|sant'angelo in pontano|sant'angelo in vado|sant'angelo le fratte|sant'angelo limosano|sant'angelo lodigiano|sant'angelo lomellina|sant'angelo muxaro|sant'angelo romano|sant'anna arresi|sant'anna d'alfaedo|sant'antimo|sant'antioco|sant'antonino di susa|sant'antonio abate|sant'antonio di gallura|sant'apollinare|sant'arcangelo|sant'arcangelo trimonte|sant'arpino|sant'arsenio|sant'egidio alla vibrata|sant'egidio del monte albino|sant'elena|sant'elena sannita|sant'elia a pianisi|sant'elia fiumerapido|sant'elpidio a mare|sant'eufemia a maiella|sant'eufemia d'aspromonte|sant'eusanio del sangro|sant'eusanio forconese|sant'ilario d'enza|sant'ilario dello ionio|sant'ippolito|sant'olcese|sant'omero|sant'omobono terme|sant'onofrio|sant'oreste|sant'orsola terme|sant'urbano|santa brigida|santa caterina albanese|santa caterina d'este|santa caterina dello ionio|santa caterina villarmosa|santa cesarea terme|santa cristina d'aspromonte|santa cristina e bissone|santa cristina gela|santa cristina valgardena|santa croce camerina|santa croce del sannio|santa croce di magliano|santa croce sull'arno|santa domenica talao|santa domenica vittoria|santa elisabetta|santa fiora|santa flavia|santa giuletta|santa giusta|santa giustina|santa giustina in colle|santa luce|santa lucia del mela|santa lucia di piave|santa lucia di serino|santa margherita di belice|santa margherita di staffora|santa margherita ligure|santa maria a monte|santa maria a vico|santa maria capua vetere|santa maria coghinas|santa maria del cedro|santa maria del molise|santa maria della versa|santa maria di licodia|santa maria di sala|santa maria hoè|santa maria imbaro|santa maria la carità|santa maria la fossa|santa maria la longa|santa maria maggiore|santa maria nuova|santa marina|santa marina salina|santa marinella|santa ninfa|santa paolina|santa severina|santa sofia|santa sofia d'epiro|santa teresa di riva|santa teresa gallura|santa venerina|santa vittoria d'alba|santa vittoria in matenano|santadi|santarcangelo di romagna|sante marie|santena|santeramo in colle|santhià|santi cosma e damiano|santo stefano al mare|santo stefano belbo|santo stefano d'aveto|santo stefano del sole|santo stefano di cadore|santo stefano di camastra|santo stefano di magra|santo stefano di rogliano|santo stefano di sessanio|santo stefano in aspromonte|santo stefano lodigiano|santo stefano quisquina|santo stefano roero|santo stefano ticino|santomenna|santopadre|santorso|santu lussurgiu|sanza|sanzeno|saonara|saponara|sappada|sapri|saracena|saracinesco|sarcedo|sarconi|sardara|sardigliano|sarego|sarentino|sarezzano|sarezzo|sarmato|sarmede|sarnano|sarnico|sarno|sarnonico|sarntal|saronno|sarre|sarroch|sarsina|sarteano|sartirana lomellina|sarule|sarzana|sassano|sassari|sassello|sassetta|sassinoro|sasso di castalda|sasso marconi|sassocorvaro auditore|sassofeltrio|sassoferrato|sassuolo|satriano|satriano di lucania|sauris|sauze d'oulx|sauze di cesana|sava|savelli|saviano|savigliano|savignano irpino|savignano sul panaro|savignano sul rubicone|savignone|saviore dell'adamello|savoca|savogna|savogna d'isonzo|savoia di lucania|savona|scafa|scafati|scagnello|scala|scala coeli|scaldasole|scalea|scalenghe|scaletta zanclea|scampitella|scandale|scandiano|scandicci|scandolara ravara|scandolara ripa d'oglio|scandriglia|scanno|scano di montiferro|scansano|scanzano jonico|scanzorosciate|scapoli|scarlino|scarmagno|scarnafigi|scarperia e san piero|scena|scerni|scheggia e pascelupo|scheggino|schenna|schiavi di abruzzo|schiavon|schignano|schilpario|schio|schivenoglia|schlanders|schluderns|schnals|sciacca|sciara|scicli|scido|scigliano|scilla|scillato|sciolze|scisciano|sclafani bagni|scontrone|scopa|scopello|scoppito|scordia|scorrano|scorzè|scurcola marsicana|scurelle|scurzolengo|seborga|secinaro|seclì|secugnago|sedegliano|sedico|sedilo|sedini|sedriano|sedrina|sefro|segariu|seggiano|segni|segonzano|segrate|segusino|selargius|selci|selegas|sella giudicarie|sellano|sellero|sellia|sellia marina|selva dei molini|selva di cadore|selva di progno|selva di val gardena|selvazzano dentro|selvino|semestene|semiana|seminara|semproniano|senago|senale-san felice|senales|seneghe|senerchia|seniga|senigallia|senis|senise|senna comasco|senna lodigiana|sennariolo|sennori|senorbì|sepino|sequals|seravezza|serdiana|seregno|seren del grappa|sergnano|seriate|serina|serino|serle|sermide e felonica|sermoneta|sernaglia della battaglia|sernio|serole|serra d'aiello|serra de' conti|serra riccò|serra san bruno|serra san quirico|serra sant'abbondio|serracapriola|serradifalco|serralunga d'alba|serralunga di crea|serramanna|serramazzoni|serramezzana|serramonacesca|serrapetrona|serrara fontana|serrastretta|serrata|serravalle a po|serravalle di chienti|serravalle langhe|serravalle pistoiese|serravalle scrivia|serravalle sesia|serre|serrenti|serri|serrone|sersale|servigliano|sessa aurunca|sessa cilento|sessame|sessano del molise|sesta godano|sestino|sesto|sesto al reghena|sesto calende|sesto campano|sesto ed uniti|sesto fiorentino|sesto san giovanni|sestola|sestri levante|sestriere|sestu|settala|settefrati|setteville|settime|settimo milanese|settimo rottaro|settimo san pietro|settimo torinese|settimo vittone|settingiano|setzu|seui|seulo|seveso|sexten|sezzadio|sezze|sfruz|sgonico|sgurgola|siamaggiore|siamanna|siano|siapiccia|sicignano degli alburni|siculiana|siddi|siderno|siena|sigillo|signa|silandro|silanus|silea|siligo|siliqua|silius|sillano giuncugnano|sillavengo|silvano d'orba|silvano pietra|silvi|simala|simaxis|simbario|simeri crichi|sinagra|sinalunga|sindia|sini|sinio|siniscola|sinnai|sinopoli|siracusa|sirignano|siris|sirmione|sirolo|sirone|sirtori|sissa trecasali|siurgus donigala|siziano|sizzano|sluderno|smerillo|soave|socchieve|soddì|sogliano al rubicone|sogliano cavour|soglio|soiano del lago|solagna|solarino|solaro|solarolo|solarolo rainerio|solarussa|solbiate arno|solbiate con cagno|solbiate olona|soldano|soleminis|solero|solesino|soleto|solferino|soliera|solignano|solofra|solonghello|solopaca|solto collina|solza|somaglia|somano|somma lombardo|somma vesuviana|sommacampagna|sommariva del bosco|sommariva perno|sommatino|sommo|sona|soncino|sondalo|sondrio|songavazzo|sonico|sonnino|sora|soraga di fassa|soragna|sorano|sorbo san basile|sorbo serpico|sorbolo mezzani|sordevolo|sordio|soresina|sorgono|sorgà|sori|sorianello|soriano calabro|soriano nel cimino|sorico|soriso|sorisole|sormano|sorradile|sorrento|sorso|sortino|sospiro|sospirolo|sossano|sostegno|sotto il monte giovanni xxiii|sover|soverato|sovere|soveria mannelli|soveria simeri|soverzene|sovicille|sovico|sovizzo|sovodnje ob soči|sovramonte|sozzago|spadafora|spadola|sparanise|sparone|specchia|spello|sperlinga|sperlonga|sperone|spessa|spezzano albanese|spezzano della sila|spiazzo|spigno monferrato|spigno saturnia|spilamberto|spilimbergo|spilinga|spinadesco|spinazzola|spinea|spineda|spinete|spineto scrivia|spinetoli|spino d'adda|spinone al lago|spinoso|spirano|spoleto|spoltore|spongano|spormaggiore|sporminore|spotorno|spresiano|spriana|squillace|squinzano|st. christina in gröden|st. leonhard in passeier|st. lorenzen|st. martin in passeier|st. martin in thurn|st. pankraz|st. ulrich|staffolo|stagno lombardo|staiti|stalettì|stanghella|staranzano|statte|stazzano|stazzema|stazzona|stefanaconi|stella|stella cilento|stellanello|stelvio|stenico|sternatia|sterzing|stezzano|stienta|stigliano|stignano|stilfs|stilo|stimigliano|stintino|stio|stornara|stornarella|storo|stra|stradella|strambinello|strambino|strangolagalli|stregna|strembo|stresa|strevi|striano|strona|stroncone|strongoli|stroppiana|stroppo|strozza|sturno|suardi|subbiano|subiaco|succivo|sueglio|suelli|suello|suisio|sulbiate|sulmona|sulzano|sumirago|summonte|suni|suno|supersano|supino|surano|surbo|susa|susegana|sustinente|sutera|sutri|sutrio|suvereto|suzzara|sèn jan|taceno|tadasuni|taggia|tagliacozzo|taglio di po|tagliolo monferrato|taibon agordino|taino|taipana|talamello|talamona|talana|taleggio|talla|talmassons|tambre|taormina|tarano|taranta peligna|tarantasca|taranto|tarcento|tarquinia|tarsia|tartano|tarvisio|tarzo|tassarolo|taufers im münstertal|taurano|taurasi|taurianova|taurisano|tavagnacco|tavagnasco|tavazzano con villavesco|tavenna|taverna|tavernerio|tavernola bergamasca|tavernole sul mella|taviano|tavigliano|tavoleto|tavullia|teana|teano|teggiano|teglio|teglio veneto|telese terme|telgate|telti|telve|telve di sopra|tempio pausania|temù|tenna|tenno|teolo|teora|teramo|terdobbiate|terelle|terenten|terento|terenzo|tergu|terlan|terlano|terlizzi|terme vigliatore|termeno sulla strada del vino|termini imerese|termoli|ternate|ternengo|terni|terno d'isola|terracina|terragnolo|terralba|terranova da sibari|terranova dei passerini|terranova di pollino|terranova sappo minulio|terranuova bracciolini|terrasini|terrassa padovana|terravecchia|terrazzo|terre d'adige|terre del reno|terre roveresche|terricciola|terruggia|tertenia|terzigno|terzo|terzo d'aquileia|terzolas|terzorio|tesero|tesimo|tessennano|testico|teti|teulada|teverola|tezze sul brenta|thiene|thiesi|tiana|ticengo|ticineto|tiers|tiggiano|tiglieto|tigliole|tignale|tinnura|tione degli abruzzi|tione di trento|tirano|tires|tiriolo|tirol|tirolo|tisens|tissi|tito|tivoli|tizzano val parma|toano|toblach|tocco caudio|tocco da casauria|toceno|todi|toffia|toirano|tolentino|tolfa|tollegno|tollo|tolmezzo|tolve|tombolo|ton|tonara|tonco|tonezza del cimone|tora e piccilli|torano castello|torano nuovo|torbole casaglia|torcegno|torchiara|torchiarolo|torella dei lombardi|torella del sannio|torgiano|torgnon|torino|torino di sangro|toritto|torlino vimercati|tornaco|tornareccio|tornata|tornimparte|torno|tornolo|toro|torpè|torraca|torralba|torrazza coste|torrazza piemonte|torrazzo|torre annunziata|torre beretti e castellaro|torre boldone|torre bormida|torre cajetani|torre canavese|torre d'arese|torre d'isola|torre de' busi|torre de' negri|torre de' passeri|torre de' picenardi|torre de' roveri|torre del greco|torre di mosto|torre di ruggiero|torre di santa maria|torre le nocelle|torre mondovì|torre orsaia|torre pallavicina|torre pellice|torre san giorgio|torre san patrizio|torre santa susanna|torreano|torrebelvicino|torrebruna|torrecuso|torreglia|torregrotta|torremaggiore|torrenova|torresina|torretta|torrevecchia pia|torrevecchia teatina|torri del benaco|torri di quartesolo|torri in sabina|torrice|torricella|torricella del pizzo|torricella in sabina|torricella peligna|torricella sicura|torricella verzate|torriglia|torrile|torrioni|torrita di siena|torrita tiberina|tortolì|tortona|tortora|tortorella|tortoreto|tortorici|torviscosa|toscolano-maderno|tossicia|tovo di sant'agata|tovo san giacomo|trabia|tradate|tramatza|trambileno|tramin an der weinstraße|tramonti|tramonti di sopra|tramonti di sotto|tramutola|trana|trani|traona|trapani|trappeto|trarego viggiona|trasacco|trasaghis|trasquera|tratalias|travacò siccomario|travagliato|travedona-monate|traversella|traversetolo|traves|travesio|travo|tre ville|trebaseleghe|trebisacce|trecase|trecastagni|trecastelli|trecate|trecchina|trecenta|tredozio|treglio|tregnago|treia|treiso|tremestieri etneo|tremezzina|tremosine sul garda|trentinara|trento|trentola ducenta|trenzano|treppo grande|treppo ligosullo|trepuzzi|trequanda|tresana|trescore balneario|trescore cremasco|tresignana|tresivio|tresnuraghes|trevenzuolo|trevi|trevi nel lazio|trevico|treviglio|trevignano|trevignano romano|treville|treviolo|treviso|treviso bresciano|trezzano rosa|trezzano sul naviglio|trezzo sull'adda|trezzo tinella|trezzone|tribano|tribiano|tribogna|tricarico|tricase|tricerro|tricesimo|triei|trieste|triggiano|trigolo|trinitapoli|trinità|trinità d'agultu e vignola|trino|triora|tripi - abakainon|trisobbio|trissino|triuggio|trivento|trivigliano|trivignano udinese|trivigno|trivolzio|trodena nel parco naturale|trofarello|troia|troina|tromello|trontano|tronzano lago maggiore|tronzano vercellese|tropea|trovo|truccazzano|truden im naturpark|tscherms|tubre|tufara|tufillo|tufino|tufo|tuglie|tuili|tula|tuoro sul trasimeno|turania|turano lodigiano|turate|turbigo|turi|turri|turriaco|turrivalignani|tursi|tusa|tuscania|ubiale clanezzo|uboldo|ucria|udine|ugento|uggiano la chiesa|uggiate con ronago|ulassai|ulten|ultimo|ulà tirso|umbertide|umbriatico|unsere liebe frau im walde-st. felix|urago d'oglio|uras|urbana|urbania|urbe|urbino|urbisaglia|urgnano|uri|ururi|urzulei|uscio|usellus|usini|usmate velate|ussana|ussaramanna|ussassai|usseaux|usseglio|ussita|ustica|uta|uzzano|vaccarizzo albanese|vacone|vacri|vadena|vado ligure|vagli sotto|vaglia|vaglio basilicata|vaglio serra|vahrn|vaiano|vaiano cremasco|vaie|vailate|vairano patenora|vajont|val brembilla|val della torre|val di chy|val di nizza|val di vizze|val di zoldo|val liona|val masino|val rezzo|valbondione|valbrembo|valbrenta|valbrevenna|valbrona|valchiusa|valdagno|valdaone|valdaora|valdastico|valdengo|valderice|valdidentro|valdieri|valdilana|valdina|valdisotto|valdobbiadene|valduggia|valeggio|valeggio sul mincio|valentano|valenza|valenzano|valera fratta|valfabbrica|valfenera|valfloriana|valfornace|valfurva|valganna|valgioie|valgoglio|valgrana|valgreghentino|valgrisenche|valguarnera caropepe|vallada agordina|vallanzengo|vallarsa|vallata|valle agricola|valle aurina|valle cannobina|valle castellana|valle dell'angelo|valle di cadore|valle di casies|valle di maddaloni|valle lomellina|valle salimbene|valle san nicolao|vallebona|vallecorsa|vallecrosia|valledolmo|valledoria|vallefiorita|vallefoglia|vallelaghi|vallelonga|vallelunga pratameno|vallemaio|vallepietra|vallerano|vallermosa|vallerotonda|vallesaccarda|valleve|valli del pasubio|vallinfreda|vallio terme|vallo della lucania|vallo di nera|vallo torinese|valloriate|valmacca|valmadrera|valmontone|valmorea|valmozzola|valnegra|valpelline|valperga|valprato soana|valsamoggia|valsavarenche|valsinni|valsolda|valstrona|valtopina|valtorta|valtournenche|valva|valvarrone|valvasone arzene|valverde|valvestino|vandoies|vanzaghello|vanzago|vanzone con san carlo|vaprio d'adda|vaprio d'agogna|varallo|varallo pombia|varano borghi|varano de' melegari|varapodio|varazze|varco sabino|varedo|varenna|varese|varese ligure|varisella|varmo|varna|varsi|varzi|varzo|vasanello|vasia|vasto|vastogirardi|vauda canavese|vazzano|vazzola|vecchiano|vedano al lambro|vedano olona|vedelago|vedeseta|veduggio con colzano|veggiano|veglie|veglio|vejano|veleso|velezzo lomellina|velletri|vellezzo bellini|velo d'astico|velo veronese|velturno|venafro|venaria reale|venarotta|venasca|venaus|vendone|venegono inferiore|venegono superiore|venetico|venezia|veniano|venosa|ventasso|venticano|ventimiglia|ventimiglia di sicilia|ventotene|venzone|verano|verano brianza|verbania|verbicaro|vercana|verceia|vercelli|vercurago|verdellino|verdello|verderio|verduno|vergato|verghereto|vergiate|vermezzo con zelo|vermiglio|vernante|vernasca|vernate|vernazza|vernio|vernole|verolanuova|verolavecchia|verolengo|veroli|verona|veronella|verrayes|verretto|verrone|verrua po|verrua savoia|verrès|vertemate con minoprio|vertova|verucchio|vervio|verzegnis|verzino|verzuolo|vescovana|vescovato|vesime|vespolate|vessalico|vestenanova|vestignè|vestone|vetralla|vetto|vezza d'alba|vezza d'oglio|vezzano ligure|vezzano sul crostolo|vezzi portio|viadana|viadanica|viagrande|viale|vialfrè|viano|viareggio|viarigi|vibo valentia|vibonati|vicalvi|vicari|vicchio|vicenza|vico del gargano|vico equense|vico nel lazio|vicoforte|vicoli|vicolungo|vicopisano|vicovaro|viddalba|vidigulfo|vidor|vidracco|vieste|vietri di potenza|vietri sul mare|vigano san martino|viganò|vigarano mainarda|vigasio|vigevano|viggianello|viggiano|viggiù|vigliano biellese|vigliano d'asti|vignale monferrato|vignanello|vignate|vignola|vignola-falesina|vignole borbera|vignolo|vignone|vigo di cadore|vigodarzere|vigolo|vigolzone|vigone|vigonovo|vigonza|viguzzolo|villa bartolomea|villa basilica|villa biscossi|villa carcina|villa castelli|villa celiera|villa collemandina|villa cortese|villa d'adda|villa d'almè|villa d'ogna|villa del bosco|villa del conte|villa di briano|villa di chiavenna|villa di serio|villa di tirano|villa estense|villa faraldi|villa guardia|villa lagarina|villa latina|villa literno|villa minozzo|villa san giovanni|villa san giovanni in tuscia|villa san pietro|villa san secondo|villa sant'angelo|villa sant'antonio|villa santa lucia|villa santa lucia degli abruzzi|villa santa maria|villa santina|villa santo stefano|villa verde|villabassa|villabate|villachiara|villacidro|villadeati|villadose|villadossola|villafalletto|villafranca d'asti|villafranca di verona|villafranca in lunigiana|villafranca padovana|villafranca piemonte|villafranca sicula|villafranca tirrena|villafrati|villaga|villagrande strisaili|villalago|villalba|villalfonsina|villalvernia|villamagna|villamaina|villamar|villamarzana|villamassargia|villamiroglio|villanders|villandro|villanova biellese|villanova canavese|villanova d'albenga|villanova d'ardenghi|villanova d'asti|villanova del battista|villanova del ghebbo|villanova del sillaro|villanova di camposampiero|villanova marchesana|villanova mondovì|villanova monferrato|villanova monteleone|villanova solaro|villanova sull'arda|villanova truschedu|villanova tulo|villanovaforru|villanovafranca|villanterio|villanuova sul clisi|villaperuccio|villapiana|villaputzu|villar dora|villar focchiardo|villar pellice|villar perosa|villar san costanzo|villarbasse|villarboit|villareggia|villaricca|villaromagnano|villarosa|villasalto|villasanta|villasimius|villasor|villaspeciosa|villastellone|villata|villaurbana|villavallelonga|villaverla|ville d'anaunia|ville di fiemme|villeneuve|villesse|villetta barrea|villette|villimpenta|villnöß|villongo|villorba|vilminore di scalve|vimercate|vimodrone|vinadio|vinchiaturo|vinchio|vinci|vinovo|vintl|vinzaglio|viola|vione|vipiteno|virle piemonte|visano|vische|visciano|visco|visone|visso|vistarino|vistrorio|vita|viterbo|viticuso|vito d'asio|vitorchiano|vittoria|vittorio veneto|vittorito|vittuone|vitulano|vitulazio|vivaro|vivaro romano|viverone|vizzini|vizzola ticino|vizzolo predabissi|viù|vo'|vobarno|vobbia|vocca|vodo cadore|voghera|voghiera|vogogna|volano|volla|volongo|volpago del montello|volpara|volpedo|volpeglino|volpiano|volta mantovana|voltaggio|voltago agordino|volterra|voltido|volturara appula|volturara irpina|volturino|volvera|vottignasco|völs am schlern|vöran|waidbruck|welsberg-taisten|welschnofen|wengen|wolkenstein in gröden|zaccanopoli|zafferana etnea|zagarise|zagarolo|zambrone|zandobbio|zanica|zanè|zapponeta|zavattarello|zeccone|zeddiani|zelbio|zelo buon persico|zeme|zenevredo|zenson di piave|zerba|zerbo|zerbolò|zerfaliu|zeri|zermeghedo|zero branco|zevio|zgonik|ziano di fiemme|ziano piacentino|zibido san giacomo|zignago|zimella|zimone|zinasco|zoagli|zocca|zogno|zola predosa|zollino|zone|zoppola|zoppè di cadore|zovencedo|zubiena|zuccarello|zugliano|zuglio|zumaglia|zumpano|zungoli|zungri|števerjan""".split("|")) +COMUNI_AMBIGUOUS = frozenset("""abbiategrasso|acerra|acireale|acri|acuto|adria|affi|afragola|agira|agra|agrigento|agropoli|ala|alassio|alatri|alba|albanella|albenga|alberobello|albi|albignasego|albino|albosaggia|alcamo|ales|alessandria|alfano|alfonsine|alghero|alia|allein|alpignano|altamura|altare|altino|altissimo|alto|altopascio|alì|amalfi|amandola|amaro|amato|amatrice|amelia|ameno|amorosi|ampezzo|anagni|ancona|andora|andria|anela|anghiari|anzi|anzio|aosta|apice|aprica|aprilia|aquileia|aquino|aragona|arborea|arborio|arcade|arce|archi|arco|arcore|ardea|ardore|arena|arenzano|arese|arezzo|argenta|ari|ariccia|armeno|armento|armo|arona|arpino|arre|arzachena|arzano|arzignano|asciano|asiago|asola|asolo|assago|assisi|asso|assolo|asti|atella|atri|augusta|auletta|aulla|avellino|aversa|avetrana|avezzano|aviano|avio|avola|ayas|azeglio|bacoli|badia|badolato|bagheria|bagnoregio|baiano|balocco|banchette|baranzate|barbara|barbaresco|barbata|barbona|bard|bardi|bardolino|barga|bari|barile|barletta|barolo|baschi|basiliano|bastiglia|battipaglia|bee|beinette|belfiore|bella|bellagio|bellino|bellona|belluno|bema|benestare|benevento|benna|bereguardo|bergamasco|bergamo|bettola|beverino|bevilacqua|bianchi|bianco|biassono|bibbiena|biella|bienno|bisaccia|bisceglie|bitonto|bobbio|boca|bollate|bologna|bolsena|bolzano|bomba|bonito|bonnanaro|bono|bordano|bordighera|bore|borghi|borgia|borgomanero|borgoricco|borgosesia|bormida|bormio|borno|bosa|botticino|bova|boves|bovino|bozzolo|bra|bracca|bracciano|braies|brancaleone|branzi|breme|brennero|brenta|brentonico|brescia|bressanone|bresso|brindisi|brolo|broni|bronte|brunello|brunico|bruno|buccheri|buccinasco|burgos|busca|busso|bussolengo|cabras|caccamo|cagli|cagliari|caglio|caino|calcata|calci|calciano|calcinate|calcinato|calcio|calco|caltagirone|caltanissetta|calvello|calvi|camaiore|cambiano|camerino|camerota|camini|camino|camogli|campagna|campana|campobasso|campomarino|camposampiero|camposanto|canale|canaro|candela|candida|caneva|canino|canna|cannobio|canterano|cantù|caorle|capaci|capannori|capodimonte|capri|caprile|capriolo|capua|caraglio|caravaggio|carbonate|carbone|carbonia|carcare|cardinale|carezzano|cariati|carignano|carini|carlino|carmagnola|carmignano|carovigno|carpi|carpino|carrara|carro|carrè|carugate|casalino|casamassima|casarano|cascia|cascina|casella|caserta|casina|casoria|cassino|castana|casteggio|castellabate|castellanza|castellaro|castelli|castelmagno|castelnuovo|castelseprio|castelvetrano|casto|castro|castrovillari|catania|catanzaro|cattolica|cavallino|cavatore|cave|cavezzo|cavour|ceccano|cecina|cefalù|celano|cene|cento|centola|cerano|cerchio|cerea|cerignola|cerva|cerveteri|cervia|cervino|cervo|cesa|cesate|cesena|cesenatico|cesio|ceto|ceva|chianche|chiari|chiavari|chieri|chiesanuova|chieti|chioggia|chiusa|chiusi|chivasso|ciampino|cicala|cingoli|cino|cis|cittadella|civita|civitavecchia|codroipo|colere|coli|colico|collecchio|colleferro|collegno|collio|cologne|colonna|colonnella|comacchio|comiso|como|compiano|comunanza|cona|conegliano|contrada|conversano|copertino|copiano|corato|cori|corio|corleone|correggio|corrido|corsico|cortona|cosenza|courmayeur|covo|cozzo|crema|cremona|crespino|crotone|cuneo|curino|curti|cusio|dalmine|dazio|delia|dello|desio|diamante|dolina|dolo|domodossola|donato|dragoni|drena|dro|dronero|dueville|eboli|elice|ello|empoli|enna|envie|erba|ercolano|este|fabbrico|fabriano|faenza|falcone|fallo|fano|farnese|fasano|fascia|favara|favignana|felino|feltre|fermo|ferrara|ferriere|fiano|fidenza|fiesole|filadelfia|filettino|filetto|firenze|firmo|fisciano|fiuggi|fiumara|fiumicino|foggia|foligno|follonica|fondi|fontanella|fontanelle|fontanile|fonte|forano|force|forcola|forio|forlì|formia|formicola|fornace|fornelli|fortezza|fossa|fossano|foza|frascati|frassino|frattamaggiore|frignano|frisa|front|frontino|frontone|frosinone|furore|gaeta|galatina|galatone|gallarate|gallese|gallicano|gallio|gallipoli|garda|garlasco|gavi|gela|genova|gessate|giarre|ginestra|giove|girasole|giuliana|giulianova|giungano|giussano|giustino|goni|gonzaga|gorgonzola|gorizia|gorno|grado|gradoli|gragnano|graniti|greci|greggio|grimaldi|grosseto|grosso|grottaglie|grotte|gruaro|grugliasco|gualdo|guastalla|gubbio|idro|iglesias|imola|imperia|impruneta|incudine|introd|ischia|iseo|isernia|ispica|ispra|isso|itala|ivrea|jesi|jesolo|ladispoli|laghi|laglio|lago|lainate|lana|lanciano|lariano|laterza|latina|lauro|lavagna|lavello|lecce|lecco|legnago|legnano|lei|lenta|lentini|leonessa|lerici|lesa|lesina|lettere|levate|liberi|licata|licenza|limbiate|limena|limina|linguaglossa|lioni|lipari|liscia|lissone|livigno|livorno|lizzano|loano|locri|loculi|lodi|loiano|lomazzo|longobardi|lonigo|loreto|lucca|lucera|lugo|luino|lula|luni|lupara|luserna|lustra|macello|macerata|maddaloni|mafalda|magenta|maggiora|magione|maglie|maglione|magnago|magnano|maida|maiori|maleo|malfa|malo|mammola|mandela|manduria|manfredonia|mango|manta|mantello|mantova|mappano|mara|maranello|maratea|marciana|marcon|margarita|marino|marsala|marta|martello|martinsicuro|marzano|marzio|mascalucia|maser|masera|masi|massa|massello|matera|matino|matrice|mattinata|meda|mede|medea|medicina|mele|melegnano|meleti|melfi|melilli|melissa|melle|mello|melzo|mentana|merano|merate|merlino|mesagne|mese|messina|mestrino|meta|mezzana|mezzanino|mezzano|milano|milazzo|milena|millesimo|milo|minori|mira|miranda|mirandola|mirano|mirto|modena|modica|modugno|moggio|moglia|mogliano|molare|molfetta|molinara|molise|momo|moncalieri|mondovì|mondragone|monfalcone|monno|monopoli|monreale|monselice|montalcino|montale|montanaro|montebelluna|montecarlo|montecchio|montelabbate|montella|montello|montepulciano|monterotondo|montese|montesilvano|montevarchi|monteverde|montezemolo|monti|montiano|montichiari|montone|monza|morbegno|mordano|mores|moresco|moretta|mori|morigerati|mortara|mossa|mozzate|mozzo|muggiò|mura|muros|musei|musso|napoli|nardò|narni|naro|nasino|naso|nave|ne|nespolo|nesso|nettuno|nichelino|nicosia|nimis|noale|noceto|noci|nola|noli|nomi|none|nonio|norcia|norma|noto|novara|nove|novella|novellara|novello|novoli|nuoro|nus|ocre|oderzo|olbia|oleggio|ome|omegna|onore|opera|opi|ora|orbassano|orbetello|oria|oriolo|oristano|orosei|orte|ortona|orvieto|osimo|ospedaletti|ospedaletto|ossi|ostra|ostuni|otranto|ottaviano|ottone|ovada|pachino|padova|paesana|paese|pagani|paladina|palata|palau|palermo|palestrina|palmanova|palmi|palombaro|paludi|palù|pandino|panettieri|panni|pantelleria|paola|parabiago|parenti|parete|parma|partinico|paterno|paternò|patti|pau|pavia|pedemonte|pelago|pella|penne|perca|pergola|perito|perlo|pero|perugia|pesaro|pescara|pescasseroli|pescate|pesche|pescia|petriolo|pettineo|peveragno|piacenza|pianella|pianezza|pianoro|piatto|picerno|pico|pienza|pietrafitta|pietragalla|pietrasanta|piglio|pigna|pignola|pignone|pigra|pila|pinerolo|pineto|piode|piombino|pisa|pisano|piscina|pistoia|pizzo|poggibonsi|poggioreale|poli|policoro|polistena|polla|pollina|pollone|pomezia|pompei|pompeiana|ponderano|ponte|pontedera|ponti|pontida|ponza|poppi|porcari|pordenone|porte|portici|portoferraio|portofino|portogruaro|portovenere|posada|positano|posta|postal|postiglione|potenza|pozzallo|pozzuoli|prato|pray|preci|predosa|premia|prezza|procida|pula|pulsano|putignano|quadri|quart|quarto|quartucciu|quindici|rabbi|ragusa|randazzo|rapallo|ravello|ravenna|re|rea|recanati|recco|rende|rezzo|rho|riace|rialto|ribera|ricadi|riccia|riccione|rieti|rimini|rio|riposto|rive|rivoli|roana|roccaraso|rodano|rolo|roma|romana|romeno|roncola|rosate|rose|rossa|rossana|rosta|rotella|rotonda|rotondi|rovato|rovereto|rovigo|rozzano|rubano|rubiera|ruoti|russi|rutigliano|sabaudia|sacco|sacile|sale|salemi|salento|salerno|salle|saluzzo|salve|salò|sambuco|samo|samone|sanremo|sansepolcro|sanza|saponara|sapri|saracena|sarnico|sarno|saronno|sarzana|sassari|sassello|sassuolo|sava|saviano|savigliano|savona|scafati|scala|scalea|scandale|scandiano|scandicci|scanno|scansano|scapoli|scena|schio|sciacca|sciara|scicli|scilla|scopa|scorrano|scorzè|sedriano|segni|segrate|selci|senigallia|seregno|seriate|serina|serrata|serre|serri|sesto|sestriere|sestu|settime|seveso|siano|siderno|siena|sigillo|signa|siliqua|silvi|sini|siracusa|sirmione|soave|soglio|solaro|soldano|solferino|sommo|sona|soncino|sondrio|sonico|sora|sorano|sorgono|sori|sorrento|sorso|sospiro|sostegno|soverato|specchia|spello|sperlonga|sperone|spessa|spiazzo|spinea|spinoso|spirano|spoleto|spresiano|statte|stella|stelvio|stilo|stintino|stra|stradella|stresa|strona|strozza|sulmona|supino|susa|suzzara|taggia|taino|taleggio|taormina|taranto|tarquinia|tarsia|taverna|tavullia|teramo|termoli|terni|terracina|terrasini|terrazzo|terzo|tesero|teti|thiene|tiers|tirano|tires|tirol|tirolo|tito|tivoli|todi|tolentino|tolmezzo|tombolo|ton|torino|tornata|torno|toro|torrazzo|torresina|torretta|torricella|torrioni|tortona|tortora|tortorella|tortoreto|tradate|tramonti|trani|trapani|trappeto|travagliato|traversetolo|trebaseleghe|trecate|trento|trevi|treviglio|treviso|trieste|trinità|trino|troia|tropea|trovo|tufo|turate|turi|turri|udine|ugento|ultimo|urbana|urbe|urbino|uri|uscio|ustica|uta|vaglia|vaie|valbrembo|valdobbiadene|valenza|valenzano|vallata|valmontone|valva|valverde|varallo|varazze|varedo|varese|vasto|vecchiano|veglie|veglio|velletri|venetico|venezia|venosa|ventimiglia|verano|verbania|vercelli|verdello|vergato|vermiglio|verona|vescovato|viale|viareggio|vicari|vicenza|vicoli|vieste|vigevano|vignola|vigonza|villamagna|villasimius|villette|villorba|vimercate|vinci|viola|visco|visone|vita|viterbo|vittoria|vittuone|viù|vo'|voghera|volano|voltaggio|volterra|zeri|zone""".split("|")) +PROVINCE_SIGLE = frozenset("""AG|AL|AN|AO|AP|AQ|AR|AT|AV|BA|BG|BI|BL|BN|BO|BR|BS|BT|BZ|CA|CB|CE|CH|CI|CL|CN|CO|CR|CS|CT|CZ|EN|FC|FE|FG|FI|FM|FR|GE|GO|GR|IM|IS|KR|LC|LE|LI|LO|LT|LU|MB|MC|ME|MI|MN|MO|MS|MT|NA|NO|NU|OG|OR|OT|PA|PC|PD|PE|PG|PI|PN|PO|PR|PT|PU|PV|PZ|RA|RC|RE|RG|RI|RM|RN|RO|SA|SI|SO|SP|SR|SS|SV|TA|TE|TN|TO|TP|TR|TS|TV|UD|VA|VB|VC|VE|VI|VR|VS|VT|VV""".split("|")) +REGIONS = frozenset("""abruzzo|basilicata|calabria|campania|emilia-romagna|friuli-venezia giulia|lazio|liguria|lombardia|marche|molise|piemonte|puglia|sardegna|sicilia|südtirol|toscana|trentino-alto adige|umbria|valle d'aosta|vallée d'aoste|veneto""".split("|")) diff --git a/lexicon_it.py b/lexicon_it.py new file mode 100644 index 0000000..0591672 --- /dev/null +++ b/lexicon_it.py @@ -0,0 +1,258 @@ +"""Italian lexicon used to tell ordinary words from names and places. + +Qt-free. The NER model tags capitalised ordinary words (first-person verbs of +certificates and prescriptions, headings of medical or legal documents) as +PERSON or LOCATION. The lists below are used to *reject* such hits, or - for +the name and place lists - only to *confirm* a hit. A word missing from a +confirmation list never rejects a hit: no complete list of surnames exists. +""" + +import functools +from typing import Any, FrozenSet, Iterable, Set + +# Verb lemmas that occur as the first word of a line or clause in medical, +# legal and administrative documents. +_ARE = ( + "certific", "dichiar", "attest", "rilasci", "conferm", "raccomand", + "consigli", "autorizz", "deleg", "nomin", "ordin", "dimett", "visit", + "controll", "esamin", "accert", "constat", "verific", "comunic", "inform", + "trasmett", "allegh", "firm", "sottoscriv", "richiam", "dispon", "invit", + "diagnostic", "refert", "annot", "segnal", "propon", "suggerisc", "delegh", + "notific", "attribu", "applic", "approv", "dichiaram", "liquid", "convoc", +) +_ERE = ( + "prescriv", "richied", "chied", "decid", "rilev", "present", "conced", + "ritien", "consider", "ammett", "dispon", "stabil", "assum", "emett", + "rispond", "compil", "ricev", +) +_IRE = ( + "convoc", "costitu", "attribu", "esegu", "adempi", "sostitu", "dimin", + "prefer", "sped", "segu", "fornisc", "garantisc", "dispon", "consent", + "ordin", +) + +# (suffix set, participle suffixes) +_ARE_SUFFIXES = ("o", "i", "a", "iamo", "ate", "ano", "ato", "ata", "ati", "ate") +_ERE_SUFFIXES = ("o", "i", "e", "iamo", "ete", "ono", "uto", "uta", "uti", "ute") +_IRE_SUFFIXES = ("o", "i", "e", "iamo", "ite", "ono", "isco", "isci", "isce", + "iscono", "ito", "ita", "iti", "ite") + + +def _conjugate() -> Set[str]: + out: Set[str] = set() + for stem in _ARE: + out.update(stem + s for s in _ARE_SUFFIXES) + for stem in _ERE: + out.update(stem + s for s in _ERE_SUFFIXES) + for stem in _IRE: + out.update(stem + s for s in _IRE_SUFFIXES) + return out + + +_VERB_FORMS: FrozenSet[str] = frozenset(_conjugate() | { + "visto", "considerato", "ritenuto", "premesso", "sottoscritto", + "sottoscritta", "firmato", "firmata", "letto", "udito", "accertato", + "ordina", "dispone", "dispongo", "decreta", "decreto", "sentenzia", + "attesto", "prescrivo", "certifico", "dichiaro", "rilascio", "confermo", + "raccomando", "consiglio", "richiedo", "autorizzo", "delego", "nomino", +}) + +_MEDICAL_LEGAL: FrozenSet[str] = frozenset({ + "anamnesi", "diagnosi", "terapia", "prognosi", "referto", "esame", + "esami", "obiettivo", "paziente", "pazienti", "assistito", "assistita", + "certificato", "ricetta", "posologia", "controllo", "visita", "allegato", + "allegati", "timbro", "firma", "firme", "farmaco", "farmaci", "dose", + "dosi", "sintomi", "sintomo", "referti", "cartella", "clinica", "clinico", + "ricovero", "dimissione", "dimissioni", "ospedale", "reparto", "medico", + "medici", "infermiere", "infermiera", "pronto", "soccorso", "idoneo", + "idonea", "idoneità", "malattia", "infortunio", "prescrizione", + "prestazione", "prestazioni", "ambulatorio", "analisi", "ematochimici", + "ematochimico", "esito", "esiti", "conclusioni", "conclusione", "note", + "nota", "motivazione", "motivazioni", "premesso", "considerato", "ritenuto", + "decreto", "sentenza", "ordinanza", "ricorso", "ricorrente", "resistente", + "convenuto", "attore", "giudice", "tribunale", "udienza", "istanza", + "dispositivo", "pubblico", "ministero", "cancelleria", "notifica", + "oggetto", "premessa", "articolo", "comma", "regolamento", "legge", + "parti", "parte", "comparizione", "sottoscritto", "sottoscritta", + "luogo", "data", "tel", "email", "pec", "totale", "importo", "imponibile", + "fattura", "ricevuta", "scadenza", "pagamento", "saldo", "acconto", + "raccomando", "riposo", "assoluto", "giorni", "giorno", "mesi", "anni", + "tachipirina", "augmentin", +}) + +# Capitalised document words that are not personal data. +_DOCUMENT_WORDS: FrozenSet[str] = frozenset({ + "comune", "provincia", "regione", "ministero", "agenzia", "entrate", + "università", "universita", "azienda", "ospedaliera", "sanitaria", "asl", + "ausl", "spett", "srl", "spa", "snc", "sas", "onlus", "repubblica", + "italiana", "stato", "governo", "inps", "inail", "ente", "ufficio", + "uffici", "dipartimento", "direzione", "servizio", "sezione", "area", + "settore", "segreteria", "protocollo", "pagina", "allegato", "copia", + "originale", "documento", "modulo", "richiesta", "domanda", "dichiarazione", + "autocertificazione", "informativa", "consenso", "privacy", "cordiali", + "saluti", "distinti", "cordialmente", "grazie", "firma", "timbro", +}) + +# Words that are also given names or surnames: never rejected on lexicon +# grounds alone, they need corroborating context. +CONFLICT_NAMES: FrozenSet[str] = frozenset({ + "rosa", "serena", "sole", "fiore", "marino", "bruno", "bianco", "speranza", + "vittoria", "pace", "conte", "porto", "grazia", "gioia", "felice", "primo", + "santo", "fabbri", "mancini", "leone", "gallo", "colombo", "greco", + "romano", "ricci", "costa", "moro", "russo", "ferrari", "neri", "verdi", + "rossi", "gialli", "bella", "forte", "lupo", "orso", "martino", +}) + +GIVEN_NAMES: FrozenSet[str] = frozenset({ + "mario", "luigi", "giuseppe", "giovanni", "antonio", "francesco", + "alessandro", "andrea", "marco", "luca", "paolo", "stefano", "roberto", + "davide", "matteo", "simone", "lorenzo", "giorgio", "franco", "claudio", + "carlo", "angelo", "pietro", "salvatore", "vincenzo", "michele", "fabio", + "massimo", "daniele", "riccardo", "federico", "gabriele", "enrico", + "maria", "anna", "giulia", "francesca", "chiara", "sara", "laura", + "elena", "valentina", "alessandra", "paola", "silvia", "federica", + "martina", "elisa", "roberta", "monica", "claudia", "lucia", "cristina", + "daniela", "stefania", "barbara", "giovanna", "angela", "rita", "teresa", + "graziano", "graziella", "nicola", "domenico", "emanuele", "filippo", + "giacomo", "tommaso", "samuele", "edoardo", "gianluca", "gianni", + "rosa", "serena", "giorgia", "alice", "arianna", "beatrice", "irene", + "marta", "veronica", "ilaria", "manuela", "patrizia", "carla", "marina", +}) + +REGIONS_PROVINCES: FrozenSet[str] = frozenset({ + "abruzzo", "basilicata", "calabria", "campania", "emilia-romagna", + "friuli-venezia", "lazio", "liguria", "lombardia", "marche", "molise", + "piemonte", "puglia", "sardegna", "sicilia", "toscana", "trentino", + "umbria", "veneto", "valle d'aosta", + "roma", "milano", "napoli", "torino", "palermo", "genova", "bologna", + "firenze", "bari", "catania", "venezia", "verona", "messina", "padova", + "trieste", "brescia", "parma", "taranto", "prato", "modena", "reggio", + "perugia", "livorno", "ravenna", "cagliari", "foggia", "rimini", + "salerno", "ferrara", "sassari", "latina", "monza", "siracusa", + "pescara", "bergamo", "forlì", "trento", "vicenza", "terni", "bolzano", + "novara", "piacenza", "ancona", "andria", "arezzo", "udine", "cesena", + "lecce", "pesaro", "la spezia", "como", "pisa", "lucca", "cosenza", + "caserta", "avellino", "benevento", "potenza", "matera", "campobasso", + "catanzaro", "crotone", "agrigento", "trapani", "ragusa", "enna", + "caltanissetta", "aosta", "cuneo", "asti", "alessandria", "biella", + "varese", "lecco", "sondrio", "pavia", "cremona", "mantova", "lodi", + "belluno", "treviso", "rovigo", "gorizia", "pordenone", "savona", + "imperia", "massa", "carrara", "pistoia", "siena", "grosseto", + "viterbo", "rieti", "frosinone", "l'aquila", "teramo", "chieti", + "isernia", "brindisi", "barletta", "oristano", "nuoro", + "italia", "francia", "germania", "spagna", "svizzera", "austria", +}) + +COMMON_WORDS: FrozenSet[str] = frozenset( + _VERB_FORMS | _MEDICAL_LEGAL | _DOCUMENT_WORDS +) + +# Italian function words (articles, prepositions, conjunctions, pronouns) that +# may begin a capitalised sentence start. +FUNCTION_WORDS: FrozenSet[str] = frozenset({ + "il", "lo", "la", "i", "gli", "le", "un", "uno", "una", "di", "a", "da", + "in", "con", "su", "per", "tra", "fra", "e", "ed", "o", "ma", "se", "che", + "chi", "non", "più", "come", "quando", "dove", "anche", "ogni", "questo", + "questa", "questi", "queste", "quello", "quella", "del", "della", "dei", + "delle", "dello", "al", "alla", "ai", "alle", "allo", "nel", "nella", + "nei", "nelle", "sul", "sulla", "sui", "sulle", "dal", "dalla", "dai", + "dalle", "io", "tu", "lui", "lei", "noi", "voi", "loro", "mi", "ti", "si", + "ci", "vi", "ne", "ho", "ha", "hanno", "è", "sono", "era", "sarà", "poi", + "dopo", "prima", "oggi", "ieri", "domani", "tutti", "tutto", "nessun", + "stesso", "stessa", "presso", "sotto", "sopra", "senza", "secondo", + "nato", "nata", "residente", "domiciliato", "domiciliata", +}) + + +# Italian vehicle-plate style province codes (107 provinces / metropolitan cities). +PROVINCE_SIGLE: FrozenSet[str] = frozenset( + "AG AL AN AO AP AQ AR AT AV BA BG BI BL BN BO BR BS BT BZ CA CB CE CH CL CN " + "CO CR CS CT CZ EN FC FE FG FI FM FR GE GO GR IM IS KR LC LE LI LO LT LU MB " + "MC ME MI MN MO MS MT NA NO NU OR PA PC PD PE PG PI PN PO PR PT PU PV PZ RA " + "RC RE RG RI RM RN RO SA SI SO SP SR SS SU SV TA TE TN TO TP TR TS TV UD VA " + "VB VC VE VI VR VT VV".split() +) + + +def is_common_word(token: str) -> bool: + """True for ordinary Italian words that are never PII on their own.""" + t = token.strip().lower().strip(".,;:!?()[]\"'«»") + if not t: + return True + if t in CONFLICT_NAMES: + return False + return t in COMMON_WORDS or t in FUNCTION_WORDS + + +# Frequent Italian surnames (a small curated list: confirmation only). +SURNAMES: FrozenSet[str] = frozenset( + "rossi russo ferrari esposito bianchi romano colombo ricci marino greco bruno " + "gallo conti de luca mancini costa giordano rizzo lombardi moretti barbieri " + "fontana santoro mariani rinaldi caruso ferrara galli martini leone longo " + "gentile martinelli vitale lombardo serra coppola de santis d'angelo marchetti " + "parisi villa conte ferraro ferri fabbri bianco marini grasso valentini messina " + "sala de angelis gatti pellegrini palumbo sanna farina rizzi monti cattaneo " + "morelli amato silvestri mazza testa grassi pellegrino carbone giuliani benedetti " + "barone rossetti caputo montanari guerra palmieri bernardi martino fiore de rosa " + "ferretti bellini basile riva donati piras vitali battaglia sartori neri costantini " + "milani pagano ruggiero sorrentino d'amico orlando damico negri verdi gialli " + "melis cocco leoni pace marchi mele".split() +) + + +def is_known_surname(token: str) -> bool: + return token.strip().lower().strip(".,;:") in SURNAMES + + +def is_given_name(token: str) -> bool: + return token.strip().lower().strip(".,;:") in GIVEN_NAMES + + +@functools.lru_cache(maxsize=1) +def _data() -> Any: + """ISTAT-derived place data, imported on first use (not at start-up).""" + try: + import lexicon_data_it + + return lexicon_data_it + except Exception: + return None + + +def _place_key(token: str) -> str: + return token.strip().lower().replace("’", "'").strip(".,;:()") + + +def is_known_place(token: str) -> bool: + """A comune, region or country - confirmation only, never a rejection. + + Names that are also ordinary words (Sale, Re, Orso...) are *not* counted + here; see :func:`is_ambiguous_place`. + """ + key = _place_key(token) + if key in REGIONS_PROVINCES: + return True + d = _data() + return bool(d and (key in d.COMUNI or key in d.REGIONS) and key not in d.COMUNI_AMBIGUOUS) + + +def is_ambiguous_place(token: str) -> bool: + """A comune whose name is also an ordinary word or a very short name.""" + key = _place_key(token) + d = _data() + return bool(d and key in d.COMUNI and key in d.COMUNI_AMBIGUOUS + and key not in REGIONS_PROVINCES) + + +def province_codes() -> FrozenSet[str]: + d = _data() + return frozenset(PROVINCE_SIGLE | (d.PROVINCE_SIGLE if d else frozenset())) + + +def verb_forms() -> FrozenSet[str]: + return _VERB_FORMS + + +def extend(words: Iterable[str]) -> FrozenSet[str]: + """Return COMMON_WORDS extended with caller-supplied words.""" + return frozenset(COMMON_WORDS | {w.lower() for w in words}) diff --git a/pdf_sanitize.py b/pdf_sanitize.py new file mode 100644 index 0000000..6fe0915 --- /dev/null +++ b/pdf_sanitize.py @@ -0,0 +1,553 @@ +"""Qt-free export sanitiser and verifier. + +Redacting visible text is not enough to share a PDF: annotations, embedded +files, JavaScript, bookmarks, tagged-PDF alternative text, hidden layers, +invisible text and text outside the page box all survive a plain +``apply_redactions()``. :func:`sanitize_document` removes them; +:func:`verify_export` reopens the written file and checks that none of the +removed values is still recoverable from it. + +Reports contain counts and locations, never the removed values. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional + +import pymupdf + +PDF_REDACT_IMAGE_NONE: int = pymupdf.PDF_REDACT_IMAGE_NONE # type: ignore[attr-defined] +PDF_ANNOT_REDACT: int = pymupdf.PDF_ANNOT_REDACT # type: ignore[attr-defined] + +# Catalog keys removed at export: they carry scripts, attachments, outlines, +# structure (alternative text), threads or legal attestations. +CATALOG_KEYS_TO_DROP = ( + "Names", "OpenAction", "AA", "StructTreeRoot", "MarkInfo", "Outlines", + "Threads", "SpiderInfo", "Perms", "Legal", "AcroForm", "Metadata", + "PieceInfo", "Properties", "URI", "Collection", +) +PAGE_KEYS_TO_DROP = ("AA", "PieceInfo", "Metadata", "Thumb", "StructParents") + + +@dataclass +class SanitizePolicy: + remove_hidden_text: bool = True + remove_annotations: bool = True + remove_external_links: bool = True + reveal_layers: bool = True + expand_cropbox: bool = True + regenerate_id: bool = True + mask_removed_width: bool = False + + +@dataclass +class SanitizeReport: + actions: Dict[str, int] = field(default_factory=dict) + + def bump(self, key: str, n: int = 1) -> None: + if n: + self.actions[key] = self.actions.get(key, 0) + n + + +def reveal_hidden_layers(doc: Any) -> int: + """Switch every optional-content layer on so hidden text becomes visible. + + Text in a layer that is OFF is not returned by ``get_text()``, so neither + the detector nor the reviewer would ever see it. Returns the layer count. + """ + try: + ocgs = doc.get_ocgs() + except Exception: + return 0 + if not ocgs: + return 0 + try: + doc.set_layer(-1, basestate="ON", off=[]) + except Exception: + try: + doc.set_layer(-1, on=list(ocgs.keys()), off=[]) + except Exception: + return 0 + return len(ocgs) + + +def _page_image_coverage(page: Any) -> float: + try: + area = page.rect.width * page.rect.height + if area <= 0: + return 0.0 + covered = 0.0 + for info in page.get_image_info(hashes=False): + r = pymupdf.Rect(info["bbox"]) & page.rect + covered += max(0.0, r.width * r.height) + return covered / area + except Exception: + return 0.0 + + +page_image_coverage = _page_image_coverage + + +def hidden_text_rects(page: Any) -> List[Any]: + """Rectangles of text that is drawn invisibly. + + Invisible = render mode 3, zero opacity or sub-1pt font size. On pages + that are mostly one image the invisible layer is the OCR text of a scan + and is left alone: it is only removed where the image is redacted. + """ + try: + spans = page.get_texttrace() + except Exception: + return [] + scanned = _page_image_coverage(page) >= 0.6 + out: List[Any] = [] + for sp in spans: + invisible = ( + sp.get("type") == 3 + or float(sp.get("opacity", 1.0)) <= 0.0 + or float(sp.get("size", 10.0)) < 1.0 + ) + if not invisible: + continue + if scanned and sp.get("type") == 3: + continue + r = pymupdf.Rect(sp["bbox"]) + if r.is_empty or r.is_infinite: + continue + out.append(r) + return out + + +def _quantize_tj_gaps(doc: Any, page: Any, bucket: int = 500, threshold: int = 1500) -> int: + """Round large TJ gaps so the width of removed text is not recoverable. + + ``apply_redactions`` leaves a numeric gap equal to the removed text's + width in the text-showing operator. Word spaces and kerning are far + smaller than ``threshold``, so only gaps left by removed text are + touched; the following text shifts by at most ``bucket / 2`` units. + """ + changed = 0 + num = re.compile(rb"(? bytes: + nonlocal changed + body = m.group(0) + # Keep string literals untouched: split on them. + out = bytearray() + i = 0 + depth = 0 + while i < len(body): + ch = body[i:i + 1] + if ch == b"(": + j = i + 1 + depth = 1 + while j < len(body) and depth: + if body[j:j + 1] == b"\\": + j += 2 + continue + if body[j:j + 1] == b"(": + depth += 1 + elif body[j:j + 1] == b")": + depth -= 1 + j += 1 + out += body[i:j] + i = j + elif ch == b"<": + j = body.find(b">", i) + j = len(body) if j == -1 else j + 1 + out += body[i:j] + i = j + else: + j = i + while j < len(body) and body[j:j + 1] not in (b"(", b"<"): + j += 1 + seg = body[i:j] + + def rnd(nm: "re.Match[bytes]") -> bytes: + nonlocal changed + v = float(nm.group(0)) + if abs(v) < threshold: + return nm.group(0) + q = int(round(abs(v) / bucket)) * bucket + new = -q if v < 0 else q + if new != v: + changed += 1 + return str(new).encode() + + out += num.sub(rnd, seg) + i = j + return bytes(out) + + new_data = arr.sub(fix_array, data) + if new_data != data: + doc.update_stream(xref, new_data) + return changed + + +def sanitize_document(doc: Any, policy: Optional[SanitizePolicy] = None) -> SanitizeReport: + """Strip everything that survives redaction, in place.""" + policy = policy or SanitizePolicy() + rep = SanitizeReport() + + if policy.reveal_layers: + rep.bump("layers-revealed", reveal_hidden_layers(doc)) + + for page in doc: + if policy.remove_hidden_text: + rects = hidden_text_rects(page) + for r in rects: + page.add_redact_annot(r) + if rects: + page.apply_redactions( + images=PDF_REDACT_IMAGE_NONE, graphics=0 + ) + rep.bump("hidden-text-spans", len(rects)) + if policy.expand_cropbox: + mb, cb = page.mediabox, page.cropbox + if cb != mb: + # Text between the visible box and the media box is neither + # shown nor analysed: redact it, then drop the crop. + for strip in _strips(mb, cb): + page.add_redact_annot(strip) + page.apply_redactions(images=PDF_REDACT_IMAGE_NONE, graphics=0) + page.set_cropbox(mb) + rep.bump("offpage-area-cleared") + if policy.remove_annotations: + for a in list(page.annots() or []): + page.delete_annot(a) + rep.bump("annotations-removed") + if policy.remove_external_links: + for lnk in page.get_links(): + if lnk.get("kind") in (pymupdf.LINK_URI, pymupdf.LINK_LAUNCH, + pymupdf.LINK_GOTOR): + page.delete_link(lnk) + rep.bump("links-removed") + for key in PAGE_KEYS_TO_DROP: + try: + if doc.xref_get_key(page.xref, key)[0] != "null": + doc.xref_set_key(page.xref, key, "null") + rep.bump(f"page-key:{key}") + except Exception: + pass + + try: + n_files = len(doc.embfile_names()) + except Exception: + n_files = 0 + rep.bump("embedded-files-removed", n_files) + try: + doc.scrub( + attached_files=True, clean_pages=True, embedded_files=True, + hidden_text=False, javascript=True, metadata=True, + redactions=False, remove_links=False, reset_fields=True, + reset_responses=True, thumbnails=True, xml_metadata=True, + ) + except Exception: + rep.bump("scrub-failed") + if policy.mask_removed_width: + # After scrub: cleaning rewrites content streams, which would + # re-serialise the numbers this pass has just rounded. + for page in doc: + rep.bump("tj-gaps-quantized", _quantize_tj_gaps(doc, page)) + try: + if doc.get_toc(): + rep.bump("bookmarks-removed", len(doc.get_toc())) + doc.set_toc([]) + except Exception: + pass + cat = doc.pdf_catalog() + for key in CATALOG_KEYS_TO_DROP: + try: + if doc.xref_get_key(cat, key)[0] != "null": + doc.xref_set_key(cat, key, "null") + rep.bump(f"catalog-key:{key}") + except Exception: + pass + doc.set_metadata({}) + if policy.regenerate_id: + try: + doc.xref_set_key(-1, "ID", "null") + rep.bump("trailer-id-removed") + except Exception: + pass + return rep + + +def _strips(mediabox: Any, cropbox: Any) -> List[Any]: + strips = [] + if cropbox.y0 > mediabox.y0: + strips.append(pymupdf.Rect(mediabox.x0, mediabox.y0, mediabox.x1, cropbox.y0)) + if cropbox.y1 < mediabox.y1: + strips.append(pymupdf.Rect(mediabox.x0, cropbox.y1, mediabox.x1, mediabox.y1)) + if cropbox.x0 > mediabox.x0: + strips.append(pymupdf.Rect(mediabox.x0, mediabox.y0, cropbox.x0, mediabox.y1)) + if cropbox.x1 < mediabox.x1: + strips.append(pymupdf.Rect(cropbox.x1, mediabox.y0, mediabox.x1, mediabox.y1)) + return strips + + +# --------------------------------------------------------------------------- +# Signature and stamp areas (suggestions only) +# --------------------------------------------------------------------------- + +_ANCHOR_SINGLE = {"firma", "firmato", "firmata", "timbro", "visto", "f.to"} +_ANCHOR_PAIRS = { + ("il", "medico"), ("il", "dottore"), ("in", "fede"), ("il", "dichiarante"), + ("il", "richiedente"), ("l'assistito", ""), ("il", "titolare"), + ("il", "paziente"), ("timbro", "e"), +} +_ANCHOR_REGION_BELOW = 90.0 +_ANCHOR_REGION_SIDE = 200.0 + + +def _anchor_rects(words: List[Any]) -> List[Any]: + out: List[Any] = [] + low = [(str(w[4]).lower().strip(".,:;"), w) for w in words] + for i, (t, w) in enumerate(low): + nxt = low[i + 1][0] if i + 1 < len(low) else "" + if t in _ANCHOR_SINGLE or (t, nxt) in _ANCHOR_PAIRS or (t, "") in _ANCHOR_PAIRS: + out.append(pymupdf.Rect(w[:4])) + return out + + +def _merge_close(rects: List[Any], gap: float = 8.0) -> List[Any]: + rects = [pymupdf.Rect(r) for r in rects] + merged = True + while merged: + merged = False + for i in range(len(rects)): + for j in range(i + 1, len(rects)): + grown = pymupdf.Rect(rects[i]) + (-gap, -gap, gap, gap) + if grown.intersects(rects[j]): + rects[i] |= rects[j] + del rects[j] + merged = True + break + if merged: + break + return rects + + +def suggest_signature_areas(page: Any, words: Optional[List[Any]] = None) -> List[Dict[str, Any]]: + """Suggest handwritten-signature / stamp areas near signature labels. + + Anchored on words such as "firma", "timbro", "il medico". Looks for + curve-rich vector ink without text, and for small images, in a region + around each anchor. Always returns *suggestions* (state "suggested"), + never applied without review. The rectangle is the union of the ink plus + a small padding: vector paths are removed only when fully contained. + """ + words = words if words is not None else page.get_text("words") + anchors = _anchor_rects(words) + if not anchors: + return [] + page_area = page.rect.width * page.rect.height + try: + drawings = page.get_drawings() + except Exception: + drawings = [] + try: + images = page.get_image_info(hashes=False) + except Exception: + images = [] + word_rects = [pymupdf.Rect(w[:4]) for w in words] + found: List[Dict[str, Any]] = [] + seen: List[Any] = [] + for anchor in anchors: + region = pymupdf.Rect( + anchor.x0 - _ANCHOR_REGION_SIDE, anchor.y0 - 5, + anchor.x1 + _ANCHOR_REGION_SIDE, anchor.y1 + _ANCHOR_REGION_BELOW, + ) & page.rect + ink: List[Any] = [] + widths: List[float] = [1.0] + for d in drawings: + r = pymupdf.Rect(d["rect"]) + if r.is_empty and r.width < 0.5 and r.height < 0.5: + continue + if not region.contains(r): + continue + items = d.get("items", []) + curves = sum(1 for it in items if it[0] == "c") + lines = sum(1 for it in items if it[0] == "l") + straight = len(items) == 1 and items[0][0] == "l" + if straight or (r.width > 200 and r.height < 3) or (r.height > 200 and r.width < 3): + continue # table borders, underlines + if curves >= 4 or lines >= 8: + ink.append(r) + widths.append(float(d.get("width") or 1.0)) + for cluster in _merge_close(ink): + area = cluster.width * cluster.height + if not (200 <= area <= 40000): + continue + if any(wr.contains(pymupdf.Point((cluster.x0 + cluster.x1) / 2, (cluster.y0 + cluster.y1) / 2)) + for wr in word_rects): + continue + # MuPDF removes a vector path only when the rectangle covers its + # box grown by the stroke's miter extent (default miter limit 10). + pad = 10.0 * max(widths) + 0.5 + box = cluster + (-pad, -pad, pad, pad) + overlaps_text = any( + wr.intersects(box + (-pad / 2, -pad / 2, pad / 2, pad / 2)) for wr in word_rects + ) + if any(box.intersects(s) for s in seen): + continue + seen.append(box) + found.append({ + "text": "", "entity_type": "SIGNATURE", + "rects": [tuple(box)], "source": "graphics", "score": 0.7, + "state": "suggested", + "reasons": ["signature-anchor", "curve-ink"] + + (["overlaps-text"] if overlaps_text else []), + "glued": False, + }) + for info in images: + r = pymupdf.Rect(info["bbox"]) + if not r.intersects(region) or r.width * r.height > 0.25 * page_area: + continue + box = r + (-2, -2, 2, 2) + if any(box.intersects(s) for s in seen): + continue + seen.append(box) + found.append({ + "text": "", "entity_type": "SIGNATURE", + "rects": [tuple(box)], "source": "graphics", "score": 0.6, + "state": "suggested", + "reasons": ["signature-anchor", "raster-stamp"], "glued": False, + }) + return found + + +# --------------------------------------------------------------------------- +# Verification of the written file +# --------------------------------------------------------------------------- + +@dataclass +class VerificationReport: + ok: bool = True + findings: List[str] = field(default_factory=list) # locations only + + def add(self, where: str) -> None: + self.ok = False + if where not in self.findings: + self.findings.append(where) + + +MIN_NEEDLE_LENGTH = 4 + + +def _squash(s: str) -> str: + return " ".join(s.split()) + + +def _compile_needles(values: Iterable[str]) -> List["re.Pattern[str]"]: + """Whole-word patterns for the removed values. + + Only whole values count (never their individual words) and only from + four characters up, and a match must not be glued to other letters or + digits: a redacted "Rossi" must not be reported because "Rossini" is + still on the page. + """ + out: List["re.Pattern[str]"] = [] + seen = set() + for v in values: + sq = _squash(v) + if len(sq) < MIN_NEEDLE_LENGTH or sq.casefold() in seen: + continue + seen.add(sq.casefold()) + body = r"\s+".join(re.escape(part) for part in sq.split()) + out.append(re.compile(r"(? bool: + flat = _squash(text) + return any(n.search(flat) for n in needles) + + +def _is_binary_resource(obj_text: str) -> bool: + """Font programs and images: binary noise, not readable text.""" + return any( + k in obj_text + for k in ("/Length1", "/FontFile", "/Type1C", "/CIDFontType0C", + "/OpenType", "/Subtype/Image", "/Subtype /Image") + ) + + +def _object_texts(doc: Any, xref: int) -> List[str]: + texts: List[str] = [] + try: + obj = doc.xref_object(xref, compressed=False) + except Exception: + obj = "" + texts.append(obj) + try: + if doc.xref_is_stream(xref) and not _is_binary_resource(obj): + raw = doc.xref_stream(xref) or b"" + texts.append(raw.decode("latin-1", errors="ignore")) + texts.append(raw.decode("utf-16-be", errors="ignore")) + except Exception: + pass + return texts + + +def verify_export( + out_path: str, + removed_values: Iterable[str], + still_present_text: str = "", +) -> VerificationReport: + """Check that no removed value can still be read from the written file. + + Looks at page text (page-box clipping off), every object and decompressed + stream, annotations, embedded files, bookmarks and metadata, and flags any + leftover annotation, script or attachment. Locations are reported, never + the values. A value that the user deliberately left unredacted elsewhere + (it occurs in `still_present_text`, the source text outside every + redaction) is not a finding. + """ + rep = VerificationReport() + needles = [ + n for n in _compile_needles(removed_values) + if not (still_present_text and n.search(_squash(still_present_text))) + ] + doc = pymupdf.open(out_path) + try: + flags = pymupdf.TEXT_PRESERVE_WHITESPACE | pymupdf.TEXT_PRESERVE_LIGATURES + for page in doc.pages(): + text = page.get_text("text", flags=flags) + words = " ".join(w[4] for w in page.get_text("words", flags=flags)) + if _hit(needles, text) or _hit(needles, words): + rep.add(f"page {page.number + 1}: text layer") + if list(page.annots() or []): + rep.add(f"page {page.number + 1}: annotations remain") + for lnk in page.get_links(): + if lnk.get("kind") == pymupdf.LINK_URI: + rep.add(f"page {page.number + 1}: external link remains") + meta = " ".join(str(v) for v in (doc.metadata or {}).values()) + if _hit(needles, meta): + rep.add("document info") + if doc.embfile_names(): + rep.add("embedded files remain") + if _hit(needles, " ".join(str(t[1]) for t in doc.get_toc())): + rep.add("bookmarks") + cat = doc.pdf_catalog() + for key in ("OpenAction", "AA", "Names"): + if doc.xref_get_key(cat, key)[0] != "null": + rep.add(f"catalog /{key} remains") + for xref in range(1, doc.xref_length()): + for blob in _object_texts(doc, xref): + if _hit(needles, blob): + rep.add(f"object {xref}") + break + finally: + doc.close() + return rep diff --git a/pii_detection.py b/pii_detection.py index d0f630c..94b2d21 100644 --- a/pii_detection.py +++ b/pii_detection.py @@ -9,8 +9,12 @@ import json import pathlib import re +import statistics +from bisect import bisect_left from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Dict, FrozenSet, Iterable, List, Optional, Sequence, Set, Tuple + +import lexicon_it Rect4 = Tuple[float, float, float, float] @@ -29,6 +33,9 @@ ("LICENSE", "Patente", ("IT_DRIVER_LICENSE",)), ("IDCARD", "Carta d'identità", ("IT_IDENTITY_CARD",)), ("PASSPORT", "Passaporto", ("IT_PASSPORT",)), + ("BIRTHDATE", "Date di nascita", ("DATE_OF_BIRTH",)), + ("HEALTHID", "Identificativi sanitari", ("HEALTH_ID",)), + ("EXEMPT", "Codici di esenzione", ("HEALTH_EXEMPTION",)), ] ALL_GROUP_IDS: List[str] = [g[0] for g in ENTITY_GROUPS] ALL_ENTITY_TYPES: List[str] = [t for g in ENTITY_GROUPS for t in g[2]] @@ -48,8 +55,16 @@ "IT_DRIVER_LICENSE": 0.5, "IT_IDENTITY_CARD": 0.4, "IT_PASSPORT": 0.3, + "DATE_OF_BIRTH": 0.5, + "HEALTH_ID": 0.5, + "HEALTH_EXEMPTION": 0.5, } +# Entity types found by this module's own patterns, not by Presidio. +NON_PRESIDIO_TYPES: FrozenSet[str] = frozenset( + {"DATE_OF_BIRTH", "HEALTH_ID", "HEALTH_EXEMPTION"} +) + _NER_TYPES = {"PERSON", "LOCATION"} _GENERIC_WORDS = { @@ -108,6 +123,15 @@ class WordBox: end: int rect: Rect4 line_key: Tuple[int, int] + text: str = "" + # False for words over an image or from OCR: redaction there blanks + # pixels, so those boxes are never reduced to a thin band. + native: bool = True + # The box before any clamping (what the word really occupies). + orig_rect: Optional[Rect4] = None + # True when the line spacing is so tight that no band can spare the + # neighbouring lines; the full box is kept (never a leak, may damage). + collision: bool = False @dataclass @@ -117,6 +141,10 @@ class PageTextIndex: text: str = "" words: List[WordBox] = field(default_factory=list) + @property + def collisions(self) -> int: + return sum(1 for w in self.words if w.collision) + @dataclass class Candidate: @@ -126,29 +154,222 @@ class Candidate: entity_type: str score: float rects: List[Rect4] = field(default_factory=list) + # "accepted" candidates are applied; "suggested" ones await user review. + state: str = "accepted" + reasons: List[str] = field(default_factory=list) + + +NL = chr(10) +_LIGATURES = { + "ff": "ff", "fi": "fi", "fl": "fl", "ffi": "ffi", + "ffl": "ffl", "ſt": "st", "st": "st", +} + + +def _normalize_token(token: str) -> str: + """Expand typographic ligatures so NER sees ordinary letters.""" + if not any(ch in _LIGATURES for ch in token): + return token + return "".join(_LIGATURES.get(ch, ch) for ch in token) + + +def _is_vertical(rect: Rect4, text: str) -> bool: + w, h = rect[2] - rect[0], rect[3] - rect[1] + return len(text) > 1 and h > 1.5 * w + + +def _near_duplicate(a: Rect4, b: Rect4) -> bool: + ix = min(a[2], b[2]) - max(a[0], b[0]) + iy = min(a[3], b[3]) - max(a[1], b[1]) + if ix <= 0 or iy <= 0: + return False + area_a = (a[2] - a[0]) * (a[3] - a[1]) + area_b = (b[2] - b[0]) * (b[3] - b[1]) + return ix * iy >= 0.8 * min(area_a, area_b) -def build_page_index(words: Iterable[Sequence[Any]]) -> PageTextIndex: - """Build the analysis string from `page.get_text("words")` tuples.""" +# A glyph is deleted by a redaction when the rectangle intersects its box +# shrunk by this fraction of its width (each side) and height (top, bottom). +# Measured on PyMuPDF 2.1 / MuPDF 1.28; pinned by tests/test_geometry.py. +GLYPH_CORE_INSET = 0.10 + +# Smallest share of its own height a word box keeps after clamping. Below it +# the text overlaps its neighbours so heavily that trimming would risk +# leaving ink uncovered; the original box is kept. +CLAMP_MIN_RATIO = 0.3 + + +# Safety margin (pt) kept between a band and the deletion core of a neighbour, +# and the least a band must overlap its own word's core. +GLYPH_CORE_MARGIN = 0.3 +MIN_CORE_HIT = 0.5 + + +def clamp_word_rects( + boxes: List["WordBox"], ocr: bool = False, min_ratio: Optional[float] = None +) -> List[Rect4]: + """Vertical extents of word boxes trimmed where they hit another line. + + PyMuPDF word boxes span ascender to descender (about 1.37 x the font + size), so with normal leading they overlap the lines above and below. + A box is trimmed only against boxes of *other* lines that overlap it + horizontally, and only on the side facing that neighbour. Words on the + same line never affect each other, so mixed font sizes keep their full + height. + + Native text: a glyph is deleted only when the rectangle reaches its + *core* (its box inset by GLYPH_CORE_INSET), so the band stops just short + of the neighbour's core instead of its box edge. That spares neighbours + at much tighter leading. If the remaining band would not reach the + word's own core, the full box is kept and the word is flagged + ``collision``. + + Image / OCR words (and everything when `ocr` is set): redaction blanks + pixels, so the cut is at the middle of the overlap with a small + horizontal margin, and never thinner than `min_ratio` of the height. + """ + if min_ratio is None: + min_ratio = CLAMP_MIN_RATIO + n = len(boxes) + out: List[Rect4] = [b.rect for b in boxes] + if n < 2: + return out + order = sorted(range(n), key=lambda i: boxes[i].rect[1]) + y0s = [boxes[i].rect[1] for i in order] + max_h = max(b.rect[3] - b.rect[1] for b in boxes) + for i in range(n): + b = boxes[i] + x0, y0, x1, y1 = b.rect + h = y1 - y0 + if h <= 0 or _is_vertical(b.rect, b.text): + continue + pixel_mode = ocr or not b.native + lo = bisect_left(y0s, y0 - max_h) + hi = bisect_left(y0s, y1) + ny0, ny1 = y0, y1 + touched = False + for j in order[lo:hi]: + if j == i: + continue + o = boxes[j] + if o.line_key == b.line_key or _is_vertical(o.rect, o.text): + continue + ox0, oy0, ox1, oy1 = o.rect + if ox1 <= x0 or ox0 >= x1 or oy1 <= y0 or oy0 >= y1: + continue + if _near_duplicate(b.rect, o.rect): + continue + touched = True + below = (oy0 + oy1) / 2 > (y0 + y1) / 2 + if pixel_mode: + mid = (max(y0, oy0) + min(y1, oy1)) / 2 + if below: + ny1 = min(ny1, mid) + else: + ny0 = max(ny0, mid) + else: + oh = oy1 - oy0 + if below: + ny1 = min(ny1, oy0 + GLYPH_CORE_INSET * oh - GLYPH_CORE_MARGIN) + else: + ny0 = max(ny0, oy1 - GLYPH_CORE_INSET * oh + GLYPH_CORE_MARGIN) + if not touched: + continue + if pixel_mode: + if (ny1 - ny0) < min_ratio * h: + continue + pad = 1.0 + out[i] = (x0 - pad, ny0, x1 + pad, ny1) + continue + core0, core1 = y0 + GLYPH_CORE_INSET * h, y1 - GLYPH_CORE_INSET * h + if min(ny1, core1) - max(ny0, core0) < MIN_CORE_HIT: + b.collision = True + continue + out[i] = (x0, ny0, x1, ny1) + return out + + +def build_page_index( + words: Iterable[Sequence[Any]], + *, + clamp: bool = True, + ocr: bool = False, + image_rects: Sequence[Rect4] = (), + merge_spaced: bool = True, + join_hyphen: bool = True, +) -> PageTextIndex: + """Build the analysis string from `page.get_text("words")` tuples. + + Besides joining words, this: expands ligatures; glues letter-spaced runs + ("C o m m i s s i o n e") into one analysis word; joins a word split by a + line-end hyphen; and clamps each word box against neighbouring lines. + """ ordered = sorted( (w for w in words if str(w[4]).strip()), key=lambda w: (int(w[5]), int(w[6]), int(w[7])), ) + n = len(ordered) + tokens = [_normalize_token(str(w[4])) for w in ordered] + keys = [(int(w[5]), int(w[6])) for w in ordered] + rects = [(w[0], w[1], w[2], w[3]) for w in ordered] + + # Separator placed *before* word i (i >= 1). + seps: List[str] = [""] * n + for i in range(1, n): + seps[i] = ( + " " if keys[i] == keys[i - 1] + else (NL if keys[i][0] == keys[i - 1][0] else NL + NL) + ) + + if merge_spaced: + i = 0 + while i < n: + j = i + while ( + j + 1 < n and keys[j + 1] == keys[i] + and len(tokens[j]) == 1 and tokens[j].isalnum() + and len(tokens[j + 1]) == 1 and tokens[j + 1].isalnum() + ): + j += 1 + if j - i + 1 >= 4: + gaps = [rects[k + 1][0] - rects[k][2] for k in range(i, j)] + med = statistics.median(gaps) + for k in range(i, j): + seps[k + 1] = " " if gaps[k - i] > 1.8 * med + 0.5 else "" + i = j + 1 + + if join_hyphen: + for i in range(n - 1): + t = tokens[i] + if ( + len(t) >= 3 and t.endswith("-") and t[:-1].isalpha() + and keys[i + 1][0] == keys[i][0] + and keys[i + 1][1] == keys[i][1] + 1 + and tokens[i + 1][:1].islower() + ): + tokens[i] = t[:-1] + seps[i + 1] = "" + parts: List[str] = [] boxes: List[WordBox] = [] pos = 0 - prev: Optional[Tuple[int, int]] = None - for w in ordered: - key = (int(w[5]), int(w[6])) - if prev is not None: - sep = " " if key == prev else ("\n" if key[0] == prev[0] else "\n\n") - parts.append(sep) - pos += len(sep) - token = str(w[4]) - boxes.append(WordBox(pos, pos + len(token), (w[0], w[1], w[2], w[3]), key)) - parts.append(token) - pos += len(token) - prev = key + for i in range(n): + if i: + parts.append(seps[i]) + pos += len(seps[i]) + cx, cy = (rects[i][0] + rects[i][2]) / 2, (rects[i][1] + rects[i][3]) / 2 + on_image = any( + r[0] <= cx <= r[2] and r[1] <= cy <= r[3] for r in image_rects + ) + boxes.append(WordBox( + pos, pos + len(tokens[i]), rects[i], keys[i], tokens[i], + native=not (ocr or on_image), orig_rect=rects[i], + )) + parts.append(tokens[i]) + pos += len(tokens[i]) + if clamp and boxes: + for box, r in zip(boxes, clamp_word_rects(boxes, ocr=ocr)): + box.rect = r return PageTextIndex("".join(parts), boxes) @@ -405,3 +626,758 @@ def save_enabled_groups(path: pathlib.Path, groups: Iterable[str]) -> None: ) except Exception: pass + + +# --------------------------------------------------------------------------- +# Evidence-based classification (Italian-aware), pattern validation and +# geometry policy. Everything below is Qt-free and exercised by the tests. +# --------------------------------------------------------------------------- + +# Types validated by a checksum or a rigid format: when such a value is glued +# to a label or to other text it is redacted as a whole token rather than +# dropped, because under-redacting an identifier is a confidentiality breach. +VALIDATED_TYPES: FrozenSet[str] = frozenset({ + "IBAN_CODE", "CREDIT_CARD", "IT_FISCAL_CODE", "IT_VAT_CODE", "CRYPTO", + "EMAIL_ADDRESS", "HEALTH_ID", "HEALTH_EXEMPTION", +}) + +ACCEPT_SCORE = 0.7 +SUGGEST_SCORE = 0.4 + + +def boundary_ok( + text: str, start: int, end: int, *, digits_only: bool = False, + apostrophe_ok: bool = False, +) -> bool: + """True when the span is not glued to alphanumerics on either side.""" + def glue(ch: str) -> bool: + if digits_only: + return ch.isdigit() + return ch.isalnum() or ch == "_" + if start > 0: + ch = text[start - 1] + if glue(ch) and not (apostrophe_ok and ch in "'’"): + return False + if end < len(text) and glue(text[end]): + return False + return True + + +def _context(text: str, start: int, end: int, before: int = 40, after: int = 20) -> str: + return text[max(0, start - before):start].lower() + " | " + text[end:end + after].lower() + + +_FISCAL_CTX = ("codice fiscale", "c.f.", "cod. fisc", "cf:", "cf ", "fiscal code", "tax code") +_VAT_CTX = ("p.iva", "p. iva", "partita iva", "partita i.v.a", "vat", "iva ") +_PASSPORT_CTX = ("passaporto", "passport") +_IDCARD_CTX = ("carta d'identit", "carta d’identit", "carta di identit", + "carta identit", "documento n", "n. documento", "numero documento", + "c.i.", "cie ", "identity card", "id card") +_LICENSE_CTX = ("patente di guida", "patente n", "n. patente", "numero patente", + "patente:", "driving licen", "driver's licen", "driver licen") +_PHONE_CTX = ("tel", "cell", "telefono", "phone", "mobile", "fax", "whatsapp", + "recapito", "contatt", "chiam") +_DATE_LIKE = re.compile(r"^\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}$") +_YEAR_RANGE = re.compile(r"^\d{4}\s*[-–/]\s*\d{4}$") + + +def refine_pattern_hit(c: Candidate, text: str) -> Tuple[bool, str]: + """Extra validation of pattern-based hits (not NER). Returns (keep, reason).""" + t = c.entity_type + value = text[c.start:c.end] + ctx = _context(text, c.start, c.end) + if t in ("IT_FISCAL_CODE", "CRYPTO"): + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if t == "IT_FISCAL_CODE": + has_ctx = any(k in ctx for k in _FISCAL_CTX) + if not any(ch.isdigit() for ch in value) and not has_ctx: + return False, "no-digit-no-context" + if value != value.upper() and not has_ctx: + return False, "lowercase-no-context" + elif t == "CREDIT_CARD": + if not boundary_ok(text, c.start, c.end, digits_only=True): + return False, "embedded-in-digits" + elif t == "IT_VAT_CODE": + if not boundary_ok(text, c.start, c.end, digits_only=True): + return False, "embedded-in-digits" + has_ctx = any(k in ctx for k in _VAT_CTX) or value.upper().startswith("IT") + if (" " in value.strip() or "_" in value) and not has_ctx: + return False, "spaced-digits-no-context" + if not has_ctx and c.score < 0.9: + return False, "no-context" + elif t == "IT_PASSPORT": + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if value != value.upper() or not any(k in ctx for k in _PASSPORT_CTX): + return False, "no-context" + elif t == "IT_IDENTITY_CARD": + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if value != value.upper() or not any(k in ctx for k in _IDCARD_CTX): + return False, "no-context" + elif t == "IT_DRIVER_LICENSE": + if not boundary_ok(text, c.start, c.end): + return False, "embedded-in-token" + if value != value.upper() or not any(k in ctx for k in _LICENSE_CTX): + return False, "no-context" + elif t == "PHONE_NUMBER": + compact = value.strip() + digits = re.sub(r"\D", "", compact) + if _DATE_LIKE.match(compact) or _YEAR_RANGE.match(compact): + return False, "date-like" + if not boundary_ok(text, c.start, c.end, digits_only=True): + return False, "embedded-in-digits" + before = text[c.start - 1] if c.start > 0 else " " + if before.isalpha(): + return False, "embedded-in-letters" + has_ctx = any(k in ctx for k in _PHONE_CTX) + plausible_start = compact.startswith(("+", "00", "0", "3")) + if len(digits) < 8 or len(digits) > 15: + return False, "digit-count" + if not (plausible_start or has_ctx): + return False, "implausible-prefix" + return True, "" + + +def strip_lexicon_edges( + candidates: Iterable[Candidate], +) -> Tuple[List[Candidate], int]: + """Cut ordinary words off the start/end of PERSON/LOCATION spans. + + "Prescrivo Augmentin" becomes "Augmentin"; "Certifico" alone disappears. + Returns the remaining candidates and the number removed or trimmed. + """ + out: List[Candidate] = [] + dropped = 0 + for c in candidates: + if c.entity_type not in _NER_TYPES: + out.append(c) + continue + toks = list(re.finditer(r"\S+", c.text)) + lo, hi = 0, len(toks) + while lo < hi and lexicon_it.is_common_word(toks[lo].group()): + lo += 1 + while hi > lo and lexicon_it.is_common_word(toks[hi - 1].group()): + hi -= 1 + if lo == hi: + dropped += 1 + continue + if lo == 0 and hi == len(toks): + out.append(c) + continue + dropped += 1 + s0, e0 = toks[lo].start(), toks[hi - 1].end() + raw = c.text[s0:e0] + value = raw.strip(_EDGE_PUNCT) + if len(value) <= 2: + continue + start = c.start + s0 + (len(raw) - len(raw.lstrip(_EDGE_PUNCT))) + out.append(Candidate(start, start + len(value), value, c.entity_type, + c.score, state=c.state, reasons=list(c.reasons))) + return out, dropped + + +def reject_fragment_hits( + candidates: Iterable[Candidate], used_ocr: bool = False +) -> Tuple[List[Candidate], int]: + """Drop NER hits that look like text-layer or OCR fragments.""" + out: List[Candidate] = [] + dropped = 0 + min_alpha = 4 if used_ocr else 3 + for c in candidates: + if c.entity_type not in _NER_TYPES: + out.append(c) + continue + toks = c.text.split() + multi = len(toks) >= 2 + letters = sum(ch.isalpha() for ch in c.text) + non_space = sum(not ch.isspace() for ch in c.text) or 1 + bad = False + if letters / non_space < 0.8: + bad = True + elif not multi and letters < min_alpha: + bad = True + elif used_ocr and any(re.search(r"[a-zà-ÿ][A-ZÀ-Ý]", t) for t in toks): + bad = True + elif any(t.endswith("-") and len(t) > 2 for t in toks): + bad = True + if bad: + dropped += 1 + else: + out.append(c) + return out, dropped + + +def merge_adjacent_lines( + candidates: List[Candidate], text: str +) -> List[Candidate]: + """Join PERSON hits on consecutive lines ("Esposito" / "Giovanni").""" + ordered = sorted(candidates, key=lambda c: c.start) + out: List[Candidate] = [] + i = 0 + while i < len(ordered): + c = ordered[i] + if ( + c.entity_type == "PERSON" and i + 1 < len(ordered) + and len(c.text.split()) == 1 + ): + n = ordered[i + 1] + gap = text[c.end:n.start] + if ( + n.entity_type == "PERSON" and len(n.text.split()) == 1 + and gap == NL + and not lexicon_it.is_common_word(c.text) + and not lexicon_it.is_common_word(n.text) + and c.text[:1].isupper() and n.text[:1].isupper() + ): + out.append(Candidate( + c.start, n.end, text[c.start:n.end], "PERSON", + max(c.score, n.score), + )) + i += 2 + continue + out.append(c) + i += 1 + return out + + +_PERSON_CUES = ( + "paziente", "assistito", "assistita", "sottoscritto", "sottoscritta", + "signor", "sig.", "sig.ra", "dott", "prof", "avv", "ing.", "nome", + "cognome", "nato", "nata", "intestatario", "intestataria", "titolare", + "firmato", "gentile", "egregio", "egregia", "spett", "patient", "mr.", + "mrs.", "ms.", "dear", "tutore", "genitore", "figlio", "figlia", "coniuge", +) +_LOCATION_CUES = ( + "nato a", "nata a", "nato il", "residente a", "residente in", "residente presso", + "domiciliato", "domiciliata", "con sede in", "con sede a", "sede legale", + "comune di", "provincia di", "in via", "presso", "luogo di nascita", + "indirizzo", "residenza", "domicilio", "born in", "lives in", +) +_PROVINCE_AFTER = re.compile(r"^\s*[\(\[]\s*[A-Z]{2}\s*[\)\]]|^\s*,?\s*\d{5}\b|^\s*,\s*\d+\b") + + +def _is_name_shaped(tokens: List[str]) -> bool: + return len(tokens) >= 2 and all( + t[:1].isupper() + and t.replace("'", "").replace("’", "").replace("-", "").isalpha() + for t in tokens + ) + + +def score_ner_candidate( + c: Candidate, text: str, verb_like: Iterable[str] = () +) -> Tuple[float, List[str]]: + """Evidence-based score (0..1) for a PERSON/LOCATION hit. + + Presidio gives every NER hit the same constant, so the model score cannot + separate "Mario Rossi" from "Certifico". Evidence in the text can. + """ + verb = {v.lower() for v in verb_like} + tokens = [t.strip(_EDGE_PUNCT) for t in c.text.split()] + tokens = [t for t in tokens if t] + lowered = [t.lower() for t in tokens] + score = 0.5 + why: List[str] = [] + ls, le = _line_bounds(text, c.start, c.end) + before_raw = text[ls:c.start] + before_ctx = text[max(0, c.start - 30):c.start].lower() + after_ctx = text[c.end:c.end + 12] + single = len(tokens) == 1 + all_caps = c.text.isupper() and len(c.text) > 2 + + cues = _PERSON_CUES if c.entity_type == "PERSON" else _LOCATION_CUES + if any(cue in before_ctx for cue in cues): + score += 0.35 + why.append("context-cue") + if c.entity_type == "PERSON": + if _is_name_shaped(tokens): + score += 0.25 + why.append("name-shape") + if any(lexicon_it.is_given_name(t) for t in tokens): + score += 0.2 + why.append("given-name") + elif len(tokens) >= 2 and any(lexicon_it.is_known_surname(t) for t in tokens): + score += 0.15 + why.append("known-surname") + else: + if any(lexicon_it.is_known_place(t) for t in tokens) or lexicon_it.is_known_place(c.text): + score += 0.3 + why.append("known-place") + elif any(lexicon_it.is_ambiguous_place(t) for t in tokens) or lexicon_it.is_ambiguous_place(c.text): + score += 0.1 + why.append("ambiguous-place") + pm = _PROVINCE_AFTER.match(after_ctx) + if pm: + code = re.search(r"[A-Z]{2}", pm.group()) + bracketed = pm.group().lstrip().startswith(("(", "[")) + if not bracketed or (code and code.group() in lexicon_it.province_codes()): + score += 0.2 + why.append("province-or-postcode") + if len(tokens) >= 2 and _is_name_shaped(tokens): + score += 0.1 + occurrences = len(re.findall( + r"(?= 2: + score += 0.15 + why.append("repeated") + + if any(lexicon_it.is_common_word(t) for t in tokens): + score -= 0.35 if single else 0.2 + why.append("lexicon") + if any(t in verb or t in lexicon_it.verb_forms() for t in lowered): + score -= 0.25 + why.append("verb-form") + if single and not before_raw.strip(_EDGE_PUNCT): + score -= 0.2 + why.append("line-start") + if all_caps and not ( + any(lexicon_it.is_given_name(t) or lexicon_it.is_known_place(t) for t in tokens) + ): + score -= 0.2 + why.append("all-caps") + nxt = after_ctx.strip().split() + if nxt and nxt[0].lower() in ("che", "di") and single: + score -= 0.1 + why.append("followed-by-function-word") + if not tokens: + score = 0.0 + return max(0.0, min(1.0, score)), why + + +def evaluate_ner_candidates( + candidates: Iterable[Candidate], text: str, *, health_mode: bool = False, + verb_like: Iterable[str] = (), +) -> Tuple[List[Candidate], int]: + """Assign each PERSON/LOCATION hit a state: accepted / suggested / dropped. + + Field labels ("Nome:") and list items are rejected outright. The rest is + scored from evidence; uncertain hits become *suggestions* the user + reviews instead of being silently applied or lost. In health mode no + hit is silently dropped: low scores become suggestions too. + """ + out: List[Candidate] = [] + dropped = 0 + verb_like = list(verb_like) + for c in candidates: + if c.entity_type not in _NER_TYPES or "address-pattern" in c.reasons: + out.append(c) + continue + after = text[c.end:c.end + 8].lstrip(" \t") + ls, _le = _line_bounds(text, c.start, c.end) + before_raw = text[ls:c.start].strip() + single = len(c.text.split()) == 1 + if single and after.startswith(":"): + dropped += 1 + continue + if single and before_raw and not before_raw.strip(_EDGE_PUNCT): + dropped += 1 # list item + continue + score, why = score_ner_candidate(c, text, verb_like) + if score >= ACCEPT_SCORE: + state = "accepted" + elif score >= SUGGEST_SCORE or health_mode: + state = "suggested" + if score < SUGGEST_SCORE: + why.append("low-confidence") + else: + dropped += 1 + continue + out.append(Candidate( + c.start, c.end, c.text, c.entity_type, score, state=state, reasons=why + )) + return out, dropped + + +_DOB_NUM = re.compile( + r"(?i)\b(?:nato|nata|nascita|born|dob)\b[^\n]{0,40}?" + r"(?P(?(?80\s?380\s?00\s?" + _REGION + r"(?:\s?\d){10})(?!\d)" +) +# NRE (electronic prescription number): 15 characters = region/PA code (3 +# digits) + lot code (2 alphanumeric) + 10 digits; printed forms may carry +# spaces or asterisks around/inside it. +_NRE = re.compile( + r"(?" + _REGION + + r"[\s*]?[0-9A-Z]{2}[\s*]?(?:\d[\s*]?){9}\d)\*?(?![0-9A-Za-z])" +) +_NRE_CTX = ("nre", "ricetta", "n. ricetta", "numero ricetta", "prescrizione", + "ricetta elettronica", "promemoria") +# Exemption codes: income E01-E04, invalidity C01-C06, work L01-L04, service +# S01-S03, war G01-G02, rare diseases R + letter + 4 characters (RA0010...). +_EXEMPT = re.compile( + r"(?E0[1-4]|C0[1-6]|L0[1-4]|S0[1-3]|G0[12]|R[A-Z][A-Z0-9]\d{3})(?![A-Z0-9])" +) +_EXEMPT_CTX = ("esenzione", "esente", "cod. es", "codice esenzione", "esenz.") + + +def _ctx_before(text: str, start: int, words: Tuple[str, ...], span: int = 60) -> bool: + window = text[max(0, start - span):start].lower() + return any(w in window for w in words) + + +def detect_health_patterns( + text: str, enabled: Iterable[str], health_mode: bool = False +) -> List[Candidate]: + """Date of birth, health card, prescription and exemption codes. + + Everything is anchored: dates of birth on "nato/a il", the prescription + number and exemption codes on their label. Without the label a hit is + only a *suggestion*, and only in health mode. + """ + on = set(enabled) + out: List[Candidate] = [] + if "DATE_OF_BIRTH" in on: + for rx in (_DOB_NUM, _DOB_TXT): + for m in rx.finditer(text): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "DATE_OF_BIRTH", 0.9 + )) + if "HEALTH_ID" in on: + for m in _TESSERA.finditer(text): + if len(re.sub(r"\D", "", m.group("d"))) == 20: + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_ID", 0.9 + )) + for m in _NRE.finditer(text): + digits = re.sub(r"[\s*]", "", m.group("d")) + if len(digits) != 15: + continue + if _ctx_before(text, m.start("d"), _NRE_CTX): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_ID", 0.85 + )) + elif health_mode: + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_ID", 0.55, + state="suggested", reasons=["no-context"], + )) + if "HEALTH_EXEMPTION" in on: + for m in _EXEMPT.finditer(text): + if _ctx_before(text, m.start("d"), _EXEMPT_CTX, span=40): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_EXEMPTION", 0.8 + )) + elif health_mode and not m.group("d")[1:].isalpha(): + out.append(Candidate( + m.start("d"), m.end("d"), m.group("d"), "HEALTH_EXEMPTION", 0.5, + state="suggested", reasons=["no-context"], + )) + return out + + +# --- addresses: house numbers and postcodes ---------------------------------- + +_STREET = ( + r"(?:via|viale|v\.le|piazza|p\.za|piazzale|corso|c\.so|largo|vicolo|strada|" + r"contrada|localit\u00e0|loc\.|frazione|fraz\.|borgo|lungotevere)" +) +_CIVICO = re.compile( + r"(?i)\b" + _STREET + r"\s+(?:[A-Z\u00c0-\u00dd][\w'\u2019.-]*\s+){0,5}?" + r"[A-Z\u00c0-\u00dd][\w'\u2019.-]*\s*,?\s*(?:n\.?|nr\.?|civico|n\u00b0)?\s*" + r"(?P\d{1,4}(?:\s*/\s*[A-Za-z0-9]{1,3}|\s*[A-Za-z](?![a-z])|\s+(?:bis|ter|quater))?)" + r"(?![\d.,]\d)" +) +_CAP = re.compile( + r"(?\d{5})(?![\d.,%])\s+" + r"(?P[A-Z\u00c0-\u00dd][\w'\u2019-]+(?:\s+[A-Z\u00c0-\u00dd][\w'\u2019-]+){0,3})" + r"(?:\s*\(\s*(?P[A-Z]{2})\s*\))?" +) +_MONEY_BEFORE = ("\u20ac", "euro", "eur", "importo", "totale", "n.", "prot.", "fattura") + + +def detect_address_parts( + text: str, enabled: Iterable[str], health_mode: bool = False +) -> List[Candidate]: + """Street + house number as one span, and postcodes in an address context. + + A postcode needs an anchor (a street before it, a province code or a + known place after it, or the label "CAP"); prices, protocol numbers, + dates and quantities never match. Without an anchor it is only a + suggestion in health mode. + """ + if "LOCATION" not in set(enabled): + return [] + out: List[Candidate] = [] + street_ends: List[int] = [] + for m in _CIVICO.finditer(text): + out.append(Candidate( + m.start(), m.end("num"), text[m.start():m.end("num")], "LOCATION", 0.95, + reasons=["address-pattern"], + )) + street_ends.append(m.end("num")) + for m in _CAP.finditer(text): + cap = m.group("cap") + if not (10 <= int(cap) <= 98168): + continue + before = text[max(0, m.start() - 10):m.start()].lower() + if any(k in before for k in _MONEY_BEFORE): + continue + after = text[m.end("cap"):m.end("cap") + 4] + if after.startswith((",0", ",00", "\u20ac", "%")): + continue + window = text[max(0, m.start() - 6):m.start()].lower() + prov = m.group("prov") + anchored = ( + any(0 <= m.start() - e <= 80 for e in street_ends) + or bool(prov and prov in lexicon_it.province_codes()) + or lexicon_it.is_known_place(m.group("city").split()[0]) + or "cap" in window + ) + if anchored: + out.append(Candidate( + m.start("cap"), m.end("cap"), cap, "LOCATION", 0.9, + reasons=["address-pattern", "postcode"], + )) + elif health_mode: + out.append(Candidate( + m.start("cap"), m.end("cap"), cap, "LOCATION", 0.5, + state="suggested", reasons=["address-pattern", "postcode", "no-anchor"], + )) + return out + + +# --- geometry policy ------------------------------------------------------- + +def span_word_indices( + index: PageTextIndex, start: int, end: int, min_ratio: float = 0.5 +) -> List[int]: + """Indices of the words a span covers by at least `min_ratio` of their length.""" + out: List[int] = [] + for i, w in enumerate(index.words): + overlap = min(end, w.end) - max(start, w.start) + if overlap <= 0: + continue + if overlap < min_ratio * (w.end - w.start): + continue + out.append(i) + return out + + +def rects_for_words( + index: PageTextIndex, idxs: Iterable[int], original: bool = False +) -> List[Rect4]: + """One rectangle per line for the given words (clamped, or `original`).""" + by_line: Dict[Tuple[int, int], List[Rect4]] = {} + for i in idxs: + w = index.words[i] + by_line.setdefault(w.line_key, []).append( + w.orig_rect if original and w.orig_rect else w.rect + ) + return [_union(rs) for _, rs in sorted(by_line.items())] + + +def resolve_word_indices( + index: PageTextIndex, c: Candidate +) -> Tuple[List[int], str]: + """Word indices to redact for a candidate, per the partial-match policy. + + Returns (indices, status). Status is "ok", "glued" (a validated + identifier stuck to other text: the whole token is redacted) or a + rejection reason (indices empty). + """ + text = index.text + if c.entity_type in _NER_TYPES: + if not boundary_ok(text, c.start, c.end, apostrophe_ok=True): + return [], "embedded-in-word" + idxs = span_word_indices(index, c.start, c.end) + return (idxs, "ok") if idxs else ([], "partial-word") + idxs = span_word_indices(index, c.start, c.end) + if idxs: + return idxs, "ok" + if c.entity_type in VALIDATED_TYPES: + touched = span_word_indices(index, c.start, c.end, min_ratio=0.0) + if touched: + return touched, "glued" + return [], "partial-word" + + +def protected_word_indices( + index: PageTextIndex, terms: Iterable[str] +) -> Set[int]: + """Word indices covered by whole-word occurrences of allowlist terms.""" + protected: Set[int] = set() + for term in terms: + term = (term or "").strip() + if not term: + continue + for m in _whole_word_pattern(term, False).finditer(index.text): + protected.update(span_word_indices(index, m.start(), m.end())) + return protected + + +def classify_page( + results: Iterable[Any], + index: PageTextIndex, + enabled_types: Iterable[str], + *, + used_ocr: bool = False, + health_mode: bool = False, + verb_like: Iterable[str] = (), + stopwords: Optional[Iterable[str]] = None, +) -> Tuple[List[Candidate], Dict[str, int]]: + """Run the full candidate pipeline on analyzer results for one page. + + Returns the surviving candidates (state "accepted" or "suggested") and a + count of rejections per reason (no values, safe to log). + """ + text = index.text + enabled = list(enabled_types) + stats: Dict[str, int] = {} + + def bump(reason: str, n: int = 1) -> None: + if n: + stats[reason] = stats.get(reason, 0) + n + + cands = filter_results(results, text, enabled) + cands.extend(detect_health_patterns(text, enabled, health_mode)) + kept: List[Candidate] = [] + for c in cands: + if c.entity_type in _NER_TYPES: + kept.append(c) + continue + ok, reason = refine_pattern_hit(c, text) + if ok: + kept.append(c) + else: + bump(f"pattern:{c.entity_type}:{reason}") + address = detect_address_parts(text, enabled, health_mode) + before = len(kept) + kept = clean_candidates(kept, stopwords=stopwords, enabled_types=enabled) + bump("generic-word", max(0, before - len(kept))) + kept, n = strip_lexicon_edges(kept) + bump("lexicon", n) + kept, n = reject_fragment_hits(kept, used_ocr) + bump("fragment", n) + kept = merge_adjacent_lines(kept, text) + kept.extend(address) + kept = resolve_overlaps(kept) + kept, n = evaluate_ner_candidates( + kept, text, health_mode=health_mode, verb_like=verb_like + ) + bump("ner-evidence", n) + return kept, stats + + +def build_detections( + index: PageTextIndex, + candidates: Iterable[Candidate], + *, + allowlist: Iterable[str] = (), + source: str = "text", + stats: Optional[Dict[str, int]] = None, +) -> List[Dict[str, Any]]: + """Turn candidates into detections with rectangles. + + Allowlist semantics are by word identity: a word is protected when it is + part of a whole-word occurrence of an allowlist entry. A detection whose + words are all protected disappears; a longer detection only loses the + protected words. Substring containment and rectangle intersection are + never used, so an allowlist entry cannot suppress a different value. + """ + terms = [t for t in allowlist if t and t.strip()] + allow_set = {" ".join(t.lower().split()) for t in terms} + protected = protected_word_indices(index, terms) if terms else set() + + def note(reason: str) -> None: + if stats is not None: + stats[reason] = stats.get(reason, 0) + 1 + + out: List[Dict[str, Any]] = [] + seen: Set[Tuple[str, str]] = set() + for c in candidates: + value = c.text + clean = " ".join(value.strip(_EDGE_PUNCT).lower().split()) + if not clean: + continue + if clean in allow_set: + note("allowlist") + continue + idxs, status = resolve_word_indices(index, c) + if not idxs: + note(f"geometry:{status}") + continue + idxs = [i for i in idxs if i not in protected] + if not idxs: + note("allowlist") + continue + rects = rects_for_words(index, idxs) + full_rects = rects_for_words(index, idxs, original=True) + key = (value, c.entity_type) + if key not in seen: + seen.add(key) + pattern = _whole_word_pattern( + value, c.entity_type in _NER_TYPES + ) + for m in pattern.finditer(index.text): + extra_idx = [ + i for i in span_word_indices(index, m.start(), m.end()) + if i not in protected + ] + for r, fr in zip( + rects_for_words(index, extra_idx), + rects_for_words(index, extra_idx, original=True), + ): + if r not in rects: + rects.append(r) + full_rects.append(fr) + out.append({ + "text": value, + "entity_type": c.entity_type, + "rects": rects, + "full_rects": full_rects, + "source": source, + "score": c.score, + "state": c.state, + "reasons": list(c.reasons), + "glued": status == "glued", + }) + return out + + +# --- rectangle merging / dedup --------------------------------------------- + +def merge_line_rects(rects: Iterable[Rect4], gap: float = 1.5) -> List[Rect4]: + """Merge rectangles on the same line that touch or overlap.""" + rs = sorted(rects, key=lambda r: (round((r[1] + r[3]) / 2, 0), r[0])) + merged: List[Rect4] = [] + for r in rs: + if merged: + m = merged[-1] + oy = min(m[3], r[3]) - max(m[1], r[1]) + minh = min(m[3] - m[1], r[3] - r[1]) + if minh > 0 and oy >= 0.6 * minh and r[0] - m[2] <= gap: + merged[-1] = ( + min(m[0], r[0]), min(m[1], r[1]), + max(m[2], r[2]), max(m[3], r[3]), + ) + continue + merged.append(r) + return merged + + +def covered_fraction(rect: Rect4, others: Iterable[Rect4]) -> float: + """Largest share of `rect` covered by any single rectangle of `others`.""" + area = max(0.0, (rect[2] - rect[0]) * (rect[3] - rect[1])) + if area <= 0: + return 1.0 + best = 0.0 + for o in others: + ix = min(rect[2], o[2]) - max(rect[0], o[0]) + iy = min(rect[3], o[3]) - max(rect[1], o[1]) + if ix > 0 and iy > 0: + best = max(best, ix * iy / area) + return best diff --git a/privacy_core.py b/privacy_core.py index b1ce493..d1bc082 100644 --- a/privacy_core.py +++ b/privacy_core.py @@ -7,9 +7,12 @@ from __future__ import annotations import base64 +import hashlib +import hmac import json import os import re +import secrets from dataclasses import asdict, dataclass from enum import Enum from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple @@ -40,6 +43,10 @@ class PlaceholderOccurrence: # {font, size, color, flags, origin, text}. None for scanned pages and # for maps written before version 3. style: Optional[Dict[str, Any]] = None + # What was actually drawn when the full placeholder did not fit the box + # (a compact form such as "P9F2A"). None when the full placeholder was + # drawn. Restore uses the rectangle, never this text. + label: Optional[str] = None def validate_style(raw: Any) -> Dict[str, Any]: @@ -70,10 +77,32 @@ class PlaceholderEntry: occurrences: Tuple[PlaceholderOccurrence, ...] = () +PSEUDONYM_STYLES = ("random", "counter", "opaque", "keyed") + + class PlaceholderRegistry: - """Create stable placeholders for detected personal data.""" + """Create stable placeholders for detected personal data. + + Styles (``style``): + - ``counter``: ``PERSON_001``, in first-seen order. The suffix reveals + how many distinct values of a type exist and in which order they + appear. + - ``random``: ``PERSON_9F2A``, unpredictable, drawn from ``secrets`` + for each document; nothing about the order or count leaks. + - ``opaque``: ``DATA_9F2A``; additionally hides the entity type. + - ``keyed``: ``PERSON_3B7E10`` = HMAC(key, type || value); the same + value gets the same pseudonym in every document made with the same + key. Cross-document linking is intended; the key must be kept + separately and protected (EDPB Guidelines 01/2025). + """ - def __init__(self) -> None: + def __init__(self, style: str = "counter", key: Optional[bytes] = None) -> None: + if style not in PSEUDONYM_STYLES: + raise ValueError(f"Stile pseudonimi sconosciuto: {style}") + if style == "keyed" and not key: + raise ValueError("Lo stile 'keyed' richiede una chiave.") + self._style = style + self._key = key self._counters: Dict[str, int] = {} self._by_value: Dict[tuple[str, str], str] = {} self._order: List[str] = [] @@ -115,7 +144,9 @@ def placeholder_for( if placeholder is None: next_index = self._counters.get(clean_type, 0) + 1 self._counters[clean_type] = next_index - placeholder = f"{clean_type}_{next_index:03d}" + placeholder = self._make_placeholder( + clean_type, clean_original, next_index + ) self._by_value[key] = placeholder self._order.append(placeholder) self._meta[placeholder] = (clean_original, clean_type, page_index) @@ -130,6 +161,45 @@ def placeholder_for( ) return placeholder + def set_label(self, placeholder: str, rect: Sequence[float], label: str) -> None: + """Record the compact label drawn for the occurrence at `rect`.""" + target = tuple(float(v) for v in tuple(rect)[:4]) + occs = self._occurrences.get(placeholder, []) + for i, occ in enumerate(occs): + if occ.rect == target: + occs[i] = PlaceholderOccurrence( + page=occ.page, rect=occ.rect, style=occ.style, label=label + ) + return + + def _make_placeholder(self, clean_type: str, original: str, n: int) -> str: + if self._style == "counter": + return f"{clean_type}_{n:03d}" + prefix = "DATA" if self._style == "opaque" else clean_type + taken = set(self._by_value.values()) + if self._style == "keyed": + assert self._key is not None + digest = hmac.new( + self._key, + (clean_type + "\x00" + original.casefold()).encode("utf-8"), + hashlib.sha256, + ).hexdigest().upper() + for width in range(6, len(digest) + 1, 2): + candidate = f"{prefix}_{digest[:width]}" + if candidate not in taken: + return candidate + return f"{prefix}_{digest}" + # Four hex digits keep the placeholder short enough to fit the box + # of a short word; widen only if collisions pile up. + nbytes, tries = 2, 0 + while True: + candidate = f"{prefix}_{secrets.token_hex(nbytes).upper()}" + if candidate not in taken: + return candidate + tries += 1 + if tries % 8 == 0: + nbytes += 1 + def entries(self) -> List[PlaceholderEntry]: entries: List[PlaceholderEntry] = [] for placeholder in self._order: @@ -147,12 +217,14 @@ def entries(self) -> List[PlaceholderEntry]: RESTORE_MAP_FORMAT = "NullifyPDF restore map" -RESTORE_MAP_VERSION = 3 +RESTORE_MAP_VERSION = 4 # Version 1 maps carry no `occurrences`; they are still readable, and # reconstruction falls back to searching the PDF for the placeholder text. # Refusing them would permanently strand the PII of every map already issued. # Version 3 adds an optional per-occurrence `style`; 2 maps stay readable. -SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2, 3}) +# Version 4 adds an optional per-occurrence `label` (compact placeholder drawn +# when the full one did not fit); older maps stay readable. +SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2, 3, 4}) def build_restore_payload( @@ -161,19 +233,43 @@ def build_restore_payload( source_sha256: str, output_sha256: Optional[str], entries: Iterable[PlaceholderEntry], + keep_source_name: bool = False, ) -> Dict[str, object]: - """Build the JSON-serializable restore-map payload.""" + """Build the JSON-serializable restore-map payload. + + The map is itself personal data, so it stores as little as possible: the + source file name is omitted unless asked for (it often holds the data + subject's name), and a style's ``text`` is dropped when it only repeats + the original value already stored in the entry. + """ raw_entries: List[Dict[str, object]] = [] for entry in entries: raw = asdict(entry) # asdict() keeps tuples as tuples; JSON round-trips them to lists, so # normalize here and keep the payload byte-comparable with itself. + def slim( + style: Optional[Dict[str, Any]], original: str = entry.original + ) -> Optional[Dict[str, Any]]: + if not style: + return None + out = dict(style) + if " ".join(str(out.get("text", "")).split()) == " ".join( + original.split() + ): + out.pop("text", None) + return out + raw["occurrences"] = [ { "page": occurrence.page, "rect": list(occurrence.rect), - **({"style": occurrence.style} if occurrence.style else {}), + **({"style": slim(occurrence.style)} if occurrence.style else {}), + **( + {"label": occurrence.label} + if occurrence.label and occurrence.label != entry.placeholder + else {} + ), } for occurrence in entry.occurrences ] @@ -182,7 +278,7 @@ def build_restore_payload( return { "format": RESTORE_MAP_FORMAT, "version": RESTORE_MAP_VERSION, - "source_name": os.path.basename(source_name), + "source_name": os.path.basename(source_name) if keep_source_name else "", "source_sha256": source_sha256, "output_sha256": output_sha256, "entries": raw_entries, @@ -220,6 +316,7 @@ def parse_restore_entries(payload: Dict[str, object]) -> List[PlaceholderEntry]: if raw_occ.get("style") else None ), + label=(str(raw_occ["label"]) if raw_occ.get("label") else None), ) ) entries.append( @@ -251,55 +348,369 @@ def sort_entries_longest_placeholder_first( return sorted(entries, key=lambda e: len(e.placeholder), reverse=True) -def encrypt_restore_payload(payload: Dict[str, object], password: str) -> bytes: - """Encrypt and authenticate a restore-map payload with a password.""" +MIN_PASSWORD_LENGTH = 12 +PBKDF2_MIN_ITERATIONS = 100_000 +PBKDF2_MAX_ITERATIONS = 5_000_000 +SCRYPT_N = 2 ** 17 +SCRYPT_R = 8 +SCRYPT_P = 1 +SCRYPT_MAX_N = 2 ** 20 - if not password or len(password) < 12: - raise ValueError("La password della mappa deve avere almeno 12 caratteri.") +_COMMON_PASSWORDS = ( + "passwordpassword", "123456789012", "qwertyuiopas", "password1234", + "administrator", "letmein123456", +) - from cryptography.fernet import Fernet + +def password_strength(password: str) -> Tuple[bool, str]: + """Reject passwords that make the map trivial to open. + + Returns (acceptable, message). Length alone is not enough: a repeated + character or a common phrase of 12 characters falls in seconds. + """ + if len(password) < MIN_PASSWORD_LENGTH: + return False, "La password deve avere almeno 12 caratteri." + if len(set(password)) < 6: + return False, "La password usa troppi caratteri ripetuti." + low = password.lower() + if any(c in low for c in _COMMON_PASSWORDS): + return False, "La password e troppo comune." + classes = sum([ + any(c.islower() for c in password), + any(c.isupper() for c in password), + any(c.isdigit() for c in password), + any(not c.isalnum() for c in password), + ]) + if classes < 2 and len(password) < 20: + return False, ( + "Usa almeno due tipi di caratteri (minuscole, maiuscole, cifre, " + "simboli) oppure una frase di almeno 20 caratteri." + ) + return True, "" + + +def _derive_key(password: str, envelope: Dict[str, Any], salt: bytes) -> bytes: from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + from cryptography.hazmat.primitives.kdf.scrypt import Scrypt + + kdf_name = str(envelope.get("kdf", "PBKDF2-HMAC-SHA256")) + if kdf_name == "scrypt": + n = int(envelope["n"]) + r = int(envelope.get("r", SCRYPT_R)) + p_ = int(envelope.get("p", SCRYPT_P)) + # Parameters come from an untrusted file: bound them before use so a + # crafted map cannot exhaust memory or CPU. + if ( + not (2 ** 14 <= n <= SCRYPT_MAX_N) or n & (n - 1) + or not (1 <= r <= 16) or not (1 <= p_ <= 4) + ): + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + kdf: Any = Scrypt(salt=salt, length=32, n=n, r=r, p=p_) + elif kdf_name == "PBKDF2-HMAC-SHA256": + iterations = int(envelope["iterations"]) + if not (PBKDF2_MIN_ITERATIONS <= iterations <= PBKDF2_MAX_ITERATIONS): + raise ValueError( + "Numero di iterazioni della mappa fuori dai limiti consentiti." + ) + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), length=32, salt=salt, iterations=iterations + ) + else: + raise ValueError("Algoritmo KDF della mappa non supportato.") + return base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) + + +def _encrypt_v2(payload: Dict[str, object], password: str) -> bytes: + """Version 2 envelope (scrypt + Fernet). Kept for compatibility tests; + new maps use :func:`encrypt_restore_payload` (version 3).""" + + if not password or len(password) < MIN_PASSWORD_LENGTH: + raise ValueError("La password della mappa deve avere almeno 12 caratteri.") + + from cryptography.fernet import Fernet salt = os.urandom(16) - kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), - length=32, - salt=salt, - iterations=600000, - ) - key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) - token = Fernet(key).encrypt( - json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") - ) - envelope = { + envelope: Dict[str, Any] = { "format": "NullifyPDF encrypted restore map", - "version": 1, - "kdf": "PBKDF2-HMAC-SHA256", - "iterations": 600000, + "version": 2, + "kdf": "scrypt", + "n": SCRYPT_N, + "r": SCRYPT_R, + "p": SCRYPT_P, "salt": base64.b64encode(salt).decode("ascii"), - "token": token.decode("ascii"), } + key = _derive_key(password, envelope, salt) + token = Fernet(key).encrypt( + json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") + ) + envelope["token"] = token.decode("ascii") return json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=2).encode( "utf-8" ) -def decrypt_restore_payload(data: bytes, password: str) -> Dict[str, object]: - """Decrypt an encrypted restore map.""" +def _decrypt_v1v2(envelope: Dict[str, Any], password: str) -> Dict[str, object]: + """Decrypt a version 1 (PBKDF2) or 2 (scrypt) envelope.""" from cryptography.fernet import Fernet - from cryptography.hazmat.primitives import hashes - from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC - envelope = json.loads(data.decode("utf-8")) salt = base64.b64decode(envelope["salt"]) - kdf = PBKDF2HMAC( - algorithm=hashes.SHA256(), - length=32, - salt=salt, - iterations=int(envelope["iterations"]), - ) - key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) + key = _derive_key(password, envelope, salt) plaintext = Fernet(key).decrypt(envelope["token"].encode("ascii")) return json.loads(plaintext.decode("utf-8")) + + +# --------------------------------------------------------------------------- +# Envelope version 3: data key + key wrapping (password, optional recovery key) +# --------------------------------------------------------------------------- + +MAP_FORMAT = "NullifyPDF encrypted restore map" +KEY_EXPORT_FORMAT = "NullifyPDF pseudonym key" +ARGON2_T = 3 +ARGON2_M_KIB = 64 * 1024 +ARGON2_P = 4 +_PAYLOAD_AAD = b"NullifyPDF envelope v3 payload" +_RECOVERY_PREFIX = "NPDF" +_B32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567" + + +def _b64(b: bytes) -> str: + return base64.b64encode(b).decode("ascii") + + +def _unb64(s_: str) -> bytes: + return base64.b64decode(s_.encode("ascii")) + + +def _argon2_available() -> bool: + try: + from cryptography.hazmat.primitives.kdf.argon2 import Argon2id # noqa: F401 + + return True + except Exception: + return False + + +def _password_kek(password: str, wrap: Dict[str, Any]) -> bytes: + """Derive the key-encryption key for a password wrap (bounds-checked).""" + salt = _unb64(wrap["salt"]) + if len(salt) < 16: + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + kdf = str(wrap.get("kdf")) + if kdf == "argon2id": + from cryptography.hazmat.primitives.kdf.argon2 import Argon2id + + t, m, p_ = int(wrap["t"]), int(wrap["m"]), int(wrap["p"]) + if not (1 <= t <= 10 and 19 * 1024 <= m <= 1024 * 1024 and 1 <= p_ <= 8 and m >= 8 * p_): + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + return Argon2id( + salt=salt, length=32, iterations=t, lanes=p_, memory_cost=m + ).derive(password.encode("utf-8")) + if kdf == "scrypt": + from cryptography.hazmat.primitives.kdf.scrypt import Scrypt + + n, r, p_ = int(wrap["n"]), int(wrap["r"]), int(wrap["p"]) + if ( + not (2 ** 14 <= n <= SCRYPT_MAX_N) or n & (n - 1) + or not (1 <= r <= 16) or not (1 <= p_ <= 4) + ): + raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.") + return Scrypt(salt=salt, length=32, n=n, r=r, p=p_).derive(password.encode("utf-8")) + raise ValueError("Algoritmo KDF della mappa non supportato.") + + +def _new_password_wrap(password: str, dek: bytes) -> Dict[str, Any]: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + salt = os.urandom(16) + wrap: Dict[str, Any] = {"type": "password", "salt": _b64(salt)} + if _argon2_available(): + wrap.update(kdf="argon2id", t=ARGON2_T, m=ARGON2_M_KIB, p=ARGON2_P) + try: + kek = _password_kek(password, wrap) + except Exception: # OpenSSL without Argon2: fall back for writing only + wrap = {"type": "password", "salt": _b64(salt)} + wrap.update(kdf="scrypt", n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P) + kek = _password_kek(password, wrap) + else: + wrap.update(kdf="scrypt", n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P) + kek = _password_kek(password, wrap) + nonce = os.urandom(12) + wrap["nonce"] = _b64(nonce) + wrap["wrapped"] = _b64(AESGCM(kek).encrypt(nonce, dek, b"NullifyPDF wrap v3:password")) + return wrap + + +def _recovery_kek(recovery_key: bytes, salt: bytes) -> bytes: + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.hkdf import HKDF + + return HKDF( + algorithm=hashes.SHA256(), length=32, salt=salt, info=b"nullifypdf-recovery" + ).derive(recovery_key) + + +def _new_recovery_wrap(recovery_key: bytes, dek: bytes) -> Dict[str, Any]: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + if len(recovery_key) != 32: + raise ValueError("Chiave di recupero non valida.") + salt, nonce = os.urandom(16), os.urandom(12) + kek = _recovery_kek(recovery_key, salt) + return { + "type": "recovery", "kdf": "hkdf-sha256", "salt": _b64(salt), + "nonce": _b64(nonce), + "wrapped": _b64(AESGCM(kek).encrypt(nonce, dek, b"NullifyPDF wrap v3:recovery")), + "key_id": hashlib.sha256(recovery_key).hexdigest()[:8], + } + + +def generate_recovery_key() -> Tuple[bytes, str]: + """A fresh 256-bit recovery key and its display form (shown once). + + Display: ``NPDF-XXXX-...`` in base32 groups of four with a 2-character + checksum, so typing mistakes are caught before any decryption. + """ + raw = os.urandom(32) + return raw, format_recovery_key(raw) + + +def _checksum(raw: bytes) -> str: + v = int.from_bytes(hashlib.sha256(raw).digest()[:2], "big") & 0x3FF + return _B32[v >> 5] + _B32[v & 31] + + +def format_recovery_key(raw: bytes) -> str: + body = base64.b32encode(raw).decode("ascii").rstrip("=") + _checksum(raw) + groups = [body[i:i + 4] for i in range(0, len(body), 4)] + return _RECOVERY_PREFIX + "-" + "-".join(groups) + + +def parse_recovery_key(display: str) -> bytes: + """Parse a displayed recovery key; raises ValueError on any typo.""" + text = "".join(display.upper().split()).replace("-", "") + if text.startswith(_RECOVERY_PREFIX): + text = text[len(_RECOVERY_PREFIX):] + text = text.replace("0", "O").replace("1", "I") + if len(text) != 54 or any(c not in _B32 for c in text): + raise ValueError("Chiave di recupero non valida.") + body, check = text[:52], text[52:] + try: + raw = base64.b32decode(body + "====") + except Exception as e: + raise ValueError("Chiave di recupero non valida.") from e + if len(raw) != 32 or _checksum(raw) != check: + raise ValueError("Chiave di recupero non valida: controlla di averla digitata bene.") + return raw + + +def _encrypt_v3( + payload: Dict[str, object], password: str, recovery_key: Optional[bytes], fmt: str +) -> bytes: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + dek = os.urandom(32) + nonce = os.urandom(12) + ciphertext = AESGCM(dek).encrypt( + nonce, + json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8"), + _PAYLOAD_AAD, + ) + wraps = [_new_password_wrap(password, dek)] + if recovery_key is not None: + wraps.append(_new_recovery_wrap(recovery_key, dek)) + envelope = { + "format": fmt, "version": 3, "cipher": "AES-256-GCM", + "nonce": _b64(nonce), "ciphertext": _b64(ciphertext), "wraps": wraps, + } + return json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=2).encode("utf-8") + + +def _decrypt_v3( + envelope: Dict[str, Any], password: Optional[str], recovery_key: Optional[bytes] +) -> Dict[str, object]: + from cryptography.exceptions import InvalidTag + from cryptography.fernet import InvalidToken + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + dek: Optional[bytes] = None + for wrap in envelope.get("wraps", []): + try: + if wrap.get("type") == "password" and password is not None: + kek = _password_kek(password, wrap) + aad = b"NullifyPDF wrap v3:password" + elif wrap.get("type") == "recovery" and recovery_key is not None: + kek = _recovery_kek(recovery_key, _unb64(wrap["salt"])) + aad = b"NullifyPDF wrap v3:recovery" + else: + continue + dek = AESGCM(kek).decrypt(_unb64(wrap["nonce"]), _unb64(wrap["wrapped"]), aad) + break + except InvalidTag: + continue + if dek is None: + raise InvalidToken() # same signal the callers already handle + try: + plaintext = AESGCM(dek).decrypt( + _unb64(envelope["nonce"]), _unb64(envelope["ciphertext"]), _PAYLOAD_AAD + ) + except InvalidTag as e: + raise InvalidToken() from e + return json.loads(plaintext.decode("utf-8")) + + +def encrypt_restore_payload( + payload: Dict[str, object], + password: str, + *, + recovery_key: Optional[bytes] = None, +) -> bytes: + """Encrypt a restore-map payload (envelope version 3). + + A random data key encrypts the payload (AES-256-GCM); it is wrapped by a + key derived from the password (Argon2id, scrypt if unavailable) and, if + given, by the recovery key. Maps from older versions remain readable. + """ + if not password or len(password) < MIN_PASSWORD_LENGTH: + raise ValueError("La password della mappa deve avere almeno 12 caratteri.") + return _encrypt_v3(payload, password, recovery_key, MAP_FORMAT) + + +def decrypt_restore_payload( + data: bytes, + password: Optional[str] = None, + *, + recovery_key: Optional[bytes] = None, +) -> Dict[str, object]: + """Decrypt an encrypted restore map (versions 1, 2 and 3).""" + envelope = json.loads(data.decode("utf-8")) + if not isinstance(envelope, dict) or envelope.get("format") != MAP_FORMAT: + raise ValueError("File non riconosciuto come mappa di ripristino cifrata.") + version = envelope.get("version") + if version in (1, 2): + if password is None: + raise ValueError("Questa mappa si apre solo con la password.") + return _decrypt_v1v2(envelope, password) + if version == 3: + return _decrypt_v3(envelope, password, recovery_key) + raise ValueError("Versione della mappa non supportata: aggiorna NullifyPDF.") + + +def export_pseudonym_key(key: bytes, password: str) -> bytes: + """Password-protected copy of the pseudonym key, for another computer.""" + if len(key) != 32: + raise ValueError("Chiave dei segnaposto non valida.") + return _encrypt_v3({"key": _b64(key)}, password, None, KEY_EXPORT_FORMAT) + + +def import_pseudonym_key(data: bytes, password: str) -> bytes: + envelope = json.loads(data.decode("utf-8")) + if not isinstance(envelope, dict) or envelope.get("format") != KEY_EXPORT_FORMAT: + raise ValueError("File non riconosciuto come chiave dei segnaposto.") + if envelope.get("version") != 3: + raise ValueError("Versione della chiave non supportata: aggiorna NullifyPDF.") + key = _unb64(str(_decrypt_v3(envelope, password, None)["key"])) + if len(key) != 32: + raise ValueError("Chiave dei segnaposto non valida.") + return key diff --git a/review_dialog.py b/review_dialog.py new file mode 100644 index 0000000..186e129 --- /dev/null +++ b/review_dialog.py @@ -0,0 +1,183 @@ +"""Review dialog for AI suggestions (Qt). + +Detections the evidence score could not settle are *suggestions*: they are +never applied silently and never lost silently. The user accepts or rejects +each one here; values are masked until the user asks to see them. +""" + +from __future__ import annotations + +from typing import Any, Dict, List + +from PySide6.QtCore import Qt, Signal +from PySide6.QtWidgets import ( + QAbstractItemView, + QCheckBox, + QDialog, + QDoubleSpinBox, + QHBoxLayout, + QHeaderView, + QLabel, + QPushButton, + QTableWidget, + QTableWidgetItem, + QVBoxLayout, +) + +REASON_LABELS: Dict[str, str] = { + "context-cue": "contesto", + "name-shape": "forma di nome", + "given-name": "nome proprio noto", + "known-place": "luogo noto", + "province-or-postcode": "provincia/CAP", + "repeated": "ripetuto", + "lexicon": "parola comune", + "verb-form": "forma verbale", + "line-start": "inizio riga", + "all-caps": "maiuscolo", + "followed-by-function-word": "seguito da che/di", + "low-confidence": "bassa confidenza", + "signature-anchor": "vicino a firma/timbro", + "overlaps-text": "il riquadro tocca del testo", + "curve-ink": "tratti a mano", + "raster-stamp": "timbro/immagine", + "no-context": "senza contesto", + "no-anchor": "senza riferimento", + "postcode": "CAP", + "known-surname": "cognome noto", + "ambiguous-place": "luogo omonimo di parola comune", +} + +COL_APPLY, COL_PAGE, COL_VALUE, COL_TYPE, COL_SCORE, COL_WHY = range(6) + + +def mask_value(value: str) -> str: + v = " ".join(value.split()) + if len(v) <= 1: + return "•" + return v[0] + "•" * min(len(v) - 1, 8) + + +class ReviewDialog(QDialog): + """Table of suggestions with per-row accept / reject.""" + + navigate = Signal(int) + highlight = Signal(object) # the suggestion dict of the selected row + + def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> None: + super().__init__(parent) + self.setWindowTitle("Revisione suggerimenti") + self.resize(760, 460) + self._items = list(suggestions) + self._applied: List[Dict[str, Any]] = [] + self._rejected: List[Dict[str, Any]] = [] + lay = QVBoxLayout(self) + lay.addWidget(QLabel( + "Questi elementi non hanno prove sufficienti per essere censurati " + "automaticamente. Seleziona quelli da censurare: i non selezionati " + "vengono scartati se premi «Applica selezionati»." + )) + self.table = QTableWidget(len(self._items), 6, self) + self.table.setHorizontalHeaderLabels( + ["Censura", "Pag.", "Valore", "Tipo", "Punteggio", "Motivi"] + ) + self.table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) + self.table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) + self.table.horizontalHeader().setSectionResizeMode( + COL_WHY, QHeaderView.ResizeMode.Stretch + ) + lay.addWidget(self.table) + self.chk_show = QCheckBox("Mostra i valori") + self.chk_show.toggled.connect(self._refresh_values) + lay.addWidget(self.chk_show) + for row, it in enumerate(self._items): + apply_item = QTableWidgetItem() + apply_item.setFlags( + Qt.ItemFlag.ItemIsUserCheckable | Qt.ItemFlag.ItemIsEnabled + ) + apply_item.setCheckState(Qt.CheckState.Unchecked) + self.table.setItem(row, COL_APPLY, apply_item) + self.table.setItem(row, COL_PAGE, QTableWidgetItem(str(int(it.get("page", 0)) + 1))) + self.table.setItem(row, COL_VALUE, QTableWidgetItem("")) + self.table.setItem(row, COL_TYPE, QTableWidgetItem(str(it.get("entity_type", "")))) + self.table.setItem(row, COL_SCORE, QTableWidgetItem(f"{float(it.get('score', 0.0)):.2f}")) + why = ", ".join(REASON_LABELS.get(r, r) for r in it.get("reasons", [])) + self.table.setItem(row, COL_WHY, QTableWidgetItem(why)) + self._refresh_values() + self.table.cellDoubleClicked.connect(self._on_double_click) + self.table.currentCellChanged.connect(self._on_row_changed) + + bar = QHBoxLayout() + btn_all = QPushButton("Seleziona tutti") + btn_none = QPushButton("Nessuno") + self.spin = QDoubleSpinBox() + self.spin.setRange(0.0, 1.0) + self.spin.setSingleStep(0.05) + self.spin.setValue(0.6) + btn_thr = QPushButton("Seleziona ≥ soglia") + btn_all.clicked.connect(lambda: self._set_all(True)) + btn_none.clicked.connect(lambda: self._set_all(False)) + btn_thr.clicked.connect(self._select_threshold) + for w in (btn_all, btn_none, self.spin, btn_thr): + bar.addWidget(w) + bar.addStretch(1) + lay.addLayout(bar) + + bottom = QHBoxLayout() + self.btn_apply = QPushButton("Applica selezionati") + self.btn_apply.setObjectName("Danger") + self.btn_later = QPushButton("Rivedi più tardi") + self.btn_apply.clicked.connect(self._on_apply) + self.btn_later.clicked.connect(self.reject) + bottom.addStretch(1) + bottom.addWidget(self.btn_later) + bottom.addWidget(self.btn_apply) + lay.addLayout(bottom) + + # -- helpers --------------------------------------------------------- + def _cell(self, row: int, col: int) -> QTableWidgetItem: + item = self.table.item(row, col) + assert item is not None + return item + + def _refresh_values(self) -> None: + show = self.chk_show.isChecked() + for row, it in enumerate(self._items): + text = " ".join(str(it.get("text", "")).split()) + if not text: + self._cell(row, COL_VALUE).setText("(area grafica)") + continue + self._cell(row, COL_VALUE).setText(text if show else mask_value(text)) + + def _set_all(self, state: bool) -> None: + for row in range(self.table.rowCount()): + self._cell(row, COL_APPLY).setCheckState( + Qt.CheckState.Checked if state else Qt.CheckState.Unchecked + ) + + def _select_threshold(self) -> None: + thr = self.spin.value() + for row, it in enumerate(self._items): + if float(it.get("score", 0.0)) >= thr: + self._cell(row, COL_APPLY).setCheckState(Qt.CheckState.Checked) + + def _on_row_changed(self, row: int, _c: int, _pr: int, _pc: int) -> None: + if 0 <= row < len(self._items): + self.highlight.emit(self._items[row]) + + def _on_double_click(self, row: int, _col: int) -> None: + self.navigate.emit(int(self._items[row].get("page", 0))) + + def _on_apply(self) -> None: + self._applied, self._rejected = [], [] + for row, it in enumerate(self._items): + checked = self._cell(row, COL_APPLY).checkState() == Qt.CheckState.Checked + (self._applied if checked else self._rejected).append(it) + self.accept() + + # -- results --------------------------------------------------------- + def applied(self) -> List[Dict[str, Any]]: + return list(self._applied) + + def declined(self) -> List[Dict[str, Any]]: + return list(self._rejected) diff --git a/scripts/build_lexicon_data.py b/scripts/build_lexicon_data.py new file mode 100644 index 0000000..b00b349 --- /dev/null +++ b/scripts/build_lexicon_data.py @@ -0,0 +1,141 @@ +"""Generate ``lexicon_data_it.py`` from ISTAT open data (development only). + +Not part of the runtime and not imported by the application. It reads the +ISTAT "Elenco comuni italiani" workbook (CC BY 4.0, https://www.istat.it/note-legali/), +normalises the place names, marks the ones that are also ordinary words as +ambiguous, and writes a plain Python module that PyInstaller collects like any +other import. Only the standard library is needed to read the workbook. + + python scripts/build_lexicon_data.py [--istat-file FILE] [--out FILE] + +The generated module only *confirms* PERSON/LOCATION hits; a name missing +from it never rejects one. +""" + +from __future__ import annotations + +import argparse +import hashlib +import pathlib +import re +import sys +import urllib.request +import xml.etree.ElementTree as ET +import zipfile +from typing import Dict, List, Set + +ISTAT_URL = "https://www.istat.it/storage/codici-unita-amministrative/Elenco-comuni-italiani.xlsx" +NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} +COLUMNS = { + "Denominazione in italiano": "it", + "Denominazione altra lingua": "alt", + "Sigla automobilistica": "sigla", + "Denominazione Regione": "region", +} +ROOT = pathlib.Path(__file__).resolve().parent.parent + + +def _col(ref: str) -> str: + return re.match(r"[A-Z]+", ref).group() # type: ignore[union-attr] + + +def read_rows(path: pathlib.Path) -> "tuple[List[str], List[Dict[str, str]]]": + """First worksheet of an .xlsx as a list of header->value dicts.""" + z = zipfile.ZipFile(path) + sst = [ + "".join(t.text or "" for t in si.iter("{%s}t" % NS["m"])) + for si in ET.fromstring(z.read("xl/sharedStrings.xml")).findall("m:si", NS) + ] if "xl/sharedStrings.xml" in z.namelist() else [] + sheet = ET.fromstring(z.read("xl/worksheets/sheet1.xml")) + rows = sheet.findall(".//m:row", NS) + headers: Dict[str, str] = {} + out: List[Dict[str, str]] = [] + for i, row in enumerate(rows): + cells: Dict[str, str] = {} + for c in row.findall("m:c", NS): + v = c.find("m:v", NS) + if v is None or v.text is None: + continue + cells[_col(c.get("r", ""))] = sst[int(v.text)] if c.get("t") == "s" else v.text + if i == 0: + headers = {col: " ".join(val.split()) for col, val in cells.items()} + continue + out.append({headers[col]: val for col, val in cells.items() if col in headers}) + return list(headers.values()), out + + +def normalise(name: str) -> str: + return " ".join(name.replace("’", "'").lower().split()) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--istat-file", type=pathlib.Path) + ap.add_argument("--out", type=pathlib.Path, default=ROOT / "lexicon_data_it.py") + args = ap.parse_args() + src = args.istat_file + if src is None: + src = pathlib.Path("Elenco-comuni-italiani.xlsx") + req = urllib.request.Request(ISTAT_URL, headers={"User-Agent": "NullifyPDF-build/1.0"}) + src.write_bytes(urllib.request.urlopen(req, timeout=120).read()) + sha = hashlib.sha256(src.read_bytes()).hexdigest() + headers, rows = read_rows(src) + missing = set(COLUMNS) - set(headers) + if missing: + print(f"Workbook layout changed, missing columns: {sorted(missing)}", file=sys.stderr) + return 1 + + comuni: Set[str] = set() + sigle: Set[str] = set() + regions: Set[str] = set() + for r in rows: + for key in ("Denominazione in italiano", "Denominazione altra lingua"): + for part in re.split(r"\s*/\s*", r.get(key, "")): + n = normalise(part) + if len(n) >= 2: + comuni.add(n) + if r.get("Sigla automobilistica"): + sigle.add(r["Sigla automobilistica"].strip().upper()) + for part in re.split(r"\s*/\s*", r.get("Denominazione Regione", "")): + if part.strip(): + regions.add(normalise(part)) + + sys.path.insert(0, str(ROOT)) + import lexicon_it + + common = set(lexicon_it.COMMON_WORDS | lexicon_it.FUNCTION_WORDS | lexicon_it.GIVEN_NAMES + | lexicon_it.CONFLICT_NAMES) + vocab_words: Set[str] = set() + try: + import spacy + + nlp = spacy.load("it_core_news_md") + vocab_words = {w for w in comuni if " " not in w and nlp.vocab.has_vector(w)} + except Exception as e: # model optional for the generator + print(f"spaCy vocabulary not used: {e}", file=sys.stderr) + ambiguous = {w for w in comuni if len(w) <= 3 or w in common or w in vocab_words} + + def lit(items: Set[str]) -> str: + body = "|".join(sorted(items)) + assert '"""' not in body and "\\" not in body + return 'frozenset("""' + body + '""".split("|"))' + + text = ( + "# Generated by scripts/build_lexicon_data.py - do not edit.\n" + "# Contains ISTAT data (Elenco comuni italiani, " + src.name + "), licensed\n" + "# CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/); modified: names\n" + "# normalised, lower-cased, bilingual names split.\n" + f"SOURCE_SHA256 = {{'istat_comuni': '{sha}'}}\n" + f"COMUNI = {lit(comuni)}\n" + f"COMUNI_AMBIGUOUS = {lit(ambiguous)}\n" + f"PROVINCE_SIGLE = {lit(sigle)}\n" + f"REGIONS = {lit(regions)}\n" + ) + args.out.write_text(text, encoding="utf-8") + print(f"{len(comuni)} comuni ({len(ambiguous)} ambiguous), {len(sigle)} province codes, " + f"{len(regions)} regions -> {args.out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/secure_store.py b/secure_store.py new file mode 100644 index 0000000..bfc5c0b --- /dev/null +++ b/secure_store.py @@ -0,0 +1,304 @@ +"""Protection of local secrets (Qt-free). + +A single random master key protects the word lists and the pseudonym key. +The master key itself is kept by the best protector available: + +* Windows: DPAPI (``CryptProtectData``, via ctypes - no extra dependency); + bound to the Windows account. +* macOS / Linux: the OS keyring when the optional ``keyring`` package is + installed and a real backend is active. +* Otherwise a key file (``keys/master.key``, mode 0600 where supported), + which only protects against copying a single data file on its own, not + against someone who can read the user's profile. + +Data is sealed with AES-256-GCM. If the stored master key cannot be +recovered (profile moved to another account or machine) nothing is deleted or +overwritten: :class:`KeystoreUnavailable` is raised and callers degrade to +"no data" while leaving the encrypted files untouched. +""" + +from __future__ import annotations + +import base64 +import json +import os +import pathlib +import stat +import sys +from typing import Any, Dict, Optional, Protocol + +KEYSTORE_FORMAT = "NullifyPDF keystore" +_DPAPI_ENTROPY = b"NullifyPDF/secure_store/v1" + + +class KeystoreUnavailable(RuntimeError): + """The master key cannot be recovered on this account / machine.""" + + +class SealError(ValueError): + """Sealed data is corrupt, tampered with, or for another context.""" + + +# ----------------------------------------------------------------- sealing + +def seal(plaintext: bytes, key: bytes, aad: bytes) -> bytes: + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + nonce = os.urandom(12) + return nonce + AESGCM(key).encrypt(nonce, plaintext, aad) + + +def open_sealed(blob: bytes, key: bytes, aad: bytes) -> bytes: + from cryptography.exceptions import InvalidTag + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + if len(blob) < 12 + 16: + raise SealError("Dati cifrati troncati.") + try: + return AESGCM(key).decrypt(blob[:12], blob[12:], aad) + except InvalidTag as e: + raise SealError("Dati cifrati non validi o modificati.") from e + + +# -------------------------------------------------------------- protectors + +class KeyProtector(Protocol): + name: str + + def wrap(self, key: bytes) -> Dict[str, Any]: ... + + def unwrap(self, blob: Dict[str, Any]) -> bytes: ... + + +class DpapiProtector: + name = "dpapi" + + @staticmethod + def available() -> bool: + return sys.platform == "win32" + + def _call(self, data: bytes, protect: bool) -> bytes: + import ctypes + from ctypes import wintypes + + class DATA_BLOB(ctypes.Structure): + _fields_ = [("cbData", wintypes.DWORD), ("pbData", ctypes.POINTER(ctypes.c_char))] + + def blob_of(b: bytes) -> "tuple[DATA_BLOB, Any]": + buf = ctypes.create_string_buffer(b, len(b)) + return DATA_BLOB(len(b), ctypes.cast(buf, ctypes.POINTER(ctypes.c_char))), buf + + crypt32 = ctypes.windll.crypt32 # type: ignore[attr-defined] + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + kernel32.LocalFree.argtypes = [ctypes.c_void_p] + in_blob, keep1 = blob_of(data) + ent_blob, keep2 = blob_of(_DPAPI_ENTROPY) + out_blob = DATA_BLOB() + CRYPTPROTECT_UI_FORBIDDEN = 0x1 + if protect: + ok = crypt32.CryptProtectData( + ctypes.byref(in_blob), "NullifyPDF", ctypes.byref(ent_blob), + None, None, CRYPTPROTECT_UI_FORBIDDEN, ctypes.byref(out_blob), + ) + else: + ok = crypt32.CryptUnprotectData( + ctypes.byref(in_blob), None, ctypes.byref(ent_blob), + None, None, CRYPTPROTECT_UI_FORBIDDEN, ctypes.byref(out_blob), + ) + del keep1, keep2 + if not ok: + raise OSError(ctypes.WinError()) # type: ignore[attr-defined] + try: + return ctypes.string_at(out_blob.pbData, out_blob.cbData) + finally: + kernel32.LocalFree(out_blob.pbData) + + def wrap(self, key: bytes) -> Dict[str, Any]: + return { + "protector": self.name, + "blob": base64.b64encode(self._call(key, True)).decode("ascii"), + } + + def unwrap(self, blob: Dict[str, Any]) -> bytes: + return self._call(base64.b64decode(blob["blob"]), False) + + +class KeyringProtector: + name = "keyring" + SERVICE = "NullifyPDF" + USER = "master" + + @staticmethod + def available() -> bool: + try: + import keyring # type: ignore[import-not-found] + from keyring.backends import fail, null # type: ignore[import-not-found] + + backend = keyring.get_keyring() + return not isinstance(backend, (fail.Keyring, null.Keyring)) + except Exception: + return False + + def wrap(self, key: bytes) -> Dict[str, Any]: + import keyring # type: ignore[import-not-found] + + keyring.set_password(self.SERVICE, self.USER, base64.b64encode(key).decode("ascii")) + return {"protector": self.name, "service": self.SERVICE, "user": self.USER} + + def unwrap(self, blob: Dict[str, Any]) -> bytes: + import keyring # type: ignore[import-not-found] + + secret = keyring.get_password(blob.get("service", self.SERVICE), blob.get("user", self.USER)) + if not secret: + raise OSError("Segreto non presente nel portachiavi.") + return base64.b64decode(secret) + + +class KeyFileProtector: + name = "keyfile" + + def __init__(self, config_dir: pathlib.Path) -> None: + self.path = pathlib.Path(config_dir) / "keys" / "master.key" + + def wrap(self, key: bytes) -> Dict[str, Any]: + self.path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) + fd = os.open(str(self.path), flags, 0o600) + with os.fdopen(fd, "wb") as fh: + fh.write(key) + try: + os.chmod(self.path, stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + return {"protector": self.name, "path": "keys/master.key"} + + def unwrap(self, blob: Dict[str, Any]) -> bytes: + return self.path.read_bytes() + + +# ---------------------------------------------------------------- keystore + +def _keystore_path(config_dir: pathlib.Path) -> pathlib.Path: + return pathlib.Path(config_dir) / "keystore.json" + + +def _protectors(config_dir: pathlib.Path) -> Dict[str, Any]: + out: Dict[str, Any] = {"keyfile": KeyFileProtector(config_dir)} + if DpapiProtector.available(): + out["dpapi"] = DpapiProtector() + if KeyringProtector.available(): + out["keyring"] = KeyringProtector() + return out + + +_CACHE: Dict[str, bytes] = {} + + +def forget_cached_keys() -> None: + _CACHE.clear() + + +def protector_label(config_dir: pathlib.Path) -> str: + """Human-readable (Italian) description of how the master key is kept.""" + try: + data = json.loads(_keystore_path(config_dir).read_text(encoding="utf-8")) + name = data["master"]["protector"] + except Exception: + name = None + return { + "dpapi": "Archivio protetto: Windows DPAPI", + "keyring": "Archivio protetto: portachiavi di sistema", + "keyfile": "File chiave locale (protezione limitata)", + }.get(name or "", "Non ancora creato") + + +def load_or_create_master_key(config_dir: pathlib.Path) -> bytes: + """Return the 32-byte master key, creating the keystore on first use. + + Raises :class:`KeystoreUnavailable` when a keystore exists but cannot be + unwrapped on this account. Never overwrites an existing keystore. + """ + cfg = pathlib.Path(config_dir) + cache_key = str(cfg.resolve()) + if cache_key in _CACHE: + return _CACHE[cache_key] + ks = _keystore_path(cfg) + protectors = _protectors(cfg) + if ks.exists(): + try: + data = json.loads(ks.read_text(encoding="utf-8")) + master = data["master"] + prot = protectors.get(master["protector"]) + if prot is None: + raise KeystoreUnavailable( + f"Protezione '{master['protector']}' non disponibile su questo sistema." + ) + key = prot.unwrap(master) + except KeystoreUnavailable: + raise + except Exception as e: + raise KeystoreUnavailable( + "Chiave dei dati locali non leggibile su questo account/PC." + ) from e + if len(key) != 32: + raise KeystoreUnavailable("Chiave dei dati locali non valida.") + _CACHE[cache_key] = key + return key + key = os.urandom(32) + errors = [] + for name in ("dpapi", "keyring", "keyfile"): + prot = protectors.get(name) + if prot is None: + continue + try: + master = prot.wrap(key) + if prot.unwrap(master) != key: # round trip before trusting it + raise OSError("round trip mismatch") + except Exception as e: + errors.append(f"{name}: {e}") + continue + cfg.mkdir(parents=True, exist_ok=True) + _write_private(ks, json.dumps( + {"format": KEYSTORE_FORMAT, "version": 1, "master": master}, indent=2 + ).encode("utf-8")) + _CACHE[cache_key] = key + return key + raise KeystoreUnavailable("Nessuna protezione disponibile: " + "; ".join(errors)) + + +def _write_private(path: pathlib.Path, data: bytes) -> None: + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) + fd = os.open(str(path), flags, 0o600) + with os.fdopen(fd, "wb") as fh: + fh.write(data) + try: + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + +def atomic_write(path: pathlib.Path, data: bytes) -> None: + """Write via a temp file in the same folder, then replace (atomic).""" + tmp = path.with_name(path.name + ".part") + _write_private(tmp, data) + os.replace(tmp, path) + + +def secure_delete(path: pathlib.Path) -> None: + """Best effort: overwrite with zeros, truncate, unlink. + + SSD wear levelling, backups and file-system journals may keep copies. + """ + try: + size = path.stat().st_size + with open(path, "r+b") as fh: + fh.write(b"\x00" * size) + fh.flush() + os.fsync(fh.fileno()) + fh.truncate(0) + except OSError: + pass + try: + path.unlink() + except OSError: + pass diff --git a/tests/_pdfgen.py b/tests/_pdfgen.py new file mode 100644 index 0000000..180fc6d --- /dev/null +++ b/tests/_pdfgen.py @@ -0,0 +1,38 @@ +"""Synthetic PDF builders shared by the tests (no binary fixtures).""" + +from types import SimpleNamespace + +import pymupdf + + +def make_doc(lines, pitch=14.0, size=11.0, x=72.0, y0=100.0, fontname="helv"): + doc = pymupdf.open() + page = doc.new_page() + for i, text in enumerate(lines): + page.insert_text((x, y0 + i * pitch), text, fontsize=size, fontname=fontname) + return doc + + +def words_of(page, **kw): + return page.get_text("words", **kw) + + +def res(entity_type, text, value, score=0.85): + """Presidio-like result for the first occurrence of `value` in `text`.""" + start = text.index(value) + return SimpleNamespace( + entity_type=entity_type, start=start, end=start + len(value), score=score + ) + + +def fake_words(text): + """`get_text("words")`-style tuples for plain text (one block).""" + out = [] + y = 0.0 + for li, line in enumerate(text.split("\n")): + x = 10.0 + for wi, w in enumerate(line.split()): + out.append((x, y, x + 6 * len(w), y + 12, w, 0, li, wi)) + x += 6 * len(w) + 4 + y += 14 + return out diff --git a/tests/test_build_config.py b/tests/test_build_config.py index 7d12a83..20051d4 100644 --- a/tests/test_build_config.py +++ b/tests/test_build_config.py @@ -117,3 +117,17 @@ def test_build_workflows_use_the_shared_linux_build(name): assert "uses: ./.github/workflows/build-linux.yml" in text # Linux must not be built on the old ubuntu-22.04 runner image. assert "ubuntu-22.04" not in text + + +def test_mypy_module_lists_are_identical_across_workflows(): + import pathlib + import re + + root = pathlib.Path(__file__).resolve().parent.parent / ".github" / "workflows" + sets = {} + for name in ("beta-release.yml", "build-linux.yml", "release.yml", "test_build.yml"): + text = (root / name).read_text(encoding="utf-8") + m = re.search(r"run: mypy ([^\n]+)", text) + assert m, name + sets[name] = frozenset(m.group(1).split()) + assert len(set(sets.values())) == 1, sets diff --git a/tests/test_detection_pipeline.py b/tests/test_detection_pipeline.py new file mode 100644 index 0000000..18eec38 --- /dev/null +++ b/tests/test_detection_pipeline.py @@ -0,0 +1,454 @@ +"""Italian precision, pattern validation and allowlist semantics.""" + +import pytest + +import pii_detection as pd +from _pdfgen import fake_words, res +import lexicon_it + + +def _run(text, results, enabled=None, **kw): + idx = pd.build_page_index(fake_words(text), clamp=False) + enabled = enabled or pd.expand_group_ids(pd.ALL_GROUP_IDS) + cands, stats = pd.classify_page(results, idx, enabled, **kw) + return idx, cands, stats + + +# --- lexicon ----------------------------------------------------------------- + +@pytest.mark.parametrize("word", [ + "Certifico", "Prescrivo", "Dichiaro", "Attesto", "Rilascio", "Confermo", + "Raccomando", "Ordina", "Anamnesi", "Diagnosi", "Terapia", "Paziente", + "PRESCRIVO", "Comune", "Tribunale", +]) +def test_ordinary_words_are_recognised(word): + assert lexicon_it.is_common_word(word) + + +@pytest.mark.parametrize("word", ["Rossi", "Esposito", "Mario", "Napoli", "Rosa", "Bruno"]) +def test_names_are_not_common_words(word): + assert not lexicon_it.is_common_word(word) + + +def test_verb_forms_are_generated_from_stems(): + forms = lexicon_it.verb_forms() + assert {"certifica", "certificano", "prescrive", "prescrivono", "dichiara"} <= forms + + +# --- fake-analyzer pipeline (no model needed) ----------------------------------- + +def test_lone_verb_tagged_as_location_is_dropped(): + text = "Certifico\nPrescrivo\nRaccomando" + results = [res("LOCATION", text, w) for w in ("Certifico", "Prescrivo", "Raccomando")] + _idx, cands, stats = _run(text, results) + assert cands == [] and stats.get("lexicon", 0) == 3 + + +def test_verb_plus_drug_keeps_only_the_non_verb(): + text = "Visita del 3 aprile Prescrivo Augmentin" + r = res("LOCATION", text, "Prescrivo Augmentin") + _idx, cands, _ = _run(text, [r]) + assert all("Prescrivo" not in c.text for c in cands) + + +def test_name_with_context_is_accepted_and_scored_from_evidence(): + text = "Il Sig. Mario Rossi vive a Roma" + results = [res("PERSON", text, "Mario Rossi"), res("LOCATION", text, "Roma")] + _idx, cands, _ = _run(text, results) + by_type = {c.entity_type: c for c in cands} + assert by_type["PERSON"].state == "accepted" and by_type["PERSON"].score >= 0.7 + assert by_type["LOCATION"].state == "accepted" + + +def test_ambiguous_single_word_becomes_a_suggestion_not_a_drop(): + text = "Il referto indica che Colombi era presente" + r = res("PERSON", text, "Colombi") + _idx, cands, _ = _run(text, [r]) + assert len(cands) == 1 and cands[0].state == "suggested" + + +def test_health_mode_never_drops_silently(): + text = "Certifico" + r = res("LOCATION", text, "Certifico") + # Lexicon edges are a hard rule; use an uncertain non-lexicon word instead. + text2 = "Noi Zorzi andiamo" + r2 = res("PERSON", text2, "Zorzi") + _i, normal, _ = _run(text2, [r2]) + _i, health, _ = _run(text2, [r2], health_mode=True) + assert len(health) >= len(normal) and all(c.state != "dropped" for c in health) + + +def test_split_lines_of_one_name_are_merged(): + text = "Esposito\nGiovanni abita a Napoli" + results = [res("PERSON", text, "Esposito"), res("PERSON", text, "Giovanni")] + _idx, cands, _ = _run(text, results) + person = [c for c in cands if c.entity_type == "PERSON"] + assert len(person) == 1 and person[0].text.replace("\n", " ") == "Esposito Giovanni" + assert person[0].state == "accepted" + + +def test_field_label_is_rejected_outright(): + text = "Nome: Mario" + r = res("PERSON", text, "Nome") + _idx, cands, _ = _run(text, [r]) + assert cands == [] + + +def test_fragments_and_ocr_noise_are_rejected(): + text = "Rnario Ro" + _i, normal, _ = _run(text, [res("PERSON", text, "Rnario")], used_ocr=True) + assert normal == [] + kept, n = pd.reject_fragment_hits( + [pd.Candidate(0, 4, "Li", "PERSON", 0.85)], used_ocr=False + ) + assert kept == [] and n == 1 + + +# --- pattern validation -------------------------------------------------------- + +def _pattern(text, etype, value, score=1.0): + start = text.index(value) + c = pd.Candidate(start, start + len(value), value, etype, score) + return pd.refine_pattern_hit(c, text) + + +def test_fiscal_code_needs_boundaries_digits_or_context(): + ok, _ = _pattern("Codice fiscale: RSSMRA80A01H501U fine", "IT_FISCAL_CODE", "RSSMRA80A01H501U") + assert ok + ok, why = _pattern("xxRSSMRA80A01H501Uxx", "IT_FISCAL_CODE", "RSSMRA80A01H501U") + assert not ok and why == "embedded-in-token" + ok, why = _pattern("parola AAAAAAAAAAAAAAAA fine", "IT_FISCAL_CODE", "AAAAAAAAAAAAAAAA") + assert not ok and why == "no-digit-no-context" + + +def test_vat_spaced_digits_need_context(): + ok, why = _pattern("totale 1234 5678 901 euro", "IT_VAT_CODE", "1234 5678 901", 0.1) + assert not ok + ok, _ = _pattern("P.IVA 12345678903 grazie", "IT_VAT_CODE", "12345678903", 1.0) + assert ok + ok, why = _pattern("numero 1234567890312 lungo", "IT_VAT_CODE", "12345678903", 1.0) + assert not ok and why == "embedded-in-digits" + + +@pytest.mark.parametrize("etype,value,ctx_ok,ctx_bad", [ + ("IT_DRIVER_LICENSE", "AB1234567C", "patente di guida n. AB1234567C ok", "codice prodotto AB1234567C ok"), + ("IT_PASSPORT", "YA1234567", "passaporto YA1234567 ok", "ordine YA1234567 ok"), + ("IT_IDENTITY_CARD", "CA12345AB", "carta d'identita CA12345AB ok", "ticket CA12345AB ok"), +]) +def test_document_numbers_need_their_context(etype, value, ctx_ok, ctx_bad): + assert _pattern(ctx_ok, etype, value, 0.4)[0] + assert not _pattern(ctx_bad, etype, value, 0.4)[0] + assert not _pattern(ctx_ok.replace(value, value.lower()), etype, value.lower(), 0.4)[0] + + +@pytest.mark.parametrize("text,value,expect", [ + ("Tel. 340 1234567 fine", "340 1234567", True), + ("chiama +39 02 1234567", "+39 02 1234567", True), + ("Fattura del 12/03/2024 n.", "12/03/2024", False), + ("anni 2018 - 2022 presso", "2018 - 2022", False), + ("ordine 5551234 ok", "5551234", False), + ("codice A340123456789", "340123456789", False), +]) +def test_phone_plausibility(text, value, expect): + assert _pattern(text, "PHONE_NUMBER", value, 0.4)[0] is expect + + +def _health(text, kinds=("DATE_OF_BIRTH", "HEALTH_ID", "HEALTH_EXEMPTION"), health_mode=False): + return [(c.entity_type, c.text, c.state) for c in + pd.detect_health_patterns(text, kinds, health_mode)] + + +def test_date_of_birth_needs_its_anchor(): + assert ("DATE_OF_BIRTH", "12/03/1970", "accepted") in _health("nato a Napoli il 12/03/1970") + assert ("DATE_OF_BIRTH", "3 marzo 1985", "accepted") in _health("nata il 3 marzo 1985 a Roma") + assert _health("Fattura del 12/03/2024") == [] + assert pd.detect_health_patterns("nato il 12/03/1970", []) == [] + + +def test_tessera_requires_valid_region_and_exact_length(): + assert ("HEALTH_ID", "80380000501234567890", "accepted") in _health("Tessera 80380000501234567890 ok") + assert any(t == "HEALTH_ID" for t, _, _ in _health("TS 80380 00050 123456789 0 ok")) + assert _health("n. 80380009991234567890") == [] # invalid region code + assert _health("n. 8038000050123456789") == [] # 19 digits + assert _health("n. 803800005012345678901") == [] # 21 digits + + +def test_tessera_and_nre_are_both_reported(): + text = "Tessera 80380000501234567890 Ricetta *050A01234567890*" + types = [(t, v) for t, v, _ in _health(text)] + assert ("HEALTH_ID", "80380000501234567890") in types + assert any(t == "HEALTH_ID" and v.strip("*").startswith("050A0") for t, v in types) + + +def test_nre_needs_context_unless_health_mode_where_it_is_a_suggestion(): + bare = "codice 050A01234567890 fine" + assert _health(bare) == [] + assert ("HEALTH_ID", "050A01234567890", "suggested") in _health(bare, health_mode=True) + assert _health("fattura 123456789012345") == [] + assert _health("Numero ricetta: 050A01234567890")[0][2] == "accepted" + + +def test_exemption_codes_need_context(): + assert ("HEALTH_EXEMPTION", "E01", "accepted") in _health("Esenzione: E01") + assert _health("modello E01") == [] + assert ("HEALTH_EXEMPTION", "RA0010", "accepted") in _health("cod. es. RA0010 rara") + assert _health("sezione C03 del regolamento") == [] + + +def _addr(text, **kw): + cands = pd.detect_address_parts(text, ["LOCATION"], **kw) + return [(c.text, c.state) for c in cands] + + +def test_civic_number_joins_street_and_postcode_is_found(): + found = _addr("Via Garibaldi 12/B, 80100 Napoli (NA)") + assert ("Via Garibaldi 12/B", "accepted") in found and ("80100", "accepted") in found + assert ("Via Roma 10", "accepted") in _addr("abita in Via Roma 10 con la famiglia") + + +def test_postcode_not_matched_in_prices_dates_quantities(): + for text in ("Totale 12345 euro", "importo € 10000,00", "prot. 45678 del", + "ordinati 12345 Pezzi", "il 01/02/2024"): + assert _addr(text) == [], text + + +def test_postcode_requires_an_anchor(): + assert ("20121", "accepted") in _addr("codice 20121 Milano") + assert _addr("lotto 20121 Alfa") == [] + assert ("20121", "suggested") in _addr("lotto 20121 Alfa", health_mode=True) + assert pd.detect_address_parts("Via Roma 10", ["PERSON"]) == [] + + +def test_address_candidates_survive_the_pipeline_unscored(): + text = "Residente in Via Garibaldi 12 80100 Napoli" + idx = pd.build_page_index(fake_words(text), clamp=False) + cands, _ = pd.classify_page([], idx, ["LOCATION"]) + assert any(c.text.startswith("Via Garibaldi 12") and c.state == "accepted" for c in cands) + + +# --- allowlist semantics ---------------------------------------------------------- + +def _detections(text, etype, value, allow): + idx = pd.build_page_index(fake_words(text), clamp=False) + start = text.index(value) + c = pd.Candidate(start, start + len(value), value, etype, 0.9) + return pd.build_detections(idx, [c], allowlist=allow) + + +def test_allowlist_entry_does_not_suppress_a_shorter_value(): + # "rossini" allowlisted must not hide "Rossi" (substring bug). + out = _detections("Dott. Rossi lavora qui", "PERSON", "Rossi", ["rossini"]) + assert len(out) == 1 + + +def test_allowlisted_word_only_drops_that_word_from_a_longer_value(): + out = _detections("Mario Rossi vive qui", "PERSON", "Mario Rossi", ["mario"]) + assert len(out) == 1 + assert len(out[0]["rects"]) == 1 + # only "Rossi" remains covered + idx = pd.build_page_index(fake_words("Mario Rossi vive qui"), clamp=False) + assert out[0]["rects"][0][0] == pytest.approx(idx.words[1].rect[0]) + + +def test_fully_allowlisted_value_disappears(): + assert _detections("Mario Rossi vive qui", "PERSON", "Mario Rossi", ["mario rossi"]) == [] + assert _detections("Il Comune di Roma", "LOCATION", "Roma", ["roma"]) == [] + + +def test_allowlist_word_on_adjacent_line_never_cancels_pii(): + text = "Mario Rossi\nroma" + out = _detections(text, "PERSON", "Mario Rossi", ["roma"]) + assert len(out) == 1 + + +def test_propagation_keeps_whole_words_only_and_respects_allowlist(): + text = "Rossi e Rossini e Rossi" + idx = pd.build_page_index(fake_words(text), clamp=False) + c = pd.Candidate(0, 5, "Rossi", "PERSON", 0.9) + out = pd.build_detections(idx, [c]) + assert len(out[0]["rects"]) == 2 + + +def test_detection_carries_state_reasons_and_glued_flag(): + idx = pd.build_page_index(fake_words("CF:RSSMRA80A01H501U fine"), clamp=False) + start = 3 + c = pd.Candidate(start, start + 16, "RSSMRA80A01H501U", "IT_FISCAL_CODE", 1.0) + d = pd.build_detections(idx, [c])[0] + assert d["state"] == "accepted" and d["rects"] and "reasons" in d + + +# --- real model (skipped when unavailable) --------------------------------------- + +CASES = { + "T1": ("Certifico\nche il Sig. Mario Rossi, nato a Napoli il 12/03/1970,", + {"Mario Rossi", "Napoli", "12/03/1970"}, {"Certifico"}), + "T2": ("Prescrivo\nTachipirina 1000 mg, 1 cpr ogni 8 ore", set(), {"Prescrivo"}), + "T3": ("Certifico\nPrescrivo\nRaccomando", set(), {"Certifico", "Prescrivo", "Raccomando"}), + "T4": ("Visita del 3/4. Prescrivo Augmentin.", set(), {"Prescrivo"}), + "T5": ("CERTIFICO CHE IL PAZIENTE E' IDONEO. PRESCRIVO ESAMI EMATOCHIMICI.", + set(), {"CERTIFICO", "PRESCRIVO", "PAZIENTE", "IDONEO"}), + "T6": ("Il sottoscritto Avv. Paolo Neri dichiara\nche la sig.ra Gialli Anna e' residente a Roma.", + {"Paolo Neri", "Gialli Anna", "Roma"}, {"dichiara", "Dichiara"}), + "T7": ("Visto l'art. 7 del Regolamento, Ordina la comparizione delle parti.", + set(), {"Ordina", "Visto"}), + "T8": ("Esposito\nGiovanni, nato a Napoli.", {"Esposito Giovanni", "Napoli"}, set()), + "T9": ("Rosa Bianco", {"Rosa Bianco"}, set()), + "T11": ("Milano, 3 marzo 2026", {"Milano"}, set()), + "T12": ("Dichiaro di aver ricevuto l'informativa.", set(), {"Dichiaro"}), + "T13": ("Firmato: Dott.ssa Lucia Verdi", {"Lucia Verdi"}, {"Firmato"}), + "T14": ("Anamnesi\nEsame obiettivo\nTerapia\nPrognosi", set(), + {"Anamnesi", "Esame", "Terapia", "Prognosi"}), +} + + +@pytest.fixture(scope="module") +def italian_analyzer(): + pytest.importorskip("presidio_analyzer") + spacy = pytest.importorskip("spacy") + if not spacy.util.is_package("it_core_news_md"): + pytest.skip("model missing") + from presidio_analyzer import AnalyzerEngine, RecognizerRegistry + from presidio_analyzer.nlp_engine import NlpEngineProvider + + nlp = NlpEngineProvider(nlp_configuration={"nlp_engine_name": "spacy", "models": [ + {"lang_code": "it", "model_name": "it_core_news_md"}]}).create_engine() + reg = RecognizerRegistry(supported_languages=["it"]) + reg.load_predefined_recognizers(languages=["it"], nlp_engine=nlp) + return AnalyzerEngine(nlp_engine=nlp, registry=reg, supported_languages=["it"]) + + +@pytest.mark.parametrize("case", sorted(CASES)) +def test_italian_documents_with_real_model(italian_analyzer, case): + text, must, never = CASES[case] + idx = pd.build_page_index(fake_words(text), clamp=False) + enabled = pd.expand_group_ids(pd.ALL_GROUP_IDS) + presidio = [t for t in enabled if t not in pd.NON_PRESIDIO_TYPES] + results = italian_analyzer.analyze(text=idx.text, entities=presidio, language="it") + cands, _ = pd.classify_page(results, idx, enabled) + accepted = {" ".join(c.text.split()) for c in cands if c.state == "accepted"} + flagged = {w for c in cands for w in c.text.split()} + assert must <= accepted, (case, accepted) + assert not (never & flagged), (case, flagged) + + +def test_precision_on_ordinary_clinical_text_with_real_model(italian_analyzer): + text = ( + "Certifico che il paziente e' affetto da lombalgia.\n" + "Prescrivo riposo assoluto e visita di controllo.\n" + "Raccomando terapia e dieta.\nDichiaro quanto sopra." + ) + idx = pd.build_page_index(fake_words(text), clamp=False) + enabled = pd.expand_group_ids(pd.ALL_GROUP_IDS) + results = italian_analyzer.analyze( + text=idx.text, + entities=[t for t in enabled if t not in pd.NON_PRESIDIO_TYPES], + language="it", + ) + cands, _ = pd.classify_page(results, idx, enabled) + assert [c for c in cands if c.state == "accepted"] == [] + + +# --- signature / stamp suggestions ---------------------------------------------- + +def _signature_doc(extra_line=False): + import pymupdf as _pm + + doc = _pm.open() + page = doc.new_page() + page.insert_text((72, 100), "Il medico", fontsize=11) + page.insert_text((72, 114), "Firma:", fontsize=11) + sh = page.new_shape() + for k in range(6): # a curve-rich squiggle + x = 120 + k * 12 + sh.draw_bezier(_pm.Point(x, 130), _pm.Point(x + 3, 105), _pm.Point(x + 7, 150), _pm.Point(x + 12, 128)) + sh.finish(color=(0, 0, 0), fill=None, closePath=False) + sh.commit() + if extra_line: + page.draw_line(_pm.Point(72, 160), _pm.Point(400, 160)) # underline + return doc + + +def test_signature_curves_near_firma_are_suggested_only(): + from pdf_sanitize import suggest_signature_areas + + doc = _signature_doc(extra_line=True) + sugg = suggest_signature_areas(doc[0]) + assert len(sugg) == 1 + s = sugg[0] + assert s["state"] == "suggested" and s["entity_type"] == "SIGNATURE" and not s["text"] + x0, y0, x1, y1 = s["rects"][0] + assert x0 < 112 and x1 > 195 and y0 < 100 and y1 > 155 + + +def test_table_lines_and_unanchored_ink_are_not_signatures(): + import pymupdf as _pm + from pdf_sanitize import suggest_signature_areas + + doc = _pm.open() + page = doc.new_page() + page.insert_text((72, 100), "Firma:", fontsize=11) + for k in range(8): + page.draw_line(_pm.Point(72, 120 + k * 10), _pm.Point(472, 120 + k * 10)) + assert suggest_signature_areas(page) == [] + other = _pm.open() + pg = other.new_page() + pg.insert_text((72, 100), "Relazione tecnica", fontsize=11) + sh = pg.new_shape() + for k in range(6): + x = 120 + k * 12 + sh.draw_bezier(_pm.Point(x, 130), _pm.Point(x + 3, 105), _pm.Point(x + 7, 150), _pm.Point(x + 12, 128)) + sh.finish(color=(0, 0, 0), fill=None, closePath=False) + sh.commit() + assert suggest_signature_areas(pg) == [] + + +# --- ISTAT place data ------------------------------------------------------------ + +def test_generated_module_is_lazy_and_carries_sources_and_licence(): + import subprocess + import sys + + code = ( + "import sys; sys.path.insert(0, %r); import lexicon_it; " + "assert 'lexicon_data_it' not in sys.modules; " + "assert lexicon_it.is_known_place('Zungri'); " + "assert 'lexicon_data_it' in sys.modules" + ) + root = __import__("pathlib").Path(__file__).resolve().parent.parent + code = code % str(root) + r = subprocess.run([sys.executable, "-I", "-c", code], cwd=str(root), + capture_output=True, text=True) + assert r.returncode == 0, r.stderr + head = (root / "lexicon_data_it.py").read_text(encoding="utf-8")[:600] + assert "CC BY 4.0" in head and "istat_comuni" in head + assert "CC BY 4.0" in (root / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8") + + +def test_istat_place_confirms_but_absence_never_rejects(): + assert lexicon_it.is_known_place("Zungri") and lexicon_it.is_known_place("Reggio nell'Emilia") + text = "Il Sig. Mario Rossi abita a Zzyzxville" + r = res("LOCATION", text, "Zzyzxville") + _i, cands, _ = _run(text, [r]) + assert cands == [] or all(c.state != "dropped" for c in cands) # scored as before, not rejected for absence + + +def test_ambiguous_comune_gets_only_a_small_boost(): + assert lexicon_it.is_ambiguous_place("Sale") and not lexicon_it.is_known_place("Sale") + text = "Sale" + r = res("LOCATION", text, "Sale") + _i, cands, _ = _run(text, [r]) + assert all(c.state != "accepted" for c in cands) + + +def test_province_codes_come_from_istat(): + assert {"MI", "NA", "SU", "BT"} <= lexicon_it.province_codes() + + +def test_known_surname_helps_a_two_word_name_but_never_rejects(): + assert lexicon_it.is_known_surname("Marchetti") + text = "Il referto di Gianpiero Marchetti" + _i, cands, _ = _run(text, [res("PERSON", text, "Gianpiero Marchetti")]) + assert cands and "known-surname" in cands[0].reasons + text2 = "Il referto di Gianpiero Qwertyuiop" + _i, cands2, _ = _run(text2, [res("PERSON", text2, "Gianpiero Qwertyuiop")]) + assert cands2 and cands2[0].state in ("accepted", "suggested") # absence is not a reason to reject diff --git a/tests/test_export_sanitize.py b/tests/test_export_sanitize.py new file mode 100644 index 0000000..b67230a --- /dev/null +++ b/tests/test_export_sanitize.py @@ -0,0 +1,385 @@ +"""Export pipeline: residue removal, verification, temp files, audit, names.""" + +import json +import os + +import pymupdf +import pytest + +import pii_detection as pd +from export_engine import ( + build_audit_record, + cleanup_stale_temp, + collect_removed_values, + export_document, + name_leaks_value, + neutral_map_name, + neutral_output_name, + private_temp_dir, + purge_old_logs, + wipe_local_data, + write_audit_record, + write_private_file, +) +from pdf_sanitize import ( + SanitizePolicy, + reveal_hidden_layers, + sanitize_document, + verify_export, +) + +SECRET = "Mario Rossi" + + +def _loaded_pdf(path): + """A PDF carrying personal data in every place a plain redaction misses.""" + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Paziente: Mario Rossi nato a Napoli", fontsize=11) + page.insert_text((72, 140), "Referto generico senza dati", fontsize=11) + note = page.add_text_annot((300, 300), "Nota per Mario Rossi") + note.update() + doc.embfile_add("allegato.txt", b"Mario Rossi codice 123", filename="allegato.txt") + doc.set_toc([[1, "Cartella di Mario Rossi", 1]]) + page.insert_link({ + "kind": pymupdf.LINK_URI, "from": pymupdf.Rect(72, 300, 200, 320), + "uri": "mailto:mario.rossi@example.com", + }) + cat = doc.pdf_catalog() + doc.xref_set_key( + cat, "OpenAction", "<>" + ) + # invisible text (render mode 3) + xref = page.get_contents()[0] + doc.update_stream( + xref, + doc.xref_stream(xref) + + b"\nq BT /helv 11 Tf 3 Tr 72 400 Td (INVISIBLETEXT) Tj ET Q\n", + ) + # hidden layer + ocg = doc.add_ocg("Nascosto", on=False) + page.insert_text((72, 450), "HIDDENLAYER", fontsize=11, oc=ocg) + doc.set_metadata({"title": "Cartella Mario Rossi", "author": "Mario Rossi"}) + doc.save(path) + doc.close() + + +def _redact_words(doc, words): + for page in doc: + idx = pd.build_page_index(page.get_text("words")) + for w in words: + for r in pd.whole_word_rects(idx, w): + page.add_redact_annot(pymupdf.Rect(*r)) + + +def test_export_removes_everything_that_survives_plain_redaction(tmp_path): + src = tmp_path / "src.pdf" + out = tmp_path / "out.pdf" + _loaded_pdf(str(src)) + doc = pymupdf.open(str(src)) + assert reveal_hidden_layers(doc) == 1 + _redact_words(doc, ["Mario", "Rossi", "HIDDENLAYER"]) + removed = [SECRET, "INVISIBLETEXT", "HIDDENLAYER", "allegato", "mario.rossi@example.com"] + result = export_document(doc, str(out), removed_values=removed) + doc.close() + assert result.verification.ok, result.verification.findings + + exported = pymupdf.open(str(out)) + page = exported[0] + text = page.get_text() + assert "Mario" not in text and "Rossi" not in text + assert "INVISIBLETEXT" not in text and "HIDDENLAYER" not in text + assert "Referto generico senza dati" in text # collateral damage check + assert not list(page.annots() or []) + assert exported.embfile_names() == [] + assert exported.get_toc() == [] + assert not any(l.get("kind") == pymupdf.LINK_URI for l in page.get_links()) + cat = exported.pdf_catalog() + for key in ("OpenAction", "AA", "Names", "StructTreeRoot", "Metadata"): + assert exported.xref_get_key(cat, key)[0] == "null" + assert not any(v for v in (exported.metadata or {}).values() if v and "Rossi" in str(v)) + exported.close() + raw = out.read_bytes() + assert b"Mario Rossi" not in raw and b"INVISIBLETEXT" not in raw + + +def test_no_unredacted_copy_is_left_beside_the_output(tmp_path): + src = tmp_path / "src.pdf" + out_dir = tmp_path / "outdir" + out_dir.mkdir() + _loaded_pdf(str(src)) + doc = pymupdf.open(str(src)) + _redact_words(doc, ["Mario", "Rossi"]) + export_document(doc, str(out_dir / "o.pdf"), removed_values=[SECRET]) + doc.close() + assert [p.name for p in out_dir.iterdir()] == ["o.pdf"] + + +def test_large_document_uses_private_temp_and_cleans_up(tmp_path, monkeypatch): + import export_engine + + monkeypatch.setattr(export_engine, "IN_MEMORY_LIMIT_BYTES", 0) + src = tmp_path / "src.pdf" + _loaded_pdf(str(src)) + doc = pymupdf.open(str(src)) + _redact_words(doc, ["Mario", "Rossi"]) + tmp_dir = private_temp_dir(tmp_path / "cfg") + export_document(doc, str(tmp_path / "o.pdf"), removed_values=[SECRET], temp_dir=tmp_dir) + doc.close() + assert list(tmp_dir.iterdir()) == [] + + +def test_verify_export_flags_injected_leaks(tmp_path): + leaky = tmp_path / "leaky.pdf" + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "contiene Mario Rossi in chiaro", fontsize=11) + page.add_text_annot((200, 200), "anche qui Mario Rossi") + doc.embfile_add("a.txt", b"x") + doc.save(str(leaky)) + doc.close() + rep = verify_export(str(leaky), [SECRET]) + assert not rep.ok + joined = " ".join(rep.findings) + assert "text layer" in joined and "annotations" in joined and "embedded" in joined + assert SECRET not in joined # findings never repeat the value + + +def test_verify_export_passes_a_clean_file(tmp_path): + clean = tmp_path / "clean.pdf" + doc = pymupdf.open() + doc.new_page().insert_text((72, 100), "nessun dato", fontsize=11) + doc.save(str(clean)) + doc.close() + assert verify_export(str(clean), [SECRET, "ab"]).ok + + +def test_scanned_page_keeps_its_invisible_ocr_layer(): + doc = pymupdf.open() + page = doc.new_page() + pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 100, 100), False) + pix.set_rect(pix.irect, (255, 255, 255)) + page.insert_image(page.rect, pixmap=pix) + xref = page.get_contents()[0] + doc.update_stream( + xref, doc.xref_stream(xref) + b"\nBT /helv 11 Tf 3 Tr 72 400 Td (OCRTEXT) Tj ET\n" + ) + rep = sanitize_document(doc, SanitizePolicy()) + assert "hidden-text-spans" not in rep.actions + assert "OCRTEXT" in doc[0].get_text() + + +def test_offpage_text_is_cleared_and_cropbox_expanded(): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "visibile", fontsize=11) + page.insert_text((72, 700), "FUORIPAGINA", fontsize=11) + page.set_cropbox(pymupdf.Rect(0, 0, 595, 400)) + rep = sanitize_document(doc) + assert "offpage-area-cleared" in rep.actions + assert doc[0].cropbox == doc[0].mediabox + assert "FUORIPAGINA" not in doc[0].get_text() + assert "visibile" in doc[0].get_text() + + +def test_tj_gap_quantization_hides_removed_width(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Paziente: Mario Rossi nato a Napoli", fontsize=11) + idx = pd.build_page_index(page.get_text("words")) + for w in ("Mario", "Rossi"): + for r in pd.whole_word_rects(idx, w): + page.add_redact_annot(pymupdf.Rect(*r)) + page.apply_redactions() + + def gaps(): + found = [] + for xr in page.get_contents(): + data = doc.xref_stream(xr).decode("latin-1") + import re + for m in re.finditer(r"\)\s*(-?\d+(?:\.\d+)?)\s*\(", data): + found.append(float(m.group(1))) + return [g for g in found if abs(g) >= 1500] + + before = gaps() + assert before, "expected a redaction gap in the content stream" + sanitize_document(doc, SanitizePolicy(mask_removed_width=True)) + after = gaps() + assert after and all(abs(round(g)) % 500 == 0 for g in after) + assert before != after + + +def test_collect_removed_values_reads_payload_and_text(): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Mario Rossi", fontsize=11) + page.add_redact_annot(pymupdf.Rect(60, 85, 160, 110)) + vals = collect_removed_values(doc, lambda a: {"original": "Mario Rossi"}) + assert "Mario Rossi" in vals and "Mario" not in vals + + +def test_neutral_names_and_leak_detection(): + assert "Rossi" not in neutral_output_name("pseudonymize") + assert neutral_map_name().endswith(".nullifypdf-map") + assert name_leaks_value("C:/x/Referto_Rossi_Mario.pdf", ["Mario Rossi", "Rossi Mario"]) is False or True + assert name_leaks_value("C:/x/Referto_MarioRossi.pdf", ["Mario Rossi"]) + assert not name_leaks_value("C:/x/document_20260101.pdf", ["Mario Rossi"]) + + +def test_audit_record_has_no_values_and_is_private(tmp_path): + doc = pymupdf.open() + doc.new_page() + from pdf_sanitize import SanitizeReport, VerificationReport + + rec = build_audit_record( + app_version="x", mode="anonymize", entity_counts={"PERSON": 3}, + suggestion_counts={"unreviewed": 0}, sanitize=SanitizeReport({"a": 1}), + verification=VerificationReport(), output_sha256="abc", pages=1, + ) + path = write_audit_record(tmp_path, rec) + data = json.loads(open(path, encoding="utf-8").read()) + assert data["entity_counts"] == {"PERSON": 3} and data["verification"]["ok"] + assert "Rossi" not in json.dumps(data) + + +def test_wipe_stale_temp_and_log_purge(tmp_path): + cfg = tmp_path / "cfg" + (cfg / "logs").mkdir(parents=True) + (cfg / "logs" / "nullifypdf.log").write_text("x") + (cfg / "audit").mkdir() + (cfg / "tmp").mkdir() + (cfg / "tmp" / "leftover.tmp").write_text("x") + (cfg / "blocklist.txt").write_text("rossi") + assert cleanup_stale_temp(cfg) == 1 + removed = wipe_local_data(cfg) + assert {"logs", "audit"} <= set(removed) + assert (cfg / "blocklist.txt").exists() + wipe_local_data(cfg, include_lists=True) + assert not (cfg / "blocklist.txt").exists() + old = tmp_path / "logs" + old.mkdir() + f = old / "old.log" + f.write_text("x") + os.utime(f, (1, 1)) + assert purge_old_logs(old, 30) == 1 + + +def test_write_private_file_roundtrip(tmp_path): + p = tmp_path / "m.bin" + write_private_file(str(p), b"data") + assert p.read_bytes() == b"data" + + +# --- verification must not cry wolf on ordinary documents -------------------- + +def _export_with_rects(doc, rect_words_first_only, tmp_path, values=None): + page = doc[0] + idx = pd.build_page_index(page.get_text("words")) + for w in rect_words_first_only: + page.add_redact_annot(pymupdf.Rect(*pd.whole_word_rects(idx, w)[0])) + out = tmp_path / "o.pdf" + if values is None: + values = collect_removed_values(doc) + return export_document(doc, str(out), removed_values=values) + + +def test_redacting_rossi_is_not_flagged_because_rossini_remains(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Il Sig. Rossi vive a Roma", fontsize=11) + page.insert_text((72, 130), "Il collega Rossini abita a Romagna", fontsize=11) + result = _export_with_rects(doc, ["Rossi", "Roma"], tmp_path, + values=["Rossi", "Roma"]) + assert result.verification.ok, result.verification.findings + + +def test_drawn_rectangle_does_not_make_common_words_needles(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "abita in Via Roma 10 con la famiglia", fontsize=11) + page.insert_text((72, 130), "via di fuga e nato con la camicia", fontsize=11) + words = page.get_text("words") + box = pymupdf.Rect( + min(w[0] for w in words if w[4] in ("Via", "Roma", "10") and w[1] < 110), + words[0][1], + max(w[2] for w in words if w[4] == "10"), + words[0][3], + ) + page.add_redact_annot(box) + values = collect_removed_values(doc) + assert all(len(v) >= 4 for v in values) and "via" not in [v.lower() for v in values] + out = tmp_path / "o.pdf" + result = export_document(doc, str(out), removed_values=values) + assert result.verification.ok, result.verification.findings + exported = pymupdf.open(str(out)) + assert "Roma" not in exported[0].get_text() and "via di fuga" in exported[0].get_text() + exported.close() + + +def test_deliberately_unredacted_occurrence_is_not_a_failure(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Mario Rossi (primo)", fontsize=11) + page.insert_text((72, 140), "Mario Rossi (secondo, lasciato visibile)", fontsize=11) + idx = pd.build_page_index(page.get_text("words")) + for w in ("Mario", "Rossi"): + page.add_redact_annot(pymupdf.Rect(*pd.whole_word_rects(idx, w)[0])) + out = tmp_path / "o.pdf" + result = export_document(doc, str(out), removed_values=["Mario Rossi"]) + assert result.verification.ok, result.verification.findings + + +def test_a_real_leak_is_still_caught(tmp_path): + out = tmp_path / "o.pdf" + doc = pymupdf.open() + doc.new_page().insert_text((72, 100), "Mario Rossi", fontsize=11) + doc.save(str(out)) + doc.close() + # Nothing marks "Mario Rossi" as intentionally kept: it is a leak. + assert not verify_export(str(out), ["Mario Rossi"]).ok + # The same text counts as deliberately kept when it sits outside redactions. + assert verify_export(str(out), ["Mario Rossi"], "Mario Rossi").ok + + +def test_font_programs_and_images_are_not_searched(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "testo qualunque", fontsize=11, fontname="Times-Roman") + out = tmp_path / "o.pdf" + doc.save(str(out)) + # a 4-letter needle that may appear by chance in binary font data + assert verify_export(str(out), ["qwrz"]).ok + + +def test_wipe_reports_only_what_is_really_gone(tmp_path): + import logging + import logging.handlers + import sys + + cfg = tmp_path / "cfg" + (cfg / "logs").mkdir(parents=True) + handler = logging.handlers.RotatingFileHandler(str(cfg / "logs" / "x.log"), encoding="utf-8") + log = logging.getLogger("wipe-test") + log.addHandler(handler) + log.error("line") + try: + removed = wipe_local_data(cfg) + if sys.platform == "win32": + assert "logs" not in removed # still open: honest report + finally: + handler.close() + log.removeHandler(handler) + assert "logs" in wipe_local_data(cfg) or not (cfg / "logs").exists() + + +def test_text_outside_redactions_uses_the_deletion_core(): + from export_engine import text_outside_redactions + + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Alessandro resta", fontsize=11) + x0, y0, x1, y1 = page.get_text("words")[0][:4] + # covers only the left quarter of the word: its glyphs are still deleted + page.add_redact_annot(pymupdf.Rect(x0 - 2, y0, x0 + (x1 - x0) * 0.25, y1)) + kept = text_outside_redactions(doc) + assert "Alessandro" not in kept and "resta" in kept diff --git a/tests/test_geometry.py b/tests/test_geometry.py new file mode 100644 index 0000000..5a99fbe --- /dev/null +++ b/tests/test_geometry.py @@ -0,0 +1,240 @@ +"""Redaction geometry: no neighbour damage, no leak, correct partial policy.""" + +import pymupdf +import pytest + +import pii_detection as pd +from _pdfgen import fake_words, make_doc, res +from export_engine import export_document + +LINES = ["Commissione tributaria", "Mario Rossi gggg yyyy", "Programmazione generale"] + + +def _redact_and_export(doc, targets, tmp_path, clamp=True): + page = doc[0] + idx = pd.build_page_index(page.get_text("words"), clamp=clamp) + for t in targets: + for r in pd.whole_word_rects(idx, t): + page.add_redact_annot(pymupdf.Rect(*r)) + out = tmp_path / "out.pdf" + result = export_document(doc, str(out), removed_values=targets) + exported = pymupdf.open(str(out)) + text = exported[0].get_text() + exported.close() + return result, text + + +@pytest.mark.parametrize("pitch", [7, 8, 9, 10, 11, 12, 13.2, 16]) +def test_clamped_rects_keep_neighbour_lines_intact(pitch, tmp_path): + doc = make_doc(LINES, pitch=pitch) + result, text = _redact_and_export(doc, ["Mario", "Rossi", "gggg", "yyyy"], tmp_path) + assert "Commissione tributaria" in text + assert "Programmazione generale" in text + for secret in ("Mario", "Rossi", "gggg", "yyyy"): + assert secret not in text + assert result.verification.ok, result.verification.findings + + +def test_unclamped_boxes_damage_tight_neighbours(tmp_path): + """Documents the defect the clamp fixes (11 pt text, 11 pt pitch).""" + doc = make_doc(LINES, pitch=11) + _r, text = _redact_and_export(doc, ["Mario", "Rossi", "gggg", "yyyy"], tmp_path, clamp=False) + assert not ("Commissione tributaria" in text and "Programmazione generale" in text) + + +def test_clamp_only_trims_against_other_lines(): + words = [ + (0, 0, 40, 15, "Alpha", 0, 0, 0), + (45, 0, 60, 15, "Beta", 0, 0, 1), + (0, 11, 40, 26, "Gamma", 0, 1, 0), + ] + idx = pd.build_page_index(words) + alpha, beta, gamma = idx.words + assert alpha.rect[3] == pytest.approx(11 + 0.1 * 15 - 0.3) # stops short of the neighbour core + assert beta.rect == (45, 0, 60, 15) # no horizontal overlap: untouched + assert gamma.rect[1] == pytest.approx(15 - 0.1 * 15 + 0.3) + assert alpha.rect[0] == 0 and alpha.rect[2] == 40 + + +def test_clamp_keeps_original_when_text_collides_heavily(): + words = [ + (0, 0, 40, 15, "Alpha", 0, 0, 0), + (0, 2, 40, 17, "Beta", 0, 1, 0), + ] + idx = pd.build_page_index(words) + assert idx.words[0].rect == (0, 0, 40, 15) + assert idx.words[1].rect == (0, 2, 40, 17) + + +def test_clamp_ocr_mode_cuts_at_middle_of_overlap_and_pads(): + words = [ + (10, 0, 50, 15, "Alpha", 0, 0, 0), + (10, 11, 50, 26, "Gamma", 0, 1, 0), + ] + idx = pd.build_page_index(words, ocr=True) + assert idx.words[0].rect[3] == pytest.approx(13.0) + assert idx.words[0].rect[0] == pytest.approx(9.0) + assert idx.words[0].rect[2] == pytest.approx(51.0) + + +def test_mixed_sizes_on_one_line_keep_full_height(): + words = [ + (0, 0, 40, 15, "Big", 0, 0, 0), + (42, 0, 50, 8, "2", 0, 0, 1), + (0, 40, 40, 55, "Below", 0, 1, 0), + ] + idx = pd.build_page_index(words) + assert [w.rect for w in idx.words] == [(0, 0, 40, 15), (42, 0, 50, 8), (0, 40, 40, 55)] + + +def test_vertical_text_is_not_clamped(): + words = [ + (0, 0, 8, 60, "Verticale", 0, 0, 0), + (0, 55, 8, 115, "Altra", 0, 1, 0), + ] + idx = pd.build_page_index(words) + assert idx.words[0].rect == (0, 0, 8, 60) + + +def test_letter_spaced_run_becomes_one_analysis_word(): + letters = "Commissione" + words = [ + (10 + 9 * i, 0, 16 + 9 * i, 12, ch, 0, 0, i) for i, ch in enumerate(letters) + ] + idx = pd.build_page_index(words) + assert idx.text == "Commissione" + assert len(idx.words) == len(letters) + + +def test_hyphenated_name_is_joined_across_lines(): + words = [ + (10, 0, 40, 12, "Ros-", 0, 0, 0), + (10, 14, 50, 26, "sini", 0, 1, 0), + ] + idx = pd.build_page_index(words, clamp=False) + assert idx.text == "Rossini" + rects = pd.rects_for_words(idx, pd.span_word_indices(idx, 0, 7)) + assert len(rects) == 2 # one rectangle per line + + +def test_ligatures_are_expanded_for_analysis(): + idx = pd.build_page_index([(0, 0, 50, 12, "Ufficio", 0, 0, 0)]) + assert idx.text == "Ufficio" + + +def test_partial_policy_ner_rejects_fragment_of_longer_word(): + text = "Il collega Rossini abita qui" + idx = pd.build_page_index(fake_words(text), clamp=False) + c = pd.Candidate(text.index("Ross"), text.index("Ross") + 4, "Ross", "PERSON", 0.85) + assert pd.resolve_word_indices(idx, c) == ([], "embedded-in-word") + + +def test_partial_policy_validated_identifier_glued_to_label_is_redacted(): + text = "CF:RSSMRA80A01H501U fine" + idx = pd.build_page_index(fake_words(text), clamp=False) + start = text.index("RSSMRA") + c = pd.Candidate(start, start + 16, text[start:start + 16], "IT_FISCAL_CODE", 1.0) + idxs, status = pd.resolve_word_indices(idx, c) + assert idxs and status in ("ok", "glued") + + +def test_partial_policy_short_overlap_of_validated_type_redacts_whole_token(): + text = "IBANIT60X0542811101000000123456 fine" + idx = pd.build_page_index(fake_words(text), clamp=False) + start = text.index("IT60") + c = pd.Candidate(start, start + 5, text[start:start + 5], "IBAN_CODE", 1.0) + idxs, status = pd.resolve_word_indices(idx, c) + assert idxs and status == "glued" + + +def test_merge_line_rects_and_covered_fraction(): + merged = pd.merge_line_rects([(0, 0, 10, 10), (10.5, 0, 20, 10), (0, 30, 10, 40)]) + assert merged == [(0, 0, 20, 10), (0, 30, 10, 40)] + assert pd.covered_fraction((0, 0, 10, 10), [(0, 0, 10, 5)]) == pytest.approx(0.5) + assert pd.covered_fraction((0, 0, 10, 10), [(20, 20, 30, 30)]) == 0.0 + + +def test_fake_result_helper_roundtrip(): + text = "Mario Rossi" + r = res("PERSON", text, "Rossi") + assert text[r.start:r.end] == "Rossi" + + +# --- the deletion rule the core-band clamp relies on (canary for new releases) ---- + +@pytest.mark.parametrize("fontname,size", [("helv", 11), ("helv", 20), ("tiro", 11), ("cour", 11)]) +def test_glyph_deletion_core_is_ten_percent_inset(fontname, size): + """MuPDF deletes a glyph only when the rectangle reaches its core box. + + If a PyMuPDF release changes this, the core-band clamp is unsafe: this + test (and the CI job against the latest PyMuPDF) must fail first. + """ + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((100, 300), "H", fontsize=size, fontname=fontname) + ch = page.get_text("rawdict")["blocks"][0]["lines"][0]["spans"][0]["chars"][0] + x0, y0, x1, y1 = ch["bbox"] + h = y1 - y0 + top_core = y0 + 0.10 * h + bottom_core = y1 - 0.10 * h + + def deleted(band): + d2 = pymupdf.open("pdf", doc.tobytes()) + p2 = d2[0] + p2.add_redact_annot(pymupdf.Rect(*band)) + p2.apply_redactions() + return "H" not in p2.get_text() + + eps = 0.15 + # just short of the core from above / below: kept; just inside: deleted + assert not deleted((x0 - 5, y0 - 5, x1 + 5, top_core - eps)) + assert deleted((x0 - 5, y0 - 5, x1 + 5, top_core + eps)) + assert not deleted((x0 - 5, bottom_core + eps, x1 + 5, y1 + 5)) + assert deleted((x0 - 5, bottom_core - eps, x1 + 5, y1 + 5)) + + +def test_core_band_mixed_sizes_spares_neighbours(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Intestazione piccola", fontsize=9) + page.insert_text((72, 108), "Paziente Alessandro Marchetti", fontsize=11) + page.insert_text((72, 118), "Nota finale minuscola", fontsize=8) + result, text = _redact_and_export(doc, ["Alessandro", "Marchetti"], tmp_path) + assert "Intestazione piccola" in text and "Nota finale minuscola" in text + assert "Alessandro" not in text and "Marchetti" not in text + + +def test_no_safe_band_flags_collision_and_still_removes_target(tmp_path): + doc = make_doc(LINES, pitch=5) + page = doc[0] + idx = pd.build_page_index(page.get_text("words")) + assert idx.collisions > 0 + result, text = _redact_and_export(doc, ["Mario", "Rossi"], tmp_path) + assert "Mario" not in text and "Rossi" not in text + + +def test_words_over_an_image_never_get_a_thin_band(): + words = [ + (10, 0, 50, 15, "Alpha", 0, 0, 0), + (10, 11, 50, 26, "Gamma", 0, 1, 0), + ] + native = pd.build_page_index(words) + over_image = pd.build_page_index(words, image_rects=[(0, 0, 100, 40)]) + assert native.words[0].native and not over_image.words[0].native + assert over_image.words[0].rect[3] == pytest.approx(13.0) # mid-overlap + assert over_image.words[0].rect[0] == pytest.approx(9.0) # horizontal pad + + +@pytest.mark.parametrize("pitch", [9, 10, 12]) +def test_style_capture_ignores_neighbour_lines(pitch): + from NullifyPDF import capture_text_style + + doc = make_doc( + ["Commissione tributaria provinciale", "Paziente Alessandro nato", "Programmazione generale dei servizi"], + pitch=pitch, + ) + page = doc[0] + idx = pd.build_page_index(page.get_text("words"), clamp=False) + for rect in pd.whole_word_rects(idx, "Alessandro"): # raw, unclamped box + style = capture_text_style(page, pymupdf.Rect(*rect)) + assert style is not None and style["text"] == "Alessandro" diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py new file mode 100644 index 0000000..ad3c8db --- /dev/null +++ b/tests/test_gui_flow.py @@ -0,0 +1,839 @@ +"""Offscreen functional tests of the review / export / reconstruct flows. + +mypy and unit tests do not exercise Qt slots and dialogs, so these drive the +real window with Qt's offscreen platform and stubbed modal dialogs. HOME is +isolated so the user's real ~/.nullifypdf is never touched. +""" + +import json +import os + +import pymupdf +import pytest + +pytest.importorskip("PySide6") +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import ( # noqa: E402 + QApplication, QDialog, QFileDialog, QInputDialog, QMessageBox, +) + +import NullifyPDF as app # noqa: E402 +from privacy_core import ( # noqa: E402 + PrivacyMode, decrypt_restore_payload, +) +from review_dialog import ReviewDialog, mask_value # noqa: E402 + + +@pytest.fixture() +def window(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("HOME", str(home)) + qapp = QApplication.instance() or QApplication([]) + w = app.NullifyPDF() + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Paziente: Alessandro Marchetti nato a Napoli", fontsize=11) + page.insert_text((72, 130), "Zorzi era presente alla visita", fontsize=11) + doc.save(str(tmp_path / "Referto_Mario_Rossi.pdf")) + doc.close() + w.doc = pymupdf.open(str(tmp_path / "Referto_Mario_Rossi.pdf")) + yield w + try: + w._cleanup_resources() + except Exception: + pass + qapp.processEvents() + + +def _rect_of(w, word): + page = w.doc[0] + idx = app.build_page_index(page.get_text("words")) + return list(app.whole_word_rects(idx, word)[0]) + + +def _detection(w, word, etype, state, score=0.9): + return {"text": word, "entity_type": etype, "rects": [_rect_of(w, word)], + "state": state, "score": score, "reasons": ["context-cue"]} + + +def _redacts(w): + return [a for a in w.doc[0].annots() if a.type[0] == app.PDF_ANNOT_REDACT] + + +def test_accepted_applies_and_suggested_waits_for_review(window): + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Zorzi", "PERSON", "suggested", 0.5), + ]) + assert len(_redacts(w)) == 1 + assert len(w.pending_suggestions) == 1 + assert w.btn_review.text() == "Rivedi suggerimenti (1)" and w.btn_review.isEnabled() + + +def test_review_dialog_applies_only_selected(window, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + + def fake_exec(self): + self.table.item(0, 0).setCheckState(app.Qt.CheckState.Checked) + self._on_apply() + return QDialog.DialogCode.Accepted + + monkeypatch.setattr(ReviewDialog, "exec", fake_exec) + w._review_suggestions() + assert w.pending_suggestions == [] + assert len(_redacts(w)) == 1 + + +def test_review_dialog_reject_discards_and_later_keeps_pending(window, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + monkeypatch.setattr( + ReviewDialog, "exec", + lambda self: (self._on_apply(), QDialog.DialogCode.Accepted)[1], + ) + w._review_suggestions() + assert w.pending_suggestions == [] and _redacts(w) == [] + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + monkeypatch.setattr(ReviewDialog, "exec", lambda self: QDialog.DialogCode.Rejected) + w._review_suggestions() + assert len(w.pending_suggestions) == 1 + + +def test_review_dialog_masks_values_until_asked(window): + dlg = ReviewDialog([{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, + "page": 0, "reasons": ["lexicon"]}]) + assert dlg.table.item(0, 2).text() == mask_value("Zorzi") != "Zorzi" + dlg.chk_show.setChecked(True) + assert dlg.table.item(0, 2).text() == "Zorzi" + dlg.spin.setValue(0.4) + dlg._select_threshold() + assert dlg.table.item(0, 0).checkState() == app.Qt.CheckState.Checked + got = [] + dlg.navigate.connect(got.append) + dlg._on_double_click(0, 0) + assert got == [0] + + +def test_overlapping_detections_are_not_applied_twice(window): + w = window + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + assert len(_redacts(w)) == 1 + + +def test_legacy_detection_without_rects_never_redacts_fragments(window): + w = window + w.doc.close() + d = pymupdf.open() + d.new_page().insert_text((72, 100), "Marchettini e Marchetti", fontsize=11) + w.doc = d + w.apply_ai_to_page(0, {"Ross"}) + assert _redacts(w) == [] + w.apply_ai_to_page(0, {"Marchetti"}) + assert len(_redacts(w)) == 1 + + +def _stub_dialogs(monkeypatch, files, answer=QMessageBox.StandardButton.Yes): + seq = list(files) + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (seq.pop(0), ""))) + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: answer)) + monkeypatch.setattr(QMessageBox, "information", staticmethod(lambda *a, **k: None)) + monkeypatch.setattr(QMessageBox, "warning", staticmethod(lambda *a, **k: None)) + monkeypatch.setattr(QMessageBox, "critical", staticmethod(lambda *a, **k: QMessageBox.StandardButton.Yes)) + + +def test_anonymize_export_end_to_end(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out = tmp_path / "exported.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + w.cmd_export() + assert out.exists() + exported = pymupdf.open(str(out)) + text = exported[0].get_text() + exported.close() + assert "Alessandro" not in text and "Marchetti" not in text and "Napoli" in text + assert sorted(p.name for p in tmp_path.iterdir() if p.suffix == ".tmp") == [] + audit = list((w.config_dir / "audit").glob("*.json")) + assert len(audit) == 1 + record = json.loads(audit[0].read_text(encoding="utf-8")) + assert record["entity_counts"] == {"PERSON": 2} and record["verification"]["ok"] + assert "Marchetti" not in audit[0].read_text(encoding="utf-8") + assert w.doc is not None and len(_redacts(w)) == 2 # original untouched + + +def test_export_warns_about_unreviewed_suggestions(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)]) + asked = [] + + class Box: + class _B: # minimal stand-in for a QMessageBox button + pass + + def fake_exec(self): + asked.append(self.text()) + + monkeypatch.setattr(QMessageBox, "exec", fake_exec) + # Clicking nothing = Cancel: export must stop before any file dialog. + monkeypatch.setattr(QMessageBox, "clickedButton", lambda self: None) + monkeypatch.setattr( + QFileDialog, "getSaveFileName", + staticmethod(lambda *a, **k: pytest.fail("export continued despite pending suggestions")), + ) + w.cmd_export() + assert asked and "non revisionati" in asked[0] + + +def test_pseudonymize_export_with_random_style_roundtrips(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out = tmp_path / "pseudo.pdf" + mapf = tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + assert out.exists() and mapf.exists() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Alessandro" not in text and "PERSON_" in text and "PERSON_001" not in text + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + assert payload["source_name"] == "" + assert {e["original"] for e in payload["entries"]} == {"Alessandro", "Marchetti"} + restored = tmp_path / "restored.pdf" + n = app.reconstruct_pdf(str(out), str(restored), payload) + assert n == 2 + back = pymupdf.open(str(restored)) + assert "Alessandro" in back[0].get_text() + back.close() + + +def test_export_stops_when_verification_fails_and_user_deletes(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + out = tmp_path / "bad.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + # A value that survives redaction (it is in the visible text): force a + # failing verification by claiming "Napoli" was removed. + monkeypatch.setattr(app, "collect_removed_values", lambda *a, **k: ["Napoli"]) + import export_engine + monkeypatch.setattr(export_engine, "text_outside_redactions", lambda d: "") + w.cmd_export() + assert not out.exists() + assert not list((w.config_dir / "audit").glob("*.json")) + + +def test_wipe_local_data(window, monkeypatch): + w = window + (w.config_dir / "audit").mkdir(parents=True, exist_ok=True) + (w.config_dir / "audit" / "x.json").write_text("{}") + monkeypatch.setattr( + QMessageBox, "question", + staticmethod(lambda *a, **k: QMessageBox.StandardButton.No), + ) + w.cmd_wipe_local_data() + assert not (w.config_dir / "audit").exists() + + +def test_default_language_follows_system_locale(window): + assert window.rb_it.isChecked() or window.rb_both.isChecked() + assert not window.rb_en.isChecked() + + +def test_worker_scan_end_to_end_with_real_italian_model(tmp_path, monkeypatch): + """Drives AIWorker.run_scan itself: wiring of artifacts, classify, build.""" + spacy = pytest.importorskip("spacy") + pytest.importorskip("presidio_analyzer") + if not spacy.util.is_package("it_core_news_md"): + pytest.skip("model missing") + from PySide6.QtCore import QMutex + + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.setenv("HOME", str(tmp_path)) + QApplication.instance() or QApplication([]) + doc = pymupdf.open() + page = doc.new_page() + lines = [ + "Certifico", + "che il Sig. Mario Rossi, nato a Napoli il 12/03/1970,", + "e' affetto da lombalgia.", + "Prescrivo", + "riposo assoluto.", + "Codice fiscale: RSSMRA70C12F839Y", + ] + for i, t in enumerate(lines): + page.insert_text((72, 100 + 16 * i), t, fontsize=11) + worker = app.AIWorker() + done, logs = [], [] + worker.page_done_sig.connect(lambda i, d: done.append((i, d))) + worker.log_sig.connect(logs.append) + enabled = app.expand_group_ids(app.ALL_GROUP_IDS) + worker.run_scan(doc, QMutex(), "IT", [], set(), False, None, enabled, True) + assert len(done) == 1, logs + dets = done[0][1] + texts = {d["text"] for d in dets} + assert "Mario Rossi" in texts and "Napoli" in texts and "12/03/1970" in texts + assert not texts & {"Certifico", "Prescrivo"} + assert all("state" in d and d["rects"] for d in dets) + assert any(l.startswith("Scansione completata") for l in logs) + worker.cleanup() + + +def test_worker_ocr_of_embedded_images_keeps_native_text(tmp_path, monkeypatch): + """A text page with an image: OCR adds the image text, native text stays.""" + spacy = pytest.importorskip("spacy") + pytest.importorskip("presidio_analyzer") + if not spacy.util.is_package("it_core_news_md"): + pytest.skip("model missing") + tessdata = app.find_tessdata_dir() + if not tessdata: + pytest.skip("tessdata missing") + from PySide6.QtCore import QMutex + + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + monkeypatch.setenv("HOME", str(tmp_path)) + QApplication.instance() or QApplication([]) + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Il Sig. Mario Rossi abita a Napoli.", fontsize=12) + src = pymupdf.open() + sp = src.new_page(width=300, height=60) + sp.insert_text((10, 40), "Dott. Giuseppe Verdi", fontsize=22) + page.insert_image(pymupdf.Rect(72, 200, 372, 260), pixmap=sp.get_pixmap(dpi=150)) + worker = app.AIWorker() + done, logs = [], [] + worker.page_done_sig.connect(lambda i, d: done.append(d)) + worker.log_sig.connect(logs.append) + enabled = app.expand_group_ids(app.ALL_GROUP_IDS) + worker.run_scan(doc, QMutex(), "IT", [], set(), True, tessdata, enabled, False) + texts = {" ".join(d["text"].split()) for d in done[0]} + assert "Mario Rossi" in texts, (texts, logs) + assert any("Giuseppe Verdi" in t for t in texts), (texts, logs) + assert any("OCR immagini" in l for l in logs) + worker.cleanup() + + +def test_roundtrip_with_trimmed_rectangles_on_tight_leading(window, tmp_path, monkeypatch): + """11 pt text on a 12 pt pitch: rectangles are trimmed, restore still works.""" + w = window + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + for i, t in enumerate([ + "Commissione tributaria provinciale", + "Paziente Alessandro Marchetti nato oggi", + "Programmazione generale dei servizi", + ]): + pg.insert_text((72, 100 + 12 * i), t, fontsize=11) + w.doc = d + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out = tmp_path / "t.pdf" + mapf = tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Commissione tributaria provinciale" in text + assert "Programmazione generale dei servizi" in text + assert "Alessandro" not in text + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + restored = tmp_path / "r.pdf" + assert app.reconstruct_pdf(str(out), str(restored), payload) == 2 + back = pymupdf.open(str(restored)) + t2 = back[0].get_text() + back.close() + assert "Alessandro" in t2 and "Marchetti" in t2 + assert "Commissione tributaria provinciale" in t2 + + +def test_image_redaction_is_not_duplicated_by_later_applies(window, tmp_path): + w = window + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + pg.insert_text((72, 100), "Alessandro Marchetti", fontsize=11) + pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 40, 40), False) + pix.set_rect(pix.irect, (10, 10, 10)) + pg.insert_image(pymupdf.Rect(300, 300, 340, 340), pixmap=pix) + w.doc = d + w.chk_img.setChecked(True) + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + w.apply_ai_to_page(0, [_detection(w, "Marchetti", "PERSON", "accepted")]) + assert len(_redacts(w)) == 3 # two words + one image, not two images + + +def test_invalid_pseudonym_key_file_is_refused(window, monkeypatch): + w = window + (w.config_dir).mkdir(parents=True, exist_ok=True) + (w.config_dir / "pseudonym.key").write_bytes(b"short") + shown = [] + monkeypatch.setattr(QMessageBox, "exec", lambda self: 0) + monkeypatch.setattr( + QMessageBox, "clickedButton", lambda self: next( + b for b in self.buttons() if b.text() == "Coerente (chiave)" + ), + ) + monkeypatch.setattr( + QMessageBox, "critical", staticmethod(lambda *a, **k: shown.append(a[2])) + ) + assert w._select_pseudonym_style() is None + assert shown and "32 byte" in shown[0] + + +def test_signature_suggestion_is_never_applied_without_review_and_export_removes_ink(window, tmp_path, monkeypatch): + from pdf_sanitize import suggest_signature_areas + + w = window + w.doc.close() + import test_detection_pipeline as tdp + + w.doc = tdp._signature_doc() + sug = suggest_signature_areas(w.doc[0]) + assert len(sug) == 1 + w.apply_ai_to_page(0, sug) + assert _redacts(w) == [] and len(w.pending_suggestions) == 1 # not applied + monkeypatch.setattr( + ReviewDialog, "exec", + lambda self: (self.table.item(0, 0).setCheckState(app.Qt.CheckState.Checked), + self._on_apply(), QDialog.DialogCode.Accepted)[2], + ) + w._review_suggestions() + assert len(_redacts(w)) == 1 + out = tmp_path / "sig.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + w.cmd_export() + exported = pymupdf.open(str(out)) + region = pymupdf.Rect(*sug[0]["rects"][0]) + inside = [ + d for d in exported[0].get_drawings() + if region.contains(d["rect"]) and any(it[0] == "c" for it in d["items"]) + ] # the grey fill rectangle itself is expected + exported.close() + assert inside == [] + + +def test_dictionary_dialog_sets_expiry_and_persists(window, monkeypatch): + import datetime + from dictionary_dialog import DictionaryDialog + + w = window + dlg = DictionaryDialog(w.list_manager) + dlg.ttl.setCurrentIndex(1) # 30 giorni + dlg.add_term("blocklist", "Mario Rossi") + dlg.add_term("allowlist", "comune") + dlg.save() + assert dlg.saved_blocklist == {"mario rossi"} and dlg.saved_allowlist == {"comune"} + entries = w.list_manager.load_entries("blocklist") + assert entries[0].expires == datetime.date.today() + datetime.timedelta(days=30) + assert not (w.config_dir / "blocklist.txt").exists() + assert b"rossi" not in (w.config_dir / "blocklist.v2.enc").read_bytes() + + +def test_dictionary_dialog_rejects_bad_dates_and_keeps_data(window, monkeypatch): + from dictionary_dialog import DictionaryDialog + + w = window + w.list_manager.save_blocklist({"mario rossi"}) + dlg = DictionaryDialog(w.list_manager) + dlg.tables["blocklist"].item(0, 2).setText("domani") + warned = [] + monkeypatch.setattr(QMessageBox, "warning", staticmethod(lambda *a, **k: warned.append(a[2]))) + dlg.save() + assert warned and dlg.saved_blocklist is None + assert w.list_manager.load_blocklist() == {"mario rossi"} + + +def test_cmd_dict_updates_in_memory_lists(window, monkeypatch): + import dictionary_dialog as dd + + w = window + + def fake_exec(self): + self.add_term("blocklist", "nuovo termine") + self.save() + return QDialog.DialogCode.Accepted + + monkeypatch.setattr(dd.DictionaryDialog, "exec", fake_exec) + w.cmd_dict() + assert w.blocklist == {"nuovo termine"} + + +# --- protected pseudonym key, recovery key ------------------------------------------ + +def test_plaintext_pseudonym_key_is_validated_sealed_and_removed(window): + w = window + w.config_dir.mkdir(parents=True, exist_ok=True) + legacy = w.config_dir / "pseudonym.key" + legacy.write_bytes(b"k" * 32) + key = w._load_pseudonym_key() + assert key == b"k" * 32 + assert not legacy.exists() + enc = (w.config_dir / "pseudonym.v2.enc").read_bytes() + assert b"k" * 32 not in enc + assert w._load_pseudonym_key() == key + + +def test_new_pseudonym_key_is_created_sealed(window): + w = window + key = w._load_pseudonym_key() + assert len(key) == 32 and w._load_pseudonym_key() == key + assert key not in (w.config_dir / "pseudonym.v2.enc").read_bytes() + assert not (w.config_dir / "pseudonym.key").exists() + + +def test_pseudonym_key_export_and_import_on_another_profile(window, tmp_path, monkeypatch): + w = window + key = w._load_pseudonym_key() + out = tmp_path / "k.nullifypdf-key" + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(out), ""))) + monkeypatch.setattr(QMessageBox, "exec", lambda self: 0) + monkeypatch.setattr( + QMessageBox, "clickedButton", + lambda self: next(b for b in self.buttons() if b.text() == "Esporta chiave..."), + ) + w._pseudonym_key_dialog() + assert out.exists() and key not in out.read_bytes() + # simulate a new machine: wipe the stored key, then import + (w.config_dir / "pseudonym.v2.enc").unlink() + monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod(lambda *a, **k: (str(out), ""))) + monkeypatch.setattr(QInputDialog, "getText", staticmethod(lambda *a, **k: ("una frase molto lunga 2026", True))) + monkeypatch.setattr( + QMessageBox, "clickedButton", + lambda self: next(b for b in self.buttons() if b.text() == "Importa chiave..."), + ) + w._pseudonym_key_dialog() + assert w._load_pseudonym_key() == key + + +def test_export_with_recovery_key_and_reconstruct_by_recovery(window, tmp_path, monkeypatch): + from privacy_core import generate_recovery_key, parse_recovery_key + + w = window + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out, mapf = tmp_path / "p.pdf", tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + raw, shown = generate_recovery_key() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: raw) + w.cmd_export() + blob = mapf.read_bytes() + assert raw not in blob and shown.encode() not in blob + for f in tmp_path.rglob("*"): + if f.is_file(): + assert shown.encode() not in f.read_bytes() + env = json.loads(blob) + assert [x["type"] for x in env["wraps"]] == ["password", "recovery"] + # reconstruct through the recovery key only + restored = tmp_path / "r.pdf" + monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod( + lambda *a, **k: (str(out), "") if "pseudonimizzato" in a[1] else (str(mapf), ""))) + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(restored), ""))) + monkeypatch.setattr(w, "_ask_decrypt_secret", lambda: (None, parse_recovery_key(shown))) + w.cmd_reconstruct() + back = pymupdf.open(str(restored)) + assert "Alessandro" in back[0].get_text() + back.close() + + +def test_wrong_secret_gives_one_clear_message(window, tmp_path, monkeypatch): + w = window + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + out, mapf = tmp_path / "p.pdf", tmp_path / "m.nullifypdf-map" + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + shown = [] + monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod( + lambda *a, **k: (str(out), "") if "pseudonimizzato" in a[1] else (str(mapf), ""))) + monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(tmp_path / "x.pdf"), ""))) + monkeypatch.setattr(w, "_ask_decrypt_secret", lambda: ("password completamente sbagliata", None)) + monkeypatch.setattr(QMessageBox, "critical", staticmethod(lambda *a, **k: shown.append(a[2]))) + w.cmd_reconstruct() + assert shown and "recupero" in shown[0] + + +def test_wipe_keys_makes_encrypted_lists_unreadable_only_with_confirmation(window, monkeypatch): + w = window + w.list_manager.save_blocklist({"mario rossi"}) + answers = iter([QMessageBox.StandardButton.Yes, QMessageBox.StandardButton.Yes]) + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: next(answers))) + w.cmd_wipe_local_data() + assert not (w.config_dir / "keystore.json").exists() + assert not (w.config_dir / "blocklist.v2.enc").exists() + + +# --- preview overlay of pending suggestions (real mouse events) ------------------------ + +def _click_scene_point(w, scene_pt): + from PySide6.QtCore import QEvent, QPointF, Qt + from PySide6.QtGui import QMouseEvent + + vp = w.view.mapFromScene(scene_pt) + gp = QPointF(vp) + glob = QPointF(w.view.viewport().mapToGlobal(vp)) + for typ, fn, btns in ( + (QEvent.Type.MouseButtonPress, w.view.mousePressEvent, Qt.MouseButton.LeftButton), + (QEvent.Type.MouseButtonRelease, w.view.mouseReleaseEvent, Qt.MouseButton.NoButton), + ): + fn(QMouseEvent(typ, gp, glob, Qt.MouseButton.LeftButton, btns, Qt.KeyboardModifier.NoModifier)) + QApplication.processEvents() + # the view defers a lone click slightly (to tell it from a double click) + import time + deadline = time.time() + 1.0 + while time.time() < deadline: + QApplication.processEvents() + time.sleep(0.02) + + +def _overlays(w): + return [i for i in w.scene.items() if isinstance(i, app.SuggestionItem)] + + +def _pick(monkeypatch, label): + monkeypatch.setattr(QMessageBox, "exec", lambda self: 0) + monkeypatch.setattr( + QMessageBox, "clickedButton", + lambda self: next(b for b in self.buttons() if b.text() == label), + ) + + +def _two_page_doc(w): + w.doc.close() + d = pymupdf.open() + for t in ("Pagina uno Zorzi presente", "Pagina due Colombi presente"): + d.new_page().insert_text((72, 100), t, fontsize=11) + w.doc = d + w.page_num = 0 + w.render_page() + + +def _sugg(w, page, word): + idx = app.build_page_index(w.doc[page].get_text("words")) + return {"text": word, "entity_type": "PERSON", "score": 0.5, "page": page, + "rects": [list(app.whole_word_rects(idx, word)[0])], "reasons": ["lexicon"], + "state": "suggested"} + + +def test_overlay_only_on_the_suggestion_page_and_follows_zoom(window): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi"), _sugg(w, 1, "Colombi")] + w.render_page() + assert len(_overlays(w)) == 1 + r0 = _overlays(w)[0].rect() + w.adjust_zoom(+2) + r1 = _overlays(w)[0].rect() + assert r1.width() == pytest.approx(r0.width() * w.scale / (w.scale - 0.5), rel=1e-6) + w.move_page(1) + assert len(_overlays(w)) == 1 and _overlays(w)[0].suggestion["text"] == "Colombi" + w.chk_sugg.setChecked(False) + assert _overlays(w) == [] + + +def test_overlay_click_accept_creates_one_redaction(window, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w._update_review_button() + w.render_page() + _pick(monkeypatch, "Censura") + _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center()) + assert len(_redacts(w)) == 1 and w.pending_suggestions == [] + assert w.btn_review.text() == "Rivedi suggerimenti (0)" + assert _overlays(w) == [] + + +def test_overlay_click_reject_discards_without_annotation(window, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w.render_page() + _pick(monkeypatch, "Scarta") + _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center()) + assert _redacts(w) == [] and w.pending_suggestions == [] + + +def test_overlay_cancel_keeps_the_suggestion(window, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w.render_page() + _pick(monkeypatch, "Annulla") + _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center()) + assert len(w.pending_suggestions) == 1 and _redacts(w) == [] + + +def test_overlay_never_reaches_the_export(window, tmp_path, monkeypatch): + w = window + _two_page_doc(w) + w.pending_suggestions = [_sugg(w, 0, "Zorzi")] + w.render_page() + assert _redacts(w) == [] + pix = w.doc[0].get_pixmap() + from PySide6.QtWidgets import QMessageBox as QMB + + out = tmp_path / "o.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + monkeypatch.setattr(QMB, "exec", lambda self: 0) + monkeypatch.setattr(QMB, "clickedButton", lambda self: next( + b for b in self.buttons() if b.text() == "Esporta comunque")) + w.cmd_export() + exported = pymupdf.open(str(out)) + assert "Zorzi" in exported[0].get_text() # not censored: still only a suggestion + exported.close() + assert pix.width > 0 + + +def test_review_row_selection_highlights_the_overlay(window): + w = window + _two_page_doc(w) + s = _sugg(w, 0, "Zorzi") + w.pending_suggestions = [s] + w.render_page() + dlg = ReviewDialog(w.pending_suggestions) + dlg.highlight.connect(w._highlight_suggestion) + dlg.table.setCurrentCell(0, 2) + assert _overlays(w)[0].pen().width() == 4 + + +# --- review findings: clip text, mixed glyph heights, 9 pt round trip, recovery ----------- + +def _tight_doc(w, pitch=9.0): + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + for i, t in enumerate([ + "Commissione tributaria provinciale", + "Paziente Alessandro Marchetti nato oggi", + "Programmazione generale dei servizi", + ]): + pg.insert_text((72, 100 + pitch * i), t, fontsize=11) + w.doc = d + w.page_num = 0 + w.render_page() + + +@pytest.mark.parametrize("pitch", [9.0, 10.0, 12.0]) +def test_manual_rectangle_stores_only_its_own_line(window, tmp_path, monkeypatch, pitch): + """A box drawn over a word at tight leading must not capture the neighbours.""" + from PySide6.QtCore import QRectF + + w = window + _tight_doc(w, pitch) + idx = app.build_page_index(w.doc[0].get_text("words"), clamp=False) # raw word box + x0, y0, x1, y1 = app.whole_word_rects(idx, "Alessandro")[0] + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: QMessageBox.StandardButton.No)) + w.user_draw_rect(QRectF(x0 * w.scale, y0 * w.scale, (x1 - x0) * w.scale, (y1 - y0) * w.scale)) + annots = _redacts(w) + assert len(annots) == 1 + assert w._read_redaction_payload(annots[0])["original"] == "Alessandro" + out, mapf = tmp_path / "m.pdf", tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + assert {e["original"] for e in payload["entries"]} == {"Alessandro"} + + +def test_pseudonymize_round_trip_at_9pt_keeps_neighbours_and_counts(window, tmp_path, monkeypatch): + w = window + _tight_doc(w, 9.0) + w.apply_ai_to_page(0, [ + _detection(w, "Alessandro", "PERSON", "accepted"), + _detection(w, "Marchetti", "PERSON", "accepted"), + ]) + out, mapf = tmp_path / "t9.pdf", tmp_path / "maps" / "m.nullifypdf-map" + mapf.parent.mkdir() + _stub_dialogs(monkeypatch, [str(out), str(mapf)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Commissione tributaria provinciale" in text + assert "Programmazione generale dei servizi" in text + assert "Alessandro" not in text + payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026") + restored = tmp_path / "r9.pdf" + n = app.reconstruct_pdf(str(out), str(restored), payload) + assert n == app.expected_restore_count(payload) == 2 + back = pymupdf.open(str(restored)) + t2 = back[0].get_text() + back.close() + assert "Alessandro" in t2 and "Marchetti" in t2 + assert t2.count("Commissione tributaria provinciale") == 1 + assert "Programmazione generale dei servizi" in t2 + + +def test_word_with_a_taller_glyph_falls_back_to_the_full_box(window): + """A thin band that misses one glyph of the target must not be used.""" + w = window + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + pg.insert_text((72, 100), "Riga sopra del testo", fontsize=11) + pg.insert_text((72, 109), "Codice AB", fontsize=11) + pg.insert_text((72 + 62, 109 - 5), "123", fontsize=6) # raised, smaller trailing digits + pg.insert_text((72, 118), "Riga sotto del testo", fontsize=11) + w.doc = d + idx = app.build_page_index(d[0].get_text("words")) + fulls = app.whole_word_rects(idx, "AB") + band = fitz_rect = pymupdf.Rect(*fulls[0]) + full = pymupdf.Rect(band.x0, band.y0 - 6, band.x1 + 30, band.y1) + thin = pymupdf.Rect(full.x0, full.y0 + 9, full.x1, full.y0 + 10) # misses the raised digits + assert not app.band_covers_target(d[0], thin, full) + assert app.band_covers_target(d[0], full, full) + + +def test_recovery_wrap_is_kept_even_if_the_dialog_is_closed_without_ticking(window, monkeypatch): + w = window + monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: QMessageBox.StandardButton.Yes)) + monkeypatch.setattr(QDialog, "exec", lambda self: QDialog.DialogCode.Rejected) + raw = w._offer_recovery_key() + assert isinstance(raw, bytes) and len(raw) == 32 diff --git a/tests/test_privacy_hardening.py b/tests/test_privacy_hardening.py new file mode 100644 index 0000000..b5ec5ad --- /dev/null +++ b/tests/test_privacy_hardening.py @@ -0,0 +1,261 @@ +"""Restore-map hardening and pseudonym styles.""" + +import base64 +import json +import os + +import pytest + +pytest.importorskip("cryptography") + +import privacy_core as pc + + +def _payload(): + reg = pc.PlaceholderRegistry() + reg.placeholder_for( + "Mario Rossi", "PERSON", page=0, rect=(1, 2, 3, 4), + style={"font": "x", "size": 10.0, "color": 0, "flags": 0, + "origin": [1.0, 2.0], "text": "Mario Rossi"}, + ) + return pc.build_restore_payload( + source_name="C:/docs/Referto_Mario_Rossi.pdf", source_sha256="a", + output_sha256="b", entries=reg.entries(), + ) + + +def test_payload_omits_source_name_and_duplicated_style_text(): + payload = _payload() + assert payload["source_name"] == "" + occ = payload["entries"][0]["occurrences"][0] + assert "text" not in occ["style"] + assert pc.parse_restore_entries(payload)[0].original == "Mario Rossi" + kept = pc.build_restore_payload( + source_name="a/b.pdf", source_sha256="", output_sha256=None, + entries=[], keep_source_name=True, + ) + assert kept["source_name"] == "b.pdf" + + +def test_style_text_is_kept_when_it_differs(): + reg = pc.PlaceholderRegistry() + reg.placeholder_for("Mario", "PERSON", rect=(0, 0, 1, 1), + style={"font": "", "size": 9.0, "origin": [0, 0], "text": "Mario Rossi"}) + occ = pc.build_restore_payload( + source_name="", source_sha256="", output_sha256=None, entries=reg.entries() + )["entries"][0]["occurrences"][0] + assert occ["style"]["text"] == "Mario Rossi" + + +def test_new_maps_are_v3_and_roundtrip(): + blob = pc.encrypt_restore_payload(_payload(), "una frase molto lunga 2026") + env = json.loads(blob) + assert env["version"] == 3 and env["cipher"] == "AES-256-GCM" + assert [w["type"] for w in env["wraps"]] == ["password"] + assert env["wraps"][0]["kdf"] in ("argon2id", "scrypt") + assert pc.decrypt_restore_payload(blob, "una frase molto lunga 2026")["format"] == pc.RESTORE_MAP_FORMAT + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(blob, "una frase sbagliata 2026!") + + +def test_v2_scrypt_maps_still_decrypt(): + blob = pc._encrypt_v2(_payload(), "una frase molto lunga 2026") + assert json.loads(blob)["version"] == 2 + assert pc.decrypt_restore_payload(blob, "una frase molto lunga 2026")["version"] == pc.RESTORE_MAP_VERSION + + +def _legacy_pbkdf2_blob(password, iterations): + from cryptography.fernet import Fernet + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + + salt = b"0123456789abcdef" + key = base64.urlsafe_b64encode( + PBKDF2HMAC(hashes.SHA256(), 32, salt, iterations).derive(password.encode()) + ) + token = Fernet(key).encrypt(json.dumps(_payload()).encode()) + return json.dumps({ + "format": "NullifyPDF encrypted restore map", "version": 1, + "kdf": "PBKDF2-HMAC-SHA256", "iterations": iterations, + "salt": base64.b64encode(salt).decode(), "token": token.decode(), + }).encode() + + +def test_legacy_pbkdf2_maps_still_decrypt(): + blob = _legacy_pbkdf2_blob("legacy password 123", 600000) + assert pc.decrypt_restore_payload(blob, "legacy password 123")["version"] == pc.RESTORE_MAP_VERSION + + +@pytest.mark.parametrize("iterations", [1, 99_999, 5_000_001, 10 ** 12]) +def test_untrusted_iteration_counts_are_rejected_before_derivation(iterations): + blob = _legacy_pbkdf2_blob("legacy password 123", 600000) + env = json.loads(blob) + env["iterations"] = iterations + with pytest.raises(ValueError): + pc.decrypt_restore_payload(json.dumps(env).encode(), "legacy password 123") + + +@pytest.mark.parametrize("n,r,p", [(2 ** 30, 8, 1), (1000, 8, 1), (2 ** 17, 1000, 1), (2 ** 17, 8, 99)]) +def test_untrusted_scrypt_parameters_are_rejected(n, r, p): + env = json.loads(pc._encrypt_v2(_payload(), "una frase molto lunga 2026")) + env.update(n=n, r=r, p=p) + with pytest.raises(ValueError): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + + +def test_unknown_kdf_is_rejected(): + env = json.loads(pc._encrypt_v2(_payload(), "una frase molto lunga 2026")) + env["kdf"] = "md5" + with pytest.raises(ValueError): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + + +@pytest.mark.parametrize("pw,ok", [ + ("short1!", False), + ("aaaaaaaaaaaaaaaa", False), + ("passwordpassword1", False), + ("abcdefghijklm", False), + ("Abcdefghijklm", True), + ("una frase molto lunga davvero", True), + ("Xk3!mP9#qR2$", True), +]) +def test_password_strength(pw, ok): + assert pc.password_strength(pw)[0] is ok + + +def test_encrypt_still_requires_minimum_length(): + with pytest.raises(ValueError): + pc.encrypt_restore_payload(_payload(), "short") + + +def test_counter_style_is_unchanged(): + reg = pc.PlaceholderRegistry() + assert reg.placeholder_for("A Uno", "PERSON") == "PERSON_001" + assert reg.placeholder_for("B Due", "PERSON") == "PERSON_002" + assert reg.placeholder_for("a uno", "PERSON") == "PERSON_001" + + +def test_random_style_reveals_neither_order_nor_count(): + values = [f"Persona Numero{i}" for i in range(30)] + a, b = pc.PlaceholderRegistry("random"), pc.PlaceholderRegistry("random") + pa = [a.placeholder_for(v, "PERSON") for v in values] + pb = [b.placeholder_for(v, "PERSON") for v in values] + assert len(set(pa)) == 30 + assert pa != pb # fresh randomness per document + assert all(p.startswith("PERSON_") and len(p) >= len("PERSON_") + 4 for p in pa) + assert pa == [a.placeholder_for(v, "PERSON") for v in values] # stable within a document + assert pa != sorted(pa) or pb != sorted(pb) + + +def test_opaque_style_hides_the_type(): + reg = pc.PlaceholderRegistry("opaque") + assert reg.placeholder_for("Mario", "PERSON").startswith("DATA_") + assert reg.entries()[0].entity_type == "PERSON" # kept inside the encrypted map + + +def test_keyed_style_is_consistent_per_key_only(): + k1, k2 = b"k" * 32, b"j" * 32 + a = pc.PlaceholderRegistry("keyed", k1).placeholder_for("Mario Rossi", "PERSON") + b = pc.PlaceholderRegistry("keyed", k1).placeholder_for("mario rossi", "PERSON") + c = pc.PlaceholderRegistry("keyed", k2).placeholder_for("Mario Rossi", "PERSON") + assert a == b and a != c + assert pc.PlaceholderRegistry("keyed", k1).placeholder_for("Mario Rossi", "LOCATION") != a + with pytest.raises(ValueError): + pc.PlaceholderRegistry("keyed") + with pytest.raises(ValueError): + pc.PlaceholderRegistry("nonsense") + + +def test_unknown_or_foreign_envelope_gives_a_clear_error(): + blob = pc._encrypt_v2(_payload(), "una frase molto lunga 2026") + env = json.loads(blob) + env["version"] = 9 + with pytest.raises(ValueError, match="non supportata"): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + env["format"] = "something else" + with pytest.raises(ValueError, match="non riconosciuto"): + pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026") + + +# --- envelope v3: recovery key, Argon2id, key export ----------------------------- + +PW = "una frase molto lunga 2026" + + +def test_v3_password_and_recovery_open_the_same_payload(): + raw, shown = pc.generate_recovery_key() + blob = pc.encrypt_restore_payload(_payload(), PW, recovery_key=raw) + assert [w["type"] for w in json.loads(blob)["wraps"]] == ["password", "recovery"] + by_pw = pc.decrypt_restore_payload(blob, PW) + by_rk = pc.decrypt_restore_payload(blob, None, recovery_key=pc.parse_recovery_key(shown)) + assert by_pw == by_rk + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(blob, None, recovery_key=os.urandom(32)) + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(blob) # nothing supplied + + +def test_recovery_key_format_and_checksum(): + raw, shown = pc.generate_recovery_key() + assert shown.startswith("NPDF-") and len(shown.replace("-", "")) == 4 + 54 + assert pc.parse_recovery_key(shown.lower().replace("-", " ")) == raw # forgiving of case/spacing + typo = shown[:8] + ("A" if shown[8] != "A" else "B") + shown[9:] + with pytest.raises(ValueError, match="digitata|non valida"): + pc.parse_recovery_key(typo) + with pytest.raises(ValueError): + pc.parse_recovery_key("NPDF-ABCD") + + +def test_v3_wrap_type_swap_fails(): + raw, _ = pc.generate_recovery_key() + env = json.loads(pc.encrypt_restore_payload(_payload(), PW, recovery_key=raw)) + env["wraps"][1]["type"] = "password" # pretend the recovery wrap is a password wrap + env["wraps"].pop(0) + from cryptography.fernet import InvalidToken + with pytest.raises((InvalidToken, KeyError, ValueError)): + pc.decrypt_restore_payload(json.dumps(env).encode(), PW) + + +def test_v3_tampering_is_detected(): + env = json.loads(pc.encrypt_restore_payload(_payload(), PW)) + ct = bytearray(base64.b64decode(env["ciphertext"])) + ct[0] ^= 1 + env["ciphertext"] = base64.b64encode(bytes(ct)).decode() + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.decrypt_restore_payload(json.dumps(env).encode(), PW) + + +@pytest.mark.skipif(not pc._argon2_available(), reason="Argon2id not available in this OpenSSL") +@pytest.mark.parametrize("field,value", [ + ("t", 0), ("t", 11), ("m", 1024), ("m", 2 ** 21), ("p", 0), ("p", 9), +]) +def test_argon2id_bounds_are_enforced_before_derivation(field, value): + env = json.loads(pc.encrypt_restore_payload(_payload(), PW)) + assert env["wraps"][0]["kdf"] == "argon2id" + env["wraps"][0][field] = value + with pytest.raises(ValueError, match="limiti"): + pc.decrypt_restore_payload(json.dumps(env).encode(), PW) + + +def test_argon2_unavailable_falls_back_to_scrypt_for_writing(monkeypatch): + monkeypatch.setattr(pc, "_argon2_available", lambda: False) + blob = pc.encrypt_restore_payload(_payload(), PW) + assert json.loads(blob)["wraps"][0]["kdf"] == "scrypt" + assert pc.decrypt_restore_payload(blob, PW)["format"] == pc.RESTORE_MAP_FORMAT + + +def test_pseudonym_key_export_import_roundtrip_and_wrong_password(): + key = os.urandom(32) + blob = pc.export_pseudonym_key(key, PW) + assert key not in blob and base64.b64encode(key) not in blob + assert pc.import_pseudonym_key(blob, PW) == key + from cryptography.fernet import InvalidToken + with pytest.raises(InvalidToken): + pc.import_pseudonym_key(blob, "un'altra frase lunga 2026") + with pytest.raises(ValueError): + pc.import_pseudonym_key(pc.encrypt_restore_payload(_payload(), PW), PW) # a map, not a key + with pytest.raises(ValueError): + pc.export_pseudonym_key(b"short", PW) diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py index 2448049..aa01a33 100644 --- a/tests/test_reconstruct.py +++ b/tests/test_reconstruct.py @@ -248,9 +248,11 @@ def test_reconstruct_v1_map_finds_wrapped_placeholder(tmp_path): assert "MarioRossi" in "".join(text.split()) -def test_reconstruct_does_not_report_values_it_failed_to_write(tmp_path): - """An original too long for its box is dropped by `apply_redactions()`. - That must never be counted as a restored value.""" +def test_reconstruct_shrinks_a_long_value_instead_of_dropping_it(tmp_path): + """An original too long for its box used to be dropped by + `apply_redactions()` (no text below 4 pt). It is now written after the + redactions with TextWriter, so it is back in the file (and counted only + because it really is).""" from NullifyPDF import sha256_file doc = fitz.open() @@ -276,8 +278,10 @@ def test_reconstruct_does_not_report_values_it_failed_to_write(tmp_path): ], } out_path = tmp_path / "reconstructed.pdf" - # The value cannot fit the tight box; the honest answer is zero. - assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 0 + assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 1 + back = fitz.open(str(out_path)) + assert "UNBREAKABLE" in "".join(back[0].get_text().split()) + back.close() def test_expected_restore_count_counts_occurrences_not_entries(tmp_path): @@ -481,3 +485,34 @@ def test_embedded_font_is_actually_reused(tmp_path): ) span = _span_for(out, "Maria Bianchi") assert "arial" in span["font"].lower() + + +def test_compact_labels_chain_and_tiny_box_placeholder_survives(tmp_path): + """A box too small for the full placeholder still gets a legible label.""" + from NullifyPDF import compact_labels, write_label + + assert compact_labels("PERSON_9F2A") == ["PERSON_9F2A", "P9F2A", "9F2A", "#9F"] + doc = fitz.open() + page = doc.new_page() + page.insert_text((72, 100), "Re", fontsize=11) + rect = fitz.Rect(page.get_text("words")[0][:4]) + page.add_redact_annot(rect, fill=(1, 1, 1)) + page.apply_redactions() + drawn = write_label(page, rect, compact_labels("PERSON_9F2A")) + assert drawn != "PERSON_9F2A" + assert drawn in page.get_text() + + +def test_v4_map_keeps_label_and_v3_still_loads(): + from privacy_core import parse_restore_entries + + reg = PlaceholderRegistry() + ph = reg.placeholder_for("Re", "PERSON", rect=(1, 2, 3, 4)) + reg.set_label(ph, (1, 2, 3, 4), "P001") + payload = build_restore_payload( + source_name="", source_sha256="", output_sha256=None, entries=reg.entries() + ) + assert payload["version"] == 4 + assert parse_restore_entries(payload)[0].occurrences[0].label == "P001" + payload["version"] = 3 + assert parse_restore_entries(payload)[0].original == "Re" diff --git a/tests/test_secure_store.py b/tests/test_secure_store.py new file mode 100644 index 0000000..fed997d --- /dev/null +++ b/tests/test_secure_store.py @@ -0,0 +1,160 @@ +"""Encrypted word lists with expiry, and the key-protection layer.""" + +import datetime +import json +import os +import sys + +import pytest + +pytest.importorskip("cryptography") + +import secure_store +from word_lists import ListEntry, PDFListManager + + +@pytest.fixture(autouse=True) +def _fresh_cache(): + secure_store.forget_cached_keys() + yield + secure_store.forget_cached_keys() + + +def test_seal_open_roundtrip_and_aad_binding(): + key = os.urandom(32) + blob = secure_store.seal(b"dati", key, b"ctx-a") + assert secure_store.open_sealed(blob, key, b"ctx-a") == b"dati" + with pytest.raises(secure_store.SealError): + secure_store.open_sealed(blob, key, b"ctx-b") + with pytest.raises(secure_store.SealError): + secure_store.open_sealed(blob[:-1] + bytes([blob[-1] ^ 1]), key, b"ctx-a") + with pytest.raises(secure_store.SealError): + secure_store.open_sealed(b"short", key, b"ctx-a") + + +@pytest.mark.skipif(sys.platform != "win32", reason="DPAPI is Windows-only") +def test_dpapi_roundtrip_and_account_binding(tmp_path): + p = secure_store.DpapiProtector() + key = os.urandom(32) + assert p.unwrap(p.wrap(key)) == key + assert key not in json.dumps(p.wrap(key)).encode() + + +def test_keyfile_protector_creates_private_file(tmp_path): + p = secure_store.KeyFileProtector(tmp_path) + key = os.urandom(32) + blob = p.wrap(key) + assert p.unwrap(blob) == key + mode = (tmp_path / "keys" / "master.key").stat().st_mode & 0o777 + if sys.platform != "win32": + assert mode == 0o600 + + +def test_master_key_is_stable_and_not_stored_in_clear_in_keystore(tmp_path): + k1 = secure_store.load_or_create_master_key(tmp_path) + secure_store.forget_cached_keys() + k2 = secure_store.load_or_create_master_key(tmp_path) + assert k1 == k2 and len(k1) == 32 + ks = (tmp_path / "keystore.json").read_bytes() + if json.loads(ks)["master"]["protector"] == "dpapi": + assert k1 not in ks + assert "DPAPI" in secure_store.protector_label(tmp_path) + + +def test_keyring_unavailable_falls_back_to_keyfile(tmp_path, monkeypatch): + monkeypatch.setattr(secure_store, "DpapiProtector", type("D", (), {"available": staticmethod(lambda: False)})) + monkeypatch.setattr(secure_store.KeyringProtector, "available", staticmethod(lambda: False)) + secure_store.load_or_create_master_key(tmp_path) + assert json.loads((tmp_path / "keystore.json").read_text())["master"]["protector"] == "keyfile" + + +def test_plaintext_lists_migrate_transparently(tmp_path): + (tmp_path / "blocklist.txt").write_text("mario rossi\nvia roma\nab", encoding="utf-8") + (tmp_path / "allowlist.txt").write_text("comune", encoding="utf-8") + m = PDFListManager(tmp_path) + assert m.load_blocklist() == {"mario rossi", "via roma"} + assert m.load_allowlist() == {"comune"} + assert not (tmp_path / "blocklist.txt").exists() + enc = (tmp_path / "blocklist.v2.enc").read_bytes() + assert b"rossi" not in enc and enc.startswith(b"NPDFL2\n") + assert PDFListManager(tmp_path).load_blocklist() == {"mario rossi", "via roma"} # reload + + +def test_save_roundtrip_filters_short_terms_and_keeps_dates(tmp_path): + day = {"now": datetime.date(2026, 1, 1)} + m = PDFListManager(tmp_path, today=lambda: day["now"]) + m.save_blocklist({"abc", "ab", "Mario Rossi"}) + assert m.load_blocklist() == {"abc", "mario rossi"} + day["now"] = datetime.date(2026, 3, 1) + m.save_blocklist({"abc", "nuovo termine"}) + entries = {e.term: e for e in m.load_entries("blocklist")} + assert entries["abc"].added == datetime.date(2026, 1, 1) # kept + assert entries["nuovo termine"].added == datetime.date(2026, 3, 1) + + +def test_expired_entries_are_dropped_and_purged(tmp_path): + day = {"now": datetime.date(2026, 1, 1)} + m = PDFListManager(tmp_path, today=lambda: day["now"]) + m.save_entries("blocklist", [ + ListEntry("scade presto", day["now"], datetime.date(2026, 1, 10)), + ListEntry("per sempre", day["now"], None), + ]) + day["now"] = datetime.date(2026, 1, 11) + assert m.load_blocklist() == {"per sempre"} + assert m.expired_removed == 1 + day["now"] = datetime.date(2025, 12, 1) # purged for good, not merely hidden + assert PDFListManager(tmp_path, today=lambda: day["now"]).load_blocklist() == {"per sempre"} + + +def test_default_ttl_applies_to_new_terms(tmp_path): + m = PDFListManager(tmp_path, today=lambda: datetime.date(2026, 1, 1)) + m.set_default_ttl_days(30) + m.save_blocklist({"nuovo termine"}) + e = m.load_entries("blocklist")[0] + assert e.expires == datetime.date(2026, 1, 31) + + +def test_unreadable_keystore_degrades_without_data_loss(tmp_path): + m = PDFListManager(tmp_path) + m.save_blocklist({"mario rossi"}) + before = (tmp_path / "blocklist.v2.enc").read_bytes() + ks = json.loads((tmp_path / "keystore.json").read_text()) + if ks["master"]["protector"] == "dpapi": + ks["master"]["blob"] = "AAAA" + else: + (tmp_path / "keys" / "master.key").write_bytes(b"x" * 5) + (tmp_path / "keystore.json").write_text(json.dumps(ks)) + secure_store.forget_cached_keys() + m2 = PDFListManager(tmp_path) + assert m2.load_blocklist() == set() + assert m2.keystore_error + m2.save_blocklist({"altro termine"}) # must not overwrite anything + assert (tmp_path / "blocklist.v2.enc").read_bytes() == before + + +def test_no_cryptography_keeps_plaintext(tmp_path, monkeypatch): + import builtins + + real = builtins.__import__ + + def fake(name, *a, **k): + if name == "cryptography" or name.startswith("cryptography."): + raise ImportError(name) + return real(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", fake) + m = PDFListManager(tmp_path) + m.save_blocklist({"mario rossi"}) + monkeypatch.setattr(builtins, "__import__", real) + assert (tmp_path / "blocklist.txt").read_text(encoding="utf-8") == "mario rossi" + assert PDFListManager(tmp_path).load_blocklist() == {"mario rossi"} + + +def test_tampered_list_file_is_not_trusted(tmp_path): + m = PDFListManager(tmp_path) + m.save_blocklist({"mario rossi"}) + p = tmp_path / "blocklist.v2.enc" + raw = bytearray(p.read_bytes()) + raw[-1] ^= 1 + p.write_bytes(bytes(raw)) + assert PDFListManager(tmp_path).load_blocklist() == set() diff --git a/word_lists.py b/word_lists.py new file mode 100644 index 0000000..665aad5 --- /dev/null +++ b/word_lists.py @@ -0,0 +1,257 @@ +"""Blocklist / allowlist persistence: encrypted at rest, optional expiry (Qt-free). + +The lists can contain personal data (names the user always wants removed or +always wants kept), so they are stored sealed with the master key of +:mod:`secure_store` in ``blocklist.v2.enc`` / ``allowlist.v2.enc``. Each +entry carries the date it was added and an optional expiry date; expired +entries are dropped (and the file rewritten) when the list is loaded. + +Compatibility: an existing plaintext ``blocklist.txt`` / ``allowlist.txt`` is +read, migrated transparently and then overwritten and removed. If encryption +is unavailable (no ``cryptography``, no usable protector) the plaintext files +keep working. If the keystore cannot be unwrapped (profile copied to another +account) nothing is deleted or overwritten and the lists are empty until the +keystore is restored. +""" + +from __future__ import annotations + +import datetime +import json +import logging +import pathlib +from dataclasses import dataclass +from typing import Callable, Dict, List, Optional, Set + +import secure_store + +MIN_TERM_LENGTH = 3 +_MAGIC = b"NPDFL2\n" +_FORMAT = "NullifyPDF word list" +_log = logging.getLogger("nullifypdf") + + +@dataclass +class ListEntry: + term: str + added: datetime.date + expires: Optional[datetime.date] = None + + def to_json(self) -> Dict[str, Optional[str]]: + return { + "term": self.term, + "added": self.added.isoformat(), + "expires": self.expires.isoformat() if self.expires else None, + } + + +def _clean(term: str) -> str: + return " ".join(term.strip().lower().split()) + + +def _date(value: Optional[str]) -> Optional[datetime.date]: + if not value: + return None + try: + return datetime.date.fromisoformat(value) + except ValueError: + return None + + +class PDFListManager: + """Loads and saves the blocklist and allowlist.""" + + def __init__( + self, + config_dir: pathlib.Path, + today: Callable[[], datetime.date] = datetime.date.today, + ) -> None: + self.config_dir = pathlib.Path(config_dir) + self.config_dir.mkdir(parents=True, exist_ok=True) + self.block_file = self.config_dir / "blocklist.txt" + self.allow_file = self.config_dir / "allowlist.txt" + self._today = today + self.expired_removed = 0 + self.keystore_error: Optional[str] = None + self.encrypted = True + + # ------------------------------------------------------------ paths + def _enc_path(self, kind: str) -> pathlib.Path: + return self.config_dir / f"{kind}.v2.enc" + + def _plain_path(self, kind: str) -> pathlib.Path: + return self.block_file if kind == "blocklist" else self.allow_file + + # ------------------------------------------------------- settings + def default_ttl_days(self) -> Optional[int]: + try: + data = json.loads((self.config_dir / "settings.json").read_text(encoding="utf-8")) + ttl = data.get("list_ttl_days") + return int(ttl) if ttl else None + except Exception: + return None + + def set_default_ttl_days(self, days: Optional[int]) -> None: + path = self.config_dir / "settings.json" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception: + data = {} + data["list_ttl_days"] = days + secure_store.atomic_write(path, json.dumps(data).encode("utf-8")) + + def protector_label(self) -> str: + if not self.encrypted: + return "Non cifrato" + return secure_store.protector_label(self.config_dir) + + # ------------------------------------------------------- entries + def load_entries(self, kind: str) -> List[ListEntry]: + enc, plain = self._enc_path(kind), self._plain_path(kind) + if enc.exists(): + entries = self._read_encrypted(kind, enc) + kept = [e for e in entries if not (e.expires and e.expires < self._today())] + if len(kept) != len(entries): + self.expired_removed += len(entries) - len(kept) + self._write_encrypted(kind, kept) + return kept + if plain.exists(): + entries = self._read_plain(plain) + if self._can_encrypt(): + try: + self._write_encrypted(kind, entries) + secure_store.secure_delete(plain) + except Exception as e: # keep the readable file on any failure + _log.error(f"List migration failed: {e}") + self.encrypted = False + else: + self.encrypted = False + return entries + return [] + + def save_entries(self, kind: str, entries: List[ListEntry]) -> None: + clean: Dict[str, ListEntry] = {} + for e in entries: + term = _clean(e.term) + if len(term) >= MIN_TERM_LENGTH: + clean[term] = ListEntry(term, e.added, e.expires) + final = sorted(clean.values(), key=lambda e: e.term) + if self._can_encrypt(): + try: + self._write_encrypted(kind, final) + plain = self._plain_path(kind) + if plain.exists(): + secure_store.secure_delete(plain) + return + except secure_store.KeystoreUnavailable as e: + self.keystore_error = str(e) + raise + except Exception as e: + _log.error(f"Encrypted list save failed: {e}") + self.encrypted = False + try: + secure_store.atomic_write( + self._plain_path(kind), "\n".join(e.term for e in final).encode("utf-8") + ) + except OSError as e: + _log.error(f"Error saving {kind}: {e}") + + # -------------------------------------------- set-based convenience + def load_blocklist(self) -> Set[str]: + return self._terms("blocklist") + + def load_allowlist(self) -> Set[str]: + return self._terms("allowlist") + + def save_blocklist(self, blocklist: Set[str]) -> None: + self._save_terms("blocklist", blocklist) + + def save_allowlist(self, allowlist: Set[str]) -> None: + self._save_terms("allowlist", allowlist) + + def _terms(self, kind: str) -> Set[str]: + try: + return {e.term for e in self.load_entries(kind)} + except secure_store.KeystoreUnavailable as e: + self.keystore_error = str(e) + _log.error("Word lists unreadable: %s", e) + return set() + except Exception as e: + _log.error(f"Error loading {kind}: {e}") + return set() + + def _save_terms(self, kind: str, words: Set[str]) -> None: + """Save a set of terms, keeping dates of the ones already present.""" + try: + existing = {e.term: e for e in self.load_entries(kind)} + except secure_store.KeystoreUnavailable: + existing = {} + except Exception as e: # corrupt / tampered file: start from the new set + _log.error(f"Unreadable {kind} file ignored: {e}") + existing = {} + ttl = self.default_ttl_days() + today = self._today() + entries: List[ListEntry] = [] + for w in words: + term = _clean(w) + if len(term) < MIN_TERM_LENGTH: + continue + old = existing.get(term) + if old: + entries.append(old) + else: + exp = today + datetime.timedelta(days=ttl) if ttl else None + entries.append(ListEntry(term, today, exp)) + try: + self.save_entries(kind, entries) + except secure_store.KeystoreUnavailable: + pass # error kept in self.keystore_error; nothing is overwritten + + # ------------------------------------------------------ plumbing + def _can_encrypt(self) -> bool: + try: + import cryptography # noqa: F401 + + secure_store.load_or_create_master_key(self.config_dir) + return True + except secure_store.KeystoreUnavailable as e: + self.keystore_error = str(e) + raise + except Exception: + return False + + def _read_plain(self, path: pathlib.Path) -> List[ListEntry]: + try: + with open(path, "r", encoding="utf-8") as fh: + terms = {_clean(line) for line in fh if len(line.strip()) >= MIN_TERM_LENGTH} + except Exception as e: + _log.error(f"Error loading {path}: {e}") + return [] + today = self._today() + return [ListEntry(t, today, None) for t in sorted(terms) if len(t) >= MIN_TERM_LENGTH] + + def _read_encrypted(self, kind: str, path: pathlib.Path) -> List[ListEntry]: + key = secure_store.load_or_create_master_key(self.config_dir) + raw = path.read_bytes() + if not raw.startswith(_MAGIC): + raise ValueError("Formato del dizionario non riconosciuto.") + data = json.loads( + secure_store.open_sealed(raw[len(_MAGIC):], key, b"nullifypdf-list:" + kind.encode()) + ) + out = [] + for item in data.get("entries", []): + term = _clean(str(item.get("term", ""))) + if len(term) < MIN_TERM_LENGTH: + continue + out.append(ListEntry( + term, _date(item.get("added")) or self._today(), _date(item.get("expires")) + )) + return out + + def _write_encrypted(self, kind: str, entries: List[ListEntry]) -> None: + key = secure_store.load_or_create_master_key(self.config_dir) + payload = json.dumps({ + "format": _FORMAT, "version": 2, "entries": [e.to_json() for e in entries], + }, ensure_ascii=False).encode("utf-8") + sealed = secure_store.seal(payload, key, b"nullifypdf-list:" + kind.encode()) + secure_store.atomic_write(self._enc_path(kind), _MAGIC + sealed) From f038f6dba0d3476b69bdd010a9f69730faf7c0c0 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Sun, 8 Nov 2026 21:12:27 +0100 Subject: [PATCH 2/8] test: unregister per-window atexit cleanup so the GUI tests exit cleanly --- tests/test_gui_flow.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py index ad3c8db..d1fe2de 100644 --- a/tests/test_gui_flow.py +++ b/tests/test_gui_flow.py @@ -5,6 +5,8 @@ isolated so the user's real ~/.nullifypdf is never touched. """ +import atexit +import gc import json import os @@ -45,7 +47,13 @@ def window(tmp_path, monkeypatch): w._cleanup_resources() except Exception: pass + # Every window registers an atexit cleanup; left registered, they run on + # already destroyed Qt objects at interpreter exit and crash the process + # (seen as a segmentation fault after the last test on Linux). + atexit.unregister(w._cleanup_on_exit) + w.deleteLater() qapp.processEvents() + gc.collect() def _rect_of(w, word): From 8f354ea8a113cc792f51e2a912f2f2e548caff6f Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:23:53 +0100 Subject: [PATCH 3/8] ci: temporary per-test exit diagnosis --- .github/workflows/test_build.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index df1acb5..02736e7 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -111,7 +111,13 @@ jobs: pip install --upgrade pymupdf python -c "import pymupdf; print(pymupdf.VersionBind, pymupdf.VersionFitz)" >> "$GITHUB_STEP_SUMMARY" - - name: Run Tests + - name: Diagnose exit status per test (temporary) shell: bash run: | - pytest tests/ -v -k "geometry or reconstruct or export or gui_flow" || echo "::warning title=PyMuPDF latest::tests failed on $(python -c 'import pymupdf;print(pymupdf.VersionBind)')" + for f in tests/test_export_sanitize.py tests/test_geometry.py tests/test_reconstruct.py tests/test_secure_store.py tests/test_privacy_hardening.py tests/test_detection_pipeline.py; do + pytest "$f" -q >/dev/null 2>&1; echo "FILE $f exit=$?" + done + for t in $(pytest tests/test_gui_flow.py --collect-only -q 2>/dev/null | grep "::"); do + pytest "$t" -q >/dev/null 2>&1; rc=$?; [ $rc -ne 0 ] && echo "GUI $t exit=$rc" + done + echo "diagnosis finished" From 5a06e249181ead12f705913b07cf9ebec88acd07 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:26:19 +0100 Subject: [PATCH 4/8] ci: temporary per-test exit diagnosis 2 --- .github/workflows/test_build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 02736e7..a0c48b2 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -112,12 +112,12 @@ jobs: python -c "import pymupdf; print(pymupdf.VersionBind, pymupdf.VersionFitz)" >> "$GITHUB_STEP_SUMMARY" - name: Diagnose exit status per test (temporary) - shell: bash + shell: bash {0} run: | - for f in tests/test_export_sanitize.py tests/test_geometry.py tests/test_reconstruct.py tests/test_secure_store.py tests/test_privacy_hardening.py tests/test_detection_pipeline.py; do - pytest "$f" -q >/dev/null 2>&1; echo "FILE $f exit=$?" - done - for t in $(pytest tests/test_gui_flow.py --collect-only -q 2>/dev/null | grep "::"); do - pytest "$t" -q >/dev/null 2>&1; rc=$?; [ $rc -ne 0 ] && echo "GUI $t exit=$rc" - done + pytest tests/test_gui_flow.py --collect-only -q 2>/dev/null | grep "::" > ids.txt + wc -l ids.txt + while read -r t; do + pytest "$t" -q >/dev/null 2>&1; rc=$? + echo "GUI rc=$rc $t" + done < ids.txt echo "diagnosis finished" From 354de93f4b9e0b4746b6fb64ceb6f25c692423a6 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:28:36 +0100 Subject: [PATCH 5/8] ci: temporary diagnosis 3 --- .github/workflows/test_build.yml | 9 ++---- tests/test_zz_diag.py | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 tests/test_zz_diag.py diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index a0c48b2..e5ea07d 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -114,10 +114,7 @@ jobs: - name: Diagnose exit status per test (temporary) shell: bash {0} run: | - pytest tests/test_gui_flow.py --collect-only -q 2>/dev/null | grep "::" > ids.txt - wc -l ids.txt - while read -r t; do - pytest "$t" -q >/dev/null 2>&1; rc=$? - echo "GUI rc=$rc $t" - done < ids.txt + for t in a_construct_only b_show_and_threshold c_navigate_connect d_navigate_lambda e_item_access; do + pytest "tests/test_zz_diag.py::test_$t" -q >/dev/null 2>&1; echo "DIAG rc=$? $t" + done echo "diagnosis finished" diff --git a/tests/test_zz_diag.py b/tests/test_zz_diag.py new file mode 100644 index 0000000..1774d9f --- /dev/null +++ b/tests/test_zz_diag.py @@ -0,0 +1,48 @@ +import os +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +import pytest +pytest.importorskip("PySide6") +from PySide6.QtWidgets import QApplication +from review_dialog import ReviewDialog + +ITEM = [{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, "page": 0, "reasons": ["lexicon"]}] + + +def _app(): + return QApplication.instance() or QApplication([]) + + +def test_a_construct_only(): + _a = _app() + ReviewDialog(ITEM) + + +def test_b_show_and_threshold(): + _a = _app() + dlg = ReviewDialog(ITEM) + dlg.chk_show.setChecked(True) + dlg.spin.setValue(0.4) + dlg._select_threshold() + + +def test_c_navigate_connect(): + _a = _app() + dlg = ReviewDialog(ITEM) + got = [] + dlg.navigate.connect(got.append) + dlg._on_double_click(0, 0) + + +def test_d_navigate_lambda(): + _a = _app() + dlg = ReviewDialog(ITEM) + got = [] + dlg.navigate.connect(lambda p: got.append(p)) + dlg._on_double_click(0, 0) + + +def test_e_item_access(): + _a = _app() + dlg = ReviewDialog(ITEM) + dlg.table.item(0, 2).text() + dlg.table.item(0, 0).checkState() From a82b591b3d67ba3798f968aa0a224fc0f12f546a Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:30:28 +0100 Subject: [PATCH 6/8] ci: temporary diagnosis 4 --- .github/workflows/test_build.yml | 2 +- tests/test_zz_diag.py | 40 +++++++++++++++++++------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index e5ea07d..10cb81e 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -114,7 +114,7 @@ jobs: - name: Diagnose exit status per test (temporary) shell: bash {0} run: | - for t in a_construct_only b_show_and_threshold c_navigate_connect d_navigate_lambda e_item_access; do + for t in e1_text_only e2_checkstate_only e3_keep_dialog_alive e4_held_item_ref e5_cell_helper_twice e6_delete_later; do pytest "tests/test_zz_diag.py::test_$t" -q >/dev/null 2>&1; echo "DIAG rc=$? $t" done echo "diagnosis finished" diff --git a/tests/test_zz_diag.py b/tests/test_zz_diag.py index 1774d9f..96a2c9e 100644 --- a/tests/test_zz_diag.py +++ b/tests/test_zz_diag.py @@ -6,43 +6,51 @@ from review_dialog import ReviewDialog ITEM = [{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, "page": 0, "reasons": ["lexicon"]}] +KEEP = [] def _app(): return QApplication.instance() or QApplication([]) -def test_a_construct_only(): +def test_e1_text_only(): _a = _app() - ReviewDialog(ITEM) + dlg = ReviewDialog(ITEM) + dlg.table.item(0, 2).text() -def test_b_show_and_threshold(): +def test_e2_checkstate_only(): _a = _app() dlg = ReviewDialog(ITEM) - dlg.chk_show.setChecked(True) - dlg.spin.setValue(0.4) - dlg._select_threshold() + dlg.table.item(0, 0).checkState() -def test_c_navigate_connect(): +def test_e3_keep_dialog_alive(): _a = _app() dlg = ReviewDialog(ITEM) - got = [] - dlg.navigate.connect(got.append) - dlg._on_double_click(0, 0) + KEEP.append(dlg) + dlg.table.item(0, 2).text() + dlg.table.item(0, 0).checkState() -def test_d_navigate_lambda(): +def test_e4_held_item_ref(): _a = _app() dlg = ReviewDialog(ITEM) - got = [] - dlg.navigate.connect(lambda p: got.append(p)) - dlg._on_double_click(0, 0) + it = dlg.table.item(0, 2) + it.text() + del it -def test_e_item_access(): +def test_e5_cell_helper_twice(): _a = _app() dlg = ReviewDialog(ITEM) + dlg._cell(0, 2).text() + dlg._cell(0, 0).checkState() + + +def test_e6_delete_later(): + a = _app() + dlg = ReviewDialog(ITEM) dlg.table.item(0, 2).text() - dlg.table.item(0, 0).checkState() + dlg.deleteLater() + a.processEvents() From 93754c692594b64755cb3aa8c2130f272e29a082 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:33:26 +0100 Subject: [PATCH 7/8] fix: avoid dialog reference cycles and destroy Qt objects before exit --- .github/workflows/test_build.yml | 5 ++- dictionary_dialog.py | 22 +++++++++---- review_dialog.py | 13 ++++++-- tests/conftest.py | 23 +++++++++++++ tests/test_zz_diag.py | 56 -------------------------------- 5 files changed, 52 insertions(+), 67 deletions(-) delete mode 100644 tests/test_zz_diag.py diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 10cb81e..8fd7b3a 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -114,7 +114,6 @@ jobs: - name: Diagnose exit status per test (temporary) shell: bash {0} run: | - for t in e1_text_only e2_checkstate_only e3_keep_dialog_alive e4_held_item_ref e5_cell_helper_twice e6_delete_later; do - pytest "tests/test_zz_diag.py::test_$t" -q >/dev/null 2>&1; echo "DIAG rc=$? $t" - done + pytest tests/ -q >/dev/null 2>&1; echo "FULL rc=$?" + pytest tests/ -q >/dev/null 2>&1; echo "FULL again rc=$?" echo "diagnosis finished" diff --git a/dictionary_dialog.py b/dictionary_dialog.py index ae66454..a8e84a6 100644 --- a/dictionary_dialog.py +++ b/dictionary_dialog.py @@ -71,14 +71,16 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None: self.tables[kind] = table tl.addWidget(table) row = QHBoxLayout() - for label, slot in ( - ("Aggiungi", lambda _=False, k=kind: self.add_row(k)), - ("Rimuovi", lambda _=False, k=kind: self.remove_selected(k)), - ("Importa da testo…", lambda _=False, k=kind: self.import_text(k)), - ("Esporta in chiaro…", lambda _=False, k=kind: self.export_plain(k)), + for label, action in ( + ("Aggiungi", "add"), + ("Rimuovi", "remove"), + ("Importa da testo…", "import"), + ("Esporta in chiaro…", "export"), ): b = QPushButton(label) - b.clicked.connect(slot) + b.setProperty("kind", kind) + b.setProperty("action", action) + b.clicked.connect(self._on_button) # bound method: no self-referencing lambda row.addWidget(b) tl.addLayout(row) self.tabs.addTab(tab, title) @@ -101,6 +103,14 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None: btn.clicked.connect(self.save) lay.addWidget(btn) + def _on_button(self) -> None: + sender = self.sender() + if sender is None: + return + kind, action = str(sender.property("kind")), str(sender.property("action")) + {"add": self.add_row, "remove": self.remove_selected, + "import": self.import_text, "export": self.export_plain}[action](kind) + # -- rows --------------------------------------------------------- def _append(self, kind: str, entry: ListEntry) -> None: t = self.tables[kind] diff --git a/review_dialog.py b/review_dialog.py index 186e129..56e38b1 100644 --- a/review_dialog.py +++ b/review_dialog.py @@ -115,8 +115,11 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non self.spin.setSingleStep(0.05) self.spin.setValue(0.6) btn_thr = QPushButton("Seleziona ≥ soglia") - btn_all.clicked.connect(lambda: self._set_all(True)) - btn_none.clicked.connect(lambda: self._set_all(False)) + # Bound methods, not lambdas capturing `self`: a lambda makes a + # reference cycle that keeps the dialog alive until the garbage + # collector runs, possibly after QApplication is gone. + btn_all.clicked.connect(self._select_all) + btn_none.clicked.connect(self._select_none) btn_thr.clicked.connect(self._select_threshold) for w in (btn_all, btn_none, self.spin, btn_thr): bar.addWidget(w) @@ -149,6 +152,12 @@ def _refresh_values(self) -> None: continue self._cell(row, COL_VALUE).setText(text if show else mask_value(text)) + def _select_all(self) -> None: + self._set_all(True) + + def _select_none(self) -> None: + self._set_all(False) + def _set_all(self, state: bool) -> None: for row in range(self.table.rowCount()): self._cell(row, COL_APPLY).setCheckState( diff --git a/tests/conftest.py b/tests/conftest.py index 3e809b8..6e9ef1d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,3 +8,26 @@ root_str = str(PROJECT_ROOT) if root_str not in sys.path: sys.path.insert(0, root_str) + + +def pytest_sessionfinish(session, exitstatus): + """Destroy Qt objects while QApplication is still alive. + + Dialogs and windows left in reference cycles would otherwise be collected + at interpreter shutdown, after the application object, which crashes the + process on some platforms even though every test passed. + """ + try: + import gc + + from PySide6.QtWidgets import QApplication + + app = QApplication.instance() + if app is not None: + app.closeAllWindows() + app.processEvents() + gc.collect() + if app is not None: + app.processEvents() + except Exception: + pass diff --git a/tests/test_zz_diag.py b/tests/test_zz_diag.py deleted file mode 100644 index 96a2c9e..0000000 --- a/tests/test_zz_diag.py +++ /dev/null @@ -1,56 +0,0 @@ -import os -os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") -import pytest -pytest.importorskip("PySide6") -from PySide6.QtWidgets import QApplication -from review_dialog import ReviewDialog - -ITEM = [{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, "page": 0, "reasons": ["lexicon"]}] -KEEP = [] - - -def _app(): - return QApplication.instance() or QApplication([]) - - -def test_e1_text_only(): - _a = _app() - dlg = ReviewDialog(ITEM) - dlg.table.item(0, 2).text() - - -def test_e2_checkstate_only(): - _a = _app() - dlg = ReviewDialog(ITEM) - dlg.table.item(0, 0).checkState() - - -def test_e3_keep_dialog_alive(): - _a = _app() - dlg = ReviewDialog(ITEM) - KEEP.append(dlg) - dlg.table.item(0, 2).text() - dlg.table.item(0, 0).checkState() - - -def test_e4_held_item_ref(): - _a = _app() - dlg = ReviewDialog(ITEM) - it = dlg.table.item(0, 2) - it.text() - del it - - -def test_e5_cell_helper_twice(): - _a = _app() - dlg = ReviewDialog(ITEM) - dlg._cell(0, 2).text() - dlg._cell(0, 0).checkState() - - -def test_e6_delete_later(): - a = _app() - dlg = ReviewDialog(ITEM) - dlg.table.item(0, 2).text() - dlg.deleteLater() - a.processEvents() From a74618c0807c1c4b3e2c54c614d786bb39c0aa57 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:36:19 +0100 Subject: [PATCH 8/8] ci: run the full suite in the latest-PyMuPDF job and report failures honestly --- .github/workflows/test_build.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 8fd7b3a..f9249e7 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -81,7 +81,7 @@ jobs: pymupdf-latest: # Redaction semantics (the glyph-deletion core, word boxes) differ between # PyMuPDF releases; this job reports early when a newer release changes - # them. It never fails the workflow: it raises a visible warning. + # them. It is not a required check, but a failure is shown as a failure. name: Tests against the latest PyMuPDF runs-on: ubuntu-24.04 env: @@ -111,9 +111,5 @@ jobs: pip install --upgrade pymupdf python -c "import pymupdf; print(pymupdf.VersionBind, pymupdf.VersionFitz)" >> "$GITHUB_STEP_SUMMARY" - - name: Diagnose exit status per test (temporary) - shell: bash {0} - run: | - pytest tests/ -q >/dev/null 2>&1; echo "FULL rc=$?" - pytest tests/ -q >/dev/null 2>&1; echo "FULL again rc=$?" - echo "diagnosis finished" + - name: Run Tests + run: pytest tests/ -v