From 96d2e6dc025150974306566c792085f56a6cc2a9 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:23:01 +0200 Subject: [PATCH 1/8] feat: precise AI PII detection with entity selection dialog (#112) Offset-based detection mapped to word boxes, whole-word propagation, per-type thresholds and generic-word filtering, address splitting for names, entity selection dialog, pymupdf import and blocking mypy in CI. --- .github/workflows/beta-release.yml | 2 +- .github/workflows/test_build.yml | 2 +- ARCHITECTURE.md | 1 + CHANGELOG.md | 12 ++ NullifyPDF.py | 209 +++++++++++++------ PDF_Checker.py | 4 +- USER_GUIDE.md | 2 +- pii_detection.py | 325 +++++++++++++++++++++++++++++ tests/test_pii_detection.py | 129 ++++++++++++ tests/test_reconstruct.py | 2 +- 10 files changed, 614 insertions(+), 74 deletions(-) create mode 100644 pii_detection.py create mode 100644 tests/test_pii_detection.py diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml index f5a4a8b..5e0140c 100644 --- a/.github/workflows/beta-release.yml +++ b/.github/workflows/beta-release.yml @@ -47,7 +47,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py --ignore-missing-imports --no-error-summary 2>&1 | head -50 || true + run: mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py - name: Resolve beta suffix from tag if: startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index cb0aed2..2cfd83d 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -47,7 +47,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py --ignore-missing-imports --no-error-summary 2>&1 | head -50 || true + run: mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py - name: Build run: python build_local.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0bb4b33..226fe26 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -26,6 +26,7 @@ It supports two privacy export modes: | πŸ“„ File | ✨ Responsibility | | ------ | ----------------- | | `NullifyPDF.py` | Main PySide6 UI, PDF rendering, AI scan orchestration, redaction, export, reconstruction, OCR integration | +| `pii_detection.py` | Qt-free AI detection helpers: word-box text index, offset-to-rect mapping, per-type score thresholds, generic-word filtering, whole-word propagation, entity selection persistence | | `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads, restore-map parsing | | `PDF_Checker.py` | Heuristic post-export inspection helper | | `build_local.py` | PyInstaller build script | diff --git a/CHANGELOG.md b/CHANGELOG.md index 41db79d..521edf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### ✨ Added + +- **Choose what the AI scan looks for**: clicking "Auto Redact (AI)" now opens a dialog with one checkbox per data type (names, locations, email, phone, IBAN, credit cards, crypto wallets, fiscal code, VAT number, driver licence, ID card, passport). The choice is remembered in `~/.nullifypdf/ai_entities.json`. + +### πŸ› Fixed + +- **AI scan selected ordinary text and parts of words**: detections were reduced to bare strings and then re-searched page-wide with a case-insensitive substring search, so one NER hit on "Rossi" also redacted "Rossini", and a stray tag spread to every occurrence on the page. Detections now use Presidio's exact character offsets mapped to word boxes, other occurrences are matched as whole words only, weak recognizers are filtered by per-type confidence thresholds, and generic PERSON/LOCATION hits (titles, months, salutations, lowercase text) are dropped. Blocklist and allowlist matching is whole-word too. A name that the NER model glues to the following address ("Rossini Marco Via Garibaldi") is now cut at the street word or number; the address part is kept as a location only when that type is enabled. + +### πŸ”§ Changed + +- **Type checking is now a blocking CI step**: `mypy` runs on `NullifyPDF.py`, `pii_detection.py`, `privacy_core.py` and `PDF_Checker.py` without `--ignore-missing-imports` or `|| true`. PyMuPDF is imported as `pymupdf` (the `fitz` alias ships no type marker), so its usage is really type-checked. + ## [2.2.0] - 2026-09-18 Validated through the 2.2.0-beta.1/beta.2/beta.3/beta.4 prereleases before this stable release, promoted from `v2.2.0-beta.4` without rebuilding; only the final stable version gets its own heading here. diff --git a/NullifyPDF.py b/NullifyPDF.py index 6eecefa..7b0bd43 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -15,7 +15,7 @@ import json import math from typing import Optional, List, Set, Dict, Any, Tuple, TypedDict, NotRequired -import fitz +import pymupdf as fitz from PySide6.QtWidgets import ( QApplication, QMainWindow, @@ -32,6 +32,7 @@ QGraphicsScene, QFileDialog, QDialog, + QDialogButtonBox, QLineEdit, QButtonGroup, QGraphicsRectItem, @@ -60,6 +61,25 @@ parse_restore_entries, sort_entries_longest_placeholder_first, ) +from pii_detection import ( + ALL_GROUP_IDS, + ENTITY_GROUPS, + build_page_index, + clean_candidates, + expand_group_ids, + filter_results, + load_enabled_groups, + propagate_whole_word, + resolve_overlaps, + save_enabled_groups, + span_to_rects, + whole_word_rects, +) + +# PyMuPDF defines these constants dynamically, which mypy cannot see; read +# them once here (with the suppression) instead of at every call site. +PDF_ANNOT_REDACT: int = fitz.PDF_ANNOT_REDACT # type: ignore[attr-defined] +PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] __version__ = "2.2.0" __version_prerelease__ = "" @@ -75,6 +95,7 @@ class DetectionDict(TypedDict): entity_type: str rects: List[Tuple[float, float, float, float]] source: NotRequired[str] + score: NotRequired[float] def setup_logging() -> logging.Logger: @@ -295,7 +316,7 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int attempted: List[Tuple[int, fitz.Rect, str]] = [] doc = fitz.open(in_path) try: - for page in doc: + for page in doc.pages(): claimed: List[fitz.Rect] = [] for entry in entries: for rect in _restore_targets(page, entry): @@ -319,7 +340,7 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int fontsize=fit_redaction_fontsize(entry.original, rect), ) attempted.append((page.number, fitz.Rect(rect), entry.original)) - page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True) + page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) doc.save(out_path, garbage=4, deflate=True, clean=True) finally: doc.close() @@ -368,7 +389,7 @@ def _verify_restored( "Reconstruction: value not restored on page %s at %s " "(it did not fit its redaction box).", page_number + 1, - tuple(round(v, 1) for v in rect), + tuple(round(v, 1) for v in (rect.x0, rect.y0, rect.x1, rect.y1)), ) finally: doc.close() @@ -568,11 +589,12 @@ def cleanup(self) -> None: except Exception as e: logging.getLogger("nullifypdf").debug(f"Error during AI cleanup: {e}") - @Slot(object, object, str, list, set, bool, object) + @Slot(object, object, str, list, set, bool, object, list) def run_scan(self, doc: Any, doc_mutex: Any, choice: str, compiled_allowlist: List[Tuple[str, Any]], allowlist_set: Set[str], use_ocr: bool, - tessdata_dir: Optional[str]) -> None: + tessdata_dir: Optional[str], + enabled_types: List[str]) -> None: """Run AI scan on PDF pages and emit detected sensitive entities. Text extraction (`page.get_text()`) is performed inside this worker so @@ -588,6 +610,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, compiled_allowlist: Pre-compiled regex patterns to skip redaction. allowlist_set: Set of allowlist entries (lowercase) for O(1) exact-match fast-path lookup before regex any() scan. + enabled_types: Presidio entity types the user chose to detect. """ try: if self._stop_requested: @@ -660,22 +683,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, ) self.loaded_langs = target_langs self.log_sig.emit("Scansione privacy in corso...") - targets = [ - "PERSON", - "LOCATION", - "EMAIL_ADDRESS", - "PHONE_NUMBER", - "IBAN_CODE", - "CREDIT_CARD", - "CRYPTO", - # Italian national-ID formats (auto-loaded by Presidio only - # for language "it", harmless to always request). - "IT_FISCAL_CODE", - "IT_VAT_CODE", - "IT_DRIVER_LICENSE", - "IT_IDENTITY_CARD", - "IT_PASSPORT", - ] + targets = list(enabled_types) # Determine page count under mutex (cheap, but doc may be replaced # mid-flight by load_path if not for the disabled UI during scan). @@ -694,6 +702,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, page = doc[i] text = page.get_text() needs_ocr = use_ocr and self._page_needs_ocr(page, text) + index = build_page_index(page.get_text("words")) if needs_ocr: self.log_sig.emit(f"OCR pagina {i + 1}...") textpage = page.get_textpage_ocr( @@ -703,58 +712,62 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, tessdata=tessdata_dir, ) text = page.get_text(textpage=textpage) + index = build_page_index( + page.get_text("words", textpage=textpage) + ) used_ocr = True except Exception as e: self.log_sig.emit( f"Avviso: estrazione testo/OCR fallita pagina {i+1}: {e}" ) text = "" - found: Dict[str, str] = {} + index = build_page_index([]) + candidates: List[Any] = [] for lang in self.loaded_langs: + if not targets or not index.text: + break res = self.analyzer.analyze( - text=text, entities=targets, language=lang + text=index.text, entities=targets, language=lang ) - for r in res: - w = text[r.start : r.end].strip() - if len(w) > 2: - found[w] = r.entity_type + candidates.extend(filter_results(res, index.text, targets)) + candidates = resolve_overlaps( + clean_candidates(candidates, enabled_types=targets) + ) detections = [] - for m, entity_type in found.items(): + seen_values: Set[Tuple[str, str]] = set() + for c in candidates: + m, entity_type = c.text, c.entity_type clean = " ".join(m.strip(string.punctuation).lower().split()) if not clean: continue - # Fast path: exact set membership is O(1). The vast majority - # of user allowlist entries are full tokens matched verbatim - # against `clean`, so this short-circuits before the O(N) - # regex any() scan over the whole allowlist. + # Fast path: exact set membership is O(1). if clean in allowlist_set: continue - # Cache the word-boundary regex for `clean` so we don't - # rebuild and recompile it once per allowlist entry. - clean_pattern = re.compile(r"\b" + re.escape(clean) + r"\b") - if not any( + clean_pattern = re.compile(r"" + re.escape(clean) + r"") + if any( f_reg.search(clean) or clean_pattern.search(a_str) for a_str, f_reg in compiled_allowlist ): - detection: DetectionDict = { - "text": m, - "entity_type": entity_type, - "rects": [], - "source": "ocr" if used_ocr else "text", - } - if used_ocr and textpage is not None: - with QMutexLocker(doc_mutex): - try: - rects = doc[i].search_for(m, textpage=textpage) - detection["rects"] = [ - (r.x0, r.y0, r.x1, r.y1) for r in rects - ] - except Exception as e: - self.log_sig.emit( - f"Avviso: coordinate OCR non disponibili " - f"pagina {i+1}: {e}" - ) - detections.append(detection) + continue + rects = span_to_rects(index, c.start, c.end) + if (m, entity_type) not in seen_values: + seen_values.add((m, entity_type)) + # Other occurrences of the same value: whole words only. + for extra in propagate_whole_word( + index, m, + case_sensitive=entity_type in ("PERSON", "LOCATION"), + ): + if extra not in rects: + rects.append(extra) + if not rects: + continue + detections.append({ + "text": m, + "entity_type": entity_type, + "rects": rects, + "source": "ocr" if used_ocr else "text", + "score": c.score, + }) self.page_done_sig.emit(i, detections) self.progress_sig.emit(i + 1, total_pages) if not self._stop_requested: @@ -866,7 +879,7 @@ class NullifyPDF(QMainWindow): config_dir: Path to user config directory (~/.nullifypdf). """ - start_scan_sig = Signal(object, object, str, list, set, bool, object) + start_scan_sig = Signal(object, object, str, list, set, bool, object, list) def __init__(self) -> None: """Initialize main application window and setup UI.""" @@ -1348,7 +1361,7 @@ def user_click_pt(self, qpt: QPointF) -> None: ans = [ a for a in (p.annots() or []) - if a.type[0] == fitz.PDF_ANNOT_REDACT and a.rect.contains(pt) + if a.type[0] == PDF_ANNOT_REDACT and a.rect.contains(pt) ] if not ans: return @@ -1388,7 +1401,7 @@ def user_click_pt(self, qpt: QPointF) -> None: ans = [ a for a in (p.annots() or []) - if a.type[0] == fitz.PDF_ANNOT_REDACT and a.rect.contains(pt) + if a.type[0] == PDF_ANNOT_REDACT and a.rect.contains(pt) ] for a in ans: p.delete_annot(a) @@ -1411,7 +1424,7 @@ def cmd_clear(self) -> None: # generator returned by p.annots(), and list-comprehension-for- # side-effects is an anti-pattern (PEP 8 / pylint W0106). to_delete = [ - a for a in (p.annots() or []) if a.type[0] == fitz.PDF_ANNOT_REDACT + a for a in (p.annots() or []) if a.type[0] == PDF_ANNOT_REDACT ] for a in to_delete: p.delete_annot(a) @@ -1500,6 +1513,9 @@ def cmd_auto_ai(self) -> None: """ if not self.doc: return + enabled_groups = self._select_ai_entities() + if not enabled_groups: + return self.btn_ai.setEnabled(False) self.prog.setValue(0) c_allow = [ @@ -1532,8 +1548,60 @@ def cmd_auto_ai(self) -> None: set(self.allowlist), use_ocr, tessdata_dir, + expand_group_ids(enabled_groups), ) + def _select_ai_entities(self) -> Optional[List[str]]: + """Ask which data types the AI scan should look for. + + Returns the chosen group ids, or None if the user cancels. The + selection is remembered in ``ai_entities.json``. + """ + store = self.config_dir / "ai_entities.json" + previous = set(load_enabled_groups(store)) + dlg = QDialog(self) + dlg.setWindowTitle("Tipi di dati da rilevare") + lay = QVBoxLayout(dlg) + lay.addWidget(QLabel("Seleziona i dati da censurare / pseudonimizzare:")) + boxes: Dict[str, QCheckBox] = {} + for gid, label, _types in ENTITY_GROUPS: + cb = QCheckBox(label) + cb.setChecked(gid in previous) + boxes[gid] = cb + lay.addWidget(cb) + row = QHBoxLayout() + btn_all = QPushButton("Seleziona tutto") + btn_none = QPushButton("Deseleziona tutto") + row.addWidget(btn_all) + row.addWidget(btn_none) + lay.addLayout(row) + bb = QDialogButtonBox( + QDialogButtonBox.StandardButton.Ok + | QDialogButtonBox.StandardButton.Cancel + ) + lay.addWidget(bb) + ok = bb.button(QDialogButtonBox.StandardButton.Ok) + + def refresh() -> None: + ok.setEnabled(any(cb.isChecked() for cb in boxes.values())) + + for cb in boxes.values(): + cb.toggled.connect(refresh) + def set_all(state: bool) -> None: + for cb in boxes.values(): + cb.setChecked(state) + + btn_all.clicked.connect(lambda: set_all(True)) + btn_none.clicked.connect(lambda: set_all(False)) + bb.accepted.connect(dlg.accept) + bb.rejected.connect(dlg.reject) + refresh() + if dlg.exec() != QDialog.DialogCode.Accepted: + return None + chosen = [gid for gid in ALL_GROUP_IDS if boxes[gid].isChecked()] + save_enabled_groups(store, chosen) + return chosen + def _select_privacy_export(self) -> Optional[PrivacyMode]: """Ask the user which privacy export mode to use.""" box = QMessageBox(self) @@ -1583,7 +1651,7 @@ def _collect_redaction_rects(self, page: Any) -> List[fitz.Rect]: """Return redaction rectangles for a page.""" return [ a.rect for a in (page.annots() or []) - if a.type[0] == fitz.PDF_ANNOT_REDACT + if a.type[0] == PDF_ANNOT_REDACT ] def _add_privacy_redaction( @@ -1634,10 +1702,10 @@ def _prepare_pseudonymized_page( page.get_text("text", clip=annot.rect).strip(), ) for annot in (page.annots() or []) - if annot.type[0] == fitz.PDF_ANNOT_REDACT + if annot.type[0] == PDF_ANNOT_REDACT ] for annot in list(page.annots() or []): - if annot.type[0] == fitz.PDF_ANNOT_REDACT: + if annot.type[0] == PDF_ANNOT_REDACT: page.delete_annot(annot) for rect, payload, clipped_text in pending: clean_original = " ".join( @@ -1700,7 +1768,7 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: page = self.doc[i] e_rects = [ a.rect for a in (page.annots() or []) - if a.type[0] == fitz.PDF_ANNOT_REDACT + if a.type[0] == PDF_ANNOT_REDACT ] if self.chk_img.isChecked(): for img in page.get_image_info(hashes=False): @@ -1713,8 +1781,9 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: fontsize=8, ) e_rects.append(ir) + page_index = build_page_index(page.get_text("words")) for bw in self.blocklist: - for r in page.search_for(bw): + for r in (fitz.Rect(*t) for t in whole_word_rects(page_index, bw)): if not any( e.contains(fitz.Point((r.x0 + r.x1) / 2, (r.y0 + r.y1) / 2)) for e in e_rects @@ -1726,7 +1795,11 @@ def apply_ai_to_page(self, i: int, detections: Any) -> None: entity_type=self._infer_entity_type(bw), ) e_rects.append(r) - p_rects = [r for aw in self.allowlist for r in page.search_for(aw)] + p_rects = [ + fitz.Rect(*t) + for aw in self.allowlist + for t in whole_word_rects(page_index, aw) + ] for detection in normalized_detections: word = str(detection.get("text", "")) entity_type = str(detection.get("entity_type", "DATA")) @@ -1847,7 +1920,7 @@ def cmd_export(self) -> None: ex_doc = fitz.open(tmp_path) # Step 3: scrub on the disk-backed copy. - for page in ex_doc: + for page in ex_doc.pages(): if mode == PrivacyMode.PSEUDONYMIZE: self._prepare_pseudonymized_page(page, registry) # page.annots() may return None for pages with no annotations @@ -1871,7 +1944,7 @@ def cmd_export(self) -> None: # whole-image redactions from "Oscura Immagini", including # under page rotation, with no visible fringe). page.apply_redactions( - images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True + images=PDF_REDACT_IMAGE_PIXELS, graphics=True ) try: # Materialize first: mutating during iteration of diff --git a/PDF_Checker.py b/PDF_Checker.py index 2abf41f..c313101 100644 --- a/PDF_Checker.py +++ b/PDF_Checker.py @@ -4,7 +4,7 @@ traces of sensitive data in text layers, metadata, and binary data. """ -import fitz +import pymupdf as fitz import os import logging from typing import Optional @@ -66,7 +66,7 @@ def inspect_pdf(file_path: str, target_word: str) -> None: alerts += 1 # 1B. Text Layer Check - for i, page in enumerate(doc): + for i, page in enumerate(doc.pages()): text = page.get_text().lower() if word_lower in text: print( diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 2613ef7..8d3701f 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -48,7 +48,7 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel ## Step 3️⃣ β€” Automatic Redaction (AI Scan) -Click the **"Auto Redact (AI)"** button. NullifyPDF will automatically find and flag: +Click the **"Auto Redact (AI)"** button. A dialog first lets you tick which data types to look for (your choice is remembered). NullifyPDF will then automatically find and flag the selected types; other occurrences of a detected value are matched as whole words only ("Rossi" does not match "Rossini"): - πŸ§‘ **Names & Surnames** - πŸ™οΈ **Cities & Addresses** diff --git a/pii_detection.py b/pii_detection.py new file mode 100644 index 0000000..f6214c2 --- /dev/null +++ b/pii_detection.py @@ -0,0 +1,325 @@ +"""Pure (Qt-free) helpers that make AI PII detection precise. + +Presidio reports character offsets into the string it analyzed. This module +builds that string from PyMuPDF word boxes so the offsets map back to exact +rectangles, filters weak or generic hits, and propagates a detected value to +its other occurrences as *whole words* only. +""" + +import json +import pathlib +import re +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +Rect4 = Tuple[float, float, float, float] + +# (entity id, Italian label) shown in the selection dialog. An id may cover +# several Presidio entity types. +ENTITY_GROUPS: List[Tuple[str, str, Tuple[str, ...]]] = [ + ("PERSON", "Nomi e cognomi", ("PERSON",)), + ("LOCATION", "Luoghi e indirizzi", ("LOCATION",)), + ("EMAIL", "Email", ("EMAIL_ADDRESS",)), + ("PHONE", "Telefono", ("PHONE_NUMBER",)), + ("IBAN", "IBAN", ("IBAN_CODE",)), + ("CARD", "Carte di credito", ("CREDIT_CARD",)), + ("CRYPTO", "Wallet crypto", ("CRYPTO",)), + ("FISCAL", "Codice fiscale", ("IT_FISCAL_CODE",)), + ("VAT", "Partita IVA", ("IT_VAT_CODE",)), + ("LICENSE", "Patente", ("IT_DRIVER_LICENSE",)), + ("IDCARD", "Carta d'identitΓ ", ("IT_IDENTITY_CARD",)), + ("PASSPORT", "Passaporto", ("IT_PASSPORT",)), +] +ALL_GROUP_IDS: List[str] = [g[0] for g in ENTITY_GROUPS] +ALL_ENTITY_TYPES: List[str] = [t for g in ENTITY_GROUPS for t in g[2]] + +# Minimum Presidio score per entity type. Checksum-backed recognizers score +# 1.0 when valid; weak pattern recognizers score low without context words. +ENTITY_SCORE_THRESHOLDS: Dict[str, float] = { + "PERSON": 0.5, + "LOCATION": 0.5, + "EMAIL_ADDRESS": 0.5, + "PHONE_NUMBER": 0.4, + "IBAN_CODE": 0.3, + "CREDIT_CARD": 0.3, + "CRYPTO": 0.3, + "IT_FISCAL_CODE": 0.3, + "IT_VAT_CODE": 0.3, + "IT_DRIVER_LICENSE": 0.5, + "IT_IDENTITY_CARD": 0.4, + "IT_PASSPORT": 0.3, +} + +_NER_TYPES = {"PERSON", "LOCATION"} + +_GENERIC_WORDS = { + "sig", "sig.ra", "sigra", "signor", "signora", "signore", "dott", "dott.ssa", + "dottssa", "dr", "dra", "prof", "prof.ssa", "ing", "avv", "geom", "rag", + "spett", "spett.le", "spettle", "gentile", "egregio", "egregia", "oggetto", + "cliente", "fattura", "via", "viale", "piazza", "corso", "largo", "vicolo", + "mr", "mrs", "ms", "miss", "dear", "subject", "re", "cc", "pec", + "gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", + "agosto", "settembre", "ottobre", "novembre", "dicembre", + "lunedΓ¬", "martedΓ¬", "mercoledΓ¬", "giovedΓ¬", "venerdΓ¬", "sabato", "domenica", + "january", "february", "march", "april", "may", "june", "july", "august", + "september", "october", "november", "december", + "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", + "italia", "italy", +} +_HONORIFICS = { + "sig", "sig.", "sig.ra", "signor", "signora", "signore", "dott", "dott.", + "dott.ssa", "dr", "dr.", "prof", "prof.", "prof.ssa", "ing", "ing.", "avv", + "avv.", "geom", "geom.", "rag", "rag.", "mr", "mr.", "mrs", "mrs.", "ms", + "ms.", "spett.le", "gentile", "egregio", "egregia", +} +_STREET_WORDS = { + "via", "viale", "v.le", "piazza", "p.za", "piazzale", "corso", "c.so", + "largo", "vicolo", "strada", "str.", "contrada", "localitΓ ", "loc.", + "frazione", "fraz.", "borgo", "lungotevere", "c/o", "street", "st.", + "avenue", "road", "rd.", "lane", +} +_EDGE_PUNCT = " \t\r\n.,;:!?()[]{}<>\"'Β«Β»β€œβ€β€˜β€™" + + +@dataclass +class WordBox: + start: int + end: int + rect: Rect4 + line_key: Tuple[int, int] + + +@dataclass +class PageTextIndex: + """Analysis text plus a map from character offsets to word rectangles.""" + + text: str = "" + words: List[WordBox] = field(default_factory=list) + + +@dataclass +class Candidate: + start: int + end: int + text: str + entity_type: str + score: float + rects: List[Rect4] = field(default_factory=list) + + +def build_page_index(words: Iterable[Sequence[Any]]) -> PageTextIndex: + """Build the analysis string from `page.get_text("words")` tuples.""" + ordered = sorted( + (w for w in words if str(w[4]).strip()), + key=lambda w: (int(w[5]), int(w[6]), int(w[7])), + ) + parts: List[str] = [] + boxes: List[WordBox] = [] + pos = 0 + prev: Optional[Tuple[int, int]] = None + for w in ordered: + key = (int(w[5]), int(w[6])) + if prev is not None: + sep = " " if key == prev else ("\n" if key[0] == prev[0] else "\n\n") + parts.append(sep) + pos += len(sep) + token = str(w[4]) + boxes.append(WordBox(pos, pos + len(token), (w[0], w[1], w[2], w[3]), key)) + parts.append(token) + pos += len(token) + prev = key + return PageTextIndex("".join(parts), boxes) + + +def _union(rects: List[Rect4]) -> Rect4: + return ( + min(r[0] for r in rects), min(r[1] for r in rects), + max(r[2] for r in rects), max(r[3] for r in rects), + ) + + +def span_to_rects(index: PageTextIndex, start: int, end: int) -> List[Rect4]: + """Rectangles (one per line) for the words a span covers. + + A word is included only when the span covers at least half of it; it is + never expanded to a whole word, so over-selection is impossible. + """ + by_line: Dict[Tuple[int, int], List[Rect4]] = {} + for w in index.words: + overlap = min(end, w.end) - max(start, w.start) + if overlap <= 0: + continue + if overlap * 2 < (w.end - w.start): + continue + by_line.setdefault(w.line_key, []).append(w.rect) + return [_union(rs) for _, rs in sorted(by_line.items())] + + +def filter_results( + results: Iterable[Any], text: str, enabled_types: Iterable[str] +) -> List[Candidate]: + """Keep results of enabled types whose score reaches the type threshold.""" + enabled = set(enabled_types) + out: List[Candidate] = [] + for r in results: + etype = r.entity_type + if etype not in enabled: + continue + if r.score < ENTITY_SCORE_THRESHOLDS.get(etype, 0.5): + continue + raw = text[r.start:r.end] + lead = len(raw) - len(raw.lstrip(_EDGE_PUNCT)) + value = raw.strip(_EDGE_PUNCT) + if len(value) <= 2: + continue + start = r.start + lead + out.append(Candidate(start, start + len(value), value, etype, float(r.score))) + return out + + +def _strip_honorifics(c: Candidate) -> Optional[Candidate]: + tokens = list(re.finditer(r"\S+", c.text)) + drop = 0 + while drop < len(tokens) and tokens[drop].group().lower() in _HONORIFICS: + drop += 1 + if drop == 0: + return c + if drop == len(tokens): + return None + shift = tokens[drop].start() + return Candidate( + c.start + shift, c.end, c.text[shift:], c.entity_type, c.score + ) + + +def _split_person_address( + c: Candidate, +) -> Tuple[Optional[Candidate], Optional[Candidate]]: + """Cut a PERSON span at the first street word or number token. + + NER often glues a name to the address that follows it ("Rossini Marco + Via Garibaldi"). Returns ``(person, address_tail)``; either may be None. + """ + tokens = list(re.finditer(r"\S+", c.text)) + for i, tok in enumerate(tokens): + word = tok.group().strip(",;:()").lower() + if word in _STREET_WORDS or any(ch.isdigit() for ch in word): + if i == 0: + return None, Candidate( + c.start, c.end, c.text, "LOCATION", c.score + ) + cut = tokens[i].start() + person_text = c.text[:cut].rstrip(_EDGE_PUNCT + " ") + person = Candidate( + c.start, c.start + len(person_text), person_text, + c.entity_type, c.score, + ) + tail_text = c.text[cut:].strip(_EDGE_PUNCT) + tail = Candidate( + c.start + cut, c.start + cut + len(tail_text), tail_text, + "LOCATION", c.score, + ) + return person, tail + return c, None + + +def clean_candidates( + candidates: Iterable[Candidate], + stopwords: Optional[Iterable[str]] = None, + enabled_types: Optional[Iterable[str]] = None, +) -> List[Candidate]: + """Drop generic/unlikely PERSON and LOCATION hits; other types pass. + + A PERSON span running into an address is cut before the address; the + address part is kept as a LOCATION only when LOCATION is enabled. + """ + stop = {s.lower() for s in (stopwords or ())} + allowed = set(enabled_types) if enabled_types is not None else None + out: List[Candidate] = [] + for c in candidates: + if c.entity_type not in _NER_TYPES: + out.append(c) + continue + cleaned: Optional[Candidate] = c + if c.entity_type == "PERSON": + cleaned = _strip_honorifics(c) + tail: Optional[Candidate] = None + if cleaned is not None: + cleaned, tail = _split_person_address(cleaned) + if tail is not None and (allowed is None or "LOCATION" in allowed): + out.extend(clean_candidates([tail], stop, allowed)) + if cleaned is None or not any(ch.isupper() for ch in cleaned.text): + continue + tokens = [t.strip(_EDGE_PUNCT).lower() for t in cleaned.text.split()] + tokens = [t for t in tokens if t] + if not tokens or all(t in _GENERIC_WORDS or t in stop for t in tokens): + continue + if len(tokens) == 1 and cleaned.text.isupper() and len(cleaned.text) > 12: + continue + out.append(cleaned) + return out + + +def resolve_overlaps(candidates: Iterable[Candidate]) -> List[Candidate]: + """Of overlapping spans keep the highest score, then the longest.""" + ranked = sorted( + candidates, key=lambda c: (-c.score, -(c.end - c.start), c.start) + ) + kept: List[Candidate] = [] + for c in ranked: + if not any(c.start < k.end and k.start < c.end for k in kept): + kept.append(c) + return sorted(kept, key=lambda c: c.start) + + +def _whole_word_pattern(value: str, case_sensitive: bool) -> "re.Pattern[str]": + parts = [re.escape(p) for p in value.split()] + return re.compile( + r"(? List[Rect4]: + """Rectangles of every whole-word occurrence of `value` on the page.""" + if not value.strip(): + return [] + rects: List[Rect4] = [] + for m in _whole_word_pattern(value, case_sensitive).finditer(index.text): + rects.extend(span_to_rects(index, m.start(), m.end())) + return rects + + +def whole_word_rects(index: PageTextIndex, term: str) -> List[Rect4]: + """Whole-word, case-insensitive search (blocklist / allowlist).""" + return propagate_whole_word(index, term, case_sensitive=False) + + +def expand_group_ids(group_ids: Iterable[str]) -> List[str]: + """Map dialog group ids to Presidio entity types.""" + wanted = set(group_ids) + return [t for gid, _, types in ENTITY_GROUPS if gid in wanted for t in types] + + +def load_enabled_groups(path: pathlib.Path) -> List[str]: + """Load the remembered selection; default (or on error) is everything.""" + try: + data = json.loads(pathlib.Path(path).read_text(encoding="utf-8")) + chosen = [g for g in data.get("enabled", []) if g in ALL_GROUP_IDS] + return chosen if chosen else list(ALL_GROUP_IDS) + except Exception: + return list(ALL_GROUP_IDS) + + +def save_enabled_groups(path: pathlib.Path, groups: Iterable[str]) -> None: + try: + p = pathlib.Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text( + json.dumps({"enabled": [g for g in groups if g in ALL_GROUP_IDS]}), + encoding="utf-8", + ) + except Exception: + pass diff --git a/tests/test_pii_detection.py b/tests/test_pii_detection.py new file mode 100644 index 0000000..64a4dd6 --- /dev/null +++ b/tests/test_pii_detection.py @@ -0,0 +1,129 @@ +import pymupdf as fitz +import pytest + +from pii_detection import ( + Candidate, + build_page_index, + clean_candidates, + expand_group_ids, + filter_results, + load_enabled_groups, + propagate_whole_word, + resolve_overlaps, + save_enabled_groups, + span_to_rects, + whole_word_rects, + ALL_GROUP_IDS, +) + + +def _index(*lines): + doc = fitz.open() + page = doc.new_page() + y = 72 + for line in lines: + page.insert_text((72, y), line, fontsize=11) + y += 20 + return build_page_index(page.get_text("words")) + + +class R: + def __init__(self, start, end, entity_type, score): + self.start, self.end, self.entity_type, self.score = start, end, entity_type, score + + +def test_offsets_map_to_words_across_lines(): + idx = _index("Mario Rossi abita", "a Roma") + start = idx.text.index("Rossi") + rects = span_to_rects(idx, start, start + 5) + assert len(rects) == 1 + multi = span_to_rects(idx, idx.text.index("abita"), idx.text.index("Roma") + 4) + assert len(multi) == 2 + + +def test_partial_word_below_half_is_dropped_not_expanded(): + idx = _index("Rossini") + assert span_to_rects(idx, 0, 3) == [] + assert len(span_to_rects(idx, 0, 5)) == 1 + + +def test_whole_word_propagation_skips_longer_words(): + idx = _index("Rossi e Rossini", "Sig. Rossi") + assert len(propagate_whole_word(idx, "Rossi")) == 2 + assert len(whole_word_rects(idx, "rossi")) == 2 + assert whole_word_rects(idx, "ross") == [] + + +def test_thresholds_and_enabled_types(): + text = "x" * 40 + res = [R(0, 10, "IT_PASSPORT", 0.01), R(0, 10, "IBAN_CODE", 1.0), + R(0, 10, "EMAIL_ADDRESS", 1.0)] + out = filter_results(res, text, ["IT_PASSPORT", "IBAN_CODE"]) + assert [c.entity_type for c in out] == ["IBAN_CODE"] + + +def test_clean_drops_generic_and_lowercase_and_strips_honorifics(): + cands = [ + Candidate(0, 3, "Sig", "PERSON", 0.85), + Candidate(0, 7, "gennaio", "LOCATION", 0.85), + Candidate(0, 7, "mario rossi", "PERSON", 0.85), + Candidate(0, 15, "Sig. Mario Rossi", "PERSON", 0.85), + Candidate(0, 6, "a@b.it", "EMAIL_ADDRESS", 1.0), + ] + out = clean_candidates(cands) + assert [c.text for c in out] == ["Mario Rossi", "a@b.it"] + + +def test_overlap_prefers_higher_score_then_longer(): + a = Candidate(0, 19, "CF RSSMRA80A01H501U", "LOCATION", 0.85) + b = Candidate(3, 19, "RSSMRA80A01H501U", "IT_FISCAL_CODE", 1.0) + assert resolve_overlaps([a, b]) == [b] + + +def test_group_persistence_and_corrupt_file(tmp_path): + f = tmp_path / "sub" / "sel.json" + assert load_enabled_groups(f) == ALL_GROUP_IDS + save_enabled_groups(f, ["IBAN", "bogus"]) + assert load_enabled_groups(f) == ["IBAN"] + f.write_text("{not json", encoding="utf-8") + assert load_enabled_groups(f) == ALL_GROUP_IDS + assert expand_group_ids(["IBAN", "PHONE"]) == ["PHONE_NUMBER", "IBAN_CODE"] + + +def test_real_analyzer_no_partial_word_redaction(): + pytest.importorskip("presidio_analyzer") + spacy = pytest.importorskip("spacy") + if not spacy.util.is_package("it_core_news_md"): + pytest.skip("model missing") + from presidio_analyzer import AnalyzerEngine, RecognizerRegistry + from presidio_analyzer.nlp_engine import NlpEngineProvider + + nlp = NlpEngineProvider(nlp_configuration={"nlp_engine_name": "spacy", "models": [ + {"lang_code": "it", "model_name": "it_core_news_md"}]}).create_engine() + reg = RecognizerRegistry(supported_languages=["it"]) + reg.load_predefined_recognizers(languages=["it"], nlp_engine=nlp) + an = AnalyzerEngine(nlp_engine=nlp, registry=reg, supported_languages=["it"]) + idx = _index("Il Sig. Mario Rossi vive a Roma.", "Il collega Rossini abita a Romano.") + res = an.analyze(text=idx.text, language="it", entities=["PERSON", "LOCATION"]) + cands = resolve_overlaps(clean_candidates(filter_results(res, idx.text, ["PERSON", "LOCATION"]))) + assert cands + for c in cands: + first = c.text.split()[0].lower() + for rect in propagate_whole_word(idx, c.text): + words = [idx.text[w.start:w.end] for w in idx.words if w.rect[1] == rect[1] and w.rect[0] >= rect[0] - 0.1 and w.rect[2] <= rect[2] + 0.1] + assert words and words[0].strip(".,").lower() == first + + +def test_person_glued_to_address_is_split(): + text = "Rossini Marco Via Garibaldi" + c = Candidate(0, len(text), text, "PERSON", 0.85) + out = clean_candidates([c]) + assert [(x.entity_type, x.text) for x in out] == [ + ("LOCATION", "Via Garibaldi"), ("PERSON", "Rossini Marco"), + ] + only_person = clean_candidates([c], enabled_types=["PERSON"]) + assert [x.text for x in only_person] == ["Rossini Marco"] + numbered = Candidate(0, 15, "Mario Rossi 12 A", "PERSON", 0.85) + assert [x.text for x in clean_candidates([numbered], enabled_types=["PERSON"])] == ["Mario Rossi"] + addr_first = Candidate(0, 13, "Via Garibaldi", "PERSON", 0.85) + assert [x.entity_type for x in clean_candidates([addr_first])] == ["LOCATION"] diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py index f772586..04cbb30 100644 --- a/tests/test_reconstruct.py +++ b/tests/test_reconstruct.py @@ -3,7 +3,7 @@ import pathlib import sys -import fitz +import pymupdf as fitz import pytest sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) From 72e568bfee5570261b96aaca54f3d49720328bb4 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:25:04 +0200 Subject: [PATCH 2/8] feat: restore original font, size and colour when reconstructing native PDFs (#113) Record the typography of each redacted value at pseudonymization export (restore map v3) and rewrite it after apply_redactions with the embedded font when usable, else the closest Base-14 font. --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 2 + NullifyPDF.py | 170 ++++++++++++++++++++++++++++++++++++-- USER_GUIDE.md | 2 +- privacy_core.py | 44 ++++++++-- tests/test_reconstruct.py | 107 ++++++++++++++++++++++++ 6 files changed, 312 insertions(+), 15 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 226fe26..1e59787 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -27,7 +27,7 @@ It supports two privacy export modes: | ------ | ----------------- | | `NullifyPDF.py` | Main PySide6 UI, PDF rendering, AI scan orchestration, redaction, export, reconstruction, OCR integration | | `pii_detection.py` | Qt-free AI detection helpers: word-box text index, offset-to-rect mapping, per-type score thresholds, generic-word filtering, whole-word propagation, entity selection persistence | -| `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads, restore-map parsing | +| `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads (version 3, optional per-occurrence text style), restore-map parsing | | `PDF_Checker.py` | Heuristic post-export inspection helper | | `build_local.py` | PyInstaller build script | | `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files bundled into every build | diff --git a/CHANGELOG.md b/CHANGELOG.md index 521edf1..c7dd1e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### ✨ Added - **Choose what the AI scan looks for**: clicking "Auto Redact (AI)" now opens a dialog with one checkbox per data type (names, locations, email, phone, IBAN, credit cards, crypto wallets, fiscal code, VAT number, driver licence, ID card, passport). The choice is remembered in `~/.nullifypdf/ai_entities.json`. +- **Reconstruction keeps the original typography on native PDFs**: for PDFs not produced by scanning (Word, Office, ...), the pseudonymization export now records the font, size, colour and baseline of each redacted value, and "Ricostruisci PDF" writes it back with that size, colour and position, reusing the original embedded font when present with all needed glyphs and otherwise the closest Base-14 font (serif/sans/mono, bold/italic). Restore maps are now version 3 (versions 1 and 2 still load); scanned pages keep the fitted-box behaviour. ### πŸ› Fixed - **AI scan selected ordinary text and parts of words**: detections were reduced to bare strings and then re-searched page-wide with a case-insensitive substring search, so one NER hit on "Rossi" also redacted "Rossini", and a stray tag spread to every occurrence on the page. Detections now use Presidio's exact character offsets mapped to word boxes, other occurrences are matched as whole words only, weak recognizers are filtered by per-type confidence thresholds, and generic PERSON/LOCATION hits (titles, months, salutations, lowercase text) are dropped. Blocklist and allowlist matching is whole-word too. A name that the NER model glues to the following address ("Rossini Marco Via Garibaldi") is now cut at the street word or number; the address part is kept as a location only when that type is enabled. +- **A value wrapped over several boxes is restored per box**: reconstruction now restores the text that actually sat inside each box instead of writing the whole value into each one. ### πŸ”§ Changed diff --git a/NullifyPDF.py b/NullifyPDF.py index 7b0bd43..a652da9 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -252,24 +252,164 @@ def _placeholder_rects_by_chars(page: Any, placeholder: str) -> List["fitz.Rect" return rects -def _restore_targets(page: Any, entry: PlaceholderEntry) -> List["fitz.Rect"]: - """Return the boxes on `page` that should get `entry.original` back. +def capture_text_style(page: Any, rect: "fitz.Rect") -> Optional[Dict[str, Any]]: + """Read the typography of the native text under `rect`. + + Returns ``{font, size, color, flags, origin, text}`` for the dominant + span (most characters), where ``text`` is the text actually inside this + box and ``origin`` the baseline start of its first line. Returns None + when the box holds no real text: no span at all, or only an invisible + OCR layer (GlyphLessFont / render mode 3). That is what distinguishes a + native PDF from a scanned one. + """ + try: + blocks = page.get_text("dict", clip=rect).get("blocks", []) + except Exception: + return None + spans: List[Dict[str, Any]] = [] + for block in blocks: + if block.get("type") != 0: + continue + for line in block.get("lines", []): + for span in line.get("spans", []): + if not str(span.get("text", "")).strip(): + continue + if "glyphless" in str(span.get("font", "")).lower(): + continue + if span.get("alpha", 255) == 0: + continue + spans.append(span) + if not spans: + return None + dominant = max(spans, key=lambda sp: len(str(sp["text"]).strip())) + text = " ".join(" ".join(str(sp["text"]) for sp in spans).split()) + return { + "font": str(dominant.get("font", "")), + "size": float(dominant["size"]), + "color": int(dominant.get("color", 0)), + "flags": int(dominant.get("flags", 0)), + "origin": [float(dominant["origin"][0]), float(dominant["origin"][1])], + "text": text, + } + + +_BASE14 = { + # (family, bold, italic) -> Base-14 font code + ("sans", False, False): "helv", ("sans", True, False): "hebo", + ("sans", False, True): "heit", ("sans", True, True): "hebi", + ("serif", False, False): "tiro", ("serif", True, False): "tibo", + ("serif", False, True): "tiit", ("serif", True, True): "tibi", + ("mono", False, False): "cour", ("mono", True, False): "cobo", + ("mono", False, True): "coit", ("mono", True, True): "cobi", +} +_SERIF_HINTS = ( + "times", "serif", "georgia", "cambria", "garamond", "book", "palatino", "minion", +) + + +def _strip_subset_prefix(name: str) -> str: + return re.sub(r"^[A-Z]{6}\+", "", name or "").lower() + + +def _same_face(basefont: str, wanted: str) -> bool: + """Loose font-name match: ignores subset prefix, case and punctuation.""" + def norm(name: str) -> str: + n = re.sub(r"[^a-z0-9]", "", _strip_subset_prefix(name)) + return re.sub(r"(regular|mt|ps)$", "", n) + + a, b = norm(basefont), norm(wanted) + return bool(a) and bool(b) and (a == b or a.startswith(b) or b.startswith(a)) + + +def closest_base14(style: Dict[str, Any]) -> str: + """Pick the Base-14 font code closest to the original face.""" + name = _strip_subset_prefix(str(style.get("font", ""))) + flags = int(style.get("flags", 0)) + if "mono" in name or "courier" in name or "consolas" in name or flags & 8: + family = "mono" + elif "sans" not in name and (flags & 4 or any(k in name for k in _SERIF_HINTS)): + family = "serif" + else: + family = "sans" + bold = bool(flags & 16) or any(k in name for k in ("bold", "black", "heavy")) + italic = bool(flags & 2) or any(k in name for k in ("italic", "oblique")) + return _BASE14[(family, bold, italic)] + + +def resolve_font( + doc: Any, page: Any, style: Dict[str, Any], text: str +) -> Tuple["fitz.Font", str]: + """Return ``(font, description)`` for restoring `text` in `style`. + + Prefers the original embedded font when the file still carries it and it + has a glyph for every character; otherwise the closest Base-14 face. + """ + wanted = _strip_subset_prefix(str(style.get("font", ""))) + if wanted: + try: + for entry in page.get_fonts(full=True): + xref, basefont = entry[0], str(entry[3]) + if xref <= 0 or not _same_face(basefont, wanted): + continue + _name, ext, _type, buffer = doc.extract_font(xref) + if not buffer or ext in ("", "n/a"): + continue + font = fitz.Font(fontbuffer=buffer) + if all(font.has_glyph(ord(ch)) for ch in text if not ch.isspace()): + return font, f"embedded:{basefont}" + except Exception: + pass + code = closest_base14(style) + return fitz.Font(code), f"base14:{code}" + + +def write_styled_text( + doc: Any, page: Any, rect: "fitz.Rect", text: str, style: Dict[str, Any] +) -> None: + """Write `text` with the original size, colour, baseline and closest font.""" + font, used = resolve_font(doc, page, style, text) + logging.getLogger("nullifypdf").debug( + "Reconstruction font: original=%s used=%s", style.get("font"), used + ) + size = float(style["size"]) + color = fitz.sRGB_to_pdf(int(style.get("color", 0))) + origin = fitz.Point(style["origin"][0], style["origin"][1]) + width = font.text_length(text, fontsize=size) + usable = max(1.0, rect.width + 1.0) + writer = fitz.TextWriter(page.rect) + if width <= usable: + writer.append(origin, text, font=font, fontsize=size) + else: + # Wider in the fallback font (or a multi-line original): shrink a + # little, and wrap inside the box if it still does not fit. + scaled = max(REDACTION_MIN_FONTSIZE, size * usable / width) + if font.text_length(text, fontsize=scaled) <= usable: + writer.append(origin, text, font=font, fontsize=scaled) + else: + writer.fill_textbox(rect, text, font=font, fontsize=scaled, align=0) + writer.write_text(page, color=color) + + +def _restore_targets( + page: Any, entry: PlaceholderEntry +) -> List[Tuple["fitz.Rect", Optional[Dict[str, Any]]]]: + """Return the ``(box, style)`` pairs on `page` that should get `entry.original` back. Prefers the coordinates recorded in the restore map (version 2+), which are the boxes the original value was taken from, so the value fits back into them. Falls back to text search only for version 1 maps. """ stored = [ - fitz.Rect(occurrence.rect) + (fitz.Rect(occurrence.rect), occurrence.style) for occurrence in entry.occurrences if occurrence.page == page.number ] if stored: return stored hits = list(page.search_for(entry.placeholder)) - if hits: - return hits - return _placeholder_rects_by_chars(page, entry.placeholder) + if not hits: + hits = _placeholder_rects_by_chars(page, entry.placeholder) + return [(hit, None) for hit in hits] def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int: @@ -318,8 +458,9 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int try: for page in doc.pages(): claimed: List[fitz.Rect] = [] + styled: List[Tuple[fitz.Rect, str, Dict[str, Any]]] = [] for entry in entries: - for rect in _restore_targets(page, entry): + for rect, style in _restore_targets(page, entry): # Entries are sorted longest-placeholder-first, so a # shorter placeholder must not re-redact a box a longer # one already claimed: searching for PERSON_001 also hits @@ -331,6 +472,15 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int if any(taken.contains(rect) for taken in claimed): continue claimed.append(fitz.Rect(rect)) + if style: + # Native text: blank the box now, write the value in + # its original typography after apply_redactions() + # (which would otherwise erase it). + text = style.get("text") or entry.original + page.add_redact_annot(rect, fill=(1, 1, 1)) + styled.append((fitz.Rect(rect), text, style)) + attempted.append((page.number, fitz.Rect(rect), text)) + continue page.add_redact_annot( rect, text=entry.original, @@ -341,6 +491,8 @@ def reconstruct_pdf(in_path: str, out_path: str, payload: Dict[str, Any]) -> int ) attempted.append((page.number, fitz.Rect(rect), entry.original)) page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) + for rect, text, style in styled: + write_styled_text(doc, page, rect, text, style) doc.save(out_path, garbage=4, deflate=True, clean=True) finally: doc.close() @@ -1700,6 +1852,7 @@ def _prepare_pseudonymized_page( annot.rect, self._read_redaction_payload(annot), page.get_text("text", clip=annot.rect).strip(), + capture_text_style(page, annot.rect), ) for annot in (page.annots() or []) if annot.type[0] == PDF_ANNOT_REDACT @@ -1707,7 +1860,7 @@ def _prepare_pseudonymized_page( for annot in list(page.annots() or []): if annot.type[0] == PDF_ANNOT_REDACT: page.delete_annot(annot) - for rect, payload, clipped_text in pending: + for rect, payload, clipped_text, style in pending: clean_original = " ".join( (payload.get("original") or clipped_text).split() ) @@ -1720,6 +1873,7 @@ def _prepare_pseudonymized_page( entity_type, page=page.number, rect=(rect.x0, rect.y0, rect.x1, rect.y1), + style=style, ) page.add_redact_annot( rect, diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 8d3701f..6caa9c9 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -139,7 +139,7 @@ If you exported a PDF with **Pseudonimizzazione reversibile**, you can later res NullifyPDF checks the SHA-256 hash recorded in the restore map against the selected PDF and refuses to proceed if they don't match, to avoid applying the wrong mapping to a document. > [!NOTE] -> Reconstruction restores the original *content*, not the original *layout*: since placeholders like `PERSON_001` are usually shorter than the original values, restored text may be clipped or visually crowd its redaction box. +> For PDFs created by other applications (Word, Office, etc. β€” not scans), the restore map also records each value's font, size, colour and baseline, and reconstruction writes the text back with the same size, colour and position, reusing the original embedded font when the file still carries it and otherwise the closest standard font (serif/sans/mono, bold/italic). Glyph shapes may therefore differ slightly from the original when the font cannot be reused. For scanned PDFs there is no font to restore, so the value is fitted into its box: it may be clipped or visually crowd it. Maps created by earlier versions keep the previous behaviour. --- diff --git a/privacy_core.py b/privacy_core.py index 3319ee8..b1ce493 100644 --- a/privacy_core.py +++ b/privacy_core.py @@ -12,7 +12,7 @@ import re from dataclasses import asdict, dataclass from enum import Enum -from typing import Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple class PrivacyMode(str, Enum): @@ -36,6 +36,27 @@ class PlaceholderOccurrence: page: int rect: Tuple[float, float, float, float] + # Typography of the original text under this box (native PDFs only): + # {font, size, color, flags, origin, text}. None for scanned pages and + # for maps written before version 3. + style: Optional[Dict[str, Any]] = None + + +def validate_style(raw: Any) -> Dict[str, Any]: + """Check a stored text style and return a normalized copy.""" + if not isinstance(raw, dict): + raise ValueError("Mappa di ripristino: stile non valido.") + origin = [float(v) for v in raw["origin"]] + if len(origin) != 2: + raise ValueError("Mappa di ripristino: stile non valido.") + return { + "font": str(raw.get("font", "")), + "size": float(raw["size"]), + "color": int(raw.get("color", 0)), + "flags": int(raw.get("flags", 0)), + "origin": origin, + "text": str(raw.get("text", "")), + } @dataclass(frozen=True) @@ -71,6 +92,7 @@ def placeholder_for( entity_type: Optional[str] = None, page: int = 0, rect: Optional[Sequence[float]] = None, + style: Optional[Dict[str, Any]] = None, ) -> str: """Return the stable placeholder for `original`, recording this hit. @@ -102,7 +124,9 @@ def placeholder_for( if rect is not None: x0, y0, x1, y1 = (float(v) for v in tuple(rect)[:4]) self._occurrences[placeholder].append( - PlaceholderOccurrence(page=page_index, rect=(x0, y0, x1, y1)) + PlaceholderOccurrence( + page=page_index, rect=(x0, y0, x1, y1), style=style + ) ) return placeholder @@ -123,11 +147,12 @@ def entries(self) -> List[PlaceholderEntry]: RESTORE_MAP_FORMAT = "NullifyPDF restore map" -RESTORE_MAP_VERSION = 2 +RESTORE_MAP_VERSION = 3 # Version 1 maps carry no `occurrences`; they are still readable, and # reconstruction falls back to searching the PDF for the placeholder text. # Refusing them would permanently strand the PII of every map already issued. -SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2}) +# Version 3 adds an optional per-occurrence `style`; 2 maps stay readable. +SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2, 3}) def build_restore_payload( @@ -145,7 +170,11 @@ def build_restore_payload( # asdict() keeps tuples as tuples; JSON round-trips them to lists, so # normalize here and keep the payload byte-comparable with itself. raw["occurrences"] = [ - {"page": occurrence.page, "rect": list(occurrence.rect)} + { + "page": occurrence.page, + "rect": list(occurrence.rect), + **({"style": occurrence.style} if occurrence.style else {}), + } for occurrence in entry.occurrences ] raw_entries.append(raw) @@ -186,6 +215,11 @@ def parse_restore_entries(payload: Dict[str, object]) -> List[PlaceholderEntry]: PlaceholderOccurrence( page=int(raw_occ["page"]), rect=(coords[0], coords[1], coords[2], coords[3]), + style=( + validate_style(raw_occ["style"]) + if raw_occ.get("style") + else None + ), ) ) entries.append( diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py index 04cbb30..2448049 100644 --- a/tests/test_reconstruct.py +++ b/tests/test_reconstruct.py @@ -374,3 +374,110 @@ def test_reconstruct_uses_stored_boxes_across_pages(tmp_path): text = "\n".join(p.get_text("text") for p in result) result.close() assert text.count("Mario Rossi") == 2 + + +def _styled_roundtrip(tmp_path, fontname, fontsize, color, value="Maria Bianchi", extra_font=None): + """Export (capture style + placeholder) then reconstruct a native PDF.""" + from NullifyPDF import capture_text_style, sha256_file + + doc = fitz.open() + page = doc.new_page() + kwargs = {"fontname": fontname, "fontsize": fontsize, "color": color} + if extra_font is not None: + page.insert_font(fontname=fontname, fontbuffer=extra_font) + page.insert_text((72, 100), f"Cliente {value} residente.", **kwargs) + page.insert_text((72, 130), "Altro testo del documento.", **kwargs) + rect = page.search_for(value)[0] + style = capture_text_style(page, rect) + registry = PlaceholderRegistry() + placeholder = registry.placeholder_for( + value, "PERSON", page=0, rect=tuple(rect), style=style + ) + page.add_redact_annot(rect, text=placeholder, fill=(1, 1, 1), fontsize=8) + page.apply_redactions(images=fitz.PDF_REDACT_IMAGE_PIXELS, graphics=True) + pseudo = tmp_path / "pseudo.pdf" + doc.save(str(pseudo), garbage=4, deflate=True, clean=True) + doc.close() + payload = build_restore_payload( + source_name="o.pdf", + source_sha256="a" * 64, + output_sha256=sha256_file(str(pseudo)), + entries=registry.entries(), + ) + out = tmp_path / "out.pdf" + count = reconstruct_pdf(str(pseudo), str(out), payload) + return style, count, out + + +def _span_for(path, needle): + doc = fitz.open(str(path)) + try: + for block in doc[0].get_text("dict")["blocks"]: + for line in block.get("lines", []): + for span in line["spans"]: + if needle in span["text"].replace("\u00a0", " "): + return span + finally: + doc.close() + return None + + +def test_native_reconstruction_keeps_size_color_and_serif(tmp_path): + grey = (0.4, 0.4, 0.4) + style, count, out = _styled_roundtrip(tmp_path, "tiro", 11, grey) + assert style is not None and style["origin"] + assert count == 1 + span = _span_for(out, "Maria Bianchi") + assert span is not None + assert span["size"] == pytest.approx(11, abs=0.1) + assert span["color"] == 0x666666 + assert "times" in span["font"].lower() or span["flags"] & 4 + assert span["origin"][1] == pytest.approx(style["origin"][1], abs=0.5) + + +def test_native_reconstruction_uses_bold_face(tmp_path): + style, count, out = _styled_roundtrip(tmp_path, "hebo", 12, (0, 0, 0)) + assert count == 1 + span = _span_for(out, "Maria Bianchi") + assert span["size"] == pytest.approx(12, abs=0.1) + assert span["flags"] & 16 or "bold" in span["font"].lower() + + +def test_native_reconstruction_reuses_embedded_font(tmp_path): + font_file = pathlib.Path("C:/Windows/Fonts/arial.ttf") + if not font_file.exists(): + pytest.skip("no system TrueType font available") + buf = font_file.read_bytes() + style, count, out = _styled_roundtrip(tmp_path, "F0", 10, (0, 0, 0), extra_font=buf) + assert count == 1 + span = _span_for(out, "Maria Bianchi") + assert span is not None + assert span["size"] == pytest.approx(10, abs=0.1) + + +def test_scanned_style_is_none_and_fallback_still_restores(tmp_path): + from NullifyPDF import capture_text_style + + doc = fitz.open() + page = doc.new_page() + assert capture_text_style(page, fitz.Rect(50, 50, 200, 80)) is None + + +def test_closest_base14_mapping(): + from NullifyPDF import closest_base14 + + assert closest_base14({"font": "ABCDEF+Calibri", "flags": 0}) == "helv" + assert closest_base14({"font": "TimesNewRomanPS-BoldMT", "flags": 20}) == "tibo" + assert closest_base14({"font": "Consolas", "flags": 8}) == "cour" + assert closest_base14({"font": "Arial-ItalicMT", "flags": 2}) == "heit" + + +def test_embedded_font_is_actually_reused(tmp_path): + font_file = pathlib.Path("C:/Windows/Fonts/arial.ttf") + if not font_file.exists(): + pytest.skip("no system TrueType font available") + style, count, out = _styled_roundtrip( + tmp_path, "F0", 10, (0, 0, 0), extra_font=font_file.read_bytes() + ) + span = _span_for(out, "Maria Bianchi") + assert "arial" in span["font"].lower() From 328ad08d2ff81456316b5263ae14a5a90ce0c2d8 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:30:24 +0200 Subject: [PATCH 3/8] docs: update docs and bump version to 2.3.0-beta.1 (#114) Document the new AI detection pipeline and restore map v3, and bump the version to 2.3.0-beta.1. --- ARCHITECTURE.md | 18 +++++++++++------- DEVELOPMENT.md | 11 ++++++----- NullifyPDF.py | 4 ++-- README.md | 3 ++- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1e59787..aeaa6a6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -89,18 +89,20 @@ Export pipeline writes a cleaned PDF copy ### πŸ€– AI Scan -1. User selects language: `EN`, `IT`, or `BOTH`. -2. User optionally enables: +1. User clicks **Auto Redact (AI)** and, in a dialog, picks which data types to look for (names, locations, email, phone, IBAN, cards, crypto, fiscal code, VAT, licence, ID card, passport). The selection is remembered in `~/.nullifypdf/ai_entities.json`. +2. User selects language (`EN`, `IT`, or `BOTH`) and optionally enables: - `OCR PDF scansionati` - `Oscura Immagini` 3. `cmd_auto_ai()` checks OCR configuration if OCR is enabled. 4. `AIWorker.run_scan()` processes pages in a background thread. 5. For each page: - - digital text is extracted with `page.get_text()` - - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` - - Presidio analyzes the extracted/OCR text - - detected entities are emitted with text, entity type, source, and optional OCR rectangles -6. `apply_ai_to_page()` runs on the UI thread and creates pending redaction annotations. + - the analysis text is built by `pii_detection.build_page_index()` from `page.get_text("words")`, recording each word's rectangle so character offsets map back to exact boxes + - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` and the same index is built from the OCR words + - Presidio analyzes the text for the enabled entity types + - results are filtered by per-type confidence thresholds, generic PERSON/LOCATION hits (titles, months, salutations, lowercase) are dropped, a name glued to a following address is cut at the street word, and overlapping spans are resolved + - each surviving value is located by its offsets, and its other occurrences on the page are added as **whole words only** + - detections are emitted with text, entity type, score, source, and their rectangles +6. `apply_ai_to_page()` runs on the UI thread and creates pending redaction annotations from the received rectangles. Blocklist and allowlist terms are matched as whole words too. ### πŸ–ŠοΈ Manual Review @@ -186,6 +188,8 @@ The restore map contains: - original values - entity types - page numbers +- the redaction box of every occurrence +- for native (non-scanned) PDFs, the typography of the redacted text: font, size, colour, flags and baseline (map version 3, optional) The map is encrypted with `cryptography` using Fernet and a key derived from the user password with PBKDF2-HMAC-SHA256. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 0436a62..cfe7ef3 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -106,6 +106,7 @@ If the GUI opens β†’ **You're ready to develop!** πŸŽ‰ NullifyPDF/ |-- NullifyPDF.py # Main PySide6 application |-- privacy_core.py # Privacy modes and encrypted restore maps +|-- pii_detection.py # Qt-free AI detection helpers (offsets, filters, whole-word matching) |-- PDF_Checker.py # Heuristic verification utility |-- setup_env.py # Environment setup script |-- build_local.py # PyInstaller build script @@ -115,6 +116,7 @@ NullifyPDF/ |-- tests/ | |-- test_validation.py | |-- test_privacy_core.py +| |-- test_pii_detection.py | |-- test_reconstruct.py | `-- test_build_config.py |-- images/ @@ -438,6 +440,7 @@ isort NullifyPDF.py | ------------------ | ----------------------- | | `NullifyPDF.py` | Main app, GUI, OCR, export logic | | `privacy_core.py` | Placeholder and restore-map logic | +| `pii_detection.py` | AI detection precision: word-box index, thresholds, filters, entity selection | | `setup_env.py` | Environment setup | | `build_local.py` | PyInstaller build | | `PDF_Checker.py` | Post-processing utility | @@ -519,11 +522,9 @@ Never hardcode API keys or passwords. ### Q: Where do I add new AI detections? **A:** -In `AIWorker.run_scan()`: -1. Use Presidio analyzer for regex patterns -2. Use spaCy models for entity recognition -3. Merge and deduplicate results -4. Filter through allowlist +1. Add the Presidio entity type (and its dialog label) to `ENTITY_GROUPS` in `pii_detection.py`, plus a threshold in `ENTITY_SCORE_THRESHOLDS`. +2. `AIWorker.run_scan()` analyzes the page text built by `build_page_index()`. Filtering (`filter_results`, `clean_candidates`, `resolve_overlaps`) and offset-to-rectangle mapping (`span_to_rects`, `propagate_whole_word`) live in `pii_detection.py`, which has no Qt dependency and is unit-tested in `tests/test_pii_detection.py`. +3. Detections are filtered through the allowlist in the worker. See [ARCHITECTURE.md](./ARCHITECTURE.md) for AI pipeline details. diff --git a/NullifyPDF.py b/NullifyPDF.py index a652da9..1cffa78 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -81,8 +81,8 @@ PDF_ANNOT_REDACT: int = fitz.PDF_ANNOT_REDACT # type: ignore[attr-defined] PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] -__version__ = "2.2.0" -__version_prerelease__ = "" +__version__ = "2.3.0" +__version_prerelease__ = "beta.1" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" if __version_prerelease__ diff --git a/README.md b/README.md index ea5cdde..6716f41 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi ## ✨ Key Features -- 🧠 **AI-Powered Redaction** β€” Automatic bilingual (EN/IT) detection of PII: names, locations, emails, phones, IBANs, credit cards, crypto addresses +- 🧠 **AI-Powered Redaction** β€” Automatic bilingual (EN/IT) detection of PII: names, locations, emails, phones, IBANs, credit cards, crypto addresses, Italian fiscal codes and IDs. A dialog lets you choose which data types to scan for, and matching is whole-word so ordinary text is not caught - πŸ—„οΈ **Fluid UI & Thread-Safe** β€” PySide6 modern dark-mode interface with zero UI freezing. Text extraction in worker thread with QMutex serialization - πŸ“– **Persistent Dictionaries** β€” Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with O(1) fast-path matching - πŸ›‘οΈ **Privacy Export Modes** β€” Choose irreversible anonymization or reversible pseudonymization with a separately encrypted restore map @@ -197,6 +197,7 @@ pytest tests/ -v - Resource path resolution - OCR configuration helpers - Privacy placeholder and encrypted restore-map primitives +- AI detection helpers (offset mapping, filtering, whole-word matching) and reconstruction with the original font - Build configuration From c7b7ab58254d30fd0fd1f0f00ed8febc587f3a54 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Sat, 19 Sep 2026 10:00:37 +0200 Subject: [PATCH 4/8] fix: bundle tldextract data in PyInstaller builds (#115) Collect tldextract in the PyInstaller recipe so Presidio's email recognizer finds its suffix snapshot in the frozen app. --- CHANGELOG.md | 1 + build_local.py | 4 ++-- tests/test_build_config.py | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7dd1e9..8ff7ed9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### πŸ› Fixed +- **AI scan crashed in packaged builds when an email was found**: Presidio's email recognizer needs `tldextract`'s `.tld_set_snapshot` data file, which PyInstaller did not bundle, so the scan aborted with `FileNotFoundError` under `_MEI... ldextract`. The package is now collected by the spec and by `build_local.py`. - **AI scan selected ordinary text and parts of words**: detections were reduced to bare strings and then re-searched page-wide with a case-insensitive substring search, so one NER hit on "Rossi" also redacted "Rossini", and a stray tag spread to every occurrence on the page. Detections now use Presidio's exact character offsets mapped to word boxes, other occurrences are matched as whole words only, weak recognizers are filtered by per-type confidence thresholds, and generic PERSON/LOCATION hits (titles, months, salutations, lowercase text) are dropped. Blocklist and allowlist matching is whole-word too. A name that the NER model glues to the following address ("Rossini Marco Via Garibaldi") is now cut at the street word or number; the address part is kept as a location only when that type is enabled. - **A value wrapped over several boxes is restored per box**: reconstruction now restores the text that actually sat inside each box instead of writing the whole value into each one. diff --git a/build_local.py b/build_local.py index 4cf7ddb..f565640 100644 --- a/build_local.py +++ b/build_local.py @@ -313,7 +313,7 @@ def build_app() -> None: datas = {datas_literal} binaries = [] hiddenimports = ['spacy', 'presidio_analyzer'] -for pkg in ['presidio_analyzer', 'spacy', 'en_core_web_md', 'it_core_news_md']: +for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] @@ -329,7 +329,7 @@ def build_app() -> None: datas = {datas_literal} binaries = [] hiddenimports = ['spacy', 'presidio_analyzer'] -for pkg in ['presidio_analyzer', 'spacy', 'en_core_web_md', 'it_core_news_md']: +for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] diff --git a/tests/test_build_config.py b/tests/test_build_config.py index 1df6682..ea94455 100644 --- a/tests/test_build_config.py +++ b/tests/test_build_config.py @@ -60,3 +60,20 @@ def test_build_includes_ocr_data(tmp_path, monkeypatch): assert ("ocr/tessdata/eng.traineddata", "ocr/tessdata") in datas assert ("ocr/tessdata/ita.traineddata", "ocr/tessdata") in datas + + +def test_tldextract_data_is_bundled_by_the_build_recipes(): + """Presidio's email recognizer needs tldextract's `.tld_set_snapshot`. + + Without it the frozen app raised FileNotFoundError on the first email + found by an AI scan, so every build recipe must collect the package. + """ + root = pathlib.Path(__file__).parent.parent + for recipe in ("build_local.py",): + assert "'tldextract'" in (root / recipe).read_text(encoding="utf-8"), recipe + + pytest.importorskip("PyInstaller") + from PyInstaller.utils.hooks import collect_all + + datas = collect_all("tldextract")[0] + assert any(src.endswith(".tld_set_snapshot") for src, _dest in datas) From d97167155a7455fba0cc8b25f1bd71a98a69974c Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Sat, 19 Sep 2026 10:01:19 +0200 Subject: [PATCH 5/8] chore: bump version to 2.3.0-beta.2 (#116) Ship the tldextract packaging fix in a new beta. --- NullifyPDF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NullifyPDF.py b/NullifyPDF.py index 1cffa78..c74b305 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -82,7 +82,7 @@ PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] __version__ = "2.3.0" -__version_prerelease__ = "beta.1" +__version_prerelease__ = "beta.2" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" if __version_prerelease__ From 52066952350242119ae54ea178c71be6f413a06f Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Sat, 19 Sep 2026 10:43:55 +0200 Subject: [PATCH 6/8] fix: stop the AI scan from flagging labels, headings and list items (#117) Judge NER spans line by line and reject single-word PERSON/LOCATION hits that are labels, list items, headings, common nouns or identifier names; strip role words from names. --- CHANGELOG.md | 1 + NullifyPDF.py | 6 ++- pii_detection.py | 96 ++++++++++++++++++++++++++++++++++--- tests/test_pii_detection.py | 83 ++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff7ed9..6d61248 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### πŸ› Fixed +- **AI scan flagged labels and headings as names or places** ("Telefono:", "- Presente", "Configurazione"): NER hits are now judged line by line (a span no longer crosses a line break, and bullets/dashes are ignored), and a single-word PERSON/LOCATION hit is dropped when it is a field label followed by a colon, a list item, a lone heading line, a word also written in lowercase elsewhere on the page, or a capitalized line start followed by a lowercase word. A list of common label, section and job-title words (IT/EN) is also ignored. Multi-word names such as "Mario Rossi" are unaffected. - **AI scan crashed in packaged builds when an email was found**: Presidio's email recognizer needs `tldextract`'s `.tld_set_snapshot` data file, which PyInstaller did not bundle, so the scan aborted with `FileNotFoundError` under `_MEI... ldextract`. The package is now collected by the spec and by `build_local.py`. - **AI scan selected ordinary text and parts of words**: detections were reduced to bare strings and then re-searched page-wide with a case-insensitive substring search, so one NER hit on "Rossi" also redacted "Rossini", and a stray tag spread to every occurrence on the page. Detections now use Presidio's exact character offsets mapped to word boxes, other occurrences are matched as whole words only, weak recognizers are filtered by per-type confidence thresholds, and generic PERSON/LOCATION hits (titles, months, salutations, lowercase text) are dropped. Blocklist and allowlist matching is whole-word too. A name that the NER model glues to the following address ("Rossini Marco Via Garibaldi") is now cut at the street word or number; the address part is kept as a location only when that type is enabled. - **A value wrapped over several boxes is restored per box**: reconstruction now restores the text that actually sat inside each box instead of writing the whole value into each one. diff --git a/NullifyPDF.py b/NullifyPDF.py index c74b305..e8f7d9b 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -70,6 +70,7 @@ filter_results, load_enabled_groups, propagate_whole_word, + reject_label_like_hits, resolve_overlaps, save_enabled_groups, span_to_rects, @@ -883,7 +884,10 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, ) candidates.extend(filter_results(res, index.text, targets)) candidates = resolve_overlaps( - clean_candidates(candidates, enabled_types=targets) + reject_label_like_hits( + clean_candidates(candidates, enabled_types=targets), + index.text, + ) ) detections = [] seen_values: Set[Tuple[str, str]] = set() diff --git a/pii_detection.py b/pii_detection.py index f6214c2..d0f630c 100644 --- a/pii_detection.py +++ b/pii_detection.py @@ -65,12 +65,33 @@ "september", "october", "november", "december", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", "italia", "italy", + # Labels, headings and list words that document/CV layouts capitalize. + "telefono", "tel", "cellulare", "cell", "fax", "email", "mail", "web", + "sito", "indirizzo", "residenza", "domicilio", "sede", "luogo", "data", + "nome", "cognome", "nato", "nata", "presente", "attuale", "configurazione", + "esperienza", "esperienze", "competenze", "formazione", "istruzione", + "lingue", "lingua", "inglese", "italiano", "francese", "tedesco", + "spagnolo", "profilo", "curriculum", "vitae", "obiettivo", "referenze", + "interessi", "hobby", "patente", "certificazioni", "progetti", "sviluppo", + "phone", "mobile", "address", "name", "surname", "present", "current", + "configuration", "experience", "skills", "education", "languages", + "english", "italian", "french", "german", "spanish", "profile", "summary", + "objective", "references", "interests", "certifications", "projects", + # Names of the identifiers themselves ("Partita IVA:", "Codice fiscale"). + "partita", "iva", "codice", "fiscale", "cf", "iban", "carta", "identitΓ ", + "documento", "passaporto", "referente", "contatto", "titolare", + # Job titles, often capitalized at the start of a CV line. + "sviluppatore", "ingegnere", "analista", "responsabile", "consulente", + "tecnico", "direttore", "amministratore", "impiegato", "impiegata", + "programmatore", "progettista", "sistemista", "developer", "engineer", + "analyst", "manager", "consultant", "technician", "director", "administrator", } _HONORIFICS = { "sig", "sig.", "sig.ra", "signor", "signora", "signore", "dott", "dott.", "dott.ssa", "dr", "dr.", "prof", "prof.", "prof.ssa", "ing", "ing.", "avv", "avv.", "geom", "geom.", "rag", "rag.", "mr", "mr.", "mrs", "mrs.", "ms", "ms.", "spett.le", "gentile", "egregio", "egregia", + "referente", "contatto", "titolare", "responsabile", } _STREET_WORDS = { "via", "viale", "v.le", "piazza", "p.za", "piazzale", "corso", "c.so", @@ -78,7 +99,7 @@ "frazione", "fraz.", "borgo", "lungotevere", "c/o", "street", "st.", "avenue", "road", "rd.", "lane", } -_EDGE_PUNCT = " \t\r\n.,;:!?()[]{}<>\"'Β«Β»β€œβ€β€˜β€™" +_EDGE_PUNCT = " \t\r\n.,;:!?()[]{}<>\"'Β«Β»β€œβ€β€˜β€™-–—‒·*β–ͺβ–Ίβ– " @dataclass @@ -167,13 +188,74 @@ def filter_results( continue if r.score < ENTITY_SCORE_THRESHOLDS.get(etype, 0.5): continue - raw = text[r.start:r.end] - lead = len(raw) - len(raw.lstrip(_EDGE_PUNCT)) - value = raw.strip(_EDGE_PUNCT) - if len(value) <= 2: + # A name or place never spans a line break; NER glues a heading or + # label ("Graziano Mariella\nTelefono") to the previous line, so + # judge each line of an NER span on its own. + if etype in _NER_TYPES: + segments = [ + (r.start + m.start(), m.group()) + for m in re.finditer(r"[^\r\n]+", text[r.start:r.end]) + ] + else: + segments = [(r.start, text[r.start:r.end])] + for seg_start, raw in segments: + lead = len(raw) - len(raw.lstrip(_EDGE_PUNCT)) + value = raw.strip(_EDGE_PUNCT) + if len(value) <= 2: + continue + start = seg_start + lead + out.append( + Candidate(start, start + len(value), value, etype, float(r.score)) + ) + return out + + +def _line_bounds(text: str, start: int, end: int) -> Tuple[int, int]: + line_start = text.rfind("\n", 0, start) + 1 + line_end = text.find("\n", end) + return line_start, len(text) if line_end == -1 else line_end + + +def reject_label_like_hits( + candidates: Iterable[Candidate], text: str +) -> List[Candidate]: + """Drop single-word PERSON/LOCATION hits that look like labels or headings. + + Document layouts capitalize words that are not names, and the NER model + tags them ("Telefono:", "- Presente", "Configurazione"). A one-word hit + is rejected when it is: + - followed by a colon (a field label); + - a list item (preceded by a bullet or dash on its line); + - a PERSON alone on its line (a heading); + - also used in lowercase elsewhere on the page (a common noun, whereas a + real name is essentially never written in lowercase). + Multi-word spans such as "Mario Rossi" are left alone. + """ + out: List[Candidate] = [] + for c in candidates: + if c.entity_type not in _NER_TYPES or len(c.text.split()) != 1: + out.append(c) + continue + after = text[c.end:c.end + 8].lstrip(" \t") + if after.startswith(":"): + continue + ls, le = _line_bounds(text, c.start, c.end) + before_raw = text[ls:c.start].strip() + rest = text[c.end:le].strip(_EDGE_PUNCT) + if before_raw and not before_raw.strip(_EDGE_PUNCT): + continue # only bullets/dashes before it: a list item + if c.entity_type == "PERSON" and not before_raw and not rest: + continue # a lone word on its own line + if c.entity_type == "PERSON" and not before_raw.strip(_EDGE_PUNCT): + following = text[c.end:le].split() + if following and following[0][:1].islower(): + continue # capitalized only because it starts the line + lowered = c.text.lower() + if lowered != c.text and re.search( + r"(? Date: Sat, 19 Sep 2026 10:44:16 +0200 Subject: [PATCH 7/8] chore: bump version to 2.3.0-beta.3 (#118) Ship the AI false-positive fixes in a new beta. --- NullifyPDF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NullifyPDF.py b/NullifyPDF.py index e8f7d9b..0be806a 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -83,7 +83,7 @@ PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] __version__ = "2.3.0" -__version_prerelease__ = "beta.2" +__version_prerelease__ = "beta.3" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" if __version_prerelease__ From dcd3d3501d119dbe8415f1f64593ac9fd843e724 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Sat, 19 Sep 2026 11:30:01 +0200 Subject: [PATCH 8/8] chore: prepare 2.3.0 stable release (#119) Update docs, move Unreleased under 2.3.0 and drop the beta suffix. --- ARCHITECTURE.md | 6 +++--- CHANGELOG.md | 4 ++++ CONTRIBUTING.md | 9 +++++---- DEVELOPMENT.md | 16 ++++++++++++++-- NullifyPDF.py | 2 +- TROUBLESHOOTING.md | 6 +++++- USER_GUIDE.md | 4 ++-- 7 files changed, 34 insertions(+), 13 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index aeaa6a6..2147510 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -99,7 +99,7 @@ Export pipeline writes a cleaned PDF copy - the analysis text is built by `pii_detection.build_page_index()` from `page.get_text("words")`, recording each word's rectangle so character offsets map back to exact boxes - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` and the same index is built from the OCR words - Presidio analyzes the text for the enabled entity types - - results are filtered by per-type confidence thresholds, generic PERSON/LOCATION hits (titles, months, salutations, lowercase) are dropped, a name glued to a following address is cut at the street word, and overlapping spans are resolved + - results are filtered by per-type confidence thresholds; NER spans are split at line breaks; generic PERSON/LOCATION hits (titles, months, salutations, lowercase, role and identifier words) are dropped; single-word hits that look like labels or headings (followed by a colon, list items, lone heading lines, words also written in lowercase on the page, capitalized line starts) are rejected; a name glued to a following address is cut at the street word; overlapping spans are resolved - each surviving value is located by its offsets, and its other occurrences on the page are added as **whole words only** - detections are emitted with text, entity type, score, source, and their rectangles 6. `apply_ai_to_page()` runs on the UI thread and creates pending redaction annotations from the received rectangles. Blocklist and allowlist terms are matched as whole words too. @@ -297,7 +297,7 @@ pytest tests/ -v ## ⚠️ Known Limits -- Detection is probabilistic. Presidio, spaCy, and OCR can miss data or create false positives. +- Detection is probabilistic. Presidio, spaCy, and OCR can miss data or create false positives. Heuristics that reject label-like single-word names trade a little recall (for example a lone first name that starts a line) for precision; such values can be marked manually. - OCR depends on Tesseract language data and scan quality. - Handwriting is not reliably supported. - Complex PDFs may contain structures not covered by automated cleanup. @@ -309,5 +309,5 @@ pytest tests/ -v --- -*Last updated: 2026-09-18* +*Last updated: 2026-09-19* *[Back to README](./README.md) | [Development β†’](./DEVELOPMENT.md)* diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d61248..be0177a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.3.0] - 2026-09-19 + +Validated through the 2.3.0-beta.1/beta.2/beta.3 prereleases before this stable release, promoted from `v2.3.0-beta.3` without rebuilding; only the final stable version gets its own heading here. + ### ✨ Added - **Choose what the AI scan looks for**: clicking "Auto Redact (AI)" now opens a dialog with one checkbox per data type (names, locations, email, phone, IBAN, credit cards, crypto wallets, fiscal code, VAT number, driver licence, ID card, passport). The choice is remembered in `~/.nullifypdf/ai_entities.json`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2decd7e..4a04ef0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -219,9 +219,10 @@ thread when scanning large PDFs concurrently." ### Before Opening PR 1. βœ… Run tests: `pytest tests/ -v` -2. βœ… Build locally: `python build_local.py` -3. βœ… Update CHANGELOG.md with your changes -4. βœ… Self-review your code (would you understand this in 6 months?) +2. βœ… Type check (blocking in CI): `mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py` +3. βœ… Build locally: `python build_local.py` +4. βœ… Update CHANGELOG.md with your changes +5. βœ… Self-review your code (would you understand this in 6 months?) 5. βœ… Check links in documentation ### PR Description Template @@ -378,5 +379,5 @@ Every contribution β€” code, docs, bug reports, ideas β€” helps make NullifyPDF --- -*Last updated: 2026-09-18* +*Last updated: 2026-09-19* *← [Back to README](./README.md) | [Architecture β†’](./ARCHITECTURE.md)* diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index cfe7ef3..3215ed1 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -225,11 +225,23 @@ pytest tests/ --cov=. --cov-report=html Opens `htmlcov/index.html` in browser. +### Type Checking + +CI runs `mypy` as a blocking step. Run the same check locally before opening a PR: + +```bash +mypy NullifyPDF.py pii_detection.py privacy_core.py PDF_Checker.py +``` + +PyMuPDF is imported as `pymupdf` (aliased to `fitz`) so it is really type-checked; its dynamically defined constants are read once near the top of `NullifyPDF.py`. + ### What's Tested - βœ… **PDFListManager** β€” File I/O, persistence - βœ… **OCR Config** β€” Tesseract language selection and tessdata discovery -- βœ… **Privacy Core** β€” Placeholder mapping and encrypted restore maps +- βœ… **Privacy Core** β€” Placeholder mapping and encrypted restore maps (v1/v2/v3) +- βœ… **AI Detection Helpers** β€” Offset-to-rectangle mapping, thresholds, label/heading filtering, whole-word matching +- βœ… **Reconstruction** β€” Placeholder restoration, original font/size/colour on native PDFs - βœ… **Build Config** β€” OCR data bundling behavior - βœ… **Resource Paths** β€” PyInstaller compatibility @@ -564,5 +576,5 @@ Ready to contribute? See [CONTRIBUTING.md](./CONTRIBUTING.md) for: --- -*Last updated: 2026-09-18* +*Last updated: 2026-09-19* *← [Troubleshooting](./TROUBLESHOOTING.md) | [Back to README β†’](./README.md)* diff --git a/NullifyPDF.py b/NullifyPDF.py index 0be806a..d319b12 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -83,7 +83,7 @@ PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] __version__ = "2.3.0" -__version_prerelease__ = "beta.3" +__version_prerelease__ = "" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" if __version_prerelease__ diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 1a114be..e666ae3 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -212,6 +212,10 @@ AI redacted text that shouldn't be redacted (false positive). ### Solution +**Option 0 β€” Scan only what you need** + +Clicking **Auto Redact (AI)** opens a dialog where you tick the data types to look for (for example untick *Luoghi e indirizzi* if cities keep being flagged). Your choice is remembered. Labels and headings such as "Telefono:" or list items are filtered automatically, and other occurrences of a value are matched as whole words only. + **Option 1 β€” Remove and Allowlist** 1. **Click once** on the black redaction box @@ -461,5 +465,5 @@ The more details, the faster we can fix it! πŸš€ --- -*Last updated: 2026-09-18* +*Last updated: 2026-09-19* *← [User Guide](./USER_GUIDE.md) | [Development β†’](./DEVELOPMENT.md)* diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 6caa9c9..44cac07 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -65,7 +65,7 @@ Click the **"Auto Redact (AI)"** button. A dialog first lets you tick which data - UI stays responsive (no freezing) > [!TIP] -> The AI isn't perfect. You'll review results in Step 4. +> The AI isn't perfect. You'll review results in Step 4. If a type produces too many false positives (for example cities), untick it in the selection dialog next time. Field labels ("Telefono:"), headings and list items are ignored automatically. --- @@ -283,5 +283,5 @@ python3.13 NullifyPDF.py --- -*Last updated: 2026-09-18* +*Last updated: 2026-09-19* *← [Back to README](./README.md) | [Troubleshooting β†’](./TROUBLESHOOTING.md)*