diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml index 6cc48b1..f71edb5 100644 --- a/.github/workflows/beta-release.yml +++ b/.github/workflows/beta-release.yml @@ -50,7 +50,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py - name: Resolve beta suffix from tag if: startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 334f0a8..09c82b4 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -82,7 +82,7 @@ jobs: run: pytest tests/ -v - name: Type check (mypy) - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py - name: Resolve beta suffix from tag if: inputs.is_beta && startsWith(github.ref, 'refs/tags/') diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7273a52..9f5ec6c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -95,7 +95,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py - name: Build run: python build_local.py diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index f9249e7..b3aa452 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -55,7 +55,7 @@ jobs: - name: Type Check (mypy) shell: bash - run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py + run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py - name: Build run: python build_local.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 15d146f..b546716 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -33,10 +33,11 @@ It supports two privacy export modes: | `pdf_sanitize.py` | Export sanitiser (annotations, attachments, scripts, hidden layers/text, off-page text, metadata, optional TJ-gap quantisation) and `verify_export()` | | `export_engine.py` | Qt-free export pipeline (`export_document`), neutral file names, private temp handling, audit record, local-data wipe | | `review_dialog.py` | Qt dialog to accept or reject suggested detections | +| `help_guide.py` | In-app guide (Italian, F1) and the tooltip table applied to every widget | | `secure_store.py` | Master key protected by DPAPI / OS keyring / key file; AES-GCM sealing; atomic private writes | | `word_lists.py` | Encrypted blocklist/allowlist with expiry (`PDFListManager`) | | `dictionary_dialog.py` | Qt editor for both lists with dates and default expiry | -| `lexicon_data_it.py` | Generated from ISTAT open data by `scripts/build_lexicon_data.py` (see `THIRD_PARTY_NOTICES.md`); imported lazily | +| `lexicon_data_it.py` | Generated by `scripts/build_lexicon_data.py` from ISTAT open data (municipalities, provinces, regions) and, with `--wikidata`, Wikidata (given names, surnames); see `THIRD_PARTY_NOTICES.md`; imported lazily and used only to confirm names and places | | `PDF_Checker.py` | Heuristic post-export inspection helper | | `build_local.py` | PyInstaller build script | | `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files bundled into every build | @@ -149,9 +150,10 @@ Clicking an existing redaction: - widgets are removed - `pdf_sanitize.sanitize_document()` removes annotations, attachments, scripts, bookmarks, structure, metadata, hidden/invisible/off-page text and external links - the final PDF is saved with cleanup options -6. `pdf_sanitize.verify_export()` re-opens the written file and checks it; a failure is reported and the file can be deleted. -7. For pseudonymization, the restore map is encrypted and written with owner-only permissions. -8. An audit record (no values) is written to `~/.nullifypdf/audit/`. +6. Before the redactions are applied, every band that stores the whole box of its word is checked; after applying them, the number of glyphs that vanished from each line must match the measured deletion rule (a glyph is deleted when the rectangle reaches its box inset by 10%), and target glyphs the band cannot reach are counted. Any survivor is reported without the value. +7. `pdf_sanitize.verify_export()` re-opens the written file and checks it; a failure is reported and the file can be deleted. +8. For pseudonymization, the restore map is encrypted and written with owner-only permissions. +9. An audit record (no values) is written to `~/.nullifypdf/audit/`. The original open document remains editable after export because the export works on a copy. @@ -320,5 +322,5 @@ pytest tests/ -v --- -*Last updated: 2026-11-08* +*Last updated: 2026-11-10* *[Back to README](./README.md) | [Development →](./DEVELOPMENT.md)* diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f799b9..465df4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### ✨ Added +- **User guide in the program**: a new "Guida" button (or F1) opens a guide in Italian that explains the whole workflow: opening a PDF, language and options, Auto Redact, review of suggestions, manual corrections, dictionaries, export modes, reconstruction, health documents, local data and shortcuts. +- **Tooltips on every element**: all buttons, options, fields, tables and indicators of the main window and of the dialogs now show a short description when the pointer rests on them. +- **Check after redaction for surviving characters**: besides searching the written file, the export now checks that every character of a redacted word was really deleted, and reports it (without the value) if one was not. - **Check for updates**: the Info dialog has a new "Controlla aggiornamenti" button. It queries the GitHub releases of this repository (on click, or at startup only if you enabled the opt-in startup check) and, if a newer version exists, shows the release link, the release notes and a **Download** button that saves the installer for the running OS into the user's system Downloads folder (resolved per platform), verifying its size and the SHA-256 published by GitHub. Linux distributions outside the Debian/Ubuntu and Fedora/RHEL families only get the release link. Stable builds are only offered newer stable releases; beta builds are offered both stable and beta releases. - **Optional check at every start**: a "Controlla ad ogni avvio" checkbox in the Info dialog (off by default, stored in `~/.nullifypdf/update_check.json`) runs the same check silently at startup and opens the update window, with the release notes, only when a newer version exists. Failures are logged, never shown. - **Repository link in the Info dialog**. @@ -24,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Signature and stamp suggestions**: with the health-document profile, curve-rich ink and small images next to "firma", "timbro", "il medico"... are proposed for review (never applied automatically). - **Postcodes and house numbers**: a street and its house number are redacted as one address, and postcodes are recognised only in an address context (never in prices, dates or quantities). Health card number, prescription number (NRE) and exemption codes follow the published structure (length, region code, lot code); the check digit is not verified, and the prescription number and exemption codes need their label. - **Suggestions in the page preview**: pending suggestions are outlined in orange on the page (view only; they never reach an export). Click one to censor or discard it; selecting a row in the review dialog highlights it. +- **Given names and surnames from Wikidata** (CC0): a few hundred given names and about one thousand surnames are used to confirm names, never to reject them; names that are also ordinary words count less. Coverage is small. - **ISTAT place names**: Italian municipalities, provinces and regions (ISTAT open data, CC BY 4.0) are used to confirm places; a place missing from the list is never rejected for that reason. - **Export audit record** in `~/.nullifypdf/audit/` (counts per type, sanitiser actions, verification result, output SHA-256; no personal data), and an "Elimina dati locali" button that wipes logs, audit records and temporary files. @@ -43,6 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Placeholders were silently dropped in small boxes**: the placeholder is now drawn after the redactions are applied, with a compact form ("P9F2A", "9F2A") when the full one does not fit, so short words keep a visible placeholder. Restore maps are version 4 (version 1-3 maps still load). - **Restore wrote neighbouring text back**: the typography captured for a redaction could include fragments of the lines above and below when the leading was tight; only the redacted line is captured, and the restore writes the original value. - **Wipe of local data left the active log file**, and "Oscura Immagini" added duplicate image redactions on every apply. +- **Labels on images and signatures survive small boxes**: the "[IMMAGINE RIMOSSA]" / "[FIRMA RIMOSSA]" labels are drawn after the redactions, with shorter forms when the area is small, instead of being dropped. +- **Master key in the system keyring on macOS and Linux**: when a keyring is available the encryption key of the dictionaries is stored there instead of in a local key file (Windows keeps using DPAPI). - **Local logs are rotated** (512 KiB, 3 files, 30 days) instead of growing indefinitely. - **Redacted text could remain in the PDF content stream**: with PyMuPDF 1.28.x, `apply_redactions()` did not remove text drawn with the `'` operator after `TL`/`Td`; the text disappeared from the extracted text but was still stored in the file. Fixed by PyMuPDF 2.x; covered by a regression test that scans every stream of the exported file. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 8a2144d..0834584 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -461,7 +461,8 @@ isort NullifyPDF.py | `secure_store.py` | Master key protection (DPAPI / keyring / key file), AES-GCM sealing | | `word_lists.py` | Encrypted blocklist/allowlist with expiry | | `dictionary_dialog.py` | Dictionary editor dialog | -| `scripts/build_lexicon_data.py` | Generates `lexicon_data_it.py` from ISTAT open data (development only) | +| `help_guide.py` | In-app guide and tooltip table (keep the guide in sync with the UI: `tests/test_gui_help.py` checks it) | +| `scripts/build_lexicon_data.py` | Generates `lexicon_data_it.py` from ISTAT open data and (with `--wikidata`) Wikidata names; development only | | `setup_env.py` | Environment setup | | `build_local.py` | PyInstaller build | | `PDF_Checker.py` | Post-processing utility | @@ -589,5 +590,5 @@ The release workflows fill each GitHub release body with `scripts/extract_releas --- -*Last updated: 2026-11-08* +*Last updated: 2026-11-10* *← [Troubleshooting](./TROUBLESHOOTING.md) | [Back to README →](./README.md)* diff --git a/NullifyPDF.py b/NullifyPDF.py index cf1fec0..a74a26f 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -42,6 +42,8 @@ QTextBrowser, ) from PySide6.QtGui import ( + QKeySequence, + QShortcut, QIcon, QPixmap, QImage, @@ -112,7 +114,7 @@ write_audit_record, write_private_file, ) -from pii_detection import GLYPH_CORE_INSET +from pii_detection import GLYPH_CORE_INSET, core_hits as _core_hits, glyph_core, target_chars from pdf_sanitize import ( SanitizePolicy, page_image_coverage, @@ -120,6 +122,7 @@ suggest_signature_areas, ) from review_dialog import ReviewDialog +from help_guide import GuideDialog, apply_tooltips from dictionary_dialog import DictionaryDialog from word_lists import PDFListManager import secure_store @@ -130,7 +133,7 @@ PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined] __version__ = "2.4.0" -__version_prerelease__ = "beta.2" +__version_prerelease__ = "beta.3" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" if __version_prerelease__ @@ -288,6 +291,11 @@ class RestoreMapMismatch(ValueError): REDACTION_FONTNAME = "helv" +# Labels (longest first) drawn on image / signature areas after redaction. +KIND_LABELS: Dict[str, List[str]] = { + "IMAGE": ["[IMMAGINE RIMOSSA]", "[IMMAGINE]", "[IMG]", "#"], + "SIGNATURE": ["[FIRMA RIMOSSA]", "[FIRMA]", "[F]", "#"], +} REDACTION_MAX_FONTSIZE = 8.0 # PyMuPDF's apply_redactions() never inserts replacement text below 4 pt, so # a smaller floor would only produce text that is silently dropped. @@ -381,23 +389,6 @@ def _placeholder_rects_by_chars(page: Any, placeholder: str) -> List["fitz.Rect" return rects -def glyph_core(bbox: Tuple[float, float, float, float]) -> Tuple[float, float, float, float]: - """The part of a character box that decides whether a redaction deletes it. - - Measured on PyMuPDF 2.1 (MuPDF 1.28): a glyph is removed when the - redaction rectangle intersects its box shrunk by 10% of its width on - each side and 10% of its height at top and bottom. - """ - x0, y0, x1, y1 = bbox - dx, dy = (x1 - x0) * GLYPH_CORE_INSET, (y1 - y0) * GLYPH_CORE_INSET - return (x0 + dx, y0 + dy, x1 - dx, y1 - dy) - - -def _core_hits(bbox: Tuple[float, float, float, float], rect: "fitz.Rect") -> bool: - c = glyph_core(bbox) - return c[0] < rect.x1 and c[2] > rect.x0 and c[1] < rect.y1 and c[3] > rect.y0 - - def capture_text_style(page: Any, rect: "fitz.Rect") -> Optional[Dict[str, Any]]: """Read the typography of the native text under `rect`. @@ -470,34 +461,6 @@ def capture_text_style(page: Any, rect: "fitz.Rect") -> Optional[Dict[str, Any]] } -def target_chars(page: Any, full: "fitz.Rect") -> List[Dict[str, Any]]: - """Characters of the line a word box was taken from (single baseline).""" - try: - blocks = page.get_text("rawdict", clip=full).get("blocks", []) - except Exception: - return [] - chars: List[Tuple[float, Dict[str, Any]]] = [] - for block in blocks: - if block.get("type") != 0: - continue - for line in block.get("lines", []): - for span in line.get("spans", []): - for ch in span.get("chars", []): - if not str(ch.get("c", "")).strip(): - continue - bx0, by0, bx1, by1 = ch["bbox"] - cx, cy = (bx0 + bx1) / 2, (by0 + by1) / 2 - if full.x0 <= cx <= full.x1 and full.y0 <= cy <= full.y1: - chars.append((round(float(ch["origin"][1]), 1), ch)) - if not chars: - return [] - centre = (full.y0 + full.y1) / 2 - bases = {b for b, _ in chars} - inside = [b for b in bases if full.y0 <= b <= full.y1] or list(bases) - base = min(inside, key=lambda b: abs(b - centre)) - return [ch for b, ch in chars if b == base] - - def band_covers_target(page: Any, band: "fitz.Rect", full: "fitz.Rect") -> bool: """True when `band` reaches the deletion core of every target character. @@ -901,6 +864,15 @@ def ocr_language_for_choice(choice: str, tessdata_dir: Optional[str]) -> str: QProgressBar { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; text-align: center; color: transparent; height: 8px;} QProgressBar::chunk { background-color: #0ea5e9; border-radius: 3px; } QGraphicsView { border: none; background-color: #020617; } +QListWidget, QTableWidget, QTextBrowser { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; color: #cbd5e1; gridline-color: #334155; selection-background-color: #0284c7; selection-color: white; } +QListWidget::item { padding: 6px; } +QHeaderView::section { background-color: #1e293b; color: #e2e8f0; border: 1px solid #334155; padding: 4px; } +QComboBox, QDoubleSpinBox { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; padding: 4px; color: #e2e8f0; } +QComboBox QAbstractItemView { background-color: #0f172a; color: #e2e8f0; selection-background-color: #0284c7; } +QTabWidget::pane { border: 1px solid #334155; } +QTabBar::tab { background-color: #1e293b; color: #cbd5e1; padding: 6px 14px; border: 1px solid #334155; } +QTabBar::tab:selected { background-color: #0284c7; color: white; } +QToolTip { background-color: #1e293b; color: #e2e8f0; border: 1px solid #0ea5e9; padding: 4px; } QRadioButton::indicator, QCheckBox::indicator { width: 16px; height: 16px; border: 1px solid #475569; border-radius: 8px; background-color: #0f172a; } QCheckBox::indicator { border-radius: 4px; } QRadioButton::indicator:checked, QCheckBox::indicator:checked { background-color: #0ea5e9; border: 1px solid #0ea5e9; } @@ -1338,7 +1310,7 @@ def __init__(self) -> None: # Detections the evidence score could not settle; never applied # silently, reviewed by the user (see review_dialog.ReviewDialog). self.pending_suggestions: List[Dict[str, Any]] = [] - self._export_labels: List[Tuple[int, Any, str, Optional[Dict[str, Any]]]] = [] + self._export_labels: List[Tuple[int, Any, List[str], Optional[Dict[str, Any]], Optional[str]]] = [] self.config_dir = pathlib.Path.home() / ".nullifypdf" # Update-check threads outlive their dialogs: keep them referenced # until they finish so a QThread is never destroyed while running. @@ -1359,6 +1331,9 @@ def __init__(self) -> None: self.start_scan_sig.connect(self.ai_worker.run_scan) self.ai_thread.start() self.build_ui() + apply_tooltips(self) + self._f1 = QShortcut(QKeySequence("F1"), self) + self._f1.activated.connect(self.cmd_guide) if self.list_manager.expired_removed: self.write_log( f"Rimossi {self.list_manager.expired_removed} termini scaduti dai dizionari." @@ -1546,6 +1521,9 @@ def build_ui(self) -> None: btn_wipe.clicked.connect(self.cmd_wipe_local_data) s_lay.addWidget(btn_wipe) s_lay.addStretch() + btn_guide = QPushButton("Guida") + btn_guide.clicked.connect(self.cmd_guide) + s_lay.addWidget(btn_guide) btn_about = QPushButton("Info") btn_about.clicked.connect(self.cmd_about) s_lay.addWidget(btn_about) @@ -1564,6 +1542,7 @@ def build_ui(self) -> None: btn_zout = QPushButton("-") btn_zout.clicked.connect(lambda: self.adjust_zoom(-1)) self.lbl_zoom = QLabel("150%") + self.lbl_zoom.setObjectName("lbl_zoom") btn_zin = QPushButton("+") btn_zin.clicked.connect(lambda: self.adjust_zoom(1)) tb_lay.addWidget(btn_zout) @@ -1573,10 +1552,12 @@ def build_ui(self) -> None: btn_prev = QPushButton("<") btn_prev.clicked.connect(lambda: self.move_page(-1)) self.le_page = QLineEdit("0") + self.le_page.setObjectName("le_page") self.le_page.setFixedWidth(40) self.le_page.setAlignment(Qt.AlignmentFlag.AlignCenter) self.le_page.returnPressed.connect(self.jump_page) self.lbl_tot = QLabel("/ 0") + self.lbl_tot.setObjectName("lbl_tot") btn_next = QPushButton(">") btn_next.clicked.connect(lambda: self.move_page(1)) tb_lay.addWidget(btn_prev) @@ -1586,6 +1567,7 @@ def build_ui(self) -> None: r_lay.addWidget(top_bar) self.scene = QGraphicsScene() self.view = PDFView(self.scene) + self.view.setObjectName("view") self.view.rect_drawn.connect(self.user_draw_rect) self.view.point_clicked.connect(self.user_click_pt) self.view.zoom_req.connect(self.adjust_zoom) @@ -1594,8 +1576,10 @@ def build_ui(self) -> None: footer.setObjectName("Panel") f_lay = QVBoxLayout(footer) self.prog = QProgressBar() + self.prog.setObjectName("prog") self.prog.setValue(0) self.log = QTextEdit() + self.log.setObjectName("log") self.log.setReadOnly(True) self.log.setFixedHeight(80) f_lay.addWidget(self.prog) @@ -1777,6 +1761,9 @@ def _decide_suggestion(self, s: Dict[str, Any]) -> None: b_yes = box.addButton("Censura", QMessageBox.ButtonRole.AcceptRole) b_no = box.addButton("Scarta", QMessageBox.ButtonRole.DestructiveRole) b_show = box.addButton("Mostra valore", QMessageBox.ButtonRole.ActionRole) + b_yes.setToolTip("Applica la censura a questo suggerimento.") + b_no.setToolTip("Scarta il suggerimento senza censurare nulla.") + b_show.setToolTip("Mostra il valore in chiaro prima di decidere.") box.addButton("Annulla", QMessageBox.ButtonRole.RejectRole) box.exec() clicked = box.clickedButton() @@ -1851,13 +1838,7 @@ def user_draw_rect(self, qrect: QRectF) -> None: p = self.doc[self.page_num] txt = text_under_rect(p, fitz.Rect(r)) if not txt: - p.add_redact_annot( - r, - text="[ IMMAGINE RIMOSSA ]", - align=1, - fill=(0.9, 0.9, 0.9), - fontsize=8, - ) + self._add_kind_redaction(p, r, "IMAGE") else: self._add_privacy_redaction( p, @@ -2014,6 +1995,10 @@ def _on_auto_update_result(self, release: Any) -> None: if release is not None: self._show_update_dialog(release, self) + def cmd_guide(self) -> None: + """Open the user guide (also bound to F1).""" + GuideDialog(self).exec() + def cmd_about(self) -> None: """Show about dialog.""" d = QDialog(self) @@ -2094,6 +2079,7 @@ def start_check() -> None: self._start_bg_thread(w) btn_upd.clicked.connect(start_check) + apply_tooltips(d) d.exec() state["closed"] = True @@ -2192,6 +2178,7 @@ def start_download() -> None: self._start_bg_thread(w) btn_dl.clicked.connect(start_download) + apply_tooltips(d) d.exec() state["closed"] = True w = state["worker"] @@ -2305,6 +2292,7 @@ def set_all(state: bool) -> None: bb.accepted.connect(dlg.accept) bb.rejected.connect(dlg.reject) refresh() + apply_tooltips(dlg) if dlg.exec() != QDialog.DialogCode.Accepted: return None chosen = [gid for gid in ALL_GROUP_IDS if boxes[gid].isChecked()] @@ -2328,9 +2316,15 @@ def _select_privacy_export(self) -> Optional[PrivacyMode]: anonymize_btn = box.addButton( "Rimozione irreversibile", QMessageBox.ButtonRole.AcceptRole ) + anonymize_btn.setToolTip( + "Elimina per sempre i valori selezionati dal file. Non si può tornare indietro." + ) pseudonymize_btn = box.addButton( "Pseudonimizzazione reversibile", QMessageBox.ButtonRole.ActionRole ) + pseudonymize_btn.setToolTip( + "Sostituisce i valori con segnaposto e crea una mappa cifrata per ripristinarli." + ) box.addButton(QMessageBox.StandardButton.Cancel) box.exec() clicked = box.clickedButton() @@ -2375,21 +2369,53 @@ def _add_privacy_redaction( original: str = "", entity_type: str = "DATA", fill: Tuple[float, float, float] = (0, 0, 0), + full: Optional[fitz.Rect] = None, ) -> None: - """Add a redaction annotation carrying optional in-memory privacy data.""" + """Add a redaction annotation carrying optional in-memory privacy data. + + `full` is the whole box of the word when `rect` is a thinner band: the + export uses it to check that no glyph of the word survived. + """ annot = page.add_redact_annot(rect, fill=fill) clean_original = " ".join((original or "").split()) if clean_original: - payload = { + payload: Dict[str, Any] = { "nullifypdf": 1, "original": clean_original, "entity_type": PlaceholderRegistry.normalize_entity_type(entity_type), } + if full is not None and fitz.Rect(full) != fitz.Rect(rect): + payload["full"] = [full.x0, full.y0, full.x1, full.y1] try: annot.set_info(content=json.dumps(payload, ensure_ascii=False)) except Exception as e: self.logger.debug(f"Could not store redaction metadata: {e}") + def _add_kind_redaction(self, page: Any, rect: fitz.Rect, kind: str) -> None: + """Redaction of a graphic area (image or signature) without overlay text. + + The label is drawn after the redactions are applied (see + `_finish_labels_page`), so it is not lost in a small box. + """ + annot = page.add_redact_annot(rect, fill=(0.9, 0.9, 0.9)) + try: + annot.set_info(content=json.dumps( + {"nullifypdf": 1, "original": "", "entity_type": kind} + )) + except Exception as e: + self.logger.debug(f"Could not store {kind} metadata: {e}") + + def _read_redaction_full(self, annot: Any) -> Optional[fitz.Rect]: + """The whole word box stored with a band annotation, if any.""" + try: + raw = annot.info.get("content", "") + payload = json.loads(raw) if raw else {} + if payload.get("nullifypdf") == 1 and payload.get("full"): + return fitz.Rect(*[float(v) for v in payload["full"]]) + except Exception: + return None + return None + def _read_redaction_payload(self, annot: Any) -> Dict[str, str]: """Read privacy data stored on a pending redaction annotation.""" try: @@ -2422,14 +2448,11 @@ def _prepare_pseudonymized_page( if annot.type[0] == PDF_ANNOT_REDACT: page.delete_annot(annot) for rect, payload, clipped_text, style in pending: - if payload.get("entity_type") == "SIGNATURE": - page.add_redact_annot( - rect, - text="[FIRMA RIMOSSA]", - fill=(1, 1, 1), - text_color=(0, 0, 0), - align=1, - fontsize=fit_redaction_fontsize("[FIRMA RIMOSSA]", rect), + if payload.get("entity_type") in ("SIGNATURE", "IMAGE"): + self._add_kind_redaction(page, rect, str(payload["entity_type"])) + self._export_labels.append( + (page.number, fitz.Rect(rect), + KIND_LABELS[str(payload["entity_type"])], None, None) ) continue clean_original = " ".join( @@ -2451,37 +2474,47 @@ def _prepare_pseudonymized_page( # the annotation is dropped when it cannot be laid out at 4 pt. page.add_redact_annot(rect, fill=(1, 1, 1)) self._export_labels.append( - (page.number, fitz.Rect(rect), placeholder, style) + (page.number, fitz.Rect(rect), compact_labels(placeholder), + style, placeholder) ) else: - page.add_redact_annot( - rect, - text="[IMAGE_REMOVED]", - fill=(1, 1, 1), - text_color=(0, 0, 0), - align=1, - fontsize=fit_redaction_fontsize("[IMAGE_REMOVED]", rect), + self._add_kind_redaction(page, rect, "IMAGE") + self._export_labels.append( + (page.number, fitz.Rect(rect), KIND_LABELS["IMAGE"], None, None) ) - def _finish_pseudonymized_page( - self, page: Any, registry: PlaceholderRegistry + def _prepare_label_annots(self, page: Any) -> None: + """Anonymize mode: remember the image / signature areas to label.""" + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + kind = self._read_redaction_payload(annot).get("entity_type") + if kind in KIND_LABELS: + self._export_labels.append( + (page.number, fitz.Rect(annot.rect), KIND_LABELS[kind], None, None) + ) + + def _finish_labels_page( + self, page: Any, registry: Optional[PlaceholderRegistry] = None ) -> None: - """Draw the placeholders of `page` once its redactions are applied.""" - for page_no, rect, placeholder, style in self._export_labels: + """Draw placeholders and image/signature labels once redactions are applied.""" + for page_no, rect, labels, style, placeholder in self._export_labels: if page_no != page.number: continue drawn = write_label( - page, - rect, - compact_labels(placeholder), - style=style, - allowance=free_space_right(page, rect), + page, rect, labels, style=style, + allowance=free_space_right(page, rect) if placeholder else 0.0, ) - if drawn != placeholder: + if placeholder and registry is not None and drawn != placeholder: registry.set_label( placeholder, (rect.x0, rect.y0, rect.x1, rect.y1), drawn ) + def _finish_pseudonymized_page( + self, page: Any, registry: PlaceholderRegistry + ) -> None: + self._finish_labels_page(page, registry) + @Slot(int, object) def apply_ai_to_page(self, i: int, detections: Any) -> None: """Apply AI-detected redactions to page. @@ -2539,13 +2572,7 @@ def already_covered(r: fitz.Rect) -> bool: ir = fitz.Rect(img["bbox"]) if already_covered(ir): continue - page.add_redact_annot( - ir, - text="[ IMMAGINE RIMOSSA ]", - align=1, - fill=(0.9, 0.9, 0.9), - fontsize=8, - ) + self._add_kind_redaction(page, ir, "IMAGE") e_rects.append(ir) try: has_images = bool(page.get_image_info(hashes=False)) @@ -2583,14 +2610,7 @@ def already_covered(r: fitz.Rect) -> bool: for r in rects: if already_covered(r): continue - annot = page.add_redact_annot(r, fill=(0.9, 0.9, 0.9)) - try: - annot.set_info(content=json.dumps({ - "nullifypdf": 1, "original": "", - "entity_type": "SIGNATURE", - })) - except Exception as e: - self.logger.debug(f"Could not store signature metadata: {e}") + self._add_kind_redaction(page, r, "SIGNATURE") e_rects.append(r) continue if not rects and word: @@ -2610,13 +2630,18 @@ def already_covered(r: fitz.Rect) -> bool: for pr in protected ) ] - for r in rects: + for k, r in enumerate(rects): if not already_covered(r): + full_box = ( + fitz.Rect(*fulls[k]) + if (k < len(fulls) and raw_rects) else None + ) self._add_privacy_redaction( page, r, original=word, entity_type=entity_type, + full=full_box, ) e_rects.append(r) @@ -2690,6 +2715,11 @@ def _select_pseudonym_style(self) -> Optional[Tuple[str, Optional[bytes]]]: b_opaque = box.addButton("Generico", QMessageBox.ButtonRole.ActionRole) b_keyed = box.addButton("Coerente (chiave)", QMessageBox.ButtonRole.ActionRole) b_manage = box.addButton("Gestisci chiave...", QMessageBox.ButtonRole.ActionRole) + b_random.setToolTip("Segnaposto casuali: non rivelano ordine né numero dei valori (consigliato).") + b_counter.setToolTip("PERSON_001, PERSON_002...: rivelano quanti valori distinti ci sono e l'ordine.") + b_opaque.setToolTip("Segnaposto generici: nascondono anche il tipo di dato.") + b_keyed.setToolTip("Lo stesso valore ha lo stesso segnaposto in ogni documento creato con la stessa chiave.") + b_manage.setToolTip("Esporta o importa la chiave dei segnaposto coerenti, protetta da password.") box.addButton(QMessageBox.StandardButton.Cancel) box.exec() clicked = box.clickedButton() @@ -2771,6 +2801,8 @@ def _pseudonym_key_dialog(self) -> None: ) b_exp = box.addButton("Esporta chiave...", QMessageBox.ButtonRole.ActionRole) b_imp = box.addButton("Importa chiave...", QMessageBox.ButtonRole.ActionRole) + b_exp.setToolTip("Salva una copia della chiave protetta da password.") + b_imp.setToolTip("Carica una copia della chiave esportata da un altro computer.") box.addButton(QMessageBox.StandardButton.Close) box.exec() clicked = box.clickedButton() @@ -2844,9 +2876,12 @@ def _offer_recovery_key(self) -> Optional[bytes]: field.setReadOnly(True) lay.addWidget(field) row = QHBoxLayout() + field.setToolTip("Chiave di recupero: mostrata una sola volta e non salvata dal programma.") btn_copy = QPushButton("Copia") + btn_copy.setToolTip("Copia la chiave negli appunti (vengono svuotati alla chiusura).") btn_copy.clicked.connect(lambda: QApplication.clipboard().setText(shown)) btn_save = QPushButton("Salva su file...") + btn_save.setToolTip("Salva la chiave in un file di testo, da conservare lontano dalla mappa.") def save_file() -> None: path, _ = QFileDialog.getSaveFileName( @@ -2862,6 +2897,7 @@ def save_file() -> None: chk = QCheckBox( "Ho conservato la chiave di recupero in un luogo sicuro, separato dalla mappa" ) + chk.setToolTip("Spunta per confermare: serve per proseguire.") lay.addWidget(chk) bb = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok) ok_btn = bb.button(QDialogButtonBox.StandardButton.Ok) @@ -2886,6 +2922,8 @@ def _ask_decrypt_secret(self) -> Optional[Tuple[Optional[str], Optional[bytes]]] field.setEchoMode(QLineEdit.EchoMode.Password) lay.addWidget(field) use_rk = QCheckBox("Usa la chiave di recupero") + use_rk.setToolTip("Apre la mappa con la chiave di recupero (NPDF-...) invece che con la password.") + field.setToolTip("Password della mappa, oppure la chiave di recupero se hai spuntato l'opzione.") def toggled(on: bool) -> None: label.setText( @@ -3004,6 +3042,8 @@ def cmd_export(self) -> None: ) b_review = box.addButton("Rivedi ora", QMessageBox.ButtonRole.AcceptRole) b_go = box.addButton("Esporta comunque", QMessageBox.ButtonRole.DestructiveRole) + b_review.setToolTip("Apre l'elenco dei suggerimenti prima di esportare.") + b_go.setToolTip("Esporta senza censurare i suggerimenti non revisionati.") box.addButton(QMessageBox.StandardButton.Cancel) box.exec() if box.clickedButton() == b_review: @@ -3100,11 +3140,11 @@ def cmd_export(self) -> None: source_sha256 = sha256_file(source_name) if os.path.exists(source_name) else "" prepare = ( (lambda page: self._prepare_pseudonymized_page(page, registry)) - if mode == PrivacyMode.PSEUDONYMIZE else None + if mode == PrivacyMode.PSEUDONYMIZE else self._prepare_label_annots ) finish = ( - (lambda page: self._finish_pseudonymized_page(page, registry)) - if mode == PrivacyMode.PSEUDONYMIZE else None + (lambda page: self._finish_labels_page(page, registry)) + if mode == PrivacyMode.PSEUDONYMIZE else self._finish_labels_page ) self._export_labels = [] with QMutexLocker(self.doc_mutex): @@ -3115,6 +3155,7 @@ def cmd_export(self) -> None: p, prepare_page=prepare, finish_page=finish, + read_full=self._read_redaction_full, policy=policy, removed_values=removed_values, temp_dir=private_temp_dir(self.config_dir), diff --git a/README.md b/README.md index e4af195..50971b2 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi - 🗄️ **Fluid UI & Thread-Safe** — PySide6 modern dark-mode interface with zero UI freezing. Text extraction in worker thread with QMutex serialization - 📖 **Persistent Dictionaries** — Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with O(1) fast-path matching - 🛡️ **Privacy Export Modes** — Choose irreversible removal or reversible pseudonymization with a separately encrypted restore map; the exported file is sanitised (annotations, attachments, scripts, metadata, hidden text) and verified +- 📖 **Built-in guide and tooltips** — Press F1 for an Italian guide to every function; every control shows a description on hover - 🔍 **Review of uncertain hits** — Detections without enough evidence are suggested for review instead of being applied or lost silently - 🔎 **OCR Support** — Bundled EN/IT Tesseract language data for scanned PDFs - 🖼️ **Blindfold Mode** — One-click image/logo censoring with professional placeholder: `[ IMAGE REMOVED ]` @@ -263,5 +264,5 @@ Want to help improve NullifyPDF? See [CONTRIBUTING.md](./CONTRIBUTING.md) for gu --- -*Last updated: 2026-11-08* +*Last updated: 2026-11-10* *[User Guide →](./USER_GUIDE.md)* diff --git a/SECURITY.md b/SECURITY.md index 3ae64aa..54bd5b9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -39,7 +39,7 @@ When you export a PDF with redactions: 1. ✅ **Redactions applied** — selected text/image areas are removed from the content stream, not just covered 2. ✅ **Sanitiser** — annotations, embedded files, external links, bookmarks, JavaScript, tagged-PDF alternative text, thumbnails, document and XMP metadata, invisible text, text outside the visible page box and the trailer ID are removed; hidden layers are shown before the scan so their text is analysed -3. ✅ **Verification** — the written file is re-opened and checked for any removed value in the text layer, every object and decompressed stream, annotations, attachments, bookmarks and metadata; findings list locations, never the values +3. ✅ **Verification** — every redacted word is checked glyph by glyph (the number of characters actually deleted must match the measured deletion rule, and characters a thin box cannot reach are reported), then the written file is re-opened and checked for any removed value in the text layer, every object and decompressed stream, annotations, attachments, bookmarks and metadata; findings list locations, never the values 4. ✅ **Pseudonymization maps encrypted** — restore maps are stored separately and encrypted with a user password (scrypt key derivation, authenticated encryption) 5. ⚙️ **Optional width masking** — the space left by removed text can be rounded so its width is not recoverable (off by default: it can shift the following text slightly) @@ -60,7 +60,7 @@ Limits you should know about: - Original PDF kept in memory only while open - AI results (detected entities) stored only in memory during the session; suggestions are never written to disk - The restore map is written with owner-only permissions where the platform supports it; keep it away from the PDF and from synchronised folders -- The blocklist and allowlist (`~/.nullifypdf/*.v2.enc`) may themselves contain personal data, so they are encrypted with AES-256-GCM under a master key protected by Windows DPAPI, the OS keyring (optional `keyring` package) or, as a fallback, a local key file. Terms can expire and are then purged. DPAPI and the keyring bind the data to the user account and machine; the key file only protects against copying a single data file, not against someone who can read the user's profile. Overwriting migrated plaintext files is best effort (SSD wear levelling and backups may keep copies) +- The blocklist and allowlist (`~/.nullifypdf/*.v2.enc`) may themselves contain personal data, so they are encrypted with AES-256-GCM under a master key protected by Windows DPAPI, the OS keyring (`keyring` package, installed on macOS and Linux; headless Linux without a Secret Service falls back to the key file) or, as a fallback, a local key file. Terms can expire and are then purged. DPAPI and the keyring bind the data to the user account and machine; the key file only protects against copying a single data file, not against someone who can read the user's profile. Overwriting migrated plaintext files is best effort (SSD wear levelling and backups may keep copies) - If the keystore cannot be read (profile copied to another account or PC) nothing is deleted or overwritten: the lists stay empty and the encrypted files untouched - The pseudonym key is stored sealed under the same master key; a password-protected copy can be exported for another computer - Restore maps (version 3 envelope): a random data key encrypts the map with AES-256-GCM; it is wrapped by a password-derived key (Argon2id, scrypt if unavailable) and optionally by a recovery key (256-bit, shown once, never stored). Key-derivation parameters read from a file are bounded before use @@ -299,5 +299,5 @@ For **security vulnerabilities only**: --- -*Last updated: 2026-11-08* +*Last updated: 2026-11-10* *← [Contributing](./CONTRIBUTING.md) | [Back to README →](./README.md)* diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index a294112..24950d6 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -15,3 +15,14 @@ https://www.istat.it/classificazione/codici-dei-comuni-delle-province-e-delle-re The file is generated by `scripts/build_lexicon_data.py`; the SHA-256 of the source workbook is recorded in the generated module. + +## Given names and surnames (Wikidata) + +`lexicon_data_it.py` also contains Italian given names and surnames taken from +Wikidata (items that have an article on the Italian Wikipedia). + +- License: Creative Commons Zero (CC0 1.0), all structured data of Wikidata + (https://www.wikidata.org/wiki/Wikidata:Licensing). +- The lists are small (a few hundred given names and about one thousand + surnames) and are only used to *confirm* a name; a name missing from them + never causes a hit to be rejected. diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 26d9630..b3cf0da 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -20,6 +20,9 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel --- +> [!TIP] +> The program has its own guide in Italian: press **F1** or the **Guida** button in the sidebar. Resting the pointer on any button, option or field shows a short description. + ## Step 1️⃣ — Load Your PDF 1. Click the blue **"Apri PDF"** button in the top left @@ -305,5 +308,5 @@ python3.13 NullifyPDF.py --- -*Last updated: 2026-11-08* +*Last updated: 2026-11-10* *← [Back to README](./README.md) | [Troubleshooting →](./TROUBLESHOOTING.md)* diff --git a/build_local.py b/build_local.py index 773527b..cf2b9b2 100644 --- a/build_local.py +++ b/build_local.py @@ -393,11 +393,12 @@ def build_app() -> None: if sys_os == "Darwin": spec_content = f"""# -*- mode: python ; coding: utf-8 -*- +import sys from PyInstaller.utils.hooks import collect_all datas = {datas_literal} binaries = [] hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it'] -for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: +for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md'] + (['keyring'] if sys.platform != 'win32' else []): t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] @@ -409,11 +410,12 @@ def build_app() -> None: """ else: spec_content = f"""# -*- mode: python ; coding: utf-8 -*- +import sys from PyInstaller.utils.hooks import collect_all datas = {datas_literal} binaries = [] hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it'] -for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']: +for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md'] + (['keyring'] if sys.platform != 'win32' else []): t = collect_all(pkg) datas += t[0]; binaries += t[1]; hiddenimports += t[2] diff --git a/dictionary_dialog.py b/dictionary_dialog.py index a8e84a6..ec26eff 100644 --- a/dictionary_dialog.py +++ b/dictionary_dialog.py @@ -23,6 +23,7 @@ QWidget, ) +from help_guide import apply_tooltips from word_lists import ListEntry, PDFListManager, MIN_TERM_LENGTH TTL_CHOICES = [("Mai", None), ("30 giorni", 30), ("90 giorni", 90), @@ -50,6 +51,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None: self.saved_allowlist: Optional[set] = None lay = QVBoxLayout(self) self.status = QLabel(manager.protector_label()) + self.status.setObjectName("dict_status") lay.addWidget(self.status) if manager.keystore_error: warn = QLabel( @@ -60,6 +62,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None: warn.setStyleSheet("color: #f59e0b;") lay.addWidget(warn) self.tabs = QTabWidget() + self.tabs.setObjectName("dict_tabs") self.tables: Dict[str, QTableWidget] = {} for kind, title in KINDS: tab = QWidget() @@ -68,6 +71,8 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None: table.setHorizontalHeaderLabels(["Termine", "Aggiunto il", "Scade il (AAAA-MM-GG)"]) table.horizontalHeader().setStretchLastSection(True) table.setColumnWidth(COL_TERM, 260) + table.setObjectName("dict_table") + table.verticalHeader().setVisible(False) self.tables[kind] = table tl.addWidget(table) row = QHBoxLayout() @@ -90,6 +95,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None: ttl_row = QHBoxLayout() ttl_row.addWidget(QLabel("Scadenza predefinita dei nuovi termini:")) self.ttl = QComboBox() + self.ttl.setObjectName("dict_ttl") current = manager.default_ttl_days() for i, (label, days) in enumerate(TTL_CHOICES): self.ttl.addItem(label, days) @@ -102,6 +108,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None: btn.setObjectName("Primary") btn.clicked.connect(self.save) lay.addWidget(btn) + apply_tooltips(self) def _on_button(self) -> None: sender = self.sender() diff --git a/export_engine.py b/export_engine.py index 629dc01..115613a 100644 --- a/export_engine.py +++ b/export_engine.py @@ -43,6 +43,7 @@ class ExportResult: sanitize: SanitizeReport = field(default_factory=SanitizeReport) verification: VerificationReport = field(default_factory=VerificationReport) pages: int = 0 + surviving_glyphs: int = 0 def neutral_output_name(mode_value: str, now: Optional[datetime.datetime] = None) -> str: @@ -149,6 +150,90 @@ def text_outside_redactions(doc: Any) -> str: return " ".join(chunks) +def _line_chars(page: Any, baseline: float, tol: float = 1.5) -> List[Dict[str, Any]]: + """Non-space characters of `page` on one baseline, left to right.""" + try: + raw = page.get_text("rawdict") + except Exception: + return [] + out: List[Dict[str, Any]] = [] + for block in raw.get("blocks", []): + if block.get("type") != 0: + continue + for line in block.get("lines", []): + for span in line.get("spans", []): + for ch in span.get("chars", []): + if str(ch.get("c", "")).strip() and abs(float(ch["origin"][1]) - baseline) <= tol: + out.append(ch) + out.sort(key=lambda c: float(c["origin"][0])) + return out + + +def _expected_deletions( + page: Any, read_full: Callable[[Any], Optional[Any]] +) -> List[Dict[str, Any]]: + """Per redaction with a stored whole-word box: what must disappear. + + After apply_redactions() every glyph of the word must be gone: a thin + band that misses one glyph (a raised or smaller character) would leave + part of the value behind, and a text search cannot see a single + character. Two independent checks, neither of which relies on glyph + positions (MuPDF rewrites and shifts the rest of the line): + + * predicted survivors: target glyphs whose deletion core the band does + not reach (the measured rule); + * rule check: on that line, the number of glyphs that actually vanish + must be at least the number the rule predicts. If a PyMuPDF release + changes the rule, this trips instead of leaving the export silently + unsafe. + """ + from pii_detection import core_hits, target_chars + + bands = [ + pymupdf.Rect(a.rect) for a in page.annots() or [] + if a.type[0] == PDF_ANNOT_REDACT + ] + expected: List[Dict[str, Any]] = [] + for annot in page.annots() or []: + if annot.type[0] != PDF_ANNOT_REDACT: + continue + full = read_full(annot) + if full is None: + continue + targets = target_chars(page, pymupdf.Rect(full)) + if not targets: + continue + baseline = float(targets[0]["origin"][1]) + line = _line_chars(page, baseline) + predicted = sum( + 1 for c in line if any(core_hits(tuple(c["bbox"]), b) for b in bands) + ) + missed = sum( + 1 for t in targets + if not any(core_hits(tuple(t["bbox"]), b) for b in bands) + ) + expected.append({ + "baseline": baseline, "n_pre": len(line), + "predicted_deleted": predicted, "missed": missed, + }) + return expected + + +def _count_survivors(page: Any, expected: List[Dict[str, Any]]) -> int: + survivors = 0 + seen_lines = set() + for item in expected: + survivors += int(item["missed"]) + key = round(item["baseline"], 1) + if key in seen_lines: + continue # several words on one line share the same count + seen_lines.add(key) + actual = item["n_pre"] - len(_line_chars(page, item["baseline"])) + if actual < item["predicted_deleted"]: + survivors += item["predicted_deleted"] - actual + return survivors + + def _open_copy( src_doc: Any, temp_dir: Optional[pathlib.Path] ) -> tuple[Any, Optional[str]]: @@ -180,6 +265,7 @@ def export_document( *, prepare_page: Optional[Callable[[Any], None]] = None, finish_page: Optional[Callable[[Any], None]] = None, + read_full: Optional[Callable[[Any], Optional[Any]]] = None, policy: Optional[SanitizePolicy] = None, removed_values: Iterable[str] = (), temp_dir: Optional[pathlib.Path] = None, @@ -190,7 +276,10 @@ def export_document( kept_text = text_outside_redactions(src_doc) ex_doc, tmp = _open_copy(src_doc, temp_dir) try: + survivors_to_check: Dict[int, List[Dict[str, Any]]] = {} for page in ex_doc.pages(): + if read_full is not None: + survivors_to_check[page.number] = _expected_deletions(page, read_full) if prepare_page is not None: prepare_page(page) redact_rects = [ @@ -207,6 +296,9 @@ def export_document( # PDF_REDACT_IMAGE_PIXELS blanks only the pixels under each rect; # IMAGE_REMOVE would delete a whole scanned page for one word. page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True) + result.surviving_glyphs += _count_survivors( + page, survivors_to_check.get(page.number, []) + ) if finish_page is not None: # Text drawn after the redactions are applied: placeholders # must not go through apply_redactions(), which silently @@ -233,6 +325,10 @@ def export_document( pass result.output_sha256 = sha256_of(out_path) result.verification = verify_export(out_path, removed_values, kept_text) + if result.surviving_glyphs: + result.verification.add( + f"{result.surviving_glyphs} glyph(s) survived redaction" + ) return result diff --git a/help_guide.py b/help_guide.py new file mode 100644 index 0000000..1ce641e --- /dev/null +++ b/help_guide.py @@ -0,0 +1,415 @@ +"""In-app user guide (Italian) and tooltips for every GUI element (Qt). + +The guide is opened from the "Guida" button or with F1. Tooltips are applied +by :func:`apply_tooltips`, which looks widgets up by their visible text or +object name; a test checks that no widget of the main window or of the +dialogs is left without one. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Tuple + +from PySide6.QtCore import Qt +from PySide6.QtWidgets import ( + QAbstractButton, + QComboBox, + QDialog, + QDoubleSpinBox, + QGraphicsView, + QHBoxLayout, + QLabel, + QLineEdit, + QListWidget, + QProgressBar, + QPushButton, + QTableWidget, + QTabWidget, + QTextBrowser, + QTextEdit, + QVBoxLayout, + QWidget, +) + +# -------------------------------------------------------------------------- +# Tooltips +# -------------------------------------------------------------------------- + +# Exact visible text (or object name) -> tooltip. +TOOLTIPS: Dict[str, str] = { + # Sidebar + "Apri PDF": "Apre un file PDF da elaborare. Puoi anche trascinarlo sulla finestra.", + "Modello lingua:": "Lingua dei modelli di riconoscimento dei dati personali.", + "EN": "Usa il modello inglese. Adatto a documenti in inglese.", + "IT": "Usa il modello italiano. Consigliato per documenti in italiano: il modello inglese aggiunge falsi positivi.", + "BOTH": "Usa entrambi i modelli (inglese e italiano). Più lento; utile per documenti misti.", + "Oscura Immagini": "Oscura anche tutte le immagini della pagina (loghi, firme, foto) quando applichi Auto Redact.", + "OCR PDF scansionati": "Legge con l'OCR le pagine scansionate e le immagini incorporate, così anche il testo nelle immagini viene analizzato.", + "Mostra suggerimenti": "Contorna in arancione, nell'anteprima, i suggerimenti ancora da rivedere. Servono solo a video: non finiscono nel file esportato.", + "Profilo documenti sanitari": "Cerca anche date di nascita, numero tessera sanitaria, ricetta e codici di esenzione, e propone firme e timbri. I casi incerti diventano suggerimenti invece di essere scartati.", + "Maschera larghezza testo rimosso": "Arrotonda lo spazio lasciato dal testo rimosso, che altrimenti ne rivela la larghezza. Può spostare di poco il testo seguente.", + "Dizionari": "Modifica le parole da censurare sempre (blocklist) e da non censurare mai (allowlist), con scadenza.", + "Auto Redact": "Analizza il documento e segna i dati personali da censurare. Nulla viene cancellato finché non esporti.", + "Rivedi suggerimenti": "Apre l'elenco dei rilevamenti incerti per decidere uno per uno se censurarli.", + "Pulisci Pagina": "Rimuove tutte le censure segnate sulla pagina corrente.", + "Esporta PDF": "Crea il PDF finale: rimozione irreversibile oppure pseudonimizzazione reversibile. Il file viene pulito e verificato.", + "Ricostruisci PDF": "Rimette i valori originali in un PDF pseudonimizzato usando la sua mappa di ripristino e la password (o la chiave di recupero).", + "Elimina dati locali": "Cancella log, registri di esportazione e file temporanei salvati nella cartella dell'utente. Chiede conferma per dizionari e chiavi.", + "Guida": "Apre la guida all'uso del programma (scorciatoia: F1).", + "Info": "Versione del programma, controllo degli aggiornamenti e collegamento al repository.", + "Esci": "Chiude il programma.", + "NullifyPDF": "NullifyPDF: censura e pseudonimizzazione di PDF in locale.", + "Privacy Edition": "Tutta l'elaborazione avviene sul tuo computer: nessun documento viene inviato in rete.", + # Viewer toolbar + "Visualizzatore Documento": "Anteprima del PDF. Trascina per segnare un'area da censurare; clicca su una censura per rimuoverla; Ctrl + rotella per lo zoom.", + "-": "Riduce lo zoom.", + "+": "Aumenta lo zoom.", + "<": "Va alla pagina precedente.", + ">": "Va alla pagina successiva.", + "le_page": "Numero della pagina corrente. Scrivine un altro e premi Invio per saltarci.", + "lbl_zoom": "Livello di zoom corrente.", + "lbl_tot": "Numero totale di pagine del documento.", + "view": "Anteprima del PDF. Trascina per segnare un'area da censurare; clicca su una censura o su un contorno arancione per decidere; Ctrl + rotella per lo zoom.", + "prog": "Avanzamento dell'analisi, pagina per pagina.", + "log": "Messaggi del programma: avanzamento, avvisi ed errori. Non contiene i dati personali trovati.", + # Review dialog + "Censura": "Applica la censura a questo elemento.", + "Pag.": "Pagina in cui è stato trovato l'elemento.", + "Valore": "Valore trovato, mascherato finché non spunti «Mostra i valori». Per le aree grafiche indica la zona.", + "Tipo": "Tipo di dato rilevato (persona, luogo, telefono, firma...).", + "Punteggio": "Affidabilità del rilevamento, da 0 a 1.", + "Motivi": "Indizi che hanno portato a questo punteggio (contesto, parola comune, inizio riga...).", + "Mostra i valori": "Mostra in chiaro i valori trovati nella tabella.", + "Seleziona tutti": "Spunta tutti gli elementi dell'elenco.", + "Nessuno": "Toglie la spunta a tutti gli elementi.", + "Seleziona ≥ soglia": "Spunta gli elementi con punteggio pari o superiore alla soglia indicata a fianco.", + "review_threshold": "Soglia di punteggio usata da «Seleziona ≥ soglia».", + "Applica selezionati": "Censura gli elementi spuntati e scarta gli altri.", + "Rivedi più tardi": "Chiude l'elenco lasciando i suggerimenti in sospeso.", + "review_table": "Rilevamenti incerti: spunta quelli da censurare. Doppio clic su una riga per andare alla pagina.", + # Dictionary dialog + "dict_status": "Come sono protetti i dizionari su questo computer.", + "Aggiungi": "Aggiunge un termine alla lista.", + "Rimuovi": "Elimina i termini selezionati dalla lista.", + "Importa da testo…": "Incolla più termini, uno per riga.", + "Esporta in chiaro…": "Salva la lista in un file di testo non cifrato (contiene dati personali).", + "Scadenza predefinita dei nuovi termini:": "Dopo quanto tempo i nuovi termini scadono e vengono eliminati da soli.", + "dict_ttl": "Scadenza predefinita dei nuovi termini (anche quelli aggiunti dalla pagina).", + "dict_tabs": "Blocklist: parole sempre censurate. Allowlist: parole mai censurate.", + "dict_table": "Termini della lista con data di inserimento e scadenza (AAAA-MM-GG oppure Mai). Doppio clic per modificare.", + "Salva": "Salva le modifiche ai dizionari.", + # Entity dialog + "Seleziona tutto": "Seleziona tutti i tipi di dato.", + "Deseleziona tutto": "Deseleziona tutti i tipi di dato.", + "Seleziona i dati da censurare / pseudonimizzare:": "Scegli quali tipi di dati cercare. La scelta viene ricordata.", + # Guide + "Chiudi": "Chiude questa finestra.", + "OK": "Conferma la scelta e prosegue.", + "Cancel": "Chiude senza confermare.", + "Annulla": "Chiude senza confermare.", + "Controlla aggiornamenti": "Cerca su GitHub se esiste una versione più recente del programma.", + "Controlla ad ogni avvio": "Controlla gli aggiornamenti a ogni avvio. Disattivato di default; mostra qualcosa solo se c'è una versione nuova.", + "Download": "Scarica nella cartella Download il file di installazione della nuova versione.", +} + +# Visible text of the entity checkboxes -> tooltip. +ENTITY_TOOLTIPS: Dict[str, str] = { + "Nomi e cognomi": "Persone citate nel testo. Le parole comuni e i titoli vengono scartati; i casi incerti vanno in revisione.", + "Luoghi e indirizzi": "Città, indirizzi con numero civico e CAP (solo in un contesto di indirizzo).", + "Email": "Indirizzi di posta elettronica.", + "Telefono": "Numeri di telefono italiani e internazionali (date e numeri d'ordine sono esclusi).", + "IBAN": "Coordinate bancarie IBAN valide.", + "Carte di credito": "Numeri di carta di credito con controllo di validità.", + "Wallet crypto": "Indirizzi di portafogli di criptovalute.", + "Codice fiscale": "Codice fiscale italiano, con controllo del carattere finale.", + "Partita IVA": "Partita IVA italiana, con controllo di validità e parola di contesto.", + "Patente": "Numero di patente di guida (richiede la parola «patente» vicino).", + "Carta d'identità": "Numero di carta d'identità (richiede il contesto).", + "Passaporto": "Numero di passaporto (richiede il contesto).", + "Date di nascita": "Date precedute da «nato/a il», «nascita»...", + "Identificativi sanitari": "Numero della tessera sanitaria (20 cifre) e della ricetta elettronica (NRE).", + "Codici di esenzione": "Codici di esenzione dal ticket (richiedono la parola «esenzione»).", +} + + +def tooltip_for(widget: QWidget) -> str: + """Tooltip text for a widget, from its object name or visible text.""" + name = widget.objectName() + if name and name in TOOLTIPS: + return TOOLTIPS[name] + text = "" + if isinstance(widget, (QAbstractButton, QLabel)): + text = widget.text() if hasattr(widget, "text") else "" + text = re.sub(r"<[^>]+>", "", text).strip() + if text in TOOLTIPS: + return TOOLTIPS[text] + if text in ENTITY_TOOLTIPS: + return ENTITY_TOOLTIPS[text] + if text.startswith("Rivedi suggerimenti"): + return TOOLTIPS["Rivedi suggerimenti"] + if text.startswith("Archivio protetto") or text.startswith("File chiave") or text == "Non cifrato" or text.startswith("Non ancora creato"): + return TOOLTIPS["dict_status"] + if text.endswith("%"): + return TOOLTIPS["lbl_zoom"] + if text.startswith("/ "): + return TOOLTIPS["lbl_tot"] + return _generic_tooltip(widget, text) + + +def _generic_tooltip(widget: QWidget, text: str) -> str: + """Texts that vary at run time (Info and update dialogs).""" + low = text.lower() + if isinstance(widget, QLabel): + if re.match(r"^v\d", low): + return "Versione del programma installata." + if "censura e pseudonimizzazione" in low: + return "Informazioni sul programma, sull'autore e sulla licenza." + if "github.com" in low or "repository" in low: + return "Pagina del progetto su GitHub." + if low.startswith("nuova versione"): + return "Versione più recente disponibile." + if low.startswith(("disponibile", "nessun aggiornamento", "controllo", "scaricato", "download")): + return "Esito dell'operazione in corso." + if low.startswith("questi elementi"): + return "Spiega perché questi elementi sono suggerimenti e non censure automatiche." + if low.startswith("http"): + return "Collegamento alla pagina della release." + return "" + + +# Widgets that deliberately have no tooltip: it would get in the way while reading. +NO_TOOLTIP = frozenset({"guide_topics", "guide_text"}) + +_INTERACTIVE: Tuple[type, ...] = ( + QAbstractButton, QLabel, QLineEdit, QProgressBar, QTextEdit, QGraphicsView, + QTableWidget, QComboBox, QDoubleSpinBox, QTabWidget, QListWidget, QTextBrowser, +) + + +def apply_tooltips(root: QWidget) -> int: + """Set a tooltip on every widget under `root` that has none. Returns the count.""" + n = 0 + for w in [root, *root.findChildren(QWidget)]: + if not isinstance(w, _INTERACTIVE) or w.toolTip() or w.objectName() in NO_TOOLTIP: + continue + tip = tooltip_for(w) + if tip: + w.setToolTip(tip) + n += 1 + return n + + +def widgets_without_tooltip(root: QWidget) -> List[str]: + """Descriptions of interactive widgets that still have no tooltip.""" + missing: List[str] = [] + for w in root.findChildren(QWidget): + if isinstance(w, _INTERACTIVE) and not w.toolTip() and w.objectName() not in NO_TOOLTIP: + # Icons and the inner widgets of composite controls carry no text. + if isinstance(w, QLabel) and not w.text().strip(): + continue + if w.parent() is not None and type(w.parent()).__name__ in ( + "QAbstractSpinBox", "QTabBar", "QHeaderView", "QScrollBar", + "QDoubleSpinBox", "QComboBox", "QTableWidget", + ): + continue + if type(w).__name__ in ("QLineEdit",) and w.parent() is not None and isinstance( + w.parent(), (QDoubleSpinBox, QComboBox) + ): + continue + missing.append(f"{type(w).__name__}({getattr(w, 'text', lambda: '')()!r}, {w.objectName()!r})") + return missing + + +# -------------------------------------------------------------------------- +# Guide +# -------------------------------------------------------------------------- + +GUIDE_TOPICS: List[Tuple[str, str]] = [ + ("Introduzione", """ +

Introduzione

+

NullifyPDF individua i dati personali in un PDF e li censura in modo irreversibile oppure li sostituisce con segnaposto reversibili (pseudonimizzazione). Tutta l'elaborazione avviene sul tuo computer: nessun documento viene inviato in rete.

+

Flusso consigliato

+
    +
  1. Apri PDF (o trascina il file sulla finestra).
    2. +
  2. Scegli la lingua del documento.
    3. +
  3. Premi Auto Redact e scegli i tipi di dati da cercare.
    4. +
  4. Rivedi i suggerimenti incerti e correggi a mano ciò che serve.
    5. +
  5. Premi Esporta PDF, scegli la modalità e controlla il risultato.
    6. +
+

Finché non esporti, le censure sono solo segni sull'anteprima: il documento originale non viene modificato.

+"""), + ("Aprire un PDF e muoversi", """ +

Aprire un PDF e muoversi

+ +"""), + ("Lingua e opzioni", """ +

Lingua e opzioni

+

Modello lingua: IT per documenti italiani (consigliato: il modello inglese su testo italiano aggiunge falsi positivi), EN per l'inglese, BOTH per documenti misti. All'avvio viene scelto IT se il sistema è in italiano.

+ +"""), + ("Auto Redact", """ +

Auto Redact

+

Premendo Auto Redact scegli quali tipi di dati cercare (nomi, luoghi, email, telefono, IBAN, carte, codice fiscale, partita IVA, patente, carta d'identità, passaporto, e con il profilo sanitario date di nascita, tessera sanitaria, ricetta, esenzioni). La scelta viene ricordata.

+

Il programma assegna a ogni nome o luogo un punteggio basato su indizi nel testo: titoli e formule come «nato a» o «residente in», forma da nome proprio, nomi e luoghi noti, ripetizioni. Verbi e parole comuni di certificati e atti («Certifico», «Prescrivo», «Anamnesi»...) vengono scartati.

+ +

I codici con controllo di validità (IBAN, carte, codice fiscale) vengono censurati interi anche se attaccati a un'etichetta.

+"""), + ("Revisione dei suggerimenti", """ +

Revisione dei suggerimenti

+

Al termine dell'analisi si apre l'elenco dei suggerimenti: pagina, valore (mascherato finché non spunti Mostra i valori), tipo, punteggio e motivi.

+ +

Sulla pagina i suggerimenti compaiono come contorni arancioni (solo a video): clicca su uno per scegliere Censura, Scarta o Mostra valore.

+

All'esportazione, se restano suggerimenti non revisionati, il programma te lo ricorda: non verranno censurati.

+"""), + ("Correzioni manuali", """ +

Correzioni manuali

+ +"""), + ("Dizionari", """ +

Dizionari

+

I Dizionari contengono due liste: la blocklist (parole sempre censurate) e l'allowlist (parole mai censurate). Il confronto avviene per parola intera: una parola in allowlist non nasconde mai un valore diverso o più lungo.

+ +"""), + ("Esportare", """ +

Esportare

+

Esporta PDF crea il file finale. Si sceglie tra:

+ +

Stile dei segnaposto: Casuale (consigliato, non rivela ordine né numero dei valori), Progressivo (PERSON_001... rivela quanti valori distinti ci sono), Generico (nasconde anche il tipo), Coerente (chiave) (lo stesso valore ha lo stesso segnaposto in tutti i documenti creati con la stessa chiave; collega i documenti tra loro). Gestisci chiave... esporta o importa la chiave protetta da password per usarla su un altro computer.

+

Mappa di ripristino: salvala lontano dal PDF e non in una cartella sincronizzata. Servono una password (almeno 12 caratteri, chiesta due volte) e, volendo, una chiave di recupero mostrata una sola volta: conservala in un luogo sicuro, il programma non la salva.

+

Cosa fa il programma in esportazione

+ +

Il nome file suggerito non contiene il nome del documento originale. Se scegli un nome che contiene un valore da rimuovere vieni avvisato.

+

Attenzione: la censura non garantisce l'anonimato. Date, luoghi o altri dettagli rimasti possono ancora identificare una persona. Controlla sempre il risultato. Un PDF pseudonimizzato e la sua mappa restano dati personali.

+"""), + ("Ricostruire un PDF", """ +

Ricostruire un PDF

+

Ricostruisci PDF rimette i valori originali in un PDF pseudonimizzato.

+
    +
  1. Scegli il PDF pseudonimizzato e la sua mappa (.nullifypdf-map).
    2. +
  2. Inserisci la password, oppure spunta Usa la chiave di recupero e digita la chiave (NPDF-...).
    3. +
  3. Scegli dove salvare il file ricostruito.
    4. +
+

Il programma controlla che la mappa corrisponda al PDF e ti dice quanti valori sono stati davvero ripristinati. Il file ricostruito contiene di nuovo tutti i dati personali: trattalo di conseguenza.

+"""), + ("Documenti sanitari", """ +

Documenti sanitari

+

Con Profilo documenti sanitari il programma cerca anche:

+ +

In questa modalità nessun caso incerto viene scartato in silenzio: diventa un suggerimento da rivedere. I dati sanitari sono dati di categoria particolare (art. 9 GDPR): controlla sempre il risultato.

+"""), + ("Dati locali e privacy", """ +

Dati locali e privacy

+

Il programma conserva nella cartella dell'utente (~/.nullifypdf): dizionari cifrati, chiave dei segnaposto cifrata, registro rotante dei messaggi, registri di esportazione (solo conteggi e hash) e, per file molto grandi, una cartella temporanea privata svuotata dopo l'esportazione.

+

Elimina dati locali cancella log, registri e file temporanei. Se confermi, elimina anche i dizionari e, con una seconda conferma, le chiavi: senza chiavi dizionari cifrati e chiave dei segnaposto diventano illeggibili.

+

Nessun valore rilevato viene scritto nei log. Nessun documento lascia il computer.

+"""), + ("Info e aggiornamenti", """ +

Info e aggiornamenti

+

La finestra Info mostra la versione e il collegamento al repository. Controlla aggiornamenti interroga le release pubblicate su GitHub; con Controlla ad ogni avvio (disattivato di default) il controllo è eseguito all'avvio e mostra qualcosa solo se esiste una versione più recente.

+"""), + ("Scorciatoie", """ +

Scorciatoie

+ + + + + + + +
F1 o il pulsante GuidaApre questa guida
Ctrl + rotellaZoom dell'anteprima
Trascina sull'anteprimaSegna un'area da censurare
Clic su una censuraLa rimuove
Clic su un contorno arancioneDecide un suggerimento
Trascina un PDF sulla finestraLo apre
+

Passando il mouse su un elemento dell'interfaccia per qualche secondo compare una breve descrizione.

+"""), + ("Limiti e problemi comuni", """ +

Limiti e problemi comuni

+ +"""), +] + + +class GuideDialog(QDialog): + """Topic list on the left, text on the right. Opened with F1.""" + + def __init__(self, parent: Any = None, topic: int = 0) -> None: + super().__init__(parent) + self.setWindowTitle("Guida - NullifyPDF") + self.resize(900, 620) + lay = QHBoxLayout(self) + self.topics = QListWidget() + self.topics.setObjectName("guide_topics") + self.topics.setFixedWidth(230) + for title, _html in GUIDE_TOPICS: + self.topics.addItem(title) + left = QVBoxLayout() + left.addWidget(self.topics) + btn = QPushButton("Chiudi") + btn.clicked.connect(self.accept) + left.addWidget(btn) + lay.addLayout(left) + self.text = QTextBrowser() + self.text.setObjectName("guide_text") + self.text.setOpenExternalLinks(True) + lay.addWidget(self.text, 1) + self.topics.currentRowChanged.connect(self._show) + self.topics.setCurrentRow(max(0, min(topic, len(GUIDE_TOPICS) - 1))) + apply_tooltips(self) # skips the topic list and the text: no hover popups while reading + + def _show(self, row: int) -> None: + if 0 <= row < len(GUIDE_TOPICS): + self.text.setHtml(GUIDE_TOPICS[row][1]) + self.text.moveCursor(self.text.textCursor().MoveOperation.Start) + + +__all__ = [ + "GUIDE_TOPICS", "GuideDialog", "TOOLTIPS", "ENTITY_TOOLTIPS", + "apply_tooltips", "tooltip_for", "widgets_without_tooltip", "Qt", +] diff --git a/lexicon_data_it.py b/lexicon_data_it.py index 7c447c3..b2189dc 100644 --- a/lexicon_data_it.py +++ b/lexicon_data_it.py @@ -7,3 +7,9 @@ COMUNI_AMBIGUOUS = frozenset("""abbiategrasso|acerra|acireale|acri|acuto|adria|affi|afragola|agira|agra|agrigento|agropoli|ala|alassio|alatri|alba|albanella|albenga|alberobello|albi|albignasego|albino|albosaggia|alcamo|ales|alessandria|alfano|alfonsine|alghero|alia|allein|alpignano|altamura|altare|altino|altissimo|alto|altopascio|alì|amalfi|amandola|amaro|amato|amatrice|amelia|ameno|amorosi|ampezzo|anagni|ancona|andora|andria|anela|anghiari|anzi|anzio|aosta|apice|aprica|aprilia|aquileia|aquino|aragona|arborea|arborio|arcade|arce|archi|arco|arcore|ardea|ardore|arena|arenzano|arese|arezzo|argenta|ari|ariccia|armeno|armento|armo|arona|arpino|arre|arzachena|arzano|arzignano|asciano|asiago|asola|asolo|assago|assisi|asso|assolo|asti|atella|atri|augusta|auletta|aulla|avellino|aversa|avetrana|avezzano|aviano|avio|avola|ayas|azeglio|bacoli|badia|badolato|bagheria|bagnoregio|baiano|balocco|banchette|baranzate|barbara|barbaresco|barbata|barbona|bard|bardi|bardolino|barga|bari|barile|barletta|barolo|baschi|basiliano|bastiglia|battipaglia|bee|beinette|belfiore|bella|bellagio|bellino|bellona|belluno|bema|benestare|benevento|benna|bereguardo|bergamasco|bergamo|bettola|beverino|bevilacqua|bianchi|bianco|biassono|bibbiena|biella|bienno|bisaccia|bisceglie|bitonto|bobbio|boca|bollate|bologna|bolsena|bolzano|bomba|bonito|bonnanaro|bono|bordano|bordighera|bore|borghi|borgia|borgomanero|borgoricco|borgosesia|bormida|bormio|borno|bosa|botticino|bova|boves|bovino|bozzolo|bra|bracca|bracciano|braies|brancaleone|branzi|breme|brennero|brenta|brentonico|brescia|bressanone|bresso|brindisi|brolo|broni|bronte|brunello|brunico|bruno|buccheri|buccinasco|burgos|busca|busso|bussolengo|cabras|caccamo|cagli|cagliari|caglio|caino|calcata|calci|calciano|calcinate|calcinato|calcio|calco|caltagirone|caltanissetta|calvello|calvi|camaiore|cambiano|camerino|camerota|camini|camino|camogli|campagna|campana|campobasso|campomarino|camposampiero|camposanto|canale|canaro|candela|candida|caneva|canino|canna|cannobio|canterano|cantù|caorle|capaci|capannori|capodimonte|capri|caprile|capriolo|capua|caraglio|caravaggio|carbonate|carbone|carbonia|carcare|cardinale|carezzano|cariati|carignano|carini|carlino|carmagnola|carmignano|carovigno|carpi|carpino|carrara|carro|carrè|carugate|casalino|casamassima|casarano|cascia|cascina|casella|caserta|casina|casoria|cassino|castana|casteggio|castellabate|castellanza|castellaro|castelli|castelmagno|castelnuovo|castelseprio|castelvetrano|casto|castro|castrovillari|catania|catanzaro|cattolica|cavallino|cavatore|cave|cavezzo|cavour|ceccano|cecina|cefalù|celano|cene|cento|centola|cerano|cerchio|cerea|cerignola|cerva|cerveteri|cervia|cervino|cervo|cesa|cesate|cesena|cesenatico|cesio|ceto|ceva|chianche|chiari|chiavari|chieri|chiesanuova|chieti|chioggia|chiusa|chiusi|chivasso|ciampino|cicala|cingoli|cino|cis|cittadella|civita|civitavecchia|codroipo|colere|coli|colico|collecchio|colleferro|collegno|collio|cologne|colonna|colonnella|comacchio|comiso|como|compiano|comunanza|cona|conegliano|contrada|conversano|copertino|copiano|corato|cori|corio|corleone|correggio|corrido|corsico|cortona|cosenza|courmayeur|covo|cozzo|crema|cremona|crespino|crotone|cuneo|curino|curti|cusio|dalmine|dazio|delia|dello|desio|diamante|dolina|dolo|domodossola|donato|dragoni|drena|dro|dronero|dueville|eboli|elice|ello|empoli|enna|envie|erba|ercolano|este|fabbrico|fabriano|faenza|falcone|fallo|fano|farnese|fasano|fascia|favara|favignana|felino|feltre|fermo|ferrara|ferriere|fiano|fidenza|fiesole|filadelfia|filettino|filetto|firenze|firmo|fisciano|fiuggi|fiumara|fiumicino|foggia|foligno|follonica|fondi|fontanella|fontanelle|fontanile|fonte|forano|force|forcola|forio|forlì|formia|formicola|fornace|fornelli|fortezza|fossa|fossano|foza|frascati|frassino|frattamaggiore|frignano|frisa|front|frontino|frontone|frosinone|furore|gaeta|galatina|galatone|gallarate|gallese|gallicano|gallio|gallipoli|garda|garlasco|gavi|gela|genova|gessate|giarre|ginestra|giove|girasole|giuliana|giulianova|giungano|giussano|giustino|goni|gonzaga|gorgonzola|gorizia|gorno|grado|gradoli|gragnano|graniti|greci|greggio|grimaldi|grosseto|grosso|grottaglie|grotte|gruaro|grugliasco|gualdo|guastalla|gubbio|idro|iglesias|imola|imperia|impruneta|incudine|introd|ischia|iseo|isernia|ispica|ispra|isso|itala|ivrea|jesi|jesolo|ladispoli|laghi|laglio|lago|lainate|lana|lanciano|lariano|laterza|latina|lauro|lavagna|lavello|lecce|lecco|legnago|legnano|lei|lenta|lentini|leonessa|lerici|lesa|lesina|lettere|levate|liberi|licata|licenza|limbiate|limena|limina|linguaglossa|lioni|lipari|liscia|lissone|livigno|livorno|lizzano|loano|locri|loculi|lodi|loiano|lomazzo|longobardi|lonigo|loreto|lucca|lucera|lugo|luino|lula|luni|lupara|luserna|lustra|macello|macerata|maddaloni|mafalda|magenta|maggiora|magione|maglie|maglione|magnago|magnano|maida|maiori|maleo|malfa|malo|mammola|mandela|manduria|manfredonia|mango|manta|mantello|mantova|mappano|mara|maranello|maratea|marciana|marcon|margarita|marino|marsala|marta|martello|martinsicuro|marzano|marzio|mascalucia|maser|masera|masi|massa|massello|matera|matino|matrice|mattinata|meda|mede|medea|medicina|mele|melegnano|meleti|melfi|melilli|melissa|melle|mello|melzo|mentana|merano|merate|merlino|mesagne|mese|messina|mestrino|meta|mezzana|mezzanino|mezzano|milano|milazzo|milena|millesimo|milo|minori|mira|miranda|mirandola|mirano|mirto|modena|modica|modugno|moggio|moglia|mogliano|molare|molfetta|molinara|molise|momo|moncalieri|mondovì|mondragone|monfalcone|monno|monopoli|monreale|monselice|montalcino|montale|montanaro|montebelluna|montecarlo|montecchio|montelabbate|montella|montello|montepulciano|monterotondo|montese|montesilvano|montevarchi|monteverde|montezemolo|monti|montiano|montichiari|montone|monza|morbegno|mordano|mores|moresco|moretta|mori|morigerati|mortara|mossa|mozzate|mozzo|muggiò|mura|muros|musei|musso|napoli|nardò|narni|naro|nasino|naso|nave|ne|nespolo|nesso|nettuno|nichelino|nicosia|nimis|noale|noceto|noci|nola|noli|nomi|none|nonio|norcia|norma|noto|novara|nove|novella|novellara|novello|novoli|nuoro|nus|ocre|oderzo|olbia|oleggio|ome|omegna|onore|opera|opi|ora|orbassano|orbetello|oria|oriolo|oristano|orosei|orte|ortona|orvieto|osimo|ospedaletti|ospedaletto|ossi|ostra|ostuni|otranto|ottaviano|ottone|ovada|pachino|padova|paesana|paese|pagani|paladina|palata|palau|palermo|palestrina|palmanova|palmi|palombaro|paludi|palù|pandino|panettieri|panni|pantelleria|paola|parabiago|parenti|parete|parma|partinico|paterno|paternò|patti|pau|pavia|pedemonte|pelago|pella|penne|perca|pergola|perito|perlo|pero|perugia|pesaro|pescara|pescasseroli|pescate|pesche|pescia|petriolo|pettineo|peveragno|piacenza|pianella|pianezza|pianoro|piatto|picerno|pico|pienza|pietrafitta|pietragalla|pietrasanta|piglio|pigna|pignola|pignone|pigra|pila|pinerolo|pineto|piode|piombino|pisa|pisano|piscina|pistoia|pizzo|poggibonsi|poggioreale|poli|policoro|polistena|polla|pollina|pollone|pomezia|pompei|pompeiana|ponderano|ponte|pontedera|ponti|pontida|ponza|poppi|porcari|pordenone|porte|portici|portoferraio|portofino|portogruaro|portovenere|posada|positano|posta|postal|postiglione|potenza|pozzallo|pozzuoli|prato|pray|preci|predosa|premia|prezza|procida|pula|pulsano|putignano|quadri|quart|quarto|quartucciu|quindici|rabbi|ragusa|randazzo|rapallo|ravello|ravenna|re|rea|recanati|recco|rende|rezzo|rho|riace|rialto|ribera|ricadi|riccia|riccione|rieti|rimini|rio|riposto|rive|rivoli|roana|roccaraso|rodano|rolo|roma|romana|romeno|roncola|rosate|rose|rossa|rossana|rosta|rotella|rotonda|rotondi|rovato|rovereto|rovigo|rozzano|rubano|rubiera|ruoti|russi|rutigliano|sabaudia|sacco|sacile|sale|salemi|salento|salerno|salle|saluzzo|salve|salò|sambuco|samo|samone|sanremo|sansepolcro|sanza|saponara|sapri|saracena|sarnico|sarno|saronno|sarzana|sassari|sassello|sassuolo|sava|saviano|savigliano|savona|scafati|scala|scalea|scandale|scandiano|scandicci|scanno|scansano|scapoli|scena|schio|sciacca|sciara|scicli|scilla|scopa|scorrano|scorzè|sedriano|segni|segrate|selci|senigallia|seregno|seriate|serina|serrata|serre|serri|sesto|sestriere|sestu|settime|seveso|siano|siderno|siena|sigillo|signa|siliqua|silvi|sini|siracusa|sirmione|soave|soglio|solaro|soldano|solferino|sommo|sona|soncino|sondrio|sonico|sora|sorano|sorgono|sori|sorrento|sorso|sospiro|sostegno|soverato|specchia|spello|sperlonga|sperone|spessa|spiazzo|spinea|spinoso|spirano|spoleto|spresiano|statte|stella|stelvio|stilo|stintino|stra|stradella|stresa|strona|strozza|sulmona|supino|susa|suzzara|taggia|taino|taleggio|taormina|taranto|tarquinia|tarsia|taverna|tavullia|teramo|termoli|terni|terracina|terrasini|terrazzo|terzo|tesero|teti|thiene|tiers|tirano|tires|tirol|tirolo|tito|tivoli|todi|tolentino|tolmezzo|tombolo|ton|torino|tornata|torno|toro|torrazzo|torresina|torretta|torricella|torrioni|tortona|tortora|tortorella|tortoreto|tradate|tramonti|trani|trapani|trappeto|travagliato|traversetolo|trebaseleghe|trecate|trento|trevi|treviglio|treviso|trieste|trinità|trino|troia|tropea|trovo|tufo|turate|turi|turri|udine|ugento|ultimo|urbana|urbe|urbino|uri|uscio|ustica|uta|vaglia|vaie|valbrembo|valdobbiadene|valenza|valenzano|vallata|valmontone|valva|valverde|varallo|varazze|varedo|varese|vasto|vecchiano|veglie|veglio|velletri|venetico|venezia|venosa|ventimiglia|verano|verbania|vercelli|verdello|vergato|vermiglio|verona|vescovato|viale|viareggio|vicari|vicenza|vicoli|vieste|vigevano|vignola|vigonza|villamagna|villasimius|villette|villorba|vimercate|vinci|viola|visco|visone|vita|viterbo|vittoria|vittuone|viù|vo'|voghera|volano|voltaggio|volterra|zeri|zone""".split("|")) PROVINCE_SIGLE = frozenset("""AG|AL|AN|AO|AP|AQ|AR|AT|AV|BA|BG|BI|BL|BN|BO|BR|BS|BT|BZ|CA|CB|CE|CH|CI|CL|CN|CO|CR|CS|CT|CZ|EN|FC|FE|FG|FI|FM|FR|GE|GO|GR|IM|IS|KR|LC|LE|LI|LO|LT|LU|MB|MC|ME|MI|MN|MO|MS|MT|NA|NO|NU|OG|OR|OT|PA|PC|PD|PE|PG|PI|PN|PO|PR|PT|PU|PV|PZ|RA|RC|RE|RG|RI|RM|RN|RO|SA|SI|SO|SP|SR|SS|SV|TA|TE|TN|TO|TP|TR|TS|TV|UD|VA|VB|VC|VE|VI|VR|VS|VT|VV""".split("|")) REGIONS = frozenset("""abruzzo|basilicata|calabria|campania|emilia-romagna|friuli-venezia giulia|lazio|liguria|lombardia|marche|molise|piemonte|puglia|sardegna|sicilia|südtirol|toscana|trentino-alto adige|umbria|valle d'aosta|vallée d'aoste|veneto""".split("|")) +# Given names and surnames: Wikidata (CC0, +# https://www.wikidata.org/wiki/Wikidata:Licensing), entries that +# have an article on the Italian Wikipedia. +NAMES_GIVEN = frozenset("""abaco|abramina|abubakr|abundancjusz|adaeze|adenike|aisha|amabile|amadi|amarachi|ambroz|amílcar|ananda|andreana|annabeth|ansovino|ansprando|antea|arbana|arduino|ariele|arleigh|armelinda|armelindo|armonia|arsinoe|atticus|aurélien|aynoor|azubuike|baldina|barnabé|biagino|bindy|blagoja|blessing|bonavita|bryon|buonaccorso|börje|cafiero|candis|carmelina|carson|cash|cass|cedrick|chandra|chus|cibele|cierra|cleonico|colbie|cristina|cvetanka|cydney|darby|deanne|deliverance|delphina|diamanto|dikran|dilia|dionigio|djurdja|dolph|doralice|dovydas|durante|dwane|edgard|egil|eliso|endika|eniko|ermelina|erodione|esmeraldo|esperina|etelfleda|euloge|evalina|evaline|ewoud|fabrizio|fara|finola|fiore|fraser|garance|gasper|gaudentius|gaudêncio|gaël|generosa|gerasimo|germinal|giairo|giannotto|gioia|gladwyn|glori|grimhild|gro|gualberto|gustavina|habakuk|hadis|hajna|harlee|hosny|hyacinth|hyacintha|iacopino|ilenja|indro|islom|ivetta|jaelen|jantine|jarod|jaylin|jaylon|jean-pierre|jedd|jerod|jill|jolan|jordyn|joël|joëlle|jérome|kailey|kajrat|kaleigh|kaley|kaouthar|karolina|kaylie|keitha|khalil|khalilah|kirby|krassimir|krassimira|lachie|laurianne|ledina|leilani|lele|leoluca|lesleigh|ljube|ljuben|lorelle|lubna|mahund|malak|marchina|marinus|martine|marvyn|matthaus|meggy|meinard|melchiade|meredyth|meriwether|micaiah|mikita|milcíades|minodora|misty|mulay|myriam|nadege|narcisio|narges|nasreen|nathan|nectarios|nicaise|niketas|nordine|nuredin|odhran|olimpiada|ondina|onelio|onyekachi|oyvind|palmiero|paziente|perrie|phillis|procopio|quinziano|raffaelo|rambaldo|ranulfo|rashed|ravshan|rayno|restituto|ricciarda|rickey|riika|rocky|rodolph|roshan|rrahman|ruaraidh|ruaridh|ryleigh|rylie|ríkharður|sage|salomone|sami|sammi|sanchia|sanjaya|sanjin|santuzza|sejad|seppe|serxhio|serxho|shannon|shanon|sharron|shea|shereen|shon|shyla|sidnie|sperindio|stacia|steeven|steevy|stilian|stiliano|styliani|submit|surayya|taisija|tarsicio|tawni|temple|tenzin|terance|tesfaye|tiffiny|timoleone|timotej|timothea|timotheos|tineke|tobina|trecia|trey|turlough|ugochukwu|usama|valérien|vando|venero|verio|vern|vincenzio|vlaho|vonda|vratislava|waltera|wandy|yair|zaffiro|zainuddin|éric|órfhlaith""".split("|")) +NAMES_SURNAMES = frozenset("""abaco|abate|abba|abbas|abbiate|abbondanza|abdullah|abeltino|abis|abozzi|abrami|abrioni|abruzzo|abubakr|abulafia|abutzu|acca|accalai|accardi|accareddu|accetto|accettura|accioni|acciu|accossu|accotzu|achena|achene|acheri|acqua|acri|adami|adani|adenike|adimari|adinolfi|adorni|agapito|agazzi|aglio|agnelli|agnese|agnoli|agosti|agozzino|agrati|agraz|agresti|ahn|aiolfi|airaghi|airaldi|al-dawasir|alba|albanese|albergo|alberi|alberici|alberti|albini|albizzati|albizzi|alboini|albonesi|alemanno|ales|alfieri|alfonsi|alifano|alinari|allevi|alonzo|altomare|alvaro|alzate|amadei|amadori|amato|ambrosini|amendola|amerighi|ammannati|amore|amoroso|anastasi|ancarani|andrei|andria|anfuso|angeli|anna|annarumma|antolini|aquenza|araújo|arecco|arese|argenti|argiolas|aricò|arienzo|armaroli|arosio|arpaia|arpino|arrighi|asghar|ashina|ashton|aspesi|astolfi|atzeni|atzeri|augelli|avola|ayame|ayasrah|azzari|azzi|bae|baek|baggio|baglioni|bagni|baiano|ballabio|ballai|balli|balsamo|bandino|barabino|baraldi|baratti|barbon|barca|bardi|barenghi|bari|barile|barocci|bartesaghi|bartoli|barzaghi|barzizza|bassani|bastoni|batmanghelidj|battisti|beati|beccari|bedarida|beltrame|beltrami|bencivenga|benedetti|benetti|benevento|beni|bennati|benni|bentivoglio|benvenuti|bernardi|bernasconi|bersani|bertelli|berti|bertoldi|bertuzzi|bevilacqua|bianchi|bibescu|bielli|biffi|bifulco|bini|bisceglie|bixio|bobbio|boccadifuoco|boccardo|bologna|bologni|bombelli|bombonati|bomboni|bonacci|bonaccorso|bonaiuti|bonamico|bonanno|bonardi|bonati|bonaventura|bondi|bonett|bonfante|bonfiglio|bongiorno|bongiovanni|boni|bonifacio|bonini|boninsegna|bonito|bonomo|bonora|bonsignore|bonucci|borboni|borghi|bortoli|boscaino|boscolo|bosio|bosisio|botero|bottai|bova|brambilla|breme|brenna|breno|brescia|briganti|brioschi|brivio|brocchi|brooks|broussard|brufani|bruno|bulgarelli|burns|buscemi|buti|buzatu|buzzi|bär|cabras|caffi|caiazzo|calabrese|calandra|caldonazzi|callegari|calogero|calvi|calò|cambi|cambiaghi|cambiaso|cameron|camozzi|campi|campo|canefri|canepa|canfora|cannas|cannata|cannavò|cannella|canonico|capodanno|cappelletto|cappelli|cappi|capra|capri|capurro|carena|cariddi|carnevale|carnielli|caro|carrobbio|carrubba|carrà|caruso|casadei|casanova|casarsa|casati|caserta|casiraghi|casirati|casoni|cassano|cassioli|castagna|castrogiovanni|cauchi|cavalleri|cavanna|cazzaniga|cazzola|ceccherelli|ceglie|celentano|ceriana|cerioli|cerne|cerri|cerutti|cervi|cha|chae|chapman|checchi|cheli|chelli|chezzi|chiacchio|chimenti|chinaglia|cho|choi|chun|chén|ciampi|ciccardini|cini|clementi|cocco|colombo|comaschi|comazzi|comi|como|coniglio|consiglio|conti|conz|cooper|coppi|corbi|corona|corradi|corrà|corte|corti|corvi|corvino|costa|cotta|coulibaly|covino|craxi|crespi|crespo|crinò|crippa|cristaldi|croce|crosio|crotta|cucchiara|cucinotta|cuffaro|cugno|cuneo|curatolo|cusumano|d'agostino|dago|danieli|danna|davide|davoli|dini|diodati|dioguardi|diotallevi|dolci|domenico|don|donati|doneda|donnini|donno|donzelli|drew|dunn|dustin|dzintars|dörfles|emiliani|emo|ermacora|esposito|fabbri|falco|fanti|fara|farina|federici|felici|ferla|fermi|ferrando|ferrara|ferrari|ferrarini|ferraro|ferrero|ferro|fiaschi|filippi|fini|fiorenza|flockhart|folliero|folloni|fontana|formiggini|fortescue|fossa|fox|fracanzani|francalancia|franceschi|frank|franzini|franzoni|freeman|frigerio|fulco|fumei|furlani|gabelli|gaber|gaekwad|gaffari|gaglioffi|gai|gallo|gambaro|gandhi|gandini|gandolfi|garibaldi|garlaschi|garlati|garzia|gaspari|gastaldi|gatti|gaudenzi|gazzolo|gemignani|gennari|genova|gentile|giacomi|gigli|gioberti|gioffré|giordano|giuli|giuliano|giussani|giusti|gjelaj|goi|gori|goria|gould|gozzadini|grandi|grassi|greco|grillo|gualtieri|guarini|guarnieri|guglielmi|guidi|gurjar|gül|haller|hamilton|han|heo|hong|howard|hwang|iaderosa|ignoti|innocenti|invernizzi|irani|israeli|ivaldi|jackson|jang|jeon|jeong|jiang|johansson|joubert|kang|katz|kazmi|kim|kong|korkut|kotes|kovács|krüger|kwak|kwon|lainati|lama|lamberti|lanzotti|lapi|laurino|laurito|lauro|lazzari|lecca|lecce|lecchi|lecci|leccio|ledda|lee|lentini|leone|lepri|levati|limiti|linati|lipari|lippi|lissoni|livraghi|locatelli|lodes|lodi|longo|lorandi|lorenzi|lorusso|lozzi|ludi|luna|lupi|lussu|luzzi|maestri|maffei|maggiore|magri|maier|mainardi|maino|mamdani|mancini|manetti|manfredi|maniero|manni|manzi|manzu|marangon|marazzi|marchesi|marchetti|marchi|marcon|marconi|mari|mariani|marino|marra|marras|martelli|martinelli|martínez|mascheroni|mauro|mazza|mazzarella|mclaughlin|melloni|merello|mereu|merloni|meuricoffre|mezzadri|micheli|migheli|min|mini|miranda|mitri|modafferi|modena|molinari|moller|momigliano|mondadori|mondaini|moon|morelli|moretti|moro|moroder|musso|muzzi|møller|najm|nam|nannoni|nazzari|nehme|nepi|neri|nervi|nerviani|nesti|neville|nicastro|nicolosi|nobili|noferi|noli|noureddine|novelli|novi|nucci|o'reilly|oddo|oggioni|ognibene|oldani|olgiati|olmi|omegna|omodei|ondedei|onesti|ongania|ongaro|onorati|orazi|orciani|orefice|oren|orlando|ornati|ornella|ortolani|ortona|osborne|ostuni|otranto|ottaviani|ottolenghi|paduano|paine|pallavicini|pallavidino|palmer|palmieri|panarello|pangallo|pantaleo|park|parra|parrella|parri|pasqua|pasquali|pasta|pastore|pastorino|paternoster|pavan|pavia|pedemonte|pedroli|pellegrini|peláez|perego|pergamo|pergola|perini|perrelli|perri|perrone|perti|perut|petracca|petroni|piccoli|piemonte|pietri|pirovano|pisani|pisanu|pittaluga|politi|postolache|pozzolengo|pradhan|premoli|preti|primavera|primo|proietti|prosperi|puglia|pugliese|punzo|quartigiani|quinn|quintavalle|quinto|raimondi|ramponi|ramsey|ranalli|ransone|rapa|rapallo|rapisarda|rassidi|ratti|ravara|ravidat|regina|ricci|rinaldi|rizzo|rocco|rodolfi|rodríguez|romano|rondelli|roos|rosa|rosi|rossetti|rossi|rota|rubini|ruggeri|ruscelli|russo|rutelli|ruvo|saccani|sacchi|sala|salierno|salm-reifferscheid-raitz|salvatori|salvidio|sambiagio|sami|sammarchi|sammarco|sammartino|sammauro|sampieri|sampietro|sampino|sanfelice|sangiorgi|sangiovanni|sanseverino|santagata|santangelo|santelia|santi|santonastaso|santoro|sanvitale|sarti|sartori|satriani|satta|saul|savini|scala|scalas|scally|scarpa|schaffgotsch|sciaccaluga|scicchitano|sclafani|scognamiglio|scopelliti|scornavacca|scorrano|scorza|seo|seong|sereni|seri|serravalle|servi|servidio|sessa|sesti|settala|sette|shin|shiono|sidney|sighinolfi|sigismondi|signori|silva|simonato|sinadeni|sinagra|sinibaldi|siniscalchi|sinisi|siri|smaldone|somaglia|somma|sommariva|son|soncini|song|soria|stabile|stafford|stanco|starace|stilo|stoppa|sullivan|sutera|sutter|suzuki|sánchez|taverna|telli|terlizzi|terrani|terzi|testa|tiberi|tokarczuk|tolomei|tomasi|tomei|tondi|tore|torrieri|torti|tozzi|tramonti|trani|trevisan|tripodi|trivulzio|tromba|trucco|tumminello|turchi|turetta|turner|ubaldi|uggeri|ugoni-longhi|urquhart|uselli|vacca|vacondio|vadalà|valda|valguarnera|valiani|vallino|varchi|varricchio|vedova|velliscig|vercelli|vercesi|verde|verlingieri|viazzi|vicentini|vieri|vigevani|vignolo|villa|villani|viola|vitale|vivaldi|volkonskaja|voltaggio|watanabe|weizmann|wells|williams|wilson|winter|woo|yair|yang|ying|yoo|yoon|zaccheo|zaghi|zampa|zampaglione|zanchi|zanini|zappa|zen|zicari|ziccardi|zilli|ziou|zocchi|zoccola|zoccoli|zoppi|zorzi|zotti|zucca|zuccalà|zuccarello|zuccheri|zulian|zuliani|zurlo|zurro""".split("|")) +NAMES_AMBIGUOUS = frozenset("""abaco|abate|abba|abbas|abbiate|abbondanza|abruzzo|acca|accetto|acqua|acri|adorni|aglio|agnelli|agnese|agresti|ahn|alba|albanese|albergo|alberi|alberti|albini|alemanno|ales|alfieri|alifano|allevi|alvaro|alzate|amabile|amendola|amore|amoroso|ananda|andrei|andria|angeli|anna|antea|arduino|arese|argenti|armonia|arpino|augelli|avola|azzi|bae|baggio|baglioni|bagni|baiano|balli|balsamo|baratti|barca|bardi|bari|barile|bassani|bastoni|battisti|beati|benevento|beni|benni|benvenuti|bersani|berti|bevilacqua|bini|bisceglie|blessing|bobbio|bologna|bonaventura|bondi|bongiorno|boni|bonifacio|bonito|bonucci|borboni|borghi|bottai|bova|brambilla|breme|brescia|briganti|brocchi|brooks|bruno|burns|buzzi|bär|cabras|calabrese|calandra|calogero|calvi|calò|cambi|cameron|campi|campo|canfora|cannata|cannella|canonico|capodanno|cappelletto|cappelli|cappi|capra|capri|carena|carnevale|caro|carrà|casanova|casarsa|casati|caserta|cash|casoni|cass|cassano|castagna|ceglie|celentano|cerri|cerutti|cervi|cha|cho|choi|chun|ciampi|clementi|colombo|comaschi|comi|como|coniglio|consiglio|cooper|coppi|corona|corte|corti|corvi|corvino|costa|cotta|craxi|crespi|crespo|cristina|croce|cucinotta|cuneo|dago|danna|davide|dini|dolci|domenico|don|donno|drew|durante|eliso|emiliani|emo|fabbri|fabrizio|falco|fanti|fara|felici|fermi|ferrari|ferrero|ferro|fiaschi|filippi|fini|fiore|fossa|fox|frank|freeman|gaber|gaglioffi|gai|gallo|gambaro|gandhi|garibaldi|gastaldi|generosa|genova|gigli|gioia|giuli|giuliano|giusti|glori|goi|gori|gould|grandi|greco|grillo|gro|guidi|gül|hamilton|han|heo|hong|howard|ignoti|innocenti|irani|jackson|jeon|jeong|jill|kang|kim|kirby|kong|kwon|lama|lauro|lazzari|lecca|lecce|lecchi|lecci|leccio|lee|lele|lentini|leone|lepri|levati|limiti|lipari|lippi|locatelli|lodi|ludi|luna|lupi|maestri|maggiore|magri|maier|mancini|manfredi|maniero|manzi|marazzi|marchesi|marcon|marconi|mari|marino|marinus|marra|martelli|mascheroni|mauro|merloni|mezzadri|min|mini|miranda|modena|molinari|mondadori|mondaini|moon|moro|musso|nam|neri|nervi|nobili|noli|novelli|novi|oggioni|olmi|omegna|ondina|onesti|onorati|orefice|ornati|ornella|ortolani|ortona|ostuni|otranto|palmer|pantaleo|park|parra|pasqua|pasquali|pasta|pastore|pavia|paziente|pedemonte|perego|pergamo|pergola|perini|perrone|piccoli|piemonte|pisani|politi|preti|primavera|primo|proietti|prosperi|puglia|pugliese|quinn|quinto|ramponi|rapa|rapallo|ratti|regina|ricci|rocco|rocky|romano|rosa|rosi|rossi|rota|rubini|ruggeri|ruscelli|russo|rutelli|sacchi|sage|salomone|salvatori|sami|santi|sarti|satta|savini|scala|scarpa|scopelliti|scorrano|scorza|seo|seppe|sereni|seri|serravalle|servi|sessa|sesti|sette|shannon|shea|shin|sidney|signori|silva|siri|somma|sommariva|son|song|soria|stabile|stanco|starace|stilo|stoppa|submit|suzuki|taverna|temple|terzi|tondi|tore|torti|tozzi|tramonti|trani|tripodi|tromba|trucco|turchi|turner|vacca|varchi|vedova|venero|vercelli|verde|vicentini|villani|viola|vivaldi|voltaggio|wells|williams|wilson|winter|woo|yang|ying|yoo|yoon|zaffiro|zampa|zappa|zen|zoccola|zoccoli|zoppi|zucca|zuccheri""".split("|")) diff --git a/lexicon_it.py b/lexicon_it.py index 0591672..44f35dd 100644 --- a/lexicon_it.py +++ b/lexicon_it.py @@ -201,11 +201,21 @@ def is_common_word(token: str) -> bool: def is_known_surname(token: str) -> bool: - return token.strip().lower().strip(".,;:") in SURNAMES + key = token.strip().lower().strip(".,;:") + if key in SURNAMES: + return True + d = _data() + return bool(d and hasattr(d, "NAMES_SURNAMES") and key in d.NAMES_SURNAMES + and key not in d.NAMES_AMBIGUOUS) def is_given_name(token: str) -> bool: - return token.strip().lower().strip(".,;:") in GIVEN_NAMES + key = token.strip().lower().strip(".,;:") + if key in GIVEN_NAMES: + return True + d = _data() + return bool(d and hasattr(d, "NAMES_GIVEN") and key in d.NAMES_GIVEN + and key not in d.NAMES_AMBIGUOUS) @functools.lru_cache(maxsize=1) diff --git a/pii_detection.py b/pii_detection.py index 94b2d21..bd53b59 100644 --- a/pii_detection.py +++ b/pii_detection.py @@ -1150,6 +1150,53 @@ def detect_address_parts( return out +# --- glyph deletion rule (shared by the app and the export checks) ----------- + +def glyph_core(bbox: Rect4) -> Rect4: + """The part of a character box that decides whether a redaction deletes it. + + Measured on PyMuPDF 2.1 (MuPDF 1.28): a glyph is removed when the + redaction rectangle intersects its box shrunk by 10% of its width on + each side and 10% of its height at top and bottom. + """ + x0, y0, x1, y1 = bbox + dx, dy = (x1 - x0) * GLYPH_CORE_INSET, (y1 - y0) * GLYPH_CORE_INSET + return (x0 + dx, y0 + dy, x1 - dx, y1 - dy) + + +def core_hits(bbox: Rect4, rect: Any) -> bool: + c = glyph_core(bbox) + return c[0] < rect.x1 and c[2] > rect.x0 and c[1] < rect.y1 and c[3] > rect.y0 + + +def target_chars(page: Any, full: Any) -> List[Dict[str, Any]]: + """Characters of the line a word box was taken from (single baseline).""" + try: + blocks = page.get_text("rawdict", clip=full).get("blocks", []) + except Exception: + return [] + chars: List[Tuple[float, Dict[str, Any]]] = [] + for block in blocks: + if block.get("type") != 0: + continue + for line in block.get("lines", []): + for span in line.get("spans", []): + for ch in span.get("chars", []): + if not str(ch.get("c", "")).strip(): + continue + bx0, by0, bx1, by1 = ch["bbox"] + cx, cy = (bx0 + bx1) / 2, (by0 + by1) / 2 + if full.x0 <= cx <= full.x1 and full.y0 <= cy <= full.y1: + chars.append((round(float(ch["origin"][1]), 1), ch)) + if not chars: + return [] + centre = (full.y0 + full.y1) / 2 + bases = {b for b, _ in chars} + inside = [b for b in bases if full.y0 <= b <= full.y1] or list(bases) + base = min(inside, key=lambda b: abs(b - centre)) + return [ch for b, ch in chars if b == base] + + # --- geometry policy ------------------------------------------------------- def span_word_indices( diff --git a/requirements.txt b/requirements.txt index 6d963fb..8ef6010 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,9 @@ PyMuPDF==2.1 presidio-analyzer==2.2.366 spacy==3.8.18 cryptography==50.0.2 +# OS keyring (macOS Keychain / Linux Secret Service) for the local master key; +# Windows uses DPAPI instead, so it is not installed there. +keyring==25.7.0; sys_platform != "win32" # spaCy language models (pinned to exact versions for reproducibility) # IMPORTANT: model minor version MUST match the spaCy core minor version # (spaCy 3.8.x <-> models 3.8.x). The models declare spacy>=3.8.0,<3.9.0. diff --git a/review_dialog.py b/review_dialog.py index 56e38b1..3919d30 100644 --- a/review_dialog.py +++ b/review_dialog.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List +from help_guide import apply_tooltips from PySide6.QtCore import Qt, Signal from PySide6.QtWidgets import ( QAbstractItemView, @@ -72,12 +73,16 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non self._applied: List[Dict[str, Any]] = [] self._rejected: List[Dict[str, Any]] = [] lay = QVBoxLayout(self) - lay.addWidget(QLabel( + intro = QLabel( "Questi elementi non hanno prove sufficienti per essere censurati " "automaticamente. Seleziona quelli da censurare: i non selezionati " "vengono scartati se premi «Applica selezionati»." - )) + ) + intro.setWordWrap(True) + lay.addWidget(intro) self.table = QTableWidget(len(self._items), 6, self) + self.table.setObjectName("review_table") + self.table.verticalHeader().setVisible(False) self.table.setHorizontalHeaderLabels( ["Censura", "Pag.", "Valore", "Tipo", "Punteggio", "Motivi"] ) @@ -114,6 +119,7 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non self.spin.setRange(0.0, 1.0) self.spin.setSingleStep(0.05) self.spin.setValue(0.6) + self.spin.setObjectName("review_threshold") btn_thr = QPushButton("Seleziona ≥ soglia") # Bound methods, not lambdas capturing `self`: a lambda makes a # reference cycle that keeps the dialog alive until the garbage @@ -136,6 +142,7 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non bottom.addWidget(self.btn_later) bottom.addWidget(self.btn_apply) lay.addLayout(bottom) + apply_tooltips(self) # -- helpers --------------------------------------------------------- def _cell(self, row: int, col: int) -> QTableWidgetItem: diff --git a/scripts/build_lexicon_data.py b/scripts/build_lexicon_data.py index b00b349..ba6cebe 100644 --- a/scripts/build_lexicon_data.py +++ b/scripts/build_lexicon_data.py @@ -24,6 +24,17 @@ import zipfile from typing import Dict, List, Set +WIKIDATA_URL = "https://query.wikidata.org/sparql" +# Names that have an article on the Italian Wikipedia: a bounded, relevant set +# (counting people per name would time out on the public endpoint). +NAME_QUERY = ( + 'SELECT DISTINCT ?l WHERE {{ ?n wdt:P31 wd:{cls}. ' + '?a schema:about ?n; schema:isPartOf . ' + '?n rdfs:label ?l. FILTER(LANG(?l)="it") }} LIMIT {page} OFFSET {off}' +) +NAME_CLASSES = {"given": "Q202444", "surnames": "Q101352"} +NAME_PAGE = 2000 +NAME_OK = re.compile(r"^[a-zà-ÿ'’-]{3,24}$") ISTAT_URL = "https://www.istat.it/storage/codici-unita-amministrative/Elenco-comuni-italiani.xlsx" NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} COLUMNS = { @@ -64,6 +75,37 @@ def read_rows(path: pathlib.Path) -> "tuple[List[str], List[Dict[str, str]]]": return list(headers.values()), out +def fetch_names(kind: str, retries: int = 6) -> Set[str]: + """Names of one kind (Wikidata, CC0), paged to stay under the query timeout.""" + import json + import time + import urllib.parse + + out: Set[str] = set() + off = 0 + while True: + q = NAME_QUERY.format(cls=NAME_CLASSES[kind], page=NAME_PAGE, off=off) + url = WIKIDATA_URL + "?format=json&query=" + urllib.parse.quote(q) + for attempt in range(retries): + try: + req = urllib.request.Request( + url, headers={"User-Agent": "NullifyPDF-build/1.0 (github.com/overwrite00/NullifyPDF)"} + ) + rows = json.load(urllib.request.urlopen(req, timeout=120))["results"]["bindings"] + break + except Exception as e: + if attempt == retries - 1: + raise RuntimeError(f"Wikidata {kind} page {off}: {e}") from e + time.sleep(10 * (attempt + 1)) + for r in rows: + n = normalise(r["l"]["value"]) + if NAME_OK.match(n): + out.add(n) + if len(rows) < NAME_PAGE: + return out + off += NAME_PAGE + + def normalise(name: str) -> str: return " ".join(name.replace("’", "'").lower().split()) @@ -71,6 +113,8 @@ def normalise(name: str) -> str: def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--istat-file", type=pathlib.Path) + ap.add_argument("--wikidata", action="store_true", + help="also fetch given names / surnames (CC0) from Wikidata") ap.add_argument("--out", type=pathlib.Path, default=ROOT / "lexicon_data_it.py") args = ap.parse_args() src = args.istat_file @@ -106,6 +150,7 @@ def main() -> int: common = set(lexicon_it.COMMON_WORDS | lexicon_it.FUNCTION_WORDS | lexicon_it.GIVEN_NAMES | lexicon_it.CONFLICT_NAMES) vocab_words: Set[str] = set() + nlp = None try: import spacy @@ -120,6 +165,28 @@ def lit(items: Set[str]) -> str: assert '"""' not in body and "\\" not in body return 'frozenset("""' + body + '""".split("|"))' + names_text = "" + if args.wikidata: + given = fetch_names("given") + surn = fetch_names("surnames") + both = given | surn + name_ambiguous = {w for w in both if w in common or len(w) <= 3} + if nlp is not None: + try: + name_ambiguous |= {w for w in both if " " not in w and nlp.vocab.has_vector(w) + and w not in lexicon_it.GIVEN_NAMES and w not in lexicon_it.SURNAMES} + except Exception: + pass + names_text = ( + "# Given names and surnames: Wikidata (CC0,\n" + "# https://www.wikidata.org/wiki/Wikidata:Licensing), entries that\n" + "# have an article on the Italian Wikipedia.\n" + f"NAMES_GIVEN = {lit(given)}\n" + f"NAMES_SURNAMES = {lit(surn)}\n" + f"NAMES_AMBIGUOUS = {lit(name_ambiguous)}\n" + ) + print(f"{len(given)} given names, {len(surn)} surnames ({len(name_ambiguous)} ambiguous)") + text = ( "# Generated by scripts/build_lexicon_data.py - do not edit.\n" "# Contains ISTAT data (Elenco comuni italiani, " + src.name + "), licensed\n" @@ -130,6 +197,7 @@ def lit(items: Set[str]) -> str: f"COMUNI_AMBIGUOUS = {lit(ambiguous)}\n" f"PROVINCE_SIGLE = {lit(sigle)}\n" f"REGIONS = {lit(regions)}\n" + + names_text ) args.out.write_text(text, encoding="utf-8") print(f"{len(comuni)} comuni ({len(ambiguous)} ambiguous), {len(sigle)} province codes, " diff --git a/tests/test_detection_pipeline.py b/tests/test_detection_pipeline.py index 18eec38..9a8dac9 100644 --- a/tests/test_detection_pipeline.py +++ b/tests/test_detection_pipeline.py @@ -452,3 +452,16 @@ def test_known_surname_helps_a_two_word_name_but_never_rejects(): text2 = "Il referto di Gianpiero Qwertyuiop" _i, cands2, _ = _run(text2, [res("PERSON", text2, "Gianpiero Qwertyuiop")]) assert cands2 and cands2[0].state in ("accepted", "suggested") # absence is not a reason to reject + + +def test_wikidata_names_confirm_but_ambiguous_ones_do_not(): + import lexicon_data_it as d + + unambiguous = sorted(d.NAMES_SURNAMES - d.NAMES_AMBIGUOUS) + assert unambiguous and lexicon_it.is_known_surname(unambiguous[0]) + ambiguous = sorted(d.NAMES_AMBIGUOUS & d.NAMES_SURNAMES) + assert ambiguous and not lexicon_it.is_known_surname(ambiguous[0]) or ambiguous[0] in lexicon_it.SURNAMES + assert not lexicon_it.is_known_surname("zzqxv") + assert "Wikidata" in open( + __import__("pathlib").Path(__file__).resolve().parent.parent / "THIRD_PARTY_NOTICES.md", + encoding="utf-8").read() diff --git a/tests/test_export_sanitize.py b/tests/test_export_sanitize.py index b67230a..1130119 100644 --- a/tests/test_export_sanitize.py +++ b/tests/test_export_sanitize.py @@ -383,3 +383,93 @@ def test_text_outside_redactions_uses_the_deletion_core(): page.add_redact_annot(pymupdf.Rect(x0 - 2, y0, x0 + (x1 - x0) * 0.25, y1)) kept = text_outside_redactions(doc) assert "Alessandro" not in kept and "resta" in kept + + +# --- post-apply glyph survival check ---------------------------------------------- + +def _doc_with_word_and_band(band_fraction): + import json as _json + + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Riga ABCDEFGH fine", fontsize=11) + x0, y0, x1, y1 = [w for w in page.get_text("words") if w[4] == "ABCDEFGH"][0][:4] + band = pymupdf.Rect(x0, y0, x0 + (x1 - x0) * band_fraction, y1) + annot = page.add_redact_annot(band) + annot.set_info(content=_json.dumps({ + "nullifypdf": 1, "original": "ABCDEFGH", "entity_type": "PERSON", + "full": [x0, y0, x1, y1], + })) + return doc + + +def _read_full(annot): + import json as _json + + try: + d = _json.loads(annot.info.get("content", "")) + return pymupdf.Rect(*d["full"]) if d.get("full") else None + except Exception: + return None + + +def test_a_band_that_misses_glyphs_is_reported(tmp_path): + doc = _doc_with_word_and_band(0.5) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs >= 3 + assert not result.verification.ok + assert any("survived redaction" in f for f in result.verification.findings) + assert "ABCD" not in " ".join(result.verification.findings) # never repeats the value + + +def test_a_band_covering_the_word_reports_nothing(tmp_path): + doc = _doc_with_word_and_band(1.0) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full, + removed_values=["ABCDEFGH"]) + assert result.surviving_glyphs == 0 and result.verification.ok, result.verification.findings + + +def test_survivor_check_is_inactive_without_a_stored_full_box(tmp_path): + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "ABCDEFGH", fontsize=11) + page.add_redact_annot(pymupdf.Rect(72, 90, 80, 110)) # manual box, no `full` + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs == 0 + + +def test_survivor_alignment_survives_mupdf_moving_the_remaining_text(tmp_path): + """The rest of the line is rewritten and shifted; positions are not reliable.""" + doc = _doc_with_word_and_band(0.5) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs == 4 # E, F, G, H + + +def test_repeated_words_on_the_line_are_not_false_survivors(tmp_path): + import json as _json + + doc = pymupdf.open() + page = doc.new_page() + page.insert_text((72, 100), "Maria Mario Maria", fontsize=11) + x0, y0, x1, y1 = [w for w in page.get_text("words") if w[4] == "Mario"][0][:4] + annot = page.add_redact_annot(pymupdf.Rect(x0, y0, x1, y1)) + annot.set_info(content=_json.dumps({ + "nullifypdf": 1, "original": "Mario", "entity_type": "PERSON", + "full": [x0 - 1, y0, x1 + 1, y1], + })) + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + assert result.surviving_glyphs == 0 + + +def test_rule_deviation_is_reported(tmp_path, monkeypatch): + """If a PyMuPDF release stops deleting what the rule predicts, the export says so.""" + import export_engine as ee + + doc = _doc_with_word_and_band(1.0) + real = ee.PDF_REDACT_IMAGE_PIXELS + orig_apply = pymupdf.Page.apply_redactions + monkeypatch.setattr(pymupdf.Page, "apply_redactions", lambda self, *a, **k: 0) # deletes nothing + result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full) + monkeypatch.setattr(pymupdf.Page, "apply_redactions", orig_apply) + assert real is not None + assert result.surviving_glyphs >= 8 and not result.verification.ok diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py index d1fe2de..b8c56ca 100644 --- a/tests/test_gui_flow.py +++ b/tests/test_gui_flow.py @@ -845,3 +845,77 @@ def test_recovery_wrap_is_kept_even_if_the_dialog_is_closed_without_ticking(wind monkeypatch.setattr(QDialog, "exec", lambda self: QDialog.DialogCode.Rejected) raw = w._offer_recovery_key() assert isinstance(raw, bytes) and len(raw) == 32 + + +# --- labels on image / signature areas survive small boxes ------------------------------- + +def _tiny_area_doc(w, height): + w.doc.close() + d = pymupdf.open() + pg = d.new_page() + pg.insert_text((72, 100), "Testo normale", fontsize=11) + pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 40, 40), False) + pix.set_rect(pix.irect, (10, 10, 10)) + pg.insert_image(pymupdf.Rect(300, 300, 300 + 24, 300 + height), pixmap=pix) + w.doc = d + w.chk_img.setChecked(True) + w.apply_ai_to_page(0, []) + + +@pytest.mark.parametrize("height", [3, 6, 20]) +@pytest.mark.parametrize("mode", [PrivacyMode.ANONYMIZE, PrivacyMode.PSEUDONYMIZE]) +def test_image_label_is_kept_even_in_a_tiny_box(window, tmp_path, monkeypatch, height, mode): + w = window + _tiny_area_doc(w, height) + assert len(_redacts(w)) == 1 + out = tmp_path / "i.pdf" + files = [str(out)] + ([str(tmp_path / "m.nullifypdf-map")] if mode == PrivacyMode.PSEUDONYMIZE else []) + _stub_dialogs(monkeypatch, files) + monkeypatch.setattr(w, "_select_privacy_export", lambda: mode) + monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None)) + monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026") + monkeypatch.setattr(w, "_offer_recovery_key", lambda: None) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert "Testo normale" in text + assert any(lab in text for lab in ("[IMMAGINE RIMOSSA]", "[IMMAGINE]", "[IMG]", "#")), text + + +def test_signature_label_in_a_small_area(window, tmp_path, monkeypatch): + w = window + w.doc.close() + d = pymupdf.open() + d.new_page().insert_text((72, 100), "Testo", fontsize=11) + w.doc = d + w._add_kind_redaction(d[0], pymupdf.Rect(200, 200, 212, 204), "SIGNATURE") + out = tmp_path / "s.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + w.cmd_export() + pdf = pymupdf.open(str(out)) + text = pdf[0].get_text() + pdf.close() + assert any(lab in text for lab in ("[FIRMA RIMOSSA]", "[FIRMA]", "[F]", "#")), text + + +def test_export_through_the_app_reports_no_survivor_for_clamped_bands(window, tmp_path, monkeypatch): + w = window + _tight_doc(w, 8.0) + w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")]) + out = tmp_path / "c.pdf" + _stub_dialogs(monkeypatch, [str(out)]) + monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE) + seen = {} + import export_engine as ee + real = ee.export_document + + def spy(*a, **k): + r = real(*a, **k) + seen["r"] = r + return r + + monkeypatch.setattr(app, "export_document", spy) + w.cmd_export() + assert seen["r"].surviving_glyphs == 0 and seen["r"].verification.ok diff --git a/tests/test_gui_help.py b/tests/test_gui_help.py new file mode 100644 index 0000000..1721453 --- /dev/null +++ b/tests/test_gui_help.py @@ -0,0 +1,141 @@ +"""Guide and tooltips: every GUI element is explained, and the guide matches the UI.""" + +import os +import re + +import pytest + +pytest.importorskip("PySide6") +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +from PySide6.QtWidgets import ( # noqa: E402 + QAbstractButton, QApplication, QCheckBox, QDialog, QPushButton, QRadioButton, +) + +import NullifyPDF as app # noqa: E402 +from help_guide import ( # noqa: E402 + ENTITY_TOOLTIPS, GUIDE_TOPICS, TOOLTIPS, GuideDialog, apply_tooltips, + widgets_without_tooltip, +) +from pii_detection import ENTITY_GROUPS # noqa: E402 +from review_dialog import ReviewDialog # noqa: E402 + + +@pytest.fixture() +def window(tmp_path, monkeypatch): + import atexit + import gc + + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("HOME", str(home)) + qapp = QApplication.instance() or QApplication([]) + w = app.NullifyPDF() + yield w + try: + w._cleanup_resources() + except Exception: + pass + atexit.unregister(w._cleanup_on_exit) + w.deleteLater() + qapp.processEvents() + gc.collect() + + +def _plain(html): + return re.sub(r"<[^>]+>", " ", html) + + +def test_every_widget_of_the_main_window_has_a_tooltip(window): + assert widgets_without_tooltip(window) == [] + + +def test_buttons_have_tooltips_too(window): + buttons = [b for b in window.findChildren(QAbstractButton)] + assert len(buttons) >= 15 + assert all(b.toolTip().strip() for b in buttons), [b.text() for b in buttons if not b.toolTip()] + + +def test_tooltips_are_in_italian_and_not_empty(): + for key, tip in {**TOOLTIPS, **ENTITY_TOOLTIPS}.items(): + assert len(tip) > 10 and tip[0].isupper(), key + + +def test_every_entity_checkbox_has_a_tooltip(): + labels = {label for _gid, label, _t in ENTITY_GROUPS} + assert labels <= set(ENTITY_TOOLTIPS), labels - set(ENTITY_TOOLTIPS) + + +def test_entity_dialog_has_tooltips(window, monkeypatch): + seen = {} + + def fake_exec(self): + seen["missing"] = widgets_without_tooltip(self) + seen["boxes"] = [c.toolTip() for c in self.findChildren(QCheckBox)] + return QDialog.DialogCode.Rejected + + monkeypatch.setattr(QDialog, "exec", fake_exec) + window._select_ai_entities() + assert seen["missing"] == [] and all(seen["boxes"]) + + +def test_review_and_dictionary_dialogs_have_tooltips(window): + from dictionary_dialog import DictionaryDialog + + items = [{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, "page": 0, "reasons": []}] + assert widgets_without_tooltip(ReviewDialog(items)) == [] + assert widgets_without_tooltip(DictionaryDialog(window.list_manager)) == [] + + +def test_about_dialog_has_tooltips(window, monkeypatch): + seen = {} + + def fake_exec(self): + seen["missing"] = widgets_without_tooltip(self) + return QDialog.DialogCode.Rejected + + monkeypatch.setattr(QDialog, "exec", fake_exec) + window.cmd_about() + assert seen["missing"] == [] + + +def test_guide_dialog_lists_every_topic_and_has_tooltips(window): + g = GuideDialog(window) + assert g.topics.count() == len(GUIDE_TOPICS) >= 10 + assert widgets_without_tooltip(g) == [] + # No hover popups on the topic list and the text: they get in the way of reading. + assert g.topics.toolTip() == "" and g.text.toolTip() == "" + assert g.topics.viewport().toolTip() == "" and g.text.viewport().toolTip() == "" + for row in range(g.topics.count()): + g.topics.setCurrentRow(row) + assert len(g.text.toPlainText()) > 80 + + +def test_f1_and_the_guide_button_open_the_guide(window, monkeypatch): + opened = [] + monkeypatch.setattr(GuideDialog, "exec", lambda self: opened.append(self.topics.count()) or 0) + window._f1.activated.emit() + btn = next(b for b in window.findChildren(QPushButton) if b.text() == "Guida") + btn.click() + assert len(opened) == 2 + + +def test_the_guide_describes_every_control_of_the_sidebar(window): + text = _plain(" ".join(html for _t, html in GUIDE_TOPICS)) + for w in window.findChildren(QAbstractButton): + label = w.text().strip() + if not label or label in {"-", "+", "<", ">", "Esci"} or label.startswith("Rivedi suggerimenti"): + continue + assert label in text, f"the guide never mentions «{label}»" + assert "Rivedi suggerimenti" in text and "F1" in text + for rb in window.findChildren(QRadioButton): + assert rb.text() in text + + +def test_the_guide_matches_what_the_program_really_does(): + text = _plain(" ".join(html for _t, html in GUIDE_TOPICS)) + for fact in ("scadenza", "chiave di recupero", "mappa di ripristino", + "Ctrl + rotella", "Revisione", "OCR", "pseudonim"): + assert fact.lower() in text.lower(), fact + assert "Wikidata" not in text and "AI" not in text.split() # no internals / no AI wording diff --git a/tests/test_secure_store.py b/tests/test_secure_store.py index fed997d..0bd9e0e 100644 --- a/tests/test_secure_store.py +++ b/tests/test_secure_store.py @@ -158,3 +158,43 @@ def test_tampered_list_file_is_not_trusted(tmp_path): raw[-1] ^= 1 p.write_bytes(bytes(raw)) assert PDFListManager(tmp_path).load_blocklist() == set() + + +def test_keyring_backend_holds_the_master_key_when_available(tmp_path, monkeypatch): + """With a real (here: in-memory) keyring backend the key never touches the keystore file.""" + keyring = pytest.importorskip("keyring") + from keyring.backend import KeyringBackend + + class MemoryKeyring(KeyringBackend): + priority = 1 + store: dict = {} + + def set_password(self, service, username, password): + self.store[(service, username)] = password + + def get_password(self, service, username): + return self.store.get((service, username)) + + def delete_password(self, service, username): + self.store.pop((service, username), None) + + old = keyring.get_keyring() + keyring.set_keyring(MemoryKeyring()) + try: + monkeypatch.setattr(secure_store.DpapiProtector, "available", staticmethod(lambda: False)) + key = secure_store.load_or_create_master_key(tmp_path) + ks = json.loads((tmp_path / "keystore.json").read_text()) + assert ks["master"]["protector"] == "keyring" + assert "portachiavi" in secure_store.protector_label(tmp_path) + import base64 + assert base64.b64encode(key).decode() not in json.dumps(ks) + assert not (tmp_path / "keys").exists() + secure_store.forget_cached_keys() + assert secure_store.load_or_create_master_key(tmp_path) == key + # losing the keyring entry = unreadable keystore, never silent re-creation + MemoryKeyring.store.clear() + secure_store.forget_cached_keys() + with pytest.raises(secure_store.KeystoreUnavailable): + secure_store.load_or_create_master_key(tmp_path) + finally: + keyring.set_keyring(old)