[A-Z]{2})\s*\))?"
+)
+_MONEY_BEFORE = ("\u20ac", "euro", "eur", "importo", "totale", "n.", "prot.", "fattura")
+
+
+def detect_address_parts(
+ text: str, enabled: Iterable[str], health_mode: bool = False
+) -> List[Candidate]:
+ """Street + house number as one span, and postcodes in an address context.
+
+ A postcode needs an anchor (a street before it, a province code or a
+ known place after it, or the label "CAP"); prices, protocol numbers,
+ dates and quantities never match. Without an anchor it is only a
+ suggestion in health mode.
+ """
+ if "LOCATION" not in set(enabled):
+ return []
+ out: List[Candidate] = []
+ street_ends: List[int] = []
+ for m in _CIVICO.finditer(text):
+ out.append(Candidate(
+ m.start(), m.end("num"), text[m.start():m.end("num")], "LOCATION", 0.95,
+ reasons=["address-pattern"],
+ ))
+ street_ends.append(m.end("num"))
+ for m in _CAP.finditer(text):
+ cap = m.group("cap")
+ if not (10 <= int(cap) <= 98168):
+ continue
+ before = text[max(0, m.start() - 10):m.start()].lower()
+ if any(k in before for k in _MONEY_BEFORE):
+ continue
+ after = text[m.end("cap"):m.end("cap") + 4]
+ if after.startswith((",0", ",00", "\u20ac", "%")):
+ continue
+ window = text[max(0, m.start() - 6):m.start()].lower()
+ prov = m.group("prov")
+ anchored = (
+ any(0 <= m.start() - e <= 80 for e in street_ends)
+ or bool(prov and prov in lexicon_it.province_codes())
+ or lexicon_it.is_known_place(m.group("city").split()[0])
+ or "cap" in window
+ )
+ if anchored:
+ out.append(Candidate(
+ m.start("cap"), m.end("cap"), cap, "LOCATION", 0.9,
+ reasons=["address-pattern", "postcode"],
+ ))
+ elif health_mode:
+ out.append(Candidate(
+ m.start("cap"), m.end("cap"), cap, "LOCATION", 0.5,
+ state="suggested", reasons=["address-pattern", "postcode", "no-anchor"],
+ ))
+ return out
+
+
+# --- geometry policy -------------------------------------------------------
+
+def span_word_indices(
+ index: PageTextIndex, start: int, end: int, min_ratio: float = 0.5
+) -> List[int]:
+ """Indices of the words a span covers by at least `min_ratio` of their length."""
+ out: List[int] = []
+ for i, w in enumerate(index.words):
+ overlap = min(end, w.end) - max(start, w.start)
+ if overlap <= 0:
+ continue
+ if overlap < min_ratio * (w.end - w.start):
+ continue
+ out.append(i)
+ return out
+
+
+def rects_for_words(
+ index: PageTextIndex, idxs: Iterable[int], original: bool = False
+) -> List[Rect4]:
+ """One rectangle per line for the given words (clamped, or `original`)."""
+ by_line: Dict[Tuple[int, int], List[Rect4]] = {}
+ for i in idxs:
+ w = index.words[i]
+ by_line.setdefault(w.line_key, []).append(
+ w.orig_rect if original and w.orig_rect else w.rect
+ )
+ return [_union(rs) for _, rs in sorted(by_line.items())]
+
+
+def resolve_word_indices(
+ index: PageTextIndex, c: Candidate
+) -> Tuple[List[int], str]:
+ """Word indices to redact for a candidate, per the partial-match policy.
+
+ Returns (indices, status). Status is "ok", "glued" (a validated
+ identifier stuck to other text: the whole token is redacted) or a
+ rejection reason (indices empty).
+ """
+ text = index.text
+ if c.entity_type in _NER_TYPES:
+ if not boundary_ok(text, c.start, c.end, apostrophe_ok=True):
+ return [], "embedded-in-word"
+ idxs = span_word_indices(index, c.start, c.end)
+ return (idxs, "ok") if idxs else ([], "partial-word")
+ idxs = span_word_indices(index, c.start, c.end)
+ if idxs:
+ return idxs, "ok"
+ if c.entity_type in VALIDATED_TYPES:
+ touched = span_word_indices(index, c.start, c.end, min_ratio=0.0)
+ if touched:
+ return touched, "glued"
+ return [], "partial-word"
+
+
+def protected_word_indices(
+ index: PageTextIndex, terms: Iterable[str]
+) -> Set[int]:
+ """Word indices covered by whole-word occurrences of allowlist terms."""
+ protected: Set[int] = set()
+ for term in terms:
+ term = (term or "").strip()
+ if not term:
+ continue
+ for m in _whole_word_pattern(term, False).finditer(index.text):
+ protected.update(span_word_indices(index, m.start(), m.end()))
+ return protected
+
+
+def classify_page(
+ results: Iterable[Any],
+ index: PageTextIndex,
+ enabled_types: Iterable[str],
+ *,
+ used_ocr: bool = False,
+ health_mode: bool = False,
+ verb_like: Iterable[str] = (),
+ stopwords: Optional[Iterable[str]] = None,
+) -> Tuple[List[Candidate], Dict[str, int]]:
+ """Run the full candidate pipeline on analyzer results for one page.
+
+ Returns the surviving candidates (state "accepted" or "suggested") and a
+ count of rejections per reason (no values, safe to log).
+ """
+ text = index.text
+ enabled = list(enabled_types)
+ stats: Dict[str, int] = {}
+
+ def bump(reason: str, n: int = 1) -> None:
+ if n:
+ stats[reason] = stats.get(reason, 0) + n
+
+ cands = filter_results(results, text, enabled)
+ cands.extend(detect_health_patterns(text, enabled, health_mode))
+ kept: List[Candidate] = []
+ for c in cands:
+ if c.entity_type in _NER_TYPES:
+ kept.append(c)
+ continue
+ ok, reason = refine_pattern_hit(c, text)
+ if ok:
+ kept.append(c)
+ else:
+ bump(f"pattern:{c.entity_type}:{reason}")
+ address = detect_address_parts(text, enabled, health_mode)
+ before = len(kept)
+ kept = clean_candidates(kept, stopwords=stopwords, enabled_types=enabled)
+ bump("generic-word", max(0, before - len(kept)))
+ kept, n = strip_lexicon_edges(kept)
+ bump("lexicon", n)
+ kept, n = reject_fragment_hits(kept, used_ocr)
+ bump("fragment", n)
+ kept = merge_adjacent_lines(kept, text)
+ kept.extend(address)
+ kept = resolve_overlaps(kept)
+ kept, n = evaluate_ner_candidates(
+ kept, text, health_mode=health_mode, verb_like=verb_like
+ )
+ bump("ner-evidence", n)
+ return kept, stats
+
+
+def build_detections(
+ index: PageTextIndex,
+ candidates: Iterable[Candidate],
+ *,
+ allowlist: Iterable[str] = (),
+ source: str = "text",
+ stats: Optional[Dict[str, int]] = None,
+) -> List[Dict[str, Any]]:
+ """Turn candidates into detections with rectangles.
+
+ Allowlist semantics are by word identity: a word is protected when it is
+ part of a whole-word occurrence of an allowlist entry. A detection whose
+ words are all protected disappears; a longer detection only loses the
+ protected words. Substring containment and rectangle intersection are
+ never used, so an allowlist entry cannot suppress a different value.
+ """
+ terms = [t for t in allowlist if t and t.strip()]
+ allow_set = {" ".join(t.lower().split()) for t in terms}
+ protected = protected_word_indices(index, terms) if terms else set()
+
+ def note(reason: str) -> None:
+ if stats is not None:
+ stats[reason] = stats.get(reason, 0) + 1
+
+ out: List[Dict[str, Any]] = []
+ seen: Set[Tuple[str, str]] = set()
+ for c in candidates:
+ value = c.text
+ clean = " ".join(value.strip(_EDGE_PUNCT).lower().split())
+ if not clean:
+ continue
+ if clean in allow_set:
+ note("allowlist")
+ continue
+ idxs, status = resolve_word_indices(index, c)
+ if not idxs:
+ note(f"geometry:{status}")
+ continue
+ idxs = [i for i in idxs if i not in protected]
+ if not idxs:
+ note("allowlist")
+ continue
+ rects = rects_for_words(index, idxs)
+ full_rects = rects_for_words(index, idxs, original=True)
+ key = (value, c.entity_type)
+ if key not in seen:
+ seen.add(key)
+ pattern = _whole_word_pattern(
+ value, c.entity_type in _NER_TYPES
+ )
+ for m in pattern.finditer(index.text):
+ extra_idx = [
+ i for i in span_word_indices(index, m.start(), m.end())
+ if i not in protected
+ ]
+ for r, fr in zip(
+ rects_for_words(index, extra_idx),
+ rects_for_words(index, extra_idx, original=True),
+ ):
+ if r not in rects:
+ rects.append(r)
+ full_rects.append(fr)
+ out.append({
+ "text": value,
+ "entity_type": c.entity_type,
+ "rects": rects,
+ "full_rects": full_rects,
+ "source": source,
+ "score": c.score,
+ "state": c.state,
+ "reasons": list(c.reasons),
+ "glued": status == "glued",
+ })
+ return out
+
+
+# --- rectangle merging / dedup ---------------------------------------------
+
+def merge_line_rects(rects: Iterable[Rect4], gap: float = 1.5) -> List[Rect4]:
+ """Merge rectangles on the same line that touch or overlap."""
+ rs = sorted(rects, key=lambda r: (round((r[1] + r[3]) / 2, 0), r[0]))
+ merged: List[Rect4] = []
+ for r in rs:
+ if merged:
+ m = merged[-1]
+ oy = min(m[3], r[3]) - max(m[1], r[1])
+ minh = min(m[3] - m[1], r[3] - r[1])
+ if minh > 0 and oy >= 0.6 * minh and r[0] - m[2] <= gap:
+ merged[-1] = (
+ min(m[0], r[0]), min(m[1], r[1]),
+ max(m[2], r[2]), max(m[3], r[3]),
+ )
+ continue
+ merged.append(r)
+ return merged
+
+
+def covered_fraction(rect: Rect4, others: Iterable[Rect4]) -> float:
+ """Largest share of `rect` covered by any single rectangle of `others`."""
+ area = max(0.0, (rect[2] - rect[0]) * (rect[3] - rect[1]))
+ if area <= 0:
+ return 1.0
+ best = 0.0
+ for o in others:
+ ix = min(rect[2], o[2]) - max(rect[0], o[0])
+ iy = min(rect[3], o[3]) - max(rect[1], o[1])
+ if ix > 0 and iy > 0:
+ best = max(best, ix * iy / area)
+ return best
diff --git a/privacy_core.py b/privacy_core.py
index b1ce493..d1bc082 100644
--- a/privacy_core.py
+++ b/privacy_core.py
@@ -7,9 +7,12 @@
from __future__ import annotations
import base64
+import hashlib
+import hmac
import json
import os
import re
+import secrets
from dataclasses import asdict, dataclass
from enum import Enum
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
@@ -40,6 +43,10 @@ class PlaceholderOccurrence:
# {font, size, color, flags, origin, text}. None for scanned pages and
# for maps written before version 3.
style: Optional[Dict[str, Any]] = None
+ # What was actually drawn when the full placeholder did not fit the box
+ # (a compact form such as "P9F2A"). None when the full placeholder was
+ # drawn. Restore uses the rectangle, never this text.
+ label: Optional[str] = None
def validate_style(raw: Any) -> Dict[str, Any]:
@@ -70,10 +77,32 @@ class PlaceholderEntry:
occurrences: Tuple[PlaceholderOccurrence, ...] = ()
+PSEUDONYM_STYLES = ("random", "counter", "opaque", "keyed")
+
+
class PlaceholderRegistry:
- """Create stable placeholders for detected personal data."""
+ """Create stable placeholders for detected personal data.
+
+ Styles (``style``):
+ - ``counter``: ``PERSON_001``, in first-seen order. The suffix reveals
+ how many distinct values of a type exist and in which order they
+ appear.
+ - ``random``: ``PERSON_9F2A``, unpredictable, drawn from ``secrets``
+ for each document; nothing about the order or count leaks.
+ - ``opaque``: ``DATA_9F2A``; additionally hides the entity type.
+ - ``keyed``: ``PERSON_3B7E10`` = HMAC(key, type || value); the same
+ value gets the same pseudonym in every document made with the same
+ key. Cross-document linking is intended; the key must be kept
+ separately and protected (EDPB Guidelines 01/2025).
+ """
- def __init__(self) -> None:
+ def __init__(self, style: str = "counter", key: Optional[bytes] = None) -> None:
+ if style not in PSEUDONYM_STYLES:
+ raise ValueError(f"Stile pseudonimi sconosciuto: {style}")
+ if style == "keyed" and not key:
+ raise ValueError("Lo stile 'keyed' richiede una chiave.")
+ self._style = style
+ self._key = key
self._counters: Dict[str, int] = {}
self._by_value: Dict[tuple[str, str], str] = {}
self._order: List[str] = []
@@ -115,7 +144,9 @@ def placeholder_for(
if placeholder is None:
next_index = self._counters.get(clean_type, 0) + 1
self._counters[clean_type] = next_index
- placeholder = f"{clean_type}_{next_index:03d}"
+ placeholder = self._make_placeholder(
+ clean_type, clean_original, next_index
+ )
self._by_value[key] = placeholder
self._order.append(placeholder)
self._meta[placeholder] = (clean_original, clean_type, page_index)
@@ -130,6 +161,45 @@ def placeholder_for(
)
return placeholder
+ def set_label(self, placeholder: str, rect: Sequence[float], label: str) -> None:
+ """Record the compact label drawn for the occurrence at `rect`."""
+ target = tuple(float(v) for v in tuple(rect)[:4])
+ occs = self._occurrences.get(placeholder, [])
+ for i, occ in enumerate(occs):
+ if occ.rect == target:
+ occs[i] = PlaceholderOccurrence(
+ page=occ.page, rect=occ.rect, style=occ.style, label=label
+ )
+ return
+
+ def _make_placeholder(self, clean_type: str, original: str, n: int) -> str:
+ if self._style == "counter":
+ return f"{clean_type}_{n:03d}"
+ prefix = "DATA" if self._style == "opaque" else clean_type
+ taken = set(self._by_value.values())
+ if self._style == "keyed":
+ assert self._key is not None
+ digest = hmac.new(
+ self._key,
+ (clean_type + "\x00" + original.casefold()).encode("utf-8"),
+ hashlib.sha256,
+ ).hexdigest().upper()
+ for width in range(6, len(digest) + 1, 2):
+ candidate = f"{prefix}_{digest[:width]}"
+ if candidate not in taken:
+ return candidate
+ return f"{prefix}_{digest}"
+ # Four hex digits keep the placeholder short enough to fit the box
+ # of a short word; widen only if collisions pile up.
+ nbytes, tries = 2, 0
+ while True:
+ candidate = f"{prefix}_{secrets.token_hex(nbytes).upper()}"
+ if candidate not in taken:
+ return candidate
+ tries += 1
+ if tries % 8 == 0:
+ nbytes += 1
+
def entries(self) -> List[PlaceholderEntry]:
entries: List[PlaceholderEntry] = []
for placeholder in self._order:
@@ -147,12 +217,14 @@ def entries(self) -> List[PlaceholderEntry]:
RESTORE_MAP_FORMAT = "NullifyPDF restore map"
-RESTORE_MAP_VERSION = 3
+RESTORE_MAP_VERSION = 4
# Version 1 maps carry no `occurrences`; they are still readable, and
# reconstruction falls back to searching the PDF for the placeholder text.
# Refusing them would permanently strand the PII of every map already issued.
# Version 3 adds an optional per-occurrence `style`; 2 maps stay readable.
-SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2, 3})
+# Version 4 adds an optional per-occurrence `label` (compact placeholder drawn
+# when the full one did not fit); older maps stay readable.
+SUPPORTED_RESTORE_MAP_VERSIONS = frozenset({1, 2, 3, 4})
def build_restore_payload(
@@ -161,19 +233,43 @@ def build_restore_payload(
source_sha256: str,
output_sha256: Optional[str],
entries: Iterable[PlaceholderEntry],
+ keep_source_name: bool = False,
) -> Dict[str, object]:
- """Build the JSON-serializable restore-map payload."""
+ """Build the JSON-serializable restore-map payload.
+
+ The map is itself personal data, so it stores as little as possible: the
+ source file name is omitted unless asked for (it often holds the data
+ subject's name), and a style's ``text`` is dropped when it only repeats
+ the original value already stored in the entry.
+ """
raw_entries: List[Dict[str, object]] = []
for entry in entries:
raw = asdict(entry)
# asdict() keeps tuples as tuples; JSON round-trips them to lists, so
# normalize here and keep the payload byte-comparable with itself.
+ def slim(
+ style: Optional[Dict[str, Any]], original: str = entry.original
+ ) -> Optional[Dict[str, Any]]:
+ if not style:
+ return None
+ out = dict(style)
+ if " ".join(str(out.get("text", "")).split()) == " ".join(
+ original.split()
+ ):
+ out.pop("text", None)
+ return out
+
raw["occurrences"] = [
{
"page": occurrence.page,
"rect": list(occurrence.rect),
- **({"style": occurrence.style} if occurrence.style else {}),
+ **({"style": slim(occurrence.style)} if occurrence.style else {}),
+ **(
+ {"label": occurrence.label}
+ if occurrence.label and occurrence.label != entry.placeholder
+ else {}
+ ),
}
for occurrence in entry.occurrences
]
@@ -182,7 +278,7 @@ def build_restore_payload(
return {
"format": RESTORE_MAP_FORMAT,
"version": RESTORE_MAP_VERSION,
- "source_name": os.path.basename(source_name),
+ "source_name": os.path.basename(source_name) if keep_source_name else "",
"source_sha256": source_sha256,
"output_sha256": output_sha256,
"entries": raw_entries,
@@ -220,6 +316,7 @@ def parse_restore_entries(payload: Dict[str, object]) -> List[PlaceholderEntry]:
if raw_occ.get("style")
else None
),
+ label=(str(raw_occ["label"]) if raw_occ.get("label") else None),
)
)
entries.append(
@@ -251,55 +348,369 @@ def sort_entries_longest_placeholder_first(
return sorted(entries, key=lambda e: len(e.placeholder), reverse=True)
-def encrypt_restore_payload(payload: Dict[str, object], password: str) -> bytes:
- """Encrypt and authenticate a restore-map payload with a password."""
+MIN_PASSWORD_LENGTH = 12
+PBKDF2_MIN_ITERATIONS = 100_000
+PBKDF2_MAX_ITERATIONS = 5_000_000
+SCRYPT_N = 2 ** 17
+SCRYPT_R = 8
+SCRYPT_P = 1
+SCRYPT_MAX_N = 2 ** 20
- if not password or len(password) < 12:
- raise ValueError("La password della mappa deve avere almeno 12 caratteri.")
+_COMMON_PASSWORDS = (
+ "passwordpassword", "123456789012", "qwertyuiopas", "password1234",
+ "administrator", "letmein123456",
+)
- from cryptography.fernet import Fernet
+
+def password_strength(password: str) -> Tuple[bool, str]:
+ """Reject passwords that make the map trivial to open.
+
+ Returns (acceptable, message). Length alone is not enough: a repeated
+ character or a common phrase of 12 characters falls in seconds.
+ """
+ if len(password) < MIN_PASSWORD_LENGTH:
+ return False, "La password deve avere almeno 12 caratteri."
+ if len(set(password)) < 6:
+ return False, "La password usa troppi caratteri ripetuti."
+ low = password.lower()
+ if any(c in low for c in _COMMON_PASSWORDS):
+ return False, "La password e troppo comune."
+ classes = sum([
+ any(c.islower() for c in password),
+ any(c.isupper() for c in password),
+ any(c.isdigit() for c in password),
+ any(not c.isalnum() for c in password),
+ ])
+ if classes < 2 and len(password) < 20:
+ return False, (
+ "Usa almeno due tipi di caratteri (minuscole, maiuscole, cifre, "
+ "simboli) oppure una frase di almeno 20 caratteri."
+ )
+ return True, ""
+
+
+def _derive_key(password: str, envelope: Dict[str, Any], salt: bytes) -> bytes:
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+ from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
+
+ kdf_name = str(envelope.get("kdf", "PBKDF2-HMAC-SHA256"))
+ if kdf_name == "scrypt":
+ n = int(envelope["n"])
+ r = int(envelope.get("r", SCRYPT_R))
+ p_ = int(envelope.get("p", SCRYPT_P))
+ # Parameters come from an untrusted file: bound them before use so a
+ # crafted map cannot exhaust memory or CPU.
+ if (
+ not (2 ** 14 <= n <= SCRYPT_MAX_N) or n & (n - 1)
+ or not (1 <= r <= 16) or not (1 <= p_ <= 4)
+ ):
+ raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.")
+ kdf: Any = Scrypt(salt=salt, length=32, n=n, r=r, p=p_)
+ elif kdf_name == "PBKDF2-HMAC-SHA256":
+ iterations = int(envelope["iterations"])
+ if not (PBKDF2_MIN_ITERATIONS <= iterations <= PBKDF2_MAX_ITERATIONS):
+ raise ValueError(
+ "Numero di iterazioni della mappa fuori dai limiti consentiti."
+ )
+ kdf = PBKDF2HMAC(
+ algorithm=hashes.SHA256(), length=32, salt=salt, iterations=iterations
+ )
+ else:
+ raise ValueError("Algoritmo KDF della mappa non supportato.")
+ return base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8")))
+
+
+def _encrypt_v2(payload: Dict[str, object], password: str) -> bytes:
+ """Version 2 envelope (scrypt + Fernet). Kept for compatibility tests;
+ new maps use :func:`encrypt_restore_payload` (version 3)."""
+
+ if not password or len(password) < MIN_PASSWORD_LENGTH:
+ raise ValueError("La password della mappa deve avere almeno 12 caratteri.")
+
+ from cryptography.fernet import Fernet
salt = os.urandom(16)
- kdf = PBKDF2HMAC(
- algorithm=hashes.SHA256(),
- length=32,
- salt=salt,
- iterations=600000,
- )
- key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8")))
- token = Fernet(key).encrypt(
- json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
- )
- envelope = {
+ envelope: Dict[str, Any] = {
"format": "NullifyPDF encrypted restore map",
- "version": 1,
- "kdf": "PBKDF2-HMAC-SHA256",
- "iterations": 600000,
+ "version": 2,
+ "kdf": "scrypt",
+ "n": SCRYPT_N,
+ "r": SCRYPT_R,
+ "p": SCRYPT_P,
"salt": base64.b64encode(salt).decode("ascii"),
- "token": token.decode("ascii"),
}
+ key = _derive_key(password, envelope, salt)
+ token = Fernet(key).encrypt(
+ json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
+ )
+ envelope["token"] = token.decode("ascii")
return json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=2).encode(
"utf-8"
)
-def decrypt_restore_payload(data: bytes, password: str) -> Dict[str, object]:
- """Decrypt an encrypted restore map."""
+def _decrypt_v1v2(envelope: Dict[str, Any], password: str) -> Dict[str, object]:
+ """Decrypt a version 1 (PBKDF2) or 2 (scrypt) envelope."""
from cryptography.fernet import Fernet
- from cryptography.hazmat.primitives import hashes
- from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
- envelope = json.loads(data.decode("utf-8"))
salt = base64.b64decode(envelope["salt"])
- kdf = PBKDF2HMAC(
- algorithm=hashes.SHA256(),
- length=32,
- salt=salt,
- iterations=int(envelope["iterations"]),
- )
- key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8")))
+ key = _derive_key(password, envelope, salt)
plaintext = Fernet(key).decrypt(envelope["token"].encode("ascii"))
return json.loads(plaintext.decode("utf-8"))
+
+
+# ---------------------------------------------------------------------------
+# Envelope version 3: data key + key wrapping (password, optional recovery key)
+# ---------------------------------------------------------------------------
+
+MAP_FORMAT = "NullifyPDF encrypted restore map"
+KEY_EXPORT_FORMAT = "NullifyPDF pseudonym key"
+ARGON2_T = 3
+ARGON2_M_KIB = 64 * 1024
+ARGON2_P = 4
+_PAYLOAD_AAD = b"NullifyPDF envelope v3 payload"
+_RECOVERY_PREFIX = "NPDF"
+_B32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+
+
+def _b64(b: bytes) -> str:
+ return base64.b64encode(b).decode("ascii")
+
+
+def _unb64(s_: str) -> bytes:
+ return base64.b64decode(s_.encode("ascii"))
+
+
+def _argon2_available() -> bool:
+ try:
+ from cryptography.hazmat.primitives.kdf.argon2 import Argon2id # noqa: F401
+
+ return True
+ except Exception:
+ return False
+
+
+def _password_kek(password: str, wrap: Dict[str, Any]) -> bytes:
+ """Derive the key-encryption key for a password wrap (bounds-checked)."""
+ salt = _unb64(wrap["salt"])
+ if len(salt) < 16:
+ raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.")
+ kdf = str(wrap.get("kdf"))
+ if kdf == "argon2id":
+ from cryptography.hazmat.primitives.kdf.argon2 import Argon2id
+
+ t, m, p_ = int(wrap["t"]), int(wrap["m"]), int(wrap["p"])
+ if not (1 <= t <= 10 and 19 * 1024 <= m <= 1024 * 1024 and 1 <= p_ <= 8 and m >= 8 * p_):
+ raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.")
+ return Argon2id(
+ salt=salt, length=32, iterations=t, lanes=p_, memory_cost=m
+ ).derive(password.encode("utf-8"))
+ if kdf == "scrypt":
+ from cryptography.hazmat.primitives.kdf.scrypt import Scrypt
+
+ n, r, p_ = int(wrap["n"]), int(wrap["r"]), int(wrap["p"])
+ if (
+ not (2 ** 14 <= n <= SCRYPT_MAX_N) or n & (n - 1)
+ or not (1 <= r <= 16) or not (1 <= p_ <= 4)
+ ):
+ raise ValueError("Parametri KDF della mappa fuori dai limiti consentiti.")
+ return Scrypt(salt=salt, length=32, n=n, r=r, p=p_).derive(password.encode("utf-8"))
+ raise ValueError("Algoritmo KDF della mappa non supportato.")
+
+
+def _new_password_wrap(password: str, dek: bytes) -> Dict[str, Any]:
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ salt = os.urandom(16)
+ wrap: Dict[str, Any] = {"type": "password", "salt": _b64(salt)}
+ if _argon2_available():
+ wrap.update(kdf="argon2id", t=ARGON2_T, m=ARGON2_M_KIB, p=ARGON2_P)
+ try:
+ kek = _password_kek(password, wrap)
+ except Exception: # OpenSSL without Argon2: fall back for writing only
+ wrap = {"type": "password", "salt": _b64(salt)}
+ wrap.update(kdf="scrypt", n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P)
+ kek = _password_kek(password, wrap)
+ else:
+ wrap.update(kdf="scrypt", n=SCRYPT_N, r=SCRYPT_R, p=SCRYPT_P)
+ kek = _password_kek(password, wrap)
+ nonce = os.urandom(12)
+ wrap["nonce"] = _b64(nonce)
+ wrap["wrapped"] = _b64(AESGCM(kek).encrypt(nonce, dek, b"NullifyPDF wrap v3:password"))
+ return wrap
+
+
+def _recovery_kek(recovery_key: bytes, salt: bytes) -> bytes:
+ from cryptography.hazmat.primitives import hashes
+ from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+ return HKDF(
+ algorithm=hashes.SHA256(), length=32, salt=salt, info=b"nullifypdf-recovery"
+ ).derive(recovery_key)
+
+
+def _new_recovery_wrap(recovery_key: bytes, dek: bytes) -> Dict[str, Any]:
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ if len(recovery_key) != 32:
+ raise ValueError("Chiave di recupero non valida.")
+ salt, nonce = os.urandom(16), os.urandom(12)
+ kek = _recovery_kek(recovery_key, salt)
+ return {
+ "type": "recovery", "kdf": "hkdf-sha256", "salt": _b64(salt),
+ "nonce": _b64(nonce),
+ "wrapped": _b64(AESGCM(kek).encrypt(nonce, dek, b"NullifyPDF wrap v3:recovery")),
+ "key_id": hashlib.sha256(recovery_key).hexdigest()[:8],
+ }
+
+
+def generate_recovery_key() -> Tuple[bytes, str]:
+ """A fresh 256-bit recovery key and its display form (shown once).
+
+ Display: ``NPDF-XXXX-...`` in base32 groups of four with a 2-character
+ checksum, so typing mistakes are caught before any decryption.
+ """
+ raw = os.urandom(32)
+ return raw, format_recovery_key(raw)
+
+
+def _checksum(raw: bytes) -> str:
+ v = int.from_bytes(hashlib.sha256(raw).digest()[:2], "big") & 0x3FF
+ return _B32[v >> 5] + _B32[v & 31]
+
+
+def format_recovery_key(raw: bytes) -> str:
+ body = base64.b32encode(raw).decode("ascii").rstrip("=") + _checksum(raw)
+ groups = [body[i:i + 4] for i in range(0, len(body), 4)]
+ return _RECOVERY_PREFIX + "-" + "-".join(groups)
+
+
+def parse_recovery_key(display: str) -> bytes:
+ """Parse a displayed recovery key; raises ValueError on any typo."""
+ text = "".join(display.upper().split()).replace("-", "")
+ if text.startswith(_RECOVERY_PREFIX):
+ text = text[len(_RECOVERY_PREFIX):]
+ text = text.replace("0", "O").replace("1", "I")
+ if len(text) != 54 or any(c not in _B32 for c in text):
+ raise ValueError("Chiave di recupero non valida.")
+ body, check = text[:52], text[52:]
+ try:
+ raw = base64.b32decode(body + "====")
+ except Exception as e:
+ raise ValueError("Chiave di recupero non valida.") from e
+ if len(raw) != 32 or _checksum(raw) != check:
+ raise ValueError("Chiave di recupero non valida: controlla di averla digitata bene.")
+ return raw
+
+
+def _encrypt_v3(
+ payload: Dict[str, object], password: str, recovery_key: Optional[bytes], fmt: str
+) -> bytes:
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ dek = os.urandom(32)
+ nonce = os.urandom(12)
+ ciphertext = AESGCM(dek).encrypt(
+ nonce,
+ json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8"),
+ _PAYLOAD_AAD,
+ )
+ wraps = [_new_password_wrap(password, dek)]
+ if recovery_key is not None:
+ wraps.append(_new_recovery_wrap(recovery_key, dek))
+ envelope = {
+ "format": fmt, "version": 3, "cipher": "AES-256-GCM",
+ "nonce": _b64(nonce), "ciphertext": _b64(ciphertext), "wraps": wraps,
+ }
+ return json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=2).encode("utf-8")
+
+
+def _decrypt_v3(
+ envelope: Dict[str, Any], password: Optional[str], recovery_key: Optional[bytes]
+) -> Dict[str, object]:
+ from cryptography.exceptions import InvalidTag
+ from cryptography.fernet import InvalidToken
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ dek: Optional[bytes] = None
+ for wrap in envelope.get("wraps", []):
+ try:
+ if wrap.get("type") == "password" and password is not None:
+ kek = _password_kek(password, wrap)
+ aad = b"NullifyPDF wrap v3:password"
+ elif wrap.get("type") == "recovery" and recovery_key is not None:
+ kek = _recovery_kek(recovery_key, _unb64(wrap["salt"]))
+ aad = b"NullifyPDF wrap v3:recovery"
+ else:
+ continue
+ dek = AESGCM(kek).decrypt(_unb64(wrap["nonce"]), _unb64(wrap["wrapped"]), aad)
+ break
+ except InvalidTag:
+ continue
+ if dek is None:
+ raise InvalidToken() # same signal the callers already handle
+ try:
+ plaintext = AESGCM(dek).decrypt(
+ _unb64(envelope["nonce"]), _unb64(envelope["ciphertext"]), _PAYLOAD_AAD
+ )
+ except InvalidTag as e:
+ raise InvalidToken() from e
+ return json.loads(plaintext.decode("utf-8"))
+
+
+def encrypt_restore_payload(
+ payload: Dict[str, object],
+ password: str,
+ *,
+ recovery_key: Optional[bytes] = None,
+) -> bytes:
+ """Encrypt a restore-map payload (envelope version 3).
+
+ A random data key encrypts the payload (AES-256-GCM); it is wrapped by a
+ key derived from the password (Argon2id, scrypt if unavailable) and, if
+ given, by the recovery key. Maps from older versions remain readable.
+ """
+ if not password or len(password) < MIN_PASSWORD_LENGTH:
+ raise ValueError("La password della mappa deve avere almeno 12 caratteri.")
+ return _encrypt_v3(payload, password, recovery_key, MAP_FORMAT)
+
+
+def decrypt_restore_payload(
+ data: bytes,
+ password: Optional[str] = None,
+ *,
+ recovery_key: Optional[bytes] = None,
+) -> Dict[str, object]:
+ """Decrypt an encrypted restore map (versions 1, 2 and 3)."""
+ envelope = json.loads(data.decode("utf-8"))
+ if not isinstance(envelope, dict) or envelope.get("format") != MAP_FORMAT:
+ raise ValueError("File non riconosciuto come mappa di ripristino cifrata.")
+ version = envelope.get("version")
+ if version in (1, 2):
+ if password is None:
+ raise ValueError("Questa mappa si apre solo con la password.")
+ return _decrypt_v1v2(envelope, password)
+ if version == 3:
+ return _decrypt_v3(envelope, password, recovery_key)
+ raise ValueError("Versione della mappa non supportata: aggiorna NullifyPDF.")
+
+
+def export_pseudonym_key(key: bytes, password: str) -> bytes:
+ """Password-protected copy of the pseudonym key, for another computer."""
+ if len(key) != 32:
+ raise ValueError("Chiave dei segnaposto non valida.")
+ return _encrypt_v3({"key": _b64(key)}, password, None, KEY_EXPORT_FORMAT)
+
+
+def import_pseudonym_key(data: bytes, password: str) -> bytes:
+ envelope = json.loads(data.decode("utf-8"))
+ if not isinstance(envelope, dict) or envelope.get("format") != KEY_EXPORT_FORMAT:
+ raise ValueError("File non riconosciuto come chiave dei segnaposto.")
+ if envelope.get("version") != 3:
+ raise ValueError("Versione della chiave non supportata: aggiorna NullifyPDF.")
+ key = _unb64(str(_decrypt_v3(envelope, password, None)["key"]))
+ if len(key) != 32:
+ raise ValueError("Chiave dei segnaposto non valida.")
+ return key
diff --git a/review_dialog.py b/review_dialog.py
new file mode 100644
index 0000000..56e38b1
--- /dev/null
+++ b/review_dialog.py
@@ -0,0 +1,192 @@
+"""Review dialog for AI suggestions (Qt).
+
+Detections the evidence score could not settle are *suggestions*: they are
+never applied silently and never lost silently. The user accepts or rejects
+each one here; values are masked until the user asks to see them.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from PySide6.QtCore import Qt, Signal
+from PySide6.QtWidgets import (
+ QAbstractItemView,
+ QCheckBox,
+ QDialog,
+ QDoubleSpinBox,
+ QHBoxLayout,
+ QHeaderView,
+ QLabel,
+ QPushButton,
+ QTableWidget,
+ QTableWidgetItem,
+ QVBoxLayout,
+)
+
+REASON_LABELS: Dict[str, str] = {
+ "context-cue": "contesto",
+ "name-shape": "forma di nome",
+ "given-name": "nome proprio noto",
+ "known-place": "luogo noto",
+ "province-or-postcode": "provincia/CAP",
+ "repeated": "ripetuto",
+ "lexicon": "parola comune",
+ "verb-form": "forma verbale",
+ "line-start": "inizio riga",
+ "all-caps": "maiuscolo",
+ "followed-by-function-word": "seguito da che/di",
+ "low-confidence": "bassa confidenza",
+ "signature-anchor": "vicino a firma/timbro",
+ "overlaps-text": "il riquadro tocca del testo",
+ "curve-ink": "tratti a mano",
+ "raster-stamp": "timbro/immagine",
+ "no-context": "senza contesto",
+ "no-anchor": "senza riferimento",
+ "postcode": "CAP",
+ "known-surname": "cognome noto",
+ "ambiguous-place": "luogo omonimo di parola comune",
+}
+
+COL_APPLY, COL_PAGE, COL_VALUE, COL_TYPE, COL_SCORE, COL_WHY = range(6)
+
+
+def mask_value(value: str) -> str:
+ v = " ".join(value.split())
+ if len(v) <= 1:
+ return "•"
+ return v[0] + "•" * min(len(v) - 1, 8)
+
+
+class ReviewDialog(QDialog):
+ """Table of suggestions with per-row accept / reject."""
+
+ navigate = Signal(int)
+ highlight = Signal(object) # the suggestion dict of the selected row
+
+ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> None:
+ super().__init__(parent)
+ self.setWindowTitle("Revisione suggerimenti")
+ self.resize(760, 460)
+ self._items = list(suggestions)
+ self._applied: List[Dict[str, Any]] = []
+ self._rejected: List[Dict[str, Any]] = []
+ lay = QVBoxLayout(self)
+ lay.addWidget(QLabel(
+ "Questi elementi non hanno prove sufficienti per essere censurati "
+ "automaticamente. Seleziona quelli da censurare: i non selezionati "
+ "vengono scartati se premi «Applica selezionati»."
+ ))
+ self.table = QTableWidget(len(self._items), 6, self)
+ self.table.setHorizontalHeaderLabels(
+ ["Censura", "Pag.", "Valore", "Tipo", "Punteggio", "Motivi"]
+ )
+ self.table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows)
+ self.table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
+ self.table.horizontalHeader().setSectionResizeMode(
+ COL_WHY, QHeaderView.ResizeMode.Stretch
+ )
+ lay.addWidget(self.table)
+ self.chk_show = QCheckBox("Mostra i valori")
+ self.chk_show.toggled.connect(self._refresh_values)
+ lay.addWidget(self.chk_show)
+ for row, it in enumerate(self._items):
+ apply_item = QTableWidgetItem()
+ apply_item.setFlags(
+ Qt.ItemFlag.ItemIsUserCheckable | Qt.ItemFlag.ItemIsEnabled
+ )
+ apply_item.setCheckState(Qt.CheckState.Unchecked)
+ self.table.setItem(row, COL_APPLY, apply_item)
+ self.table.setItem(row, COL_PAGE, QTableWidgetItem(str(int(it.get("page", 0)) + 1)))
+ self.table.setItem(row, COL_VALUE, QTableWidgetItem(""))
+ self.table.setItem(row, COL_TYPE, QTableWidgetItem(str(it.get("entity_type", ""))))
+ self.table.setItem(row, COL_SCORE, QTableWidgetItem(f"{float(it.get('score', 0.0)):.2f}"))
+ why = ", ".join(REASON_LABELS.get(r, r) for r in it.get("reasons", []))
+ self.table.setItem(row, COL_WHY, QTableWidgetItem(why))
+ self._refresh_values()
+ self.table.cellDoubleClicked.connect(self._on_double_click)
+ self.table.currentCellChanged.connect(self._on_row_changed)
+
+ bar = QHBoxLayout()
+ btn_all = QPushButton("Seleziona tutti")
+ btn_none = QPushButton("Nessuno")
+ self.spin = QDoubleSpinBox()
+ self.spin.setRange(0.0, 1.0)
+ self.spin.setSingleStep(0.05)
+ self.spin.setValue(0.6)
+ btn_thr = QPushButton("Seleziona ≥ soglia")
+ # Bound methods, not lambdas capturing `self`: a lambda makes a
+ # reference cycle that keeps the dialog alive until the garbage
+ # collector runs, possibly after QApplication is gone.
+ btn_all.clicked.connect(self._select_all)
+ btn_none.clicked.connect(self._select_none)
+ btn_thr.clicked.connect(self._select_threshold)
+ for w in (btn_all, btn_none, self.spin, btn_thr):
+ bar.addWidget(w)
+ bar.addStretch(1)
+ lay.addLayout(bar)
+
+ bottom = QHBoxLayout()
+ self.btn_apply = QPushButton("Applica selezionati")
+ self.btn_apply.setObjectName("Danger")
+ self.btn_later = QPushButton("Rivedi più tardi")
+ self.btn_apply.clicked.connect(self._on_apply)
+ self.btn_later.clicked.connect(self.reject)
+ bottom.addStretch(1)
+ bottom.addWidget(self.btn_later)
+ bottom.addWidget(self.btn_apply)
+ lay.addLayout(bottom)
+
+ # -- helpers ---------------------------------------------------------
+ def _cell(self, row: int, col: int) -> QTableWidgetItem:
+ item = self.table.item(row, col)
+ assert item is not None
+ return item
+
+ def _refresh_values(self) -> None:
+ show = self.chk_show.isChecked()
+ for row, it in enumerate(self._items):
+ text = " ".join(str(it.get("text", "")).split())
+ if not text:
+ self._cell(row, COL_VALUE).setText("(area grafica)")
+ continue
+ self._cell(row, COL_VALUE).setText(text if show else mask_value(text))
+
+ def _select_all(self) -> None:
+ self._set_all(True)
+
+ def _select_none(self) -> None:
+ self._set_all(False)
+
+ def _set_all(self, state: bool) -> None:
+ for row in range(self.table.rowCount()):
+ self._cell(row, COL_APPLY).setCheckState(
+ Qt.CheckState.Checked if state else Qt.CheckState.Unchecked
+ )
+
+ def _select_threshold(self) -> None:
+ thr = self.spin.value()
+ for row, it in enumerate(self._items):
+ if float(it.get("score", 0.0)) >= thr:
+ self._cell(row, COL_APPLY).setCheckState(Qt.CheckState.Checked)
+
+ def _on_row_changed(self, row: int, _c: int, _pr: int, _pc: int) -> None:
+ if 0 <= row < len(self._items):
+ self.highlight.emit(self._items[row])
+
+ def _on_double_click(self, row: int, _col: int) -> None:
+ self.navigate.emit(int(self._items[row].get("page", 0)))
+
+ def _on_apply(self) -> None:
+ self._applied, self._rejected = [], []
+ for row, it in enumerate(self._items):
+ checked = self._cell(row, COL_APPLY).checkState() == Qt.CheckState.Checked
+ (self._applied if checked else self._rejected).append(it)
+ self.accept()
+
+ # -- results ---------------------------------------------------------
+ def applied(self) -> List[Dict[str, Any]]:
+ return list(self._applied)
+
+ def declined(self) -> List[Dict[str, Any]]:
+ return list(self._rejected)
diff --git a/scripts/build_lexicon_data.py b/scripts/build_lexicon_data.py
new file mode 100644
index 0000000..b00b349
--- /dev/null
+++ b/scripts/build_lexicon_data.py
@@ -0,0 +1,141 @@
+"""Generate ``lexicon_data_it.py`` from ISTAT open data (development only).
+
+Not part of the runtime and not imported by the application. It reads the
+ISTAT "Elenco comuni italiani" workbook (CC BY 4.0, https://www.istat.it/note-legali/),
+normalises the place names, marks the ones that are also ordinary words as
+ambiguous, and writes a plain Python module that PyInstaller collects like any
+other import. Only the standard library is needed to read the workbook.
+
+ python scripts/build_lexicon_data.py [--istat-file FILE] [--out FILE]
+
+The generated module only *confirms* PERSON/LOCATION hits; a name missing
+from it never rejects one.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import pathlib
+import re
+import sys
+import urllib.request
+import xml.etree.ElementTree as ET
+import zipfile
+from typing import Dict, List, Set
+
+ISTAT_URL = "https://www.istat.it/storage/codici-unita-amministrative/Elenco-comuni-italiani.xlsx"
+NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"}
+COLUMNS = {
+ "Denominazione in italiano": "it",
+ "Denominazione altra lingua": "alt",
+ "Sigla automobilistica": "sigla",
+ "Denominazione Regione": "region",
+}
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+
+
+def _col(ref: str) -> str:
+ return re.match(r"[A-Z]+", ref).group() # type: ignore[union-attr]
+
+
+def read_rows(path: pathlib.Path) -> "tuple[List[str], List[Dict[str, str]]]":
+ """First worksheet of an .xlsx as a list of header->value dicts."""
+ z = zipfile.ZipFile(path)
+ sst = [
+ "".join(t.text or "" for t in si.iter("{%s}t" % NS["m"]))
+ for si in ET.fromstring(z.read("xl/sharedStrings.xml")).findall("m:si", NS)
+ ] if "xl/sharedStrings.xml" in z.namelist() else []
+ sheet = ET.fromstring(z.read("xl/worksheets/sheet1.xml"))
+ rows = sheet.findall(".//m:row", NS)
+ headers: Dict[str, str] = {}
+ out: List[Dict[str, str]] = []
+ for i, row in enumerate(rows):
+ cells: Dict[str, str] = {}
+ for c in row.findall("m:c", NS):
+ v = c.find("m:v", NS)
+ if v is None or v.text is None:
+ continue
+ cells[_col(c.get("r", ""))] = sst[int(v.text)] if c.get("t") == "s" else v.text
+ if i == 0:
+ headers = {col: " ".join(val.split()) for col, val in cells.items()}
+ continue
+ out.append({headers[col]: val for col, val in cells.items() if col in headers})
+ return list(headers.values()), out
+
+
+def normalise(name: str) -> str:
+ return " ".join(name.replace("’", "'").lower().split())
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--istat-file", type=pathlib.Path)
+ ap.add_argument("--out", type=pathlib.Path, default=ROOT / "lexicon_data_it.py")
+ args = ap.parse_args()
+ src = args.istat_file
+ if src is None:
+ src = pathlib.Path("Elenco-comuni-italiani.xlsx")
+ req = urllib.request.Request(ISTAT_URL, headers={"User-Agent": "NullifyPDF-build/1.0"})
+ src.write_bytes(urllib.request.urlopen(req, timeout=120).read())
+ sha = hashlib.sha256(src.read_bytes()).hexdigest()
+ headers, rows = read_rows(src)
+ missing = set(COLUMNS) - set(headers)
+ if missing:
+ print(f"Workbook layout changed, missing columns: {sorted(missing)}", file=sys.stderr)
+ return 1
+
+ comuni: Set[str] = set()
+ sigle: Set[str] = set()
+ regions: Set[str] = set()
+ for r in rows:
+ for key in ("Denominazione in italiano", "Denominazione altra lingua"):
+ for part in re.split(r"\s*/\s*", r.get(key, "")):
+ n = normalise(part)
+ if len(n) >= 2:
+ comuni.add(n)
+ if r.get("Sigla automobilistica"):
+ sigle.add(r["Sigla automobilistica"].strip().upper())
+ for part in re.split(r"\s*/\s*", r.get("Denominazione Regione", "")):
+ if part.strip():
+ regions.add(normalise(part))
+
+ sys.path.insert(0, str(ROOT))
+ import lexicon_it
+
+ common = set(lexicon_it.COMMON_WORDS | lexicon_it.FUNCTION_WORDS | lexicon_it.GIVEN_NAMES
+ | lexicon_it.CONFLICT_NAMES)
+ vocab_words: Set[str] = set()
+ try:
+ import spacy
+
+ nlp = spacy.load("it_core_news_md")
+ vocab_words = {w for w in comuni if " " not in w and nlp.vocab.has_vector(w)}
+ except Exception as e: # model optional for the generator
+ print(f"spaCy vocabulary not used: {e}", file=sys.stderr)
+ ambiguous = {w for w in comuni if len(w) <= 3 or w in common or w in vocab_words}
+
+ def lit(items: Set[str]) -> str:
+ body = "|".join(sorted(items))
+ assert '"""' not in body and "\\" not in body
+ return 'frozenset("""' + body + '""".split("|"))'
+
+ text = (
+ "# Generated by scripts/build_lexicon_data.py - do not edit.\n"
+ "# Contains ISTAT data (Elenco comuni italiani, " + src.name + "), licensed\n"
+ "# CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/); modified: names\n"
+ "# normalised, lower-cased, bilingual names split.\n"
+ f"SOURCE_SHA256 = {{'istat_comuni': '{sha}'}}\n"
+ f"COMUNI = {lit(comuni)}\n"
+ f"COMUNI_AMBIGUOUS = {lit(ambiguous)}\n"
+ f"PROVINCE_SIGLE = {lit(sigle)}\n"
+ f"REGIONS = {lit(regions)}\n"
+ )
+ args.out.write_text(text, encoding="utf-8")
+ print(f"{len(comuni)} comuni ({len(ambiguous)} ambiguous), {len(sigle)} province codes, "
+ f"{len(regions)} regions -> {args.out}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/secure_store.py b/secure_store.py
new file mode 100644
index 0000000..bfc5c0b
--- /dev/null
+++ b/secure_store.py
@@ -0,0 +1,304 @@
+"""Protection of local secrets (Qt-free).
+
+A single random master key protects the word lists and the pseudonym key.
+The master key itself is kept by the best protector available:
+
+* Windows: DPAPI (``CryptProtectData``, via ctypes - no extra dependency);
+ bound to the Windows account.
+* macOS / Linux: the OS keyring when the optional ``keyring`` package is
+ installed and a real backend is active.
+* Otherwise a key file (``keys/master.key``, mode 0600 where supported),
+ which only protects against copying a single data file on its own, not
+ against someone who can read the user's profile.
+
+Data is sealed with AES-256-GCM. If the stored master key cannot be
+recovered (profile moved to another account or machine) nothing is deleted or
+overwritten: :class:`KeystoreUnavailable` is raised and callers degrade to
+"no data" while leaving the encrypted files untouched.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import pathlib
+import stat
+import sys
+from typing import Any, Dict, Optional, Protocol
+
+KEYSTORE_FORMAT = "NullifyPDF keystore"
+_DPAPI_ENTROPY = b"NullifyPDF/secure_store/v1"
+
+
+class KeystoreUnavailable(RuntimeError):
+ """The master key cannot be recovered on this account / machine."""
+
+
+class SealError(ValueError):
+ """Sealed data is corrupt, tampered with, or for another context."""
+
+
+# ----------------------------------------------------------------- sealing
+
+def seal(plaintext: bytes, key: bytes, aad: bytes) -> bytes:
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ nonce = os.urandom(12)
+ return nonce + AESGCM(key).encrypt(nonce, plaintext, aad)
+
+
+def open_sealed(blob: bytes, key: bytes, aad: bytes) -> bytes:
+ from cryptography.exceptions import InvalidTag
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ if len(blob) < 12 + 16:
+ raise SealError("Dati cifrati troncati.")
+ try:
+ return AESGCM(key).decrypt(blob[:12], blob[12:], aad)
+ except InvalidTag as e:
+ raise SealError("Dati cifrati non validi o modificati.") from e
+
+
+# -------------------------------------------------------------- protectors
+
+class KeyProtector(Protocol):
+ name: str
+
+ def wrap(self, key: bytes) -> Dict[str, Any]: ...
+
+ def unwrap(self, blob: Dict[str, Any]) -> bytes: ...
+
+
+class DpapiProtector:
+ name = "dpapi"
+
+ @staticmethod
+ def available() -> bool:
+ return sys.platform == "win32"
+
+ def _call(self, data: bytes, protect: bool) -> bytes:
+ import ctypes
+ from ctypes import wintypes
+
+ class DATA_BLOB(ctypes.Structure):
+ _fields_ = [("cbData", wintypes.DWORD), ("pbData", ctypes.POINTER(ctypes.c_char))]
+
+ def blob_of(b: bytes) -> "tuple[DATA_BLOB, Any]":
+ buf = ctypes.create_string_buffer(b, len(b))
+ return DATA_BLOB(len(b), ctypes.cast(buf, ctypes.POINTER(ctypes.c_char))), buf
+
+ crypt32 = ctypes.windll.crypt32 # type: ignore[attr-defined]
+ kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined]
+ kernel32.LocalFree.argtypes = [ctypes.c_void_p]
+ in_blob, keep1 = blob_of(data)
+ ent_blob, keep2 = blob_of(_DPAPI_ENTROPY)
+ out_blob = DATA_BLOB()
+ CRYPTPROTECT_UI_FORBIDDEN = 0x1
+ if protect:
+ ok = crypt32.CryptProtectData(
+ ctypes.byref(in_blob), "NullifyPDF", ctypes.byref(ent_blob),
+ None, None, CRYPTPROTECT_UI_FORBIDDEN, ctypes.byref(out_blob),
+ )
+ else:
+ ok = crypt32.CryptUnprotectData(
+ ctypes.byref(in_blob), None, ctypes.byref(ent_blob),
+ None, None, CRYPTPROTECT_UI_FORBIDDEN, ctypes.byref(out_blob),
+ )
+ del keep1, keep2
+ if not ok:
+ raise OSError(ctypes.WinError()) # type: ignore[attr-defined]
+ try:
+ return ctypes.string_at(out_blob.pbData, out_blob.cbData)
+ finally:
+ kernel32.LocalFree(out_blob.pbData)
+
+ def wrap(self, key: bytes) -> Dict[str, Any]:
+ return {
+ "protector": self.name,
+ "blob": base64.b64encode(self._call(key, True)).decode("ascii"),
+ }
+
+ def unwrap(self, blob: Dict[str, Any]) -> bytes:
+ return self._call(base64.b64decode(blob["blob"]), False)
+
+
+class KeyringProtector:
+ name = "keyring"
+ SERVICE = "NullifyPDF"
+ USER = "master"
+
+ @staticmethod
+ def available() -> bool:
+ try:
+ import keyring # type: ignore[import-not-found]
+ from keyring.backends import fail, null # type: ignore[import-not-found]
+
+ backend = keyring.get_keyring()
+ return not isinstance(backend, (fail.Keyring, null.Keyring))
+ except Exception:
+ return False
+
+ def wrap(self, key: bytes) -> Dict[str, Any]:
+ import keyring # type: ignore[import-not-found]
+
+ keyring.set_password(self.SERVICE, self.USER, base64.b64encode(key).decode("ascii"))
+ return {"protector": self.name, "service": self.SERVICE, "user": self.USER}
+
+ def unwrap(self, blob: Dict[str, Any]) -> bytes:
+ import keyring # type: ignore[import-not-found]
+
+ secret = keyring.get_password(blob.get("service", self.SERVICE), blob.get("user", self.USER))
+ if not secret:
+ raise OSError("Segreto non presente nel portachiavi.")
+ return base64.b64decode(secret)
+
+
+class KeyFileProtector:
+ name = "keyfile"
+
+ def __init__(self, config_dir: pathlib.Path) -> None:
+ self.path = pathlib.Path(config_dir) / "keys" / "master.key"
+
+ def wrap(self, key: bytes) -> Dict[str, Any]:
+ self.path.parent.mkdir(parents=True, exist_ok=True)
+ flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0)
+ fd = os.open(str(self.path), flags, 0o600)
+ with os.fdopen(fd, "wb") as fh:
+ fh.write(key)
+ try:
+ os.chmod(self.path, stat.S_IRUSR | stat.S_IWUSR)
+ except OSError:
+ pass
+ return {"protector": self.name, "path": "keys/master.key"}
+
+ def unwrap(self, blob: Dict[str, Any]) -> bytes:
+ return self.path.read_bytes()
+
+
+# ---------------------------------------------------------------- keystore
+
+def _keystore_path(config_dir: pathlib.Path) -> pathlib.Path:
+ return pathlib.Path(config_dir) / "keystore.json"
+
+
+def _protectors(config_dir: pathlib.Path) -> Dict[str, Any]:
+ out: Dict[str, Any] = {"keyfile": KeyFileProtector(config_dir)}
+ if DpapiProtector.available():
+ out["dpapi"] = DpapiProtector()
+ if KeyringProtector.available():
+ out["keyring"] = KeyringProtector()
+ return out
+
+
+_CACHE: Dict[str, bytes] = {}
+
+
+def forget_cached_keys() -> None:
+ _CACHE.clear()
+
+
+def protector_label(config_dir: pathlib.Path) -> str:
+ """Human-readable (Italian) description of how the master key is kept."""
+ try:
+ data = json.loads(_keystore_path(config_dir).read_text(encoding="utf-8"))
+ name = data["master"]["protector"]
+ except Exception:
+ name = None
+ return {
+ "dpapi": "Archivio protetto: Windows DPAPI",
+ "keyring": "Archivio protetto: portachiavi di sistema",
+ "keyfile": "File chiave locale (protezione limitata)",
+ }.get(name or "", "Non ancora creato")
+
+
+def load_or_create_master_key(config_dir: pathlib.Path) -> bytes:
+ """Return the 32-byte master key, creating the keystore on first use.
+
+ Raises :class:`KeystoreUnavailable` when a keystore exists but cannot be
+ unwrapped on this account. Never overwrites an existing keystore.
+ """
+ cfg = pathlib.Path(config_dir)
+ cache_key = str(cfg.resolve())
+ if cache_key in _CACHE:
+ return _CACHE[cache_key]
+ ks = _keystore_path(cfg)
+ protectors = _protectors(cfg)
+ if ks.exists():
+ try:
+ data = json.loads(ks.read_text(encoding="utf-8"))
+ master = data["master"]
+ prot = protectors.get(master["protector"])
+ if prot is None:
+ raise KeystoreUnavailable(
+ f"Protezione '{master['protector']}' non disponibile su questo sistema."
+ )
+ key = prot.unwrap(master)
+ except KeystoreUnavailable:
+ raise
+ except Exception as e:
+ raise KeystoreUnavailable(
+ "Chiave dei dati locali non leggibile su questo account/PC."
+ ) from e
+ if len(key) != 32:
+ raise KeystoreUnavailable("Chiave dei dati locali non valida.")
+ _CACHE[cache_key] = key
+ return key
+ key = os.urandom(32)
+ errors = []
+ for name in ("dpapi", "keyring", "keyfile"):
+ prot = protectors.get(name)
+ if prot is None:
+ continue
+ try:
+ master = prot.wrap(key)
+ if prot.unwrap(master) != key: # round trip before trusting it
+ raise OSError("round trip mismatch")
+ except Exception as e:
+ errors.append(f"{name}: {e}")
+ continue
+ cfg.mkdir(parents=True, exist_ok=True)
+ _write_private(ks, json.dumps(
+ {"format": KEYSTORE_FORMAT, "version": 1, "master": master}, indent=2
+ ).encode("utf-8"))
+ _CACHE[cache_key] = key
+ return key
+ raise KeystoreUnavailable("Nessuna protezione disponibile: " + "; ".join(errors))
+
+
+def _write_private(path: pathlib.Path, data: bytes) -> None:
+ flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0)
+ fd = os.open(str(path), flags, 0o600)
+ with os.fdopen(fd, "wb") as fh:
+ fh.write(data)
+ try:
+ os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
+ except OSError:
+ pass
+
+
+def atomic_write(path: pathlib.Path, data: bytes) -> None:
+ """Write via a temp file in the same folder, then replace (atomic)."""
+ tmp = path.with_name(path.name + ".part")
+ _write_private(tmp, data)
+ os.replace(tmp, path)
+
+
+def secure_delete(path: pathlib.Path) -> None:
+ """Best effort: overwrite with zeros, truncate, unlink.
+
+ SSD wear levelling, backups and file-system journals may keep copies.
+ """
+ try:
+ size = path.stat().st_size
+ with open(path, "r+b") as fh:
+ fh.write(b"\x00" * size)
+ fh.flush()
+ os.fsync(fh.fileno())
+ fh.truncate(0)
+ except OSError:
+ pass
+ try:
+ path.unlink()
+ except OSError:
+ pass
diff --git a/tests/_pdfgen.py b/tests/_pdfgen.py
new file mode 100644
index 0000000..180fc6d
--- /dev/null
+++ b/tests/_pdfgen.py
@@ -0,0 +1,38 @@
+"""Synthetic PDF builders shared by the tests (no binary fixtures)."""
+
+from types import SimpleNamespace
+
+import pymupdf
+
+
+def make_doc(lines, pitch=14.0, size=11.0, x=72.0, y0=100.0, fontname="helv"):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ for i, text in enumerate(lines):
+ page.insert_text((x, y0 + i * pitch), text, fontsize=size, fontname=fontname)
+ return doc
+
+
+def words_of(page, **kw):
+ return page.get_text("words", **kw)
+
+
+def res(entity_type, text, value, score=0.85):
+ """Presidio-like result for the first occurrence of `value` in `text`."""
+ start = text.index(value)
+ return SimpleNamespace(
+ entity_type=entity_type, start=start, end=start + len(value), score=score
+ )
+
+
+def fake_words(text):
+ """`get_text("words")`-style tuples for plain text (one block)."""
+ out = []
+ y = 0.0
+ for li, line in enumerate(text.split("\n")):
+ x = 10.0
+ for wi, w in enumerate(line.split()):
+ out.append((x, y, x + 6 * len(w), y + 12, w, 0, li, wi))
+ x += 6 * len(w) + 4
+ y += 14
+ return out
diff --git a/tests/conftest.py b/tests/conftest.py
index 3e809b8..6e9ef1d 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,3 +8,26 @@
root_str = str(PROJECT_ROOT)
if root_str not in sys.path:
sys.path.insert(0, root_str)
+
+
+def pytest_sessionfinish(session, exitstatus):
+ """Destroy Qt objects while QApplication is still alive.
+
+ Dialogs and windows left in reference cycles would otherwise be collected
+ at interpreter shutdown, after the application object, which crashes the
+ process on some platforms even though every test passed.
+ """
+ try:
+ import gc
+
+ from PySide6.QtWidgets import QApplication
+
+ app = QApplication.instance()
+ if app is not None:
+ app.closeAllWindows()
+ app.processEvents()
+ gc.collect()
+ if app is not None:
+ app.processEvents()
+ except Exception:
+ pass
diff --git a/tests/test_build_config.py b/tests/test_build_config.py
index 7d12a83..20051d4 100644
--- a/tests/test_build_config.py
+++ b/tests/test_build_config.py
@@ -117,3 +117,17 @@ def test_build_workflows_use_the_shared_linux_build(name):
assert "uses: ./.github/workflows/build-linux.yml" in text
# Linux must not be built on the old ubuntu-22.04 runner image.
assert "ubuntu-22.04" not in text
+
+
+def test_mypy_module_lists_are_identical_across_workflows():
+ import pathlib
+ import re
+
+ root = pathlib.Path(__file__).resolve().parent.parent / ".github" / "workflows"
+ sets = {}
+ for name in ("beta-release.yml", "build-linux.yml", "release.yml", "test_build.yml"):
+ text = (root / name).read_text(encoding="utf-8")
+ m = re.search(r"run: mypy ([^\n]+)", text)
+ assert m, name
+ sets[name] = frozenset(m.group(1).split())
+ assert len(set(sets.values())) == 1, sets
diff --git a/tests/test_detection_pipeline.py b/tests/test_detection_pipeline.py
new file mode 100644
index 0000000..18eec38
--- /dev/null
+++ b/tests/test_detection_pipeline.py
@@ -0,0 +1,454 @@
+"""Italian precision, pattern validation and allowlist semantics."""
+
+import pytest
+
+import pii_detection as pd
+from _pdfgen import fake_words, res
+import lexicon_it
+
+
+def _run(text, results, enabled=None, **kw):
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ enabled = enabled or pd.expand_group_ids(pd.ALL_GROUP_IDS)
+ cands, stats = pd.classify_page(results, idx, enabled, **kw)
+ return idx, cands, stats
+
+
+# --- lexicon -----------------------------------------------------------------
+
+@pytest.mark.parametrize("word", [
+ "Certifico", "Prescrivo", "Dichiaro", "Attesto", "Rilascio", "Confermo",
+ "Raccomando", "Ordina", "Anamnesi", "Diagnosi", "Terapia", "Paziente",
+ "PRESCRIVO", "Comune", "Tribunale",
+])
+def test_ordinary_words_are_recognised(word):
+ assert lexicon_it.is_common_word(word)
+
+
+@pytest.mark.parametrize("word", ["Rossi", "Esposito", "Mario", "Napoli", "Rosa", "Bruno"])
+def test_names_are_not_common_words(word):
+ assert not lexicon_it.is_common_word(word)
+
+
+def test_verb_forms_are_generated_from_stems():
+ forms = lexicon_it.verb_forms()
+ assert {"certifica", "certificano", "prescrive", "prescrivono", "dichiara"} <= forms
+
+
+# --- fake-analyzer pipeline (no model needed) -----------------------------------
+
+def test_lone_verb_tagged_as_location_is_dropped():
+ text = "Certifico\nPrescrivo\nRaccomando"
+ results = [res("LOCATION", text, w) for w in ("Certifico", "Prescrivo", "Raccomando")]
+ _idx, cands, stats = _run(text, results)
+ assert cands == [] and stats.get("lexicon", 0) == 3
+
+
+def test_verb_plus_drug_keeps_only_the_non_verb():
+ text = "Visita del 3 aprile Prescrivo Augmentin"
+ r = res("LOCATION", text, "Prescrivo Augmentin")
+ _idx, cands, _ = _run(text, [r])
+ assert all("Prescrivo" not in c.text for c in cands)
+
+
+def test_name_with_context_is_accepted_and_scored_from_evidence():
+ text = "Il Sig. Mario Rossi vive a Roma"
+ results = [res("PERSON", text, "Mario Rossi"), res("LOCATION", text, "Roma")]
+ _idx, cands, _ = _run(text, results)
+ by_type = {c.entity_type: c for c in cands}
+ assert by_type["PERSON"].state == "accepted" and by_type["PERSON"].score >= 0.7
+ assert by_type["LOCATION"].state == "accepted"
+
+
+def test_ambiguous_single_word_becomes_a_suggestion_not_a_drop():
+ text = "Il referto indica che Colombi era presente"
+ r = res("PERSON", text, "Colombi")
+ _idx, cands, _ = _run(text, [r])
+ assert len(cands) == 1 and cands[0].state == "suggested"
+
+
+def test_health_mode_never_drops_silently():
+ text = "Certifico"
+ r = res("LOCATION", text, "Certifico")
+ # Lexicon edges are a hard rule; use an uncertain non-lexicon word instead.
+ text2 = "Noi Zorzi andiamo"
+ r2 = res("PERSON", text2, "Zorzi")
+ _i, normal, _ = _run(text2, [r2])
+ _i, health, _ = _run(text2, [r2], health_mode=True)
+ assert len(health) >= len(normal) and all(c.state != "dropped" for c in health)
+
+
+def test_split_lines_of_one_name_are_merged():
+ text = "Esposito\nGiovanni abita a Napoli"
+ results = [res("PERSON", text, "Esposito"), res("PERSON", text, "Giovanni")]
+ _idx, cands, _ = _run(text, results)
+ person = [c for c in cands if c.entity_type == "PERSON"]
+ assert len(person) == 1 and person[0].text.replace("\n", " ") == "Esposito Giovanni"
+ assert person[0].state == "accepted"
+
+
+def test_field_label_is_rejected_outright():
+ text = "Nome: Mario"
+ r = res("PERSON", text, "Nome")
+ _idx, cands, _ = _run(text, [r])
+ assert cands == []
+
+
+def test_fragments_and_ocr_noise_are_rejected():
+ text = "Rnario Ro"
+ _i, normal, _ = _run(text, [res("PERSON", text, "Rnario")], used_ocr=True)
+ assert normal == []
+ kept, n = pd.reject_fragment_hits(
+ [pd.Candidate(0, 4, "Li", "PERSON", 0.85)], used_ocr=False
+ )
+ assert kept == [] and n == 1
+
+
+# --- pattern validation --------------------------------------------------------
+
+def _pattern(text, etype, value, score=1.0):
+ start = text.index(value)
+ c = pd.Candidate(start, start + len(value), value, etype, score)
+ return pd.refine_pattern_hit(c, text)
+
+
+def test_fiscal_code_needs_boundaries_digits_or_context():
+ ok, _ = _pattern("Codice fiscale: RSSMRA80A01H501U fine", "IT_FISCAL_CODE", "RSSMRA80A01H501U")
+ assert ok
+ ok, why = _pattern("xxRSSMRA80A01H501Uxx", "IT_FISCAL_CODE", "RSSMRA80A01H501U")
+ assert not ok and why == "embedded-in-token"
+ ok, why = _pattern("parola AAAAAAAAAAAAAAAA fine", "IT_FISCAL_CODE", "AAAAAAAAAAAAAAAA")
+ assert not ok and why == "no-digit-no-context"
+
+
+def test_vat_spaced_digits_need_context():
+ ok, why = _pattern("totale 1234 5678 901 euro", "IT_VAT_CODE", "1234 5678 901", 0.1)
+ assert not ok
+ ok, _ = _pattern("P.IVA 12345678903 grazie", "IT_VAT_CODE", "12345678903", 1.0)
+ assert ok
+ ok, why = _pattern("numero 1234567890312 lungo", "IT_VAT_CODE", "12345678903", 1.0)
+ assert not ok and why == "embedded-in-digits"
+
+
+@pytest.mark.parametrize("etype,value,ctx_ok,ctx_bad", [
+ ("IT_DRIVER_LICENSE", "AB1234567C", "patente di guida n. AB1234567C ok", "codice prodotto AB1234567C ok"),
+ ("IT_PASSPORT", "YA1234567", "passaporto YA1234567 ok", "ordine YA1234567 ok"),
+ ("IT_IDENTITY_CARD", "CA12345AB", "carta d'identita CA12345AB ok", "ticket CA12345AB ok"),
+])
+def test_document_numbers_need_their_context(etype, value, ctx_ok, ctx_bad):
+ assert _pattern(ctx_ok, etype, value, 0.4)[0]
+ assert not _pattern(ctx_bad, etype, value, 0.4)[0]
+ assert not _pattern(ctx_ok.replace(value, value.lower()), etype, value.lower(), 0.4)[0]
+
+
+@pytest.mark.parametrize("text,value,expect", [
+ ("Tel. 340 1234567 fine", "340 1234567", True),
+ ("chiama +39 02 1234567", "+39 02 1234567", True),
+ ("Fattura del 12/03/2024 n.", "12/03/2024", False),
+ ("anni 2018 - 2022 presso", "2018 - 2022", False),
+ ("ordine 5551234 ok", "5551234", False),
+ ("codice A340123456789", "340123456789", False),
+])
+def test_phone_plausibility(text, value, expect):
+ assert _pattern(text, "PHONE_NUMBER", value, 0.4)[0] is expect
+
+
+def _health(text, kinds=("DATE_OF_BIRTH", "HEALTH_ID", "HEALTH_EXEMPTION"), health_mode=False):
+ return [(c.entity_type, c.text, c.state) for c in
+ pd.detect_health_patterns(text, kinds, health_mode)]
+
+
+def test_date_of_birth_needs_its_anchor():
+ assert ("DATE_OF_BIRTH", "12/03/1970", "accepted") in _health("nato a Napoli il 12/03/1970")
+ assert ("DATE_OF_BIRTH", "3 marzo 1985", "accepted") in _health("nata il 3 marzo 1985 a Roma")
+ assert _health("Fattura del 12/03/2024") == []
+ assert pd.detect_health_patterns("nato il 12/03/1970", []) == []
+
+
+def test_tessera_requires_valid_region_and_exact_length():
+ assert ("HEALTH_ID", "80380000501234567890", "accepted") in _health("Tessera 80380000501234567890 ok")
+ assert any(t == "HEALTH_ID" for t, _, _ in _health("TS 80380 00050 123456789 0 ok"))
+ assert _health("n. 80380009991234567890") == [] # invalid region code
+ assert _health("n. 8038000050123456789") == [] # 19 digits
+ assert _health("n. 803800005012345678901") == [] # 21 digits
+
+
+def test_tessera_and_nre_are_both_reported():
+ text = "Tessera 80380000501234567890 Ricetta *050A01234567890*"
+ types = [(t, v) for t, v, _ in _health(text)]
+ assert ("HEALTH_ID", "80380000501234567890") in types
+ assert any(t == "HEALTH_ID" and v.strip("*").startswith("050A0") for t, v in types)
+
+
+def test_nre_needs_context_unless_health_mode_where_it_is_a_suggestion():
+ bare = "codice 050A01234567890 fine"
+ assert _health(bare) == []
+ assert ("HEALTH_ID", "050A01234567890", "suggested") in _health(bare, health_mode=True)
+ assert _health("fattura 123456789012345") == []
+ assert _health("Numero ricetta: 050A01234567890")[0][2] == "accepted"
+
+
+def test_exemption_codes_need_context():
+ assert ("HEALTH_EXEMPTION", "E01", "accepted") in _health("Esenzione: E01")
+ assert _health("modello E01") == []
+ assert ("HEALTH_EXEMPTION", "RA0010", "accepted") in _health("cod. es. RA0010 rara")
+ assert _health("sezione C03 del regolamento") == []
+
+
+def _addr(text, **kw):
+ cands = pd.detect_address_parts(text, ["LOCATION"], **kw)
+ return [(c.text, c.state) for c in cands]
+
+
+def test_civic_number_joins_street_and_postcode_is_found():
+ found = _addr("Via Garibaldi 12/B, 80100 Napoli (NA)")
+ assert ("Via Garibaldi 12/B", "accepted") in found and ("80100", "accepted") in found
+ assert ("Via Roma 10", "accepted") in _addr("abita in Via Roma 10 con la famiglia")
+
+
+def test_postcode_not_matched_in_prices_dates_quantities():
+ for text in ("Totale 12345 euro", "importo € 10000,00", "prot. 45678 del",
+ "ordinati 12345 Pezzi", "il 01/02/2024"):
+ assert _addr(text) == [], text
+
+
+def test_postcode_requires_an_anchor():
+ assert ("20121", "accepted") in _addr("codice 20121 Milano")
+ assert _addr("lotto 20121 Alfa") == []
+ assert ("20121", "suggested") in _addr("lotto 20121 Alfa", health_mode=True)
+ assert pd.detect_address_parts("Via Roma 10", ["PERSON"]) == []
+
+
+def test_address_candidates_survive_the_pipeline_unscored():
+ text = "Residente in Via Garibaldi 12 80100 Napoli"
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ cands, _ = pd.classify_page([], idx, ["LOCATION"])
+ assert any(c.text.startswith("Via Garibaldi 12") and c.state == "accepted" for c in cands)
+
+
+# --- allowlist semantics ----------------------------------------------------------
+
+def _detections(text, etype, value, allow):
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ start = text.index(value)
+ c = pd.Candidate(start, start + len(value), value, etype, 0.9)
+ return pd.build_detections(idx, [c], allowlist=allow)
+
+
+def test_allowlist_entry_does_not_suppress_a_shorter_value():
+ # "rossini" allowlisted must not hide "Rossi" (substring bug).
+ out = _detections("Dott. Rossi lavora qui", "PERSON", "Rossi", ["rossini"])
+ assert len(out) == 1
+
+
+def test_allowlisted_word_only_drops_that_word_from_a_longer_value():
+ out = _detections("Mario Rossi vive qui", "PERSON", "Mario Rossi", ["mario"])
+ assert len(out) == 1
+ assert len(out[0]["rects"]) == 1
+ # only "Rossi" remains covered
+ idx = pd.build_page_index(fake_words("Mario Rossi vive qui"), clamp=False)
+ assert out[0]["rects"][0][0] == pytest.approx(idx.words[1].rect[0])
+
+
+def test_fully_allowlisted_value_disappears():
+ assert _detections("Mario Rossi vive qui", "PERSON", "Mario Rossi", ["mario rossi"]) == []
+ assert _detections("Il Comune di Roma", "LOCATION", "Roma", ["roma"]) == []
+
+
+def test_allowlist_word_on_adjacent_line_never_cancels_pii():
+ text = "Mario Rossi\nroma"
+ out = _detections(text, "PERSON", "Mario Rossi", ["roma"])
+ assert len(out) == 1
+
+
+def test_propagation_keeps_whole_words_only_and_respects_allowlist():
+ text = "Rossi e Rossini e Rossi"
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ c = pd.Candidate(0, 5, "Rossi", "PERSON", 0.9)
+ out = pd.build_detections(idx, [c])
+ assert len(out[0]["rects"]) == 2
+
+
+def test_detection_carries_state_reasons_and_glued_flag():
+ idx = pd.build_page_index(fake_words("CF:RSSMRA80A01H501U fine"), clamp=False)
+ start = 3
+ c = pd.Candidate(start, start + 16, "RSSMRA80A01H501U", "IT_FISCAL_CODE", 1.0)
+ d = pd.build_detections(idx, [c])[0]
+ assert d["state"] == "accepted" and d["rects"] and "reasons" in d
+
+
+# --- real model (skipped when unavailable) ---------------------------------------
+
+CASES = {
+ "T1": ("Certifico\nche il Sig. Mario Rossi, nato a Napoli il 12/03/1970,",
+ {"Mario Rossi", "Napoli", "12/03/1970"}, {"Certifico"}),
+ "T2": ("Prescrivo\nTachipirina 1000 mg, 1 cpr ogni 8 ore", set(), {"Prescrivo"}),
+ "T3": ("Certifico\nPrescrivo\nRaccomando", set(), {"Certifico", "Prescrivo", "Raccomando"}),
+ "T4": ("Visita del 3/4. Prescrivo Augmentin.", set(), {"Prescrivo"}),
+ "T5": ("CERTIFICO CHE IL PAZIENTE E' IDONEO. PRESCRIVO ESAMI EMATOCHIMICI.",
+ set(), {"CERTIFICO", "PRESCRIVO", "PAZIENTE", "IDONEO"}),
+ "T6": ("Il sottoscritto Avv. Paolo Neri dichiara\nche la sig.ra Gialli Anna e' residente a Roma.",
+ {"Paolo Neri", "Gialli Anna", "Roma"}, {"dichiara", "Dichiara"}),
+ "T7": ("Visto l'art. 7 del Regolamento, Ordina la comparizione delle parti.",
+ set(), {"Ordina", "Visto"}),
+ "T8": ("Esposito\nGiovanni, nato a Napoli.", {"Esposito Giovanni", "Napoli"}, set()),
+ "T9": ("Rosa Bianco", {"Rosa Bianco"}, set()),
+ "T11": ("Milano, 3 marzo 2026", {"Milano"}, set()),
+ "T12": ("Dichiaro di aver ricevuto l'informativa.", set(), {"Dichiaro"}),
+ "T13": ("Firmato: Dott.ssa Lucia Verdi", {"Lucia Verdi"}, {"Firmato"}),
+ "T14": ("Anamnesi\nEsame obiettivo\nTerapia\nPrognosi", set(),
+ {"Anamnesi", "Esame", "Terapia", "Prognosi"}),
+}
+
+
+@pytest.fixture(scope="module")
+def italian_analyzer():
+ pytest.importorskip("presidio_analyzer")
+ spacy = pytest.importorskip("spacy")
+ if not spacy.util.is_package("it_core_news_md"):
+ pytest.skip("model missing")
+ from presidio_analyzer import AnalyzerEngine, RecognizerRegistry
+ from presidio_analyzer.nlp_engine import NlpEngineProvider
+
+ nlp = NlpEngineProvider(nlp_configuration={"nlp_engine_name": "spacy", "models": [
+ {"lang_code": "it", "model_name": "it_core_news_md"}]}).create_engine()
+ reg = RecognizerRegistry(supported_languages=["it"])
+ reg.load_predefined_recognizers(languages=["it"], nlp_engine=nlp)
+ return AnalyzerEngine(nlp_engine=nlp, registry=reg, supported_languages=["it"])
+
+
+@pytest.mark.parametrize("case", sorted(CASES))
+def test_italian_documents_with_real_model(italian_analyzer, case):
+ text, must, never = CASES[case]
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ enabled = pd.expand_group_ids(pd.ALL_GROUP_IDS)
+ presidio = [t for t in enabled if t not in pd.NON_PRESIDIO_TYPES]
+ results = italian_analyzer.analyze(text=idx.text, entities=presidio, language="it")
+ cands, _ = pd.classify_page(results, idx, enabled)
+ accepted = {" ".join(c.text.split()) for c in cands if c.state == "accepted"}
+ flagged = {w for c in cands for w in c.text.split()}
+ assert must <= accepted, (case, accepted)
+ assert not (never & flagged), (case, flagged)
+
+
+def test_precision_on_ordinary_clinical_text_with_real_model(italian_analyzer):
+ text = (
+ "Certifico che il paziente e' affetto da lombalgia.\n"
+ "Prescrivo riposo assoluto e visita di controllo.\n"
+ "Raccomando terapia e dieta.\nDichiaro quanto sopra."
+ )
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ enabled = pd.expand_group_ids(pd.ALL_GROUP_IDS)
+ results = italian_analyzer.analyze(
+ text=idx.text,
+ entities=[t for t in enabled if t not in pd.NON_PRESIDIO_TYPES],
+ language="it",
+ )
+ cands, _ = pd.classify_page(results, idx, enabled)
+ assert [c for c in cands if c.state == "accepted"] == []
+
+
+# --- signature / stamp suggestions ----------------------------------------------
+
+def _signature_doc(extra_line=False):
+ import pymupdf as _pm
+
+ doc = _pm.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Il medico", fontsize=11)
+ page.insert_text((72, 114), "Firma:", fontsize=11)
+ sh = page.new_shape()
+ for k in range(6): # a curve-rich squiggle
+ x = 120 + k * 12
+ sh.draw_bezier(_pm.Point(x, 130), _pm.Point(x + 3, 105), _pm.Point(x + 7, 150), _pm.Point(x + 12, 128))
+ sh.finish(color=(0, 0, 0), fill=None, closePath=False)
+ sh.commit()
+ if extra_line:
+ page.draw_line(_pm.Point(72, 160), _pm.Point(400, 160)) # underline
+ return doc
+
+
+def test_signature_curves_near_firma_are_suggested_only():
+ from pdf_sanitize import suggest_signature_areas
+
+ doc = _signature_doc(extra_line=True)
+ sugg = suggest_signature_areas(doc[0])
+ assert len(sugg) == 1
+ s = sugg[0]
+ assert s["state"] == "suggested" and s["entity_type"] == "SIGNATURE" and not s["text"]
+ x0, y0, x1, y1 = s["rects"][0]
+ assert x0 < 112 and x1 > 195 and y0 < 100 and y1 > 155
+
+
+def test_table_lines_and_unanchored_ink_are_not_signatures():
+ import pymupdf as _pm
+ from pdf_sanitize import suggest_signature_areas
+
+ doc = _pm.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Firma:", fontsize=11)
+ for k in range(8):
+ page.draw_line(_pm.Point(72, 120 + k * 10), _pm.Point(472, 120 + k * 10))
+ assert suggest_signature_areas(page) == []
+ other = _pm.open()
+ pg = other.new_page()
+ pg.insert_text((72, 100), "Relazione tecnica", fontsize=11)
+ sh = pg.new_shape()
+ for k in range(6):
+ x = 120 + k * 12
+ sh.draw_bezier(_pm.Point(x, 130), _pm.Point(x + 3, 105), _pm.Point(x + 7, 150), _pm.Point(x + 12, 128))
+ sh.finish(color=(0, 0, 0), fill=None, closePath=False)
+ sh.commit()
+ assert suggest_signature_areas(pg) == []
+
+
+# --- ISTAT place data ------------------------------------------------------------
+
+def test_generated_module_is_lazy_and_carries_sources_and_licence():
+ import subprocess
+ import sys
+
+ code = (
+ "import sys; sys.path.insert(0, %r); import lexicon_it; "
+ "assert 'lexicon_data_it' not in sys.modules; "
+ "assert lexicon_it.is_known_place('Zungri'); "
+ "assert 'lexicon_data_it' in sys.modules"
+ )
+ root = __import__("pathlib").Path(__file__).resolve().parent.parent
+ code = code % str(root)
+ r = subprocess.run([sys.executable, "-I", "-c", code], cwd=str(root),
+ capture_output=True, text=True)
+ assert r.returncode == 0, r.stderr
+ head = (root / "lexicon_data_it.py").read_text(encoding="utf-8")[:600]
+ assert "CC BY 4.0" in head and "istat_comuni" in head
+ assert "CC BY 4.0" in (root / "THIRD_PARTY_NOTICES.md").read_text(encoding="utf-8")
+
+
+def test_istat_place_confirms_but_absence_never_rejects():
+ assert lexicon_it.is_known_place("Zungri") and lexicon_it.is_known_place("Reggio nell'Emilia")
+ text = "Il Sig. Mario Rossi abita a Zzyzxville"
+ r = res("LOCATION", text, "Zzyzxville")
+ _i, cands, _ = _run(text, [r])
+ assert cands == [] or all(c.state != "dropped" for c in cands) # scored as before, not rejected for absence
+
+
+def test_ambiguous_comune_gets_only_a_small_boost():
+ assert lexicon_it.is_ambiguous_place("Sale") and not lexicon_it.is_known_place("Sale")
+ text = "Sale"
+ r = res("LOCATION", text, "Sale")
+ _i, cands, _ = _run(text, [r])
+ assert all(c.state != "accepted" for c in cands)
+
+
+def test_province_codes_come_from_istat():
+ assert {"MI", "NA", "SU", "BT"} <= lexicon_it.province_codes()
+
+
+def test_known_surname_helps_a_two_word_name_but_never_rejects():
+ assert lexicon_it.is_known_surname("Marchetti")
+ text = "Il referto di Gianpiero Marchetti"
+ _i, cands, _ = _run(text, [res("PERSON", text, "Gianpiero Marchetti")])
+ assert cands and "known-surname" in cands[0].reasons
+ text2 = "Il referto di Gianpiero Qwertyuiop"
+ _i, cands2, _ = _run(text2, [res("PERSON", text2, "Gianpiero Qwertyuiop")])
+ assert cands2 and cands2[0].state in ("accepted", "suggested") # absence is not a reason to reject
diff --git a/tests/test_export_sanitize.py b/tests/test_export_sanitize.py
new file mode 100644
index 0000000..b67230a
--- /dev/null
+++ b/tests/test_export_sanitize.py
@@ -0,0 +1,385 @@
+"""Export pipeline: residue removal, verification, temp files, audit, names."""
+
+import json
+import os
+
+import pymupdf
+import pytest
+
+import pii_detection as pd
+from export_engine import (
+ build_audit_record,
+ cleanup_stale_temp,
+ collect_removed_values,
+ export_document,
+ name_leaks_value,
+ neutral_map_name,
+ neutral_output_name,
+ private_temp_dir,
+ purge_old_logs,
+ wipe_local_data,
+ write_audit_record,
+ write_private_file,
+)
+from pdf_sanitize import (
+ SanitizePolicy,
+ reveal_hidden_layers,
+ sanitize_document,
+ verify_export,
+)
+
+SECRET = "Mario Rossi"
+
+
+def _loaded_pdf(path):
+ """A PDF carrying personal data in every place a plain redaction misses."""
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Paziente: Mario Rossi nato a Napoli", fontsize=11)
+ page.insert_text((72, 140), "Referto generico senza dati", fontsize=11)
+ note = page.add_text_annot((300, 300), "Nota per Mario Rossi")
+ note.update()
+ doc.embfile_add("allegato.txt", b"Mario Rossi codice 123", filename="allegato.txt")
+ doc.set_toc([[1, "Cartella di Mario Rossi", 1]])
+ page.insert_link({
+ "kind": pymupdf.LINK_URI, "from": pymupdf.Rect(72, 300, 200, 320),
+ "uri": "mailto:mario.rossi@example.com",
+ })
+ cat = doc.pdf_catalog()
+ doc.xref_set_key(
+ cat, "OpenAction", "<>"
+ )
+ # invisible text (render mode 3)
+ xref = page.get_contents()[0]
+ doc.update_stream(
+ xref,
+ doc.xref_stream(xref)
+ + b"\nq BT /helv 11 Tf 3 Tr 72 400 Td (INVISIBLETEXT) Tj ET Q\n",
+ )
+ # hidden layer
+ ocg = doc.add_ocg("Nascosto", on=False)
+ page.insert_text((72, 450), "HIDDENLAYER", fontsize=11, oc=ocg)
+ doc.set_metadata({"title": "Cartella Mario Rossi", "author": "Mario Rossi"})
+ doc.save(path)
+ doc.close()
+
+
+def _redact_words(doc, words):
+ for page in doc:
+ idx = pd.build_page_index(page.get_text("words"))
+ for w in words:
+ for r in pd.whole_word_rects(idx, w):
+ page.add_redact_annot(pymupdf.Rect(*r))
+
+
+def test_export_removes_everything_that_survives_plain_redaction(tmp_path):
+ src = tmp_path / "src.pdf"
+ out = tmp_path / "out.pdf"
+ _loaded_pdf(str(src))
+ doc = pymupdf.open(str(src))
+ assert reveal_hidden_layers(doc) == 1
+ _redact_words(doc, ["Mario", "Rossi", "HIDDENLAYER"])
+ removed = [SECRET, "INVISIBLETEXT", "HIDDENLAYER", "allegato", "mario.rossi@example.com"]
+ result = export_document(doc, str(out), removed_values=removed)
+ doc.close()
+ assert result.verification.ok, result.verification.findings
+
+ exported = pymupdf.open(str(out))
+ page = exported[0]
+ text = page.get_text()
+ assert "Mario" not in text and "Rossi" not in text
+ assert "INVISIBLETEXT" not in text and "HIDDENLAYER" not in text
+ assert "Referto generico senza dati" in text # collateral damage check
+ assert not list(page.annots() or [])
+ assert exported.embfile_names() == []
+ assert exported.get_toc() == []
+ assert not any(l.get("kind") == pymupdf.LINK_URI for l in page.get_links())
+ cat = exported.pdf_catalog()
+ for key in ("OpenAction", "AA", "Names", "StructTreeRoot", "Metadata"):
+ assert exported.xref_get_key(cat, key)[0] == "null"
+ assert not any(v for v in (exported.metadata or {}).values() if v and "Rossi" in str(v))
+ exported.close()
+ raw = out.read_bytes()
+ assert b"Mario Rossi" not in raw and b"INVISIBLETEXT" not in raw
+
+
+def test_no_unredacted_copy_is_left_beside_the_output(tmp_path):
+ src = tmp_path / "src.pdf"
+ out_dir = tmp_path / "outdir"
+ out_dir.mkdir()
+ _loaded_pdf(str(src))
+ doc = pymupdf.open(str(src))
+ _redact_words(doc, ["Mario", "Rossi"])
+ export_document(doc, str(out_dir / "o.pdf"), removed_values=[SECRET])
+ doc.close()
+ assert [p.name for p in out_dir.iterdir()] == ["o.pdf"]
+
+
+def test_large_document_uses_private_temp_and_cleans_up(tmp_path, monkeypatch):
+ import export_engine
+
+ monkeypatch.setattr(export_engine, "IN_MEMORY_LIMIT_BYTES", 0)
+ src = tmp_path / "src.pdf"
+ _loaded_pdf(str(src))
+ doc = pymupdf.open(str(src))
+ _redact_words(doc, ["Mario", "Rossi"])
+ tmp_dir = private_temp_dir(tmp_path / "cfg")
+ export_document(doc, str(tmp_path / "o.pdf"), removed_values=[SECRET], temp_dir=tmp_dir)
+ doc.close()
+ assert list(tmp_dir.iterdir()) == []
+
+
+def test_verify_export_flags_injected_leaks(tmp_path):
+ leaky = tmp_path / "leaky.pdf"
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "contiene Mario Rossi in chiaro", fontsize=11)
+ page.add_text_annot((200, 200), "anche qui Mario Rossi")
+ doc.embfile_add("a.txt", b"x")
+ doc.save(str(leaky))
+ doc.close()
+ rep = verify_export(str(leaky), [SECRET])
+ assert not rep.ok
+ joined = " ".join(rep.findings)
+ assert "text layer" in joined and "annotations" in joined and "embedded" in joined
+ assert SECRET not in joined # findings never repeat the value
+
+
+def test_verify_export_passes_a_clean_file(tmp_path):
+ clean = tmp_path / "clean.pdf"
+ doc = pymupdf.open()
+ doc.new_page().insert_text((72, 100), "nessun dato", fontsize=11)
+ doc.save(str(clean))
+ doc.close()
+ assert verify_export(str(clean), [SECRET, "ab"]).ok
+
+
+def test_scanned_page_keeps_its_invisible_ocr_layer():
+ doc = pymupdf.open()
+ page = doc.new_page()
+ pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 100, 100), False)
+ pix.set_rect(pix.irect, (255, 255, 255))
+ page.insert_image(page.rect, pixmap=pix)
+ xref = page.get_contents()[0]
+ doc.update_stream(
+ xref, doc.xref_stream(xref) + b"\nBT /helv 11 Tf 3 Tr 72 400 Td (OCRTEXT) Tj ET\n"
+ )
+ rep = sanitize_document(doc, SanitizePolicy())
+ assert "hidden-text-spans" not in rep.actions
+ assert "OCRTEXT" in doc[0].get_text()
+
+
+def test_offpage_text_is_cleared_and_cropbox_expanded():
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "visibile", fontsize=11)
+ page.insert_text((72, 700), "FUORIPAGINA", fontsize=11)
+ page.set_cropbox(pymupdf.Rect(0, 0, 595, 400))
+ rep = sanitize_document(doc)
+ assert "offpage-area-cleared" in rep.actions
+ assert doc[0].cropbox == doc[0].mediabox
+ assert "FUORIPAGINA" not in doc[0].get_text()
+ assert "visibile" in doc[0].get_text()
+
+
+def test_tj_gap_quantization_hides_removed_width(tmp_path):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Paziente: Mario Rossi nato a Napoli", fontsize=11)
+ idx = pd.build_page_index(page.get_text("words"))
+ for w in ("Mario", "Rossi"):
+ for r in pd.whole_word_rects(idx, w):
+ page.add_redact_annot(pymupdf.Rect(*r))
+ page.apply_redactions()
+
+ def gaps():
+ found = []
+ for xr in page.get_contents():
+ data = doc.xref_stream(xr).decode("latin-1")
+ import re
+ for m in re.finditer(r"\)\s*(-?\d+(?:\.\d+)?)\s*\(", data):
+ found.append(float(m.group(1)))
+ return [g for g in found if abs(g) >= 1500]
+
+ before = gaps()
+ assert before, "expected a redaction gap in the content stream"
+ sanitize_document(doc, SanitizePolicy(mask_removed_width=True))
+ after = gaps()
+ assert after and all(abs(round(g)) % 500 == 0 for g in after)
+ assert before != after
+
+
+def test_collect_removed_values_reads_payload_and_text():
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Mario Rossi", fontsize=11)
+ page.add_redact_annot(pymupdf.Rect(60, 85, 160, 110))
+ vals = collect_removed_values(doc, lambda a: {"original": "Mario Rossi"})
+ assert "Mario Rossi" in vals and "Mario" not in vals
+
+
+def test_neutral_names_and_leak_detection():
+ assert "Rossi" not in neutral_output_name("pseudonymize")
+ assert neutral_map_name().endswith(".nullifypdf-map")
+ assert name_leaks_value("C:/x/Referto_Rossi_Mario.pdf", ["Mario Rossi", "Rossi Mario"]) is False or True
+ assert name_leaks_value("C:/x/Referto_MarioRossi.pdf", ["Mario Rossi"])
+ assert not name_leaks_value("C:/x/document_20260101.pdf", ["Mario Rossi"])
+
+
+def test_audit_record_has_no_values_and_is_private(tmp_path):
+ doc = pymupdf.open()
+ doc.new_page()
+ from pdf_sanitize import SanitizeReport, VerificationReport
+
+ rec = build_audit_record(
+ app_version="x", mode="anonymize", entity_counts={"PERSON": 3},
+ suggestion_counts={"unreviewed": 0}, sanitize=SanitizeReport({"a": 1}),
+ verification=VerificationReport(), output_sha256="abc", pages=1,
+ )
+ path = write_audit_record(tmp_path, rec)
+ data = json.loads(open(path, encoding="utf-8").read())
+ assert data["entity_counts"] == {"PERSON": 3} and data["verification"]["ok"]
+ assert "Rossi" not in json.dumps(data)
+
+
+def test_wipe_stale_temp_and_log_purge(tmp_path):
+ cfg = tmp_path / "cfg"
+ (cfg / "logs").mkdir(parents=True)
+ (cfg / "logs" / "nullifypdf.log").write_text("x")
+ (cfg / "audit").mkdir()
+ (cfg / "tmp").mkdir()
+ (cfg / "tmp" / "leftover.tmp").write_text("x")
+ (cfg / "blocklist.txt").write_text("rossi")
+ assert cleanup_stale_temp(cfg) == 1
+ removed = wipe_local_data(cfg)
+ assert {"logs", "audit"} <= set(removed)
+ assert (cfg / "blocklist.txt").exists()
+ wipe_local_data(cfg, include_lists=True)
+ assert not (cfg / "blocklist.txt").exists()
+ old = tmp_path / "logs"
+ old.mkdir()
+ f = old / "old.log"
+ f.write_text("x")
+ os.utime(f, (1, 1))
+ assert purge_old_logs(old, 30) == 1
+
+
+def test_write_private_file_roundtrip(tmp_path):
+ p = tmp_path / "m.bin"
+ write_private_file(str(p), b"data")
+ assert p.read_bytes() == b"data"
+
+
+# --- verification must not cry wolf on ordinary documents --------------------
+
+def _export_with_rects(doc, rect_words_first_only, tmp_path, values=None):
+ page = doc[0]
+ idx = pd.build_page_index(page.get_text("words"))
+ for w in rect_words_first_only:
+ page.add_redact_annot(pymupdf.Rect(*pd.whole_word_rects(idx, w)[0]))
+ out = tmp_path / "o.pdf"
+ if values is None:
+ values = collect_removed_values(doc)
+ return export_document(doc, str(out), removed_values=values)
+
+
+def test_redacting_rossi_is_not_flagged_because_rossini_remains(tmp_path):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Il Sig. Rossi vive a Roma", fontsize=11)
+ page.insert_text((72, 130), "Il collega Rossini abita a Romagna", fontsize=11)
+ result = _export_with_rects(doc, ["Rossi", "Roma"], tmp_path,
+ values=["Rossi", "Roma"])
+ assert result.verification.ok, result.verification.findings
+
+
+def test_drawn_rectangle_does_not_make_common_words_needles(tmp_path):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "abita in Via Roma 10 con la famiglia", fontsize=11)
+ page.insert_text((72, 130), "via di fuga e nato con la camicia", fontsize=11)
+ words = page.get_text("words")
+ box = pymupdf.Rect(
+ min(w[0] for w in words if w[4] in ("Via", "Roma", "10") and w[1] < 110),
+ words[0][1],
+ max(w[2] for w in words if w[4] == "10"),
+ words[0][3],
+ )
+ page.add_redact_annot(box)
+ values = collect_removed_values(doc)
+ assert all(len(v) >= 4 for v in values) and "via" not in [v.lower() for v in values]
+ out = tmp_path / "o.pdf"
+ result = export_document(doc, str(out), removed_values=values)
+ assert result.verification.ok, result.verification.findings
+ exported = pymupdf.open(str(out))
+ assert "Roma" not in exported[0].get_text() and "via di fuga" in exported[0].get_text()
+ exported.close()
+
+
+def test_deliberately_unredacted_occurrence_is_not_a_failure(tmp_path):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Mario Rossi (primo)", fontsize=11)
+ page.insert_text((72, 140), "Mario Rossi (secondo, lasciato visibile)", fontsize=11)
+ idx = pd.build_page_index(page.get_text("words"))
+ for w in ("Mario", "Rossi"):
+ page.add_redact_annot(pymupdf.Rect(*pd.whole_word_rects(idx, w)[0]))
+ out = tmp_path / "o.pdf"
+ result = export_document(doc, str(out), removed_values=["Mario Rossi"])
+ assert result.verification.ok, result.verification.findings
+
+
+def test_a_real_leak_is_still_caught(tmp_path):
+ out = tmp_path / "o.pdf"
+ doc = pymupdf.open()
+ doc.new_page().insert_text((72, 100), "Mario Rossi", fontsize=11)
+ doc.save(str(out))
+ doc.close()
+ # Nothing marks "Mario Rossi" as intentionally kept: it is a leak.
+ assert not verify_export(str(out), ["Mario Rossi"]).ok
+ # The same text counts as deliberately kept when it sits outside redactions.
+ assert verify_export(str(out), ["Mario Rossi"], "Mario Rossi").ok
+
+
+def test_font_programs_and_images_are_not_searched(tmp_path):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "testo qualunque", fontsize=11, fontname="Times-Roman")
+ out = tmp_path / "o.pdf"
+ doc.save(str(out))
+ # a 4-letter needle that may appear by chance in binary font data
+ assert verify_export(str(out), ["qwrz"]).ok
+
+
+def test_wipe_reports_only_what_is_really_gone(tmp_path):
+ import logging
+ import logging.handlers
+ import sys
+
+ cfg = tmp_path / "cfg"
+ (cfg / "logs").mkdir(parents=True)
+ handler = logging.handlers.RotatingFileHandler(str(cfg / "logs" / "x.log"), encoding="utf-8")
+ log = logging.getLogger("wipe-test")
+ log.addHandler(handler)
+ log.error("line")
+ try:
+ removed = wipe_local_data(cfg)
+ if sys.platform == "win32":
+ assert "logs" not in removed # still open: honest report
+ finally:
+ handler.close()
+ log.removeHandler(handler)
+ assert "logs" in wipe_local_data(cfg) or not (cfg / "logs").exists()
+
+
+def test_text_outside_redactions_uses_the_deletion_core():
+ from export_engine import text_outside_redactions
+
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Alessandro resta", fontsize=11)
+ x0, y0, x1, y1 = page.get_text("words")[0][:4]
+ # covers only the left quarter of the word: its glyphs are still deleted
+ page.add_redact_annot(pymupdf.Rect(x0 - 2, y0, x0 + (x1 - x0) * 0.25, y1))
+ kept = text_outside_redactions(doc)
+ assert "Alessandro" not in kept and "resta" in kept
diff --git a/tests/test_geometry.py b/tests/test_geometry.py
new file mode 100644
index 0000000..5a99fbe
--- /dev/null
+++ b/tests/test_geometry.py
@@ -0,0 +1,240 @@
+"""Redaction geometry: no neighbour damage, no leak, correct partial policy."""
+
+import pymupdf
+import pytest
+
+import pii_detection as pd
+from _pdfgen import fake_words, make_doc, res
+from export_engine import export_document
+
+LINES = ["Commissione tributaria", "Mario Rossi gggg yyyy", "Programmazione generale"]
+
+
+def _redact_and_export(doc, targets, tmp_path, clamp=True):
+ page = doc[0]
+ idx = pd.build_page_index(page.get_text("words"), clamp=clamp)
+ for t in targets:
+ for r in pd.whole_word_rects(idx, t):
+ page.add_redact_annot(pymupdf.Rect(*r))
+ out = tmp_path / "out.pdf"
+ result = export_document(doc, str(out), removed_values=targets)
+ exported = pymupdf.open(str(out))
+ text = exported[0].get_text()
+ exported.close()
+ return result, text
+
+
+@pytest.mark.parametrize("pitch", [7, 8, 9, 10, 11, 12, 13.2, 16])
+def test_clamped_rects_keep_neighbour_lines_intact(pitch, tmp_path):
+ doc = make_doc(LINES, pitch=pitch)
+ result, text = _redact_and_export(doc, ["Mario", "Rossi", "gggg", "yyyy"], tmp_path)
+ assert "Commissione tributaria" in text
+ assert "Programmazione generale" in text
+ for secret in ("Mario", "Rossi", "gggg", "yyyy"):
+ assert secret not in text
+ assert result.verification.ok, result.verification.findings
+
+
+def test_unclamped_boxes_damage_tight_neighbours(tmp_path):
+ """Documents the defect the clamp fixes (11 pt text, 11 pt pitch)."""
+ doc = make_doc(LINES, pitch=11)
+ _r, text = _redact_and_export(doc, ["Mario", "Rossi", "gggg", "yyyy"], tmp_path, clamp=False)
+ assert not ("Commissione tributaria" in text and "Programmazione generale" in text)
+
+
+def test_clamp_only_trims_against_other_lines():
+ words = [
+ (0, 0, 40, 15, "Alpha", 0, 0, 0),
+ (45, 0, 60, 15, "Beta", 0, 0, 1),
+ (0, 11, 40, 26, "Gamma", 0, 1, 0),
+ ]
+ idx = pd.build_page_index(words)
+ alpha, beta, gamma = idx.words
+ assert alpha.rect[3] == pytest.approx(11 + 0.1 * 15 - 0.3) # stops short of the neighbour core
+ assert beta.rect == (45, 0, 60, 15) # no horizontal overlap: untouched
+ assert gamma.rect[1] == pytest.approx(15 - 0.1 * 15 + 0.3)
+ assert alpha.rect[0] == 0 and alpha.rect[2] == 40
+
+
+def test_clamp_keeps_original_when_text_collides_heavily():
+ words = [
+ (0, 0, 40, 15, "Alpha", 0, 0, 0),
+ (0, 2, 40, 17, "Beta", 0, 1, 0),
+ ]
+ idx = pd.build_page_index(words)
+ assert idx.words[0].rect == (0, 0, 40, 15)
+ assert idx.words[1].rect == (0, 2, 40, 17)
+
+
+def test_clamp_ocr_mode_cuts_at_middle_of_overlap_and_pads():
+ words = [
+ (10, 0, 50, 15, "Alpha", 0, 0, 0),
+ (10, 11, 50, 26, "Gamma", 0, 1, 0),
+ ]
+ idx = pd.build_page_index(words, ocr=True)
+ assert idx.words[0].rect[3] == pytest.approx(13.0)
+ assert idx.words[0].rect[0] == pytest.approx(9.0)
+ assert idx.words[0].rect[2] == pytest.approx(51.0)
+
+
+def test_mixed_sizes_on_one_line_keep_full_height():
+ words = [
+ (0, 0, 40, 15, "Big", 0, 0, 0),
+ (42, 0, 50, 8, "2", 0, 0, 1),
+ (0, 40, 40, 55, "Below", 0, 1, 0),
+ ]
+ idx = pd.build_page_index(words)
+ assert [w.rect for w in idx.words] == [(0, 0, 40, 15), (42, 0, 50, 8), (0, 40, 40, 55)]
+
+
+def test_vertical_text_is_not_clamped():
+ words = [
+ (0, 0, 8, 60, "Verticale", 0, 0, 0),
+ (0, 55, 8, 115, "Altra", 0, 1, 0),
+ ]
+ idx = pd.build_page_index(words)
+ assert idx.words[0].rect == (0, 0, 8, 60)
+
+
+def test_letter_spaced_run_becomes_one_analysis_word():
+ letters = "Commissione"
+ words = [
+ (10 + 9 * i, 0, 16 + 9 * i, 12, ch, 0, 0, i) for i, ch in enumerate(letters)
+ ]
+ idx = pd.build_page_index(words)
+ assert idx.text == "Commissione"
+ assert len(idx.words) == len(letters)
+
+
+def test_hyphenated_name_is_joined_across_lines():
+ words = [
+ (10, 0, 40, 12, "Ros-", 0, 0, 0),
+ (10, 14, 50, 26, "sini", 0, 1, 0),
+ ]
+ idx = pd.build_page_index(words, clamp=False)
+ assert idx.text == "Rossini"
+ rects = pd.rects_for_words(idx, pd.span_word_indices(idx, 0, 7))
+ assert len(rects) == 2 # one rectangle per line
+
+
+def test_ligatures_are_expanded_for_analysis():
+ idx = pd.build_page_index([(0, 0, 50, 12, "Ufficio", 0, 0, 0)])
+ assert idx.text == "Ufficio"
+
+
+def test_partial_policy_ner_rejects_fragment_of_longer_word():
+ text = "Il collega Rossini abita qui"
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ c = pd.Candidate(text.index("Ross"), text.index("Ross") + 4, "Ross", "PERSON", 0.85)
+ assert pd.resolve_word_indices(idx, c) == ([], "embedded-in-word")
+
+
+def test_partial_policy_validated_identifier_glued_to_label_is_redacted():
+ text = "CF:RSSMRA80A01H501U fine"
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ start = text.index("RSSMRA")
+ c = pd.Candidate(start, start + 16, text[start:start + 16], "IT_FISCAL_CODE", 1.0)
+ idxs, status = pd.resolve_word_indices(idx, c)
+ assert idxs and status in ("ok", "glued")
+
+
+def test_partial_policy_short_overlap_of_validated_type_redacts_whole_token():
+ text = "IBANIT60X0542811101000000123456 fine"
+ idx = pd.build_page_index(fake_words(text), clamp=False)
+ start = text.index("IT60")
+ c = pd.Candidate(start, start + 5, text[start:start + 5], "IBAN_CODE", 1.0)
+ idxs, status = pd.resolve_word_indices(idx, c)
+ assert idxs and status == "glued"
+
+
+def test_merge_line_rects_and_covered_fraction():
+ merged = pd.merge_line_rects([(0, 0, 10, 10), (10.5, 0, 20, 10), (0, 30, 10, 40)])
+ assert merged == [(0, 0, 20, 10), (0, 30, 10, 40)]
+ assert pd.covered_fraction((0, 0, 10, 10), [(0, 0, 10, 5)]) == pytest.approx(0.5)
+ assert pd.covered_fraction((0, 0, 10, 10), [(20, 20, 30, 30)]) == 0.0
+
+
+def test_fake_result_helper_roundtrip():
+ text = "Mario Rossi"
+ r = res("PERSON", text, "Rossi")
+ assert text[r.start:r.end] == "Rossi"
+
+
+# --- the deletion rule the core-band clamp relies on (canary for new releases) ----
+
+@pytest.mark.parametrize("fontname,size", [("helv", 11), ("helv", 20), ("tiro", 11), ("cour", 11)])
+def test_glyph_deletion_core_is_ten_percent_inset(fontname, size):
+ """MuPDF deletes a glyph only when the rectangle reaches its core box.
+
+ If a PyMuPDF release changes this, the core-band clamp is unsafe: this
+ test (and the CI job against the latest PyMuPDF) must fail first.
+ """
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((100, 300), "H", fontsize=size, fontname=fontname)
+ ch = page.get_text("rawdict")["blocks"][0]["lines"][0]["spans"][0]["chars"][0]
+ x0, y0, x1, y1 = ch["bbox"]
+ h = y1 - y0
+ top_core = y0 + 0.10 * h
+ bottom_core = y1 - 0.10 * h
+
+ def deleted(band):
+ d2 = pymupdf.open("pdf", doc.tobytes())
+ p2 = d2[0]
+ p2.add_redact_annot(pymupdf.Rect(*band))
+ p2.apply_redactions()
+ return "H" not in p2.get_text()
+
+ eps = 0.15
+ # just short of the core from above / below: kept; just inside: deleted
+ assert not deleted((x0 - 5, y0 - 5, x1 + 5, top_core - eps))
+ assert deleted((x0 - 5, y0 - 5, x1 + 5, top_core + eps))
+ assert not deleted((x0 - 5, bottom_core + eps, x1 + 5, y1 + 5))
+ assert deleted((x0 - 5, bottom_core - eps, x1 + 5, y1 + 5))
+
+
+def test_core_band_mixed_sizes_spares_neighbours(tmp_path):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Intestazione piccola", fontsize=9)
+ page.insert_text((72, 108), "Paziente Alessandro Marchetti", fontsize=11)
+ page.insert_text((72, 118), "Nota finale minuscola", fontsize=8)
+ result, text = _redact_and_export(doc, ["Alessandro", "Marchetti"], tmp_path)
+ assert "Intestazione piccola" in text and "Nota finale minuscola" in text
+ assert "Alessandro" not in text and "Marchetti" not in text
+
+
+def test_no_safe_band_flags_collision_and_still_removes_target(tmp_path):
+ doc = make_doc(LINES, pitch=5)
+ page = doc[0]
+ idx = pd.build_page_index(page.get_text("words"))
+ assert idx.collisions > 0
+ result, text = _redact_and_export(doc, ["Mario", "Rossi"], tmp_path)
+ assert "Mario" not in text and "Rossi" not in text
+
+
+def test_words_over_an_image_never_get_a_thin_band():
+ words = [
+ (10, 0, 50, 15, "Alpha", 0, 0, 0),
+ (10, 11, 50, 26, "Gamma", 0, 1, 0),
+ ]
+ native = pd.build_page_index(words)
+ over_image = pd.build_page_index(words, image_rects=[(0, 0, 100, 40)])
+ assert native.words[0].native and not over_image.words[0].native
+ assert over_image.words[0].rect[3] == pytest.approx(13.0) # mid-overlap
+ assert over_image.words[0].rect[0] == pytest.approx(9.0) # horizontal pad
+
+
+@pytest.mark.parametrize("pitch", [9, 10, 12])
+def test_style_capture_ignores_neighbour_lines(pitch):
+ from NullifyPDF import capture_text_style
+
+ doc = make_doc(
+ ["Commissione tributaria provinciale", "Paziente Alessandro nato", "Programmazione generale dei servizi"],
+ pitch=pitch,
+ )
+ page = doc[0]
+ idx = pd.build_page_index(page.get_text("words"), clamp=False)
+ for rect in pd.whole_word_rects(idx, "Alessandro"): # raw, unclamped box
+ style = capture_text_style(page, pymupdf.Rect(*rect))
+ assert style is not None and style["text"] == "Alessandro"
diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py
new file mode 100644
index 0000000..d1fe2de
--- /dev/null
+++ b/tests/test_gui_flow.py
@@ -0,0 +1,847 @@
+"""Offscreen functional tests of the review / export / reconstruct flows.
+
+mypy and unit tests do not exercise Qt slots and dialogs, so these drive the
+real window with Qt's offscreen platform and stubbed modal dialogs. HOME is
+isolated so the user's real ~/.nullifypdf is never touched.
+"""
+
+import atexit
+import gc
+import json
+import os
+
+import pymupdf
+import pytest
+
+pytest.importorskip("PySide6")
+os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
+
+from PySide6.QtWidgets import ( # noqa: E402
+ QApplication, QDialog, QFileDialog, QInputDialog, QMessageBox,
+)
+
+import NullifyPDF as app # noqa: E402
+from privacy_core import ( # noqa: E402
+ PrivacyMode, decrypt_restore_payload,
+)
+from review_dialog import ReviewDialog, mask_value # noqa: E402
+
+
+@pytest.fixture()
+def window(tmp_path, monkeypatch):
+ home = tmp_path / "home"
+ home.mkdir()
+ monkeypatch.setenv("USERPROFILE", str(home))
+ monkeypatch.setenv("HOME", str(home))
+ qapp = QApplication.instance() or QApplication([])
+ w = app.NullifyPDF()
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Paziente: Alessandro Marchetti nato a Napoli", fontsize=11)
+ page.insert_text((72, 130), "Zorzi era presente alla visita", fontsize=11)
+ doc.save(str(tmp_path / "Referto_Mario_Rossi.pdf"))
+ doc.close()
+ w.doc = pymupdf.open(str(tmp_path / "Referto_Mario_Rossi.pdf"))
+ yield w
+ try:
+ w._cleanup_resources()
+ except Exception:
+ pass
+ # Every window registers an atexit cleanup; left registered, they run on
+ # already destroyed Qt objects at interpreter exit and crash the process
+ # (seen as a segmentation fault after the last test on Linux).
+ atexit.unregister(w._cleanup_on_exit)
+ w.deleteLater()
+ qapp.processEvents()
+ gc.collect()
+
+
+def _rect_of(w, word):
+ page = w.doc[0]
+ idx = app.build_page_index(page.get_text("words"))
+ return list(app.whole_word_rects(idx, word)[0])
+
+
+def _detection(w, word, etype, state, score=0.9):
+ return {"text": word, "entity_type": etype, "rects": [_rect_of(w, word)],
+ "state": state, "score": score, "reasons": ["context-cue"]}
+
+
+def _redacts(w):
+ return [a for a in w.doc[0].annots() if a.type[0] == app.PDF_ANNOT_REDACT]
+
+
+def test_accepted_applies_and_suggested_waits_for_review(window):
+ w = window
+ w.apply_ai_to_page(0, [
+ _detection(w, "Alessandro", "PERSON", "accepted"),
+ _detection(w, "Zorzi", "PERSON", "suggested", 0.5),
+ ])
+ assert len(_redacts(w)) == 1
+ assert len(w.pending_suggestions) == 1
+ assert w.btn_review.text() == "Rivedi suggerimenti (1)" and w.btn_review.isEnabled()
+
+
+def test_review_dialog_applies_only_selected(window, monkeypatch):
+ w = window
+ w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)])
+
+ def fake_exec(self):
+ self.table.item(0, 0).setCheckState(app.Qt.CheckState.Checked)
+ self._on_apply()
+ return QDialog.DialogCode.Accepted
+
+ monkeypatch.setattr(ReviewDialog, "exec", fake_exec)
+ w._review_suggestions()
+ assert w.pending_suggestions == []
+ assert len(_redacts(w)) == 1
+
+
+def test_review_dialog_reject_discards_and_later_keeps_pending(window, monkeypatch):
+ w = window
+ w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)])
+ monkeypatch.setattr(
+ ReviewDialog, "exec",
+ lambda self: (self._on_apply(), QDialog.DialogCode.Accepted)[1],
+ )
+ w._review_suggestions()
+ assert w.pending_suggestions == [] and _redacts(w) == []
+ w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)])
+ monkeypatch.setattr(ReviewDialog, "exec", lambda self: QDialog.DialogCode.Rejected)
+ w._review_suggestions()
+ assert len(w.pending_suggestions) == 1
+
+
+def test_review_dialog_masks_values_until_asked(window):
+ dlg = ReviewDialog([{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5,
+ "page": 0, "reasons": ["lexicon"]}])
+ assert dlg.table.item(0, 2).text() == mask_value("Zorzi") != "Zorzi"
+ dlg.chk_show.setChecked(True)
+ assert dlg.table.item(0, 2).text() == "Zorzi"
+ dlg.spin.setValue(0.4)
+ dlg._select_threshold()
+ assert dlg.table.item(0, 0).checkState() == app.Qt.CheckState.Checked
+ got = []
+ dlg.navigate.connect(got.append)
+ dlg._on_double_click(0, 0)
+ assert got == [0]
+
+
+def test_overlapping_detections_are_not_applied_twice(window):
+ w = window
+ w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")])
+ w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")])
+ assert len(_redacts(w)) == 1
+
+
+def test_legacy_detection_without_rects_never_redacts_fragments(window):
+ w = window
+ w.doc.close()
+ d = pymupdf.open()
+ d.new_page().insert_text((72, 100), "Marchettini e Marchetti", fontsize=11)
+ w.doc = d
+ w.apply_ai_to_page(0, {"Ross"})
+ assert _redacts(w) == []
+ w.apply_ai_to_page(0, {"Marchetti"})
+ assert len(_redacts(w)) == 1
+
+
+def _stub_dialogs(monkeypatch, files, answer=QMessageBox.StandardButton.Yes):
+ seq = list(files)
+ monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (seq.pop(0), "")))
+ monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: answer))
+ monkeypatch.setattr(QMessageBox, "information", staticmethod(lambda *a, **k: None))
+ monkeypatch.setattr(QMessageBox, "warning", staticmethod(lambda *a, **k: None))
+ monkeypatch.setattr(QMessageBox, "critical", staticmethod(lambda *a, **k: QMessageBox.StandardButton.Yes))
+
+
+def test_anonymize_export_end_to_end(window, tmp_path, monkeypatch):
+ w = window
+ w.apply_ai_to_page(0, [
+ _detection(w, "Alessandro", "PERSON", "accepted"),
+ _detection(w, "Marchetti", "PERSON", "accepted"),
+ ])
+ out = tmp_path / "exported.pdf"
+ _stub_dialogs(monkeypatch, [str(out)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE)
+ w.cmd_export()
+ assert out.exists()
+ exported = pymupdf.open(str(out))
+ text = exported[0].get_text()
+ exported.close()
+ assert "Alessandro" not in text and "Marchetti" not in text and "Napoli" in text
+ assert sorted(p.name for p in tmp_path.iterdir() if p.suffix == ".tmp") == []
+ audit = list((w.config_dir / "audit").glob("*.json"))
+ assert len(audit) == 1
+ record = json.loads(audit[0].read_text(encoding="utf-8"))
+ assert record["entity_counts"] == {"PERSON": 2} and record["verification"]["ok"]
+ assert "Marchetti" not in audit[0].read_text(encoding="utf-8")
+ assert w.doc is not None and len(_redacts(w)) == 2 # original untouched
+
+
+def test_export_warns_about_unreviewed_suggestions(window, tmp_path, monkeypatch):
+ w = window
+ w.apply_ai_to_page(0, [_detection(w, "Zorzi", "PERSON", "suggested", 0.5)])
+ asked = []
+
+ class Box:
+ class _B: # minimal stand-in for a QMessageBox button
+ pass
+
+ def fake_exec(self):
+ asked.append(self.text())
+
+ monkeypatch.setattr(QMessageBox, "exec", fake_exec)
+ # Clicking nothing = Cancel: export must stop before any file dialog.
+ monkeypatch.setattr(QMessageBox, "clickedButton", lambda self: None)
+ monkeypatch.setattr(
+ QFileDialog, "getSaveFileName",
+ staticmethod(lambda *a, **k: pytest.fail("export continued despite pending suggestions")),
+ )
+ w.cmd_export()
+ assert asked and "non revisionati" in asked[0]
+
+
+def test_pseudonymize_export_with_random_style_roundtrips(window, tmp_path, monkeypatch):
+ w = window
+ w.apply_ai_to_page(0, [
+ _detection(w, "Alessandro", "PERSON", "accepted"),
+ _detection(w, "Marchetti", "PERSON", "accepted"),
+ ])
+ out = tmp_path / "pseudo.pdf"
+ mapf = tmp_path / "maps" / "m.nullifypdf-map"
+ mapf.parent.mkdir()
+ _stub_dialogs(monkeypatch, [str(out), str(mapf)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE)
+ monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None))
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(w, "_offer_recovery_key", lambda: None)
+ w.cmd_export()
+ assert out.exists() and mapf.exists()
+ pdf = pymupdf.open(str(out))
+ text = pdf[0].get_text()
+ pdf.close()
+ assert "Alessandro" not in text and "PERSON_" in text and "PERSON_001" not in text
+ payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026")
+ assert payload["source_name"] == ""
+ assert {e["original"] for e in payload["entries"]} == {"Alessandro", "Marchetti"}
+ restored = tmp_path / "restored.pdf"
+ n = app.reconstruct_pdf(str(out), str(restored), payload)
+ assert n == 2
+ back = pymupdf.open(str(restored))
+ assert "Alessandro" in back[0].get_text()
+ back.close()
+
+
+def test_export_stops_when_verification_fails_and_user_deletes(window, tmp_path, monkeypatch):
+ w = window
+ w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")])
+ out = tmp_path / "bad.pdf"
+ _stub_dialogs(monkeypatch, [str(out)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE)
+ # A value that survives redaction (it is in the visible text): force a
+ # failing verification by claiming "Napoli" was removed.
+ monkeypatch.setattr(app, "collect_removed_values", lambda *a, **k: ["Napoli"])
+ import export_engine
+ monkeypatch.setattr(export_engine, "text_outside_redactions", lambda d: "")
+ w.cmd_export()
+ assert not out.exists()
+ assert not list((w.config_dir / "audit").glob("*.json"))
+
+
+def test_wipe_local_data(window, monkeypatch):
+ w = window
+ (w.config_dir / "audit").mkdir(parents=True, exist_ok=True)
+ (w.config_dir / "audit" / "x.json").write_text("{}")
+ monkeypatch.setattr(
+ QMessageBox, "question",
+ staticmethod(lambda *a, **k: QMessageBox.StandardButton.No),
+ )
+ w.cmd_wipe_local_data()
+ assert not (w.config_dir / "audit").exists()
+
+
+def test_default_language_follows_system_locale(window):
+ assert window.rb_it.isChecked() or window.rb_both.isChecked()
+ assert not window.rb_en.isChecked()
+
+
+def test_worker_scan_end_to_end_with_real_italian_model(tmp_path, monkeypatch):
+ """Drives AIWorker.run_scan itself: wiring of artifacts, classify, build."""
+ spacy = pytest.importorskip("spacy")
+ pytest.importorskip("presidio_analyzer")
+ if not spacy.util.is_package("it_core_news_md"):
+ pytest.skip("model missing")
+ from PySide6.QtCore import QMutex
+
+ monkeypatch.setenv("USERPROFILE", str(tmp_path))
+ monkeypatch.setenv("HOME", str(tmp_path))
+ QApplication.instance() or QApplication([])
+ doc = pymupdf.open()
+ page = doc.new_page()
+ lines = [
+ "Certifico",
+ "che il Sig. Mario Rossi, nato a Napoli il 12/03/1970,",
+ "e' affetto da lombalgia.",
+ "Prescrivo",
+ "riposo assoluto.",
+ "Codice fiscale: RSSMRA70C12F839Y",
+ ]
+ for i, t in enumerate(lines):
+ page.insert_text((72, 100 + 16 * i), t, fontsize=11)
+ worker = app.AIWorker()
+ done, logs = [], []
+ worker.page_done_sig.connect(lambda i, d: done.append((i, d)))
+ worker.log_sig.connect(logs.append)
+ enabled = app.expand_group_ids(app.ALL_GROUP_IDS)
+ worker.run_scan(doc, QMutex(), "IT", [], set(), False, None, enabled, True)
+ assert len(done) == 1, logs
+ dets = done[0][1]
+ texts = {d["text"] for d in dets}
+ assert "Mario Rossi" in texts and "Napoli" in texts and "12/03/1970" in texts
+ assert not texts & {"Certifico", "Prescrivo"}
+ assert all("state" in d and d["rects"] for d in dets)
+ assert any(l.startswith("Scansione completata") for l in logs)
+ worker.cleanup()
+
+
+def test_worker_ocr_of_embedded_images_keeps_native_text(tmp_path, monkeypatch):
+ """A text page with an image: OCR adds the image text, native text stays."""
+ spacy = pytest.importorskip("spacy")
+ pytest.importorskip("presidio_analyzer")
+ if not spacy.util.is_package("it_core_news_md"):
+ pytest.skip("model missing")
+ tessdata = app.find_tessdata_dir()
+ if not tessdata:
+ pytest.skip("tessdata missing")
+ from PySide6.QtCore import QMutex
+
+ monkeypatch.setenv("USERPROFILE", str(tmp_path))
+ monkeypatch.setenv("HOME", str(tmp_path))
+ QApplication.instance() or QApplication([])
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Il Sig. Mario Rossi abita a Napoli.", fontsize=12)
+ src = pymupdf.open()
+ sp = src.new_page(width=300, height=60)
+ sp.insert_text((10, 40), "Dott. Giuseppe Verdi", fontsize=22)
+ page.insert_image(pymupdf.Rect(72, 200, 372, 260), pixmap=sp.get_pixmap(dpi=150))
+ worker = app.AIWorker()
+ done, logs = [], []
+ worker.page_done_sig.connect(lambda i, d: done.append(d))
+ worker.log_sig.connect(logs.append)
+ enabled = app.expand_group_ids(app.ALL_GROUP_IDS)
+ worker.run_scan(doc, QMutex(), "IT", [], set(), True, tessdata, enabled, False)
+ texts = {" ".join(d["text"].split()) for d in done[0]}
+ assert "Mario Rossi" in texts, (texts, logs)
+ assert any("Giuseppe Verdi" in t for t in texts), (texts, logs)
+ assert any("OCR immagini" in l for l in logs)
+ worker.cleanup()
+
+
+def test_roundtrip_with_trimmed_rectangles_on_tight_leading(window, tmp_path, monkeypatch):
+ """11 pt text on a 12 pt pitch: rectangles are trimmed, restore still works."""
+ w = window
+ w.doc.close()
+ d = pymupdf.open()
+ pg = d.new_page()
+ for i, t in enumerate([
+ "Commissione tributaria provinciale",
+ "Paziente Alessandro Marchetti nato oggi",
+ "Programmazione generale dei servizi",
+ ]):
+ pg.insert_text((72, 100 + 12 * i), t, fontsize=11)
+ w.doc = d
+ w.apply_ai_to_page(0, [
+ _detection(w, "Alessandro", "PERSON", "accepted"),
+ _detection(w, "Marchetti", "PERSON", "accepted"),
+ ])
+ out = tmp_path / "t.pdf"
+ mapf = tmp_path / "maps" / "m.nullifypdf-map"
+ mapf.parent.mkdir()
+ _stub_dialogs(monkeypatch, [str(out), str(mapf)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE)
+ monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None))
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(w, "_offer_recovery_key", lambda: None)
+ w.cmd_export()
+ pdf = pymupdf.open(str(out))
+ text = pdf[0].get_text()
+ pdf.close()
+ assert "Commissione tributaria provinciale" in text
+ assert "Programmazione generale dei servizi" in text
+ assert "Alessandro" not in text
+ payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026")
+ restored = tmp_path / "r.pdf"
+ assert app.reconstruct_pdf(str(out), str(restored), payload) == 2
+ back = pymupdf.open(str(restored))
+ t2 = back[0].get_text()
+ back.close()
+ assert "Alessandro" in t2 and "Marchetti" in t2
+ assert "Commissione tributaria provinciale" in t2
+
+
+def test_image_redaction_is_not_duplicated_by_later_applies(window, tmp_path):
+ w = window
+ w.doc.close()
+ d = pymupdf.open()
+ pg = d.new_page()
+ pg.insert_text((72, 100), "Alessandro Marchetti", fontsize=11)
+ pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 40, 40), False)
+ pix.set_rect(pix.irect, (10, 10, 10))
+ pg.insert_image(pymupdf.Rect(300, 300, 340, 340), pixmap=pix)
+ w.doc = d
+ w.chk_img.setChecked(True)
+ w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")])
+ w.apply_ai_to_page(0, [_detection(w, "Marchetti", "PERSON", "accepted")])
+ assert len(_redacts(w)) == 3 # two words + one image, not two images
+
+
+def test_invalid_pseudonym_key_file_is_refused(window, monkeypatch):
+ w = window
+ (w.config_dir).mkdir(parents=True, exist_ok=True)
+ (w.config_dir / "pseudonym.key").write_bytes(b"short")
+ shown = []
+ monkeypatch.setattr(QMessageBox, "exec", lambda self: 0)
+ monkeypatch.setattr(
+ QMessageBox, "clickedButton", lambda self: next(
+ b for b in self.buttons() if b.text() == "Coerente (chiave)"
+ ),
+ )
+ monkeypatch.setattr(
+ QMessageBox, "critical", staticmethod(lambda *a, **k: shown.append(a[2]))
+ )
+ assert w._select_pseudonym_style() is None
+ assert shown and "32 byte" in shown[0]
+
+
+def test_signature_suggestion_is_never_applied_without_review_and_export_removes_ink(window, tmp_path, monkeypatch):
+ from pdf_sanitize import suggest_signature_areas
+
+ w = window
+ w.doc.close()
+ import test_detection_pipeline as tdp
+
+ w.doc = tdp._signature_doc()
+ sug = suggest_signature_areas(w.doc[0])
+ assert len(sug) == 1
+ w.apply_ai_to_page(0, sug)
+ assert _redacts(w) == [] and len(w.pending_suggestions) == 1 # not applied
+ monkeypatch.setattr(
+ ReviewDialog, "exec",
+ lambda self: (self.table.item(0, 0).setCheckState(app.Qt.CheckState.Checked),
+ self._on_apply(), QDialog.DialogCode.Accepted)[2],
+ )
+ w._review_suggestions()
+ assert len(_redacts(w)) == 1
+ out = tmp_path / "sig.pdf"
+ _stub_dialogs(monkeypatch, [str(out)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE)
+ w.cmd_export()
+ exported = pymupdf.open(str(out))
+ region = pymupdf.Rect(*sug[0]["rects"][0])
+ inside = [
+ d for d in exported[0].get_drawings()
+ if region.contains(d["rect"]) and any(it[0] == "c" for it in d["items"])
+ ] # the grey fill rectangle itself is expected
+ exported.close()
+ assert inside == []
+
+
+def test_dictionary_dialog_sets_expiry_and_persists(window, monkeypatch):
+ import datetime
+ from dictionary_dialog import DictionaryDialog
+
+ w = window
+ dlg = DictionaryDialog(w.list_manager)
+ dlg.ttl.setCurrentIndex(1) # 30 giorni
+ dlg.add_term("blocklist", "Mario Rossi")
+ dlg.add_term("allowlist", "comune")
+ dlg.save()
+ assert dlg.saved_blocklist == {"mario rossi"} and dlg.saved_allowlist == {"comune"}
+ entries = w.list_manager.load_entries("blocklist")
+ assert entries[0].expires == datetime.date.today() + datetime.timedelta(days=30)
+ assert not (w.config_dir / "blocklist.txt").exists()
+ assert b"rossi" not in (w.config_dir / "blocklist.v2.enc").read_bytes()
+
+
+def test_dictionary_dialog_rejects_bad_dates_and_keeps_data(window, monkeypatch):
+ from dictionary_dialog import DictionaryDialog
+
+ w = window
+ w.list_manager.save_blocklist({"mario rossi"})
+ dlg = DictionaryDialog(w.list_manager)
+ dlg.tables["blocklist"].item(0, 2).setText("domani")
+ warned = []
+ monkeypatch.setattr(QMessageBox, "warning", staticmethod(lambda *a, **k: warned.append(a[2])))
+ dlg.save()
+ assert warned and dlg.saved_blocklist is None
+ assert w.list_manager.load_blocklist() == {"mario rossi"}
+
+
+def test_cmd_dict_updates_in_memory_lists(window, monkeypatch):
+ import dictionary_dialog as dd
+
+ w = window
+
+ def fake_exec(self):
+ self.add_term("blocklist", "nuovo termine")
+ self.save()
+ return QDialog.DialogCode.Accepted
+
+ monkeypatch.setattr(dd.DictionaryDialog, "exec", fake_exec)
+ w.cmd_dict()
+ assert w.blocklist == {"nuovo termine"}
+
+
+# --- protected pseudonym key, recovery key ------------------------------------------
+
+def test_plaintext_pseudonym_key_is_validated_sealed_and_removed(window):
+ w = window
+ w.config_dir.mkdir(parents=True, exist_ok=True)
+ legacy = w.config_dir / "pseudonym.key"
+ legacy.write_bytes(b"k" * 32)
+ key = w._load_pseudonym_key()
+ assert key == b"k" * 32
+ assert not legacy.exists()
+ enc = (w.config_dir / "pseudonym.v2.enc").read_bytes()
+ assert b"k" * 32 not in enc
+ assert w._load_pseudonym_key() == key
+
+
+def test_new_pseudonym_key_is_created_sealed(window):
+ w = window
+ key = w._load_pseudonym_key()
+ assert len(key) == 32 and w._load_pseudonym_key() == key
+ assert key not in (w.config_dir / "pseudonym.v2.enc").read_bytes()
+ assert not (w.config_dir / "pseudonym.key").exists()
+
+
+def test_pseudonym_key_export_and_import_on_another_profile(window, tmp_path, monkeypatch):
+ w = window
+ key = w._load_pseudonym_key()
+ out = tmp_path / "k.nullifypdf-key"
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(out), "")))
+ monkeypatch.setattr(QMessageBox, "exec", lambda self: 0)
+ monkeypatch.setattr(
+ QMessageBox, "clickedButton",
+ lambda self: next(b for b in self.buttons() if b.text() == "Esporta chiave..."),
+ )
+ w._pseudonym_key_dialog()
+ assert out.exists() and key not in out.read_bytes()
+ # simulate a new machine: wipe the stored key, then import
+ (w.config_dir / "pseudonym.v2.enc").unlink()
+ monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod(lambda *a, **k: (str(out), "")))
+ monkeypatch.setattr(QInputDialog, "getText", staticmethod(lambda *a, **k: ("una frase molto lunga 2026", True)))
+ monkeypatch.setattr(
+ QMessageBox, "clickedButton",
+ lambda self: next(b for b in self.buttons() if b.text() == "Importa chiave..."),
+ )
+ w._pseudonym_key_dialog()
+ assert w._load_pseudonym_key() == key
+
+
+def test_export_with_recovery_key_and_reconstruct_by_recovery(window, tmp_path, monkeypatch):
+ from privacy_core import generate_recovery_key, parse_recovery_key
+
+ w = window
+ w.apply_ai_to_page(0, [
+ _detection(w, "Alessandro", "PERSON", "accepted"),
+ _detection(w, "Marchetti", "PERSON", "accepted"),
+ ])
+ out, mapf = tmp_path / "p.pdf", tmp_path / "maps" / "m.nullifypdf-map"
+ mapf.parent.mkdir()
+ raw, shown = generate_recovery_key()
+ _stub_dialogs(monkeypatch, [str(out), str(mapf)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE)
+ monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None))
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(w, "_offer_recovery_key", lambda: raw)
+ w.cmd_export()
+ blob = mapf.read_bytes()
+ assert raw not in blob and shown.encode() not in blob
+ for f in tmp_path.rglob("*"):
+ if f.is_file():
+ assert shown.encode() not in f.read_bytes()
+ env = json.loads(blob)
+ assert [x["type"] for x in env["wraps"]] == ["password", "recovery"]
+ # reconstruct through the recovery key only
+ restored = tmp_path / "r.pdf"
+ monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod(
+ lambda *a, **k: (str(out), "") if "pseudonimizzato" in a[1] else (str(mapf), "")))
+ monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(restored), "")))
+ monkeypatch.setattr(w, "_ask_decrypt_secret", lambda: (None, parse_recovery_key(shown)))
+ w.cmd_reconstruct()
+ back = pymupdf.open(str(restored))
+ assert "Alessandro" in back[0].get_text()
+ back.close()
+
+
+def test_wrong_secret_gives_one_clear_message(window, tmp_path, monkeypatch):
+ w = window
+ w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")])
+ out, mapf = tmp_path / "p.pdf", tmp_path / "m.nullifypdf-map"
+ _stub_dialogs(monkeypatch, [str(out), str(mapf)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE)
+ monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None))
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(w, "_offer_recovery_key", lambda: None)
+ w.cmd_export()
+ shown = []
+ monkeypatch.setattr(QFileDialog, "getOpenFileName", staticmethod(
+ lambda *a, **k: (str(out), "") if "pseudonimizzato" in a[1] else (str(mapf), "")))
+ monkeypatch.setattr(QFileDialog, "getSaveFileName", staticmethod(lambda *a, **k: (str(tmp_path / "x.pdf"), "")))
+ monkeypatch.setattr(w, "_ask_decrypt_secret", lambda: ("password completamente sbagliata", None))
+ monkeypatch.setattr(QMessageBox, "critical", staticmethod(lambda *a, **k: shown.append(a[2])))
+ w.cmd_reconstruct()
+ assert shown and "recupero" in shown[0]
+
+
+def test_wipe_keys_makes_encrypted_lists_unreadable_only_with_confirmation(window, monkeypatch):
+ w = window
+ w.list_manager.save_blocklist({"mario rossi"})
+ answers = iter([QMessageBox.StandardButton.Yes, QMessageBox.StandardButton.Yes])
+ monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: next(answers)))
+ w.cmd_wipe_local_data()
+ assert not (w.config_dir / "keystore.json").exists()
+ assert not (w.config_dir / "blocklist.v2.enc").exists()
+
+
+# --- preview overlay of pending suggestions (real mouse events) ------------------------
+
+def _click_scene_point(w, scene_pt):
+ from PySide6.QtCore import QEvent, QPointF, Qt
+ from PySide6.QtGui import QMouseEvent
+
+ vp = w.view.mapFromScene(scene_pt)
+ gp = QPointF(vp)
+ glob = QPointF(w.view.viewport().mapToGlobal(vp))
+ for typ, fn, btns in (
+ (QEvent.Type.MouseButtonPress, w.view.mousePressEvent, Qt.MouseButton.LeftButton),
+ (QEvent.Type.MouseButtonRelease, w.view.mouseReleaseEvent, Qt.MouseButton.NoButton),
+ ):
+ fn(QMouseEvent(typ, gp, glob, Qt.MouseButton.LeftButton, btns, Qt.KeyboardModifier.NoModifier))
+ QApplication.processEvents()
+ # the view defers a lone click slightly (to tell it from a double click)
+ import time
+ deadline = time.time() + 1.0
+ while time.time() < deadline:
+ QApplication.processEvents()
+ time.sleep(0.02)
+
+
+def _overlays(w):
+ return [i for i in w.scene.items() if isinstance(i, app.SuggestionItem)]
+
+
+def _pick(monkeypatch, label):
+ monkeypatch.setattr(QMessageBox, "exec", lambda self: 0)
+ monkeypatch.setattr(
+ QMessageBox, "clickedButton",
+ lambda self: next(b for b in self.buttons() if b.text() == label),
+ )
+
+
+def _two_page_doc(w):
+ w.doc.close()
+ d = pymupdf.open()
+ for t in ("Pagina uno Zorzi presente", "Pagina due Colombi presente"):
+ d.new_page().insert_text((72, 100), t, fontsize=11)
+ w.doc = d
+ w.page_num = 0
+ w.render_page()
+
+
+def _sugg(w, page, word):
+ idx = app.build_page_index(w.doc[page].get_text("words"))
+ return {"text": word, "entity_type": "PERSON", "score": 0.5, "page": page,
+ "rects": [list(app.whole_word_rects(idx, word)[0])], "reasons": ["lexicon"],
+ "state": "suggested"}
+
+
+def test_overlay_only_on_the_suggestion_page_and_follows_zoom(window):
+ w = window
+ _two_page_doc(w)
+ w.pending_suggestions = [_sugg(w, 0, "Zorzi"), _sugg(w, 1, "Colombi")]
+ w.render_page()
+ assert len(_overlays(w)) == 1
+ r0 = _overlays(w)[0].rect()
+ w.adjust_zoom(+2)
+ r1 = _overlays(w)[0].rect()
+ assert r1.width() == pytest.approx(r0.width() * w.scale / (w.scale - 0.5), rel=1e-6)
+ w.move_page(1)
+ assert len(_overlays(w)) == 1 and _overlays(w)[0].suggestion["text"] == "Colombi"
+ w.chk_sugg.setChecked(False)
+ assert _overlays(w) == []
+
+
+def test_overlay_click_accept_creates_one_redaction(window, monkeypatch):
+ w = window
+ _two_page_doc(w)
+ w.pending_suggestions = [_sugg(w, 0, "Zorzi")]
+ w._update_review_button()
+ w.render_page()
+ _pick(monkeypatch, "Censura")
+ _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center())
+ assert len(_redacts(w)) == 1 and w.pending_suggestions == []
+ assert w.btn_review.text() == "Rivedi suggerimenti (0)"
+ assert _overlays(w) == []
+
+
+def test_overlay_click_reject_discards_without_annotation(window, monkeypatch):
+ w = window
+ _two_page_doc(w)
+ w.pending_suggestions = [_sugg(w, 0, "Zorzi")]
+ w.render_page()
+ _pick(monkeypatch, "Scarta")
+ _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center())
+ assert _redacts(w) == [] and w.pending_suggestions == []
+
+
+def test_overlay_cancel_keeps_the_suggestion(window, monkeypatch):
+ w = window
+ _two_page_doc(w)
+ w.pending_suggestions = [_sugg(w, 0, "Zorzi")]
+ w.render_page()
+ _pick(monkeypatch, "Annulla")
+ _click_scene_point(w, _overlays(w)[0].sceneBoundingRect().center())
+ assert len(w.pending_suggestions) == 1 and _redacts(w) == []
+
+
+def test_overlay_never_reaches_the_export(window, tmp_path, monkeypatch):
+ w = window
+ _two_page_doc(w)
+ w.pending_suggestions = [_sugg(w, 0, "Zorzi")]
+ w.render_page()
+ assert _redacts(w) == []
+ pix = w.doc[0].get_pixmap()
+ from PySide6.QtWidgets import QMessageBox as QMB
+
+ out = tmp_path / "o.pdf"
+ _stub_dialogs(monkeypatch, [str(out)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE)
+ monkeypatch.setattr(QMB, "exec", lambda self: 0)
+ monkeypatch.setattr(QMB, "clickedButton", lambda self: next(
+ b for b in self.buttons() if b.text() == "Esporta comunque"))
+ w.cmd_export()
+ exported = pymupdf.open(str(out))
+ assert "Zorzi" in exported[0].get_text() # not censored: still only a suggestion
+ exported.close()
+ assert pix.width > 0
+
+
+def test_review_row_selection_highlights_the_overlay(window):
+ w = window
+ _two_page_doc(w)
+ s = _sugg(w, 0, "Zorzi")
+ w.pending_suggestions = [s]
+ w.render_page()
+ dlg = ReviewDialog(w.pending_suggestions)
+ dlg.highlight.connect(w._highlight_suggestion)
+ dlg.table.setCurrentCell(0, 2)
+ assert _overlays(w)[0].pen().width() == 4
+
+
+# --- review findings: clip text, mixed glyph heights, 9 pt round trip, recovery -----------
+
+def _tight_doc(w, pitch=9.0):
+ w.doc.close()
+ d = pymupdf.open()
+ pg = d.new_page()
+ for i, t in enumerate([
+ "Commissione tributaria provinciale",
+ "Paziente Alessandro Marchetti nato oggi",
+ "Programmazione generale dei servizi",
+ ]):
+ pg.insert_text((72, 100 + pitch * i), t, fontsize=11)
+ w.doc = d
+ w.page_num = 0
+ w.render_page()
+
+
+@pytest.mark.parametrize("pitch", [9.0, 10.0, 12.0])
+def test_manual_rectangle_stores_only_its_own_line(window, tmp_path, monkeypatch, pitch):
+ """A box drawn over a word at tight leading must not capture the neighbours."""
+ from PySide6.QtCore import QRectF
+
+ w = window
+ _tight_doc(w, pitch)
+ idx = app.build_page_index(w.doc[0].get_text("words"), clamp=False) # raw word box
+ x0, y0, x1, y1 = app.whole_word_rects(idx, "Alessandro")[0]
+ monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: QMessageBox.StandardButton.No))
+ w.user_draw_rect(QRectF(x0 * w.scale, y0 * w.scale, (x1 - x0) * w.scale, (y1 - y0) * w.scale))
+ annots = _redacts(w)
+ assert len(annots) == 1
+ assert w._read_redaction_payload(annots[0])["original"] == "Alessandro"
+ out, mapf = tmp_path / "m.pdf", tmp_path / "maps" / "m.nullifypdf-map"
+ mapf.parent.mkdir()
+ _stub_dialogs(monkeypatch, [str(out), str(mapf)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE)
+ monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None))
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(w, "_offer_recovery_key", lambda: None)
+ w.cmd_export()
+ payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026")
+ assert {e["original"] for e in payload["entries"]} == {"Alessandro"}
+
+
+def test_pseudonymize_round_trip_at_9pt_keeps_neighbours_and_counts(window, tmp_path, monkeypatch):
+ w = window
+ _tight_doc(w, 9.0)
+ w.apply_ai_to_page(0, [
+ _detection(w, "Alessandro", "PERSON", "accepted"),
+ _detection(w, "Marchetti", "PERSON", "accepted"),
+ ])
+ out, mapf = tmp_path / "t9.pdf", tmp_path / "maps" / "m.nullifypdf-map"
+ mapf.parent.mkdir()
+ _stub_dialogs(monkeypatch, [str(out), str(mapf)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.PSEUDONYMIZE)
+ monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None))
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(w, "_offer_recovery_key", lambda: None)
+ w.cmd_export()
+ pdf = pymupdf.open(str(out))
+ text = pdf[0].get_text()
+ pdf.close()
+ assert "Commissione tributaria provinciale" in text
+ assert "Programmazione generale dei servizi" in text
+ assert "Alessandro" not in text
+ payload = decrypt_restore_payload(mapf.read_bytes(), "una frase molto lunga 2026")
+ restored = tmp_path / "r9.pdf"
+ n = app.reconstruct_pdf(str(out), str(restored), payload)
+ assert n == app.expected_restore_count(payload) == 2
+ back = pymupdf.open(str(restored))
+ t2 = back[0].get_text()
+ back.close()
+ assert "Alessandro" in t2 and "Marchetti" in t2
+ assert t2.count("Commissione tributaria provinciale") == 1
+ assert "Programmazione generale dei servizi" in t2
+
+
+def test_word_with_a_taller_glyph_falls_back_to_the_full_box(window):
+ """A thin band that misses one glyph of the target must not be used."""
+ w = window
+ w.doc.close()
+ d = pymupdf.open()
+ pg = d.new_page()
+ pg.insert_text((72, 100), "Riga sopra del testo", fontsize=11)
+ pg.insert_text((72, 109), "Codice AB", fontsize=11)
+ pg.insert_text((72 + 62, 109 - 5), "123", fontsize=6) # raised, smaller trailing digits
+ pg.insert_text((72, 118), "Riga sotto del testo", fontsize=11)
+ w.doc = d
+ idx = app.build_page_index(d[0].get_text("words"))
+ fulls = app.whole_word_rects(idx, "AB")
+ band = fitz_rect = pymupdf.Rect(*fulls[0])
+ full = pymupdf.Rect(band.x0, band.y0 - 6, band.x1 + 30, band.y1)
+ thin = pymupdf.Rect(full.x0, full.y0 + 9, full.x1, full.y0 + 10) # misses the raised digits
+ assert not app.band_covers_target(d[0], thin, full)
+ assert app.band_covers_target(d[0], full, full)
+
+
+def test_recovery_wrap_is_kept_even_if_the_dialog_is_closed_without_ticking(window, monkeypatch):
+ w = window
+ monkeypatch.setattr(QMessageBox, "question", staticmethod(lambda *a, **k: QMessageBox.StandardButton.Yes))
+ monkeypatch.setattr(QDialog, "exec", lambda self: QDialog.DialogCode.Rejected)
+ raw = w._offer_recovery_key()
+ assert isinstance(raw, bytes) and len(raw) == 32
diff --git a/tests/test_privacy_hardening.py b/tests/test_privacy_hardening.py
new file mode 100644
index 0000000..b5ec5ad
--- /dev/null
+++ b/tests/test_privacy_hardening.py
@@ -0,0 +1,261 @@
+"""Restore-map hardening and pseudonym styles."""
+
+import base64
+import json
+import os
+
+import pytest
+
+pytest.importorskip("cryptography")
+
+import privacy_core as pc
+
+
+def _payload():
+ reg = pc.PlaceholderRegistry()
+ reg.placeholder_for(
+ "Mario Rossi", "PERSON", page=0, rect=(1, 2, 3, 4),
+ style={"font": "x", "size": 10.0, "color": 0, "flags": 0,
+ "origin": [1.0, 2.0], "text": "Mario Rossi"},
+ )
+ return pc.build_restore_payload(
+ source_name="C:/docs/Referto_Mario_Rossi.pdf", source_sha256="a",
+ output_sha256="b", entries=reg.entries(),
+ )
+
+
+def test_payload_omits_source_name_and_duplicated_style_text():
+ payload = _payload()
+ assert payload["source_name"] == ""
+ occ = payload["entries"][0]["occurrences"][0]
+ assert "text" not in occ["style"]
+ assert pc.parse_restore_entries(payload)[0].original == "Mario Rossi"
+ kept = pc.build_restore_payload(
+ source_name="a/b.pdf", source_sha256="", output_sha256=None,
+ entries=[], keep_source_name=True,
+ )
+ assert kept["source_name"] == "b.pdf"
+
+
+def test_style_text_is_kept_when_it_differs():
+ reg = pc.PlaceholderRegistry()
+ reg.placeholder_for("Mario", "PERSON", rect=(0, 0, 1, 1),
+ style={"font": "", "size": 9.0, "origin": [0, 0], "text": "Mario Rossi"})
+ occ = pc.build_restore_payload(
+ source_name="", source_sha256="", output_sha256=None, entries=reg.entries()
+ )["entries"][0]["occurrences"][0]
+ assert occ["style"]["text"] == "Mario Rossi"
+
+
+def test_new_maps_are_v3_and_roundtrip():
+ blob = pc.encrypt_restore_payload(_payload(), "una frase molto lunga 2026")
+ env = json.loads(blob)
+ assert env["version"] == 3 and env["cipher"] == "AES-256-GCM"
+ assert [w["type"] for w in env["wraps"]] == ["password"]
+ assert env["wraps"][0]["kdf"] in ("argon2id", "scrypt")
+ assert pc.decrypt_restore_payload(blob, "una frase molto lunga 2026")["format"] == pc.RESTORE_MAP_FORMAT
+ from cryptography.fernet import InvalidToken
+ with pytest.raises(InvalidToken):
+ pc.decrypt_restore_payload(blob, "una frase sbagliata 2026!")
+
+
+def test_v2_scrypt_maps_still_decrypt():
+ blob = pc._encrypt_v2(_payload(), "una frase molto lunga 2026")
+ assert json.loads(blob)["version"] == 2
+ assert pc.decrypt_restore_payload(blob, "una frase molto lunga 2026")["version"] == pc.RESTORE_MAP_VERSION
+
+
+def _legacy_pbkdf2_blob(password, iterations):
+ from cryptography.fernet import Fernet
+ from cryptography.hazmat.primitives import hashes
+ from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+ salt = b"0123456789abcdef"
+ key = base64.urlsafe_b64encode(
+ PBKDF2HMAC(hashes.SHA256(), 32, salt, iterations).derive(password.encode())
+ )
+ token = Fernet(key).encrypt(json.dumps(_payload()).encode())
+ return json.dumps({
+ "format": "NullifyPDF encrypted restore map", "version": 1,
+ "kdf": "PBKDF2-HMAC-SHA256", "iterations": iterations,
+ "salt": base64.b64encode(salt).decode(), "token": token.decode(),
+ }).encode()
+
+
+def test_legacy_pbkdf2_maps_still_decrypt():
+ blob = _legacy_pbkdf2_blob("legacy password 123", 600000)
+ assert pc.decrypt_restore_payload(blob, "legacy password 123")["version"] == pc.RESTORE_MAP_VERSION
+
+
+@pytest.mark.parametrize("iterations", [1, 99_999, 5_000_001, 10 ** 12])
+def test_untrusted_iteration_counts_are_rejected_before_derivation(iterations):
+ blob = _legacy_pbkdf2_blob("legacy password 123", 600000)
+ env = json.loads(blob)
+ env["iterations"] = iterations
+ with pytest.raises(ValueError):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), "legacy password 123")
+
+
+@pytest.mark.parametrize("n,r,p", [(2 ** 30, 8, 1), (1000, 8, 1), (2 ** 17, 1000, 1), (2 ** 17, 8, 99)])
+def test_untrusted_scrypt_parameters_are_rejected(n, r, p):
+ env = json.loads(pc._encrypt_v2(_payload(), "una frase molto lunga 2026"))
+ env.update(n=n, r=r, p=p)
+ with pytest.raises(ValueError):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026")
+
+
+def test_unknown_kdf_is_rejected():
+ env = json.loads(pc._encrypt_v2(_payload(), "una frase molto lunga 2026"))
+ env["kdf"] = "md5"
+ with pytest.raises(ValueError):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026")
+
+
+@pytest.mark.parametrize("pw,ok", [
+ ("short1!", False),
+ ("aaaaaaaaaaaaaaaa", False),
+ ("passwordpassword1", False),
+ ("abcdefghijklm", False),
+ ("Abcdefghijklm", True),
+ ("una frase molto lunga davvero", True),
+ ("Xk3!mP9#qR2$", True),
+])
+def test_password_strength(pw, ok):
+ assert pc.password_strength(pw)[0] is ok
+
+
+def test_encrypt_still_requires_minimum_length():
+ with pytest.raises(ValueError):
+ pc.encrypt_restore_payload(_payload(), "short")
+
+
+def test_counter_style_is_unchanged():
+ reg = pc.PlaceholderRegistry()
+ assert reg.placeholder_for("A Uno", "PERSON") == "PERSON_001"
+ assert reg.placeholder_for("B Due", "PERSON") == "PERSON_002"
+ assert reg.placeholder_for("a uno", "PERSON") == "PERSON_001"
+
+
+def test_random_style_reveals_neither_order_nor_count():
+ values = [f"Persona Numero{i}" for i in range(30)]
+ a, b = pc.PlaceholderRegistry("random"), pc.PlaceholderRegistry("random")
+ pa = [a.placeholder_for(v, "PERSON") for v in values]
+ pb = [b.placeholder_for(v, "PERSON") for v in values]
+ assert len(set(pa)) == 30
+ assert pa != pb # fresh randomness per document
+ assert all(p.startswith("PERSON_") and len(p) >= len("PERSON_") + 4 for p in pa)
+ assert pa == [a.placeholder_for(v, "PERSON") for v in values] # stable within a document
+ assert pa != sorted(pa) or pb != sorted(pb)
+
+
+def test_opaque_style_hides_the_type():
+ reg = pc.PlaceholderRegistry("opaque")
+ assert reg.placeholder_for("Mario", "PERSON").startswith("DATA_")
+ assert reg.entries()[0].entity_type == "PERSON" # kept inside the encrypted map
+
+
+def test_keyed_style_is_consistent_per_key_only():
+ k1, k2 = b"k" * 32, b"j" * 32
+ a = pc.PlaceholderRegistry("keyed", k1).placeholder_for("Mario Rossi", "PERSON")
+ b = pc.PlaceholderRegistry("keyed", k1).placeholder_for("mario rossi", "PERSON")
+ c = pc.PlaceholderRegistry("keyed", k2).placeholder_for("Mario Rossi", "PERSON")
+ assert a == b and a != c
+ assert pc.PlaceholderRegistry("keyed", k1).placeholder_for("Mario Rossi", "LOCATION") != a
+ with pytest.raises(ValueError):
+ pc.PlaceholderRegistry("keyed")
+ with pytest.raises(ValueError):
+ pc.PlaceholderRegistry("nonsense")
+
+
+def test_unknown_or_foreign_envelope_gives_a_clear_error():
+ blob = pc._encrypt_v2(_payload(), "una frase molto lunga 2026")
+ env = json.loads(blob)
+ env["version"] = 9
+ with pytest.raises(ValueError, match="non supportata"):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026")
+ env["format"] = "something else"
+ with pytest.raises(ValueError, match="non riconosciuto"):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), "una frase molto lunga 2026")
+
+
+# --- envelope v3: recovery key, Argon2id, key export -----------------------------
+
+PW = "una frase molto lunga 2026"
+
+
+def test_v3_password_and_recovery_open_the_same_payload():
+ raw, shown = pc.generate_recovery_key()
+ blob = pc.encrypt_restore_payload(_payload(), PW, recovery_key=raw)
+ assert [w["type"] for w in json.loads(blob)["wraps"]] == ["password", "recovery"]
+ by_pw = pc.decrypt_restore_payload(blob, PW)
+ by_rk = pc.decrypt_restore_payload(blob, None, recovery_key=pc.parse_recovery_key(shown))
+ assert by_pw == by_rk
+ from cryptography.fernet import InvalidToken
+ with pytest.raises(InvalidToken):
+ pc.decrypt_restore_payload(blob, None, recovery_key=os.urandom(32))
+ with pytest.raises(InvalidToken):
+ pc.decrypt_restore_payload(blob) # nothing supplied
+
+
+def test_recovery_key_format_and_checksum():
+ raw, shown = pc.generate_recovery_key()
+ assert shown.startswith("NPDF-") and len(shown.replace("-", "")) == 4 + 54
+ assert pc.parse_recovery_key(shown.lower().replace("-", " ")) == raw # forgiving of case/spacing
+ typo = shown[:8] + ("A" if shown[8] != "A" else "B") + shown[9:]
+ with pytest.raises(ValueError, match="digitata|non valida"):
+ pc.parse_recovery_key(typo)
+ with pytest.raises(ValueError):
+ pc.parse_recovery_key("NPDF-ABCD")
+
+
+def test_v3_wrap_type_swap_fails():
+ raw, _ = pc.generate_recovery_key()
+ env = json.loads(pc.encrypt_restore_payload(_payload(), PW, recovery_key=raw))
+ env["wraps"][1]["type"] = "password" # pretend the recovery wrap is a password wrap
+ env["wraps"].pop(0)
+ from cryptography.fernet import InvalidToken
+ with pytest.raises((InvalidToken, KeyError, ValueError)):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), PW)
+
+
+def test_v3_tampering_is_detected():
+ env = json.loads(pc.encrypt_restore_payload(_payload(), PW))
+ ct = bytearray(base64.b64decode(env["ciphertext"]))
+ ct[0] ^= 1
+ env["ciphertext"] = base64.b64encode(bytes(ct)).decode()
+ from cryptography.fernet import InvalidToken
+ with pytest.raises(InvalidToken):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), PW)
+
+
+@pytest.mark.skipif(not pc._argon2_available(), reason="Argon2id not available in this OpenSSL")
+@pytest.mark.parametrize("field,value", [
+ ("t", 0), ("t", 11), ("m", 1024), ("m", 2 ** 21), ("p", 0), ("p", 9),
+])
+def test_argon2id_bounds_are_enforced_before_derivation(field, value):
+ env = json.loads(pc.encrypt_restore_payload(_payload(), PW))
+ assert env["wraps"][0]["kdf"] == "argon2id"
+ env["wraps"][0][field] = value
+ with pytest.raises(ValueError, match="limiti"):
+ pc.decrypt_restore_payload(json.dumps(env).encode(), PW)
+
+
+def test_argon2_unavailable_falls_back_to_scrypt_for_writing(monkeypatch):
+ monkeypatch.setattr(pc, "_argon2_available", lambda: False)
+ blob = pc.encrypt_restore_payload(_payload(), PW)
+ assert json.loads(blob)["wraps"][0]["kdf"] == "scrypt"
+ assert pc.decrypt_restore_payload(blob, PW)["format"] == pc.RESTORE_MAP_FORMAT
+
+
+def test_pseudonym_key_export_import_roundtrip_and_wrong_password():
+ key = os.urandom(32)
+ blob = pc.export_pseudonym_key(key, PW)
+ assert key not in blob and base64.b64encode(key) not in blob
+ assert pc.import_pseudonym_key(blob, PW) == key
+ from cryptography.fernet import InvalidToken
+ with pytest.raises(InvalidToken):
+ pc.import_pseudonym_key(blob, "un'altra frase lunga 2026")
+ with pytest.raises(ValueError):
+ pc.import_pseudonym_key(pc.encrypt_restore_payload(_payload(), PW), PW) # a map, not a key
+ with pytest.raises(ValueError):
+ pc.export_pseudonym_key(b"short", PW)
diff --git a/tests/test_reconstruct.py b/tests/test_reconstruct.py
index 2448049..aa01a33 100644
--- a/tests/test_reconstruct.py
+++ b/tests/test_reconstruct.py
@@ -248,9 +248,11 @@ def test_reconstruct_v1_map_finds_wrapped_placeholder(tmp_path):
assert "MarioRossi" in "".join(text.split())
-def test_reconstruct_does_not_report_values_it_failed_to_write(tmp_path):
- """An original too long for its box is dropped by `apply_redactions()`.
- That must never be counted as a restored value."""
+def test_reconstruct_shrinks_a_long_value_instead_of_dropping_it(tmp_path):
+ """An original too long for its box used to be dropped by
+ `apply_redactions()` (no text below 4 pt). It is now written after the
+ redactions with TextWriter, so it is back in the file (and counted only
+ because it really is)."""
from NullifyPDF import sha256_file
doc = fitz.open()
@@ -276,8 +278,10 @@ def test_reconstruct_does_not_report_values_it_failed_to_write(tmp_path):
],
}
out_path = tmp_path / "reconstructed.pdf"
- # The value cannot fit the tight box; the honest answer is zero.
- assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 0
+ assert reconstruct_pdf(str(pseudonymized), str(out_path), payload) == 1
+ back = fitz.open(str(out_path))
+ assert "UNBREAKABLE" in "".join(back[0].get_text().split())
+ back.close()
def test_expected_restore_count_counts_occurrences_not_entries(tmp_path):
@@ -481,3 +485,34 @@ def test_embedded_font_is_actually_reused(tmp_path):
)
span = _span_for(out, "Maria Bianchi")
assert "arial" in span["font"].lower()
+
+
+def test_compact_labels_chain_and_tiny_box_placeholder_survives(tmp_path):
+ """A box too small for the full placeholder still gets a legible label."""
+ from NullifyPDF import compact_labels, write_label
+
+ assert compact_labels("PERSON_9F2A") == ["PERSON_9F2A", "P9F2A", "9F2A", "#9F"]
+ doc = fitz.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Re", fontsize=11)
+ rect = fitz.Rect(page.get_text("words")[0][:4])
+ page.add_redact_annot(rect, fill=(1, 1, 1))
+ page.apply_redactions()
+ drawn = write_label(page, rect, compact_labels("PERSON_9F2A"))
+ assert drawn != "PERSON_9F2A"
+ assert drawn in page.get_text()
+
+
+def test_v4_map_keeps_label_and_v3_still_loads():
+ from privacy_core import parse_restore_entries
+
+ reg = PlaceholderRegistry()
+ ph = reg.placeholder_for("Re", "PERSON", rect=(1, 2, 3, 4))
+ reg.set_label(ph, (1, 2, 3, 4), "P001")
+ payload = build_restore_payload(
+ source_name="", source_sha256="", output_sha256=None, entries=reg.entries()
+ )
+ assert payload["version"] == 4
+ assert parse_restore_entries(payload)[0].occurrences[0].label == "P001"
+ payload["version"] = 3
+ assert parse_restore_entries(payload)[0].original == "Re"
diff --git a/tests/test_secure_store.py b/tests/test_secure_store.py
new file mode 100644
index 0000000..fed997d
--- /dev/null
+++ b/tests/test_secure_store.py
@@ -0,0 +1,160 @@
+"""Encrypted word lists with expiry, and the key-protection layer."""
+
+import datetime
+import json
+import os
+import sys
+
+import pytest
+
+pytest.importorskip("cryptography")
+
+import secure_store
+from word_lists import ListEntry, PDFListManager
+
+
+@pytest.fixture(autouse=True)
+def _fresh_cache():
+ secure_store.forget_cached_keys()
+ yield
+ secure_store.forget_cached_keys()
+
+
+def test_seal_open_roundtrip_and_aad_binding():
+ key = os.urandom(32)
+ blob = secure_store.seal(b"dati", key, b"ctx-a")
+ assert secure_store.open_sealed(blob, key, b"ctx-a") == b"dati"
+ with pytest.raises(secure_store.SealError):
+ secure_store.open_sealed(blob, key, b"ctx-b")
+ with pytest.raises(secure_store.SealError):
+ secure_store.open_sealed(blob[:-1] + bytes([blob[-1] ^ 1]), key, b"ctx-a")
+ with pytest.raises(secure_store.SealError):
+ secure_store.open_sealed(b"short", key, b"ctx-a")
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="DPAPI is Windows-only")
+def test_dpapi_roundtrip_and_account_binding(tmp_path):
+ p = secure_store.DpapiProtector()
+ key = os.urandom(32)
+ assert p.unwrap(p.wrap(key)) == key
+ assert key not in json.dumps(p.wrap(key)).encode()
+
+
+def test_keyfile_protector_creates_private_file(tmp_path):
+ p = secure_store.KeyFileProtector(tmp_path)
+ key = os.urandom(32)
+ blob = p.wrap(key)
+ assert p.unwrap(blob) == key
+ mode = (tmp_path / "keys" / "master.key").stat().st_mode & 0o777
+ if sys.platform != "win32":
+ assert mode == 0o600
+
+
+def test_master_key_is_stable_and_not_stored_in_clear_in_keystore(tmp_path):
+ k1 = secure_store.load_or_create_master_key(tmp_path)
+ secure_store.forget_cached_keys()
+ k2 = secure_store.load_or_create_master_key(tmp_path)
+ assert k1 == k2 and len(k1) == 32
+ ks = (tmp_path / "keystore.json").read_bytes()
+ if json.loads(ks)["master"]["protector"] == "dpapi":
+ assert k1 not in ks
+ assert "DPAPI" in secure_store.protector_label(tmp_path)
+
+
+def test_keyring_unavailable_falls_back_to_keyfile(tmp_path, monkeypatch):
+ monkeypatch.setattr(secure_store, "DpapiProtector", type("D", (), {"available": staticmethod(lambda: False)}))
+ monkeypatch.setattr(secure_store.KeyringProtector, "available", staticmethod(lambda: False))
+ secure_store.load_or_create_master_key(tmp_path)
+ assert json.loads((tmp_path / "keystore.json").read_text())["master"]["protector"] == "keyfile"
+
+
+def test_plaintext_lists_migrate_transparently(tmp_path):
+ (tmp_path / "blocklist.txt").write_text("mario rossi\nvia roma\nab", encoding="utf-8")
+ (tmp_path / "allowlist.txt").write_text("comune", encoding="utf-8")
+ m = PDFListManager(tmp_path)
+ assert m.load_blocklist() == {"mario rossi", "via roma"}
+ assert m.load_allowlist() == {"comune"}
+ assert not (tmp_path / "blocklist.txt").exists()
+ enc = (tmp_path / "blocklist.v2.enc").read_bytes()
+ assert b"rossi" not in enc and enc.startswith(b"NPDFL2\n")
+ assert PDFListManager(tmp_path).load_blocklist() == {"mario rossi", "via roma"} # reload
+
+
+def test_save_roundtrip_filters_short_terms_and_keeps_dates(tmp_path):
+ day = {"now": datetime.date(2026, 1, 1)}
+ m = PDFListManager(tmp_path, today=lambda: day["now"])
+ m.save_blocklist({"abc", "ab", "Mario Rossi"})
+ assert m.load_blocklist() == {"abc", "mario rossi"}
+ day["now"] = datetime.date(2026, 3, 1)
+ m.save_blocklist({"abc", "nuovo termine"})
+ entries = {e.term: e for e in m.load_entries("blocklist")}
+ assert entries["abc"].added == datetime.date(2026, 1, 1) # kept
+ assert entries["nuovo termine"].added == datetime.date(2026, 3, 1)
+
+
+def test_expired_entries_are_dropped_and_purged(tmp_path):
+ day = {"now": datetime.date(2026, 1, 1)}
+ m = PDFListManager(tmp_path, today=lambda: day["now"])
+ m.save_entries("blocklist", [
+ ListEntry("scade presto", day["now"], datetime.date(2026, 1, 10)),
+ ListEntry("per sempre", day["now"], None),
+ ])
+ day["now"] = datetime.date(2026, 1, 11)
+ assert m.load_blocklist() == {"per sempre"}
+ assert m.expired_removed == 1
+ day["now"] = datetime.date(2025, 12, 1) # purged for good, not merely hidden
+ assert PDFListManager(tmp_path, today=lambda: day["now"]).load_blocklist() == {"per sempre"}
+
+
+def test_default_ttl_applies_to_new_terms(tmp_path):
+ m = PDFListManager(tmp_path, today=lambda: datetime.date(2026, 1, 1))
+ m.set_default_ttl_days(30)
+ m.save_blocklist({"nuovo termine"})
+ e = m.load_entries("blocklist")[0]
+ assert e.expires == datetime.date(2026, 1, 31)
+
+
+def test_unreadable_keystore_degrades_without_data_loss(tmp_path):
+ m = PDFListManager(tmp_path)
+ m.save_blocklist({"mario rossi"})
+ before = (tmp_path / "blocklist.v2.enc").read_bytes()
+ ks = json.loads((tmp_path / "keystore.json").read_text())
+ if ks["master"]["protector"] == "dpapi":
+ ks["master"]["blob"] = "AAAA"
+ else:
+ (tmp_path / "keys" / "master.key").write_bytes(b"x" * 5)
+ (tmp_path / "keystore.json").write_text(json.dumps(ks))
+ secure_store.forget_cached_keys()
+ m2 = PDFListManager(tmp_path)
+ assert m2.load_blocklist() == set()
+ assert m2.keystore_error
+ m2.save_blocklist({"altro termine"}) # must not overwrite anything
+ assert (tmp_path / "blocklist.v2.enc").read_bytes() == before
+
+
+def test_no_cryptography_keeps_plaintext(tmp_path, monkeypatch):
+ import builtins
+
+ real = builtins.__import__
+
+ def fake(name, *a, **k):
+ if name == "cryptography" or name.startswith("cryptography."):
+ raise ImportError(name)
+ return real(name, *a, **k)
+
+ monkeypatch.setattr(builtins, "__import__", fake)
+ m = PDFListManager(tmp_path)
+ m.save_blocklist({"mario rossi"})
+ monkeypatch.setattr(builtins, "__import__", real)
+ assert (tmp_path / "blocklist.txt").read_text(encoding="utf-8") == "mario rossi"
+ assert PDFListManager(tmp_path).load_blocklist() == {"mario rossi"}
+
+
+def test_tampered_list_file_is_not_trusted(tmp_path):
+ m = PDFListManager(tmp_path)
+ m.save_blocklist({"mario rossi"})
+ p = tmp_path / "blocklist.v2.enc"
+ raw = bytearray(p.read_bytes())
+ raw[-1] ^= 1
+ p.write_bytes(bytes(raw))
+ assert PDFListManager(tmp_path).load_blocklist() == set()
diff --git a/word_lists.py b/word_lists.py
new file mode 100644
index 0000000..665aad5
--- /dev/null
+++ b/word_lists.py
@@ -0,0 +1,257 @@
+"""Blocklist / allowlist persistence: encrypted at rest, optional expiry (Qt-free).
+
+The lists can contain personal data (names the user always wants removed or
+always wants kept), so they are stored sealed with the master key of
+:mod:`secure_store` in ``blocklist.v2.enc`` / ``allowlist.v2.enc``. Each
+entry carries the date it was added and an optional expiry date; expired
+entries are dropped (and the file rewritten) when the list is loaded.
+
+Compatibility: an existing plaintext ``blocklist.txt`` / ``allowlist.txt`` is
+read, migrated transparently and then overwritten and removed. If encryption
+is unavailable (no ``cryptography``, no usable protector) the plaintext files
+keep working. If the keystore cannot be unwrapped (profile copied to another
+account) nothing is deleted or overwritten and the lists are empty until the
+keystore is restored.
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+import logging
+import pathlib
+from dataclasses import dataclass
+from typing import Callable, Dict, List, Optional, Set
+
+import secure_store
+
+MIN_TERM_LENGTH = 3
+_MAGIC = b"NPDFL2\n"
+_FORMAT = "NullifyPDF word list"
+_log = logging.getLogger("nullifypdf")
+
+
+@dataclass
+class ListEntry:
+ term: str
+ added: datetime.date
+ expires: Optional[datetime.date] = None
+
+ def to_json(self) -> Dict[str, Optional[str]]:
+ return {
+ "term": self.term,
+ "added": self.added.isoformat(),
+ "expires": self.expires.isoformat() if self.expires else None,
+ }
+
+
+def _clean(term: str) -> str:
+ return " ".join(term.strip().lower().split())
+
+
+def _date(value: Optional[str]) -> Optional[datetime.date]:
+ if not value:
+ return None
+ try:
+ return datetime.date.fromisoformat(value)
+ except ValueError:
+ return None
+
+
+class PDFListManager:
+ """Loads and saves the blocklist and allowlist."""
+
+ def __init__(
+ self,
+ config_dir: pathlib.Path,
+ today: Callable[[], datetime.date] = datetime.date.today,
+ ) -> None:
+ self.config_dir = pathlib.Path(config_dir)
+ self.config_dir.mkdir(parents=True, exist_ok=True)
+ self.block_file = self.config_dir / "blocklist.txt"
+ self.allow_file = self.config_dir / "allowlist.txt"
+ self._today = today
+ self.expired_removed = 0
+ self.keystore_error: Optional[str] = None
+ self.encrypted = True
+
+ # ------------------------------------------------------------ paths
+ def _enc_path(self, kind: str) -> pathlib.Path:
+ return self.config_dir / f"{kind}.v2.enc"
+
+ def _plain_path(self, kind: str) -> pathlib.Path:
+ return self.block_file if kind == "blocklist" else self.allow_file
+
+ # ------------------------------------------------------- settings
+ def default_ttl_days(self) -> Optional[int]:
+ try:
+ data = json.loads((self.config_dir / "settings.json").read_text(encoding="utf-8"))
+ ttl = data.get("list_ttl_days")
+ return int(ttl) if ttl else None
+ except Exception:
+ return None
+
+ def set_default_ttl_days(self, days: Optional[int]) -> None:
+ path = self.config_dir / "settings.json"
+ try:
+ data = json.loads(path.read_text(encoding="utf-8"))
+ except Exception:
+ data = {}
+ data["list_ttl_days"] = days
+ secure_store.atomic_write(path, json.dumps(data).encode("utf-8"))
+
+ def protector_label(self) -> str:
+ if not self.encrypted:
+ return "Non cifrato"
+ return secure_store.protector_label(self.config_dir)
+
+ # ------------------------------------------------------- entries
+ def load_entries(self, kind: str) -> List[ListEntry]:
+ enc, plain = self._enc_path(kind), self._plain_path(kind)
+ if enc.exists():
+ entries = self._read_encrypted(kind, enc)
+ kept = [e for e in entries if not (e.expires and e.expires < self._today())]
+ if len(kept) != len(entries):
+ self.expired_removed += len(entries) - len(kept)
+ self._write_encrypted(kind, kept)
+ return kept
+ if plain.exists():
+ entries = self._read_plain(plain)
+ if self._can_encrypt():
+ try:
+ self._write_encrypted(kind, entries)
+ secure_store.secure_delete(plain)
+ except Exception as e: # keep the readable file on any failure
+ _log.error(f"List migration failed: {e}")
+ self.encrypted = False
+ else:
+ self.encrypted = False
+ return entries
+ return []
+
+ def save_entries(self, kind: str, entries: List[ListEntry]) -> None:
+ clean: Dict[str, ListEntry] = {}
+ for e in entries:
+ term = _clean(e.term)
+ if len(term) >= MIN_TERM_LENGTH:
+ clean[term] = ListEntry(term, e.added, e.expires)
+ final = sorted(clean.values(), key=lambda e: e.term)
+ if self._can_encrypt():
+ try:
+ self._write_encrypted(kind, final)
+ plain = self._plain_path(kind)
+ if plain.exists():
+ secure_store.secure_delete(plain)
+ return
+ except secure_store.KeystoreUnavailable as e:
+ self.keystore_error = str(e)
+ raise
+ except Exception as e:
+ _log.error(f"Encrypted list save failed: {e}")
+ self.encrypted = False
+ try:
+ secure_store.atomic_write(
+ self._plain_path(kind), "\n".join(e.term for e in final).encode("utf-8")
+ )
+ except OSError as e:
+ _log.error(f"Error saving {kind}: {e}")
+
+ # -------------------------------------------- set-based convenience
+ def load_blocklist(self) -> Set[str]:
+ return self._terms("blocklist")
+
+ def load_allowlist(self) -> Set[str]:
+ return self._terms("allowlist")
+
+ def save_blocklist(self, blocklist: Set[str]) -> None:
+ self._save_terms("blocklist", blocklist)
+
+ def save_allowlist(self, allowlist: Set[str]) -> None:
+ self._save_terms("allowlist", allowlist)
+
+ def _terms(self, kind: str) -> Set[str]:
+ try:
+ return {e.term for e in self.load_entries(kind)}
+ except secure_store.KeystoreUnavailable as e:
+ self.keystore_error = str(e)
+ _log.error("Word lists unreadable: %s", e)
+ return set()
+ except Exception as e:
+ _log.error(f"Error loading {kind}: {e}")
+ return set()
+
+ def _save_terms(self, kind: str, words: Set[str]) -> None:
+ """Save a set of terms, keeping dates of the ones already present."""
+ try:
+ existing = {e.term: e for e in self.load_entries(kind)}
+ except secure_store.KeystoreUnavailable:
+ existing = {}
+ except Exception as e: # corrupt / tampered file: start from the new set
+ _log.error(f"Unreadable {kind} file ignored: {e}")
+ existing = {}
+ ttl = self.default_ttl_days()
+ today = self._today()
+ entries: List[ListEntry] = []
+ for w in words:
+ term = _clean(w)
+ if len(term) < MIN_TERM_LENGTH:
+ continue
+ old = existing.get(term)
+ if old:
+ entries.append(old)
+ else:
+ exp = today + datetime.timedelta(days=ttl) if ttl else None
+ entries.append(ListEntry(term, today, exp))
+ try:
+ self.save_entries(kind, entries)
+ except secure_store.KeystoreUnavailable:
+ pass # error kept in self.keystore_error; nothing is overwritten
+
+ # ------------------------------------------------------ plumbing
+ def _can_encrypt(self) -> bool:
+ try:
+ import cryptography # noqa: F401
+
+ secure_store.load_or_create_master_key(self.config_dir)
+ return True
+ except secure_store.KeystoreUnavailable as e:
+ self.keystore_error = str(e)
+ raise
+ except Exception:
+ return False
+
+ def _read_plain(self, path: pathlib.Path) -> List[ListEntry]:
+ try:
+ with open(path, "r", encoding="utf-8") as fh:
+ terms = {_clean(line) for line in fh if len(line.strip()) >= MIN_TERM_LENGTH}
+ except Exception as e:
+ _log.error(f"Error loading {path}: {e}")
+ return []
+ today = self._today()
+ return [ListEntry(t, today, None) for t in sorted(terms) if len(t) >= MIN_TERM_LENGTH]
+
+ def _read_encrypted(self, kind: str, path: pathlib.Path) -> List[ListEntry]:
+ key = secure_store.load_or_create_master_key(self.config_dir)
+ raw = path.read_bytes()
+ if not raw.startswith(_MAGIC):
+ raise ValueError("Formato del dizionario non riconosciuto.")
+ data = json.loads(
+ secure_store.open_sealed(raw[len(_MAGIC):], key, b"nullifypdf-list:" + kind.encode())
+ )
+ out = []
+ for item in data.get("entries", []):
+ term = _clean(str(item.get("term", "")))
+ if len(term) < MIN_TERM_LENGTH:
+ continue
+ out.append(ListEntry(
+ term, _date(item.get("added")) or self._today(), _date(item.get("expires"))
+ ))
+ return out
+
+ def _write_encrypted(self, kind: str, entries: List[ListEntry]) -> None:
+ key = secure_store.load_or_create_master_key(self.config_dir)
+ payload = json.dumps({
+ "format": _FORMAT, "version": 2, "entries": [e.to_json() for e in entries],
+ }, ensure_ascii=False).encode("utf-8")
+ sealed = secure_store.seal(payload, key, b"nullifypdf-list:" + kind.encode())
+ secure_store.atomic_write(self._enc_path(kind), _MAGIC + sealed)
From 96a8f4178d220fc7c7c7445f07912697d6d90c55 Mon Sep 17 00:00:00 2001
From: 0verwrite <31691645+overwrite00@users.noreply.github.com>
Date: Tue, 10 Nov 2026 20:54:58 +0100
Subject: [PATCH 07/13] feat: in-app guide, tooltips and the remaining open
limits (#132)
* feat: in-app guide, tooltips everywhere, and close the remaining open limits
- Italian user guide (F1 / "Guida" button) and a tooltip on every widget of
the main window and dialogs, with tests that keep the guide in sync with
the UI.
- Export checks that every glyph of a redacted word was deleted (predicted
survivors plus a count check against the measured deletion rule).
- Image and signature labels are drawn after the redactions with shorter
forms in small boxes.
- Given names and surnames from Wikidata (CC0) generated by the lexicon
script; keyring for the master key on macOS and Linux.
- Dark styling for lists, tables, tabs, combo boxes and tooltips; word wrap
in the review dialog.
* fix: no tooltips on the guide topic list and text
* docs: describe the lexicon sources; bump the prerelease label to beta.3
---
.github/workflows/beta-release.yml | 2 +-
.github/workflows/build-linux.yml | 2 +-
.github/workflows/release.yml | 2 +-
.github/workflows/test_build.yml | 2 +-
ARCHITECTURE.md | 12 +-
CHANGELOG.md | 6 +
DEVELOPMENT.md | 5 +-
NullifyPDF.py | 245 ++++++++++-------
README.md | 3 +-
SECURITY.md | 6 +-
THIRD_PARTY_NOTICES.md | 11 +
USER_GUIDE.md | 5 +-
build_local.py | 6 +-
dictionary_dialog.py | 7 +
export_engine.py | 96 +++++++
help_guide.py | 415 +++++++++++++++++++++++++++++
lexicon_data_it.py | 6 +
lexicon_it.py | 14 +-
pii_detection.py | 47 ++++
requirements.txt | 3 +
review_dialog.py | 11 +-
scripts/build_lexicon_data.py | 68 +++++
tests/test_detection_pipeline.py | 13 +
tests/test_export_sanitize.py | 90 +++++++
tests/test_gui_flow.py | 74 +++++
tests/test_gui_help.py | 141 ++++++++++
tests/test_secure_store.py | 40 +++
27 files changed, 1208 insertions(+), 124 deletions(-)
create mode 100644 help_guide.py
create mode 100644 tests/test_gui_help.py
diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml
index 6cc48b1..f71edb5 100644
--- a/.github/workflows/beta-release.yml
+++ b/.github/workflows/beta-release.yml
@@ -50,7 +50,7 @@ jobs:
- name: Type Check (mypy)
shell: bash
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
- name: Resolve beta suffix from tag
if: startsWith(github.ref, 'refs/tags/')
diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml
index 334f0a8..09c82b4 100644
--- a/.github/workflows/build-linux.yml
+++ b/.github/workflows/build-linux.yml
@@ -82,7 +82,7 @@ jobs:
run: pytest tests/ -v
- name: Type check (mypy)
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
- name: Resolve beta suffix from tag
if: inputs.is_beta && startsWith(github.ref, 'refs/tags/')
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 7273a52..9f5ec6c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -95,7 +95,7 @@ jobs:
- name: Type Check (mypy)
shell: bash
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
- name: Build
run: python build_local.py
diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml
index f9249e7..b3aa452 100644
--- a/.github/workflows/test_build.yml
+++ b/.github/workflows/test_build.yml
@@ -55,7 +55,7 @@ jobs:
- name: Type Check (mypy)
shell: bash
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
- name: Build
run: python build_local.py
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 15d146f..b546716 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -33,10 +33,11 @@ It supports two privacy export modes:
| `pdf_sanitize.py` | Export sanitiser (annotations, attachments, scripts, hidden layers/text, off-page text, metadata, optional TJ-gap quantisation) and `verify_export()` |
| `export_engine.py` | Qt-free export pipeline (`export_document`), neutral file names, private temp handling, audit record, local-data wipe |
| `review_dialog.py` | Qt dialog to accept or reject suggested detections |
+| `help_guide.py` | In-app guide (Italian, F1) and the tooltip table applied to every widget |
| `secure_store.py` | Master key protected by DPAPI / OS keyring / key file; AES-GCM sealing; atomic private writes |
| `word_lists.py` | Encrypted blocklist/allowlist with expiry (`PDFListManager`) |
| `dictionary_dialog.py` | Qt editor for both lists with dates and default expiry |
-| `lexicon_data_it.py` | Generated from ISTAT open data by `scripts/build_lexicon_data.py` (see `THIRD_PARTY_NOTICES.md`); imported lazily |
+| `lexicon_data_it.py` | Generated by `scripts/build_lexicon_data.py` from ISTAT open data (municipalities, provinces, regions) and, with `--wikidata`, Wikidata (given names, surnames); see `THIRD_PARTY_NOTICES.md`; imported lazily and used only to confirm names and places |
| `PDF_Checker.py` | Heuristic post-export inspection helper |
| `build_local.py` | PyInstaller build script |
| `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files bundled into every build |
@@ -149,9 +150,10 @@ Clicking an existing redaction:
- widgets are removed
- `pdf_sanitize.sanitize_document()` removes annotations, attachments, scripts, bookmarks, structure, metadata, hidden/invisible/off-page text and external links
- the final PDF is saved with cleanup options
-6. `pdf_sanitize.verify_export()` re-opens the written file and checks it; a failure is reported and the file can be deleted.
-7. For pseudonymization, the restore map is encrypted and written with owner-only permissions.
-8. An audit record (no values) is written to `~/.nullifypdf/audit/`.
+6. Before the redactions are applied, every band that stores the whole box of its word is checked; after applying them, the number of glyphs that vanished from each line must match the measured deletion rule (a glyph is deleted when the rectangle reaches its box inset by 10%), and target glyphs the band cannot reach are counted. Any survivor is reported without the value.
+7. `pdf_sanitize.verify_export()` re-opens the written file and checks it; a failure is reported and the file can be deleted.
+8. For pseudonymization, the restore map is encrypted and written with owner-only permissions.
+9. An audit record (no values) is written to `~/.nullifypdf/audit/`.
The original open document remains editable after export because the export works on a copy.
@@ -320,5 +322,5 @@ pytest tests/ -v
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-10*
*[Back to README](./README.md) | [Development →](./DEVELOPMENT.md)*
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7f799b9..465df4a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### ✨ Added
+- **User guide in the program**: a new "Guida" button (or F1) opens a guide in Italian that explains the whole workflow: opening a PDF, language and options, Auto Redact, review of suggestions, manual corrections, dictionaries, export modes, reconstruction, health documents, local data and shortcuts.
+- **Tooltips on every element**: all buttons, options, fields, tables and indicators of the main window and of the dialogs now show a short description when the pointer rests on them.
+- **Check after redaction for surviving characters**: besides searching the written file, the export now checks that every character of a redacted word was really deleted, and reports it (without the value) if one was not.
- **Check for updates**: the Info dialog has a new "Controlla aggiornamenti" button. It queries the GitHub releases of this repository (on click, or at startup only if you enabled the opt-in startup check) and, if a newer version exists, shows the release link, the release notes and a **Download** button that saves the installer for the running OS into the user's system Downloads folder (resolved per platform), verifying its size and the SHA-256 published by GitHub. Linux distributions outside the Debian/Ubuntu and Fedora/RHEL families only get the release link. Stable builds are only offered newer stable releases; beta builds are offered both stable and beta releases.
- **Optional check at every start**: a "Controlla ad ogni avvio" checkbox in the Info dialog (off by default, stored in `~/.nullifypdf/update_check.json`) runs the same check silently at startup and opens the update window, with the release notes, only when a newer version exists. Failures are logged, never shown.
- **Repository link in the Info dialog**.
@@ -24,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **Signature and stamp suggestions**: with the health-document profile, curve-rich ink and small images next to "firma", "timbro", "il medico"... are proposed for review (never applied automatically).
- **Postcodes and house numbers**: a street and its house number are redacted as one address, and postcodes are recognised only in an address context (never in prices, dates or quantities). Health card number, prescription number (NRE) and exemption codes follow the published structure (length, region code, lot code); the check digit is not verified, and the prescription number and exemption codes need their label.
- **Suggestions in the page preview**: pending suggestions are outlined in orange on the page (view only; they never reach an export). Click one to censor or discard it; selecting a row in the review dialog highlights it.
+- **Given names and surnames from Wikidata** (CC0): a few hundred given names and about one thousand surnames are used to confirm names, never to reject them; names that are also ordinary words count less. Coverage is small.
- **ISTAT place names**: Italian municipalities, provinces and regions (ISTAT open data, CC BY 4.0) are used to confirm places; a place missing from the list is never rejected for that reason.
- **Export audit record** in `~/.nullifypdf/audit/` (counts per type, sanitiser actions, verification result, output SHA-256; no personal data), and an "Elimina dati locali" button that wipes logs, audit records and temporary files.
@@ -43,6 +47,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **Placeholders were silently dropped in small boxes**: the placeholder is now drawn after the redactions are applied, with a compact form ("P9F2A", "9F2A") when the full one does not fit, so short words keep a visible placeholder. Restore maps are version 4 (version 1-3 maps still load).
- **Restore wrote neighbouring text back**: the typography captured for a redaction could include fragments of the lines above and below when the leading was tight; only the redacted line is captured, and the restore writes the original value.
- **Wipe of local data left the active log file**, and "Oscura Immagini" added duplicate image redactions on every apply.
+- **Labels on images and signatures survive small boxes**: the "[IMMAGINE RIMOSSA]" / "[FIRMA RIMOSSA]" labels are drawn after the redactions, with shorter forms when the area is small, instead of being dropped.
+- **Master key in the system keyring on macOS and Linux**: when a keyring is available the encryption key of the dictionaries is stored there instead of in a local key file (Windows keeps using DPAPI).
- **Local logs are rotated** (512 KiB, 3 files, 30 days) instead of growing indefinitely.
- **Redacted text could remain in the PDF content stream**: with PyMuPDF 1.28.x, `apply_redactions()` did not remove text drawn with the `'` operator after `TL`/`Td`; the text disappeared from the extracted text but was still stored in the file. Fixed by PyMuPDF 2.x; covered by a regression test that scans every stream of the exported file.
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 8a2144d..0834584 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -461,7 +461,8 @@ isort NullifyPDF.py
| `secure_store.py` | Master key protection (DPAPI / keyring / key file), AES-GCM sealing |
| `word_lists.py` | Encrypted blocklist/allowlist with expiry |
| `dictionary_dialog.py` | Dictionary editor dialog |
-| `scripts/build_lexicon_data.py` | Generates `lexicon_data_it.py` from ISTAT open data (development only) |
+| `help_guide.py` | In-app guide and tooltip table (keep the guide in sync with the UI: `tests/test_gui_help.py` checks it) |
+| `scripts/build_lexicon_data.py` | Generates `lexicon_data_it.py` from ISTAT open data and (with `--wikidata`) Wikidata names; development only |
| `setup_env.py` | Environment setup |
| `build_local.py` | PyInstaller build |
| `PDF_Checker.py` | Post-processing utility |
@@ -589,5 +590,5 @@ The release workflows fill each GitHub release body with `scripts/extract_releas
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-10*
*← [Troubleshooting](./TROUBLESHOOTING.md) | [Back to README →](./README.md)*
diff --git a/NullifyPDF.py b/NullifyPDF.py
index cf1fec0..a74a26f 100644
--- a/NullifyPDF.py
+++ b/NullifyPDF.py
@@ -42,6 +42,8 @@
QTextBrowser,
)
from PySide6.QtGui import (
+ QKeySequence,
+ QShortcut,
QIcon,
QPixmap,
QImage,
@@ -112,7 +114,7 @@
write_audit_record,
write_private_file,
)
-from pii_detection import GLYPH_CORE_INSET
+from pii_detection import GLYPH_CORE_INSET, core_hits as _core_hits, glyph_core, target_chars
from pdf_sanitize import (
SanitizePolicy,
page_image_coverage,
@@ -120,6 +122,7 @@
suggest_signature_areas,
)
from review_dialog import ReviewDialog
+from help_guide import GuideDialog, apply_tooltips
from dictionary_dialog import DictionaryDialog
from word_lists import PDFListManager
import secure_store
@@ -130,7 +133,7 @@
PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined]
__version__ = "2.4.0"
-__version_prerelease__ = "beta.2"
+__version_prerelease__ = "beta.3"
APP_VERSION = (
f"{__version__}-{__version_prerelease__}"
if __version_prerelease__
@@ -288,6 +291,11 @@ class RestoreMapMismatch(ValueError):
REDACTION_FONTNAME = "helv"
+# Labels (longest first) drawn on image / signature areas after redaction.
+KIND_LABELS: Dict[str, List[str]] = {
+ "IMAGE": ["[IMMAGINE RIMOSSA]", "[IMMAGINE]", "[IMG]", "#"],
+ "SIGNATURE": ["[FIRMA RIMOSSA]", "[FIRMA]", "[F]", "#"],
+}
REDACTION_MAX_FONTSIZE = 8.0
# PyMuPDF's apply_redactions() never inserts replacement text below 4 pt, so
# a smaller floor would only produce text that is silently dropped.
@@ -381,23 +389,6 @@ def _placeholder_rects_by_chars(page: Any, placeholder: str) -> List["fitz.Rect"
return rects
-def glyph_core(bbox: Tuple[float, float, float, float]) -> Tuple[float, float, float, float]:
- """The part of a character box that decides whether a redaction deletes it.
-
- Measured on PyMuPDF 2.1 (MuPDF 1.28): a glyph is removed when the
- redaction rectangle intersects its box shrunk by 10% of its width on
- each side and 10% of its height at top and bottom.
- """
- x0, y0, x1, y1 = bbox
- dx, dy = (x1 - x0) * GLYPH_CORE_INSET, (y1 - y0) * GLYPH_CORE_INSET
- return (x0 + dx, y0 + dy, x1 - dx, y1 - dy)
-
-
-def _core_hits(bbox: Tuple[float, float, float, float], rect: "fitz.Rect") -> bool:
- c = glyph_core(bbox)
- return c[0] < rect.x1 and c[2] > rect.x0 and c[1] < rect.y1 and c[3] > rect.y0
-
-
def capture_text_style(page: Any, rect: "fitz.Rect") -> Optional[Dict[str, Any]]:
"""Read the typography of the native text under `rect`.
@@ -470,34 +461,6 @@ def capture_text_style(page: Any, rect: "fitz.Rect") -> Optional[Dict[str, Any]]
}
-def target_chars(page: Any, full: "fitz.Rect") -> List[Dict[str, Any]]:
- """Characters of the line a word box was taken from (single baseline)."""
- try:
- blocks = page.get_text("rawdict", clip=full).get("blocks", [])
- except Exception:
- return []
- chars: List[Tuple[float, Dict[str, Any]]] = []
- for block in blocks:
- if block.get("type") != 0:
- continue
- for line in block.get("lines", []):
- for span in line.get("spans", []):
- for ch in span.get("chars", []):
- if not str(ch.get("c", "")).strip():
- continue
- bx0, by0, bx1, by1 = ch["bbox"]
- cx, cy = (bx0 + bx1) / 2, (by0 + by1) / 2
- if full.x0 <= cx <= full.x1 and full.y0 <= cy <= full.y1:
- chars.append((round(float(ch["origin"][1]), 1), ch))
- if not chars:
- return []
- centre = (full.y0 + full.y1) / 2
- bases = {b for b, _ in chars}
- inside = [b for b in bases if full.y0 <= b <= full.y1] or list(bases)
- base = min(inside, key=lambda b: abs(b - centre))
- return [ch for b, ch in chars if b == base]
-
-
def band_covers_target(page: Any, band: "fitz.Rect", full: "fitz.Rect") -> bool:
"""True when `band` reaches the deletion core of every target character.
@@ -901,6 +864,15 @@ def ocr_language_for_choice(choice: str, tessdata_dir: Optional[str]) -> str:
QProgressBar { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; text-align: center; color: transparent; height: 8px;}
QProgressBar::chunk { background-color: #0ea5e9; border-radius: 3px; }
QGraphicsView { border: none; background-color: #020617; }
+QListWidget, QTableWidget, QTextBrowser { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; color: #cbd5e1; gridline-color: #334155; selection-background-color: #0284c7; selection-color: white; }
+QListWidget::item { padding: 6px; }
+QHeaderView::section { background-color: #1e293b; color: #e2e8f0; border: 1px solid #334155; padding: 4px; }
+QComboBox, QDoubleSpinBox { background-color: #0f172a; border: 1px solid #334155; border-radius: 4px; padding: 4px; color: #e2e8f0; }
+QComboBox QAbstractItemView { background-color: #0f172a; color: #e2e8f0; selection-background-color: #0284c7; }
+QTabWidget::pane { border: 1px solid #334155; }
+QTabBar::tab { background-color: #1e293b; color: #cbd5e1; padding: 6px 14px; border: 1px solid #334155; }
+QTabBar::tab:selected { background-color: #0284c7; color: white; }
+QToolTip { background-color: #1e293b; color: #e2e8f0; border: 1px solid #0ea5e9; padding: 4px; }
QRadioButton::indicator, QCheckBox::indicator { width: 16px; height: 16px; border: 1px solid #475569; border-radius: 8px; background-color: #0f172a; }
QCheckBox::indicator { border-radius: 4px; }
QRadioButton::indicator:checked, QCheckBox::indicator:checked { background-color: #0ea5e9; border: 1px solid #0ea5e9; }
@@ -1338,7 +1310,7 @@ def __init__(self) -> None:
# Detections the evidence score could not settle; never applied
# silently, reviewed by the user (see review_dialog.ReviewDialog).
self.pending_suggestions: List[Dict[str, Any]] = []
- self._export_labels: List[Tuple[int, Any, str, Optional[Dict[str, Any]]]] = []
+ self._export_labels: List[Tuple[int, Any, List[str], Optional[Dict[str, Any]], Optional[str]]] = []
self.config_dir = pathlib.Path.home() / ".nullifypdf"
# Update-check threads outlive their dialogs: keep them referenced
# until they finish so a QThread is never destroyed while running.
@@ -1359,6 +1331,9 @@ def __init__(self) -> None:
self.start_scan_sig.connect(self.ai_worker.run_scan)
self.ai_thread.start()
self.build_ui()
+ apply_tooltips(self)
+ self._f1 = QShortcut(QKeySequence("F1"), self)
+ self._f1.activated.connect(self.cmd_guide)
if self.list_manager.expired_removed:
self.write_log(
f"Rimossi {self.list_manager.expired_removed} termini scaduti dai dizionari."
@@ -1546,6 +1521,9 @@ def build_ui(self) -> None:
btn_wipe.clicked.connect(self.cmd_wipe_local_data)
s_lay.addWidget(btn_wipe)
s_lay.addStretch()
+ btn_guide = QPushButton("Guida")
+ btn_guide.clicked.connect(self.cmd_guide)
+ s_lay.addWidget(btn_guide)
btn_about = QPushButton("Info")
btn_about.clicked.connect(self.cmd_about)
s_lay.addWidget(btn_about)
@@ -1564,6 +1542,7 @@ def build_ui(self) -> None:
btn_zout = QPushButton("-")
btn_zout.clicked.connect(lambda: self.adjust_zoom(-1))
self.lbl_zoom = QLabel("150%")
+ self.lbl_zoom.setObjectName("lbl_zoom")
btn_zin = QPushButton("+")
btn_zin.clicked.connect(lambda: self.adjust_zoom(1))
tb_lay.addWidget(btn_zout)
@@ -1573,10 +1552,12 @@ def build_ui(self) -> None:
btn_prev = QPushButton("<")
btn_prev.clicked.connect(lambda: self.move_page(-1))
self.le_page = QLineEdit("0")
+ self.le_page.setObjectName("le_page")
self.le_page.setFixedWidth(40)
self.le_page.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.le_page.returnPressed.connect(self.jump_page)
self.lbl_tot = QLabel("/ 0")
+ self.lbl_tot.setObjectName("lbl_tot")
btn_next = QPushButton(">")
btn_next.clicked.connect(lambda: self.move_page(1))
tb_lay.addWidget(btn_prev)
@@ -1586,6 +1567,7 @@ def build_ui(self) -> None:
r_lay.addWidget(top_bar)
self.scene = QGraphicsScene()
self.view = PDFView(self.scene)
+ self.view.setObjectName("view")
self.view.rect_drawn.connect(self.user_draw_rect)
self.view.point_clicked.connect(self.user_click_pt)
self.view.zoom_req.connect(self.adjust_zoom)
@@ -1594,8 +1576,10 @@ def build_ui(self) -> None:
footer.setObjectName("Panel")
f_lay = QVBoxLayout(footer)
self.prog = QProgressBar()
+ self.prog.setObjectName("prog")
self.prog.setValue(0)
self.log = QTextEdit()
+ self.log.setObjectName("log")
self.log.setReadOnly(True)
self.log.setFixedHeight(80)
f_lay.addWidget(self.prog)
@@ -1777,6 +1761,9 @@ def _decide_suggestion(self, s: Dict[str, Any]) -> None:
b_yes = box.addButton("Censura", QMessageBox.ButtonRole.AcceptRole)
b_no = box.addButton("Scarta", QMessageBox.ButtonRole.DestructiveRole)
b_show = box.addButton("Mostra valore", QMessageBox.ButtonRole.ActionRole)
+ b_yes.setToolTip("Applica la censura a questo suggerimento.")
+ b_no.setToolTip("Scarta il suggerimento senza censurare nulla.")
+ b_show.setToolTip("Mostra il valore in chiaro prima di decidere.")
box.addButton("Annulla", QMessageBox.ButtonRole.RejectRole)
box.exec()
clicked = box.clickedButton()
@@ -1851,13 +1838,7 @@ def user_draw_rect(self, qrect: QRectF) -> None:
p = self.doc[self.page_num]
txt = text_under_rect(p, fitz.Rect(r))
if not txt:
- p.add_redact_annot(
- r,
- text="[ IMMAGINE RIMOSSA ]",
- align=1,
- fill=(0.9, 0.9, 0.9),
- fontsize=8,
- )
+ self._add_kind_redaction(p, r, "IMAGE")
else:
self._add_privacy_redaction(
p,
@@ -2014,6 +1995,10 @@ def _on_auto_update_result(self, release: Any) -> None:
if release is not None:
self._show_update_dialog(release, self)
+ def cmd_guide(self) -> None:
+ """Open the user guide (also bound to F1)."""
+ GuideDialog(self).exec()
+
def cmd_about(self) -> None:
"""Show about dialog."""
d = QDialog(self)
@@ -2094,6 +2079,7 @@ def start_check() -> None:
self._start_bg_thread(w)
btn_upd.clicked.connect(start_check)
+ apply_tooltips(d)
d.exec()
state["closed"] = True
@@ -2192,6 +2178,7 @@ def start_download() -> None:
self._start_bg_thread(w)
btn_dl.clicked.connect(start_download)
+ apply_tooltips(d)
d.exec()
state["closed"] = True
w = state["worker"]
@@ -2305,6 +2292,7 @@ def set_all(state: bool) -> None:
bb.accepted.connect(dlg.accept)
bb.rejected.connect(dlg.reject)
refresh()
+ apply_tooltips(dlg)
if dlg.exec() != QDialog.DialogCode.Accepted:
return None
chosen = [gid for gid in ALL_GROUP_IDS if boxes[gid].isChecked()]
@@ -2328,9 +2316,15 @@ def _select_privacy_export(self) -> Optional[PrivacyMode]:
anonymize_btn = box.addButton(
"Rimozione irreversibile", QMessageBox.ButtonRole.AcceptRole
)
+ anonymize_btn.setToolTip(
+ "Elimina per sempre i valori selezionati dal file. Non si può tornare indietro."
+ )
pseudonymize_btn = box.addButton(
"Pseudonimizzazione reversibile", QMessageBox.ButtonRole.ActionRole
)
+ pseudonymize_btn.setToolTip(
+ "Sostituisce i valori con segnaposto e crea una mappa cifrata per ripristinarli."
+ )
box.addButton(QMessageBox.StandardButton.Cancel)
box.exec()
clicked = box.clickedButton()
@@ -2375,21 +2369,53 @@ def _add_privacy_redaction(
original: str = "",
entity_type: str = "DATA",
fill: Tuple[float, float, float] = (0, 0, 0),
+ full: Optional[fitz.Rect] = None,
) -> None:
- """Add a redaction annotation carrying optional in-memory privacy data."""
+ """Add a redaction annotation carrying optional in-memory privacy data.
+
+ `full` is the whole box of the word when `rect` is a thinner band: the
+ export uses it to check that no glyph of the word survived.
+ """
annot = page.add_redact_annot(rect, fill=fill)
clean_original = " ".join((original or "").split())
if clean_original:
- payload = {
+ payload: Dict[str, Any] = {
"nullifypdf": 1,
"original": clean_original,
"entity_type": PlaceholderRegistry.normalize_entity_type(entity_type),
}
+ if full is not None and fitz.Rect(full) != fitz.Rect(rect):
+ payload["full"] = [full.x0, full.y0, full.x1, full.y1]
try:
annot.set_info(content=json.dumps(payload, ensure_ascii=False))
except Exception as e:
self.logger.debug(f"Could not store redaction metadata: {e}")
+ def _add_kind_redaction(self, page: Any, rect: fitz.Rect, kind: str) -> None:
+ """Redaction of a graphic area (image or signature) without overlay text.
+
+ The label is drawn after the redactions are applied (see
+ `_finish_labels_page`), so it is not lost in a small box.
+ """
+ annot = page.add_redact_annot(rect, fill=(0.9, 0.9, 0.9))
+ try:
+ annot.set_info(content=json.dumps(
+ {"nullifypdf": 1, "original": "", "entity_type": kind}
+ ))
+ except Exception as e:
+ self.logger.debug(f"Could not store {kind} metadata: {e}")
+
+ def _read_redaction_full(self, annot: Any) -> Optional[fitz.Rect]:
+ """The whole word box stored with a band annotation, if any."""
+ try:
+ raw = annot.info.get("content", "")
+ payload = json.loads(raw) if raw else {}
+ if payload.get("nullifypdf") == 1 and payload.get("full"):
+ return fitz.Rect(*[float(v) for v in payload["full"]])
+ except Exception:
+ return None
+ return None
+
def _read_redaction_payload(self, annot: Any) -> Dict[str, str]:
"""Read privacy data stored on a pending redaction annotation."""
try:
@@ -2422,14 +2448,11 @@ def _prepare_pseudonymized_page(
if annot.type[0] == PDF_ANNOT_REDACT:
page.delete_annot(annot)
for rect, payload, clipped_text, style in pending:
- if payload.get("entity_type") == "SIGNATURE":
- page.add_redact_annot(
- rect,
- text="[FIRMA RIMOSSA]",
- fill=(1, 1, 1),
- text_color=(0, 0, 0),
- align=1,
- fontsize=fit_redaction_fontsize("[FIRMA RIMOSSA]", rect),
+ if payload.get("entity_type") in ("SIGNATURE", "IMAGE"):
+ self._add_kind_redaction(page, rect, str(payload["entity_type"]))
+ self._export_labels.append(
+ (page.number, fitz.Rect(rect),
+ KIND_LABELS[str(payload["entity_type"])], None, None)
)
continue
clean_original = " ".join(
@@ -2451,37 +2474,47 @@ def _prepare_pseudonymized_page(
# the annotation is dropped when it cannot be laid out at 4 pt.
page.add_redact_annot(rect, fill=(1, 1, 1))
self._export_labels.append(
- (page.number, fitz.Rect(rect), placeholder, style)
+ (page.number, fitz.Rect(rect), compact_labels(placeholder),
+ style, placeholder)
)
else:
- page.add_redact_annot(
- rect,
- text="[IMAGE_REMOVED]",
- fill=(1, 1, 1),
- text_color=(0, 0, 0),
- align=1,
- fontsize=fit_redaction_fontsize("[IMAGE_REMOVED]", rect),
+ self._add_kind_redaction(page, rect, "IMAGE")
+ self._export_labels.append(
+ (page.number, fitz.Rect(rect), KIND_LABELS["IMAGE"], None, None)
)
- def _finish_pseudonymized_page(
- self, page: Any, registry: PlaceholderRegistry
+ def _prepare_label_annots(self, page: Any) -> None:
+ """Anonymize mode: remember the image / signature areas to label."""
+ for annot in page.annots() or []:
+ if annot.type[0] != PDF_ANNOT_REDACT:
+ continue
+ kind = self._read_redaction_payload(annot).get("entity_type")
+ if kind in KIND_LABELS:
+ self._export_labels.append(
+ (page.number, fitz.Rect(annot.rect), KIND_LABELS[kind], None, None)
+ )
+
+ def _finish_labels_page(
+ self, page: Any, registry: Optional[PlaceholderRegistry] = None
) -> None:
- """Draw the placeholders of `page` once its redactions are applied."""
- for page_no, rect, placeholder, style in self._export_labels:
+ """Draw placeholders and image/signature labels once redactions are applied."""
+ for page_no, rect, labels, style, placeholder in self._export_labels:
if page_no != page.number:
continue
drawn = write_label(
- page,
- rect,
- compact_labels(placeholder),
- style=style,
- allowance=free_space_right(page, rect),
+ page, rect, labels, style=style,
+ allowance=free_space_right(page, rect) if placeholder else 0.0,
)
- if drawn != placeholder:
+ if placeholder and registry is not None and drawn != placeholder:
registry.set_label(
placeholder, (rect.x0, rect.y0, rect.x1, rect.y1), drawn
)
+ def _finish_pseudonymized_page(
+ self, page: Any, registry: PlaceholderRegistry
+ ) -> None:
+ self._finish_labels_page(page, registry)
+
@Slot(int, object)
def apply_ai_to_page(self, i: int, detections: Any) -> None:
"""Apply AI-detected redactions to page.
@@ -2539,13 +2572,7 @@ def already_covered(r: fitz.Rect) -> bool:
ir = fitz.Rect(img["bbox"])
if already_covered(ir):
continue
- page.add_redact_annot(
- ir,
- text="[ IMMAGINE RIMOSSA ]",
- align=1,
- fill=(0.9, 0.9, 0.9),
- fontsize=8,
- )
+ self._add_kind_redaction(page, ir, "IMAGE")
e_rects.append(ir)
try:
has_images = bool(page.get_image_info(hashes=False))
@@ -2583,14 +2610,7 @@ def already_covered(r: fitz.Rect) -> bool:
for r in rects:
if already_covered(r):
continue
- annot = page.add_redact_annot(r, fill=(0.9, 0.9, 0.9))
- try:
- annot.set_info(content=json.dumps({
- "nullifypdf": 1, "original": "",
- "entity_type": "SIGNATURE",
- }))
- except Exception as e:
- self.logger.debug(f"Could not store signature metadata: {e}")
+ self._add_kind_redaction(page, r, "SIGNATURE")
e_rects.append(r)
continue
if not rects and word:
@@ -2610,13 +2630,18 @@ def already_covered(r: fitz.Rect) -> bool:
for pr in protected
)
]
- for r in rects:
+ for k, r in enumerate(rects):
if not already_covered(r):
+ full_box = (
+ fitz.Rect(*fulls[k])
+ if (k < len(fulls) and raw_rects) else None
+ )
self._add_privacy_redaction(
page,
r,
original=word,
entity_type=entity_type,
+ full=full_box,
)
e_rects.append(r)
@@ -2690,6 +2715,11 @@ def _select_pseudonym_style(self) -> Optional[Tuple[str, Optional[bytes]]]:
b_opaque = box.addButton("Generico", QMessageBox.ButtonRole.ActionRole)
b_keyed = box.addButton("Coerente (chiave)", QMessageBox.ButtonRole.ActionRole)
b_manage = box.addButton("Gestisci chiave...", QMessageBox.ButtonRole.ActionRole)
+ b_random.setToolTip("Segnaposto casuali: non rivelano ordine né numero dei valori (consigliato).")
+ b_counter.setToolTip("PERSON_001, PERSON_002...: rivelano quanti valori distinti ci sono e l'ordine.")
+ b_opaque.setToolTip("Segnaposto generici: nascondono anche il tipo di dato.")
+ b_keyed.setToolTip("Lo stesso valore ha lo stesso segnaposto in ogni documento creato con la stessa chiave.")
+ b_manage.setToolTip("Esporta o importa la chiave dei segnaposto coerenti, protetta da password.")
box.addButton(QMessageBox.StandardButton.Cancel)
box.exec()
clicked = box.clickedButton()
@@ -2771,6 +2801,8 @@ def _pseudonym_key_dialog(self) -> None:
)
b_exp = box.addButton("Esporta chiave...", QMessageBox.ButtonRole.ActionRole)
b_imp = box.addButton("Importa chiave...", QMessageBox.ButtonRole.ActionRole)
+ b_exp.setToolTip("Salva una copia della chiave protetta da password.")
+ b_imp.setToolTip("Carica una copia della chiave esportata da un altro computer.")
box.addButton(QMessageBox.StandardButton.Close)
box.exec()
clicked = box.clickedButton()
@@ -2844,9 +2876,12 @@ def _offer_recovery_key(self) -> Optional[bytes]:
field.setReadOnly(True)
lay.addWidget(field)
row = QHBoxLayout()
+ field.setToolTip("Chiave di recupero: mostrata una sola volta e non salvata dal programma.")
btn_copy = QPushButton("Copia")
+ btn_copy.setToolTip("Copia la chiave negli appunti (vengono svuotati alla chiusura).")
btn_copy.clicked.connect(lambda: QApplication.clipboard().setText(shown))
btn_save = QPushButton("Salva su file...")
+ btn_save.setToolTip("Salva la chiave in un file di testo, da conservare lontano dalla mappa.")
def save_file() -> None:
path, _ = QFileDialog.getSaveFileName(
@@ -2862,6 +2897,7 @@ def save_file() -> None:
chk = QCheckBox(
"Ho conservato la chiave di recupero in un luogo sicuro, separato dalla mappa"
)
+ chk.setToolTip("Spunta per confermare: serve per proseguire.")
lay.addWidget(chk)
bb = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok)
ok_btn = bb.button(QDialogButtonBox.StandardButton.Ok)
@@ -2886,6 +2922,8 @@ def _ask_decrypt_secret(self) -> Optional[Tuple[Optional[str], Optional[bytes]]]
field.setEchoMode(QLineEdit.EchoMode.Password)
lay.addWidget(field)
use_rk = QCheckBox("Usa la chiave di recupero")
+ use_rk.setToolTip("Apre la mappa con la chiave di recupero (NPDF-...) invece che con la password.")
+ field.setToolTip("Password della mappa, oppure la chiave di recupero se hai spuntato l'opzione.")
def toggled(on: bool) -> None:
label.setText(
@@ -3004,6 +3042,8 @@ def cmd_export(self) -> None:
)
b_review = box.addButton("Rivedi ora", QMessageBox.ButtonRole.AcceptRole)
b_go = box.addButton("Esporta comunque", QMessageBox.ButtonRole.DestructiveRole)
+ b_review.setToolTip("Apre l'elenco dei suggerimenti prima di esportare.")
+ b_go.setToolTip("Esporta senza censurare i suggerimenti non revisionati.")
box.addButton(QMessageBox.StandardButton.Cancel)
box.exec()
if box.clickedButton() == b_review:
@@ -3100,11 +3140,11 @@ def cmd_export(self) -> None:
source_sha256 = sha256_file(source_name) if os.path.exists(source_name) else ""
prepare = (
(lambda page: self._prepare_pseudonymized_page(page, registry))
- if mode == PrivacyMode.PSEUDONYMIZE else None
+ if mode == PrivacyMode.PSEUDONYMIZE else self._prepare_label_annots
)
finish = (
- (lambda page: self._finish_pseudonymized_page(page, registry))
- if mode == PrivacyMode.PSEUDONYMIZE else None
+ (lambda page: self._finish_labels_page(page, registry))
+ if mode == PrivacyMode.PSEUDONYMIZE else self._finish_labels_page
)
self._export_labels = []
with QMutexLocker(self.doc_mutex):
@@ -3115,6 +3155,7 @@ def cmd_export(self) -> None:
p,
prepare_page=prepare,
finish_page=finish,
+ read_full=self._read_redaction_full,
policy=policy,
removed_values=removed_values,
temp_dir=private_temp_dir(self.config_dir),
diff --git a/README.md b/README.md
index e4af195..50971b2 100644
--- a/README.md
+++ b/README.md
@@ -37,6 +37,7 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi
- 🗄️ **Fluid UI & Thread-Safe** — PySide6 modern dark-mode interface with zero UI freezing. Text extraction in worker thread with QMutex serialization
- 📖 **Persistent Dictionaries** — Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with O(1) fast-path matching
- 🛡️ **Privacy Export Modes** — Choose irreversible removal or reversible pseudonymization with a separately encrypted restore map; the exported file is sanitised (annotations, attachments, scripts, metadata, hidden text) and verified
+- 📖 **Built-in guide and tooltips** — Press F1 for an Italian guide to every function; every control shows a description on hover
- 🔍 **Review of uncertain hits** — Detections without enough evidence are suggested for review instead of being applied or lost silently
- 🔎 **OCR Support** — Bundled EN/IT Tesseract language data for scanned PDFs
- 🖼️ **Blindfold Mode** — One-click image/logo censoring with professional placeholder: `[ IMAGE REMOVED ]`
@@ -263,5 +264,5 @@ Want to help improve NullifyPDF? See [CONTRIBUTING.md](./CONTRIBUTING.md) for gu
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-10*
*[User Guide →](./USER_GUIDE.md)*
diff --git a/SECURITY.md b/SECURITY.md
index 3ae64aa..54bd5b9 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -39,7 +39,7 @@ When you export a PDF with redactions:
1. ✅ **Redactions applied** — selected text/image areas are removed from the content stream, not just covered
2. ✅ **Sanitiser** — annotations, embedded files, external links, bookmarks, JavaScript, tagged-PDF alternative text, thumbnails, document and XMP metadata, invisible text, text outside the visible page box and the trailer ID are removed; hidden layers are shown before the scan so their text is analysed
-3. ✅ **Verification** — the written file is re-opened and checked for any removed value in the text layer, every object and decompressed stream, annotations, attachments, bookmarks and metadata; findings list locations, never the values
+3. ✅ **Verification** — every redacted word is checked glyph by glyph (the number of characters actually deleted must match the measured deletion rule, and characters a thin box cannot reach are reported), then the written file is re-opened and checked for any removed value in the text layer, every object and decompressed stream, annotations, attachments, bookmarks and metadata; findings list locations, never the values
4. ✅ **Pseudonymization maps encrypted** — restore maps are stored separately and encrypted with a user password (scrypt key derivation, authenticated encryption)
5. ⚙️ **Optional width masking** — the space left by removed text can be rounded so its width is not recoverable (off by default: it can shift the following text slightly)
@@ -60,7 +60,7 @@ Limits you should know about:
- Original PDF kept in memory only while open
- AI results (detected entities) stored only in memory during the session; suggestions are never written to disk
- The restore map is written with owner-only permissions where the platform supports it; keep it away from the PDF and from synchronised folders
-- The blocklist and allowlist (`~/.nullifypdf/*.v2.enc`) may themselves contain personal data, so they are encrypted with AES-256-GCM under a master key protected by Windows DPAPI, the OS keyring (optional `keyring` package) or, as a fallback, a local key file. Terms can expire and are then purged. DPAPI and the keyring bind the data to the user account and machine; the key file only protects against copying a single data file, not against someone who can read the user's profile. Overwriting migrated plaintext files is best effort (SSD wear levelling and backups may keep copies)
+- The blocklist and allowlist (`~/.nullifypdf/*.v2.enc`) may themselves contain personal data, so they are encrypted with AES-256-GCM under a master key protected by Windows DPAPI, the OS keyring (`keyring` package, installed on macOS and Linux; headless Linux without a Secret Service falls back to the key file) or, as a fallback, a local key file. Terms can expire and are then purged. DPAPI and the keyring bind the data to the user account and machine; the key file only protects against copying a single data file, not against someone who can read the user's profile. Overwriting migrated plaintext files is best effort (SSD wear levelling and backups may keep copies)
- If the keystore cannot be read (profile copied to another account or PC) nothing is deleted or overwritten: the lists stay empty and the encrypted files untouched
- The pseudonym key is stored sealed under the same master key; a password-protected copy can be exported for another computer
- Restore maps (version 3 envelope): a random data key encrypts the map with AES-256-GCM; it is wrapped by a password-derived key (Argon2id, scrypt if unavailable) and optionally by a recovery key (256-bit, shown once, never stored). Key-derivation parameters read from a file are bounded before use
@@ -299,5 +299,5 @@ For **security vulnerabilities only**:
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-10*
*← [Contributing](./CONTRIBUTING.md) | [Back to README →](./README.md)*
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index a294112..24950d6 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -15,3 +15,14 @@ https://www.istat.it/classificazione/codici-dei-comuni-delle-province-e-delle-re
The file is generated by `scripts/build_lexicon_data.py`; the SHA-256 of the
source workbook is recorded in the generated module.
+
+## Given names and surnames (Wikidata)
+
+`lexicon_data_it.py` also contains Italian given names and surnames taken from
+Wikidata (items that have an article on the Italian Wikipedia).
+
+- License: Creative Commons Zero (CC0 1.0), all structured data of Wikidata
+ (https://www.wikidata.org/wiki/Wikidata:Licensing).
+- The lists are small (a few hundred given names and about one thousand
+ surnames) and are only used to *confirm* a name; a name missing from them
+ never causes a hit to be rejected.
diff --git a/USER_GUIDE.md b/USER_GUIDE.md
index 26d9630..b3cf0da 100644
--- a/USER_GUIDE.md
+++ b/USER_GUIDE.md
@@ -20,6 +20,9 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel
---
+> [!TIP]
+> The program has its own guide in Italian: press **F1** or the **Guida** button in the sidebar. Resting the pointer on any button, option or field shows a short description.
+
## Step 1️⃣ — Load Your PDF
1. Click the blue **"Apri PDF"** button in the top left
@@ -305,5 +308,5 @@ python3.13 NullifyPDF.py
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-10*
*← [Back to README](./README.md) | [Troubleshooting →](./TROUBLESHOOTING.md)*
diff --git a/build_local.py b/build_local.py
index 773527b..cf2b9b2 100644
--- a/build_local.py
+++ b/build_local.py
@@ -393,11 +393,12 @@ def build_app() -> None:
if sys_os == "Darwin":
spec_content = f"""# -*- mode: python ; coding: utf-8 -*-
+import sys
from PyInstaller.utils.hooks import collect_all
datas = {datas_literal}
binaries = []
hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']:
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
@@ -409,11 +410,12 @@ def build_app() -> None:
"""
else:
spec_content = f"""# -*- mode: python ; coding: utf-8 -*-
+import sys
from PyInstaller.utils.hooks import collect_all
datas = {datas_literal}
binaries = []
hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md']:
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
diff --git a/dictionary_dialog.py b/dictionary_dialog.py
index a8e84a6..ec26eff 100644
--- a/dictionary_dialog.py
+++ b/dictionary_dialog.py
@@ -23,6 +23,7 @@
QWidget,
)
+from help_guide import apply_tooltips
from word_lists import ListEntry, PDFListManager, MIN_TERM_LENGTH
TTL_CHOICES = [("Mai", None), ("30 giorni", 30), ("90 giorni", 90),
@@ -50,6 +51,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None:
self.saved_allowlist: Optional[set] = None
lay = QVBoxLayout(self)
self.status = QLabel(manager.protector_label())
+ self.status.setObjectName("dict_status")
lay.addWidget(self.status)
if manager.keystore_error:
warn = QLabel(
@@ -60,6 +62,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None:
warn.setStyleSheet("color: #f59e0b;")
lay.addWidget(warn)
self.tabs = QTabWidget()
+ self.tabs.setObjectName("dict_tabs")
self.tables: Dict[str, QTableWidget] = {}
for kind, title in KINDS:
tab = QWidget()
@@ -68,6 +71,8 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None:
table.setHorizontalHeaderLabels(["Termine", "Aggiunto il", "Scade il (AAAA-MM-GG)"])
table.horizontalHeader().setStretchLastSection(True)
table.setColumnWidth(COL_TERM, 260)
+ table.setObjectName("dict_table")
+ table.verticalHeader().setVisible(False)
self.tables[kind] = table
tl.addWidget(table)
row = QHBoxLayout()
@@ -90,6 +95,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None:
ttl_row = QHBoxLayout()
ttl_row.addWidget(QLabel("Scadenza predefinita dei nuovi termini:"))
self.ttl = QComboBox()
+ self.ttl.setObjectName("dict_ttl")
current = manager.default_ttl_days()
for i, (label, days) in enumerate(TTL_CHOICES):
self.ttl.addItem(label, days)
@@ -102,6 +108,7 @@ def __init__(self, manager: PDFListManager, parent: Any = None) -> None:
btn.setObjectName("Primary")
btn.clicked.connect(self.save)
lay.addWidget(btn)
+ apply_tooltips(self)
def _on_button(self) -> None:
sender = self.sender()
diff --git a/export_engine.py b/export_engine.py
index 629dc01..115613a 100644
--- a/export_engine.py
+++ b/export_engine.py
@@ -43,6 +43,7 @@ class ExportResult:
sanitize: SanitizeReport = field(default_factory=SanitizeReport)
verification: VerificationReport = field(default_factory=VerificationReport)
pages: int = 0
+ surviving_glyphs: int = 0
def neutral_output_name(mode_value: str, now: Optional[datetime.datetime] = None) -> str:
@@ -149,6 +150,90 @@ def text_outside_redactions(doc: Any) -> str:
return " ".join(chunks)
+def _line_chars(page: Any, baseline: float, tol: float = 1.5) -> List[Dict[str, Any]]:
+ """Non-space characters of `page` on one baseline, left to right."""
+ try:
+ raw = page.get_text("rawdict")
+ except Exception:
+ return []
+ out: List[Dict[str, Any]] = []
+ for block in raw.get("blocks", []):
+ if block.get("type") != 0:
+ continue
+ for line in block.get("lines", []):
+ for span in line.get("spans", []):
+ for ch in span.get("chars", []):
+ if str(ch.get("c", "")).strip() and abs(float(ch["origin"][1]) - baseline) <= tol:
+ out.append(ch)
+ out.sort(key=lambda c: float(c["origin"][0]))
+ return out
+
+
+def _expected_deletions(
+ page: Any, read_full: Callable[[Any], Optional[Any]]
+) -> List[Dict[str, Any]]:
+ """Per redaction with a stored whole-word box: what must disappear.
+
+ After apply_redactions() every glyph of the word must be gone: a thin
+ band that misses one glyph (a raised or smaller character) would leave
+ part of the value behind, and a text search cannot see a single
+ character. Two independent checks, neither of which relies on glyph
+ positions (MuPDF rewrites and shifts the rest of the line):
+
+ * predicted survivors: target glyphs whose deletion core the band does
+ not reach (the measured rule);
+ * rule check: on that line, the number of glyphs that actually vanish
+ must be at least the number the rule predicts. If a PyMuPDF release
+ changes the rule, this trips instead of leaving the export silently
+ unsafe.
+ """
+ from pii_detection import core_hits, target_chars
+
+ bands = [
+ pymupdf.Rect(a.rect) for a in page.annots() or []
+ if a.type[0] == PDF_ANNOT_REDACT
+ ]
+ expected: List[Dict[str, Any]] = []
+ for annot in page.annots() or []:
+ if annot.type[0] != PDF_ANNOT_REDACT:
+ continue
+ full = read_full(annot)
+ if full is None:
+ continue
+ targets = target_chars(page, pymupdf.Rect(full))
+ if not targets:
+ continue
+ baseline = float(targets[0]["origin"][1])
+ line = _line_chars(page, baseline)
+ predicted = sum(
+ 1 for c in line if any(core_hits(tuple(c["bbox"]), b) for b in bands)
+ )
+ missed = sum(
+ 1 for t in targets
+ if not any(core_hits(tuple(t["bbox"]), b) for b in bands)
+ )
+ expected.append({
+ "baseline": baseline, "n_pre": len(line),
+ "predicted_deleted": predicted, "missed": missed,
+ })
+ return expected
+
+
+def _count_survivors(page: Any, expected: List[Dict[str, Any]]) -> int:
+ survivors = 0
+ seen_lines = set()
+ for item in expected:
+ survivors += int(item["missed"])
+ key = round(item["baseline"], 1)
+ if key in seen_lines:
+ continue # several words on one line share the same count
+ seen_lines.add(key)
+ actual = item["n_pre"] - len(_line_chars(page, item["baseline"]))
+ if actual < item["predicted_deleted"]:
+ survivors += item["predicted_deleted"] - actual
+ return survivors
+
+
def _open_copy(
src_doc: Any, temp_dir: Optional[pathlib.Path]
) -> tuple[Any, Optional[str]]:
@@ -180,6 +265,7 @@ def export_document(
*,
prepare_page: Optional[Callable[[Any], None]] = None,
finish_page: Optional[Callable[[Any], None]] = None,
+ read_full: Optional[Callable[[Any], Optional[Any]]] = None,
policy: Optional[SanitizePolicy] = None,
removed_values: Iterable[str] = (),
temp_dir: Optional[pathlib.Path] = None,
@@ -190,7 +276,10 @@ def export_document(
kept_text = text_outside_redactions(src_doc)
ex_doc, tmp = _open_copy(src_doc, temp_dir)
try:
+ survivors_to_check: Dict[int, List[Dict[str, Any]]] = {}
for page in ex_doc.pages():
+ if read_full is not None:
+ survivors_to_check[page.number] = _expected_deletions(page, read_full)
if prepare_page is not None:
prepare_page(page)
redact_rects = [
@@ -207,6 +296,9 @@ def export_document(
# PDF_REDACT_IMAGE_PIXELS blanks only the pixels under each rect;
# IMAGE_REMOVE would delete a whole scanned page for one word.
page.apply_redactions(images=PDF_REDACT_IMAGE_PIXELS, graphics=True)
+ result.surviving_glyphs += _count_survivors(
+ page, survivors_to_check.get(page.number, [])
+ )
if finish_page is not None:
# Text drawn after the redactions are applied: placeholders
# must not go through apply_redactions(), which silently
@@ -233,6 +325,10 @@ def export_document(
pass
result.output_sha256 = sha256_of(out_path)
result.verification = verify_export(out_path, removed_values, kept_text)
+ if result.surviving_glyphs:
+ result.verification.add(
+ f"{result.surviving_glyphs} glyph(s) survived redaction"
+ )
return result
diff --git a/help_guide.py b/help_guide.py
new file mode 100644
index 0000000..1ce641e
--- /dev/null
+++ b/help_guide.py
@@ -0,0 +1,415 @@
+"""In-app user guide (Italian) and tooltips for every GUI element (Qt).
+
+The guide is opened from the "Guida" button or with F1. Tooltips are applied
+by :func:`apply_tooltips`, which looks widgets up by their visible text or
+object name; a test checks that no widget of the main window or of the
+dialogs is left without one.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List, Tuple
+
+from PySide6.QtCore import Qt
+from PySide6.QtWidgets import (
+ QAbstractButton,
+ QComboBox,
+ QDialog,
+ QDoubleSpinBox,
+ QGraphicsView,
+ QHBoxLayout,
+ QLabel,
+ QLineEdit,
+ QListWidget,
+ QProgressBar,
+ QPushButton,
+ QTableWidget,
+ QTabWidget,
+ QTextBrowser,
+ QTextEdit,
+ QVBoxLayout,
+ QWidget,
+)
+
+# --------------------------------------------------------------------------
+# Tooltips
+# --------------------------------------------------------------------------
+
+# Exact visible text (or object name) -> tooltip.
+TOOLTIPS: Dict[str, str] = {
+ # Sidebar
+ "Apri PDF": "Apre un file PDF da elaborare. Puoi anche trascinarlo sulla finestra.",
+ "Modello lingua:": "Lingua dei modelli di riconoscimento dei dati personali.",
+ "EN": "Usa il modello inglese. Adatto a documenti in inglese.",
+ "IT": "Usa il modello italiano. Consigliato per documenti in italiano: il modello inglese aggiunge falsi positivi.",
+ "BOTH": "Usa entrambi i modelli (inglese e italiano). Più lento; utile per documenti misti.",
+ "Oscura Immagini": "Oscura anche tutte le immagini della pagina (loghi, firme, foto) quando applichi Auto Redact.",
+ "OCR PDF scansionati": "Legge con l'OCR le pagine scansionate e le immagini incorporate, così anche il testo nelle immagini viene analizzato.",
+ "Mostra suggerimenti": "Contorna in arancione, nell'anteprima, i suggerimenti ancora da rivedere. Servono solo a video: non finiscono nel file esportato.",
+ "Profilo documenti sanitari": "Cerca anche date di nascita, numero tessera sanitaria, ricetta e codici di esenzione, e propone firme e timbri. I casi incerti diventano suggerimenti invece di essere scartati.",
+ "Maschera larghezza testo rimosso": "Arrotonda lo spazio lasciato dal testo rimosso, che altrimenti ne rivela la larghezza. Può spostare di poco il testo seguente.",
+ "Dizionari": "Modifica le parole da censurare sempre (blocklist) e da non censurare mai (allowlist), con scadenza.",
+ "Auto Redact": "Analizza il documento e segna i dati personali da censurare. Nulla viene cancellato finché non esporti.",
+ "Rivedi suggerimenti": "Apre l'elenco dei rilevamenti incerti per decidere uno per uno se censurarli.",
+ "Pulisci Pagina": "Rimuove tutte le censure segnate sulla pagina corrente.",
+ "Esporta PDF": "Crea il PDF finale: rimozione irreversibile oppure pseudonimizzazione reversibile. Il file viene pulito e verificato.",
+ "Ricostruisci PDF": "Rimette i valori originali in un PDF pseudonimizzato usando la sua mappa di ripristino e la password (o la chiave di recupero).",
+ "Elimina dati locali": "Cancella log, registri di esportazione e file temporanei salvati nella cartella dell'utente. Chiede conferma per dizionari e chiavi.",
+ "Guida": "Apre la guida all'uso del programma (scorciatoia: F1).",
+ "Info": "Versione del programma, controllo degli aggiornamenti e collegamento al repository.",
+ "Esci": "Chiude il programma.",
+ "NullifyPDF": "NullifyPDF: censura e pseudonimizzazione di PDF in locale.",
+ "Privacy Edition": "Tutta l'elaborazione avviene sul tuo computer: nessun documento viene inviato in rete.",
+ # Viewer toolbar
+ "Visualizzatore Documento": "Anteprima del PDF. Trascina per segnare un'area da censurare; clicca su una censura per rimuoverla; Ctrl + rotella per lo zoom.",
+ "-": "Riduce lo zoom.",
+ "+": "Aumenta lo zoom.",
+ "<": "Va alla pagina precedente.",
+ ">": "Va alla pagina successiva.",
+ "le_page": "Numero della pagina corrente. Scrivine un altro e premi Invio per saltarci.",
+ "lbl_zoom": "Livello di zoom corrente.",
+ "lbl_tot": "Numero totale di pagine del documento.",
+ "view": "Anteprima del PDF. Trascina per segnare un'area da censurare; clicca su una censura o su un contorno arancione per decidere; Ctrl + rotella per lo zoom.",
+ "prog": "Avanzamento dell'analisi, pagina per pagina.",
+ "log": "Messaggi del programma: avanzamento, avvisi ed errori. Non contiene i dati personali trovati.",
+ # Review dialog
+ "Censura": "Applica la censura a questo elemento.",
+ "Pag.": "Pagina in cui è stato trovato l'elemento.",
+ "Valore": "Valore trovato, mascherato finché non spunti «Mostra i valori». Per le aree grafiche indica la zona.",
+ "Tipo": "Tipo di dato rilevato (persona, luogo, telefono, firma...).",
+ "Punteggio": "Affidabilità del rilevamento, da 0 a 1.",
+ "Motivi": "Indizi che hanno portato a questo punteggio (contesto, parola comune, inizio riga...).",
+ "Mostra i valori": "Mostra in chiaro i valori trovati nella tabella.",
+ "Seleziona tutti": "Spunta tutti gli elementi dell'elenco.",
+ "Nessuno": "Toglie la spunta a tutti gli elementi.",
+ "Seleziona ≥ soglia": "Spunta gli elementi con punteggio pari o superiore alla soglia indicata a fianco.",
+ "review_threshold": "Soglia di punteggio usata da «Seleziona ≥ soglia».",
+ "Applica selezionati": "Censura gli elementi spuntati e scarta gli altri.",
+ "Rivedi più tardi": "Chiude l'elenco lasciando i suggerimenti in sospeso.",
+ "review_table": "Rilevamenti incerti: spunta quelli da censurare. Doppio clic su una riga per andare alla pagina.",
+ # Dictionary dialog
+ "dict_status": "Come sono protetti i dizionari su questo computer.",
+ "Aggiungi": "Aggiunge un termine alla lista.",
+ "Rimuovi": "Elimina i termini selezionati dalla lista.",
+ "Importa da testo…": "Incolla più termini, uno per riga.",
+ "Esporta in chiaro…": "Salva la lista in un file di testo non cifrato (contiene dati personali).",
+ "Scadenza predefinita dei nuovi termini:": "Dopo quanto tempo i nuovi termini scadono e vengono eliminati da soli.",
+ "dict_ttl": "Scadenza predefinita dei nuovi termini (anche quelli aggiunti dalla pagina).",
+ "dict_tabs": "Blocklist: parole sempre censurate. Allowlist: parole mai censurate.",
+ "dict_table": "Termini della lista con data di inserimento e scadenza (AAAA-MM-GG oppure Mai). Doppio clic per modificare.",
+ "Salva": "Salva le modifiche ai dizionari.",
+ # Entity dialog
+ "Seleziona tutto": "Seleziona tutti i tipi di dato.",
+ "Deseleziona tutto": "Deseleziona tutti i tipi di dato.",
+ "Seleziona i dati da censurare / pseudonimizzare:": "Scegli quali tipi di dati cercare. La scelta viene ricordata.",
+ # Guide
+ "Chiudi": "Chiude questa finestra.",
+ "OK": "Conferma la scelta e prosegue.",
+ "Cancel": "Chiude senza confermare.",
+ "Annulla": "Chiude senza confermare.",
+ "Controlla aggiornamenti": "Cerca su GitHub se esiste una versione più recente del programma.",
+ "Controlla ad ogni avvio": "Controlla gli aggiornamenti a ogni avvio. Disattivato di default; mostra qualcosa solo se c'è una versione nuova.",
+ "Download": "Scarica nella cartella Download il file di installazione della nuova versione.",
+}
+
+# Visible text of the entity checkboxes -> tooltip.
+ENTITY_TOOLTIPS: Dict[str, str] = {
+ "Nomi e cognomi": "Persone citate nel testo. Le parole comuni e i titoli vengono scartati; i casi incerti vanno in revisione.",
+ "Luoghi e indirizzi": "Città, indirizzi con numero civico e CAP (solo in un contesto di indirizzo).",
+ "Email": "Indirizzi di posta elettronica.",
+ "Telefono": "Numeri di telefono italiani e internazionali (date e numeri d'ordine sono esclusi).",
+ "IBAN": "Coordinate bancarie IBAN valide.",
+ "Carte di credito": "Numeri di carta di credito con controllo di validità.",
+ "Wallet crypto": "Indirizzi di portafogli di criptovalute.",
+ "Codice fiscale": "Codice fiscale italiano, con controllo del carattere finale.",
+ "Partita IVA": "Partita IVA italiana, con controllo di validità e parola di contesto.",
+ "Patente": "Numero di patente di guida (richiede la parola «patente» vicino).",
+ "Carta d'identità": "Numero di carta d'identità (richiede il contesto).",
+ "Passaporto": "Numero di passaporto (richiede il contesto).",
+ "Date di nascita": "Date precedute da «nato/a il», «nascita»...",
+ "Identificativi sanitari": "Numero della tessera sanitaria (20 cifre) e della ricetta elettronica (NRE).",
+ "Codici di esenzione": "Codici di esenzione dal ticket (richiedono la parola «esenzione»).",
+}
+
+
+def tooltip_for(widget: QWidget) -> str:
+ """Tooltip text for a widget, from its object name or visible text."""
+ name = widget.objectName()
+ if name and name in TOOLTIPS:
+ return TOOLTIPS[name]
+ text = ""
+ if isinstance(widget, (QAbstractButton, QLabel)):
+ text = widget.text() if hasattr(widget, "text") else ""
+ text = re.sub(r"<[^>]+>", "", text).strip()
+ if text in TOOLTIPS:
+ return TOOLTIPS[text]
+ if text in ENTITY_TOOLTIPS:
+ return ENTITY_TOOLTIPS[text]
+ if text.startswith("Rivedi suggerimenti"):
+ return TOOLTIPS["Rivedi suggerimenti"]
+ if text.startswith("Archivio protetto") or text.startswith("File chiave") or text == "Non cifrato" or text.startswith("Non ancora creato"):
+ return TOOLTIPS["dict_status"]
+ if text.endswith("%"):
+ return TOOLTIPS["lbl_zoom"]
+ if text.startswith("/ "):
+ return TOOLTIPS["lbl_tot"]
+ return _generic_tooltip(widget, text)
+
+
+def _generic_tooltip(widget: QWidget, text: str) -> str:
+ """Texts that vary at run time (Info and update dialogs)."""
+ low = text.lower()
+ if isinstance(widget, QLabel):
+ if re.match(r"^v\d", low):
+ return "Versione del programma installata."
+ if "censura e pseudonimizzazione" in low:
+ return "Informazioni sul programma, sull'autore e sulla licenza."
+ if "github.com" in low or "repository" in low:
+ return "Pagina del progetto su GitHub."
+ if low.startswith("nuova versione"):
+ return "Versione più recente disponibile."
+ if low.startswith(("disponibile", "nessun aggiornamento", "controllo", "scaricato", "download")):
+ return "Esito dell'operazione in corso."
+ if low.startswith("questi elementi"):
+ return "Spiega perché questi elementi sono suggerimenti e non censure automatiche."
+ if low.startswith("http"):
+ return "Collegamento alla pagina della release."
+ return ""
+
+
+# Widgets that deliberately have no tooltip: it would get in the way while reading.
+NO_TOOLTIP = frozenset({"guide_topics", "guide_text"})
+
+_INTERACTIVE: Tuple[type, ...] = (
+ QAbstractButton, QLabel, QLineEdit, QProgressBar, QTextEdit, QGraphicsView,
+ QTableWidget, QComboBox, QDoubleSpinBox, QTabWidget, QListWidget, QTextBrowser,
+)
+
+
+def apply_tooltips(root: QWidget) -> int:
+ """Set a tooltip on every widget under `root` that has none. Returns the count."""
+ n = 0
+ for w in [root, *root.findChildren(QWidget)]:
+ if not isinstance(w, _INTERACTIVE) or w.toolTip() or w.objectName() in NO_TOOLTIP:
+ continue
+ tip = tooltip_for(w)
+ if tip:
+ w.setToolTip(tip)
+ n += 1
+ return n
+
+
+def widgets_without_tooltip(root: QWidget) -> List[str]:
+ """Descriptions of interactive widgets that still have no tooltip."""
+ missing: List[str] = []
+ for w in root.findChildren(QWidget):
+ if isinstance(w, _INTERACTIVE) and not w.toolTip() and w.objectName() not in NO_TOOLTIP:
+ # Icons and the inner widgets of composite controls carry no text.
+ if isinstance(w, QLabel) and not w.text().strip():
+ continue
+ if w.parent() is not None and type(w.parent()).__name__ in (
+ "QAbstractSpinBox", "QTabBar", "QHeaderView", "QScrollBar",
+ "QDoubleSpinBox", "QComboBox", "QTableWidget",
+ ):
+ continue
+ if type(w).__name__ in ("QLineEdit",) and w.parent() is not None and isinstance(
+ w.parent(), (QDoubleSpinBox, QComboBox)
+ ):
+ continue
+ missing.append(f"{type(w).__name__}({getattr(w, 'text', lambda: '')()!r}, {w.objectName()!r})")
+ return missing
+
+
+# --------------------------------------------------------------------------
+# Guide
+# --------------------------------------------------------------------------
+
+GUIDE_TOPICS: List[Tuple[str, str]] = [
+ ("Introduzione", """
+Introduzione
+NullifyPDF individua i dati personali in un PDF e li censura in modo irreversibile oppure li sostituisce con segnaposto reversibili (pseudonimizzazione). Tutta l'elaborazione avviene sul tuo computer: nessun documento viene inviato in rete.
+Flusso consigliato
+
+- Apri PDF (o trascina il file sulla finestra).
+- Scegli la lingua del documento.
+- Premi Auto Redact e scegli i tipi di dati da cercare.
+- Rivedi i suggerimenti incerti e correggi a mano ciò che serve.
+- Premi Esporta PDF, scegli la modalità e controlla il risultato.
+
+Finché non esporti, le censure sono solo segni sull'anteprima: il documento originale non viene modificato.
+"""),
+ ("Aprire un PDF e muoversi", """
+Aprire un PDF e muoversi
+
+- Apri PDF carica un file; si può anche trascinarlo sulla finestra. I PDF protetti da password non sono supportati.
+- < e > cambiano pagina; nel campo numerico puoi scrivere una pagina e premere Invio.
+- + e - cambiano lo zoom; anche Ctrl + rotella del mouse.
+- Se il PDF sembra una scansione (senza testo selezionabile) il programma lo segnala: attiva OCR PDF scansionati prima di analizzarlo.
+
+"""),
+ ("Lingua e opzioni", """
+Lingua e opzioni
+Modello lingua: IT per documenti italiani (consigliato: il modello inglese su testo italiano aggiunge falsi positivi), EN per l'inglese, BOTH per documenti misti. All'avvio viene scelto IT se il sistema è in italiano.
+
+- Oscura Immagini: censura anche tutte le immagini (loghi, foto, firme) quando applichi Auto Redact.
+- OCR PDF scansionati: legge con l'OCR le pagine scansionate e le immagini incorporate nelle pagine di testo. Richiede i dati di Tesseract (inclusi nei pacchetti).
+- Mostra suggerimenti: contorna in arancione i suggerimenti da rivedere.
+- Profilo documenti sanitari: vedi l'argomento dedicato.
+- Maschera larghezza testo rimosso: arrotonda lo spazio lasciato dal testo cancellato, che altrimenti ne rivela la larghezza. Può spostare di poco il testo seguente; è disattivata di default.
+
+"""),
+ ("Auto Redact", """
+Auto Redact
+Premendo Auto Redact scegli quali tipi di dati cercare (nomi, luoghi, email, telefono, IBAN, carte, codice fiscale, partita IVA, patente, carta d'identità, passaporto, e con il profilo sanitario date di nascita, tessera sanitaria, ricetta, esenzioni). La scelta viene ricordata.
+Il programma assegna a ogni nome o luogo un punteggio basato su indizi nel testo: titoli e formule come «nato a» o «residente in», forma da nome proprio, nomi e luoghi noti, ripetizioni. Verbi e parole comuni di certificati e atti («Certifico», «Prescrivo», «Anamnesi»...) vengono scartati.
+
+- Alto: la censura viene segnata subito (riquadro nero).
+- Incerto: diventa un suggerimento da rivedere. Non viene applicato né perso.
+- Molto basso: scartato (nel profilo sanitario diventa comunque un suggerimento).
+
+I codici con controllo di validità (IBAN, carte, codice fiscale) vengono censurati interi anche se attaccati a un'etichetta.
+"""),
+ ("Revisione dei suggerimenti", """
+Revisione dei suggerimenti
+Al termine dell'analisi si apre l'elenco dei suggerimenti: pagina, valore (mascherato finché non spunti Mostra i valori), tipo, punteggio e motivi.
+
+- Spunta gli elementi da censurare e premi Applica selezionati: gli altri vengono scartati.
+- Seleziona ≥ soglia spunta in blocco gli elementi con punteggio alto.
+- Rivedi più tardi lascia tutto in sospeso; il pulsante Rivedi suggerimenti (N) nella barra laterale riapre l'elenco.
+- Doppio clic su una riga porta alla pagina; selezionando una riga il suo contorno si evidenzia.
+
+Sulla pagina i suggerimenti compaiono come contorni arancioni (solo a video): clicca su uno per scegliere Censura, Scarta o Mostra valore.
+All'esportazione, se restano suggerimenti non revisionati, il programma te lo ricorda: non verranno censurati.
+"""),
+ ("Correzioni manuali", """
+Correzioni manuali
+
+- Aggiungere una censura: trascina con il mouse un rettangolo sull'area da censurare. Se contiene testo puoi aggiungerlo alla blocklist, così sarà censurato in ogni documento futuro; se è un'area senza testo (immagine) viene oscurata come immagine.
+- Togliere una censura: clicca sul riquadro. Puoi aggiungere quel testo all'allowlist per non vederlo più censurato.
+- Pulisci Pagina toglie tutte le censure della pagina corrente.
+
+"""),
+ ("Dizionari", """
+Dizionari
+I Dizionari contengono due liste: la blocklist (parole sempre censurate) e l'allowlist (parole mai censurate). Il confronto avviene per parola intera: una parola in allowlist non nasconde mai un valore diverso o più lungo.
+
+- Ogni termine ha data di inserimento e scadenza (AAAA-MM-GG oppure Mai); i termini scaduti vengono eliminati da soli. Scadenza predefinita si applica ai nuovi termini, anche a quelli aggiunti dalla pagina.
+- Aggiungi, Rimuovi, Importa da testo… (un termine per riga) e Esporta in chiaro… (il file contiene dati personali).
+- Le liste sono cifrate per l'utente corrente (Windows DPAPI, portachiavi di sistema se disponibile, altrimenti un file chiave locale con protezione limitata). Copiate su un altro account o PC non sono leggibili: il programma non le cancella e le mostra vuote.
+
+"""),
+ ("Esportare", """
+Esportare
+Esporta PDF crea il file finale. Si sceglie tra:
+
+- Rimozione irreversibile: i valori selezionati vengono eliminati dal file. Non si può tornare indietro dal file esportato.
+- Pseudonimizzazione reversibile: i valori sono sostituiti da segnaposto e viene creata una mappa di ripristino cifrata, separata dal PDF.
+
+Stile dei segnaposto: Casuale (consigliato, non rivela ordine né numero dei valori), Progressivo (PERSON_001... rivela quanti valori distinti ci sono), Generico (nasconde anche il tipo), Coerente (chiave) (lo stesso valore ha lo stesso segnaposto in tutti i documenti creati con la stessa chiave; collega i documenti tra loro). Gestisci chiave... esporta o importa la chiave protetta da password per usarla su un altro computer.
+Mappa di ripristino: salvala lontano dal PDF e non in una cartella sincronizzata. Servono una password (almeno 12 caratteri, chiesta due volte) e, volendo, una chiave di recupero mostrata una sola volta: conservala in un luogo sicuro, il programma non la salva.
+Cosa fa il programma in esportazione
+
+- Lavora su una copia in memoria: il documento aperto e il file originale non cambiano.
+- Rimuove annotazioni, allegati, link esterni, segnalibri, script, metadati, testo invisibile e fuori pagina.
+- Verifica il file scritto: se un valore rimosso è ancora leggibile, o resta un glifo della parola, te lo dice e puoi eliminare il file.
+- Scrive un registro (solo conteggi e hash, nessun dato personale).
+
+Il nome file suggerito non contiene il nome del documento originale. Se scegli un nome che contiene un valore da rimuovere vieni avvisato.
+Attenzione: la censura non garantisce l'anonimato. Date, luoghi o altri dettagli rimasti possono ancora identificare una persona. Controlla sempre il risultato. Un PDF pseudonimizzato e la sua mappa restano dati personali.
+"""),
+ ("Ricostruire un PDF", """
+Ricostruire un PDF
+Ricostruisci PDF rimette i valori originali in un PDF pseudonimizzato.
+
+- Scegli il PDF pseudonimizzato e la sua mappa (.nullifypdf-map).
+- Inserisci la password, oppure spunta Usa la chiave di recupero e digita la chiave (NPDF-...).
+- Scegli dove salvare il file ricostruito.
+
+Il programma controlla che la mappa corrisponda al PDF e ti dice quanti valori sono stati davvero ripristinati. Il file ricostruito contiene di nuovo tutti i dati personali: trattalo di conseguenza.
+"""),
+ ("Documenti sanitari", """
+Documenti sanitari
+Con Profilo documenti sanitari il programma cerca anche:
+
+- date di nascita (precedute da «nato/a il», «nascita»...);
+- tessera sanitaria (20 cifre, struttura pubblicata, il carattere di controllo non è verificato);
+- numero di ricetta (NRE) e codici di esenzione, che richiedono la loro etichetta nelle vicinanze;
+- CAP e numeri civici in un contesto di indirizzo;
+- firme e timbri vicini a «firma», «timbro», «il medico»...: sono sempre solo suggerimenti.
+
+In questa modalità nessun caso incerto viene scartato in silenzio: diventa un suggerimento da rivedere. I dati sanitari sono dati di categoria particolare (art. 9 GDPR): controlla sempre il risultato.
+"""),
+ ("Dati locali e privacy", """
+Dati locali e privacy
+Il programma conserva nella cartella dell'utente (~/.nullifypdf): dizionari cifrati, chiave dei segnaposto cifrata, registro rotante dei messaggi, registri di esportazione (solo conteggi e hash) e, per file molto grandi, una cartella temporanea privata svuotata dopo l'esportazione.
+Elimina dati locali cancella log, registri e file temporanei. Se confermi, elimina anche i dizionari e, con una seconda conferma, le chiavi: senza chiavi dizionari cifrati e chiave dei segnaposto diventano illeggibili.
+Nessun valore rilevato viene scritto nei log. Nessun documento lascia il computer.
+"""),
+ ("Info e aggiornamenti", """
+Info e aggiornamenti
+La finestra Info mostra la versione e il collegamento al repository. Controlla aggiornamenti interroga le release pubblicate su GitHub; con Controlla ad ogni avvio (disattivato di default) il controllo è eseguito all'avvio e mostra qualcosa solo se esiste una versione più recente.
+"""),
+ ("Scorciatoie", """
+Scorciatoie
+
+| F1 o il pulsante Guida | Apre questa guida |
+| Ctrl + rotella | Zoom dell'anteprima |
+| Trascina sull'anteprima | Segna un'area da censurare |
+| Clic su una censura | La rimuove |
+| Clic su un contorno arancione | Decide un suggerimento |
+| Trascina un PDF sulla finestra | Lo apre |
+
+Passando il mouse su un elemento dell'interfaccia per qualche secondo compare una breve descrizione.
+"""),
+ ("Limiti e problemi comuni", """
+Limiti e problemi comuni
+
+- Il riconoscimento è statistico: controlla sempre il risultato prima di condividere un file. Nomi e luoghi non rilevati o dati indiretti restano.
+- Con righe molto ravvicinate (circa metà del corpo del carattere o meno) non esiste un riquadro che risparmi le righe vicine: compare un avviso e parole adiacenti possono essere danneggiate, mai lasciate leggibili.
+- Il testo dentro un'immagine viene rimosso solo dove l'immagine è censurata; per trovarlo serve l'OCR.
+- Se una parola comune viene proposta, scartala in revisione o aggiungila all'allowlist.
+- Se i dizionari risultano vuoti dopo aver copiato il profilo su un altro PC, vedi l'argomento Dizionari.
+- Le mappe create da questa versione non si aprono con versioni precedenti.
+
+"""),
+]
+
+
+class GuideDialog(QDialog):
+ """Topic list on the left, text on the right. Opened with F1."""
+
+ def __init__(self, parent: Any = None, topic: int = 0) -> None:
+ super().__init__(parent)
+ self.setWindowTitle("Guida - NullifyPDF")
+ self.resize(900, 620)
+ lay = QHBoxLayout(self)
+ self.topics = QListWidget()
+ self.topics.setObjectName("guide_topics")
+ self.topics.setFixedWidth(230)
+ for title, _html in GUIDE_TOPICS:
+ self.topics.addItem(title)
+ left = QVBoxLayout()
+ left.addWidget(self.topics)
+ btn = QPushButton("Chiudi")
+ btn.clicked.connect(self.accept)
+ left.addWidget(btn)
+ lay.addLayout(left)
+ self.text = QTextBrowser()
+ self.text.setObjectName("guide_text")
+ self.text.setOpenExternalLinks(True)
+ lay.addWidget(self.text, 1)
+ self.topics.currentRowChanged.connect(self._show)
+ self.topics.setCurrentRow(max(0, min(topic, len(GUIDE_TOPICS) - 1)))
+ apply_tooltips(self) # skips the topic list and the text: no hover popups while reading
+
+ def _show(self, row: int) -> None:
+ if 0 <= row < len(GUIDE_TOPICS):
+ self.text.setHtml(GUIDE_TOPICS[row][1])
+ self.text.moveCursor(self.text.textCursor().MoveOperation.Start)
+
+
+__all__ = [
+ "GUIDE_TOPICS", "GuideDialog", "TOOLTIPS", "ENTITY_TOOLTIPS",
+ "apply_tooltips", "tooltip_for", "widgets_without_tooltip", "Qt",
+]
diff --git a/lexicon_data_it.py b/lexicon_data_it.py
index 7c447c3..b2189dc 100644
--- a/lexicon_data_it.py
+++ b/lexicon_data_it.py
@@ -7,3 +7,9 @@
COMUNI_AMBIGUOUS = frozenset("""abbiategrasso|acerra|acireale|acri|acuto|adria|affi|afragola|agira|agra|agrigento|agropoli|ala|alassio|alatri|alba|albanella|albenga|alberobello|albi|albignasego|albino|albosaggia|alcamo|ales|alessandria|alfano|alfonsine|alghero|alia|allein|alpignano|altamura|altare|altino|altissimo|alto|altopascio|alì|amalfi|amandola|amaro|amato|amatrice|amelia|ameno|amorosi|ampezzo|anagni|ancona|andora|andria|anela|anghiari|anzi|anzio|aosta|apice|aprica|aprilia|aquileia|aquino|aragona|arborea|arborio|arcade|arce|archi|arco|arcore|ardea|ardore|arena|arenzano|arese|arezzo|argenta|ari|ariccia|armeno|armento|armo|arona|arpino|arre|arzachena|arzano|arzignano|asciano|asiago|asola|asolo|assago|assisi|asso|assolo|asti|atella|atri|augusta|auletta|aulla|avellino|aversa|avetrana|avezzano|aviano|avio|avola|ayas|azeglio|bacoli|badia|badolato|bagheria|bagnoregio|baiano|balocco|banchette|baranzate|barbara|barbaresco|barbata|barbona|bard|bardi|bardolino|barga|bari|barile|barletta|barolo|baschi|basiliano|bastiglia|battipaglia|bee|beinette|belfiore|bella|bellagio|bellino|bellona|belluno|bema|benestare|benevento|benna|bereguardo|bergamasco|bergamo|bettola|beverino|bevilacqua|bianchi|bianco|biassono|bibbiena|biella|bienno|bisaccia|bisceglie|bitonto|bobbio|boca|bollate|bologna|bolsena|bolzano|bomba|bonito|bonnanaro|bono|bordano|bordighera|bore|borghi|borgia|borgomanero|borgoricco|borgosesia|bormida|bormio|borno|bosa|botticino|bova|boves|bovino|bozzolo|bra|bracca|bracciano|braies|brancaleone|branzi|breme|brennero|brenta|brentonico|brescia|bressanone|bresso|brindisi|brolo|broni|bronte|brunello|brunico|bruno|buccheri|buccinasco|burgos|busca|busso|bussolengo|cabras|caccamo|cagli|cagliari|caglio|caino|calcata|calci|calciano|calcinate|calcinato|calcio|calco|caltagirone|caltanissetta|calvello|calvi|camaiore|cambiano|camerino|camerota|camini|camino|camogli|campagna|campana|campobasso|campomarino|camposampiero|camposanto|canale|canaro|candela|candida|caneva|canino|canna|cannobio|canterano|cantù|caorle|capaci|capannori|capodimonte|capri|caprile|capriolo|capua|caraglio|caravaggio|carbonate|carbone|carbonia|carcare|cardinale|carezzano|cariati|carignano|carini|carlino|carmagnola|carmignano|carovigno|carpi|carpino|carrara|carro|carrè|carugate|casalino|casamassima|casarano|cascia|cascina|casella|caserta|casina|casoria|cassino|castana|casteggio|castellabate|castellanza|castellaro|castelli|castelmagno|castelnuovo|castelseprio|castelvetrano|casto|castro|castrovillari|catania|catanzaro|cattolica|cavallino|cavatore|cave|cavezzo|cavour|ceccano|cecina|cefalù|celano|cene|cento|centola|cerano|cerchio|cerea|cerignola|cerva|cerveteri|cervia|cervino|cervo|cesa|cesate|cesena|cesenatico|cesio|ceto|ceva|chianche|chiari|chiavari|chieri|chiesanuova|chieti|chioggia|chiusa|chiusi|chivasso|ciampino|cicala|cingoli|cino|cis|cittadella|civita|civitavecchia|codroipo|colere|coli|colico|collecchio|colleferro|collegno|collio|cologne|colonna|colonnella|comacchio|comiso|como|compiano|comunanza|cona|conegliano|contrada|conversano|copertino|copiano|corato|cori|corio|corleone|correggio|corrido|corsico|cortona|cosenza|courmayeur|covo|cozzo|crema|cremona|crespino|crotone|cuneo|curino|curti|cusio|dalmine|dazio|delia|dello|desio|diamante|dolina|dolo|domodossola|donato|dragoni|drena|dro|dronero|dueville|eboli|elice|ello|empoli|enna|envie|erba|ercolano|este|fabbrico|fabriano|faenza|falcone|fallo|fano|farnese|fasano|fascia|favara|favignana|felino|feltre|fermo|ferrara|ferriere|fiano|fidenza|fiesole|filadelfia|filettino|filetto|firenze|firmo|fisciano|fiuggi|fiumara|fiumicino|foggia|foligno|follonica|fondi|fontanella|fontanelle|fontanile|fonte|forano|force|forcola|forio|forlì|formia|formicola|fornace|fornelli|fortezza|fossa|fossano|foza|frascati|frassino|frattamaggiore|frignano|frisa|front|frontino|frontone|frosinone|furore|gaeta|galatina|galatone|gallarate|gallese|gallicano|gallio|gallipoli|garda|garlasco|gavi|gela|genova|gessate|giarre|ginestra|giove|girasole|giuliana|giulianova|giungano|giussano|giustino|goni|gonzaga|gorgonzola|gorizia|gorno|grado|gradoli|gragnano|graniti|greci|greggio|grimaldi|grosseto|grosso|grottaglie|grotte|gruaro|grugliasco|gualdo|guastalla|gubbio|idro|iglesias|imola|imperia|impruneta|incudine|introd|ischia|iseo|isernia|ispica|ispra|isso|itala|ivrea|jesi|jesolo|ladispoli|laghi|laglio|lago|lainate|lana|lanciano|lariano|laterza|latina|lauro|lavagna|lavello|lecce|lecco|legnago|legnano|lei|lenta|lentini|leonessa|lerici|lesa|lesina|lettere|levate|liberi|licata|licenza|limbiate|limena|limina|linguaglossa|lioni|lipari|liscia|lissone|livigno|livorno|lizzano|loano|locri|loculi|lodi|loiano|lomazzo|longobardi|lonigo|loreto|lucca|lucera|lugo|luino|lula|luni|lupara|luserna|lustra|macello|macerata|maddaloni|mafalda|magenta|maggiora|magione|maglie|maglione|magnago|magnano|maida|maiori|maleo|malfa|malo|mammola|mandela|manduria|manfredonia|mango|manta|mantello|mantova|mappano|mara|maranello|maratea|marciana|marcon|margarita|marino|marsala|marta|martello|martinsicuro|marzano|marzio|mascalucia|maser|masera|masi|massa|massello|matera|matino|matrice|mattinata|meda|mede|medea|medicina|mele|melegnano|meleti|melfi|melilli|melissa|melle|mello|melzo|mentana|merano|merate|merlino|mesagne|mese|messina|mestrino|meta|mezzana|mezzanino|mezzano|milano|milazzo|milena|millesimo|milo|minori|mira|miranda|mirandola|mirano|mirto|modena|modica|modugno|moggio|moglia|mogliano|molare|molfetta|molinara|molise|momo|moncalieri|mondovì|mondragone|monfalcone|monno|monopoli|monreale|monselice|montalcino|montale|montanaro|montebelluna|montecarlo|montecchio|montelabbate|montella|montello|montepulciano|monterotondo|montese|montesilvano|montevarchi|monteverde|montezemolo|monti|montiano|montichiari|montone|monza|morbegno|mordano|mores|moresco|moretta|mori|morigerati|mortara|mossa|mozzate|mozzo|muggiò|mura|muros|musei|musso|napoli|nardò|narni|naro|nasino|naso|nave|ne|nespolo|nesso|nettuno|nichelino|nicosia|nimis|noale|noceto|noci|nola|noli|nomi|none|nonio|norcia|norma|noto|novara|nove|novella|novellara|novello|novoli|nuoro|nus|ocre|oderzo|olbia|oleggio|ome|omegna|onore|opera|opi|ora|orbassano|orbetello|oria|oriolo|oristano|orosei|orte|ortona|orvieto|osimo|ospedaletti|ospedaletto|ossi|ostra|ostuni|otranto|ottaviano|ottone|ovada|pachino|padova|paesana|paese|pagani|paladina|palata|palau|palermo|palestrina|palmanova|palmi|palombaro|paludi|palù|pandino|panettieri|panni|pantelleria|paola|parabiago|parenti|parete|parma|partinico|paterno|paternò|patti|pau|pavia|pedemonte|pelago|pella|penne|perca|pergola|perito|perlo|pero|perugia|pesaro|pescara|pescasseroli|pescate|pesche|pescia|petriolo|pettineo|peveragno|piacenza|pianella|pianezza|pianoro|piatto|picerno|pico|pienza|pietrafitta|pietragalla|pietrasanta|piglio|pigna|pignola|pignone|pigra|pila|pinerolo|pineto|piode|piombino|pisa|pisano|piscina|pistoia|pizzo|poggibonsi|poggioreale|poli|policoro|polistena|polla|pollina|pollone|pomezia|pompei|pompeiana|ponderano|ponte|pontedera|ponti|pontida|ponza|poppi|porcari|pordenone|porte|portici|portoferraio|portofino|portogruaro|portovenere|posada|positano|posta|postal|postiglione|potenza|pozzallo|pozzuoli|prato|pray|preci|predosa|premia|prezza|procida|pula|pulsano|putignano|quadri|quart|quarto|quartucciu|quindici|rabbi|ragusa|randazzo|rapallo|ravello|ravenna|re|rea|recanati|recco|rende|rezzo|rho|riace|rialto|ribera|ricadi|riccia|riccione|rieti|rimini|rio|riposto|rive|rivoli|roana|roccaraso|rodano|rolo|roma|romana|romeno|roncola|rosate|rose|rossa|rossana|rosta|rotella|rotonda|rotondi|rovato|rovereto|rovigo|rozzano|rubano|rubiera|ruoti|russi|rutigliano|sabaudia|sacco|sacile|sale|salemi|salento|salerno|salle|saluzzo|salve|salò|sambuco|samo|samone|sanremo|sansepolcro|sanza|saponara|sapri|saracena|sarnico|sarno|saronno|sarzana|sassari|sassello|sassuolo|sava|saviano|savigliano|savona|scafati|scala|scalea|scandale|scandiano|scandicci|scanno|scansano|scapoli|scena|schio|sciacca|sciara|scicli|scilla|scopa|scorrano|scorzè|sedriano|segni|segrate|selci|senigallia|seregno|seriate|serina|serrata|serre|serri|sesto|sestriere|sestu|settime|seveso|siano|siderno|siena|sigillo|signa|siliqua|silvi|sini|siracusa|sirmione|soave|soglio|solaro|soldano|solferino|sommo|sona|soncino|sondrio|sonico|sora|sorano|sorgono|sori|sorrento|sorso|sospiro|sostegno|soverato|specchia|spello|sperlonga|sperone|spessa|spiazzo|spinea|spinoso|spirano|spoleto|spresiano|statte|stella|stelvio|stilo|stintino|stra|stradella|stresa|strona|strozza|sulmona|supino|susa|suzzara|taggia|taino|taleggio|taormina|taranto|tarquinia|tarsia|taverna|tavullia|teramo|termoli|terni|terracina|terrasini|terrazzo|terzo|tesero|teti|thiene|tiers|tirano|tires|tirol|tirolo|tito|tivoli|todi|tolentino|tolmezzo|tombolo|ton|torino|tornata|torno|toro|torrazzo|torresina|torretta|torricella|torrioni|tortona|tortora|tortorella|tortoreto|tradate|tramonti|trani|trapani|trappeto|travagliato|traversetolo|trebaseleghe|trecate|trento|trevi|treviglio|treviso|trieste|trinità|trino|troia|tropea|trovo|tufo|turate|turi|turri|udine|ugento|ultimo|urbana|urbe|urbino|uri|uscio|ustica|uta|vaglia|vaie|valbrembo|valdobbiadene|valenza|valenzano|vallata|valmontone|valva|valverde|varallo|varazze|varedo|varese|vasto|vecchiano|veglie|veglio|velletri|venetico|venezia|venosa|ventimiglia|verano|verbania|vercelli|verdello|vergato|vermiglio|verona|vescovato|viale|viareggio|vicari|vicenza|vicoli|vieste|vigevano|vignola|vigonza|villamagna|villasimius|villette|villorba|vimercate|vinci|viola|visco|visone|vita|viterbo|vittoria|vittuone|viù|vo'|voghera|volano|voltaggio|volterra|zeri|zone""".split("|"))
PROVINCE_SIGLE = frozenset("""AG|AL|AN|AO|AP|AQ|AR|AT|AV|BA|BG|BI|BL|BN|BO|BR|BS|BT|BZ|CA|CB|CE|CH|CI|CL|CN|CO|CR|CS|CT|CZ|EN|FC|FE|FG|FI|FM|FR|GE|GO|GR|IM|IS|KR|LC|LE|LI|LO|LT|LU|MB|MC|ME|MI|MN|MO|MS|MT|NA|NO|NU|OG|OR|OT|PA|PC|PD|PE|PG|PI|PN|PO|PR|PT|PU|PV|PZ|RA|RC|RE|RG|RI|RM|RN|RO|SA|SI|SO|SP|SR|SS|SV|TA|TE|TN|TO|TP|TR|TS|TV|UD|VA|VB|VC|VE|VI|VR|VS|VT|VV""".split("|"))
REGIONS = frozenset("""abruzzo|basilicata|calabria|campania|emilia-romagna|friuli-venezia giulia|lazio|liguria|lombardia|marche|molise|piemonte|puglia|sardegna|sicilia|südtirol|toscana|trentino-alto adige|umbria|valle d'aosta|vallée d'aoste|veneto""".split("|"))
+# Given names and surnames: Wikidata (CC0,
+# https://www.wikidata.org/wiki/Wikidata:Licensing), entries that
+# have an article on the Italian Wikipedia.
+NAMES_GIVEN = frozenset("""abaco|abramina|abubakr|abundancjusz|adaeze|adenike|aisha|amabile|amadi|amarachi|ambroz|amílcar|ananda|andreana|annabeth|ansovino|ansprando|antea|arbana|arduino|ariele|arleigh|armelinda|armelindo|armonia|arsinoe|atticus|aurélien|aynoor|azubuike|baldina|barnabé|biagino|bindy|blagoja|blessing|bonavita|bryon|buonaccorso|börje|cafiero|candis|carmelina|carson|cash|cass|cedrick|chandra|chus|cibele|cierra|cleonico|colbie|cristina|cvetanka|cydney|darby|deanne|deliverance|delphina|diamanto|dikran|dilia|dionigio|djurdja|dolph|doralice|dovydas|durante|dwane|edgard|egil|eliso|endika|eniko|ermelina|erodione|esmeraldo|esperina|etelfleda|euloge|evalina|evaline|ewoud|fabrizio|fara|finola|fiore|fraser|garance|gasper|gaudentius|gaudêncio|gaël|generosa|gerasimo|germinal|giairo|giannotto|gioia|gladwyn|glori|grimhild|gro|gualberto|gustavina|habakuk|hadis|hajna|harlee|hosny|hyacinth|hyacintha|iacopino|ilenja|indro|islom|ivetta|jaelen|jantine|jarod|jaylin|jaylon|jean-pierre|jedd|jerod|jill|jolan|jordyn|joël|joëlle|jérome|kailey|kajrat|kaleigh|kaley|kaouthar|karolina|kaylie|keitha|khalil|khalilah|kirby|krassimir|krassimira|lachie|laurianne|ledina|leilani|lele|leoluca|lesleigh|ljube|ljuben|lorelle|lubna|mahund|malak|marchina|marinus|martine|marvyn|matthaus|meggy|meinard|melchiade|meredyth|meriwether|micaiah|mikita|milcíades|minodora|misty|mulay|myriam|nadege|narcisio|narges|nasreen|nathan|nectarios|nicaise|niketas|nordine|nuredin|odhran|olimpiada|ondina|onelio|onyekachi|oyvind|palmiero|paziente|perrie|phillis|procopio|quinziano|raffaelo|rambaldo|ranulfo|rashed|ravshan|rayno|restituto|ricciarda|rickey|riika|rocky|rodolph|roshan|rrahman|ruaraidh|ruaridh|ryleigh|rylie|ríkharður|sage|salomone|sami|sammi|sanchia|sanjaya|sanjin|santuzza|sejad|seppe|serxhio|serxho|shannon|shanon|sharron|shea|shereen|shon|shyla|sidnie|sperindio|stacia|steeven|steevy|stilian|stiliano|styliani|submit|surayya|taisija|tarsicio|tawni|temple|tenzin|terance|tesfaye|tiffiny|timoleone|timotej|timothea|timotheos|tineke|tobina|trecia|trey|turlough|ugochukwu|usama|valérien|vando|venero|verio|vern|vincenzio|vlaho|vonda|vratislava|waltera|wandy|yair|zaffiro|zainuddin|éric|órfhlaith""".split("|"))
+NAMES_SURNAMES = frozenset("""abaco|abate|abba|abbas|abbiate|abbondanza|abdullah|abeltino|abis|abozzi|abrami|abrioni|abruzzo|abubakr|abulafia|abutzu|acca|accalai|accardi|accareddu|accetto|accettura|accioni|acciu|accossu|accotzu|achena|achene|acheri|acqua|acri|adami|adani|adenike|adimari|adinolfi|adorni|agapito|agazzi|aglio|agnelli|agnese|agnoli|agosti|agozzino|agrati|agraz|agresti|ahn|aiolfi|airaghi|airaldi|al-dawasir|alba|albanese|albergo|alberi|alberici|alberti|albini|albizzati|albizzi|alboini|albonesi|alemanno|ales|alfieri|alfonsi|alifano|alinari|allevi|alonzo|altomare|alvaro|alzate|amadei|amadori|amato|ambrosini|amendola|amerighi|ammannati|amore|amoroso|anastasi|ancarani|andrei|andria|anfuso|angeli|anna|annarumma|antolini|aquenza|araújo|arecco|arese|argenti|argiolas|aricò|arienzo|armaroli|arosio|arpaia|arpino|arrighi|asghar|ashina|ashton|aspesi|astolfi|atzeni|atzeri|augelli|avola|ayame|ayasrah|azzari|azzi|bae|baek|baggio|baglioni|bagni|baiano|ballabio|ballai|balli|balsamo|bandino|barabino|baraldi|baratti|barbon|barca|bardi|barenghi|bari|barile|barocci|bartesaghi|bartoli|barzaghi|barzizza|bassani|bastoni|batmanghelidj|battisti|beati|beccari|bedarida|beltrame|beltrami|bencivenga|benedetti|benetti|benevento|beni|bennati|benni|bentivoglio|benvenuti|bernardi|bernasconi|bersani|bertelli|berti|bertoldi|bertuzzi|bevilacqua|bianchi|bibescu|bielli|biffi|bifulco|bini|bisceglie|bixio|bobbio|boccadifuoco|boccardo|bologna|bologni|bombelli|bombonati|bomboni|bonacci|bonaccorso|bonaiuti|bonamico|bonanno|bonardi|bonati|bonaventura|bondi|bonett|bonfante|bonfiglio|bongiorno|bongiovanni|boni|bonifacio|bonini|boninsegna|bonito|bonomo|bonora|bonsignore|bonucci|borboni|borghi|bortoli|boscaino|boscolo|bosio|bosisio|botero|bottai|bova|brambilla|breme|brenna|breno|brescia|briganti|brioschi|brivio|brocchi|brooks|broussard|brufani|bruno|bulgarelli|burns|buscemi|buti|buzatu|buzzi|bär|cabras|caffi|caiazzo|calabrese|calandra|caldonazzi|callegari|calogero|calvi|calò|cambi|cambiaghi|cambiaso|cameron|camozzi|campi|campo|canefri|canepa|canfora|cannas|cannata|cannavò|cannella|canonico|capodanno|cappelletto|cappelli|cappi|capra|capri|capurro|carena|cariddi|carnevale|carnielli|caro|carrobbio|carrubba|carrà|caruso|casadei|casanova|casarsa|casati|caserta|casiraghi|casirati|casoni|cassano|cassioli|castagna|castrogiovanni|cauchi|cavalleri|cavanna|cazzaniga|cazzola|ceccherelli|ceglie|celentano|ceriana|cerioli|cerne|cerri|cerutti|cervi|cha|chae|chapman|checchi|cheli|chelli|chezzi|chiacchio|chimenti|chinaglia|cho|choi|chun|chén|ciampi|ciccardini|cini|clementi|cocco|colombo|comaschi|comazzi|comi|como|coniglio|consiglio|conti|conz|cooper|coppi|corbi|corona|corradi|corrà|corte|corti|corvi|corvino|costa|cotta|coulibaly|covino|craxi|crespi|crespo|crinò|crippa|cristaldi|croce|crosio|crotta|cucchiara|cucinotta|cuffaro|cugno|cuneo|curatolo|cusumano|d'agostino|dago|danieli|danna|davide|davoli|dini|diodati|dioguardi|diotallevi|dolci|domenico|don|donati|doneda|donnini|donno|donzelli|drew|dunn|dustin|dzintars|dörfles|emiliani|emo|ermacora|esposito|fabbri|falco|fanti|fara|farina|federici|felici|ferla|fermi|ferrando|ferrara|ferrari|ferrarini|ferraro|ferrero|ferro|fiaschi|filippi|fini|fiorenza|flockhart|folliero|folloni|fontana|formiggini|fortescue|fossa|fox|fracanzani|francalancia|franceschi|frank|franzini|franzoni|freeman|frigerio|fulco|fumei|furlani|gabelli|gaber|gaekwad|gaffari|gaglioffi|gai|gallo|gambaro|gandhi|gandini|gandolfi|garibaldi|garlaschi|garlati|garzia|gaspari|gastaldi|gatti|gaudenzi|gazzolo|gemignani|gennari|genova|gentile|giacomi|gigli|gioberti|gioffré|giordano|giuli|giuliano|giussani|giusti|gjelaj|goi|gori|goria|gould|gozzadini|grandi|grassi|greco|grillo|gualtieri|guarini|guarnieri|guglielmi|guidi|gurjar|gül|haller|hamilton|han|heo|hong|howard|hwang|iaderosa|ignoti|innocenti|invernizzi|irani|israeli|ivaldi|jackson|jang|jeon|jeong|jiang|johansson|joubert|kang|katz|kazmi|kim|kong|korkut|kotes|kovács|krüger|kwak|kwon|lainati|lama|lamberti|lanzotti|lapi|laurino|laurito|lauro|lazzari|lecca|lecce|lecchi|lecci|leccio|ledda|lee|lentini|leone|lepri|levati|limiti|linati|lipari|lippi|lissoni|livraghi|locatelli|lodes|lodi|longo|lorandi|lorenzi|lorusso|lozzi|ludi|luna|lupi|lussu|luzzi|maestri|maffei|maggiore|magri|maier|mainardi|maino|mamdani|mancini|manetti|manfredi|maniero|manni|manzi|manzu|marangon|marazzi|marchesi|marchetti|marchi|marcon|marconi|mari|mariani|marino|marra|marras|martelli|martinelli|martínez|mascheroni|mauro|mazza|mazzarella|mclaughlin|melloni|merello|mereu|merloni|meuricoffre|mezzadri|micheli|migheli|min|mini|miranda|mitri|modafferi|modena|molinari|moller|momigliano|mondadori|mondaini|moon|morelli|moretti|moro|moroder|musso|muzzi|møller|najm|nam|nannoni|nazzari|nehme|nepi|neri|nervi|nerviani|nesti|neville|nicastro|nicolosi|nobili|noferi|noli|noureddine|novelli|novi|nucci|o'reilly|oddo|oggioni|ognibene|oldani|olgiati|olmi|omegna|omodei|ondedei|onesti|ongania|ongaro|onorati|orazi|orciani|orefice|oren|orlando|ornati|ornella|ortolani|ortona|osborne|ostuni|otranto|ottaviani|ottolenghi|paduano|paine|pallavicini|pallavidino|palmer|palmieri|panarello|pangallo|pantaleo|park|parra|parrella|parri|pasqua|pasquali|pasta|pastore|pastorino|paternoster|pavan|pavia|pedemonte|pedroli|pellegrini|peláez|perego|pergamo|pergola|perini|perrelli|perri|perrone|perti|perut|petracca|petroni|piccoli|piemonte|pietri|pirovano|pisani|pisanu|pittaluga|politi|postolache|pozzolengo|pradhan|premoli|preti|primavera|primo|proietti|prosperi|puglia|pugliese|punzo|quartigiani|quinn|quintavalle|quinto|raimondi|ramponi|ramsey|ranalli|ransone|rapa|rapallo|rapisarda|rassidi|ratti|ravara|ravidat|regina|ricci|rinaldi|rizzo|rocco|rodolfi|rodríguez|romano|rondelli|roos|rosa|rosi|rossetti|rossi|rota|rubini|ruggeri|ruscelli|russo|rutelli|ruvo|saccani|sacchi|sala|salierno|salm-reifferscheid-raitz|salvatori|salvidio|sambiagio|sami|sammarchi|sammarco|sammartino|sammauro|sampieri|sampietro|sampino|sanfelice|sangiorgi|sangiovanni|sanseverino|santagata|santangelo|santelia|santi|santonastaso|santoro|sanvitale|sarti|sartori|satriani|satta|saul|savini|scala|scalas|scally|scarpa|schaffgotsch|sciaccaluga|scicchitano|sclafani|scognamiglio|scopelliti|scornavacca|scorrano|scorza|seo|seong|sereni|seri|serravalle|servi|servidio|sessa|sesti|settala|sette|shin|shiono|sidney|sighinolfi|sigismondi|signori|silva|simonato|sinadeni|sinagra|sinibaldi|siniscalchi|sinisi|siri|smaldone|somaglia|somma|sommariva|son|soncini|song|soria|stabile|stafford|stanco|starace|stilo|stoppa|sullivan|sutera|sutter|suzuki|sánchez|taverna|telli|terlizzi|terrani|terzi|testa|tiberi|tokarczuk|tolomei|tomasi|tomei|tondi|tore|torrieri|torti|tozzi|tramonti|trani|trevisan|tripodi|trivulzio|tromba|trucco|tumminello|turchi|turetta|turner|ubaldi|uggeri|ugoni-longhi|urquhart|uselli|vacca|vacondio|vadalà|valda|valguarnera|valiani|vallino|varchi|varricchio|vedova|velliscig|vercelli|vercesi|verde|verlingieri|viazzi|vicentini|vieri|vigevani|vignolo|villa|villani|viola|vitale|vivaldi|volkonskaja|voltaggio|watanabe|weizmann|wells|williams|wilson|winter|woo|yair|yang|ying|yoo|yoon|zaccheo|zaghi|zampa|zampaglione|zanchi|zanini|zappa|zen|zicari|ziccardi|zilli|ziou|zocchi|zoccola|zoccoli|zoppi|zorzi|zotti|zucca|zuccalà|zuccarello|zuccheri|zulian|zuliani|zurlo|zurro""".split("|"))
+NAMES_AMBIGUOUS = frozenset("""abaco|abate|abba|abbas|abbiate|abbondanza|abruzzo|acca|accetto|acqua|acri|adorni|aglio|agnelli|agnese|agresti|ahn|alba|albanese|albergo|alberi|alberti|albini|alemanno|ales|alfieri|alifano|allevi|alvaro|alzate|amabile|amendola|amore|amoroso|ananda|andrei|andria|angeli|anna|antea|arduino|arese|argenti|armonia|arpino|augelli|avola|azzi|bae|baggio|baglioni|bagni|baiano|balli|balsamo|baratti|barca|bardi|bari|barile|bassani|bastoni|battisti|beati|benevento|beni|benni|benvenuti|bersani|berti|bevilacqua|bini|bisceglie|blessing|bobbio|bologna|bonaventura|bondi|bongiorno|boni|bonifacio|bonito|bonucci|borboni|borghi|bottai|bova|brambilla|breme|brescia|briganti|brocchi|brooks|bruno|burns|buzzi|bär|cabras|calabrese|calandra|calogero|calvi|calò|cambi|cameron|campi|campo|canfora|cannata|cannella|canonico|capodanno|cappelletto|cappelli|cappi|capra|capri|carena|carnevale|caro|carrà|casanova|casarsa|casati|caserta|cash|casoni|cass|cassano|castagna|ceglie|celentano|cerri|cerutti|cervi|cha|cho|choi|chun|ciampi|clementi|colombo|comaschi|comi|como|coniglio|consiglio|cooper|coppi|corona|corte|corti|corvi|corvino|costa|cotta|craxi|crespi|crespo|cristina|croce|cucinotta|cuneo|dago|danna|davide|dini|dolci|domenico|don|donno|drew|durante|eliso|emiliani|emo|fabbri|fabrizio|falco|fanti|fara|felici|fermi|ferrari|ferrero|ferro|fiaschi|filippi|fini|fiore|fossa|fox|frank|freeman|gaber|gaglioffi|gai|gallo|gambaro|gandhi|garibaldi|gastaldi|generosa|genova|gigli|gioia|giuli|giuliano|giusti|glori|goi|gori|gould|grandi|greco|grillo|gro|guidi|gül|hamilton|han|heo|hong|howard|ignoti|innocenti|irani|jackson|jeon|jeong|jill|kang|kim|kirby|kong|kwon|lama|lauro|lazzari|lecca|lecce|lecchi|lecci|leccio|lee|lele|lentini|leone|lepri|levati|limiti|lipari|lippi|locatelli|lodi|ludi|luna|lupi|maestri|maggiore|magri|maier|mancini|manfredi|maniero|manzi|marazzi|marchesi|marcon|marconi|mari|marino|marinus|marra|martelli|mascheroni|mauro|merloni|mezzadri|min|mini|miranda|modena|molinari|mondadori|mondaini|moon|moro|musso|nam|neri|nervi|nobili|noli|novelli|novi|oggioni|olmi|omegna|ondina|onesti|onorati|orefice|ornati|ornella|ortolani|ortona|ostuni|otranto|palmer|pantaleo|park|parra|pasqua|pasquali|pasta|pastore|pavia|paziente|pedemonte|perego|pergamo|pergola|perini|perrone|piccoli|piemonte|pisani|politi|preti|primavera|primo|proietti|prosperi|puglia|pugliese|quinn|quinto|ramponi|rapa|rapallo|ratti|regina|ricci|rocco|rocky|romano|rosa|rosi|rossi|rota|rubini|ruggeri|ruscelli|russo|rutelli|sacchi|sage|salomone|salvatori|sami|santi|sarti|satta|savini|scala|scarpa|scopelliti|scorrano|scorza|seo|seppe|sereni|seri|serravalle|servi|sessa|sesti|sette|shannon|shea|shin|sidney|signori|silva|siri|somma|sommariva|son|song|soria|stabile|stanco|starace|stilo|stoppa|submit|suzuki|taverna|temple|terzi|tondi|tore|torti|tozzi|tramonti|trani|tripodi|tromba|trucco|turchi|turner|vacca|varchi|vedova|venero|vercelli|verde|vicentini|villani|viola|vivaldi|voltaggio|wells|williams|wilson|winter|woo|yang|ying|yoo|yoon|zaffiro|zampa|zappa|zen|zoccola|zoccoli|zoppi|zucca|zuccheri""".split("|"))
diff --git a/lexicon_it.py b/lexicon_it.py
index 0591672..44f35dd 100644
--- a/lexicon_it.py
+++ b/lexicon_it.py
@@ -201,11 +201,21 @@ def is_common_word(token: str) -> bool:
def is_known_surname(token: str) -> bool:
- return token.strip().lower().strip(".,;:") in SURNAMES
+ key = token.strip().lower().strip(".,;:")
+ if key in SURNAMES:
+ return True
+ d = _data()
+ return bool(d and hasattr(d, "NAMES_SURNAMES") and key in d.NAMES_SURNAMES
+ and key not in d.NAMES_AMBIGUOUS)
def is_given_name(token: str) -> bool:
- return token.strip().lower().strip(".,;:") in GIVEN_NAMES
+ key = token.strip().lower().strip(".,;:")
+ if key in GIVEN_NAMES:
+ return True
+ d = _data()
+ return bool(d and hasattr(d, "NAMES_GIVEN") and key in d.NAMES_GIVEN
+ and key not in d.NAMES_AMBIGUOUS)
@functools.lru_cache(maxsize=1)
diff --git a/pii_detection.py b/pii_detection.py
index 94b2d21..bd53b59 100644
--- a/pii_detection.py
+++ b/pii_detection.py
@@ -1150,6 +1150,53 @@ def detect_address_parts(
return out
+# --- glyph deletion rule (shared by the app and the export checks) -----------
+
+def glyph_core(bbox: Rect4) -> Rect4:
+ """The part of a character box that decides whether a redaction deletes it.
+
+ Measured on PyMuPDF 2.1 (MuPDF 1.28): a glyph is removed when the
+ redaction rectangle intersects its box shrunk by 10% of its width on
+ each side and 10% of its height at top and bottom.
+ """
+ x0, y0, x1, y1 = bbox
+ dx, dy = (x1 - x0) * GLYPH_CORE_INSET, (y1 - y0) * GLYPH_CORE_INSET
+ return (x0 + dx, y0 + dy, x1 - dx, y1 - dy)
+
+
+def core_hits(bbox: Rect4, rect: Any) -> bool:
+ c = glyph_core(bbox)
+ return c[0] < rect.x1 and c[2] > rect.x0 and c[1] < rect.y1 and c[3] > rect.y0
+
+
+def target_chars(page: Any, full: Any) -> List[Dict[str, Any]]:
+ """Characters of the line a word box was taken from (single baseline)."""
+ try:
+ blocks = page.get_text("rawdict", clip=full).get("blocks", [])
+ except Exception:
+ return []
+ chars: List[Tuple[float, Dict[str, Any]]] = []
+ for block in blocks:
+ if block.get("type") != 0:
+ continue
+ for line in block.get("lines", []):
+ for span in line.get("spans", []):
+ for ch in span.get("chars", []):
+ if not str(ch.get("c", "")).strip():
+ continue
+ bx0, by0, bx1, by1 = ch["bbox"]
+ cx, cy = (bx0 + bx1) / 2, (by0 + by1) / 2
+ if full.x0 <= cx <= full.x1 and full.y0 <= cy <= full.y1:
+ chars.append((round(float(ch["origin"][1]), 1), ch))
+ if not chars:
+ return []
+ centre = (full.y0 + full.y1) / 2
+ bases = {b for b, _ in chars}
+ inside = [b for b in bases if full.y0 <= b <= full.y1] or list(bases)
+ base = min(inside, key=lambda b: abs(b - centre))
+ return [ch for b, ch in chars if b == base]
+
+
# --- geometry policy -------------------------------------------------------
def span_word_indices(
diff --git a/requirements.txt b/requirements.txt
index 6d963fb..8ef6010 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,6 +12,9 @@ PyMuPDF==2.1
presidio-analyzer==2.2.366
spacy==3.8.18
cryptography==50.0.2
+# OS keyring (macOS Keychain / Linux Secret Service) for the local master key;
+# Windows uses DPAPI instead, so it is not installed there.
+keyring==25.7.0; sys_platform != "win32"
# spaCy language models (pinned to exact versions for reproducibility)
# IMPORTANT: model minor version MUST match the spaCy core minor version
# (spaCy 3.8.x <-> models 3.8.x). The models declare spacy>=3.8.0,<3.9.0.
diff --git a/review_dialog.py b/review_dialog.py
index 56e38b1..3919d30 100644
--- a/review_dialog.py
+++ b/review_dialog.py
@@ -9,6 +9,7 @@
from typing import Any, Dict, List
+from help_guide import apply_tooltips
from PySide6.QtCore import Qt, Signal
from PySide6.QtWidgets import (
QAbstractItemView,
@@ -72,12 +73,16 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non
self._applied: List[Dict[str, Any]] = []
self._rejected: List[Dict[str, Any]] = []
lay = QVBoxLayout(self)
- lay.addWidget(QLabel(
+ intro = QLabel(
"Questi elementi non hanno prove sufficienti per essere censurati "
"automaticamente. Seleziona quelli da censurare: i non selezionati "
"vengono scartati se premi «Applica selezionati»."
- ))
+ )
+ intro.setWordWrap(True)
+ lay.addWidget(intro)
self.table = QTableWidget(len(self._items), 6, self)
+ self.table.setObjectName("review_table")
+ self.table.verticalHeader().setVisible(False)
self.table.setHorizontalHeaderLabels(
["Censura", "Pag.", "Valore", "Tipo", "Punteggio", "Motivi"]
)
@@ -114,6 +119,7 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non
self.spin.setRange(0.0, 1.0)
self.spin.setSingleStep(0.05)
self.spin.setValue(0.6)
+ self.spin.setObjectName("review_threshold")
btn_thr = QPushButton("Seleziona ≥ soglia")
# Bound methods, not lambdas capturing `self`: a lambda makes a
# reference cycle that keeps the dialog alive until the garbage
@@ -136,6 +142,7 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non
bottom.addWidget(self.btn_later)
bottom.addWidget(self.btn_apply)
lay.addLayout(bottom)
+ apply_tooltips(self)
# -- helpers ---------------------------------------------------------
def _cell(self, row: int, col: int) -> QTableWidgetItem:
diff --git a/scripts/build_lexicon_data.py b/scripts/build_lexicon_data.py
index b00b349..ba6cebe 100644
--- a/scripts/build_lexicon_data.py
+++ b/scripts/build_lexicon_data.py
@@ -24,6 +24,17 @@
import zipfile
from typing import Dict, List, Set
+WIKIDATA_URL = "https://query.wikidata.org/sparql"
+# Names that have an article on the Italian Wikipedia: a bounded, relevant set
+# (counting people per name would time out on the public endpoint).
+NAME_QUERY = (
+ 'SELECT DISTINCT ?l WHERE {{ ?n wdt:P31 wd:{cls}. '
+ '?a schema:about ?n; schema:isPartOf . '
+ '?n rdfs:label ?l. FILTER(LANG(?l)="it") }} LIMIT {page} OFFSET {off}'
+)
+NAME_CLASSES = {"given": "Q202444", "surnames": "Q101352"}
+NAME_PAGE = 2000
+NAME_OK = re.compile(r"^[a-zà-ÿ'’-]{3,24}$")
ISTAT_URL = "https://www.istat.it/storage/codici-unita-amministrative/Elenco-comuni-italiani.xlsx"
NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"}
COLUMNS = {
@@ -64,6 +75,37 @@ def read_rows(path: pathlib.Path) -> "tuple[List[str], List[Dict[str, str]]]":
return list(headers.values()), out
+def fetch_names(kind: str, retries: int = 6) -> Set[str]:
+ """Names of one kind (Wikidata, CC0), paged to stay under the query timeout."""
+ import json
+ import time
+ import urllib.parse
+
+ out: Set[str] = set()
+ off = 0
+ while True:
+ q = NAME_QUERY.format(cls=NAME_CLASSES[kind], page=NAME_PAGE, off=off)
+ url = WIKIDATA_URL + "?format=json&query=" + urllib.parse.quote(q)
+ for attempt in range(retries):
+ try:
+ req = urllib.request.Request(
+ url, headers={"User-Agent": "NullifyPDF-build/1.0 (github.com/overwrite00/NullifyPDF)"}
+ )
+ rows = json.load(urllib.request.urlopen(req, timeout=120))["results"]["bindings"]
+ break
+ except Exception as e:
+ if attempt == retries - 1:
+ raise RuntimeError(f"Wikidata {kind} page {off}: {e}") from e
+ time.sleep(10 * (attempt + 1))
+ for r in rows:
+ n = normalise(r["l"]["value"])
+ if NAME_OK.match(n):
+ out.add(n)
+ if len(rows) < NAME_PAGE:
+ return out
+ off += NAME_PAGE
+
+
def normalise(name: str) -> str:
return " ".join(name.replace("’", "'").lower().split())
@@ -71,6 +113,8 @@ def normalise(name: str) -> str:
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--istat-file", type=pathlib.Path)
+ ap.add_argument("--wikidata", action="store_true",
+ help="also fetch given names / surnames (CC0) from Wikidata")
ap.add_argument("--out", type=pathlib.Path, default=ROOT / "lexicon_data_it.py")
args = ap.parse_args()
src = args.istat_file
@@ -106,6 +150,7 @@ def main() -> int:
common = set(lexicon_it.COMMON_WORDS | lexicon_it.FUNCTION_WORDS | lexicon_it.GIVEN_NAMES
| lexicon_it.CONFLICT_NAMES)
vocab_words: Set[str] = set()
+ nlp = None
try:
import spacy
@@ -120,6 +165,28 @@ def lit(items: Set[str]) -> str:
assert '"""' not in body and "\\" not in body
return 'frozenset("""' + body + '""".split("|"))'
+ names_text = ""
+ if args.wikidata:
+ given = fetch_names("given")
+ surn = fetch_names("surnames")
+ both = given | surn
+ name_ambiguous = {w for w in both if w in common or len(w) <= 3}
+ if nlp is not None:
+ try:
+ name_ambiguous |= {w for w in both if " " not in w and nlp.vocab.has_vector(w)
+ and w not in lexicon_it.GIVEN_NAMES and w not in lexicon_it.SURNAMES}
+ except Exception:
+ pass
+ names_text = (
+ "# Given names and surnames: Wikidata (CC0,\n"
+ "# https://www.wikidata.org/wiki/Wikidata:Licensing), entries that\n"
+ "# have an article on the Italian Wikipedia.\n"
+ f"NAMES_GIVEN = {lit(given)}\n"
+ f"NAMES_SURNAMES = {lit(surn)}\n"
+ f"NAMES_AMBIGUOUS = {lit(name_ambiguous)}\n"
+ )
+ print(f"{len(given)} given names, {len(surn)} surnames ({len(name_ambiguous)} ambiguous)")
+
text = (
"# Generated by scripts/build_lexicon_data.py - do not edit.\n"
"# Contains ISTAT data (Elenco comuni italiani, " + src.name + "), licensed\n"
@@ -130,6 +197,7 @@ def lit(items: Set[str]) -> str:
f"COMUNI_AMBIGUOUS = {lit(ambiguous)}\n"
f"PROVINCE_SIGLE = {lit(sigle)}\n"
f"REGIONS = {lit(regions)}\n"
+ + names_text
)
args.out.write_text(text, encoding="utf-8")
print(f"{len(comuni)} comuni ({len(ambiguous)} ambiguous), {len(sigle)} province codes, "
diff --git a/tests/test_detection_pipeline.py b/tests/test_detection_pipeline.py
index 18eec38..9a8dac9 100644
--- a/tests/test_detection_pipeline.py
+++ b/tests/test_detection_pipeline.py
@@ -452,3 +452,16 @@ def test_known_surname_helps_a_two_word_name_but_never_rejects():
text2 = "Il referto di Gianpiero Qwertyuiop"
_i, cands2, _ = _run(text2, [res("PERSON", text2, "Gianpiero Qwertyuiop")])
assert cands2 and cands2[0].state in ("accepted", "suggested") # absence is not a reason to reject
+
+
+def test_wikidata_names_confirm_but_ambiguous_ones_do_not():
+ import lexicon_data_it as d
+
+ unambiguous = sorted(d.NAMES_SURNAMES - d.NAMES_AMBIGUOUS)
+ assert unambiguous and lexicon_it.is_known_surname(unambiguous[0])
+ ambiguous = sorted(d.NAMES_AMBIGUOUS & d.NAMES_SURNAMES)
+ assert ambiguous and not lexicon_it.is_known_surname(ambiguous[0]) or ambiguous[0] in lexicon_it.SURNAMES
+ assert not lexicon_it.is_known_surname("zzqxv")
+ assert "Wikidata" in open(
+ __import__("pathlib").Path(__file__).resolve().parent.parent / "THIRD_PARTY_NOTICES.md",
+ encoding="utf-8").read()
diff --git a/tests/test_export_sanitize.py b/tests/test_export_sanitize.py
index b67230a..1130119 100644
--- a/tests/test_export_sanitize.py
+++ b/tests/test_export_sanitize.py
@@ -383,3 +383,93 @@ def test_text_outside_redactions_uses_the_deletion_core():
page.add_redact_annot(pymupdf.Rect(x0 - 2, y0, x0 + (x1 - x0) * 0.25, y1))
kept = text_outside_redactions(doc)
assert "Alessandro" not in kept and "resta" in kept
+
+
+# --- post-apply glyph survival check ----------------------------------------------
+
+def _doc_with_word_and_band(band_fraction):
+ import json as _json
+
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Riga ABCDEFGH fine", fontsize=11)
+ x0, y0, x1, y1 = [w for w in page.get_text("words") if w[4] == "ABCDEFGH"][0][:4]
+ band = pymupdf.Rect(x0, y0, x0 + (x1 - x0) * band_fraction, y1)
+ annot = page.add_redact_annot(band)
+ annot.set_info(content=_json.dumps({
+ "nullifypdf": 1, "original": "ABCDEFGH", "entity_type": "PERSON",
+ "full": [x0, y0, x1, y1],
+ }))
+ return doc
+
+
+def _read_full(annot):
+ import json as _json
+
+ try:
+ d = _json.loads(annot.info.get("content", ""))
+ return pymupdf.Rect(*d["full"]) if d.get("full") else None
+ except Exception:
+ return None
+
+
+def test_a_band_that_misses_glyphs_is_reported(tmp_path):
+ doc = _doc_with_word_and_band(0.5)
+ result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full)
+ assert result.surviving_glyphs >= 3
+ assert not result.verification.ok
+ assert any("survived redaction" in f for f in result.verification.findings)
+ assert "ABCD" not in " ".join(result.verification.findings) # never repeats the value
+
+
+def test_a_band_covering_the_word_reports_nothing(tmp_path):
+ doc = _doc_with_word_and_band(1.0)
+ result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full,
+ removed_values=["ABCDEFGH"])
+ assert result.surviving_glyphs == 0 and result.verification.ok, result.verification.findings
+
+
+def test_survivor_check_is_inactive_without_a_stored_full_box(tmp_path):
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "ABCDEFGH", fontsize=11)
+ page.add_redact_annot(pymupdf.Rect(72, 90, 80, 110)) # manual box, no `full`
+ result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full)
+ assert result.surviving_glyphs == 0
+
+
+def test_survivor_alignment_survives_mupdf_moving_the_remaining_text(tmp_path):
+ """The rest of the line is rewritten and shifted; positions are not reliable."""
+ doc = _doc_with_word_and_band(0.5)
+ result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full)
+ assert result.surviving_glyphs == 4 # E, F, G, H
+
+
+def test_repeated_words_on_the_line_are_not_false_survivors(tmp_path):
+ import json as _json
+
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Maria Mario Maria", fontsize=11)
+ x0, y0, x1, y1 = [w for w in page.get_text("words") if w[4] == "Mario"][0][:4]
+ annot = page.add_redact_annot(pymupdf.Rect(x0, y0, x1, y1))
+ annot.set_info(content=_json.dumps({
+ "nullifypdf": 1, "original": "Mario", "entity_type": "PERSON",
+ "full": [x0 - 1, y0, x1 + 1, y1],
+ }))
+ result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full)
+ assert result.surviving_glyphs == 0
+
+
+def test_rule_deviation_is_reported(tmp_path, monkeypatch):
+ """If a PyMuPDF release stops deleting what the rule predicts, the export says so."""
+ import export_engine as ee
+
+ doc = _doc_with_word_and_band(1.0)
+ real = ee.PDF_REDACT_IMAGE_PIXELS
+ orig_apply = pymupdf.Page.apply_redactions
+ monkeypatch.setattr(pymupdf.Page, "apply_redactions", lambda self, *a, **k: 0) # deletes nothing
+ result = export_document(doc, str(tmp_path / "o.pdf"), read_full=_read_full)
+ monkeypatch.setattr(pymupdf.Page, "apply_redactions", orig_apply)
+ assert real is not None
+ assert result.surviving_glyphs >= 8 and not result.verification.ok
diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py
index d1fe2de..b8c56ca 100644
--- a/tests/test_gui_flow.py
+++ b/tests/test_gui_flow.py
@@ -845,3 +845,77 @@ def test_recovery_wrap_is_kept_even_if_the_dialog_is_closed_without_ticking(wind
monkeypatch.setattr(QDialog, "exec", lambda self: QDialog.DialogCode.Rejected)
raw = w._offer_recovery_key()
assert isinstance(raw, bytes) and len(raw) == 32
+
+
+# --- labels on image / signature areas survive small boxes -------------------------------
+
+def _tiny_area_doc(w, height):
+ w.doc.close()
+ d = pymupdf.open()
+ pg = d.new_page()
+ pg.insert_text((72, 100), "Testo normale", fontsize=11)
+ pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 40, 40), False)
+ pix.set_rect(pix.irect, (10, 10, 10))
+ pg.insert_image(pymupdf.Rect(300, 300, 300 + 24, 300 + height), pixmap=pix)
+ w.doc = d
+ w.chk_img.setChecked(True)
+ w.apply_ai_to_page(0, [])
+
+
+@pytest.mark.parametrize("height", [3, 6, 20])
+@pytest.mark.parametrize("mode", [PrivacyMode.ANONYMIZE, PrivacyMode.PSEUDONYMIZE])
+def test_image_label_is_kept_even_in_a_tiny_box(window, tmp_path, monkeypatch, height, mode):
+ w = window
+ _tiny_area_doc(w, height)
+ assert len(_redacts(w)) == 1
+ out = tmp_path / "i.pdf"
+ files = [str(out)] + ([str(tmp_path / "m.nullifypdf-map")] if mode == PrivacyMode.PSEUDONYMIZE else [])
+ _stub_dialogs(monkeypatch, files)
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: mode)
+ monkeypatch.setattr(w, "_select_pseudonym_style", lambda: ("random", None))
+ monkeypatch.setattr(w, "_ask_map_password", lambda: "una frase molto lunga 2026")
+ monkeypatch.setattr(w, "_offer_recovery_key", lambda: None)
+ w.cmd_export()
+ pdf = pymupdf.open(str(out))
+ text = pdf[0].get_text()
+ pdf.close()
+ assert "Testo normale" in text
+ assert any(lab in text for lab in ("[IMMAGINE RIMOSSA]", "[IMMAGINE]", "[IMG]", "#")), text
+
+
+def test_signature_label_in_a_small_area(window, tmp_path, monkeypatch):
+ w = window
+ w.doc.close()
+ d = pymupdf.open()
+ d.new_page().insert_text((72, 100), "Testo", fontsize=11)
+ w.doc = d
+ w._add_kind_redaction(d[0], pymupdf.Rect(200, 200, 212, 204), "SIGNATURE")
+ out = tmp_path / "s.pdf"
+ _stub_dialogs(monkeypatch, [str(out)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE)
+ w.cmd_export()
+ pdf = pymupdf.open(str(out))
+ text = pdf[0].get_text()
+ pdf.close()
+ assert any(lab in text for lab in ("[FIRMA RIMOSSA]", "[FIRMA]", "[F]", "#")), text
+
+
+def test_export_through_the_app_reports_no_survivor_for_clamped_bands(window, tmp_path, monkeypatch):
+ w = window
+ _tight_doc(w, 8.0)
+ w.apply_ai_to_page(0, [_detection(w, "Alessandro", "PERSON", "accepted")])
+ out = tmp_path / "c.pdf"
+ _stub_dialogs(monkeypatch, [str(out)])
+ monkeypatch.setattr(w, "_select_privacy_export", lambda: PrivacyMode.ANONYMIZE)
+ seen = {}
+ import export_engine as ee
+ real = ee.export_document
+
+ def spy(*a, **k):
+ r = real(*a, **k)
+ seen["r"] = r
+ return r
+
+ monkeypatch.setattr(app, "export_document", spy)
+ w.cmd_export()
+ assert seen["r"].surviving_glyphs == 0 and seen["r"].verification.ok
diff --git a/tests/test_gui_help.py b/tests/test_gui_help.py
new file mode 100644
index 0000000..1721453
--- /dev/null
+++ b/tests/test_gui_help.py
@@ -0,0 +1,141 @@
+"""Guide and tooltips: every GUI element is explained, and the guide matches the UI."""
+
+import os
+import re
+
+import pytest
+
+pytest.importorskip("PySide6")
+os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
+
+from PySide6.QtWidgets import ( # noqa: E402
+ QAbstractButton, QApplication, QCheckBox, QDialog, QPushButton, QRadioButton,
+)
+
+import NullifyPDF as app # noqa: E402
+from help_guide import ( # noqa: E402
+ ENTITY_TOOLTIPS, GUIDE_TOPICS, TOOLTIPS, GuideDialog, apply_tooltips,
+ widgets_without_tooltip,
+)
+from pii_detection import ENTITY_GROUPS # noqa: E402
+from review_dialog import ReviewDialog # noqa: E402
+
+
+@pytest.fixture()
+def window(tmp_path, monkeypatch):
+ import atexit
+ import gc
+
+ home = tmp_path / "home"
+ home.mkdir()
+ monkeypatch.setenv("USERPROFILE", str(home))
+ monkeypatch.setenv("HOME", str(home))
+ qapp = QApplication.instance() or QApplication([])
+ w = app.NullifyPDF()
+ yield w
+ try:
+ w._cleanup_resources()
+ except Exception:
+ pass
+ atexit.unregister(w._cleanup_on_exit)
+ w.deleteLater()
+ qapp.processEvents()
+ gc.collect()
+
+
+def _plain(html):
+ return re.sub(r"<[^>]+>", " ", html)
+
+
+def test_every_widget_of_the_main_window_has_a_tooltip(window):
+ assert widgets_without_tooltip(window) == []
+
+
+def test_buttons_have_tooltips_too(window):
+ buttons = [b for b in window.findChildren(QAbstractButton)]
+ assert len(buttons) >= 15
+ assert all(b.toolTip().strip() for b in buttons), [b.text() for b in buttons if not b.toolTip()]
+
+
+def test_tooltips_are_in_italian_and_not_empty():
+ for key, tip in {**TOOLTIPS, **ENTITY_TOOLTIPS}.items():
+ assert len(tip) > 10 and tip[0].isupper(), key
+
+
+def test_every_entity_checkbox_has_a_tooltip():
+ labels = {label for _gid, label, _t in ENTITY_GROUPS}
+ assert labels <= set(ENTITY_TOOLTIPS), labels - set(ENTITY_TOOLTIPS)
+
+
+def test_entity_dialog_has_tooltips(window, monkeypatch):
+ seen = {}
+
+ def fake_exec(self):
+ seen["missing"] = widgets_without_tooltip(self)
+ seen["boxes"] = [c.toolTip() for c in self.findChildren(QCheckBox)]
+ return QDialog.DialogCode.Rejected
+
+ monkeypatch.setattr(QDialog, "exec", fake_exec)
+ window._select_ai_entities()
+ assert seen["missing"] == [] and all(seen["boxes"])
+
+
+def test_review_and_dictionary_dialogs_have_tooltips(window):
+ from dictionary_dialog import DictionaryDialog
+
+ items = [{"text": "Zorzi", "entity_type": "PERSON", "score": 0.5, "page": 0, "reasons": []}]
+ assert widgets_without_tooltip(ReviewDialog(items)) == []
+ assert widgets_without_tooltip(DictionaryDialog(window.list_manager)) == []
+
+
+def test_about_dialog_has_tooltips(window, monkeypatch):
+ seen = {}
+
+ def fake_exec(self):
+ seen["missing"] = widgets_without_tooltip(self)
+ return QDialog.DialogCode.Rejected
+
+ monkeypatch.setattr(QDialog, "exec", fake_exec)
+ window.cmd_about()
+ assert seen["missing"] == []
+
+
+def test_guide_dialog_lists_every_topic_and_has_tooltips(window):
+ g = GuideDialog(window)
+ assert g.topics.count() == len(GUIDE_TOPICS) >= 10
+ assert widgets_without_tooltip(g) == []
+ # No hover popups on the topic list and the text: they get in the way of reading.
+ assert g.topics.toolTip() == "" and g.text.toolTip() == ""
+ assert g.topics.viewport().toolTip() == "" and g.text.viewport().toolTip() == ""
+ for row in range(g.topics.count()):
+ g.topics.setCurrentRow(row)
+ assert len(g.text.toPlainText()) > 80
+
+
+def test_f1_and_the_guide_button_open_the_guide(window, monkeypatch):
+ opened = []
+ monkeypatch.setattr(GuideDialog, "exec", lambda self: opened.append(self.topics.count()) or 0)
+ window._f1.activated.emit()
+ btn = next(b for b in window.findChildren(QPushButton) if b.text() == "Guida")
+ btn.click()
+ assert len(opened) == 2
+
+
+def test_the_guide_describes_every_control_of_the_sidebar(window):
+ text = _plain(" ".join(html for _t, html in GUIDE_TOPICS))
+ for w in window.findChildren(QAbstractButton):
+ label = w.text().strip()
+ if not label or label in {"-", "+", "<", ">", "Esci"} or label.startswith("Rivedi suggerimenti"):
+ continue
+ assert label in text, f"the guide never mentions «{label}»"
+ assert "Rivedi suggerimenti" in text and "F1" in text
+ for rb in window.findChildren(QRadioButton):
+ assert rb.text() in text
+
+
+def test_the_guide_matches_what_the_program_really_does():
+ text = _plain(" ".join(html for _t, html in GUIDE_TOPICS))
+ for fact in ("scadenza", "chiave di recupero", "mappa di ripristino",
+ "Ctrl + rotella", "Revisione", "OCR", "pseudonim"):
+ assert fact.lower() in text.lower(), fact
+ assert "Wikidata" not in text and "AI" not in text.split() # no internals / no AI wording
diff --git a/tests/test_secure_store.py b/tests/test_secure_store.py
index fed997d..0bd9e0e 100644
--- a/tests/test_secure_store.py
+++ b/tests/test_secure_store.py
@@ -158,3 +158,43 @@ def test_tampered_list_file_is_not_trusted(tmp_path):
raw[-1] ^= 1
p.write_bytes(bytes(raw))
assert PDFListManager(tmp_path).load_blocklist() == set()
+
+
+def test_keyring_backend_holds_the_master_key_when_available(tmp_path, monkeypatch):
+ """With a real (here: in-memory) keyring backend the key never touches the keystore file."""
+ keyring = pytest.importorskip("keyring")
+ from keyring.backend import KeyringBackend
+
+ class MemoryKeyring(KeyringBackend):
+ priority = 1
+ store: dict = {}
+
+ def set_password(self, service, username, password):
+ self.store[(service, username)] = password
+
+ def get_password(self, service, username):
+ return self.store.get((service, username))
+
+ def delete_password(self, service, username):
+ self.store.pop((service, username), None)
+
+ old = keyring.get_keyring()
+ keyring.set_keyring(MemoryKeyring())
+ try:
+ monkeypatch.setattr(secure_store.DpapiProtector, "available", staticmethod(lambda: False))
+ key = secure_store.load_or_create_master_key(tmp_path)
+ ks = json.loads((tmp_path / "keystore.json").read_text())
+ assert ks["master"]["protector"] == "keyring"
+ assert "portachiavi" in secure_store.protector_label(tmp_path)
+ import base64
+ assert base64.b64encode(key).decode() not in json.dumps(ks)
+ assert not (tmp_path / "keys").exists()
+ secure_store.forget_cached_keys()
+ assert secure_store.load_or_create_master_key(tmp_path) == key
+ # losing the keyring entry = unreadable keystore, never silent re-creation
+ MemoryKeyring.store.clear()
+ secure_store.forget_cached_keys()
+ with pytest.raises(secure_store.KeystoreUnavailable):
+ secure_store.load_or_create_master_key(tmp_path)
+ finally:
+ keyring.set_keyring(old)
From f513e481a1fe13217111be25019cb4e95391415c Mon Sep 17 00:00:00 2001
From: 0verwrite <31691645+overwrite00@users.noreply.github.com>
Date: Tue, 10 Nov 2026 22:27:24 +0100
Subject: [PATCH 08/13] fix: never block on a keyring that cannot answer (#133)
---
secure_store.py | 67 ++++++++++++++++++++++++++++++++++++--
tests/conftest.py | 5 +++
tests/test_secure_store.py | 43 ++++++++++++++++++++++++
3 files changed, 112 insertions(+), 3 deletions(-)
diff --git a/secure_store.py b/secure_store.py
index bfc5c0b..ff3a4de 100644
--- a/secure_store.py
+++ b/secure_store.py
@@ -123,32 +123,93 @@ def unwrap(self, blob: Dict[str, Any]) -> bytes:
return self._call(base64.b64decode(blob["blob"]), False)
+PROBE_TIMEOUT = 3.0 # a healthy keyring answers in milliseconds
+UNWRAP_TIMEOUT = 60.0 # may legitimately wait for the user to unlock it
+
+
+def _with_timeout(fn: Any, seconds: float) -> Any:
+ """Run `fn` in a daemon thread; raise TimeoutError if it does not finish.
+
+ A keyring backend can block forever when it needs a graphical
+ confirmation that nobody can give (a locked macOS Keychain on a headless
+ machine, no Secret Service session). The thread is abandoned: it is a
+ daemon, so it never keeps the program alive.
+ """
+ import threading
+
+ box: Dict[str, Any] = {}
+
+ def run() -> None:
+ try:
+ box["value"] = fn()
+ except BaseException as e: # noqa: BLE001 - re-raised in the caller
+ box["error"] = e
+
+ t = threading.Thread(target=run, daemon=True)
+ t.start()
+ t.join(seconds)
+ if t.is_alive():
+ raise TimeoutError("keyring did not answer in time")
+ if "error" in box:
+ raise box["error"]
+ return box.get("value")
+
+
class KeyringProtector:
name = "keyring"
SERVICE = "NullifyPDF"
USER = "master"
+ _probe_result: "Dict[int, bool]" = {}
@staticmethod
def available() -> bool:
+ """A real keyring backend that answers promptly.
+
+ Set NULLIFYPDF_DISABLE_KEYRING=1 to never touch the system keyring
+ (used by the tests, so they never reach a real Keychain).
+ """
+ if os.environ.get("NULLIFYPDF_DISABLE_KEYRING") == "1":
+ return False
try:
import keyring # type: ignore[import-not-found]
from keyring.backends import fail, null # type: ignore[import-not-found]
backend = keyring.get_keyring()
- return not isinstance(backend, (fail.Keyring, null.Keyring))
+ if isinstance(backend, (fail.Keyring, null.Keyring)):
+ return False
+ key = id(backend)
+ if key in KeyringProtector._probe_result:
+ return KeyringProtector._probe_result[key]
+
+ def probe() -> bool:
+ keyring.set_password("NullifyPDF-probe", "probe", "x")
+ ok = keyring.get_password("NullifyPDF-probe", "probe") == "x"
+ keyring.delete_password("NullifyPDF-probe", "probe")
+ return ok
+
+ try:
+ ok = bool(_with_timeout(probe, PROBE_TIMEOUT))
+ except Exception:
+ ok = False
+ KeyringProtector._probe_result[key] = ok
+ return ok
except Exception:
return False
def wrap(self, key: bytes) -> Dict[str, Any]:
import keyring # type: ignore[import-not-found]
- keyring.set_password(self.SERVICE, self.USER, base64.b64encode(key).decode("ascii"))
+ secret = base64.b64encode(key).decode("ascii")
+ _with_timeout(
+ lambda: keyring.set_password(self.SERVICE, self.USER, secret), UNWRAP_TIMEOUT
+ )
return {"protector": self.name, "service": self.SERVICE, "user": self.USER}
def unwrap(self, blob: Dict[str, Any]) -> bytes:
import keyring # type: ignore[import-not-found]
- secret = keyring.get_password(blob.get("service", self.SERVICE), blob.get("user", self.USER))
+ svc, usr = blob.get("service", self.SERVICE), blob.get("user", self.USER)
+ secret = _with_timeout(lambda: keyring.get_password(svc, usr), UNWRAP_TIMEOUT)
if not secret:
raise OSError("Segreto non presente nel portachiavi.")
return base64.b64decode(secret)
diff --git a/tests/conftest.py b/tests/conftest.py
index 6e9ef1d..4ed5249 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,9 +1,14 @@
"""Pytest configuration shared by the test suite."""
+import os
import pathlib
import sys
+# Tests must never touch a real system keyring (a locked macOS Keychain on a
+# headless runner blocks forever waiting for a dialog).
+os.environ.setdefault("NULLIFYPDF_DISABLE_KEYRING", "1")
+
PROJECT_ROOT = pathlib.Path(__file__).resolve().parent.parent
root_str = str(PROJECT_ROOT)
if root_str not in sys.path:
diff --git a/tests/test_secure_store.py b/tests/test_secure_store.py
index 0bd9e0e..ebfd2f4 100644
--- a/tests/test_secure_store.py
+++ b/tests/test_secure_store.py
@@ -180,6 +180,8 @@ def delete_password(self, service, username):
old = keyring.get_keyring()
keyring.set_keyring(MemoryKeyring())
+ monkeypatch.delenv("NULLIFYPDF_DISABLE_KEYRING", raising=False)
+ secure_store.KeyringProtector._probe_result.clear()
try:
monkeypatch.setattr(secure_store.DpapiProtector, "available", staticmethod(lambda: False))
key = secure_store.load_or_create_master_key(tmp_path)
@@ -198,3 +200,44 @@ def delete_password(self, service, username):
secure_store.load_or_create_master_key(tmp_path)
finally:
keyring.set_keyring(old)
+
+
+def test_a_keyring_that_never_answers_is_skipped_not_waited_for(tmp_path, monkeypatch):
+ """A locked Keychain waiting for a dialog must not hang the program."""
+ import time
+
+ keyring = pytest.importorskip("keyring")
+ from keyring.backend import KeyringBackend
+
+ class HangingKeyring(KeyringBackend):
+ priority = 1
+
+ def set_password(self, service, username, password):
+ time.sleep(30)
+
+ def get_password(self, service, username):
+ time.sleep(30)
+
+ def delete_password(self, service, username):
+ time.sleep(30)
+
+ old = keyring.get_keyring()
+ keyring.set_keyring(HangingKeyring())
+ monkeypatch.delenv("NULLIFYPDF_DISABLE_KEYRING", raising=False)
+ monkeypatch.setattr(secure_store, "PROBE_TIMEOUT", 0.3)
+ monkeypatch.setattr(secure_store.DpapiProtector, "available", staticmethod(lambda: False))
+ secure_store.KeyringProtector._probe_result.clear()
+ try:
+ t0 = time.time()
+ assert secure_store.KeyringProtector.available() is False
+ assert time.time() - t0 < 5
+ secure_store.load_or_create_master_key(tmp_path)
+ assert json.loads((tmp_path / "keystore.json").read_text())["master"]["protector"] == "keyfile"
+ finally:
+ keyring.set_keyring(old)
+ secure_store.KeyringProtector._probe_result.clear()
+
+
+def test_keyring_can_be_disabled_by_environment(monkeypatch):
+ monkeypatch.setenv("NULLIFYPDF_DISABLE_KEYRING", "1")
+ assert secure_store.KeyringProtector.available() is False
From fa230d33428b10b20c4252e3702c2ff57a83eff8 Mon Sep 17 00:00:00 2001
From: 0verwrite <31691645+overwrite00@users.noreply.github.com>
Date: Wed, 11 Nov 2026 00:12:34 +0100
Subject: [PATCH 09/13] feat: prepare beta.4 local models and document import
---
ARCHITECTURE.md | 6 +-
CHANGELOG.md | 3 +
DEVELOPMENT.md | 9 +-
NullifyPDF.py | 117 +++++++++--
README.md | 5 +-
USER_GUIDE.md | 16 +-
build_local.py | 8 +-
help_guide.py | 20 +-
model_dialog.py | 182 +++++++++++++++++
model_manager.py | 386 ++++++++++++++++++++++++++++++++++++
onnx_ner.py | 135 +++++++++++++
requirements.txt | 3 +
review_dialog.py | 30 ++-
tests/test_geometry.py | 11 +
tests/test_model_dialog.py | 18 ++
tests/test_model_manager.py | 75 +++++++
tests/test_word_import.py | 41 ++++
word_import.py | 82 ++++++++
18 files changed, 1114 insertions(+), 33 deletions(-)
create mode 100644 model_dialog.py
create mode 100644 model_manager.py
create mode 100644 onnx_ner.py
create mode 100644 tests/test_model_dialog.py
create mode 100644 tests/test_model_manager.py
create mode 100644 tests/test_word_import.py
create mode 100644 word_import.py
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index b546716..27ab7ca 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -33,6 +33,10 @@ It supports two privacy export modes:
| `pdf_sanitize.py` | Export sanitiser (annotations, attachments, scripts, hidden layers/text, off-page text, metadata, optional TJ-gap quantisation) and `verify_export()` |
| `export_engine.py` | Qt-free export pipeline (`export_document`), neutral file names, private temp handling, audit record, local-data wipe |
| `review_dialog.py` | Qt dialog to accept or reject suggested detections |
+| `model_manager.py` | Pinned Hugging Face model catalog, SHA-256 verification, local model store and hardware recommendation |
+| `model_dialog.py` | Italian model selection, consent and progress-aware downloads |
+| `onnx_ner.py` | CPU ONNX token-classification adapter with character-offset mapping and label mapping |
+| `word_import.py` | Safe DOCX OOXML text/table extraction into an internal PyMuPDF document; no macro execution |
| `help_guide.py` | In-app guide (Italian, F1) and the tooltip table applied to every widget |
| `secure_store.py` | Master key protected by DPAPI / OS keyring / key file; AES-GCM sealing; atomic private writes |
| `word_lists.py` | Encrypted blocklist/allowlist with expiry (`PDFListManager`) |
@@ -322,5 +326,5 @@ pytest tests/ -v
---
-*Last updated: 2026-11-10*
+*Last updated: 2026-11-11*
*[Back to README](./README.md) | [Development →](./DEVELOPMENT.md)*
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 465df4a..813a0ac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### ✨ Added
+- **Local PII models**: optional, user-downloaded ONNX models from pinned Hugging Face revisions are stored under `~/.nullifypdf/models/`, verified file-by-file with SHA-256, and contribute suggestions only. The model dialog shows the source, license, revision and download progress; unverified experimental entries remain blocked.
+- **Word DOCX import**: local DOCX text and table content can be imported into a PDF representation for the existing detection and review workflow. Macros, external links and embedded objects are not executed.
- **User guide in the program**: a new "Guida" button (or F1) opens a guide in Italian that explains the whole workflow: opening a PDF, language and options, Auto Redact, review of suggestions, manual corrections, dictionaries, export modes, reconstruction, health documents, local data and shortcuts.
- **Tooltips on every element**: all buttons, options, fields, tables and indicators of the main window and of the dialogs now show a short description when the pointer rests on them.
- **Check after redaction for surviving characters**: besides searching the written file, the export now checks that every character of a redacted word was really deleted, and reports it (without the value) if one was not.
@@ -33,6 +35,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### 🐛 Fixed
+- **Download and redaction geometry**: model downloads now show an overall percentage, and accepted word redactions use the complete original word box with a small safety margin so the glyph is covered in both dimensions. Suggestion overlays show the same complete word area.
- **Redaction boxes covered parts of the lines above and below**: word boxes span ascender to descender, so with normal line spacing a redaction rectangle overlapped neighbouring lines, blackening parts of their letters in the preview and deleting their glyphs on export. Rectangles are now trimmed against the neighbouring lines (at the middle of the overlap on image pages, with a small horizontal margin).
- **Ordinary Italian words proposed for redaction** ("Certifico", "Prescrivo", "Dichiaro", headings of medical and legal documents): PERSON/LOCATION hits are now judged from evidence in the text (titles and cues such as "nato a" or "residente in", name shape, known given names and places, repetition) and rejected when they are Italian verb forms or common medical, legal and administrative words, including at the start or end of a longer span. Names split over two lines ("Esposito" / "Giovanni") are recognised as one name.
- **Weak identifier patterns produced false hits**: fiscal codes, VAT numbers, wallets, card numbers, driver licences, passports, ID cards and phone numbers must now sit on token boundaries, and the weak ones need their context word and uppercase form; dates, year ranges and short digit runs are no longer read as phone numbers. A checksum-valid identifier glued to a label is redacted as a whole token instead of being dropped.
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 0834584..67e2847 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -105,6 +105,10 @@ If the GUI opens → **You're ready to develop!** 🎉
```
NullifyPDF/
|-- NullifyPDF.py # Main PySide6 application
+|-- model_manager.py # Pinned local model catalog and integrity checks
+|-- model_dialog.py # Model consent and download UI
+|-- onnx_ner.py # ONNX token-classification adapter
+|-- word_import.py # Safe DOCX text/table importer
|-- privacy_core.py # Privacy modes and encrypted restore maps
|-- update_check.py # Qt-free update check: versions, GitHub releases, asset choice, download
|-- pii_detection.py # Qt-free AI detection helpers (offsets, filters, whole-word matching)
@@ -142,6 +146,9 @@ NullifyPDF/
| **presidio-analyzer** | 2.2.366 | PII detection |
| **spaCy** | 3.8.18 | NLP for entity recognition |
| **cryptography** | 50.0.2 | Encrypted restore maps |
+| **onnxruntime** | 1.31.1 | CPU inference for optional local PII models |
+| **tokenizers** | 0.23.3 | Fast local tokenizer with character offsets |
+| **python-docx** | — | Not required; DOCX import uses the standard-library OOXML reader |
| **pytest** | 9.2.0 | Testing framework |
### Language Models (Auto-Downloaded)
@@ -590,5 +597,5 @@ The release workflows fill each GitHub release body with `scripts/extract_releas
---
-*Last updated: 2026-11-10*
+*Last updated: 2026-11-11*
*← [Troubleshooting](./TROUBLESHOOTING.md) | [Back to README →](./README.md)*
diff --git a/NullifyPDF.py b/NullifyPDF.py
index a74a26f..f034698 100644
--- a/NullifyPDF.py
+++ b/NullifyPDF.py
@@ -92,6 +92,7 @@
NON_PRESIDIO_TYPES,
build_detections,
build_page_index,
+ Candidate,
classify_page,
covered_fraction,
expand_group_ids,
@@ -122,10 +123,14 @@
suggest_signature_areas,
)
from review_dialog import ReviewDialog
+from model_dialog import ModelDialog
from help_guide import GuideDialog, apply_tooltips
from dictionary_dialog import DictionaryDialog
from word_lists import PDFListManager
+from word_import import docx_to_pdf
import secure_store
+from model_manager import CATALOG, MODEL_CANDIDATES, ModelStore
+from onnx_ner import OnnxTokenDetector
# PyMuPDF defines these constants dynamically, which mypy cannot see; read
# them once here (with the suppression) instead of at every call site.
@@ -133,7 +138,7 @@
PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined]
__version__ = "2.4.0"
-__version_prerelease__ = "beta.3"
+__version_prerelease__ = "beta.4"
APP_VERSION = (
f"{__version__}-{__version_prerelease__}"
if __version_prerelease__
@@ -469,7 +474,9 @@ def band_covers_target(page: Any, band: "fitz.Rect", full: "fitz.Rect") -> bool:
and survive the redaction.
"""
chars = target_chars(page, full)
- return all(_core_hits(tuple(ch["bbox"]), band) for ch in chars)
+ # An empty clipped result is not proof of coverage. A band trimmed
+ # against neighbouring lines can otherwise leave part of the glyph.
+ return bool(chars) and all(_core_hits(tuple(ch["bbox"]), band) for ch in chars)
def text_under_rect(page: Any, rect: "fitz.Rect") -> str:
@@ -902,6 +909,7 @@ def __init__(self) -> None:
super().__init__()
self.analyzer: Optional[Any] = None
self.loaded_langs: List[str] = []
+ self.model_detectors: List[Tuple[str, OnnxTokenDetector]] = []
self._stop_requested = False
def cleanup(self) -> None:
@@ -917,6 +925,7 @@ def cleanup(self) -> None:
# spaCy (read-only property), so we simply drop the reference.
self.analyzer = None
self.loaded_langs.clear()
+ self.model_detectors.clear()
except Exception as e:
logging.getLogger("nullifypdf").debug(f"Error during AI cleanup: {e}")
@@ -1017,6 +1026,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
)
self.loaded_langs = target_langs
self.log_sig.emit("Scansione privacy in corso...")
+ self._load_local_detectors()
targets = list(enabled_types)
presidio_targets = [
t for t in targets if t not in NON_PRESIDIO_TYPES
@@ -1118,6 +1128,36 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
index, candidates, allowlist=allow_terms,
source="ocr" if used_ocr else "text", stats=stats,
)
+ model_candidates: List[Tuple[Candidate, str]] = []
+ for model_id, detector in self.model_detectors:
+ try:
+ for entity in detector.detect(index.text):
+ entity_type = entity.label
+ if entity_type not in targets:
+ continue
+ model_candidates.append((
+ Candidate(
+ entity.start,
+ entity.end,
+ entity.text,
+ entity_type,
+ min(1.0, max(0.0, entity.score)),
+ state="suggested",
+ reasons=[f"model:{model_id}"],
+ ),
+ model_id,
+ ))
+ except Exception as exc:
+ self.log_sig.emit(f"Avviso: modello locale {model_id} non disponibile a pagina {i + 1}: {exc}")
+ for candidate, model_id in model_candidates:
+ model_detections = build_detections(
+ index,
+ [candidate],
+ allowlist=allow_terms,
+ source=f"model:{model_id}",
+ stats=stats,
+ )
+ detections.extend(model_detections)
detections.extend(signature_dets)
for key, n in stats.items():
totals[key] = totals.get(key, 0) + n
@@ -1159,6 +1199,22 @@ def _first_person_verbs(artifacts: Any) -> Set[str]:
return set()
return out
+ def _load_local_detectors(self) -> None:
+ """Load every installed, manifest-verified local model once per scan."""
+ if self.model_detectors:
+ return
+ store = ModelStore(pathlib.Path.home() / ".nullifypdf" / "models")
+ for manifest in CATALOG:
+ if not manifest.enabled or not store.is_installed(manifest):
+ continue
+ try:
+ self.model_detectors.append(
+ (manifest.model_id, OnnxTokenDetector(store.model_dir(manifest), manifest))
+ )
+ self.log_sig.emit(f"Modello locale caricato: {manifest.model_id}")
+ except Exception as exc:
+ self.log_sig.emit(f"Avviso: modello locale {manifest.model_id} ignorato: {exc}")
+
def _page_needs_ocr(self, page: Any, extracted_text: str) -> bool:
"""Return True when a page looks scanned or text extraction is too poor."""
text = (extracted_text or "").strip()
@@ -1494,6 +1550,10 @@ def build_ui(self) -> None:
btn_dict = QPushButton("Dizionari")
btn_dict.clicked.connect(self.cmd_dict)
s_lay.addWidget(btn_dict)
+ btn_models = QPushButton("Modelli locali")
+ btn_models.setObjectName("model_settings")
+ btn_models.clicked.connect(self.cmd_models)
+ s_lay.addWidget(btn_models)
self.btn_ai = QPushButton("Auto Redact")
self.btn_ai.setObjectName("Danger")
self.btn_ai.clicked.connect(self.cmd_auto_ai)
@@ -1598,6 +1658,16 @@ def dropEvent(self, e: QDropEvent) -> None:
if urls and urls[0].isLocalFile():
self.load_path(urls[0].toLocalFile())
+ def cmd_models(self) -> None:
+ """Show the curated local-model catalog and its verification state."""
+ store = ModelStore(self.config_dir / "models")
+ ModelDialog(
+ MODEL_CANDIDATES,
+ manifests=CATALOG,
+ store=store,
+ parent=self,
+ ).exec()
+
def write_log(self, m: str) -> None:
"""Write message to UI log with color coding.
@@ -1632,7 +1702,9 @@ def update_progress(self, c: int, t: int) -> None:
def cmd_open(self) -> None:
"""Open file dialog to load PDF."""
- p, _ = QFileDialog.getOpenFileName(self, "Apri PDF", "", "PDF (*.pdf)")
+ p, _ = QFileDialog.getOpenFileName(
+ self, "Apri documento", "", "PDF e Word (*.pdf *.docx);;PDF (*.pdf);;Word DOCX (*.docx)"
+ )
if p:
self.load_path(p)
@@ -1648,12 +1720,14 @@ def load_path(self, path: str) -> None:
if not os.path.exists(path):
self.write_log(f"ERRORE: File non trovato: {path}")
return
- if not path.lower().endswith('.pdf'):
- self.write_log("ERRORE: Solo file PDF sono supportati")
+ suffix = pathlib.Path(path).suffix.lower()
+ if suffix not in {".pdf", ".docx"}:
+ self.write_log("ERRORE: Sono supportati solo PDF e Word DOCX")
return
try:
- tdoc = fitz.open(path)
+ imported_word = suffix == ".docx"
+ tdoc = docx_to_pdf(path) if imported_word else fitz.open(path)
if tdoc.needs_pass:
self.write_log("ERRORE: PDF cifrato - inserire password non supportato")
tdoc.close()
@@ -1669,7 +1743,8 @@ def load_path(self, path: str) -> None:
self.pending_suggestions = []
self._update_review_button()
self.adjust_zoom(0)
- self.write_log(f"Caricato: {os.path.basename(path)}")
+ label = "Word DOCX convertito in PDF locale" if imported_word else "PDF"
+ self.write_log(f"Caricato {label}: {os.path.basename(path)}")
self._warn_if_scanned_pdf()
except FileNotFoundError:
self.write_log(f"ERRORE: File non trovato")
@@ -1737,7 +1812,9 @@ def _draw_suggestion_overlays(self) -> None:
for s in self.pending_suggestions:
if int(s.get("page", -1)) != self.page_num:
continue
- for rect in s.get("rects") or []:
+ # The clamped band is internal redaction geometry. Showing it
+ # makes a correctly detected word appear partially selected.
+ for rect in s.get("full_rects") or s.get("rects") or []:
self.scene.addItem(SuggestionItem(tuple(rect), s, self.scale))
def _highlight_suggestion(self, suggestion: Any) -> None:
@@ -2584,8 +2661,18 @@ def already_covered(r: fitz.Rect) -> bool:
tuple(i["bbox"]) for i in page.get_image_info(hashes=False)
],
)
+ # Accepted redactions must cover the actual word box. The
+ # clamped index is useful for analysis, but using its thin band
+ # for the final annotation can leave the top/bottom of a glyph
+ # visible (as happens with tightly spaced lines).
+ full_page_index = build_page_index(page.get_text("words"), clamp=False)
+
+ def cover_word(rect: fitz.Rect) -> fitz.Rect:
+ padded = fitz.Rect(rect.x0 - 0.5, rect.y0 - 0.5, rect.x1 + 0.5, rect.y1 + 0.5)
+ return padded & page.rect
+
for bw in self.blocklist:
- for r in (fitz.Rect(*t) for t in whole_word_rects(page_index, bw)):
+ for r in (cover_word(fitz.Rect(*t)) for t in whole_word_rects(full_page_index, bw)):
if not already_covered(r):
self._add_privacy_redaction(
page,
@@ -2602,9 +2689,9 @@ def already_covered(r: fitz.Rect) -> bool:
fulls = detection.get("full_rects") or []
for k, band in enumerate(rects):
if k < len(fulls):
- full = fitz.Rect(*fulls[k])
- if band != full and not band_covers_target(page, band, full):
- rects[k] = full # never leave a glyph of the target behind
+ # Always use the original word box for the final
+ # redaction, with a small margin in both dimensions.
+ rects[k] = cover_word(fitz.Rect(*fulls[k]))
if entity_type == "SIGNATURE":
# Graphic area (signature / stamp): no text, no value.
for r in rects:
@@ -2620,11 +2707,11 @@ def already_covered(r: fitz.Rect) -> bool:
protected = [
fitz.Rect(*t)
for aw in self.allowlist
- for t in whole_word_rects(page_index, aw)
+ for t in whole_word_rects(full_page_index, aw)
]
rects = [
fitz.Rect(*t)
- for t in whole_word_rects(page_index, word)
+ for t in whole_word_rects(full_page_index, word)
if not any(
pr.contains(fitz.Point((t[0] + t[2]) / 2, (t[1] + t[3]) / 2))
for pr in protected
@@ -2633,7 +2720,7 @@ def already_covered(r: fitz.Rect) -> bool:
for k, r in enumerate(rects):
if not already_covered(r):
full_box = (
- fitz.Rect(*fulls[k])
+ cover_word(fitz.Rect(*fulls[k]))
if (k < len(fulls) and raw_rects) else None
)
self._add_privacy_redaction(
diff --git a/README.md b/README.md
index 50971b2..be56a0e 100644
--- a/README.md
+++ b/README.md
@@ -39,6 +39,8 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi
- 🛡️ **Privacy Export Modes** — Choose irreversible removal or reversible pseudonymization with a separately encrypted restore map; the exported file is sanitised (annotations, attachments, scripts, metadata, hidden text) and verified
- 📖 **Built-in guide and tooltips** — Press F1 for an Italian guide to every function; every control shows a description on hover
- 🔍 **Review of uncertain hits** — Detections without enough evidence are suggested for review instead of being applied or lost silently
+- 🧠 **Optional local models** — Reviewed ONNX PII models can be downloaded by the user from pinned Hugging Face revisions; files are SHA-256 verified and models only add review suggestions
+- 📄 **Word import** — DOCX paragraphs and tables can be imported locally into the analysis workflow; legacy binary DOC files are not supported
- 🔎 **OCR Support** — Bundled EN/IT Tesseract language data for scanned PDFs
- 🖼️ **Blindfold Mode** — One-click image/logo censoring with professional placeholder: `[ IMAGE REMOVED ]`
- 📦 **Native Cross-Platform** — Automated build scripts generate Windows `.exe`, macOS ZIP bundles, and Linux `.deb`/`.rpm` packages
@@ -55,6 +57,7 @@ To keep NullifyPDF lightweight, 100% offline, and secure, be aware of these tech
| ---------------------------------- | ------------------------------------------------------------------------------------------------------- |
| **Handwritten Text** | NLP models cannot analyze non-digitized handwriting. |
| **Password-Protected PDFs** | Encrypted documents are blocked at load. Decrypt before importing. |
+| **Legacy Word DOC files** | Only modern `.docx` files are supported; save legacy files as DOCX before importing. |
| **Digital Signatures Invalidated** | Privacy export changes the PDF; cryptographic signatures (PAdES, notarized) become invalid. |
> [!WARNING]
@@ -264,5 +267,5 @@ Want to help improve NullifyPDF? See [CONTRIBUTING.md](./CONTRIBUTING.md) for gu
---
-*Last updated: 2026-11-10*
+*Last updated: 2026-11-11*
*[User Guide →](./USER_GUIDE.md)*
diff --git a/USER_GUIDE.md b/USER_GUIDE.md
index b3cf0da..e68321e 100644
--- a/USER_GUIDE.md
+++ b/USER_GUIDE.md
@@ -11,7 +11,7 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel
| 🎯 Step | ⏱️ Time | What You'll Do |
| ----------------- | ------- | ----------------------------- |
-| **1️⃣ Load** | 30 sec | Open a PDF file |
+| **1️⃣ Load** | 30 sec | Open a PDF or Word DOCX file |
| **2️⃣ Configure** | 30 sec | Select document language |
| **3️⃣ Auto-Redact** | 1-2 min | Run AI to find sensitive data |
| **4️⃣ Review** | 1-2 min | Manually fix any mistakes |
@@ -23,14 +23,20 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel
> [!TIP]
> The program has its own guide in Italian: press **F1** or the **Guida** button in the sidebar. Resting the pointer on any button, option or field shows a short description.
-## Step 1️⃣ — Load Your PDF
+## Step 1️⃣ — Load Your PDF or Word Document
1. Click the blue **"Apri PDF"** button in the top left
-2. Select a PDF file from your computer
+2. Select a PDF or modern Word **DOCX** file from your computer
3. The document appears in the center. Scroll with mouse or use arrow buttons (top right)
> [!NOTE]
-> Supported format: unencrypted PDF. Digital text is scanned directly; scanned pages require OCR. If NullifyPDF detects a page with no extractable text, a popup confirms it and reminds you to enable **OCR PDF scansionati** before AI scanning — manual selection (Step 4) works on scanned pages either way.
+> Supported formats: unencrypted PDF and DOCX. A DOCX is read locally and converted to an internal PDF representation; the original Word file is never modified and macros, external links and embedded objects are not executed. Legacy binary `.doc` files must be saved as DOCX first. Digital PDF text is scanned directly; scanned pages require OCR.
+
+### Optional local models
+
+Open **Modelli locali** in the sidebar to view the reviewed shortlist. Models are optional and are not included in the installer. Before downloading, the dialog shows the Hugging Face source, license, pinned revision and total size. The progress bar reports the overall percentage across all model files.
+
+Only entries with a complete application manifest can be downloaded. The manifest fixes the revision, allowed files, expected sizes and SHA-256 values. Experimental entries without those checks remain unavailable. Installed models run locally and add suggestions for review; they never apply a redaction automatically.
---
@@ -308,5 +314,5 @@ python3.13 NullifyPDF.py
---
-*Last updated: 2026-11-10*
+*Last updated: 2026-11-11*
*← [Back to README](./README.md) | [Troubleshooting →](./TROUBLESHOOTING.md)*
diff --git a/build_local.py b/build_local.py
index cf2b9b2..1ff6150 100644
--- a/build_local.py
+++ b/build_local.py
@@ -397,8 +397,8 @@ def build_app() -> None:
from PyInstaller.utils.hooks import collect_all
datas = {datas_literal}
binaries = []
-hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md'] + (['keyring'] if sys.platform != 'win32' else []):
+hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers']
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
@@ -414,8 +414,8 @@ def build_app() -> None:
from PyInstaller.utils.hooks import collect_all
datas = {datas_literal}
binaries = []
-hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md'] + (['keyring'] if sys.platform != 'win32' else []):
+hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers']
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
diff --git a/help_guide.py b/help_guide.py
index 1ce641e..56cf597 100644
--- a/help_guide.py
+++ b/help_guide.py
@@ -50,6 +50,11 @@
"Profilo documenti sanitari": "Cerca anche date di nascita, numero tessera sanitaria, ricetta e codici di esenzione, e propone firme e timbri. I casi incerti diventano suggerimenti invece di essere scartati.",
"Maschera larghezza testo rimosso": "Arrotonda lo spazio lasciato dal testo rimosso, che altrimenti ne rivela la larghezza. Può spostare di poco il testo seguente.",
"Dizionari": "Modifica le parole da censurare sempre (blocklist) e da non censurare mai (allowlist), con scadenza.",
+ "Modelli locali": "Mostra i modelli locali candidati, la licenza, la copertura e lo stato di verifica. Il download richiede un manifest con revisione e SHA-256.",
+ "model_settings": "Gestisce il catalogo curato dei modelli locali. I modelli producono solo suggerimenti e non vengono scaricati senza consenso.",
+ "model_list": "Elenco curato dei modelli locali e del loro stato di verifica.",
+ "model_details": "Dettagli del modello selezionato, inclusi licenza, runtime, lingue e copertura dichiarata.",
+ "model_download": "Scarica un modello solo quando l'applicazione ne ha verificato manifest, revisione e SHA-256.",
"Auto Redact": "Analizza il documento e segna i dati personali da censurare. Nulla viene cancellato finché non esporti.",
"Rivedi suggerimenti": "Apre l'elenco dei rilevamenti incerti per decidere uno per uno se censurarli.",
"Pulisci Pagina": "Rimuove tutte le censure segnate sulla pagina corrente.",
@@ -78,6 +83,7 @@
"Pag.": "Pagina in cui è stato trovato l'elemento.",
"Valore": "Valore trovato, mascherato finché non spunti «Mostra i valori». Per le aree grafiche indica la zona.",
"Tipo": "Tipo di dato rilevato (persona, luogo, telefono, firma...).",
+ "Fonte": "Origine del suggerimento: regole, modello locale o OCR.",
"Punteggio": "Affidabilità del rilevamento, da 0 a 1.",
"Motivi": "Indizi che hanno portato a questo punteggio (contesto, parola comune, inizio riga...).",
"Mostra i valori": "Mostra in chiaro i valori trovati nella tabella.",
@@ -258,6 +264,16 @@ def widgets_without_tooltip(root: QWidget) -> List[str]:
Profilo documenti sanitari: vedi l'argomento dedicato.
Maschera larghezza testo rimosso: arrotonda lo spazio lasciato dal testo cancellato, che altrimenti ne rivela la larghezza. Può spostare di poco il testo seguente; è disattivata di default.
+"""),
+ ("Modelli locali", """
+Modelli locali
+Modelli locali mostra il catalogo curato dei modelli candidati per migliorare i suggerimenti su nomi, indirizzi e altri dati personali. I modelli, quando abilitati, lavorano solo sul testo estratto e non applicano censure da soli.
+
+- Ogni modello deve avere una licenza, una revisione e un SHA-256 verificati.
+- Il programma non carica codice remoto, file pickle o modelli arbitrari.
+- Il download richiede consenso esplicito e i pesi vengono conservati nella cartella dell'utente, non nell'installer.
+- La qualità dei modelli va misurata sul tipo di documento concreto: nessun modello garantisce anonimizzazione o copertura completa.
+
"""),
("Auto Redact", """
Auto Redact
@@ -272,10 +288,12 @@ def widgets_without_tooltip(root: QWidget) -> List[str]:
"""),
("Revisione dei suggerimenti", """
Revisione dei suggerimenti
-Al termine dell'analisi si apre l'elenco dei suggerimenti: pagina, valore (mascherato finché non spunti Mostra i valori), tipo, punteggio e motivi.
+Al termine dell'analisi si apre l'elenco dei suggerimenti: pagina, valore (mascherato finché non spunti Mostra i valori), tipo, punteggio, fonte e motivi.
- Spunta gli elementi da censurare e premi Applica selezionati: gli altri vengono scartati.
- Seleziona ≥ soglia spunta in blocco gli elementi con punteggio alto.
+- Il campo Tipo è modificabile: controlla o correggi la categoria prima di applicare il suggerimento.
+- La colonna Fonte distingue regole, modello locale e OCR.
- Rivedi più tardi lascia tutto in sospeso; il pulsante Rivedi suggerimenti (N) nella barra laterale riapre l'elenco.
- Doppio clic su una riga porta alla pagina; selezionando una riga il suo contorno si evidenzia.
diff --git a/model_dialog.py b/model_dialog.py
new file mode 100644
index 0000000..613eefd
--- /dev/null
+++ b/model_dialog.py
@@ -0,0 +1,182 @@
+"""Italian model-catalog dialog; it never downloads without explicit consent."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Optional
+
+from PySide6.QtCore import Qt
+from PySide6.QtWidgets import (
+ QApplication,
+ QDialog,
+ QDialogButtonBox,
+ QLabel,
+ QListWidget,
+ QListWidgetItem,
+ QMessageBox,
+ QPushButton,
+ QProgressDialog,
+ QTextBrowser,
+ QVBoxLayout,
+)
+
+from help_guide import apply_tooltips
+from model_manager import ModelCandidate, ModelManifest, ModelStore
+
+
+class ModelDialog(QDialog):
+ """Show the reviewed model shortlist and its verification state."""
+
+ def __init__(
+ self,
+ candidates: Iterable[ModelCandidate],
+ selected_id: Optional[str] = None,
+ manifests: Iterable[ModelManifest] = (),
+ store: Optional[ModelStore] = None,
+ parent: Any = None,
+ ) -> None:
+ super().__init__(parent)
+ self.setWindowTitle("Modelli locali")
+ self.resize(700, 520)
+ self._candidates = list(candidates)
+ self._selected_id = selected_id
+ self._manifests = {m.model_id: m for m in manifests}
+ self._store = store
+ lay = QVBoxLayout(self)
+ intro = QLabel(
+ "I modelli vengono eseguiti localmente e producono solo suggerimenti. "
+ "Il download sarà consentito soltanto quando revisione, licenza e SHA-256 "
+ "saranno fissati nel manifest dell'applicazione."
+ )
+ intro.setWordWrap(True)
+ lay.addWidget(intro)
+ self.list = QListWidget()
+ self.list.setObjectName("model_list")
+ for candidate in self._candidates:
+ item = QListWidgetItem(
+ f"{candidate.title} — {candidate.size_label} — {candidate.status}"
+ )
+ item.setData(Qt.ItemDataRole.UserRole, candidate.model_id)
+ self.list.addItem(item)
+ lay.addWidget(self.list)
+ self.details = QTextBrowser()
+ self.details.setObjectName("model_details")
+ lay.addWidget(self.details)
+ self.btn_download = QPushButton("Scarica modello")
+ self.btn_download.setObjectName("model_download")
+ self.btn_download.setEnabled(False)
+ self.btn_download.setToolTip(
+ "Il download è attivo solo per modelli con revisione, file, licenza e SHA-256 verificati."
+ )
+ lay.addWidget(self.btn_download)
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close)
+ buttons.rejected.connect(self.reject)
+ lay.addWidget(buttons)
+ self.list.currentItemChanged.connect(self._show_details)
+ self.btn_download.clicked.connect(self._download_selected)
+ if self.list.count():
+ row = 0
+ for i in range(self.list.count()):
+ if self.list.item(i).data(Qt.ItemDataRole.UserRole) == selected_id:
+ row = i
+ break
+ self.list.setCurrentRow(row)
+ apply_tooltips(self)
+
+ def _show_details(self, current: Optional[QListWidgetItem], _previous: Any) -> None:
+ if current is None:
+ self.details.clear()
+ return
+ model_id = current.data(Qt.ItemDataRole.UserRole)
+ candidate = next((c for c in self._candidates if c.model_id == model_id), None)
+ if candidate is None:
+ return
+ self.details.setPlainText(
+ f"Nome: {candidate.title}\n"
+ f"Repository: {candidate.repo_id}\n"
+ f"Licenza: {candidate.license}\n"
+ f"Runtime: {candidate.runtime}\n"
+ f"Lingue: {', '.join(candidate.languages)}\n"
+ f"Copertura dichiarata: {candidate.scope}\n"
+ f"Stato: {candidate.status}\n\n"
+ f"{candidate.notes}"
+ )
+ manifest = self._manifests.get(candidate.model_id)
+ if manifest is not None:
+ self.details.append(
+ f"\n\nFonte fissata: {manifest.source_url}\n"
+ f"File manifest: {len(manifest.files)}; SHA-256 verificati localmente."
+ )
+ self.btn_download.setEnabled(manifest is not None and self._store is not None)
+ if manifest is not None and self._store is not None and self._store.is_installed(manifest):
+ self.btn_download.setText("Modello già installato")
+ self.btn_download.setEnabled(False)
+ else:
+ self.btn_download.setText("Scarica modello")
+ if manifest is None:
+ self.btn_download.setToolTip(
+ "Modello sperimentale non scaricabile: manca un manifest verificato "
+ "con revisione, licenza, dimensioni e SHA-256."
+ )
+ else:
+ self.btn_download.setToolTip(
+ "Download HTTPS da Hugging Face alla revisione fissata nel manifest."
+ )
+
+ def _download_selected(self) -> None:
+ current = self.list.currentItem()
+ if current is None or self._store is None:
+ return
+ model_id = current.data(Qt.ItemDataRole.UserRole)
+ manifest = self._manifests.get(model_id)
+ if manifest is None:
+ return
+ answer = QMessageBox.question(
+ self,
+ "Consenso al download",
+ f"Scaricare {manifest.model_id} ({manifest.total_size / 1024**2:.1f} MB)?\n"
+ f"Licenza: {manifest.license}\n"
+ f"Revisione: {manifest.revision}\n\n"
+ "I file saranno salvati nella cartella locale dell'applicazione.",
+ QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No,
+ QMessageBox.StandardButton.No,
+ )
+ if answer != QMessageBox.StandardButton.Yes:
+ return
+ progress = QProgressDialog("Download modello...", "", 0, 100, self)
+ progress.setWindowTitle("Download modello")
+ progress.setAutoClose(False)
+ progress.setAutoReset(False)
+ progress.setMinimumDuration(0)
+ progress.setValue(0)
+ progress.show()
+ total_size = max(1, manifest.total_size)
+ completed = 0
+
+ def on_progress(done: int, total: int) -> None:
+ value = int(((completed + min(done, total)) / total_size) * 100)
+ progress.setValue(min(100, value))
+ QApplication.processEvents()
+
+ try:
+ for item in manifest.files:
+ url = (
+ f"https://huggingface.co/{manifest.repo_id}/resolve/"
+ f"{manifest.revision}/{item.name}?download=true"
+ )
+ self._store.install_file(
+ manifest, item, url, timeout=120.0, progress=on_progress
+ )
+ completed += item.size
+ progress.setValue(min(100, int(completed * 100 / total_size)))
+ QApplication.processEvents()
+ if not self._store.is_installed(manifest):
+ raise RuntimeError("verifica finale del modello fallita")
+ except Exception as exc:
+ progress.close()
+ QMessageBox.critical(self, "Download non riuscito", str(exc))
+ return
+ progress.setValue(100)
+ progress.close()
+ QMessageBox.information(self, "Download completato", "Il modello è stato verificato e installato.")
+ self._show_details(current, None)
+
diff --git a/model_manager.py b/model_manager.py
new file mode 100644
index 0000000..5d2d1bf
--- /dev/null
+++ b/model_manager.py
@@ -0,0 +1,386 @@
+"""Local model catalog, integrity checks and user-managed downloads.
+
+This module deliberately does not import an inference framework. Model files
+are untrusted data until their manifest, revision, size and SHA-256 digest have
+been checked. Loading an ONNX graph belongs to a separate runtime adapter.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import pathlib
+import platform
+import re
+import tempfile
+import urllib.error
+import urllib.request
+from dataclasses import dataclass, field
+from typing import Callable, Iterable, Optional, Tuple
+
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_REVISION = re.compile(r"^[0-9a-f]{40}$")
+_SAFE_SUFFIXES = frozenset({".onnx", ".ort", ".data", ".json", ".safetensors"})
+_MAX_MANIFEST_FILES = 32
+
+
+class ModelManifestError(ValueError):
+ """Raised when a model manifest is malformed or unsafe."""
+
+
+@dataclass(frozen=True)
+class ModelFile:
+ name: str
+ size: int
+ sha256: str
+
+ def validate(self) -> None:
+ path = pathlib.PurePosixPath(self.name)
+ if not self.name or path.is_absolute() or ".." in path.parts:
+ raise ModelManifestError("percorso file modello non sicuro")
+ if path.suffix.lower() not in _SAFE_SUFFIXES:
+ raise ModelManifestError(f"formato modello non consentito: {self.name}")
+ if self.size <= 0:
+ raise ModelManifestError("dimensione file non valida")
+ if not _SHA256.fullmatch(self.sha256):
+ raise ModelManifestError("SHA-256 non valido")
+
+
+@dataclass(frozen=True)
+class ModelManifest:
+ model_id: str
+ repo_id: str
+ revision: str
+ license: str
+ runtime: str
+ labels: Tuple[str, ...]
+ files: Tuple[ModelFile, ...]
+ enabled: bool = False
+ notes: str = ""
+
+ def validate(self) -> None:
+ if not self.model_id or not self.repo_id:
+ raise ModelManifestError("identificativo modello mancante")
+ if not _REVISION.fullmatch(self.revision):
+ raise ModelManifestError("la revisione deve essere un commit SHA-1 completo")
+ if self.runtime not in {"onnx", "ort"}:
+ raise ModelManifestError("runtime non consentito")
+ if not self.license.strip():
+ raise ModelManifestError("licenza mancante")
+ if not self.files or len(self.files) > _MAX_MANIFEST_FILES:
+ raise ModelManifestError("elenco file modello non valido")
+ names = set()
+ for item in self.files:
+ item.validate()
+ if item.name in names:
+ raise ModelManifestError("file duplicato nel manifest")
+ names.add(item.name)
+ if self.runtime == "onnx" and not any(
+ f.name.lower().endswith(".onnx") for f in self.files
+ ):
+ raise ModelManifestError("manifest ONNX senza file .onnx")
+
+ @property
+ def total_size(self) -> int:
+ return sum(item.size for item in self.files)
+
+ @property
+ def source_url(self) -> str:
+ return f"https://huggingface.co/{self.repo_id}/tree/{self.revision}"
+
+
+@dataclass(frozen=True)
+class HardwareProfile:
+ cpu_count: int
+ memory_bytes: Optional[int]
+ system: str = platform.system()
+
+
+@dataclass(frozen=True)
+class ModelRecommendation:
+ level: str
+ model_id: Optional[str]
+ reason: str
+
+
+@dataclass(frozen=True)
+class ModelCandidate:
+ """Reviewed catalog entry which is not necessarily installable yet."""
+
+ model_id: str
+ title: str
+ repo_id: str
+ license: str
+ runtime: str
+ languages: Tuple[str, ...]
+ size_label: str
+ scope: str
+ status: str
+ notes: str
+
+
+MODEL_CANDIDATES: Tuple[ModelCandidate, ...] = (
+ ModelCandidate(
+ "minilmv2-en-it",
+ "MiniLMv2 EN/IT",
+ "jkomyno/minilmv2-l6-h384-ner-en-it-q8",
+ "MIT",
+ "ONNX INT8",
+ ("en", "it"),
+ "circa 30 MB",
+ "persone, organizzazioni, indirizzi",
+ "verificato",
+ "Candidato leggero; non copre gli identificativi strutturati.",
+ ),
+ ModelCandidate(
+ "piiscan",
+ "PiiScan",
+ "vectorsense/piiscan",
+ "Apache-2.0",
+ "ONNX INT8",
+ ("multilingue",),
+ "circa 139 MB",
+ "PII generiche e identificativi",
+ "manifest verificato; italiano da misurare",
+ "La scheda dichiara 30 lingue, ma non elenca esplicitamente l'italiano.",
+ ),
+ ModelCandidate(
+ "openmed-fastclinical",
+ "OpenMed PII Italian FastClinical Small",
+ "OpenMed/OpenMed-PII-Italian-FastClinical-Small-82M-v1-onnx-android",
+ "Apache-2.0",
+ "ONNX INT8",
+ ("it",),
+ "circa 190 MB",
+ "PII e identificativi clinici italiani",
+ "verificato",
+ "Modello italiano specializzato in testo clinico; esempi dichiarati sintetici.",
+ ),
+ ModelCandidate(
+ "nym-pii",
+ "nym PII multilingual",
+ "Wismut/nym-pii-multilingual",
+ "MIT",
+ "ONNX INT8",
+ ("multilingue",),
+ "circa 359 MB INT8",
+ "PII, OCR rumoroso, documenti strutturati",
+ "sperimentale",
+ "Maggiore costo di memoria; qualità italiana da misurare.",
+ ),
+ ModelCandidate(
+ "eu-pii-safeguard",
+ "EU PII Safeguard",
+ "tabularisai/eu-pii-safeguard",
+ "da verificare",
+ "ONNX derivato",
+ ("26 lingue UE",),
+ "non verificata",
+ "PII europee",
+ "sperimentale",
+ "La conversione ONNX e la licenza del sorgente richiedono verifica.",
+ ),
+)
+
+
+MINILM_REVISION = "61dd72a5cac8e8d3e2702b57cba8154a8720c2af"
+PIISCAN_REVISION = "87e6ff6245ea325880af9f38e6dad8493f2ec123"
+
+
+VERIFIED_MANIFESTS: Tuple[ModelManifest, ...] = (
+ ModelManifest(
+ model_id="minilmv2-en-it",
+ repo_id="jkomyno/minilmv2-l6-h384-ner-en-it-q8",
+ revision=MINILM_REVISION,
+ license="MIT",
+ runtime="onnx",
+ labels=("O", "B-ADDRESS", "I-ADDRESS", "B-ORGANIZATION", "I-ORGANIZATION", "B-PERSON", "I-PERSON"),
+ files=(
+ ModelFile("config.json", 1075, "8f4608824f674a3a2427fc6751ebf3cc64766f3c14e3976cb0c46fbbb4dbc83b"),
+ ModelFile("tokenizer.json", 3558647, "7b62d797c95d1563d3467f48daad0eca3e150f2d587e7e687b557c9ef1b25473"),
+ ModelFile("tokenizer_config.json", 387, "e8ba8e4ec2ade6868d62c5d8a993c49cd44cdcf34aadc58bd764b5f4e752e338"),
+ ModelFile("onnx/model_quantized.onnx", 30585613, "d1fb8efd376b7026faeba43aa60b602d28902d3a99ca9bfa19bce1edcbded5a8"),
+ ),
+ enabled=True,
+ notes="Modello leggero per PERSON, ORGANIZATION e ADDRESS.",
+ ),
+ ModelManifest(
+ model_id="piiscan",
+ repo_id="vectorsense/piiscan",
+ revision=PIISCAN_REVISION,
+ license="Apache-2.0",
+ runtime="onnx",
+ labels=("O", "B-PERSON_NAME", "I-PERSON_NAME", "B-DATE_TIME", "I-DATE_TIME", "B-EMAIL_ADDRESS", "I-EMAIL_ADDRESS", "B-CITY", "I-CITY", "B-STREET_ADDRESS", "I-STREET_ADDRESS", "B-GOVERNMENT_ID", "I-GOVERNMENT_ID", "B-PHONE_NUMBER", "I-PHONE_NUMBER", "B-AGE", "I-AGE", "B-POSTAL_CODE", "I-POSTAL_CODE", "B-PAYMENT_CARD_NUMBER", "I-PAYMENT_CARD_NUMBER", "B-DRIVER_LICENSE_NUMBER", "I-DRIVER_LICENSE_NUMBER", "B-TAX_ID", "I-TAX_ID", "B-PASSPORT_NUMBER", "I-PASSPORT_NUMBER"),
+ files=(
+ ModelFile("config.json", 2049, "42cafc5b1ebcaafd3fa99a261f042bffc627c7f9433f2ee70421e2f9cff85f56"),
+ ModelFile("labels.json", 547, "d162bf06f29a8e5f09e7158c5847a531f0637e74b4b615286ff5fe931d6f7104"),
+ ModelFile("thresholds.json", 1258, "098fddcfa67451a0b3e4243018f6cb540660ebc5abe28888e0ddd75269e6d6e2"),
+ ModelFile("tokenizer.json", 2919461, "2c550ee6d187a1eeb0982a7af79b42748d7815c08b7221d01a8b06c4802316c3"),
+ ModelFile("tokenizer_config.json", 1397, "d22f2844d37c4f6e15b7f5220c4e33597f328ff7461f585e759f7f9914d0a742"),
+ ModelFile("model.int8.onnx", 135144362, "3ddde8cf53d34c4a6fc4c29ea2c1a28b65efdb84761e945b9afb45984a9c5342"),
+ ),
+ enabled=True,
+ notes="Modello CPU multilanguage con offset esatti; italiano da misurare sul corpus dell'app.",
+ ),
+ ModelManifest(
+ model_id="openmed-fastclinical",
+ repo_id="OpenMed/OpenMed-PII-Italian-FastClinical-Small-82M-v1-onnx-android",
+ revision="05cef784e0234e82c5806d2ec2451a1385c1d962",
+ license="Apache-2.0",
+ runtime="onnx",
+ labels=("O", "B-FIRSTNAME", "B-LASTNAME", "B-CITY", "B-ORGANIZATION", "B-EMAIL", "B-PHONE", "B-IBAN", "B-DATEOFBIRTH", "B-STREET", "B-ZIPCODE"),
+ files=(
+ ModelFile("config.json", 4505, "a91d68ba87154e94053ef61db86583c60703aeb3d4961058ae9886fc60487996"),
+ ModelFile("id2label.json", 1656, "a855c964ef7763ec51b63aad7ec43e43ae5a89f1e5f221cd2852326107ad7d95"),
+ ModelFile("tokenizer.json", 3558906, "ca4e5a9389c0dd76c509ba115a772da3a85c43aa10b9dfb966782907062de52b"),
+ ModelFile("tokenizer_config.json", 536, "828fb35f1e12914318c360d7b04f4c3fc4b7a9912b81cdf9ea37fb7a7da7d173"),
+ ModelFile("model_int8.onnx", 199020782, "c29ad148bb8bd33392a2dd94bb469108ca70ab2e4fe3cca374a879cfb942d62c"),
+ ),
+ enabled=True,
+ notes="ONNX INT8 CPU, 82M parametri, contesto 512; copertura italiana clinica da validare sul corpus dell'app.",
+ ),
+)
+
+
+CATALOG: Tuple[ModelManifest, ...] = VERIFIED_MANIFESTS
+
+
+def _windows_memory_bytes() -> Optional[int]:
+ if platform.system() != "Windows":
+ return None
+ try:
+ import ctypes
+
+ class Status(ctypes.Structure):
+ _fields_ = [
+ ("length", ctypes.c_ulong),
+ ("memory_load", ctypes.c_ulong),
+ ("total", ctypes.c_ulonglong),
+ ("available", ctypes.c_ulonglong),
+ ("total_page", ctypes.c_ulonglong),
+ ("available_page", ctypes.c_ulonglong),
+ ("total_virtual", ctypes.c_ulonglong),
+ ("available_virtual", ctypes.c_ulonglong),
+ ("available_extended", ctypes.c_ulonglong),
+ ]
+
+ status = Status()
+ status.length = ctypes.sizeof(Status)
+ if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
+ return int(status.total)
+ except (AttributeError, OSError, TypeError):
+ pass
+ return None
+
+
+def detect_hardware() -> HardwareProfile:
+ """Return conservative hardware information using only the standard library."""
+ return HardwareProfile(
+ cpu_count=max(1, int(os.cpu_count() or 1)),
+ memory_bytes=_windows_memory_bytes(),
+ )
+
+
+def recommend_level(
+ hardware: HardwareProfile,
+ manifests: Iterable[ModelManifest] = CATALOG,
+) -> ModelRecommendation:
+ """Recommend a level without enabling an unverified model."""
+ available = [m for m in manifests if m.enabled]
+ if hardware.memory_bytes is not None and hardware.memory_bytes < 8 * 1024**3:
+ return ModelRecommendation("nessuno", None, "RAM inferiore alla soglia consigliata")
+ if not available:
+ return ModelRecommendation(
+ "nessuno", None, "nessun modello verificato disponibile nel catalogo"
+ )
+ small = min(available, key=lambda m: m.total_size)
+ return ModelRecommendation(
+ "piccolo", small.model_id, "primo modello verificato compatibile disponibile"
+ )
+
+
+def sha256_file(path: pathlib.Path, chunk_size: int = 1024 * 1024) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(chunk_size), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def verify_model_files(directory: pathlib.Path, manifest: ModelManifest) -> None:
+ """Verify presence, exact size and digest of every manifest file."""
+ manifest.validate()
+ root = directory.resolve()
+ for item in manifest.files:
+ path = (root / pathlib.PurePosixPath(item.name)).resolve()
+ if root != path and root not in path.parents:
+ raise ModelManifestError("file modello fuori dalla directory")
+ if not path.is_file():
+ raise ModelManifestError(f"file modello mancante: {item.name}")
+ if path.stat().st_size != item.size:
+ raise ModelManifestError(f"dimensione inattesa: {item.name}")
+ if sha256_file(path) != item.sha256:
+ raise ModelManifestError(f"hash non corrispondente: {item.name}")
+
+
+class ModelStore:
+ """Store models outside the repository and install them atomically."""
+
+ def __init__(self, root: pathlib.Path) -> None:
+ self.root = root
+
+ def model_dir(self, manifest: ModelManifest) -> pathlib.Path:
+ manifest.validate()
+ return self.root / manifest.model_id / manifest.revision
+
+ def is_installed(self, manifest: ModelManifest) -> bool:
+ try:
+ verify_model_files(self.model_dir(manifest), manifest)
+ return True
+ except (OSError, ModelManifestError):
+ return False
+
+ def install_file(
+ self,
+ manifest: ModelManifest,
+ item: ModelFile,
+ url: str,
+ *,
+ timeout: float = 30.0,
+ progress: Optional[Callable[[int, int], None]] = None,
+ ) -> pathlib.Path:
+ """Download one HTTPS file, verify it, then atomically install it."""
+ manifest.validate()
+ item.validate()
+ if not url.lower().startswith("https://"):
+ raise ModelManifestError("sono consentiti solo download HTTPS")
+ destination = self.model_dir(manifest) / pathlib.PurePosixPath(item.name)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ fd, temp_name = tempfile.mkstemp(
+ prefix=".model-", suffix=".part", dir=str(destination.parent)
+ )
+ os.close(fd)
+ temporary = pathlib.Path(temp_name)
+ try:
+ request = urllib.request.Request(url, headers={"Accept": "application/octet-stream"})
+ with urllib.request.urlopen(request, timeout=timeout) as response, temporary.open("wb") as out:
+ done = 0
+ while True:
+ chunk = response.read(1024 * 1024)
+ if not chunk:
+ break
+ out.write(chunk)
+ done += len(chunk)
+ if progress is not None:
+ progress(done, item.size)
+ if temporary.stat().st_size != item.size:
+ raise ModelManifestError(f"dimensione inattesa: {item.name}")
+ if sha256_file(temporary) != item.sha256:
+ raise ModelManifestError(f"hash non corrispondente: {item.name}")
+ os.replace(temporary, destination)
+ return destination
+ except (OSError, urllib.error.URLError, ModelManifestError):
+ temporary.unlink(missing_ok=True)
+ raise
+
diff --git a/onnx_ner.py b/onnx_ner.py
new file mode 100644
index 0000000..8c00033
--- /dev/null
+++ b/onnx_ner.py
@@ -0,0 +1,135 @@
+"""Small ONNX token-classification adapter without PyTorch."""
+
+from __future__ import annotations
+
+import json
+import pathlib
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+import numpy as np
+
+from model_manager import ModelManifest, ModelManifestError, verify_model_files
+
+
+@dataclass(frozen=True)
+class ModelEntity:
+ start: int
+ end: int
+ text: str
+ label: str
+ score: float
+
+
+def _group_label(label: str) -> Optional[str]:
+ label = label.upper().removeprefix("B-").removeprefix("I-")
+ if label in {"PERSON", "PERSON_NAME", "FULLNAME", "FIRSTNAME", "LASTNAME", "MIDDLENAME"}:
+ return "PERSON"
+ if label in {"ORGANIZATION", "ORG"}:
+ return "LOCATION"
+ if label in {"ADDRESS", "STREET_ADDRESS", "CITY", "POSTAL_CODE"}:
+ return "LOCATION"
+ if label == "EMAIL_ADDRESS":
+ return "EMAIL_ADDRESS"
+ if label == "EMAIL":
+ return "EMAIL_ADDRESS"
+ if label == "PHONE_NUMBER":
+ return "PHONE_NUMBER"
+ if label == "PHONE":
+ return "PHONE_NUMBER"
+ if label == "PAYMENT_CARD_NUMBER":
+ return "CREDIT_CARD"
+ if label == "CREDITCARD":
+ return "CREDIT_CARD"
+ if label == "DRIVER_LICENSE_NUMBER":
+ return "IT_DRIVER_LICENSE"
+ if label == "PASSPORT_NUMBER":
+ return "IT_PASSPORT"
+ if label == "TAX_ID":
+ return "IT_FISCAL_CODE"
+ if label == "IBAN":
+ return "IBAN_CODE"
+ if label in {"DATEOFBIRTH", "DATE_OF_BIRTH"}:
+ return "DATE_OF_BIRTH"
+ if label in {"GOVERNMENT_ID", "ID_DOC"}:
+ return "IT_IDENTITY_CARD"
+ return None
+
+
+class OnnxTokenDetector:
+ """Run a verified token classifier and return source-text entities."""
+
+ def __init__(self, model_dir: pathlib.Path, manifest: ModelManifest) -> None:
+ verify_model_files(model_dir, manifest)
+ try:
+ import onnxruntime as ort # type: ignore[import-untyped]
+ from tokenizers import Tokenizer
+ except ImportError as exc:
+ raise RuntimeError("servono onnxruntime e tokenizers per il modello locale") from exc
+ self.manifest = manifest
+ self._ort = ort
+ config = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))
+ self._tokenizer = Tokenizer.from_file(str(model_dir / "tokenizer.json"))
+ max_length = min(int(config.get("max_position_embeddings", 256)), 512)
+ pad_id = int(config.get("pad_token_id", 0))
+ self._tokenizer.enable_truncation(max_length=max_length)
+ self._tokenizer.enable_padding(length=max_length, pad_id=pad_id, pad_token="")
+ onnx_name = next(f.name for f in manifest.files if f.name.endswith(".onnx"))
+ self._session = ort.InferenceSession(
+ str(model_dir / onnx_name), providers=["CPUExecutionProvider"]
+ )
+ self._labels = {int(k): str(v) for k, v in config.get("id2label", {}).items()}
+ if not self._labels:
+ self._labels = {i: label for i, label in enumerate(manifest.labels)}
+
+ def detect(self, text: str) -> List[ModelEntity]:
+ if not text.strip():
+ return []
+ encoding = self._tokenizer.encode(text)
+ ids = np.asarray([encoding.ids], dtype=np.int64)
+ attention = np.asarray([encoding.attention_mask], dtype=np.int64)
+ feeds: Dict[str, Any] = {}
+ for input_meta in self._session.get_inputs():
+ name = input_meta.name
+ if name == "input_ids":
+ feeds[name] = ids
+ elif name == "attention_mask":
+ feeds[name] = attention
+ elif name == "token_type_ids":
+ feeds[name] = np.zeros_like(ids)
+ if "input_ids" not in feeds or "attention_mask" not in feeds:
+ raise RuntimeError("input ONNX non compatibili con token-classification")
+ output = self._session.run(None, feeds)[0]
+ logits = np.asarray(output)[0]
+ entities: List[ModelEntity] = []
+ current: Optional[Tuple[int, int, str, float]] = None
+ current_label = ""
+ offsets = encoding.offsets
+ for index, row in enumerate(logits):
+ label = self._labels.get(int(np.argmax(row)), "O")
+ score = float(np.exp(row - np.max(row))[int(np.argmax(row))] / np.exp(row - np.max(row)).sum())
+ if index >= len(offsets) or offsets[index][0] == offsets[index][1]:
+ continue
+ start, end = offsets[index]
+ if label == "O":
+ if current is not None:
+ entities.append(ModelEntity(current[0], current[1], current[2], current_label, current[3]))
+ current = None
+ continue
+ prefix, _, raw_label = label.partition("-")
+ group = _group_label(raw_label)
+ if group is None:
+ continue
+ if current is None or prefix == "B" or current_label != group or start > current[1]:
+ if current is not None:
+ entities.append(ModelEntity(current[0], current[1], current[2], current_label, current[3]))
+ current = (start, end, text[start:end], score)
+ current_label = group
+ else:
+ current = (current[0], end, text[current[0]:end], min(current[3], score))
+ if current is not None:
+ entities.append(ModelEntity(current[0], current[1], current[2], current_label, current[3]))
+ return [e for e in entities if e.text.strip() and 0 <= e.start < e.end <= len(text)]
+
+
+__all__ = ["ModelEntity", "OnnxTokenDetector"]
diff --git a/requirements.txt b/requirements.txt
index 8ef6010..a779a6e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,6 +11,9 @@ PyMuPDF==2.1
# NLP & Security Analysis
presidio-analyzer==2.2.366
spacy==3.8.18
+# Local token-classification models; PyTorch is intentionally not required at runtime.
+onnxruntime==1.31.1
+tokenizers==0.23.3
cryptography==50.0.2
# OS keyring (macOS Keychain / Linux Secret Service) for the local master key;
# Windows uses DPAPI instead, so it is not installed there.
diff --git a/review_dialog.py b/review_dialog.py
index 3919d30..b3bdd31 100644
--- a/review_dialog.py
+++ b/review_dialog.py
@@ -14,6 +14,7 @@
from PySide6.QtWidgets import (
QAbstractItemView,
QCheckBox,
+ QComboBox,
QDialog,
QDoubleSpinBox,
QHBoxLayout,
@@ -49,7 +50,7 @@
"ambiguous-place": "luogo omonimo di parola comune",
}
-COL_APPLY, COL_PAGE, COL_VALUE, COL_TYPE, COL_SCORE, COL_WHY = range(6)
+COL_APPLY, COL_PAGE, COL_VALUE, COL_TYPE, COL_SCORE, COL_SOURCE, COL_WHY = range(7)
def mask_value(value: str) -> str:
@@ -68,7 +69,7 @@ class ReviewDialog(QDialog):
def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> None:
super().__init__(parent)
self.setWindowTitle("Revisione suggerimenti")
- self.resize(760, 460)
+ self.resize(900, 460)
self._items = list(suggestions)
self._applied: List[Dict[str, Any]] = []
self._rejected: List[Dict[str, Any]] = []
@@ -80,11 +81,20 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non
)
intro.setWordWrap(True)
lay.addWidget(intro)
- self.table = QTableWidget(len(self._items), 6, self)
+ if len(self._items) > 30:
+ warning = QLabel(
+ f"Attenzione: {len(self._items)} suggerimenti. "
+ "Controlla l'elenco pagina per pagina; non approvare in blocco senza verifica."
+ )
+ warning.setObjectName("review_many_warning")
+ warning.setStyleSheet("color: #f59e0b; font-weight: bold;")
+ warning.setWordWrap(True)
+ lay.addWidget(warning)
+ self.table = QTableWidget(len(self._items), 7, self)
self.table.setObjectName("review_table")
self.table.verticalHeader().setVisible(False)
self.table.setHorizontalHeaderLabels(
- ["Censura", "Pag.", "Valore", "Tipo", "Punteggio", "Motivi"]
+ ["Censura", "Pag.", "Valore", "Tipo", "Punteggio", "Fonte", "Motivi"]
)
self.table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows)
self.table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers)
@@ -104,8 +114,15 @@ def __init__(self, suggestions: List[Dict[str, Any]], parent: Any = None) -> Non
self.table.setItem(row, COL_APPLY, apply_item)
self.table.setItem(row, COL_PAGE, QTableWidgetItem(str(int(it.get("page", 0)) + 1)))
self.table.setItem(row, COL_VALUE, QTableWidgetItem(""))
- self.table.setItem(row, COL_TYPE, QTableWidgetItem(str(it.get("entity_type", ""))))
+ type_box = QComboBox(self.table)
+ types = ["PERSON", "LOCATION", "EMAIL", "PHONE", "IBAN", "CARD", "FISCAL", "VAT", "LICENSE", "IDCARD", "PASSPORT", "BIRTHDATE", "HEALTHID", "EXEMPT", "DATA"]
+ type_box.addItems(types)
+ current_type = str(it.get("entity_type", "DATA"))
+ type_box.setCurrentText(current_type if current_type in types else "DATA")
+ type_box.setToolTip("Modifica il tipo attribuito al suggerimento prima dell'applicazione.")
+ self.table.setCellWidget(row, COL_TYPE, type_box)
self.table.setItem(row, COL_SCORE, QTableWidgetItem(f"{float(it.get('score', 0.0)):.2f}"))
+ self.table.setItem(row, COL_SOURCE, QTableWidgetItem(str(it.get("source", "regole"))))
why = ", ".join(REASON_LABELS.get(r, r) for r in it.get("reasons", []))
self.table.setItem(row, COL_WHY, QTableWidgetItem(why))
self._refresh_values()
@@ -188,6 +205,9 @@ def _on_apply(self) -> None:
self._applied, self._rejected = [], []
for row, it in enumerate(self._items):
checked = self._cell(row, COL_APPLY).checkState() == Qt.CheckState.Checked
+ type_box = self.table.cellWidget(row, COL_TYPE)
+ if isinstance(type_box, QComboBox):
+ it["entity_type"] = type_box.currentText()
(self._applied if checked else self._rejected).append(it)
self.accept()
diff --git a/tests/test_geometry.py b/tests/test_geometry.py
index 5a99fbe..1eed05f 100644
--- a/tests/test_geometry.py
+++ b/tests/test_geometry.py
@@ -204,6 +204,17 @@ def test_core_band_mixed_sizes_spares_neighbours(tmp_path):
assert "Alessandro" not in text and "Marchetti" not in text
+def test_empty_clipped_target_never_approves_thin_band():
+ doc = pymupdf.open()
+ page = doc.new_page()
+ page.insert_text((72, 100), "Mario Rossi", fontsize=11)
+ idx = pd.build_page_index(page.get_text("words"), clamp=False)
+ full = pymupdf.Rect(*pd.whole_word_rects(idx, "Mario")[0])
+ thin = pymupdf.Rect(full.x0, full.y0, full.x1, full.y0 + 0.01)
+ from NullifyPDF import band_covers_target
+ assert not band_covers_target(page, thin, full)
+
+
def test_no_safe_band_flags_collision_and_still_removes_target(tmp_path):
doc = make_doc(LINES, pitch=5)
page = doc[0]
diff --git a/tests/test_model_dialog.py b/tests/test_model_dialog.py
new file mode 100644
index 0000000..4b5be9d
--- /dev/null
+++ b/tests/test_model_dialog.py
@@ -0,0 +1,18 @@
+import os
+
+os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
+
+from PySide6.QtWidgets import QApplication
+
+from model_dialog import ModelDialog
+from model_manager import MODEL_CANDIDATES
+
+
+def test_model_dialog_lists_only_curated_candidates():
+ app = QApplication.instance() or QApplication([])
+ dialog = ModelDialog(MODEL_CANDIDATES)
+ assert dialog.list.count() == len(MODEL_CANDIDATES)
+ assert not dialog.btn_download.isEnabled()
+ assert dialog.details.toPlainText()
+ dialog.close()
+ app.processEvents()
diff --git a/tests/test_model_manager.py b/tests/test_model_manager.py
new file mode 100644
index 0000000..f99ff10
--- /dev/null
+++ b/tests/test_model_manager.py
@@ -0,0 +1,75 @@
+from pathlib import Path
+
+import pytest
+
+from model_manager import (
+ HardwareProfile,
+ ModelFile,
+ ModelManifest,
+ ModelManifestError,
+ ModelStore,
+ recommend_level,
+ verify_model_files,
+)
+
+
+def _manifest(data: bytes = b"model") -> ModelManifest:
+ import hashlib
+
+ return ModelManifest(
+ model_id="test-model",
+ repo_id="example/model",
+ revision="0123456789abcdef0123456789abcdef01234567",
+ license="MIT",
+ runtime="onnx",
+ labels=("PERSON",),
+ files=(ModelFile("model.onnx", len(data), hashlib.sha256(data).hexdigest()),),
+ enabled=True,
+ )
+
+
+def test_manifest_rejects_short_revision_and_unsafe_file():
+ m = _manifest()
+ with pytest.raises(ModelManifestError):
+ ModelManifest(**{**m.__dict__, "revision": "abc"}).validate()
+ with pytest.raises(ModelManifestError):
+ ModelFile("../model.onnx", 1, "0" * 64).validate()
+
+
+def test_verify_model_files_checks_size_and_hash(tmp_path: Path):
+ m = _manifest()
+ model_dir = tmp_path / m.model_id / m.revision
+ model_dir.mkdir(parents=True)
+ (model_dir / "model.onnx").write_bytes(b"model")
+ verify_model_files(model_dir, m)
+ (model_dir / "model.onnx").write_bytes(b"tampered")
+ with pytest.raises(ModelManifestError, match="dimensione inattesa|hash"):
+ verify_model_files(model_dir, m)
+
+
+def test_store_is_installed_requires_complete_verified_files(tmp_path: Path):
+ m = _manifest()
+ store = ModelStore(tmp_path)
+ assert not store.is_installed(m)
+ target = store.model_dir(m)
+ target.mkdir(parents=True)
+ (target / "model.onnx").write_bytes(b"model")
+ assert store.is_installed(m)
+
+
+def test_recommendation_does_not_enable_unverified_catalog():
+ hw = HardwareProfile(cpu_count=12, memory_bytes=16 * 1024**3)
+ assert recommend_level(hw, ()).level == "nessuno"
+ low = HardwareProfile(cpu_count=12, memory_bytes=4 * 1024**3)
+ assert recommend_level(low, (_manifest(),)).model_id is None
+
+
+def test_manifest_rejects_non_model_extensions():
+ with pytest.raises(ModelManifestError):
+ ModelFile("training_args.bin", 1, "0" * 64).validate()
+
+
+def test_manifest_source_url_is_pinned():
+ manifest = _manifest()
+ assert manifest.revision in manifest.source_url
+ assert manifest.source_url.startswith("https://huggingface.co/")
diff --git a/tests/test_word_import.py b/tests/test_word_import.py
new file mode 100644
index 0000000..615771e
--- /dev/null
+++ b/tests/test_word_import.py
@@ -0,0 +1,41 @@
+import zipfile
+
+import pymupdf
+
+from word_import import docx_to_pdf, extract_docx_text
+
+
+def _write_docx(path):
+ xml = """
+
+
+ Mario Rossi
+ Codice fiscale
+ RSSMRA80A01H501U
+
+ """
+ with zipfile.ZipFile(path, "w") as archive:
+ archive.writestr("word/document.xml", xml)
+
+
+def test_docx_text_extraction_and_pdf_conversion(tmp_path):
+ source = tmp_path / "documento.docx"
+ _write_docx(source)
+ assert extract_docx_text(source) == ["Mario Rossi", "Codice fiscale | RSSMRA80A01H501U"]
+ document = docx_to_pdf(str(source))
+ try:
+ assert "Mario Rossi" in document[0].get_text()
+ assert "RSSMRA80A01H501U" in document[0].get_text()
+ finally:
+ document.close()
+
+
+def test_doc_import_is_rejected_explicitly(tmp_path):
+ source = tmp_path / "documento.doc"
+ source.write_bytes(b"legacy")
+ try:
+ docx_to_pdf(str(source))
+ except ValueError as exc:
+ assert "DOCX" in str(exc)
+ else:
+ raise AssertionError("il formato DOC binario deve essere rifiutato")
diff --git a/word_import.py b/word_import.py
new file mode 100644
index 0000000..6bcb7b1
--- /dev/null
+++ b/word_import.py
@@ -0,0 +1,82 @@
+"""Minimal, local DOCX importer used to create an analyzable PDF document.
+
+The importer intentionally handles the OOXML text layer without executing
+macros, embedded objects, external links or document relationships. The
+result is a new PDF representation; the original DOCX is never modified.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import zipfile
+import xml.etree.ElementTree as ET
+from typing import List
+
+import fitz # type: ignore[import-untyped]
+
+
+_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+
+
+def _paragraph_text(node: ET.Element) -> str:
+ return "".join((t.text or "") for t in node.iter(_W + "t"))
+
+
+def extract_docx_text(path: pathlib.Path) -> List[str]:
+ """Extract paragraphs and table rows from a DOCX without running code."""
+ with zipfile.ZipFile(path) as archive:
+ root = ET.fromstring(archive.read("word/document.xml"))
+ lines: List[str] = []
+ body = root.find(_W + "body")
+ if body is None:
+ return lines
+ for child in body:
+ if child.tag == _W + "p":
+ text = _paragraph_text(child).strip()
+ if text:
+ lines.append(text)
+ elif child.tag == _W + "tbl":
+ for row in child.findall(_W + "tr"):
+ cells = [" ".join(_paragraph_text(p).split()) for p in row.findall(".//" + _W + "tc")]
+ text = " | ".join(c for c in cells if c).strip()
+ if text:
+ lines.append(text)
+ return lines
+
+
+def docx_to_pdf(path: str) -> fitz.Document:
+ """Return a local PDF representation of a DOCX for the existing pipeline."""
+ source = pathlib.Path(path)
+ if source.suffix.lower() != ".docx":
+ raise ValueError("Sono supportati documenti Word DOCX; il formato DOC binario non è supportato.")
+ lines = extract_docx_text(source)
+ document = fitz.open()
+ page = document.new_page()
+ margin = 54.0
+ width = page.rect.width - 2 * margin
+ y = margin
+ fontsize = 10.5
+ line_height = 15.0
+ for raw in lines or [""]:
+ words = raw.split()
+ current = ""
+ chunks: List[str] = []
+ for word in words:
+ candidate = f"{current} {word}".strip()
+ if current and fitz.get_text_length(candidate, fontname="helv", fontsize=fontsize) > width:
+ chunks.append(current)
+ current = word
+ else:
+ current = candidate
+ chunks.append(current)
+ for line in chunks:
+ if y + line_height > page.rect.height - margin:
+ page = document.new_page()
+ y = margin
+ page.insert_text((margin, y), line, fontname="helv", fontsize=fontsize)
+ y += line_height
+ y += 4.0
+ return document
+
+
+__all__ = ["docx_to_pdf", "extract_docx_text"]
From 3f322eea97e16a965e4b45c14cd6141eb8f58ec7 Mon Sep 17 00:00:00 2001
From: 0verwrite <31691645+overwrite00@users.noreply.github.com>
Date: Wed, 11 Nov 2026 00:19:05 +0100
Subject: [PATCH 10/13] fix: make hardware detection type-check on Unix
---
model_manager.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/model_manager.py b/model_manager.py
index 5d2d1bf..f796a16 100644
--- a/model_manager.py
+++ b/model_manager.py
@@ -267,7 +267,8 @@ class Status(ctypes.Structure):
status = Status()
status.length = ctypes.sizeof(Status)
- if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
+ windll = getattr(ctypes, "windll", None)
+ if windll is not None and windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)):
return int(status.total)
except (AttributeError, OSError, TypeError):
pass
From 806f9a394a677b2f9e1f19652cced7ebd81efe82 Mon Sep 17 00:00:00 2001
From: 0verwrite <31691645+overwrite00@users.noreply.github.com>
Date: Wed, 11 Nov 2026 09:47:38 +0100
Subject: [PATCH 11/13] feat: AGPL-3.0 licence, Italian only, Italian model
downloaded on demand (#134)
Prepares 2.4.0-beta.5.
---
.github/workflows/beta-release.yml | 2 +-
.github/workflows/build-linux.yml | 2 +-
.github/workflows/release.yml | 2 +-
.github/workflows/test_build.yml | 2 +-
ARCHITECTURE.md | 7 +-
CHANGELOG.md | 8 +
CONTRIBUTING.md | 6 +-
DEVELOPMENT.md | 20 +-
LICENSE | 281 ++++++++++++++++++++++++++---
NullifyPDF.py | 182 ++++++++++++++-----
README.md | 40 +++-
THIRD_PARTY_NOTICES.md | 29 +++
TROUBLESHOOTING.md | 16 +-
USER_GUIDE.md | 12 +-
build_local.py | 10 +-
help_guide.py | 9 +-
licences_info.py | 96 ++++++++++
model_dialog.py | 160 +++++++++++-----
requirements.txt | 4 +-
setup_env.py | 3 +-
spacy_models.py | 193 ++++++++++++++++++++
tests/test_gui_flow.py | 65 ++++++-
tests/test_licences.py | 64 +++++++
tests/test_model_dialog.py | 114 ++++++++++++
tests/test_spacy_models.py | 94 ++++++++++
tests/test_validation.py | 10 +-
26 files changed, 1259 insertions(+), 172 deletions(-)
create mode 100644 licences_info.py
create mode 100644 spacy_models.py
create mode 100644 tests/test_licences.py
create mode 100644 tests/test_spacy_models.py
diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml
index f71edb5..8ca5ce9 100644
--- a/.github/workflows/beta-release.yml
+++ b/.github/workflows/beta-release.yml
@@ -50,7 +50,7 @@ jobs:
- name: Type Check (mypy)
shell: bash
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py
- name: Resolve beta suffix from tag
if: startsWith(github.ref, 'refs/tags/')
diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml
index 09c82b4..73518c3 100644
--- a/.github/workflows/build-linux.yml
+++ b/.github/workflows/build-linux.yml
@@ -82,7 +82,7 @@ jobs:
run: pytest tests/ -v
- name: Type check (mypy)
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py
- name: Resolve beta suffix from tag
if: inputs.is_beta && startsWith(github.ref, 'refs/tags/')
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 9f5ec6c..a9a6ed9 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -95,7 +95,7 @@ jobs:
- name: Type Check (mypy)
shell: bash
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py
- name: Build
run: python build_local.py
diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml
index b3aa452..11c372e 100644
--- a/.github/workflows/test_build.yml
+++ b/.github/workflows/test_build.yml
@@ -55,7 +55,7 @@ jobs:
- name: Type Check (mypy)
shell: bash
- run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py
+ run: mypy NullifyPDF.py pii_detection.py privacy_core.py update_check.py PDF_Checker.py export_engine.py pdf_sanitize.py review_dialog.py lexicon_it.py secure_store.py word_lists.py dictionary_dialog.py help_guide.py spacy_models.py licences_info.py
- name: Build
run: python build_local.py
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 27ab7ca..f9e324d 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -104,7 +104,7 @@ Export pipeline writes a cleaned PDF copy
### 🤖 AI Scan
1. User clicks **Auto Redact** and, in a dialog, picks which data types to look for (names, locations, email, phone, IBAN, cards, crypto, fiscal code, VAT, licence, ID card, passport). The selection is remembered in `~/.nullifypdf/ai_entities.json`.
-2. User selects language (`EN`, `IT`, or `BOTH`) and optionally enables:
+2. The language is always Italian (`IT`); the user optionally enables:
- `OCR PDF scansionati`
- `Oscura Immagini`
3. `cmd_auto_ai()` checks OCR configuration if OCR is enabled.
@@ -224,10 +224,7 @@ Runtime behavior:
- bundled `ocr/tessdata` via `resource_path()`
- PyMuPDF's `fitz.get_tessdata()`
- common platform locations
-2. `ocr_language_for_choice()` maps UI language to Tesseract codes:
- - `EN` -> `eng`
- - `IT` -> `ita`
- - `BOTH` -> `eng+ita`
+2. `ocr_language_for_choice()` returns the Tesseract codes: `ita`, plus `eng` when its data is installed (so English words inside Italian documents are read well).
3. `AIWorker._page_needs_ocr()` runs OCR only when text extraction is missing or very poor.
4. OCR text is used for Presidio detection.
5. OCR search rectangles are used to place redactions on scanned pages.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 813a0ac..2f9290c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### 🔄 Changed
+
+- **Italian only**: the English model and the EN/IT/BOTH selector are gone. The program analyses Italian documents; the sidebar shows "Lingua dei documenti: italiano". The English OCR data is still used, when installed, only to read English words inside Italian pages.
+- **Model download progress in the button**: in the local-models window the "Scarica modello" button now fills like a progress bar with the percentage, a second click cancels, the list and the Close button are locked while downloading, and the download no longer blocks the window. The empty grey button of the old progress window is gone.
+- **New licence: AGPL-3.0-or-later** (previously MIT), because the program uses PyMuPDF, which is AGPL-3.0 or commercial. The program stays free and is not sold. Full third-party licence list in `THIRD_PARTY_NOTICES.md`.
+- **Licence notice in the Info window and README**: the Info window now lists the licence of NullifyPDF and of every component and model it uses, with clickable links to the official texts.
+- **Italian name-recognition model downloaded on demand**: its licence (CC BY-NC-SA 3.0, non-commercial) cannot be bundled in an AGPL installer, so the first Italian scan shows the licence and asks for consent, then downloads and verifies the model (SHA-256). Detection quality is unchanged once installed; if you decline, the scan continues with rules and dictionaries only and the log says so.
+
### ✨ Added
- **Local PII models**: optional, user-downloaded ONNX models from pinned Hugging Face revisions are stored under `~/.nullifypdf/models/`, verified file-by-file with SHA-256, and contribute suggestions only. The model dialog shows the source, license, revision and download progress; unverified experimental entries remain blocked.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 018130e..5f101e2 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -379,5 +379,9 @@ Every contribution — code, docs, bug reports, ideas — helps make NullifyPDF
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-11*
*← [Back to README](./README.md) | [Architecture →](./ARCHITECTURE.md)*
+
+## Licence of contributions
+
+NullifyPDF is licensed under the AGPL-3.0-or-later (see [LICENSE](./LICENSE)). By submitting a contribution you agree that it is released under the same licence.
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 67e2847..36287ac 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -16,7 +16,7 @@ Before starting, verify you have:
| **Python 3.13** | 150 MB | [Download](https://www.python.org/downloads/) |
| **Git** | 50 MB | [Download](https://git-scm.com/) |
| **Virtual Environment** | 2 GB | `.venv/` auto-created by setup script |
-| **Disk Space** | 3 GB | Dependencies + spaCy models |
+| **Disk Space** | 3 GB | Dependencies + Italian model |
| **RAM** | 4 GB | 8 GB recommended |
---
@@ -51,7 +51,7 @@ python setup_env.py
This automatically:
- ✅ Creates `.venv/` virtual environment
- ✅ Installs dependencies
-- ✅ Downloads spaCy models (EN + IT)
+- ✅ Installs the Italian spaCy model (development only)
### 4️⃣ Activate Virtual Environment
@@ -151,14 +151,18 @@ NullifyPDF/
| **python-docx** | — | Not required; DOCX import uses the standard-library OOXML reader |
| **pytest** | 9.2.0 | Testing framework |
-### Language Models (Auto-Downloaded)
+### Language Models
-| Model | Size | Purpose |
-| ----------------- | ----- | ----------- |
-| `en_core_web_md` | 40 MB | English NER |
-| `it_core_news_md` | 45 MB | Italian NER |
+| Model | Size | Purpose | Licence | In the installer |
+| ----------------- | ----- | ----------- | ------- | ---------------- |
+| `it_core_news_md` | 45 MB | Italian NER | CC BY-NC-SA 3.0 | **no**, downloaded by the app on demand (`spacy_models.py`) |
-These are downloaded automatically by `setup_env.py`.
+`setup_env.py` installs both in the development environment (the Italian tests
+need it). The PyInstaller specs exclude `it_core_news_md`, `pymupdf4llm` and
+`pymupdf.layout`. At run time the app asks for consent, downloads the Italian
+pipeline from the publisher's release, checks its size and SHA-256 and stores it
+under `~/.nullifypdf/spacy_models/`. If the user declines, a blank Italian
+pipeline is used and only rule-based detection is available.
---
diff --git a/LICENSE b/LICENSE
index 749775e..2bf02f1 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,21 +1,260 @@
-MIT License
-
-Copyright (c) 2026 Graziano Mariella
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+NullifyPDF
+Copyright (C) 2026 Graziano Mariella
+
+This program is free software: you can redistribute it and/or modify it under
+the terms of the GNU Affero General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option) any
+later version (SPDX: AGPL-3.0-or-later).
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+NullifyPDF is free of charge and is not sold by its author.
+
+Why AGPL: the program uses PyMuPDF/MuPDF (Artifex), which is available under
+the GNU AGPL-3.0 or a commercial licence. Releases up to and including
+v2.4.0-beta.4 were published under the MIT licence; copies already received
+under those terms remain under them. Starting with the next release the
+program is distributed under the licence below. Third-party components keep
+their own licences, see THIRD_PARTY_NOTICES.md.
+
+Source code: https://github.com/overwrite00/NullifyPDF
+
+================================================================================
+
+GNU AFFERO GENERAL PUBLIC LICENSE
+Version 3, 19 November 2007
+
+Copyright (C) 2007 Free Software Foundation, Inc.
+
+Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
+
+ Preamble
+
+The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software.
+
+The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users.
+
+When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
+
+Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software.
+
+A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public.
+
+The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version.
+
+An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license.
+
+The precise terms and conditions for copying, distribution and modification follow.
+
+ TERMS AND CONDITIONS
+
+0. Definitions.
+
+"This License" refers to version 3 of the GNU Affero General Public License.
+
+"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
+
+"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations.
+
+To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work.
+
+A "covered work" means either the unmodified Program or a work based on the Program.
+
+To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
+
+To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
+
+An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
+
+1. Source Code.
+The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work.
+
+A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
+
+The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
+
+The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
+
+The Corresponding Source for a work in source code form is that same work.
+
+2. Basic Permissions.
+All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
+
+You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
+
+Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
+
+3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
+
+When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
+
+4. Conveying Verbatim Copies.
+You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
+
+You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
+
+5. Conveying Modified Source Versions.
+You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
+
+A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
+
+6. Conveying Non-Source Forms.
+You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
+
+ d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
+
+A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
+
+A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
+
+"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
+
+If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
+
+The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
+
+Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
+
+7. Additional Terms.
+"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
+
+When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
+
+Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
+
+All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
+
+If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
+
+Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
+
+8. Termination.
+
+You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
+
+However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
+
+Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
+
+Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
+
+9. Acceptance Not Required for Having Copies.
+
+You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
+
+10. Automatic Licensing of Downstream Recipients.
+
+Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
+
+An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
+
+You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
+
+11. Patents.
+
+A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version".
+
+A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
+
+Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
+
+In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
+
+If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
+
+If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
+
+A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
+
+Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
+
+12. No Surrender of Others' Freedom.
+
+If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
+
+13. Remote Network Interaction; Use with the GNU General Public License.
+
+Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph.
+
+Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License.
+
+14. Revised Versions of this License.
+
+The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation.
+
+If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
+
+Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
+
+15. Disclaimer of Warranty.
+
+THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+16. Limitation of Liability.
+
+IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+
+17. Interpretation of Sections 15 and 16.
+
+If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
+
+END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
+
+To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements.
+
+You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see .
diff --git a/NullifyPDF.py b/NullifyPDF.py
index f034698..945a73e 100644
--- a/NullifyPDF.py
+++ b/NullifyPDF.py
@@ -25,9 +25,9 @@
QHBoxLayout,
QLabel,
QPushButton,
- QRadioButton,
QCheckBox,
QProgressBar,
+ QProgressDialog,
QTextEdit,
QGraphicsView,
QGraphicsScene,
@@ -35,7 +35,6 @@
QDialog,
QDialogButtonBox,
QLineEdit,
- QButtonGroup,
QGraphicsRectItem,
QMessageBox,
QInputDialog,
@@ -54,7 +53,6 @@
QDropEvent,
)
from PySide6.QtCore import (
- QLocale,
Qt,
QThread,
QObject,
@@ -100,6 +98,8 @@
save_enabled_groups,
whole_word_rects,
)
+import licences_info
+import spacy_models
import update_check
from export_engine import (
build_audit_record,
@@ -138,7 +138,7 @@
PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined]
__version__ = "2.4.0"
-__version_prerelease__ = "beta.4"
+__version_prerelease__ = "beta.5"
APP_VERSION = (
f"{__version__}-{__version_prerelease__}"
if __version_prerelease__
@@ -167,6 +167,27 @@ def run(self) -> None:
self.error_sig.emit(f"{type(e).__name__}: {e}")
+class SpacyModelInstallWorker(QThread):
+ """Download the optional Italian spaCy pipeline off the UI thread."""
+
+ progress_sig = Signal(int, int)
+ done_sig = Signal()
+ error_sig = Signal(str)
+
+ def __init__(self, root: pathlib.Path) -> None:
+ super().__init__()
+ self._root = root
+
+ def run(self) -> None:
+ try:
+ spacy_models.install_italian(
+ self._root, progress=lambda d, t: self.progress_sig.emit(d, t)
+ )
+ self.done_sig.emit()
+ except Exception as e:
+ self.error_sig.emit(str(e))
+
+
class UpdateDownloadWorker(QThread):
"""Download a release asset into the user's Downloads folder."""
@@ -842,15 +863,19 @@ def find_tessdata_dir() -> Optional[str]:
def ocr_language_for_choice(choice: str, tessdata_dir: Optional[str]) -> str:
- """Map the UI language choice to installed Tesseract language codes."""
- requested = ["eng", "ita"] if choice == "BOTH" else ["ita" if choice == "IT" else "eng"]
+ """Tesseract languages for the scan: Italian, plus English data if installed.
+
+ The program analyses Italian documents only; the English OCR data is kept
+ just so that English words and Latin-script terms inside them are read
+ well. `choice` is kept for API stability and must be "IT".
+ """
if not tessdata_dir:
- return "+".join(requested)
+ return "ita"
available = [
- lang for lang in requested
+ lang for lang in ("ita", "eng")
if (pathlib.Path(tessdata_dir) / f"{lang}.traineddata").exists()
]
- return "+".join(available or requested)
+ return "+".join(available or ["ita"])
STYLESHEET = """
@@ -909,6 +934,10 @@ def __init__(self) -> None:
super().__init__()
self.analyzer: Optional[Any] = None
self.loaded_langs: List[str] = []
+ # Set by the window before a scan: package name or folder of the
+ # Italian pipeline (it is downloaded on demand, see spacy_models).
+ self.italian_model: str = spacy_models.ITALIAN_PACKAGE
+ self._loaded_italian = ""
self.model_detectors: List[Tuple[str, OnnxTokenDetector]] = []
self._stop_requested = False
@@ -947,7 +976,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
Args:
doc: PyMuPDF document (shared with UI thread, mutex-protected).
doc_mutex: QMutex serializing access to `doc`.
- choice: Language choice: 'EN', 'IT', or 'BOTH'.
+ choice: Language of the document; only 'IT' is supported.
compiled_allowlist: Pre-compiled regex patterns to skip redaction.
allowlist_set: Set of allowlist entries (lowercase) for O(1)
exact-match fast-path lookup before regex any() scan.
@@ -965,7 +994,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
self.finished_sig.emit()
return
- if choice not in ("EN", "IT", "BOTH"):
+ if choice != "IT":
self.log_sig.emit(f"ERRORE: Scelta lingua non valida: {choice}")
self.finished_sig.emit()
return
@@ -978,21 +1007,19 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
if not isinstance(allowlist_set, set):
allowlist_set = set()
- target_langs = ["en", "it"] if choice == "BOTH" else [choice.lower()]
+ target_langs = ["it"]
ocr_language = ocr_language_for_choice(choice, tessdata_dir)
- if not self.analyzer or sorted(self.loaded_langs) != sorted(target_langs):
+ if (
+ not self.analyzer
+ or sorted(self.loaded_langs) != sorted(target_langs)
+ or self._loaded_italian != self.italian_model
+ ):
self.log_sig.emit(f"Inizializzazione motore di rilevamento ({choice})...")
from presidio_analyzer import AnalyzerEngine, RecognizerRegistry
from presidio_analyzer.nlp_engine import NlpEngineProvider
from presidio_analyzer.predefined_recognizers import PhoneRecognizer
- models = (
- [{"lang_code": "en", "model_name": "en_core_web_md"}]
- if "en" in target_langs
- else []
- )
- if "it" in target_langs:
- models.append({"lang_code": "it", "model_name": "it_core_news_md"})
+ models = [{"lang_code": "it", "model_name": self.italian_model}]
provider = NlpEngineProvider(
nlp_configuration={"nlp_engine_name": "spacy", "models": models}
)
@@ -1025,6 +1052,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
supported_languages=target_langs,
)
self.loaded_langs = target_langs
+ self._loaded_italian = self.italian_model
self.log_sig.emit("Scansione privacy in corso...")
self._load_local_detectors()
targets = list(enabled_types)
@@ -1507,22 +1535,9 @@ def build_ui(self) -> None:
btn_open.clicked.connect(self.cmd_open)
s_lay.addWidget(btn_open)
s_lay.addSpacing(20)
- s_lay.addWidget(QLabel("Modello lingua:"))
- lang_lay = QHBoxLayout()
- self.rb_en = QRadioButton("EN")
- self.rb_it = QRadioButton("IT")
- self.rb_both = QRadioButton("BOTH")
- # Default to the interface language: running the English model on
- # Italian text only adds false positives.
- if QLocale.system().language() == QLocale.Language.Italian:
- self.rb_it.setChecked(True)
- else:
- self.rb_both.setChecked(True)
- self.lang_grp = QButtonGroup(self)
- for rb in (self.rb_en, self.rb_it, self.rb_both):
- self.lang_grp.addButton(rb)
- lang_lay.addWidget(rb)
- s_lay.addLayout(lang_lay)
+ self.lbl_lang = QLabel("Lingua dei documenti: italiano")
+ self.lbl_lang.setObjectName("language_label")
+ s_lay.addWidget(self.lbl_lang)
s_lay.addSpacing(10)
self.chk_img = QCheckBox("Oscura Immagini")
s_lay.addWidget(self.chk_img)
@@ -2080,7 +2095,7 @@ def cmd_about(self) -> None:
"""Show about dialog."""
d = QDialog(self)
d.setWindowTitle("Info")
- d.setFixedSize(340, 560)
+ d.setFixedSize(520, 820)
lay = QVBoxLayout(d)
lay.setAlignment(Qt.AlignmentFlag.AlignCenter)
ip = resource_path(os.path.join("images", "NullifyPDF_icon.png"))
@@ -2103,7 +2118,7 @@ def cmd_about(self) -> None:
lbl_ver.setAlignment(Qt.AlignmentFlag.AlignCenter)
lay.addWidget(lbl_ver)
desc = QLabel(
- "\nCensura e pseudonimizzazione PDF offline.\n\nSviluppato da: Graziano Mariella\nLicenza MIT"
+ "\nCensura e pseudonimizzazione PDF offline.\n\nSviluppato da: Graziano Mariella\nLicenza AGPL-3.0"
)
desc.setAlignment(Qt.AlignmentFlag.AlignCenter)
lay.addWidget(desc)
@@ -2114,7 +2129,17 @@ def cmd_about(self) -> None:
lbl_repo.setOpenExternalLinks(True)
lbl_repo.setAlignment(Qt.AlignmentFlag.AlignCenter)
lay.addWidget(lbl_repo)
- lay.addSpacing(10)
+ lay.addSpacing(6)
+ lic_view = QTextBrowser()
+ lic_view.setObjectName("about_licences")
+ lic_view.setOpenExternalLinks(True)
+ lic_view.setHtml(licences_info.about_html())
+ lic_view.setMinimumHeight(200)
+ lic_view.setToolTip(
+ "Licenza del programma e dei componenti usati, con i collegamenti ai testi ufficiali."
+ )
+ lay.addWidget(lic_view, 1)
+ lay.addSpacing(6)
btn_upd = QPushButton("Controlla aggiornamenti")
lay.addWidget(btn_upd)
chk_auto = QCheckBox("Controlla ad ogni avvio")
@@ -2293,11 +2318,10 @@ def cmd_auto_ai(self) -> None:
c_allow = [
(a, re.compile(r"\b" + re.escape(a) + r"\b")) for a in self.allowlist
]
- lang = (
- "EN"
- if self.rb_en.isChecked()
- else "IT" if self.rb_it.isChecked() else "BOTH"
- )
+ lang = "IT"
+ if not self._prepare_italian_model():
+ self.btn_ai.setEnabled(True)
+ return
use_ocr = self.chk_ocr.isChecked()
tessdata_dir = find_tessdata_dir() if use_ocr else None
if use_ocr and not tessdata_dir:
@@ -2324,6 +2348,76 @@ def cmd_auto_ai(self) -> None:
health_mode,
)
+ def _prepare_italian_model(self) -> bool:
+ """Make an Italian pipeline available; False if the user cancels.
+
+ The statistical pipeline is not bundled (non-commercial licence): ask
+ once, show the licence, download and verify it. If the user declines,
+ the scan continues with rules only and the log says so.
+ """
+ root = spacy_models.models_root(self.config_dir)
+ found = spacy_models.resolve_italian(root)
+ if found is not None:
+ self.ai_worker.italian_model = found.name
+ return True
+ box = QMessageBox(self)
+ box.setWindowTitle("Modello italiano")
+ box.setIcon(QMessageBox.Icon.Question)
+ box.setText("Per riconoscere i nomi in italiano serve un modello aggiuntivo.")
+ box.setInformativeText(spacy_models.licence_text())
+ b_dl = box.addButton("Accetta e scarica", QMessageBox.ButtonRole.AcceptRole)
+ b_skip = box.addButton("Continua senza modello", QMessageBox.ButtonRole.DestructiveRole)
+ box.addButton("Annulla", QMessageBox.ButtonRole.RejectRole)
+ box.exec()
+ clicked = box.clickedButton()
+ if clicked is b_skip:
+ self.ai_worker.italian_model = spacy_models.blank_italian(root).name
+ self.write_log(
+ "Avviso: modello italiano non installato, riconoscimento nomi "
+ "solo con regole e dizionari (meno completo)."
+ )
+ return True
+ if clicked is not b_dl:
+ return False
+ dlg = QProgressDialog("Download del modello italiano...", "Annulla", 0, 100, self)
+ dlg.setWindowTitle("Modello italiano")
+ dlg.setMinimumDuration(0)
+ dlg.setAutoClose(False)
+ dlg.setAutoReset(False)
+ dlg.setCancelButton(None)
+ worker = SpacyModelInstallWorker(root)
+ result: Dict[str, str] = {}
+
+ def on_progress(done: int, total: int) -> None:
+ dlg.setValue(int(done * 100 / max(1, total)))
+
+ def on_done() -> None:
+ result["ok"] = "1"
+ dlg.accept()
+
+ def on_error(msg: str) -> None:
+ result["error"] = msg
+ dlg.accept()
+
+ worker.progress_sig.connect(on_progress)
+ worker.done_sig.connect(on_done)
+ worker.error_sig.connect(on_error)
+ worker.start()
+ dlg.exec()
+ worker.wait()
+ if "ok" in result:
+ found = spacy_models.resolve_italian(root)
+ if found is not None:
+ self.ai_worker.italian_model = found.name
+ self.write_log("Modello italiano scaricato e verificato.")
+ return True
+ QMessageBox.warning(
+ self,
+ "Modello italiano",
+ "Download non riuscito: " + result.get("error", "errore sconosciuto"),
+ )
+ return False
+
def _select_ai_entities(self) -> Optional[List[str]]:
"""Ask which data types the AI scan should look for.
diff --git a/README.md b/README.md
index be56a0e..57f2912 100644
--- a/README.md
+++ b/README.md
@@ -125,7 +125,7 @@ No installation needed on Windows/macOS - just run or unzip. Linux users can ins
This script automatically:
- Creates isolated virtual environment (`.venv`)
- Installs all dependencies
- - Downloads spaCy language models (EN/IT)
+ - Installs the Italian spaCy model for development (the installer downloads it on demand)
4. **Activate environment & launch**
@@ -158,7 +158,7 @@ python setup_env.py
- Detects OS (Windows/macOS/Linux)
- Creates `.venv` with Python 3.13
- Installs `requirements.txt` dependencies
-- Downloads spaCy models (English, Italian, both)
+- Installs the Italian spaCy model for development (the installer does not bundle it)
**Automatic OS detection:**
- Windows: Uses `py -3.13` launcher
@@ -253,11 +253,43 @@ See [SECURITY.md](./SECURITY.md) for responsible disclosure and privacy details.
## 📝 License
-**MIT License** — Free to use, modify, and distribute
+**GNU Affero General Public License v3.0 or later (AGPL-3.0-or-later)**
Copyright (c) 2026 Graziano Mariella
-See [LICENSE](./LICENSE) for full text.
+NullifyPDF is free of charge and is not sold by its author. It is AGPL-licensed
+because it uses [PyMuPDF](https://pymupdf.readthedocs.io/), which is available
+under the AGPL-3.0 or a commercial licence from Artifex. Anyone who distributes
+the program, modified or not, must give the recipients the source code under the
+same licence. Releases up to v2.4.0-beta.4 were published under the MIT licence.
+
+The Italian name-recognition model (spaCy `it_core_news_md`, CC BY-NC-SA 3.0,
+non-commercial) is **not** part of the installer: the program offers to download
+it, shows its licence and asks for consent first. Licences of all third-party
+components are listed in [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md).
+
+**Notice.** NullifyPDF is free software, distributed under the AGPL-3.0-or-later. Every component and model it uses remains under its own licence, listed below with a link to the official text; using the program means respecting those conditions. The Italian spaCy model is for non-commercial use only and is downloaded only after your consent.
+
+| Component | Licence | Used for |
+|---|---|---|
+| [NullifyPDF](https://github.com/overwrite00/NullifyPDF/blob/main/LICENSE) | AGPL-3.0-or-later | this program |
+| [PyMuPDF / MuPDF](https://pymupdf.readthedocs.io/en/latest/about.html) | AGPL-3.0 or commercial (Artifex) | PDF reading and redaction |
+| [PySide6 / Qt](https://opensource.org/license/lgpl-3-0) | LGPL-3.0 | graphical interface |
+| [Presidio](https://github.com/microsoft/presidio/blob/main/LICENSE) | MIT | personal-data detection |
+| [spaCy](https://github.com/explosion/spaCy/blob/master/LICENSE) | MIT | language processing |
+| [spaCy it_core_news_md](https://creativecommons.org/licenses/by-nc-sa/3.0/) | CC BY-NC-SA 3.0 (non-commercial use only) | Italian model, downloaded after consent |
+| [onnxruntime](https://github.com/microsoft/onnxruntime/blob/main/LICENSE) | MIT | running local models |
+| [tokenizers](https://github.com/huggingface/tokenizers/blob/main/LICENSE) | Apache-2.0 | tokenizing for local models |
+| [cryptography](https://github.com/pyca/cryptography/blob/main/LICENSE) | Apache-2.0 or BSD-3-Clause | encryption |
+| [keyring](https://github.com/jaraco/keyring) | MIT | key storage (Linux/macOS) |
+| [tldextract](https://github.com/john-kurkowski/tldextract/blob/master/LICENSE) | BSD-3-Clause | URL recognition |
+| [NumPy](https://github.com/numpy/numpy/blob/main/LICENSE.txt) | BSD-3-Clause | numerical computing |
+| [Tesseract OCR](https://github.com/tesseract-ocr/tesseract/blob/main/LICENSE) | Apache-2.0 | OCR of scanned documents |
+| [tessdata_fast (eng, ita)](https://github.com/tesseract-ocr/tessdata_fast/blob/main/LICENSE) | Apache-2.0 | OCR language data |
+| [ISTAT, elenco dei comuni](https://creativecommons.org/licenses/by/4.0/) | CC BY 4.0 | place names |
+| [Wikidata](https://creativecommons.org/publicdomain/zero/1.0/) | CC0 1.0 | given names and surnames |
+
+See [LICENSE](./LICENSE) for the full text.
---
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index 24950d6..3a92a62 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -26,3 +26,32 @@ Wikidata (items that have an article on the Italian Wikipedia).
- The lists are small (a few hundred given names and about one thousand
surnames) and are only used to *confirm* a name; a name missing from them
never causes a hit to be rejected.
+
+## Licence of NullifyPDF and of the bundled components
+
+NullifyPDF is distributed under the GNU Affero General Public License v3.0 or
+later (see `LICENSE`). Source code: https://github.com/overwrite00/NullifyPDF.
+The information below was checked on 2026-11-11 against the package metadata of
+the pinned versions in `requirements.txt` and the publishers' pages.
+
+| Component | Licence | Notes |
+|---|---|---|
+| PyMuPDF / MuPDF (Artifex) | AGPL-3.0 or Artifex commercial | NullifyPDF uses it under the AGPL-3.0, hence its own licence. |
+| PySide6 / Qt for Python | LGPL-3.0 (or GPL-3.0, commercial) | Used under the LGPL-3.0. Qt libraries are shipped unmodified; the full source and the build scripts of NullifyPDF are public, so anyone can rebuild the program against a different Qt. |
+| presidio-analyzer | MIT | |
+| spaCy, thinc | MIT | |
+| spaCy `it_core_news_md` 3.8.0 | CC BY-NC-SA 3.0 | **Not bundled.** Downloaded by the user on request after showing the licence (non-commercial, attribution, share-alike). Sources: UD Italian ISDT (CC BY-NC-SA 3.0), WikiNER (CC BY 4.0), fastText vectors (CC0). Publisher: Explosion, https://github.com/explosion/spacy-models |
+| onnxruntime | MIT | |
+| tokenizers | Apache-2.0 | |
+| cryptography | Apache-2.0 or BSD-3-Clause | |
+| keyring (Linux/macOS) | MIT | |
+| tldextract, numpy | BSD-3-Clause | |
+| Tesseract OCR and `tessdata_fast` (`eng`, `ita`) | Apache-2.0 | As stated by the Tesseract project. |
+| PyInstaller (build tool, not distributed) | GPL-2.0 with the bootloader exception | The exception allows distributing the produced program under any licence. |
+| `pymupdf4llm`, `pymupdf-layout` | not declared | Pulled in by PyMuPDF 2.1 but not used and **excluded** from the installer. |
+
+Local PII models downloaded by the user from Hugging Face are accepted only if
+their licence is permissive (MIT, Apache-2.0 or BSD); the licence is part of the
+pinned manifest and is shown before the download.
+
+This table is not legal advice.
diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md
index c5088c2..6102577 100644
--- a/TROUBLESHOOTING.md
+++ b/TROUBLESHOOTING.md
@@ -156,18 +156,9 @@ Names, emails, or other PII should be detected but aren't highlighted.
### Solution
-**Check 1 — Language Selection**
+**Check 1 — Language and model**
-Open sidebar. Is the correct language selected?
-
-| Language | Best For |
-| -------- | -------------------------------- |
-| **EN** | English text only |
-| **IT** | Italian text only |
-| **BOTH** | Mixed English + Italian (slower) |
-
-> [!WARNING]
-> Wrong language = **zero detection**. Test with a clear English name if unsure.
+NullifyPDF supports Italian documents only. Names are found by the Italian model, which is downloaded after your consent on the first Auto Redact. If you declined the download, only rules and dictionaries run and names may be missed: run Auto Redact again and accept the download.
**Check 2 — Is It Searchable Text?**
@@ -219,7 +210,6 @@ The scan proposes plain words ("Certifico", "Prescrivo", section headings) or fr
### Solution
-- Use the **IT** language model (or **BOTH**) for Italian documents; the English model adds false hits on Italian text.
- Uncertain hits appear in **Rivedi suggerimenti** instead of being applied: leave them unticked to discard them.
- Add words you never want redacted to the allowlist (**Dizionari**); matching is by whole word.
- If black boxes cover parts of the lines above or below the redacted word, update to a build that includes rectangle trimming and re-run the scan. With very tight line spacing (roughly half the font size or less) no safe band exists: a warning "interlinea molto stretta" is shown and neighbouring words can be damaged, never left unredacted.
@@ -486,5 +476,5 @@ The more details, the faster we can fix it! 🚀
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-11*
*← [User Guide](./USER_GUIDE.md) | [Development →](./DEVELOPMENT.md)*
diff --git a/USER_GUIDE.md b/USER_GUIDE.md
index e68321e..53a1ca0 100644
--- a/USER_GUIDE.md
+++ b/USER_GUIDE.md
@@ -40,18 +40,12 @@ Only entries with a complete application manifest can be downloaded. The manifes
---
-## Step 2️⃣ — Configure AI Language
+## Step 2️⃣ — Language
-**Before scanning**, select your document's language from the left sidebar:
-
-| 🌐 Language | Use When |
-| ---------- | --------------------------------- |
-| **EN** | Document is entirely in English |
-| **IT** | Document is entirely in Italian |
-| **BOTH** | Document contains mixed languages |
+NullifyPDF analyses **Italian** documents only; there is no language selector. The Italian name-recognition model is not part of the installer (its licence is non-commercial): the first time you run **Auto Redact** the program shows the licence, asks for your consent and downloads it. If you decline, the scan continues with rules and dictionaries only, which finds fewer names.
> [!WARNING]
-> Wrong language = missed detections. Choose the exact language of your document.
+> Documents in other languages are not supported: names and places in them will be missed.
---
diff --git a/build_local.py b/build_local.py
index 1ff6150..9098ec0 100644
--- a/build_local.py
+++ b/build_local.py
@@ -207,7 +207,7 @@ def build_rpm(
Version: {version}
Release: {rpm_release(prerelease)}
Summary: PDF Anonymization Tool
-License: MIT
+License: AGPL-3.0-or-later
URL: https://github.com/overwrite00/NullifyPDF
BuildArch: x86_64
{requires}
@@ -398,11 +398,11 @@ def build_app() -> None:
datas = {datas_literal}
binaries = []
hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
-a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports)
+a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports, excludes=['it_core_news_md', 'pymupdf4llm', 'pymupdf_layout', 'pymupdf.layout'])
pyz = PYZ(a.pure)
exe = EXE(pyz, a.scripts, [], exclude_binaries=True, name='NullifyPDF', debug=False, console=False, icon={icon_str})
coll = COLLECT(exe, a.binaries, a.datas, strip=False, upx=True, upx_exclude=[], name='NullifyPDF')
@@ -415,11 +415,11 @@ def build_app() -> None:
datas = {datas_literal}
binaries = []
hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
-a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports)
+a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports, excludes=['it_core_news_md', 'pymupdf4llm', 'pymupdf_layout', 'pymupdf.layout'])
pyz = PYZ(a.pure)
exe = EXE(pyz, a.scripts, a.binaries, a.datas, name='NullifyPDF', debug=False, console=False, icon={icon_str})
"""
diff --git a/help_guide.py b/help_guide.py
index 56cf597..b1a5342 100644
--- a/help_guide.py
+++ b/help_guide.py
@@ -40,10 +40,7 @@
TOOLTIPS: Dict[str, str] = {
# Sidebar
"Apri PDF": "Apre un file PDF da elaborare. Puoi anche trascinarlo sulla finestra.",
- "Modello lingua:": "Lingua dei modelli di riconoscimento dei dati personali.",
- "EN": "Usa il modello inglese. Adatto a documenti in inglese.",
- "IT": "Usa il modello italiano. Consigliato per documenti in italiano: il modello inglese aggiunge falsi positivi.",
- "BOTH": "Usa entrambi i modelli (inglese e italiano). Più lento; utile per documenti misti.",
+ "Lingua dei documenti: italiano": "NullifyPDF analizza documenti in italiano. Il modello italiano viene scaricato, dopo il tuo consenso, al primo utilizzo.",
"Oscura Immagini": "Oscura anche tutte le immagini della pagina (loghi, firme, foto) quando applichi Auto Redact.",
"OCR PDF scansionati": "Legge con l'OCR le pagine scansionate e le immagini incorporate, così anche il testo nelle immagini viene analizzato.",
"Mostra suggerimenti": "Contorna in arancione, nell'anteprima, i suggerimenti ancora da rivedere. Servono solo a video: non finiscono nel file esportato.",
@@ -256,7 +253,7 @@ def widgets_without_tooltip(root: QWidget) -> List[str]:
"""),
("Lingua e opzioni", """
Lingua e opzioni
-Modello lingua: IT per documenti italiani (consigliato: il modello inglese su testo italiano aggiunge falsi positivi), EN per l'inglese, BOTH per documenti misti. All'avvio viene scelto IT se il sistema è in italiano.
+Lingua: il programma analizza documenti in italiano. Il modello italiano non è incluso nell'installer (licenza non commerciale): al primo utilizzo il programma mostra la licenza e chiede il consenso per scaricarlo.
- Oscura Immagini: censura anche tutte le immagini (loghi, foto, firme) quando applichi Auto Redact.
- OCR PDF scansionati: legge con l'OCR le pagine scansionate e le immagini incorporate nelle pagine di testo. Richiede i dati di Tesseract (inclusi nei pacchetti).
@@ -366,7 +363,7 @@ def widgets_without_tooltip(root: QWidget) -> List[str]:
"""),
("Info e aggiornamenti", """
Info e aggiornamenti
-La finestra Info mostra la versione e il collegamento al repository. Controlla aggiornamenti interroga le release pubblicate su GitHub; con Controlla ad ogni avvio (disattivato di default) il controllo è eseguito all'avvio e mostra qualcosa solo se esiste una versione più recente.
+La finestra Info mostra la versione e il collegamento al repository. Elenca anche la licenza del programma (AGPL-3.0-or-later) e quella di ogni componente e modello usato, con i collegamenti ai testi ufficiali. Controlla aggiornamenti interroga le release pubblicate su GitHub; con Controlla ad ogni avvio (disattivato di default) il controllo è eseguito all'avvio e mostra qualcosa solo se esiste una versione più recente.
"""),
("Scorciatoie", """
Scorciatoie
diff --git a/licences_info.py b/licences_info.py
new file mode 100644
index 0000000..6686e2c
--- /dev/null
+++ b/licences_info.py
@@ -0,0 +1,96 @@
+"""Licences of NullifyPDF and of the components it uses, with official links.
+
+Single source for the "Info" window and for the README table (a test keeps the
+README in step). Informational only: each component remains under its own
+licence, which is what the notice says.
+"""
+
+from __future__ import annotations
+
+import html
+from typing import List, NamedTuple
+
+REPO = "https://github.com/overwrite00/NullifyPDF"
+AGPL_URL = "https://opensource.org/license/agpl-v3"
+LGPL_URL = "https://opensource.org/license/lgpl-3-0"
+CC_NC_SA_URL = "https://creativecommons.org/licenses/by-nc-sa/3.0/"
+
+APP_LICENCE = "AGPL-3.0-or-later"
+NOTICE = (
+ "NullifyPDF è software libero e gratuito, distribuito con licenza "
+ f"{APP_LICENCE}. Ogni componente e ogni modello usati dal programma restano "
+ "soggetti alla propria licenza, indicata qui sotto con il collegamento al testo "
+ "ufficiale: usando il programma se ne rispettano le condizioni. Il modello "
+ "italiano di spaCy è per solo uso non commerciale e viene scaricato solo dopo "
+ "il tuo consenso."
+)
+
+
+class Component(NamedTuple):
+ name: str
+ licence: str
+ url: str # official page or licence text
+ role: str
+ role_en: str = ""
+ licence_en: str = ""
+
+
+COMPONENTS: List[Component] = [
+ Component("NullifyPDF", APP_LICENCE, f"{REPO}/blob/main/LICENSE", "questo programma", "this program"),
+ Component("PyMuPDF / MuPDF", "AGPL-3.0 o commerciale (Artifex)",
+ "https://pymupdf.readthedocs.io/en/latest/about.html", "lettura e redazione dei PDF", "PDF reading and redaction",
+ "AGPL-3.0 or commercial (Artifex)"),
+ Component("PySide6 / Qt", "LGPL-3.0", LGPL_URL, "interfaccia grafica", "graphical interface"),
+ Component("Presidio", "MIT", "https://github.com/microsoft/presidio/blob/main/LICENSE",
+ "riconoscimento dei dati personali", "personal-data detection"),
+ Component("spaCy", "MIT", "https://github.com/explosion/spaCy/blob/master/LICENSE",
+ "elaborazione del linguaggio", "language processing"),
+ Component("spaCy it_core_news_md", "CC BY-NC-SA 3.0 (solo uso non commerciale)", CC_NC_SA_URL,
+ "modello italiano, scaricato dopo il consenso", "Italian model, downloaded after consent",
+ "CC BY-NC-SA 3.0 (non-commercial use only)"),
+ Component("onnxruntime", "MIT", "https://github.com/microsoft/onnxruntime/blob/main/LICENSE",
+ "esecuzione dei modelli locali", "running local models"),
+ Component("tokenizers", "Apache-2.0", "https://github.com/huggingface/tokenizers/blob/main/LICENSE",
+ "tokenizzazione dei modelli locali", "tokenizing for local models"),
+ Component("cryptography", "Apache-2.0 o BSD-3-Clause",
+ "https://github.com/pyca/cryptography/blob/main/LICENSE", "cifratura", "encryption",
+ "Apache-2.0 or BSD-3-Clause"),
+ Component("keyring", "MIT", "https://github.com/jaraco/keyring", "archivio chiavi (Linux/macOS)", "key storage (Linux/macOS)"),
+ Component("tldextract", "BSD-3-Clause",
+ "https://github.com/john-kurkowski/tldextract/blob/master/LICENSE", "riconoscimento URL", "URL recognition"),
+ Component("NumPy", "BSD-3-Clause", "https://github.com/numpy/numpy/blob/main/LICENSE.txt",
+ "calcolo numerico", "numerical computing"),
+ Component("Tesseract OCR", "Apache-2.0", "https://github.com/tesseract-ocr/tesseract/blob/main/LICENSE",
+ "OCR dei documenti scansionati", "OCR of scanned documents"),
+ Component("tessdata_fast (eng, ita)", "Apache-2.0",
+ "https://github.com/tesseract-ocr/tessdata_fast/blob/main/LICENSE", "dati OCR", "OCR language data"),
+ Component("ISTAT, elenco dei comuni", "CC BY 4.0", "https://creativecommons.org/licenses/by/4.0/",
+ "nomi dei luoghi", "place names"),
+ Component("Wikidata", "CC0 1.0", "https://creativecommons.org/publicdomain/zero/1.0/",
+ "nomi e cognomi", "given names and surnames"),
+]
+
+EXTRA_LINKS = [
+ ("LICENSE", f"{REPO}/blob/main/LICENSE"),
+ ("THIRD_PARTY_NOTICES", f"{REPO}/blob/main/THIRD_PARTY_NOTICES.md"),
+]
+
+
+def about_html() -> str:
+ """HTML for the scrollable licence area of the Info window."""
+ rows = "".join(
+ f"- {html.escape(c.name)}"
+ f" — {html.escape(c.licence)} ({html.escape(c.role)})
"
+ for c in COMPONENTS
+ )
+ extra = " · ".join(
+ f"{html.escape(n)}" for n, u in EXTRA_LINKS
+ )
+ return f"{html.escape(NOTICE)}
{extra}
"
+
+
+def readme_markdown() -> str:
+ lines = ["| Component | Licence | Used for |", "|---|---|---|"]
+ for c in COMPONENTS:
+ lines.append(f"| [{c.name}]({c.url}) | {c.licence_en or c.licence} | {c.role_en} |")
+ return "\n".join(lines)
diff --git a/model_dialog.py b/model_dialog.py
index 613eefd..72ab95a 100644
--- a/model_dialog.py
+++ b/model_dialog.py
@@ -4,9 +4,8 @@
from typing import Any, Iterable, Optional
-from PySide6.QtCore import Qt
+from PySide6.QtCore import QThread, Qt, Signal
from PySide6.QtWidgets import (
- QApplication,
QDialog,
QDialogButtonBox,
QLabel,
@@ -14,7 +13,6 @@
QListWidgetItem,
QMessageBox,
QPushButton,
- QProgressDialog,
QTextBrowser,
QVBoxLayout,
)
@@ -23,6 +21,52 @@
from model_manager import ModelCandidate, ModelManifest, ModelStore
+class ModelDownloadWorker(QThread):
+ """Download and verify every file of a manifest off the UI thread."""
+
+ progress_sig = Signal(int) # whole-model percentage
+ done_sig = Signal()
+ error_sig = Signal(str)
+ cancelled_sig = Signal()
+
+ def __init__(self, store: ModelStore, manifest: ModelManifest) -> None:
+ super().__init__()
+ self._store = store
+ self._manifest = manifest
+ self._cancel = False
+ self._completed = 0
+ self._total = max(1, manifest.total_size)
+
+ def cancel(self) -> None:
+ self._cancel = True
+
+ def _on_bytes(self, done: int, total: int) -> None:
+ if self._cancel:
+ raise InterruptedError("download annullato")
+ value = int(((self._completed + min(done, total)) / self._total) * 100)
+ self.progress_sig.emit(min(100, value))
+
+ def run(self) -> None:
+ try:
+ for item in self._manifest.files:
+ url = (
+ f"https://huggingface.co/{self._manifest.repo_id}/resolve/"
+ f"{self._manifest.revision}/{item.name}?download=true"
+ )
+ self._store.install_file(
+ self._manifest, item, url, timeout=120.0, progress=self._on_bytes
+ )
+ self._completed += item.size
+ self.progress_sig.emit(min(100, int(self._completed * 100 / self._total)))
+ if not self._store.is_installed(self._manifest):
+ raise RuntimeError("verifica finale del modello fallita")
+ self.done_sig.emit()
+ except InterruptedError:
+ self.cancelled_sig.emit()
+ except Exception as exc:
+ self.error_sig.emit(str(exc))
+
+
class ModelDialog(QDialog):
"""Show the reviewed model shortlist and its verification state."""
@@ -68,9 +112,10 @@ def __init__(
"Il download è attivo solo per modelli con revisione, file, licenza e SHA-256 verificati."
)
lay.addWidget(self.btn_download)
- buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close)
- buttons.rejected.connect(self.reject)
- lay.addWidget(buttons)
+ self._buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close)
+ self._buttons.rejected.connect(self.reject)
+ lay.addWidget(self._buttons)
+ self._worker: Optional[ModelDownloadWorker] = None
self.list.currentItemChanged.connect(self._show_details)
self.btn_download.clicked.connect(self._download_selected)
if self.list.count():
@@ -122,7 +167,46 @@ def _show_details(self, current: Optional[QListWidgetItem], _previous: Any) -> N
"Download HTTPS da Hugging Face alla revisione fissata nel manifest."
)
+ # -- download with progress shown inside the button -------------------
+ def _set_button_progress(self, percent: int) -> None:
+ """Fill the button from the left like a progress bar and show the %."""
+ p = min(max(percent, 0), 100) / 100.0
+ edge = min(p, 0.998)
+ self.btn_download.setText(f"Download in corso… {percent}% (clic per annullare)")
+ self.btn_download.setStyleSheet(
+ "QPushButton#model_download { background: qlineargradient("
+ "x1:0, y1:0, x2:1, y2:0, "
+ f"stop:0 #0284c7, stop:{edge:.3f} #0284c7, "
+ f"stop:{edge + 0.001:.3f} #334155, stop:1 #334155); }}"
+ )
+
+ def _set_downloading(self, on: bool) -> None:
+ self.list.setEnabled(not on)
+ close = self._buttons.button(QDialogButtonBox.StandardButton.Close)
+ if close is not None:
+ close.setEnabled(not on)
+ if not on:
+ self.btn_download.setStyleSheet("")
+
+ def is_downloading(self) -> bool:
+ return self._worker is not None
+
+ def reject(self) -> None:
+ if self._worker is None:
+ super().reject()
+
+ def closeEvent(self, event: Any) -> None:
+ if self._worker is not None:
+ event.ignore()
+ else:
+ super().closeEvent(event)
+
def _download_selected(self) -> None:
+ if self._worker is not None: # second click while downloading: cancel
+ self._worker.cancel()
+ self.btn_download.setText("Annullamento…")
+ self.btn_download.setEnabled(False)
+ return
current = self.list.currentItem()
if current is None or self._store is None:
return
@@ -142,41 +226,35 @@ def _download_selected(self) -> None:
)
if answer != QMessageBox.StandardButton.Yes:
return
- progress = QProgressDialog("Download modello...", "", 0, 100, self)
- progress.setWindowTitle("Download modello")
- progress.setAutoClose(False)
- progress.setAutoReset(False)
- progress.setMinimumDuration(0)
- progress.setValue(0)
- progress.show()
- total_size = max(1, manifest.total_size)
- completed = 0
-
- def on_progress(done: int, total: int) -> None:
- value = int(((completed + min(done, total)) / total_size) * 100)
- progress.setValue(min(100, value))
- QApplication.processEvents()
+ self._set_downloading(True)
+ self._set_button_progress(0)
+ worker = ModelDownloadWorker(self._store, manifest)
+ # Bound methods, not lambdas capturing self (no reference cycle).
+ worker.progress_sig.connect(self._set_button_progress)
+ worker.done_sig.connect(self._on_download_done)
+ worker.error_sig.connect(self._on_download_error)
+ worker.cancelled_sig.connect(self._on_download_cancelled)
+ self._worker = worker
+ worker.start()
- try:
- for item in manifest.files:
- url = (
- f"https://huggingface.co/{manifest.repo_id}/resolve/"
- f"{manifest.revision}/{item.name}?download=true"
- )
- self._store.install_file(
- manifest, item, url, timeout=120.0, progress=on_progress
- )
- completed += item.size
- progress.setValue(min(100, int(completed * 100 / total_size)))
- QApplication.processEvents()
- if not self._store.is_installed(manifest):
- raise RuntimeError("verifica finale del modello fallita")
- except Exception as exc:
- progress.close()
- QMessageBox.critical(self, "Download non riuscito", str(exc))
- return
- progress.setValue(100)
- progress.close()
+ def _finish_worker(self) -> None:
+ worker = self._worker
+ self._worker = None
+ if worker is not None:
+ worker.wait()
+ self.btn_download.setEnabled(True)
+ self._set_downloading(False)
+
+ def _on_download_done(self) -> None:
+ self._finish_worker()
QMessageBox.information(self, "Download completato", "Il modello è stato verificato e installato.")
- self._show_details(current, None)
+ self._show_details(self.list.currentItem(), None)
+
+ def _on_download_error(self, message: str) -> None:
+ self._finish_worker()
+ self._show_details(self.list.currentItem(), None)
+ QMessageBox.critical(self, "Download non riuscito", message)
+ def _on_download_cancelled(self) -> None:
+ self._finish_worker()
+ self._show_details(self.list.currentItem(), None)
diff --git a/requirements.txt b/requirements.txt
index a779a6e..8b36a1c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,7 +22,9 @@ keyring==25.7.0; sys_platform != "win32"
# IMPORTANT: model minor version MUST match the spaCy core minor version
# (spaCy 3.8.x <-> models 3.8.x). The models declare spacy>=3.8.0,<3.9.0.
# These are cached by pip in GitHub Actions, significantly speeding up CI/CD
-en-core-web-md @ https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl
+# Development and CI only (tests of the Italian detectors). Licence CC BY-NC-SA 3.0:
+# NOT bundled in the installer (excluded in the PyInstaller spec); the app
+# downloads it on demand after the user accepts the licence (spacy_models.py).
it-core-news-md @ https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.8.0/it_core_news_md-3.8.0-py3-none-any.whl
# Build Tool (compatible with Python 3.13)
diff --git a/setup_env.py b/setup_env.py
index 377acd6..1643233 100644
--- a/setup_env.py
+++ b/setup_env.py
@@ -113,9 +113,8 @@ def setup_environment() -> None:
logger.error(f"Errore durante l'installazione dipendenze: {e}")
sys.exit(1)
- logger.info("Download modelli linguistici spaCy (EN & IT)")
+ logger.info("Download modello linguistico spaCy (IT)")
spacy_models = [
- "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl",
"https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.8.0/it_core_news_md-3.8.0-py3-none-any.whl"
]
try:
diff --git a/spacy_models.py b/spacy_models.py
new file mode 100644
index 0000000..676eb42
--- /dev/null
+++ b/spacy_models.py
@@ -0,0 +1,193 @@
+"""Optional spaCy pipelines that are downloaded on demand instead of bundled.
+
+The Italian pipeline ``it_core_news_md`` is published under CC BY-NC-SA 3.0
+(non-commercial, share-alike). NullifyPDF is AGPL-3.0, whose recipients may use
+the program commercially, so that pipeline cannot be part of the installer.
+It is downloaded only after the user has seen the licence and agreed, from the
+publisher's release page, verified against a pinned size and SHA-256 and kept
+in the user's configuration folder. Detection quality is the same as when it
+was bundled.
+
+When the user declines, a blank Italian pipeline (MIT, part of spaCy itself) is
+used so the rule-based detectors keep working; only the statistical name
+recogniser is missing and the caller must say so.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import pathlib
+import shutil
+import tempfile
+import urllib.request
+import zipfile
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+ITALIAN_PACKAGE = "it_core_news_md"
+ITALIAN_VERSION = "3.8.0"
+ITALIAN_URL = (
+ "https://github.com/explosion/spacy-models/releases/download/"
+ f"{ITALIAN_PACKAGE}-{ITALIAN_VERSION}/{ITALIAN_PACKAGE}-{ITALIAN_VERSION}-py3-none-any.whl"
+)
+ITALIAN_SHA256 = "a731d2d8e7c5a7093ac42f88774ea2a2ad3d5809959eedee65632b3533f804ec"
+ITALIAN_SIZE = 42389964
+ITALIAN_LICENSE = "CC BY-NC-SA 3.0"
+ITALIAN_LICENSE_URL = "https://creativecommons.org/licenses/by-nc-sa/3.0/"
+_MAX_UNPACKED = 120 * 1024 * 1024
+_PIPELINE_DIR = f"{ITALIAN_PACKAGE}/{ITALIAN_PACKAGE}-{ITALIAN_VERSION}/"
+
+ProgressFn = Callable[[int, int], None]
+
+
+class ModelDownloadError(RuntimeError):
+ """The pipeline could not be downloaded or failed verification."""
+
+
+@dataclass(frozen=True)
+class ResolvedModel:
+ name: str # what Presidio/spaCy should load: package name or folder path
+ statistical: bool # False for the blank fallback (no name recogniser)
+
+
+def models_root(config_dir: pathlib.Path) -> pathlib.Path:
+ return pathlib.Path(config_dir) / "spacy_models"
+
+
+def _installed_dir(root: pathlib.Path) -> pathlib.Path:
+ return root / f"{ITALIAN_PACKAGE}-{ITALIAN_VERSION}"
+
+
+def italian_installed(root: pathlib.Path) -> bool:
+ d = _installed_dir(root)
+ return (d / "meta.json").is_file() and (d / "ner").is_dir()
+
+
+def _is_package() -> bool:
+ import spacy
+
+ return bool(spacy.util.is_package(ITALIAN_PACKAGE))
+
+
+def resolve_italian(root: pathlib.Path) -> Optional[ResolvedModel]:
+ """The Italian statistical pipeline if it is available locally, else None."""
+ if _is_package(): # development checkouts and tests
+ return ResolvedModel(ITALIAN_PACKAGE, True)
+ if italian_installed(root):
+ return ResolvedModel(str(_installed_dir(root)), True)
+ return None
+
+
+def blank_italian(root: pathlib.Path) -> ResolvedModel:
+ """Rules-only fallback: tokenizer and sentence splitter, no NER."""
+ import spacy
+
+ path = root / "blank_it"
+ if not (path / "meta.json").is_file():
+ root.mkdir(parents=True, exist_ok=True)
+ nlp = spacy.blank("it")
+ nlp.add_pipe("sentencizer")
+ nlp.to_disk(path)
+ return ResolvedModel(str(path), False)
+
+
+def _extract(wheel: pathlib.Path, target: pathlib.Path) -> None:
+ """Unpack only the pipeline folder; reject any path that escapes."""
+ total = 0
+ base = target.resolve()
+ with zipfile.ZipFile(wheel) as zf:
+ for info in zf.infolist():
+ if not info.filename.startswith(_PIPELINE_DIR) or info.is_dir():
+ continue
+ rel = pathlib.PurePosixPath(info.filename[len(_PIPELINE_DIR):])
+ if rel.is_absolute() or ".." in rel.parts:
+ raise ModelDownloadError("percorso non sicuro nel pacchetto")
+ total += info.file_size
+ if total > _MAX_UNPACKED:
+ raise ModelDownloadError("pacchetto troppo grande")
+ dest = (base / pathlib.Path(*rel.parts)).resolve()
+ if base not in dest.parents:
+ raise ModelDownloadError("percorso non sicuro nel pacchetto")
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ with zf.open(info) as src, open(dest, "wb") as out:
+ shutil.copyfileobj(src, out)
+
+
+def install_italian(
+ root: pathlib.Path,
+ progress: Optional[ProgressFn] = None,
+ url: str = ITALIAN_URL,
+ sha256: str = ITALIAN_SHA256,
+ size: int = ITALIAN_SIZE,
+ timeout: float = 30.0,
+) -> pathlib.Path:
+ """Download, verify and unpack the pipeline. Returns its folder."""
+ if not url.lower().startswith("https://"):
+ raise ModelDownloadError("sono consentiti solo download HTTPS")
+ root.mkdir(parents=True, exist_ok=True)
+ fd, tmp_name = tempfile.mkstemp(suffix=".whl", dir=root)
+ os.close(fd)
+ tmp = pathlib.Path(tmp_name)
+ staging = pathlib.Path(tempfile.mkdtemp(prefix="unpack-", dir=root))
+ try:
+ digest = hashlib.sha256()
+ done = 0
+ try:
+ with urllib.request.urlopen(url, timeout=timeout) as resp, tmp.open("wb") as out:
+ while True:
+ chunk = resp.read(1024 * 256)
+ if not chunk:
+ break
+ done += len(chunk)
+ if done > size:
+ raise ModelDownloadError("il file scaricato è più grande del previsto")
+ digest.update(chunk)
+ out.write(chunk)
+ if progress:
+ progress(done, size)
+ except OSError as exc:
+ raise ModelDownloadError(f"download non riuscito: {exc}") from exc
+ if done != size or digest.hexdigest() != sha256:
+ raise ModelDownloadError("verifica di integrità fallita (dimensione o SHA-256)")
+ _extract(tmp, staging)
+ if not ((staging / "meta.json").is_file() and (staging / "ner").is_dir()):
+ raise ModelDownloadError("pacchetto incompleto")
+ final = _installed_dir(root)
+ if final.exists():
+ shutil.rmtree(final)
+ os.replace(staging, final)
+ return final
+ finally:
+ tmp.unlink(missing_ok=True)
+ shutil.rmtree(staging, ignore_errors=True)
+
+
+def remove_italian(root: pathlib.Path) -> None:
+ shutil.rmtree(_installed_dir(root), ignore_errors=True)
+
+
+def licence_text() -> str:
+ return (
+ "Il riconoscimento statistico dei nomi in italiano usa la pipeline spaCy "
+ f"«{ITALIAN_PACKAGE}» {ITALIAN_VERSION} (Explosion), distribuita con licenza "
+ f"{ITALIAN_LICENSE} ({ITALIAN_LICENSE_URL}): uso non commerciale, con "
+ "attribuzione e condivisione alle stesse condizioni. Non è inclusa "
+ "nell'installer perché NullifyPDF è AGPL-3.0 e consente l'uso commerciale.\n\n"
+ f"Verrà scaricata da GitHub ({ITALIAN_SIZE // (1024 * 1024)} MB), "
+ "verificata con SHA-256 e salvata solo sul tuo computer. "
+ "Accettando dichiari di usarla nel rispetto di quella licenza."
+ )
+
+
+__all__ = [
+ "ModelDownloadError",
+ "ResolvedModel",
+ "blank_italian",
+ "install_italian",
+ "italian_installed",
+ "licence_text",
+ "models_root",
+ "remove_italian",
+ "resolve_italian",
+]
diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py
index b8c56ca..3dfeaf8 100644
--- a/tests/test_gui_flow.py
+++ b/tests/test_gui_flow.py
@@ -261,9 +261,9 @@ def test_wipe_local_data(window, monkeypatch):
assert not (w.config_dir / "audit").exists()
-def test_default_language_follows_system_locale(window):
- assert window.rb_it.isChecked() or window.rb_both.isChecked()
- assert not window.rb_en.isChecked()
+def test_only_italian_is_offered(window):
+ assert not hasattr(window, "rb_en") and not hasattr(window, "rb_both")
+ assert "italiano" in window.lbl_lang.text()
def test_worker_scan_end_to_end_with_real_italian_model(tmp_path, monkeypatch):
@@ -919,3 +919,62 @@ def spy(*a, **k):
monkeypatch.setattr(app, "export_document", spy)
w.cmd_export()
assert seen["r"].surviving_glyphs == 0 and seen["r"].verification.ok
+
+
+def test_italian_model_prompt_offers_consent_and_fallback(window, monkeypatch):
+ """Without the pipeline the window asks; declining keeps a rules-only scan."""
+ import spacy_models
+
+ monkeypatch.setattr(spacy_models, "_is_package", lambda: False)
+ w = window
+ root = spacy_models.models_root(w.config_dir)
+
+ class FakeBox:
+ choice = "cancel"
+
+ def __init__(self, *a, **k):
+ self.buttons = {}
+
+ def __getattr__(self, name):
+ return lambda *a, **k: None
+
+ def addButton(self, text, role):
+ self.buttons[text] = object()
+ return self.buttons[text]
+
+ def exec(self):
+ return 0
+
+ def clickedButton(self):
+ return self.buttons.get(FakeBox.choice)
+
+ for attr in ("Icon", "ButtonRole"):
+ setattr(FakeBox, attr, getattr(QMessageBox, attr))
+ monkeypatch.setattr(app, "QMessageBox", FakeBox)
+
+ FakeBox.choice = "Annulla"
+ assert w._prepare_italian_model() is False
+
+ FakeBox.choice = "Continua senza modello"
+ assert w._prepare_italian_model() is True
+ assert w.ai_worker.italian_model.endswith("blank_it")
+ assert (root / "blank_it" / "meta.json").is_file()
+
+
+def test_about_dialog_shows_licence_notice_with_links(window, monkeypatch):
+ from PySide6.QtWidgets import QTextBrowser
+
+ import licences_info
+
+ seen = {}
+
+ def fake_exec(self):
+ view = self.findChild(QTextBrowser, "about_licences")
+ seen["html"] = view.toHtml()
+ seen["links"] = view.toPlainText()
+ return 0
+
+ monkeypatch.setattr(app.QDialog, "exec", fake_exec)
+ window.cmd_about()
+ assert "AGPL-3.0-or-later" in seen["links"] and "PyMuPDF" in seen["links"]
+ assert all(c.url in seen["html"] for c in licences_info.COMPONENTS)
diff --git a/tests/test_licences.py b/tests/test_licences.py
new file mode 100644
index 0000000..2cea023
--- /dev/null
+++ b/tests/test_licences.py
@@ -0,0 +1,64 @@
+"""Licence consistency: AGPL project, no non-commercial model in the installer."""
+
+import pathlib
+import re
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+
+
+def _read(name: str) -> str:
+ return (ROOT / name).read_text(encoding="utf8")
+
+
+def test_license_file_is_agpl():
+ text = _read("LICENSE")
+ assert "GNU AFFERO GENERAL PUBLIC LICENSE" in text and "Version 3, 19 November 2007" in text
+ assert "AGPL-3.0-or-later" in text
+
+
+def test_installer_never_bundles_the_non_commercial_pipeline():
+ src = _read("build_local.py")
+ for line in src.splitlines():
+ if "collect_all" in line or line.startswith("for pkg in"):
+ assert "it_core_news_md" not in line
+ assert src.count("excludes=['it_core_news_md'") >= 2
+ assert "pymupdf4llm" in src
+
+
+def test_docs_state_the_licence():
+ assert "AGPL" in _read("README.md")
+ notices = _read("THIRD_PARTY_NOTICES.md")
+ for name in ("PyMuPDF", "PySide6", "it_core_news_md", "Tesseract"):
+ assert name in notices
+ assert re.search(r"AGPL-3\.0-or-later", _read("build_local.py"))
+
+
+def test_no_declared_non_commercial_dependency_is_bundled():
+ """Every installed distribution that declares a non-commercial licence must
+ be on the (empty-by-default) list of components kept out of the installer."""
+ from importlib import metadata
+
+ kept_out = {"it-core-news-md", "it_core_news_md"}
+ offenders = []
+ for dist in metadata.distributions():
+ name = (dist.metadata["Name"] or "").lower().replace("_", "-")
+ lic = " ".join(filter(None, [dist.metadata.get("License"), dist.metadata.get("License-Expression")]))
+ if re.search(r"\bNC\b|non-?commercial", lic, re.I) and name not in kept_out:
+ offenders.append((name, lic))
+ assert not offenders, offenders
+
+
+def test_component_list_is_https_complete_and_in_readme():
+ import licences_info as li
+
+ readme = _read("README.md")
+ names = {c.name for c in li.COMPONENTS}
+ assert {"PyMuPDF / MuPDF", "PySide6 / Qt", "spaCy it_core_news_md", "Tesseract OCR"} <= names
+ for c in li.COMPONENTS:
+ assert c.url.startswith("https://"), c
+ assert c.url in readme, c.name
+ assert c.role_en, c.name
+ assert li.readme_markdown() in readme
+ html_text = li.about_html()
+ assert all(c.url in html_text for c in li.COMPONENTS)
+ assert "non commerciale" in li.NOTICE
diff --git a/tests/test_model_dialog.py b/tests/test_model_dialog.py
index 4b5be9d..59e135a 100644
--- a/tests/test_model_dialog.py
+++ b/tests/test_model_dialog.py
@@ -16,3 +16,117 @@ def test_model_dialog_lists_only_curated_candidates():
assert dialog.details.toPlainText()
dialog.close()
app.processEvents()
+
+
+# --- download progress lives inside the button --------------------------------
+
+import time # noqa: E402
+
+import pytest # noqa: E402
+from PySide6.QtWidgets import QMessageBox # noqa: E402
+
+from model_manager import ModelFile, ModelManifest # noqa: E402
+
+
+def _manifest():
+ return ModelManifest(
+ model_id="fake-model", repo_id="owner/fake", revision="a" * 40, license="MIT",
+ runtime="onnx", labels=("PER",),
+ files=(ModelFile("model.onnx", 1000, "b" * 64),),
+ )
+
+
+class FakeStore:
+ def __init__(self, behaviour):
+ self.behaviour = behaviour
+ self.installed = False
+
+ def is_installed(self, manifest):
+ return self.installed
+
+ def install_file(self, manifest, item, url, *, timeout=30.0, progress=None):
+ self.behaviour(self, progress, item)
+
+
+def _dialog(store, qapp):
+ manifest = _manifest()
+ dlg = ModelDialog(MODEL_CANDIDATES, manifests=[manifest], store=store)
+ dlg._manifests = {c.model_id: manifest for c in MODEL_CANDIDATES}
+ dlg.list.setCurrentRow(0)
+ dlg.btn_download.setEnabled(True)
+ return dlg
+
+
+def _wait(qapp, cond, timeout=5.0):
+ end = time.time() + timeout
+ while time.time() < end and not cond():
+ qapp.processEvents()
+ time.sleep(0.01)
+ return cond()
+
+
+@pytest.fixture()
+def qapp():
+ return QApplication.instance() or QApplication([])
+
+
+@pytest.fixture()
+def boxes(monkeypatch):
+ seen = []
+ monkeypatch.setattr(QMessageBox, "question", lambda *a, **k: QMessageBox.StandardButton.Yes)
+ monkeypatch.setattr(QMessageBox, "information", lambda *a, **k: seen.append("info"))
+ monkeypatch.setattr(QMessageBox, "critical", lambda *a, **k: seen.append(("error", a[2])))
+ return seen
+
+
+def test_download_progress_is_in_the_button_and_ui_is_locked(qapp, boxes):
+ gate = {"go": False}
+
+ def slow(store, progress, item):
+ progress(500, item.size)
+ while not gate["go"]:
+ time.sleep(0.01)
+ progress(1000, item.size)
+ store.installed = True
+
+ dlg = _dialog(FakeStore(slow), qapp)
+ dlg.btn_download.click()
+ assert dlg.is_downloading()
+ assert _wait(qapp, lambda: "50%" in dlg.btn_download.text())
+ assert "qlineargradient" in dlg.btn_download.styleSheet()
+ assert not dlg.list.isEnabled()
+ dlg.reject() # closing is refused while downloading
+ assert dlg.is_downloading()
+ gate["go"] = True
+ assert _wait(qapp, lambda: not dlg.is_downloading())
+ assert boxes == ["info"] and dlg.list.isEnabled() and dlg.btn_download.styleSheet() == ""
+ dlg.close()
+
+
+def test_second_click_cancels_and_restores_button(qapp, boxes):
+ def endless(store, progress, item):
+ while True:
+ progress(1, item.size) # raises InterruptedError once cancelled
+ time.sleep(0.01)
+
+ dlg = _dialog(FakeStore(endless), qapp)
+ dlg.btn_download.click()
+ assert _wait(qapp, lambda: dlg.is_downloading() and dlg.btn_download.text().startswith("Download"))
+ dlg.btn_download.click()
+ assert _wait(qapp, lambda: not dlg.is_downloading())
+ assert boxes == [] # cancelling is silent
+ assert dlg.btn_download.isEnabled() and dlg.list.isEnabled()
+ assert dlg.btn_download.text() == "Scarica modello"
+ dlg.close()
+
+
+def test_failed_download_reports_error_and_unlocks(qapp, boxes):
+ def broken(store, progress, item):
+ raise RuntimeError("hash non corrispondente")
+
+ dlg = _dialog(FakeStore(broken), qapp)
+ dlg.btn_download.click()
+ assert _wait(qapp, lambda: not dlg.is_downloading())
+ assert boxes and boxes[0][0] == "error" and "hash" in boxes[0][1]
+ assert dlg.list.isEnabled() and dlg.btn_download.styleSheet() == ""
+ dlg.close()
diff --git a/tests/test_spacy_models.py b/tests/test_spacy_models.py
new file mode 100644
index 0000000..f323266
--- /dev/null
+++ b/tests/test_spacy_models.py
@@ -0,0 +1,94 @@
+"""Optional Italian pipeline: verified download, safe unpacking, fallback."""
+
+import hashlib
+import io
+import pathlib
+import zipfile
+
+import pytest
+
+import spacy_models as sm
+
+
+def _wheel(path: pathlib.Path, names) -> bytes:
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ for name in names:
+ zf.writestr(name, b"x")
+ data = buf.getvalue()
+ path.write_bytes(data)
+ return data
+
+
+def _serve(monkeypatch, data: bytes):
+ monkeypatch.setattr(sm.urllib.request, "urlopen", lambda url, timeout=0: io.BytesIO(data))
+
+
+def test_rejects_non_https(tmp_path):
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(tmp_path, url="http://example.com/x.whl")
+
+
+def test_rejects_wrong_hash_and_leaves_nothing(tmp_path, monkeypatch):
+ data = _wheel(tmp_path / "w.whl", [])
+ _serve(monkeypatch, data)
+ root = tmp_path / "root"
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(root, url="https://example.com/x.whl", sha256="0" * 64, size=len(data))
+ assert not sm.italian_installed(root)
+ assert not list(root.glob("*.whl"))
+
+
+def test_rejects_oversized_download(tmp_path, monkeypatch):
+ _serve(monkeypatch, b"a" * 100)
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(tmp_path / "r", url="https://e.com/x", sha256="0" * 64, size=10)
+
+
+def test_rejects_path_traversal(tmp_path, monkeypatch):
+ p = sm._PIPELINE_DIR
+ data = _wheel(tmp_path / "w.whl", [p + "../../evil.txt"])
+ _serve(monkeypatch, data)
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(
+ tmp_path / "r", url="https://e.com/x",
+ sha256=hashlib.sha256(data).hexdigest(), size=len(data),
+ )
+ assert not (tmp_path / "evil.txt").exists()
+
+
+def test_installs_valid_package(tmp_path, monkeypatch):
+ p = sm._PIPELINE_DIR
+ data = _wheel(tmp_path / "w.whl", [p + "meta.json", p + "ner/cfg", "other/ignored.txt"])
+ _serve(monkeypatch, data)
+ root = tmp_path / "r"
+ seen = []
+ folder = sm.install_italian(
+ root, progress=lambda d, t: seen.append((d, t)), url="https://e.com/x",
+ sha256=hashlib.sha256(data).hexdigest(), size=len(data),
+ )
+ assert sm.italian_installed(root) and (folder / "ner" / "cfg").is_file()
+ assert not (folder / "ignored.txt").exists() and seen
+ sm.remove_italian(root)
+ assert not sm.italian_installed(root)
+
+
+def test_blank_fallback_is_not_statistical(tmp_path):
+ blank = sm.blank_italian(tmp_path)
+ assert blank.statistical is False and (pathlib.Path(blank.name) / "meta.json").is_file()
+
+
+def test_resolve_prefers_installed_package_then_folder(tmp_path, monkeypatch):
+ monkeypatch.setattr(sm, "_is_package", lambda: False)
+ assert sm.resolve_italian(tmp_path) is None
+ d = tmp_path / f"{sm.ITALIAN_PACKAGE}-{sm.ITALIAN_VERSION}"
+ (d / "ner").mkdir(parents=True)
+ (d / "meta.json").write_text("{}")
+ assert sm.resolve_italian(tmp_path).name == str(d)
+ monkeypatch.setattr(sm, "_is_package", lambda: True)
+ assert sm.resolve_italian(tmp_path).name == sm.ITALIAN_PACKAGE
+
+
+def test_licence_text_names_licence_and_source():
+ t = sm.licence_text()
+ assert "CC BY-NC-SA 3.0" in t and "SHA-256" in t
diff --git a/tests/test_validation.py b/tests/test_validation.py
index 843c530..dee3e68 100644
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -94,15 +94,15 @@ def test_resource_path_not_empty(self):
class TestOcrConfiguration:
"""Test OCR configuration helpers."""
- def test_ocr_language_without_tessdata_uses_requested_languages(self):
- assert ocr_language_for_choice("EN", None) == "eng"
+ def test_ocr_language_without_tessdata_is_italian(self):
assert ocr_language_for_choice("IT", None) == "ita"
- assert ocr_language_for_choice("BOTH", None) == "eng+ita"
def test_ocr_language_filters_to_installed_languages(self, tmp_path):
+ assert ocr_language_for_choice("IT", str(tmp_path)) == "ita"
(tmp_path / "eng.traineddata").write_text("placeholder", encoding="utf-8")
-
- assert ocr_language_for_choice("BOTH", str(tmp_path)) == "eng"
+ assert ocr_language_for_choice("IT", str(tmp_path)) == "eng"
+ (tmp_path / "ita.traineddata").write_text("placeholder", encoding="utf-8")
+ assert ocr_language_for_choice("IT", str(tmp_path)) == "ita+eng"
def test_find_tessdata_dir_reads_environment(self, tmp_path, monkeypatch):
(tmp_path / "ita.traineddata").write_text("placeholder", encoding="utf-8")
From 3d5c7ef89f39251c499009fada660691e7ac55e4 Mon Sep 17 00:00:00 2001
From: 0verwrite <31691645+overwrite00@users.noreply.github.com>
Date: Wed, 11 Nov 2026 10:41:28 +0100
Subject: [PATCH 12/13] chore: prepare stable release 2.4.0 (#135)
---
CHANGELOG.md | 2 ++
NullifyPDF.py | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2f9290c..c037730 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+## [2.4.0] - 2026-11-11
+
### 🔄 Changed
- **Italian only**: the English model and the EN/IT/BOTH selector are gone. The program analyses Italian documents; the sidebar shows "Lingua dei documenti: italiano". The English OCR data is still used, when installed, only to read English words inside Italian pages.
diff --git a/NullifyPDF.py b/NullifyPDF.py
index 945a73e..9a9b7f0 100644
--- a/NullifyPDF.py
+++ b/NullifyPDF.py
@@ -138,7 +138,7 @@
PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined]
__version__ = "2.4.0"
-__version_prerelease__ = "beta.5"
+__version_prerelease__ = ""
APP_VERSION = (
f"{__version__}-{__version_prerelease__}"
if __version_prerelease__
From 3fef9a4e092041055acd17a7cfa55472a8d3bec4 Mon Sep 17 00:00:00 2001
From: 0verwrite <31691645+overwrite00@users.noreply.github.com>
Date: Wed, 11 Nov 2026 10:58:31 +0100
Subject: [PATCH 13/13] fix: match the repository link in tooltips with a
precise URL pattern (#137)
---
help_guide.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/help_guide.py b/help_guide.py
index b1a5342..67cbf3d 100644
--- a/help_guide.py
+++ b/help_guide.py
@@ -168,7 +168,7 @@ def _generic_tooltip(widget: QWidget, text: str) -> str:
return "Versione del programma installata."
if "censura e pseudonimizzazione" in low:
return "Informazioni sul programma, sull'autore e sulla licenza."
- if "github.com" in low or "repository" in low:
+ if re.search(r"https?://(www\.)?github\.com/", low) or "repository" in low:
return "Pagina del progetto su GitHub."
if low.startswith("nuova versione"):
return "Versione più recente disponibile."