+
+ This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements.
+
+You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see .
diff --git a/NullifyPDF.py b/NullifyPDF.py
index f034698..945a73e 100644
--- a/NullifyPDF.py
+++ b/NullifyPDF.py
@@ -25,9 +25,9 @@
QHBoxLayout,
QLabel,
QPushButton,
- QRadioButton,
QCheckBox,
QProgressBar,
+ QProgressDialog,
QTextEdit,
QGraphicsView,
QGraphicsScene,
@@ -35,7 +35,6 @@
QDialog,
QDialogButtonBox,
QLineEdit,
- QButtonGroup,
QGraphicsRectItem,
QMessageBox,
QInputDialog,
@@ -54,7 +53,6 @@
QDropEvent,
)
from PySide6.QtCore import (
- QLocale,
Qt,
QThread,
QObject,
@@ -100,6 +98,8 @@
save_enabled_groups,
whole_word_rects,
)
+import licences_info
+import spacy_models
import update_check
from export_engine import (
build_audit_record,
@@ -138,7 +138,7 @@
PDF_REDACT_IMAGE_PIXELS: int = fitz.PDF_REDACT_IMAGE_PIXELS # type: ignore[attr-defined]
__version__ = "2.4.0"
-__version_prerelease__ = "beta.4"
+__version_prerelease__ = "beta.5"
APP_VERSION = (
f"{__version__}-{__version_prerelease__}"
if __version_prerelease__
@@ -167,6 +167,27 @@ def run(self) -> None:
self.error_sig.emit(f"{type(e).__name__}: {e}")
+class SpacyModelInstallWorker(QThread):
+ """Download the optional Italian spaCy pipeline off the UI thread."""
+
+ progress_sig = Signal(int, int)
+ done_sig = Signal()
+ error_sig = Signal(str)
+
+ def __init__(self, root: pathlib.Path) -> None:
+ super().__init__()
+ self._root = root
+
+ def run(self) -> None:
+ try:
+ spacy_models.install_italian(
+ self._root, progress=lambda d, t: self.progress_sig.emit(d, t)
+ )
+ self.done_sig.emit()
+ except Exception as e:
+ self.error_sig.emit(str(e))
+
+
class UpdateDownloadWorker(QThread):
"""Download a release asset into the user's Downloads folder."""
@@ -842,15 +863,19 @@ def find_tessdata_dir() -> Optional[str]:
def ocr_language_for_choice(choice: str, tessdata_dir: Optional[str]) -> str:
- """Map the UI language choice to installed Tesseract language codes."""
- requested = ["eng", "ita"] if choice == "BOTH" else ["ita" if choice == "IT" else "eng"]
+ """Tesseract languages for the scan: Italian, plus English data if installed.
+
+ The program analyses Italian documents only; the English OCR data is kept
+ just so that English words and Latin-script terms inside them are read
+ well. `choice` is kept for API stability and must be "IT".
+ """
if not tessdata_dir:
- return "+".join(requested)
+ return "ita"
available = [
- lang for lang in requested
+ lang for lang in ("ita", "eng")
if (pathlib.Path(tessdata_dir) / f"{lang}.traineddata").exists()
]
- return "+".join(available or requested)
+ return "+".join(available or ["ita"])
STYLESHEET = """
@@ -909,6 +934,10 @@ def __init__(self) -> None:
super().__init__()
self.analyzer: Optional[Any] = None
self.loaded_langs: List[str] = []
+ # Set by the window before a scan: package name or folder of the
+ # Italian pipeline (it is downloaded on demand, see spacy_models).
+ self.italian_model: str = spacy_models.ITALIAN_PACKAGE
+ self._loaded_italian = ""
self.model_detectors: List[Tuple[str, OnnxTokenDetector]] = []
self._stop_requested = False
@@ -947,7 +976,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
Args:
doc: PyMuPDF document (shared with UI thread, mutex-protected).
doc_mutex: QMutex serializing access to `doc`.
- choice: Language choice: 'EN', 'IT', or 'BOTH'.
+ choice: Language of the document; only 'IT' is supported.
compiled_allowlist: Pre-compiled regex patterns to skip redaction.
allowlist_set: Set of allowlist entries (lowercase) for O(1)
exact-match fast-path lookup before regex any() scan.
@@ -965,7 +994,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
self.finished_sig.emit()
return
- if choice not in ("EN", "IT", "BOTH"):
+ if choice != "IT":
self.log_sig.emit(f"ERRORE: Scelta lingua non valida: {choice}")
self.finished_sig.emit()
return
@@ -978,21 +1007,19 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
if not isinstance(allowlist_set, set):
allowlist_set = set()
- target_langs = ["en", "it"] if choice == "BOTH" else [choice.lower()]
+ target_langs = ["it"]
ocr_language = ocr_language_for_choice(choice, tessdata_dir)
- if not self.analyzer or sorted(self.loaded_langs) != sorted(target_langs):
+ if (
+ not self.analyzer
+ or sorted(self.loaded_langs) != sorted(target_langs)
+ or self._loaded_italian != self.italian_model
+ ):
self.log_sig.emit(f"Inizializzazione motore di rilevamento ({choice})...")
from presidio_analyzer import AnalyzerEngine, RecognizerRegistry
from presidio_analyzer.nlp_engine import NlpEngineProvider
from presidio_analyzer.predefined_recognizers import PhoneRecognizer
- models = (
- [{"lang_code": "en", "model_name": "en_core_web_md"}]
- if "en" in target_langs
- else []
- )
- if "it" in target_langs:
- models.append({"lang_code": "it", "model_name": "it_core_news_md"})
+ models = [{"lang_code": "it", "model_name": self.italian_model}]
provider = NlpEngineProvider(
nlp_configuration={"nlp_engine_name": "spacy", "models": models}
)
@@ -1025,6 +1052,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str,
supported_languages=target_langs,
)
self.loaded_langs = target_langs
+ self._loaded_italian = self.italian_model
self.log_sig.emit("Scansione privacy in corso...")
self._load_local_detectors()
targets = list(enabled_types)
@@ -1507,22 +1535,9 @@ def build_ui(self) -> None:
btn_open.clicked.connect(self.cmd_open)
s_lay.addWidget(btn_open)
s_lay.addSpacing(20)
- s_lay.addWidget(QLabel("Modello lingua:"))
- lang_lay = QHBoxLayout()
- self.rb_en = QRadioButton("EN")
- self.rb_it = QRadioButton("IT")
- self.rb_both = QRadioButton("BOTH")
- # Default to the interface language: running the English model on
- # Italian text only adds false positives.
- if QLocale.system().language() == QLocale.Language.Italian:
- self.rb_it.setChecked(True)
- else:
- self.rb_both.setChecked(True)
- self.lang_grp = QButtonGroup(self)
- for rb in (self.rb_en, self.rb_it, self.rb_both):
- self.lang_grp.addButton(rb)
- lang_lay.addWidget(rb)
- s_lay.addLayout(lang_lay)
+ self.lbl_lang = QLabel("Lingua dei documenti: italiano")
+ self.lbl_lang.setObjectName("language_label")
+ s_lay.addWidget(self.lbl_lang)
s_lay.addSpacing(10)
self.chk_img = QCheckBox("Oscura Immagini")
s_lay.addWidget(self.chk_img)
@@ -2080,7 +2095,7 @@ def cmd_about(self) -> None:
"""Show about dialog."""
d = QDialog(self)
d.setWindowTitle("Info")
- d.setFixedSize(340, 560)
+ d.setFixedSize(520, 820)
lay = QVBoxLayout(d)
lay.setAlignment(Qt.AlignmentFlag.AlignCenter)
ip = resource_path(os.path.join("images", "NullifyPDF_icon.png"))
@@ -2103,7 +2118,7 @@ def cmd_about(self) -> None:
lbl_ver.setAlignment(Qt.AlignmentFlag.AlignCenter)
lay.addWidget(lbl_ver)
desc = QLabel(
- "\nCensura e pseudonimizzazione PDF offline.\n\nSviluppato da: Graziano Mariella\nLicenza MIT"
+ "\nCensura e pseudonimizzazione PDF offline.\n\nSviluppato da: Graziano Mariella\nLicenza AGPL-3.0"
)
desc.setAlignment(Qt.AlignmentFlag.AlignCenter)
lay.addWidget(desc)
@@ -2114,7 +2129,17 @@ def cmd_about(self) -> None:
lbl_repo.setOpenExternalLinks(True)
lbl_repo.setAlignment(Qt.AlignmentFlag.AlignCenter)
lay.addWidget(lbl_repo)
- lay.addSpacing(10)
+ lay.addSpacing(6)
+ lic_view = QTextBrowser()
+ lic_view.setObjectName("about_licences")
+ lic_view.setOpenExternalLinks(True)
+ lic_view.setHtml(licences_info.about_html())
+ lic_view.setMinimumHeight(200)
+ lic_view.setToolTip(
+ "Licenza del programma e dei componenti usati, con i collegamenti ai testi ufficiali."
+ )
+ lay.addWidget(lic_view, 1)
+ lay.addSpacing(6)
btn_upd = QPushButton("Controlla aggiornamenti")
lay.addWidget(btn_upd)
chk_auto = QCheckBox("Controlla ad ogni avvio")
@@ -2293,11 +2318,10 @@ def cmd_auto_ai(self) -> None:
c_allow = [
(a, re.compile(r"\b" + re.escape(a) + r"\b")) for a in self.allowlist
]
- lang = (
- "EN"
- if self.rb_en.isChecked()
- else "IT" if self.rb_it.isChecked() else "BOTH"
- )
+ lang = "IT"
+ if not self._prepare_italian_model():
+ self.btn_ai.setEnabled(True)
+ return
use_ocr = self.chk_ocr.isChecked()
tessdata_dir = find_tessdata_dir() if use_ocr else None
if use_ocr and not tessdata_dir:
@@ -2324,6 +2348,76 @@ def cmd_auto_ai(self) -> None:
health_mode,
)
+ def _prepare_italian_model(self) -> bool:
+ """Make an Italian pipeline available; False if the user cancels.
+
+ The statistical pipeline is not bundled (non-commercial licence): ask
+ once, show the licence, download and verify it. If the user declines,
+ the scan continues with rules only and the log says so.
+ """
+ root = spacy_models.models_root(self.config_dir)
+ found = spacy_models.resolve_italian(root)
+ if found is not None:
+ self.ai_worker.italian_model = found.name
+ return True
+ box = QMessageBox(self)
+ box.setWindowTitle("Modello italiano")
+ box.setIcon(QMessageBox.Icon.Question)
+ box.setText("Per riconoscere i nomi in italiano serve un modello aggiuntivo.")
+ box.setInformativeText(spacy_models.licence_text())
+ b_dl = box.addButton("Accetta e scarica", QMessageBox.ButtonRole.AcceptRole)
+ b_skip = box.addButton("Continua senza modello", QMessageBox.ButtonRole.DestructiveRole)
+ box.addButton("Annulla", QMessageBox.ButtonRole.RejectRole)
+ box.exec()
+ clicked = box.clickedButton()
+ if clicked is b_skip:
+ self.ai_worker.italian_model = spacy_models.blank_italian(root).name
+ self.write_log(
+ "Avviso: modello italiano non installato, riconoscimento nomi "
+ "solo con regole e dizionari (meno completo)."
+ )
+ return True
+ if clicked is not b_dl:
+ return False
+ dlg = QProgressDialog("Download del modello italiano...", "Annulla", 0, 100, self)
+ dlg.setWindowTitle("Modello italiano")
+ dlg.setMinimumDuration(0)
+ dlg.setAutoClose(False)
+ dlg.setAutoReset(False)
+ dlg.setCancelButton(None)
+ worker = SpacyModelInstallWorker(root)
+ result: Dict[str, str] = {}
+
+ def on_progress(done: int, total: int) -> None:
+ dlg.setValue(int(done * 100 / max(1, total)))
+
+ def on_done() -> None:
+ result["ok"] = "1"
+ dlg.accept()
+
+ def on_error(msg: str) -> None:
+ result["error"] = msg
+ dlg.accept()
+
+ worker.progress_sig.connect(on_progress)
+ worker.done_sig.connect(on_done)
+ worker.error_sig.connect(on_error)
+ worker.start()
+ dlg.exec()
+ worker.wait()
+ if "ok" in result:
+ found = spacy_models.resolve_italian(root)
+ if found is not None:
+ self.ai_worker.italian_model = found.name
+ self.write_log("Modello italiano scaricato e verificato.")
+ return True
+ QMessageBox.warning(
+ self,
+ "Modello italiano",
+ "Download non riuscito: " + result.get("error", "errore sconosciuto"),
+ )
+ return False
+
def _select_ai_entities(self) -> Optional[List[str]]:
"""Ask which data types the AI scan should look for.
diff --git a/README.md b/README.md
index be56a0e..57f2912 100644
--- a/README.md
+++ b/README.md
@@ -125,7 +125,7 @@ No installation needed on Windows/macOS - just run or unzip. Linux users can ins
This script automatically:
- Creates isolated virtual environment (`.venv`)
- Installs all dependencies
- - Downloads spaCy language models (EN/IT)
+ - Installs the Italian spaCy model for development (the installer downloads it on demand)
4. **Activate environment & launch**
@@ -158,7 +158,7 @@ python setup_env.py
- Detects OS (Windows/macOS/Linux)
- Creates `.venv` with Python 3.13
- Installs `requirements.txt` dependencies
-- Downloads spaCy models (English, Italian, both)
+- Installs the Italian spaCy model for development (the installer does not bundle it)
**Automatic OS detection:**
- Windows: Uses `py -3.13` launcher
@@ -253,11 +253,43 @@ See [SECURITY.md](./SECURITY.md) for responsible disclosure and privacy details.
## 📝 License
-**MIT License** — Free to use, modify, and distribute
+**GNU Affero General Public License v3.0 or later (AGPL-3.0-or-later)**
Copyright (c) 2026 Graziano Mariella
-See [LICENSE](./LICENSE) for full text.
+NullifyPDF is free of charge and is not sold by its author. It is AGPL-licensed
+because it uses [PyMuPDF](https://pymupdf.readthedocs.io/), which is available
+under the AGPL-3.0 or a commercial licence from Artifex. Anyone who distributes
+the program, modified or not, must give the recipients the source code under the
+same licence. Releases up to v2.4.0-beta.4 were published under the MIT licence.
+
+The Italian name-recognition model (spaCy `it_core_news_md`, CC BY-NC-SA 3.0,
+non-commercial) is **not** part of the installer: the program offers to download
+it, shows its licence and asks for consent first. Licences of all third-party
+components are listed in [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md).
+
+**Notice.** NullifyPDF is free software, distributed under the AGPL-3.0-or-later. Every component and model it uses remains under its own licence, listed below with a link to the official text; using the program means respecting those conditions. The Italian spaCy model is for non-commercial use only and is downloaded only after your consent.
+
+| Component | Licence | Used for |
+|---|---|---|
+| [NullifyPDF](https://github.com/overwrite00/NullifyPDF/blob/main/LICENSE) | AGPL-3.0-or-later | this program |
+| [PyMuPDF / MuPDF](https://pymupdf.readthedocs.io/en/latest/about.html) | AGPL-3.0 or commercial (Artifex) | PDF reading and redaction |
+| [PySide6 / Qt](https://opensource.org/license/lgpl-3-0) | LGPL-3.0 | graphical interface |
+| [Presidio](https://github.com/microsoft/presidio/blob/main/LICENSE) | MIT | personal-data detection |
+| [spaCy](https://github.com/explosion/spaCy/blob/master/LICENSE) | MIT | language processing |
+| [spaCy it_core_news_md](https://creativecommons.org/licenses/by-nc-sa/3.0/) | CC BY-NC-SA 3.0 (non-commercial use only) | Italian model, downloaded after consent |
+| [onnxruntime](https://github.com/microsoft/onnxruntime/blob/main/LICENSE) | MIT | running local models |
+| [tokenizers](https://github.com/huggingface/tokenizers/blob/main/LICENSE) | Apache-2.0 | tokenizing for local models |
+| [cryptography](https://github.com/pyca/cryptography/blob/main/LICENSE) | Apache-2.0 or BSD-3-Clause | encryption |
+| [keyring](https://github.com/jaraco/keyring) | MIT | key storage (Linux/macOS) |
+| [tldextract](https://github.com/john-kurkowski/tldextract/blob/master/LICENSE) | BSD-3-Clause | URL recognition |
+| [NumPy](https://github.com/numpy/numpy/blob/main/LICENSE.txt) | BSD-3-Clause | numerical computing |
+| [Tesseract OCR](https://github.com/tesseract-ocr/tesseract/blob/main/LICENSE) | Apache-2.0 | OCR of scanned documents |
+| [tessdata_fast (eng, ita)](https://github.com/tesseract-ocr/tessdata_fast/blob/main/LICENSE) | Apache-2.0 | OCR language data |
+| [ISTAT, elenco dei comuni](https://creativecommons.org/licenses/by/4.0/) | CC BY 4.0 | place names |
+| [Wikidata](https://creativecommons.org/publicdomain/zero/1.0/) | CC0 1.0 | given names and surnames |
+
+See [LICENSE](./LICENSE) for the full text.
---
diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md
index 24950d6..3a92a62 100644
--- a/THIRD_PARTY_NOTICES.md
+++ b/THIRD_PARTY_NOTICES.md
@@ -26,3 +26,32 @@ Wikidata (items that have an article on the Italian Wikipedia).
- The lists are small (a few hundred given names and about one thousand
surnames) and are only used to *confirm* a name; a name missing from them
never causes a hit to be rejected.
+
+## Licence of NullifyPDF and of the bundled components
+
+NullifyPDF is distributed under the GNU Affero General Public License v3.0 or
+later (see `LICENSE`). Source code: https://github.com/overwrite00/NullifyPDF.
+The information below was checked on 2026-11-11 against the package metadata of
+the pinned versions in `requirements.txt` and the publishers' pages.
+
+| Component | Licence | Notes |
+|---|---|---|
+| PyMuPDF / MuPDF (Artifex) | AGPL-3.0 or Artifex commercial | NullifyPDF uses it under the AGPL-3.0, hence its own licence. |
+| PySide6 / Qt for Python | LGPL-3.0 (or GPL-3.0, commercial) | Used under the LGPL-3.0. Qt libraries are shipped unmodified; the full source and the build scripts of NullifyPDF are public, so anyone can rebuild the program against a different Qt. |
+| presidio-analyzer | MIT | |
+| spaCy, thinc | MIT | |
+| spaCy `it_core_news_md` 3.8.0 | CC BY-NC-SA 3.0 | **Not bundled.** Downloaded by the user on request after showing the licence (non-commercial, attribution, share-alike). Sources: UD Italian ISDT (CC BY-NC-SA 3.0), WikiNER (CC BY 4.0), fastText vectors (CC0). Publisher: Explosion, https://github.com/explosion/spacy-models |
+| onnxruntime | MIT | |
+| tokenizers | Apache-2.0 | |
+| cryptography | Apache-2.0 or BSD-3-Clause | |
+| keyring (Linux/macOS) | MIT | |
+| tldextract, numpy | BSD-3-Clause | |
+| Tesseract OCR and `tessdata_fast` (`eng`, `ita`) | Apache-2.0 | As stated by the Tesseract project. |
+| PyInstaller (build tool, not distributed) | GPL-2.0 with the bootloader exception | The exception allows distributing the produced program under any licence. |
+| `pymupdf4llm`, `pymupdf-layout` | not declared | Pulled in by PyMuPDF 2.1 but not used and **excluded** from the installer. |
+
+Local PII models downloaded by the user from Hugging Face are accepted only if
+their licence is permissive (MIT, Apache-2.0 or BSD); the licence is part of the
+pinned manifest and is shown before the download.
+
+This table is not legal advice.
diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md
index c5088c2..6102577 100644
--- a/TROUBLESHOOTING.md
+++ b/TROUBLESHOOTING.md
@@ -156,18 +156,9 @@ Names, emails, or other PII should be detected but aren't highlighted.
### Solution
-**Check 1 — Language Selection**
+**Check 1 — Language and model**
-Open sidebar. Is the correct language selected?
-
-| Language | Best For |
-| -------- | -------------------------------- |
-| **EN** | English text only |
-| **IT** | Italian text only |
-| **BOTH** | Mixed English + Italian (slower) |
-
-> [!WARNING]
-> Wrong language = **zero detection**. Test with a clear English name if unsure.
+NullifyPDF supports Italian documents only. Names are found by the Italian model, which is downloaded after your consent on the first Auto Redact. If you declined the download, only rules and dictionaries run and names may be missed: run Auto Redact again and accept the download.
**Check 2 — Is It Searchable Text?**
@@ -219,7 +210,6 @@ The scan proposes plain words ("Certifico", "Prescrivo", section headings) or fr
### Solution
-- Use the **IT** language model (or **BOTH**) for Italian documents; the English model adds false hits on Italian text.
- Uncertain hits appear in **Rivedi suggerimenti** instead of being applied: leave them unticked to discard them.
- Add words you never want redacted to the allowlist (**Dizionari**); matching is by whole word.
- If black boxes cover parts of the lines above or below the redacted word, update to a build that includes rectangle trimming and re-run the scan. With very tight line spacing (roughly half the font size or less) no safe band exists: a warning "interlinea molto stretta" is shown and neighbouring words can be damaged, never left unredacted.
@@ -486,5 +476,5 @@ The more details, the faster we can fix it! 🚀
---
-*Last updated: 2026-11-08*
+*Last updated: 2026-11-11*
*← [User Guide](./USER_GUIDE.md) | [Development →](./DEVELOPMENT.md)*
diff --git a/USER_GUIDE.md b/USER_GUIDE.md
index e68321e..53a1ca0 100644
--- a/USER_GUIDE.md
+++ b/USER_GUIDE.md
@@ -40,18 +40,12 @@ Only entries with a complete application manifest can be downloaded. The manifes
---
-## Step 2️⃣ — Configure AI Language
+## Step 2️⃣ — Language
-**Before scanning**, select your document's language from the left sidebar:
-
-| 🌐 Language | Use When |
-| ---------- | --------------------------------- |
-| **EN** | Document is entirely in English |
-| **IT** | Document is entirely in Italian |
-| **BOTH** | Document contains mixed languages |
+NullifyPDF analyses **Italian** documents only; there is no language selector. The Italian name-recognition model is not part of the installer (its licence is non-commercial): the first time you run **Auto Redact** the program shows the licence, asks for your consent and downloads it. If you decline, the scan continues with rules and dictionaries only, which finds fewer names.
> [!WARNING]
-> Wrong language = missed detections. Choose the exact language of your document.
+> Documents in other languages are not supported: names and places in them will be missed.
---
diff --git a/build_local.py b/build_local.py
index 1ff6150..9098ec0 100644
--- a/build_local.py
+++ b/build_local.py
@@ -207,7 +207,7 @@ def build_rpm(
Version: {version}
Release: {rpm_release(prerelease)}
Summary: PDF Anonymization Tool
-License: MIT
+License: AGPL-3.0-or-later
URL: https://github.com/overwrite00/NullifyPDF
BuildArch: x86_64
{requires}
@@ -398,11 +398,11 @@ def build_app() -> None:
datas = {datas_literal}
binaries = []
hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
-a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports)
+a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports, excludes=['it_core_news_md', 'pymupdf4llm', 'pymupdf_layout', 'pymupdf.layout'])
pyz = PYZ(a.pure)
exe = EXE(pyz, a.scripts, [], exclude_binaries=True, name='NullifyPDF', debug=False, console=False, icon={icon_str})
coll = COLLECT(exe, a.binaries, a.datas, strip=False, upx=True, upx_exclude=[], name='NullifyPDF')
@@ -415,11 +415,11 @@ def build_app() -> None:
datas = {datas_literal}
binaries = []
hiddenimports = ['spacy', 'presidio_analyzer', 'lexicon_data_it', 'onnxruntime', 'tokenizers']
-for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'en_core_web_md', 'it_core_news_md', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
+for pkg in ['presidio_analyzer', 'spacy', 'tldextract', 'onnxruntime', 'tokenizers'] + (['keyring'] if sys.platform != 'win32' else []):
t = collect_all(pkg)
datas += t[0]; binaries += t[1]; hiddenimports += t[2]
-a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports)
+a = Analysis(['NullifyPDF.py'], datas=datas, hiddenimports=hiddenimports, excludes=['it_core_news_md', 'pymupdf4llm', 'pymupdf_layout', 'pymupdf.layout'])
pyz = PYZ(a.pure)
exe = EXE(pyz, a.scripts, a.binaries, a.datas, name='NullifyPDF', debug=False, console=False, icon={icon_str})
"""
diff --git a/help_guide.py b/help_guide.py
index 56cf597..b1a5342 100644
--- a/help_guide.py
+++ b/help_guide.py
@@ -40,10 +40,7 @@
TOOLTIPS: Dict[str, str] = {
# Sidebar
"Apri PDF": "Apre un file PDF da elaborare. Puoi anche trascinarlo sulla finestra.",
- "Modello lingua:": "Lingua dei modelli di riconoscimento dei dati personali.",
- "EN": "Usa il modello inglese. Adatto a documenti in inglese.",
- "IT": "Usa il modello italiano. Consigliato per documenti in italiano: il modello inglese aggiunge falsi positivi.",
- "BOTH": "Usa entrambi i modelli (inglese e italiano). Più lento; utile per documenti misti.",
+ "Lingua dei documenti: italiano": "NullifyPDF analizza documenti in italiano. Il modello italiano viene scaricato, dopo il tuo consenso, al primo utilizzo.",
"Oscura Immagini": "Oscura anche tutte le immagini della pagina (loghi, firme, foto) quando applichi Auto Redact.",
"OCR PDF scansionati": "Legge con l'OCR le pagine scansionate e le immagini incorporate, così anche il testo nelle immagini viene analizzato.",
"Mostra suggerimenti": "Contorna in arancione, nell'anteprima, i suggerimenti ancora da rivedere. Servono solo a video: non finiscono nel file esportato.",
@@ -256,7 +253,7 @@ def widgets_without_tooltip(root: QWidget) -> List[str]:
"""),
("Lingua e opzioni", """
Lingua e opzioni
-Modello lingua: IT per documenti italiani (consigliato: il modello inglese su testo italiano aggiunge falsi positivi), EN per l'inglese, BOTH per documenti misti. All'avvio viene scelto IT se il sistema è in italiano.
+Lingua: il programma analizza documenti in italiano. Il modello italiano non è incluso nell'installer (licenza non commerciale): al primo utilizzo il programma mostra la licenza e chiede il consenso per scaricarlo.
- Oscura Immagini: censura anche tutte le immagini (loghi, foto, firme) quando applichi Auto Redact.
- OCR PDF scansionati: legge con l'OCR le pagine scansionate e le immagini incorporate nelle pagine di testo. Richiede i dati di Tesseract (inclusi nei pacchetti).
@@ -366,7 +363,7 @@ def widgets_without_tooltip(root: QWidget) -> List[str]:
"""),
("Info e aggiornamenti", """
Info e aggiornamenti
-La finestra Info mostra la versione e il collegamento al repository. Controlla aggiornamenti interroga le release pubblicate su GitHub; con Controlla ad ogni avvio (disattivato di default) il controllo è eseguito all'avvio e mostra qualcosa solo se esiste una versione più recente.
+La finestra Info mostra la versione e il collegamento al repository. Elenca anche la licenza del programma (AGPL-3.0-or-later) e quella di ogni componente e modello usato, con i collegamenti ai testi ufficiali. Controlla aggiornamenti interroga le release pubblicate su GitHub; con Controlla ad ogni avvio (disattivato di default) il controllo è eseguito all'avvio e mostra qualcosa solo se esiste una versione più recente.
"""),
("Scorciatoie", """
Scorciatoie
diff --git a/licences_info.py b/licences_info.py
new file mode 100644
index 0000000..6686e2c
--- /dev/null
+++ b/licences_info.py
@@ -0,0 +1,96 @@
+"""Licences of NullifyPDF and of the components it uses, with official links.
+
+Single source for the "Info" window and for the README table (a test keeps the
+README in step). Informational only: each component remains under its own
+licence, which is what the notice says.
+"""
+
+from __future__ import annotations
+
+import html
+from typing import List, NamedTuple
+
+REPO = "https://github.com/overwrite00/NullifyPDF"
+AGPL_URL = "https://opensource.org/license/agpl-v3"
+LGPL_URL = "https://opensource.org/license/lgpl-3-0"
+CC_NC_SA_URL = "https://creativecommons.org/licenses/by-nc-sa/3.0/"
+
+APP_LICENCE = "AGPL-3.0-or-later"
+NOTICE = (
+ "NullifyPDF è software libero e gratuito, distribuito con licenza "
+ f"{APP_LICENCE}. Ogni componente e ogni modello usati dal programma restano "
+ "soggetti alla propria licenza, indicata qui sotto con il collegamento al testo "
+ "ufficiale: usando il programma se ne rispettano le condizioni. Il modello "
+ "italiano di spaCy è per solo uso non commerciale e viene scaricato solo dopo "
+ "il tuo consenso."
+)
+
+
+class Component(NamedTuple):
+ name: str
+ licence: str
+ url: str # official page or licence text
+ role: str
+ role_en: str = ""
+ licence_en: str = ""
+
+
+COMPONENTS: List[Component] = [
+ Component("NullifyPDF", APP_LICENCE, f"{REPO}/blob/main/LICENSE", "questo programma", "this program"),
+ Component("PyMuPDF / MuPDF", "AGPL-3.0 o commerciale (Artifex)",
+ "https://pymupdf.readthedocs.io/en/latest/about.html", "lettura e redazione dei PDF", "PDF reading and redaction",
+ "AGPL-3.0 or commercial (Artifex)"),
+ Component("PySide6 / Qt", "LGPL-3.0", LGPL_URL, "interfaccia grafica", "graphical interface"),
+ Component("Presidio", "MIT", "https://github.com/microsoft/presidio/blob/main/LICENSE",
+ "riconoscimento dei dati personali", "personal-data detection"),
+ Component("spaCy", "MIT", "https://github.com/explosion/spaCy/blob/master/LICENSE",
+ "elaborazione del linguaggio", "language processing"),
+ Component("spaCy it_core_news_md", "CC BY-NC-SA 3.0 (solo uso non commerciale)", CC_NC_SA_URL,
+ "modello italiano, scaricato dopo il consenso", "Italian model, downloaded after consent",
+ "CC BY-NC-SA 3.0 (non-commercial use only)"),
+ Component("onnxruntime", "MIT", "https://github.com/microsoft/onnxruntime/blob/main/LICENSE",
+ "esecuzione dei modelli locali", "running local models"),
+ Component("tokenizers", "Apache-2.0", "https://github.com/huggingface/tokenizers/blob/main/LICENSE",
+ "tokenizzazione dei modelli locali", "tokenizing for local models"),
+ Component("cryptography", "Apache-2.0 o BSD-3-Clause",
+ "https://github.com/pyca/cryptography/blob/main/LICENSE", "cifratura", "encryption",
+ "Apache-2.0 or BSD-3-Clause"),
+ Component("keyring", "MIT", "https://github.com/jaraco/keyring", "archivio chiavi (Linux/macOS)", "key storage (Linux/macOS)"),
+ Component("tldextract", "BSD-3-Clause",
+ "https://github.com/john-kurkowski/tldextract/blob/master/LICENSE", "riconoscimento URL", "URL recognition"),
+ Component("NumPy", "BSD-3-Clause", "https://github.com/numpy/numpy/blob/main/LICENSE.txt",
+ "calcolo numerico", "numerical computing"),
+ Component("Tesseract OCR", "Apache-2.0", "https://github.com/tesseract-ocr/tesseract/blob/main/LICENSE",
+ "OCR dei documenti scansionati", "OCR of scanned documents"),
+ Component("tessdata_fast (eng, ita)", "Apache-2.0",
+ "https://github.com/tesseract-ocr/tessdata_fast/blob/main/LICENSE", "dati OCR", "OCR language data"),
+ Component("ISTAT, elenco dei comuni", "CC BY 4.0", "https://creativecommons.org/licenses/by/4.0/",
+ "nomi dei luoghi", "place names"),
+ Component("Wikidata", "CC0 1.0", "https://creativecommons.org/publicdomain/zero/1.0/",
+ "nomi e cognomi", "given names and surnames"),
+]
+
+EXTRA_LINKS = [
+ ("LICENSE", f"{REPO}/blob/main/LICENSE"),
+ ("THIRD_PARTY_NOTICES", f"{REPO}/blob/main/THIRD_PARTY_NOTICES.md"),
+]
+
+
+def about_html() -> str:
+ """HTML for the scrollable licence area of the Info window."""
+ rows = "".join(
+ f"- {html.escape(c.name)}"
+ f" — {html.escape(c.licence)} ({html.escape(c.role)})
"
+ for c in COMPONENTS
+ )
+ extra = " · ".join(
+ f"{html.escape(n)}" for n, u in EXTRA_LINKS
+ )
+ return f"{html.escape(NOTICE)}
{extra}
"
+
+
+def readme_markdown() -> str:
+ lines = ["| Component | Licence | Used for |", "|---|---|---|"]
+ for c in COMPONENTS:
+ lines.append(f"| [{c.name}]({c.url}) | {c.licence_en or c.licence} | {c.role_en} |")
+ return "\n".join(lines)
diff --git a/model_dialog.py b/model_dialog.py
index 613eefd..72ab95a 100644
--- a/model_dialog.py
+++ b/model_dialog.py
@@ -4,9 +4,8 @@
from typing import Any, Iterable, Optional
-from PySide6.QtCore import Qt
+from PySide6.QtCore import QThread, Qt, Signal
from PySide6.QtWidgets import (
- QApplication,
QDialog,
QDialogButtonBox,
QLabel,
@@ -14,7 +13,6 @@
QListWidgetItem,
QMessageBox,
QPushButton,
- QProgressDialog,
QTextBrowser,
QVBoxLayout,
)
@@ -23,6 +21,52 @@
from model_manager import ModelCandidate, ModelManifest, ModelStore
+class ModelDownloadWorker(QThread):
+ """Download and verify every file of a manifest off the UI thread."""
+
+ progress_sig = Signal(int) # whole-model percentage
+ done_sig = Signal()
+ error_sig = Signal(str)
+ cancelled_sig = Signal()
+
+ def __init__(self, store: ModelStore, manifest: ModelManifest) -> None:
+ super().__init__()
+ self._store = store
+ self._manifest = manifest
+ self._cancel = False
+ self._completed = 0
+ self._total = max(1, manifest.total_size)
+
+ def cancel(self) -> None:
+ self._cancel = True
+
+ def _on_bytes(self, done: int, total: int) -> None:
+ if self._cancel:
+ raise InterruptedError("download annullato")
+ value = int(((self._completed + min(done, total)) / self._total) * 100)
+ self.progress_sig.emit(min(100, value))
+
+ def run(self) -> None:
+ try:
+ for item in self._manifest.files:
+ url = (
+ f"https://huggingface.co/{self._manifest.repo_id}/resolve/"
+ f"{self._manifest.revision}/{item.name}?download=true"
+ )
+ self._store.install_file(
+ self._manifest, item, url, timeout=120.0, progress=self._on_bytes
+ )
+ self._completed += item.size
+ self.progress_sig.emit(min(100, int(self._completed * 100 / self._total)))
+ if not self._store.is_installed(self._manifest):
+ raise RuntimeError("verifica finale del modello fallita")
+ self.done_sig.emit()
+ except InterruptedError:
+ self.cancelled_sig.emit()
+ except Exception as exc:
+ self.error_sig.emit(str(exc))
+
+
class ModelDialog(QDialog):
"""Show the reviewed model shortlist and its verification state."""
@@ -68,9 +112,10 @@ def __init__(
"Il download è attivo solo per modelli con revisione, file, licenza e SHA-256 verificati."
)
lay.addWidget(self.btn_download)
- buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close)
- buttons.rejected.connect(self.reject)
- lay.addWidget(buttons)
+ self._buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Close)
+ self._buttons.rejected.connect(self.reject)
+ lay.addWidget(self._buttons)
+ self._worker: Optional[ModelDownloadWorker] = None
self.list.currentItemChanged.connect(self._show_details)
self.btn_download.clicked.connect(self._download_selected)
if self.list.count():
@@ -122,7 +167,46 @@ def _show_details(self, current: Optional[QListWidgetItem], _previous: Any) -> N
"Download HTTPS da Hugging Face alla revisione fissata nel manifest."
)
+ # -- download with progress shown inside the button -------------------
+ def _set_button_progress(self, percent: int) -> None:
+ """Fill the button from the left like a progress bar and show the %."""
+ p = min(max(percent, 0), 100) / 100.0
+ edge = min(p, 0.998)
+ self.btn_download.setText(f"Download in corso… {percent}% (clic per annullare)")
+ self.btn_download.setStyleSheet(
+ "QPushButton#model_download { background: qlineargradient("
+ "x1:0, y1:0, x2:1, y2:0, "
+ f"stop:0 #0284c7, stop:{edge:.3f} #0284c7, "
+ f"stop:{edge + 0.001:.3f} #334155, stop:1 #334155); }}"
+ )
+
+ def _set_downloading(self, on: bool) -> None:
+ self.list.setEnabled(not on)
+ close = self._buttons.button(QDialogButtonBox.StandardButton.Close)
+ if close is not None:
+ close.setEnabled(not on)
+ if not on:
+ self.btn_download.setStyleSheet("")
+
+ def is_downloading(self) -> bool:
+ return self._worker is not None
+
+ def reject(self) -> None:
+ if self._worker is None:
+ super().reject()
+
+ def closeEvent(self, event: Any) -> None:
+ if self._worker is not None:
+ event.ignore()
+ else:
+ super().closeEvent(event)
+
def _download_selected(self) -> None:
+ if self._worker is not None: # second click while downloading: cancel
+ self._worker.cancel()
+ self.btn_download.setText("Annullamento…")
+ self.btn_download.setEnabled(False)
+ return
current = self.list.currentItem()
if current is None or self._store is None:
return
@@ -142,41 +226,35 @@ def _download_selected(self) -> None:
)
if answer != QMessageBox.StandardButton.Yes:
return
- progress = QProgressDialog("Download modello...", "", 0, 100, self)
- progress.setWindowTitle("Download modello")
- progress.setAutoClose(False)
- progress.setAutoReset(False)
- progress.setMinimumDuration(0)
- progress.setValue(0)
- progress.show()
- total_size = max(1, manifest.total_size)
- completed = 0
-
- def on_progress(done: int, total: int) -> None:
- value = int(((completed + min(done, total)) / total_size) * 100)
- progress.setValue(min(100, value))
- QApplication.processEvents()
+ self._set_downloading(True)
+ self._set_button_progress(0)
+ worker = ModelDownloadWorker(self._store, manifest)
+ # Bound methods, not lambdas capturing self (no reference cycle).
+ worker.progress_sig.connect(self._set_button_progress)
+ worker.done_sig.connect(self._on_download_done)
+ worker.error_sig.connect(self._on_download_error)
+ worker.cancelled_sig.connect(self._on_download_cancelled)
+ self._worker = worker
+ worker.start()
- try:
- for item in manifest.files:
- url = (
- f"https://huggingface.co/{manifest.repo_id}/resolve/"
- f"{manifest.revision}/{item.name}?download=true"
- )
- self._store.install_file(
- manifest, item, url, timeout=120.0, progress=on_progress
- )
- completed += item.size
- progress.setValue(min(100, int(completed * 100 / total_size)))
- QApplication.processEvents()
- if not self._store.is_installed(manifest):
- raise RuntimeError("verifica finale del modello fallita")
- except Exception as exc:
- progress.close()
- QMessageBox.critical(self, "Download non riuscito", str(exc))
- return
- progress.setValue(100)
- progress.close()
+ def _finish_worker(self) -> None:
+ worker = self._worker
+ self._worker = None
+ if worker is not None:
+ worker.wait()
+ self.btn_download.setEnabled(True)
+ self._set_downloading(False)
+
+ def _on_download_done(self) -> None:
+ self._finish_worker()
QMessageBox.information(self, "Download completato", "Il modello è stato verificato e installato.")
- self._show_details(current, None)
+ self._show_details(self.list.currentItem(), None)
+
+ def _on_download_error(self, message: str) -> None:
+ self._finish_worker()
+ self._show_details(self.list.currentItem(), None)
+ QMessageBox.critical(self, "Download non riuscito", message)
+ def _on_download_cancelled(self) -> None:
+ self._finish_worker()
+ self._show_details(self.list.currentItem(), None)
diff --git a/requirements.txt b/requirements.txt
index a779a6e..8b36a1c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -22,7 +22,9 @@ keyring==25.7.0; sys_platform != "win32"
# IMPORTANT: model minor version MUST match the spaCy core minor version
# (spaCy 3.8.x <-> models 3.8.x). The models declare spacy>=3.8.0,<3.9.0.
# These are cached by pip in GitHub Actions, significantly speeding up CI/CD
-en-core-web-md @ https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl
+# Development and CI only (tests of the Italian detectors). Licence CC BY-NC-SA 3.0:
+# NOT bundled in the installer (excluded in the PyInstaller spec); the app
+# downloads it on demand after the user accepts the licence (spacy_models.py).
it-core-news-md @ https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.8.0/it_core_news_md-3.8.0-py3-none-any.whl
# Build Tool (compatible with Python 3.13)
diff --git a/setup_env.py b/setup_env.py
index 377acd6..1643233 100644
--- a/setup_env.py
+++ b/setup_env.py
@@ -113,9 +113,8 @@ def setup_environment() -> None:
logger.error(f"Errore durante l'installazione dipendenze: {e}")
sys.exit(1)
- logger.info("Download modelli linguistici spaCy (EN & IT)")
+ logger.info("Download modello linguistico spaCy (IT)")
spacy_models = [
- "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0-py3-none-any.whl",
"https://github.com/explosion/spacy-models/releases/download/it_core_news_md-3.8.0/it_core_news_md-3.8.0-py3-none-any.whl"
]
try:
diff --git a/spacy_models.py b/spacy_models.py
new file mode 100644
index 0000000..676eb42
--- /dev/null
+++ b/spacy_models.py
@@ -0,0 +1,193 @@
+"""Optional spaCy pipelines that are downloaded on demand instead of bundled.
+
+The Italian pipeline ``it_core_news_md`` is published under CC BY-NC-SA 3.0
+(non-commercial, share-alike). NullifyPDF is AGPL-3.0, whose recipients may use
+the program commercially, so that pipeline cannot be part of the installer.
+It is downloaded only after the user has seen the licence and agreed, from the
+publisher's release page, verified against a pinned size and SHA-256 and kept
+in the user's configuration folder. Detection quality is the same as when it
+was bundled.
+
+When the user declines, a blank Italian pipeline (MIT, part of spaCy itself) is
+used so the rule-based detectors keep working; only the statistical name
+recogniser is missing and the caller must say so.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import pathlib
+import shutil
+import tempfile
+import urllib.request
+import zipfile
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+ITALIAN_PACKAGE = "it_core_news_md"
+ITALIAN_VERSION = "3.8.0"
+ITALIAN_URL = (
+ "https://github.com/explosion/spacy-models/releases/download/"
+ f"{ITALIAN_PACKAGE}-{ITALIAN_VERSION}/{ITALIAN_PACKAGE}-{ITALIAN_VERSION}-py3-none-any.whl"
+)
+ITALIAN_SHA256 = "a731d2d8e7c5a7093ac42f88774ea2a2ad3d5809959eedee65632b3533f804ec"
+ITALIAN_SIZE = 42389964
+ITALIAN_LICENSE = "CC BY-NC-SA 3.0"
+ITALIAN_LICENSE_URL = "https://creativecommons.org/licenses/by-nc-sa/3.0/"
+_MAX_UNPACKED = 120 * 1024 * 1024
+_PIPELINE_DIR = f"{ITALIAN_PACKAGE}/{ITALIAN_PACKAGE}-{ITALIAN_VERSION}/"
+
+ProgressFn = Callable[[int, int], None]
+
+
+class ModelDownloadError(RuntimeError):
+ """The pipeline could not be downloaded or failed verification."""
+
+
+@dataclass(frozen=True)
+class ResolvedModel:
+ name: str # what Presidio/spaCy should load: package name or folder path
+ statistical: bool # False for the blank fallback (no name recogniser)
+
+
+def models_root(config_dir: pathlib.Path) -> pathlib.Path:
+ return pathlib.Path(config_dir) / "spacy_models"
+
+
+def _installed_dir(root: pathlib.Path) -> pathlib.Path:
+ return root / f"{ITALIAN_PACKAGE}-{ITALIAN_VERSION}"
+
+
+def italian_installed(root: pathlib.Path) -> bool:
+ d = _installed_dir(root)
+ return (d / "meta.json").is_file() and (d / "ner").is_dir()
+
+
+def _is_package() -> bool:
+ import spacy
+
+ return bool(spacy.util.is_package(ITALIAN_PACKAGE))
+
+
+def resolve_italian(root: pathlib.Path) -> Optional[ResolvedModel]:
+ """The Italian statistical pipeline if it is available locally, else None."""
+ if _is_package(): # development checkouts and tests
+ return ResolvedModel(ITALIAN_PACKAGE, True)
+ if italian_installed(root):
+ return ResolvedModel(str(_installed_dir(root)), True)
+ return None
+
+
+def blank_italian(root: pathlib.Path) -> ResolvedModel:
+ """Rules-only fallback: tokenizer and sentence splitter, no NER."""
+ import spacy
+
+ path = root / "blank_it"
+ if not (path / "meta.json").is_file():
+ root.mkdir(parents=True, exist_ok=True)
+ nlp = spacy.blank("it")
+ nlp.add_pipe("sentencizer")
+ nlp.to_disk(path)
+ return ResolvedModel(str(path), False)
+
+
+def _extract(wheel: pathlib.Path, target: pathlib.Path) -> None:
+ """Unpack only the pipeline folder; reject any path that escapes."""
+ total = 0
+ base = target.resolve()
+ with zipfile.ZipFile(wheel) as zf:
+ for info in zf.infolist():
+ if not info.filename.startswith(_PIPELINE_DIR) or info.is_dir():
+ continue
+ rel = pathlib.PurePosixPath(info.filename[len(_PIPELINE_DIR):])
+ if rel.is_absolute() or ".." in rel.parts:
+ raise ModelDownloadError("percorso non sicuro nel pacchetto")
+ total += info.file_size
+ if total > _MAX_UNPACKED:
+ raise ModelDownloadError("pacchetto troppo grande")
+ dest = (base / pathlib.Path(*rel.parts)).resolve()
+ if base not in dest.parents:
+ raise ModelDownloadError("percorso non sicuro nel pacchetto")
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ with zf.open(info) as src, open(dest, "wb") as out:
+ shutil.copyfileobj(src, out)
+
+
+def install_italian(
+ root: pathlib.Path,
+ progress: Optional[ProgressFn] = None,
+ url: str = ITALIAN_URL,
+ sha256: str = ITALIAN_SHA256,
+ size: int = ITALIAN_SIZE,
+ timeout: float = 30.0,
+) -> pathlib.Path:
+ """Download, verify and unpack the pipeline. Returns its folder."""
+ if not url.lower().startswith("https://"):
+ raise ModelDownloadError("sono consentiti solo download HTTPS")
+ root.mkdir(parents=True, exist_ok=True)
+ fd, tmp_name = tempfile.mkstemp(suffix=".whl", dir=root)
+ os.close(fd)
+ tmp = pathlib.Path(tmp_name)
+ staging = pathlib.Path(tempfile.mkdtemp(prefix="unpack-", dir=root))
+ try:
+ digest = hashlib.sha256()
+ done = 0
+ try:
+ with urllib.request.urlopen(url, timeout=timeout) as resp, tmp.open("wb") as out:
+ while True:
+ chunk = resp.read(1024 * 256)
+ if not chunk:
+ break
+ done += len(chunk)
+ if done > size:
+ raise ModelDownloadError("il file scaricato è più grande del previsto")
+ digest.update(chunk)
+ out.write(chunk)
+ if progress:
+ progress(done, size)
+ except OSError as exc:
+ raise ModelDownloadError(f"download non riuscito: {exc}") from exc
+ if done != size or digest.hexdigest() != sha256:
+ raise ModelDownloadError("verifica di integrità fallita (dimensione o SHA-256)")
+ _extract(tmp, staging)
+ if not ((staging / "meta.json").is_file() and (staging / "ner").is_dir()):
+ raise ModelDownloadError("pacchetto incompleto")
+ final = _installed_dir(root)
+ if final.exists():
+ shutil.rmtree(final)
+ os.replace(staging, final)
+ return final
+ finally:
+ tmp.unlink(missing_ok=True)
+ shutil.rmtree(staging, ignore_errors=True)
+
+
+def remove_italian(root: pathlib.Path) -> None:
+ shutil.rmtree(_installed_dir(root), ignore_errors=True)
+
+
+def licence_text() -> str:
+ return (
+ "Il riconoscimento statistico dei nomi in italiano usa la pipeline spaCy "
+ f"«{ITALIAN_PACKAGE}» {ITALIAN_VERSION} (Explosion), distribuita con licenza "
+ f"{ITALIAN_LICENSE} ({ITALIAN_LICENSE_URL}): uso non commerciale, con "
+ "attribuzione e condivisione alle stesse condizioni. Non è inclusa "
+ "nell'installer perché NullifyPDF è AGPL-3.0 e consente l'uso commerciale.\n\n"
+ f"Verrà scaricata da GitHub ({ITALIAN_SIZE // (1024 * 1024)} MB), "
+ "verificata con SHA-256 e salvata solo sul tuo computer. "
+ "Accettando dichiari di usarla nel rispetto di quella licenza."
+ )
+
+
+__all__ = [
+ "ModelDownloadError",
+ "ResolvedModel",
+ "blank_italian",
+ "install_italian",
+ "italian_installed",
+ "licence_text",
+ "models_root",
+ "remove_italian",
+ "resolve_italian",
+]
diff --git a/tests/test_gui_flow.py b/tests/test_gui_flow.py
index b8c56ca..3dfeaf8 100644
--- a/tests/test_gui_flow.py
+++ b/tests/test_gui_flow.py
@@ -261,9 +261,9 @@ def test_wipe_local_data(window, monkeypatch):
assert not (w.config_dir / "audit").exists()
-def test_default_language_follows_system_locale(window):
- assert window.rb_it.isChecked() or window.rb_both.isChecked()
- assert not window.rb_en.isChecked()
+def test_only_italian_is_offered(window):
+ assert not hasattr(window, "rb_en") and not hasattr(window, "rb_both")
+ assert "italiano" in window.lbl_lang.text()
def test_worker_scan_end_to_end_with_real_italian_model(tmp_path, monkeypatch):
@@ -919,3 +919,62 @@ def spy(*a, **k):
monkeypatch.setattr(app, "export_document", spy)
w.cmd_export()
assert seen["r"].surviving_glyphs == 0 and seen["r"].verification.ok
+
+
+def test_italian_model_prompt_offers_consent_and_fallback(window, monkeypatch):
+ """Without the pipeline the window asks; declining keeps a rules-only scan."""
+ import spacy_models
+
+ monkeypatch.setattr(spacy_models, "_is_package", lambda: False)
+ w = window
+ root = spacy_models.models_root(w.config_dir)
+
+ class FakeBox:
+ choice = "cancel"
+
+ def __init__(self, *a, **k):
+ self.buttons = {}
+
+ def __getattr__(self, name):
+ return lambda *a, **k: None
+
+ def addButton(self, text, role):
+ self.buttons[text] = object()
+ return self.buttons[text]
+
+ def exec(self):
+ return 0
+
+ def clickedButton(self):
+ return self.buttons.get(FakeBox.choice)
+
+ for attr in ("Icon", "ButtonRole"):
+ setattr(FakeBox, attr, getattr(QMessageBox, attr))
+ monkeypatch.setattr(app, "QMessageBox", FakeBox)
+
+ FakeBox.choice = "Annulla"
+ assert w._prepare_italian_model() is False
+
+ FakeBox.choice = "Continua senza modello"
+ assert w._prepare_italian_model() is True
+ assert w.ai_worker.italian_model.endswith("blank_it")
+ assert (root / "blank_it" / "meta.json").is_file()
+
+
+def test_about_dialog_shows_licence_notice_with_links(window, monkeypatch):
+ from PySide6.QtWidgets import QTextBrowser
+
+ import licences_info
+
+ seen = {}
+
+ def fake_exec(self):
+ view = self.findChild(QTextBrowser, "about_licences")
+ seen["html"] = view.toHtml()
+ seen["links"] = view.toPlainText()
+ return 0
+
+ monkeypatch.setattr(app.QDialog, "exec", fake_exec)
+ window.cmd_about()
+ assert "AGPL-3.0-or-later" in seen["links"] and "PyMuPDF" in seen["links"]
+ assert all(c.url in seen["html"] for c in licences_info.COMPONENTS)
diff --git a/tests/test_licences.py b/tests/test_licences.py
new file mode 100644
index 0000000..2cea023
--- /dev/null
+++ b/tests/test_licences.py
@@ -0,0 +1,64 @@
+"""Licence consistency: AGPL project, no non-commercial model in the installer."""
+
+import pathlib
+import re
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+
+
+def _read(name: str) -> str:
+ return (ROOT / name).read_text(encoding="utf8")
+
+
+def test_license_file_is_agpl():
+ text = _read("LICENSE")
+ assert "GNU AFFERO GENERAL PUBLIC LICENSE" in text and "Version 3, 19 November 2007" in text
+ assert "AGPL-3.0-or-later" in text
+
+
+def test_installer_never_bundles_the_non_commercial_pipeline():
+ src = _read("build_local.py")
+ for line in src.splitlines():
+ if "collect_all" in line or line.startswith("for pkg in"):
+ assert "it_core_news_md" not in line
+ assert src.count("excludes=['it_core_news_md'") >= 2
+ assert "pymupdf4llm" in src
+
+
+def test_docs_state_the_licence():
+ assert "AGPL" in _read("README.md")
+ notices = _read("THIRD_PARTY_NOTICES.md")
+ for name in ("PyMuPDF", "PySide6", "it_core_news_md", "Tesseract"):
+ assert name in notices
+ assert re.search(r"AGPL-3\.0-or-later", _read("build_local.py"))
+
+
+def test_no_declared_non_commercial_dependency_is_bundled():
+ """Every installed distribution that declares a non-commercial licence must
+ be on the (empty-by-default) list of components kept out of the installer."""
+ from importlib import metadata
+
+ kept_out = {"it-core-news-md", "it_core_news_md"}
+ offenders = []
+ for dist in metadata.distributions():
+ name = (dist.metadata["Name"] or "").lower().replace("_", "-")
+ lic = " ".join(filter(None, [dist.metadata.get("License"), dist.metadata.get("License-Expression")]))
+ if re.search(r"\bNC\b|non-?commercial", lic, re.I) and name not in kept_out:
+ offenders.append((name, lic))
+ assert not offenders, offenders
+
+
+def test_component_list_is_https_complete_and_in_readme():
+ import licences_info as li
+
+ readme = _read("README.md")
+ names = {c.name for c in li.COMPONENTS}
+ assert {"PyMuPDF / MuPDF", "PySide6 / Qt", "spaCy it_core_news_md", "Tesseract OCR"} <= names
+ for c in li.COMPONENTS:
+ assert c.url.startswith("https://"), c
+ assert c.url in readme, c.name
+ assert c.role_en, c.name
+ assert li.readme_markdown() in readme
+ html_text = li.about_html()
+ assert all(c.url in html_text for c in li.COMPONENTS)
+ assert "non commerciale" in li.NOTICE
diff --git a/tests/test_model_dialog.py b/tests/test_model_dialog.py
index 4b5be9d..59e135a 100644
--- a/tests/test_model_dialog.py
+++ b/tests/test_model_dialog.py
@@ -16,3 +16,117 @@ def test_model_dialog_lists_only_curated_candidates():
assert dialog.details.toPlainText()
dialog.close()
app.processEvents()
+
+
+# --- download progress lives inside the button --------------------------------
+
+import time # noqa: E402
+
+import pytest # noqa: E402
+from PySide6.QtWidgets import QMessageBox # noqa: E402
+
+from model_manager import ModelFile, ModelManifest # noqa: E402
+
+
+def _manifest():
+ return ModelManifest(
+ model_id="fake-model", repo_id="owner/fake", revision="a" * 40, license="MIT",
+ runtime="onnx", labels=("PER",),
+ files=(ModelFile("model.onnx", 1000, "b" * 64),),
+ )
+
+
+class FakeStore:
+ def __init__(self, behaviour):
+ self.behaviour = behaviour
+ self.installed = False
+
+ def is_installed(self, manifest):
+ return self.installed
+
+ def install_file(self, manifest, item, url, *, timeout=30.0, progress=None):
+ self.behaviour(self, progress, item)
+
+
+def _dialog(store, qapp):
+ manifest = _manifest()
+ dlg = ModelDialog(MODEL_CANDIDATES, manifests=[manifest], store=store)
+ dlg._manifests = {c.model_id: manifest for c in MODEL_CANDIDATES}
+ dlg.list.setCurrentRow(0)
+ dlg.btn_download.setEnabled(True)
+ return dlg
+
+
+def _wait(qapp, cond, timeout=5.0):
+ end = time.time() + timeout
+ while time.time() < end and not cond():
+ qapp.processEvents()
+ time.sleep(0.01)
+ return cond()
+
+
+@pytest.fixture()
+def qapp():
+ return QApplication.instance() or QApplication([])
+
+
+@pytest.fixture()
+def boxes(monkeypatch):
+ seen = []
+ monkeypatch.setattr(QMessageBox, "question", lambda *a, **k: QMessageBox.StandardButton.Yes)
+ monkeypatch.setattr(QMessageBox, "information", lambda *a, **k: seen.append("info"))
+ monkeypatch.setattr(QMessageBox, "critical", lambda *a, **k: seen.append(("error", a[2])))
+ return seen
+
+
+def test_download_progress_is_in_the_button_and_ui_is_locked(qapp, boxes):
+ gate = {"go": False}
+
+ def slow(store, progress, item):
+ progress(500, item.size)
+ while not gate["go"]:
+ time.sleep(0.01)
+ progress(1000, item.size)
+ store.installed = True
+
+ dlg = _dialog(FakeStore(slow), qapp)
+ dlg.btn_download.click()
+ assert dlg.is_downloading()
+ assert _wait(qapp, lambda: "50%" in dlg.btn_download.text())
+ assert "qlineargradient" in dlg.btn_download.styleSheet()
+ assert not dlg.list.isEnabled()
+ dlg.reject() # closing is refused while downloading
+ assert dlg.is_downloading()
+ gate["go"] = True
+ assert _wait(qapp, lambda: not dlg.is_downloading())
+ assert boxes == ["info"] and dlg.list.isEnabled() and dlg.btn_download.styleSheet() == ""
+ dlg.close()
+
+
+def test_second_click_cancels_and_restores_button(qapp, boxes):
+ def endless(store, progress, item):
+ while True:
+ progress(1, item.size) # raises InterruptedError once cancelled
+ time.sleep(0.01)
+
+ dlg = _dialog(FakeStore(endless), qapp)
+ dlg.btn_download.click()
+ assert _wait(qapp, lambda: dlg.is_downloading() and dlg.btn_download.text().startswith("Download"))
+ dlg.btn_download.click()
+ assert _wait(qapp, lambda: not dlg.is_downloading())
+ assert boxes == [] # cancelling is silent
+ assert dlg.btn_download.isEnabled() and dlg.list.isEnabled()
+ assert dlg.btn_download.text() == "Scarica modello"
+ dlg.close()
+
+
+def test_failed_download_reports_error_and_unlocks(qapp, boxes):
+ def broken(store, progress, item):
+ raise RuntimeError("hash non corrispondente")
+
+ dlg = _dialog(FakeStore(broken), qapp)
+ dlg.btn_download.click()
+ assert _wait(qapp, lambda: not dlg.is_downloading())
+ assert boxes and boxes[0][0] == "error" and "hash" in boxes[0][1]
+ assert dlg.list.isEnabled() and dlg.btn_download.styleSheet() == ""
+ dlg.close()
diff --git a/tests/test_spacy_models.py b/tests/test_spacy_models.py
new file mode 100644
index 0000000..f323266
--- /dev/null
+++ b/tests/test_spacy_models.py
@@ -0,0 +1,94 @@
+"""Optional Italian pipeline: verified download, safe unpacking, fallback."""
+
+import hashlib
+import io
+import pathlib
+import zipfile
+
+import pytest
+
+import spacy_models as sm
+
+
+def _wheel(path: pathlib.Path, names) -> bytes:
+ buf = io.BytesIO()
+ with zipfile.ZipFile(buf, "w") as zf:
+ for name in names:
+ zf.writestr(name, b"x")
+ data = buf.getvalue()
+ path.write_bytes(data)
+ return data
+
+
+def _serve(monkeypatch, data: bytes):
+ monkeypatch.setattr(sm.urllib.request, "urlopen", lambda url, timeout=0: io.BytesIO(data))
+
+
+def test_rejects_non_https(tmp_path):
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(tmp_path, url="http://example.com/x.whl")
+
+
+def test_rejects_wrong_hash_and_leaves_nothing(tmp_path, monkeypatch):
+ data = _wheel(tmp_path / "w.whl", [])
+ _serve(monkeypatch, data)
+ root = tmp_path / "root"
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(root, url="https://example.com/x.whl", sha256="0" * 64, size=len(data))
+ assert not sm.italian_installed(root)
+ assert not list(root.glob("*.whl"))
+
+
+def test_rejects_oversized_download(tmp_path, monkeypatch):
+ _serve(monkeypatch, b"a" * 100)
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(tmp_path / "r", url="https://e.com/x", sha256="0" * 64, size=10)
+
+
+def test_rejects_path_traversal(tmp_path, monkeypatch):
+ p = sm._PIPELINE_DIR
+ data = _wheel(tmp_path / "w.whl", [p + "../../evil.txt"])
+ _serve(monkeypatch, data)
+ with pytest.raises(sm.ModelDownloadError):
+ sm.install_italian(
+ tmp_path / "r", url="https://e.com/x",
+ sha256=hashlib.sha256(data).hexdigest(), size=len(data),
+ )
+ assert not (tmp_path / "evil.txt").exists()
+
+
+def test_installs_valid_package(tmp_path, monkeypatch):
+ p = sm._PIPELINE_DIR
+ data = _wheel(tmp_path / "w.whl", [p + "meta.json", p + "ner/cfg", "other/ignored.txt"])
+ _serve(monkeypatch, data)
+ root = tmp_path / "r"
+ seen = []
+ folder = sm.install_italian(
+ root, progress=lambda d, t: seen.append((d, t)), url="https://e.com/x",
+ sha256=hashlib.sha256(data).hexdigest(), size=len(data),
+ )
+ assert sm.italian_installed(root) and (folder / "ner" / "cfg").is_file()
+ assert not (folder / "ignored.txt").exists() and seen
+ sm.remove_italian(root)
+ assert not sm.italian_installed(root)
+
+
+def test_blank_fallback_is_not_statistical(tmp_path):
+ blank = sm.blank_italian(tmp_path)
+ assert blank.statistical is False and (pathlib.Path(blank.name) / "meta.json").is_file()
+
+
+def test_resolve_prefers_installed_package_then_folder(tmp_path, monkeypatch):
+ monkeypatch.setattr(sm, "_is_package", lambda: False)
+ assert sm.resolve_italian(tmp_path) is None
+ d = tmp_path / f"{sm.ITALIAN_PACKAGE}-{sm.ITALIAN_VERSION}"
+ (d / "ner").mkdir(parents=True)
+ (d / "meta.json").write_text("{}")
+ assert sm.resolve_italian(tmp_path).name == str(d)
+ monkeypatch.setattr(sm, "_is_package", lambda: True)
+ assert sm.resolve_italian(tmp_path).name == sm.ITALIAN_PACKAGE
+
+
+def test_licence_text_names_licence_and_source():
+ t = sm.licence_text()
+ assert "CC BY-NC-SA 3.0" in t and "SHA-256" in t
diff --git a/tests/test_validation.py b/tests/test_validation.py
index 843c530..dee3e68 100644
--- a/tests/test_validation.py
+++ b/tests/test_validation.py
@@ -94,15 +94,15 @@ def test_resource_path_not_empty(self):
class TestOcrConfiguration:
"""Test OCR configuration helpers."""
- def test_ocr_language_without_tessdata_uses_requested_languages(self):
- assert ocr_language_for_choice("EN", None) == "eng"
+ def test_ocr_language_without_tessdata_is_italian(self):
assert ocr_language_for_choice("IT", None) == "ita"
- assert ocr_language_for_choice("BOTH", None) == "eng+ita"
def test_ocr_language_filters_to_installed_languages(self, tmp_path):
+ assert ocr_language_for_choice("IT", str(tmp_path)) == "ita"
(tmp_path / "eng.traineddata").write_text("placeholder", encoding="utf-8")
-
- assert ocr_language_for_choice("BOTH", str(tmp_path)) == "eng"
+ assert ocr_language_for_choice("IT", str(tmp_path)) == "eng"
+ (tmp_path / "ita.traineddata").write_text("placeholder", encoding="utf-8")
+ assert ocr_language_for_choice("IT", str(tmp_path)) == "ita+eng"
def test_find_tessdata_dir_reads_environment(self, tmp_path, monkeypatch):
(tmp_path / "ita.traineddata").write_text("placeholder", encoding="utf-8")