From 5f0e350741750c7f3c59402f4ad58ab82432077a Mon Sep 17 00:00:00 2001 From: Odinn Adalsteinsson Date: Thu, 29 Oct 2026 12:23:56 +0100 Subject: [PATCH 1/3] fix: repair word-final Danish mojibake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recognize CP850 æ misdecoded as µ at the end of a word. Add the HAVEN subject regression and prepare golded-ftn 1.5.1. --- .github/workflows/publish.yml | 2 +- CHANGELOG.md | 5 +++++ README.md | 6 +++--- pyproject.toml | 2 +- src/golded_ftn/mojibake.py | 4 +++- tests/test_mojibake_regressions.py | 7 +++++++ 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 53a9c1c..be4f06d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,7 +5,7 @@ on: tag: description: Reviewed GitHub release tag required: true - default: v1.5.0 + default: v1.5.1 type: string permissions: contents: read diff --git a/CHANGELOG.md b/CHANGELOG.md index 0846642..fb5dcd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 1.5.1 — 2026-10-29 + +- Repair word-final CP850 æ misdecoded as µ, such as Kastanjetrµ, during + opt-in mojibake repair. + ## 1.5.0 — 2026-10-27 - Rename ParsedMessage.area_code to area, including serialized message fields. diff --git a/README.md b/README.md index 7c55014..e71a83c 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ Repository: [`golded-ftn-python`](https://github.com/golded-dev/golded-ftn-python). The distribution remains `golded-ftn`; imports use `golded_ftn`. -The source is public on GitHub. [Version 1.5.0 is available on PyPI](https://pypi.org/project/golded-ftn/1.5.0/). +The source is public on GitHub. [Version 1.5.1 is available on PyPI](https://pypi.org/project/golded-ftn/1.5.1/). Install with Python 3.12 or newer: ```sh -python -m pip install golded-ftn==1.5.0 +python -m pip install golded-ftn==1.5.1 ``` Shared FTN message values, reader/writer protocols and text helpers for Python @@ -21,7 +21,7 @@ git clone https://github.com/golded-dev/golded-ftn-python.git cd golded-ftn-python uv sync --locked uv build -python -m pip install dist/golded_ftn-1.5.0-py3-none-any.whl +python -m pip install dist/golded_ftn-1.5.1-py3-none-any.whl ``` ```python diff --git a/pyproject.toml b/pyproject.toml index 609e7a9..8ee7a54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "golded-ftn" -version = "1.5.0" +version = "1.5.1" description = "Shared FTN message contracts, values and text helpers" readme = "README.md" requires-python = ">=3.12" diff --git a/src/golded_ftn/mojibake.py b/src/golded_ftn/mojibake.py index 981e63d..59b9684 100644 --- a/src/golded_ftn/mojibake.py +++ b/src/golded_ftn/mojibake.py @@ -28,7 +28,9 @@ def _score(text: str) -> float: _LITERAL_DEGREE.sub("", text).count("°") if c == "°" else text.count(c) for c in _DAMAGE ) - damage += len(re.findall(r"[A-Za-z]µ(?=[A-Za-z])", text)) + # µ after a letter is usually æ (CP850→Latin-1), including word-final + # subjects like Kastanjetrµ. Keep "10 µm" alone: space before µ. + damage += len(re.findall(r"[A-Za-z]µ", text)) return ( -2.5 * damage + 1.5 * sum(text.count(c) for c in _PLAUSIBLE) diff --git a/tests/test_mojibake_regressions.py b/tests/test_mojibake_regressions.py index 82fa4d6..6c25353 100644 --- a/tests/test_mojibake_regressions.py +++ b/tests/test_mojibake_regressions.py @@ -1,6 +1,13 @@ from golded_ftn import repair_mojibake +def test_repairs_trailing_mu_in_danish_subject() -> None: + """HAVEN msg 64 subject: CP850 æ stored as Latin-1 µ at end of word.""" + result = repair_mojibake("Kastanjetrµ") + assert result.text == "Kastanjetræ" + assert result.changed + + def test_preserves_spaced_temperature() -> None: result = repair_mojibake("20 °C") assert result.text == "20 °C" From f70063dba8d4ecdb9aaa8f4e5aaf1c274f6b670d Mon Sep 17 00:00:00 2001 From: Odinn Adalsteinsson Date: Thu, 29 Oct 2026 12:25:01 +0100 Subject: [PATCH 2/3] chore: record 1.5.1 release archives --- RELEASE-SHA256.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE-SHA256.txt b/RELEASE-SHA256.txt index 887b3ee..40a4db5 100644 --- a/RELEASE-SHA256.txt +++ b/RELEASE-SHA256.txt @@ -1,2 +1,2 @@ -2119b573a56f7779ab013589b23d004ef4d4f2390a4c0530269a57dd1e6d1b17 dist/golded_ftn-1.5.0-py3-none-any.whl -77edb452254be66992dd3894175a4dd4c2964403d125ed5eb616bdf648bb1be0 dist/golded_ftn-1.5.0.tar.gz +571761bb5a07e07e44046a0ab5af82f933c676b22b11dc40a193b2c7187803e2 dist/golded_ftn-1.5.1-py3-none-any.whl +f283afdafde06983e57d6707475ef301b704d237ec57a10d611501b84b1e6c0c dist/golded_ftn-1.5.1.tar.gz From 786e3f35b8459801fba7a5e418f083219994236f Mon Sep 17 00:00:00 2001 From: Odinn Adalsteinsson Date: Thu, 29 Oct 2026 12:26:00 +0100 Subject: [PATCH 3/3] docs: clarify the mojibake encoding direction --- CHANGELOG.md | 2 +- RELEASE-SHA256.txt | 4 ++-- src/golded_ftn/mojibake.py | 2 +- tests/test_mojibake_regressions.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb5dcd1..f26d34c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 1.5.1 — 2026-10-29 -- Repair word-final CP850 æ misdecoded as µ, such as Kastanjetrµ, during +- Repair word-final Latin-1 æ misdecoded as CP850 µ, such as Kastanjetrµ, during opt-in mojibake repair. ## 1.5.0 — 2026-10-27 diff --git a/RELEASE-SHA256.txt b/RELEASE-SHA256.txt index 40a4db5..563ddbc 100644 --- a/RELEASE-SHA256.txt +++ b/RELEASE-SHA256.txt @@ -1,2 +1,2 @@ -571761bb5a07e07e44046a0ab5af82f933c676b22b11dc40a193b2c7187803e2 dist/golded_ftn-1.5.1-py3-none-any.whl -f283afdafde06983e57d6707475ef301b704d237ec57a10d611501b84b1e6c0c dist/golded_ftn-1.5.1.tar.gz +a5f8d0cae62acfc377201c10151ca8db0cb8454f531a5d63f5208adb6246ba8a dist/golded_ftn-1.5.1-py3-none-any.whl +c0975141e2ef5ddcc3aaca5e79cc16f63f83730e674364c14fce6910fabcbfb5 dist/golded_ftn-1.5.1.tar.gz diff --git a/src/golded_ftn/mojibake.py b/src/golded_ftn/mojibake.py index 59b9684..be35f17 100644 --- a/src/golded_ftn/mojibake.py +++ b/src/golded_ftn/mojibake.py @@ -28,7 +28,7 @@ def _score(text: str) -> float: _LITERAL_DEGREE.sub("", text).count("°") if c == "°" else text.count(c) for c in _DAMAGE ) - # µ after a letter is usually æ (CP850→Latin-1), including word-final + # µ after a letter is usually æ (Latin-1 read as CP850), including word-final # subjects like Kastanjetrµ. Keep "10 µm" alone: space before µ. damage += len(re.findall(r"[A-Za-z]µ", text)) return ( diff --git a/tests/test_mojibake_regressions.py b/tests/test_mojibake_regressions.py index 6c25353..54f9896 100644 --- a/tests/test_mojibake_regressions.py +++ b/tests/test_mojibake_regressions.py @@ -2,7 +2,7 @@ def test_repairs_trailing_mu_in_danish_subject() -> None: - """HAVEN msg 64 subject: CP850 æ stored as Latin-1 µ at end of word.""" + """HAVEN msg 64 subject: Latin-1 æ read as CP850 µ at end of word.""" result = repair_mojibake("Kastanjetrµ") assert result.text == "Kastanjetræ" assert result.changed