Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/scripts/i18n/apply_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@

DEFAULT_EXPECTED_LOCALES = (
"zh-cn=zh-CN zh-tw=zh-TW ja-jp=ja-JP es=es pt-br=pt-BR ko=ko de=de fr=fr "
"hi=hi ar=ar it=it vi=vi nl=nl fa=fa ru=ru tr=tr uk=uk id=id pl=pl th=th"
"hi=hi ar=ar it=it vi=vi nl=nl sv=sv fa=fa ru=ru tr=tr uk=uk id=id pl=pl th=th"
)
SOURCE_HASH_RE = re.compile(r"^x-i18n:\n(?:[ \t]+.*\n)*?[ \t]+source_hash: ([0-9a-f]{64})$", re.M)

Expand Down
24 changes: 20 additions & 4 deletions .github/scripts/i18n/build_pending_manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

Environment:
LOCALE, LOCALE_SLUG, SHARD_INDEX, SHARD_TOTAL, optional FORCE_RETRANSLATE,
PENDING_LIMIT, CANARY_SOURCE_PATH, and GITHUB_OUTPUT.
PENDING_LIMIT, CANARY_SOURCE_PATH, CANARY_PUBLISH_REQUIRED, and GITHUB_OUTPUT.

Outputs:
Writes docs-i18n-<locale_slug>-s<index>of<total>.txt under --openclaw-sync-dir.
Expand All @@ -39,6 +39,7 @@

SOURCE_HASH_RE = re.compile(r"^x-i18n:\n(?:[ \t]+.*\n)*?[ \t]+source_hash: ([0-9a-f]{64})$", re.M)
MODEL_METADATA_RE = re.compile(r"^x-i18n:\n(?:[ \t]+.*\n)*?[ \t]+(?:model|provider):", re.M)
LOCALE_ENTRY_SOURCES = ("index.md", "start/getting-started.md")


@dataclass(frozen=True)
Expand All @@ -48,6 +49,7 @@ class PendingResult:
pending_count: int
pending_path: Path
shard_files: list[Path]
canary_bootstrap: bool = False


def translation_matches_source(path: Path, source_hash: str) -> bool:
Expand Down Expand Up @@ -104,6 +106,7 @@ def build_pending_manifest(
pending_limit: int = 0,
canary_source_path: str = "",
force_retranslate: bool = False,
canary_publish_required: bool = False,
) -> PendingResult:
locale_dirs = {path.name for path in docs_root.iterdir() if is_locale_dir(path)}
pending_path = openclaw_sync_dir / f"docs-i18n-{locale_slug}-s{shard_index}of{shard_total}.txt"
Expand Down Expand Up @@ -131,10 +134,20 @@ def build_pending_manifest(

pending_files = sorted(pending_files)
shard_files = [file for index, file in enumerate(pending_files) if index % shard_total == shard_index]
canary_bootstrap = False
if pending_limit:
if canary_source_path:
canary_sources: list[Path] = []
for source_path in parse_canary_source_paths(canary_source_path):
source_paths = parse_canary_source_paths(canary_source_path)
if canary_publish_required:
# Publishing a locale's first page activates it in the renderer.
# Include missing entry pages so its homepage and smoke sample work.
for entry in LOCALE_ENTRY_SOURCES:
if not (docs_root / locale / entry).is_file():
canary_bootstrap = True
if entry not in source_paths:
source_paths.append(entry)
for source_path in source_paths:
canary_source = (docs_root / source_path).resolve()
try:
canary_source.relative_to(docs_root.resolve())
Expand All @@ -145,8 +158,8 @@ def build_pending_manifest(
if canary_source in canary_sources:
raise SystemExit("configured canary sources must resolve to unique paths")
canary_sources.append(canary_source)
# A diagnostic canary may exercise a bounded explicit page set in
# one non-publishing job; every path still passes the normal gates.
# Explicit diagnostics and first-locale publication may sample a
# bounded page set; every path still passes the normal hash gates.
shard_files = canary_sources
else:
# Routine canaries probe pending pages without publishing. Choose the
Expand All @@ -165,6 +178,7 @@ def build_pending_manifest(
pending_count=len(shard_files),
pending_path=pending_path,
shard_files=shard_files,
canary_bootstrap=canary_bootstrap,
)


Expand All @@ -176,6 +190,7 @@ def append_output(result: PendingResult) -> None:
fh.write(f"all_count={result.all_count}\n")
fh.write(f"total_pending_count={result.total_pending_count}\n")
fh.write(f"pending_count={result.pending_count}\n")
fh.write(f"canary_bootstrap={str(result.canary_bootstrap).lower()}\n")


def parse_args() -> argparse.Namespace:
Expand Down Expand Up @@ -209,6 +224,7 @@ def main() -> None:
pending_limit=pending_limit,
canary_source_path=os.environ.get("CANARY_SOURCE_PATH", ""),
force_retranslate=os.environ.get("FORCE_RETRANSLATE", "false") == "true",
canary_publish_required=os.environ.get("CANARY_PUBLISH_REQUIRED", "false") == "true",
)
append_output(result)

Expand Down
1 change: 1 addition & 0 deletions .github/scripts/i18n/prepare.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
"pl",
"pt-BR",
"ru",
"sv",
"th",
"tr",
"uk",
Expand Down
111 changes: 101 additions & 10 deletions .github/scripts/i18n/tests/test_i18n_scripts.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,11 @@ def test_full_workflow_keeps_only_weekly_and_manual_triggers(self) -> None:
self.assertIn("schedule:", text)
self.assertIn("workflow_dispatch:", text)
self.assertIn("target_locale:", text)
options = re.search(r"target_locale:.*?options:\n((?: - [^\n]*\n)+)", text, re.S).group(1)
self.assertEqual(
{"all", *(locale.locale_slug for locale in translation_plan.all_locales())},
set(re.findall(r"- (\S+)", options)),
)
self.assertIn("resume_run_id:", text)
self.assertIn("canary_only:", text)
self.assertRegex(text, r"force_retranslate:[\s\S]*?default: false")
Expand Down Expand Up @@ -298,7 +303,11 @@ def test_full_workflow_gates_batches_after_canary(self) -> None:
reusable,
)
self.assertNotIn("inputs.artifact_role == 'canary' || steps.apply.outputs.changed_count != '0'", reusable)
self.assertIn("inputs.artifact_role != 'canary' && steps.apply.outputs.changed_count != '0'", reusable)
self.assertEqual(5, reusable.count("(inputs.artifact_role != 'canary' || needs.translate.outputs.canary_bootstrap == 'true') && steps.apply.outputs.changed_count != '0'"))
self.assertIn("canary_bootstrap: ${{ steps.pending.outputs.canary_bootstrap }}", reusable)
self.assertIn("CANARY_PUBLISH_REQUIRED: ${{ inputs.canary_publish_required }}", reusable)
self.assertIn("if: steps.pending.outputs.canary_bootstrap == 'true' && github.ref != 'refs/heads/main'", reusable)
self.assertLess(reusable.index("Require main for locale bootstrap publication"), reusable.index("Translate changed docs into locale"))
self.assertIn("inputs.commit_locale && steps.apply.outputs.changed_count != '0'", reusable)
self.assertIn("Fail uncommitted locale refresh", reusable)
self.assertIn(
Expand Down Expand Up @@ -578,9 +587,13 @@ def test_translation_workflows_pin_latest_codex_and_tier_effort(self) -> None:
self.assertNotIn('thinking_effort: "max"', incremental)

def test_prepare_path_selection_matches_incremental_rules(self) -> None:
self.assertEqual(translation_plan.LOCALE_NAMES, prepare.LOCALES)
self.assertTrue(prepare.is_translatable_doc_path("docs/guide/setup.mdx"))
self.assertTrue(prepare.is_translatable_doc_path("docs/reference/test.md"))
self.assertFalse(prepare.is_translatable_doc_path("docs/fr/guide/setup.mdx"))
for locale in translation_plan.all_locales():
with self.subTest(locale=locale.locale):
self.assertFalse(prepare.is_translatable_doc_path(f"docs/{locale.locale}/guide/setup.mdx"))
self.assertFalse(prepare.is_translatable_doc_path("docs/.i18n/glossary.fr.json"))
self.assertFalse(prepare.is_translatable_doc_path("docs/.generated/api.md"))
self.assertEqual("3600", prepare.default_cooldown("incremental", "push", "", "3600"))
Expand Down Expand Up @@ -905,11 +918,11 @@ def test_supported_locale_dirs_are_never_source_docs_without_markers(self) -> No
def test_full_plan_all_uses_canary_and_small_batches(self) -> None:
result = plan_full.plan_full("all", 4, FIXTURES / "pending-docs" / "docs")
self.assertEqual("es", result["canary"]["locale"])
self.assertEqual(5, len(result["batches"]))
self.assertEqual(6, len(result["batches"]))
self.assertEqual(1, result["shard_total"])
self.assertEqual(20, len(result["expected_locales"].split()))
self.assertEqual(21, len(result["expected_locales"].split()))
self.assertLessEqual(max(len(batch) for batch in result["batches"]), 4)
self.assertEqual(20, sum(len(batch) for batch in result["batches"]))
self.assertEqual(21, sum(len(batch) for batch in result["batches"]))

def test_translation_plan_shared_shard_policy(self) -> None:
self.assertEqual(1, translation_plan.shard_total_for_doc_count(0, 250, 4))
Expand Down Expand Up @@ -1111,10 +1124,10 @@ def test_full_plan_excludes_supported_locale_dirs_without_marker(self) -> None:
def test_incremental_plan_reuses_shared_locale_and_shard_policy(self) -> None:
result = plan_incremental.plan_incremental(FIXTURES / "pending-docs" / "docs", target_docs_per_shard=1, max_shards=4)

self.assertEqual(20, result["locale_count"])
self.assertEqual(21, result["locale_count"])
self.assertEqual(2, result["source_doc_count"])
self.assertEqual(2, result["shard_total"])
self.assertEqual(40, len(result["matrix"]["include"]))
self.assertEqual(42, len(result["matrix"]["include"]))
self.assertEqual(
[
{"locale": "es", "locale_slug": "es", "shard_index": "0", "shard_total": "2"},
Expand All @@ -1139,6 +1152,7 @@ def test_incremental_plan_reuses_shared_locale_and_shard_policy(self) -> None:
"vi",
"pl",
"nl",
"sv",
"uk",
"th",
"ar",
Expand All @@ -1162,12 +1176,14 @@ def test_incremental_plan_excludes_supported_locale_dirs_without_marker(self) ->

self.assertEqual(1, result["source_doc_count"])
self.assertEqual(1, result["shard_total"])
self.assertEqual(20, len(result["matrix"]["include"]))
self.assertEqual(21, len(result["matrix"]["include"]))

def test_full_plan_manual_single_locale_only_selects_target(self) -> None:
result = plan_full.plan_full("fr", 3, FIXTURES / "pending-docs" / "docs")
self.assertEqual({"locale": "fr", "locale_slug": "fr"}, result["canary"])
self.assertEqual([[{"locale": "fr", "locale_slug": "fr", "shard_index": "0", "shard_total": "1"}]], result["batches"])
for locale in ("fr", "sv"):
with self.subTest(locale=locale):
result = plan_full.plan_full(locale, 3, FIXTURES / "pending-docs" / "docs")
self.assertEqual({"locale": locale, "locale_slug": locale}, result["canary"])
self.assertEqual([[{"locale": locale, "locale_slug": locale, "shard_index": "0", "shard_total": "1"}]], result["batches"])
with self.assertRaises(SystemExit):
plan_full.plan_full("xx", 3, FIXTURES / "pending-docs" / "docs")

Expand Down Expand Up @@ -1456,6 +1472,81 @@ def test_pending_manifest_canary_supports_multiple_configured_source_pages(self)
[file.relative_to((tmp_path / "docs").resolve()).as_posix() for file in result.shard_files],
)

def test_publishing_canary_bootstraps_only_missing_locale_entry_pages(self) -> None:
for present, expected, bootstrap in (
((), ["channels/line.md", "index.md", "start/getting-started.md"], True),
(("index.md",), ["channels/line.md", "start/getting-started.md"], True),
(("index.md", "start/getting-started.md"), ["channels/line.md"], False),
):
with self.subTest(present=present), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp).resolve()
docs = root / "docs"
for relative in ("channels/line.md", *pending.LOCALE_ENTRY_SOURCES, "unrelated.md"):
source = docs / relative
source.parent.mkdir(parents=True, exist_ok=True)
source.write_text(f"# {relative}\n")
for relative in present:
translated = docs / "sv" / relative
translated.parent.mkdir(parents=True, exist_ok=True)
translated.write_text(self._translated_page(docs / relative, "# Befintlig sida\n"))
result = pending.build_pending_manifest(
docs, root / ".openclaw-sync", "sv", "sv", 0, 1,
pending_limit=1, canary_source_path="channels/line.md",
force_retranslate=True, canary_publish_required=True,
)
self.assertEqual(expected, [file.relative_to(docs).as_posix() for file in result.shard_files])
self.assertEqual(bootstrap, result.canary_bootstrap)
self.assertEqual(4, result.total_pending_count)
output = root / "output"
with env({"GITHUB_OUTPUT": str(output)}):
pending.append_output(result)
self.assertIn(f"canary_bootstrap={str(bootstrap).lower()}\n", output.read_text())
allowed = package_artifact.read_pending_allowed(root, "sv", "sv", 0, 1)
self.assertEqual({f"docs/sv/{page}" for page in expected} | {"docs/.i18n/sv.tm.jsonl"}, allowed)
for source in result.shard_files:
relative = source.relative_to(docs)
translated = root / "translated.md"
translated.write_text(self._translated_page(source, "# Svensk sida\n"))
with chdir(root):
self.assertEqual((True, ""), apply_artifacts.should_apply_changed(True, "sv", f"docs/sv/{relative.as_posix()}", translated))
source.write_text(source.read_text() + "Changed source\n")
self.assertFalse(apply_artifacts.should_apply_changed(True, "sv", f"docs/sv/{relative.as_posix()}", translated)[0])

def test_publishing_canary_requires_bootstrap_sources_and_diagnostics_do_not_expand(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
docs = root / "docs"
(docs / "channels").mkdir(parents=True)
(docs / "channels/line.md").write_text("# LINE\n")
args = dict(docs_root=docs, openclaw_sync_dir=root / ".openclaw-sync", locale="sv", locale_slug="sv", shard_index=0, shard_total=1, pending_limit=1, canary_source_path="channels/line.md")
diagnostic = pending.build_pending_manifest(**args)
self.assertEqual(1, diagnostic.pending_count)
self.assertFalse(diagnostic.canary_bootstrap)
with self.assertRaisesRegex(SystemExit, "not pending in this shard: index.md"):
pending.build_pending_manifest(**args, canary_publish_required=True)

def test_publishing_canary_deploy_scope_refreshes_new_locale_menus(self) -> None:
workflow = (REPO_ROOT / ".github/workflows/translate-locale-reusable.yml").read_text()
block = workflow.split(" - name: Dispatch locale docs deploy\n", 1)[1].split(" - name:", 1)[0]
script = "\n".join(line[10:] for line in block.split(" run: |\n", 1)[1].splitlines())
values = {"github.ref_name": "main", "inputs.locale": "sv", "inputs.canary_live_path": "channels/line", "inputs.canary_publish_required": "true"}
script = workflow_shell_check.GITHUB_EXPRESSION_RE.sub(lambda match: values[match.group()[3:-2].strip()], script)
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
(root / "python").symlink_to(sys.executable)
(root / "dispatch_r2_pages.py").write_text("import json, os, sys\nfrom pathlib import Path\nPath(os.environ['CAPTURE']).write_text(json.dumps(sys.argv[1:]))\n")
for bootstrap, scope in (("false", "page"), ("true", "full")):
with self.subTest(bootstrap=bootstrap):
process_env = {**os.environ, "PATH": f"{root}{os.pathsep}{os.environ['PATH']}", "I18N_SCRIPT_DIR": str(root), "CAPTURE": str(root / "args.json"), "CANARY_BOOTSTRAP": bootstrap, "ARTIFACT_ROLE": "canary", "CANARY_LIVE_URL": "https://docs.openclaw.ai/sv/channels/line", "CANARY_EXPECTED_H1": "LINE"}
result = subprocess.run(["/bin/bash", "-eu", "-c", script], env=process_env, text=True, capture_output=True)
self.assertEqual(0, result.returncode, result.stderr)
args = json.loads((root / "args.json").read_text())
self.assertEqual(scope, args[args.index("--artifact-scope") + 1])
self.assertEqual("sv", args[args.index("--locale") + 1])
self.assertEqual(scope == "page", "--page-path" in args)
self.assertEqual("https://docs.openclaw.ai/sv/channels/line", args[args.index("--live-url") + 1])
self.assertNotIn("--no-wait", args)

def test_pending_manifest_canary_rejects_duplicate_configured_source_pages(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
tmp_path = Path(tmp)
Expand Down
1 change: 1 addition & 0 deletions .github/scripts/i18n/translation_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
("vi", "vi"),
("pl", "pl"),
("nl", "nl"),
("sv", "sv"),
("uk", "uk"),
("th", "th"),
("ar", "ar"),
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/translate-all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ on:
- it
- vi
- nl
- sv
- fa
- ru
- tr
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/translate-incremental.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ on:
- "!docs/pl/**"
- "!docs/pt-BR/**"
- "!docs/ru/**"
- "!docs/sv/**"
- "!docs/th/**"
- "!docs/tr/**"
- "!docs/uk/**"
Expand Down
Loading
Loading