diff --git a/.ci/unit-tests/python/test_py2cs/test_build_plan.py b/.ci/unit-tests/python/test_py2cs/test_build_plan.py new file mode 100644 index 0000000..4052b98 --- /dev/null +++ b/.ci/unit-tests/python/test_py2cs/test_build_plan.py @@ -0,0 +1,144 @@ +"""The build plan of Build-Repositories.ps1: the key of each repository, the build order and the restore from the build cache, with no network and no build.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +REPO = Path(__file__).resolve().parents[4] +SCRIPT = REPO / ".github" / "scripts" / "Build-Repositories.ps1" +SHELL = shutil.which("pwsh") or shutil.which("powershell") +COMMITS = {"BHoM/BHoM": "a" * 40, "BHoM/BHoM_Engine": "b" * 40, "BHoM/JSONSchema_Toolkit": "c" * 40, "BHoM/CSharpSchema_Test": "d" * 40} +DEPENDENCIES = {"BHoM_Engine": ["BHoM/BHoM"], "JSONSchema_Toolkit": ["BHoM/BHoM", "BHoM/BHoM_Engine", "BHoM/Unlisted_Toolkit"], "CSharpSchema_Test": ["BHoM/BHoM"]} + +pytestmark = pytest.mark.skipif(SHELL is None, reason="PowerShell is not installed") + +def run(folder: Path, *arguments: str) -> subprocess.CompletedProcess[str]: + """Runs Build-Repositories.ps1 in the folder with the given arguments, without the GitHub output file of an enclosing workflow step. + + Every test here uses it, and fails when the script fails. + """ + environment = {key: value for key, value in os.environ.items() if key != "GITHUB_OUTPUT"} + result = subprocess.run([str(SHELL), "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", str(SCRIPT), *arguments], + cwd=folder, env=environment, capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stdout + result.stderr + return result + +def plan(folder: Path, commits: dict[str, str] | None = None, os_name: str = "Windows", tools: str = "tools 1", salt: str = "v1") -> dict[str, Any]: + """Writes the list, the commits and the dependencies.txt files into the folder, runs the plan and returns it as read from the plan file. + + The tests compare the keys of plans made from small changes to these inputs. + """ + commits = commits or COMMITS + (folder / "list.txt").write_text("\n".join(commits) + "\n", encoding="utf-8") + (folder / "commits.txt").write_text("".join(f"{repository} {commit}\n" for repository, commit in commits.items()), encoding="utf-8") + for name, dependencies in DEPENDENCIES.items(): + (folder / "dependencies" / name).mkdir(parents=True, exist_ok=True) + (folder / "dependencies" / name / "dependencies.txt").write_text("\n".join(dependencies) + "\n", encoding="utf-8") + run(folder, "-List", "list.txt", "-Plan", "-PlanFile", "plan.json", "-CommitsFile", "commits.txt", "-DependenciesRoot", "dependencies", + "-OS", os_name, "-ToolsVersion", tools, "-Salt", salt) + return json.loads((folder / "plan.json").read_text(encoding="utf-8-sig")) + +def keys(built: dict[str, Any]) -> dict[str, str]: + """Returns the key of each repository of a plan, by name.""" + return {repository["name"]: repository["key"] for repository in built["repositories"]} + +def test_the_order_puts_each_repository_after_its_listed_dependencies(tmp_path: Path): + """Checks that the plan orders the repositories after their dependencies, keeps only the listed dependencies, and names each entry after the repository and its key. + + Protects the build, where a repository reads its dependencies' assemblies from the installation. + """ + built = plan(tmp_path) + order = [repository["name"] for repository in built["repositories"]] + assert order.index("BHoM") < order.index("BHoM_Engine") < order.index("JSONSchema_Toolkit") + assert order.index("BHoM") < order.index("CSharpSchema_Test") + toolkit = next(r for r in built["repositories"] if r["name"] == "JSONSchema_Toolkit") + assert toolkit["dependencies"] == ["BHoM", "BHoM_Engine"], "an unlisted dependency is left out" + assert all(r["entry"] == f"{r['name']}-{r['key'][:16]}" for r in built["repositories"]) + +def test_a_new_commit_changes_the_key_of_the_repository_and_of_what_depends_on_it_only(tmp_path: Path): + """Checks that a new commit in one repository changes exactly its own key and the keys of the repositories that depend on it, directly or not. + + Protects the point of the cache: a change rebuilds only what it can affect. + """ + (tmp_path / "before").mkdir() + before = keys(plan(tmp_path / "before")) + cases = { + "BHoM/CSharpSchema_Test": {"CSharpSchema_Test"}, + "BHoM/BHoM_Engine": {"BHoM_Engine", "JSONSchema_Toolkit"}, + "BHoM/BHoM": {"BHoM", "BHoM_Engine", "JSONSchema_Toolkit", "CSharpSchema_Test"}, + } + for repository, expected in cases.items(): + folder = tmp_path / repository.split("/")[1] + folder.mkdir() + after = keys(plan(folder, {**COMMITS, repository: "e" * 40})) + assert {name for name in before if before[name] != after[name]} == expected, repository + +def test_the_salt_the_os_and_the_tools_change_every_key(tmp_path: Path): + """Checks that a new salt, OS or build tools change the key of every repository, and that only the tools leave the state prefix unchanged. + + Protects the cache from restoring what other tools built, and the refresh gate, which looks the state up on ubuntu without the Windows tools. + """ + (tmp_path / "base").mkdir() + base = plan(tmp_path / "base") + for name, change in (("salt", {"salt": "v2"}), ("os", {"os_name": "Linux"}), ("tools", {"tools": "tools 2"})): + (tmp_path / name).mkdir() + changed = plan(tmp_path / name, **change) + assert all(keys(base)[r] != keys(changed)[r] for r in keys(base)), name + assert changed["cacheKey"] != base["cacheKey"], name + assert (changed["statePrefix"] == base["statePrefix"]) == (name == "tools"), name + assert base["cacheKey"].startswith(base["statePrefix"]) and base["statePrefix"].startswith(base["cachePrefix"]) + assert base["cachePrefix"] == "bhom-build-cache-v1-Windows-" + +def test_only_the_msbuild_major_minor_and_the_sdk_feature_band_change_the_keys(tmp_path: Path): + """Checks that a new patch version of MSBuild or of the .NET SDK keeps every key, while a new MSBuild minor version or SDK feature band changes them. + + Protects the cache from a full rebuild whenever a re-run lands on a runner whose image is a patch ahead or behind. + """ + cases = { + "base": "msbuild 18.11.1.51303, dotnet 10.0.402", + "patch": "msbuild 18.11.2.10001, dotnet 10.0.405", + "msbuild": "msbuild 18.12.0.1, dotnet 10.0.402", + "band": "msbuild 18.11.1.51303, dotnet 10.0.300", + } + built = {} + for name, tools in cases.items(): + (tmp_path / name).mkdir() + built[name] = plan(tmp_path / name, tools=tools) + assert built["base"]["tools"] == "msbuild 18.11, dotnet 10.0.4xx" + assert built["patch"]["cacheKey"] == built["base"]["cacheKey"] + assert built["msbuild"]["cacheKey"] != built["base"]["cacheKey"] + assert built["band"]["cacheKey"] != built["base"]["cacheKey"] + +@pytest.mark.skipif(sys.platform != "win32", reason="the restore copies with robocopy") +def test_a_run_with_every_entry_cached_copies_them_clones_nothing_and_prunes_old_entries(tmp_path: Path): + """Checks that when the folder holds every planned entry, the build copies them into the installation in build order, clones and builds nothing, deletes the entries of other keys, and writes commits.txt. + + Protects the fast path of the PR checks and the size of the cache. + """ + built = plan(tmp_path) + cache = tmp_path / "cache" + for repository in built["repositories"]: + assemblies = cache / repository["entry"] / "Assemblies" + assemblies.mkdir(parents=True) + (assemblies / f"{repository['name']}.dll").write_text(repository["name"], encoding="utf-8") + # Shared by two entries: the one built later wins, as in a real build + (assemblies / "Shared.dll").write_text(repository["name"], encoding="utf-8") + (cache / "BHoM-0000000000000000").mkdir() + result = run(tmp_path, "-List", "list.txt", "-PlanFile", "plan.json", "-CacheFolder", "cache", "-BHoMRoot", "installation", "-ReposRoot", "repos") + installation = tmp_path / "installation" + assert all((installation / "Assemblies" / f"{name}.dll").exists() for name in keys(built)) + assert (installation / "Assemblies" / "Shared.dll").read_text(encoding="utf-8") == built["repositories"][-1]["name"] + assert all((installation / folder).is_dir() for folder in ("Datasets", "Upgrades", "Settings", "Developer")) + assert sorted(p.name for p in cache.iterdir()) == sorted(r["entry"] for r in built["repositories"]) + assert [p.name for p in (tmp_path / "repos").iterdir()] == ["commits.txt"], "nothing is cloned" + commits = (tmp_path / "repos" / "commits.txt").read_text(encoding="utf-8").splitlines() + assert commits == [f"{r['repository']} {r['commit']}" for r in built["repositories"]] + assert "Built: nothing" in result.stdout diff --git a/.ci/unit-tests/python/test_py2cs/test_workflows.py b/.ci/unit-tests/python/test_py2cs/test_workflows.py index c1ee923..d684c1f 100644 --- a/.ci/unit-tests/python/test_py2cs/test_workflows.py +++ b/.ci/unit-tests/python/test_py2cs/test_workflows.py @@ -36,6 +36,7 @@ def test_all_workflows_are_valid_and_have_the_expected_jobs(): assert list(load("pr-checks.yml")["jobs"]) == ["checks", "alignment"] assert list(load("linked-csharp-pr.yml")["jobs"]) == ["linked"], "one job, so the pull request page shows one line" assert list(load("linked-csharp-pr-close.yml")["jobs"]) == ["close"] + assert list(load("build-cache.yml")["jobs"]) == ["gate", "refresh"] def test_the_weekly_generation_commit_cannot_start_the_linked_pr_workflow(): """Checks that neither linked-PR workflow has a push or schedule trigger, that they run for pull requests only on src/bhom_schema paths, and that only the close workflow reacts to a closed pull request. @@ -70,11 +71,13 @@ def test_workflows_start_with_read_only_permissions_and_raise_them_per_job(): Protects the token permissions of the workflows. """ - for name in ("linked-csharp-pr.yml", "linked-csharp-pr-close.yml", "pr-checks.yml"): + for name in ("linked-csharp-pr.yml", "linked-csharp-pr-close.yml", "pr-checks.yml", "build-cache.yml"): assert load(name)["permissions"] == {"contents": "read"}, name assert load("linked-csharp-pr.yml")["jobs"]["linked"]["permissions"] == {"contents": "read", "pull-requests": "write"} assert "permissions" not in load("linked-csharp-pr-close.yml")["jobs"]["close"] + assert load("build-cache.yml")["jobs"]["gate"]["permissions"] == {"contents": "read", "actions": "read"} + assert "permissions" not in load("build-cache.yml")["jobs"]["refresh"] def test_the_app_token_is_made_only_for_the_repositories_the_change_touches(): @@ -152,13 +155,14 @@ def test_every_listed_converted_repository_is_also_built(): assert all(r.lower() in built for r in lines("Converted-repositories.txt")) def test_the_generation_and_the_checks_build_the_same_repositories(): - """Checks that the generation and PR checks workflows clone and build the repositories from the same list. + """Checks that the generation and PR checks workflows build the repositories from the same list, the generation from its own clones and the PR checks through the build cache. Protects align, which compares the PR with what the generator would write. """ - for path in (WORKFLOWS / "python-schema-generation.yml", WORKFLOWS.parent / "actions" / "bhom-assemblies" / "action.yml"): - text = path.read_text(encoding="utf-8") - assert "Clone-Repositories.ps1 -List Included-repositories.txt" in text and "Build-Repositories.ps1 -List Included-repositories.txt" in text + generation = (WORKFLOWS / "python-schema-generation.yml").read_text(encoding="utf-8") + assert "Clone-Repositories.ps1 -List Included-repositories.txt" in generation and "Build-Repositories.ps1 -List Included-repositories.txt" in generation + action = (WORKFLOWS.parent / "actions" / "bhom-assemblies" / "action.yml").read_text(encoding="utf-8") + assert "Build-Repositories.ps1 -List Included-repositories.txt" in action and 'List = "Included-repositories.txt"' in action def test_the_release_workflow_is_manual_and_only_the_publish_job_gets_an_identity_token(): """Checks that the release workflow starts only by hand, that contents are read-only by default, and that only the TestPyPI job can ask for an identity token. @@ -189,22 +193,67 @@ def test_the_release_workflow_stores_no_secret_and_nothing_is_published_in_a_dry assert publishing and all(s["if"] == "${{ !inputs.dry_run }}" for s in publishing) -def test_the_pr_checks_build_only_when_the_cache_of_assemblies_misses(): - """Checks that the shared assemblies action caches the built BHoM assemblies under a key made from the current commits, builds only on a cache miss, and saves the cache right after the build. +def test_the_assemblies_action_plans_restores_builds_and_saves_in_that_order(): + """Checks that the shared assemblies action plans the build first, restores the build cache by its exact key then by its prefix, builds from the saved plan with the cache folder, and saves under the exact key only when an entry was added. - Protects the speed of the PR checks from a change that rebuilds every time, and the cache from becoming stale. + Protects the speed of the PR checks from a change that rebuilds every time, and the cache from being saved under a key that does not match its content. """ action = yaml.safe_load((WORKFLOWS.parent / "actions" / "bhom-assemblies" / "action.yml").read_text(encoding="utf-8")) steps = action["runs"]["steps"] by_name = {s.get("name", s.get("uses", "")): s for s in steps} - restore = by_name["Restore the built BHoM assemblies"] - assert restore["with"]["key"] == "${{ steps.commits.outputs.key }}" - for name in ("Clone the repositories", "Build the repositories in dependency order", "Save the built BHoM assemblies"): - assert by_name[name]["if"] == "steps.assemblies.outputs.cache-hit != 'true'" - assert "git ls-remote" in by_name["Find the current commit of each repository"]["run"] - order = [s.get("name", "") for s in steps] - assert order.index("Build the repositories in dependency order") < order.index("Save the built BHoM assemblies") + order = list(by_name) + assert order.index("microsoft/setup-msbuild@v2") < order.index("Plan the build"), "the MSBuild version is part of the keys" + assert order.index("Plan the build") < order.index("Restore the build cache") < order.index("Build the repositories in dependency order") < order.index("Save the build cache") + assert "-Plan -PlanFile" in by_name["Plan the build"]["run"] + restore = by_name["Restore the build cache"] + assert restore["uses"].startswith("actions/cache/restore@") + assert restore["with"] == {"path": ".build-cache", "key": "${{ steps.plan.outputs.key }}", "restore-keys": "${{ steps.plan.outputs.prefix }}"} + build = by_name["Build the repositories in dependency order"]["run"] + assert 'PlanFile = "$env:RUNNER_TEMP/build-plan.json"' in build and 'CacheFolder = ".build-cache"' in build, "the build uses the planned commits" + save = by_name["Save the build cache"] + assert save["uses"].startswith("actions/cache/save@") and save["if"] == "steps.build.outputs.added != ''" + assert save["with"] == {"path": ".build-cache", "key": "${{ steps.plan.outputs.key }}"} + +def test_the_alignment_job_waits_for_the_tests_and_still_runs_when_they_fail(): + """Checks that the C# alignment job needs the build and tests job and runs unless the run is cancelled. + + Protects the PR checks from building the same changed C# repository twice, and the alignment result from being skipped when a test fails. + """ + alignment = load("pr-checks.yml")["jobs"]["alignment"] + assert alignment["needs"] == "checks" + assert alignment["if"] == "${{ !cancelled() }}" + +def test_the_build_cache_refresh_runs_after_the_pr_checks_and_builds_from_develop(): + """Checks that the refresh workflow starts only after the PR workflow or by hand, that its gate rejects forks and manual runs off develop and looks the current state up on develop, and that the refresh builds from develop. + Protects the develop cache, which every pull request reads, from what a pull request or a fork built. + """ + workflow = load("build-cache.yml") + assert set(workflow["on"]) == {"workflow_run", "workflow_dispatch"} + assert workflow["on"]["workflow_run"] == {"workflows": ["PR"], "types": ["completed"]} + assert load("pr-checks.yml")["name"] == "PR" + assert workflow["concurrency"]["cancel-in-progress"] is False + gate = workflow["jobs"]["gate"] + assert "github.event.workflow_run.head_repository.full_name == github.repository" in gate["if"] + assert "github.ref == 'refs/heads/develop'" in gate["if"] + assert gate["runs-on"] == "ubuntu-latest" + assert not any("bhom-assemblies" in str(s.get("uses", "")) or "actions/cache" in str(s.get("uses", "")) for s in gate["steps"]), "the gate writes no cache" + lookup = next(s for s in gate["steps"] if s.get("id") == "lookup") + assert "--ref refs/heads/develop" in lookup["run"] and "$env:STATE_PREFIX" in lookup["run"] + refresh = workflow["jobs"]["refresh"] + assert refresh["needs"] == "gate" and refresh["if"] == "needs.gate.outputs.refresh == 'true'" + checkout = next(s for s in refresh["steps"] if str(s.get("uses", "")).startswith("actions/checkout")) + assert checkout["with"]["ref"] == "develop" + assert any(s.get("uses") == "./.github/actions/bhom-assemblies" for s in refresh["steps"]) + +def test_the_release_and_the_generation_build_clean(): + """Checks that the release and generation workflows do not use the build cache yet. + + Protects the published wheel and the weekly commit to develop from a cache that has not proved itself on the PR checks. + """ + for name in ("release.yml", "python-schema-generation.yml"): + text = (WORKFLOWS / name).read_text(encoding="utf-8") + assert "bhom-assemblies" not in text and "actions/cache" not in text and "-CacheFolder" not in text, name def test_the_alignment_check_is_its_own_job_with_a_name_that_says_what_is_pending(): """Checks that the C# alignment runs as its own job of pr-checks, named after the linked C# PR, using the shared assemblies action and explaining a failure in the job summary, and that the main job no longer runs align. diff --git a/.github/actions/bhom-assemblies/action.yml b/.github/actions/bhom-assemblies/action.yml index ca11dff..b0b406d 100644 --- a/.github/actions/bhom-assemblies/action.yml +++ b/.github/actions/bhom-assemblies/action.yml @@ -1,60 +1,56 @@ name: Built BHoM assemblies description: > - Installs the built assemblies of the repositories in Included-repositories.txt into C:\ProgramData\BHoM\Assemblies, from a cache when none of them - has a new commit, otherwise by cloning and building them (and saving the cache). + Installs the built assemblies of the repositories in Included-repositories.txt into C:\ProgramData\BHoM, building only the repositories whose + commit, or whose dependencies' commits, changed since the build cache was saved. + +inputs: + no-cache: + description: Ignore the build cache and build every repository + default: 'false' + +outputs: + built: + description: The names of the repositories that were built, separated by commas (empty when all came from the cache) + value: ${{ steps.build.outputs.built }} runs: using: composite steps: - # The built assemblies only change when one of the cloned repositories does, so they are cached under the current commit of each - # repository (the same branch choice as Clone-Repositories.ps1). A new commit anywhere gives a new key, so the cache is never stale. - - name: Find the current commit of each repository - id: commits - shell: pwsh - run: | - $lines = foreach ($repository in (Get-Content Included-repositories.txt | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" -and -not $_.StartsWith("#") })) { - $commit = "" - foreach ($branch in "develop", "main", "master") { - $found = git ls-remote "https://github.com/$repository.git" "refs/heads/$branch" - if ($found) { $commit = ($found -split "\s+")[0]; break } - } - - if (-not $commit) { throw "Could not find a develop, main or master branch of $repository" } - "$repository $commit" - } - - $text = $lines -join "`n" - Write-Host $text - $hash = [System.BitConverter]::ToString([System.Security.Cryptography.SHA256]::Create().ComputeHash([System.Text.Encoding]::UTF8.GetBytes($text))).Replace("-", "").ToLower() - "key=bhom-assemblies-v1-$hash" >> $env:GITHUB_OUTPUT - - - name: Restore the built BHoM assemblies - id: assemblies - uses: actions/cache/restore@v4 - with: - path: C:\ProgramData\BHoM\Assemblies - key: ${{ steps.commits.outputs.key }} - + # Before the plan, because the MSBuild version is part of the keys - uses: microsoft/setup-msbuild@v2 - if: steps.assemblies.outputs.cache-hit != 'true' - uses: NuGet/setup-nuget@v2 - if: steps.assemblies.outputs.cache-hit != 'true' - - name: Clone the repositories - if: steps.assemblies.outputs.cache-hit != 'true' + # The current commit and dependencies of each repository give it a key; the plan is saved so the build uses the same commits + - name: Plan the build + id: plan shell: pwsh - run: ./.github/scripts/Clone-Repositories.ps1 -List Included-repositories.txt + run: ./.github/scripts/Build-Repositories.ps1 -List Included-repositories.txt -Plan -PlanFile "$env:RUNNER_TEMP/build-plan.json" + + # The exact key means nothing changed; otherwise the most recent folder gives the entries that are still current + - name: Restore the build cache + if: inputs.no-cache != 'true' + uses: actions/cache/restore@v4 + with: + path: .build-cache + key: ${{ steps.plan.outputs.key }} + restore-keys: ${{ steps.plan.outputs.prefix }} - name: Build the repositories in dependency order - if: steps.assemblies.outputs.cache-hit != 'true' + id: build shell: pwsh - run: ./.github/scripts/Build-Repositories.ps1 -List Included-repositories.txt + env: + NO_CACHE: ${{ inputs.no-cache }} + run: | + $arguments = @{ List = "Included-repositories.txt"; PlanFile = "$env:RUNNER_TEMP/build-plan.json"; CacheFolder = ".build-cache" } + if ($env:NO_CACHE -eq "true") { $arguments.NoCache = $true } + + ./.github/scripts/Build-Repositories.ps1 @arguments # Saved right after the build, so a failure later in the job does not lose it - - name: Save the built BHoM assemblies - if: steps.assemblies.outputs.cache-hit != 'true' + - name: Save the build cache + if: steps.build.outputs.added != '' uses: actions/cache/save@v4 with: - path: C:\ProgramData\BHoM\Assemblies - key: ${{ steps.commits.outputs.key }} + path: .build-cache + key: ${{ steps.plan.outputs.key }} diff --git a/.github/scripts/Build-Repositories.ps1 b/.github/scripts/Build-Repositories.ps1 index 29643f2..f8edf57 100644 --- a/.github/scripts/Build-Repositories.ps1 +++ b/.github/scripts/Build-Repositories.ps1 @@ -1,64 +1,353 @@ -# Builds the cloned repositories in dependency order, with the same build.ps1 as the BHoM_JSONSchema weekly job. +# Builds the listed repositories in dependency order, with the same build.ps1 as the BHoM_JSONSchema weekly job. # # Build-Repositories.ps1 -List Included-repositories.txt [-ReposRoot .repos] [-OrderOnly] +# Build-Repositories.ps1 -List Included-repositories.txt -Plan [-PlanFile plan.json] [-CacheFolder .build-cache] +# Build-Repositories.ps1 -List Included-repositories.txt -CacheFolder .build-cache [-PlanFile plan.json] [-NoCache] # -# The order comes from the dependencies.txt of each repository (lines like BHoM/BHoM), among the listed repositories only. Building a -# BHoM repository copies its assemblies to %ProgramData%\BHoM\Assemblies, where the next repository and the generator find them. -# With -OrderOnly the order is printed and nothing is built. +# Without -Plan or -CacheFolder it builds the clones already in (see Clone-Repositories.ps1). The order comes from the +# dependencies.txt of each repository (lines like BHoM/BHoM), among the listed repositories only. Building a BHoM repository copies its +# assemblies to %ProgramData%\BHoM\Assemblies, where the next repository and the generator find them. With -OrderOnly the order is printed +# and nothing is built. +# +# -Plan reads the current commit of each repository (develop, else main, else master) and its dependencies.txt at that commit, without +# cloning, and gives each repository a key made from its commit, the keys of its dependencies and the build tools. It prints the plan, +# writes it to -PlanFile, and stops. +# +# -CacheFolder runs the plan (read from -PlanFile when that file exists). A repository whose entry - is in the folder is copied +# from it; any other is cloned at the planned commit and built, and the files its build added or changed under %ProgramData%\BHoM become +# its entry. Entries of other keys are deleted, and /commits.txt is written. -NoCache builds every repository and leaves the +# folder alone. param( [Parameter(Mandatory = $true)][string]$List, [string]$ReposRoot = ".repos", - [switch]$OrderOnly + [switch]$OrderOnly, + [switch]$Plan, + [string]$PlanFile, + [string]$CacheFolder, + [switch]$NoCache, + # For tests: "Owner/Name " lines instead of git ls-remote + [string]$CommitsFile, + # For tests: /dependencies.txt files instead of the ones on GitHub + [string]$DependenciesRoot, + # The build tools in the keys; read from msbuild and dotnet when not given + [string]$ToolsVersion, + # The OS of the runner that builds, so that a plan made on ubuntu gives the keys of the Windows build + [string]$OS = $(if ($env:RUNNER_OS) { $env:RUNNER_OS } else { "Windows" }), + [string]$BHoMRoot = $(if ($env:ProgramData) { Join-Path $env:ProgramData "BHoM" } else { "" }), + # Bump to throw every cache away + [string]$Salt = "v1" ) $ErrorActionPreference = "Stop" +$buildScript = Join-Path $PSScriptRoot "../workflows/build.ps1" + +function Read-Lines($lines) { + @($lines | ForEach-Object { "$_".Trim() } | Where-Object { $_ -ne "" -and -not $_.StartsWith("#") }) +} + +function Get-Hash([string]$text) { + $sha = [System.Security.Cryptography.SHA256]::Create() + [System.BitConverter]::ToString($sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($text))).Replace("-", "").ToLower() +} + +function Invoke-WithRetry([scriptblock]$body, [string]$what) { + for ($attempt = 1; ; $attempt++) { + try { + return & $body + } catch { + if ($attempt -ge 3) { throw "$what failed after 3 attempts: $_" } -$names = Get-Content $List | ForEach-Object { $_.Trim() } | Where-Object { $_ -ne "" -and -not $_.StartsWith("#") } | - ForEach-Object { $_.Split("/")[-1] } | Select-Object -Unique - -$dependencies = @{} -foreach ($name in $names) { - $dependencies[$name] = @() - $file = Join-Path $ReposRoot "$name/dependencies.txt" - if (Test-Path $file) { - $dependencies[$name] = @(Get-Content $file | ForEach-Object { $_.Trim() } | - Where-Object { $_ -ne "" -and -not $_.StartsWith("#") } | - ForEach-Object { $_.Split("/")[-1] } | - Where-Object { $names -contains $_ -and $_ -ne $name }) + Write-Host "$what failed, retrying: $_" + Start-Sleep -Seconds (5 * $attempt) + } } } -$script:order = @() -$script:state = @{} +# The listed dependencies of one repository, by name, among the listed repositories only +function Select-Dependencies([string]$name, $lines) { + @(Read-Lines $lines | ForEach-Object { $_.Split("/")[-1] } | Where-Object { $script:names -contains $_ -and $_ -ne $name } | Select-Object -Unique) +} + +function Get-Order($dependencies) { + $script:order = @() + $script:state = @{} + foreach ($name in $script:names) { Visit $name $dependencies } + + $script:order +} -function Visit([string]$name) { +function Visit([string]$name, $dependencies) { if ($script:state[$name] -eq "done") { return } if ($script:state[$name] -eq "visiting") { throw "Circular dependency involving $name" } + $script:state[$name] = "visiting" - foreach ($dependency in $dependencies[$name]) { Visit $dependency } + foreach ($dependency in $dependencies[$name]) { Visit $dependency $dependencies } + $script:state[$name] = "done" $script:order += $name } -foreach ($name in $names) { Visit $name } +function Get-Commit([string]$repository) { + $heads = Invoke-WithRetry { + $found = git ls-remote --heads "https://github.com/$repository.git" develop main master + if ($LASTEXITCODE -ne 0) { throw "git ls-remote exited with $LASTEXITCODE" } -Write-Host "Build order: $($script:order -join ', ')" -if ($OrderOnly) { return } + $found + } "Reading the branches of $repository" -$buildScript = Join-Path $PSScriptRoot "../workflows/build.ps1" -foreach ($name in $script:order) { - Push-Location (Join-Path $ReposRoot $name) + foreach ($branch in "develop", "main", "master") { + $line = @($heads | Where-Object { $_ -match "\srefs/heads/$branch$" }) | Select-Object -First 1 + if ($line) { return ($line -split "\s+")[0] } + } + + throw "Could not find a develop, main or master branch of $repository" +} + +function Get-DependencyLines([string]$repository, [string]$commit) { + if ($DependenciesRoot) { + $file = Join-Path $DependenciesRoot "$($repository.Split('/')[-1])/dependencies.txt" + if (Test-Path $file) { return Get-Content $file } + + return @() + } + + $url = "https://raw.githubusercontent.com/$repository/$commit/dependencies.txt" + Invoke-WithRetry { + try { + (Invoke-WebRequest $url -UseBasicParsing).Content -split "`r?`n" + } catch { + $status = 0 + if ($_.Exception.Response) { $status = [int]$_.Exception.Response.StatusCode } + + # A repository without the file has no dependencies + if ($status -ne 404) { throw } + + @() + } + } "Reading dependencies.txt of $repository" +} + +function Get-ToolsVersion { + $versions = @() + if (Get-Command msbuild -ErrorAction SilentlyContinue) { $versions += "msbuild " + "$(msbuild -nologo -version | Select-Object -Last 1)".Trim() } + + if (Get-Command dotnet -ErrorAction SilentlyContinue) { $versions += "dotnet " + "$(dotnet --version)".Trim() } + + if ($versions.Count -eq 0) { return "none" } + + $versions -join ", " +} + +# Hosted runners in the middle of an image rollout differ in patch versions, so only MSBuild major.minor and the SDK feature band count +function Get-ToolsName([string]$tools) { + $tools = $tools -replace "msbuild (\d+\.\d+)[\d.]*", 'msbuild $1' + $tools -replace "dotnet (\d+\.\d+\.\d)\d*[^\s,]*", 'dotnet $1xx' +} + +function New-BuildPlan { + $commits = @{} + if ($CommitsFile) { + foreach ($line in (Read-Lines (Get-Content $CommitsFile))) { + $parts = $line -split "\s+" + $commits[$parts[0]] = $parts[1] + } + } + + foreach ($repository in $script:repositories) { + if (-not $commits[$repository]) { $commits[$repository] = Get-Commit $repository } + } + + $dependencies = @{} + foreach ($repository in $script:repositories) { + $name = $repository.Split("/")[-1] + $dependencies[$name] = @(Select-Dependencies $name (Get-DependencyLines $repository $commits[$repository])) + } + + $tools = $ToolsVersion + if (-not $tools) { $tools = Get-ToolsVersion } + + $tools = Get-ToolsName $tools + + # Line endings depend on the checkout, so they are not part of the hash + $toolsHash = Get-Hash "$tools|$((Get-Content $buildScript -Raw).Replace("`r`n", "`n"))" + + # The source key is the plan's key without the tools, so that ubuntu can find the state of a Windows build + $sourceKeys = @{} + $entries = @() + foreach ($name in (Get-Order $dependencies)) { + $repository = $script:fullNames[$name] + $dependencyKeys = @($dependencies[$name] | ForEach-Object { $sourceKeys[$_] } | Sort-Object) + $sourceKeys[$name] = Get-Hash ((@($Salt, $OS, $repository, $commits[$repository]) + $dependencyKeys) -join "|") + $key = Get-Hash "$($sourceKeys[$name])|$toolsHash" + $entries += [pscustomobject]@{ + repository = $repository + name = $name + commit = $commits[$repository] + dependencies = @($dependencies[$name]) + key = $key + entry = "$name-$($key.Substring(0, 16))" + } + } + + $state = Get-Hash ((@($sourceKeys.Values) | Sort-Object) -join "|") + $prefix = "bhom-build-cache-$Salt-$OS-" + [pscustomobject]@{ + salt = $Salt + os = $OS + tools = $tools + cacheKey = "$prefix$($state.Substring(0, 16))-$($toolsHash.Substring(0, 16))" + cachePrefix = $prefix + statePrefix = "$prefix$($state.Substring(0, 16))-" + repositories = $entries + } +} + +function Set-StepOutput([string]$name, [string]$value) { + if ($env:GITHUB_OUTPUT) { Add-Content -Path $env:GITHUB_OUTPUT -Value "$name=$value" } +} + +function Build-Repository([string]$name, [string]$folder) { + Push-Location $folder try { Write-Host "Building $name..." $solution = Get-ChildItem -Filter *.sln | Select-Object -First 1 - if ($solution) { - & $buildScript -SolutionOrProjectPath $solution.FullName - } else { - foreach ($project in Get-ChildItem -Recurse -Filter *.csproj) { - & $buildScript -SolutionOrProjectPath $project.FullName - } + $paths = if ($solution) { @($solution.FullName) } else { @(Get-ChildItem -Recurse -Filter *.csproj | ForEach-Object { $_.FullName }) } + foreach ($path in $paths) { + & $buildScript -SolutionOrProjectPath $path + if ($LASTEXITCODE -ne 0) { throw "Building $name failed (exit code $LASTEXITCODE)" } } } finally { Pop-Location } } + +# Clones only the planned commit, so that a push during the run cannot make the build differ from its key +function Get-Clone([string]$repository, [string]$commit, [string]$folder) { + if (-not (Test-Path (Join-Path $folder ".git"))) { + git init --quiet $folder + if ($LASTEXITCODE -ne 0) { throw "git init $folder failed" } + } + + Write-Host "Cloning $repository at $commit..." + Invoke-WithRetry { + git -C $folder fetch --depth 1 --quiet "https://github.com/$repository.git" $commit + if ($LASTEXITCODE -ne 0) { throw "git fetch exited with $LASTEXITCODE" } + } "Fetching $repository" + + git -C $folder checkout --quiet --detach FETCH_HEAD + if ($LASTEXITCODE -ne 0) { throw "Could not check out $repository at $commit" } +} + +# Relative path to "size|last write time" of every file under the folder +function Get-Snapshot([string]$root) { + $snapshot = @{} + $length = (Get-Item $root).FullName.TrimEnd("\", "/").Length + 1 + foreach ($file in (Get-ChildItem $root -Recurse -File -Force)) { + $snapshot[$file.FullName.Substring($length)] = "$($file.Length)|$($file.LastWriteTimeUtc.Ticks)" + } + + $snapshot +} + +function Copy-Folder([string]$source, [string]$target) { + robocopy $source $target /E /COPY:DAT /R:2 /W:1 /NFL /NDL /NJH /NJS /NP | Out-Null + if ($LASTEXITCODE -ge 8) { throw "Copying $source to $target failed (robocopy exit code $LASTEXITCODE)" } + + $global:LASTEXITCODE = 0 +} + +$script:repositories = @(Read-Lines (Get-Content $List) | Select-Object -Unique) +$script:names = @($script:repositories | ForEach-Object { $_.Split("/")[-1] }) +$script:fullNames = @{} +foreach ($repository in $script:repositories) { $script:fullNames[$repository.Split("/")[-1]] = $repository } + +if (-not $Plan -and -not $CacheFolder) { + $dependencies = @{} + foreach ($name in $script:names) { + $dependencies[$name] = @() + $file = Join-Path $ReposRoot "$name/dependencies.txt" + if (Test-Path $file) { $dependencies[$name] = @(Select-Dependencies $name (Get-Content $file)) } + } + + $order = Get-Order $dependencies + Write-Host "Build order: $($order -join ', ')" + if ($OrderOnly) { return } + + foreach ($name in $order) { Build-Repository $name (Join-Path $ReposRoot $name) } + + return +} + +if (-not $Plan -and $PlanFile -and (Test-Path $PlanFile)) { + $buildPlan = Get-Content $PlanFile -Raw | ConvertFrom-Json +} else { + $buildPlan = New-BuildPlan +} + +$useCache = $CacheFolder -and -not $NoCache +$buildPlan.repositories | ForEach-Object { + $cache = if (-not $CacheFolder) { "-" } elseif (-not $useCache) { "ignored" } elseif (Test-Path (Join-Path $CacheFolder $_.entry)) { "hit" } else { "miss" } + [pscustomobject]@{ Repository = $_.repository; Commit = $_.commit.Substring(0, 10); Entry = $_.entry; Cache = $cache } +} | Format-Table -AutoSize | Out-String | Write-Host +Write-Host "Cache key: $($buildPlan.cacheKey) (tools: $($buildPlan.tools))" +if ($env:ImageOS) { Write-Host "Runner image: $env:ImageOS $env:ImageVersion" } + + +if ($Plan) { + if ($PlanFile) { $buildPlan | ConvertTo-Json -Depth 5 | Set-Content $PlanFile -Encoding utf8 } + + Set-StepOutput "key" $buildPlan.cacheKey + Set-StepOutput "prefix" $buildPlan.cachePrefix + Set-StepOutput "state-prefix" $buildPlan.statePrefix + return +} + +# The empty folders that a build creates next to the assemblies +foreach ($folder in "Assemblies", "Datasets", "Upgrades", "Settings", "Developer") { + New-Item -ItemType Directory -Force (Join-Path $BHoMRoot $folder) | Out-Null +} + +New-Item -ItemType Directory -Force $ReposRoot | Out-Null +if ($useCache) { New-Item -ItemType Directory -Force $CacheFolder | Out-Null } + +$built = @() +foreach ($repository in $buildPlan.repositories) { + $entryFolder = Join-Path $CacheFolder $repository.entry + if ($useCache -and (Test-Path $entryFolder)) { + Write-Host "Restoring $($repository.name) from the cache..." + Copy-Folder $entryFolder $BHoMRoot + continue + } + + $clone = Join-Path $ReposRoot $repository.name + Get-Clone $repository.repository $repository.commit $clone + $before = Get-Snapshot $BHoMRoot + Build-Repository $repository.name $clone + $built += $repository.name + if (-not $useCache) { continue } + + # Comparing size and time is enough: a build writes its files anew + $after = Get-Snapshot $BHoMRoot + New-Item -ItemType Directory -Force $entryFolder | Out-Null + foreach ($path in $after.Keys) { + if ($before[$path] -eq $after[$path]) { continue } + + $target = Join-Path $entryFolder $path + New-Item -ItemType Directory -Force (Split-Path -Parent $target) | Out-Null + Copy-Item -LiteralPath (Join-Path $BHoMRoot $path) -Destination $target + } +} + +# Only the current heads can be needed again, so the folder never holds more than one entry per repository +if ($useCache) { + $kept = @($buildPlan.repositories | ForEach-Object { $_.entry }) + Get-ChildItem $CacheFolder -Directory | Where-Object { $kept -notcontains $_.Name } | ForEach-Object { + Write-Host "Deleting the old entry $($_.Name)" + Remove-Item -LiteralPath $_.FullName -Recurse -Force + } +} + +$buildPlan.repositories | ForEach-Object { "$($_.repository) $($_.commit)" } | Set-Content (Join-Path $ReposRoot "commits.txt") +Write-Host "Built: $(if ($built) { $built -join ', ' } else { 'nothing' })" +Set-StepOutput "built" ($built -join ",") +Set-StepOutput "added" $(if ($useCache) { $built -join "," } else { "" }) diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml new file mode 100644 index 0000000..b66ea47 --- /dev/null +++ b/.github/workflows/build-cache.yml @@ -0,0 +1,73 @@ +name: Build cache + +# Saves the build cache of the C# repositories on develop, where every pull request can read it. A cache saved by a pull request run is only +# visible to that pull request, so after a PR run this checks whether develop already has the current state, and builds it from develop when not. +# It never uses what the pull request built: the refresh clones and builds the C# repositories itself, with the code on develop. + +on: + workflow_run: + workflows: [PR] + types: [completed] + workflow_dispatch: + +permissions: + contents: read + +# The running refresh finishes, and a newer pending one replaces an older pending one +concurrency: + group: build-cache + cancel-in-progress: false + +jobs: + gate: + name: Does develop need a refresh? + # Never for a pull request from a fork, and only for develop when started by hand + if: >- + (github.event_name == 'workflow_run' && github.event.workflow_run.head_repository.full_name == github.repository) + || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/develop') + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + outputs: + refresh: ${{ steps.lookup.outputs.refresh }} + + steps: + - uses: actions/checkout@v4 + with: + ref: develop + + # The plan is made for the Windows build. Its state prefix leaves out the build tools, which only the Windows runner knows + - name: Plan the build + id: plan + shell: pwsh + run: ./.github/scripts/Build-Repositories.ps1 -List Included-repositories.txt -Plan -OS Windows -ToolsVersion ubuntu + + - name: Look for the current state on develop + id: lookup + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + STATE_PREFIX: ${{ steps.plan.outputs.state-prefix }} + MANUAL: ${{ github.event_name == 'workflow_dispatch' }} + run: | + $found = gh cache list --repo $env:GITHUB_REPOSITORY --ref refs/heads/develop --key $env:STATE_PREFIX --json key --jq '.[].key' + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + $refresh = $env:MANUAL -eq "true" -or -not $found + Write-Host $(if ($found) { "develop has: $found" } else { "develop has no entry starting with $env:STATE_PREFIX" }) + "refresh=$("$refresh".ToLower())" >> $env:GITHUB_OUTPUT + + refresh: + name: Build and save on develop + needs: gate + if: needs.gate.outputs.refresh == 'true' + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + with: + ref: develop + + - name: Install the built BHoM assemblies + uses: ./.github/actions/bhom-assemblies diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index b5a007e..73bb4ea 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -73,8 +73,11 @@ jobs: # Its own job, so that the pull request page names what is pending. The Python PR is a proposal: this only passes once the linked C# pull # request is merged, because the assemblies built here come from the C# develop branches. After that merge, re-run this job. + # It waits for the first job, which saves the build cache, so that a change in a C# repository is built once; it runs even when the tests fail. alignment: name: C# alignment (fails until the linked C# PR is merged) + needs: checks + if: ${{ !cancelled() }} runs-on: windows-latest env: PYTHONPATH: ${{ github.workspace }}/.ci/generation diff --git a/.gitignore b/.gitignore index 7157884..665fa92 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,5 @@ dist/ /.claude # Read-only clones made by the scripts .repos/ +# Build cache entries made by Build-Repositories.ps1 +.build-cache/ diff --git a/docs/Build-Cache-Plan.md b/docs/Build-Cache-Plan.md index be5a5b1..59c550a 100644 --- a/docs/Build-Cache-Plan.md +++ b/docs/Build-Cache-Plan.md @@ -1,7 +1,7 @@ -# Speeding up the PR checks with a build cache (for later) +# Speeding up the PR checks with a build cache: background -Status: **not implemented**. This records what we learned and what to do if the PR checks need to be faster. The cache that exists today is -described in [Workflows.md](Workflows.md). +Status: **superseded** by [Per-Repository-Build-Cache-Plan.md](Per-Repository-Build-Cache-Plan.md) (a build cache per repository, refreshed on `develop` +when it does not have the current state yet), which is implemented. This page keeps the measurements; the cache as it works now is described in [Workflows.md](Workflows.md). ## Where the time goes @@ -17,71 +17,23 @@ described in [Workflows.md](Workflows.md). The log of the build step shows `BHoM` alone restoring about 30 projects (`Geometry_oM`, `Structure_oM`, and so on) before compiling `BHoM.dll`. The log we have stops there and has no per-repository timings, so how the minute splits between `BHoM`, `BHoM_Engine`, -`JSONSchema_Toolkit` and `CSharpSchema_Test` is **not measured**. `BHoM` is probably most of it. +`JSONSchema_Toolkit` and `CSharpSchema_Test` is **not measured**. ## What exists today `pr-checks.yml` caches `C:\ProgramData\BHoM\Assemblies` under one key made from the current `develop` commit of all four repositories. On a hit it skips the clone and the build. -Two limits: +Two limits, which the per-repository plan addresses: -1. **One commit anywhere is a miss.** A new commit in any of the four repositories gives a new key, and everything is rebuilt. That includes - `CSharpSchema_Test`, which the loop changes on purpose with every exercise, although it is small and nothing depends on it. -2. **A PR's cache is only visible to that PR.** GitHub scopes a cache to the ref that saved it, and a PR run can also read caches saved on its base - branch. `pr-checks.yml` never runs on `develop`, so nothing is saved there and the first run of every new PR misses. +1. **One commit anywhere is a miss.** A new commit in any of the four repositories gives a new key, and everything is rebuilt, including `CSharpSchema_Test`, + which the loop changes on purpose with every exercise. +2. **A PR's cache is only visible to that PR.** GitHub scopes a cache to the ref that saved it. A PR run can also read caches saved on its base branch, + but `pr-checks.yml` never runs on `develop`, so nothing is saved there and the first run of every new PR misses. -## A cache per repository +## Decisions taken since the first version of this page -The idea is to cache each repository's build separately, so that a change in `BHoM_Engine` rebuilds one repository, not four. - -This needs a dependency rule. A repository is compiled against the assemblies of the ones it depends on (`dependencies.txt`), so its key must -contain its own commit **and** the keys of everything it depends on: - -| What changes | What must be rebuilt | -|---|---| -| `CSharpSchema_Test` | only itself | -| `JSONSchema_Toolkit` | only itself | -| `BHoM_Engine` | itself and `JSONSchema_Toolkit` | -| `BHoM` | all four | - -`BHoM` is the biggest repository and changes most often, and any change there rebuilds everything. A full per-repository cache therefore helps less -than it first appears. It would also need real work in `Build-Repositories.ps1`: snapshot what each build adds to `%ProgramData%\BHoM`, zip it -per repository under its key, restore in dependency order, and clone only the repositories that must be rebuilt. It cannot be tested locally without -changing the BHoM installation on the machine, because the repositories' post-build step copies into `%ProgramData%`. - -## Recommendation: two groups - -Split the cache in two and leave the rest alone: - -1. **Upstream group** (`BHoM`, `BHoM_Engine`, `JSONSchema_Toolkit`): cached together, keyed by their three commits. This is the slow part and - it changes least. -2. **`CSharpSchema_Test`**: not cached. It is built on top of the restored assemblies on every run. It is a leaf repository that is small and is the - one the loop changes, so a merge there no longer invalidates the cache. - -How to do it: - -- Make a second list file containing only `BHoM/CSharpSchema_Test` (for example `Leaf-repositories.txt`), and make the upstream list the other three. - `Build-Repositories.ps1` orders only among the listed repositories, so building the leaf list uses what is already installed. The generation and - release workflows can keep building from `Included-repositories.txt`. -- In `pr-checks.yml`, compute the cache key from the upstream list only, restore, build the upstream list on a miss and save, then always clone and - build the leaf list. -- Keep the key's version prefix (`bhom-assemblies-v1-`) so the cache can be reset by changing it. - -## Warm-up on `develop` - -To make the first run of a new PR hit, add a workflow that runs on `develop` (on a schedule such as daily, and by hand) and does only the -restore, build and save steps with the same key logic. A PR can restore caches saved on `develop`. - -- It hits only while none of the cached repositories has moved since the last warm-up. `BHoM` changes often, so expect partial hit rates. -- GitHub deletes caches unused for 7 days and limits a repository to 10 GB. The assemblies folder should be far below that. -- A restore from an older key (a prefix fallback with `restore-keys`) would give hits more often, but would restore assemblies built from other - commits and rebuild on top of them. Do not do that without a check that the result equals a clean build. - -## Before doing any of this - -1. Get per-repository build times from a run (a `Building ...` line is printed for each, but the log needs timestamps; step timing can be - switched on in the run's log settings). If `BHoM` is nearly all of the minute, the two-group split gains little for ordinary PRs and the warm-up is - the better investment. If `CSharpSchema_Test` takes a noticeable part, the split pays off on every loop exercise. -2. Check how often `BHoM` really changes in a week, since that sets the hit rate of any cache. -3. Check the first run after a change: a cache saved by the run itself never shows a saving on that run. Compare the second run. +- A **full per-repository cache** is wanted, not a split into two groups: it works for any list of repositories. `BHoM` does not change that often, and when it does, + everything is rebuilt anyway. +- A scheduled warm-up on `develop` is not wanted. `develop` is refreshed only when a PR run built something. +- Changing the BHoM installation on the development machine is acceptable for testing. diff --git a/docs/Per-Repository-Build-Cache-Plan.md b/docs/Per-Repository-Build-Cache-Plan.md new file mode 100644 index 0000000..a13cd01 --- /dev/null +++ b/docs/Per-Repository-Build-Cache-Plan.md @@ -0,0 +1,223 @@ +# Plan: a build cache per repository + +Status: **implemented**, except the later step 7 (generation). It replaces the "two groups" recommendation of [Build-Cache-Plan.md](Build-Cache-Plan.md), which stays as the +background (where the time goes, what exists today). + +## Goal + +Rebuild only the C# repositories whose own commit, or whose dependencies' commits, changed since the last build. It must work for any list in +`Included-repositories.txt`, not just today's four, so adding `BHoM/BHoM` or a toolkit needs no change to the cache. + +It is adopted in steps (see "Where it is used"): `pr-checks.yml` first, `python-schema-generation.yml` once the cache has proved itself, and +`release.yml` keeps a clean build. + +## What we corrected from the earlier notes + +| Earlier note | Now | +|---|---| +| "`BHoM` changes most often, so a per-repository cache helps little" | `BHoM` does not change that often. When it does, everything is rebuilt (as today). In between, a change in a dependent repository rebuilds only that repository and what depends on it | +| "It cannot be tested locally without changing the BHoM installation" | Changing the installation on the development machine is acceptable, so the scripts can be tested for real. Note that some projects write to a fixed `C:\ProgramData\BHoM\Assemblies` (for example `BHoM_Engine`), so redirecting `%ProgramData%` would not isolate a test | +| "Add a warm-up workflow on `develop`" | Not as a schedule. A refresh runs on `develop` only when `develop` does not have the current state (see "Refreshing `develop`") | +| "The last PR that builds updates the cache and the next PR uses it" | GitHub scopes a cache to the ref that saved it: a PR can read its own caches and those of its base branch (`develop`), but not another PR's. A cache saved by a PR run is therefore invisible to the next PR. That is why a run on `develop` is needed to share it | + +## What we know about the builds + +- A repository's build installs its assemblies into `C:\ProgramData\BHoM\Assemblies` through a post-build `xcopy`, and creates the empty folders + `Datasets`, `Upgrades`, `Settings` and `Developer` next to it. `BHoM_Engine` also copies `DeepCloner.dll`. +- Later repositories read the earlier ones' assemblies from there (`HintPath` `$(ProgramData)\BHoM\Assemblies\...`). That is the dependency that matters. +- The dependencies are listed in each repository's `dependencies.txt`, one `Owner/Name` per line: `BHoM` has none, `BHoM_Engine` lists `BHoM`, + `JSONSchema_Toolkit` lists `BHoM` and `BHoM_Engine`. `Build-Repositories.ps1` already orders the build with them, among the listed repositories only. +- Nothing after the build reads the C# sources in `.repos`: the generator and the tests use the installed assemblies, and the generation commit + message uses `.repos/commits.txt`. So a repository that is restored from the cache does not need to be cloned. +- All assemblies on the development machine are 145 MB in 567 files. The four repositories of CI are a fraction of that, so the entries stay small. + To be measured on a runner. + +## Design + +### 1. One key per repository + +``` +source(R) = sha256( "v1" | runner OS | R | commit(R) | sorted [ source(D) for D in dependencies(R) that are in the list ] ) +key(R) = sha256( source(R) | tools ) +``` + +- `commit(R)` is the head of `develop` (else `main`, else `master`), the same choice as `Clone-Repositories.ps1`, read with `git ls-remote`. +- Because `source(D)` is part of `source(R)`, a change in a dependency changes every key that depends on it, and nothing else. This is the table of the + earlier notes: a `CSharpSchema_Test` commit rebuilds one repository, a `BHoM_Engine` commit rebuilds it and `JSONSchema_Toolkit`, a `BHoM` commit rebuilds all. +- `tools` is the hash of `.github/workflows/build.ps1`, the MSBuild major.minor version and the .NET SDK feature band (`10.0.4xx`). A change to + how the repositories are built, or a new MSBuild on the runner image, then gives new keys without anybody having to remember it. Patch versions + are left out: during an image rollout two attempts of the same run can land on runners a patch apart, which with exact versions rebuilt + everything on a re-run. The plan logs the runner image next to the tools. The tools are kept out of `source(R)` because only + the Windows runner knows them, and the refresh gate runs on ubuntu (see "Refreshing `develop`"). +- The leading `v1` is a manual salt: bump it to throw every cache away, for example after a change to the cache scripts themselves. +- Only the dependencies that are in the list count. A dependency through a repository that is not listed (A depends on an unlisted X, which depends on + a listed B) is not seen, so `key(A)` ignores B. `Build-Repositories.ps1` has the same limitation today. + +### 2. Reading the dependencies before cloning + +The keys are needed before any clone, so each `dependencies.txt` is fetched alone, at the commit just read, from +`https://raw.githubusercontent.com///dependencies.txt`. A missing file means no dependencies. This is one small request per repository, +and a hit then needs no clone at all. + +The repositories are public, so no token is needed. A private repository would need a GitHub App token like the one `linked-csharp-pr.yml` makes, +because the workflow token can only read this repository. + +A failed `git ls-remote` or fetch is retried a few times, then fails the run. It never falls back to a partial plan. + +### 3. What one repository's entry holds + +A folder `-/` holding the files the repository's build added or changed under `%ProgramData%\BHoM`, with paths relative to that +folder. It is a plain folder, not a zip: `actions/cache` already compresses the whole cache, and copying a folder is faster than `Expand-Archive`. + +- The files are found by a snapshot of the `BHoM` folder (relative path, size, last write time) before and after that repository's build. + Comparing size and time is enough and fast; no hashing of 567 files. +- Restoring is copying the folder into `%ProgramData%\BHoM` (`robocopy`), in build order and overwriting, so a later repository's version of a + shared file wins, as in a real build. The four empty folders are always created. +- A file that two repositories copy with the same size and time (a third-party DLL keeps its package's timestamp through `xcopy`) is only in the + entry of the first one built. That is fine while both are in the list, but see "Risks" for removing a repository. + +### 4. Where the entries are stored: one folder, one GitHub cache entry + +GitHub's `actions/cache` is a workflow step and cannot be called from a loop over a list read at run time, so one step per repository is not possible for +a list-driven design. Instead the entries live together in one folder, `.build-cache/`, which is one GitHub cache entry: + +| Step | Key | +|---|---| +| Restore | exact key `bhom-build-cache-v1---`, where `` hashes every `source(R)` and `` the tools; if that misses, `restore-keys` prefix `bhom-build-cache-v1--`, which brings the most recent folder in scope | +| Save | the exact key above, only when it was not an exact hit and an entry was added | + +- An exact hit means nothing changed: nothing is built and nothing is saved. +- A prefix hit gives the folder of an earlier state. The script then finds, repository by repository, whether `-/` is in it. +- Cache keys are immutable, so a second save of the same key fails harmlessly when two runs race. +- Before saving, entries whose key is not among this run's keys are deleted. The current heads of the repositories are the only ones a run can need, so + the folder never grows beyond one entry per repository. + +### 5. The run, in order + +1. **Plan:** read the list, the commits and the dependencies, compute every key, order the repositories (existing topological order). Print a table of + repository, commit, entry, hit or miss. A `-Plan` mode stops here and writes the plan to a file, which the build then reads, so the build uses + the commits the key was made from. It takes the OS and the tools as parameters, so it also runs on ubuntu. +2. **Restore** the folder (exact key, then prefix). +3. **For each repository in order:** on a hit, copy its entry. On a miss, clone it at the planned commit (`git init`, `git fetch --depth 1 origin `, + `git checkout FETCH_HEAD`; GitHub allows fetching a commit by its hash), snapshot, build, snapshot, copy the difference into the folder. Cloning the + planned commit instead of the branch head means a push during the run cannot make the build differ from its key. A build that fails stops the + run, so a failed build never becomes an entry. +4. **Write** `.repos/commits.txt` from the plan, in the same format as `Clone-Repositories.ps1`. +5. **Save** the folder under the exact key when something was built. + +Hits skip the clone as well, so a run with every repository cached does not clone anything. + +### 6. Refreshing `develop` + +A PR run saves under the PR's own ref. For the next PR to benefit, the same state must be saved on `develop`. A new workflow `build-cache.yml` does this +only when `develop` does not have it yet: + +| Part | Detail | +|---|---| +| Trigger | `workflow_run` when the `PR` workflow completes, plus `workflow_dispatch`. A `workflow_run` workflow runs on the default branch, so what it saves is visible to every PR | +| Job 1, `gate` (ubuntu, a few seconds, `actions: read`) | Continues only if the PR comes from this repository, not a fork (`github.event.workflow_run.head_repository.full_name`), or for a manual run on `develop`. Then runs `-Plan` for Windows and looks for the state prefix `bhom-build-cache-v1-Windows--` on `develop` (`gh cache list --ref refs/heads/develop --key `). Continues only if it is not there, or for a manual run | +| Job 2, `refresh` (windows) | Runs the same action: plan, restore, build the misses, save. It builds from the trusted code on `develop`, never from the PR | +| Concurrency | One group, `cancel-in-progress: false`: the running refresh finishes, and a newer pending one replaces an older pending one. This avoids building the same state twice in parallel | + +The gate asks `develop` directly rather than whether the PR run built something. A PR's second push can hit the PR's own cache after a refresh failed +or was cancelled, and would then never trigger a new one. + +Why it rebuilds, instead of passing the PR's entries over: an entry from a PR run is code that the PR built. Putting it in the `develop` cache would let +one branch affect every later run, and forks could do the same. The rebuild costs one extra build per change in an upstream repository, not one per PR, +and the next PR hits. + +Alternatives that were rejected: + +| Alternative | Why not | +|---|---| +| A scheduled warm-up | Not event-driven: it either wastes runs or leaves a window with misses | +| The PR run dispatches the refresh with `gh workflow run` | Needs `actions: write` in the PR job, which also runs the PR's own code | +| Run on `push` to `develop` | A push to this repository does not happen when only an upstream repository changed | +| Pass the PR's entries as an artifact | The poisoning risk above | +| Gate on an artifact the PR run uploads when it built something | Misses the case above, where a PR run hits its own cache but `develop` was never refreshed | + +The gate cannot see the tools, so after a new MSBuild on the runner image `develop` keeps its entry of the old tools until a repository changes +(or a manual run of `build-cache.yml`); in between, pull requests rebuild everything, as before this cache. + +Until the generation workflow adopts the cache, nothing else refreshes `develop`. If no PR runs for 7 days, GitHub deletes the unused entry and the next +PR builds everything once (about 1 min 15 s), then the refresh restores it. + +### 7. Where it is used + +The composite action `.github/actions/bhom-assemblies` is rewritten for this. It is adopted in steps, because a wrong cache does very different harm depending on where +it is used: + +| Workflow | When | Why | +|---|---|---| +| `pr-checks.yml`, both jobs | First change | A wrong cache gives a wrong test result on a pull request. It is seen, re-run with `-NoCache`, and nothing is published | +| `python-schema-generation.yml` | After about four weekly runs without a discrepancy | It commits straight to `develop`, with no pull request. The worst failure is silent: a stale entry gives Python that matches the old C#, `py2cs verify` and Pyright pass, and `commits.txt` names the new commits | +| `release.yml` | Not adopted: it keeps a clean build | It publishes a wheel and runs a few times a year, so saving a minute is worth nothing. A `workflow_dispatch` run can also read the caches of the ref it was dispatched from, so a release dispatched from a branch could use what that branch built | + +In `pr-checks.yml`, the alignment job waits for the unit-test job (`needs: checks`, `if: ${{ !cancelled() }}`), so a miss is built once: the unit-test +job saves the cache right after the build, and the alignment job then hits it. Alignment still runs when the tests fail, and re-running only the +alignment job after the C# merge still works, because a single-job re-run does not re-run its `needs`. The price is an alignment result that comes +after the tests instead of beside them. + +While generation still builds clean, it can also restore the cache and compare the two `Assemblies` folders (file list, sizes and assembly versions), +writing the differences to the step summary without using the cache. That makes the waiting period a real test. + +## What it should cost, to be measured + +| Situation | Today (one key) | Per repository | +|---|---|---| +| Nothing changed | restore, about 20 s | plan (a few requests) + restore + copy every entry, expected 30 to 45 s | +| `CSharpSchema_Test` changed | rebuild all, about 1 min 15 s | build one repository | +| `BHoM_Engine` changed | rebuild all | build it and `JSONSchema_Toolkit` | +| `BHoM` changed | rebuild all | rebuild all, as today | +| First run of a new PR | miss: only PR refs save the cache today, and a PR cannot read another PR's | hit, once `develop` has the cache | + +These are estimates. The per-repository build times are not measured yet (the earlier log has no timings), so the second row onward is only as good as +that measurement. + +## Risks and how they are handled + +| Risk | Handling | +|---|---| +| A stale entry is used | The key contains the commit, the dependencies' keys, the OS, the build script and the MSBuild and .NET SDK versions. The `v1` salt (`-Salt` of `Build-Repositories.ps1`) covers the rest | +| A build fails but its files are cached | `Build-Repositories.ps1` stops at the first build that exits with an error, before the entry is complete and before the save | +| A file shared by two repositories is only in the first one's entry | Fine while both are listed, since every run installs every listed repository. After removing a repository from the list, bump the salt | +| A build has side effects outside `%ProgramData%\BHoM` | None known. The plan checks this by comparing the folder before and after on a real run | +| Two jobs build the same repository at once | Avoided in `pr-checks.yml` by `needs`. Elsewhere, wasted time, not a wrong result: the second save of the same key fails harmlessly | +| The cache grows or is evicted | One entry per repository, pruned on every save. Unused entries are deleted after 7 days and the repository limit is 10 GB, far above this | +| `git ls-remote` or a `dependencies.txt` fetch fails | Retried, then the run fails. It never builds from a partial plan | +| A fork spams the refresh workflow | The gate rejects runs whose head repository is not this repository, and runs on an ubuntu runner for a few seconds | +| The script is wrong and builds from stale assemblies | `-NoCache` ignores the cache completely, and bumping the salt resets it; both are one-line changes. Generation adopts the cache only after a comparison period, and release never does | + +## Implementation steps + +Steps 1 to 6 are done. + +1. **Planning script.** Add `-Plan` to `Build-Repositories.ps1`: commits, dependencies, keys, order, hit or miss. A `-CommitsFile` + parameter replaces `git ls-remote` and `-DependenciesRoot` the fetch of `dependencies.txt`, so tests need no network. +2. **Build with the folder.** `-CacheFolder` for `Build-Repositories.ps1`: copy on a hit, clone at the planned commit and build on a miss, snapshot, copy the + difference, prune, write `.repos/commits.txt`. `-NoCache` for the old behaviour. +3. **Composite action.** Plan, restore (exact then prefix), build, save when something was built; outputs which repositories were built. +4. **Use it** in `pr-checks.yml`, with the alignment job waiting for the unit-test job. +5. **`build-cache.yml`** with the gate and refresh jobs above. +6. **Tests and docs**, listed below, then update [Workflows.md](Workflows.md) and mark [Build-Cache-Plan.md](Build-Cache-Plan.md) as superseded. +7. **Later, a separate change:** the comparison in `python-schema-generation.yml`, then the switch to the cache once it has shown no difference. + +## How we will test it + +- **The plan logic, with no network and no build:** a test that feeds `-Plan` a fake commits file and fake `dependencies.txt` files and checks the keys: a change in one + repository changes exactly its own key and those that depend on it, the order is topological, and the salt, the OS and the tools change every key. NUnit or pytest can start `pwsh`. +- **Restore, on a real machine:** run the build twice on the development machine (the installation may be changed): the second run must copy everything and build + nothing; change one commit (or a fake commit file) and check that only that repository and its dependents are rebuilt; compare the resulting `Assemblies` folder with a + clean build by file list, sizes and assembly versions (the builds are not byte-for-byte deterministic). +- **Workflow structure tests** (the style of `test_workflows.py`): the action restores before building and saves after; the alignment job needs the unit-test job and + runs unless cancelled; the refresh workflow triggers only on `workflow_run` and manual runs; the gate checks the head repository and looks the key up on `develop`; + only the refresh job writes to the `develop` cache; `release.yml` does not use the action. +- **On GitHub:** a first run that builds and saves, a second that restores everything, then a PR that touches one upstream repository's commit (or a fake) and rebuilds only that one, + and a check that the refresh run makes the next PR hit. + +The plan logic and the restore are tested in `test_build_plan.py`, and the workflow structure in `test_workflows.py`. The run on the development +machine with real builds and the runs on GitHub are still to do. + +## Still to measure + +1. Build time per repository from a real run (the `Building ...` lines with timestamps), to confirm the expected gain. diff --git a/docs/Reusing-oM-Classes-2.md b/docs/Reusing-oM-Classes-2.md new file mode 100644 index 0000000..c09d9c1 --- /dev/null +++ b/docs/Reusing-oM-Classes-2.md @@ -0,0 +1,126 @@ +Updated todo list + +Searched for files matching `**/Environment_oM/**/Project*.cs`, no matches + +Searched for files matching `**/Physical_oM/**/*.cs`, 20 matches + +Searched for files matching `**/Structure_oM/**/*.cs`, 20 matches + +No matches found. Your search pattern might be excluded completely by either the search.exclude settings or .*ignore files. +If you believe that it should have results, you can check into the .*ignore files and the exclude setting (here are some excluded patterns for reference:[**/node_modules,**/bower_components,**/*.code-search]). +Then if you want to include those files you can call the tool again by setting "includeIgnoredFiles" to true. + +Searched for regex `class Storey\b|class Project\b|class Floor\b|class Grid\b"` (`**/Physical_oM/**`), 1 result + +Searched for regex `class Column\b|class Beam\b|class Slab\b|class Panel\b|class Foundation\b|class Pile\b|class Material\b|class StructuralSystem\b|class GravitySystem\b|class LateralSystem\b|class ShearWall\b"` (`**/Structure_oM/**`), 2 results + +Read + +Read + +Read + +Searched for regex `class Column|class Beam|class IFoundation|class Foundation|class Material|class StructuralSystem|class GravitySystem|class LateralSystem|class ShearWall` (`**/Structure_oM/**`), 1 result + +Read + +Read + +No matches found. Your search pattern might be excluded completely by either the search.exclude settings or .*ignore files. +If you believe that it should have results, you can check into the .*ignore files and the exclude setting (here are some excluded patterns for reference:[**/node_modules,**/bower_components,**/*.code-search]). +Then if you want to include those files you can call the tool again by setting "includeIgnoredFiles" to true. + +Searched for regex `class.*Storey|class.*Level\b`, 4 results + +Read + +Read + +Read + +Searched for regex `class Grid\b|GridLine|class.*Grid`, 5 results + +Read + +Read + +Read + +Read + +Read + +Read + +No matches found. Your search pattern might be excluded completely by either the search.exclude settings or .*ignore files. +If you believe that it should have results, you can check into the .*ignore files and the exclude setting (here are some excluded patterns for reference:[**/node_modules,**/bower_components,**/*.code-search]). +Then if you want to include those files you can call the tool again by setting "includeIgnoredFiles" to true. + +Searched for regex `GrossFloorArea|FloorArea|RoofArea|StoreysAboveGround|StoreysBelowGround|FloorToFloorHeight`, 3 results + +Read + +Searched for regex `ProjectName|ProjectNumber|JobNumber|ProjectID|ProjectId`, 12 results + +Read + +No matches found. Your search pattern might be excluded completely by either the search.exclude settings or .*ignore files. +If you believe that it should have results, you can check into the .*ignore files and the exclude setting (here are some excluded patterns for reference:[**/node_modules,**/bower_components,**/*.code-search]). +Then if you want to include those files you can call the tool again by setting "includeIgnoredFiles" to true. + +Read + +No matches found. Your search pattern might be excluded completely by either the search.exclude settings or .*ignore files. +If you believe that it should have results, you can check into the .*ignore files and the exclude setting (here are some excluded patterns for reference:[**/node_modules,**/bower_components,**/*.code-search]). +Then if you want to include those files you can call the tool again by setting "includeIgnoredFiles" to true. + +Read + +Read + +No matches found. Your search pattern might be excluded completely by either the search.exclude settings or .*ignore files. +If you believe that it should have results, you can check into the .*ignore files and the exclude setting (here are some excluded patterns for reference:[**/node_modules,**/bower_components,**/*.code-search]). +Then if you want to include those files you can call the tool again by setting "includeIgnoredFiles" to true. + +Read + +Read , lines 1 to 55 + +Read , lines 1 to 60 + +Now I have enough to map the whole document against the existing BHoM object model. Here's the breakdown: + +## Summary + +The required schema describes **project-level benchmarking/summary metrics** (e.g. "typical column size", "stories above ground", "material volume distribution"). BHoM's existing oM classes instead model **individual geometric/analytical elements** (an actual `Column` with a `Location` and section `Property`, an actual `Panel`/`PadFoundation` with real geometry). Because of this mismatch in level of abstraction, almost none of the requested properties already exist — only a few concepts have a loose, partial analogue. + +### Environment_oM → `project` +| Required property | Status | +|---|---| +| `Project_number` | **New** — no `Project` class exists in `Environment_oM`. Closest analogues are `ProjectNumber` in `Audit.cs:74` and `ProjectName` in `LifeCycleAssessmentScope.cs:71`, but neither lives in `Environment_oM` and neither is a dedicated `Project` class with both fields. | +| `Project_name` | **New** (same as above) | + +→ You'll need a brand-new `BH.oM.Environment.Elements.Project` class (or similar) with both properties. + +### Physical_oM +| Required class/property | Status | +|---|---| +| `Storey` (typical_slab_to_slab_height, stories_above_ground, stories_below_ground) | **New.** No `Storey` class exists in `Physical_oM`. `Level.cs` only models a single elevation plane, not storey counts. | +| `Floor_area` (gross_slab_area_non_roof, roof_area) | **New.** `Floor.cs` is a geometric surface element (`ISurface`), it has no area/roof summary properties. Closest is `FloorArea`/`ConstructedFloorArea` in `IStructEEvaluationConfig.cs:43`, but that's a single generic value, not split gross/roof. | +| `Grid_spacing` (typical_x_spacing_m, typical_y_spacing_m) | **New.** `Grid.cs` models an actual 1D gridline (geometry), not a typical-spacing summary value. | + +### Structure_oM (tool-focused) +| Required class/property | Status | +|---|---| +| `Base_structural_element.Is_refurbishment` | **New.** No common base class with this flag exists. | +| `Column` (typical_material/size/count_per_slab/material_distribution) | **Partial reuse of class identity only.** `Column.cs` exists but only has `Location` + `Property` (a real section/material for one physical column) — none of the "typical/aggregate" properties exist. | +| `Beam` (same pattern) | Same as Column — `Beam.cs` exists but lacks these aggregate fields. | +| `Slab` (typical_type/roof_type/size) + depth/ground-slab/material_distribution fields | **Partial reuse.** Slabs are modeled as `Panel.cs` with an `ISurfaceProperty` (e.g. `ConstantThickness.cs` has `Thickness` + `Material` + `PanelType`), which is conceptually close to depth/type/material, but there's no "typical/benchmark" wrapper class matching the doc's flat property list. | +| `Foundation` (foundation_type, pile_count, pile_volume) | **New.** `Pile.cs`, `PadFoundation.cs`, `PileFoundation.cs` model actual individual foundation geometry, not count/volume/type summary stats. | +| `Material` (material_volumes_above/below_ground) | **New.** No such aggregate class exists in `Structure_oM`, `Matter_oM`, or `LifeCycleAssessment_oM`. | +| `Structural_system` → `Gravity_system`, `Lateral_system` (incl. shear wall thickness) | **New.** No `StructuralSystem`, `GravitySystem`, `LateralSystem`, or shear-wall classes exist anywhere in the repo. | + +## Conclusion +None of the leaf properties listed in the doc already exist as named properties anywhere in BHoM — they describe early-stage/benchmarking summary data, while existing `Environment_oM`/`Physical_oM`/`Structure_oM` classes describe concrete, individual building elements. You can **reuse the class names/concepts** (`Column`, `Beam`, `Panel`-as-slab, `Pile`/`PadFoundation`-as-foundation) as a naming/semantic starting point, and reuse `BuildingBenchmarkingData`-style patterns (see `BuildingBenchmarkingData.cs`) for the High/Average/Low benchmarking idiom, but every property in your doc will need to be **newly authored**, most likely as a new set of "early-stage benchmarking" classes (e.g. under a new project or inside `LifeCycleAssessment_oM`/`Structure_oM`) rather than by extending the existing geometric element classes directly. + +Created 2 todos \ No newline at end of file diff --git a/docs/Testing-On-GitHub.md b/docs/Testing-On-GitHub.md index 8a8431d..123da42 100644 --- a/docs/Testing-On-GitHub.md +++ b/docs/Testing-On-GitHub.md @@ -9,7 +9,8 @@ A procedure for proving, on GitHub, the whole loop between this repository and [ | B. Update a class | a property of `Beam` (added, then changed or removed) | the lines of that property change in the existing file; breaking changes are labelled | | C. Add a class | a new class and its file | one new file in the right folder and namespace | -This procedure has not been run yet. Part 0 is done once; the three exercises can be done in any order. How each workflow works is in +Separately, "Testing the build cache" proves that the PR checks build only the C# repositories that changed. This procedure has not been run +yet. Part 0 is done once; the three exercises can be done in any order. How each workflow works is in [Workflows.md](Workflows.md); the Python authoring rules are checked by `py2cs check`. ## Part 0: before the first exercise @@ -155,6 +156,71 @@ prints what it would do and proves the App token; unticked it does it. A pull request that is closed without merging closes its draft with a comment; delete the `python-schema/pr-` branch in C# by hand. +## Testing the build cache + +The PR checks keep a build cache with one entry per C# repository (see [Per-Repository-Build-Cache-Plan.md](Per-Repository-Build-Cache-Plan.md)). +These tests prove that a repository is built only when its commit, or a dependency's commit, changed, and that `develop` shares the build +with every pull request. + +**Part D1 can run from the `ImproveCaching` pull request before it is merged. Part D2 needs it merged into `develop` first.** A pull request runs +the `pr-checks.yml` of its own branch, so D1 tests the new action as it is. `build-cache.yml` is different: GitHub starts a `workflow_run` +workflow, and lists a workflow under *Run workflow*, only from the default branch, so before the merge it never runs. And only a run on +`develop` can save the cache that every pull request reads. + +### Where to look + +| Where | What it tells you | +|---|---| +| Step *Plan the build* of the job | The commit and entry name of each repository, and the cache key | +| Step *Restore the build cache* | `Cache restored from key: `; when that key differs from the planned key it was a prefix hit, an earlier state. `Cache not found` means nothing could be restored | +| Step *Build the repositories in dependency order* | The same table with `hit` or `miss` per repository, then `Restoring from the cache...` for a hit, `Cloning at ...` and `Building ...` for a miss, `Deleting the old entry `, and finally `Built: ` or `Built: nothing` | +| Step *Save the build cache* | Skipped when nothing was built | +| Actions tab, *Caches* (under Management) | Each `bhom-build-cache-v1-Windows-...` entry and its branch: `refs/pull//merge` for a pull request, `refs/heads/develop` for `develop` | + +### Part D1: on the `ImproveCaching` pull request (before merging) + +1. **First run, everything is built.** Open the pull request from `ImproveCaching` to `develop`. In **PR / Build and tests**: the restore + says `Cache not found` (the old `bhom-assemblies-v1-...` entries do not match the new prefix), the build clones the four repositories at + the commits of the plan and ends with `Built: BHoM, BHoM_Engine, JSONSchema_Toolkit, CSharpSchema_Test`, and the save runs. Every test + passes, which shows that the assemblies installed this way are the ones the tests need. *Caches* shows one new entry on `refs/pull//merge`. +2. **The alignment job waits.** **PR / C# alignment** starts only once **Build and tests** has finished. This pull request does not change + `src/bhom_schema`, so its steps are skipped; D2 step 5 tests it with a build. +3. **A re-run builds nothing.** Re-run all jobs. The restore names the planned key exactly, the build table says `hit` for all four, the log + has four `Restoring ... from the cache...` lines and `Built: nothing`, and the save is skipped. Note how long *Build the repositories* takes + compared with step 1. +4. **One changed repository is built alone.** Make a commit on `develop` of CSharpSchema_Test (for example merge the C# draft of an exercise, + or a small pull request there), then re-run the jobs. The table says `miss` for CSharpSchema_Test and `hit` for the others, the log ends with + `Built: CSharpSchema_Test`, the old entry of CSharpSchema_Test is deleted, and the save runs under a new key. + If BHoM_Engine or BHoM changed upstream in the meantime, that repository and the ones that depend on it are built too. Compare the commits + in the table with the repositories to check it. +5. **Nothing runs on `develop` yet.** No *Build cache* run appears in the Actions tab. That is expected before the merge. + +### Part D2: after merging into `develop` + +1. **Fill the `develop` cache by hand.** Actions, *Build cache*, Run workflow, branch `develop`. The gate prints `develop has no entry + starting with bhom-build-cache-v1-Windows-...`, then **Build and save on develop** builds the four repositories (`develop` cannot read + a pull request's cache) and saves. *Caches* shows an entry on `refs/heads/develop`. +2. **Try a manual run from another branch.** Run *Build cache* from `ImproveCaching` or any other branch: both jobs are skipped, so + nothing but `develop` can write to the `develop` cache. +3. **A new pull request builds nothing.** Open a new pull request that changes `src/bhom_schema` (Exercise A is a good one). Its first run + restores the exact key from `develop` and logs `Built: nothing` in **Build and tests**, and **C# alignment** logs `Built: nothing` too. +4. **No refresh when `develop` is up to date.** When that PR run completes, *Build cache* starts by itself. The gate prints `develop has: + ` and the refresh job is skipped. +5. **One changed repository is built once, then shared.** Make a commit on `develop` of CSharpSchema_Test (closing the loop of an exercise does + this), then push to the pull request or re-run its jobs. + - **Build and tests** builds only CSharpSchema_Test. + - **C# alignment** waits for it and logs `Built: nothing`, because it restores what the first job just saved. Without the wait it would + have built CSharpSchema_Test a second time. + - The *Build cache* run that follows finds no entry for the new state on `develop`. Its refresh builds only CSharpSchema_Test, from the + `develop` entries of the others, and saves on `develop`. + - Open, or re-run, a second pull request. It restores that `develop` entry and builds nothing. +6. **The C# alignment still runs when a test fails** (optional). Push a commit that makes one pytest test fail. **Build and tests** fails, + and **C# alignment** still runs. +7. **A fork gets no refresh** (optional, needs a fork). A pull request from a fork runs the PR checks with its own cache, and the *Build cache* + run that follows skips both jobs. +8. **Switching the cache off** (optional). On a branch, give the action `with: no-cache: 'true'` in `pr-checks.yml`. The restore and save are + skipped, and every repository is built, as before this cache. Do not merge that change. + ## If something goes wrong | Symptom | Likely cause | @@ -165,6 +231,11 @@ A pull request that is closed without merging closes its draft with a comment; d | the draft changes more than the lines you changed | report it: that is a bug in the patcher, with the file and your Python diff | | `py2cs align` still fails after the C# merge | the C# `develop` clone in the workflow is older than the merge, or the C# review changed the class; read the listed differences | | the weekly run commits after everything merged | the generator writes the class differently from what `py2cs` drafted: look at the commit | +| *Build cache* never runs after a PR run | `build-cache.yml` is not on `develop` yet, or the PR workflow was renamed (the trigger names `PR`) | +| the gate fails on `gh cache list` | the `gh` of the ubuntu runner is too old for `--ref`; the log shows the error | +| every pull request builds everything | `develop` has no entry, or one made with other build tools after an MSBuild update on the runner: run *Build cache* by hand on `develop` | +| `Building failed` | the C# of that repository does not build at its planned commit. The run stops so a failed build never becomes an entry | +| tests fail after a restore but pass with `no-cache` | a wrong entry: bump `-Salt` (the `v1`) in `Build-Repositories.ps1` to throw every cache away, and report it with both logs | ## Record the result diff --git a/docs/Workflows.md b/docs/Workflows.md index 7fed964..f008ffc 100644 --- a/docs/Workflows.md +++ b/docs/Workflows.md @@ -7,7 +7,8 @@ written to.** BHoM, BHoM_Engine and JSONSchema_Toolkit are only cloned and built | Workflow | When | What it does | |---|---|---| | [python-schema-generation.yml](../.github/workflows/python-schema-generation.yml) | Every Saturday at 4 AM UTC, and by hand | Clones and builds the C# repositories, generates `src/bhom_schema` from the C#, checks it, and commits it straight to `develop`. Not a pull request. If a check fails, nothing is committed | -| [pr-checks.yml](../.github/workflows/pr-checks.yml) | Every pull request to `develop` | Two jobs. **Build and tests**: installs the built assemblies of the repositories in `Included-repositories.txt` (the shared action `.github/actions/bhom-assemblies` caches them under the current commit of each repository, so unchanged repositories are not rebuilt), clones BHoM_JSONSchema read only as the test oracle for the JSON, runs `run-all.ps1` (build, NUnit, pytest, Pyright) and, for a PR that changes `src/bhom_schema`, `py2cs check`. **C# alignment (fails until the linked C# PR is merged)**: for such a PR, builds the generator and runs `py2cs align`; when it fails it explains in the job summary that the linked C# pull request has to be merged first, then the job re-run | +| [pr-checks.yml](../.github/workflows/pr-checks.yml) | Every pull request to `develop` | Two jobs. **Build and tests**: installs the built assemblies of the repositories in `Included-repositories.txt` (the shared action `.github/actions/bhom-assemblies` keeps a build cache with one entry per repository, keyed by its commit, its dependencies' keys and the build tools, so only a changed repository and what depends on it are rebuilt), clones BHoM_JSONSchema read only as the test oracle for the JSON, runs `run-all.ps1` (build, NUnit, pytest, Pyright) and, for a PR that changes `src/bhom_schema`, `py2cs check`. **C# alignment (fails until the linked C# PR is merged)**: after the first job (so a changed C# repository is built once), and even when its tests fail; for such a PR, builds the generator and runs `py2cs align`; when it fails it explains in the job summary that the linked C# pull request has to be merged first, then the job re-run | +| [build-cache.yml](../.github/workflows/build-cache.yml) | After every run of `pr-checks.yml` from this repository, and by hand on `develop` | A cache saved by a pull request is only visible to that pull request. **Does develop need a refresh?** (ubuntu, a few seconds) plans the build and looks for the current state among the caches of `develop`. When it is missing, **Build and save on develop** clones and builds the changed repositories from `develop`, never from the pull request, and saves the cache where every pull request can read it | | [linked-csharp-pr.yml](../.github/workflows/linked-csharp-pr.yml) | A pull request that changes `src/bhom_schema`, and by hand | One job, **Raise or update draft**: finds the C# repositories the change touches, raises or updates a linked **draft** pull request in each, and comments on the Python PR with its link. The shared action `.github/actions/linked-pr-context` does the finding | | [linked-csharp-pr-close.yml](../.github/workflows/linked-csharp-pr-close.yml) | A Python pull request that changes `src/bhom_schema` is closed, and by hand | Closes the linked C# draft with a comment when the Python PR was closed unmerged, or only comments when it was merged. A separate workflow so that nothing of it shows on an open pull request | | [release.yml](../.github/workflows/release.yml) | By hand only (dry run by default) | Regenerates with `--release X.Y`, checks, builds the wheel and the release notes; when not a dry run, pushes branch `vX.Y` and tag `vX.Y.0`, creates the GitHub Release and publishes to TestPyPI. See "Making a release" | @@ -35,36 +36,43 @@ written to.** BHoM, BHoM_Engine and JSONSchema_Toolkit are only cloned and built │ on every PR to develop │ │ on a PR that changes src/bhom_schema │ │ build, NUnit, pytest, │ │ │ │ Pyright; py2cs check; │ │ one job: raise or update the draft, and add │ -│ C# alignment (own job) │ │ a comment on the Python PR with its link │ -│ (writes nothing) │ │ │ -└──────────────────────────────┘ │ PR closed: linked-csharp-pr-close.yml │ - │ │ - │ │ - │ │ - └─────────────────────┬───────────────────────────┘ - │ the C# draft is reviewed and merged in C# - ▼ - next weekly generation: the C# is the source of truth, - so src/bhom_schema ends up matching the merged C# +│ C# alignment (own job, │ │ a comment on the Python PR with its link │ +│ after the tests) │ │ │ +│ writes only the build cache │ │ PR closed: linked-csharp-pr-close.yml │ +│ of its own PR │ │ │ +└──────────────┬───────────────┘ │ │ + │ when the run completes │ │ + ▼ └─────────────────────┬───────────────────────────┘ +┌──────────────────────────────┐ │ the C# draft is reviewed and merged in C# +│ build-cache.yml │ ▼ +│ gate (ubuntu): is the │ next weekly generation: the C# is the source of truth, +│ current state on develop? │ so src/bhom_schema ends up matching the merged C# +│ if not, refresh (windows): │ +│ build from develop and save │ +│ the cache on develop, where │ +│ every PR can read it │ +└──────────────────────────────┘ End of a milestone, by hand: ┌──────────────────────────────────────────────────────────────────────────────────────────────────────┐ │ release.yml (input X.Y, dry run by default) │ │ job build: clone + build ─► generate --release X.Y ─► verify + Pyright ─► wheel + notes │ │ not a dry run: branch vX.Y, tag vX.Y.0, GitHub Release (wheel + notes) │ -│ job publish-testpypi (not a dry run): the wheel ─► TestPyPI, by trusted publishing │ +│ job publish-testpypi (not a dry run): the wheel ─► TestPyPI, by trusted publishing │ └──────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` Order in one line: **generation** (weekly) → a Python **pull request** → **pr-checks** and **linked-csharp-pr** run side by side → the C# -draft is merged → the next **generation** brings the Python back in line → **release** (by hand) freezes it as `vX.Y`. +draft is merged → the next **generation** brings the Python back in line → **release** (by hand) freezes it as `vX.Y`. After each +**pr-checks** run, **build-cache** saves the built C# repositories on `develop` when they changed, so the next pull request does not build them. ## What each one reads and writes | | Reads | Writes | |---|---|---| | Generation | `Included-repositories.txt` (what to build), `Converted-repositories.txt` (what to convert), `.ci/generation/warnings-baseline.txt` | `src/bhom_schema` on `develop` | -| PR checks | the same | nothing | +| PR checks | the same, and the build cache of its pull request or of `develop` | the build cache of its own pull request, which no other pull request sees | +| Build cache | `Included-repositories.txt`, the `dependencies.txt` of each C# repository, and the caches of `develop` | the build cache on `develop` | | Linked C# PR | `Converted-repositories.txt` | a branch `python-schema/pr-` and a draft PR in the C# repository, and one comment on the Python PR | `Converted-repositories.txt` is also the list of repositories a C# pull request may be raised in. A Python class names its own