diff --git a/.github/workflows/compat-prune-monthly.yml b/.github/workflows/compat-prune-monthly.yml
new file mode 100644
index 0000000..4e8c7b4
--- /dev/null
+++ b/.github/workflows/compat-prune-monthly.yml
@@ -0,0 +1,32 @@
+name: Compat prune (monthly)
+
+# The organization's monthly [compat] prune: every CT* package audited together, one PR per
+# repository with something to drop, one tracking issue here in CTActions. The logic is the
+# reusable compat-prune.yml; see docs/WORKFLOWS.md §3.2.
+
+on:
+ schedule:
+ - cron: '0 4 1 * *'
+ workflow_dispatch:
+ inputs:
+ dry_run:
+ description: 'Only audit and report: no PR, no issue'
+ type: boolean
+ default: true
+
+concurrency:
+ group: compat-prune-monthly
+ cancel-in-progress: false
+
+jobs:
+ prune:
+ uses: ./.github/workflows/compat-prune.yml
+ with:
+ # Dry run on every trigger until the GitHub App is set up; then the schedule
+ # publishes: `github.event_name == 'workflow_dispatch' && inputs.dry_run`.
+ dry_run: ${{ github.event_name != 'workflow_dispatch' || inputs.dry_run }}
+ # the scripts of the same commit as this workflow
+ ctactions_ref: ${{ github.sha }}
+ secrets:
+ CT_APP_CLIENT_ID: ${{ secrets.CT_APP_CLIENT_ID }}
+ CT_APP_PRIVATE_KEY: ${{ secrets.CT_APP_PRIVATE_KEY }}
diff --git a/.github/workflows/compat-prune-tests.yml b/.github/workflows/compat-prune-tests.yml
new file mode 100644
index 0000000..e2f434f
--- /dev/null
+++ b/.github/workflows/compat-prune-tests.yml
@@ -0,0 +1,51 @@
+name: Compat prune tests
+
+# Self-test of compat-prune, in CTActions itself: the test suite of scripts/compat-prune on
+# every audited Julia version (unit, non-regression and integration tests), then a real
+# dry-run audit of the ecosystem through the reusable workflow. Runs when the scripts or the
+# workflows change; nothing is written anywhere (dry run).
+
+on:
+ pull_request:
+ paths:
+ - 'scripts/compat-prune/**'
+ - '.github/workflows/compat-prune*.yml'
+ push:
+ branches: [main]
+ paths:
+ - 'scripts/compat-prune/**'
+ - '.github/workflows/compat-prune*.yml'
+ workflow_dispatch:
+
+concurrency:
+ group: compat-prune-tests-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ tests:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ version: ['1.10', '1.12', '1.13']
+ steps:
+ - uses: actions/checkout@v6
+ - uses: julia-actions/setup-julia@v2
+ with:
+ version: ${{ matrix.version }}
+ - name: Add the General registry
+ run: julia --startup-file=no -e 'using Pkg; Pkg.Registry.add("General")'
+ - name: Tests (with integration)
+ env:
+ COMPAT_PRUNE_INTEGRATION: '1'
+ run: julia --startup-file=no scripts/compat-prune/test/runtests.jl
+
+ dry-run:
+ needs: tests
+ uses: ./.github/workflows/compat-prune.yml
+ with:
+ dry_run: true
+ ctactions_ref: ${{ github.event.pull_request.head.sha || github.sha }}
diff --git a/.github/workflows/compat-prune.yml b/.github/workflows/compat-prune.yml
new file mode 100644
index 0000000..bee0417
--- /dev/null
+++ b/.github/workflows/compat-prune.yml
@@ -0,0 +1,185 @@
+name: Compat prune
+
+# Finds the [compat] specifiers that no supported Julia version ever selects across the
+# control-toolbox packages, proves that dropping them leaves every resolved manifest
+# unchanged, and (unless dry_run) opens one PR per affected repository plus one tracking
+# issue (in CTActions by default), both updated in place by later runs. The prune is the
+# organization's: it is called monthly by `compat-prune-monthly.yml`, here in CTActions, and
+# by the self-test `compat-prune-tests.yml`; see docs/WORKFLOWS.md §3.2.
+#
+# The `audit` job is read-only: it resolves the joint, test and docs environments of every
+# package on every Julia version (Pkg's resolver only: nothing is downloaded or installed),
+# classifies the multi-specifier entries, validates the prunes, applies them to fresh
+# checkouts of the default branches, and uploads the result (`compat-prune` artifact,
+# report in the job summary). Scripts and tests: scripts/compat-prune/.
+#
+# The `publish` job (skipped on dry runs) is the only one that writes, with a token of the
+# organization's GitHub App (secrets CT_APP_CLIENT_ID, CT_APP_PRIVATE_KEY; permissions:
+# contents, pull requests, issues). It re-applies the prunes to fresh checkouts, then keeps
+# one branch `ct/compat-prune` and one PR per repository, and one tracking issue in
+# `tracking_repo`: a rerun updates them in place, and closes what has nothing left.
+
+on:
+ workflow_call:
+ inputs:
+ packages:
+ description: 'JSON list of the audited packages (registered names)'
+ required: false
+ type: string
+ default: '["CTBase","CTParser","CTModels","CTLie","CTFlows","CTDirect","CTSolvers","OptimalControl"]'
+ julia_versions:
+ description: 'JSON list of the Julia versions (juliaup channels); the last one is the newest'
+ required: false
+ type: string
+ default: '["1.10","1.12","1.13"]'
+ dry_run:
+ description: 'Only audit and report: no PR, no issue'
+ required: false
+ type: boolean
+ default: true
+ ctactions_ref:
+ description: 'Ref of CTActions the scripts are checked out from'
+ required: false
+ type: string
+ default: main
+ tracking_repo:
+ description: 'Repository of the tracking issue'
+ required: false
+ type: string
+ default: control-toolbox/CTActions
+ secrets:
+ CT_APP_CLIENT_ID:
+ description: 'Client ID of the GitHub App that publishes (not needed for a dry run)'
+ required: false
+ CT_APP_PRIVATE_KEY:
+ description: 'Private key of that GitHub App'
+ required: false
+
+jobs:
+ audit:
+ runs-on: ubuntu-latest
+ timeout-minutes: 120
+ permissions:
+ contents: read
+ env:
+ PACKAGES: ${{ inputs.packages }}
+ JULIA_VERSIONS: ${{ inputs.julia_versions }}
+ SCRIPTS: ctactions/scripts/compat-prune
+ OUT: ${{ github.workspace }}/compat-prune
+ GH_TOKEN: ${{ github.token }}
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ repository: control-toolbox/CTActions
+ ref: ${{ inputs.ctactions_ref }}
+ path: ctactions
+
+ - uses: julia-actions/install-juliaup@v3
+ with:
+ channel: release
+
+ - name: Install the audited Julia versions
+ run: |
+ for v in $(jq -r '.[]' <<< "$JULIA_VERSIONS"); do juliaup add "$v"; done
+ juliaup status
+
+ # ct-registry is public: added over HTTPS, no SSH key needed. The registries are
+ # updated once more by audit.jl, then frozen for the whole audit.
+ - name: Add the registries
+ run: |
+ julia --startup-file=no -e 'using Pkg
+ Pkg.Registry.add("General")
+ Pkg.Registry.add(url="https://github.com/control-toolbox/ct-registry.git")'
+
+ - name: Test the scripts
+ run: julia --startup-file=no "$SCRIPTS/test/runtests.jl"
+
+ - name: Audit
+ run: julia --startup-file=no "$SCRIPTS/audit.jl" "$PACKAGES" "$JULIA_VERSIONS" "$OUT"
+
+ - name: Apply the prunes to fresh checkouts of the default branches
+ run: |
+ "$SCRIPTS/checkout.sh" "$OUT" checkouts
+ julia --startup-file=no "$SCRIPTS/edit.jl" "$OUT" checkouts
+ for d in checkouts/*/; do
+ git -C "$d" diff --src-prefix="a/$(basename "$d")/" --dst-prefix="b/$(basename "$d")/"
+ done > "$OUT/edits.diff"
+
+ - name: Report
+ run: |
+ RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
+ julia --startup-file=no "$SCRIPTS/report.jl" "$OUT"
+ {
+ cat "$OUT/report.md"
+ echo
+ echo "Edits on the default branches (diff)
"
+ echo
+ echo '```diff'
+ cat "$OUT/edits.diff"
+ echo '```'
+ echo "Entries left as they are ($(length(left)))
\n\n" *
+ table(["entry", "compat", "status", "why"], [[entry(r), code(r["before"]), r["status"], why(r)] for r in left]) *
+ "\n\nEntries left as they are (2)
+
+| entry | compat | status | why |
+| --- | --- | --- | --- |
+| Alpha `test/Project.toml` `Gpu` | `5, 6` | idle | not resolved where the entry applies |
+| Beta `Project.toml` `Dep3` | `0.9, 1` | keep | every lane in use: 1.2.0 on 1.12; 0.9.4 on 1.10 |
+
+", b)
+ @test occursin("| to prune (open PRs) | 0 |", b)
+ for h in ("## To prune", "## Merged", "## Drift", "## Rejected", "## Withheld", "## Blocked")
+ @test !occursin(h, b)
+ end
+ @test !occursin("[run]", b)
+ @test occursin("## Audited versions", b)
+ end
+
+ @testset "helpers" begin
+ @test table(["a", "b"], [["1", "2"]]) == "| a | b |\n| --- | --- |\n| 1 | 2 |"
+ @test selected(Dict("0.9" => ["CTBase/test@1.10=0.9.18"], "0.12" => ["CTBase/test@1.12=0.12.3", "joint@1.13=0.12.3"])) ==
+ "0.12.3 on 1.12, 1.13; 0.9.18 on 1.10"
+ @test selected(Dict{String,Any}()) == ""
+ end
+end
diff --git a/scripts/compat-prune/test/test_validate.jl b/scripts/compat-prune/test/test_validate.jl
new file mode 100644
index 0000000..819fd24
--- /dev/null
+++ b/scripts/compat-prune/test/test_validate.jl
@@ -0,0 +1,107 @@
+# validate.jl: departures, rejection of failing prunes (fake simulator), blockers.
+
+"""
+A fake simulator: the baseline `versions` of each environment, departed from (`Tool` moves to
+9.9.9) when `breaks(name, narrowings)` holds. Records every call.
+"""
+function fake_sim(baseline; breaks=(name, n) -> false, error_for=Dict())
+ calls = []
+ sim = function (name, env, n, uuids)
+ push!(calls, (name, copy(n)))
+ haskey(error_for, name) && return failed(error_for[name])
+ versions = copy(baseline[name]["versions"])
+ breaks(name, n) && (versions["Tool"] = "9.9.9")
+ return Dict("ok" => true, "versions" => versions)
+ end
+ return sim, calls
+end
+
+@testset "validate" begin
+ @testset "departure" begin
+ base = Dict("A" => "1.0.0", "B" => "2.0.0")
+ @test departure(ok("A" => "1.0.0", "B" => "2.0.0"), base) === nothing
+ @test departure(ok("A" => "1.0.0", "B" => "2.1.0"), base) == "manifest differs: B 2.0.0 → 2.1.0"
+ @test departure(ok("A" => "1.0.0"), base) == "manifest differs: B 2.0.0 → —"
+ @test departure(ok("A" => "1.0.0", "B" => "2.0.0", "C" => "3.0.0"), base) == "manifest differs: C — → 3.0.0"
+ @test departure(failed("empty compat intersection for CUDSS"), base) == "empty compat intersection for CUDSS"
+ many = Dict("P$i" => "1.0.$i" for i in 1:9)
+ @test endswith(departure(ok(), many), ", …")
+ end
+
+ @testset "blockers" begin
+ @test blockers(CONFLICT_ON_DEP, "DocStringExtensions") == ["SciMLBase"]
+ @test blockers(CONFLICT_ON_OTHER, "DocStringExtensions") == ["DocumenterVitepress"]
+ @test isempty(blockers("empty compat intersection for CUDSS", "CUDSS"))
+ end
+
+ with_collected() do root
+ envs, audited, uuids = load_collected(root)
+ baseline = Dict(name => r for (k, r) in fixture_results() for name in (first(split(k, "@")),) if endswith(k, "@1.12"))
+ dep1 = prune("Alpha", "Project.toml", "Dep1", "0.5, 0.6, 0.7", "0.6, 0.7")
+ tool = prune("Alpha", "test/Project.toml", "Tool", "1, 2", "2")
+
+ @testset "nothing departs" begin
+ sim, calls = fake_sim(baseline)
+ @test isempty(validate_prunes(envs, audited, uuids, [dep1, tool], baseline, "1.12"; sim))
+ # one combined simulation per environment where a prune applies: Dep1 wherever
+ # Alpha is (not Beta/test), Tool in Alpha/test only
+ @test calls == [("Alpha/docs", Dict("Dep1" => "0.6, 0.7")),
+ ("Alpha/test", Dict("Dep1" => "0.6, 0.7", "Tool" => "2")),
+ ("joint", Dict("Dep1" => "0.6, 0.7"))]
+ end
+
+ @testset "a culprit fails alone" begin
+ sim, calls = fake_sim(baseline; breaks=(name, n) -> get(n, "Tool", "") == "2")
+ rejected = validate_prunes(envs, audited, uuids, [dep1, tool], baseline, "1.12"; sim)
+ @test length(rejected) == 1
+ r = only(rejected)
+ @test entry_key(r) == entry_key(tool)
+ @test r["where"] == "Alpha/test@1.12"
+ @test r["reason"] == "manifest differs: Tool 2.0.0 → 9.9.9"
+ end
+
+ @testset "prunes that fail only together are all rejected" begin
+ sim, _ = fake_sim(baseline; breaks=(name, n) -> haskey(n, "Tool") && haskey(n, "Dep1"))
+ rejected = validate_prunes(envs, audited, uuids, [dep1, tool], baseline, "1.12"; sim)
+ @test sort(entry_key.(rejected)) == sort(entry_key.([dep1, tool]))
+ @test all(r -> startswith(r["reason"], "fails only combined with other prunes"), rejected)
+ end
+
+ @testset "a rejected prune is not simulated again in later environments" begin
+ # Dep1 breaks everywhere: rejected in the first environment (sorted: Alpha/docs),
+ # then absent from the simulations of the next ones.
+ sim, calls = fake_sim(baseline; breaks=(name, n) -> haskey(n, "Dep1"))
+ rejected = validate_prunes(envs, audited, uuids, [dep1], baseline, "1.12"; sim)
+ @test only(rejected)["where"] == "Alpha/docs@1.12"
+ # combined + alone in Alpha/docs, then nothing left to simulate
+ @test length(calls) == 2
+ end
+
+ @testset "environments that failed at baseline are skipped" begin
+ b = copy(baseline)
+ b["joint"] = failed()
+ sim, calls = fake_sim(baseline)
+ validate_prunes(envs, audited, uuids, [dep1], b, "1.12"; sim)
+ @test !("joint" in first.(calls))
+ end
+
+ @testset "probe_blockers" begin
+ blocked = [Dict("owner" => "Alpha", "file" => "docs/Project.toml", "dep" => "Doc",
+ "newest" => "1", "applies_in" => ["Alpha/docs", "Alpha/test"])]
+ msg = replace(CONFLICT_ON_OTHER, "DocStringExtensions" => "Doc")
+ sim, calls = fake_sim(baseline; error_for=Dict("Alpha/docs" => msg))
+ found = probe_blockers(envs, uuids, blocked, baseline, "1.12"; sim)
+ # Alpha/test does not have Doc: not probed
+ @test calls == [("Alpha/docs", Dict("Doc" => "1"))]
+ f = only(found)
+ @test f["where"] == "Alpha/docs@1.12"
+ @test f["blockers"] == ["DocumenterVitepress"]
+ @test f["newest"] == "1"
+ # restricted to the newest, it resolves: the resolver merely prefers other versions
+ sim, _ = fake_sim(baseline)
+ f = only(probe_blockers(envs, uuids, blocked, baseline, "1.12"; sim))
+ @test isempty(f["blockers"])
+ @test f["detail"] == "resolves, but the resolver prefers other versions"
+ end
+ end
+end
diff --git a/scripts/compat-prune/validate.jl b/scripts/compat-prune/validate.jl
new file mode 100644
index 0000000..9bc678d
--- /dev/null
+++ b/scripts/compat-prune/validate.jl
@@ -0,0 +1,119 @@
+# Validate the prunes and identify the blockers, on the running Julia.
+#
+# usage: julia validate.jl [--blockers]
+#
+# Prunes: all of them (minus those already rejected) are simulated together; an environment
+# whose manifest differs from the baseline, or that no longer resolves, is a failure. The
+# prunes that apply in a failing environment are then simulated one by one there: those that
+# fail alone are rejected; if none does, the failure comes from their combination and all of
+# them are rejected. `audit.jl` repeats until no Julia version rejects anything.
+#
+# Blockers (`--blockers`, given by `audit.jl` on the newest Julia version only, as they are
+# information and cost one resolution per environment): for each blocked entry, each
+# environment where it applies is resolved with the dependency restricted to the newest
+# specifier; the resolver's explanation names the packages that cap it.
+
+include("common.jl")
+
+"Short description of how `r` departs from the baseline versions `base`."
+function departure(r, base)
+ r["ok"] || return first(r["error"], 600)
+ d = sort(["$k $(get(base, k, "—")) → $(get(r["versions"], k, "—"))"
+ for k in union(keys(base), keys(r["versions"])) if get(base, k, "") != get(r["versions"], k, "")])
+ isempty(d) ? nothing : "manifest differs: " * join(first(d, 6), ", ") * (length(d) > 6 ? ", …" : "")
+end
+
+"""
+ blockers(msg, dep) -> Vector{String}
+
+Packages that cap `dep`, read from the resolver's error when `dep` is restricted to its newest
+specifier. The resolver reports the first conflict it meets, not every one: when that conflict
+is on another package, that package is the blocker (its compat excludes the newest `dep`);
+when it is on `dep` itself, the blockers are the first-level `restricted by compatibility
+requirements with …` lines of its log (deeper lines explain those blockers, they are not
+blockers of `dep`).
+"""
+function blockers(msg, dep)
+ head = match(r"Unsatisfiable requirements detected for package (\S+) \[", msg)
+ head === nothing && return String[]
+ head[1] == dep || return [String(head[1])]
+ first_level = r"^ [├└]─restricted by compatibility requirements with (\S+) \["m
+ return sort(unique(String[m[1] for m in eachmatch(first_level, msg)]))
+end
+
+
+"""
+ validate_prunes(envs, audited, uuids, prunes, baseline, julia; sim=simulate) -> rejected
+
+Simulate `prunes` together in every environment of `envs` that resolved at `baseline`; in an
+environment that departs from its baseline, simulate the prunes that apply there one by one
+and reject those that fail alone, or all of them when none does. Returns the rejected
+records (`owner`, `file`, `dep`, `where`, `reason`).
+"""
+function validate_prunes(envs, audited, uuids, prunes, baseline, julia; sim=simulate)
+ prunes = copy(prunes)
+ rejected = Dict{String,Any}[]
+ for name in sort(collect(keys(envs)))
+ base = baseline[name]
+ base["ok"] || continue
+ n = narrowings(prunes, name, base["versions"], audited)
+ isempty(n) && continue
+ why = departure(sim(name, envs[name], n, uuids), base["versions"])
+ why === nothing && continue
+ println(julia, " ", rpad(name, 24), "fails together: ", why)
+ suspects = [p for p in prunes if !isempty(narrowings([p], name, base["versions"], audited))]
+ alone = Dict{Any,String}()
+ for p in suspects
+ w = departure(sim(name, envs[name], narrowings([p], name, base["versions"], audited), uuids), base["versions"])
+ w === nothing || (alone[entry_key(p)] = w)
+ end
+ culprits = isempty(alone) ? suspects : [p for p in suspects if haskey(alone, entry_key(p))]
+ for p in culprits
+ reason = get(alone, entry_key(p), "fails only combined with other prunes: $why")
+ push!(rejected, Dict("owner" => p["owner"], "file" => p["file"], "dep" => p["dep"],
+ "where" => "$name@$julia", "reason" => reason))
+ println(" rejected: $(p["owner"]) $(p["file"]) $(p["dep"]) → $(p["after"])")
+ end
+ gone = Set(entry_key.(culprits))
+ filter!(p -> !(entry_key(p) in gone), prunes)
+ end
+ return rejected
+end
+
+"""
+ probe_blockers(envs, uuids, blocked, baseline, julia; sim=simulate) -> probes
+
+For each blocked entry, resolve every environment where it applies with the dependency
+restricted to its newest specifier, and read the blockers from the resolver's error.
+"""
+function probe_blockers(envs, uuids, blocked, baseline, julia; sim=simulate)
+ found = Dict{String,Any}[]
+ for b in blocked, name in b["applies_in"]
+ base = baseline[name]
+ base["ok"] && haskey(base["versions"], b["dep"]) || continue
+ r = sim(name, envs[name], Dict(b["dep"] => b["newest"]), uuids)
+ push!(found, Dict("owner" => b["owner"], "file" => b["file"], "dep" => b["dep"],
+ "newest" => b["newest"], "where" => "$name@$julia",
+ "blockers" => r["ok"] ? String[] : blockers(r["error"], b["dep"]),
+ "detail" => r["ok"] ? "resolves, but the resolver prefers other versions" : first(r["error"], 2000)))
+ println(julia, " blocked: $(b["owner"]) $(b["file"]) $(b["dep"]) $(b["newest"]) in $name, by ",
+ r["ok"] ? "nothing (resolver preference)" : join(found[end]["blockers"], ", "))
+ end
+ return found
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+ # usage: … [--blockers]
+ root, classified, basefile, outfile, rejectedfile = ARGS[1:5]
+ rejected_before = Set(entry_key.(get(TOML.parsefile(rejectedfile), "rejected", [])))
+ envs, audited, uuids = load_collected(root)
+ c = TOML.parsefile(classified)
+ prunes = [p for p in c["prune"] if !(entry_key(p) in rejected_before)]
+ baseline = TOML.parsefile(basefile)["env"]
+ julia = join(split(string(VERSION), ".")[1:2], ".")
+ rejected = validate_prunes(envs, audited, uuids, prunes, baseline, julia)
+ blocked = [b for b in get(c, "other", []) if b["status"] == "blocked"]
+ found = "--blockers" in ARGS ? probe_blockers(envs, uuids, blocked, baseline, julia) : Dict{String,Any}[]
+ open(io -> TOML.print(io, Dict("julia" => string(VERSION), "rejected" => rejected, "blocked" => found)), outfile, "w")
+ println(julia, " ", length(rejected), " prune(s) rejected, ", length(found), " blocker probe(s)")
+end