From 87b52177241c239a54e738ad44e2fbca02e1a5f7 Mon Sep 17 00:00:00 2001 From: Kirby <996@itdog.icu> Date: Thu, 5 Nov 2026 01:17:48 +0800 Subject: [PATCH] feat: schedule verified consumer synchronization with explicit source coverage Co-Authored-By: Claude Opus 5 --- .github/workflows/consumer-sync.yml | 53 ++++++++++++++ .github/workflows/validate.yml | 30 ++++++++ consumer-sources.json | 29 ++++++++ docs/CONSUMER_SYNC.md | 49 +++++++++++++ scripts/consumer-sync/pr.mjs | 33 +++++++++ scripts/consumer-sync/protocol.mjs | 63 +++++++++++++++++ scripts/consumer-sync/sync.mjs | 105 ++++++++++++++++++++++++++++ 7 files changed, 362 insertions(+) create mode 100644 .github/workflows/consumer-sync.yml create mode 100644 consumer-sources.json create mode 100644 docs/CONSUMER_SYNC.md create mode 100644 scripts/consumer-sync/pr.mjs create mode 100644 scripts/consumer-sync/protocol.mjs create mode 100644 scripts/consumer-sync/sync.mjs diff --git a/.github/workflows/consumer-sync.yml b/.github/workflows/consumer-sync.yml new file mode 100644 index 0000000..5906893 --- /dev/null +++ b/.github/workflows/consumer-sync.yml @@ -0,0 +1,53 @@ +name: Synchronize published consumers +on: + schedule: + - cron: '43 * * * *' + workflow_dispatch: +permissions: + contents: write + pull-requests: write + actions: write +concurrency: + group: consumer-sync + cancel-in-progress: false +jobs: + synchronize: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-node@v7 + with: + node-version: 24 + - name: Reuse one open synchronization PR + env: + GH_TOKEN: ${{ github.token }} + run: | + set -eu + BRANCH=$(gh pr list --state open --json headRefName --jq '[.[] | select(.headRefName | startswith("automation/consumer-sync-"))][0].headRefName // ""') + if [ -z "$BRANCH" ]; then + # Recover a pushed branch whose PR creation failed on the prior run. + for REF in $(git for-each-ref --sort=-refname --format='%(refname:short)' 'refs/remotes/origin/automation/consumer-sync-*'); do + CANDIDATE=${REF#origin/} + CLOSED=$(gh pr list --state all --head "$CANDIDATE" --json number --jq length) + if [ "$CLOSED" = 0 ] && ! git merge-base --is-ancestor "$REF" origin/main; then BRANCH=$CANDIDATE; break; fi + done + fi + if [ -n "$BRANCH" ]; then + printf '%s\n' "$BRANCH" | grep -Eq '^automation/consumer-sync-[0-9]+$' || exit 1 + git fetch origin "$BRANCH" + git checkout -b "$BRANCH" FETCH_HEAD + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git merge origin/main --no-edit + else + BRANCH="automation/consumer-sync-$(date -u +%Y%m%d%H%M%S)" + fi + echo "SYNC_BRANCH=$BRANCH" >> "$GITHUB_ENV" + - name: Verify registered feeds and stage complete consumer updates + run: node scripts/consumer-sync/sync.mjs + - name: Create or update normal PR and dispatch exact-head validation + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/consumer-sync/pr.mjs diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 48abb1f..9b5b358 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -2,6 +2,12 @@ name: Validate catalog on: push: pull_request: + workflow_dispatch: + inputs: + expected_head: + description: 'Exact synchronization commit to validate' + required: true + type: string permissions: contents: read jobs: @@ -9,7 +15,31 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Bind dispatched validation to its exact head + if: github.event_name == 'workflow_dispatch' + env: + EXPECTED_HEAD: ${{ inputs.expected_head }} + ACTUAL_HEAD: ${{ github.sha }} + GH_TOKEN: ${{ github.token }} + run: | + set -eu + test "$EXPECTED_HEAD" = "$ACTUAL_HEAD" + test "$(git rev-parse HEAD)" = "$EXPECTED_HEAD" + REMOTE=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$GITHUB_REF_NAME" --jq .object.sha) + test "$REMOTE" = "$EXPECTED_HEAD" - uses: actions/setup-node@v4 with: node-version: '24' - run: npm test + - name: Check synchronization tooling syntax + run: | + for file in scripts/consumer-sync/*.mjs; do node --check "$file"; done + - name: Reject a moved synchronization branch + if: github.event_name == 'workflow_dispatch' + env: + EXPECTED_HEAD: ${{ inputs.expected_head }} + GH_TOKEN: ${{ github.token }} + run: | + set -eu + REMOTE=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/$GITHUB_REF_NAME" --jq .object.sha) + test "$REMOTE" = "$EXPECTED_HEAD" diff --git a/consumer-sources.json b/consumer-sources.json new file mode 100644 index 0000000..eefd1d3 --- /dev/null +++ b/consumer-sources.json @@ -0,0 +1,29 @@ +[ + { + "market": "plugins", + "channel": "incremental-prerelease", + "source": "cortex-epd", + "ids": [ + "cortex-epd" + ], + "archivePrefixes": [ + "/experts/cortex-epd/v" + ] + }, + { + "source": "cortex-org-wiki", + "blocked": "Compatible consumer publisher/discovery not established; current pins preserved. Recovery: HiQ-AI/plugins#8." + }, + { + "source": "hiq-editor", + "blocked": "Compatible consumer publisher/discovery not established; current pins preserved. Recovery: HiQ-AI/plugins#8." + }, + { + "source": "jimu-lca", + "blocked": "Compatible consumer publisher/discovery not established; current pins preserved. Recovery: HiQ-AI/plugins#8." + }, + { + "source": "cortex-lca-expert", + "blocked": "Compatible consumer publisher/discovery not established; current pins preserved. Recovery: HiQ-AI/plugins#8." + } +] diff --git a/docs/CONSUMER_SYNC.md b/docs/CONSUMER_SYNC.md new file mode 100644 index 0000000..77278da --- /dev/null +++ b/docs/CONSUMER_SYNC.md @@ -0,0 +1,49 @@ +# Automatic consumer synchronization + +This market checks its registered public consumer feed every hour at minute 43 +and supports a manual `Synchronize published consumers` run. It updates complete +plugin packages and exact CLI pins through one normal pull request. Automatic +merge is disabled; reviewing and merging remains separate from discovering an +update. Updating the market does not upgrade a user's installed CLI or restart a +session. + +`consumer-sources.json` registers source identity, release channel, plugin IDs and +allowed archive prefixes. `consumer-lock.json` records accepted consumer sequence, +producer revision and bundle digest. The producer publishes an immutable public +bundle first and updates its short-cache discovery pointer last. Consumer package +versions are independent of the CLI version, without changing executable version or rebuilding native programs. This market +currently registers EPD incremental prereleases. EPD skills remain bound to the +published executable archive; independent EPD guidance updates are not supported +until a compatible host-guidance contract exists. + +The synchronization script verifies identity, channel, SHA256, safe paths, +immutable revision and nondecreasing CLI versions. Complete source updates are +staged and checked with the existing catalog generator before replacing current +files. Downloaded scripts are not run. A bad feed preserves the existing source; +independent valid feeds may still produce an update. An unchanged feed produces +no empty commit or new PR. + +The bot uses this repository's GITHUB_TOKEN only. Because token-created PR events +do not start normal PR CI, it explicitly dispatches the existing static validation +workflow on the exact pushed head. Validation checks that head before and after +running the catalog check. No synthetic successful status or bypass is used. The +workflow neither approves nor merges PRs. If repository policy denies PR creation, +the pushed automation branch is retained and reused on the next run; no personal +or cross-repository token is substituted. + +For recovery, inspect the workflow summary and producer release. Fix the producer +and publish a new valid consumer revision rather than rewriting immutable bytes +or deleting lock entries. Retain source/channel boundaries during rollback; a CLI +downgrade requires a deliberate reviewed change outside automatic synchronization. + +The producer implementation and authority are maintained in +[HiQ-AI/cortex](https://github.com/HiQ-AI/cortex/blob/main/docs/PUBLIC_CONSUMER_SYNC.md). +This repository carries only reviewed consumer tooling and public packages. + +## Unconnected supply sources + +The registered cortex-org-wiki, hiq-editor, jimu-lca and cortex-lca-expert entries +are explicitly blocked, not automatically updated. Their current pins and files +are retained. The workflow summary lists them separately from actual feed errors; +a healthy EPD no-op is not failed because these dependencies remain open. See +[compatible publisher recovery](https://github.com/HiQ-AI/plugins/issues/8). diff --git a/scripts/consumer-sync/pr.mjs b/scripts/consumer-sync/pr.mjs new file mode 100644 index 0000000..df9ab9e --- /dev/null +++ b/scripts/consumer-sync/pr.mjs @@ -0,0 +1,33 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import assert from 'node:assert/strict'; + +const run = (command, args, allowFailure = false) => { + const result = spawnSync(command, args, { encoding: 'utf8' }); + if (!allowFailure) assert.equal(result.status, 0, result.stderr || result.stdout); + return result; +}; +const branch = process.env.SYNC_BRANCH; +assert.match(branch, /^automation\/consumer-sync-[0-9]+$/); +run('git', ['config', 'user.name', 'github-actions[bot]']); +run('git', ['config', 'user.email', '41898282+github-actions[bot]@users.noreply.github.com']); +const status = run('git', ['status', '--porcelain']).stdout; +const ahead = run('git', ['rev-list', '--count', 'origin/main..HEAD']).stdout.trim(); +if (!status.trim() && ahead === '0') { console.log('No consumer changes; no PR or dispatch.'); process.exit(0); } +const files = status.split('\n').filter(Boolean).map(line => line.slice(3)); +assert(files.every(file => file.startsWith('plugins/') || ['marketplace.json', 'cli-release.json', 'consumer-lock.json', 'README.md', '.claude-plugin/marketplace.json'].includes(file)), 'Unexpected sync change'); +run('git', ['checkout', '-B', branch]); +if (files.length) { + run('git', ['add', '--', ...files]); + run('git', ['commit', '-m', 'chore: synchronize published consumer packages']); +} +const head = run('git', ['rev-parse', 'HEAD']).stdout.trim(); +run('git', ['push', 'origin', `HEAD:refs/heads/${branch}`]); +const repo = process.env.GITHUB_REPOSITORY; +const existing = JSON.parse(run('gh', ['pr', 'list', '--repo', repo, '--head', branch, '--state', 'open', '--json', 'number']).stdout); +const body = 'Generated from registered immutable CDN consumer packages. Review content and pins before merging. Automatic merge is disabled. Static validation is explicitly dispatched on this exact automation branch because GITHUB_TOKEN-created PR events do not trigger ordinary PR CI.\n\nHead: '+head; +if (existing.length) run('gh', ['pr', 'edit', String(existing[0].number), '--repo', repo, '--body', body]); +else run('gh', ['pr', 'create', '--repo', repo, '--head', branch, '--base', 'main', '--title', 'Synchronize published plugin consumer packages', '--body', body]); +assert.equal(run('git', ['ls-remote', 'origin', `refs/heads/${branch}`]).stdout.split(/\s/)[0], head, 'Head moved before validation dispatch'); +run('gh', ['workflow', 'run', 'validate.yml', '--repo', repo, '--ref', branch, '-f', `expected_head=${head}`]); +console.log(`Dispatched existing validation for exact head ${head}; merge remains manual.`); diff --git a/scripts/consumer-sync/protocol.mjs b/scripts/consumer-sync/protocol.mjs new file mode 100644 index 0000000..ec65796 --- /dev/null +++ b/scripts/consumer-sync/protocol.mjs @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; + +export const origin = 'https://download.hiq.earth'; +export const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +export const encode = value => Buffer.from(JSON.stringify(value, null, 2) + '\n'); +export function allowedPath(file, ids) { + if (typeof file !== 'string' || file.includes('\\') || file.split('/').some(part => !part || part === '.' || part === '..')) return false; + if (file === 'cli-release.json') return ids.length === 20; + return ids.some(id => file === `plugins/${id}/README.md` || file === `plugins/${id}/.claude-plugin/plugin.json` || + new RegExp(`^plugins/${id}/(?:skills/[a-z0-9-]+/(?:SKILL\\.md|references/[a-z0-9-]+\\.(?:md|json))|agents/[a-z0-9-]+\\.md|SOURCES\\.md)$`).test(file)); +} +export function validateBundle(bundle, registration) { + assert.equal(bundle.schemaVersion, 1); + for (const key of ['market', 'channel', 'source']) assert.equal(bundle[key], registration[key], `Wrong ${key}`); + assert.match(bundle.revision, /^[a-f0-9]{40}$/); + assert.match(bundle.sequence, /^\d{14}$/); + assert.deepEqual(bundle.plugins.map(p => p.id).sort(), [...registration.ids].sort()); + for (const plugin of bundle.plugins) { + assert.equal(plugin.path, `plugins/${plugin.id}`); + assert.equal(plugin.components.cli.name, plugin.id, 'Program identity mismatch'); + assert(Object.keys(plugin.components).every(key => ['cli', 'skills', 'agents'].includes(key)), 'Unknown executable component'); + const cli = plugin.components.cli; + assert(Object.keys(cli).every(key => ['name', 'version', 'format', 'executable', 'skillPaths', 'releases'].includes(key)), 'Unknown CLI metadata'); + assert(['expert-release', undefined].includes(cli.format)); + for (const file of [cli.executable, ...Object.values(cli.skillPaths ?? {})]) assert(typeof file === 'string' && !file.startsWith('/') && !file.includes('\\') && !file.split('/').includes('..'), 'Unsafe archive entrypoint'); + if (registration.channel === 'incremental-prerelease') assert.match(plugin.components.cli.version, /^\d+\.\d+\.\d+-dev\./); + else assert.match(plugin.components.cli.version, /^\d+\.\d+\.\d+$/); + for (const component of [...(plugin.components.skills ?? []), ...(plugin.components.agents ?? [])]) assert.match(component.id, /^[a-z0-9]+(?:-[a-z0-9]+)*$/); + assert.match(plugin.version, /^\d+\.\d+\.\d+(?:[-.][a-zA-Z0-9.-]+)?$/); + for (const release of Object.values(plugin.components.cli.releases)) { + const url = new URL(release.url); + assert.equal(url.origin, origin); + assert(!url.username && !url.password && !url.search && !url.hash, 'Unexpected archive URL credentials or suffix'); + assert(registration.archivePrefixes.some(prefix => url.pathname.startsWith(prefix)), 'Archive outside registered channel'); + assert(url.pathname.endsWith('.tar.gz'), 'Unsupported archive type'); + assert.match(release.sha256, /^[a-f0-9]{64}$/); + } + } + assert(Object.keys(bundle.files).length > 0); + for (const [file, entry] of Object.entries(bundle.files)) { + assert(allowedPath(file, registration.ids), `Disallowed consumer path: ${file}`); + assert.equal(typeof entry.content, 'string'); + assert.equal(hash(entry.content), entry.sha256, `File digest mismatch: ${file}`); + } + for (const plugin of bundle.plugins) { + const manifest = JSON.parse(bundle.files[`${plugin.path}/.claude-plugin/plugin.json`].content); + assert.equal(manifest.name, plugin.id); + assert.equal(manifest.version, plugin.version); + assert(Object.keys(manifest).every(key => ['name', 'displayName', 'version', 'description', 'author', 'repository', 'license', 'skills', 'agents'].includes(key)), 'Executable plugin configuration is not consumer content'); + if (manifest.skills !== undefined) assert.equal(manifest.skills, './skills/'); + if (manifest.agents !== undefined) assert.deepEqual(manifest.agents, (plugin.components.agents ?? []).map(agent => `./agents/${agent.id}.md`)); + for (const skill of plugin.components.skills ?? []) assert(bundle.files[`${plugin.path}/skills/${skill.id}/SKILL.md`]); + for (const agent of plugin.components.agents ?? []) assert(bundle.files[`${plugin.path}/agents/${agent.id}.md`]); + } + return bundle; +} +export async function download(url, prefix) { + assert(url.startsWith(`${origin}${prefix}`), 'URL outside registered feed'); + const response = await fetch(url, { redirect: 'error', cache: 'no-store' }); + assert(response.ok, `HTTP ${response.status}: ${url}`); + return Buffer.from(await response.arrayBuffer()); +} diff --git a/scripts/consumer-sync/sync.mjs b/scripts/consumer-sync/sync.mjs new file mode 100644 index 0000000..2cdd329 --- /dev/null +++ b/scripts/consumer-sync/sync.mjs @@ -0,0 +1,105 @@ +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync, mkdirSync, rmSync, lstatSync, existsSync, renameSync, cpSync } from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { download, hash, validateBundle, encode } from './protocol.mjs'; + +export async function synchronize(root, registry, get = download) { + const catalogPath = path.join(root, 'marketplace.json'); + let catalog = JSON.parse(readFileSync(catalogPath)); + const statePath = path.join(root, 'consumer-lock.json'); + const state = existsSync(statePath) ? JSON.parse(readFileSync(statePath)) : {}; + const failures = [], changed = [], blocked = []; + for (const registration of registry) { + if (registration.blocked) { blocked.push(`${registration.source}: ${registration.blocked}`); continue; } + try { + const prefix = `/plugin-consumers/${registration.market}/${registration.channel}/`; + const pointer = JSON.parse(await get(`https://download.hiq.earth${prefix}latest.json`, prefix)); + assert.equal(pointer.schemaVersion, 1); + for (const key of ['market', 'channel', 'source']) assert.equal(pointer[key], registration[key], `Pointer ${key} mismatch`); + assert.match(pointer.sha256, /^[a-f0-9]{64}$/); + assert.match(pointer.revision, /^[a-f0-9]{40}$/); + assert.match(pointer.sequence, /^\d{14}$/); + assert.equal(pointer.url, `https://download.hiq.earth${prefix}revisions/${pointer.sequence}-${pointer.revision}/bundle.json`); + const previous = state[registration.source]; + if (previous) { + assert(pointer.sequence >= previous.sequence, 'Consumer rollback rejected'); + if (pointer.sequence === previous.sequence) assert.equal(pointer.sha256, previous.sha256, 'Immutable revision changed'); + } + const bytes = await get(pointer.url, prefix); + assert.equal(hash(bytes), pointer.sha256, 'Bundle digest mismatch'); + const bundle = validateBundle(JSON.parse(bytes), registration); + assert.equal(bundle.revision, pointer.revision); + assert.equal(bundle.sequence, pointer.sequence); + if (previous?.sha256 === pointer.sha256) continue; + for (const plugin of bundle.plugins) { + const current = catalog.plugins.find(p => p.id === plugin.id); + assert(current, 'New plugin requires explicit registration'); + const natural = value => value.split(/[^0-9]+/).filter(Boolean).map(Number); + const compare = (a, b) => { + const x = natural(a), y = natural(b); + for (let i = 0; i < Math.max(x.length, y.length); i++) if ((x[i] ?? 0) !== (y[i] ?? 0)) return (x[i] ?? 0) - (y[i] ?? 0); + return 0; + }; + assert(compare(plugin.components.cli.version, current.components.cli.version) >= 0, 'CLI downgrade rejected'); + if (plugin.components.cli.version === current.components.cli.version) assert.deepEqual(plugin.components.cli, current.components.cli, 'Same CLI version changed pins'); + } + // Validate every destination ancestor before staging any source files. + for (const file of Object.keys(bundle.files)) { + let target = root; + for (const component of file.split('/')) { + target = path.join(target, component); + if (existsSync(target)) assert(!lstatSync(target).isSymbolicLink(), 'Linked consumer destination rejected'); + } + } + const stage = path.join(root, '.consumer-stage'); + rmSync(stage, { recursive: true, force: true }); + mkdirSync(stage); + try { + cpSync(path.join(root, 'plugins'), path.join(stage, 'plugins'), { recursive: true }); + mkdirSync(path.join(stage, 'scripts')); + cpSync(path.join(root, 'scripts/catalog.mjs'), path.join(stage, 'scripts/catalog.mjs')); + cpSync(path.join(root, 'README.md'), path.join(stage, 'README.md')); + if (existsSync(path.join(root, 'cli-release.json'))) cpSync(path.join(root, 'cli-release.json'), path.join(stage, 'cli-release.json')); + for (const plugin of bundle.plugins) rmSync(path.join(stage, plugin.path), { recursive: true }); + for (const [file, entry] of Object.entries(bundle.files)) { + const destination = path.join(stage, file); + mkdirSync(path.dirname(destination), { recursive: true }); + writeFileSync(destination, entry.content); + } + const nextCatalog = { ...catalog, plugins: catalog.plugins.map(plugin => bundle.plugins.find(p => p.id === plugin.id) ?? plugin) }; + writeFileSync(path.join(stage, 'marketplace.json'), encode(nextCatalog)); + const validate = spawnSync(process.execPath, ['scripts/catalog.mjs', '--write'], { cwd: stage, encoding: 'utf8' }); + assert.equal(validate.status, 0, validate.stderr); + const verify = spawnSync(process.execPath, ['scripts/catalog.mjs'], { cwd: stage, encoding: 'utf8' }); + assert.equal(verify.status, 0, verify.stderr); + for (const plugin of bundle.plugins) { + const destination = path.join(root, plugin.path); + rmSync(destination, { recursive: true }); + renameSync(path.join(stage, plugin.path), destination); + } + if (bundle.files['cli-release.json']) renameSync(path.join(stage, 'cli-release.json'), path.join(root, 'cli-release.json')); + } finally { rmSync(stage, { recursive: true, force: true }); } + catalog = { ...catalog, plugins: catalog.plugins.map(plugin => bundle.plugins.find(p => p.id === plugin.id) ?? plugin) }; + state[registration.source] = { sequence: pointer.sequence, revision: pointer.revision, sha256: pointer.sha256 }; + changed.push(registration.source); + } catch (error) { failures.push(`${registration.source}: ${error.message}`); } + } + if (changed.length) { + writeFileSync(catalogPath, encode(catalog)); + writeFileSync(statePath, encode(state)); + const result = spawnSync(process.execPath, ['scripts/catalog.mjs', '--write'], { cwd: root, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + const check = spawnSync(process.execPath, ['scripts/catalog.mjs'], { cwd: root, encoding: 'utf8' }); + assert.equal(check.status, 0, check.stderr); + } + return { changed, failures, blocked }; +} +if (process.argv[1]?.endsWith('/sync.mjs')) { + const root = process.cwd(); + const result = await synchronize(root, JSON.parse(readFileSync('consumer-sources.json'))); + console.log(JSON.stringify(result, null, 2)); + if (process.env.GITHUB_STEP_SUMMARY) writeFileSync(process.env.GITHUB_STEP_SUMMARY, `Consumer sync\n\nChanged: ${result.changed.join(', ') || 'none'}\n\n${result.failures.map(f => `- Failed: ${f}`).join('\n')}\n${result.blocked.map(f => `- Not connected: ${f}`).join('\n')}\n`, { flag: 'a' }); + // Partial source failures are visible, while independent valid updates may still get a PR. + if (result.failures.length && !result.changed.length) process.exitCode = 1; +}