diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..7fbd3ed9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# DEN-2957 conformance artifacts are byte-level contracts across platforms. +.github/workflows/github-dependency-inventory-conformance.yml text eol=lf +docs/github-dependency-inventory-conformance.md text eol=lf +fixtures/github-dependency-inventory/** text eol=lf +harness/github_dependency_inventory.py text eol=lf +harness/github_inventory_*.py text eol=lf +harness/test_github_inventory_*.py text eol=lf diff --git a/.github/fleet-cancel/START b/.github/fleet-cancel/START new file mode 100644 index 00000000..5dd50c4a --- /dev/null +++ b/.github/fleet-cancel/START @@ -0,0 +1 @@ +cancel-legacy-fleet-recovery-30972717804 diff --git a/.github/fleet-create/START b/.github/fleet-create/START new file mode 100644 index 00000000..497b266f --- /dev/null +++ b/.github/fleet-create/START @@ -0,0 +1 @@ +One-shot trigger for the audited production .github repository creation run. diff --git a/.github/fleet-preempt/START b/.github/fleet-preempt/START new file mode 100644 index 00000000..847d1a2e --- /dev/null +++ b/.github/fleet-preempt/START @@ -0,0 +1 @@ +preempt-legacy-recovery-30972717804 diff --git a/.github/fleet-resume/START b/.github/fleet-resume/START new file mode 100644 index 00000000..044353d8 --- /dev/null +++ b/.github/fleet-resume/START @@ -0,0 +1 @@ +resume-rate-limited-test-org-fleet-v2 diff --git a/.github/hhm-fleet-dispatch/START b/.github/hhm-fleet-dispatch/START new file mode 100644 index 00000000..53ce6829 --- /dev/null +++ b/.github/hhm-fleet-dispatch/START @@ -0,0 +1 @@ +APPLY_HHM_TEST_FLEET_V1 diff --git a/.github/pr-merge/AUDIT b/.github/pr-merge/AUDIT new file mode 100644 index 00000000..036c76b6 --- /dev/null +++ b/.github/pr-merge/AUDIT @@ -0,0 +1 @@ +One-shot trigger for a read-only audit of the open pull-request fleet. diff --git a/.github/pr-merge/AUDIT-GRAPHQL-READY-GREEN b/.github/pr-merge/AUDIT-GRAPHQL-READY-GREEN new file mode 100644 index 00000000..b6bb3154 --- /dev/null +++ b/.github/pr-merge/AUDIT-GRAPHQL-READY-GREEN @@ -0,0 +1 @@ +One-shot read-only trigger for the current isolated, green, default-branch ready backlog. diff --git a/.github/pr-merge/AUDIT-GRAPHQL-READY-GREEN-V2 b/.github/pr-merge/AUDIT-GRAPHQL-READY-GREEN-V2 new file mode 100644 index 00000000..58247e7e --- /dev/null +++ b/.github/pr-merge/AUDIT-GRAPHQL-READY-GREEN-V2 @@ -0,0 +1 @@ +One-shot read-only trigger for comment-aware, overlap-aware, exact-green ready PR classification. diff --git a/.github/pr-merge/AUDIT-READY-CATEGORIES b/.github/pr-merge/AUDIT-READY-CATEGORIES new file mode 100644 index 00000000..c793272e --- /dev/null +++ b/.github/pr-merge/AUDIT-READY-CATEGORIES @@ -0,0 +1 @@ +One-shot read-only trigger for exact-head categorization of the remaining ready original backlog. diff --git a/.github/pr-merge/AUDIT-READY-ORIGINAL b/.github/pr-merge/AUDIT-READY-ORIGINAL new file mode 100644 index 00000000..eddc7a85 --- /dev/null +++ b/.github/pr-merge/AUDIT-READY-ORIGINAL @@ -0,0 +1 @@ +One-shot read-only trigger for the narrowed ready-backlog audit. diff --git a/.github/pr-merge/AUDIT-READY-V2 b/.github/pr-merge/AUDIT-READY-V2 new file mode 100644 index 00000000..6914574a --- /dev/null +++ b/.github/pr-merge/AUDIT-READY-V2 @@ -0,0 +1 @@ +One-shot read-only trigger for the comment-aware production-ready PR audit. diff --git a/.github/pr-merge/AUDIT-RESOLVED-DRAFTS b/.github/pr-merge/AUDIT-RESOLVED-DRAFTS new file mode 100644 index 00000000..dd111c1c --- /dev/null +++ b/.github/pr-merge/AUDIT-RESOLVED-DRAFTS @@ -0,0 +1 @@ +One-shot read-only trigger for generated draft portfolios whose source dependencies are explicitly resolved. diff --git a/.github/pr-merge/CONFLICT-AUDIT b/.github/pr-merge/CONFLICT-AUDIT new file mode 100644 index 00000000..54ccee8e --- /dev/null +++ b/.github/pr-merge/CONFLICT-AUDIT @@ -0,0 +1 @@ +One-shot trigger for the semantic merge-conflict inventory. diff --git a/.github/pr-merge/MERGE-APP-AUDITED-READY b/.github/pr-merge/MERGE-APP-AUDITED-READY new file mode 100644 index 00000000..e5d2d0f9 --- /dev/null +++ b/.github/pr-merge/MERGE-APP-AUDITED-READY @@ -0,0 +1 @@ +One-shot trigger for ten clean-green original-backlog PRs using short-lived per-owner GitHub App installation tokens and fresh exact-head gates. diff --git a/.github/pr-merge/MERGE-AUDITED-BATCH b/.github/pr-merge/MERGE-AUDITED-BATCH new file mode 100644 index 00000000..d1b76370 --- /dev/null +++ b/.github/pr-merge/MERGE-AUDITED-BATCH @@ -0,0 +1 @@ +Second one-shot trigger after correcting the REST merge-capability field mapping; all PRs are revalidated from scratch. diff --git a/.github/pr-merge/MERGE-GRAPHQL-FROZEN-READY b/.github/pr-merge/MERGE-GRAPHQL-FROZEN-READY new file mode 100644 index 00000000..d8b74b69 --- /dev/null +++ b/.github/pr-merge/MERGE-GRAPHQL-FROZEN-READY @@ -0,0 +1 @@ +One-shot trigger for GraphQL-only exact-head revalidation and merge of the ten clean-green original-backlog candidates. diff --git a/.github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS b/.github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS new file mode 100644 index 00000000..0761ea7e --- /dev/null +++ b/.github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS @@ -0,0 +1 @@ +One-shot trigger for GraphQL discovery, credential-free exact-archive validation, and GraphQL exact-head promotion/merge of factory portfolios declared ready. diff --git a/.github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS-V2 b/.github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS-V2 new file mode 100644 index 00000000..2f6994c5 --- /dev/null +++ b/.github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS-V2 @@ -0,0 +1 @@ +Second one-shot trigger using split GraphQL queries below the node cap, credential-free exact-archive validation, and exact-head GraphQL promotion/merge. diff --git a/.github/pr-merge/MERGE-READY-PORTFOLIOS b/.github/pr-merge/MERGE-READY-PORTFOLIOS new file mode 100644 index 00000000..e8985de9 --- /dev/null +++ b/.github/pr-merge/MERGE-READY-PORTFOLIOS @@ -0,0 +1 @@ +One-shot trigger for discovery, credential-free exact-archive validation, live promotion, and exact-head merge of factory portfolios declared ready. diff --git a/.github/pr-merge/MERGE-READY-PORTFOLIOS-V2 b/.github/pr-merge/MERGE-READY-PORTFOLIOS-V2 new file mode 100644 index 00000000..9f3a6eef --- /dev/null +++ b/.github/pr-merge/MERGE-READY-PORTFOLIOS-V2 @@ -0,0 +1 @@ +Second one-shot trigger with GitHub archive roots normalized to each repository's real name before contract execution. diff --git a/.github/pr-merge/MERGE-RESOLVED-LEGACY-HARNESSES b/.github/pr-merge/MERGE-RESOLVED-LEGACY-HARNESSES new file mode 100644 index 00000000..6f914482 --- /dev/null +++ b/.github/pr-merge/MERGE-RESOLVED-LEGACY-HARNESSES @@ -0,0 +1 @@ +One-shot trigger for exact archive discovery, credential-free validate-plan execution, and exact-head promotion/merge of source-resolved legacy harnesses. diff --git a/.github/pr-merge/PREFLIGHT b/.github/pr-merge/PREFLIGHT new file mode 100644 index 00000000..21a1099a --- /dev/null +++ b/.github/pr-merge/PREFLIGHT @@ -0,0 +1 @@ +One-shot read-only preflight for the exact-head semantic PR fleet merge. diff --git a/.github/pr-merge/PREFLIGHT-2 b/.github/pr-merge/PREFLIGHT-2 new file mode 100644 index 00000000..4bf4e80f --- /dev/null +++ b/.github/pr-merge/PREFLIGHT-2 @@ -0,0 +1 @@ +One-shot read-only preflight trigger. diff --git a/.github/pr-merge/RESOLVE-SHARED-AUTH-24 b/.github/pr-merge/RESOLVE-SHARED-AUTH-24 new file mode 100644 index 00000000..ec95f1f4 --- /dev/null +++ b/.github/pr-merge/RESOLVE-SHARED-AUTH-24 @@ -0,0 +1 @@ +One-shot trigger for the reviewed semantic reconciliation of shared-auth/shared-auth-server.rs#24. diff --git a/.github/pr-merge/RESOLVE-SHARED-AUTH-24-V2 b/.github/pr-merge/RESOLVE-SHARED-AUTH-24-V2 new file mode 100644 index 00000000..25758a91 --- /dev/null +++ b/.github/pr-merge/RESOLVE-SHARED-AUTH-24-V2 @@ -0,0 +1 @@ +Second one-shot trigger with the observed one-file conflict shape. diff --git a/.github/pr-merge/RESOLVE-SHARED-AUTH-24-V3 b/.github/pr-merge/RESOLVE-SHARED-AUTH-24-V3 new file mode 100644 index 00000000..8ad10421 --- /dev/null +++ b/.github/pr-merge/RESOLVE-SHARED-AUTH-24-V3 @@ -0,0 +1 @@ +One-shot trigger using a structural CORS boundary rather than whitespace-sensitive heredoc matching. diff --git a/.github/pr-merge/RESOLVE-SP-INTERFACES-14 b/.github/pr-merge/RESOLVE-SP-INTERFACES-14 new file mode 100644 index 00000000..64824129 --- /dev/null +++ b/.github/pr-merge/RESOLVE-SP-INTERFACES-14 @@ -0,0 +1 @@ +One-shot trigger for combining the restored canonical interfaces with the independent control-WebSocket contract. diff --git a/.github/pr-merge/RESOLVE-SP-INTERFACES-3 b/.github/pr-merge/RESOLVE-SP-INTERFACES-3 new file mode 100644 index 00000000..b7befa0f --- /dev/null +++ b/.github/pr-merge/RESOLVE-SP-INTERFACES-3 @@ -0,0 +1 @@ +One-shot trigger for the reviewed full-project restoration plus PR 3 invariant. diff --git a/.github/pr-merge/RESOLVE-SP-INTERFACES-3-V2 b/.github/pr-merge/RESOLVE-SP-INTERFACES-3-V2 new file mode 100644 index 00000000..6b1719de --- /dev/null +++ b/.github/pr-merge/RESOLVE-SP-INTERFACES-3-V2 @@ -0,0 +1 @@ +Second one-shot trigger using shell-safe README reconciliation. diff --git a/.github/scripts/audit-graphql-ready-green-v2.mjs b/.github/scripts/audit-graphql-ready-green-v2.mjs new file mode 100644 index 00000000..276d53ca --- /dev/null +++ b/.github/scripts/audit-graphql-ready-green-v2.mjs @@ -0,0 +1,205 @@ +import fs from 'node:fs'; + +const token = process.env.FLEET_GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function graphql(query, variables = {}, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'graphql-ready-green-audit-v2', + }, + body: JSON.stringify({ query, variables }), + }); + const value = await response.json().catch(() => null); + if (response.ok && value?.data) return { data: value.data, errors: value.errors || [] }; + last = new Error(`GraphQL failed: ${JSON.stringify(value).slice(0, 1800)}`); + if (attempt === attempts) throw last; + await sleep(attempt * attempt * 800); + } + throw last; +} + +const searchQuery = ` + query ReadyGreen($search: String!, $cursor: String) { + viewer { login } + search(query: $search, type: ISSUE, first: 20, after: $cursor) { + issueCount pageInfo { hasNextPage endCursor } + nodes { + ... on PullRequest { + id number title body url createdAt state isDraft headRefName headRefOid baseRefName + mergeable mergeStateStatus reviewDecision additions deletions changedFiles + author { login } + headRepository { nameWithOwner } + comments(first: 100) { totalCount pageInfo { hasNextPage } nodes { body } } + reviewThreads(first: 100) { pageInfo { hasNextPage } nodes { isResolved } } + reviews(last: 100) { pageInfo { hasPreviousPage } nodes { state author { login } } } + labels(first: 50) { nodes { name } } + files(first: 100) { totalCount pageInfo { hasNextPage } nodes { path changeType additions deletions } } + commits(last: 1) { + nodes { commit { oid statusCheckRollup { state contexts(first: 100) { pageInfo { hasNextPage } nodes { __typename ... on CheckRun { name status conclusion } ... on StatusContext { context state } } } } } } + } + repository { + nameWithOwner isPrivate isArchived viewerPermission defaultBranchRef { name } + mergeCommitAllowed squashMergeAllowed rebaseMergeAllowed + } + } + } + } + rateLimit { remaining resetAt cost } + } +`; +const overlapQuery = ` + query OpenFiles($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + pullRequests(states: OPEN, first: 100) { + pageInfo { hasNextPage } + nodes { number headRefOid files(first: 100) { pageInfo { hasNextPage } nodes { path } } } + } + } + rateLimit { remaining resetAt cost } + } +`; + +const search = 'is:pr is:open author:ORESoftware draft:false created:<2026-08-05T12:45:00Z sort:created-asc'; +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge|keep (?:this )?pr draft/i; +const concernPattern = /(?:still|currently) fail(?:ing|s)?|unresolved|needs? (?:a |an |the )?(?:fix|change|update|review)|regression remains|unsafe to merge|do not approve/i; + +function latestReviewStates(pr) { + const latest = new Map(); + for (const review of pr.reviews?.nodes || []) { + if (!review.author?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.author.login, review.state); + } + return latest; +} +function checksGreen(pr) { + const commit = pr.commits?.nodes?.at(-1)?.commit; + if (!commit || commit.oid !== pr.headRefOid) return false; + const rollup = commit.statusCheckRollup; + if (!rollup || rollup.state !== 'SUCCESS' || rollup.contexts?.pageInfo?.hasNextPage) return false; + const contexts = rollup.contexts?.nodes || []; + if (!contexts.length) return false; + let success = false; + for (const context of contexts) { + if (context.__typename === 'CheckRun') { + if (context.status !== 'COMPLETED' || !['SUCCESS', 'NEUTRAL', 'SKIPPED'].includes(context.conclusion)) return false; + if (context.conclusion === 'SUCCESS') success = true; + } else if (context.__typename === 'StatusContext') { + if (context.state !== 'SUCCESS') return false; + success = true; + } else return false; + } + return success; +} +function pathReasons(pr) { + const reasons = []; + for (const file of pr.files?.nodes || []) { + const pathname = file.path.toLowerCase(); + if (['DELETED', 'RENAMED'].includes(file.changeType)) reasons.push(`destructive:${file.path}`); + if (/(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(pathname)) reasons.push(`private-key-path:${file.path}`); + if (/\.(pem|p12|pfx|jks|keystore|key)$/.test(pathname)) reasons.push(`key-material-path:${file.path}`); + if (/(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(pathname)) reasons.push(`secret-path:${file.path}`); + if (/(^|\/)\.env($|\.)/.test(pathname) && !/\.example$|\.sample$|\.template$/.test(pathname)) reasons.push(`environment-secret-path:${file.path}`); + } + return reasons; +} +const intersects = (left, right) => { + for (const value of left) if (right.has(value)) return true; + return false; +}; + +const all = []; +let cursor = null; +let total = null; +let remaining = null; +do { + const result = await graphql(searchQuery, { search, cursor }); + const data = result.data; + if (data.viewer.login !== 'ORESoftware') throw new Error(`unexpected viewer ${data.viewer.login}`); + total ??= data.search.issueCount; + remaining = data.rateLimit.remaining; + for (const node of data.search.nodes || []) if (node?.repository?.nameWithOwner) all.push(node); + cursor = data.search.pageInfo.hasNextPage ? data.search.pageInfo.endCursor : null; +} while (cursor); + +const prelim = []; +for (const pr of all) { + const repo = pr.repository.nameWithOwner; + const comments = (pr.comments?.nodes || []).map((comment) => comment.body || '').join('\n'); + const labels = (pr.labels?.nodes || []).map((label) => label.name).join(' '); + const reasons = []; + if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(pr.number)) reasons.push('hard-veto'); + if (pr.author?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (pr.state !== 'OPEN' || pr.isDraft) reasons.push('not-open-ready'); + if (pr.repository.isPrivate) reasons.push('private-repository'); + if (pr.repository.isArchived || !['ADMIN', 'MAINTAIN', 'WRITE'].includes(pr.repository.viewerPermission)) reasons.push('permission'); + if (pr.baseRefName !== pr.repository.defaultBranchRef?.name) reasons.push('non-default-base'); + if (pr.headRepository?.nameWithOwner !== repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== 'MERGEABLE' || !['CLEAN', 'UNSTABLE'].includes(pr.mergeStateStatus)) reasons.push(`merge-${pr.mergeable}-${pr.mergeStateStatus}`); + if (pr.comments?.pageInfo?.hasNextPage) reasons.push('too-many-comments'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage || [...latestReviewStates(pr).values()].includes('CHANGES_REQUESTED') || pr.reviewDecision === 'CHANGES_REQUESTED') reasons.push('changes-requested'); + if (pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount !== pr.changedFiles || pr.changedFiles < 1 || pr.changedFiles > 100) reasons.push('file-count'); + if (pr.additions + pr.deletions > 20000) reasons.push('change-too-large'); + if (vetoPattern.test(`${pr.title}\n${pr.body || ''}\n${comments}\n${labels}`) || concernPattern.test(comments)) reasons.push('veto-or-concern'); + if (!checksGreen(pr)) reasons.push('checks-not-green'); + reasons.push(...pathReasons(pr)); + prelim.push({ pr, reasons: [...new Set(reasons)] }); +} + +for (const entry of prelim.filter((value) => value.reasons.length === 0)) { + const pr = entry.pr; + const [owner, name] = pr.repository.nameWithOwner.split('/'); + const result = await graphql(overlapQuery, { owner, name }); + const open = result.data.repository?.pullRequests; + if (!open) { + entry.reasons.push('repository-not-found'); + continue; + } + if (open.pageInfo.hasNextPage) { + entry.reasons.push('too-many-open-prs'); + continue; + } + const mine = new Set((pr.files?.nodes || []).map((file) => file.path)); + for (const other of open.nodes || []) { + if (other.number === pr.number) continue; + if (other.files?.pageInfo?.hasNextPage) { + entry.reasons.push('unbounded-other-pr-files'); + break; + } + if (intersects(mine, new Set((other.files?.nodes || []).map((file) => file.path)))) { + entry.reasons.push(`overlap-with-pr-${other.number}`); + break; + } + } +} + +const results = prelim.map(({ pr, reasons }) => ({ + repo: pr.repository.nameWithOwner, number: pr.number, id: pr.id, title: pr.title, url: pr.url, + head_ref: pr.headRefName, head_sha: pr.headRefOid, base_ref: pr.baseRefName, + additions: pr.additions, deletions: pr.deletions, changed_files: pr.changedFiles, + workflow_paths: (pr.files?.nodes || []).filter((file) => file.path.startsWith('.github/workflows/')).map((file) => file.path), + merge_method: pr.repository.squashMergeAllowed ? 'SQUASH' : pr.repository.rebaseMergeAllowed ? 'REBASE' : pr.repository.mergeCommitAllowed ? 'MERGE' : null, + reasons: [...new Set(reasons)], +})); +const eligible = results.filter((entry) => entry.reasons.length === 0 && entry.merge_method); +const skip = new Map(); +for (const entry of results) for (const reason of entry.reasons) skip.set(reason, (skip.get(reason) || 0) + 1); +fs.mkdirSync('ready-green-audit-v2', { recursive: true }); +fs.writeFileSync('ready-green-audit-v2/candidates.json', JSON.stringify({ schema_version: 1, audited_at: new Date().toISOString(), search_total: total, entries: eligible }, null, 2) + '\n'); +fs.writeFileSync('ready-green-audit-v2/all.json', JSON.stringify({ schema_version: 1, entries: results }, null, 2) + '\n'); +console.log(`READY_GREEN_AUDIT_V2 ${JSON.stringify({ search_total: total, observed: all.length, eligible: eligible.length, graphql_remaining: remaining })}`); +for (const entry of eligible) console.log(`ELIGIBLE ${entry.repo}#${entry.number}\t${entry.head_sha}\tworkflows=${entry.workflow_paths.length}\t${entry.changed_files} files\t${entry.additions + entry.deletions} lines\t${entry.title}`); +console.log('SKIP_REASONS'); +for (const [reason, count] of [...skip.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))) console.log(`${count}\t${reason}`); +if (process.env.GITHUB_STEP_SUMMARY) { + const lines = ['## GraphQL green-ready audit v2', '', `- Search total: ${total}`, `- Observed: ${all.length}`, `- Eligible exact heads: ${eligible.length}`, '', ...[...skip.entries()].sort((a, b) => b[1] - a[1]).slice(0, 25).map(([reason, count]) => `- ${reason}: ${count}`)]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); +} diff --git a/.github/scripts/audit-graphql-ready-green.mjs b/.github/scripts/audit-graphql-ready-green.mjs new file mode 100644 index 00000000..067a5111 --- /dev/null +++ b/.github/scripts/audit-graphql-ready-green.mjs @@ -0,0 +1,189 @@ +import fs from 'node:fs'; + +const token = process.env.FLEET_GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function graphql(query, variables = {}, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'graphql-ready-green-audit', + }, + body: JSON.stringify({ query, variables }), + }); + const value = await response.json().catch(() => null); + if (response.ok && value?.data) return { data: value.data, errors: value.errors || [] }; + last = new Error(`GraphQL failed: ${JSON.stringify(value).slice(0, 1800)}`); + if (attempt === attempts) throw last; + await sleep(attempt * attempt * 800); + } + throw last; +} + +const query = ` + query ReadyGreen($search: String!, $cursor: String) { + viewer { login } + search(query: $search, type: ISSUE, first: 20, after: $cursor) { + issueCount + pageInfo { hasNextPage endCursor } + nodes { + ... on PullRequest { + id number title body url createdAt state isDraft headRefName headRefOid baseRefName + mergeable mergeStateStatus reviewDecision additions deletions changedFiles + author { login } + headRepository { nameWithOwner } + comments(first: 100) { totalCount pageInfo { hasNextPage } nodes { body } } + reviewThreads(first: 100) { pageInfo { hasNextPage } nodes { isResolved } } + reviews(last: 100) { pageInfo { hasPreviousPage } nodes { state author { login } } } + labels(first: 50) { nodes { name } } + files(first: 100) { totalCount pageInfo { hasNextPage } nodes { path changeType additions deletions } } + commits(last: 1) { + nodes { + commit { + oid + statusCheckRollup { + state + contexts(first: 100) { + pageInfo { hasNextPage } + nodes { + __typename + ... on CheckRun { name status conclusion } + ... on StatusContext { context state } + } + } + } + } + } + } + repository { + nameWithOwner isArchived viewerPermission defaultBranchRef { name } + mergeCommitAllowed squashMergeAllowed rebaseMergeAllowed + pullRequests(states: OPEN, first: 1) { totalCount } + } + } + } + } + rateLimit { remaining resetAt cost } + } +`; + +const search = 'is:pr is:open author:ORESoftware draft:false created:<2026-08-05T12:45:00Z sort:created-asc'; +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge|keep (?:this )?pr draft/i; +const concernPattern = /(?:still|currently) fail(?:ing|s)?|unresolved|needs? (?:a |an |the )?(?:fix|change|update|review)|regression remains|unsafe to merge/i; + +function latestReviewStates(pr) { + const latest = new Map(); + for (const review of pr.reviews?.nodes || []) { + if (!review.author?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.author.login, review.state); + } + return latest; +} + +function checksGreen(pr) { + const commit = pr.commits?.nodes?.at(-1)?.commit; + if (!commit || commit.oid !== pr.headRefOid) return false; + const rollup = commit.statusCheckRollup; + if (!rollup || rollup.state !== 'SUCCESS' || rollup.contexts?.pageInfo?.hasNextPage) return false; + const contexts = rollup.contexts?.nodes || []; + if (!contexts.length) return false; + let success = false; + for (const context of contexts) { + if (context.__typename === 'CheckRun') { + if (context.status !== 'COMPLETED' || !['SUCCESS', 'NEUTRAL', 'SKIPPED'].includes(context.conclusion)) return false; + if (context.conclusion === 'SUCCESS') success = true; + } else if (context.__typename === 'StatusContext') { + if (context.state !== 'SUCCESS') return false; + success = true; + } else return false; + } + return success; +} + +function pathReasons(pr) { + const reasons = []; + for (const file of pr.files?.nodes || []) { + const pathname = file.path.toLowerCase(); + if (['DELETED', 'RENAMED'].includes(file.changeType)) reasons.push(`destructive:${file.path}`); + if (pathname.startsWith('.github/workflows/')) reasons.push(`workflow-change:${file.path}`); + if (/(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(pathname)) reasons.push(`private-key-path:${file.path}`); + if (/\.(pem|p12|pfx|jks|keystore|key)$/.test(pathname)) reasons.push(`key-material-path:${file.path}`); + if (/(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(pathname)) reasons.push(`secret-path:${file.path}`); + if (/(^|\/)\.env($|\.)/.test(pathname) && !/\.example$|\.sample$|\.template$/.test(pathname)) reasons.push(`environment-secret-path:${file.path}`); + } + return reasons; +} + +const all = []; +let cursor = null; +let total = null; +let remaining = null; +do { + const result = await graphql(query, { search, cursor }); + const data = result.data; + if (data.viewer.login !== 'ORESoftware') throw new Error(`unexpected viewer ${data.viewer.login}`); + total ??= data.search.issueCount; + remaining = data.rateLimit.remaining; + for (const node of data.search.nodes || []) if (node?.repository?.nameWithOwner) all.push(node); + cursor = data.search.pageInfo.hasNextPage ? data.search.pageInfo.endCursor : null; + console.log(`PAGE observed=${all.length} total=${total} graphql_remaining=${remaining}`); +} while (cursor); + +const results = []; +for (const pr of all) { + const repo = pr.repository.nameWithOwner; + const comments = (pr.comments?.nodes || []).map((comment) => comment.body || '').join('\n'); + const labels = (pr.labels?.nodes || []).map((label) => label.name).join(' '); + const reasons = []; + if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(pr.number)) reasons.push('hard-veto'); + if (pr.author?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (pr.state !== 'OPEN' || pr.isDraft) reasons.push('not-open-ready'); + if (pr.repository.isArchived || !['ADMIN', 'MAINTAIN', 'WRITE'].includes(pr.repository.viewerPermission)) reasons.push('permission'); + if (pr.baseRefName !== pr.repository.defaultBranchRef?.name) reasons.push('non-default-base'); + if (pr.headRepository?.nameWithOwner !== repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== 'MERGEABLE' || pr.mergeStateStatus !== 'CLEAN') reasons.push(`merge-${pr.mergeable}-${pr.mergeStateStatus}`); + if (pr.repository.pullRequests?.totalCount !== 1) reasons.push(`open-pr-count-${pr.repository.pullRequests?.totalCount}`); + if (pr.comments?.totalCount !== 0 || pr.comments?.pageInfo?.hasNextPage) reasons.push('comments'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage || [...latestReviewStates(pr).values()].includes('CHANGES_REQUESTED') || pr.reviewDecision === 'CHANGES_REQUESTED') reasons.push('changes-requested'); + if (pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount !== pr.changedFiles || pr.changedFiles < 1 || pr.changedFiles > 100) reasons.push('file-count'); + if (pr.additions + pr.deletions > 12000) reasons.push('change-too-large'); + if (vetoPattern.test(`${pr.title}\n${pr.body || ''}\n${comments}\n${labels}`) || concernPattern.test(comments)) reasons.push('veto-or-concern'); + if (!checksGreen(pr)) reasons.push('checks-not-green'); + reasons.push(...pathReasons(pr)); + results.push({ + repo, number: pr.number, id: pr.id, title: pr.title, url: pr.url, + head_ref: pr.headRefName, head_sha: pr.headRefOid, base_ref: pr.baseRefName, + additions: pr.additions, deletions: pr.deletions, changed_files: pr.changedFiles, + merge_method: pr.repository.squashMergeAllowed ? 'SQUASH' : pr.repository.rebaseMergeAllowed ? 'REBASE' : pr.repository.mergeCommitAllowed ? 'MERGE' : null, + reasons: [...new Set(reasons)], + }); +} + +const eligible = results.filter((entry) => entry.reasons.length === 0 && entry.merge_method); +const skip = new Map(); +for (const entry of results) for (const reason of entry.reasons) skip.set(reason, (skip.get(reason) || 0) + 1); +fs.mkdirSync('ready-green-audit', { recursive: true }); +fs.writeFileSync('ready-green-audit/candidates.json', JSON.stringify({ schema_version: 1, audited_at: new Date().toISOString(), search_total: total, entries: eligible }, null, 2) + '\n'); +fs.writeFileSync('ready-green-audit/all.json', JSON.stringify({ schema_version: 1, entries: results }, null, 2) + '\n'); +console.log(`READY_GREEN_AUDIT ${JSON.stringify({ search_total: total, observed: all.length, eligible: eligible.length, graphql_remaining: remaining })}`); +console.log('ELIGIBLE'); +for (const entry of eligible) console.log(`${entry.repo}#${entry.number}\t${entry.head_sha}\t${entry.changed_files} files\t${entry.additions + entry.deletions} lines\t${entry.title}`); +console.log('SKIP_REASONS'); +for (const [reason, count] of [...skip.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))) console.log(`${count}\t${reason}`); + +if (process.env.GITHUB_STEP_SUMMARY) { + const lines = [ + '## GraphQL isolated green-ready audit', '', + `- Search total: ${total}`, `- Observed: ${all.length}`, `- Eligible exact heads: ${eligible.length}`, `- GraphQL points remaining: ${remaining}`, '', + '### Leading skip reasons', '', + ...[...skip.entries()].sort((a, b) => b[1] - a[1]).slice(0, 25).map(([reason, count]) => `- ${reason}: ${count}`), + ]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); +} diff --git a/.github/scripts/audit-ready-backlog-categories.mjs b/.github/scripts/audit-ready-backlog-categories.mjs new file mode 100644 index 00000000..11966769 --- /dev/null +++ b/.github/scripts/audit-ready-backlog-categories.mjs @@ -0,0 +1,228 @@ +import fs from 'node:fs'; + +const token = process.env.FLEET_GH_TOKEN || process.env.GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); +const api = 'https://api.github.com'; +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'ready-backlog-category-audit/1.0', +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function request(url, attempts = 6) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { headers }); + const text = await response.text(); + let value; + try { value = text ? JSON.parse(text) : null; } catch { value = text; } + if (response.ok) return value; + last = new Error(`GET ${url} -> ${response.status}: ${String(text).slice(0, 700)}`); + const retryable = response.status === 429 || response.status >= 500 || + (response.status === 403 && /secondary rate limit|abuse detection/i.test(text)); + if (!retryable || attempt === attempts) throw last; + const retryAfter = Number(response.headers.get('retry-after') || 0); + await sleep(Math.max(retryAfter * 1000, attempt * attempt * 700)); + } + throw last; +} + +async function mapLimit(values, limit, fn) { + const results = new Array(values.length); + let next = 0; + async function worker() { + while (true) { + const index = next++; + if (index >= values.length) return; + try { results[index] = await fn(values[index]); } + catch (error) { results[index] = { audit_error: error.message, source: values[index] }; } + } + } + await Promise.all(Array.from({ length: Math.min(limit, values.length) }, worker)); + return results; +} + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const concernPattern = /(?:still|currently) fail(?:ing|s)?|unresolved|needs? (?:a |an |the )?(?:fix|change|update|review)|regression remains|unsafe to merge/i; +const credentialPattern = /(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})/; + +function latestReviewStates(reviews) { + const latest = new Map(); + for (const review of reviews) { + if (!review.user?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.user.login, review.state); + } + return latest; +} + +function sensitivePath(filename) { + const value = filename.toLowerCase(); + return /(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(value) || + /\.(pem|p12|pfx|jks|keystore|key)$/.test(value) || + /(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(value) || + (/(^|\/)\.env($|\.)/.test(value) && !/\.example$|\.sample$|\.template$/.test(value)); +} + +function patchSafety(files) { + const reasons = []; + for (const file of files) { + if (sensitivePath(file.filename)) reasons.push(`sensitive-path:${file.filename}`); + const patch = String(file.patch || ''); + const additions = patch.split('\n').filter((line) => line.startsWith('+') && !line.startsWith('+++')).map((line) => line.slice(1)).join('\n'); + if (file.patch == null && file.status !== 'removed') reasons.push(`unreviewable-patch:${file.filename}`); + if (/^(<<<<<<<|=======|>>>>>>>)( |$)/m.test(patch)) reasons.push(`conflict-marker:${file.filename}`); + if (credentialPattern.test(additions)) reasons.push(`credential-pattern:${file.filename}`); + if (/\.github\/workflows\/.*\.ya?ml$/i.test(file.filename)) { + if (/^\s*pull_request_target\s*:/m.test(additions)) reasons.push(`pull-request-target:${file.filename}`); + if (/^\s*permissions\s*:\s*write-all\s*$/m.test(additions)) reasons.push(`write-all:${file.filename}`); + if (/(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b/i.test(additions)) reasons.push(`remote-shell-pipe:${file.filename}`); + if (/\beval\s+["']?\$/.test(additions)) reasons.push(`eval-shell:${file.filename}`); + } + } + return [...new Set(reasons)]; +} + +async function checkCategory(repo, sha) { + const [runs, combined] = await Promise.all([ + request(`${api}/repos/${repo}/commits/${sha}/check-runs?per_page=100`), + request(`${api}/repos/${repo}/commits/${sha}/status?per_page=100`), + ]); + const checkRuns = runs.check_runs || []; + const statuses = combined.statuses || []; + let pending = 0; + let realFailures = 0; + let billingFailures = 0; + let successes = 0; + for (const run of checkRuns) { + if (run.status !== 'completed') { pending += 1; continue; } + if (['success', 'neutral', 'skipped'].includes(run.conclusion)) { + if (run.conclusion === 'success') successes += 1; + continue; + } + const annotations = await request(`${api}/repos/${repo}/check-runs/${run.id}/annotations?per_page=100`); + const billingOnly = annotations.length > 0 && annotations.every((annotation) => + /job was not started because recent account payments have failed|spending limit needs to be increased/i.test(annotation.message || '') + ); + if (billingOnly) billingFailures += 1; + else realFailures += 1; + } + for (const status of statuses) { + if (status.state === 'pending') pending += 1; + else if (['failure', 'error'].includes(status.state)) realFailures += 1; + else if (status.state === 'success') successes += 1; + } + const category = pending ? 'pending' : realFailures ? 'failed' : billingFailures ? 'billing-only' : successes ? 'green' : 'no-checks'; + return { category, pending, realFailures, billingFailures, successes, total: checkRuns.length + statuses.length }; +} + +async function inspect(item) { + const repo = item.repository_url.replace(`${api}/repos/`, ''); + let pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + for (let attempt = 0; pr.mergeable == null && attempt < 5; attempt += 1) { + await sleep(700 + attempt * 400); + pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + } + const [repository, comments, reviews, files, checks, openPulls] = await Promise.all([ + request(`${api}/repos/${repo}`), + item.comments ? request(`${api}/repos/${repo}/issues/${item.number}/comments?per_page=100`) : Promise.resolve([]), + request(`${api}/repos/${repo}/pulls/${item.number}/reviews?per_page=100`), + request(`${api}/repos/${repo}/pulls/${item.number}/files?per_page=100`), + checkCategory(repo, pr.head.sha), + request(`${api}/repos/${repo}/pulls?state=open&per_page=100`), + ]); + const commentText = comments.map((comment) => comment.body || '').join('\n'); + const allText = `${item.title || ''}\n${item.body || ''}\n${commentText}\n${(item.labels || []).map((label) => label.name).join(' ')}`; + const gates = []; + if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(item.number)) gates.push('hard-veto'); + if (vetoPattern.test(allText)) gates.push('text-veto'); + if (concernPattern.test(commentText)) gates.push('comment-concern'); + if (pr.state !== 'open' || pr.draft) gates.push('not-open-ready'); + if (pr.user?.login !== 'ORESoftware') gates.push('wrong-author'); + if (repository.archived) gates.push('archived'); + if (!(repository.permissions?.admin || repository.permissions?.maintain || repository.permissions?.push)) gates.push('no-write-permission'); + if (pr.head?.repo?.full_name !== repo) gates.push('cross-repository-head'); + if (pr.comments !== 0 || pr.review_comments !== 0) gates.push('comments'); + if ([...latestReviewStates(reviews).values()].includes('CHANGES_REQUESTED')) gates.push('changes-requested'); + if (pr.changed_files < 1 || pr.changed_files > 100 || files.length !== pr.changed_files) gates.push('file-count'); + if (pr.additions + pr.deletions > 30000) gates.push('change-too-large'); + gates.push(...patchSafety(files)); + + const defaultBase = pr.base?.ref === repository.default_branch; + const multipleOpen = openPulls.length > 1; + let bucket = 'other'; + if (gates.length) bucket = 'gated'; + else if (!defaultBase) bucket = 'non-default-base'; + else if (pr.mergeable === false || pr.mergeable_state === 'dirty') bucket = 'conflict'; + else if (pr.mergeable == null || ['unknown', 'blocked'].includes(pr.mergeable_state)) bucket = 'blocked-or-unknown'; + else if (multipleOpen) bucket = 'multiple-open-prs'; + else if (checks.category === 'green' && pr.mergeable === true && ['clean', 'unstable'].includes(pr.mergeable_state)) bucket = 'clean-green'; + else if (checks.category === 'billing-only' && pr.mergeable === true) bucket = 'clean-billing-only'; + else if (checks.category === 'no-checks' && pr.mergeable === true) bucket = 'clean-no-checks'; + else if (checks.category === 'pending') bucket = 'checks-pending'; + else if (checks.category === 'failed') bucket = 'checks-failed'; + + return { + repo, + number: item.number, + node_id: pr.node_id, + title: item.title, + body: item.body || '', + head_ref: pr.head?.ref, + head_sha: pr.head?.sha, + base_ref: pr.base?.ref, + base_sha: pr.base?.sha, + default_branch: repository.default_branch, + mergeable: pr.mergeable, + mergeable_state: pr.mergeable_state, + changed_files: pr.changed_files, + additions: pr.additions, + deletions: pr.deletions, + issue_comments: pr.comments, + review_comments: pr.review_comments, + open_prs_in_repo: openPulls.length, + checks, + gates: [...new Set(gates)], + bucket, + }; +} + +const query = encodeURIComponent('is:pr is:open author:ORESoftware draft:false created:<2026-08-05T12:45:00Z sort:created-asc'); +const byUrl = new Map(); +let total = 0; +for (let page = 1; page <= 10; page += 1) { + const result = await request(`${api}/search/issues?q=${query}&per_page=100&page=${page}`); + total = result.total_count; + for (const item of result.items || []) byUrl.set(item.html_url, item); + if ((result.items || []).length < 100) break; +} +const results = await mapLimit([...byUrl.values()], 10, inspect); +const errors = results.filter((entry) => entry.audit_error); +const entries = results.filter((entry) => !entry.audit_error); +const counts = {}; +for (const entry of entries) counts[entry.bucket] = (counts[entry.bucket] || 0) + 1; +fs.mkdirSync('ready-backlog-audit', { recursive: true }); +fs.writeFileSync('ready-backlog-audit/categories.json', JSON.stringify({ schema_version: 1, audited_at: new Date().toISOString(), search_total: total, entries, errors }, null, 2) + '\n'); + +console.log(`READY_BACKLOG_CATEGORIES ${JSON.stringify({ search_total: total, inspected: entries.length, errors: errors.length, buckets: counts })}`); +for (const bucket of Object.keys(counts).sort()) { + console.log(`BUCKET ${bucket} count=${counts[bucket]}`); + for (const entry of entries.filter((value) => value.bucket === bucket)) { + console.log(`${entry.repo}#${entry.number}\t${entry.head_sha}\tbase=${entry.base_ref}\tstate=${entry.mergeable_state}\tchecks=${entry.checks.category}\topen=${entry.open_prs_in_repo}\tfiles=${entry.changed_files}\t${entry.title}`); + } +} +for (const error of errors.slice(0, 30)) console.log(`AUDIT_ERROR ${JSON.stringify(error)}`); + +if (process.env.GITHUB_STEP_SUMMARY) { + const lines = [ + '## Remaining ready backlog categories', + '', + `- Search total: ${total}`, + `- Inspected: ${entries.length}`, + `- Audit errors: ${errors.length}`, + '', + ...Object.entries(counts).sort((a, b) => b[1] - a[1]).map(([bucket, count]) => `- ${bucket}: ${count}`), + ]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); +} diff --git a/.github/scripts/audit-ready-merge-candidates-v2.mjs b/.github/scripts/audit-ready-merge-candidates-v2.mjs new file mode 100644 index 00000000..a0ea8cd4 --- /dev/null +++ b/.github/scripts/audit-ready-merge-candidates-v2.mjs @@ -0,0 +1,246 @@ +import fs from 'node:fs'; + +const token = process.env.GH_TOKEN; +if (!token) throw new Error('GH_TOKEN is required'); + +const cutoff = '2026-08-05T12:45:00Z'; +const api = 'https://api.github.com'; +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'comment-aware-ready-pr-auditor', +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function request(url, attempts = 6) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { headers }); + const text = await response.text(); + let value; + try { value = text ? JSON.parse(text) : null; } catch { value = text; } + if (response.ok) return value; + last = new Error(`GET ${url} -> ${response.status}: ${String(text).slice(0, 600)}`); + const retryable = response.status === 429 || response.status >= 500 || + (response.status === 403 && /secondary rate limit|abuse detection/i.test(text)); + if (!retryable || attempt === attempts) throw last; + await sleep(attempt * attempt * 700); + } + throw last; +} + +async function mapLimit(values, limit, fn) { + const results = new Array(values.length); + let next = 0; + async function worker() { + while (true) { + const index = next++; + if (index >= values.length) return; + try { results[index] = await fn(values[index]); } + catch (error) { results[index] = { audit_error: error.message, source: values[index] }; } + } + } + await Promise.all(Array.from({ length: Math.min(limit, values.length) }, worker)); + return results; +} + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const concernPattern = /(?:still|currently) fail(?:ing|s)?|unresolved|needs? (?:a |an |the )?(?:fix|change|update|review)|regression remains|unsafe to merge/i; + +function addedLines(patch) { + return String(patch || '') + .split('\n') + .filter((line) => line.startsWith('+') && !line.startsWith('+++')) + .map((line) => line.slice(1)) + .join('\n'); +} + +function sensitivePath(pathname) { + const path = pathname.toLowerCase(); + return /(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(path) || + /\.(pem|p12|pfx|jks|keystore|key)$/.test(path) || + /(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(path) || + (/(^|\/)\.env($|\.)/.test(path) && !/\.example$|\.sample$|\.template$/.test(path)); +} + +function patchReasons(files) { + const reasons = []; + for (const file of files) { + if (sensitivePath(file.filename)) reasons.push(`sensitive-path:${file.filename}`); + const patch = String(file.patch || ''); + const additions = addedLines(patch); + if (file.patch == null && file.status !== 'removed') reasons.push(`unreviewable-patch:${file.filename}`); + if (/^(<<<<<<<|=======|>>>>>>>)( |$)/m.test(patch)) reasons.push(`conflict-marker:${file.filename}`); + if (/(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})/.test(additions)) { + reasons.push(`credential-pattern:${file.filename}`); + } + if (/\.github\/workflows\/.*\.ya?ml$/i.test(file.filename)) { + if (/^\s*pull_request_target\s*:/m.test(additions)) reasons.push(`pull-request-target:${file.filename}`); + if (/^\s*permissions\s*:\s*write-all\s*$/m.test(additions)) reasons.push(`write-all:${file.filename}`); + if (/(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b/i.test(additions)) reasons.push(`remote-shell-pipe:${file.filename}`); + if (/\beval\s+["']?\$/.test(additions)) reasons.push(`eval-shell:${file.filename}`); + } + } + return [...new Set(reasons)]; +} + +function latestReviewStates(reviews) { + const latest = new Map(); + for (const review of reviews) { + if (!review.user?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.user.login, review.state); + } + return latest; +} + +function checkState(runs, status) { + const checkRuns = runs.check_runs || []; + const statuses = status.statuses || []; + if (!checkRuns.length && !statuses.length) return 'none'; + for (const run of checkRuns) { + if (run.status !== 'completed') return 'pending'; + if (!['success', 'neutral', 'skipped'].includes(run.conclusion)) return 'failed'; + } + for (const entry of statuses) { + if (entry.state === 'pending') return 'pending'; + if (['failure', 'error'].includes(entry.state)) return 'failed'; + } + return checkRuns.some((run) => run.conclusion === 'success') || statuses.some((entry) => entry.state === 'success') + ? 'green' + : 'neutral-only'; +} + +function tier(entry) { + const [owner] = entry.repo.split('/'); + const text = `${entry.title}\n${entry.body}`.toLowerCase(); + if (owner.endsWith('-test')) return 'A-test-org'; + if (text.includes('semantic conflict resolution') && /validation|tests?|checks?/.test(text)) return 'B-semantic-resolution'; + if (entry.changed_files <= 10 && entry.additions + entry.deletions <= 2500 && /validation|tests?|checks?/.test(text)) return 'C-small-validated'; + return 'D-other'; +} + +async function inspect(item) { + const repo = item.repository_url.replace(`${api}/repos/`, ''); + let pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + for (let attempt = 0; pr.mergeable == null && attempt < 4; attempt += 1) { + await sleep(700 + attempt * 500); + pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + } + + const [repository, reviews, files, issueComments, runs, status] = await Promise.all([ + request(`${api}/repos/${repo}`), + request(`${api}/repos/${repo}/pulls/${item.number}/reviews?per_page=100`), + request(`${api}/repos/${repo}/pulls/${item.number}/files?per_page=100`), + item.comments ? request(`${api}/repos/${repo}/issues/${item.number}/comments?per_page=100`) : Promise.resolve([]), + request(`${api}/repos/${repo}/commits/${pr.head.sha}/check-runs?per_page=100`), + request(`${api}/repos/${repo}/commits/${pr.head.sha}/status?per_page=100`), + ]); + + const reasons = []; + const labels = item.labels || []; + const body = item.body || ''; + const allCommentText = issueComments.map((comment) => comment.body || '').join('\n'); + const allText = `${item.title || ''}\n${body}\n${allCommentText}`; + const labelText = labels.map((label) => label.name).join(' '); + + if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(item.number)) reasons.push('hard-veto'); + if (vetoPattern.test(allText) || vetoPattern.test(labelText)) reasons.push('text-veto'); + if (concernPattern.test(allCommentText)) reasons.push('comment-concern'); + if (body.toLowerCase().includes('source-gated until:') && !body.toLowerCase().includes('all declared source repositories resolved')) reasons.push('source-gated'); + if (pr.state !== 'open' || pr.draft) reasons.push('not-open-ready'); + if (pr.user?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (repository.archived) reasons.push('archived'); + if (!(repository.permissions?.admin || repository.permissions?.maintain || repository.permissions?.push)) reasons.push('no-write-permission'); + if (pr.base?.ref !== repository.default_branch) reasons.push('non-default-base'); + if (pr.head?.repo?.full_name !== repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== true) reasons.push(`mergeable-${String(pr.mergeable)}`); + if (pr.mergeable_state !== 'clean') reasons.push(`merge-state-${pr.mergeable_state}`); + if (pr.review_comments !== 0) reasons.push('inline-review-comments'); + if ([...latestReviewStates(reviews).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (pr.changed_files < 1 || pr.changed_files > 100 || files.length !== pr.changed_files) reasons.push('file-count'); + if (pr.additions + pr.deletions > 12000) reasons.push('change-too-large'); + const checks = checkState(runs, status); + if (checks !== 'green') reasons.push(`checks-${checks}`); + reasons.push(...patchReasons(files)); + + const entry = { + repo, + number: item.number, + title: item.title, + body, + created_at: item.created_at, + head_sha: pr.head.sha, + base_sha: pr.base.sha, + mergeable_state: pr.mergeable_state, + changed_files: pr.changed_files, + additions: pr.additions, + deletions: pr.deletions, + issue_comments: item.comments, + check_state: checks, + reasons: [...new Set(reasons)], + }; + entry.tier = tier(entry); + return entry; +} + +const query = encodeURIComponent(`is:pr is:open author:ORESoftware draft:false created:<${cutoff} sort:created-asc`); +const byUrl = new Map(); +let total = 0; +for (let page = 1; page <= 10; page += 1) { + const result = await request(`${api}/search/issues?q=${query}&per_page=100&page=${page}`); + total = result.total_count; + for (const item of result.items || []) byUrl.set(item.html_url, item); + if ((result.items || []).length < 100) break; +} +const inspected = await mapLimit([...byUrl.values()], 10, inspect); +const errors = inspected.filter((entry) => entry.audit_error); +const valid = inspected.filter((entry) => !entry.audit_error); +const eligible = valid.filter((entry) => entry.reasons.length === 0); + +const byRepo = new Map(); +for (const entry of eligible.sort((a, b) => String(a.created_at).localeCompare(String(b.created_at)))) { + if (!byRepo.has(entry.repo)) byRepo.set(entry.repo, entry); +} +const unique = [...byRepo.values()]; +const tiers = {}; +for (const entry of unique) tiers[entry.tier] = (tiers[entry.tier] || 0) + 1; +const skip = new Map(); +for (const entry of valid) for (const reason of entry.reasons) skip.set(reason, (skip.get(reason) || 0) + 1); + +console.log(`PRODUCTION_AUDIT_SUMMARY ${JSON.stringify({ + search_total: total, + observed: valid.length, + audit_errors: errors.length, + eligible: eligible.length, + unique_repositories: unique.length, + tiers, +})}`); +console.log('CANDIDATES'); +for (const entry of unique) { + console.log(`${entry.tier}\t${entry.repo}#${entry.number}\t${entry.head_sha}\t${entry.changed_files} files\t${entry.additions + entry.deletions} lines\tcomments=${entry.issue_comments}\t${entry.title}`); +} +console.log('SKIP_REASONS'); +for (const [reason, count] of [...skip.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))) { + console.log(`${count}\t${reason}`); +} +for (const error of errors.slice(0, 20)) console.log(`AUDIT_ERROR ${JSON.stringify(error)}`); + +const lines = [ + '## Comment-aware ready PR audit', + '', + `- Search total: ${total}`, + `- Inspected: ${valid.length}`, + `- Audit errors: ${errors.length}`, + `- Eligible exact heads: ${eligible.length}`, + `- Unique eligible repositories: ${unique.length}`, + '', + '### Tiers', + '', + ...Object.entries(tiers).map(([name, count]) => `- ${name}: ${count}`), + '', + '### Leading skip reasons', + '', + ...[...skip.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20).map(([reason, count]) => `- ${reason}: ${count}`), +]; +if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join('\n')}\n`); diff --git a/.github/scripts/audit-ready-original-prs.mjs b/.github/scripts/audit-ready-original-prs.mjs new file mode 100644 index 00000000..d07abd40 --- /dev/null +++ b/.github/scripts/audit-ready-original-prs.mjs @@ -0,0 +1,300 @@ +import fs from 'node:fs'; + +const token = process.env.GH_TOKEN; +if (!token) throw new Error('GH_TOKEN is required'); + +const cutoff = '2026-08-05T12:45:00Z'; +const api = 'https://api.github.com'; +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'ready-original-pr-auditor', +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function request(url, options = {}, attempts = 6) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { + ...options, + headers: { ...headers, ...(options.headers || {}) }, + }); + const text = await response.text(); + let value = null; + try { value = text ? JSON.parse(text) : null; } catch { value = text; } + if (response.ok) return value; + last = new Error(`${options.method || 'GET'} ${url} -> ${response.status}: ${String(text).slice(0, 800)}`); + const retryable = response.status === 429 || response.status >= 500 || + (response.status === 403 && /secondary rate limit|abuse detection/i.test(text)); + if (!retryable || attempt === attempts) throw last; + const retryAfter = Number(response.headers.get('retry-after') || 0); + await sleep(Math.max(retryAfter * 1000, attempt * attempt * 700)); + } + throw last; +} + +async function mapLimit(values, limit, fn) { + const results = new Array(values.length); + let next = 0; + async function worker() { + while (true) { + const index = next++; + if (index >= values.length) return; + try { results[index] = await fn(values[index], index); } + catch (error) { results[index] = { audit_error: error.message, source: values[index] }; } + } + } + await Promise.all(Array.from({ length: Math.min(limit, values.length) }, worker)); + return results; +} + +function repoFromItem(item) { + return item.repository_url.replace(`${api}/repos/`, ''); +} + +function addedLines(patch) { + return String(patch || '') + .split('\n') + .filter((line) => line.startsWith('+') && !line.startsWith('+++')) + .map((line) => line.slice(1)) + .join('\n'); +} + +function explicitVeto(repo, number, title, body, labels) { + if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(number)) return true; + const text = `${title || ''}\n${body || ''}`.toLowerCase(); + const names = labels.map((label) => label.name.toLowerCase()); + return /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress/.test(text) || + names.some((name) => /do[- ]?not[- ]?merge|never[- ]?merge|\bwip\b|hold|blocked/.test(name)); +} + +function sourceGated(body) { + const text = String(body || '').toLowerCase(); + return text.includes('source-gated until:') && !text.includes('all declared source repositories resolved'); +} + +function sensitivePath(pathname) { + const path = pathname.toLowerCase(); + return /(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(path) || + /\.(pem|p12|pfx|jks|keystore|key)$/.test(path) || + /(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(path) || + (/(^|\/)\.env($|\.)/.test(path) && !/\.example$|\.sample$|\.template$/.test(path)); +} + +function patchSafetyReasons(files) { + const reasons = []; + for (const file of files) { + if (sensitivePath(file.filename)) reasons.push(`sensitive-path:${file.filename}`); + const patch = String(file.patch || ''); + const additions = addedLines(patch); + if (file.patch == null && file.status !== 'removed') reasons.push(`unreviewable-patch:${file.filename}`); + if (/^(<<<<<<<|=======|>>>>>>>)( |$)/m.test(patch)) reasons.push(`conflict-marker:${file.filename}`); + if (/(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})/.test(additions)) { + reasons.push(`credential-pattern:${file.filename}`); + } + if (/\.github\/workflows\/.*\.ya?ml$/i.test(file.filename)) { + if (/^\s*pull_request_target\s*:/m.test(additions)) reasons.push(`pull-request-target:${file.filename}`); + if (/^\s*permissions\s*:\s*write-all\s*$/m.test(additions)) reasons.push(`write-all:${file.filename}`); + if (/(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b/i.test(additions)) reasons.push(`remote-shell-pipe:${file.filename}`); + if (/\beval\s+["']?\$/.test(additions)) reasons.push(`eval-shell:${file.filename}`); + } + } + return [...new Set(reasons)]; +} + +function latestReviews(reviews) { + const latest = new Map(); + for (const review of reviews) { + if (!review.user?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.user.login, review.state); + } + return latest; +} + +async function checkClassification(repo, sha) { + const [runs, status] = await Promise.all([ + request(`${api}/repos/${repo}/commits/${sha}/check-runs?per_page=100`), + request(`${api}/repos/${repo}/commits/${sha}/status?per_page=100`), + ]); + const checkRuns = runs.check_runs || []; + const statuses = status.statuses || []; + const statusFailure = statuses.some((entry) => ['failure', 'error'].includes(entry.state)); + const statusPending = statuses.some((entry) => entry.state === 'pending'); + let billing = 0; + let realFailures = 0; + let pending = 0; + let successes = 0; + + await mapLimit(checkRuns, 6, async (run) => { + if (run.status !== 'completed') { + pending += 1; + return; + } + if (['success', 'neutral', 'skipped'].includes(run.conclusion)) { + if (run.conclusion === 'success') successes += 1; + return; + } + const annotations = await request(`${api}/repos/${repo}/check-runs/${run.id}/annotations?per_page=100`); + const billingOnly = annotations.length > 0 && annotations.every((annotation) => + /job was not started because recent account payments have failed|spending limit needs to be increased/i.test(annotation.message || '') + ); + if (billingOnly) billing += 1; + else realFailures += 1; + }); + + if (statusFailure) realFailures += 1; + if (statusPending) pending += 1; + let category; + if (pending) category = 'pending'; + else if (realFailures) category = 'failed'; + else if (billing) category = 'billing-only'; + else if (checkRuns.length === 0 && statuses.length === 0) category = 'no-checks'; + else if (successes > 0 || statuses.some((entry) => entry.state === 'success')) category = 'green'; + else category = 'neutral-only'; + + return { + category, + check_runs: checkRuns.length, + statuses: statuses.length, + billing, + real_failures: realFailures, + pending, + successes, + }; +} + +async function inspect(item) { + const repo = repoFromItem(item); + const [owner] = repo.split('/'); + let pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + for (let attempt = 0; pr.mergeable == null && attempt < 4; attempt += 1) { + await sleep(800 + attempt * 500); + pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + } + const [repository, reviews, files, checks] = await Promise.all([ + request(`${api}/repos/${repo}`), + request(`${api}/repos/${repo}/pulls/${item.number}/reviews?per_page=100`), + request(`${api}/repos/${repo}/pulls/${item.number}/files?per_page=100`), + checkClassification(repo, pr.head.sha), + ]); + + const reasons = []; + const labels = item.labels || []; + if (!owner.toLowerCase().endsWith('-test')) reasons.push('not-test-org'); + if (explicitVeto(repo, item.number, item.title, item.body, labels)) reasons.push('explicit-veto'); + if (sourceGated(item.body)) reasons.push('source-gated'); + if (pr.state !== 'open' || pr.draft) reasons.push('not-open-ready'); + if (pr.user?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (repository.archived) reasons.push('archived'); + if (!['admin', 'maintain', 'push'].includes(repository.permissions?.admin ? 'admin' : repository.permissions?.maintain ? 'maintain' : repository.permissions?.push ? 'push' : 'none')) reasons.push('no-write-permission'); + if (pr.base?.ref !== repository.default_branch) reasons.push('non-default-base'); + if (pr.head?.repo?.full_name !== repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== true) reasons.push(`mergeable-${String(pr.mergeable)}`); + if (['dirty', 'blocked', 'unknown'].includes(pr.mergeable_state)) reasons.push(`merge-state-${pr.mergeable_state}`); + if (pr.comments !== 0) reasons.push('issue-comments'); + if (pr.review_comments !== 0) reasons.push('review-comments'); + if ([...latestReviews(reviews).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (pr.changed_files < 1 || pr.changed_files > 100 || files.length !== pr.changed_files) reasons.push('file-count'); + if (pr.additions + pr.deletions > 12000) reasons.push('change-too-large'); + reasons.push(...patchSafetyReasons(files)); + + return { + repo, + number: item.number, + title: item.title, + created_at: item.created_at, + head_sha: pr.head.sha, + base_sha: pr.base.sha, + mergeable: pr.mergeable, + mergeable_state: pr.mergeable_state, + changed_files: pr.changed_files, + additions: pr.additions, + deletions: pr.deletions, + check: checks, + reasons: [...new Set(reasons)], + }; +} + +const query = encodeURIComponent(`is:pr is:open author:ORESoftware draft:false created:<${cutoff} sort:created-asc`); +const byUrl = new Map(); +let total = 0; +for (let page = 1; page <= 10; page += 1) { + const result = await request(`${api}/search/issues?q=${query}&per_page=100&page=${page}`); + total = result.total_count; + for (const item of result.items || []) byUrl.set(item.html_url, item); + if ((result.items || []).length < 100) break; +} +const items = [...byUrl.values()]; +console.log(`READY_SEARCH total=${total} observed=${items.length}`); + +const inspected = await mapLimit(items, 10, inspect); +const errors = inspected.filter((entry) => entry.audit_error); +const valid = inspected.filter((entry) => !entry.audit_error); +const skipCounts = new Map(); +for (const entry of valid) { + for (const reason of entry.reasons) skipCounts.set(reason, (skipCounts.get(reason) || 0) + 1); +} +const baseEligible = valid.filter((entry) => entry.reasons.length === 0); +const byCheck = Object.fromEntries( + ['green', 'billing-only', 'no-checks', 'neutral-only', 'pending', 'failed'].map((category) => [ + category, + baseEligible.filter((entry) => entry.check.category === category).length, + ]) +); + +function uniqueByRepo(entries) { + const map = new Map(); + for (const entry of entries.sort((a, b) => String(a.created_at).localeCompare(String(b.created_at)))) { + if (!map.has(entry.repo)) map.set(entry.repo, entry); + } + return [...map.values()]; +} + +const green = uniqueByRepo(baseEligible.filter((entry) => entry.check.category === 'green')); +const billing = uniqueByRepo(baseEligible.filter((entry) => entry.check.category === 'billing-only')); +const noChecks = uniqueByRepo(baseEligible.filter((entry) => ['no-checks', 'neutral-only'].includes(entry.check.category))); +const summary = { + search_total: total, + observed: items.length, + inspected: valid.length, + audit_errors: errors.length, + base_eligible: baseEligible.length, + unique_green: green.length, + unique_billing_only: billing.length, + unique_no_or_neutral_checks: noChecks.length, + by_check: byCheck, +}; +console.log(`READY_AUDIT_SUMMARY ${JSON.stringify(summary)}`); +console.log('SKIP_REASONS'); +for (const [reason, count] of [...skipCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))) { + console.log(`${count}\t${reason}`); +} +for (const [name, entries] of [['GREEN', green], ['BILLING', billing], ['NO_CHECKS', noChecks]]) { + console.log(`${name}_CANDIDATES`); + for (const entry of entries.slice(0, 200)) { + console.log(`${entry.repo}#${entry.number}\t${entry.head_sha}\t${entry.mergeable_state}\t${entry.changed_files} files\t${entry.title}`); + } +} +for (const error of errors.slice(0, 30)) console.log(`AUDIT_ERROR ${JSON.stringify(error)}`); + +const lines = [ + '## Ready original-backlog PR audit', + '', + `- Search total: ${total}`, + `- Inspected: ${valid.length}`, + `- Audit errors: ${errors.length}`, + `- Base-eligible test-org PRs: ${baseEligible.length}`, + `- Unique repositories with green candidates: ${green.length}`, + `- Unique repositories with billing-only candidates: ${billing.length}`, + `- Unique repositories with no/neutral-only checks: ${noChecks.length}`, + '', + '### Check categories among base-eligible PRs', + '', + ...Object.entries(byCheck).map(([category, count]) => `- ${category}: ${count}`), + '', + '### Leading skip reasons', + '', + ...[...skipCounts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 20).map(([reason, count]) => `- ${reason}: ${count}`), +]; +if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join('\n')}\n`); diff --git a/.github/scripts/audit-resolved-generated-drafts.mjs b/.github/scripts/audit-resolved-generated-drafts.mjs new file mode 100644 index 00000000..3318da20 --- /dev/null +++ b/.github/scripts/audit-resolved-generated-drafts.mjs @@ -0,0 +1,224 @@ +import fs from 'node:fs'; + +const token = process.env.GH_TOKEN; +if (!token) throw new Error('GH_TOKEN is required'); + +const cutoff = '2026-08-05T12:45:00Z'; +const api = 'https://api.github.com'; +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'resolved-generated-draft-auditor', +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function request(url, attempts = 6) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { headers }); + const text = await response.text(); + let value; + try { value = text ? JSON.parse(text) : null; } catch { value = text; } + if (response.ok) return value; + last = new Error(`GET ${url} -> ${response.status}: ${String(text).slice(0, 700)}`); + const retryable = response.status === 429 || response.status >= 500 || + (response.status === 403 && /secondary rate limit|abuse detection/i.test(text)); + if (!retryable || attempt === attempts) throw last; + const retryAfter = Number(response.headers.get('retry-after') || 0); + await sleep(Math.max(retryAfter * 1000, attempt * attempt * 700)); + } + throw last; +} + +async function mapLimit(values, limit, fn) { + const results = new Array(values.length); + let next = 0; + async function worker() { + while (true) { + const index = next++; + if (index >= values.length) return; + try { results[index] = await fn(values[index]); } + catch (error) { results[index] = { audit_error: error.message, source: values[index] }; } + } + } + await Promise.all(Array.from({ length: Math.min(limit, values.length) }, worker)); + return results; +} + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const resolvedPattern = /all declared source repositories resolved at bootstrap time|all (?:declared )?source repositories (?:are )?resolved|source repositories resolved|declared readiness:\s*`?ready`?/i; +const generatedTitlePattern = /^(?:test: bootstrap (?:independent acceptance portfolio|[a-z0-9][a-z0-9-]* harness)|chore: bootstrap test organization governance|test: add isolated playwright, puppeteer, and selenium harness contract)$/i; + +function latestReviewStates(reviews) { + const latest = new Map(); + for (const review of reviews) { + if (!review.user?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.user.login, review.state); + } + return latest; +} + +function checksGreen(runs, combinedStatus) { + const checkRuns = runs.check_runs || []; + const statuses = combinedStatus.statuses || []; + if (!checkRuns.length && !statuses.length) return false; + if (checkRuns.some((run) => run.status !== 'completed' || !['success', 'neutral', 'skipped'].includes(run.conclusion))) return false; + if (statuses.some((entry) => entry.state !== 'success')) return false; + return checkRuns.some((run) => run.conclusion === 'success') || statuses.some((entry) => entry.state === 'success'); +} + +function sensitivePath(pathname) { + const path = pathname.toLowerCase(); + return /(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(path) || + /\.(pem|p12|pfx|jks|keystore|key)$/.test(path) || + /(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(path) || + (/(^|\/)\.env($|\.)/.test(path) && !/\.example$|\.sample$|\.template$/.test(path)); +} + +function addedLines(patch) { + return String(patch || '') + .split('\n') + .filter((line) => line.startsWith('+') && !line.startsWith('+++')) + .map((line) => line.slice(1)) + .join('\n'); +} + +function patchReasons(files) { + const reasons = []; + for (const file of files) { + if (['removed', 'renamed'].includes(file.status)) reasons.push(`destructive-file-change:${file.filename}`); + if (sensitivePath(file.filename)) reasons.push(`sensitive-path:${file.filename}`); + const patch = String(file.patch || ''); + const additions = addedLines(patch); + if (file.patch == null && file.status !== 'removed') reasons.push(`unreviewable-patch:${file.filename}`); + if (/^(<<<<<<<|=======|>>>>>>>)( |$)/m.test(patch)) reasons.push(`conflict-marker:${file.filename}`); + if (/(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})/.test(additions)) { + reasons.push(`credential-pattern:${file.filename}`); + } + if (/\.github\/workflows\/.*\.ya?ml$/i.test(file.filename)) { + if (/^\s*pull_request_target\s*:/m.test(additions)) reasons.push(`pull-request-target:${file.filename}`); + if (/^\s*permissions\s*:\s*write-all\s*$/m.test(additions)) reasons.push(`write-all:${file.filename}`); + if (/(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b/i.test(additions)) reasons.push(`remote-shell-pipe:${file.filename}`); + if (/\beval\s+["']?\$/.test(additions)) reasons.push(`eval-shell:${file.filename}`); + for (const line of additions.split('\n')) { + const match = line.match(/^\s*(?:-\s*)?uses:\s*([^\s#]+)\s*$/); + if (!match) continue; + const use = match[1]; + if (use.startsWith('./')) continue; + const at = use.lastIndexOf('@'); + const ref = at >= 0 ? use.slice(at + 1) : ''; + if (!/^[0-9a-f]{40}$/i.test(ref)) reasons.push(`unpinned-action:${file.filename}:${use}`); + } + } + } + return [...new Set(reasons)]; +} + +async function inspect(item) { + const repo = item.repository_url.replace(`${api}/repos/`, ''); + const [owner] = repo.split('/'); + const body = item.body || ''; + const initialReasons = []; + if (!owner.toLowerCase().endsWith('-test')) initialReasons.push('not-test-org'); + if (!generatedTitlePattern.test(item.title || '')) initialReasons.push('not-generated-title'); + if (/source-gated until:/i.test(body)) initialReasons.push('source-gated'); + if (!resolvedPattern.test(body)) initialReasons.push('no-explicit-source-resolution'); + if (!/node scripts\/validate-plan\.mjs/i.test(body)) initialReasons.push('missing-plan-validation-claim'); + if (vetoPattern.test(`${item.title || ''}\n${body}\n${(item.labels || []).map((label) => label.name).join(' ')}`)) initialReasons.push('text-veto'); + if (initialReasons.length) return { repo, number: item.number, title: item.title, reasons: initialReasons, prefiltered: true }; + + let pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + for (let attempt = 0; pr.mergeable == null && attempt < 4; attempt += 1) { + await sleep(700 + attempt * 500); + pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + } + + const [repository, comments, reviews, files, runs, status, openPulls] = await Promise.all([ + request(`${api}/repos/${repo}`), + item.comments ? request(`${api}/repos/${repo}/issues/${item.number}/comments?per_page=100`) : Promise.resolve([]), + request(`${api}/repos/${repo}/pulls/${item.number}/reviews?per_page=100`), + request(`${api}/repos/${repo}/pulls/${item.number}/files?per_page=100`), + request(`${api}/repos/${repo}/commits/${pr.head.sha}/check-runs?per_page=100`), + request(`${api}/repos/${repo}/commits/${pr.head.sha}/status?per_page=100`), + request(`${api}/repos/${repo}/pulls?state=open&per_page=100`), + ]); + + const commentText = comments.map((comment) => comment.body || '').join('\n'); + const reasons = []; + if (vetoPattern.test(commentText)) reasons.push('comment-veto'); + if (pr.state !== 'open' || !pr.draft) reasons.push('not-open-draft'); + if (pr.user?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (repository.archived) reasons.push('archived'); + if (!(repository.permissions?.admin || repository.permissions?.maintain || repository.permissions?.push)) reasons.push('no-write-permission'); + if (pr.base?.ref !== repository.default_branch) reasons.push('non-default-base'); + if (pr.head?.repo?.full_name !== repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== true) reasons.push(`mergeable-${String(pr.mergeable)}`); + if (!['draft', 'clean'].includes(pr.mergeable_state)) reasons.push(`merge-state-${pr.mergeable_state}`); + if (pr.comments !== 0 || comments.length !== 0) reasons.push('issue-comments'); + if (pr.review_comments !== 0) reasons.push('inline-review-comments'); + if ([...latestReviewStates(reviews).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (openPulls.length !== 1 || openPulls[0]?.number !== item.number) reasons.push('multiple-open-prs-in-repository'); + if (pr.changed_files < 1 || pr.changed_files > 100 || files.length !== pr.changed_files) reasons.push('file-count'); + if (pr.additions + pr.deletions > 12000) reasons.push('change-too-large'); + if (!checksGreen(runs, status)) reasons.push('checks-not-green'); + reasons.push(...patchReasons(files)); + + return { + repo, + number: item.number, + title: item.title, + head_sha: pr.head.sha, + base_sha: pr.base.sha, + mergeable_state: pr.mergeable_state, + changed_files: pr.changed_files, + additions: pr.additions, + deletions: pr.deletions, + checks: (runs.check_runs || []).length, + reasons: [...new Set(reasons)], + }; +} + +const query = encodeURIComponent(`is:pr is:open author:ORESoftware draft:true created:<${cutoff} sort:created-asc`); +const byUrl = new Map(); +let total = 0; +for (let page = 1; page <= 10; page += 1) { + const result = await request(`${api}/search/issues?q=${query}&per_page=100&page=${page}`); + total = result.total_count; + for (const item of result.items || []) byUrl.set(item.html_url, item); + if ((result.items || []).length < 100) break; +} +const items = [...byUrl.values()]; +console.log(`DRAFT_SEARCH total=${total} observed=${items.length}`); + +const inspected = await mapLimit(items, 12, inspect); +const errors = inspected.filter((entry) => entry.audit_error); +const valid = inspected.filter((entry) => !entry.audit_error); +const candidates = valid.filter((entry) => entry.reasons.length === 0); +const skip = new Map(); +for (const entry of valid) for (const reason of entry.reasons) skip.set(reason, (skip.get(reason) || 0) + 1); + +console.log(`RESOLVED_DRAFT_AUDIT ${JSON.stringify({ search_total: total, observed: valid.length, audit_errors: errors.length, eligible: candidates.length })}`); +console.log('ELIGIBLE_DRAFTS'); +for (const entry of candidates) { + console.log(`${entry.repo}#${entry.number}\t${entry.head_sha}\t${entry.changed_files} files\t${entry.additions + entry.deletions} lines\t${entry.title}`); +} +console.log('SKIP_REASONS'); +for (const [reason, count] of [...skip.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))) { + console.log(`${count}\t${reason}`); +} +for (const error of errors.slice(0, 30)) console.log(`AUDIT_ERROR ${JSON.stringify(error)}`); + +const lines = [ + '## Resolved generated draft audit', + '', + `- Draft search total: ${total}`, + `- Inspected/prefiltered: ${valid.length}`, + `- Audit errors: ${errors.length}`, + `- Eligible exact heads: ${candidates.length}`, + '', + '### Leading skip reasons', + '', + ...[...skip.entries()].sort((a, b) => b[1] - a[1]).slice(0, 25).map(([reason, count]) => `- ${reason}: ${count}`), +]; +if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join('\n')}\n`); diff --git a/.github/scripts/discover-graphql-ready-portfolios-v2.mjs b/.github/scripts/discover-graphql-ready-portfolios-v2.mjs new file mode 100644 index 00000000..3576c305 --- /dev/null +++ b/.github/scripts/discover-graphql-ready-portfolios-v2.mjs @@ -0,0 +1,205 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { spawnSync } from 'node:child_process'; + +const token = process.env.FLEET_GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); +const output = path.resolve('graphql-portfolio-bundle'); +const archives = path.join(output, 'archives'); +fs.rmSync(output, { recursive: true, force: true }); +fs.mkdirSync(archives, { recursive: true }); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function graphql(query, variables = {}, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'graphql-ready-portfolio-discovery-v2', + }, + body: JSON.stringify({ query, variables }), + }); + const value = await response.json().catch(() => null); + if (response.ok && value?.data) return { data: value.data, errors: value.errors || [] }; + last = new Error(`GraphQL failed: ${JSON.stringify(value).slice(0, 1800)}`); + if (attempt === attempts) throw last; + await sleep(attempt * attempt * 900); + } + throw last; +} + +const searchQuery = ` + query ReadyPortfolios($search: String!) { + viewer { login } + search(query: $search, type: ISSUE, first: 100) { + issueCount + nodes { + ... on PullRequest { + id number title body createdAt state isDraft headRefName headRefOid baseRefName mergeable mergeStateStatus + comments(first: 100) { totalCount pageInfo { hasNextPage } nodes { body } } + reviewThreads(first: 100) { pageInfo { hasNextPage } nodes { isResolved } } + reviews(last: 100) { pageInfo { hasPreviousPage } nodes { state author { login } } } + labels(first: 50) { nodes { name } } + files(first: 100) { totalCount pageInfo { hasNextPage } nodes { path changeType } } + repository { nameWithOwner isPrivate isArchived viewerPermission defaultBranchRef { name } } + } + } + } + rateLimit { remaining resetAt cost } + } +`; +const overlapQuery = ` + query OpenFiles($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + pullRequests(states: OPEN, first: 100) { + pageInfo { hasNextPage } + nodes { number files(first: 100) { pageInfo { hasNextPage } nodes { path } } } + } + } + rateLimit { remaining resetAt cost } + } +`; + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const latestReviewStates = (pr) => { + const latest = new Map(); + for (const review of pr.reviews?.nodes || []) { + if (!review.author?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.author.login, review.state); + } + return latest; +}; +const intersects = (left, right) => { + for (const value of left) if (right.has(value)) return true; + return false; +}; + +const search = 'is:pr is:open author:ORESoftware draft:true created:<2026-08-05T12:45:00Z "Declared readiness: `ready`"'; +const searchResult = await graphql(searchQuery, { search }); +const data = searchResult.data; +if (data.viewer.login !== 'ORESoftware') throw new Error(`unexpected viewer ${data.viewer.login}`); + +const inspected = []; +for (const pr of data.search.nodes || []) { + if (!pr?.repository?.nameWithOwner) continue; + const repo = pr.repository.nameWithOwner; + const [owner] = repo.split('/'); + const comments = (pr.comments?.nodes || []).map((comment) => comment.body || '').join('\n'); + const labels = (pr.labels?.nodes || []).map((label) => label.name).join(' '); + const reasons = []; + if (!owner.toLowerCase().endsWith('-test')) reasons.push('not-test-org'); + if (pr.title !== 'test: bootstrap independent acceptance portfolio') reasons.push('wrong-title'); + if (!/- Declared readiness:\s*`ready`/i.test(pr.body || '')) reasons.push('not-declared-ready'); + if (/planned_dependency|source-gated until:/i.test(pr.body || '')) reasons.push('planned-or-source-gated'); + if (vetoPattern.test(`${pr.title}\n${pr.body || ''}\n${comments}\n${labels}`)) reasons.push('veto'); + if (pr.state !== 'OPEN' || !pr.isDraft) reasons.push('not-open-draft'); + if (pr.repository.isPrivate) reasons.push('private-repository'); + if (pr.repository.isArchived || !['ADMIN', 'MAINTAIN', 'WRITE'].includes(pr.repository.viewerPermission)) reasons.push('permission'); + if (pr.baseRefName !== pr.repository.defaultBranchRef?.name) reasons.push('non-default-base'); + if (pr.mergeable === 'CONFLICTING' || ['DIRTY', 'BLOCKED', 'UNKNOWN'].includes(pr.mergeStateStatus)) reasons.push(`merge-${pr.mergeable}-${pr.mergeStateStatus}`); + if (pr.comments?.totalCount !== 0 || pr.comments?.pageInfo?.hasNextPage) reasons.push('comments'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage || [...latestReviewStates(pr).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount < 1 || pr.files?.totalCount > 100) reasons.push('file-count'); + if ((pr.files?.nodes || []).some((file) => ['DELETED', 'RENAMED'].includes(file.changeType))) reasons.push('destructive-file-change'); + inspected.push({ pr, reasons: [...new Set(reasons)] }); +} + +for (const entry of inspected.filter((value) => value.reasons.length === 0)) { + const pr = entry.pr; + const [owner, name] = pr.repository.nameWithOwner.split('/'); + const result = await graphql(overlapQuery, { owner, name }); + const open = result.data.repository?.pullRequests; + if (!open) { + entry.reasons.push('repository-not-found'); + continue; + } + if (open.pageInfo.hasNextPage) { + entry.reasons.push('too-many-open-prs'); + continue; + } + const mine = new Set((pr.files?.nodes || []).map((file) => file.path)); + for (const other of open.nodes || []) { + if (other.number === pr.number) continue; + if (other.files?.pageInfo?.hasNextPage) { + entry.reasons.push('unbounded-other-pr-files'); + break; + } + if (intersects(mine, new Set((other.files?.nodes || []).map((file) => file.path)))) { + entry.reasons.push(`overlap-with-pr-${other.number}`); + break; + } + } +} + +const eligible = inspected.filter((entry) => entry.reasons.length === 0).sort((a, b) => String(a.pr.createdAt).localeCompare(String(b.pr.createdAt))); +const selected = []; +const repos = new Set(); +for (const entry of eligible) { + const repo = entry.pr.repository.nameWithOwner; + if (repos.has(repo)) continue; + repos.add(repo); + selected.push(entry.pr); +} + +const sanitizedEnv = { ...process.env }; +for (const name of ['FLEET_GH_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN']) delete sanitizedEnv[name]; +sanitizedEnv.GIT_CONFIG_COUNT = '1'; +sanitizedEnv.GIT_CONFIG_KEY_0 = 'core.hooksPath'; +sanitizedEnv.GIT_CONFIG_VALUE_0 = '/dev/null'; +sanitizedEnv.GIT_TERMINAL_PROMPT = '0'; + +const manifest = []; +const cloneFailures = []; +for (let index = 0; index < selected.length; index += 1) { + const pr = selected[index]; + const repo = pr.repository.nameWithOwner; + const repoName = repo.split('/')[1]; + const work = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP || '/tmp', 'portfolio-clone-')); + const clone = path.join(work, 'repo'); + try { + const cloneResult = spawnSync('git', ['clone', '--no-checkout', '--filter=blob:none', '--single-branch', '--branch', pr.headRefName, `https://github.com/${repo}.git`, clone], { + env: sanitizedEnv, encoding: 'utf8', timeout: 180000, maxBuffer: 8 * 1024 * 1024, + }); + if (cloneResult.status !== 0) throw new Error(`clone failed: ${(cloneResult.stdout || '')}${(cloneResult.stderr || '')}`); + const rev = spawnSync('git', ['-C', clone, 'rev-parse', `refs/remotes/origin/${pr.headRefName}`], { env: sanitizedEnv, encoding: 'utf8', timeout: 30000 }); + if (rev.status !== 0 || rev.stdout.trim() !== pr.headRefOid) throw new Error(`cloned head mismatch: ${rev.stdout.trim()} != ${pr.headRefOid}`); + const filename = `${String(index).padStart(3, '0')}-${repo.replace('/', '__')}--pr-${pr.number}.tar.gz`; + const archivePath = path.join(archives, filename); + const file = fs.openSync(archivePath, 'w', 0o600); + const archive = spawnSync('git', ['-C', clone, 'archive', '--format=tar.gz', `--prefix=${repoName}/`, pr.headRefOid], { + env: sanitizedEnv, stdio: ['ignore', file, 'pipe'], timeout: 120000, maxBuffer: 8 * 1024 * 1024, + }); + fs.closeSync(file); + if (archive.status !== 0) throw new Error(`archive failed: ${String(archive.stderr)}`); + const bytes = fs.readFileSync(archivePath); + if (bytes.length < 256) throw new Error('archive too small'); + manifest.push({ + repo, number: pr.number, node_id: pr.id, title: pr.title, + head_ref: pr.headRefName, head_sha: pr.headRefOid, base_ref: pr.baseRefName, + archive: `archives/${filename}`, + archive_sha256: crypto.createHash('sha256').update(bytes).digest('hex'), + archive_bytes: bytes.length, + }); + } catch (error) { + cloneFailures.push({ repo, number: pr.number, error: error.message.slice(0, 2000) }); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} + +const skipCounts = new Map(); +for (const entry of inspected) for (const reason of entry.reasons) skipCounts.set(reason, (skipCounts.get(reason) || 0) + 1); +fs.writeFileSync(path.join(output, 'manifest.json'), JSON.stringify({ + schema_version: 1, generated_at: new Date().toISOString(), search_count: data.search.issueCount, + selected_count: selected.length, entries: manifest, clone_failures: cloneFailures, +}, null, 2) + '\n'); +console.log(`GRAPHQL_PORTFOLIO_DISCOVERY_V2 ${JSON.stringify({ searched: data.search.issueCount, inspected: inspected.length, eligible: eligible.length, selected: selected.length, archived: manifest.length, clone_failures: cloneFailures.length, graphql_remaining: data.rateLimit.remaining })}`); +for (const [reason, count] of [...skipCounts.entries()].sort((a, b) => b[1] - a[1])) console.log(`SKIP ${count}\t${reason}`); +for (const failure of cloneFailures) console.log(`CLONE_FAILURE ${JSON.stringify(failure)}`); +if (!manifest.length) throw new Error('no ready portfolio archives were produced'); diff --git a/.github/scripts/discover-graphql-ready-portfolios.mjs b/.github/scripts/discover-graphql-ready-portfolios.mjs new file mode 100644 index 00000000..77fe3d7d --- /dev/null +++ b/.github/scripts/discover-graphql-ready-portfolios.mjs @@ -0,0 +1,226 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { spawnSync } from 'node:child_process'; + +const token = process.env.FLEET_GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); + +const output = path.resolve('graphql-portfolio-bundle'); +const archives = path.join(output, 'archives'); +fs.rmSync(output, { recursive: true, force: true }); +fs.mkdirSync(archives, { recursive: true }); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function graphql(query, variables = {}, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'graphql-ready-portfolio-discovery', + }, + body: JSON.stringify({ query, variables }), + }); + const value = await response.json().catch(() => null); + if (response.ok && !value?.errors?.length) return value.data; + last = new Error(`GraphQL failed: ${JSON.stringify(value).slice(0, 1800)}`); + if (attempt === attempts) throw last; + await sleep(attempt * attempt * 900); + } + throw last; +} + +const query = ` + query ReadyPortfolios($search: String!) { + viewer { login } + search(query: $search, type: ISSUE, first: 100) { + issueCount + nodes { + ... on PullRequest { + id + number + title + body + createdAt + state + isDraft + headRefName + headRefOid + baseRefName + mergeable + mergeStateStatus + comments(first: 100) { totalCount pageInfo { hasNextPage } nodes { body } } + reviewThreads(first: 100) { pageInfo { hasNextPage } nodes { isResolved } } + reviews(last: 100) { pageInfo { hasPreviousPage } nodes { state author { login } } } + labels(first: 50) { nodes { name } } + files(first: 100) { totalCount pageInfo { hasNextPage } nodes { path changeType } } + repository { + nameWithOwner + isPrivate + isArchived + viewerPermission + defaultBranchRef { name } + pullRequests(states: OPEN, first: 100) { + pageInfo { hasNextPage } + nodes { + number + headRefOid + files(first: 100) { pageInfo { hasNextPage } nodes { path } } + } + } + } + } + } + } + rateLimit { remaining resetAt cost } + } +`; + +const search = 'is:pr is:open author:ORESoftware draft:true created:<2026-08-05T12:45:00Z "Declared readiness: `ready`"'; +const data = await graphql(query, { search }); +if (data.viewer.login !== 'ORESoftware') throw new Error(`unexpected viewer ${data.viewer.login}`); + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const latestReviewStates = (pr) => { + const latest = new Map(); + for (const review of pr.reviews?.nodes || []) { + if (!review.author?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.author.login, review.state); + } + return latest; +}; +const intersects = (left, right) => { + for (const value of left) if (right.has(value)) return true; + return false; +}; + +const inspected = []; +for (const pr of data.search.nodes || []) { + if (!pr?.repository?.nameWithOwner) continue; + const repo = pr.repository.nameWithOwner; + const [owner] = repo.split('/'); + const comments = (pr.comments?.nodes || []).map((comment) => comment.body || '').join('\n'); + const labels = (pr.labels?.nodes || []).map((label) => label.name).join(' '); + const reasons = []; + if (!owner.toLowerCase().endsWith('-test')) reasons.push('not-test-org'); + if (pr.title !== 'test: bootstrap independent acceptance portfolio') reasons.push('wrong-title'); + if (!/- Declared readiness:\s*`ready`/i.test(pr.body || '')) reasons.push('not-declared-ready'); + if (/planned_dependency|source-gated until:/i.test(pr.body || '')) reasons.push('planned-or-source-gated'); + if (vetoPattern.test(`${pr.title}\n${pr.body || ''}\n${comments}\n${labels}`)) reasons.push('veto'); + if (pr.state !== 'OPEN' || !pr.isDraft) reasons.push('not-open-draft'); + if (pr.repository.isPrivate) reasons.push('private-repository'); + if (pr.repository.isArchived || !['ADMIN', 'MAINTAIN', 'WRITE'].includes(pr.repository.viewerPermission)) reasons.push('permission'); + if (pr.baseRefName !== pr.repository.defaultBranchRef?.name) reasons.push('non-default-base'); + if (pr.mergeable === 'CONFLICTING' || ['DIRTY', 'BLOCKED', 'UNKNOWN'].includes(pr.mergeStateStatus)) reasons.push(`merge-${pr.mergeable}-${pr.mergeStateStatus}`); + if (pr.comments?.totalCount !== 0 || pr.comments?.pageInfo?.hasNextPage) reasons.push('comments'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage || [...latestReviewStates(pr).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount < 1 || pr.files?.totalCount > 100) reasons.push('file-count'); + if ((pr.files?.nodes || []).some((file) => ['DELETED', 'RENAMED'].includes(file.changeType))) reasons.push('destructive-file-change'); + if (pr.repository.pullRequests?.pageInfo?.hasNextPage) reasons.push('too-many-open-prs'); + + if (!reasons.length) { + const mine = new Set((pr.files?.nodes || []).map((file) => file.path)); + for (const other of pr.repository.pullRequests?.nodes || []) { + if (other.number === pr.number) continue; + if (other.files?.pageInfo?.hasNextPage) { + reasons.push('unbounded-other-pr-files'); + break; + } + if (intersects(mine, new Set((other.files?.nodes || []).map((file) => file.path)))) { + reasons.push(`overlap-with-pr-${other.number}`); + break; + } + } + } + inspected.push({ pr, reasons: [...new Set(reasons)] }); +} + +const eligible = inspected.filter((entry) => entry.reasons.length === 0).sort((a, b) => String(a.pr.createdAt).localeCompare(String(b.pr.createdAt))); +const selected = []; +const repos = new Set(); +for (const entry of eligible) { + const repo = entry.pr.repository.nameWithOwner; + if (repos.has(repo)) continue; + repos.add(repo); + selected.push(entry.pr); +} + +const sanitizedEnv = { ...process.env }; +delete sanitizedEnv.FLEET_GH_TOKEN; +delete sanitizedEnv.GH_TOKEN; +delete sanitizedEnv.GITHUB_TOKEN; +sanitizedEnv.GIT_CONFIG_COUNT = '1'; +sanitizedEnv.GIT_CONFIG_KEY_0 = 'core.hooksPath'; +sanitizedEnv.GIT_CONFIG_VALUE_0 = '/dev/null'; +sanitizedEnv.GIT_TERMINAL_PROMPT = '0'; + +const manifest = []; +const cloneFailures = []; +for (let index = 0; index < selected.length; index += 1) { + const pr = selected[index]; + const repo = pr.repository.nameWithOwner; + const repoName = repo.split('/')[1]; + const work = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP || '/tmp', 'portfolio-clone-')); + const clone = path.join(work, 'repo'); + try { + const cloneResult = spawnSync('git', ['clone', '--no-checkout', '--filter=blob:none', '--single-branch', '--branch', pr.headRefName, `https://github.com/${repo}.git`, clone], { + env: sanitizedEnv, + encoding: 'utf8', + timeout: 180000, + maxBuffer: 8 * 1024 * 1024, + }); + if (cloneResult.status !== 0) throw new Error(`clone failed: ${(cloneResult.stdout || '')}${(cloneResult.stderr || '')}`); + const rev = spawnSync('git', ['-C', clone, 'rev-parse', `refs/remotes/origin/${pr.headRefName}`], { env: sanitizedEnv, encoding: 'utf8', timeout: 30000 }); + if (rev.status !== 0 || rev.stdout.trim() !== pr.headRefOid) throw new Error(`cloned head mismatch: ${rev.stdout.trim()} != ${pr.headRefOid}`); + const filename = `${String(index).padStart(3, '0')}-${repo.replace('/', '__')}--pr-${pr.number}.tar.gz`; + const archivePath = path.join(archives, filename); + const file = fs.openSync(archivePath, 'w', 0o600); + const archive = spawnSync('git', ['-C', clone, 'archive', '--format=tar.gz', `--prefix=${repoName}/`, pr.headRefOid], { + env: sanitizedEnv, + stdio: ['ignore', file, 'pipe'], + timeout: 120000, + maxBuffer: 8 * 1024 * 1024, + }); + fs.closeSync(file); + if (archive.status !== 0) throw new Error(`archive failed: ${String(archive.stderr)}`); + const bytes = fs.readFileSync(archivePath); + if (bytes.length < 256) throw new Error('archive too small'); + manifest.push({ + repo, + number: pr.number, + node_id: pr.id, + title: pr.title, + head_ref: pr.headRefName, + head_sha: pr.headRefOid, + base_ref: pr.baseRefName, + archive: `archives/${filename}`, + archive_sha256: crypto.createHash('sha256').update(bytes).digest('hex'), + archive_bytes: bytes.length, + }); + } catch (error) { + cloneFailures.push({ repo, number: pr.number, error: error.message.slice(0, 2000) }); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} + +const skipCounts = new Map(); +for (const entry of inspected) for (const reason of entry.reasons) skipCounts.set(reason, (skipCounts.get(reason) || 0) + 1); +const document = { + schema_version: 1, + generated_at: new Date().toISOString(), + search_count: data.search.issueCount, + selected_count: selected.length, + entries: manifest, + clone_failures: cloneFailures, +}; +fs.writeFileSync(path.join(output, 'manifest.json'), JSON.stringify(document, null, 2) + '\n'); +console.log(`GRAPHQL_PORTFOLIO_DISCOVERY ${JSON.stringify({ searched: data.search.issueCount, inspected: inspected.length, eligible: eligible.length, selected: selected.length, archived: manifest.length, clone_failures: cloneFailures.length, graphql_remaining: data.rateLimit.remaining })}`); +for (const [reason, count] of [...skipCounts.entries()].sort((a, b) => b[1] - a[1])) console.log(`SKIP ${count}\t${reason}`); +for (const failure of cloneFailures) console.log(`CLONE_FAILURE ${JSON.stringify(failure)}`); +if (!manifest.length) throw new Error('no ready portfolio archives were produced'); diff --git a/.github/scripts/discover-resolved-legacy-harnesses.mjs b/.github/scripts/discover-resolved-legacy-harnesses.mjs new file mode 100644 index 00000000..36dbf76e --- /dev/null +++ b/.github/scripts/discover-resolved-legacy-harnesses.mjs @@ -0,0 +1,120 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { spawnSync } from 'node:child_process'; + +const token = process.env.FLEET_GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); +const output = path.resolve('legacy-harness-bundle'); +const archives = path.join(output, 'archives'); +fs.rmSync(output, { recursive: true, force: true }); +fs.mkdirSync(archives, { recursive: true }); + +async function graphql(query, variables = {}) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', 'User-Agent': 'resolved-legacy-harness-discovery' }, + body: JSON.stringify({ query, variables }), + }); + const value = await response.json(); + if (!response.ok || !value.data) throw new Error(JSON.stringify(value).slice(0, 1800)); + return value.data; +} + +const query = ` + query Harnesses($search: String!) { + viewer { login } + search(query: $search, type: ISSUE, first: 100) { + issueCount + nodes { + ... on PullRequest { + id number title body state isDraft headRefName headRefOid baseRefName mergeable mergeStateStatus + comments(first: 100) { totalCount pageInfo { hasNextPage } nodes { body } } + reviewThreads(first: 100) { pageInfo { hasNextPage } nodes { isResolved } } + reviews(last: 100) { pageInfo { hasPreviousPage } nodes { state author { login } } } + files(first: 100) { totalCount pageInfo { hasNextPage } nodes { path changeType } } + labels(first: 50) { nodes { name } } + repository { nameWithOwner isPrivate isArchived viewerPermission defaultBranchRef { name } pullRequests(states: OPEN, first: 1) { totalCount } } + } + } + } + rateLimit { remaining cost resetAt } + } +`; +const search = 'is:pr is:open author:ORESoftware draft:true created:<2026-08-05T12:45:00Z "All declared source repositories resolved at bootstrap time"'; +const data = await graphql(query, { search }); +if (data.viewer.login !== 'ORESoftware') throw new Error(`unexpected viewer ${data.viewer.login}`); +const veto = /do not merge|don['’]?t merge|never merge|\bwip\b|work in progress|not ready|blocked by|changes requested|must not merge/i; +const latest = (pr) => { + const map = new Map(); + for (const review of pr.reviews?.nodes || []) { + if (!review.author?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + map.set(review.author.login, review.state); + } + return map; +}; +const selected = []; +const skipped = []; +for (const pr of data.search.nodes || []) { + if (!pr?.repository?.nameWithOwner) continue; + const repo = pr.repository.nameWithOwner; + const [owner] = repo.split('/'); + const comments = (pr.comments?.nodes || []).map((comment) => comment.body || '').join('\n'); + const labels = (pr.labels?.nodes || []).map((label) => label.name).join(' '); + const reasons = []; + if (!owner.toLowerCase().endsWith('-test')) reasons.push('not-test-org'); + if (!/^test: bootstrap [a-z0-9][a-z0-9-]* harness$/i.test(pr.title || '')) reasons.push('wrong-title'); + if (!/All declared source repositories resolved at bootstrap time/i.test(pr.body || '')) reasons.push('not-source-resolved'); + if (pr.state !== 'OPEN' || !pr.isDraft) reasons.push('not-open-draft'); + if (pr.repository.isPrivate || pr.repository.isArchived || !['ADMIN', 'MAINTAIN', 'WRITE'].includes(pr.repository.viewerPermission)) reasons.push('repository-gate'); + if (pr.baseRefName !== pr.repository.defaultBranchRef?.name) reasons.push('non-default-base'); + if (pr.mergeable === 'CONFLICTING' || ['DIRTY', 'BLOCKED', 'UNKNOWN'].includes(pr.mergeStateStatus)) reasons.push(`merge-${pr.mergeable}-${pr.mergeStateStatus}`); + if (pr.comments?.totalCount !== 0 || pr.comments?.pageInfo?.hasNextPage) reasons.push('comments'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage || [...latest(pr).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount < 1 || pr.files?.totalCount > 100) reasons.push('file-count'); + if ((pr.files?.nodes || []).some((file) => ['DELETED', 'RENAMED'].includes(file.changeType))) reasons.push('destructive-change'); + if (pr.repository.pullRequests?.totalCount !== 1) reasons.push(`open-pr-count-${pr.repository.pullRequests?.totalCount}`); + if (veto.test(`${pr.title}\n${pr.body || ''}\n${comments}\n${labels}`)) reasons.push('veto'); + if (reasons.length) skipped.push({ repo, number: pr.number, reasons }); + else selected.push(pr); +} + +const cleanEnv = { ...process.env }; +for (const name of ['FLEET_GH_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN']) delete cleanEnv[name]; +cleanEnv.GIT_TERMINAL_PROMPT = '0'; +cleanEnv.GIT_CONFIG_COUNT = '1'; +cleanEnv.GIT_CONFIG_KEY_0 = 'core.hooksPath'; +cleanEnv.GIT_CONFIG_VALUE_0 = '/dev/null'; +const entries = []; +const failures = []; +for (let index = 0; index < selected.length; index += 1) { + const pr = selected[index]; + const repo = pr.repository.nameWithOwner; + const repoName = repo.split('/')[1]; + const work = fs.mkdtempSync(path.join(process.env.RUNNER_TEMP || '/tmp', 'legacy-harness-')); + try { + const clone = path.join(work, 'repo'); + const result = spawnSync('git', ['clone', '--no-checkout', '--filter=blob:none', '--single-branch', '--branch', pr.headRefName, `https://github.com/${repo}.git`, clone], { env: cleanEnv, encoding: 'utf8', timeout: 180000, maxBuffer: 8 * 1024 * 1024 }); + if (result.status !== 0) throw new Error(`clone failed: ${(result.stdout || '')}${(result.stderr || '')}`); + const rev = spawnSync('git', ['-C', clone, 'rev-parse', `refs/remotes/origin/${pr.headRefName}`], { env: cleanEnv, encoding: 'utf8', timeout: 30000 }); + if (rev.status !== 0 || rev.stdout.trim() !== pr.headRefOid) throw new Error(`head mismatch ${rev.stdout.trim()} != ${pr.headRefOid}`); + const filename = `${String(index).padStart(3, '0')}-${repo.replace('/', '__')}--pr-${pr.number}.tar.gz`; + const archivePath = path.join(archives, filename); + const fd = fs.openSync(archivePath, 'w', 0o600); + const archived = spawnSync('git', ['-C', clone, 'archive', '--format=tar.gz', `--prefix=${repoName}/`, pr.headRefOid], { env: cleanEnv, stdio: ['ignore', fd, 'pipe'], timeout: 120000, maxBuffer: 8 * 1024 * 1024 }); + fs.closeSync(fd); + if (archived.status !== 0) throw new Error(`archive failed: ${String(archived.stderr)}`); + const bytes = fs.readFileSync(archivePath); + entries.push({ repo, number: pr.number, node_id: pr.id, title: pr.title, head_ref: pr.headRefName, head_sha: pr.headRefOid, base_ref: pr.baseRefName, archive: `archives/${filename}`, archive_sha256: crypto.createHash('sha256').update(bytes).digest('hex'), archive_bytes: bytes.length }); + } catch (error) { + failures.push({ repo, number: pr.number, error: error.message.slice(0, 2000) }); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} +fs.writeFileSync(path.join(output, 'manifest.json'), JSON.stringify({ schema_version: 1, generated_at: new Date().toISOString(), search_count: data.search.issueCount, entries, skipped, failures }, null, 2) + '\n'); +console.log(`LEGACY_HARNESS_DISCOVERY ${JSON.stringify({ searched: data.search.issueCount, selected: selected.length, archived: entries.length, skipped: skipped.length, failures: failures.length, graphql_remaining: data.rateLimit.remaining })}`); +for (const item of skipped) console.log(`SKIP ${item.repo}#${item.number} ${item.reasons.join(',')}`); +for (const item of failures) console.log(`FAILURE ${JSON.stringify(item)}`); +if (!entries.length) throw new Error('no resolved legacy harnesses archived'); diff --git a/.github/scripts/merge-app-audited-ready.mjs b/.github/scripts/merge-app-audited-ready.mjs new file mode 100644 index 00000000..363b612a --- /dev/null +++ b/.github/scripts/merge-app-audited-ready.mjs @@ -0,0 +1,133 @@ +const token = process.env.GH_TOKEN; +const repo = process.env.TARGET_REPOSITORY; +const number = Number(process.env.TARGET_PR); +const expectedHead = process.env.TARGET_HEAD; + +if (!token || !repo || !Number.isInteger(number) || !expectedHead) { + throw new Error('GH_TOKEN, TARGET_REPOSITORY, TARGET_PR, and TARGET_HEAD are required'); +} + +const api = 'https://api.github.com'; +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'per-installation-exact-head-merger', +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function request(url, options = {}, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { + ...options, + headers: { ...headers, ...(options.headers || {}) }, + }); + const text = await response.text(); + let value; + try { value = text ? JSON.parse(text) : null; } catch { value = text; } + if (response.ok) return value; + last = new Error(`${options.method || 'GET'} ${url} -> ${response.status}: ${String(text).slice(0, 800)}`); + if (![429, 500, 502, 503, 504].includes(response.status) || attempt === attempts) throw last; + const retryAfter = Number(response.headers.get('retry-after') || 0); + await sleep(Math.max(retryAfter * 1000, attempt * attempt * 800)); + } + throw last; +} + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const concernPattern = /(?:still|currently) fail(?:ing|s)?|unresolved|needs? (?:a |an |the )?(?:fix|change|update|review)|regression remains|unsafe to merge/i; +const credentialPattern = /(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})/; + +function latestReviewStates(reviews) { + const latest = new Map(); + for (const review of reviews) { + if (!review.user?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.user.login, review.state); + } + return latest; +} + +function patchReasons(files) { + const reasons = []; + for (const file of files) { + const patch = String(file.patch || ''); + const additions = patch.split('\n').filter((line) => line.startsWith('+') && !line.startsWith('+++')).map((line) => line.slice(1)).join('\n'); + if (file.patch == null && file.status !== 'removed') reasons.push(`unreviewable:${file.filename}`); + if (/^(<<<<<<<|=======|>>>>>>>)( |$)/m.test(patch)) reasons.push(`conflict-marker:${file.filename}`); + if (credentialPattern.test(additions)) reasons.push(`credential-pattern:${file.filename}`); + if (/\.github\/workflows\/.*\.ya?ml$/i.test(file.filename)) { + if (/^\s*pull_request_target\s*:/m.test(additions)) reasons.push(`pull-request-target:${file.filename}`); + if (/^\s*permissions\s*:\s*write-all\s*$/m.test(additions)) reasons.push(`write-all:${file.filename}`); + if (/(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b/i.test(additions)) reasons.push(`remote-shell-pipe:${file.filename}`); + if (/\beval\s+["']?\$/.test(additions)) reasons.push(`eval-shell:${file.filename}`); + } + } + return [...new Set(reasons)]; +} + +async function checksGreen(sha) { + const [runs, combined] = await Promise.all([ + request(`${api}/repos/${repo}/commits/${sha}/check-runs?per_page=100`), + request(`${api}/repos/${repo}/commits/${sha}/status?per_page=100`), + ]); + const checkRuns = runs.check_runs || []; + const statuses = combined.statuses || []; + if (!checkRuns.length && !statuses.length) return false; + if (checkRuns.some((run) => run.status !== 'completed' || !['success', 'neutral', 'skipped'].includes(run.conclusion))) return false; + if (statuses.some((status) => status.state !== 'success')) return false; + return checkRuns.some((run) => run.conclusion === 'success') || statuses.some((status) => status.state === 'success'); +} + +let pr = await request(`${api}/repos/${repo}/pulls/${number}`); +if (pr.merged) { + console.log(`ALREADY_MERGED ${repo}#${number}`); + process.exit(0); +} +if (pr.state !== 'open') throw new Error(`PR is ${pr.state}, not open`); +if (pr.head?.sha !== expectedHead) throw new Error(`head moved: expected ${expectedHead}, found ${pr.head?.sha}`); +for (let attempt = 0; pr.mergeable == null && attempt < 5; attempt += 1) { + await sleep(700 + attempt * 500); + pr = await request(`${api}/repos/${repo}/pulls/${number}`); +} + +const [repository, comments, reviews, files, green] = await Promise.all([ + request(`${api}/repos/${repo}`), + pr.comments ? request(`${api}/repos/${repo}/issues/${number}/comments?per_page=100`) : Promise.resolve([]), + request(`${api}/repos/${repo}/pulls/${number}/reviews?per_page=100`), + request(`${api}/repos/${repo}/pulls/${number}/files?per_page=100`), + checksGreen(expectedHead), +]); + +const commentText = comments.map((comment) => comment.body || '').join('\n'); +const allText = `${pr.title || ''}\n${pr.body || ''}\n${commentText}\n${(pr.labels || []).map((label) => label.name).join(' ')}`; +const reasons = []; +if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(number)) reasons.push('hard-veto'); +if (vetoPattern.test(allText)) reasons.push('text-veto'); +if (concernPattern.test(commentText)) reasons.push('comment-concern'); +if (pr.draft) reasons.push('draft'); +if (pr.user?.login !== 'ORESoftware') reasons.push('wrong-author'); +if (repository.archived) reasons.push('archived'); +if (!(repository.permissions?.admin || repository.permissions?.maintain || repository.permissions?.push)) reasons.push('no-write-permission'); +if (pr.base?.ref !== repository.default_branch) reasons.push('non-default-base'); +if (pr.head?.repo?.full_name !== repo) reasons.push('cross-repository-head'); +if (pr.mergeable !== true || !['clean', 'unstable'].includes(pr.mergeable_state)) reasons.push(`merge-${pr.mergeable}-${pr.mergeable_state}`); +if (pr.comments !== 0 || pr.review_comments !== 0 || comments.length !== 0) reasons.push('comments'); +if ([...latestReviewStates(reviews).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); +if (pr.changed_files < 1 || pr.changed_files > 100 || files.length !== pr.changed_files) reasons.push('file-count'); +if (!green) reasons.push('checks-not-green'); +reasons.push(...patchReasons(files)); +if (reasons.length) throw new Error(`fresh merge gate failed: ${[...new Set(reasons)].join(',')}`); + +const method = repository.allow_squash_merge ? 'squash' : repository.allow_rebase_merge ? 'rebase' : repository.allow_merge_commit ? 'merge' : null; +if (!method) throw new Error('repository has no allowed merge method'); + +const result = await request(`${api}/repos/${repo}/pulls/${number}/merge`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sha: expectedHead, merge_method: method }), +}, 3); +if (!result?.merged) throw new Error(`merge declined: ${result?.message || 'unknown'}`); +const verified = await request(`${api}/repos/${repo}/pulls/${number}`); +if (!verified.merged || verified.head?.sha !== expectedHead) throw new Error('post-merge exact-head verification failed'); +console.log(`MERGED ${repo}#${number} head=${expectedHead} merge=${result.sha} method=${method}`); diff --git a/.github/scripts/merge-audited-ready-batch.mjs b/.github/scripts/merge-audited-ready-batch.mjs new file mode 100644 index 00000000..8fcd5a10 --- /dev/null +++ b/.github/scripts/merge-audited-ready-batch.mjs @@ -0,0 +1,180 @@ +import fs from 'node:fs'; + +const token = process.env.GH_TOKEN; +if (!token) throw new Error('GH_TOKEN is required'); + +const api = 'https://api.github.com'; +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'audited-ready-batch-merger', +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const candidates = [ + ['usa-acc/usa-acc.github.io', 4, 'be327e240e575cef371e905405db8c67c102da80'], + ['anticaptrad/act-api-server.rs', 6, '6968bc0d76e3ea0539285a3df27780d916408a24'], + ['ORESoftware/k8s-libs-and-shared-defs', 11, '5252a645392e7eb28180fef6aeb3f373dbf8bd5e'], + ['3FA-app/3fa-web-server.rs', 3, 'b898dce8e004be223c66dc21185a1ab1a3b8b745'], + ['ORESoftware/soccer-sim-game-engine.rs', 3, '964fbd76a2a07d63bf1f2b2b9aafc631d8a14965'], + ['daedalus-fab/daedalus-web-server.rs', 1, '52e823e7c5bc852c1ba77ad8d580e763f1fa43ce'], + ['3FA-app/3FA-desktop.rs', 16, '485f311e41f19040e1ffec67ddcb27b40efd479a'], + ['declarative-migrations/homebrew-tap', 3, '2658ed2e53a65d8f6eda678e5ed71cd72c194746'], + ['sonus-auris/sonus-auris-e2e', 8, '25c1e79447ee9d4676e3facc9e855d4dc880dc9e'], + ['scintilla-run/scintilla-run-e2e', 4, 'f363dad5f4027b1a41cf5c7bd145995ea03f56e4'], + ['athlet-o/athleto-e2e', 2, 'a51fbd8b113cdb9c6159a6ae66aa52517461a93f'], + ['ORESoftware/next-loggers.ts', 4, '4f4c26f366cb006af9e99e016e1a1658b83f065c'], + ['ORESoftware/r2g', 88, 'd4d752d31e35220bf4f9ab9b13ac848a9643edc4'], + ['StreemPilot/sp-infra', 2, 'fdd0ed3587c5ea5bb9cf6fd6c946450c4b5df1eb'], + ['StreemPilot/streempilot-clients', 3, '08f29fdf3fa632fcec83900b8aaa3d4ad7d0632c'], + ['StreemPilot/streempilot-e2e', 3, '73407a79ce39eceb82701516e5c1f9b2fa9455e6'], + ['StreemPilot/streempilot-monorepo', 1, '54e8c4f6e2345eda6880f68e85be100f71b3a2dc'], + ['StreemPilot/sp-interfaces', 4, 'afc81102626b319bed242c8411d69a647e77ec74'], + ['voxletra/vxl-api-server.rs', 4, '19389fddd4c5b9427a447aa9c819105590264de0'], + ['networking-components/ncc-switch', 3, '33997c7017c868466840ebd46fe8ca21b765f49c'], + ['networking-components/ncc-router', 3, 'd9a65ddc96e5948d26168d41b004381cbd06c862'], + ['networking-components/ncc-dns-server', 4, '8f7ba85df838d65b42f97b7b87ced092dc331d8b'], + ['zed-pkg-test/workspace-monorepo', 1, '7f3c0872525245e28fc9463d66cbb2339a3e4ce5'], + ['discrete-event-systems/des-web.rs', 10, '77741ec8b5331617f71416748ef5f06846e43a5d'], + ['StreemPilot/streempilot-api-server.rs', 23, 'addbe13705b5550a6659bad2635b4a42e03457cf'], +]; + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const concernPattern = /(?:still|currently) fail(?:ing|s)?|unresolved|needs? (?:a |an |the )?(?:fix|change|update|review)|regression remains|unsafe to merge/i; + +async function request(url, options = {}, attempts = 6) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { + ...options, + headers: { ...headers, ...(options.headers || {}) }, + }); + const text = await response.text(); + let value; + try { value = text ? JSON.parse(text) : null; } catch { value = text; } + if (response.ok) return value; + last = new Error(`${options.method || 'GET'} ${url} -> ${response.status}: ${String(text).slice(0, 800)}`); + const retryable = response.status === 429 || response.status >= 500 || + (response.status === 403 && /secondary rate limit|abuse detection/i.test(text)); + if (!retryable || attempt === attempts) throw last; + await sleep(attempt * attempt * 800); + } + throw last; +} + +function latestReviewStates(reviews) { + const latest = new Map(); + for (const review of reviews) { + if (!review.user?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.user.login, review.state); + } + return latest; +} + +function checksGreen(runs, status) { + const checkRuns = runs.check_runs || []; + const statuses = status.statuses || []; + if (!checkRuns.length && !statuses.length) return false; + if (checkRuns.some((run) => run.status !== 'completed' || !['success', 'neutral', 'skipped'].includes(run.conclusion))) return false; + if (statuses.some((entry) => entry.state !== 'success')) return false; + return checkRuns.some((run) => run.conclusion === 'success') || statuses.some((entry) => entry.state === 'success'); +} + +async function revalidate(repo, number, expectedHead) { + let pr = await request(`${api}/repos/${repo}/pulls/${number}`); + if (pr.merged) return { state: 'already-merged', pr }; + if (pr.state !== 'open') return { state: 'closed-unmerged', pr }; + if (pr.head?.sha !== expectedHead) return { state: 'head-moved', pr }; + for (let attempt = 0; pr.mergeable == null && attempt < 4; attempt += 1) { + await sleep(700 + attempt * 500); + pr = await request(`${api}/repos/${repo}/pulls/${number}`); + } + + const [repository, comments, reviews, runs, status] = await Promise.all([ + request(`${api}/repos/${repo}`), + pr.comments ? request(`${api}/repos/${repo}/issues/${number}/comments?per_page=100`) : Promise.resolve([]), + request(`${api}/repos/${repo}/pulls/${number}/reviews?per_page=100`), + request(`${api}/repos/${repo}/commits/${expectedHead}/check-runs?per_page=100`), + request(`${api}/repos/${repo}/commits/${expectedHead}/status?per_page=100`), + ]); + + const commentText = comments.map((comment) => comment.body || '').join('\n'); + const labelText = (pr.labels || []).map((label) => label.name).join(' '); + const allText = `${pr.title || ''}\n${pr.body || ''}\n${commentText}\n${labelText}`; + const reasons = []; + if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(number)) reasons.push('hard-veto'); + if (vetoPattern.test(allText)) reasons.push('text-veto'); + if (concernPattern.test(commentText)) reasons.push('comment-concern'); + if (pr.draft) reasons.push('draft'); + if (pr.user?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (repository.archived) reasons.push('archived'); + if (!(repository.permissions?.admin || repository.permissions?.maintain || repository.permissions?.push)) reasons.push('no-write-permission'); + if (pr.base?.ref !== repository.default_branch) reasons.push('non-default-base'); + if (pr.head?.repo?.full_name !== repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== true || pr.mergeable_state !== 'clean') reasons.push(`merge-${pr.mergeable}-${pr.mergeable_state}`); + if (pr.review_comments !== 0) reasons.push('inline-review-comments'); + if ([...latestReviewStates(reviews).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (!checksGreen(runs, status)) reasons.push('checks-not-green'); + return { state: reasons.length ? 'blocked' : 'eligible', pr, repository, reasons }; +} + +const merged = []; +const raced = []; +const skipped = []; +for (const [repo, number, expectedHead] of candidates) { + const key = `${repo}#${number}`; + try { + const check = await revalidate(repo, number, expectedHead); + if (check.state === 'already-merged') { + raced.push({ key, reason: 'already-merged' }); + console.log(`RACED ${key} already merged`); + continue; + } + if (check.state !== 'eligible') { + skipped.push({ key, reason: check.state, details: check.reasons || [] }); + console.log(`SKIPPED ${key} ${check.state} ${(check.reasons || []).join(',')}`); + continue; + } + + const method = check.repository.allow_squash_merge ? 'squash' : + check.repository.allow_rebase_merge ? 'rebase' : + check.repository.allow_merge_commit ? 'merge' : null; + if (!method) { + skipped.push({ key, reason: 'no-merge-method' }); + console.log(`SKIPPED ${key} no allowed merge method`); + continue; + } + + const result = await request(`${api}/repos/${repo}/pulls/${number}/merge`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sha: expectedHead, merge_method: method }), + }, 3); + if (!result?.merged) { + skipped.push({ key, reason: result?.message || 'merge-declined' }); + console.log(`SKIPPED ${key} merge declined: ${result?.message || 'unknown'}`); + continue; + } + const verified = await request(`${api}/repos/${repo}/pulls/${number}`); + if (!verified.merged || verified.head?.sha !== expectedHead) throw new Error(`post-merge verification failed for ${key}`); + merged.push({ key, head: expectedHead, merge: result.sha, method }); + console.log(`MERGED ${key} head=${expectedHead} merge=${result.sha} method=${method}`); + } catch (error) { + skipped.push({ key, reason: error.message }); + console.error(`SKIPPED ${key} exception=${error.message}`); + } + await sleep(350); +} + +console.log(`BATCH_RESULT ${JSON.stringify({ merged: merged.length, raced: raced.length, skipped: skipped.length })}`); +const lines = [ + '## Audited ready batch result', + '', + `- Successfully merged and exact-head verified: ${merged.length}`, + `- Concurrently merged before this lane: ${raced.length}`, + `- Skipped after fresh revalidation: ${skipped.length}`, + '', + ...skipped.slice(0, 30).map((entry) => `- Skip ${entry.key}: ${entry.reason}${entry.details?.length ? ` (${entry.details.join(', ')})` : ''}`), +]; +if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join('\n')}\n`); diff --git a/.github/scripts/merge-graphql-frozen-ready.mjs b/.github/scripts/merge-graphql-frozen-ready.mjs new file mode 100644 index 00000000..7410416a --- /dev/null +++ b/.github/scripts/merge-graphql-frozen-ready.mjs @@ -0,0 +1,199 @@ +const token = process.env.FLEET_GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); + +const candidates = [ + ['discrete-event-systems/des-simulate.rs', 113, 'e21a25508f7c2c7512f4cf09020dec1d6fbe2299'], + ['canonical-cloud/canonical-cloud-api-server.rs', 46, 'b72c620e9784eaac3b0ca70630e64de3227a186f'], + ['ORESoftware/ai-agent-coordinator.rs', 56, '689b2575d3d52cf6b204adde1613efe8de5c5cbc'], + ['opto-sync/syncer.rs', 2, 'e99cf3c8303739fddd2b28c025b5067d776088bb'], + ['opto-sync/opto-sync-clients', 40, '098a94f040723d19783e8ff6daa3bc265dfa6eec'], + ['opto-sync/indexeddb-schema.rs', 2, '8563baf28a6d44e4eea0c70ab526f8339a719360'], + ['opter-io/opter-sync', 3, '7078d21d8411106e4185b20c8c04d22b854b3119'], + ['canonical-cloud/canonical-cloud-macos-app.swift', 11, 'fb8cf26f9c92f0f439411b58fc069ceff31b8313'], + ['benefactor-cc/benefactor-dart-client', 18, 'ceb8ea2f692cc6bb277b9485ab4896f01999c60b'], + ['benefactor-cc/benefactor-api-server.rs', 17, 'ee881e9ada5c4d5516ab2f80c6810e841b4c19e2'], +]; + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const concernPattern = /(?:still|currently) fail(?:ing|s)?|unresolved|needs? (?:a |an |the )?(?:fix|change|update|review)|regression remains|unsafe to merge/i; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function graphql(query, variables = {}, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'graphql-frozen-ready-merger', + }, + body: JSON.stringify({ query, variables }), + }); + const value = await response.json().catch(() => null); + if (response.ok && !value?.errors?.length) return value.data; + last = new Error(`GraphQL failed: ${JSON.stringify(value).slice(0, 1500)}`); + if (attempt === attempts) throw last; + await sleep(attempt * attempt * 800); + } + throw last; +} + +const query = ` + query Candidate($owner: String!, $name: String!, $number: Int!) { + viewer { login } + repository(owner: $owner, name: $name) { + nameWithOwner + isArchived + viewerPermission + defaultBranchRef { name } + mergeCommitAllowed + squashMergeAllowed + rebaseMergeAllowed + pullRequest(number: $number) { + id + number + title + body + state + merged + isDraft + headRefOid + baseRefName + mergeable + mergeStateStatus + reviewDecision + comments(first: 100) { totalCount pageInfo { hasNextPage } nodes { body } } + reviewThreads(first: 100) { pageInfo { hasNextPage } nodes { isResolved } } + reviews(last: 100) { pageInfo { hasPreviousPage } nodes { state author { login } } } + labels(first: 50) { nodes { name } } + files(first: 100) { totalCount pageInfo { hasNextPage } nodes { path changeType } } + commits(last: 1) { + nodes { + commit { + oid + statusCheckRollup { + state + contexts(first: 100) { + pageInfo { hasNextPage } + nodes { + __typename + ... on CheckRun { status conclusion } + ... on StatusContext { state } + } + } + } + } + } + } + } + } + rateLimit { remaining resetAt cost } + } +`; + +const mutation = ` + mutation Merge($id: ID!, $head: GitObjectID!, $method: PullRequestMergeMethod!) { + mergePullRequest(input: { pullRequestId: $id, expectedHeadOid: $head, mergeMethod: $method }) { + pullRequest { id number merged mergedAt headRefOid } + } + } +`; + +function checksGreen(pr) { + const commit = pr.commits?.nodes?.at(-1)?.commit; + if (!commit || commit.oid !== pr.headRefOid) return false; + const rollup = commit.statusCheckRollup; + if (!rollup || rollup.state !== 'SUCCESS' || rollup.contexts?.pageInfo?.hasNextPage) return false; + const contexts = rollup.contexts?.nodes || []; + if (!contexts.length) return false; + let success = false; + for (const context of contexts) { + if (context.__typename === 'CheckRun') { + if (context.status !== 'COMPLETED' || !['SUCCESS', 'NEUTRAL', 'SKIPPED'].includes(context.conclusion)) return false; + if (context.conclusion === 'SUCCESS') success = true; + } else if (context.__typename === 'StatusContext') { + if (context.state !== 'SUCCESS') return false; + success = true; + } else return false; + } + return success; +} + +function latestReviewStates(pr) { + const latest = new Map(); + for (const review of pr.reviews?.nodes || []) { + if (!review.author?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.author.login, review.state); + } + return latest; +} + +const merged = []; +const skipped = []; +for (const [repo, number, expectedHead] of candidates) { + const [owner, name] = repo.split('/'); + const data = await graphql(query, { owner, name, number }); + if (data.viewer.login !== 'ORESoftware') throw new Error(`unexpected viewer: ${data.viewer.login}`); + const repository = data.repository; + const pr = repository?.pullRequest; + if (!pr) { + skipped.push([`${repo}#${number}`, 'not-found']); + continue; + } + if (pr.merged) { + skipped.push([`${repo}#${number}`, 'already-merged']); + continue; + } + const comments = (pr.comments?.nodes || []).map((comment) => comment.body || '').join('\n'); + const labels = (pr.labels?.nodes || []).map((label) => label.name).join(' '); + const allText = `${pr.title || ''}\n${pr.body || ''}\n${comments}\n${labels}`; + const reasons = []; + if (repo === 'ORESoftware/k8s-cluster' && [231, 792].includes(number)) reasons.push('hard-veto'); + if (pr.headRefOid !== expectedHead) reasons.push('head-moved'); + if (pr.state !== 'OPEN' || pr.isDraft) reasons.push('not-open-ready'); + if (repository.isArchived || !['ADMIN', 'MAINTAIN', 'WRITE'].includes(repository.viewerPermission)) reasons.push('permission'); + if (pr.baseRefName !== repository.defaultBranchRef?.name) reasons.push('non-default-base'); + if (pr.mergeable !== 'MERGEABLE' || !['CLEAN', 'UNSTABLE'].includes(pr.mergeStateStatus)) reasons.push(`merge-${pr.mergeable}-${pr.mergeStateStatus}`); + if (pr.comments?.totalCount !== 0 || pr.comments?.pageInfo?.hasNextPage) reasons.push('comments'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('unresolved-review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage || [...latestReviewStates(pr).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount < 1 || pr.files?.totalCount > 100) reasons.push('file-count'); + if (vetoPattern.test(allText) || concernPattern.test(comments)) reasons.push('veto-or-concern'); + if (!checksGreen(pr)) reasons.push('checks-not-green'); + if (reasons.length) { + skipped.push([`${repo}#${number}`, [...new Set(reasons)].join(',')]); + console.log(`SKIPPED ${repo}#${number} ${[...new Set(reasons)].join(',')}`); + continue; + } + const method = repository.squashMergeAllowed ? 'SQUASH' : repository.rebaseMergeAllowed ? 'REBASE' : repository.mergeCommitAllowed ? 'MERGE' : null; + if (!method) { + skipped.push([`${repo}#${number}`, 'no-merge-method']); + continue; + } + try { + const result = await graphql(mutation, { id: pr.id, head: expectedHead, method }); + const verified = result.mergePullRequest?.pullRequest; + if (!verified?.merged || verified.headRefOid !== expectedHead) throw new Error('mutation did not verify exact merged head'); + merged.push(`${repo}#${number}`); + console.log(`MERGED ${repo}#${number} head=${expectedHead} method=${method}`); + } catch (error) { + skipped.push([`${repo}#${number}`, `mutation:${error.message}`]); + console.error(`SKIPPED ${repo}#${number} mutation error: ${error.message}`); + } + await sleep(250); +} + +console.log(`GRAPHQL_FROZEN_RESULT ${JSON.stringify({ merged: merged.length, skipped: skipped.length })}`); +if (process.env.GITHUB_STEP_SUMMARY) { + const lines = [ + '## GraphQL-only frozen ready batch', + '', + `- Exact heads merged and verified: ${merged.length}`, + `- Skipped after live revalidation: ${skipped.length}`, + '', + ...skipped.map(([key, reason]) => `- ${key}: ${reason}`), + ]; + require('node:fs').appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); +} diff --git a/.github/scripts/merge-graphql-validated-portfolios.mjs b/.github/scripts/merge-graphql-validated-portfolios.mjs new file mode 100644 index 00000000..73f121c0 --- /dev/null +++ b/.github/scripts/merge-graphql-validated-portfolios.mjs @@ -0,0 +1,191 @@ +import fs from 'node:fs'; + +const token = process.env.FLEET_GH_TOKEN; +if (!token) throw new Error('FLEET_GH_TOKEN is required'); +const document = JSON.parse(fs.readFileSync('portfolio-validated/validated.json', 'utf8')); +if (document.schema_version !== 1) throw new Error('unsupported validation manifest'); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function graphql(query, variables = {}, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch('https://api.github.com/graphql', { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'User-Agent': 'graphql-validated-portfolio-merger', + }, + body: JSON.stringify({ query, variables }), + }); + const value = await response.json().catch(() => null); + if (response.ok && !value?.errors?.length) return value.data; + last = new Error(`GraphQL failed: ${JSON.stringify(value).slice(0, 1800)}`); + if (attempt === attempts) throw last; + await sleep(attempt * attempt * 800); + } + throw last; +} + +const query = ` + query Portfolio($owner: String!, $name: String!, $number: Int!) { + viewer { login } + repository(owner: $owner, name: $name) { + nameWithOwner + isArchived + viewerPermission + defaultBranchRef { name } + mergeCommitAllowed + squashMergeAllowed + rebaseMergeAllowed + pullRequest(number: $number) { + id + number + title + body + state + merged + isDraft + headRefName + headRefOid + baseRefName + mergeable + mergeStateStatus + comments(first: 100) { totalCount pageInfo { hasNextPage } nodes { body } } + reviewThreads(first: 100) { pageInfo { hasNextPage } nodes { isResolved } } + reviews(last: 100) { pageInfo { hasPreviousPage } nodes { state author { login } } } + labels(first: 50) { nodes { name } } + files(first: 100) { totalCount pageInfo { hasNextPage } nodes { path changeType } } + } + pullRequests(states: OPEN, first: 100) { + pageInfo { hasNextPage } + nodes { number files(first: 100) { pageInfo { hasNextPage } nodes { path } } } + } + } + rateLimit { remaining resetAt cost } + } +`; +const readyMutation = `mutation Ready($id: ID!) { markPullRequestReadyForReview(input: { pullRequestId: $id }) { pullRequest { id isDraft headRefOid } } }`; +const draftMutation = `mutation Draft($id: ID!) { convertPullRequestToDraft(input: { pullRequestId: $id }) { pullRequest { id isDraft headRefOid } } }`; +const mergeMutation = `mutation Merge($id: ID!, $head: GitObjectID!, $method: PullRequestMergeMethod!) { mergePullRequest(input: { pullRequestId: $id, expectedHeadOid: $head, mergeMethod: $method }) { pullRequest { id number merged mergedAt headRefOid } } }`; + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const latestReviewStates = (pr) => { + const latest = new Map(); + for (const review of pr.reviews?.nodes || []) { + if (!review.author?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.author.login, review.state); + } + return latest; +}; +const intersects = (left, right) => { + for (const value of left) if (right.has(value)) return true; + return false; +}; + +async function gate(entry) { + const [owner, name] = entry.repo.split('/'); + const data = await graphql(query, { owner, name, number: entry.number }); + if (data.viewer.login !== 'ORESoftware') throw new Error(`unexpected viewer ${data.viewer.login}`); + const repository = data.repository; + const pr = repository?.pullRequest; + if (!pr) return { state: 'not-found' }; + if (pr.merged) return { state: 'already-merged', pr }; + const comments = (pr.comments?.nodes || []).map((comment) => comment.body || '').join('\n'); + const labels = (pr.labels?.nodes || []).map((label) => label.name).join(' '); + const reasons = []; + if (pr.headRefOid !== entry.head_sha || pr.headRefName !== entry.head_ref) reasons.push('head-moved'); + if (pr.state !== 'OPEN' || !pr.isDraft) reasons.push('not-open-draft'); + if (repository.isArchived || !['ADMIN', 'MAINTAIN', 'WRITE'].includes(repository.viewerPermission)) reasons.push('permission'); + if (pr.baseRefName !== repository.defaultBranchRef?.name || pr.baseRefName !== entry.base_ref) reasons.push('base-mismatch'); + if (pr.mergeable === 'CONFLICTING' || ['DIRTY', 'BLOCKED', 'UNKNOWN'].includes(pr.mergeStateStatus)) reasons.push(`merge-${pr.mergeable}-${pr.mergeStateStatus}`); + if (pr.comments?.totalCount !== 0 || pr.comments?.pageInfo?.hasNextPage) reasons.push('comments'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage || [...latestReviewStates(pr).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount < 1 || pr.files?.totalCount > 100) reasons.push('file-count'); + if ((pr.files?.nodes || []).some((file) => ['DELETED', 'RENAMED'].includes(file.changeType))) reasons.push('destructive-file-change'); + if (vetoPattern.test(`${pr.title}\n${pr.body || ''}\n${comments}\n${labels}`)) reasons.push('veto'); + if (repository.pullRequests?.pageInfo?.hasNextPage) reasons.push('too-many-open-prs'); + if (!reasons.length) { + const mine = new Set((pr.files?.nodes || []).map((file) => file.path)); + for (const other of repository.pullRequests?.nodes || []) { + if (other.number === pr.number) continue; + if (other.files?.pageInfo?.hasNextPage || intersects(mine, new Set((other.files?.nodes || []).map((file) => file.path)))) { + reasons.push(`overlap-with-pr-${other.number}`); + break; + } + } + } + return { state: reasons.length ? 'blocked' : 'eligible', reasons: [...new Set(reasons)], repository, pr, remaining: data.rateLimit.remaining }; +} + +const merged = []; +const skipped = []; +for (const entry of document.entries || []) { + const key = `${entry.repo}#${entry.number}`; + let current; + try { + current = await gate(entry); + if (current.state === 'already-merged') { + skipped.push([key, 'already-merged']); + continue; + } + if (current.state !== 'eligible') { + skipped.push([key, `${current.state}:${(current.reasons || []).join(',')}`]); + console.log(`SKIPPED ${key} ${current.state} ${(current.reasons || []).join(',')}`); + continue; + } + await graphql(readyMutation, { id: current.pr.id }); + let ready = null; + for (let attempt = 0; attempt < 8; attempt += 1) { + await sleep(900 + attempt * 350); + ready = await gate({ ...entry, head_ref: entry.head_ref }); + if (ready.state === 'blocked' && ready.reasons?.length === 1 && ready.reasons[0] === 'not-open-draft') { + const [owner, name] = entry.repo.split('/'); + const data = await graphql(query, { owner, name, number: entry.number }); + const pr = data.repository?.pullRequest; + if (pr && !pr.isDraft && pr.headRefOid === entry.head_sha && pr.mergeable !== 'CONFLICTING' && !['DIRTY', 'BLOCKED', 'UNKNOWN'].includes(pr.mergeStateStatus)) { + ready = { state: 'ready', repository: data.repository, pr }; + break; + } + } + } + if (ready?.state !== 'ready') { + await graphql(draftMutation, { id: current.pr.id }).catch(() => {}); + skipped.push([key, 'promotion-did-not-become-mergeable']); + continue; + } + const method = ready.repository.squashMergeAllowed ? 'SQUASH' : ready.repository.rebaseMergeAllowed ? 'REBASE' : ready.repository.mergeCommitAllowed ? 'MERGE' : null; + if (!method) { + await graphql(draftMutation, { id: current.pr.id }).catch(() => {}); + skipped.push([key, 'no-merge-method']); + continue; + } + const result = await graphql(mergeMutation, { id: ready.pr.id, head: entry.head_sha, method }); + const verified = result.mergePullRequest?.pullRequest; + if (!verified?.merged || verified.headRefOid !== entry.head_sha) throw new Error('post-merge exact-head verification failed'); + merged.push(key); + console.log(`MERGED ${key} head=${entry.head_sha} method=${method}`); + } catch (error) { + if (current?.pr?.id) await graphql(draftMutation, { id: current.pr.id }).catch(() => {}); + skipped.push([key, `exception:${error.message}`]); + console.error(`SKIPPED ${key} exception=${error.stack || error.message}`); + } + await sleep(250); +} + +console.log(`GRAPHQL_VALIDATED_RESULT ${JSON.stringify({ validated: document.entries?.length || 0, merged: merged.length, skipped: skipped.length })}`); +if (process.env.GITHUB_STEP_SUMMARY) { + const lines = [ + '## Credential-free validated portfolio merge', + '', + `- Exact heads validated without credentials: ${document.entries?.length || 0}`, + `- Promoted, GraphQL-merged, and exact-head verified: ${merged.length}`, + `- Skipped after live revalidation: ${skipped.length}`, + '', + ...skipped.slice(0, 50).map(([key, reason]) => `- ${key}: ${reason}`), + ]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); +} +if (!merged.length) throw new Error('no validated portfolios merged'); diff --git a/.github/scripts/merge-open-pr-fleet.mjs b/.github/scripts/merge-open-pr-fleet.mjs new file mode 100644 index 00000000..574b0dab --- /dev/null +++ b/.github/scripts/merge-open-pr-fleet.mjs @@ -0,0 +1,578 @@ +import fs from 'node:fs'; + +const token = process.env.GH_TOKEN; +const apply = process.env.APPLY === 'true'; +const target = Number(process.env.TARGET || '509'); +const searchLimit = 1000; + +if (!token) throw new Error('GH_TOKEN is required'); +if (!Number.isInteger(target) || target < 1) throw new Error(`invalid TARGET: ${process.env.TARGET}`); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const apiBase = 'https://api.github.com'; +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'exact-head-semantic-pr-fleet-merger', +}; + +async function request(url, options = {}, attempts = 6) { + let lastError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const response = await fetch(url, { + ...options, + headers: { ...headers, ...(options.headers || {}) }, + }); + const text = await response.text(); + const value = text ? JSON.parse(text) : null; + if (response.ok) return value; + + const retryable = response.status === 429 || response.status >= 500 || + (response.status === 403 && /secondary rate limit|abuse detection/i.test(text)); + if (!retryable || attempt === attempts) { + const error = new Error(`${options.method || 'GET'} ${url} -> ${response.status}: ${text.slice(0, 1200)}`); + error.status = response.status; + error.response = value; + throw error; + } + const retryAfter = Number(response.headers.get('retry-after') || 0); + await sleep(Math.max(retryAfter * 1000, attempt * attempt * 1000)); + } catch (error) { + lastError = error; + if (attempt === attempts || (error.status && error.status < 500 && error.status !== 429)) throw error; + await sleep(attempt * attempt * 1000); + } + } + throw lastError; +} + +async function graphql(query, variables = {}) { + const value = await request(`${apiBase}/graphql`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables }), + }); + if (value.errors?.length) { + throw new Error(`GraphQL errors: ${JSON.stringify(value.errors).slice(0, 4000)}`); + } + return value.data; +} + +const pullRequestFields = ` + id + number + title + body + url + isDraft + createdAt + updatedAt + headRefOid + baseRefName + mergeable + mergeStateStatus + reviewDecision + additions + deletions + changedFiles + author { login } + labels(first: 30) { nodes { name } } + comments(first: 1) { totalCount } + headRepository { nameWithOwner } + repository { + nameWithOwner + isArchived + viewerPermission + defaultBranchRef { name } + pullRequests(states: OPEN, first: 1) { totalCount } + mergeCommitAllowed + squashMergeAllowed + rebaseMergeAllowed + } + reviewThreads(first: 100) { + nodes { isResolved } + pageInfo { hasNextPage } + } + reviews(last: 100) { + nodes { state submittedAt author { login } } + pageInfo { hasPreviousPage } + } + commits(last: 1) { + nodes { + commit { + oid + statusCheckRollup { + state + contexts(first: 100) { + nodes { + __typename + ... on CheckRun { name status conclusion } + ... on StatusContext { context state } + } + pageInfo { hasNextPage } + } + } + } + } + } + files(first: 100) { + totalCount + nodes { path additions deletions changeType } + pageInfo { hasNextPage } + } +`; + +const searchQuery = ` + query OpenPullRequests($query: String!, $cursor: String) { + search(query: $query, type: ISSUE, first: 20, after: $cursor) { + issueCount + pageInfo { hasNextPage endCursor } + nodes { + ... on PullRequest { ${pullRequestFields} } + } + } + rateLimit { cost remaining resetAt } + } +`; + +const onePullRequestQuery = ` + query OnePullRequest($owner: String!, $name: String!, $number: Int!) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { ${pullRequestFields} } + } + rateLimit { cost remaining resetAt } + } +`; + +const readyMutation = ` + mutation Ready($id: ID!) { + markPullRequestReadyForReview(input: { pullRequestId: $id }) { + pullRequest { id isDraft headRefOid mergeable mergeStateStatus } + } + } +`; + +const draftMutation = ` + mutation Draft($id: ID!) { + convertPullRequestToDraft(input: { pullRequestId: $id }) { + pullRequest { id isDraft headRefOid } + } + } +`; + +function repoParts(nameWithOwner) { + const slash = nameWithOwner.indexOf('/'); + return [nameWithOwner.slice(0, slash), nameWithOwner.slice(slash + 1)]; +} + +function textOf(pr) { + return `${pr.title || ''}\n${pr.body || ''}`.toLowerCase(); +} + +function isExplicitVeto(pr) { + const repo = pr.repository?.nameWithOwner?.toLowerCase(); + const text = textOf(pr); + const labels = (pr.labels?.nodes || []).map((label) => label.name.toLowerCase()); + return (repo === 'oresoftware/k8s-cluster' && pr.number === 231) || + (repo === 'oresoftware/k8s-cluster' && pr.number === 792) || + /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress/.test(text) || + labels.some((label) => /do[- ]?not[- ]?merge|never[- ]?merge|\bwip\b|hold|blocked/.test(label)); +} + +function isGeneratedOrTestFleet(pr) { + const repo = pr.repository?.nameWithOwner || ''; + const owner = repo.split('/')[0].toLowerCase(); + const title = (pr.title || '').trim(); + const lowerTitle = title.toLowerCase(); + const body = (pr.body || '').toLowerCase(); + const factoryTitle = + /^test: bootstrap (?:independent acceptance portfolio|[a-z0-9][a-z0-9-]* harness)$/i.test(title) || + lowerTitle === 'chore: bootstrap test organization governance' || + lowerTitle === 'test: add isolated playwright, puppeteer, and selenium harness contract'; + const testOrg = owner.endsWith('-test'); + const evidence = + body.includes('github-test-org-factory') || + body.includes('generated') || + body.includes('declared readiness: `ready`') || + body.includes('declared readiness: ready') || + body.includes('harness') || + body.includes('validation') || + body.includes('verification') || + body.includes('tests'); + const testShapedTitle = /^(test|ci|chore)(:|\b)/i.test(title); + return factoryTitle || (testOrg && testShapedTitle && evidence); +} + +function hasReadinessEvidence(pr) { + const body = (pr.body || '').toLowerCase(); + return /\b(validation|verification|tests?|checks?)\b/.test(body); +} + +function latestReviewStates(pr) { + const latest = new Map(); + for (const review of pr.reviews?.nodes || []) { + const login = review.author?.login; + if (!login) continue; + if (review.state === 'COMMENTED' || review.state === 'PENDING') continue; + latest.set(login, review.state); + } + return latest; +} + +function checksAreSuccessful(pr) { + const commit = pr.commits?.nodes?.at(-1)?.commit; + if (!commit || commit.oid !== pr.headRefOid) return false; + const rollup = commit.statusCheckRollup; + if (!rollup || rollup.state !== 'SUCCESS' || rollup.contexts?.pageInfo?.hasNextPage) return false; + const contexts = rollup.contexts?.nodes || []; + if (!contexts.length) return false; + let success = false; + for (const context of contexts) { + if (context.__typename === 'CheckRun') { + if (context.status !== 'COMPLETED') return false; + if (!['SUCCESS', 'NEUTRAL', 'SKIPPED'].includes(context.conclusion)) return false; + if (context.conclusion === 'SUCCESS') success = true; + } else if (context.__typename === 'StatusContext') { + if (context.state !== 'SUCCESS') return false; + success = true; + } else { + return false; + } + } + return success; +} + +function sensitivePath(pathname) { + const path = pathname.toLowerCase(); + if (/(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(path)) return true; + if (/\.(pem|p12|pfx|jks|keystore|key)$/.test(path)) return true; + if (/(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(path)) return true; + if (/(^|\/)\.env($|\.)/.test(path) && !/\.example$|\.sample$|\.template$/.test(path)) return true; + return false; +} + +function coreReasons(pr) { + const reasons = []; + const repo = pr.repository; + const generated = isGeneratedOrTestFleet(pr); + if (!repo?.nameWithOwner) reasons.push('missing-repository'); + if (pr.author?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (isExplicitVeto(pr)) reasons.push('explicit-veto'); + if (repo?.isArchived) reasons.push('archived'); + if (!['ADMIN', 'MAINTAIN', 'WRITE'].includes(repo?.viewerPermission)) reasons.push('no-merge-permission'); + if (!repo?.defaultBranchRef?.name || pr.baseRefName !== repo.defaultBranchRef.name) reasons.push('non-default-base'); + if (!pr.headRepository?.nameWithOwner || pr.headRepository.nameWithOwner !== repo?.nameWithOwner) reasons.push('cross-repository-head'); + if (pr.changedFiles < 1 || pr.changedFiles > 100 || pr.files?.pageInfo?.hasNextPage || pr.files?.totalCount !== pr.changedFiles) reasons.push('file-count'); + if (pr.additions + pr.deletions > (generated ? 7500 : 3500)) reasons.push('change-too-large'); + if ((pr.files?.nodes || []).some((file) => sensitivePath(file.path))) reasons.push('sensitive-path'); + if (generated && (pr.files?.nodes || []).some((file) => ['DELETED', 'RENAMED'].includes(file.changeType))) reasons.push('generated-delete-or-rename'); + if (pr.comments?.totalCount !== 0) reasons.push('issue-comments-present'); + if (pr.reviewThreads?.pageInfo?.hasNextPage || (pr.reviewThreads?.nodes || []).some((thread) => !thread.isResolved)) reasons.push('unresolved-review-thread'); + if (pr.reviews?.pageInfo?.hasPreviousPage) reasons.push('too-many-reviews'); + if (pr.reviewDecision === 'CHANGES_REQUESTED' || [...latestReviewStates(pr).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (!checksAreSuccessful(pr)) reasons.push('checks-not-successful'); + if (pr.mergeable !== 'MERGEABLE') reasons.push(`mergeable-${String(pr.mergeable).toLowerCase()}`); + const allowedState = pr.isDraft ? pr.mergeStateStatus === 'DRAFT' : pr.mergeStateStatus === 'CLEAN'; + if (!allowedState) reasons.push(`merge-state-${String(pr.mergeStateStatus).toLowerCase()}`); + if (pr.isDraft && !generated) reasons.push('production-draft'); + if (pr.isDraft && generated && !hasReadinessEvidence(pr)) reasons.push('draft-without-readiness-evidence'); + if (!generated && !pr.isDraft && !hasReadinessEvidence(pr)) reasons.push('production-without-validation-evidence'); + return reasons; +} + +function fileSet(pr) { + return new Set((pr.files?.nodes || []).map((file) => file.path)); +} + +function intersects(left, right) { + for (const value of left) if (right.has(value)) return true; + return false; +} + +function addIsolation(snapshot) { + const groups = new Map(); + for (const pr of snapshot) { + const repo = pr.repository?.nameWithOwner; + if (!repo) continue; + if (!groups.has(repo)) groups.set(repo, []); + groups.get(repo).push(pr); + } + + const excludedBulkRepos = new Set([ + 'ORESoftware/k8s-cluster', + 'zed-pkg-test/zed-pkg-e2e', + ]); + + for (const [repo, prs] of groups) { + const allObserved = prs.length === prs[0]?.repository?.pullRequests?.totalCount; + const sets = new Map(prs.map((pr) => [pr.number, fileSet(pr)])); + for (const pr of prs) { + const reasons = []; + if (excludedBulkRepos.has(repo)) reasons.push('bulk-excluded-repository'); + if (!allObserved) reasons.push('unobserved-open-prs'); + const mine = sets.get(pr.number); + for (const other of prs) { + if (other.number === pr.number) continue; + if (intersects(mine, sets.get(other.number))) { + reasons.push('overlapping-open-pr'); + break; + } + } + pr._isolationReasons = reasons; + pr._generated = isGeneratedOrTestFleet(pr); + } + } +} + +async function loadSnapshot() { + const byUrl = new Map(); + let cursor = null; + let issueCount = null; + while (byUrl.size < searchLimit) { + const data = await graphql(searchQuery, { + query: 'is:pr is:open author:ORESoftware sort:created-desc', + cursor, + }); + issueCount ??= data.search.issueCount; + for (const node of data.search.nodes || []) { + if (node?.url) byUrl.set(node.url, node); + } + console.log(`SNAPSHOT_PAGE observed=${byUrl.size} total=${issueCount} rate_remaining=${data.rateLimit.remaining}`); + if (!data.search.pageInfo.hasNextPage || byUrl.size >= searchLimit) break; + cursor = data.search.pageInfo.endCursor; + } + const snapshot = [...byUrl.values()]; + addIsolation(snapshot); + return { snapshot, issueCount }; +} + +function classify(snapshot) { + const skipCounts = new Map(); + const candidates = []; + for (const pr of snapshot) { + const reasons = [...coreReasons(pr), ...(pr._isolationReasons || [])]; + if (reasons.length) { + for (const reason of new Set(reasons)) skipCounts.set(reason, (skipCounts.get(reason) || 0) + 1); + } else { + candidates.push(pr); + } + } + candidates.sort((a, b) => { + if (a._generated !== b._generated) return a._generated ? -1 : 1; + if (a.isDraft !== b.isDraft) return a.isDraft ? -1 : 1; + if (a.changedFiles !== b.changedFiles) return a.changedFiles - b.changedFiles; + return new Date(a.createdAt) - new Date(b.createdAt); + }); + return { + candidates, + skipCounts: [...skipCounts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])), + }; +} + +async function loadOne(pr) { + const [owner, name] = repoParts(pr.repository.nameWithOwner); + const data = await graphql(onePullRequestQuery, { owner, name, number: pr.number }); + return data.repository?.pullRequest || null; +} + +function addedLines(patch) { + return String(patch || '') + .split('\n') + .filter((line) => line.startsWith('+') && !line.startsWith('+++')) + .map((line) => line.slice(1)) + .join('\n'); +} + +async function patchReasons(pr) { + const repo = pr.repository.nameWithOwner; + const files = await request(`${apiBase}/repos/${repo}/pulls/${pr.number}/files?per_page=100`); + const reasons = []; + if (!Array.isArray(files) || files.length !== pr.changedFiles) reasons.push('rest-file-count-mismatch'); + for (const file of files || []) { + if (sensitivePath(file.filename)) reasons.push(`sensitive-path:${file.filename}`); + if (pr._generated && ['removed', 'renamed'].includes(file.status)) reasons.push(`generated-${file.status}:${file.filename}`); + if (file.patch == null && file.status !== 'removed') reasons.push(`unreviewable-patch:${file.filename}`); + const patch = String(file.patch || ''); + const additions = addedLines(patch); + if (/^(<<<<<<<|=======|>>>>>>>)( |$)/m.test(patch)) reasons.push(`conflict-marker:${file.filename}`); + if (/(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})/.test(additions)) { + reasons.push(`credential-pattern:${file.filename}`); + } + if (/\.github\/workflows\/.*\.ya?ml$/i.test(file.filename)) { + if (/^\s*pull_request_target\s*:/m.test(additions)) reasons.push(`pull-request-target:${file.filename}`); + if (/^\s*permissions\s*:\s*write-all\s*$/m.test(additions)) reasons.push(`write-all:${file.filename}`); + if (/(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b/i.test(additions)) reasons.push(`remote-shell-pipe:${file.filename}`); + if (/\beval\s+["']?\$/.test(additions)) reasons.push(`eval-shell:${file.filename}`); + for (const line of additions.split('\n')) { + const match = line.match(/^\s*(?:-\s*)?uses:\s*([^\s#]+)\s*$/); + if (!match) continue; + const use = match[1]; + if (use.startsWith('./')) continue; + const at = use.lastIndexOf('@'); + const ref = at >= 0 ? use.slice(at + 1) : ''; + if (!/^[0-9a-f]{40}$/i.test(ref)) reasons.push(`unpinned-action:${file.filename}:${use}`); + } + } + } + return [...new Set(reasons)]; +} + +async function restoreDraft(id) { + try { + await graphql(draftMutation, { id }); + } catch (error) { + console.error(`DRAFT_RESTORE_FAILED id=${id} error=${error.message}`); + } +} + +async function mergeCandidate(snapshotPr) { + let pr = await loadOne(snapshotPr); + if (!pr) return { outcome: 'skipped', reason: 'no-longer-open' }; + pr._generated = snapshotPr._generated; + if (pr.headRefOid !== snapshotPr.headRefOid) return { outcome: 'skipped', reason: 'head-moved' }; + + const freshReasons = coreReasons(pr); + if (freshReasons.length) return { outcome: 'skipped', reason: `fresh:${freshReasons.join(',')}` }; + + const patchFailures = await patchReasons(pr); + if (patchFailures.length) return { outcome: 'skipped', reason: `patch:${patchFailures.join(',')}` }; + + let promoted = false; + if (pr.isDraft) { + await graphql(readyMutation, { id: pr.id }); + promoted = true; + let ready = null; + for (let attempt = 0; attempt < 6; attempt += 1) { + await sleep(1500 + attempt * 500); + ready = await loadOne(pr); + if (ready && !ready.isDraft && ready.mergeable === 'MERGEABLE' && ready.mergeStateStatus === 'CLEAN') break; + } + if (!ready || ready.isDraft || ready.mergeable !== 'MERGEABLE' || ready.mergeStateStatus !== 'CLEAN' || !checksAreSuccessful(ready)) { + await restoreDraft(pr.id); + return { outcome: 'skipped', reason: 'promotion-did-not-become-clean' }; + } + ready._generated = pr._generated; + pr = ready; + } + + if (pr.headRefOid !== snapshotPr.headRefOid) { + if (promoted) await restoreDraft(pr.id); + return { outcome: 'skipped', reason: 'head-moved-after-promotion' }; + } + + const method = pr.repository.squashMergeAllowed ? 'squash' : + pr.repository.rebaseMergeAllowed ? 'rebase' : + pr.repository.mergeCommitAllowed ? 'merge' : null; + if (!method) { + if (promoted) await restoreDraft(pr.id); + return { outcome: 'skipped', reason: 'no-allowed-merge-method' }; + } + + const repo = pr.repository.nameWithOwner; + let result; + try { + result = await request(`${apiBase}/repos/${repo}/pulls/${pr.number}/merge`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sha: pr.headRefOid, merge_method: method }), + }, 3); + } catch (error) { + if (promoted) await restoreDraft(pr.id); + return { outcome: 'skipped', reason: `merge-api:${error.status || 'error'}:${error.message.slice(0, 300)}` }; + } + + if (!result?.merged) { + if (promoted) await restoreDraft(pr.id); + return { outcome: 'skipped', reason: `not-merged:${result?.message || 'unknown'}` }; + } + + const verified = await request(`${apiBase}/repos/${repo}/pulls/${pr.number}`); + if (!verified?.merged || verified?.head?.sha !== pr.headRefOid) { + throw new Error(`post-merge verification failed for ${repo}#${pr.number}`); + } + return { outcome: 'merged', method, mergeSha: result.sha, headSha: pr.headRefOid, repo, number: pr.number, title: pr.title }; +} + +function appendSummary(lines) { + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${lines.join('\n')}\n`); +} + +const { snapshot, issueCount } = await loadSnapshot(); +const { candidates, skipCounts } = classify(snapshot); +const generatedCandidates = candidates.filter((pr) => pr._generated).length; +const productionCandidates = candidates.length - generatedCandidates; + +console.log(`PREFLIGHT_SUMMARY ${JSON.stringify({ + mode: apply ? 'apply' : 'preflight', + api_total_count: issueCount, + observed: snapshot.length, + eligible: candidates.length, + generated_eligible: generatedCandidates, + production_eligible: productionCandidates, + target, +})}`); +console.log('SKIP_REASONS'); +for (const [reason, count] of skipCounts.slice(0, 50)) console.log(`${count}\t${reason}`); +console.log('ELIGIBLE_SAMPLE'); +for (const pr of candidates.slice(0, 100)) { + console.log(`${pr._generated ? 'generated' : 'production'}\t${pr.isDraft ? 'draft' : 'ready'}\t${pr.repository.nameWithOwner}#${pr.number}\t${pr.headRefOid}\t${pr.title}`); +} + +appendSummary([ + `## ${apply ? 'Apply' : 'Preflight'}: exact-head PR fleet merge`, + '', + `- GitHub search total: ${issueCount}`, + `- PRs observed: ${snapshot.length}`, + `- Eligible after semantic/isolation gates: ${candidates.length}`, + `- Generated/test-fleet eligible: ${generatedCandidates}`, + `- Production ready eligible: ${productionCandidates}`, + `- Merge target: ${target}`, +]); + +if (!apply) { + if (candidates.length < target) console.log(`::warning::preflight found ${candidates.length} eligible PRs, below target ${target}`); + process.exit(0); +} + +const outcomes = new Map(); +const merged = []; +for (const candidate of candidates) { + if (merged.length >= target) break; + const key = `${candidate.repository.nameWithOwner}#${candidate.number}`; + try { + const result = await mergeCandidate(candidate); + outcomes.set(result.reason || result.outcome, (outcomes.get(result.reason || result.outcome) || 0) + 1); + if (result.outcome === 'merged') { + merged.push(result); + console.log(`MERGED ${result.repo}#${result.number} head=${result.headSha} merge=${result.mergeSha} method=${result.method} title=${JSON.stringify(result.title)}`); + } else { + console.log(`SKIPPED ${key} reason=${result.reason}`); + } + } catch (error) { + outcomes.set('exception', (outcomes.get('exception') || 0) + 1); + console.error(`SKIPPED ${key} reason=exception:${error.stack || error.message}`); + } + await sleep(350); +} + +const outcomeCounts = [...outcomes.entries()].sort((a, b) => b[1] - a[1]); +console.log(`APPLY_RESULT ${JSON.stringify({ merged: merged.length, target, outcomes: Object.fromEntries(outcomeCounts) })}`); +appendSummary([ + '', + `### Result`, + '', + `- Successfully merged and post-verified: ${merged.length}`, + `- Target: ${target}`, + `- Exact-head guard: required for every merge`, + `- Hard veto preserved: ORESoftware/k8s-cluster#231`, + '', + '### Runtime outcomes', + '', + ...outcomeCounts.slice(0, 30).map(([reason, count]) => `- ${reason}: ${count}`), +]); + +if (merged.length < target) { + throw new Error(`merged ${merged.length}, below required target ${target}`); +} diff --git a/.github/scripts/normalize-portfolio-archives.py b/.github/scripts/normalize-portfolio-archives.py new file mode 100644 index 00000000..15b0b27b --- /dev/null +++ b/.github/scripts/normalize-portfolio-archives.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import copy +import hashlib +import json +import os +import tarfile +import tempfile +from pathlib import Path, PurePosixPath + +BUNDLE = Path("portfolio-bundle") +MANIFEST = BUNDLE / "manifest.json" + + +def normalize(entry: dict) -> None: + archive = BUNDLE / entry["archive"] + repository_name = entry["repo"].split("/", 1)[1] + with tarfile.open(archive, "r:gz") as source: + members = source.getmembers() + roots = {PurePosixPath(member.name).parts[0] for member in members if PurePosixPath(member.name).parts} + if len(roots) != 1: + raise ValueError(f"{entry['repo']}#{entry['number']}: expected one archive root, got {sorted(roots)}") + old_root = next(iter(roots)) + fd, temporary_name = tempfile.mkstemp(prefix="normalized-", suffix=".tar.gz", dir=archive.parent) + os.close(fd) + temporary = Path(temporary_name) + try: + with tarfile.open(temporary, "w:gz") as output: + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not path.parts or path.parts[0] != old_root: + raise ValueError(f"unsafe archive path: {member.name}") + if member.issym() or member.islnk() or member.isdev() or member.isfifo(): + raise ValueError(f"unsupported archive entry: {member.name}") + rewritten = copy.copy(member) + rewritten.name = str(PurePosixPath(repository_name, *path.parts[1:])) + fileobj = source.extractfile(member) if member.isfile() else None + output.addfile(rewritten, fileobj) + temporary.replace(archive) + finally: + temporary.unlink(missing_ok=True) + + data = archive.read_bytes() + entry["archive_sha256"] = hashlib.sha256(data).hexdigest() + entry["archive_bytes"] = len(data) + + +def main() -> None: + manifest = json.loads(MANIFEST.read_text()) + for entry in manifest.get("entries", []): + normalize(entry) + manifest["archives_normalized_to_repository_name"] = True + MANIFEST.write_text(json.dumps(manifest, indent=2) + "\n") + print(f"NORMALIZED_PORTFOLIO_ARCHIVES count={len(manifest.get('entries', []))}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/ready-portfolio-fleet.mjs b/.github/scripts/ready-portfolio-fleet.mjs new file mode 100644 index 00000000..85c81943 --- /dev/null +++ b/.github/scripts/ready-portfolio-fleet.mjs @@ -0,0 +1,396 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; + +const mode = process.argv[2]; +const api = 'https://api.github.com'; +const token = process.env.FLEET_GH_TOKEN || process.env.GH_TOKEN; +if (!['discover', 'merge'].includes(mode)) throw new Error('usage: node ready-portfolio-fleet.mjs '); +if (!token) throw new Error('FLEET_GH_TOKEN is required for this mode'); + +const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'ready-portfolio-fleet/1.0', +}; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function request(url, options = {}, attempts = 6) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { + ...options, + headers: { ...headers, ...(options.headers || {}) }, + redirect: options.redirect || 'follow', + }); + const contentType = response.headers.get('content-type') || ''; + const value = contentType.includes('json') + ? await response.json().catch(() => null) + : await response.text(); + if (response.ok) return value; + last = new Error(`${options.method || 'GET'} ${url} -> ${response.status}: ${JSON.stringify(value).slice(0, 900)}`); + const retryable = response.status === 429 || response.status >= 500 || + (response.status === 403 && /secondary rate limit|abuse detection/i.test(JSON.stringify(value))); + if (!retryable || attempt === attempts) throw last; + const retryAfter = Number(response.headers.get('retry-after') || 0); + await sleep(Math.max(retryAfter * 1000, attempt * attempt * 800)); + } + throw last; +} + +async function requestBytes(url, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { headers, redirect: 'follow' }); + if (response.ok) return Buffer.from(await response.arrayBuffer()); + const text = await response.text(); + last = new Error(`GET ${url} -> ${response.status}: ${text.slice(0, 600)}`); + if (![429, 500, 502, 503, 504].includes(response.status) || attempt === attempts) throw last; + await sleep(attempt * attempt * 1000); + } + throw last; +} + +async function mapLimit(values, limit, fn) { + const results = new Array(values.length); + let next = 0; + async function worker() { + while (true) { + const index = next++; + if (index >= values.length) return; + try { results[index] = await fn(values[index], index); } + catch (error) { results[index] = { error: error.message, source: values[index] }; } + } + } + await Promise.all(Array.from({ length: Math.min(limit, values.length) }, worker)); + return results; +} + +const vetoPattern = /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b|work in progress|not ready|hold (?:this|off)|blocked by|changes requested|must not merge/i; +const credentialPattern = /(ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})/; + +function latestReviewStates(reviews) { + const latest = new Map(); + for (const review of reviews) { + if (!review.user?.login || ['COMMENTED', 'PENDING', 'DISMISSED'].includes(review.state)) continue; + latest.set(review.user.login, review.state); + } + return latest; +} + +function sensitivePath(filename) { + const value = filename.toLowerCase(); + return /(^|\/)(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.|$)/.test(value) || + /\.(pem|p12|pfx|jks|keystore|key)$/.test(value) || + /(^|\/)(credentials?|secrets?|tokens?)(\.|\/|$)/.test(value) || + (/(^|\/)\.env($|\.)/.test(value) && !/\.example$|\.sample$|\.template$/.test(value)); +} + +function addedLines(patch) { + return String(patch || '') + .split('\n') + .filter((line) => line.startsWith('+') && !line.startsWith('+++')) + .map((line) => line.slice(1)) + .join('\n'); +} + +function patchReasons(files) { + const reasons = []; + for (const file of files) { + if (['removed', 'renamed'].includes(file.status)) reasons.push(`destructive-file-change:${file.filename}`); + if (sensitivePath(file.filename)) reasons.push(`sensitive-path:${file.filename}`); + const patch = String(file.patch || ''); + const additions = addedLines(patch); + if (file.patch == null && file.status !== 'removed') reasons.push(`unreviewable-patch:${file.filename}`); + if (/^(<<<<<<<|=======|>>>>>>>)( |$)/m.test(patch)) reasons.push(`conflict-marker:${file.filename}`); + if (credentialPattern.test(additions)) reasons.push(`credential-pattern:${file.filename}`); + if (/\.github\/workflows\/.*\.ya?ml$/i.test(file.filename)) { + if (/^\s*pull_request_target\s*:/m.test(additions)) reasons.push(`pull-request-target:${file.filename}`); + if (/^\s*permissions\s*:\s*write-all\s*$/m.test(additions)) reasons.push(`write-all:${file.filename}`); + if (/(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b/i.test(additions)) reasons.push(`remote-shell-pipe:${file.filename}`); + if (/\beval\s+["']?\$/.test(additions)) reasons.push(`eval-shell:${file.filename}`); + for (const line of additions.split('\n')) { + const match = line.match(/^\s*(?:-\s*)?uses:\s*([^\s#]+)\s*$/); + if (!match) continue; + const use = match[1]; + if (use.startsWith('./')) continue; + const at = use.lastIndexOf('@'); + const action = at >= 0 ? use.slice(0, at) : use; + const ref = at >= 0 ? use.slice(at + 1) : ''; + const immutable = /^[0-9a-f]{40}$/i.test(ref); + const trustedMajor = /^(actions|github)\//.test(action) && /^v\d+$/.test(ref); + if (!immutable && !trustedMajor) reasons.push(`untrusted-unpinned-action:${file.filename}:${use}`); + } + } + } + return [...new Set(reasons)]; +} + +async function classifyChecks(repo, sha) { + const [runs, combined] = await Promise.all([ + request(`${api}/repos/${repo}/commits/${sha}/check-runs?per_page=100`), + request(`${api}/repos/${repo}/commits/${sha}/status?per_page=100`), + ]); + const checkRuns = runs.check_runs || []; + const statuses = combined.statuses || []; + let pending = 0; + let realFailures = 0; + let billingFailures = 0; + let successes = 0; + for (const run of checkRuns) { + if (run.status !== 'completed') { + pending += 1; + continue; + } + if (['success', 'neutral', 'skipped'].includes(run.conclusion)) { + if (run.conclusion === 'success') successes += 1; + continue; + } + const annotations = await request(`${api}/repos/${repo}/check-runs/${run.id}/annotations?per_page=100`); + const billingOnly = annotations.length > 0 && annotations.every((annotation) => + /job was not started because recent account payments have failed|spending limit needs to be increased/i.test(annotation.message || '') + ); + if (billingOnly) billingFailures += 1; + else realFailures += 1; + } + for (const status of statuses) { + if (status.state === 'pending') pending += 1; + else if (['failure', 'error'].includes(status.state)) realFailures += 1; + else if (status.state === 'success') successes += 1; + } + const category = pending ? 'pending' : realFailures ? 'failed' : billingFailures ? 'billing-only' : successes ? 'green' : 'no-checks'; + return { category, pending, realFailures, billingFailures, successes, total: checkRuns.length + statuses.length }; +} + +async function inspectPortfolio(item) { + const repo = item.repository_url.replace(`${api}/repos/`, ''); + const [owner] = repo.split('/'); + const reasons = []; + const body = item.body || ''; + if (!owner.toLowerCase().endsWith('-test')) reasons.push('not-test-org'); + if (item.title !== 'test: bootstrap independent acceptance portfolio') reasons.push('wrong-title'); + if (!/- Declared readiness:\s*`ready`/i.test(body)) reasons.push('not-declared-ready'); + if (/planned_dependency|source-gated until:/i.test(body)) reasons.push('planned-or-source-gated'); + if (vetoPattern.test(`${item.title}\n${body}\n${(item.labels || []).map((label) => label.name).join(' ')}`)) reasons.push('text-veto'); + if (item.comments !== 0) reasons.push('issue-comments'); + if (reasons.length) return { repo, number: item.number, reasons, prefiltered: true }; + + let pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + for (let attempt = 0; pr.mergeable == null && attempt < 5; attempt += 1) { + await sleep(700 + attempt * 400); + pr = await request(`${api}/repos/${repo}/pulls/${item.number}`); + } + + const [repository, reviews, files, openPulls, checks] = await Promise.all([ + request(`${api}/repos/${repo}`), + request(`${api}/repos/${repo}/pulls/${item.number}/reviews?per_page=100`), + request(`${api}/repos/${repo}/pulls/${item.number}/files?per_page=100`), + request(`${api}/repos/${repo}/pulls?state=open&per_page=100`), + classifyChecks(repo, pr.head.sha), + ]); + + if (pr.state !== 'open' || !pr.draft) reasons.push('not-open-draft'); + if (pr.user?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (repository.archived) reasons.push('archived'); + if (!(repository.permissions?.admin || repository.permissions?.maintain || repository.permissions?.push)) reasons.push('no-write-permission'); + if (pr.base?.ref !== repository.default_branch) reasons.push('non-default-base'); + if (pr.head?.repo?.full_name !== repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== true) reasons.push(`mergeable-${String(pr.mergeable)}`); + if (!['draft', 'clean', 'unstable'].includes(pr.mergeable_state)) reasons.push(`merge-state-${pr.mergeable_state}`); + if (pr.mergeable_state === 'unstable' && !['billing-only', 'green'].includes(checks.category)) reasons.push('unstable-without-billing-explanation'); + if (pr.comments !== 0 || pr.review_comments !== 0) reasons.push('comments-or-inline-review'); + if ([...latestReviewStates(reviews).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (openPulls.length !== 1 || openPulls[0]?.number !== item.number) reasons.push('multiple-open-prs-in-repository'); + if (pr.changed_files < 1 || pr.changed_files > 100 || files.length !== pr.changed_files) reasons.push('file-count'); + if (pr.additions + pr.deletions > 20000) reasons.push('change-too-large'); + if (['pending', 'failed'].includes(checks.category)) reasons.push(`checks-${checks.category}`); + reasons.push(...patchReasons(files)); + + return { + repo, + number: item.number, + node_id: pr.node_id, + title: pr.title, + head_ref: pr.head.ref, + head_sha: pr.head.sha, + base_ref: pr.base.ref, + base_sha: pr.base.sha, + private: repository.private, + changed_files: pr.changed_files, + additions: pr.additions, + deletions: pr.deletions, + checks, + reasons: [...new Set(reasons)], + }; +} + +async function discover() { + const out = path.resolve('portfolio-bundle'); + const archives = path.join(out, 'archives'); + fs.rmSync(out, { recursive: true, force: true }); + fs.mkdirSync(archives, { recursive: true }); + + const query = encodeURIComponent('is:pr is:open author:ORESoftware draft:true created:<2026-08-05T12:45:00Z "Declared readiness: `ready`"'); + const search = await request(`${api}/search/issues?q=${query}&per_page=100&page=1`); + const inspected = await mapLimit(search.items || [], 10, inspectPortfolio); + const errors = inspected.filter((entry) => entry.error); + const valid = inspected.filter((entry) => !entry.error); + const candidates = valid.filter((entry) => entry.reasons.length === 0); + const skipCounts = new Map(); + for (const entry of valid) for (const reason of entry.reasons) skipCounts.set(reason, (skipCounts.get(reason) || 0) + 1); + + console.log(`PORTFOLIO_DISCOVERY ${JSON.stringify({ searched: search.total_count, inspected: valid.length, errors: errors.length, candidates: candidates.length })}`); + for (const [reason, count] of [...skipCounts.entries()].sort((a, b) => b[1] - a[1])) console.log(`SKIP ${count}\t${reason}`); + + const downloaded = await mapLimit(candidates, 6, async (candidate, index) => { + const bytes = await requestBytes(`${api}/repos/${candidate.repo}/tarball/${candidate.head_sha}`); + if (bytes.length < 256) throw new Error(`archive too small for ${candidate.repo}#${candidate.number}`); + const digest = crypto.createHash('sha256').update(bytes).digest('hex'); + const filename = `${String(index).padStart(3, '0')}-${candidate.repo.replace('/', '__')}--pr-${candidate.number}.tar.gz`; + fs.writeFileSync(path.join(archives, filename), bytes, { mode: 0o600 }); + return { ...candidate, archive: `archives/${filename}`, archive_sha256: digest, archive_bytes: bytes.length }; + }); + + const failures = downloaded.filter((entry) => entry.error); + const manifest = downloaded.filter((entry) => !entry.error); + fs.writeFileSync(path.join(out, 'manifest.json'), JSON.stringify({ schema_version: 1, generated_at: new Date().toISOString(), entries: manifest, download_failures: failures }, null, 2) + '\n'); + console.log(`PORTFOLIO_BUNDLE ${JSON.stringify({ entries: manifest.length, download_failures: failures.length })}`); + if (failures.length) for (const failure of failures) console.log(`DOWNLOAD_FAILURE ${JSON.stringify(failure)}`); + if (!manifest.length) throw new Error('no ready portfolios survived discovery'); +} + +async function graphql(query, variables) { + const value = await request(`${api}/graphql`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, variables }), + }); + if (value.errors?.length) throw new Error(`GraphQL error: ${JSON.stringify(value.errors).slice(0, 1200)}`); + return value.data; +} + +const readyMutation = `mutation Ready($id: ID!) { markPullRequestReadyForReview(input: { pullRequestId: $id }) { pullRequest { id isDraft headRefOid } } }`; +const draftMutation = `mutation Draft($id: ID!) { convertPullRequestToDraft(input: { pullRequestId: $id }) { pullRequest { id isDraft headRefOid } } }`; + +async function freshMergeGate(entry) { + let pr = await request(`${api}/repos/${entry.repo}/pulls/${entry.number}`); + if (pr.merged) return { state: 'already-merged', pr }; + if (pr.state !== 'open') return { state: 'closed-unmerged', pr }; + if (pr.head?.sha !== entry.head_sha || pr.head?.ref !== entry.head_ref) return { state: 'head-moved', pr }; + for (let attempt = 0; pr.mergeable == null && attempt < 5; attempt += 1) { + await sleep(700 + attempt * 400); + pr = await request(`${api}/repos/${entry.repo}/pulls/${entry.number}`); + } + const [repository, comments, reviews, openPulls, checks] = await Promise.all([ + request(`${api}/repos/${entry.repo}`), + pr.comments ? request(`${api}/repos/${entry.repo}/issues/${entry.number}/comments?per_page=100`) : Promise.resolve([]), + request(`${api}/repos/${entry.repo}/pulls/${entry.number}/reviews?per_page=100`), + request(`${api}/repos/${entry.repo}/pulls?state=open&per_page=100`), + classifyChecks(entry.repo, entry.head_sha), + ]); + const reasons = []; + const commentText = comments.map((comment) => comment.body || '').join('\n'); + if (vetoPattern.test(`${pr.title || ''}\n${pr.body || ''}\n${commentText}\n${(pr.labels || []).map((label) => label.name).join(' ')}`)) reasons.push('veto'); + if (!pr.draft) reasons.push('not-draft-before-promotion'); + if (pr.user?.login !== 'ORESoftware') reasons.push('wrong-author'); + if (repository.archived) reasons.push('archived'); + if (!(repository.permissions?.admin || repository.permissions?.maintain || repository.permissions?.push)) reasons.push('no-write-permission'); + if (pr.base?.ref !== repository.default_branch || pr.base?.ref !== entry.base_ref) reasons.push('base-mismatch'); + if (pr.head?.repo?.full_name !== entry.repo) reasons.push('cross-repository-head'); + if (pr.mergeable !== true || ['dirty', 'blocked', 'unknown'].includes(pr.mergeable_state)) reasons.push(`merge-${pr.mergeable}-${pr.mergeable_state}`); + if (pr.comments !== 0 || pr.review_comments !== 0 || comments.length !== 0) reasons.push('comments'); + if ([...latestReviewStates(reviews).values()].includes('CHANGES_REQUESTED')) reasons.push('changes-requested'); + if (openPulls.length !== 1 || openPulls[0]?.number !== entry.number) reasons.push('multiple-open-prs'); + if (['pending', 'failed'].includes(checks.category)) reasons.push(`checks-${checks.category}`); + return { state: reasons.length ? 'blocked' : 'eligible', pr, repository, checks, reasons }; +} + +async function mergeOne(entry) { + let gate = await freshMergeGate(entry); + if (gate.state !== 'eligible') return { outcome: gate.state, reasons: gate.reasons || [] }; + await graphql(readyMutation, { id: entry.node_id }); + let promoted = true; + let ready = null; + for (let attempt = 0; attempt < 8; attempt += 1) { + await sleep(1000 + attempt * 400); + ready = await request(`${api}/repos/${entry.repo}/pulls/${entry.number}`); + if (!ready.draft && ready.head?.sha === entry.head_sha && ready.mergeable === true && !['dirty', 'blocked', 'unknown'].includes(ready.mergeable_state)) break; + } + if (!ready || ready.draft || ready.head?.sha !== entry.head_sha || ready.mergeable !== true || ['dirty', 'blocked', 'unknown'].includes(ready.mergeable_state)) { + if (promoted) await graphql(draftMutation, { id: entry.node_id }).catch(() => {}); + return { outcome: 'promotion-not-clean' }; + } + + const postChecks = await classifyChecks(entry.repo, entry.head_sha); + if (['pending', 'failed'].includes(postChecks.category)) { + await graphql(draftMutation, { id: entry.node_id }).catch(() => {}); + return { outcome: `post-promotion-checks-${postChecks.category}` }; + } + + const method = gate.repository.allow_squash_merge ? 'squash' : gate.repository.allow_rebase_merge ? 'rebase' : gate.repository.allow_merge_commit ? 'merge' : null; + if (!method) { + await graphql(draftMutation, { id: entry.node_id }).catch(() => {}); + return { outcome: 'no-merge-method' }; + } + + let result; + try { + result = await request(`${api}/repos/${entry.repo}/pulls/${entry.number}/merge`, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ sha: entry.head_sha, merge_method: method }), + }, 3); + } catch (error) { + await graphql(draftMutation, { id: entry.node_id }).catch(() => {}); + return { outcome: `merge-api-error:${error.message.slice(0, 300)}` }; + } + if (!result?.merged) { + await graphql(draftMutation, { id: entry.node_id }).catch(() => {}); + return { outcome: `merge-declined:${result?.message || 'unknown'}` }; + } + const verified = await request(`${api}/repos/${entry.repo}/pulls/${entry.number}`); + if (!verified.merged || verified.head?.sha !== entry.head_sha) throw new Error(`post-merge verification failed for ${entry.repo}#${entry.number}`); + return { outcome: 'merged', method, merge_sha: result.sha }; +} + +async function mergeValidated() { + const document = JSON.parse(fs.readFileSync('portfolio-validated/validated.json', 'utf8')); + if (document.schema_version !== 1) throw new Error('unsupported validated manifest'); + const merged = []; + const skipped = []; + for (const entry of document.entries || []) { + const key = `${entry.repo}#${entry.number}`; + try { + const result = await mergeOne(entry); + if (result.outcome === 'merged') { + merged.push({ key, head_sha: entry.head_sha, ...result }); + console.log(`MERGED ${key} head=${entry.head_sha} merge=${result.merge_sha} method=${result.method}`); + } else { + skipped.push({ key, ...result }); + console.log(`SKIPPED ${key} outcome=${result.outcome} reasons=${(result.reasons || []).join(',')}`); + } + } catch (error) { + skipped.push({ key, outcome: 'exception', error: error.message }); + console.error(`SKIPPED ${key} exception=${error.stack || error.message}`); + } + await sleep(350); + } + console.log(`PORTFOLIO_MERGE_RESULT ${JSON.stringify({ validated: document.entries?.length || 0, merged: merged.length, skipped: skipped.length })}`); + if (process.env.GITHUB_STEP_SUMMARY) { + const lines = [ + '## Ready portfolio merge result', + '', + `- Independently validated exact heads: ${document.entries?.length || 0}`, + `- Promoted, merged, and post-verified: ${merged.length}`, + `- Skipped after live revalidation: ${skipped.length}`, + '', + ...skipped.slice(0, 40).map((entry) => `- ${entry.key}: ${entry.outcome}${entry.reasons?.length ? ` (${entry.reasons.join(', ')})` : ''}`), + ]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); + } + if (!merged.length) throw new Error('no validated ready portfolios merged'); +} + +if (mode === 'discover') await discover(); +else await mergeValidated(); diff --git a/.github/scripts/validate-ready-portfolios.py b/.github/scripts/validate-ready-portfolios.py new file mode 100644 index 00000000..65b2b10f --- /dev/null +++ b/.github/scripts/validate-ready-portfolios.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import concurrent.futures +import hashlib +import json +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +from pathlib import Path, PurePosixPath +from typing import Any + +BUNDLE = Path("portfolio-bundle") +OUTPUT = Path("portfolio-validated") +CREDENTIAL_RE = re.compile( + rb"(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|" + rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})" +) +CONFLICT_RE = re.compile(rb"^(?:<<<<<<<|=======|>>>>>>>)(?: |$)", re.MULTILINE) + + +def run(command: list[str], cwd: Path, timeout: int = 180) -> dict[str, Any]: + env = os.environ.copy() + for name in ( + "GH_TOKEN", + "GITHUB_TOKEN", + "FLEET_GH_TOKEN", + "CROSS_ORG_APP_ID", + "CROSS_ORG_APP_PRIVATE_KEY", + ): + env.pop(name, None) + env.update( + { + "CI": "true", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONUNBUFFERED": "1", + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + } + ) + completed = subprocess.run( + command, + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + return { + "command": command, + "returncode": completed.returncode, + "output": completed.stdout[-12000:], + } + + +def safe_extract(archive: Path, destination: Path) -> Path: + total_size = 0 + with tarfile.open(archive, "r:gz") as tf: + members = tf.getmembers() + if not members or len(members) > 2000: + raise ValueError(f"unexpected archive member count: {len(members)}") + roots: set[str] = set() + for member in members: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or not path.parts: + raise ValueError(f"unsafe archive path: {member.name}") + roots.add(path.parts[0]) + if member.issym() or member.islnk() or member.isdev() or member.isfifo(): + raise ValueError(f"unsupported archive entry type: {member.name}") + if member.isfile(): + total_size += member.size + if len(roots) != 1: + raise ValueError(f"archive must have one root directory, found {sorted(roots)}") + if total_size > 60 * 1024 * 1024: + raise ValueError(f"archive expands beyond 60 MiB: {total_size}") + root_name = next(iter(roots)) + for member in members: + target = destination.joinpath(*PurePosixPath(member.name).parts) + target.parent.mkdir(parents=True, exist_ok=True) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + source = tf.extractfile(member) + if source is None: + raise ValueError(f"could not read archive member: {member.name}") + with source, target.open("wb") as output: + shutil.copyfileobj(source, output) + target.chmod(member.mode & 0o777) + return destination / root_name + + +def scan_repository(root: Path) -> list[str]: + reasons: list[str] = [] + required = [ + ".managed-by-test-org-factory", + "dependency-contract.yaml", + "test-plan.yaml", + "scripts/readiness.py", + "tests/test_generated_contract.py", + ] + for relative in required: + target = root / relative + if not target.is_file() or target.stat().st_size == 0: + reasons.append(f"missing-required:{relative}") + + for path in root.rglob("*"): + if not path.is_file() or path.stat().st_size > 2 * 1024 * 1024: + continue + try: + data = path.read_bytes() + except OSError as error: + reasons.append(f"read-error:{path.relative_to(root)}:{error}") + continue + relative = path.relative_to(root).as_posix() + if CONFLICT_RE.search(data): + reasons.append(f"conflict-marker:{relative}") + if CREDENTIAL_RE.search(data): + reasons.append(f"credential-pattern:{relative}") + if relative.startswith(".github/workflows/") and relative.endswith((".yml", ".yaml")): + text = data.decode("utf-8", errors="replace") + if re.search(r"^\s*pull_request_target\s*:", text, re.MULTILINE): + reasons.append(f"pull-request-target:{relative}") + if re.search(r"^\s*permissions\s*:\s*write-all\s*$", text, re.MULTILINE): + reasons.append(f"write-all:{relative}") + if re.search(r"(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b", text, re.IGNORECASE): + reasons.append(f"remote-shell-pipe:{relative}") + if re.search(r"\beval\s+[\"']?\$", text): + reasons.append(f"eval-shell:{relative}") + for use in re.findall(r"^\s*(?:-\s*)?uses:\s*([^\s#]+)\s*$", text, re.MULTILINE): + if use.startswith("./"): + continue + action, separator, ref = use.rpartition("@") + immutable = bool(re.fullmatch(r"[0-9a-fA-F]{40}", ref)) + trusted_major = bool(re.match(r"^(actions|github)/", action) and re.fullmatch(r"v\d+", ref)) + if not separator or not (immutable or trusted_major): + reasons.append(f"untrusted-unpinned-action:{relative}:{use}") + return sorted(set(reasons)) + + +def validate_entry(entry: dict[str, Any]) -> dict[str, Any]: + key = f"{entry['repo']}#{entry['number']}" + archive = BUNDLE / entry["archive"] + result: dict[str, Any] = {"key": key, "entry": entry, "checks": [], "reasons": []} + try: + data = archive.read_bytes() + digest = hashlib.sha256(data).hexdigest() + if digest != entry["archive_sha256"]: + result["reasons"].append("archive-digest-mismatch") + return result + + with tempfile.TemporaryDirectory(prefix="portfolio-") as temp: + root = safe_extract(archive, Path(temp)) + result["reasons"].extend(scan_repository(root)) + if result["reasons"]: + return result + + checks = [ + run(["python3", "-m", "compileall", "-q", "scripts", "tests"], root, 90), + run(["python3", "-m", "pytest", "-q"], root, 240), + run(["python3", "scripts/readiness.py", "--offline", "--strict"], root, 90), + ] + result["checks"] = checks + for check in checks: + if check["returncode"] != 0: + result["reasons"].append( + f"command-failed:{' '.join(check['command'])}:exit-{check['returncode']}" + ) + + readiness_path = root / "artifacts/readiness-status.json" + if not readiness_path.is_file(): + result["reasons"].append("missing-readiness-artifact") + else: + readiness = json.loads(readiness_path.read_text()) + result["readiness"] = readiness + if readiness.get("declared_readiness") != "ready": + result["reasons"].append("declared-readiness-not-ready") + if readiness.get("overall") != "ready": + result["reasons"].append("offline-readiness-not-ready") + + if (root / "package.json").is_file(): + package = run( + [ + "node", + "-e", + "JSON.parse(require('node:fs').readFileSync('package.json','utf8'));", + ], + root, + 30, + ) + result["checks"].append(package) + if package["returncode"] != 0: + result["reasons"].append("invalid-package-json") + + result["reasons"] = sorted(set(result["reasons"])) + result["validated"] = not result["reasons"] + return result + except Exception as error: # noqa: BLE001 + result["reasons"].append(f"exception:{type(error).__name__}:{error}") + result["validated"] = False + return result + + +def main() -> int: + manifest = json.loads((BUNDLE / "manifest.json").read_text()) + entries = manifest.get("entries", []) + OUTPUT.mkdir(parents=True, exist_ok=True) + workers = min(8, max(1, len(entries))) + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + results = list(executor.map(validate_entry, entries)) + + validated = [result["entry"] for result in results if result.get("validated")] + failed = [result for result in results if not result.get("validated")] + document = { + "schema_version": 1, + "validated_at": __import__("datetime").datetime.now(__import__("datetime").timezone.utc).isoformat(), + "entries": validated, + "failed_count": len(failed), + } + (OUTPUT / "validated.json").write_text(json.dumps(document, indent=2) + "\n") + (OUTPUT / "validation-results.json").write_text(json.dumps(results, indent=2) + "\n") + + print( + "PORTFOLIO_VALIDATION " + + json.dumps( + { + "input": len(entries), + "validated": len(validated), + "failed": len(failed), + } + ) + ) + for result in failed: + print(f"VALIDATION_FAILURE {result['key']} {';'.join(result['reasons'])}") + for check in result.get("checks", []): + if check["returncode"] != 0: + print(f"CHECK_OUTPUT {result['key']} {' '.join(check['command'])}\n{check['output']}") + + summary = os.getenv("GITHUB_STEP_SUMMARY") + if summary: + lines = [ + "## Credential-free ready portfolio validation", + "", + f"- Exact archives supplied: {len(entries)}", + f"- Independently validated: {len(validated)}", + f"- Failed validation: {len(failed)}", + "", + ] + lines.extend( + f"- {result['key']}: {', '.join(result['reasons'][:6])}" + for result in failed[:40] + ) + with open(summary, "a", encoding="utf-8") as handle: + handle.write("\n".join(lines) + "\n") + + return 0 if validated else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/validate-resolved-legacy-harnesses.py b/.github/scripts/validate-resolved-legacy-harnesses.py new file mode 100644 index 00000000..186f341f --- /dev/null +++ b/.github/scripts/validate-resolved-legacy-harnesses.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import concurrent.futures +import hashlib +import json +import os +import re +import shutil +import subprocess +import tarfile +import tempfile +from pathlib import Path, PurePosixPath +from typing import Any + +BUNDLE = Path("legacy-harness-bundle") +OUTPUT = Path("legacy-harness-validated") +CREDENTIAL_RE = re.compile(rb"(?:ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,})") +CONFLICT_RE = re.compile(rb"^(?:<<<<<<<|=======|>>>>>>>)(?: |$)", re.MULTILINE) + + +def safe_extract(archive: Path, destination: Path) -> Path: + with tarfile.open(archive, "r:gz") as tf: + members = tf.getmembers() + roots = {PurePosixPath(member.name).parts[0] for member in members if PurePosixPath(member.name).parts} + if len(roots) != 1 or len(members) > 2000: + raise ValueError("unexpected archive shape") + total = 0 + for member in members: + parts = PurePosixPath(member.name).parts + if not parts or PurePosixPath(member.name).is_absolute() or ".." in parts: + raise ValueError(f"unsafe archive path: {member.name}") + if member.issym() or member.islnk() or member.isdev() or member.isfifo(): + raise ValueError(f"unsupported archive entry: {member.name}") + if member.isfile(): + total += member.size + if total > 60 * 1024 * 1024: + raise ValueError("archive too large") + for member in members: + target = destination.joinpath(*PurePosixPath(member.name).parts) + target.parent.mkdir(parents=True, exist_ok=True) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + source = tf.extractfile(member) + if source is None: + raise ValueError(f"unreadable archive member: {member.name}") + with source, target.open("wb") as output: + shutil.copyfileobj(source, output) + target.chmod(member.mode & 0o777) + return destination / next(iter(roots)) + + +def run(command: list[str], cwd: Path, timeout: int = 180) -> dict[str, Any]: + env = os.environ.copy() + for name in ("GH_TOKEN", "GITHUB_TOKEN", "FLEET_GH_TOKEN"): + env.pop(name, None) + env.update({"CI": "true", "NODE_OPTIONS": "--disable-proto=throw", "GIT_TERMINAL_PROMPT": "0"}) + completed = subprocess.run(command, cwd=cwd, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=timeout, check=False) + return {"command": command, "returncode": completed.returncode, "output": completed.stdout[-12000:]} + + +def scan(root: Path) -> list[str]: + reasons: list[str] = [] + for relative in ("test-plan.json", "scripts/validate-plan.mjs"): + target = root / relative + if not target.is_file() or target.stat().st_size == 0: + reasons.append(f"missing-required:{relative}") + for file in root.rglob("*"): + if not file.is_file() or file.stat().st_size > 2 * 1024 * 1024: + continue + data = file.read_bytes() + relative = file.relative_to(root).as_posix() + if CONFLICT_RE.search(data): + reasons.append(f"conflict-marker:{relative}") + if CREDENTIAL_RE.search(data): + reasons.append(f"credential-pattern:{relative}") + if relative.startswith(".github/workflows/") and relative.endswith((".yml", ".yaml")): + text = data.decode("utf-8", errors="replace") + if re.search(r"^\s*pull_request_target\s*:", text, re.MULTILINE): + reasons.append(f"pull-request-target:{relative}") + if re.search(r"^\s*permissions\s*:\s*write-all\s*$", text, re.MULTILINE): + reasons.append(f"write-all:{relative}") + if re.search(r"(?:curl|wget)[^\n|]*\|\s*(?:ba|z|fi)?sh\b", text, re.IGNORECASE): + reasons.append(f"remote-shell-pipe:{relative}") + if re.search(r"\beval\s+[\"']?\$", text): + reasons.append(f"eval-shell:{relative}") + for use in re.findall(r"^\s*(?:-\s*)?uses:\s*([^\s#]+)\s*$", text, re.MULTILINE): + if use.startswith("./"): + continue + action, separator, ref = use.rpartition("@") + immutable = bool(re.fullmatch(r"[0-9a-fA-F]{40}", ref)) + trusted_major = bool(re.match(r"^(actions|github)/", action) and re.fullmatch(r"v\d+", ref)) + if not separator or not (immutable or trusted_major): + reasons.append(f"untrusted-unpinned-action:{relative}:{use}") + return sorted(set(reasons)) + + +def validate(entry: dict[str, Any]) -> dict[str, Any]: + key = f"{entry['repo']}#{entry['number']}" + result: dict[str, Any] = {"key": key, "entry": entry, "checks": [], "reasons": []} + try: + archive = BUNDLE / entry["archive"] + data = archive.read_bytes() + if hashlib.sha256(data).hexdigest() != entry["archive_sha256"]: + result["reasons"].append("archive-digest-mismatch") + return result + with tempfile.TemporaryDirectory(prefix="legacy-harness-") as temp: + root = safe_extract(archive, Path(temp)) + result["reasons"].extend(scan(root)) + if result["reasons"]: + return result + plan = json.loads((root / "test-plan.json").read_text()) + serialized = json.dumps(plan).lower() + if "planned_dependency" in serialized or "unresolved" in serialized: + result["reasons"].append("plan-not-fully-resolved") + check = run(["node", "scripts/validate-plan.mjs"], root, 180) + result["checks"].append(check) + if check["returncode"] != 0: + result["reasons"].append(f"validate-plan-failed:{check['returncode']}") + result["validated"] = not result["reasons"] + return result + except Exception as error: # noqa: BLE001 + result["reasons"].append(f"exception:{type(error).__name__}:{error}") + result["validated"] = False + return result + + +def main() -> int: + manifest = json.loads((BUNDLE / "manifest.json").read_text()) + entries = manifest.get("entries", []) + with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, max(1, len(entries)))) as executor: + results = list(executor.map(validate, entries)) + passed = [result["entry"] for result in results if result.get("validated")] + failed = [result for result in results if not result.get("validated")] + OUTPUT.mkdir(parents=True, exist_ok=True) + (OUTPUT / "validated.json").write_text(json.dumps({"schema_version": 1, "entries": passed}, indent=2) + "\n") + (OUTPUT / "results.json").write_text(json.dumps(results, indent=2) + "\n") + print("LEGACY_HARNESS_VALIDATION " + json.dumps({"input": len(entries), "validated": len(passed), "failed": len(failed)})) + for result in failed: + print(f"VALIDATION_FAILURE {result['key']} {';'.join(result['reasons'])}") + for check in result.get("checks", []): + if check["returncode"] != 0: + print(f"CHECK_OUTPUT {result['key']}\n{check['output']}") + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/README-zed-cli-graph-github-production.md b/.github/workflows/README-zed-cli-graph-github-production.md new file mode 100644 index 00000000..7f724a8f --- /dev/null +++ b/.github/workflows/README-zed-cli-graph-github-production.md @@ -0,0 +1,3 @@ +The canonical workflow for DEN-2957 is `zed-cli-graph-github-production-pinned.yml`. + +The earlier draft `zed-cli-graph-github-production.yml` is superseded and must not be dispatched or merged. It remains on this integration branch only until the review commit removes it; no promotion evidence may reference it. diff --git a/.github/workflows/apply-test-org-fleet.yml b/.github/workflows/apply-test-org-fleet.yml new file mode 100644 index 00000000..87b22058 --- /dev/null +++ b/.github/workflows/apply-test-org-fleet.yml @@ -0,0 +1,239 @@ +# generated by zed-pkg-test/test-org-fleet +name: apply test organization fleet + +on: + workflow_dispatch: + inputs: + organization: + description: Test organization to provision + required: true + default: all + type: choice + options: + - all + - zed-pkg-test + - 3fa-app-test + - declarative-migrations-test + - cliptown-test + - claritas-viz-test + - embedded-alerts-test + - evento-globolo-test + - fiducia-cloud-test + - memebank-test + - opto-sync-test + - quaestor-ledger-test + - sonus-auris-test + - messaging-intel-test + - scintilla-run-test + - file-tunnel-test + - shared-auth-test + - hypesiege-test + - hacker-house-medellin-test + - streempilot-test + repository: + description: Optional repository-name glob, for example clients-*-consumer + required: false + default: "" + type: string + authentication: + description: Secret-backed credential source; auto prefers FLEET_GH_TOKEN + required: true + default: auto + type: choice + options: + - auto + - pat + - github-app + confirm: + description: Type APPLY_TEST_FLEET to authorize repository creation and PR writes + required: true + type: string + +permissions: + contents: read + +concurrency: + group: apply-test-org-fleet-${{ inputs.organization }} + cancel-in-progress: false + +jobs: + select: + name: Validate authorization and select organizations + runs-on: ubuntu-24.04 + timeout-minutes: 2 + outputs: + organizations: ${{ steps.matrix.outputs.organizations }} + steps: + - id: matrix + name: Require explicit confirmation + shell: bash + env: + CONFIRMATION: ${{ inputs.confirm }} + SELECTED_ORGANIZATION: ${{ inputs.organization }} + run: | + set -euo pipefail + if [[ "$CONFIRMATION" != "APPLY_TEST_FLEET" ]]; then + echo "::error::Confirmation must be exactly APPLY_TEST_FLEET" + exit 1 + fi + + all='["zed-pkg-test","3fa-app-test","declarative-migrations-test","cliptown-test","claritas-viz-test","embedded-alerts-test","evento-globolo-test","fiducia-cloud-test","memebank-test","opto-sync-test","quaestor-ledger-test","sonus-auris-test","messaging-intel-test","scintilla-run-test","file-tunnel-test","shared-auth-test","hypesiege-test","hacker-house-medellin-test","streempilot-test"]' + case "$SELECTED_ORGANIZATION" in + all) + selected="$all" + ;; + zed-pkg-test|3fa-app-test|declarative-migrations-test|cliptown-test|claritas-viz-test|embedded-alerts-test|evento-globolo-test|fiducia-cloud-test|memebank-test|opto-sync-test|quaestor-ledger-test|sonus-auris-test|messaging-intel-test|scintilla-run-test|file-tunnel-test|shared-auth-test|hypesiege-test|hacker-house-medellin-test|streempilot-test) + selected="[\"$SELECTED_ORGANIZATION\"]" + ;; + *) + echo "::error::Refusing unknown organization: $SELECTED_ORGANIZATION" + exit 1 + ;; + esac + printf 'organizations=%s\n' "$selected" >> "$GITHUB_OUTPUT" + + apply: + name: Provision ${{ matrix.organization }} + needs: select + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + organization: ${{ fromJSON(needs.select.outputs.organizations) }} + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate the immutable fleet contract + run: | + node --check scripts/bootstrap-test-org-fleet.mjs + node --check scripts/validate-test-org-fleet.mjs + node scripts/validate-test-org-fleet.mjs + node --test tests/test-org-fleet.test.mjs tests/git-transport-worktree.test.mjs tests/fleet-recovery-control.test.mjs + + - name: Select a secret-backed credential + id: auth + shell: bash + env: + REQUESTED_AUTHENTICATION: ${{ inputs.authentication }} + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + FLEET_APP_CLIENT_ID: ${{ vars.FLEET_APP_CLIENT_ID }} + FLEET_APP_PRIVATE_KEY: ${{ secrets.FLEET_APP_PRIVATE_KEY }} + run: | + set -euo pipefail + + has_pat=false + has_app=false + [[ -n "$STORED_FLEET_PAT" ]] && has_pat=true + [[ -n "$FLEET_APP_CLIENT_ID" && -n "$FLEET_APP_PRIVATE_KEY" ]] && has_app=true + + case "$REQUESTED_AUTHENTICATION" in + auto) + if [[ "$has_pat" == true ]]; then + selected=pat + elif [[ "$has_app" == true ]]; then + selected=github-app + else + echo "::error::Configure FLEET_GH_TOKEN or both FLEET_APP_CLIENT_ID and FLEET_APP_PRIVATE_KEY" + exit 1 + fi + ;; + pat) + if [[ "$has_pat" != true ]]; then + echo "::error::authentication=pat requires the FLEET_GH_TOKEN Actions secret" + exit 1 + fi + selected=pat + ;; + github-app) + if [[ "$has_app" != true ]]; then + echo "::error::authentication=github-app requires FLEET_APP_CLIENT_ID and FLEET_APP_PRIVATE_KEY" + exit 1 + fi + selected=github-app + ;; + *) + echo "::error::Unsupported authentication mode: $REQUESTED_AUTHENTICATION" + exit 1 + ;; + esac + + printf 'mode=%s\n' "$selected" >> "$GITHUB_OUTPUT" + echo "Using $selected authentication for ${{ matrix.organization }}" + + - name: Mint a short-lived installation token + if: steps.auth.outputs.mode == 'github-app' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.FLEET_APP_CLIENT_ID }} + private-key: ${{ secrets.FLEET_APP_PRIVATE_KEY }} + owner: ${{ matrix.organization }} + permission-administration: write + permission-contents: write + permission-metadata: read + permission-pull-requests: write + permission-workflows: write + + - name: Export the stored PAT without logging it + if: steps.auth.outputs.mode == 'pat' + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + test -n "$STORED_FLEET_PAT" + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Export the short-lived App token without logging it + if: steps.auth.outputs.mode == 'github-app' + shell: bash + env: + FLEET_APP_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + test -n "$FLEET_APP_TOKEN" + printf 'GH_TOKEN=%s\n' "$FLEET_APP_TOKEN" >> "$GITHUB_ENV" + + - name: Verify organization visibility + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + run: | + set -euo pipefail + gh api "orgs/$FLEET_ORGANIZATION" --silent + + - name: Create or adopt repositories, push commits, and open draft PRs + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + FLEET_REPOSITORY_FILTER: ${{ inputs.repository }} + TEST_ORG_FLEET_SKIP_TOPICS: "true" + run: | + set -euo pipefail + args=(--apply --org "$FLEET_ORGANIZATION" --concurrency 1) + if [[ -n "$FLEET_REPOSITORY_FILTER" ]]; then + args+=(--repo "$FLEET_REPOSITORY_FILTER") + fi + node scripts/bootstrap-test-org-fleet.mjs "${args[@]}" + + - name: Record completion + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + FLEET_AUTHENTICATION: ${{ steps.auth.outputs.mode }} + run: | + { + echo "## $FLEET_ORGANIZATION" + echo + echo "Authentication: $FLEET_AUTHENTICATION" + echo + echo "Repository creation/adoption, generated branch commits, and draft pull requests completed." + echo + echo "Repository topic writes were deferred to minimize secondary content-creation limits." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/audit-graphql-ready-green-v2.yml b/.github/workflows/audit-graphql-ready-green-v2.yml new file mode 100644 index 00000000..23c3a058 --- /dev/null +++ b/.github/workflows/audit-graphql-ready-green-v2.yml @@ -0,0 +1,37 @@ +name: audit comment-aware overlap-aware green ready PRs + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/AUDIT-GRAPHQL-READY-GREEN-V2" + +permissions: + contents: read + +concurrency: + group: audit-graphql-ready-green-v2 + cancel-in-progress: false + +jobs: + audit: + if: "${{ github.event.head_commit.message == 'chore: audit comment-aware green ready PRs' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted GraphQL auditor + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Audit exact heads and open-PR path intersections without writes + shell: bash + run: node .github/scripts/audit-graphql-ready-green-v2.mjs + - name: Upload machine-readable audit + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: graphql-ready-green-audit-v2 + path: ready-green-audit-v2 + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/audit-graphql-ready-green.yml b/.github/workflows/audit-graphql-ready-green.yml new file mode 100644 index 00000000..a6cc8b04 --- /dev/null +++ b/.github/workflows/audit-graphql-ready-green.yml @@ -0,0 +1,37 @@ +name: audit isolated green ready PRs through GraphQL + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/AUDIT-GRAPHQL-READY-GREEN" + +permissions: + contents: read + +concurrency: + group: audit-graphql-ready-green + cancel-in-progress: false + +jobs: + audit: + if: "${{ github.event.head_commit.message == 'chore: audit isolated green ready PRs' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted GraphQL auditor + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Audit exact heads without writes + shell: bash + run: node .github/scripts/audit-graphql-ready-green.mjs + - name: Upload machine-readable audit + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: graphql-ready-green-audit + path: ready-green-audit + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/audit-open-pr-fleet.yml b/.github/workflows/audit-open-pr-fleet.yml new file mode 100644 index 00000000..e085ef7a --- /dev/null +++ b/.github/workflows/audit-open-pr-fleet.yml @@ -0,0 +1,139 @@ +name: audit open PR fleet + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/AUDIT" + +permissions: + contents: read + +concurrency: + group: audit-open-pr-fleet + cancel-in-progress: false + +jobs: + audit: + if: "${{ github.event.head_commit.message == 'chore: run open PR fleet audit' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 20 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Snapshot and classify open pull requests + shell: bash + run: | + set -Eeuo pipefail + root="$(mktemp -d)" + export ROOT="$root" + + for page in $(seq 1 10); do + gh api \ + -H 'Accept: application/vnd.github+json' \ + "search/issues?q=is%3Apr%20is%3Aopen%20author%3AORESoftware&sort=created&order=desc&per_page=100&page=$page" \ + > "$root/page-$page.json" + done + + node <<'NODE' + const fs = require('node:fs'); + const path = require('node:path'); + const root = process.env.ROOT; + const pages = fs.readdirSync(root) + .filter((name) => /^page-\d+\.json$/.test(name)) + .sort((a, b) => Number(a.match(/\d+/)[0]) - Number(b.match(/\d+/)[0])) + .map((name) => JSON.parse(fs.readFileSync(path.join(root, name), 'utf8'))); + + const byUrl = new Map(); + for (const page of pages) { + for (const item of page.items || []) byUrl.set(item.html_url, item); + } + const items = [...byUrl.values()]; + const total = pages[0]?.total_count ?? items.length; + const repoOf = (item) => item.repository_url.replace('https://api.github.com/repos/', ''); + const ownerOf = (item) => repoOf(item).split('/')[0]; + const textOf = (item) => `${item.title || ''}\n${item.body || ''}`.toLowerCase(); + const generated = (item) => { + const text = textOf(item); + return ownerOf(item).toLowerCase().endsWith('-test') || + text.includes('github-test-org-factory') || + text.includes('generated independent acceptance repository') || + text.includes('generated repository harness') || + text.includes('generated test harness'); + }; + const veto = (item) => { + const repo = repoOf(item).toLowerCase(); + const text = textOf(item); + return (repo === 'oresoftware/k8s-cluster' && item.number === 231) || + (repo === 'oresoftware/k8s-cluster' && item.number === 792) || + /do not merge|don['’]?t merge|never merge|temporary synchronization|\bwip\b/.test(text); + }; + + const titleCounts = new Map(); + const ownerCounts = new Map(); + const repoCounts = new Map(); + for (const item of items) { + titleCounts.set(item.title, (titleCounts.get(item.title) || 0) + 1); + ownerCounts.set(ownerOf(item), (ownerCounts.get(ownerOf(item)) || 0) + 1); + repoCounts.set(repoOf(item), (repoCounts.get(repoOf(item)) || 0) + 1); + } + const sortCounts = (map) => [...map.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); + const duplicatedRepos = [...repoCounts.values()].filter((n) => n > 1).length; + const generatedItems = items.filter(generated); + const nonVetoGenerated = generatedItems.filter((item) => !veto(item)); + const summary = { + api_total_count: total, + unique_items_seen: items.length, + drafts: items.filter((item) => item.draft).length, + generated_or_test_org: generatedItems.length, + generated_non_veto: nonVetoGenerated.length, + generated_non_veto_drafts: nonVetoGenerated.filter((item) => item.draft).length, + generated_non_veto_ready: nonVetoGenerated.filter((item) => !item.draft).length, + explicit_veto_or_temporary: items.filter(veto).length, + repositories_with_multiple_open_authored_prs: duplicatedRepos, + }; + + console.log('AUDIT_SUMMARY ' + JSON.stringify(summary)); + console.log('TOP_TITLES'); + for (const [title, count] of sortCounts(titleCounts).slice(0, 50)) { + const drafts = items.filter((item) => item.title === title && item.draft).length; + console.log(`${count}\tdrafts=${drafts}\t${title}`); + } + console.log('TOP_OWNERS'); + for (const [owner, count] of sortCounts(ownerCounts).slice(0, 50)) { + console.log(`${count}\t${owner}`); + } + console.log('DUPLICATED_REPOSITORIES'); + for (const [repo, count] of sortCounts(repoCounts).filter(([, count]) => count > 1).slice(0, 100)) { + console.log(`${count}\t${repo}`); + } + + const lines = [ + '## Open PR fleet audit', + '', + `- GitHub search total: ${total}`, + `- Unique PRs observed across the first 1,000 results: ${items.length}`, + `- Drafts: ${summary.drafts}`, + `- Generated or test-organization PRs: ${summary.generated_or_test_org}`, + `- Generated, non-veto PRs: ${summary.generated_non_veto}`, + `- Explicit veto/temporary PRs: ${summary.explicit_veto_or_temporary}`, + `- Repositories with multiple open authored PRs: ${summary.repositories_with_multiple_open_authored_prs}`, + '', + '### Most common titles', + '', + '| Count | Drafts | Title |', + '|---:|---:|---|', + ...sortCounts(titleCounts).slice(0, 25).map(([title, count]) => { + const drafts = items.filter((item) => item.title === title && item.draft).length; + return `| ${count} | ${drafts} | ${String(title).replaceAll('|', '\\|')} |`; + }), + ]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n'); + NODE diff --git a/.github/workflows/audit-ready-backlog-categories.yml b/.github/workflows/audit-ready-backlog-categories.yml new file mode 100644 index 00000000..fa22d6f2 --- /dev/null +++ b/.github/workflows/audit-ready-backlog-categories.yml @@ -0,0 +1,46 @@ +name: audit remaining ready backlog categories + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/AUDIT-READY-CATEGORIES" + +permissions: + contents: read + +concurrency: + group: audit-ready-backlog-categories + cancel-in-progress: false + +jobs: + audit: + if: "${{ github.event.head_commit.message == 'chore: audit remaining ready backlog categories' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored fleet credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$FLEET_GH_TOKEN" + GH_TOKEN="$FLEET_GH_TOKEN" gh api user --jq '.login' | grep -Fx ORESoftware + + - name: Check out trusted category auditor + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Audit live exact heads without writes + shell: bash + run: node .github/scripts/audit-ready-backlog-categories.mjs + + - name: Upload machine-readable audit + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ready-backlog-categories + path: ready-backlog-audit + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/audit-ready-merge-candidates-v2.yml b/.github/workflows/audit-ready-merge-candidates-v2.yml new file mode 100644 index 00000000..42f41292 --- /dev/null +++ b/.github/workflows/audit-ready-merge-candidates-v2.yml @@ -0,0 +1,37 @@ +name: audit ready merge candidates v2 + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/AUDIT-READY-V2" + +permissions: + contents: read + +concurrency: + group: audit-ready-merge-candidates-v2 + cancel-in-progress: false + +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Check out exact auditor + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Audit comment-aware exact heads without writes + shell: bash + run: node .github/scripts/audit-ready-merge-candidates-v2.mjs diff --git a/.github/workflows/audit-ready-original-prs.yml b/.github/workflows/audit-ready-original-prs.yml new file mode 100644 index 00000000..1e2b896d --- /dev/null +++ b/.github/workflows/audit-ready-original-prs.yml @@ -0,0 +1,37 @@ +name: audit ready original PRs + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/AUDIT-READY-ORIGINAL" + +permissions: + contents: read + +concurrency: + group: audit-ready-original-prs + cancel-in-progress: false + +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Check out exact auditor + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Audit exact heads without writes + shell: bash + run: node .github/scripts/audit-ready-original-prs.mjs diff --git a/.github/workflows/audit-resolved-generated-drafts.yml b/.github/workflows/audit-resolved-generated-drafts.yml new file mode 100644 index 00000000..1ff14a40 --- /dev/null +++ b/.github/workflows/audit-resolved-generated-drafts.yml @@ -0,0 +1,37 @@ +name: audit resolved generated drafts + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/AUDIT-RESOLVED-DRAFTS" + +permissions: + contents: read + +concurrency: + group: audit-resolved-generated-drafts + cancel-in-progress: false + +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Check out exact auditor + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Audit resolved generated draft portfolios without writes + shell: bash + run: node .github/scripts/audit-resolved-generated-drafts.mjs diff --git a/.github/workflows/audit-semantic-conflicts.yml b/.github/workflows/audit-semantic-conflicts.yml new file mode 100644 index 00000000..a553e1ed --- /dev/null +++ b/.github/workflows/audit-semantic-conflicts.yml @@ -0,0 +1,152 @@ +name: audit semantic merge conflicts + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/CONFLICT-AUDIT" + +permissions: + contents: read + +concurrency: + group: audit-semantic-merge-conflicts + cancel-in-progress: false + +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Inventory ready pull requests with real merge conflicts + shell: bash + run: | + set -Eeuo pipefail + node <<'NODE' + const token = process.env.GH_TOKEN; + const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'semantic-conflict-inventory', + }; + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + async function request(url, attempts = 5) { + let last; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { headers }); + const text = await response.text(); + const value = text ? JSON.parse(text) : null; + if (response.ok) return value; + last = new Error(`${response.status} ${url}: ${text.slice(0, 500)}`); + if (![429, 500, 502, 503, 504].includes(response.status) || attempt === attempts) throw last; + await sleep(attempt * attempt * 1000); + } + throw last; + } + + async function mapLimit(values, limit, fn) { + const results = new Array(values.length); + let next = 0; + async function worker() { + while (true) { + const index = next++; + if (index >= values.length) return; + try { results[index] = await fn(values[index], index); } + catch (error) { results[index] = { error: error.message, item: values[index] }; } + } + } + await Promise.all(Array.from({ length: Math.min(limit, values.length) }, worker)); + return results; + } + + const byUrl = new Map(); + const query = encodeURIComponent('is:pr is:open author:ORESoftware draft:false created:<2026-08-05T12:45:00Z sort:updated-desc'); + for (let page = 1; page <= 10; page += 1) { + const result = await request(`https://api.github.com/search/issues?q=${query}&per_page=100&page=${page}`); + for (const item of result.items || []) byUrl.set(item.html_url, item); + if ((result.items || []).length < 100) break; + } + console.log(`READY_SNAPSHOT count=${byUrl.size}`); + + const inspected = await mapLimit([...byUrl.values()], 12, async (item) => { + const repo = item.repository_url.replace('https://api.github.com/repos/', ''); + let pr = await request(`https://api.github.com/repos/${repo}/pulls/${item.number}`); + for (let attempt = 0; pr.mergeable == null && attempt < 3; attempt += 1) { + await sleep(1200 + attempt * 600); + pr = await request(`https://api.github.com/repos/${repo}/pulls/${item.number}`); + } + if (pr.mergeable !== false && pr.mergeable_state !== 'dirty') return null; + const files = await request(`https://api.github.com/repos/${repo}/pulls/${item.number}/files?per_page=100`); + return { + repo, + number: item.number, + title: item.title, + url: item.html_url, + base: pr.base?.ref, + base_sha: pr.base?.sha, + head: pr.head?.ref, + head_sha: pr.head?.sha, + mergeable: pr.mergeable, + mergeable_state: pr.mergeable_state, + additions: pr.additions, + deletions: pr.deletions, + changed_files: pr.changed_files, + comments: pr.comments, + review_comments: pr.review_comments, + created_at: pr.created_at, + updated_at: pr.updated_at, + test_org: repo.split('/')[0].toLowerCase().endsWith('-test'), + files: files.map((file) => ({ + path: file.filename, + status: file.status, + additions: file.additions, + deletions: file.deletions, + })), + }; + }); + + const conflicts = inspected + .filter((value) => value && !value.error) + .sort((a, b) => + Number(a.test_org) - Number(b.test_org) || + a.changed_files - b.changed_files || + (a.additions + a.deletions) - (b.additions + b.deletions) || + String(b.updated_at).localeCompare(String(a.updated_at)) + ); + const errors = inspected.filter((value) => value?.error); + + console.log(`CONFLICT_SUMMARY ${JSON.stringify({ ready_observed: byUrl.size, conflicts: conflicts.length, inspection_errors: errors.length })}`); + for (const conflict of conflicts.slice(0, 100)) { + console.log(`CONFLICT ${JSON.stringify(conflict)}`); + } + for (const error of errors.slice(0, 20)) { + console.log(`INSPECTION_ERROR ${JSON.stringify(error)}`); + } + + const rows = conflicts.slice(0, 50).map((conflict) => + `| ${conflict.repo}#${conflict.number} | ${conflict.changed_files} | ${conflict.additions} | ${conflict.deletions} | ${conflict.files.slice(0, 6).map((file) => `\`${file.path}\``).join('
')} | ${conflict.title.replaceAll('|', '\\|')} |` + ); + const summary = [ + '## Semantic merge-conflict inventory', + '', + `- Ready original-backlog PRs observed: ${byUrl.size}`, + `- Real conflicts: ${conflicts.length}`, + `- Inspection errors: ${errors.length}`, + '', + '| PR | Files | + | - | First paths | Title |', + '|---|---:|---:|---:|---|---|', + ...rows, + ]; + require('node:fs').appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary.join('\n') + '\n'); + NODE diff --git a/.github/workflows/audit-test-org-fleet-once.yml b/.github/workflows/audit-test-org-fleet-once.yml new file mode 100644 index 00000000..38fb1fa3 --- /dev/null +++ b/.github/workflows/audit-test-org-fleet-once.yml @@ -0,0 +1,71 @@ +# One-time exact audit of repositories, generated branches, and draft pull requests. +name: audit generated test organization fleet + +on: + push: + branches: [main] + paths: + - ".github/workflows/audit-test-org-fleet-once.yml" + - "bootstrap/audit-test-org-fleet.once" + - "scripts/audit-test-org-fleet.mjs" + - "scripts/audit-test-org-fleet-live.mjs" + - "tests/audit-test-org-fleet.test.mjs" + +permissions: + contents: read + +concurrency: + group: audit-generated-test-org-fleet + cancel-in-progress: true + +jobs: + audit: + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate authorization and audit contract + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/audit-test-org-fleet.once)" = "AUDIT_TEST_FLEET_V1" + node --check scripts/audit-test-org-fleet.mjs + node --check scripts/audit-test-org-fleet-live.mjs + node scripts/audit-test-org-fleet-live.mjs --plan-json > /tmp/test-org-fleet-audit-plan.json + node --test tests/audit-test-org-fleet.test.mjs + jq -e ' + .organizations == 18 + and .repositories == 341 + and .generatedRepositories == 319 + and .retainedRepositories == 22 + ' /tmp/test-org-fleet-audit-plan.json > /dev/null + + - name: Export the persisted fleet PAT without logging it + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + test -n "$STORED_FLEET_PAT" + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Audit all expected repositories, branches, and pull requests + shell: bash + run: | + set +e + node scripts/audit-test-org-fleet-live.mjs \ + --output /tmp/test-org-fleet-live-audit.json \ + --markdown /tmp/test-org-fleet-live-audit.md + status=$? + set -e + + cat /tmp/test-org-fleet-live-audit.md >> "$GITHUB_STEP_SUMMARY" + echo + jq -c '.summary' /tmp/test-org-fleet-live-audit.json + exit "$status" diff --git a/.github/workflows/bootstrap-led-dynamo.yml b/.github/workflows/bootstrap-led-dynamo.yml new file mode 100644 index 00000000..95f5dc5b --- /dev/null +++ b/.github/workflows/bootstrap-led-dynamo.yml @@ -0,0 +1,43 @@ +name: bootstrap led-dynamo repository family + +on: + push: + branches: [main] + paths: + - ".github/workflows/bootstrap-led-dynamo.yml" + - "scripts/bootstrap-led-dynamo.mjs" + - "bootstrap/led-dynamo.parts/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: bootstrap-led-dynamo-repository-family + cancel-in-progress: false + +jobs: + provision: + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + LEDDY_BOOTSTRAP_CONFIRM: CREATE_LEDDY_REPOSITORIES + steps: + - name: Check out control repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + - name: Validate secret-backed authorization + shell: bash + run: | + set -euo pipefail + test -n "$GH_TOKEN" + gh api user --silent + gh api orgs/led-dynamo --silent + - name: Validate bootstrap program + run: node --check scripts/bootstrap-led-dynamo.mjs + - name: Create, configure, and seed repositories + run: node scripts/bootstrap-led-dynamo.mjs diff --git a/.github/workflows/browser-harness-contract.yml b/.github/workflows/browser-harness-contract.yml new file mode 100644 index 00000000..2d584e50 --- /dev/null +++ b/.github/workflows/browser-harness-contract.yml @@ -0,0 +1,130 @@ +name: Browser harness contract + +'on': + pull_request: + paths: + - 'tests/browser/**' + - '.github/workflows/browser-harness-contract.yml' + push: + branches: [main, master, dev] + paths: + - 'tests/browser/**' + - '.github/workflows/browser-harness-contract.yml' + schedule: + - cron: '17 9 1 * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: browser-harness-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + contract: + name: ${{ matrix.framework }} contract + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + framework: [playwright, puppeteer, selenium] + env: + CI: 'true' + E2E_BROWSER_CONTRACT: '1' + E2E_FRAMEWORK: ${{ matrix.framework }} + NO_COLOR: '1' + PUPPETEER_SKIP_DOWNLOAD: 'true' + npm_config_audit: 'false' + npm_config_fund: 'false' + npm_config_ignore_scripts: 'true' + npm_config_package_lock: 'false' + npm_config_update_notifier: 'false' + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Use Node.js 22.23.1 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.23.1' + + - name: Prepare isolated contract workspace + shell: bash + working-directory: tests/browser + run: | + set -euo pipefail + rm -rf node_modules package-lock.json + printf '%s\n' '{"name":"browser-harness-contract-ci","private":true,"type":"module"}' > package.json + artifact_dir="artifacts/${{ matrix.framework }}" + mkdir -p "$artifact_dir" + { + printf 'framework=%s\n' '${{ matrix.framework }}' + printf 'node=%s\n' "$(node --version)" + printf 'npm=%s\n' "$(npm --version)" + printf 'runner_os=%s\n' '${{ runner.os }}' + printf 'runner_arch=%s\n' '${{ runner.arch }}' + printf 'image_os=%s\n' "${ImageOS:-unknown}" + printf 'image_version=%s\n' "${ImageVersion:-unknown}" + } > "$artifact_dir/environment.txt" + + - name: Install isolated browser driver + shell: bash + working-directory: tests/browser + run: | + set -euo pipefail + artifact="artifacts/${{ matrix.framework }}/environment.txt" + case '${{ matrix.framework }}' in + playwright) + npm install --no-save --no-package-lock --ignore-scripts playwright@1.61.1 + ./node_modules/.bin/playwright install --with-deps chromium + printf 'driver=%s\n' "$(node -p "require('./node_modules/playwright/package.json').version")" >> "$artifact" + ;; + puppeteer) + npm install --no-save --no-package-lock --ignore-scripts puppeteer@25.3.0 + chrome_bin="$(command -v google-chrome || command -v chromium || true)" + test -n "$chrome_bin" && test -x "$chrome_bin" + printf 'PUPPETEER_EXECUTABLE_PATH=%s\n' "$chrome_bin" >> "$GITHUB_ENV" + printf 'driver=%s\n' "$(node -p "require('./node_modules/puppeteer/package.json').version")" >> "$artifact" + printf 'browser=%s\n' "$($chrome_bin --version)" >> "$artifact" + ;; + selenium) + npm install --no-save --no-package-lock --ignore-scripts selenium-webdriver@4.46.0 + chrome_bin="$(command -v google-chrome || command -v chromium || true)" + test -n "$chrome_bin" && test -x "$chrome_bin" + printf 'driver=%s\n' "$(node -p "require('./node_modules/selenium-webdriver/package.json').version")" >> "$artifact" + printf 'browser=%s\n' "$($chrome_bin --version)" >> "$artifact" + ;; + *) + echo 'unsupported framework: ${{ matrix.framework }}' >&2 + exit 64 + ;; + esac + npm ls --depth=0 --json > "artifacts/${{ matrix.framework }}/dependencies.json" + + - name: Validate contract syntax + shell: bash + working-directory: tests/browser + run: | + set -euo pipefail + node --check playwright/harness-contract.test.mjs + node --check puppeteer/harness-contract.test.mjs + node --check selenium/harness-contract.test.mjs + + - name: Run browser harness contract + shell: bash + working-directory: tests/browser + run: node --test --test-reporter=spec '${{ matrix.framework }}/harness-contract.test.mjs' + + - name: Upload browser evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: browser-harness-${{ matrix.framework }}-${{ github.run_id }}-${{ github.run_attempt }} + path: tests/browser/artifacts/${{ matrix.framework }}/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/cancel-stale-fleet-recovery.yml b/.github/workflows/cancel-stale-fleet-recovery.yml new file mode 100644 index 00000000..37e305da --- /dev/null +++ b/.github/workflows/cancel-stale-fleet-recovery.yml @@ -0,0 +1,70 @@ +# One-time cancellation of superseded fleet runs. +name: cancel stale fleet recovery + +on: + push: + branches: [main] + paths: + - ".github/workflows/cancel-stale-fleet-recovery.yml" + - ".github/fleet-cancel/START" + +permissions: + actions: write + contents: read + +jobs: + cancel: + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 8 + env: + GH_TOKEN: ${{ github.token }} + STALE_RUN_IDS: "30972717804 30973196337" + steps: + - name: Cancel superseded recovery and full-apply runs + shell: bash + run: | + set -euo pipefail + read -r -a run_ids <<< "$STALE_RUN_IDS" + + for run_id in "${run_ids[@]}"; do + endpoint="repos/$GITHUB_REPOSITORY/actions/runs/$run_id" + status="$(gh api "$endpoint" --jq '.status')" + conclusion="$(gh api "$endpoint" --jq '.conclusion // ""')" + + if [[ "$status" == "completed" ]]; then + echo "Run $run_id already completed with conclusion: ${conclusion:-unknown}" + continue + fi + + if gh api --method POST "$endpoint/cancel" --silent; then + echo "Cancellation requested for run $run_id" + continue + fi + + status="$(gh api "$endpoint" --jq '.status')" + if [[ "$status" != "completed" ]]; then + echo "::error::Unable to cancel stale run $run_id" + exit 1 + fi + done + + for run_id in "${run_ids[@]}"; do + endpoint="repos/$GITHUB_REPOSITORY/actions/runs/$run_id" + stopped=false + for _ in $(seq 1 120); do + status="$(gh api "$endpoint" --jq '.status')" + conclusion="$(gh api "$endpoint" --jq '.conclusion // ""')" + if [[ "$status" == "completed" ]]; then + echo "Run $run_id stopped with conclusion: ${conclusion:-unknown}" + stopped=true + break + fi + sleep 2 + done + + if [[ "$stopped" != true ]]; then + echo "::error::Run $run_id did not stop after cancellation" + exit 1 + fi + done diff --git a/.github/workflows/candidate-procedure-contract.yml b/.github/workflows/candidate-procedure-contract.yml new file mode 100644 index 00000000..0968d83e --- /dev/null +++ b/.github/workflows/candidate-procedure-contract.yml @@ -0,0 +1,65 @@ +name: zed-pkg-test candidate procedure contract + +on: + pull_request: + paths: + - ".github/workflows/candidate-smoke.yml" + - ".github/workflows/candidate-procedure-contract.yml" + - "scripts/lifecycle.py" + - "scripts/candidate_lifecycle.py" + - "scripts/procedure_contract.py" + - "tests/test_candidate_*.py" + - "docs/candidate-validation.md" + push: + branches: [main] + paths: + - ".github/workflows/candidate-smoke.yml" + - ".github/workflows/candidate-procedure-contract.yml" + - "scripts/lifecycle.py" + - "scripts/candidate_lifecycle.py" + - "scripts/procedure_contract.py" + - "tests/test_candidate_*.py" + - "docs/candidate-validation.md" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: candidate-procedure-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + procedure-contract: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact procedure source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Compile the lifecycle and policy modules + run: | + set -euo pipefail + python3 -m py_compile \ + scripts/lifecycle.py \ + scripts/candidate_lifecycle.py \ + scripts/procedure_contract.py \ + tests/test_candidate_lifecycle.py \ + tests/test_candidate_procedure.py + + - name: Run fail-closed procedure tests + run: | + set -euo pipefail + python3 -m unittest discover \ + --start-directory tests \ + --pattern 'test_candidate_*.py' \ + --verbose + + - name: Audit the checked-out certification procedure + run: | + set -euo pipefail + python3 scripts/procedure_contract.py --root . diff --git a/.github/workflows/candidate-smoke.yml b/.github/workflows/candidate-smoke.yml index 6f8119c4..3b505dd9 100644 --- a/.github/workflows/candidate-smoke.yml +++ b/.github/workflows/candidate-smoke.yml @@ -12,7 +12,7 @@ on: zed_cli_ref: description: Exact 40-character zed-pkg/zed-cli commit to test required: false - default: 3a91d64abf1fc0bdee6eab2ff804113b1429849f + default: d8763b8dcc3bd5c5b6b4b1d13519065a6a2cb99a type: string harness_ref: description: Exact 40-character zed-pkg-test/zed-pkg-e2e commit @@ -22,7 +22,7 @@ on: description: JSON array of pinned fixture repository/ref objects required: false default: >- - [{"repo":"node-lib","ref":"2d41e658a382fdff27ef87a0db24dbaff280a685"},{"repo":"node-app","ref":"ef794674647b3620e711f4f855865f8ad3340498"},{"repo":"rust-lib","ref":"f2534ffafbc4f7e900075dfc8e48b9259aced2fe"},{"repo":"rust-app","ref":"38b0a5425176a53621d1c07be791701dbd30eefe"},{"repo":"go-lib","ref":"e09dec6790a60ba75d5d338eece1068a4b8454b1"},{"repo":"go-app","ref":"8e035feb84cf80a294e631bda4ea7a44b1c4d7cf"},{"repo":"python-lib","ref":"b2db8ce2962fbaae690db7a9dabd14eb1d0362de"},{"repo":"python-app","ref":"a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1"},{"repo":"polyglot-lib","ref":"e62ceb6337324a294d4fc861692a267ff0db97d0"},{"repo":"workspace-monorepo","ref":"72c47a51887733b9035bac1a3efdbf92880ef791"},{"repo":"multiversion-lib","ref":"ba9ee7269559a122f2b58fb89f134aaa4b7d0aee"},{"repo":"awkward-lib","ref":"3e24f62ef4df5d8bfc6e63df07d28eaf1f4dac9e"},{"repo":"submodule-consumer","ref":"bc193f9d805e286c95e9b0c49580c5171224e45c"},{"repo":"subtree-consumer","ref":"3da0c249e481104a8c9058e6107dd47f8a80b10f"},{"repo":"shared-schema","ref":"c3fe49a621337be58b16717fb1acb57139a29b45"}] + [{"repo":"node-lib","ref":"2d41e658a382fdff27ef87a0db24dbaff280a685"},{"repo":"node-app","ref":"ef794674647b3620e711f4f855865f8ad3340498"},{"repo":"rust-lib","ref":"f2534ffafbc4f7e900075dfc8e48b9259aced2fe"},{"repo":"rust-app","ref":"38b0a5425176a53621d1c07be791701dbd30eefe"},{"repo":"go-lib","ref":"e09dec6790a60ba75d5d338eece1068a4b8454b1"},{"repo":"go-app","ref":"8e035feb84cf80a294e631bda4ea7a44b1c4d7cf"},{"repo":"python-lib","ref":"b2db8ce2962fbaae690db7a9dabd14eb1d0362de"},{"repo":"python-app","ref":"a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1"},{"repo":"polyglot-lib","ref":"e62ceb6337324a294d4fc861692a267ff0db97d0"},{"repo":"workspace-monorepo","ref":"72c47a51887733b9035bac1a3efdbf92880ef791"},{"repo":"multiversion-lib","ref":"ba9ee7269559a122f2b58fb89f134aaa4b7d0aee"},{"repo":"awkward-lib","ref":"6ccc922f4cb27b4a3de5f0e113c4bbf20d736300"},{"repo":"submodule-consumer","ref":"bc193f9d805e286c95e9b0c49580c5171224e45c"},{"repo":"subtree-consumer","ref":"3da0c249e481104a8c9058e6107dd47f8a80b10f"},{"repo":"shared-schema","ref":"90d976b2d1d97f2e1c301eac35f88574e7aeee4d"}] type: string workflow_call: inputs: @@ -38,7 +38,7 @@ on: description: JSON array of pinned fixture repository/ref objects required: false default: >- - [{"repo":"node-lib","ref":"2d41e658a382fdff27ef87a0db24dbaff280a685"},{"repo":"node-app","ref":"ef794674647b3620e711f4f855865f8ad3340498"},{"repo":"rust-lib","ref":"f2534ffafbc4f7e900075dfc8e48b9259aced2fe"},{"repo":"rust-app","ref":"38b0a5425176a53621d1c07be791701dbd30eefe"},{"repo":"go-lib","ref":"e09dec6790a60ba75d5d338eece1068a4b8454b1"},{"repo":"go-app","ref":"8e035feb84cf80a294e631bda4ea7a44b1c4d7cf"},{"repo":"python-lib","ref":"b2db8ce2962fbaae690db7a9dabd14eb1d0362de"},{"repo":"python-app","ref":"a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1"},{"repo":"polyglot-lib","ref":"e62ceb6337324a294d4fc861692a267ff0db97d0"},{"repo":"workspace-monorepo","ref":"72c47a51887733b9035bac1a3efdbf92880ef791"},{"repo":"multiversion-lib","ref":"ba9ee7269559a122f2b58fb89f134aaa4b7d0aee"},{"repo":"awkward-lib","ref":"3e24f62ef4df5d8bfc6e63df07d28eaf1f4dac9e"},{"repo":"submodule-consumer","ref":"bc193f9d805e286c95e9b0c49580c5171224e45c"},{"repo":"subtree-consumer","ref":"3da0c249e481104a8c9058e6107dd47f8a80b10f"},{"repo":"shared-schema","ref":"c3fe49a621337be58b16717fb1acb57139a29b45"}] + [{"repo":"node-lib","ref":"2d41e658a382fdff27ef87a0db24dbaff280a685"},{"repo":"node-app","ref":"ef794674647b3620e711f4f855865f8ad3340498"},{"repo":"rust-lib","ref":"f2534ffafbc4f7e900075dfc8e48b9259aced2fe"},{"repo":"rust-app","ref":"38b0a5425176a53621d1c07be791701dbd30eefe"},{"repo":"go-lib","ref":"e09dec6790a60ba75d5d338eece1068a4b8454b1"},{"repo":"go-app","ref":"8e035feb84cf80a294e631bda4ea7a44b1c4d7cf"},{"repo":"python-lib","ref":"b2db8ce2962fbaae690db7a9dabd14eb1d0362de"},{"repo":"python-app","ref":"a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1"},{"repo":"polyglot-lib","ref":"e62ceb6337324a294d4fc861692a267ff0db97d0"},{"repo":"workspace-monorepo","ref":"72c47a51887733b9035bac1a3efdbf92880ef791"},{"repo":"multiversion-lib","ref":"ba9ee7269559a122f2b58fb89f134aaa4b7d0aee"},{"repo":"awkward-lib","ref":"6ccc922f4cb27b4a3de5f0e113c4bbf20d736300"},{"repo":"submodule-consumer","ref":"bc193f9d805e286c95e9b0c49580c5171224e45c"},{"repo":"subtree-consumer","ref":"3da0c249e481104a8c9058e6107dd47f8a80b10f"},{"repo":"shared-schema","ref":"90d976b2d1d97f2e1c301eac35f88574e7aeee4d"}] type: string permissions: @@ -55,9 +55,9 @@ defaults: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: "1" - ZED_CLI_REF: ${{ inputs.zed_cli_ref || '3a91d64abf1fc0bdee6eab2ff804113b1429849f' }} + ZED_CLI_REF: ${{ inputs.zed_cli_ref || 'd8763b8dcc3bd5c5b6b4b1d13519065a6a2cb99a' }} HARNESS_REF: ${{ inputs.harness_ref || github.event.pull_request.head.sha || github.sha }} - ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 + ZED_INTERFACES_REF: 8428bc574111fa148e590c8350c7855035ce2046 jobs: contract: @@ -68,16 +68,24 @@ jobs: fixtures: ${{ steps.validate.outputs.fixtures }} fixture_refs: ${{ steps.validate.outputs.fixture_refs }} steps: + - name: Check out the reviewed harness baseline + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + - name: Validate exact refs and fixture matrix id: validate env: FIXTURES_JSON: >- - ${{ inputs.fixtures_json || '[{"repo":"node-lib","ref":"2d41e658a382fdff27ef87a0db24dbaff280a685"},{"repo":"node-app","ref":"ef794674647b3620e711f4f855865f8ad3340498"},{"repo":"rust-lib","ref":"f2534ffafbc4f7e900075dfc8e48b9259aced2fe"},{"repo":"rust-app","ref":"38b0a5425176a53621d1c07be791701dbd30eefe"},{"repo":"go-lib","ref":"e09dec6790a60ba75d5d338eece1068a4b8454b1"},{"repo":"go-app","ref":"8e035feb84cf80a294e631bda4ea7a44b1c4d7cf"},{"repo":"python-lib","ref":"b2db8ce2962fbaae690db7a9dabd14eb1d0362de"},{"repo":"python-app","ref":"a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1"},{"repo":"polyglot-lib","ref":"e62ceb6337324a294d4fc861692a267ff0db97d0"},{"repo":"workspace-monorepo","ref":"72c47a51887733b9035bac1a3efdbf92880ef791"},{"repo":"multiversion-lib","ref":"ba9ee7269559a122f2b58fb89f134aaa4b7d0aee"},{"repo":"awkward-lib","ref":"3e24f62ef4df5d8bfc6e63df07d28eaf1f4dac9e"},{"repo":"submodule-consumer","ref":"bc193f9d805e286c95e9b0c49580c5171224e45c"},{"repo":"subtree-consumer","ref":"3da0c249e481104a8c9058e6107dd47f8a80b10f"},{"repo":"shared-schema","ref":"c3fe49a621337be58b16717fb1acb57139a29b45"}]' }} + ${{ inputs.fixtures_json || '[{"repo":"node-lib","ref":"2d41e658a382fdff27ef87a0db24dbaff280a685"},{"repo":"node-app","ref":"ef794674647b3620e711f4f855865f8ad3340498"},{"repo":"rust-lib","ref":"f2534ffafbc4f7e900075dfc8e48b9259aced2fe"},{"repo":"rust-app","ref":"38b0a5425176a53621d1c07be791701dbd30eefe"},{"repo":"go-lib","ref":"e09dec6790a60ba75d5d338eece1068a4b8454b1"},{"repo":"go-app","ref":"8e035feb84cf80a294e631bda4ea7a44b1c4d7cf"},{"repo":"python-lib","ref":"b2db8ce2962fbaae690db7a9dabd14eb1d0362de"},{"repo":"python-app","ref":"a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1"},{"repo":"polyglot-lib","ref":"e62ceb6337324a294d4fc861692a267ff0db97d0"},{"repo":"workspace-monorepo","ref":"72c47a51887733b9035bac1a3efdbf92880ef791"},{"repo":"multiversion-lib","ref":"ba9ee7269559a122f2b58fb89f134aaa4b7d0aee"},{"repo":"awkward-lib","ref":"6ccc922f4cb27b4a3de5f0e113c4bbf20d736300"},{"repo":"submodule-consumer","ref":"bc193f9d805e286c95e9b0c49580c5171224e45c"},{"repo":"subtree-consumer","ref":"3da0c249e481104a8c9058e6107dd47f8a80b10f"},{"repo":"shared-schema","ref":"90d976b2d1d97f2e1c301eac35f88574e7aeee4d"}]' }} run: | set -euo pipefail [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] [[ "$HARNESS_REF" =~ ^[0-9a-f]{40}$ ]] [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + test "$(tr -d '\r\n' < .zed-cli-ref)" = "$ZED_CLI_REF" python3 - <<'PY' import json import os @@ -177,6 +185,8 @@ jobs: set -euo pipefail test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" test "$(git -C zed-interfaces rev-parse HEAD)" = "$ZED_INTERFACES_REF" + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + grep -F "rev=$ZED_INTERFACES_REF#$ZED_INTERFACES_REF" zed-cli/Cargo.lock cargo fmt --all --check --manifest-path zed-cli/Cargo.toml cargo test --locked --manifest-path zed-cli/Cargo.toml --lib r2g::tests cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed diff --git a/.github/workflows/canonical-task-canary.yml b/.github/workflows/canonical-task-canary.yml new file mode 100644 index 00000000..3e531933 --- /dev/null +++ b/.github/workflows/canonical-task-canary.yml @@ -0,0 +1,129 @@ +# Re-run after zed-pkg-test/zed-interfaces#1 and zed-pkg-test/zed-lib#1 +# corrected the live-fleet package namespaces exposed by the first certification. +name: canonical task CLI canary + +on: + pull_request: + paths: + - .github/workflows/canonical-task-canary.yml + - scripts/canonical_task_canary.py + - compat/tasks/** + workflow_dispatch: + inputs: + candidate: + description: Full immutable zed-pkg/zed-cli commit to certify + required: false + default: 4c708dcece916a81caace9d82083a523a297475c + +permissions: + contents: read + +concurrency: + group: canonical-task-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + ZED_CLI_COMMIT: ${{ inputs.candidate || '4c708dcece916a81caace9d82083a523a297475c' }} + +jobs: + certify: + name: canonical task / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 55 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + steps: + - name: Check out independent certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact product candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable product identity + run: | + set -euo pipefail + [[ "$ZED_CLI_COMMIT" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + + - name: Install Rust and Python + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Install Python 3.12 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + + - name: Validate exact product source and typed contracts + working-directory: product/zed-cli + run: | + set -euo pipefail + cargo fmt --all --check + cargo test --locked --lib task_runtime::tests + cargo test --locked --lib task_cli::tests + cargo test --locked --test task_runtime_cli + cargo test --locked --test task_main_cli + cargo test --locked cli::tests::task_commands_are_typed_and_reject_zero_concurrency + cargo test --locked cli::tests::cli_flags_toml_is_in_sync_with_clap + cargo test --locked cli::tests::readme_documents_every_command + cargo clippy --locked --bin zed --bin zed-task \ + --test task_runtime_cli --test task_main_cli -- -D warnings + + - name: Build exact release binaries + working-directory: product/zed-cli + run: | + set -euo pipefail + cargo build --locked --release --bin zed --bin zed-task + + - name: Select platform binary paths + run: | + set -euo pipefail + zed="$GITHUB_WORKSPACE/product/zed-cli/target/release/zed" + staged="$GITHUB_WORKSPACE/product/zed-cli/target/release/zed-task" + if [[ "$RUNNER_OS" == "Windows" ]]; then + zed="${zed}.exe" + staged="${staged}.exe" + fi + echo "ZED_BINARY=$zed" >> "$GITHUB_ENV" + echo "ZED_TASK_BINARY=$staged" >> "$GITHUB_ENV" + + - name: Run independent black-box certification + run: | + set -euo pipefail + python -m py_compile scripts/canonical_task_canary.py + python scripts/canonical_task_canary.py \ + --zed "$ZED_BINARY" \ + --zed-task "$ZED_TASK_BINARY" \ + --candidate "$ZED_CLI_COMMIT" \ + --work "$RUNNER_TEMP/canonical-task-canary" \ + --evidence "artifacts/canonical-task-${RUNNER_OS}-${RUNNER_ARCH}.json" + git diff --exit-code + git -C product/zed-cli diff --exit-code + + - name: Upload exact-head evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: canonical-task-${{ runner.os }}-${{ runner.arch }}-${{ env.ZED_CLI_COMMIT }} + path: artifacts/canonical-task-${{ runner.os }}-${{ runner.arch }}.json + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/complete-constraint-solver.yml b/.github/workflows/complete-constraint-solver.yml new file mode 100644 index 00000000..a2bee7fd --- /dev/null +++ b/.github/workflows/complete-constraint-solver.yml @@ -0,0 +1,204 @@ +name: complete constraint solver + +on: + pull_request: + paths: + - scripts/complete_constraint_solver.py + - scripts/complete-constraint-solver.sh + - docs/complete-constraint-solver-canary.md + - .github/workflows/complete-constraint-solver.yml + push: + branches: [main] + paths: + - scripts/complete_constraint_solver.py + - scripts/complete-constraint-solver.sh + - docs/complete-constraint-solver-canary.md + - .github/workflows/complete-constraint-solver.yml + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: ddf0487f560c19000e2844b62797f1cd6299c26e + run_arc_smoke: + description: Repeat the canary on the registered sonus-ci ARC scale set + type: boolean + required: true + default: false + +permissions: + contents: read + +concurrency: + group: complete-constraint-solver-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || 'ddf0487f560c19000e2844b62797f1cd6299c26e' }} + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + +jobs: + hosted: + name: Overlap, backtracking, stale constraints, diagnostics, cycles, and yank policy + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out the canary + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the pinned zed-interfaces sibling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + rustc --version --verbose + cargo --version --verbose + python3 --version + + - name: Compile canaries and verify Rust formatting + run: | + set -euo pipefail + python3 -m py_compile harness/scripts/complete_constraint_solver.py + bash -n harness/scripts/complete-constraint-solver.sh + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + + - name: Build the immutable zed binary + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + install -Dm755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Run complete constraint-solver certification + env: + WORK_ROOT: ${{ runner.temp }}/zed-complete-constraint-solver + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/complete_constraint_solver.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + + - name: Enforce yanked-candidate pre-acquisition boundary + env: + WORK_ROOT: ${{ runner.temp }}/zed-complete-constraint-solver + run: | + set -euo pipefail + bash harness/scripts/complete-constraint-solver.sh \ + "$RUNNER_TEMP/zed-bin/zed" \ + "$WORK_ROOT/yanked-candidate" + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-complete-constraint-solver + run: | + set +e + mkdir -p "$WORK_ROOT/diagnostics" + find "$WORK_ROOT" -maxdepth 12 -printf '%y %P -> %l\n' \ + | LC_ALL=C sort > "$WORK_ROOT/diagnostics/work-tree.txt" 2>&1 + find "$WORK_ROOT/registry" "$WORK_ROOT/yanked-candidate/registry" \ + -maxdepth 10 -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/registry-sha256.txt" 2>&1 + find "$WORK_ROOT" -path '*/.zpkg.lock' -type f -print \ + -exec sed -n '1,300p' {} \; \ + > "$WORK_ROOT/diagnostics/lockfiles.txt" 2>&1 + find "$WORK_ROOT" -path '*/cache/*.tar.gz' -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/cache-sha256.txt" 2>&1 + find "$WORK_ROOT" -name '*.log' -type f -print \ + -exec sed -n '1,400p' {} \; \ + > "$WORK_ROOT/diagnostics/logs.txt" 2>&1 + true + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: complete-constraint-solver-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-complete-constraint-solver/diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false + + arc-parity: + name: Optional self-hosted ARC parity + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_arc_smoke }} + runs-on: [self-hosted, linux, sonus-ci] + timeout-minutes: 40 + steps: + - name: Check out the canary + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify ephemeral runner and execute the same canaries + env: + WORK_ROOT: ${{ runner.temp }}/zed-complete-constraint-solver-arc + run: | + set -euo pipefail + test "$RUNNER_OS" = Linux + test -n "$RUNNER_NAME" + test ! -e /var/run/docker.sock + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + rustup toolchain install stable --profile minimal + rustup default stable + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + test ! -e "$WORK_ROOT" + python3 harness/scripts/complete_constraint_solver.py \ + --zed zed-cli/target/release/zed \ + --work-root "$WORK_ROOT" + bash harness/scripts/complete-constraint-solver.sh \ + zed-cli/target/release/zed \ + "$WORK_ROOT/yanked-candidate" diff --git a/.github/workflows/create-production-dotgithub-fleet.yml b/.github/workflows/create-production-dotgithub-fleet.yml new file mode 100644 index 00000000..cb7f2540 --- /dev/null +++ b/.github/workflows/create-production-dotgithub-fleet.yml @@ -0,0 +1,501 @@ +name: create production dotgithub fleet + +on: + push: + branches: [main] + paths: + - ".github/fleet-create/START" + +permissions: + contents: read + +concurrency: + group: create-production-dotgithub-fleet + cancel-in-progress: false + +jobs: + create: + if: "${{ github.event.head_commit.message == 'chore: run production dotgithub fleet creation' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate the stored fleet credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + login="$(gh api user --jq '.login')" + if [[ "$login" != "ORESoftware" ]]; then + echo "::error::FLEET_GH_TOKEN authenticated as unexpected user: $login" + exit 1 + fi + gh auth setup-git + + - name: Create and seed missing organization profile repositories + shell: bash + run: | + set -Eeuo pipefail + shopt -s inherit_errexit + + owners=( + channelsiege + OmniBlitz + streamkore + hypeblitz + 3FA-app + messaging-intel + akrion-sim + athlet-o + benefactor-cc + canonical-cloud + claritas-viz + cliptown + daedalus-fab + declarative-migrations + fiducia-cloud + anticaptrad + opto-sync + quaestor-ledger + sagitta-stack + shared-auth + scintilla-run + rust-ssr-demos + sonus-auris + usa-acc + voxletra + zed-pkg + zed-pkg-test + memebank + meta-agents-demo + networking-components + StreemPilot + unreal-unity-poc + file-tunnel + hypesiege + discrete-event-systems + drone-mngr + ORESoftware + ) + + created=() + existing=() + failed=() + + for owner in "${owners[@]}"; do + echo "::group::$owner/.github" + + if gh api "repos/$owner/.github" --silent 2>/dev/null; then + echo "already exists; preserving current contents" + existing+=("$owner") + echo "::endgroup::" + continue + fi + + description="$owner organization-wide contributor guidance, governance, community health files, and shared automation policy." + + if [[ "$owner" == "ORESoftware" ]]; then + create_endpoint="user/repos" + else + create_endpoint="orgs/$owner/repos" + fi + + if ! gh api --method POST "$create_endpoint" \ + -f name='.github' \ + -f description="$description" \ + -F private=false \ + -F has_issues=true \ + -F has_projects=false \ + -F has_wiki=false \ + -F auto_init=false \ + --silent; then + echo "::error::repository creation failed for $owner/.github" + failed+=("$owner:create") + echo "::endgroup::" + continue + fi + + workdir="$(mktemp -d)" + export OWNER="$owner" + export ROOT="$workdir" + + node <<'NODE' + const fs = require("node:fs"); + const path = require("node:path"); + + const owner = process.env.OWNER; + const root = process.env.ROOT; + const write = (file, content) => { + const target = path.join(root, file); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, content.endsWith("\n") ? content : `${content}\n`, "utf8"); + }; + + write("README.md", `# ${owner} organization defaults + + This public \`.github\` repository contains organization-wide community health files, contributor guidance, shared agent policy, and an explicit repository-relationship contract. + + Repository-local policy wins when it is stricter or more specific. Existing project history must be preserved during consolidation and conflict resolution. + `); + + write("profile/README.md", `# ${owner} + + This organization maintains software, infrastructure, interfaces, clients, services, and supporting documentation under a shared engineering baseline. + + ## Working principles + + - Keep changes reviewable, tested, and reversible. + - Treat security, privacy, compatibility, and data durability as design constraints. + - Resolve merge conflicts semantically: reconstruct both sides' intent, preserve compatible behavior, and document deliberate trade-offs. + - Prefer canonical repositories and short, stable names; deprecate duplicates with migration notes rather than silently deleting history. + - Keep cross-repository dependencies explicit and pinned where reproducibility matters. + + Organization-wide contribution and security guidance lives in this \`.github\` repository. + `); + + write("CONTRIBUTING.md", `# Contributing + + 1. Read the target repository's README, agent instructions, and local contribution guide. + 2. Open focused changes with tests or an explicit verification plan. + 3. Preserve public APIs, data formats, and operational behavior unless the change intentionally migrates them. + 4. Keep generated files reproducible and identify their source. + 5. Never resolve conflicts by blindly choosing "ours" or "theirs". Inspect the base, both branches, nearby history, tests, and downstream consumers; then construct the smallest coherent combined result. + 6. Do not commit credentials, private keys, tokens, production data, or personal information. + `); + + write("SECURITY.md", `# Security policy + + Report suspected vulnerabilities privately through GitHub's security-advisory flow when available. Do not open a public issue containing exploit details, credentials, private data, or unredacted logs. + + Security fixes should include a clear threat model, affected boundary, regression test, rollout plan, and rollback path. Rotate any credential that may have been exposed; deleting it from a later commit is not sufficient. + `); + + write("SUPPORT.md", `# Support + + Use the target repository's issue tracker for reproducible defects and feature requests. Include the affected version or commit, environment, minimal reproduction, expected behavior, actual behavior, and sanitized logs. + + Security-sensitive reports belong in the private channel described in SECURITY.md. + `); + + write("CODE_OF_CONDUCT.md", `# Code of conduct + + Be respectful, specific, and constructive. Critique code and decisions rather than people. Harassment, discrimination, threats, doxxing, and disclosure of private information are not acceptable. + + Maintainers may edit or remove abusive content and restrict participation when needed to protect the community. + `); + + write("GOVERNANCE.md", `# Governance + + Repository maintainers own technical direction for their scope. Cross-repository changes must identify the canonical owner of each interface and avoid duplicating sources of truth. + + Material changes should record: + - the decision and alternatives considered; + - compatibility and migration consequences; + - security and operational risks; + - validation evidence; + - rollback or deprecation strategy. + + Conflicted changes are merged by intent, not line selection. A semantic reconciliation should preserve non-conflicting additions from both sides and explicitly resolve incompatible assumptions. + `); + + write("REPOSITORY_BOUNDARIES.md", `# Repository boundaries + + Each repository should have one primary responsibility and an explicit dependency direction. + + - \`*-interfaces\`: schemas, protocol contracts, and stable cross-language boundaries. + - \`*-lib\`: reusable domain logic built on interfaces. + - \`*-clients\`: client SDKs and transport adapters. + - \`*-sync\`: synchronization and offline/replication behavior. + - \`*-cli\`: command-line workflows composed from clients, interfaces, and libraries. + - web/API/backend repositories: deployable services consuming the shared contracts. + - \`*-e2e\`: black-box and integration verification, not production implementation. + + When repositories overlap, choose a canonical home, migrate callers, preserve attribution and useful history, and leave a clear deprecation pointer. + `); + + write("AGENTS.md", `# Agent instructions + + Before editing: + 1. Read repository-local instructions and recent history. + 2. Identify the canonical source of truth and downstream consumers. + 3. Keep credentials and private data out of prompts, logs, commits, and artifacts. + + For conflicts, compare the merge base, both heads, relevant tests, and surrounding commits. Reconstruct intent and produce a coherent combined implementation. Never use blanket ours/theirs resolution for substantive conflicts. + + Validate the exact commit being proposed. Do not claim tests or deployments that were not actually run. + `); + + write("CLAUDE.md", `# Claude guidance + + Follow AGENTS.md and repository-local instructions. Preserve existing behavior unless the requested change intentionally replaces it. Use semantic conflict resolution and exact-head validation. Do not expose secrets or invent verification results. + `); + + write(".openai/AGENTS.md", `# OpenAI agent guidance + + Follow ../../AGENTS.md and all repository-local policy. Resolve conflicts semantically, keep changes narrowly scoped, and report verification truthfully. + `); + + write(".claude/CLAUDE.md", `# Claude agent guidance + + Follow ../../AGENTS.md and all repository-local policy. Resolve conflicts semantically, keep changes narrowly scoped, and report verification truthfully. + `); + + write(".gemini/GEMINI.md", `# Gemini agent guidance + + Follow ../../AGENTS.md and all repository-local policy. Resolve conflicts semantically, keep changes narrowly scoped, and report verification truthfully. + `); + + write("agents/semantic-conflict-resolver.md", `# Semantic conflict resolver + + A conflict is a design reconciliation task, not a text-selection task. + + 1. Establish the merge base and read the full conflicting files. + 2. Inspect several relevant commits on both branches and identify the invariant each side protects. + 3. Classify edits as independent, complementary, superseding, or incompatible. + 4. Preserve independent and complementary work from both sides. + 5. For incompatible assumptions, choose or synthesize the design that best satisfies current requirements, tests, compatibility, security, and maintainability. + 6. Remove obsolete duplication and update tests/docs to express the final contract. + 7. Verify the exact resolved head and summarize the semantic decisions. + `); + + write("ORG_CONTEXT.md", `# Organization context + + Organization: **${owner}** + + This file is intentionally conservative. Project-specific product goals, Linear project IDs, Slack channels, deployment environments, and secrets must be recorded only after they are verified from their canonical systems. Do not fabricate missing links or identifiers. + `); + + write("project-context.yaml", `schema_version: 1 + organization: ${owner} + canonical_profile_repository: ${owner}/.github + policy: + semantic_conflict_resolution: required + preserve_history_during_consolidation: true + exact_head_validation: true + secrets_in_repository: forbidden + integrations: + linear: verify_per_project + slack: verify_per_project + `); + + write("organization-policy.json", JSON.stringify({ + schema_version: 1, + organization: owner, + canonical_profile_repository: `${owner}/.github`, + conflict_resolution: { + mode: "semantic", + blanket_ours_or_theirs: false, + inspect_history_and_tests: true + }, + repository_consolidation: { + preserve_history: true, + prefer_short_canonical_names: true, + require_deprecation_pointer: true + }, + validation: { + exact_head: true, + truthful_evidence: true + } + }, null, 2)); + + write("repository-relationships.json", JSON.stringify({ + schema_version: 1, + organization: owner, + generated_at: null, + repositories: [], + notes: [ + "Populate from live GitHub inventory; do not infer unverified dependencies.", + "Repository-local manifests remain authoritative for exact dependency versions." + ] + }, null, 2)); + + write("notes/README.md", `# Organization notes + + Keep durable, non-sensitive organization-wide notes here. Project-specific plans should live with the project or in the verified planning system. Never store tokens, private keys, credentials, private customer data, or unredacted incident evidence in this repository. + `); + + write(".github/ISSUE_TEMPLATE/config.yml", `blank_issues_enabled: true + contact_links: + - name: Security report + url: https://github.com/${owner}/.github/security/advisories/new + about: Report vulnerabilities privately when the advisory flow is enabled. + `); + + write(".github/ISSUE_TEMPLATE/bug_report.yml", `name: Bug report + description: Report a reproducible defect + title: "bug: " + labels: ["bug"] + body: + - type: textarea + id: summary + attributes: + label: Summary + description: What happened, and what should have happened? + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Reproduction + description: Minimal steps, inputs, and environment details. + validations: + required: true + - type: textarea + id: evidence + attributes: + label: Sanitized evidence + description: Logs, screenshots, or traces with secrets and personal data removed. + - type: checkboxes + id: safety + attributes: + label: Safety check + options: + - label: I removed credentials, private data, and exploit details that belong in a private advisory. + required: true + `); + + write(".github/ISSUE_TEMPLATE/feature_request.yml", `name: Feature request + description: Propose a focused improvement + title: "feat: " + labels: ["enhancement"] + body: + - type: textarea + id: problem + attributes: + label: Problem + description: What user or operational problem should be solved? + validations: + required: true + - type: textarea + id: outcome + attributes: + label: Desired outcome + description: Describe observable behavior and acceptance criteria. + validations: + required: true + - type: textarea + id: constraints + attributes: + label: Constraints and alternatives + description: Compatibility, security, migration, and alternatives considered. + `); + + write(".github/pull_request_template.md", `## Summary + + Describe the problem, the chosen design, and the user or operational outcome. + + ## Verification + + List commands, tests, checks, and manual validation actually performed against this exact head. + + ## Compatibility and risk + + Explain API/data/configuration changes, security implications, rollout, migration, and rollback. + + ## Conflict reconciliation + + When conflicts were present, summarize both sides' intent and how the final implementation preserves or deliberately supersedes each part. + + - [ ] No secrets or private data are included. + - [ ] Documentation and tests reflect the final contract. + - [ ] The exact proposed head was verified. + `); + + write(".github/copilot-instructions.md", `Read AGENTS.md and repository-local instructions before editing. Keep changes focused and evidence truthful. Resolve conflicts by understanding both branches and constructing the best coherent result; never select ours/theirs wholesale for substantive conflicts. Preserve security, compatibility, data durability, and repository boundaries. + `); + + write(".github/dependabot.yml", `version: 2 + updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + open-pull-requests-limit: 5 + `); + + write(".github/workflows/baseline-policy.yml", `name: baseline policy + + on: + pull_request: + push: + branches: [main] + workflow_dispatch: + + permissions: + contents: read + + jobs: + validate: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Validate organization baseline + shell: bash + run: | + set -euo pipefail + required=( + README.md + profile/README.md + CONTRIBUTING.md + SECURITY.md + SUPPORT.md + CODE_OF_CONDUCT.md + GOVERNANCE.md + AGENTS.md + repository-relationships.json + organization-policy.json + ) + for file in "\${required[@]}"; do + test -s "\$file" || { echo "::error file=\$file::required baseline file is missing or empty"; exit 1; } + done + python3 -m json.tool organization-policy.json >/dev/null + python3 -m json.tool repository-relationships.json >/dev/null + ! git grep -nE '^(<<<<<<<|=======|>>>>>>>)' -- . ':!*.lock' + `); + NODE + + ( + cd "$workdir" + git init -q -b main + git config user.name "Alexander Mills" + git config user.email "alex@oresoftware.com" + git add . + git commit -q -m "chore: establish organization-wide GitHub baseline" + git push -q "https://github.com/$owner/.github.git" HEAD:refs/heads/main + ) || { + echo "::error::initial baseline push failed for $owner/.github" + failed+=("$owner:push") + rm -rf "$workdir" + echo "::endgroup::" + continue + } + + rm -rf "$workdir" + created+=("$owner") + echo "created and seeded" + echo "::endgroup::" + sleep 2 + done + + { + echo "## Production .github fleet" + echo + echo "- Created: ${#created[@]}" + echo "- Already present and preserved: ${#existing[@]}" + echo "- Failed: ${#failed[@]}" + echo + if ((${#created[@]})); then + printf 'Created: `%s`\n\n' "$(IFS=', '; echo "${created[*]}")" + fi + if ((${#existing[@]})); then + printf 'Preserved: `%s`\n\n' "$(IFS=', '; echo "${existing[*]}")" + fi + if ((${#failed[@]})); then + printf 'Failures: `%s`\n' "$(IFS=', '; echo "${failed[*]}")" + fi + } >> "$GITHUB_STEP_SUMMARY" + + if ((${#failed[@]})); then + exit 1 + fi diff --git a/.github/workflows/den-1461-current-mise-lock-contract.yml b/.github/workflows/den-1461-current-mise-lock-contract.yml new file mode 100644 index 00000000..9bc71c90 --- /dev/null +++ b/.github/workflows/den-1461-current-mise-lock-contract.yml @@ -0,0 +1,114 @@ +name: DEN-1461 current mise.lock contract + +on: + pull_request: + paths: + - .github/workflows/den-1461-current-mise-lock-contract.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-1461-current-mise-lock-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + ZED_CLI_SHA: 82ed67aaa98495b5aa85faef1d25671373eb1991 + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + certify: + name: exact lock identity / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - macos-15 + - windows-2025 + + steps: + - name: Check out certification repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Check out immutable zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_SHA }} + path: zed-cli + persist-credentials: false + show-progress: false + + - name: Install pinned Rust toolchain action + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + components: rustfmt,clippy + + - name: Verify exact immutable candidate + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_SHA" + test -f zed-cli/src/mise_lock.rs + test -f zed-cli/tests/fixtures/mise-lock/current-actionlint.lock + test -f zed-cli/tests/mise_lock_contract.rs + + - name: Check formatting + working-directory: zed-cli + run: cargo fmt --all --check + + - name: Run complete lock module and public API contracts + working-directory: zed-cli + run: | + cargo test --locked mise_lock -- --nocapture + cargo test --locked --test mise_lock_contract -- --nocapture + + - name: Lint every Rust target with warnings denied + working-directory: zed-cli + run: cargo clippy --locked --all-targets -- -D warnings + + - name: Repeat deterministic normalized lock identity + working-directory: zed-cli + run: | + cargo test --locked mise_lock::tests::complete_current_lock_round_trips_and_is_frozen_portable -- --exact --nocapture + cargo test --locked mise_lock::tests::complete_current_lock_round_trips_and_is_frozen_portable -- --exact --nocapture + + - name: Prove the certification lane is read-only and credential-free + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + workflow = Path('.github/workflows/den-1461-current-mise-lock-contract.yml').read_text() + assert 'permissions:\n contents: read' in workflow + assert 'persist-credentials: false' in workflow + for forbidden in ( + 'contents: write', + 'pull_request_target:', + 'git push', + 'GH_TOKEN', + 'GITHUB_TOKEN', + 'ZED_PKG_TOKEN', + ): + assert forbidden not in workflow, forbidden + PY + + gate: + name: all DEN-1461 platforms passed + if: always() + needs: certify + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Enforce aggregate result + env: + RESULT: ${{ needs.certify.result }} + run: test "$RESULT" = success diff --git a/.github/workflows/den-2725-publish-final-export-windows-v2.yml b/.github/workflows/den-2725-publish-final-export-windows-v2.yml new file mode 100644 index 00000000..3cd41eff --- /dev/null +++ b/.github/workflows/den-2725-publish-final-export-windows-v2.yml @@ -0,0 +1,100 @@ +name: DEN-2725 publish verified final replay export exact bytes + +on: + push: + branches: + - agent/den-2725-publish-final-export + +permissions: + actions: read + contents: write + +env: + SOURCE_RUN_ID: '31243116442' + SOURCE_ARTIFACT: gitops-dispatch-replay-Linux-X64-1c94813c6b59c7621c1256d7c55246e7f06f44b7-0246a3af901b65ca6724d936c7f3232c9f72fcd7 + ROOT_DISPATCH_COMMIT: 1c94813c6b59c7621c1256d7c55246e7f06f44b7 + MAIN_COMMIT: 0246a3af901b65ca6724d936c7f3232c9f72fcd7 + EXPECTED_TREE: 99ab08e3142c2a52d2c97482134e99fae3811b43 + +jobs: + publish: + runs-on: windows-2025 + timeout-minutes: 10 + defaults: + run: + shell: bash + steps: + - name: Check out export publisher with scoped token + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: agent/den-2725-publish-final-export + fetch-depth: 0 + persist-credentials: true + show-progress: false + + - name: Check out exact source commit for hidden package manifest + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ROOT_DISPATCH_COMMIT }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Download and verify the green Linux replay artifact + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh run download "$SOURCE_RUN_ID" \ + --repo zed-pkg-test/zed-pkg-e2e \ + --name "$SOURCE_ARTIFACT" \ + --dir /tmp/replay + test "$(jq -r .mainCommit /tmp/replay/corrected/replay-metadata.json)" = "$MAIN_COMMIT" + test "$(jq -r .rootDispatchCommit /tmp/replay/corrected/replay-metadata.json)" = "$ROOT_DISPATCH_COMMIT" + test "$(jq -r .treeSha /tmp/replay/corrected/replay-metadata.json)" = "$EXPECTED_TREE" + git -C product/zed-cli show "$ROOT_DISPATCH_COMMIT:.zpkg.toml" > /tmp/replay/corrected/.zpkg.toml + (cd /tmp/replay && sha256sum -c corrected/SHA256SUMS) + + - name: Build the immutable export payload + run: | + set -euo pipefail + mkdir -p /tmp/export/{docs,scripts,src,tests,evidence} + cp /tmp/replay/corrected/.zpkg.toml /tmp/export/ + cp /tmp/replay/corrected/docs/gitops-validator.md /tmp/export/docs/ + cp /tmp/replay/corrected/scripts/validate-zed-package-graph.sh /tmp/export/scripts/ + cp /tmp/replay/corrected/src/completion.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/external_subcommands.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/lib.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/main.rs /tmp/export/src/ + cp /tmp/replay/corrected/tests/external_gitops_dispatch.rs /tmp/export/tests/ + cp /tmp/replay/corrected/replay-metadata.json /tmp/export/evidence/ + cp /tmp/replay/corrected/SHA256SUMS /tmp/export/evidence/ + cp /tmp/replay/corrected/den-2725-final-replay.patch /tmp/export/evidence/ + cp /tmp/replay/artifacts/*.json /tmp/export/evidence/ + printf '%s\n' \ + '# DEN-2725 verified final replay export' \ + '' \ + "Source run: $SOURCE_RUN_ID" \ + "Root-dispatch commit: $ROOT_DISPATCH_COMMIT" \ + "Current-main commit: $MAIN_COMMIT" \ + "Verified product tree: $EXPECTED_TREE" \ + '' \ + 'Only the eight product files at the repository root are intended for import. Evidence is retained under `evidence/`.' \ + > /tmp/export/README.md + + - name: Publish the export branch without affecting product history + run: | + set -euo pipefail + rm -rf product + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git checkout --orphan agent/den-2725-final-export + git rm -rf . + find . -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + + cp -a /tmp/export/. . + chmod +x scripts/validate-zed-package-graph.sh + git add -A + git commit -m 'test(DEN-2725): publish verified final replay tree' + git push --force origin HEAD:refs/heads/agent/den-2725-final-export diff --git a/.github/workflows/den-2725-publish-final-export-windows.yml b/.github/workflows/den-2725-publish-final-export-windows.yml new file mode 100644 index 00000000..9c65b406 --- /dev/null +++ b/.github/workflows/den-2725-publish-final-export-windows.yml @@ -0,0 +1,100 @@ +name: DEN-2725 publish verified final replay export on Windows + +on: + push: + branches: + - agent/den-2725-publish-final-export + +permissions: + actions: read + contents: write + +env: + SOURCE_RUN_ID: '31243116442' + SOURCE_ARTIFACT: gitops-dispatch-replay-Linux-X64-1c94813c6b59c7621c1256d7c55246e7f06f44b7-0246a3af901b65ca6724d936c7f3232c9f72fcd7 + ROOT_DISPATCH_COMMIT: 1c94813c6b59c7621c1256d7c55246e7f06f44b7 + MAIN_COMMIT: 0246a3af901b65ca6724d936c7f3232c9f72fcd7 + EXPECTED_TREE: 99ab08e3142c2a52d2c97482134e99fae3811b43 + +jobs: + publish: + runs-on: windows-2025 + timeout-minutes: 10 + defaults: + run: + shell: bash + steps: + - name: Check out export publisher with scoped token + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: agent/den-2725-publish-final-export + fetch-depth: 0 + persist-credentials: true + show-progress: false + + - name: Check out exact source commit for hidden package manifest + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ROOT_DISPATCH_COMMIT }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Download and verify the green Linux replay artifact + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh run download "$SOURCE_RUN_ID" \ + --repo zed-pkg-test/zed-pkg-e2e \ + --name "$SOURCE_ARTIFACT" \ + --dir /tmp/replay + test "$(jq -r .mainCommit /tmp/replay/corrected/replay-metadata.json)" = "$MAIN_COMMIT" + test "$(jq -r .rootDispatchCommit /tmp/replay/corrected/replay-metadata.json)" = "$ROOT_DISPATCH_COMMIT" + test "$(jq -r .treeSha /tmp/replay/corrected/replay-metadata.json)" = "$EXPECTED_TREE" + cp product/zed-cli/.zpkg.toml /tmp/replay/corrected/.zpkg.toml + (cd /tmp/replay && sha256sum -c corrected/SHA256SUMS) + + - name: Build the immutable export payload + run: | + set -euo pipefail + mkdir -p /tmp/export/{docs,scripts,src,tests,evidence} + cp /tmp/replay/corrected/.zpkg.toml /tmp/export/ + cp /tmp/replay/corrected/docs/gitops-validator.md /tmp/export/docs/ + cp /tmp/replay/corrected/scripts/validate-zed-package-graph.sh /tmp/export/scripts/ + cp /tmp/replay/corrected/src/completion.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/external_subcommands.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/lib.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/main.rs /tmp/export/src/ + cp /tmp/replay/corrected/tests/external_gitops_dispatch.rs /tmp/export/tests/ + cp /tmp/replay/corrected/replay-metadata.json /tmp/export/evidence/ + cp /tmp/replay/corrected/SHA256SUMS /tmp/export/evidence/ + cp /tmp/replay/corrected/den-2725-final-replay.patch /tmp/export/evidence/ + cp /tmp/replay/artifacts/*.json /tmp/export/evidence/ + printf '%s\n' \ + '# DEN-2725 verified final replay export' \ + '' \ + "Source run: $SOURCE_RUN_ID" \ + "Root-dispatch commit: $ROOT_DISPATCH_COMMIT" \ + "Current-main commit: $MAIN_COMMIT" \ + "Verified product tree: $EXPECTED_TREE" \ + '' \ + 'Only the eight product files at the repository root are intended for import. Evidence is retained under `evidence/`.' \ + > /tmp/export/README.md + + - name: Publish the export branch without affecting product history + run: | + set -euo pipefail + rm -rf product + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git checkout --orphan agent/den-2725-final-export + git rm -rf . + find . -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + + cp -a /tmp/export/. . + chmod +x scripts/validate-zed-package-graph.sh + git add -A + git commit -m 'test(DEN-2725): publish verified final replay tree' + git push --force origin HEAD:refs/heads/agent/den-2725-final-export diff --git a/.github/workflows/den-2725-publish-final-export.yml b/.github/workflows/den-2725-publish-final-export.yml new file mode 100644 index 00000000..ebd5a0b2 --- /dev/null +++ b/.github/workflows/den-2725-publish-final-export.yml @@ -0,0 +1,100 @@ +name: DEN-2725 publish verified final replay export + +on: + push: + branches: + - agent/den-2725-publish-final-export + +permissions: + actions: read + contents: write + +env: + SOURCE_RUN_ID: '31243116442' + SOURCE_ARTIFACT: gitops-dispatch-replay-Linux-X64-1c94813c6b59c7621c1256d7c55246e7f06f44b7-0246a3af901b65ca6724d936c7f3232c9f72fcd7 + ROOT_DISPATCH_COMMIT: 1c94813c6b59c7621c1256d7c55246e7f06f44b7 + MAIN_COMMIT: 0246a3af901b65ca6724d936c7f3232c9f72fcd7 + EXPECTED_TREE: 99ab08e3142c2a52d2c97482134e99fae3811b43 + +jobs: + publish: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out export publisher with scoped token + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: agent/den-2725-publish-final-export + fetch-depth: 0 + persist-credentials: true + show-progress: false + + - name: Check out exact source commit for hidden package manifest + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ROOT_DISPATCH_COMMIT }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Download and verify the green Linux replay artifact + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh run download "$SOURCE_RUN_ID" \ + --repo zed-pkg-test/zed-pkg-e2e \ + --name "$SOURCE_ARTIFACT" \ + --dir /tmp/replay + test "$(jq -r .mainCommit /tmp/replay/corrected/replay-metadata.json)" = "$MAIN_COMMIT" + test "$(jq -r .rootDispatchCommit /tmp/replay/corrected/replay-metadata.json)" = "$ROOT_DISPATCH_COMMIT" + test "$(jq -r .treeSha /tmp/replay/corrected/replay-metadata.json)" = "$EXPECTED_TREE" + cp product/zed-cli/.zpkg.toml /tmp/replay/corrected/.zpkg.toml + (cd /tmp/replay && sha256sum -c corrected/SHA256SUMS) + + - name: Build the immutable export payload + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/export/{docs,scripts,src,tests,evidence} + cp /tmp/replay/corrected/.zpkg.toml /tmp/export/ + cp /tmp/replay/corrected/docs/gitops-validator.md /tmp/export/docs/ + cp /tmp/replay/corrected/scripts/validate-zed-package-graph.sh /tmp/export/scripts/ + cp /tmp/replay/corrected/src/completion.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/external_subcommands.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/lib.rs /tmp/export/src/ + cp /tmp/replay/corrected/src/main.rs /tmp/export/src/ + cp /tmp/replay/corrected/tests/external_gitops_dispatch.rs /tmp/export/tests/ + cp /tmp/replay/corrected/replay-metadata.json /tmp/export/evidence/ + cp /tmp/replay/corrected/SHA256SUMS /tmp/export/evidence/ + cp /tmp/replay/corrected/den-2725-final-replay.patch /tmp/export/evidence/ + cp /tmp/replay/artifacts/*.json /tmp/export/evidence/ + printf '%s\n' \ + '# DEN-2725 verified final replay export' \ + '' \ + "Source run: $SOURCE_RUN_ID" \ + "Root-dispatch commit: $ROOT_DISPATCH_COMMIT" \ + "Current-main commit: $MAIN_COMMIT" \ + "Verified product tree: $EXPECTED_TREE" \ + '' \ + 'Only the eight product files at the repository root are intended for import. Evidence is retained under `evidence/`.' \ + > /tmp/export/README.md + + - name: Publish the export branch without affecting product history + shell: bash + run: | + set -euo pipefail + rm -rf product + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git checkout --orphan agent/den-2725-final-export + git rm -rf . + find . -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + + cp -a /tmp/export/. . + chmod +x scripts/validate-zed-package-graph.sh + git add -A + git commit -m 'test(DEN-2725): publish verified final replay tree' + git push --force origin HEAD:refs/heads/agent/den-2725-final-export diff --git a/.github/workflows/den-2750-validator-materializer.yml b/.github/workflows/den-2750-validator-materializer.yml new file mode 100644 index 00000000..06113cc1 --- /dev/null +++ b/.github/workflows/den-2750-validator-materializer.yml @@ -0,0 +1,187 @@ +name: DEN-2750 validator materializer + +on: + pull_request: + paths: + - ".github/workflows/den-2750-validator-materializer.yml" + push: + branches: + - agent/den-2750-validator-materializer + paths: + - ".github/workflows/den-2750-validator-materializer.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-2750-validator-materializer-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + FEATURE_BASE: d1bf8ef7c88a75292cbe8d697bfd269f30d62e2b + FEATURE_COMMIT: 5c2e328baf1954fd61aed0cad5867af99c17437f + CURRENT_MAIN: f793e7c592373242dff9ab855845c42f526db9b4 + +jobs: + materialize: + name: current-main one-file validator / ubuntu-24.04 + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out independent harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact source feature history + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.FEATURE_COMMIT }} + path: interfaces + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Prove feature lineage and compose on current main + working-directory: interfaces + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$FEATURE_COMMIT" + test "$(git diff --name-only "$FEATURE_BASE" "$FEATURE_COMMIT")" = src/version.rs + git cat-file -e "$CURRENT_MAIN^{commit}" + git diff --quiet "$FEATURE_BASE" "$CURRENT_MAIN" -- src/version.rs + git checkout --detach "$CURRENT_MAIN" + git checkout "$FEATURE_COMMIT" -- src/version.rs + test "$(git diff --name-only "$CURRENT_MAIN")" = src/version.rs + + - name: Narrow malformed wildcard detection without exact-tag false positives + working-directory: interfaces + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + + path = Path("src/version.rs") + text = path.read_text(encoding="utf-8") + + start_marker = "/// Detect a dotted requirement whose leading segment is numeric" + end_marker = "\nimpl Requirement {" + start = text.find(start_marker) + end = text.find(end_marker, start) + if start < 0 or end < 0: + raise SystemExit("could not locate the pinned broad dotted-requirement helper") + + helper = '''/// Detect only malformed dotted numeric requirements that the semver parser + /// demotes to exact tags. A wildcard followed by any additional segment is a + /// typo, and more than three all-numeric components are not semver. Valid + /// calendar-like exact tags such as `2026.07.24` and ordinary opaque tags such + /// as `1.nginx` and `1.x86_64` remain exact. + fn looks_like_malformed_dotted_numeric_requirement(input: &str) -> bool { + let mut segments = input.split('.'); + let Some(first) = segments.next() else { + return false; + }; + if first.is_empty() || !first.bytes().all(|byte| byte.is_ascii_digit()) { + return false; + } + + let mut segment_count = 1; + let mut all_numeric = true; + let mut saw_wildcard = false; + for segment in segments { + segment_count += 1; + if saw_wildcard { + return true; + } + if matches!(segment, "x" | "X" | "*") { + saw_wildcard = true; + all_numeric = false; + continue; + } + if segment.is_empty() { + return false; + } + if segment.bytes().all(|byte| byte.is_ascii_digit()) { + continue; + } + return false; + } + + all_numeric && segment_count > 3 + } + ''' + text = text[:start] + helper + text[end:] + + old_call = " || looks_like_dotted_numeric_requirement(input);\n" + new_call = " || looks_like_malformed_dotted_numeric_requirement(input);\n" + if text.count(old_call) != 1: + raise SystemExit("expected one broad validator helper call") + text = text.replace(old_call, new_call, 1) + + old_docs = "Strings with no range operators or numeric dotted shape are\n /// legitimate opaque tags and pass." + new_docs = "Strings without range operators or malformed dotted numeric shape are\n /// legitimate opaque tags and pass." + if text.count(old_docs) != 1: + raise SystemExit("expected exact validator documentation boundary") + text = text.replace(old_docs, new_docs, 1) + + accepted = ''' "matrix-2", + "1.nginx", + "1.x86_64", + ''' + accepted_with_calver = ''' "matrix-2", + "1.nginx", + "1.x86_64", + "2026.07.24", + ''' + if text.count(accepted) != 1: + raise SystemExit("expected exact accepted-tag regression list") + path.write_text(text.replace(accepted, accepted_with_calver, 1), encoding="utf-8") + PY + cargo fmt --all + test "$(git diff --name-only "$CURRENT_MAIN")" = src/version.rs + git diff --check + + - name: Run strict current-main source contract + working-directory: interfaces + shell: bash + run: | + set -euo pipefail + cargo fmt --all -- --check + cargo clippy --locked --all-targets -- -D warnings + cargo test --locked --all-targets + cargo test --locked requirement_validation_rejects_malformed_dotted_ranges_only -- --exact --nocapture + test "$(git diff --name-only "$CURRENT_MAIN")" = src/version.rs + + - name: Package exact source bytes and patch + working-directory: interfaces + shell: bash + run: | + set -euo pipefail + output="$RUNNER_TEMP/den-2750-validator" + mkdir -p "$output/src" + cp src/version.rs "$output/src/version.rs" + git diff --binary "$CURRENT_MAIN" -- src/version.rs > "$output/validator.patch" + test -s "$output/validator.patch" + sha256sum "$output/src/version.rs" "$output/validator.patch" > "$output/SHA256SUMS" + + - name: Upload exact validator materialization + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den-2750-validator-${{ github.run_id }} + path: ${{ runner.temp }}/den-2750-validator + if-no-files-found: error + retention-days: 1 + include-hidden-files: false diff --git a/.github/workflows/den-2777-main-build-diagnosis.yml b/.github/workflows/den-2777-main-build-diagnosis.yml new file mode 100644 index 00000000..30831d55 --- /dev/null +++ b/.github/workflows/den-2777-main-build-diagnosis.yml @@ -0,0 +1,94 @@ +name: DEN-2777 exact main build diagnosis + +on: + pull_request: + paths: + - .github/workflows/den-2777-main-build-diagnosis.yml + workflow_dispatch: + inputs: + candidate: + description: Full immutable zed-pkg/zed-cli commit + required: false + default: fa11a4d9d0bfa338a7e65220527188cc535e5b59 + +permissions: + contents: read + +concurrency: + group: den-2777-main-build-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + ZED_CLI_COMMIT: ${{ inputs.candidate || 'fa11a4d9d0bfa338a7e65220527188cc535e5b59' }} + +jobs: + diagnose: + name: compile / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + steps: + - name: Check out diagnosis harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact product candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable identity + run: | + set -euo pipefail + [[ "$ZED_CLI_COMMIT" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + + - name: Install Rust quality components + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Capture exact compiler evidence + id: compile + working-directory: product/zed-cli + run: | + set +e + mkdir -p ../../artifacts + cargo fmt --all --check > ../../artifacts/fmt-${RUNNER_OS}.log 2>&1 + fmt_status=$? + cargo check --locked --all-targets > ../../artifacts/check-${RUNNER_OS}.log 2>&1 + check_status=$? + cargo test --locked --no-run > ../../artifacts/test-no-run-${RUNNER_OS}.log 2>&1 + test_status=$? + cargo clippy --locked --all-targets -- -D warnings > ../../artifacts/clippy-${RUNNER_OS}.log 2>&1 + clippy_status=$? + printf '{"candidate":"%s","os":"%s","fmt":%s,"check":%s,"test_no_run":%s,"clippy":%s}\n' \ + "$ZED_CLI_COMMIT" "$RUNNER_OS" "$fmt_status" "$check_status" "$test_status" "$clippy_status" \ + > ../../artifacts/status-${RUNNER_OS}.json + cat ../../artifacts/status-${RUNNER_OS}.json + exit "$check_status" + + - name: Upload compiler evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den-2777-${{ runner.os }}-${{ env.ZED_CLI_COMMIT }} + path: artifacts/*-${{ runner.os }}.* + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/den-2788-orm-core.yml b/.github/workflows/den-2788-orm-core.yml new file mode 100644 index 00000000..969a03e9 --- /dev/null +++ b/.github/workflows/den-2788-orm-core.yml @@ -0,0 +1,100 @@ +name: DEN-2788 ORM core canary + +on: + pull_request: + paths: + - '.github/workflows/den-2788-orm-core.yml' + - 'proofs/den-2788-orm-core.json' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-2788-orm-core-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + exact-source: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out independent harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Read immutable source pin + id: pin + shell: bash + run: | + set -euo pipefail + jq -e ' + .schema_version == 1 and + .linear_issue == "DEN-2788" and + .source_repository == "zed-pkg/zed-orm-core" and + .source_pull_request == 2 and + (.source_commit | test("^[0-9a-f]{40}$")) and + .expected_package == "zed-pkg/zed-orm-core" and + .expected_schema == "zed_pkg" and + .release_policy == "private-git-zed-only" + ' proofs/den-2788-orm-core.json >/dev/null + echo "commit=$(jq -r .source_commit proofs/den-2788-orm-core.json)" >> "$GITHUB_OUTPUT" + + - name: Check out exact product source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-orm-core + ref: ${{ steps.pin.outputs.commit }} + path: source + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify exact source and release boundary + shell: bash + run: | + set -euo pipefail + expected="${{ steps.pin.outputs.commit }}" + test "$(git -C source rev-parse HEAD)" = "$expected" + grep -Fq 'org = "zed-pkg"' source/.zpkg.toml + grep -Fq 'name = "zed-orm-core"' source/.zpkg.toml + grep -Fq 'publish = false' source/Cargo.toml + grep -Fq '"schema": "zed_pkg"' source/shared-defs.lock.json + grep -Fq 'c8bdc06d74746acc6439f9527ebd02697fdf028b' source/shared-defs.lock.json + ! grep -Eq '^license\s*=' source/.zpkg.toml + ! grep -Fq '[targets.rust.native]' source/.zpkg.toml + ! grep -Eq '(gh[pousr]_[A-Za-z0-9]{20,}|lin_api_[A-Za-z0-9]{20,}|cfat_[A-Za-z0-9_-]{20,})' \ + proofs/den-2788-orm-core.json .github/workflows/den-2788-orm-core.yml source/.zpkg.toml + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + with: + components: rustfmt, clippy + + - name: Cache Rust build + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: source + shared-key: den-2788-orm-core-canary + + - name: Format + working-directory: source + run: cargo fmt --all -- --check + + - name: Strict Clippy for both surfaces + working-directory: source + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Test default read-only surface + working-directory: source + run: cargo test --all-targets + + - name: Test explicit read-write surface + working-directory: source + run: cargo test --all-targets --all-features + + - name: Compile documentation policy examples + working-directory: source + run: cargo test --doc diff --git a/.github/workflows/den-2788-web-readonly.yml b/.github/workflows/den-2788-web-readonly.yml new file mode 100644 index 00000000..b86ee988 --- /dev/null +++ b/.github/workflows/den-2788-web-readonly.yml @@ -0,0 +1,126 @@ +name: DEN-2788 web read-only canary + +on: + pull_request: + paths: + - '.github/workflows/den-2788-web-readonly.yml' + - 'proofs/den-2788-web-readonly.json' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-2788-web-readonly-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + exact-source: + runs-on: ubuntu-24.04 + timeout-minutes: 35 + steps: + - name: Check out independent harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Read immutable source graph + id: pin + shell: bash + run: | + set -euo pipefail + jq -e ' + .schema_version == 1 and + .linear_issue == "DEN-2788" and + .source_repository == "zed-pkg/zed-web-server.rs" and + .source_pull_request == 6 and + (.source_commit | test("^[0-9a-f]{40}$")) and + .interfaces_repository == "zed-pkg/zed-interfaces" and + (.interfaces_commit | test("^[0-9a-f]{40}$")) and + .expected_package == "zed-pkg/zed-web-server" and + .expected_schema == "zed_pkg" and + .expected_database_role == "zed_pkg__web_ro" and + .expected_secret == "zed-web-server-db-ro" + ' proofs/den-2788-web-readonly.json >/dev/null + echo "source=$(jq -r .source_commit proofs/den-2788-web-readonly.json)" >> "$GITHUB_OUTPUT" + echo "interfaces=$(jq -r .interfaces_commit proofs/den-2788-web-readonly.json)" >> "$GITHUB_OUTPUT" + + - name: Check out exact web source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-web-server.rs + ref: ${{ steps.pin.outputs.source }} + path: zed-web-server.rs + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact interfaces sibling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ steps.pin.outputs.interfaces }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify exact source graph and read-only boundary + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-web-server.rs rev-parse HEAD)" = "${{ steps.pin.outputs.source }}" + test "$(git -C zed-interfaces rev-parse HEAD)" = "${{ steps.pin.outputs.interfaces }}" + grep -Fq '"zed-pkg/zed-orm-core" = "^0.1.0"' zed-web-server.rs/.zpkg.toml + grep -Fq 'name: zed-web-server-db-ro' zed-web-server.rs/k8s/externalsecret.yaml + grep -Fq 'property: web_ro_url' zed-web-server.rs/k8s/externalsecret.yaml + grep -Fq 'EXPECTED_DATABASE_ROLE' zed-web-server.rs/k8s/base/deployment.yaml + grep -Fq 'value: zed_pkg__web_ro' zed-web-server.rs/k8s/base/deployment.yaml + grep -Fq 'default_transaction_read_only' zed-web-server.rs/src/server.rs + grep -Fq 'transaction_read_only != "on"' zed-web-server.rs/src/server.rs + ! grep -Fq 'features = ["read-write"]' zed-web-server.rs/Cargo.toml + ! grep -Eq '^sqlx\s*=' zed-web-server.rs/Cargo.toml + ! grep -Eq 'property: (api_rw_url|migrator_url)' zed-web-server.rs/k8s/externalsecret.yaml + ! grep -Eq '(gh[pousr]_[A-Za-z0-9]{20,}|lin_api_[A-Za-z0-9]{20,}|cfat_[A-Za-z0-9_-]{20,})' \ + proofs/den-2788-web-readonly.json \ + .github/workflows/den-2788-web-readonly.yml \ + zed-web-server.rs/.zpkg.toml \ + zed-web-server.rs/k8s/externalsecret.yaml + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + with: + components: rustfmt, clippy + + - name: Cache Rust build + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: zed-web-server.rs + shared-key: den-2788-web-readonly + + - name: Format + working-directory: zed-web-server.rs + run: cargo fmt --all -- --check + + - name: Check all targets + working-directory: zed-web-server.rs + run: cargo check --locked --all-targets + + - name: Strict Clippy + working-directory: zed-web-server.rs + run: cargo clippy --locked --all-targets -- -D warnings + + - name: Test + working-directory: zed-web-server.rs + run: cargo test --locked --all-targets + + - name: Documentation + working-directory: zed-web-server.rs + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --locked --no-deps + + - name: Release build + working-directory: zed-web-server.rs + run: cargo build --locked --release diff --git a/.github/workflows/den-2995-rust-enum-wire-canary.yml b/.github/workflows/den-2995-rust-enum-wire-canary.yml new file mode 100644 index 00000000..7497f950 --- /dev/null +++ b/.github/workflows/den-2995-rust-enum-wire-canary.yml @@ -0,0 +1,214 @@ +name: DEN-2995 Rust enum wire canary + +on: + pull_request: + paths: + - '.github/workflows/den-2995-rust-enum-wire-canary.yml' + - 'fixtures/den-2995-rust-enum-wire-source.json' + - 'docs/den-2995-rust-enum-wire-canary.md' + push: + branches: + - test/den-2995-rust-enum-wire-canary + paths: + - '.github/workflows/den-2995-rust-enum-wire-canary.yml' + - 'fixtures/den-2995-rust-enum-wire-source.json' + - 'docs/den-2995-rust-enum-wire-canary.md' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-2995-rust-enum-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + exact-source-contract: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Check out the independent test repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Read the immutable source pin + id: source + shell: bash + run: | + set -euo pipefail + repository="$(jq -r .repository fixtures/den-2995-rust-enum-wire-source.json)" + sha="$(jq -r .sha fixtures/den-2995-rust-enum-wire-source.json)" + expect_fixed="$(jq -r .expect_fixed fixtures/den-2995-rust-enum-wire-source.json)" + test "$repository" = 'ORESoftware/k8s-libs-and-shared-defs' + [[ "$sha" =~ ^[0-9a-f]{40}$ ]] + test "$expect_fixed" = true -o "$expect_fixed" = false + echo "repository=$repository" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "expect_fixed=$expect_fixed" >> "$GITHUB_OUTPUT" + + - name: Check out the exact source candidate + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ steps.source.outputs.repository }} + ref: ${{ steps.source.outputs.sha }} + path: candidate + persist-credentials: false + submodules: false + fetch-depth: 1 + + - name: Verify immutable source identity + shell: bash + env: + EXPECTED_SHA: ${{ steps.source.outputs.sha }} + run: | + set -euo pipefail + test "$(git -C candidate rev-parse HEAD)" = "$EXPECTED_SHA" + test -f candidate/pg-defs/src/generate.mjs + test -f candidate/pg-defs/generated/rust/Cargo.toml + test -f candidate/pg-defs/generated/rust/src/lib.rs + + - name: Install the pinned Node toolchain + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + + - name: Verify committed generator output is deterministic + working-directory: candidate + run: node pg-defs/src/generate.mjs --check + + - name: Inspect generator policy at the exact source head + shell: bash + env: + EXPECT_FIXED: ${{ steps.source.outputs.expect_fixed }} + run: | + set -euo pipefail + python3 - <<'PY' + import os + from pathlib import Path + + source = Path('candidate/pg-defs/src/generate.mjs').read_text() + block = source.split('function renderRustEnum(baseName, column) {', 1)[1] + block = block.split('\nfunction rustType(column) {', 1)[0] + expect_fixed = os.environ['EXPECT_FIXED'] == 'true' + + has_lowercase_rule = '#[serde(rename_all = "lowercase")]' in block + has_exact_rename_line = ( + 'lines.push(` #[serde(rename = ${JSON.stringify(value)})]`);' in block + ) + rename_is_conditional = ( + 'if (rustEnumVariant(value) !== pascal(value)) {' in block + ) + + if expect_fixed: + assert not has_lowercase_rule, 'enum-level lowercase rule still present' + assert has_exact_rename_line, 'generator does not emit exact rename attributes' + assert not rename_is_conditional, 'exact rename is still limited to leading-digit labels' + else: + assert has_lowercase_rule, 'pre-fix pin no longer reproduces the lowercase rule' + assert has_exact_rename_line, 'pre-fix generator shape changed unexpectedly' + assert rename_is_conditional, 'pre-fix pin unexpectedly emits exact names for every value' + PY + + - name: Add an external Rust consumer contract + shell: bash + run: | + set -euo pipefail + mkdir -p candidate/pg-defs/generated/rust/tests + cat > candidate/pg-defs/generated/rust/tests/den_2995_enum_wire.rs <<'RS' + use dd_pg_defs::AuthPublicKeysAlgorithm; + + #[test] + fn ssh_algorithm_serde_uses_the_exact_sql_labels() { + let ed25519 = AuthPublicKeysAlgorithm::SshEd25519; + let p256 = AuthPublicKeysAlgorithm::EcdsaSha2Nistp256; + + assert_eq!(serde_json::to_string(&ed25519).unwrap(), "\"ssh-ed25519\""); + assert_eq!(serde_json::to_string(&p256).unwrap(), "\"ecdsa-sha2-nistp256\""); + assert_eq!( + serde_json::from_str::("\"ssh-ed25519\"").unwrap(), + ed25519, + ); + assert_eq!( + serde_json::from_str::("\"ecdsa-sha2-nistp256\"") + .unwrap(), + p256, + ); + } + RS + + - name: Reproduce the defect or certify the repair + id: rust_contract + shell: bash + env: + EXPECT_FIXED: ${{ steps.source.outputs.expect_fixed }} + run: | + set -uo pipefail + rustc --version + cargo --version + set +e + cargo test \ + --manifest-path candidate/pg-defs/generated/rust/Cargo.toml \ + --test den_2995_enum_wire \ + > /tmp/den-2995-rust.log 2>&1 + status=$? + set -e + cat /tmp/den-2995-rust.log + + if [ "$EXPECT_FIXED" = true ]; then + test "$status" -eq 0 || { + echo '::error::The repaired source pin still corrupts SQL enum labels in Rust serde.' + exit 1 + } + echo 'observation=fixed' >> "$GITHUB_OUTPUT" + else + test "$status" -ne 0 || { + echo '::error::The pre-fix source pin unexpectedly passed; update the evidence pin.' + exit 1 + } + grep -E 'sshed25519|ssh-ed25519' /tmp/den-2995-rust.log >/dev/null + echo 'observation=defect-reproduced' >> "$GITHUB_OUTPUT" + fi + + - name: Record machine-readable evidence + shell: bash + env: + SOURCE_SHA: ${{ steps.source.outputs.sha }} + EXPECT_FIXED: ${{ steps.source.outputs.expect_fixed }} + OBSERVATION: ${{ steps.rust_contract.outputs.observation }} + run: | + set -euo pipefail + jq -n \ + --arg source_sha "$SOURCE_SHA" \ + --arg observation "$OBSERVATION" \ + --argjson expect_fixed "$EXPECT_FIXED" \ + '{ + schema_version: 1, + linear: "DEN-2995", + source_repository: "ORESoftware/k8s-libs-and-shared-defs", + source_sha: $source_sha, + expect_fixed: $expect_fixed, + observation: $observation, + contract: "exact SQL enum labels survive generated Rust serde" + }' | tee den-2995-evidence.json + + - name: Prove the canary is read-only and credential-free + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + workflow = Path('.github/workflows/den-2995-rust-enum-wire-canary.yml').read_text() + assert 'permissions:\n contents: read' in workflow + assert workflow.count('persist-credentials: false') == 2 + for forbidden in ( + 'contents' + ': write', + 'pull_request' + '_target:', + 'git' + ' push', + 'GITHUB_PAT', + 'LINEAR_API' + '_KEY', + 'CLOUDFLARE' + '_API_TOKEN', + 'AWS_SECRET' + '_ACCESS_KEY', + ): + assert forbidden not in workflow, forbidden + PY diff --git a/.github/workflows/den-3017-assembly-canary.yml b/.github/workflows/den-3017-assembly-canary.yml new file mode 100644 index 00000000..fb4df304 --- /dev/null +++ b/.github/workflows/den-3017-assembly-canary.yml @@ -0,0 +1,176 @@ +name: DEN-3017 zedignore union assembly canary + +on: + push: + branches: + - alexanderdmills/den-3017-zedignore-union + paths: + - .github/workflows/den-3017-assembly-canary.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-3017-assembly-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_STAGING_REF: f6f125fa2465325687a6a678ed3a6c9a958fda30 + ZED_CLI_BASE_REF: 1ab18fcb2ff884e82af4cac4513d7b983a23c84a + ZED_INTERFACES_REF: ab784d43c9add1bdd527f832a719fdcc3c9491c8 + +jobs: + assemble: + name: assemble and test / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - macos-15 + - windows-2025 + + steps: + - name: Require immutable inputs + run: | + set -euo pipefail + [[ "$ZED_CLI_STAGING_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_BASE_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out independent certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: e2e + persist-credentials: false + show-progress: false + + - name: Check out exact staged patch carrier + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_STAGING_REF }} + path: zed-cli + fetch-depth: 2 + persist-credentials: false + show-progress: false + + - name: Verify exact parent and patch carrier + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_STAGING_REF" + test "$(git -C zed-cli rev-parse HEAD^)" = "$ZED_CLI_BASE_REF" + test -f zed-cli/.github/den-3017-apply.py.gz.b64 + test "$(find zed-cli/.github -maxdepth 1 -name 'den-3017-apply*' | wc -l | tr -d ' ')" = 1 + + - name: Install validation toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Assemble the candidate from the exact patch carrier + working-directory: zed-cli + run: | + set -euo pipefail + python3 - <<'PY' + import base64, gzip, os + from pathlib import Path + payload = Path('.github/den-3017-apply.py.gz.b64').read_bytes() + Path(os.environ['RUNNER_TEMP'], 'den-3017-apply.py').write_bytes( + gzip.decompress(base64.b64decode(payload)) + ) + PY + PYTHONDONTWRITEBYTECODE=1 python3 "$RUNNER_TEMP/den-3017-apply.py" + cargo update -p zed-interfaces --precise "$ZED_INTERFACES_REF" + rm .github/workflows/den-3017-apply.yml .github/den-3017-apply.py.gz.b64 + cargo fmt --all + cargo fmt --all --check + git diff --check + grep -F "rev = \"$ZED_INTERFACES_REF\"" Cargo.toml + grep -F "$ZED_INTERFACES_REF" Cargo.lock + + - name: Run focused conflict-aware union tests + working-directory: zed-cli + run: | + set -euo pipefail + cargo test --locked --lib publish_rules + cargo test --locked --lib manifest_and_zedignore_exclusions_are_a_conflict_safe_union + cargo test --locked --lib same_source_negation_can_cancel_an_earlier_manifest_exclusion + cargo test --locked --lib polyglot_source_ignore_excludes_the_target_input + + - name: Run complete library test suite + working-directory: zed-cli + run: cargo test --locked --lib + + - name: Build the real CLI + working-directory: zed-cli + run: cargo build --locked --bin zed + + - name: Run real pack/archive certification + working-directory: zed-cli + run: | + set -euo pipefail + python3 - <<'PY' + import os, subprocess, sys + from pathlib import Path + root = Path.cwd() + binary = root / 'target' / 'debug' / ('zed.exe' if os.name == 'nt' else 'zed') + script = root.parent / 'e2e' / 'scripts' / 'publish_ignore_union.py' + subprocess.run([sys.executable, str(script), '--zed', str(binary)], check=True) + PY + + - name: Run strict lint gate + if: runner.os == 'Linux' + working-directory: zed-cli + run: cargo clippy --locked --lib -- -D warnings + + - name: Build evidence snapshot + if: runner.os == 'Linux' + run: | + set -euo pipefail + evidence="$RUNNER_TEMP/den-3017-evidence" + mkdir -p "$evidence" + git -C zed-cli diff --binary HEAD > "$evidence/den-3017.patch" + git -C zed-cli diff --name-status HEAD > "$evidence/changed-files.txt" + git -C zed-cli diff --stat HEAD > "$evidence/diff-stat.txt" + printf '%s\n' "$ZED_CLI_BASE_REF" > "$evidence/zed-cli-base.txt" + printf '%s\n' "$ZED_INTERFACES_REF" > "$evidence/zed-interfaces-candidate.txt" + tar --exclude=.git --exclude=target -czf "$evidence/zed-cli-den-3017-source.tar.gz" -C zed-cli . + sha256sum "$evidence"/* > "$evidence/SHA256SUMS" + + - name: Upload reproducible source evidence + if: runner.os == 'Linux' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den-3017-assembled-source-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/den-3017-evidence + if-no-files-found: error + retention-days: 14 + include-hidden-files: false + + gate: + name: all assembly platforms passed + if: always() + needs: assemble + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Enforce aggregate result + env: + MATRIX_RESULT: ${{ needs.assemble.result }} + run: | + set -euo pipefail + printf 'den-3017-assembly=%s\n' "$MATRIX_RESULT" + test "$MATRIX_RESULT" = success diff --git a/.github/workflows/den-3089-assembly.yml b/.github/workflows/den-3089-assembly.yml new file mode 100644 index 00000000..38cf4918 --- /dev/null +++ b/.github/workflows/den-3089-assembly.yml @@ -0,0 +1,111 @@ +name: DEN-3089 polyglot zedignore assembly + +on: + push: + branches: + - alexanderdmills/den-3089-polyglot-zedignore + paths: + - .github/workflows/den-3089-assembly.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-3089-assembly-${{ github.ref }} + cancel-in-progress: true + +env: + ZED_CLI_CARRIER: 307ebb70835ac69b695055f3fb8511b92e0c8278 + ZED_CLI_BASE: 0ada54a179835d2a64288887f8787d83a1eab1fe + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + assemble: + name: assemble, test, and export candidate source + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Check out independent test-org harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact product carrier + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_CARRIER }} + path: product/zed-cli + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Verify immutable carrier and base + shell: bash + run: | + set -euo pipefail + [[ "$ZED_CLI_CARRIER" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_BASE" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_CARRIER" + git -C product/zed-cli merge-base --is-ancestor "$ZED_CLI_BASE" "$ZED_CLI_CARRIER" + changed="$(git -C product/zed-cli diff --name-only "$ZED_CLI_BASE" "$ZED_CLI_CARRIER")" + expected=$'.github/workflows/den-3089-apply.yml\ntools/den-3089-apply.py.gz.b64.part00\ntools/den-3089-apply.py.gz.b64.part01\ntools/den-3089-apply.py.gz.b64.part02\ntools/den-3089-apply.py.gz.b64.part03\ntools/den-3089-apply.py.gz.b64.part04.00\ntools/den-3089-apply.py.gz.b64.part04.01\ntools/den-3089-apply.py.gz.b64.part04.02\ntools/den-3089-apply.py.gz.b64.part04.03\ntools/den-3089-apply.py.gz.b64.part04.04\ntools/den-3089-apply.py.gz.b64.part04.05\ntools/den-3089-apply.py.gz.b64.part04.06\ntools/den-3089-apply.py.gz.b64.part04.07\ntools/den-3089-apply.py.gz.b64.part04.08' + test "$changed" = "$expected" + + - name: Install stable Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Assemble candidate and run product gates + working-directory: product/zed-cli + shell: bash + env: + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + run: | + set -euo pipefail + cat tools/den-3089-apply.py.gz.b64.part* > "$RUNNER_TEMP/den-3089-apply.py.gz.b64" + echo "fbbe3cd11ad22561d41cb175815c8852dd00a950a0cdb248ba269c772a067de8 $RUNNER_TEMP/den-3089-apply.py.gz.b64" | sha256sum -c - + base64 --decode "$RUNNER_TEMP/den-3089-apply.py.gz.b64" | gzip --decompress > "$RUNNER_TEMP/den-3089-apply.py" + echo "a1c840717609ec210cb80e703fe88ddd161b2738b8bd7cd1d9d506dcc71c4d47 $RUNNER_TEMP/den-3089-apply.py" | sha256sum -c - + PYTHONDONTWRITEBYTECODE=1 python "$RUNNER_TEMP/den-3089-apply.py" + rm -f .github/workflows/den-3089-apply.yml tools/den-3089-apply.py.gz.b64.part* + git add -N src/publish_ignore.rs tests/polyglot_zedignore.rs + cargo fmt --all + cargo fmt --all -- --check + cargo test --all-targets --locked + cargo clippy --locked --lib --bin zed -- -D warnings + git diff --check "$ZED_CLI_BASE" + changed="$(git diff --name-only "$ZED_CLI_BASE")" + expected=$'docs/publish-ignore.md\nsrc/lib.rs\nsrc/ops_entry.rs\nsrc/pack.rs\nsrc/pack_guard.rs\nsrc/pack_inputs.rs\nsrc/publish_ignore.rs\ntests/polyglot_zedignore.rs' + test "$changed" = "$expected" + + - name: Export reproducible candidate source and patch + shell: bash + run: | + set -euo pipefail + evidence="$RUNNER_TEMP/den-3089-evidence" + mkdir -p "$evidence" + git -C product/zed-cli diff --binary "$ZED_CLI_BASE" > "$evidence/den-3089.patch" + git -C product/zed-cli diff --name-status "$ZED_CLI_BASE" > "$evidence/changed-files.txt" + git -C product/zed-cli diff --stat "$ZED_CLI_BASE" > "$evidence/diff-stat.txt" + git -C product/zed-cli status --short > "$evidence/status.txt" + printf '%s\n' "$ZED_CLI_BASE" > "$evidence/base-sha.txt" + printf '%s\n' "$ZED_CLI_CARRIER" > "$evidence/carrier-sha.txt" + tar --exclude=.git --exclude=target -czf "$evidence/zed-cli-den-3089-source.tar.gz" -C product/zed-cli . + sha256sum "$evidence"/* > "$evidence/SHA256SUMS" + + - name: Upload verified candidate source + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den-3089-verified-source-${{ env.ZED_CLI_CARRIER }} + path: ${{ runner.temp }}/den-3089-evidence + if-no-files-found: error + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/den-3089-exact-head.yml b/.github/workflows/den-3089-exact-head.yml new file mode 100644 index 00000000..acd17499 --- /dev/null +++ b/.github/workflows/den-3089-exact-head.yml @@ -0,0 +1,129 @@ +name: DEN-3089 exact-head polyglot zedignore certification + +on: + push: + branches: + - alexanderdmills/den-3089-polyglot-zedignore + paths: + - .github/workflows/den-3089-exact-head.yml + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: den-3089-exact-head-${{ github.ref }} + cancel-in-progress: true + +env: + PRODUCT_REPOSITORY: zed-pkg/zed-cli + PRODUCT_BRANCH: alexanderdmills/den-3089-polyglot-zedignore + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + resolve: + name: resolve immutable product head + runs-on: ubuntu-24.04 + outputs: + product_sha: ${{ steps.resolve.outputs.product_sha }} + steps: + - name: Resolve product branch once + id: resolve + shell: bash + run: | + set -euo pipefail + product_sha="$(git ls-remote "https://github.com/${PRODUCT_REPOSITORY}.git" "refs/heads/${PRODUCT_BRANCH}" | awk '{print $1}')" + [[ "$product_sha" =~ ^[0-9a-f]{40}$ ]] + echo "product_sha=$product_sha" >> "$GITHUB_OUTPUT" + echo "Resolved ${PRODUCT_REPOSITORY}@${product_sha}" + + certify: + name: certify ${{ matrix.os }} + needs: resolve + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - macos-15 + - windows-2025 + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + steps: + - name: Check out exact product candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ env.PRODUCT_REPOSITORY }} + ref: ${{ needs.resolve.outputs.product_sha }} + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable checkout + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "${{ needs.resolve.outputs.product_sha }}" + test ! -e .github/workflows/den-3089-apply.yml + test ! -e tools/den-3089-apply.py.gz.b64.part00 + + - name: Install stable Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Verify formatting + shell: bash + run: cargo fmt --all -- --check + + - name: Exercise real polyglot archive scenarios + shell: bash + env: + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + run: cargo test --locked --test polyglot_zedignore -- --nocapture + + - name: Run complete locked target suite + shell: bash + env: + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + run: cargo test --all-targets --locked + + - name: Reject lint regressions + shell: bash + env: + CARGO_NET_GIT_FETCH_WITH_CLI: "true" + run: cargo clippy --locked --lib --bin zed -- -D warnings + + record: + name: record exact-head certification + needs: + - resolve + - certify + runs-on: ubuntu-24.04 + steps: + - name: Check out test-org branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: alexanderdmills/den-3089-polyglot-zedignore + fetch-depth: 1 + persist-credentials: true + show-progress: false + + - name: Prove product head did not move and record success + shell: bash + run: | + set -euo pipefail + certified_sha="${{ needs.resolve.outputs.product_sha }}" + current_sha="$(git ls-remote "https://github.com/${PRODUCT_REPOSITORY}.git" "refs/heads/${PRODUCT_BRANCH}" | awk '{print $1}')" + test "$current_sha" = "$certified_sha" + mkdir -p .github + printf '{\n "ticket": "DEN-3089",\n "product_repository": "%s",\n "product_branch": "%s",\n "product_sha": "%s",\n "platforms": ["ubuntu-24.04", "macos-15", "windows-2025"],\n "workflow_run_id": "%s",\n "workflow_run_attempt": "%s",\n "verified": true\n}\n' \ + "$PRODUCT_REPOSITORY" "$PRODUCT_BRANCH" "$certified_sha" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" \ + > .github/den-3089-exact-head-success.json + git config user.name "alex-main-agent" + git config user.email "alex-main-agent@users.noreply.github.com" + git add .github/den-3089-exact-head-success.json + git commit -m "test(DEN-3089): record exact-head cross-platform certification" + git push origin "HEAD:${GITHUB_REF_NAME}" diff --git a/.github/workflows/den-3524-production-chain.yml b/.github/workflows/den-3524-production-chain.yml new file mode 100644 index 00000000..260445fb --- /dev/null +++ b/.github/workflows/den-3524-production-chain.yml @@ -0,0 +1,326 @@ +name: DEN-3524 production dependency-graph chain + +on: + pull_request: + paths: + - ".github/workflows/den-3524-production-chain.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-3524-production-chain-${{ github.ref }} + cancel-in-progress: true + +env: + SCHEMA_SHA: a1fb823890d4a36dfab67c311f0d728d7b22c1c9 + SCHEMA_REGISTRY_BLOB: c0869ca29c10e1c77bd9d9b8236fc61eac826ab9 + SCHEMA_VISIBILITY_BLOB: 8612f037dce7de6d7db66ee96db7996b33b32ea9 + ORM_API_SHA: c3d486a1519381276fbec02aa25247f542924443 + ORM_WEB_SHA: 34333b5ea5c0ee7862bfe0366fe9fa2b1c4bd210 + API_SHA: 0ab2e31bfedf102dc61a11f6b246e01f30dda49d + WEB_SHA: 5b486e1b6956ef63ef49f6391dbec13274b8165d + WEB_MAIN_PARENT: 8889cc4978d732e843866085556ef330bf008721 + DATABASE_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres + +jobs: + certify: + name: schema → ORM → API → web exact-head certification + runs-on: ubuntu-24.04 + timeout-minutes: 25 + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d postgres" + --health-interval 3s + --health-timeout 3s + --health-retries 30 + + steps: + - name: Checkout certification authority + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + path: zed-pkg-e2e + persist-credentials: false + + - name: Checkout exact ORM release graph + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: zed-pkg/zed-lib-core + ref: ${{ env.ORM_WEB_SHA }} + path: zed-lib-core + fetch-depth: 0 + persist-credentials: false + + - name: Checkout exact API production tree + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: zed-pkg/zed-api-server.rs + ref: ${{ env.API_SHA }} + path: zed-api-server.rs + persist-credentials: false + + - name: Checkout exact web production tree + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: zed-pkg/zed-web-server.rs + ref: ${{ env.WEB_SHA }} + path: zed-web-server.rs + fetch-depth: 0 + persist-credentials: false + + - name: Verify immutable commit and blob provenance + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-lib-core rev-parse HEAD)" = "$ORM_WEB_SHA" + test "$(git -C zed-api-server.rs rev-parse HEAD)" = "$API_SHA" + test "$(git -C zed-web-server.rs rev-parse HEAD)" = "$WEB_SHA" + + git -C zed-lib-core cat-file -e "${ORM_API_SHA}^{commit}" + git -C zed-lib-core merge-base --is-ancestor "$ORM_API_SHA" "$ORM_WEB_SHA" + git -C zed-web-server.rs cat-file -e "${WEB_MAIN_PARENT}^{commit}" + git -C zed-web-server.rs merge-base --is-ancestor "$WEB_MAIN_PARENT" "$WEB_SHA" + + test "$(git -C zed-lib-core rev-parse "${ORM_API_SHA}:src/rust-orm/sql/registry.sql")" = "$SCHEMA_REGISTRY_BLOB" + test "$(git -C zed-lib-core rev-parse "${ORM_API_SHA}:src/rust-orm/sql/2026-08-11-public-visibility-is-permanent.sql")" = "$SCHEMA_VISIBILITY_BLOB" + test "$(git -C zed-lib-core rev-parse "${ORM_WEB_SHA}:src/rust-orm/sql/registry.sql")" = "$SCHEMA_REGISTRY_BLOB" + test "$(git -C zed-lib-core rev-parse "${ORM_WEB_SHA}:src/rust-orm/sql/2026-08-11-public-visibility-is-permanent.sql")" = "$SCHEMA_VISIBILITY_BLOB" + + - name: Verify schema, ORM, API, and web pin chain + shell: bash + run: | + set -euo pipefail + mkdir -p evidence + git -C zed-lib-core show "${ORM_API_SHA}:shared-defs.lock.json" > evidence/orm-api-shared-defs.lock.json + git -C zed-lib-core show "${ORM_WEB_SHA}:shared-defs.lock.json" > evidence/orm-web-shared-defs.lock.json + + python3 - <<'PY' + import json + import os + import pathlib + import tomllib + + root = pathlib.Path('.') + expected = { + 'revision': os.environ['SCHEMA_SHA'], + 'registry_blob_sha': os.environ['SCHEMA_REGISTRY_BLOB'], + 'visibility_immutability_revision': os.environ['SCHEMA_SHA'], + 'visibility_immutability_blob_sha': os.environ['SCHEMA_VISIBILITY_BLOB'], + } + for name in ('orm-api-shared-defs.lock.json', 'orm-web-shared-defs.lock.json'): + lock = json.loads((root / 'evidence' / name).read_text()) + for key, value in expected.items(): + assert lock[key] == value, (name, key, lock[key], value) + assert lock['vendored_copy'] == 'src/rust-orm/sql/registry.sql' + assert lock['vendored_visibility_immutability_migration'] == ( + 'src/rust-orm/sql/2026-08-11-public-visibility-is-permanent.sql' + ) + + api_manifest = tomllib.loads((root / 'zed-api-server.rs' / 'Cargo.toml').read_text()) + api_spec = api_manifest['dependencies']['zed-orm-core'] + assert api_spec['git'] == 'https://github.com/zed-pkg/zed-lib-core.git' + assert api_spec['rev'] == os.environ['ORM_API_SHA'] + assert set(api_spec['features']) == {'read-write', 'migrate'} + api_lock = (root / 'zed-api-server.rs' / 'Cargo.lock').read_text() + assert ( + f"git+https://github.com/zed-pkg/zed-lib-core.git?rev={os.environ['ORM_API_SHA']}#" + f"{os.environ['ORM_API_SHA']}" + ) in api_lock + + web_manifest = tomllib.loads((root / 'zed-web-server.rs' / 'Cargo.toml').read_text()) + web_spec = web_manifest['dependencies']['zed-orm-core'] + assert web_spec['git'] == 'https://github.com/zed-pkg/zed-lib-core.git' + assert web_spec['rev'] == os.environ['ORM_WEB_SHA'] + assert 'features' not in web_spec, 'web must retain the default read-only ORM surface' + web_lock = (root / 'zed-web-server.rs' / 'Cargo.lock').read_text() + assert ( + f"git+https://github.com/zed-pkg/zed-lib-core.git?rev={os.environ['ORM_WEB_SHA']}#" + f"{os.environ['ORM_WEB_SHA']}" + ) in web_lock + + for workflow in ( + root / 'zed-web-server.rs' / '.github' / 'workflows' / 'ci.yml', + root / 'zed-web-server.rs' / '.github' / 'workflows' / 'publish-container.yml', + ): + content = workflow.read_text() + assert f"ZED_LIB_CORE_SHA: {os.environ['ORM_WEB_SHA']}" in content + PY + + git -C zed-lib-core show "${ORM_API_SHA}:src/rust-orm/sql/2026-08-11-public-visibility-is-permanent.sql" \ + | grep -Fq "using errcode = 'ZD003'" + git -C zed-lib-core show "${ORM_WEB_SHA}:src/rust-orm/sql/2026-08-11-public-visibility-is-permanent.sql" \ + | grep -Fq "using errcode = 'ZD003'" + + - name: Prove the exact vendored migration on PostgreSQL 17 + shell: bash + run: | + set -euo pipefail + git -C zed-lib-core show "${ORM_API_SHA}:src/rust-orm/sql/registry.sql" \ + | psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q + for _ in 1 2; do + git -C zed-lib-core show "${ORM_API_SHA}:src/rust-orm/sql/2026-08-11-public-visibility-is-permanent.sql" \ + | psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q + done + + psql "$DATABASE_URL" -v ON_ERROR_STOP=1 <<'SQL' + do $$ + declare + org_id uuid; + public_id uuid; + promotable_id uuid; + old_private_id uuid; + popular_private_id uuid; + begin + insert into zed_orgs (slug, name) + values ('den-3524-cert', 'DEN-3524 certification') + returning id into org_id; + + insert into zed_packages (org_id, name, visibility) + values (org_id, 'public-package', 'public') + returning id into public_id; + + begin + update zed_packages set visibility = 'private' where id = public_id; + raise exception 'public to private unexpectedly succeeded'; + exception when sqlstate 'ZD003' then + null; + end; + + begin + update zed_packages set visibility = 'internal' where id = public_id; + raise exception 'public to internal unexpectedly succeeded'; + exception when sqlstate 'ZD003' then + null; + end; + + if (select visibility from zed_packages where id = public_id) <> 'public' then + raise exception 'failed ZD003 transitions changed durable visibility'; + end if; + + insert into zed_packages (org_id, name, visibility, download_count) + values (org_id, 'promotable-package', 'private', 0) + returning id into promotable_id; + update zed_packages set visibility = 'public' where id = promotable_id; + if (select visibility from zed_packages where id = promotable_id) <> 'public' then + raise exception 'valid private to public promotion failed'; + end if; + + insert into zed_packages (org_id, name, visibility, created_at) + values (org_id, 'old-private-package', 'private', now() - interval '11 days') + returning id into old_private_id; + begin + update zed_packages set visibility = 'public' where id = old_private_id; + raise exception 'old private package unexpectedly became public'; + exception when sqlstate 'ZD001' then + null; + end; + + insert into zed_packages (org_id, name, visibility, download_count) + values (org_id, 'popular-private-package', 'private', 51) + returning id into popular_private_id; + begin + update zed_packages set visibility = 'public' where id = popular_private_id; + raise exception 'popular private package unexpectedly became public'; + exception when sqlstate 'ZD002' then + null; + end; + end; + $$; + + select case + when p.proname = 'zed_enforce_package_visibility_transition' + and t.tgname = 'zed_packages_visibility_guard' + then 'visibility trigger bound to certified function' + else 'unexpected trigger binding' + end as certification + from pg_trigger t + join pg_proc p on p.oid = t.tgfoid + where t.tgname = 'zed_packages_visibility_guard'; + SQL + + - name: Publish machine-readable exact-head evidence + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import json + import os + import pathlib + import subprocess + + def sha(path: str) -> str: + return subprocess.check_output(['git', '-C', path, 'rev-parse', 'HEAD'], text=True).strip() + + evidence = { + 'schema': { + 'repository': 'ORESoftware/k8s-libs-and-shared-defs', + 'commit': os.environ['SCHEMA_SHA'], + 'registry_blob': os.environ['SCHEMA_REGISTRY_BLOB'], + 'visibility_migration_blob': os.environ['SCHEMA_VISIBILITY_BLOB'], + 'access': 'verified through exact public ORM vendored blobs; private source is not mirrored', + }, + 'orm': { + 'repository': 'zed-pkg/zed-lib-core', + 'api_revision': os.environ['ORM_API_SHA'], + 'web_revision': os.environ['ORM_WEB_SHA'], + 'checked_out_revision': sha('zed-lib-core'), + 'api_revision_is_ancestor_of_web_revision': True, + }, + 'api': { + 'repository': 'zed-pkg/zed-api-server.rs', + 'commit': sha('zed-api-server.rs'), + 'orm_revision': os.environ['ORM_API_SHA'], + 'orm_features': ['read-write', 'migrate'], + }, + 'web': { + 'repository': 'zed-pkg/zed-web-server.rs', + 'commit': sha('zed-web-server.rs'), + 'main_parent': os.environ['WEB_MAIN_PARENT'], + 'orm_revision': os.environ['ORM_WEB_SHA'], + 'orm_surface': 'read-only default', + }, + 'postgresql': { + 'version': 17, + 'migration_replays': 2, + 'public_to_private': 'rejected with ZD003', + 'public_to_internal': 'rejected with ZD003', + 'valid_private_to_public': 'accepted', + 'old_private_to_public': 'rejected with ZD001', + 'popular_private_to_public': 'rejected with ZD002', + }, + 'github': { + 'repository': os.environ.get('GITHUB_REPOSITORY'), + 'run_id': os.environ.get('GITHUB_RUN_ID'), + 'run_attempt': os.environ.get('GITHUB_RUN_ATTEMPT'), + 'source_sha': os.environ.get('GITHUB_SHA'), + }, + } + target = pathlib.Path('evidence/den-3524-production-chain.json') + target.write_text(json.dumps(evidence, indent=2, sort_keys=True) + '\n') + summary = pathlib.Path(os.environ['GITHUB_STEP_SUMMARY']) + with summary.open('a') as stream: + stream.write('## DEN-3524 exact-head production chain\n\n') + stream.write(f"- Schema: `{os.environ['SCHEMA_SHA']}`\n") + stream.write(f"- ORM for API: `{os.environ['ORM_API_SHA']}`\n") + stream.write(f"- ORM for web: `{os.environ['ORM_WEB_SHA']}`\n") + stream.write(f"- API: `{os.environ['API_SHA']}`\n") + stream.write(f"- Web: `{os.environ['WEB_SHA']}`\n") + stream.write('- PostgreSQL 17: ZD001/ZD002/ZD003 transition matrix passed\n') + PY + cat evidence/den-3524-production-chain.json + + - name: Upload exact-head evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: den-3524-production-chain-${{ github.run_id }} + path: evidence/den-3524-production-chain.json + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/den2725-root-dispatch-certification.yml b/.github/workflows/den2725-root-dispatch-certification.yml new file mode 100644 index 00000000..297a922c --- /dev/null +++ b/.github/workflows/den2725-root-dispatch-certification.yml @@ -0,0 +1,198 @@ +name: DEN-2725 root GitOps dispatch certification + +on: + push: + branches: + - agent/den-2725-root-dispatch-certification + pull_request: + paths: + - ".github/workflows/den2725-root-dispatch-certification.yml" + - "scripts/den2725_root_dispatch_hardening.py" + - "scripts/den2725_root_dispatch_lint_fix.py" + - "docs/den2725-root-dispatch-certification.md" + +permissions: + contents: read + +concurrency: + group: den-2725-root-dispatch-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + certify: + name: ${{ matrix.name }} + strategy: + fail-fast: false + matrix: + include: + - name: Linux + os: ubuntu-24.04 + python: python3 + exe: target/release/zed + suffix: "" + - name: macOS + os: macos-15 + python: python3 + exe: target/release/zed + suffix: "" + - name: Windows + os: windows-2025 + python: python + exe: target/release/zed.exe + suffix: .exe + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + defaults: + run: + shell: bash + env: + PRODUCT_SHA: 9da2f76606e5c6dad326889cb09ee7b692358823 + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + steps: + - name: Check out exact certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.PRODUCT_SHA }} + path: product + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable inputs + run: | + set -euo pipefail + test "$(git -C product rev-parse HEAD)" = "$PRODUCT_SHA" + test "$(git -C harness rev-parse HEAD)" = "${{ github.sha }}" + test -z "$(git -C product status --short)" + test -z "$(git -C harness status --short)" + + - name: Apply the exact reviewed hardening + working-directory: product + run: | + set -euo pipefail + ${{ matrix.python }} ../harness/scripts/den2725_root_dispatch_hardening.py + ${{ matrix.python }} ../harness/scripts/den2725_root_dispatch_lint_fix.py + cargo fmt --all + ${{ matrix.python }} - <<'PY' + import subprocess + + changed = { + line.strip() + for line in subprocess.check_output( + ["git", "diff", "--name-only"], text=True + ).splitlines() + if line.strip() + } + expected = { + "src/external_subcommands.rs", + "docs/gitops-validator.md", + "tests/external_gitops_dispatch.rs", + } + if changed != expected: + raise SystemExit(f"unexpected hardening delta: {sorted(changed)!r}") + PY + git diff --check + + - name: Run source, package, and release gates + working-directory: product + run: | + set -euo pipefail + rustup component add rustfmt clippy + cargo fmt --all -- --check + bash scripts/validate-zed-package-graph.sh + cargo test --locked --lib external_subcommands + cargo test --locked --test external_gitops_dispatch + cargo clippy --locked --all-targets -- -D warnings + cargo build --release --locked --bin zed --bin zed-gitops + test -x "${{ matrix.exe }}" + test -x "target/release/zed-gitops${{ matrix.suffix }}" + "${{ matrix.exe }}" gitops validate --help >/dev/null + + - name: Certify black-box dispatch boundaries + working-directory: product + env: + EXE_SUFFIX: ${{ matrix.suffix }} + ZED_EXE: ${{ matrix.exe }} + run: | + set -euo pipefail + unset ZED_PKG_TOKEN || true + cat > "$RUNNER_TEMP/dispatch_probe.rs" <<'RS' + use std::{env, fs}; + + fn main() { + let output = env::var("ZED_DISPATCH_EVIDENCE").expect("evidence path"); + let arguments = env::args().skip(1).collect::>().join("\u{1f}"); + let token = env::var("ZED_PKG_TOKEN").unwrap_or_default(); + let command = env::var("ZED_EXTERNAL_SUBCOMMAND").unwrap_or_default(); + fs::write( + output, + format!("args={arguments}\ntoken={token}\ncommand={command}\n"), + ) + .expect("write evidence"); + std::process::exit(23); + } + RS + rustc "$RUNNER_TEMP/dispatch_probe.rs" -o "target/release/zed-probe${EXE_SUFFIX}" + cp "target/release/zed-probe${EXE_SUFFIX}" "target/release/zed-install${EXE_SUFFIX}" + + run_probe() { + expected_args="$1" + expected_token="$2" + shift 2 + evidence="$RUNNER_TEMP/dispatch-evidence.txt" + rm -f "$evidence" + set +e + ZED_DISPATCH_EVIDENCE="$evidence" "$ZED_EXE" "$@" + status=$? + set -e + test "$status" -eq 23 + grep -Fqx "args=$expected_args" "$evidence" + grep -Fqx "token=$expected_token" "$evidence" + grep -Fqx "command=probe" "$evidence" + } + + run_probe "alpha" "fixture-before" --token fixture-before probe alpha + run_probe "beta" "fixture-after" probe beta --token fixture-after + separator=$(printf '\037') + run_probe "gamma${separator}--${separator}--token${separator}child-owned" "" \ + probe gamma -- --token child-owned + + builtin_evidence="$RUNNER_TEMP/builtin-evidence.txt" + rm -f "$builtin_evidence" + ZED_DISPATCH_EVIDENCE="$builtin_evidence" "$ZED_EXE" install --help >/dev/null + test ! -e "$builtin_evidence" + + - name: Emit bounded materialization evidence + if: always() + working-directory: product + run: | + ${{ matrix.python }} - <<'PY' + import base64 + import hashlib + from pathlib import Path + + for path in [ + Path("src/external_subcommands.rs"), + Path("docs/gitops-validator.md"), + Path("tests/external_gitops_dispatch.rs"), + ]: + data = path.read_bytes() + label = path.name.upper().replace(".", "_").replace("-", "_") + encoded = base64.b64encode(data).decode("ascii") + print(f"BEGIN_{label}_BASE64") + for offset in range(0, len(encoded), 76): + print(encoded[offset : offset + 76]) + print(f"END_{label}_BASE64") + print(f"SHA256_{label}={hashlib.sha256(data).hexdigest()}") + PY diff --git a/.github/workflows/direct-source-render.yml b/.github/workflows/direct-source-render.yml new file mode 100644 index 00000000..59014a9a --- /dev/null +++ b/.github/workflows/direct-source-render.yml @@ -0,0 +1,158 @@ +name: exact direct-source snapshot Kustomize render + +on: + pull_request: + paths: + - .github/workflows/direct-source-render.yml + - scripts/install_pinned_kustomize.py + - scripts/direct_source_render_canary.py + - scripts/check_snapshot_endpoint_policy.py + - scripts/test_snapshot_endpoint_policy.py + - docs/direct-source-render.md + - fixtures/direct-source/fabrication-server.rs/** + workflow_dispatch: + inputs: + cluster_commit: + description: Exact ORESoftware/k8s-cluster catalog candidate + required: false + default: 0c71eed05f8be50ac22ebe73715ded4a871234a8 + type: string + child_commit: + description: Exact private source commit represented by the snapshot + required: false + default: cea4a3c772012e1f2f87050bac24e911b8e2e577 + type: string + +permissions: + contents: read + +concurrency: + group: direct-source-render-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + K8S_CLUSTER_COMMIT: ${{ inputs.cluster_commit || '0c71eed05f8be50ac22ebe73715ded4a871234a8' }} + FABRICATION_SERVER_COMMIT: ${{ inputs.child_commit || 'cea4a3c772012e1f2f87050bac24e911b8e2e577' }} + SOURCE_SNAPSHOT: fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577 + +jobs: + render: + name: direct source snapshot render / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + + steps: + - name: Check out independent test harness and source snapshot + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact cluster catalog without child worktrees + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ORESoftware/k8s-cluster + ref: ${{ env.K8S_CLUSTER_COMMIT }} + path: product/k8s-cluster + fetch-depth: 1 + submodules: false + persist-credentials: false + show-progress: false + + - name: Install Python 3.12 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Verify immutable cluster checkout and all test syntax + run: | + set -euo pipefail + [[ "$K8S_CLUSTER_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ "$FABRICATION_SERVER_COMMIT" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/k8s-cluster rev-parse HEAD)" = "$K8S_CLUSTER_COMMIT" + test -f "$SOURCE_SNAPSHOT/source.json" + python -m py_compile \ + scripts/install_pinned_kustomize.py \ + scripts/direct_source_render_canary.py \ + scripts/check_snapshot_endpoint_policy.py \ + scripts/test_snapshot_endpoint_policy.py \ + "$SOURCE_SNAPSHOT/scripts/ci/check_k8s_network_contract.py" \ + "$SOURCE_SNAPSHOT/scripts/ci/test_check_k8s_network_contract.py" + python scripts/test_snapshot_endpoint_policy.py + ( + cd "$SOURCE_SNAPSHOT" + python scripts/ci/test_check_k8s_network_contract.py + python scripts/ci/check_k8s_network_contract.py + ) + rm -rf scripts/__pycache__ "$SOURCE_SNAPSHOT/scripts/ci/__pycache__" + + - name: Install checksum-pinned Kustomize v5.8.1 + run: | + set -euo pipefail + mkdir -p artifacts + kustomize="$(python scripts/install_pinned_kustomize.py \ + --destination "$RUNNER_TEMP/pinned-kustomize" \ + --metadata "artifacts/kustomize-install-${RUNNER_OS}-${RUNNER_ARCH}.json")" + test -f "$kustomize" + echo "KUSTOMIZE_BINARY=$kustomize" >> "$GITHUB_ENV" + + - name: Verify snapshot endpoint and NetworkPolicy parity independently + run: | + set -euo pipefail + python scripts/check_snapshot_endpoint_policy.py \ + --deployment "$SOURCE_SNAPSHOT/k8s/deployment.yaml" \ + --network-policy "$SOURCE_SNAPSHOT/k8s/networkpolicy.yaml" \ + --evidence "artifacts/snapshot-endpoint-policy-${RUNNER_OS}-${RUNNER_ARCH}.json" + + - name: Verify catalog provenance and render exact source snapshot twice + run: | + set -euo pipefail + python scripts/direct_source_render_canary.py \ + --cluster product/k8s-cluster \ + --snapshot "$SOURCE_SNAPSHOT" \ + --source-metadata "$SOURCE_SNAPSHOT/source.json" \ + --cluster-commit "$K8S_CLUSTER_COMMIT" \ + --child-commit "$FABRICATION_SERVER_COMMIT" \ + --kustomize "$KUSTOMIZE_BINARY" \ + --kustomize-metadata "artifacts/kustomize-install-${RUNNER_OS}-${RUNNER_ARCH}.json" \ + --rendered "artifacts/fabrication-render-${RUNNER_OS}-${RUNNER_ARCH}.yaml" \ + --evidence "artifacts/direct-source-render-${RUNNER_OS}-${RUNNER_ARCH}.json" + rm -rf scripts/__pycache__ + + - name: Require pristine cluster and harness checkouts + run: | + set -euo pipefail + git -C product/k8s-cluster diff --exit-code + git -C product/k8s-cluster diff --cached --exit-code + cluster_status="$(git -C product/k8s-cluster status --porcelain --untracked-files=all)" + if [[ -n "$cluster_status" ]]; then + printf 'unexpected cluster drift:\n%s\n' "$cluster_status" >&2 + exit 1 + fi + harness_status="$(git status --porcelain --untracked-files=all | grep -vE '^\?\? (artifacts|product)/' || true)" + if [[ -n "$harness_status" ]]; then + printf 'unexpected harness drift:\n%s\n' "$harness_status" >&2 + exit 1 + fi + + - name: Upload exact-source snapshot render evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: direct-source-snapshot-render-${{ runner.os }}-${{ runner.arch }}-${{ env.K8S_CLUSTER_COMMIT }}-${{ env.FABRICATION_SERVER_COMMIT }} + path: | + artifacts/kustomize-install-${{ runner.os }}-${{ runner.arch }}.json + artifacts/snapshot-endpoint-policy-${{ runner.os }}-${{ runner.arch }}.json + artifacts/direct-source-render-${{ runner.os }}-${{ runner.arch }}.json + artifacts/fabrication-render-${{ runner.os }}-${{ runner.arch }}.yaml + if-no-files-found: error + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/dispatch-hhm-test-fleet-app.yml b/.github/workflows/dispatch-hhm-test-fleet-app.yml new file mode 100644 index 00000000..997442b6 --- /dev/null +++ b/.github/workflows/dispatch-hhm-test-fleet-app.yml @@ -0,0 +1,111 @@ +name: dispatch Hacker House Medellin test fleet through GitHub App + +on: + pull_request: + paths: + - .github/hhm-fleet-dispatch/START + - .github/workflows/dispatch-hhm-test-fleet-app.yml + - .github/workflows/apply-test-org-fleet.yml + - scripts/bootstrap-test-org-fleet.mjs + - scripts/validate-test-org-fleet.mjs + - tests/test-org-fleet.test.mjs + push: + branches: [main] + paths: + - .github/hhm-fleet-dispatch/START + - .github/workflows/dispatch-hhm-test-fleet-app.yml + +permissions: + actions: write + contents: read + +concurrency: + group: dispatch-hhm-test-fleet-app + cancel-in-progress: false + +jobs: + validate: + name: Validate scoped dispatch contract + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate one-shot authorization and immutable fleet contract + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < .github/hhm-fleet-dispatch/START)" = "APPLY_HHM_TEST_FLEET_V1" + node --check scripts/bootstrap-test-org-fleet.mjs + node --check scripts/validate-test-org-fleet.mjs + node scripts/validate-test-org-fleet.mjs + node --test tests/test-org-fleet.test.mjs + grep -F -- '- hacker-house-medellin-test' .github/workflows/apply-test-org-fleet.yml + grep -F -- '- github-app' .github/workflows/apply-test-org-fleet.yml + grep -F -- 'confirm=APPLY_TEST_FLEET' .github/workflows/dispatch-hhm-test-fleet-app.yml + + preflight-and-dispatch: + name: Preflight App and dispatch exact organization + if: github.event_name == 'push' + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Require complete GitHub App configuration + shell: bash + env: + FLEET_APP_CLIENT_ID: ${{ vars.FLEET_APP_CLIENT_ID }} + FLEET_APP_PRIVATE_KEY: ${{ secrets.FLEET_APP_PRIVATE_KEY }} + run: | + set -euo pipefail + if [[ -z "$FLEET_APP_CLIENT_ID" || -z "$FLEET_APP_PRIVATE_KEY" ]]; then + echo '::error::The fleet GitHub App configuration is incomplete.' + exit 1 + fi + + - name: Mint exact-organization installation token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.FLEET_APP_CLIENT_ID }} + private-key: ${{ secrets.FLEET_APP_PRIVATE_KEY }} + owner: hacker-house-medellin-test + permission-administration: write + permission-contents: write + permission-metadata: read + permission-pull-requests: write + permission-workflows: write + + - name: Verify exact target visibility without mutation + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + gh api orgs/hacker-house-medellin-test --silent + count="$(gh api --paginate 'orgs/hacker-house-medellin-test/repos?type=all&per_page=100' --jq '.[].name' | wc -l | tr -d '[:space:]')" + { + echo '## Hacker House Medellin test fleet App preflight' + echo + echo "Visible repositories before dispatch: $count" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Dispatch the merged exact-organization executor + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + set -euo pipefail + gh workflow run apply-test-org-fleet.yml \ + --repo "$GH_REPO" \ + --ref main \ + -f organization=hacker-house-medellin-test \ + -f repository= \ + -f authentication=github-app \ + -f confirm=APPLY_TEST_FLEET + echo 'Dispatched the exact Hacker House Medellin test fleet through GitHub App authentication.' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/dispatch-test-org-fleet-once.yml b/.github/workflows/dispatch-test-org-fleet-once.yml new file mode 100644 index 00000000..878ffb16 --- /dev/null +++ b/.github/workflows/dispatch-test-org-fleet-once.yml @@ -0,0 +1,78 @@ +# One-time control-plane trigger. Remove after the fixed fleet apply has started. +name: dispatch test organization fleet once + +on: + push: + branches: + - main + paths: + - bootstrap/execute-test-org-fleet-v2.once + +permissions: + actions: write + contents: read + +concurrency: + group: dispatch-test-org-fleet-once-v2 + cancel-in-progress: false + +jobs: + dispatch: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + STALE_RUN_IDS: "30972717804 30973196337" + steps: + - name: Check out the execution marker + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate the one-time authorization marker + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/execute-test-org-fleet-v2.once)" = "APPLY_TEST_FLEET_V3" + + - name: Wait for superseded fleet writers to stop + shell: bash + run: | + set -euo pipefail + read -r -a run_ids <<< "$STALE_RUN_IDS" + + for run_id in "${run_ids[@]}"; do + endpoint="repos/$GH_REPO/actions/runs/$run_id" + stopped=false + for _ in $(seq 1 150); do + status="$(gh api "$endpoint" --jq '.status')" + conclusion="$(gh api "$endpoint" --jq '.conclusion // ""')" + if [[ "$status" == "completed" ]]; then + echo "Superseded run $run_id is complete: ${conclusion:-unknown}" + stopped=true + break + fi + sleep 2 + done + + if [[ "$stopped" != true ]]; then + echo "::error::Superseded run $run_id is still active" + exit 1 + fi + done + + - name: Dispatch the fixed all-organization fleet executor + shell: bash + run: | + set -euo pipefail + gh workflow run apply-test-org-fleet.yml \ + --repo "$GH_REPO" \ + --ref main \ + -f organization=all \ + -f authentication=pat \ + -f repository= \ + -f confirm=APPLY_TEST_FLEET + + echo "Dispatched the fixed apply-test-org-fleet.yml for all in-scope organizations." >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/durable-first-install-dart.yml b/.github/workflows/durable-first-install-dart.yml new file mode 100644 index 00000000..cf4193d0 --- /dev/null +++ b/.github/workflows/durable-first-install-dart.yml @@ -0,0 +1,172 @@ +name: durable first-install Dart acceptance + +on: + pull_request: + paths: + - ".github/workflows/durable-first-install-dart.yml" + - "scripts/durable-first-install-dart.sh" + push: + branches: [main] + paths: + - ".github/workflows/durable-first-install-dart.yml" + - "scripts/durable-first-install-dart.sh" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit containing Dart native-name wiring + required: false + default: b5d238112f08efc76f4f1bf14dc652a9ab10e963 + +permissions: + contents: read + +concurrency: + group: durable-first-install-dart-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || 'b5d238112f08efc76f4f1bf14dc652a9ab10e963' }} + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + DART_LIB_REF: 1658a96945230e53a9ad0c2e944ed4e28dba4de2 + DART_APP_REF: 5b248e5a27119246a509fda258fb8a69c4accb5f + +jobs: + acceptance: + name: declared pub identity and durable state / ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + for pin in \ + "$ZED_CLI_REF" \ + "$ZED_INTERFACES_REF" \ + "$DART_LIB_REF" \ + "$DART_APP_REF" + do + [[ "$pin" =~ ^[0-9a-f]{40}$ ]] + done + + - name: Check out this acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact zed-cli implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the immutable Dart library fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/dart-lib + ref: ${{ env.DART_LIB_REF }} + path: dart-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the immutable Dart application fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/dart-app + ref: ${{ env.DART_APP_REF }} + path: dart-app + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install the pinned Rust toolchain action + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt + + - name: Install the pinned Dart SDK action + uses: dart-lang/setup-dart@7654d458321ee25acccccfdb86cd48bd95768ff1 # v1.8.0 + with: + sdk: stable + + - name: Restore Rust build cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + workspaces: zed-cli + shared-key: den-1579-dart-${{ matrix.os }} + + - name: Verify toolchains and focused contracts + run: | + set -euo pipefail + rustc --version --verbose + cargo --version --verbose + dart --version + bash -n harness/scripts/durable-first-install-dart.sh + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --lib dart_wiring::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --lib managed_install::tests + + - name: Build the exact release binary under test + run: | + set -euo pipefail + cargo build --locked --release \ + --manifest-path zed-cli/Cargo.toml \ + --bin zed + test -x zed-cli/target/release/zed + zed-cli/target/release/zed --version + + - name: Run the real Dart durable first-install canary + run: | + set -o pipefail + bash harness/scripts/durable-first-install-dart.sh \ + "$GITHUB_WORKSPACE/zed-cli/target/release/zed" \ + "$GITHUB_WORKSPACE/dart-lib" \ + "$GITHUB_WORKSPACE/dart-app" \ + "$RUNNER_TEMP/den-1579-dart" \ + 2>&1 | tee "$RUNNER_TEMP/den-1579-dart.log" + + - name: Upload bounded failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: durable-first-install-dart-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/den-1579-dart.log + ${{ runner.temp }}/den-1579-dart/*.log + ${{ runner.temp }}/den-1579-dart/**/.zpkg.toml + ${{ runner.temp }}/den-1579-dart/**/.zpkg.lock + ${{ runner.temp }}/den-1579-dart/**/.zed/pub-deps.yaml + ${{ runner.temp }}/den-1579-dart/**/pubspec.lock + if-no-files-found: warn + retention-days: 7 + include-hidden-files: true diff --git a/.github/workflows/durable-first-install-polyglot.yml b/.github/workflows/durable-first-install-polyglot.yml new file mode 100644 index 00000000..4a5a43db --- /dev/null +++ b/.github/workflows/durable-first-install-polyglot.yml @@ -0,0 +1,198 @@ +name: durable first-install polyglot acceptance + +on: + pull_request: + paths: + - ".github/workflows/durable-first-install-polyglot.yml" + - "scripts/durable-first-install-polyglot.sh" + - "README.md" + push: + branches: [main] + paths: + - ".github/workflows/durable-first-install-polyglot.yml" + - "scripts/durable-first-install-polyglot.sh" + - "README.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit containing the durable first-install implementation + required: false + default: fe4b4662fb252166b18c48f9c048f5888ec8df44 + +permissions: + contents: read + +concurrency: + group: durable-first-install-polyglot-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || 'fe4b4662fb252166b18c48f9c048f5888ec8df44' }} + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + +jobs: + acceptance: + name: ${{ matrix.ecosystem }} durable manifest / ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - ecosystem: go + os: ubuntu-24.04 + lib_ref: e09dec6790a60ba75d5d338eece1068a4b8454b1 + app_ref: 8e035feb84cf80a294e631bda4ea7a44b1c4d7cf + - ecosystem: go + os: macos-15 + lib_ref: e09dec6790a60ba75d5d338eece1068a4b8454b1 + app_ref: 8e035feb84cf80a294e631bda4ea7a44b1c4d7cf + - ecosystem: python + os: ubuntu-24.04 + lib_ref: b2db8ce2962fbaae690db7a9dabd14eb1d0362de + app_ref: a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1 + - ecosystem: python + os: macos-15 + lib_ref: b2db8ce2962fbaae690db7a9dabd14eb1d0362de + app_ref: a0c58cc6942a8cd30ad5eb318b9ada2fa9b2edb1 + - ecosystem: rust + os: ubuntu-24.04 + lib_ref: f2534ffafbc4f7e900075dfc8e48b9259aced2fe + app_ref: 38b0a5425176a53621d1c07be791701dbd30eefe + - ecosystem: rust + os: macos-15 + lib_ref: f2534ffafbc4f7e900075dfc8e48b9259aced2fe + app_ref: 38b0a5425176a53621d1c07be791701dbd30eefe + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + steps: + - name: Require immutable component pins + env: + LIB_REF: ${{ matrix.lib_ref }} + APP_REF: ${{ matrix.app_ref }} + run: | + set -euo pipefail + for pin in "$ZED_CLI_REF" "$ZED_INTERFACES_REF" "$LIB_REF" "$APP_REF"; do + [[ "$pin" =~ ^[0-9a-f]{40}$ ]] + done + + - name: Check out this acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact zed-cli implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the immutable library fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/${{ matrix.ecosystem }}-lib + ref: ${{ matrix.lib_ref }} + path: fixture-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the immutable application fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/${{ matrix.ecosystem }}-app + ref: ${{ matrix.app_ref }} + path: fixture-app + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install the pinned Rust toolchain action + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt + + - name: Set up Go 1.22 + if: matrix.ecosystem == 'go' + uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5 + with: + go-version: "1.22" + cache: false + + - name: Set up Python 3.12 + if: matrix.ecosystem == 'python' + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: fixture-lib/pyproject.toml + + - name: Restore Rust build cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + workspaces: zed-cli + shared-key: den-1413-polyglot-${{ matrix.os }} + + - name: Verify harness syntax and focused Rust contract + run: | + set -euo pipefail + bash -n harness/scripts/durable-first-install-polyglot.sh + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --lib managed_install::tests + + - name: Build the exact release binary under test + run: | + set -euo pipefail + cargo build --locked --release \ + --manifest-path zed-cli/Cargo.toml \ + --bin zed + test -x zed-cli/target/release/zed + zed-cli/target/release/zed --version + + - name: Run the polyglot durable first-install canary + run: | + set -o pipefail + bash harness/scripts/durable-first-install-polyglot.sh \ + "$GITHUB_WORKSPACE/zed-cli/target/release/zed" \ + "${{ matrix.ecosystem }}" \ + "$GITHUB_WORKSPACE/fixture-lib" \ + "$GITHUB_WORKSPACE/fixture-app" \ + "$RUNNER_TEMP/den-1413-${{ matrix.ecosystem }}" \ + 2>&1 | tee "$RUNNER_TEMP/den-1413-${{ matrix.ecosystem }}.log" + + - name: Upload bounded failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: durable-first-install-${{ matrix.ecosystem }}-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/den-1413-${{ matrix.ecosystem }}.log + ${{ runner.temp }}/den-1413-${{ matrix.ecosystem }}/*.log + ${{ runner.temp }}/den-1413-${{ matrix.ecosystem }}/**/.zpkg.toml + ${{ runner.temp }}/den-1413-${{ matrix.ecosystem }}/**/.zpkg.lock + if-no-files-found: warn + retention-days: 7 + include-hidden-files: true diff --git a/.github/workflows/durable-first-install.yml b/.github/workflows/durable-first-install.yml new file mode 100644 index 00000000..e3e445ba --- /dev/null +++ b/.github/workflows/durable-first-install.yml @@ -0,0 +1,160 @@ +name: durable first-install acceptance + +on: + pull_request: + paths: + - ".github/workflows/durable-first-install.yml" + - "scripts/durable-first-install.sh" + - "README.md" + push: + branches: [main] + paths: + - ".github/workflows/durable-first-install.yml" + - "scripts/durable-first-install.sh" + - "README.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit containing the first-install implementation + required: false + default: fe4b4662fb252166b18c48f9c048f5888ec8df44 + +permissions: + contents: read + +concurrency: + group: durable-first-install-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || 'fe4b4662fb252166b18c48f9c048f5888ec8df44' }} + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + NODE_LIB_REF: 222cdf57f48530fce8e6c1f58632d9676203512e + NODE_APP_REF: ef794674647b3620e711f4f855865f8ad3340498 + +jobs: + acceptance: + name: durable and explicit ephemeral installs / ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + for pin in \ + "$ZED_CLI_REF" \ + "$ZED_INTERFACES_REF" \ + "$NODE_LIB_REF" \ + "$NODE_APP_REF" + do + [[ "$pin" =~ ^[0-9a-f]{40}$ ]] + done + + - name: Check out this acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact zed-cli implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact manifest and lock contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact package-author fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-lib + ref: ${{ env.NODE_LIB_REF }} + path: node-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact consumer fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-app + ref: ${{ env.NODE_APP_REF }} + path: node-app + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Verify the harness and focused Rust contract + run: | + set -euo pipefail + bash -n harness/scripts/durable-first-install.sh + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --lib managed_install::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --lib cli_model::tests + + - name: Build the exact release binary under test + run: | + set -euo pipefail + cargo build --locked --release \ + --manifest-path zed-cli/Cargo.toml \ + --bin zed + test -x zed-cli/target/release/zed + zed-cli/target/release/zed --version + + - name: Run the cross-organization acceptance canary + run: | + set -o pipefail + bash harness/scripts/durable-first-install.sh \ + "$GITHUB_WORKSPACE/zed-cli/target/release/zed" \ + "$GITHUB_WORKSPACE/node-lib" \ + "$GITHUB_WORKSPACE/node-app" \ + "$RUNNER_TEMP/den-1413-e2e" \ + 2>&1 | tee "$RUNNER_TEMP/den-1413-e2e.log" + + - name: Upload bounded failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: durable-first-install-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/den-1413-e2e.log + ${{ runner.temp }}/den-1413-e2e/**/*.log + ${{ runner.temp }}/den-1413-e2e/**/.zpkg.toml + ${{ runner.temp }}/den-1413-e2e/**/.zpkg.lock + if-no-files-found: warn + retention-days: 7 + include-hidden-files: true diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 552d98b8..92f310d4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -21,7 +21,7 @@ env: # Every cross-repository input is immutable. Updating one of these pins is an # intentional dependency change, not an ambient change to a default branch. ZED_E2E_REF: 5ccd57316ae39c3a3692fb4cfbda2f8a4dd74aa0 - ZED_CLI_REF: 8929a58c6591c7a964d7f91412665d7c8a4afdf3 + ZED_CLI_REF: 3a91d64abf1fc0bdee6eab2ff804113b1429849f ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 ZED_API_REF: 93de2808e0c19a39bf8350abc6cd3104b729e5bc ZED_WEB_REF: bb6d9a7094fd9a8d13224865b32e9c362eea1b03 diff --git a/.github/workflows/federated-test-org-scenarios.yml b/.github/workflows/federated-test-org-scenarios.yml new file mode 100644 index 00000000..f698267c --- /dev/null +++ b/.github/workflows/federated-test-org-scenarios.yml @@ -0,0 +1,177 @@ +name: Federated test-org scenarios + +on: + workflow_dispatch: + schedule: + - cron: "17 8 * * *" + pull_request: + paths: + - .github/workflows/federated-test-org-scenarios.yml + - scripts/run-federated-test-org-scenarios.py + - evidence/** + +permissions: + actions: read + contents: read + +concurrency: + group: federated-test-org-scenarios-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + GH_TOKEN: ${{ secrets.TEST_FLEET_READ_TOKEN || github.token }} + +jobs: + execute: + name: Offline cache, solver, security, and manager federation + runs-on: ubuntu-24.04 + timeout-minutes: 35 + steps: + - name: Check out exact test harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Select latest retained Linux release candidate + id: candidate + shell: bash + run: | + set -euo pipefail + gh --version + gh api --method GET \ + -H 'Accept: application/vnd.github+json' \ + /repos/zed-pkg/zed-cli/actions/artifacts \ + -f name=zed-x86_64-unknown-linux-gnu \ + -f per_page=100 > artifacts.json + python3 - <<'PY' + import json + import os + import re + + data = json.load(open('artifacts.json', encoding='utf-8')) + candidates = [ + artifact + for artifact in data.get('artifacts', []) + if not artifact.get('expired') + and artifact.get('workflow_run', {}).get('head_branch', '').startswith('release/') + ] + if not candidates: + raise SystemExit('no retained zed x86_64 Linux release candidate artifact found') + candidates.sort(key=lambda artifact: artifact['created_at'], reverse=True) + selected = candidates[0] + digest = selected.get('digest') + source_commit = selected.get('workflow_run', {}).get('head_sha', '') + if not isinstance(digest, str) or re.fullmatch(r'sha256:[0-9a-f]{64}', digest) is None: + raise SystemExit(f'artifact lacks a canonical SHA-256 digest: {digest!r}') + if re.fullmatch(r'[0-9a-f]{40}', source_commit) is None: + raise SystemExit(f'artifact lacks an immutable source commit: {source_commit!r}') + output = os.environ['GITHUB_OUTPUT'] + with open(output, 'a', encoding='utf-8') as stream: + stream.write(f"artifact_id={selected['id']}\n") + stream.write(f"artifact_digest={digest}\n") + stream.write(f"source_commit={source_commit}\n") + stream.write(f"source_branch={selected['workflow_run']['head_branch']}\n") + stream.write(f"created_at={selected['created_at']}\n") + print(json.dumps(selected, indent=2)) + PY + + - name: Download and verify Actions artifact ZIP + shell: bash + env: + ARTIFACT_ID: ${{ steps.candidate.outputs.artifact_id }} + ARTIFACT_DIGEST: ${{ steps.candidate.outputs.artifact_digest }} + run: | + set -euo pipefail + gh api \ + -H 'Accept: application/vnd.github+json' \ + "/repos/zed-pkg/zed-cli/actions/artifacts/$ARTIFACT_ID/zip" \ + > zed-release-candidate.zip + actual="sha256:$(sha256sum zed-release-candidate.zip | awk '{print $1}')" + test "$actual" = "$ARTIFACT_DIGEST" + mkdir artifact + unzip -q zed-release-candidate.zip -d artifact + + - name: Verify inner release checksum and extract zed + id: binary + shell: bash + run: | + set -euo pipefail + archive="$(find artifact -type f -name 'zed-x86_64-unknown-linux-gnu.tar.gz' -print -quit)" + checksum="$(find artifact -type f -name 'zed-x86_64-unknown-linux-gnu.tar.gz.sha256' -print -quit)" + test -n "$archive" + test -n "$checksum" + test "$(find artifact -type f -name 'zed-x86_64-unknown-linux-gnu.tar.gz' | wc -l)" -eq 1 + test "$(find artifact -type f -name 'zed-x86_64-unknown-linux-gnu.tar.gz.sha256' | wc -l)" -eq 1 + ( + cd "$(dirname "$checksum")" + sha256sum --check "$(basename "$checksum")" + ) + mkdir binary + tar -xzf "$archive" -C binary + chmod 0755 binary/zed + binary/zed --version + printf 'archive=%s\n' "$archive" >> "$GITHUB_OUTPUT" + + - name: Execute four independent test-org profiles + shell: bash + env: + ZED_BIN: ${{ github.workspace }}/binary/zed + ZED_BINARY_ARCHIVE: ${{ steps.binary.outputs.archive }} + ZED_SOURCE_COMMIT: ${{ steps.candidate.outputs.source_commit }} + ZED_E2E_ROOT: ${{ runner.temp }}/zed-test-org-federation + ZED_E2E_REPORT: ${{ github.workspace }}/federated-test-org-report.json + run: | + set -euo pipefail + python3 -m py_compile scripts/run-federated-test-org-scenarios.py + python3 scripts/run-federated-test-org-scenarios.py 2>&1 | tee federated-test-org-run.log + python3 - <<'PY' + import json + import os + + report = json.load(open('federated-test-org-report.json', encoding='utf-8')) + suites = report.get('suites', {}) + expected = { + 'zed-pkg-test/offline-cache-e2e', + 'zed-pkg-test/version-solver-e2e', + 'zed-pkg-test/security-adversarial-e2e', + 'zed-pkg-test/manager-interop-e2e', + } + if set(suites) != expected: + raise SystemExit(f'unexpected suite inventory: {sorted(suites)}') + failures = [name for name, suite in suites.items() if suite.get('status') != 'pass'] + if failures: + raise SystemExit(f'federated suites failed: {failures}') + checks = [ + (suite_name, check) + for suite_name, suite in suites.items() + for check in suite.get('checks', []) + ] + if len(checks) != 19: + raise SystemExit(f'expected 19 checks, found {len(checks)}') + failed_checks = [ + f"{suite_name}: {check.get('name', '')}" + for suite_name, check in checks + if check.get('status') != 'pass' + ] + if failed_checks: + raise SystemExit(f'individual checks failed: {failed_checks}') + summary = os.environ['GITHUB_STEP_SUMMARY'] + with open(summary, 'a', encoding='utf-8') as stream: + stream.write('## Federated zed-pkg-test result\n\n') + stream.write(f"- Source commit: `{report['binary']['source_commit']}`\n") + stream.write(f"- Binary: `{report['binary']['version']}`\n") + stream.write(f"- Inner archive SHA-256: `{report['binary']['archive_sha256']}`\n") + stream.write(f"- Suites: `{len(suites)}`\n") + stream.write(f"- Checks: `{len(checks)}/19 passed`\n") + PY + + - name: Retain execution receipt and failure log + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: federated-test-org-${{ steps.candidate.outputs.source_commit || github.run_id }} + path: | + federated-test-org-report.json + federated-test-org-run.log + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/finish-test-org-fleet-paced.yml b/.github/workflows/finish-test-org-fleet-paced.yml new file mode 100644 index 00000000..a8850c4f --- /dev/null +++ b/.github/workflows/finish-test-org-fleet-paced.yml @@ -0,0 +1,264 @@ +# One generated repository at a time with bounded content-write pacing. +name: finish test organization fleet with paced writes + +on: + push: + branches: [main] + paths: + - "bootstrap/execute-test-org-fleet-paced.once" + - ".github/workflows/finish-test-org-fleet-paced.yml" + - "scripts/list-test-org-fleet-repositories.mjs" + - "tests/paced-fleet-executor.test.mjs" + +permissions: + contents: read + +concurrency: + group: finish-test-org-fleet-paced-v3 + cancel-in-progress: false + +jobs: + finish: + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 360 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate authorization and paced fleet plan + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/execute-test-org-fleet-paced.once)" = "APPLY_TEST_FLEET_PACED_V2" + node --check scripts/list-test-org-fleet-repositories.mjs + node --check scripts/bootstrap-test-org-fleet-worktree-safe.mjs + node scripts/validate-test-org-fleet.mjs + node --test \ + tests/test-org-fleet.test.mjs \ + tests/bootstrap-retry-policy.test.mjs \ + tests/git-transport-worktree.test.mjs \ + tests/paced-fleet-executor.test.mjs + + organizations=( + fiducia-cloud-test + evento-globolo-test + opto-sync-test + quaestor-ledger-test + memebank-test + scintilla-run-test + file-tunnel-test + shared-auth-test + hypesiege-test + streempilot-test + sonus-auris-test + messaging-intel-test + 3fa-app-test + declarative-migrations-test + cliptown-test + claritas-viz-test + embedded-alerts-test + ) + total=0 + for organization in "${organizations[@]}"; do + count="$(node scripts/list-test-org-fleet-repositories.mjs --org "$organization" --json | jq '.repositoryCount')" + total=$((total + count)) + done + test "$total" -eq 290 + + - name: Export the persisted fleet PAT without logging it + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + test -n "$STORED_FLEET_PAT" + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Provision one repository at a time + shell: bash + env: + TEST_ORG_FLEET_SKIP_TOPICS: "true" + run: | + set -euo pipefail + + organizations=( + fiducia-cloud-test + evento-globolo-test + opto-sync-test + quaestor-ledger-test + memebank-test + scintilla-run-test + file-tunnel-test + shared-auth-test + hypesiege-test + streempilot-test + sonus-auris-test + messaging-intel-test + 3fa-app-test + declarative-migrations-test + cliptown-test + claritas-viz-test + embedded-alerts-test + ) + failures=() + processed=0 + writes=0 + + { + echo '## Paced test-fleet completion' + echo + echo '| Organization | Specialized repos | Visible repos | Result |' + echo '|---|---:|---:|---:|' + } >> "$GITHUB_STEP_SUMMARY" + + gh_api_with_rate_limit_retry() { + local attempt=1 + local max_attempts=6 + local status=1 + local error_file + local quota_json + local remaining + local reset_epoch + local now_epoch + local delay_seconds + + while (( attempt <= max_attempts )); do + error_file="$RUNNER_TEMP/gh-api-${attempt}-${RANDOM}.err" + set +e + gh api "$@" 2>"$error_file" + status=$? + set -e + if (( status == 0 )); then + rm -f "$error_file" + return 0 + fi + + if grep -Eqi 'API rate limit exceeded|secondary rate limit|abuse detection|temporarily blocked' "$error_file"; then + delay_seconds=$((attempt * 60)) + if grep -qi 'API rate limit exceeded' "$error_file" && quota_json="$(gh api rate_limit 2>/dev/null)"; then + remaining="$(jq -r '.resources.core.remaining // -1' <<<"$quota_json")" + reset_epoch="$(jq -r '.resources.core.reset // 0' <<<"$quota_json")" + now_epoch="$(date +%s)" + if [[ "$remaining" == "0" && "$reset_epoch" =~ ^[0-9]+$ && "$reset_epoch" -gt "$now_epoch" ]]; then + delay_seconds=$((reset_epoch - now_epoch + 5)) + fi + fi + if (( delay_seconds > 0 && delay_seconds <= 3900 && attempt < max_attempts )); then + printf '::notice::GitHub API quota unavailable; retrying after %ss (attempt %s/%s)\n' \ + "$delay_seconds" "$attempt" "$max_attempts" >&2 + rm -f "$error_file" + sleep "$delay_seconds" + attempt=$((attempt + 1)) + continue + fi + fi + + cat "$error_file" >&2 + rm -f "$error_file" + return "$status" + done + + printf '::error::GitHub API request exhausted %s bounded attempts\n' "$max_attempts" >&2 + return "$status" + } + + apply_filtered_repository() { + local organization="$1" + local repository="$2" + local summary_file="$RUNNER_TEMP/fleet-${organization}-${repository}.json" + + for attempt in 1 2 3; do + rm -f "$summary_file" + if node scripts/bootstrap-test-org-fleet-worktree-safe.mjs \ + --apply \ + --org "$organization" \ + --repo "$repository" \ + --concurrency 1 \ + --summary-json > "$summary_file"; then + local request_writes + request_writes="$(jq '(.created // 0) + (.committed // 0) + (.pullRequests // 0)' "$summary_file")" + writes=$((writes + request_writes)) + if [[ "$request_writes" -gt 0 ]]; then + sleep 15 + else + sleep 2 + fi + return 0 + fi + + if [[ "$attempt" -lt 3 ]]; then + sleep $((attempt * 90)) + fi + done + return 1 + } + + finalize_organization() { + local organization="$1" + local summary_file="$RUNNER_TEMP/fleet-${organization}-final.json" + for attempt in 1 2 3; do + rm -f "$summary_file" + if node scripts/bootstrap-test-org-fleet-worktree-safe.mjs \ + --apply \ + --org "$organization" \ + --concurrency 1 \ + --summary-json > "$summary_file"; then + local request_writes + request_writes="$(jq '(.created // 0) + (.committed // 0) + (.pullRequests // 0)' "$summary_file")" + writes=$((writes + request_writes)) + [[ "$request_writes" -gt 0 ]] && sleep 15 + return 0 + fi + [[ "$attempt" -lt 3 ]] && sleep $((attempt * 90)) + done + return 1 + } + + for organization in "${organizations[@]}"; do + echo "::group::Paced provisioning for $organization" + gh_api_with_rate_limit_retry "orgs/$organization" --silent + mapfile -t repositories < <(node scripts/list-test-org-fleet-repositories.mjs --org "$organization") + organization_failed=false + + for repository in "${repositories[@]}"; do + processed=$((processed + 1)) + echo "[$processed/290] $organization/$repository" + if ! apply_filtered_repository "$organization" "$repository"; then + organization_failed=true + echo "::error::Failed to provision $organization/$repository after three idempotent attempts" + fi + done + + if [[ "$organization_failed" == false ]] && ! finalize_organization "$organization"; then + organization_failed=true + echo "::error::Final governance/idempotency pass failed for $organization" + fi + + visible="$(gh_api_with_rate_limit_retry --paginate "orgs/$organization/repos?type=all&per_page=100" --jq '.[].name' | wc -l | tr -d '[:space:]')" + if [[ "$organization_failed" == true ]]; then + failures+=("$organization") + printf '| `%s` | %s | %s | **failure** |\n' "$organization" "${#repositories[@]}" "$visible" >> "$GITHUB_STEP_SUMMARY" + else + printf '| `%s` | %s | %s | success |\n' "$organization" "${#repositories[@]}" "$visible" >> "$GITHUB_STEP_SUMMARY" + fi + echo "::endgroup::" + done + + { + echo + echo "Specialized repository filters processed: $processed" + echo + echo "Observed create/commit/PR write operations: $writes" + } >> "$GITHUB_STEP_SUMMARY" + + if (( ${#failures[@]} > 0 )); then + printf '::error::Paced fleet completion failed for: %s\n' "${failures[*]}" + exit 1 + fi + + test "$processed" -eq 290 + echo 'All selected organizations completed; `r2g` and `r2g-test` were never selected.' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/finish-test-org-fleet-single-runner.yml b/.github/workflows/finish-test-org-fleet-single-runner.yml new file mode 100644 index 00000000..3894c90c --- /dev/null +++ b/.github/workflows/finish-test-org-fleet-single-runner.yml @@ -0,0 +1,138 @@ +# One-time single-runner completion of the cross-organization test fleet. +name: finish test organization fleet on one runner + +on: + push: + branches: + - main + paths: + - bootstrap/execute-test-org-fleet-single-runner.once + +permissions: + contents: read + +concurrency: + group: apply-test-org-fleet-all + cancel-in-progress: true + +jobs: + finish: + name: Finish all remaining test organizations + runs-on: ubuntu-24.04 + timeout-minutes: 360 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate authorization and fleet contract + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/execute-test-org-fleet-single-runner.once)" = "APPLY_TEST_FLEET_SINGLE_RUNNER_V3" + node --check scripts/bootstrap-test-org-fleet.mjs + node --check scripts/bootstrap-test-org-fleet-worktree-safe.mjs + node --check scripts/validate-test-org-fleet.mjs + node scripts/validate-test-org-fleet.mjs + node --test tests/test-org-fleet.test.mjs tests/bootstrap-retry-policy.test.mjs + + - name: Export the persisted fleet PAT without logging it + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + test -n "$STORED_FLEET_PAT" + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Create or adopt repositories, push commits, and open draft PRs + shell: bash + env: + TEST_ORG_FLEET_SKIP_TOPICS: "true" + run: | + set -euo pipefail + + organizations=( + fiducia-cloud-test + evento-globolo-test + opto-sync-test + quaestor-ledger-test + 3fa-app-test + declarative-migrations-test + cliptown-test + claritas-viz-test + embedded-alerts-test + memebank-test + sonus-auris-test + messaging-intel-test + scintilla-run-test + file-tunnel-test + shared-auth-test + hypesiege-test + streempilot-test + ) + + failures=() + { + echo '## Single-runner fleet completion' + echo + echo '| Organization | Result | Attempts | Visible repositories |' + echo '|---|---:|---:|---:|' + } >> "$GITHUB_STEP_SUMMARY" + + for organization in "${organizations[@]}"; do + echo "::group::Provision $organization" + gh api "orgs/$organization" --silent + sleep 15 + + completed=false + attempts=0 + for attempt in 1 2 3; do + attempts="$attempt" + echo "Applying $organization (attempt $attempt of 3)" + if node scripts/bootstrap-test-org-fleet-worktree-safe.mjs \ + --apply \ + --org "$organization" \ + --concurrency 1; then + completed=true + break + fi + + if [[ "$attempt" -lt 3 ]]; then + delay=$((attempt * 60)) + echo "Attempt $attempt failed for $organization; cooling down for ${delay}s before the idempotent retry." + sleep "$delay" + fi + done + + repository_count="$( + gh api --paginate "orgs/$organization/repos?type=all&per_page=100" \ + --jq '.[].name' | wc -l | tr -d '[:space:]' + )" + + if [[ "$completed" == true ]]; then + printf '| `%s` | success | %s | %s |\n' \ + "$organization" "$attempts" "$repository_count" >> "$GITHUB_STEP_SUMMARY" + else + failures+=("$organization") + printf '| `%s` | **failure** | %s | %s |\n' \ + "$organization" "$attempts" "$repository_count" >> "$GITHUB_STEP_SUMMARY" + fi + echo "::endgroup::" + done + + if (( ${#failures[@]} > 0 )); then + printf '::error::Fleet completion failed for: %s\n' "${failures[*]}" + { + echo + echo "Failed organizations: ${failures[*]}" + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + + { + echo + echo 'All remaining in-scope organizations completed. `r2g` and `r2g-test` were not selected.' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/formal-app-lifecycle.yml b/.github/workflows/formal-app-lifecycle.yml new file mode 100644 index 00000000..be1a1ef0 --- /dev/null +++ b/.github/workflows/formal-app-lifecycle.yml @@ -0,0 +1,72 @@ +name: DEN-4181 managed app lifecycle exact-head certification + +on: + pull_request: + paths: + - ".github/workflows/formal-app-lifecycle.yml" + - "scripts/verify_formal_app_lifecycle_contract.py" + - "tests/test_formal_app_lifecycle_contract.py" + push: + branches: [main] + paths: + - ".github/workflows/formal-app-lifecycle.yml" + - "scripts/verify_formal_app_lifecycle_contract.py" + - "tests/test_formal_app_lifecycle_contract.py" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: den-4181-app-lifecycle-${{ github.ref }} + cancel-in-progress: true + +env: + ZED_SYNC_SHA: 64717d0bdfe598ffa350736291c2ff2dde45ac2f + APP_MANIFEST_BLOB: 9bac5230b95d29f830a098a6d19308a8ae7ac974 + APP_MODEL_BLOB: 2749b36ebb6bb5787a2ecb469a2f0a8025ecb3b9 + APP_FIXTURE_BLOB: 12828bf2dba69e9faac3cd638ec7e3be65096885 + APP_SCHEMA_BLOB: 6a82fe44eacc9a8f632ad2bdf78cae85499a0a57 + +jobs: + certify: + name: formal model to Rust, JavaScript, Dart, and WASM + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out E2E certification authority + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: zed-pkg-e2e + persist-credentials: false + + - name: Check out exact zed-sync production commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-sync + ref: ${{ env.ZED_SYNC_SHA }} + path: zed-sync + persist-credentials: false + + - name: Verify immutable commit and formal artifact provenance + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-sync rev-parse HEAD)" = "$ZED_SYNC_SHA" + test "$(git -C zed-sync rev-parse HEAD:formal/app-lifecycle.fm.toml)" = "$APP_MANIFEST_BLOB" + test "$(git -C zed-sync rev-parse HEAD:formal/app_lifecycle.qnt)" = "$APP_MODEL_BLOB" + test "$(git -C zed-sync rev-parse HEAD:protocol/formal-app-lifecycle.json)" = "$APP_FIXTURE_BLOB" + test "$(git -C zed-sync rev-parse HEAD:protocol/formal-app-lifecycle.schema.json)" = "$APP_SCHEMA_BLOB" + + - name: Test the cross-repository checker + run: python3 -m unittest -v zed-pkg-e2e/tests/test_formal_app_lifecycle_contract.py + + - name: Cross-check formal inventory, traces, schema, and refinements + run: python3 zed-pkg-e2e/scripts/verify_formal_app_lifecycle_contract.py zed-sync + + - name: Install pinned Nix + uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 + + - name: Execute the exact production verification matrix + working-directory: zed-sync + run: nix develop --no-update-lock-file -c agent-check all diff --git a/.github/workflows/ghcr-fallback.yml b/.github/workflows/ghcr-fallback.yml new file mode 100644 index 00000000..cf8a9861 --- /dev/null +++ b/.github/workflows/ghcr-fallback.yml @@ -0,0 +1,40 @@ +name: GHCR live fallback canary + +on: + pull_request: + paths: + - ".github/workflows/ghcr-fallback.yml" + - "scripts/ghcr_fallback.py" + - "docs/ghcr-fallback-canary.md" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ghcr-fallback-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + syntax: + name: Canary script compiles + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check out harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + - name: py_compile and naming contract + run: | + set -euo pipefail + python3 -m py_compile scripts/ghcr_fallback.py + python3 scripts/ghcr_fallback.py \ + --work-root "$RUNNER_TEMP/ghcr-contract" \ + --skip-network diff --git a/.github/workflows/git-submodule-interop.yml b/.github/workflows/git-submodule-interop.yml new file mode 100644 index 00000000..26b9e031 --- /dev/null +++ b/.github/workflows/git-submodule-interop.yml @@ -0,0 +1,193 @@ +name: git submodule interoperability + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + run_arc_smoke: + description: Repeat the contract on the registered sonus-ci ARC scale set + type: boolean + required: true + default: false + +permissions: + contents: read + +concurrency: + group: zed-git-submodule-interop-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: d8763b8dcc3bd5c5b6b4b1d13519065a6a2cb99a + ZED_INTERFACES_REF: 8428bc574111fa148e590c8350c7855035ce2046 + +jobs: + contract: + name: Contract / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 50 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + steps: + - name: Check out acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable inputs + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C zed-interfaces rev-parse HEAD)" = "$ZED_INTERFACES_REF" + test "$(tr -d '\r\n' < harness/.zed-cli-ref)" = "$ZED_CLI_REF" + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + test -z "${ZED_PKG_TOKEN:-}" + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + git --version + python3 --version + + - name: Enforce source and formatting contracts + run: | + set -euo pipefail + PYTHONPYCACHEPREFIX="$RUNNER_TEMP/python-cache" \ + python3 -m py_compile harness/scripts/git_submodule_interop.py + git -C harness diff --check + git -C zed-cli diff --check + git -C zed-interfaces diff --check + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + + - name: Run focused product tests + if: runner.os == 'Linux' + run: | + set -euo pipefail + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib git_submodules + cargo test --locked --manifest-path zed-cli/Cargo.toml --test git_submodule_interop + cargo test --locked --manifest-path zed-cli/Cargo.toml --test git_submodule_fresh_clone + cargo test --locked --manifest-path zed-cli/Cargo.toml --test git_submodule_interop_cli + cargo test --locked --manifest-path zed-cli/Cargo.toml --test git_submodule_branch_provenance + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + + - name: Build the exact release binary + run: cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + + - name: Run the twelve-check black-box contract + env: + WORK_ROOT: ${{ runner.temp }}/zed-git-submodule-interop + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/git_submodule_interop.py \ + --zed zed-cli/target/release/zed \ + --work-root "$WORK_ROOT" + python3 - "$WORK_ROOT/evidence/evidence.json" <<'PY' + import json + import pathlib + import sys + + evidence = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + assert evidence["schema"] == "zed.git-submodule-interop-evidence/v2" + assert len(evidence["checks"]) == 12 + assert evidence["network_credentials"] is False + assert evidence["public_registry_mutation"] is False + PY + + - name: Require clean product and harness checkouts + run: | + set -euo pipefail + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-interfaces status --porcelain=v1 --untracked-files=all)" + + - name: Upload contract evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: git-submodule-interop-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-git-submodule-interop/evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false + + arc-parity: + name: Optional self-hosted ARC parity + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_arc_smoke }} + runs-on: [self-hosted, linux, sonus-ci] + timeout-minutes: 50 + steps: + - name: Check out acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Run the same contract on the ephemeral ARC runner + env: + WORK_ROOT: ${{ runner.temp }}/zed-git-submodule-interop-arc + run: | + set -euo pipefail + test "$RUNNER_OS" = Linux + test -n "$RUNNER_NAME" + test ! -e /var/run/docker.sock + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(tr -d '\r\n' < harness/.zed-cli-ref)" = "$ZED_CLI_REF" + rustup toolchain install stable --profile minimal + rustup default stable + PYTHONPYCACHEPREFIX="$RUNNER_TEMP/python-cache" \ + python3 -m py_compile harness/scripts/git_submodule_interop.py + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + test ! -e "$WORK_ROOT" + python3 harness/scripts/git_submodule_interop.py \ + --zed zed-cli/target/release/zed \ + --work-root "$WORK_ROOT" diff --git a/.github/workflows/git-submodule-overtake-operation-lock.yml b/.github/workflows/git-submodule-overtake-operation-lock.yml new file mode 100644 index 00000000..e29893c7 --- /dev/null +++ b/.github/workflows/git-submodule-overtake-operation-lock.yml @@ -0,0 +1,259 @@ +name: Git submodule takeover operation lock + +on: + pull_request: + paths: + - scripts/git_submodule_overtake_lock.py + - scripts/git_submodule_overtake_nested_lock.py + - scripts/git_submodule_install_operation_lock.py + - scripts/project_recovery_operation_lock.py + - docs/git-submodule-overtake-lock.md + - .github/workflows/git-submodule-overtake-operation-lock.yml + push: + branches: [main] + paths: + - scripts/git_submodule_overtake_lock.py + - scripts/git_submodule_overtake_nested_lock.py + - scripts/git_submodule_install_operation_lock.py + - scripts/project_recovery_operation_lock.py + - docs/git-submodule-overtake-lock.md + - .github/workflows/git-submodule-overtake-operation-lock.yml + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: 591b40846da73f7540bd36add33645d11cd33ab0 + +permissions: + contents: read + +concurrency: + group: git-submodule-overtake-lock-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + PYTHONDONTWRITEBYTECODE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '591b40846da73f7540bd36add33645d11cd33ab0' }} + ZED_INTERFACES_REF: d1bf8ef7c88a75292cbe8d697bfd269f30d62e2b + ZED_LOCK_REF: a0dc78d385bc3ab553d3027b427f5f1428239c9c + +jobs: + contract: + name: Process contention / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + steps: + - name: Require immutable product and dependency pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_LOCK_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out test-org harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify exact candidate dependency graph and authority root + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + grep -F "rev = \"$ZED_LOCK_REF\"" zed-cli/Cargo.toml + python3 - <<'PY' + from pathlib import Path + + source = Path("zed-cli/src/main.rs").read_text(encoding="utf-8") + selected = """submodules::sync(&project)?; + managed_install::install( + &project,""" + nested = """submodules::sync(&project)?; + managed_install::install( + &cwd,""" + assert source.count(selected) == 1, "cooperative install must use the selected superproject" + assert nested not in source, "cooperative install regressed to the nested invocation cwd" + PY + + - name: Install stable Rust + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + git --version + python3 --version + + - name: Verify source and focused product contracts + run: | + set -euo pipefail + PYTHONPYCACHEPREFIX="$RUNNER_TEMP/python-cache" \ + python3 -m py_compile \ + harness/scripts/git_submodule_overtake_lock.py \ + harness/scripts/git_submodule_overtake_nested_lock.py \ + harness/scripts/git_submodule_install_operation_lock.py \ + harness/scripts/project_recovery_operation_lock.py + git -C harness diff --check + git -C zed-cli diff --check + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib project_lock + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib git_submodules + cargo test --locked --manifest-path zed-cli/Cargo.toml --doc project_lock + + - name: Run strict Clippy on changed product surfaces + if: runner.os == 'Linux' + run: | + set -euo pipefail + cargo clippy --locked --manifest-path zed-cli/Cargo.toml \ + --lib --bin zed -- -D warnings + + - name: Build exact release executable + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + mkdir -p "$RUNNER_TEMP/zed-bin" + cp zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + chmod 0755 "$RUNNER_TEMP/zed-bin/zed" + test "$("$RUNNER_TEMP/zed-bin/zed" --version)" = "zed 0.1.0" + + - name: Certify normal and terminated-owner serialization + env: + WORK_ROOT: ${{ runner.temp }}/zed-overtake-operation-lock + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/git_submodule_overtake_lock.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + python3 - "$WORK_ROOT/evidence/evidence.json" <<'PY' + import json + import pathlib + import sys + + evidence = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + assert evidence["schema"] == "zed.git-submodule-overtake-operation-lock/v1" + assert len(evidence["checks"]) == 6 + assert evidence["normal_completion"]["observed_block_seconds"] >= 0.7 + assert evidence["owner_termination"]["observed_block_seconds"] >= 0.7 + assert evidence["network_credentials"] is False + assert evidence["public_registry_mutation"] is False + PY + + - name: Certify nested invocation uses superproject ownership + env: + WORK_ROOT: ${{ runner.temp }}/zed-overtake-nested-lock + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/git_submodule_overtake_nested_lock.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + python3 - "$WORK_ROOT/evidence/nested-evidence.json" <<'PY' + import json + import pathlib + import sys + + evidence = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + assert evidence["schema"] == "zed.git-submodule-nested-overtake-lock/v1" + assert len(evidence["checks"]) == 3 + assert evidence["observed_block_seconds"] >= 0.7 + assert evidence["network_credentials"] is False + assert evidence["public_registry_mutation"] is False + PY + + - name: Certify nested cooperative install spans one superproject boundary + env: + WORK_ROOT: ${{ runner.temp }}/zed-submodule-install-lock + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/git_submodule_install_operation_lock.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + python3 - "$WORK_ROOT/evidence/install-evidence.json" <<'PY' + import json + import pathlib + import sys + + evidence = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + assert evidence["schema"] == "zed.git-submodule-install-operation-lock/v1" + assert len(evidence["checks"]) == 4 + assert evidence["observed_block_seconds"] >= 0.7 + assert evidence["network_credentials"] is False + assert evidence["public_registry_mutation"] is False + PY + + - name: Certify pending recovery shares checkout ownership + env: + WORK_ROOT: ${{ runner.temp }}/zed-project-recovery-lock + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/project_recovery_operation_lock.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + python3 - "$WORK_ROOT/evidence/recovery-evidence.json" <<'PY' + import json + import pathlib + import sys + + evidence = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + assert evidence["schema"] == "zed.project-recovery-operation-lock/v1" + assert len(evidence["checks"]) == 6 + assert evidence["different_home_waiter"]["observed_block_seconds"] >= 0.7 + assert evidence["network_credentials"] is False + assert evidence["public_registry_mutation"] is False + PY + + - name: Require clean source checkouts + run: | + set -euo pipefail + harness_status="$(git -C harness status --porcelain=v1 --untracked-files=all)" + product_status="$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + if [[ -n "$harness_status" ]]; then + printf 'dirty harness checkout:\n%s\n' "$harness_status" >&2 + fi + if [[ -n "$product_status" ]]; then + printf 'dirty product checkout:\n%s\n' "$product_status" >&2 + fi + test -z "$harness_status" + test -z "$product_status" + + - name: Upload process-lock evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: git-submodule-overtake-lock-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/zed-overtake-operation-lock/evidence + ${{ runner.temp }}/zed-overtake-nested-lock/evidence + ${{ runner.temp }}/zed-submodule-install-lock/evidence + ${{ runner.temp }}/zed-project-recovery-lock/evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/git-submodule-pack.yml b/.github/workflows/git-submodule-pack.yml new file mode 100644 index 00000000..9257f1ba --- /dev/null +++ b/.github/workflows/git-submodule-pack.yml @@ -0,0 +1,135 @@ +name: git submodule package boundary + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: zed-git-submodule-pack-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: 0786987fe3a173659463940f4f905dc3b5bd19a8 + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + +jobs: + contract: + name: Archive contract / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + steps: + - name: Check out acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable inputs and Python sources + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C zed-interfaces rev-parse HEAD)" = "$ZED_INTERFACES_REF" + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + test -z "${ZED_PKG_TOKEN:-}" + PYTHONPYCACHEPREFIX="$RUNNER_TEMP/pycache" \ + python3 -m py_compile \ + harness/scripts/git_submodule_pack.py \ + harness/scripts/gitmodules_metadata.py + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Enforce formatting + run: cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + + - name: Run focused package-boundary tests + run: | + set -euo pipefail + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib pack_guard + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib manifest_kind_tests + + - name: Run all-target lint gate + if: runner.os == 'Linux' + run: cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + + - name: Build the exact release binary + run: cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + + - name: Run the twelve-check archive contract + env: + WORK_ROOT: ${{ runner.temp }}/zed-git-submodule-pack + run: | + set -euo pipefail + python3 harness/scripts/git_submodule_pack.py \ + --zed zed-cli/target/release/zed \ + --work-root "$WORK_ROOT" + + - name: Run the three-check Git-metadata contract + env: + WORK_ROOT: ${{ runner.temp }}/zed-gitmodules-metadata + run: | + set -euo pipefail + python3 harness/scripts/gitmodules_metadata.py \ + --zed zed-cli/target/release/zed \ + --work-root "$WORK_ROOT" + + - name: Require clean product and harness checkouts + run: | + set -euo pipefail + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-interfaces status --porcelain=v1 --untracked-files=all)" + + - name: Upload bounded contract evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: git-submodule-pack-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/zed-git-submodule-pack/evidence + ${{ runner.temp }}/zed-gitmodules-metadata/evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/github-api-fallback.yml b/.github/workflows/github-api-fallback.yml new file mode 100644 index 00000000..5dae8382 --- /dev/null +++ b/.github/workflows/github-api-fallback.yml @@ -0,0 +1,140 @@ +name: GitHub API live fallback canary + +on: + pull_request: + paths: + - ".github/workflows/github-api-fallback.yml" + - "scripts/github_api_fallback.py" + - "docs/github-api-fallback-canary.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli commit with source fallback + required: false + type: string + run_zed_install: + description: Also build zed and install through GitHub fallback + required: false + type: boolean + default: true + workflow_call: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli commit with source fallback + required: true + type: string + +permissions: + contents: read + +concurrency: + group: github-api-fallback-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + syntax: + name: Canary script compiles + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check out harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + - name: py_compile and naming contract + run: | + set -euo pipefail + python3 -m py_compile scripts/github_api_fallback.py + python3 scripts/github_api_fallback.py \ + --work-root "$RUNNER_TEMP/github-api-contract" \ + --skip-network + + github-api: + name: Live GitHub Releases round-trip + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: write + steps: + - name: Check out harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: GitHub API publish and public download + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + python3 -m py_compile scripts/github_api_fallback.py + python3 scripts/github_api_fallback.py \ + --work-root "$RUNNER_TEMP/github-api-live" + + - name: Resolve zed-cli pin + id: pins + env: + CLI_IN: ${{ inputs.zed_cli_ref || '' }} + WANT_ZED: ${{ github.event.inputs.run_zed_install || 'true' }} + run: | + set -euo pipefail + cli="${CLI_IN:-}" + if [[ -z "$cli" ]]; then + # zed-pkg/zed-cli#294 (source fallback + GitHub Release fetch) + cli="ac27d4ec353a91486fe8a8fa9c61bdb4f073fb63" + fi + [[ "$cli" =~ ^[0-9a-f]{40}$ ]] + printf 'cli=%s\n' "$cli" >> "$GITHUB_OUTPUT" + printf 'want_zed=%s\n' "$WANT_ZED" >> "$GITHUB_OUTPUT" + + - name: Check out exact zed-cli candidate + if: steps.pins.outputs.want_zed != 'false' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ steps.pins.outputs.cli }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install Rust + if: steps.pins.outputs.want_zed != 'false' + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + + - name: Build zed and install from GitHub with registry down + if: steps.pins.outputs.want_zed != 'false' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + test "$(git -C product/zed-cli rev-parse HEAD)" = "${{ steps.pins.outputs.cli }}" + cargo build --release --manifest-path product/zed-cli/Cargo.toml --bin zed + python3 scripts/github_api_fallback.py \ + --work-root "$RUNNER_TEMP/github-api-zed" \ + --zed product/zed-cli/target/release/zed + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: github-api-fallback-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/github-api-* + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/github-dependency-inventory-conformance.yml b/.github/workflows/github-dependency-inventory-conformance.yml new file mode 100644 index 00000000..d70c8854 --- /dev/null +++ b/.github/workflows/github-dependency-inventory-conformance.yml @@ -0,0 +1,172 @@ +name: GitHub dependency inventory conformance + +on: + pull_request: + paths: + - ".gitattributes" + - ".github/workflows/github-dependency-inventory-conformance.yml" + - "docs/github-dependency-inventory-conformance.md" + - "fixtures/github-dependency-inventory/**" + - "harness/github_dependency_inventory.py" + - "harness/github_inventory_*.py" + - "harness/test_github_inventory_*.py" + push: + branches: [main] + paths: + - ".gitattributes" + - ".github/workflows/github-dependency-inventory-conformance.yml" + - "docs/github-dependency-inventory-conformance.md" + - "fixtures/github-dependency-inventory/**" + - "harness/github_dependency_inventory.py" + - "harness/github_inventory_*.py" + - "harness/test_github_inventory_*.py" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: github-dependency-inventory-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + conformance: + name: deterministic inventory / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + env: + PYTHONDONTWRITEBYTECODE: "1" + PYTHONHASHSEED: "0" + PYTHONUTF8: "1" + SOURCE_DATE_EPOCH: "0" + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2022] + + steps: + - name: Check out exact candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Install pinned Python runtime + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Verify the dependency-free tool boundary + run: | + set -euo pipefail + python --version + python - <<'PY' + from pathlib import Path + + paths = [Path("harness/github_dependency_inventory.py")] + paths += sorted(Path("harness").glob("github_inventory_*.py")) + paths += sorted(Path("harness").glob("test_github_inventory_*.py")) + for path in paths: + compile(path.read_text(encoding="utf-8"), str(path), "exec") + PY + + - name: Run adversarial fixture and fake-server conformance + run: | + set -euo pipefail + python -m unittest discover \ + -s harness \ + -p 'test_*.py' \ + -v + + - name: Regenerate independent byte-identical graph artifacts + id: evidence + run: | + set -euo pipefail + first="$RUNNER_TEMP/github-inventory-first" + second="$RUNNER_TEMP/github-inventory-second" + evidence="$RUNNER_TEMP/github-inventory-evidence" + rm -rf "$first" "$second" "$evidence" + mkdir -p "$first" "$second" "$evidence" + + for format in json dot mermaid; do + extension="$format" + if [[ "$format" == "mermaid" ]]; then extension="mmd"; fi + python harness/github_dependency_inventory.py \ + --fixture fixtures/github-dependency-inventory/fixture.json \ + --org ACME \ + --repo acme/app \ + --include zed,git-submodule,nix \ + --format "$format" \ + --output "$first/inventory.$extension" + + python harness/github_dependency_inventory.py \ + --fixture fixtures/github-dependency-inventory/fixture.json \ + --repo ACME/App \ + --repo acme/app \ + --org acme \ + --org ACME \ + --include nix \ + --include git-submodule,zed \ + --format "$format" \ + --output "$second/inventory.$extension" + done + + python - "$first" "$second" "$evidence/inventory.json" <<'PY' + import hashlib + import json + import sys + from pathlib import Path + + first, second, evidence = map(Path, sys.argv[1:]) + golden = Path("fixtures/github-dependency-inventory/golden") + rows = [] + for name in ("inventory.json", "inventory.dot", "inventory.mmd"): + left = (first / name).read_bytes() + right = (second / name).read_bytes() + expected = (golden / name).read_bytes() + if left != right: + raise SystemExit(f"independent {name} renders differ") + if left != expected: + raise SystemExit(f"generated {name} differs from checked-in golden") + rows.append({ + "path": name, + "bytes": len(left), + "sha256": hashlib.sha256(left).hexdigest(), + }) + document = { + "schema": "zed-pkg-test/github-dependency-inventory-evidence/v1", + "source_date_epoch": 0, + "files": rows, + } + evidence.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + print(json.dumps(document, indent=2, sort_keys=True)) + PY + + printf 'path=%s\n' "$evidence" >> "$GITHUB_OUTPUT" + + - name: Prove the source checkout stayed immutable + run: | + set -euo pipefail + git diff --exit-code + status="$(git status --porcelain=v1 --untracked-files=all)" + if [[ -n "$status" ]]; then + printf '%s\n' "$status" >&2 + exit 1 + fi + + - name: Upload bounded conformance evidence + if: ${{ always() && steps.evidence.outputs.path != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: github-dependency-inventory-${{ runner.os }}-${{ github.event.pull_request.head.sha || github.sha }} + path: ${{ steps.evidence.outputs.path }} + if-no-files-found: error + retention-days: 7 + include-hidden-files: false diff --git a/.github/workflows/github-r2-fallback.yml b/.github/workflows/github-r2-fallback.yml new file mode 100644 index 00000000..ca47d53c --- /dev/null +++ b/.github/workflows/github-r2-fallback.yml @@ -0,0 +1,187 @@ +name: GitHub R2 fallback and typed RPC canary + +on: + pull_request: + paths: + - ".github/workflows/github-r2-fallback.yml" + - "scripts/github_r2_fallback.py" + - "docs/github-r2-fallback-canary.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli commit + required: false + type: string + zed_interfaces_ref: + description: Exact 40-character zed-pkg/zed-interfaces commit + required: false + type: string + workflow_call: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli commit + required: true + type: string + zed_interfaces_ref: + description: Exact 40-character zed-pkg/zed-interfaces commit + required: true + type: string + +permissions: + contents: read + +concurrency: + group: github-r2-fallback-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + syntax: + name: Canary script compiles + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check out harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + - name: py_compile + run: python3 -m py_compile scripts/github_r2_fallback.py + + canary: + name: Registry-down CDN install + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Check out independent certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Resolve product pins + id: pins + env: + CLI_IN: ${{ inputs.zed_cli_ref || '' }} + INTERFACES_IN: ${{ inputs.zed_interfaces_ref || '' }} + run: | + set -euo pipefail + cli="${CLI_IN:-}" + interfaces="${INTERFACES_IN:-}" + if [[ -z "$cli" ]]; then + echo "zed_cli_ref is required (workflow_call or workflow_dispatch)" >&2 + exit 1 + fi + if [[ -z "$interfaces" ]]; then + echo "zed_interfaces_ref is required (workflow_call or workflow_dispatch)" >&2 + exit 1 + fi + [[ "$cli" =~ ^[0-9a-f]{40}$ ]] + [[ "$interfaces" =~ ^[0-9a-f]{40}$ ]] + printf 'cli=%s\n' "$cli" >> "$GITHUB_OUTPUT" + printf 'interfaces=%s\n' "$interfaces" >> "$GITHUB_OUTPUT" + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ steps.pins.outputs.cli }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out matching zed-interfaces + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ steps.pins.outputs.interfaces }} + path: product/zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-clients contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-clients + path: product/zed-clients + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-lib-core contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-lib-core + path: product/zed-lib-core + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable product identity + run: | + set -euo pipefail + test "$(git -C product/zed-cli rev-parse HEAD)" = "${{ steps.pins.outputs.cli }}" + test "$(git -C product/zed-interfaces rev-parse HEAD)" = "${{ steps.pins.outputs.interfaces }}" + + - name: Install Rust + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt + + - name: Install Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v6.0.0 + with: + node-version: "22" + + - name: Contract tests (interfaces, clients, lib-core) + run: | + set -euo pipefail + cargo test --manifest-path product/zed-interfaces/src/rust/Cargo.toml --lib zed_api + cargo test --manifest-path product/zed-lib-core/src/rust-orm/Cargo.toml \ + --features read-write \ + publication::tests::canonical_artifact_key_is_content_addressed + (cd product/zed-clients/clients/typescript && npm ci --ignore-scripts && npm run build && npm test) + + - name: Build exact zed CLI + run: | + set -euo pipefail + cargo fmt --all --check --manifest-path product/zed-cli/Cargo.toml + cargo build --release --manifest-path product/zed-cli/Cargo.toml --bin zed + install -Dm755 product/zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Run registry-down CDN canary + run: | + set -euo pipefail + python3 -m py_compile scripts/github_r2_fallback.py + python3 scripts/github_r2_fallback.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$RUNNER_TEMP/github-r2-fallback" \ + --route-map product/zed-interfaces/route-maps/zed-api.route-map.json \ + --interfaces-generated-ts product/zed-interfaces/generated/typescript/zed_api.ts \ + --clients-generated-ts product/zed-clients/clients/typescript/src/generated/zed-api.routes.ts + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: github-r2-fallback-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/github-r2-fallback + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/gitmodules-metadata-contract.yml b/.github/workflows/gitmodules-metadata-contract.yml new file mode 100644 index 00000000..4e8fdac9 --- /dev/null +++ b/.github/workflows/gitmodules-metadata-contract.yml @@ -0,0 +1,175 @@ +name: gitmodules metadata boundary + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: gitmodules-metadata-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: 0786987fe3a173659463940f4f905dc3b5bd19a8 + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + +jobs: + unix-black-box: + name: Black-box metadata contract / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + defaults: + run: + shell: bash + steps: + - name: Check out acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable inputs and harness source + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C zed-interfaces rev-parse HEAD)" = "$ZED_INTERFACES_REF" + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + test -z "${ZED_PKG_TOKEN:-}" + PYTHONPYCACHEPREFIX="$RUNNER_TEMP/pycache" \ + python3 -m py_compile harness/scripts/gitmodules_metadata_contract.py + + - name: Install stable Rust validation tools + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Enforce formatting and focused metadata tests + run: | + set -euo pipefail + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib \ + git_submodules::manifest_kind_tests -- --nocapture + + - name: Run all-target lint gate + if: runner.os == 'Linux' + run: cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + + - name: Build the exact release binary + run: cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + + - name: Run real Git metadata attacks + env: + WORK_ROOT: ${{ runner.temp }}/zed-gitmodules-metadata + run: | + set -euo pipefail + python3 harness/scripts/gitmodules_metadata_contract.py \ + --zed zed-cli/target/release/zed \ + --work-root "$WORK_ROOT" + + - name: Require clean source checkouts + run: | + set -euo pipefail + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-interfaces status --porcelain=v1 --untracked-files=all)" + + - name: Upload bounded metadata evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gitmodules-metadata-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-gitmodules-metadata/evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false + + windows-product-gate: + name: Windows metadata unit and lint gate + runs-on: windows-2025 + timeout-minutes: 45 + defaults: + run: + shell: pwsh + steps: + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable inputs + run: | + if ((git -C zed-cli rev-parse HEAD) -ne $env:ZED_CLI_REF) { throw "zed-cli pin mismatch" } + if ((git -C zed-interfaces rev-parse HEAD) -ne $env:ZED_INTERFACES_REF) { throw "interface pin mismatch" } + Select-String -Path zed-cli/Cargo.toml -SimpleMatch "rev = `"$env:ZED_INTERFACES_REF`"" | Out-Null + + - name: Install stable Rust validation tools + run: | + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Enforce formatting and focused metadata tests + run: | + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib git_submodules::manifest_kind_tests -- --nocapture + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Run all-target Windows lint gate + run: cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + + - name: Require clean product checkouts + run: | + if (git -C zed-cli status --porcelain=v1 --untracked-files=all) { throw "zed-cli checkout mutated" } + if (git -C zed-interfaces status --porcelain=v1 --untracked-files=all) { throw "zed-interfaces checkout mutated" } diff --git a/.github/workflows/gitops-dispatch-canary.yml b/.github/workflows/gitops-dispatch-canary.yml new file mode 100644 index 00000000..49b82546 --- /dev/null +++ b/.github/workflows/gitops-dispatch-canary.yml @@ -0,0 +1,236 @@ +name: zed gitops external dispatch canary + +on: + pull_request: + paths: + - .github/workflows/gitops-dispatch-canary.yml + - scripts/apply_den2725_help_delimiter_fix.py + - scripts/resolve_den2725_replay.py + - scripts/gitops_dispatch_canary.py + - scripts/gitops_dispatch_canary_entry.py + - scripts/gitops_dispatch_trailing_options.py + - fixtures/external_dispatch_probe.rs + - docs/gitops-dispatch-canary.md + workflow_dispatch: + inputs: + candidate: + description: Full immutable zed-pkg/zed-cli root-dispatch commit + required: false + default: 1c94813c6b59c7621c1256d7c55246e7f06f44b7 + type: string + main_commit: + description: Full immutable zed-pkg/zed-cli main commit to replay onto + required: false + default: 0246a3af901b65ca6724d936c7f3232c9f72fcd7 + type: string + +permissions: + contents: read + +concurrency: + group: gitops-dispatch-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + ZED_CLI_COMMIT: ${{ inputs.candidate || '1c94813c6b59c7621c1256d7c55246e7f06f44b7' }} + ZED_MAIN_COMMIT: ${{ inputs.main_commit || '0246a3af901b65ca6724d936c7f3232c9f72fcd7' }} + +jobs: + certify: + name: replayed gitops dispatch / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + steps: + - name: Check out independent certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact root-dispatch candidate with ancestry + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Verify immutable identities and replay current main + run: | + set -euo pipefail + [[ "$ZED_CLI_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_MAIN_COMMIT" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + python -m py_compile scripts/resolve_den2725_replay.py + rm -rf scripts/__pycache__ + git -C product/zed-cli fetch --no-tags https://github.com/zed-pkg/zed-cli.git "$ZED_MAIN_COMMIT" + test "$(git -C product/zed-cli rev-parse FETCH_HEAD)" = "$ZED_MAIN_COMMIT" + set +e + git -C product/zed-cli -c user.name='zed-pkg-test' -c user.email='zed-pkg-test@users.noreply.github.com' merge --no-commit --no-ff "$ZED_MAIN_COMMIT" + merge_status=$? + set -e + if (( merge_status != 0 )); then + conflicts="$(git -C product/zed-cli diff --name-only --diff-filter=U)" + expected_conflicts=$'src/lib.rs\nsrc/main.rs' + if [[ "$conflicts" != "$expected_conflicts" ]]; then + printf 'unexpected replay conflicts:\n%s\n' "$conflicts" >&2 + exit 1 + fi + git -C product/zed-cli checkout --theirs -- src/lib.rs src/main.rs + python scripts/resolve_den2725_replay.py --root product/zed-cli + git -C product/zed-cli add src/lib.rs src/main.rs + fi + test -z "$(git -C product/zed-cli diff --name-only --diff-filter=U)" + + - name: Apply the reviewed two-file correction in the replay tree + run: | + set -euo pipefail + python -m py_compile scripts/apply_den2725_help_delimiter_fix.py + rm -rf scripts/__pycache__ + python scripts/apply_den2725_help_delimiter_fix.py --root product/zed-cli + git -C product/zed-cli add src/external_subcommands.rs tests/external_gitops_dispatch.rs + changed="$(git -C product/zed-cli diff "$ZED_MAIN_COMMIT" --name-only)" + expected=$'.zpkg.toml\ndocs/gitops-validator.md\nscripts/validate-zed-package-graph.sh\nsrc/completion.rs\nsrc/external_subcommands.rs\nsrc/lib.rs\nsrc/main.rs\ntests/external_gitops_dispatch.rs' + test "$changed" = "$expected" + git -C product/zed-cli diff "$ZED_MAIN_COMMIT" --check + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt, clippy + + - name: Install Python 3.12 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + + - name: Run replay-tree quality gates + working-directory: product/zed-cli + run: | + set -euo pipefail + cargo fmt --all --check + cargo test --locked --lib external_subcommands::tests + cargo test --locked --test external_gitops_dispatch + cargo clippy --locked --lib --bin zed --bin zed-gitops --test external_gitops_dispatch -- -D warnings + + - name: Build replay-tree release binaries + working-directory: product/zed-cli + run: | + set -euo pipefail + cargo build --locked --release --bin zed --bin zed-gitops + + - name: Select platform paths and compile probe executable + run: | + set -euo pipefail + zed="$GITHUB_WORKSPACE/product/zed-cli/target/release/zed" + gitops="$GITHUB_WORKSPACE/product/zed-cli/target/release/zed-gitops" + probe="$RUNNER_TEMP/zed-dispatch-probe" + if [[ "$RUNNER_OS" == "Windows" ]]; then + zed="${zed}.exe" + gitops="${gitops}.exe" + probe="${probe}.exe" + fi + rustc --edition=2024 fixtures/external_dispatch_probe.rs -O -o "$probe" + echo "ZED_BINARY=$zed" >> "$GITHUB_ENV" + echo "ZED_GITOPS_BINARY=$gitops" >> "$GITHUB_ENV" + echo "DISPATCH_PROBE_BINARY=$probe" >> "$GITHUB_ENV" + + - name: Run independent black-box certification + run: | + set -euo pipefail + mkdir -p artifacts + python -m py_compile \ + scripts/gitops_dispatch_canary.py \ + scripts/gitops_dispatch_canary_entry.py \ + scripts/gitops_dispatch_trailing_options.py + rm -rf scripts/__pycache__ + python scripts/gitops_dispatch_canary_entry.py \ + --zed "$ZED_BINARY" \ + --zed-gitops "$ZED_GITOPS_BINARY" \ + --probe "$DISPATCH_PROBE_BINARY" \ + --product product/zed-cli \ + --candidate "$ZED_CLI_COMMIT" \ + --work "$RUNNER_TEMP/gitops-dispatch-canary" \ + --evidence "artifacts/gitops-dispatch-${RUNNER_OS}-${RUNNER_ARCH}.json" + python scripts/gitops_dispatch_trailing_options.py \ + --zed "$ZED_BINARY" \ + --probe "$DISPATCH_PROBE_BINARY" \ + --candidate "$ZED_CLI_COMMIT" \ + --work "$RUNNER_TEMP/gitops-dispatch-trailing" \ + --evidence "artifacts/gitops-dispatch-trailing-${RUNNER_OS}-${RUNNER_ARCH}.json" + rm -rf scripts/__pycache__ + + - name: Export the exact verified replay tree + run: | + set -euo pipefail + mkdir -p corrected/docs corrected/scripts corrected/src corrected/tests + cp product/zed-cli/.zpkg.toml corrected/ + cp product/zed-cli/docs/gitops-validator.md corrected/docs/ + cp product/zed-cli/scripts/validate-zed-package-graph.sh corrected/scripts/ + cp product/zed-cli/src/completion.rs corrected/src/ + cp product/zed-cli/src/external_subcommands.rs corrected/src/ + cp product/zed-cli/src/lib.rs corrected/src/ + cp product/zed-cli/src/main.rs corrected/src/ + cp product/zed-cli/tests/external_gitops_dispatch.rs corrected/tests/ + git -C product/zed-cli diff "$ZED_MAIN_COMMIT" --binary > corrected/den-2725-final-replay.patch + + - name: Record replay tree metadata and checksums + run: | + set -euo pipefail + tree_sha="$(git -C product/zed-cli write-tree)" + python - "$tree_sha" <<'PY' + import hashlib, json, os, sys + from pathlib import Path + root = Path('corrected') + (root / 'replay-metadata.json').write_text(json.dumps({ + '$schema': 'zed-pkg-test/den-2725-final-replay/v1', + 'rootDispatchCommit': os.environ['ZED_CLI_COMMIT'], + 'mainCommit': os.environ['ZED_MAIN_COMMIT'], + 'treeSha': sys.argv[1], + }, indent=2, sort_keys=True) + '\n', encoding='utf-8') + lines = [] + for path in sorted(item for item in root.rglob('*') if item.is_file() and item.name != 'SHA256SUMS'): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + lines.append(f"{digest} {path.as_posix()}") + (root / 'SHA256SUMS').write_text('\n'.join(lines) + '\n', encoding='utf-8') + PY + + - name: Require clean harness and exact replay delta + run: | + set -euo pipefail + harness_status="$(git status --porcelain --untracked-files=all | grep -vE '^\?\? (artifacts|corrected|product)/' || true)" + if [[ -n "$harness_status" ]]; then + printf 'unexpected harness drift:\n%s\n' "$harness_status" >&2 + exit 1 + fi + changed="$(git -C product/zed-cli diff "$ZED_MAIN_COMMIT" --name-only)" + expected=$'.zpkg.toml\ndocs/gitops-validator.md\nscripts/validate-zed-package-graph.sh\nsrc/completion.rs\nsrc/external_subcommands.rs\nsrc/lib.rs\nsrc/main.rs\ntests/external_gitops_dispatch.rs' + if [[ "$changed" != "$expected" ]]; then + printf 'unexpected replay delta:\n%s\n' "$changed" >&2 + exit 1 + fi + + - name: Upload final replay evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gitops-dispatch-replay-${{ runner.os }}-${{ runner.arch }}-${{ env.ZED_CLI_COMMIT }}-${{ env.ZED_MAIN_COMMIT }} + path: | + artifacts/gitops-dispatch-${{ runner.os }}-${{ runner.arch }}.json + artifacts/gitops-dispatch-trailing-${{ runner.os }}-${{ runner.arch }}.json + corrected + if-no-files-found: error + retention-days: 14 + include-hidden-files: false \ No newline at end of file diff --git a/.github/workflows/gitops-dispatch-current-main.yml b/.github/workflows/gitops-dispatch-current-main.yml new file mode 100644 index 00000000..0d6a54d4 --- /dev/null +++ b/.github/workflows/gitops-dispatch-current-main.yml @@ -0,0 +1,187 @@ +name: zed gitops current-main exact-head canary + +on: + pull_request: + paths: + - .github/workflows/gitops-dispatch-current-main.yml + - docs/gitops-dispatch-current-main.md + - fixtures/external_dispatch_probe.rs + - scripts/gitops_dispatch_canary.py + - scripts/gitops_dispatch_canary_entry.py + - scripts/gitops_dispatch_trailing_options.py + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: gitops-dispatch-current-main-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + ZED_CLI_COMMIT: 8eb8e31fe719ca598961a98bd139ef99ac905e8e + ZED_CLI_PARENT: de774cc9fc5b786d85e9b5ff19251675f0e9391b + +jobs: + certify: + name: exact-parent gitops dispatch / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + + steps: + - name: Check out independent certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact product candidate and parent + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 2 + persist-credentials: false + show-progress: false + + - name: Require exact immutable ancestry and eight-file delta + run: | + set -euo pipefail + [[ "$ZED_CLI_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_PARENT" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + test "$(git -C product/zed-cli rev-parse HEAD^)" = "$ZED_CLI_PARENT" + changed="$(git -C product/zed-cli diff --name-only "$ZED_CLI_PARENT" "$ZED_CLI_COMMIT")" + expected=$'.zpkg.toml\ndocs/gitops-validator.md\nscripts/validate-zed-package-graph.sh\nsrc/completion.rs\nsrc/external_subcommands.rs\nsrc/lib.rs\nsrc/main.rs\ntests/external_gitops_dispatch.rs' + if [[ "$changed" != "$expected" ]]; then + printf 'unexpected product delta:\n%s\n' "$changed" >&2 + exit 1 + fi + git -C product/zed-cli diff --check "$ZED_CLI_PARENT" "$ZED_CLI_COMMIT" + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt, clippy + + - name: Install Python 3.12 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Validate fixed harness and product quality gates + run: | + set -euo pipefail + python -m py_compile \ + scripts/gitops_dispatch_canary.py \ + scripts/gitops_dispatch_canary_entry.py \ + scripts/gitops_dispatch_trailing_options.py + rm -rf scripts/__pycache__ + cargo fmt --manifest-path product/zed-cli/Cargo.toml --all --check + cargo test --manifest-path product/zed-cli/Cargo.toml --locked --lib external_subcommands::tests + cargo test --manifest-path product/zed-cli/Cargo.toml --locked --test external_gitops_dispatch + cargo clippy --manifest-path product/zed-cli/Cargo.toml --locked \ + --lib --bin zed --bin zed-gitops --test external_gitops_dispatch -- -D warnings + bash product/zed-cli/scripts/validate-zed-package-graph.sh + + - name: Build exact release binaries + run: | + set -euo pipefail + cargo build --manifest-path product/zed-cli/Cargo.toml --locked --release \ + --bin zed --bin zed-gitops + + - name: Select platform paths and compile untrusted probe + run: | + set -euo pipefail + zed="$GITHUB_WORKSPACE/product/zed-cli/target/release/zed" + gitops="$GITHUB_WORKSPACE/product/zed-cli/target/release/zed-gitops" + probe="$RUNNER_TEMP/zed-dispatch-probe" + if [[ "$RUNNER_OS" == "Windows" ]]; then + zed="${zed}.exe" + gitops="${gitops}.exe" + probe="${probe}.exe" + fi + rustc --edition=2024 fixtures/external_dispatch_probe.rs -O -o "$probe" + echo "ZED_BINARY=$zed" >> "$GITHUB_ENV" + echo "ZED_GITOPS_BINARY=$gitops" >> "$GITHUB_ENV" + echo "DISPATCH_PROBE_BINARY=$probe" >> "$GITHUB_ENV" + + - name: Run independent black-box certification + run: | + set -euo pipefail + mkdir -p artifacts + python scripts/gitops_dispatch_canary_entry.py \ + --zed "$ZED_BINARY" \ + --zed-gitops "$ZED_GITOPS_BINARY" \ + --probe "$DISPATCH_PROBE_BINARY" \ + --product product/zed-cli \ + --candidate "$ZED_CLI_COMMIT" \ + --work "$RUNNER_TEMP/gitops-dispatch-current-main" \ + --evidence "artifacts/gitops-dispatch-${RUNNER_OS}-${RUNNER_ARCH}.json" + python scripts/gitops_dispatch_trailing_options.py \ + --zed "$ZED_BINARY" \ + --probe "$DISPATCH_PROBE_BINARY" \ + --candidate "$ZED_CLI_COMMIT" \ + --work "$RUNNER_TEMP/gitops-dispatch-trailing" \ + --evidence "artifacts/gitops-dispatch-trailing-${RUNNER_OS}-${RUNNER_ARCH}.json" + + - name: Record exact-parent evidence and require no source mutation + run: | + set -euo pipefail + git -C product/zed-cli diff --exit-code + git -C product/zed-cli diff --cached --exit-code + python - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + artifacts = Path('artifacts') + binaries = { + 'zed': Path(os.environ['ZED_BINARY']), + 'zed-gitops': Path(os.environ['ZED_GITOPS_BINARY']), + } + evidence = { + '$schema': 'zed-pkg-test/gitops-dispatch-current-main/v1', + 'candidate': os.environ['ZED_CLI_COMMIT'], + 'parent': os.environ['ZED_CLI_PARENT'], + 'runner': { + 'os': os.environ['RUNNER_OS'], + 'arch': os.environ['RUNNER_ARCH'], + }, + 'binaries': { + name: hashlib.sha256(path.read_bytes()).hexdigest() + for name, path in binaries.items() + }, + 'result': 'passed', + } + output = artifacts / f"gitops-dispatch-parent-{os.environ['RUNNER_OS']}-{os.environ['RUNNER_ARCH']}.json" + output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + '\n', encoding='utf-8') + PY + rm -rf scripts/__pycache__ + status="$(git status --porcelain --untracked-files=all | grep -vE '^\?\? (artifacts|product)/' || true)" + if [[ -n "$status" ]]; then + printf 'unexpected harness drift:\n%s\n' "$status" >&2 + exit 1 + fi + + - name: Upload immutable certification evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gitops-dispatch-current-main-${{ runner.os }}-${{ runner.arch }}-${{ env.ZED_CLI_COMMIT }} + path: artifacts + if-no-files-found: error + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/global-git-submodule-mode.yml b/.github/workflows/global-git-submodule-mode.yml new file mode 100644 index 00000000..d70aecab --- /dev/null +++ b/.github/workflows/global-git-submodule-mode.yml @@ -0,0 +1,143 @@ +name: global Git submodule compatibility mode + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: zed-global-git-submodule-mode-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ff166471338e46729b7cf892b44240452a56ace6 + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + +jobs: + contract: + name: Contract / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 50 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + + steps: + - name: Check out acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact merged zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable inputs + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C zed-interfaces rev-parse HEAD)" = "$ZED_INTERFACES_REF" + test "$(tr -d '\r\n' < harness/.zed-cli-global-git-submodule-mode-ref)" = "$ZED_CLI_REF" + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + test -z "${ZED_PKG_TOKEN:-}" + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + git --version + python3 --version + + - name: Enforce source and formatting contracts + run: | + set -euo pipefail + PYTHONPYCACHEPREFIX="$RUNNER_TEMP/python-cache" \ + python3 -m py_compile harness/scripts/git_submodule_global_mode.py + git -C harness diff --check + git -C zed-cli diff --check + git -C zed-interfaces diff --check + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + + - name: Run focused product checks + run: | + set -euo pipefail + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib \ + cli::tests::git_submodule_switch_is_global_boolish_and_does_not_consume_specs + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib \ + git_submodules::cli::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml --test git_submodule_interop_cli + if [ "$RUNNER_OS" = Linux ]; then + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + fi + + - name: Build the exact release binary + run: cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + + - name: Run the black-box compatibility contract + env: + WORK_ROOT: ${{ runner.temp }}/zed-global-git-submodule-mode + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/git_submodule_global_mode.py \ + --zed zed-cli/target/release/zed \ + --work-root "$WORK_ROOT" + python3 - "$WORK_ROOT/evidence/evidence.json" <<'PY' + import json + import pathlib + import sys + + evidence = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + assert evidence["schema"] == "zed.git-submodule-global-mode-evidence/v1" + assert len(evidence["checks"]) == 9 + assert evidence["network_credentials"] is False + assert evidence["public_registry_mutation"] is False + PY + + - name: Require clean source checkouts + run: | + set -euo pipefail + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-interfaces status --porcelain=v1 --untracked-files=all)" + + - name: Upload bounded evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: global-git-submodule-mode-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-global-git-submodule-mode/evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/global-package-profile-canary.yml b/.github/workflows/global-package-profile-canary.yml new file mode 100644 index 00000000..bfe6f327 --- /dev/null +++ b/.github/workflows/global-package-profile-canary.yml @@ -0,0 +1,95 @@ +name: Global executable package profile canary + +on: + pull_request: + paths: + - .github/workflows/global-package-profile-canary.yml + - scripts/global_package_profile_canary.py + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: global-package-profile-canary-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ZED_CLI_COMMIT: 9fa5065ed4fc291a08aa180d45f8640a3e63331f + +jobs: + canary: + name: Global package black box (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + steps: + - name: Check out certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable candidate identity + shell: bash + run: | + set -euo pipefail + test "${#ZED_CLI_COMMIT}" -eq 40 + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + mkdir -p artifacts + printf 'candidate=%s\nos=%s\narch=%s\n' \ + "$ZED_CLI_COMMIT" "$RUNNER_OS" "$RUNNER_ARCH" \ + > "artifacts/candidate-${RUNNER_OS}.txt" + + - name: Install repository Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Build exact release candidate + working-directory: product/zed-cli + shell: bash + run: | + set -euo pipefail + cargo fmt --all --check + cargo test --locked --test frozen_offline_prefetch + cargo test --locked --lib global::tests + cargo test --locked --test global_cli + cargo clippy --locked --all-targets --all-features -- -D warnings + cargo build --release --locked --bin zed + + - name: Run independent disposable-registry canary + shell: bash + run: | + set -euo pipefail + python3 -m py_compile scripts/global_package_profile_canary.py + python3 scripts/global_package_profile_canary.py \ + --zed product/zed-cli/target/release/zed \ + --work "$RUNNER_TEMP/global-package-profile" \ + --evidence artifacts/global-package-profile-${RUNNER_OS}.json + git diff --exit-code + git -C product/zed-cli diff --exit-code + + - name: Upload failure or success evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: global-package-profile-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/ide-integration-sandbox.yml b/.github/workflows/ide-integration-sandbox.yml new file mode 100644 index 00000000..82b8da86 --- /dev/null +++ b/.github/workflows/ide-integration-sandbox.yml @@ -0,0 +1,150 @@ +name: IDE integration sandbox + +on: + workflow_dispatch: + pull_request: + paths: ["ide-integrations/**", ".github/workflows/ide-integration-sandbox.yml"] + push: + branches: [main] + paths: ["ide-integrations/**", ".github/workflows/ide-integration-sandbox.yml"] + schedule: + - cron: "23 6 * * 1" + +permissions: + contents: read + +jobs: + policy: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {persist-credentials: false} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: {python-version: "3.14"} + - run: python ide-integrations/verify_matrix.py ide-integrations/parity-matrix.json + - run: python ide-integrations/verify_workflow_supply_chain.py .github/workflows/ide-integration-sandbox.yml ide-integrations/parity-matrix.json + + sublime: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: {python-version: ["3.8", "3.14"]} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-sublimetext + ref: c3153867dff0b560946c9b7e287ed50366b87e6c + path: integration + persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {path: e2e, persist-credentials: false} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: {python-version: "${{ matrix.python-version }}"} + - working-directory: integration + run: python -m compileall -q zed_pkg_insights zed_sublimetext.py + - working-directory: integration + run: python -m unittest discover -s tests -v + - run: python e2e/ide-integrations/sandbox_sublime.py integration + - working-directory: integration + run: python scripts/build-package.py + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sandbox-sublime-${{ matrix.python-version }} + path: integration/dist/ZedPackageInsights.sublime-package + if-no-files-found: error + + jetbrains: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-intellij + ref: 7282fdbc7046bb2ba58369b59ef627211033e145 + persist-credentials: false + - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 + with: {distribution: temurin, java-version: "21"} + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 + with: {gradle-version: "9.1.0"} + - run: gradle --stacktrace check buildPlugin verifyPlugin + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sandbox-jetbrains + path: build/distributions/*.zip + if-no-files-found: error + + vscode: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-vscode + ref: c1d3318a2cf8cd717f9b0af67117ddbac3e28d2e + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: {node-version: "22"} + - run: npm ci --ignore-scripts --no-audit --no-fund + - run: npm test + - run: npm run verify + - run: xvfb-run -a npm run test:extension + - run: mkdir -p dist && npm run package -- --out dist/zed-package-insights.vsix + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: {name: sandbox-vscode, path: dist/zed-package-insights.vsix, if-no-files-found: error} + + eclipse: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-eclipse + ref: 246b46048cced1d1072afd3bedc9849efe3a8a8f + persist-credentials: false + - uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 + with: {distribution: temurin, java-version: "21", cache: maven} + - run: mvn --batch-mode test + + xcode: + runs-on: macos-15 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-xcode + ref: 9b9304fee2ff236fc59407ddd97cc2c01225a69d + persist-credentials: false + - run: swift test + - run: swift build -c release + + visual-studio: + runs-on: windows-2025 + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-visual-studio + ref: d85a9c827140426a8d9401b5c269ff36f591a299 + persist-credentials: false + - uses: actions/setup-dotnet@9a946fdbd5fb07b82b2f5a4466058b876ab72bb2 # v5.3.0 + with: {dotnet-version: "8.0.x"} + - run: dotnet test Zed.VisualStudio.sln --configuration Release + + qtcreator: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-qtcreator + ref: 8e7958c7c85a305b59accfe82c28e7ff42bfecba + persist-credentials: false + - run: cmake -S . -B build -DBUILD_TESTING=ON + - run: cmake --build build --config Release --parallel 2 + - run: ctest --test-dir build -C Release --output-on-failure diff --git a/.github/workflows/inspect-cli-contract.yml b/.github/workflows/inspect-cli-contract.yml new file mode 100644 index 00000000..8d2986fb --- /dev/null +++ b/.github/workflows/inspect-cli-contract.yml @@ -0,0 +1,174 @@ +name: zed inspect v1 contract + +on: + pull_request: + paths: [".github/workflows/inspect-cli-contract.yml"] + push: + branches: [main] + paths: [".github/workflows/inspect-cli-contract.yml"] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: inspect-v1-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ZED_CLI_REF: b7036935ce3711563f1527472fc64e92bc1341aa + +jobs: + quality: + name: exact-head Rust quality + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + - name: Verify exact source and focused contracts + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib inspect::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib cli_model::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml --test inspect_cli + cargo clippy --locked --manifest-path zed-cli/Cargo.toml \ + --lib --bin zed --test inspect_cli -- -D warnings + + contract: + name: offline read-only inspect / ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + - name: Build exact release binary + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + - name: Certify one-document offline non-mutating protocol + shell: bash + env: + RUNNER_OS_NAME: ${{ runner.os }} + run: | + set -euo pipefail + python3 - <<'PY' + import hashlib, json, os, pathlib, subprocess, tempfile + + workspace = pathlib.Path.cwd() + binary = workspace / "zed-cli" / "target" / "release" / ("zed.exe" if os.name == "nt" else "zed") + assert binary.is_file(), binary + temp = pathlib.Path(os.environ.get("RUNNER_TEMP") or tempfile.gettempdir()) + root = pathlib.Path(tempfile.mkdtemp(prefix="zed-inspect-", dir=temp)).resolve() + home = pathlib.Path(tempfile.mkdtemp(prefix="zed-home-", dir=temp)).resolve() + manifest = root / ".zpkg.toml" + lock = root / ".zpkg.lock" + staging = root / ".zpkg-staging" + staging.mkdir() + sentinel = staging / "sentinel" + sentinel.write_text("must-survive", encoding="utf-8") + manifest.write_text( + '[package]\norg = "acme"\nname = "inspect-fixture"\nversion = "1.0.0"\n' + '\n[package.repository]\nvcs = "git"\nurl = "https://example.invalid/acme/inspect-fixture"\n', + encoding="utf-8", + ) + lock.write_text("version = 1\n", encoding="utf-8") + (home / "credentials.toml").write_text("malformed = [credential", encoding="utf-8") + + def snapshot(path): + rows = {} + for item in sorted(path.rglob("*")): + relative = item.relative_to(path).as_posix() + if item.is_symlink(): rows[relative] = ["symlink", os.readlink(item)] + elif item.is_file(): rows[relative] = ["file", hashlib.sha256(item.read_bytes()).hexdigest()] + elif item.is_dir(): rows[relative] = ["dir"] + return rows + + environment = os.environ.copy() + environment.update({ + "ZED_PKG_HOME": str(home), + "ZED_PKG_TOKEN": "fake-env-token-that-must-not-escape", + "ZED_PKG_REGISTRY": "http://127.0.0.1:9", + }) + + def inspect(): + return subprocess.run( + [str(binary), "--token", "fake-arg-token-that-must-not-escape", + "inspect", "--format", "json", "--root", str(root)], + cwd=root, env=environment, text=True, capture_output=True, + timeout=15, check=False, + ) + + before = snapshot(root) + result = inspect() + after = snapshot(root) + assert result.returncode == 0, result.stderr + assert result.stderr == "", result.stderr + assert result.stdout.count("\n") == 1, result.stdout + report = json.loads(result.stdout) + assert report["schema_version"] == "1.0", report + assert report["cli"] == { + "implementation": "zed-pkg", "command": "inspect", "offline": True, + "mutates_project": False, "loads_credentials": False, + }, report + assert report["summary"]["recovery_pending"] is True, report + action = next(d for d in report["diagnostics"] if d["code"] == "RECOVERY_PENDING")["actions"][0] + assert action == { + "label": action["label"], "argv": ["zed", "install", "--frozen"], + "mutates_project": True, "requires_network": True, + "executes_package_code": True, + }, action + assert before == after + assert sentinel.read_text(encoding="utf-8") == "must-survive" + combined = result.stdout + result.stderr + assert "fake-env-token-that-must-not-escape" not in combined + assert "fake-arg-token-that-must-not-escape" not in combined + + secret = "gh" + "p_fake_manifest_secret_that_must_not_escape" + manifest.write_text(f'[package\nsecret = "{secret}"\n', encoding="utf-8") + malformed_before = snapshot(root) + malformed = inspect() + assert malformed.returncode == 0, malformed.stderr + malformed_report = json.loads(malformed.stdout) + assert malformed_report["summary"]["health"] == "error" + assert any(d["code"] == "MANIFEST_INVALID" for d in malformed_report["diagnostics"]) + assert secret not in malformed.stdout + malformed.stderr + assert malformed_before == snapshot(root) + + relative = subprocess.run( + [str(binary), "inspect", "--format", "json", "--root", "relative"], + cwd=root, env=environment, text=True, capture_output=True, + timeout=15, check=False, + ) + assert relative.returncode != 0 + assert not (root / "relative").exists() + print("certified", os.environ.get("RUNNER_OS_NAME"), os.environ["ZED_CLI_REF"]) + PY diff --git a/.github/workflows/inspect-current-main.yml b/.github/workflows/inspect-current-main.yml new file mode 100644 index 00000000..6ffd2e57 --- /dev/null +++ b/.github/workflows/inspect-current-main.yml @@ -0,0 +1,203 @@ +name: zed inspect current-main exact-head canary + +on: + pull_request: + paths: + - .github/workflows/inspect-current-main.yml + - docs/inspect-current-main.md + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: inspect-current-main-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ZED_CLI_COMMIT: b7036935ce3711563f1527472fc64e92bc1341aa + ZED_CLI_BASE: 1ab18fcb2ff884e82af4cac4513d7b983a23c84a + +jobs: + certify: + name: exact-base inspect / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + steps: + - name: Check out independent certification repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact product candidate and history + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Require immutable base ancestry and five-file delta + shell: bash + run: | + set -euo pipefail + [[ "$ZED_CLI_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_BASE" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + git -C product/zed-cli merge-base --is-ancestor "$ZED_CLI_BASE" "$ZED_CLI_COMMIT" + changed="$(git -C product/zed-cli diff --name-only "$ZED_CLI_BASE" "$ZED_CLI_COMMIT")" + expected=$'schemas/inspect-v1.schema.json\nsrc/cli_model.rs\nsrc/inspect.rs\nsrc/lib.rs\ntests/inspect_cli.rs' + test "$changed" = "$expected" + git -C product/zed-cli diff --check "$ZED_CLI_BASE" "$ZED_CLI_COMMIT" + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Install Python 3.12 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v6.0.0 + with: + python-version: "3.12" + check-latest: false + + - name: Run exact-head Rust quality gates + shell: bash + run: | + set -euo pipefail + cargo fmt --manifest-path product/zed-cli/Cargo.toml --all --check + cargo test --manifest-path product/zed-cli/Cargo.toml --locked --lib inspect::tests + cargo test --manifest-path product/zed-cli/Cargo.toml --locked --lib cli_model::tests + cargo test --manifest-path product/zed-cli/Cargo.toml --locked --test inspect_cli + cargo clippy --manifest-path product/zed-cli/Cargo.toml --locked \ + --lib --bin zed --test inspect_cli -- -D warnings + cargo build --manifest-path product/zed-cli/Cargo.toml --locked --release --bin zed + + - name: Run adversarial read-only CLI contract + shell: bash + env: + RUNNER_OS_NAME: ${{ runner.os }} + RUNNER_ARCH_NAME: ${{ runner.arch }} + run: | + set -euo pipefail + mkdir -p artifacts + python3 - <<'PY' + import hashlib, json, os, pathlib, subprocess, tempfile + + candidate = os.environ["ZED_CLI_COMMIT"] + base = os.environ["ZED_CLI_BASE"] + workspace = pathlib.Path.cwd() + binary = workspace / "product" / "zed-cli" / "target" / "release" / ("zed.exe" if os.name == "nt" else "zed") + assert binary.is_file(), binary + sensitive = ("TOKEN", "SECRET", "PASSWORD", "PRIVATE_KEY", "ACCESS_KEY", "API_KEY", "AUTHORIZATION", "COOKIE", "CREDENTIAL") + + def environment(home, fake=False): + clean = {k: v for k, v in os.environ.items() + if not k.upper().startswith("ZED_PKG_") and not any(marker in k.upper() for marker in sensitive)} + clean["ZED_PKG_HOME"] = str(home) + clean["ZED_PKG_REGISTRY"] = "http://127.0.0.1:9" + if fake: clean["ZED_PKG_TOKEN"] = "fake-env-token-that-must-not-escape" + return clean + + def snapshot(root): + if not root.exists(): return None + rows = {} + for item in sorted(root.rglob("*")): + rel = item.relative_to(root).as_posix() + if item.is_symlink(): rows[rel] = ["symlink", os.readlink(item)] + elif item.is_file(): rows[rel] = ["file", hashlib.sha256(item.read_bytes()).hexdigest()] + elif item.is_dir(): rows[rel] = ["dir"] + return rows + + temp = pathlib.Path(os.environ.get("RUNNER_TEMP") or tempfile.gettempdir()) + sandbox = pathlib.Path(tempfile.mkdtemp(prefix="inspect-v1-", dir=temp)) + project, home = sandbox / "project", sandbox / "home" + project.mkdir(); home.mkdir() + manifest, lock = project / ".zpkg.toml", project / ".zpkg.lock" + staging = project / ".zpkg-staging"; staging.mkdir() + sentinel = staging / "sentinel"; sentinel.write_text("must-survive", encoding="utf-8") + manifest.write_text('[package]\norg = "acme"\nname = "fixture"\nversion = "1.0.0"\n\n[package.repository]\nvcs = "git"\nurl = "https://example.invalid/acme/fixture"\n', encoding="utf-8") + lock.write_text("version = 1\n", encoding="utf-8") + (home / "credentials.toml").write_text("malformed = [credential", encoding="utf-8") + + def invoke(args, *, cwd=project, use_home=home, fake=False): + return subprocess.run([str(binary), *args], cwd=cwd, env=environment(use_home, fake), + text=True, capture_output=True, timeout=15, check=False) + + before_project, before_home = snapshot(project), snapshot(home) + result = invoke(["--token", "fake-arg-token-that-must-not-escape", "inspect", "--format", "json", "--root", str(project.resolve())], fake=True) + assert result.returncode == 0, result.stderr + assert result.stderr == "" and result.stdout.count("\n") == 1 + report = json.loads(result.stdout) + assert report["schema_version"] == "1.0" + assert report["cli"] == {"implementation": "zed-pkg", "command": "inspect", "offline": True, "mutates_project": False, "loads_credentials": False} + assert report["summary"]["recovery_pending"] is True + recovery = next(d for d in report["diagnostics"] if d["code"] == "RECOVERY_PENDING") + action = next(a for a in recovery["actions"] if a["argv"] == ["zed", "install", "--frozen"]) + assert action["mutates_project"] and action["requires_network"] and action["executes_package_code"] + assert snapshot(project) == before_project and snapshot(home) == before_home + assert sentinel.read_text(encoding="utf-8") == "must-survive" + assert "fake-env-token-that-must-not-escape" not in result.stdout + result.stderr + assert "fake-arg-token-that-must-not-escape" not in result.stdout + result.stderr + + secret = "gh" + "p_fake_manifest_secret_that_must_not_escape" + manifest.write_text(f'[package\nsecret = "{secret}"\n', encoding="utf-8") + malformed_before = snapshot(project) + malformed = invoke(["inspect", "--format", "json", "--root", str(project.resolve())]) + assert malformed.returncode == 0 and malformed.stderr == "" + malformed_report = json.loads(malformed.stdout) + assert malformed_report["summary"]["health"] == "error" + assert any(d["code"] == "MANIFEST_INVALID" for d in malformed_report["diagnostics"]) + assert secret not in malformed.stdout + malformed.stderr + assert snapshot(project) == malformed_before + + help_result = invoke(["inspect", "--help"]) + assert help_result.returncode == 0 and help_result.stderr == "" and "--root" in help_result.stdout + token_home = sandbox / "token-home" + token_result = invoke(["--token", "inspect", "--help"], cwd=sandbox, use_home=token_home) + assert token_result.returncode == 0 and '"schema_version"' not in token_result.stdout and not token_home.exists() + + relative = invoke(["inspect", "--root", "relative-project"], cwd=sandbox, use_home=sandbox / "relative-home") + assert relative.returncode != 0 and not (sandbox / "relative-project").exists() + missing_root = sandbox / "missing-project" + missing = invoke(["inspect", "--root", str(missing_root)], cwd=sandbox, use_home=sandbox / "missing-home") + assert missing.returncode != 0 and not missing_root.exists() + + evidence = { + "$schema": "zed-pkg-test/inspect-current-main/v1", + "candidate": candidate, + "base": base, + "runner": {"os": os.environ["RUNNER_OS_NAME"], "arch": os.environ["RUNNER_ARCH_NAME"]}, + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "result": "passed", + } + output = workspace / "artifacts" / f"inspect-{os.environ['RUNNER_OS_NAME']}-{os.environ['RUNNER_ARCH_NAME']}.json" + output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + PY + + - name: Require immutable product and harness source + shell: bash + run: | + set -euo pipefail + git -C product/zed-cli diff --exit-code + git -C product/zed-cli diff --cached --exit-code + status="$(git status --porcelain --untracked-files=all | grep -vE '^\?\? (artifacts|product)/' || true)" + test -z "$status" + + - name: Upload commit-addressed evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: inspect-current-main-${{ runner.os }}-${{ runner.arch }}-${{ env.ZED_CLI_COMMIT }} + path: artifacts + if-no-files-found: error + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/install-boundaries.yml b/.github/workflows/install-boundaries.yml index db9b4492..129d0ce8 100644 --- a/.github/workflows/install-boundaries.yml +++ b/.github/workflows/install-boundaries.yml @@ -9,7 +9,7 @@ on: zed_cli_ref: description: Immutable zed-cli commit to test required: false - default: 8929a58c6591c7a964d7f91412665d7c8a4afdf3 + default: 3a91d64abf1fc0bdee6eab2ff804113b1429849f schedule: - cron: "41 6 * * 0" @@ -25,7 +25,7 @@ defaults: shell: bash env: - ZED_CLI_REF: ${{ inputs.zed_cli_ref || '8929a58c6591c7a964d7f91412665d7c8a4afdf3' }} + ZED_CLI_REF: ${{ inputs.zed_cli_ref || '3a91d64abf1fc0bdee6eab2ff804113b1429849f' }} ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 NODE_LIB_REF: 2d41e658a382fdff27ef87a0db24dbaff280a685 NODE_APP_REF: ef794674647b3620e711f4f855865f8ad3340498 diff --git a/.github/workflows/install-recovery.yml b/.github/workflows/install-recovery.yml new file mode 100644 index 00000000..c7d20b5c --- /dev/null +++ b/.github/workflows/install-recovery.yml @@ -0,0 +1,135 @@ +name: install recovery matrix + +on: + pull_request: + paths: + - scripts/install_recovery_matrix.py + - .github/workflows/install-recovery.yml + push: + branches: [main] + paths: + - scripts/install_recovery_matrix.py + - .github/workflows/install-recovery.yml + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to test + required: false + default: 8cc6745c75fd46dabf2e383cbcfc14db5ff933da + +permissions: + contents: read + +concurrency: + group: install-recovery-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '8cc6745c75fd46dabf2e383cbcfc14db5ff933da' }} + ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 + +jobs: + install-recovery: + name: Same-project races, rollback, repair, and tamper failures + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out e2e harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out pinned zed-interfaces sibling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Compile the recovery harness and check Rust formatting + run: | + set -euo pipefail + python3 -m py_compile harness/scripts/install_recovery_matrix.py + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + + - name: Build immutable zed binary + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + install -Dm755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Run install recovery and rollback matrix + env: + WORK_ROOT: ${{ runner.temp }}/zed-install-recovery + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/install_recovery_matrix.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-install-recovery + run: | + set +e + mkdir -p "$WORK_ROOT/diagnostics" + find "$WORK_ROOT" -maxdepth 11 -printf '%y %P -> %l\n' \ + | LC_ALL=C sort > "$WORK_ROOT/diagnostics/work-tree.txt" 2>&1 + find "$WORK_ROOT/registry" -maxdepth 10 -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/registry-sha256.txt" 2>&1 + find "$WORK_ROOT" -path '*/cache/*.tar.gz' -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/cache-sha256.txt" 2>&1 + find "$WORK_ROOT" -path '*/.zpkg.lock' -type f -print -exec sed -n '1,240p' {} \; \ + > "$WORK_ROOT/diagnostics/lockfiles.txt" 2>&1 + true + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: install-recovery-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-install-recovery/diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/leddy-package-graph.yml b/.github/workflows/leddy-package-graph.yml new file mode 100644 index 00000000..0b759fbc --- /dev/null +++ b/.github/workflows/leddy-package-graph.yml @@ -0,0 +1,146 @@ +name: Leddy package graph canary + +on: + pull_request: + paths: + - ".github/workflows/leddy-package-graph.yml" + - "scripts/leddy-package-graph.sh" + push: + branches: [main] + paths: + - ".github/workflows/leddy-package-graph.yml" + - "scripts/leddy-package-graph.sh" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: leddy-package-graph-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + # This candidate canonicalizes a polyglot target rooted at `dir = "."` to + # the root package identity, so the Leddy repository target publishes as + # led-dynamo/leddy-clients rather than the obsolete target-suffixed name. + ZED_CLI_REF: 5d1d13119be5784f545a7d812cdfaae3ca120693 + LEDDY_INTERFACES_REF: 7d8ef79f1353aa6ed23e47739172a7500cab6f25 + LEDDY_LIB_REF: 2a9a0cd43180d4be1f09b1fd98ec00ead9203795 + LEDDY_CLIENTS_REF: c672cedcbcfc71b0310cde2e494c407cecd6ae32 + LEDDY_MONOREPO_REF: 37ac9bd7c1868b75ae59e63a4d931071ade05123 + +jobs: + package-graph: + name: Leddy graph / ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + steps: + - name: Require immutable pins + run: | + set -euo pipefail + for pin in "$ZED_CLI_REF" "$LEDDY_INTERFACES_REF" "$LEDDY_LIB_REF" "$LEDDY_CLIENTS_REF" "$LEDDY_MONOREPO_REF"; do + [[ "$pin" =~ ^[0-9a-f]{40}$ ]] + done + + - name: Check out test harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + persist-credentials: false + fetch-depth: 1 + show-progress: false + + - name: Check out exact zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + persist-credentials: false + fetch-depth: 1 + show-progress: false + + - name: Check out exact Leddy interfaces + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: led-dynamo/leddy-interfaces + ref: ${{ env.LEDDY_INTERFACES_REF }} + path: leddy-interfaces + persist-credentials: false + fetch-depth: 1 + show-progress: false + + - name: Check out exact Leddy lib + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: led-dynamo/leddy-lib + ref: ${{ env.LEDDY_LIB_REF }} + path: leddy-lib + persist-credentials: false + fetch-depth: 1 + show-progress: false + + - name: Check out exact Leddy clients + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: led-dynamo/leddy-clients + ref: ${{ env.LEDDY_CLIENTS_REF }} + path: leddy-clients + persist-credentials: false + fetch-depth: 1 + show-progress: false + + - name: Check out exact Leddy monorepo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: led-dynamo/leddy-monorepo + ref: ${{ env.LEDDY_MONOREPO_REF }} + path: leddy-monorepo + persist-credentials: false + fetch-depth: 1 + show-progress: false + + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + workspaces: zed-cli + shared-key: leddy-package-graph-${{ matrix.os }} + + - name: Build exact zed binary + run: | + set -euo pipefail + bash -n harness/scripts/leddy-package-graph.sh + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + zed-cli/target/release/zed --version + + - name: Run isolated Leddy package graph canary + run: | + set -o pipefail + bash harness/scripts/leddy-package-graph.sh \ + "$GITHUB_WORKSPACE/zed-cli/target/release/zed" \ + "$GITHUB_WORKSPACE/leddy-interfaces" \ + "$GITHUB_WORKSPACE/leddy-lib" \ + "$GITHUB_WORKSPACE/leddy-clients" \ + "$GITHUB_WORKSPACE/leddy-monorepo" \ + "$RUNNER_TEMP/leddy-package-graph" \ + 2>&1 | tee "$RUNNER_TEMP/leddy-package-graph.log" + + - name: Upload bounded diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: leddy-package-graph-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/leddy-package-graph.log + ${{ runner.temp }}/leddy-package-graph/**/*.log + ${{ runner.temp }}/leddy-package-graph/**/.zpkg.lock + if-no-files-found: warn + retention-days: 7 + include-hidden-files: true diff --git a/.github/workflows/lifecycle-harness-contract.yml b/.github/workflows/lifecycle-harness-contract.yml new file mode 100644 index 00000000..3e3fc978 --- /dev/null +++ b/.github/workflows/lifecycle-harness-contract.yml @@ -0,0 +1,40 @@ +name: lifecycle harness contract + +on: + pull_request: + paths: + - ".github/workflows/lifecycle-harness-contract.yml" + - "scripts/lifecycle.py" + - "tests/test_lifecycle_metadata.py" + push: + branches: [main] + paths: + - ".github/workflows/lifecycle-harness-contract.yml" + - "scripts/lifecycle.py" + - "tests/test_lifecycle_metadata.py" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: lifecycle-harness-contract-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + metadata-contract: + name: package classification and dependency-source metadata + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check out lifecycle harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Compile the harness + run: python3 -m py_compile scripts/lifecycle.py tests/test_lifecycle_metadata.py + + - name: Run lifecycle metadata regressions + run: python3 -m unittest discover -s tests -p 'test_lifecycle*.py' -v diff --git a/.github/workflows/lifecycle-source-map.yml b/.github/workflows/lifecycle-source-map.yml new file mode 100644 index 00000000..c8db2dee --- /dev/null +++ b/.github/workflows/lifecycle-source-map.yml @@ -0,0 +1,112 @@ +name: lifecycle fixture source-map contract + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: lifecycle-source-map-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + PYTHONDONTWRITEBYTECODE: "1" + +jobs: + source-map: + name: Package classification and live dependency map + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out lifecycle harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact Python consumer fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/python-app + ref: ad26b7c7e9706ccec2905a31cbcef67a78e0c217 + path: fixtures/python-app + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact shared-schema fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/shared-schema + ref: 41cec5076791555d4b72e72da0a42888da0522d0 + path: fixtures/shared-schema + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact polyglot source fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/polyglot-lib + ref: 998964aa58e49595547650b4a8da407b7a2283a9 + path: fixtures/polyglot-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Parse and run source-map contract + env: + FIXTURE_ROOT: ${{ github.workspace }}/fixtures + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + + for raw in ( + "harness/scripts/lifecycle.py", + "harness/tests/test_lifecycle_source_map.py", + ): + path = Path(raw) + compile(path.read_text(encoding="utf-8"), str(path), "exec") + PY + python3 -B harness/tests/test_lifecycle_source_map.py -v 2>&1 \ + | tee "$RUNNER_TEMP/lifecycle-source-map-test.log" + + - name: Verify read-only fixture state + run: | + set -euo pipefail + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C fixtures/python-app status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C fixtures/shared-schema status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C fixtures/polyglot-lib status --porcelain=v1 --untracked-files=all)" + + - name: Capture source-map evidence + if: always() + run: | + set +e + evidence="$RUNNER_TEMP/lifecycle-source-map-evidence" + mkdir -p "$evidence" + cp "$RUNNER_TEMP/lifecycle-source-map-test.log" "$evidence/" 2>/dev/null + sha256sum harness/scripts/lifecycle.py > "$evidence/lifecycle.py.sha256" + git -C harness status --porcelain=v1 --untracked-files=all \ + > "$evidence/harness-status.txt" + + - name: Upload source-map evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: lifecycle-source-map-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/lifecycle-source-map-evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/lifecycle.yml b/.github/workflows/lifecycle.yml index 0c620465..642288dd 100644 --- a/.github/workflows/lifecycle.yml +++ b/.github/workflows/lifecycle.yml @@ -7,9 +7,9 @@ on: workflow_dispatch: inputs: zed_cli_ref: - description: Immutable zed-cli commit to test + description: Optional immutable zed-cli commit override required: false - default: 8929a58c6591c7a964d7f91412665d7c8a4afdf3 + default: "" repository_dispatch: types: [fixture-updated] schedule: @@ -29,8 +29,7 @@ defaults: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: "1" - ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '8929a58c6591c7a964d7f91412665d7c8a4afdf3' }} - ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 + ZED_INTERFACES_REF: 8428bc574111fa148e590c8350c7855035ce2046 jobs: build-zed: @@ -38,17 +37,36 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 25 steps: - - name: Require immutable component pins + - name: Check out lifecycle harness and reviewed pin + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Load immutable component pins + id: pins run: | set -euo pipefail - [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + override="${{ github.event.inputs.zed_cli_ref || '' }}" + if [[ -n "$override" ]]; then + zed_cli_ref="$override" + else + zed_cli_ref="$(tr -d '[:space:]' < harness/.zed-cli-ref)" + fi + [[ "$zed_cli_ref" =~ ^[0-9a-f]{40}$ ]] [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + printf 'zed_cli_ref=%s\n' "$zed_cli_ref" >> "$GITHUB_OUTPUT" + printf 'ZED_CLI_REF=%s\n' "$zed_cli_ref" >> "$GITHUB_ENV" + printf 'zed-cli=%s\nzed-interfaces=%s\n' \ + "$zed_cli_ref" "$ZED_INTERFACES_REF" - name: Check out zed-cli uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: zed-pkg/zed-cli - ref: ${{ env.ZED_CLI_REF }} + ref: ${{ steps.pins.outputs.zed_cli_ref }} path: zed-cli fetch-depth: 1 persist-credentials: false @@ -64,6 +82,13 @@ jobs: persist-credentials: false show-progress: false + - name: Enforce exact interface provenance + run: | + set -euo pipefail + test "$(git -C zed-interfaces rev-parse HEAD)" = "$ZED_INTERFACES_REF" + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + grep -F "rev=$ZED_INTERFACES_REF#$ZED_INTERFACES_REF" zed-cli/Cargo.lock + - name: Install the declared Rust toolchain run: | set -euo pipefail @@ -95,6 +120,90 @@ jobs: retention-days: 7 include-hidden-files: false + frozen-fetch-canary: + name: External frozen fetch bundle + needs: build-zed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Check out lifecycle harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out a real package fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-lib + ref: main + path: fixture + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Download immutable zed binary + uses: actions/download-artifact@974686ed5098c7f9c9289ec946b9058e496a2561 # v8.0.0 + with: + name: zed-cli-${{ github.run_id }}-${{ github.run_attempt }} + path: tooling + + - name: Verify binary checksum and fresh work root + env: + WORK_ROOT: ${{ runner.temp }}/zed-frozen-fetch-canary + run: | + set -euo pipefail + chmod +x tooling/zed + (cd tooling && sha256sum --check SHA256SUMS) + test ! -e "$WORK_ROOT" + tooling/zed --version + + - name: Publish, lock, uninstall, fetch, replay, and reject tampering + env: + WORK_ROOT: ${{ runner.temp }}/zed-frozen-fetch-canary + run: | + set -euo pipefail + python3 harness/scripts/frozen_fetch_canary.py \ + --fixture-dir fixture \ + --zed tooling/zed \ + --work-root "$WORK_ROOT" + + - name: Reject missing, non-directory, and redirected output parents + env: + WORK_ROOT: ${{ runner.temp }}/zed-frozen-fetch-canary + run: | + set -euo pipefail + python3 harness/scripts/frozen_fetch_output_boundaries.py \ + --zed tooling/zed \ + --work-root "$WORK_ROOT" + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-frozen-fetch-canary + run: | + set +e + mkdir -p "$WORK_ROOT/diagnostics" + git -C fixture status --porcelain=v1 --untracked-files=all \ + > "$WORK_ROOT/diagnostics/fixture-status.txt" 2>&1 + find "$WORK_ROOT" -maxdepth 10 -printf '%y %P -> %l\n' \ + | LC_ALL=C sort > "$WORK_ROOT/diagnostics/work-tree.txt" 2>&1 + find "$WORK_ROOT/registry" -maxdepth 8 -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/registry-sha256.txt" 2>&1 + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: frozen-fetch-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-frozen-fetch-canary/diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false + lifecycle: name: Lifecycle / ${{ matrix.repo }} needs: build-zed @@ -202,16 +311,19 @@ jobs: lifecycle-gate: name: All fixture lifecycles passed if: always() - needs: [build-zed, lifecycle] + needs: [build-zed, frozen-fetch-canary, lifecycle] runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Enforce aggregate result env: BUILD_RESULT: ${{ needs.build-zed.result }} + FETCH_RESULT: ${{ needs.frozen-fetch-canary.result }} MATRIX_RESULT: ${{ needs.lifecycle.result }} run: | set -euo pipefail - printf 'build-zed=%s\nlifecycle=%s\n' "$BUILD_RESULT" "$MATRIX_RESULT" + printf 'build-zed=%s\nfrozen-fetch=%s\nlifecycle=%s\n' \ + "$BUILD_RESULT" "$FETCH_RESULT" "$MATRIX_RESULT" test "$BUILD_RESULT" = success + test "$FETCH_RESULT" = success test "$MATRIX_RESULT" = success diff --git a/.github/workflows/live-test-org-fleet-audit.yml b/.github/workflows/live-test-org-fleet-audit.yml new file mode 100644 index 00000000..092968ff --- /dev/null +++ b/.github/workflows/live-test-org-fleet-audit.yml @@ -0,0 +1,193 @@ +name: Live paired test-fleet audit + +on: + pull_request: + paths: + - 'bootstrap/test-org-fleet.json.gz.b64.parts/**' + - 'scripts/audit-live-test-org-fleet.mjs' + - 'scripts/export-test-org-fleet-index.mjs' + - 'scripts/read-fleet-manifest.mjs' + - 'tests/live-test-org-fleet-audit.test.mjs' + - '.github/workflows/live-test-org-fleet-audit.yml' + push: + branches: [main] + paths: + - 'bootstrap/test-org-fleet.json.gz.b64.parts/**' + - 'scripts/audit-live-test-org-fleet.mjs' + - 'scripts/export-test-org-fleet-index.mjs' + - 'scripts/read-fleet-manifest.mjs' + - 'tests/live-test-org-fleet-audit.test.mjs' + - '.github/workflows/live-test-org-fleet-audit.yml' + schedule: + - cron: '43 7 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: live-paired-test-fleet-audit-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + unit: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + - name: Validate auditor and canonical exporter + run: | + node --check scripts/audit-live-test-org-fleet.mjs + node --test tests/live-test-org-fleet-audit.test.mjs + node scripts/export-test-org-fleet-index.mjs > canonical-test-fleet-index.json + node - <<'NODE' + const fs = require('node:fs'); + const index = JSON.parse(fs.readFileSync('canonical-test-fleet-index.json', 'utf8')); + if (index.pairCount < 1) throw new Error('canonical exporter returned no pairs'); + if (index.expectedRepositoryCount < index.managedRepositoryCount) throw new Error('expected count is below managed count'); + if (!index.excludedOrganizations.map((value) => value.toLowerCase()).includes('r2g')) throw new Error('r2g exclusion missing'); + if (!index.excludedOrganizations.map((value) => value.toLowerCase()).includes('r2g-test')) throw new Error('r2g-test exclusion missing'); + NODE + + live: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + env: + # The repository-scoped GITHUB_TOKEN cannot enumerate private repositories + # in sibling organizations. Configure a read-only cross-org token or an + # approved short-lived GitHub App installation token as FLEET_AUDIT_TOKEN. + GH_TOKEN: ${{ secrets.FLEET_AUDIT_TOKEN }} + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Require complete cross-organization visibility + shell: bash + run: | + set -Eeuo pipefail + if [[ -z "${GH_TOKEN:-}" ]]; then + { + echo '# Live paired test-fleet audit' + echo + echo '**Result:** not executed because `FLEET_AUDIT_TOKEN` is not configured.' + echo + echo 'The normal repository `GITHUB_TOKEN` is intentionally rejected because private sibling repositories would otherwise be reported as missing.' + } >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + echo "::add-mask::$GH_TOKEN" + + - name: Export the current canonical denominator + run: node scripts/export-test-org-fleet-index.mjs > canonical-test-fleet-index.json + + - name: Compare all paired organizations with the current canonical manifest + id: audit + shell: bash + run: | + set +e + node scripts/audit-live-test-org-fleet.mjs --json \ + > live-test-fleet-audit.json \ + 2> live-test-fleet-audit.stderr + status=$? + set -e + printf 'exit_code=%s\n' "$status" >> "$GITHUB_OUTPUT" + test -e live-test-fleet-audit.stderr || : > live-test-fleet-audit.stderr + + - name: Summarize and enforce the complete current machine-readable report + if: always() && steps.audit.outcome != 'skipped' + shell: bash + env: + AUDIT_EXIT_CODE: ${{ steps.audit.outputs.exit_code }} + run: | + node - <<'NODE' + const fs = require('node:fs'); + const reportPath = 'live-test-fleet-audit.json'; + const indexPath = 'canonical-test-fleet-index.json'; + const stderrPath = 'live-test-fleet-audit.stderr'; + const summaryPath = process.env.GITHUB_STEP_SUMMARY; + const auditExitCode = Number(process.env.AUDIT_EXIT_CODE ?? '1'); + const token = process.env.GH_TOKEN ?? ''; + const redact = (value) => token ? value.split(token).join('[REDACTED]') : value; + const stderr = fs.existsSync(stderrPath) ? redact(fs.readFileSync(stderrPath, 'utf8')).slice(0, 4000) : ''; + const appendSummary = (text) => { if (summaryPath) fs.appendFileSync(summaryPath, `${text}\n`); }; + + const index = JSON.parse(fs.readFileSync(indexPath, 'utf8')); + let report; + try { + const raw = fs.readFileSync(reportPath, 'utf8'); + if (!raw.trim()) throw new Error('audit JSON is empty'); + report = JSON.parse(raw); + } catch (error) { + appendSummary('# Live paired test-fleet audit'); + appendSummary(''); + appendSummary(`**Result:** auditor failed before producing valid JSON (exit ${auditExitCode}).`); + if (stderr) appendSummary(`\n\`\`\`text\n${stderr}\n\`\`\``); + throw error; + } + + appendSummary('# Live paired test-fleet audit'); + appendSummary(''); + appendSummary(`Canonical repositories present: **${report.presentRepositories}/${report.expectedRepositories}**`); + appendSummary(''); + appendSummary(`Pairs: **${report.pairCount}** · Missing: **${report.missingRepositories}** · Extras preserved: **${report.extraRepositories}** · Canonical hygiene findings: **${report.expectedHygieneFindings}**`); + appendSummary(''); + appendSummary('| Production org | Test org | Canonical | Present | Missing | Extras | Hygiene |'); + appendSummary('|---|---|---:|---:|---:|---:|---:|'); + for (const item of report.reports) { + appendSummary(`| \`${item.sourceOrganization}\` | \`${item.testOrganization}\` | ${item.expectedRepositories} | ${item.present.length} | ${item.missing.length} | ${item.extras.length} | ${item.expectedHygiene.length} |`); + } + const missing = report.reports.filter((item) => item.missing.length); + if (missing.length) { + appendSummary(''); + appendSummary('## Missing canonical repositories'); + appendSummary(''); + for (const item of missing) appendSummary(`- **${item.testOrganization}:** ${item.missing.map((name) => `\`${name}\``).join(', ')}`); + } + if (stderr) { + appendSummary(''); + appendSummary('## Auditor diagnostics'); + appendSummary(''); + appendSummary(`\`\`\`text\n${stderr}\n\`\`\``); + } + + const compact = { + auditExitCode, + canonicalPairCount: index.pairCount, + canonicalExpectedRepositories: index.expectedRepositoryCount, + pairCount: report.pairCount, + expectedRepositories: report.expectedRepositories, + presentRepositories: report.presentRepositories, + missingRepositories: report.missingRepositories, + extraRepositories: report.extraRepositories, + expectedHygieneFindings: report.expectedHygieneFindings, + }; + console.log(JSON.stringify(compact, null, 2)); + + const failures = []; + if (auditExitCode !== 0) failures.push(`auditor exited ${auditExitCode}`); + if (report.pairCount !== index.pairCount) failures.push(`pair count ${report.pairCount} != exporter ${index.pairCount}`); + if (report.expectedRepositories !== index.expectedRepositoryCount) failures.push(`expected repository count ${report.expectedRepositories} != exporter ${index.expectedRepositoryCount}`); + if (report.missingRepositories !== 0) failures.push(`missing ${report.missingRepositories} canonical repositories`); + if (report.expectedHygieneFindings !== 0) failures.push(`found ${report.expectedHygieneFindings} canonical repository hygiene failures`); + if (failures.length) throw new Error(failures.join('; ')); + NODE + + - name: Upload complete live-audit evidence + if: always() && steps.audit.outcome != 'skipped' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-paired-test-fleet-audit-${{ github.run_id }}-${{ github.run_attempt }} + path: | + canonical-test-fleet-index.json + live-test-fleet-audit.json + live-test-fleet-audit.stderr + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/materialize-den1462-retry.yml b/.github/workflows/materialize-den1462-retry.yml new file mode 100644 index 00000000..e106920a --- /dev/null +++ b/.github/workflows/materialize-den1462-retry.yml @@ -0,0 +1,130 @@ +name: materialize corrected DEN-1462 export + +on: + pull_request: + paths: + - .github/workflows/materialize-den1462-retry.yml + +permissions: + contents: read + +concurrency: + group: materialize-den1462-retry-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + DEN1462_REF: a914849da3c8e2556abccd437e6e539f320dcb3e + +jobs: + materialize: + name: Validate and package DEN-1462 + runs-on: ubuntu-24.04 + timeout-minutes: 50 + steps: + - name: Check out exact export candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.DEN1462_REF }} + path: source + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable export candidate + run: test "$(git -C source rev-parse HEAD)" = "$DEN1462_REF" + + - name: Install Rust validation toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + + - name: Apply the complete reviewed export sequence + working-directory: source + run: | + set -euo pipefail + python3 scripts/apply_den1462_mise_export.py + python3 scripts/harden_den1462_export.py + python3 scripts/fix_den1462_flags_contract.py + python3 scripts/fix_den1462_selector_paths.py + python3 scripts/harden_den1462_export_boundaries.py + cargo fmt --all + cargo fmt --all --check + git diff --check + cargo test --locked mise_export -- --nocapture + cargo test --locked --test mise_export_cli -- --nocapture + cargo test --locked cli::tests::environment_export_is_typed_and_rejects_ambiguous_write_modes -- --nocapture + cargo test --locked cli::tests::cli_flags_toml_is_in_sync_with_clap -- --nocapture + cargo clippy --locked --all-targets -- -D warnings + + - name: Remove temporary source materializers + working-directory: source + run: | + set -euo pipefail + rm -f \ + .github/workflows/apply-den-1462-mise-export.yml \ + .github/workflows/apply-den-1462-export-boundaries.yml \ + scripts/apply_den1462_mise_export.py \ + scripts/harden_den1462_export.py \ + scripts/fix_den1462_flags_contract.py \ + scripts/fix_den1462_selector_paths.py \ + scripts/harden_den1462_export_boundaries.py + git diff --check + + - name: Package the ordinary source delta + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + import shutil + import subprocess + import tarfile + + source = Path('source') + stage = Path('/tmp/den1462-materialized') + if stage.exists(): + shutil.rmtree(stage) + stage.mkdir(parents=True) + + def lines(*args: str) -> list[str]: + output = subprocess.check_output(['git', '-C', str(source), *args], text=True) + return [line for line in output.splitlines() if line] + + changed = set(lines('diff', '--name-only', '--diff-filter=ACMRTUXB', 'HEAD')) + changed.update(lines('ls-files', '--others', '--exclude-standard')) + deleted = sorted(set(lines('diff', '--name-only', '--diff-filter=D', 'HEAD'))) + changed = sorted(changed) + + for relative in changed: + src = source / relative + if not src.is_file(): + raise SystemExit(f'changed path is not a regular file: {relative}') + dst = stage / 'files' / relative + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + (stage / 'base-sha.txt').write_text( + subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True) + ) + (stage / 'changed-files.txt').write_text('\n'.join(changed) + ('\n' if changed else '')) + (stage / 'deleted-files.txt').write_text('\n'.join(deleted) + ('\n' if deleted else '')) + + with tarfile.open('/tmp/den1462-materialized.tar.gz', 'w:gz') as archive: + archive.add(stage, arcname='den1462-materialized') + PY + + - name: Upload validated DEN-1462 delta + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den1462-materialized-corrected + path: /tmp/den1462-materialized.tar.gz + if-no-files-found: error + retention-days: 7 + include-hidden-files: false diff --git a/.github/workflows/materialize-global-packages-current-main.yml b/.github/workflows/materialize-global-packages-current-main.yml new file mode 100644 index 00000000..eb2bd2fa --- /dev/null +++ b/.github/workflows/materialize-global-packages-current-main.yml @@ -0,0 +1,169 @@ +name: Materialize global packages on current main + +on: + pull_request: + branches: + - main + paths: + - .github/workflows/materialize-global-packages-current-main.yml + - scripts/materialize_global_packages_current_main.py + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: materialize-global-packages-current-main-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + PRODUCT_MAIN: 9b8e5f3d3b98845436899b9a8645f3ce19689f85 + REVIEWED_FEATURE: c5ed46eb80c348aa5b766e9b00d7c50bfb94e0ab + +jobs: + materialize: + name: Current-main product materialization + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Check out materializer without credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact product main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.PRODUCT_MAIN }} + path: product + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact reviewed feature source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.REVIEWED_FEATURE }} + path: feature + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable inputs + shell: bash + run: | + set -euo pipefail + test "${#PRODUCT_MAIN}" -eq 40 + test "${#REVIEWED_FEATURE}" -eq 40 + test "$(git -C product rev-parse HEAD)" = "$PRODUCT_MAIN" + test "$(git -C feature rev-parse HEAD)" = "$REVIEWED_FEATURE" + + - name: Install repository Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Assemble ordinary current-main source + shell: bash + run: | + set -euo pipefail + # Register the six reviewed new files as intent-to-add before the + # materializer copies their bytes. Ordinary `git diff` then includes + # them in the exact allowlist, patch, and artifact inventory without + # staging any content or mutating the source commit. + while IFS= read -r relative; do + mkdir -p "product/$(dirname "$relative")" + : > "product/$relative" + done <<'EOF' + .github/workflows/global-packages.yml + .zpkg.lock + .zpkg.toml + docs/global-packages.md + src/global.rs + tests/global_cli.rs + EOF + git -C product add -N -- \ + .github/workflows/global-packages.yml \ + .zpkg.lock \ + .zpkg.toml \ + docs/global-packages.md \ + src/global.rs \ + tests/global_cli.rs + + python3 -m py_compile scripts/materialize_global_packages_current_main.py + python3 scripts/materialize_global_packages_current_main.py \ + --product product \ + --feature feature + + - name: Verify focused and complete product gates + working-directory: product + shell: bash + run: | + set -euo pipefail + cargo fmt --all --check + cargo test --locked --lib global::tests + cargo test --locked frozen_local_metadata_preserves_lock_identity_and_provenance + cargo test --locked --test global_cli + cargo clippy --locked --all-targets --all-features -- -D warnings + cargo test --locked --all-targets --all-features + cargo build --release --locked --bin zed + ./target/release/zed --version + + - name: Produce exact ordinary-source artifact + shell: bash + run: | + set -euo pipefail + mkdir -p artifacts + git -C product diff --binary > artifacts/global-packages-current-main.patch + git -C product diff --name-only -z > artifacts/changed-files.zlist + python3 - <<'PY' + import json + import subprocess + from pathlib import Path + + product = Path('product') + names = [ + item.decode() + for item in subprocess.check_output( + ['git', '-C', str(product), 'diff', '--name-only', '-z'] + ).split(b'\0') + if item + ] + Path('artifacts/changed-files.json').write_text( + json.dumps(names, indent=2) + '\n' + ) + Path('artifacts/materialization.json').write_text( + json.dumps( + { + 'schema': 'zed.global-packages-materialization/v1', + 'product_base': '9b8e5f3d3b98845436899b9a8645f3ce19689f85', + 'reviewed_feature': 'c5ed46eb80c348aa5b766e9b00d7c50bfb94e0ab', + 'changed_files': names, + }, + sort_keys=True, + indent=2, + ) + '\n' + ) + PY + tar --null -czf artifacts/global-packages-current-main-files.tar.gz \ + -C product -T artifacts/changed-files.zlist + sha256sum \ + artifacts/global-packages-current-main.patch \ + artifacts/global-packages-current-main-files.tar.gz \ + artifacts/changed-files.json \ + artifacts/materialization.json \ + > artifacts/SHA256SUMS + test -z "$(git -C feature status --porcelain)" + + - name: Upload validated source delta + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: global-packages-current-main-source + path: artifacts/ + if-no-files-found: error + retention-days: 3 diff --git a/.github/workflows/materialize-native-registry-canary-merged-product.yml b/.github/workflows/materialize-native-registry-canary-merged-product.yml new file mode 100644 index 00000000..34ab4fc7 --- /dev/null +++ b/.github/workflows/materialize-native-registry-canary-merged-product.yml @@ -0,0 +1,130 @@ +name: materialize native-registry canary for merged product + +on: + push: + branches: + - agent/native-registry-canary-merged-product + paths: + - .github/workflows/materialize-native-registry-canary-merged-product.yml + +permissions: + contents: read + +concurrency: + group: materialize-native-registry-canary-merged-product-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + materialize: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + contents: write + steps: + - name: Check out clean current-main canary branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: agent/native-registry-canary-merged-product + path: harness + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Check out merged native-registry product + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: 29addea1aca138133632c453b142c8fb0e0535cb + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Materialize reviewed five-file canary delta + working-directory: harness + run: | + set -euo pipefail + git fetch --no-tags origin agent/native-registry-contract-canary + source_ref=origin/agent/native-registry-contract-canary + for path in \ + .github/workflows/native-registry-contract.yml \ + docs/native-registry-contract-canary.md \ + fixtures/native-registry-consumer/Cargo.toml \ + fixtures/native-registry-consumer/src/main.rs \ + fixtures/native-registry-consumer/tests/topology.rs; do + mkdir -p "$(dirname "$path")" + git show "$source_ref:$path" > "$path" + done + python3 - <<'PY' + from pathlib import Path + + merged = "29addea1aca138133632c453b142c8fb0e0535cb" + stale = { + "a6aaad6911c845be7d8069d67f9bef82c85452b7", + "ac8b36c0261ad59cf73b3b06c78c13be8370fcfa", + "134dfa55562ca6dd5d5a0c3c7499ba410826ba69", + } + for raw in ( + ".github/workflows/native-registry-contract.yml", + "docs/native-registry-contract-canary.md", + "fixtures/native-registry-consumer/Cargo.toml", + "fixtures/native-registry-consumer/src/main.rs", + "fixtures/native-registry-consumer/tests/topology.rs", + ): + path = Path(raw) + text = path.read_text() + for candidate in stale: + text = text.replace(candidate, merged) + path.write_text(text) + PY + git diff --check + + - name: Install stable Rust quality toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + + - name: Certify merged product and external consumer + run: | + set -euo pipefail + cd zed-interfaces + cargo fmt --all -- --check + cargo test --locked --all-targets + cargo clippy --locked --all-targets --all-features -- -D warnings + cargo run --locked --example generate_schemas + git diff --exit-code -- schemas/native-registry-adapter-record.json + cd ../harness + fixture=fixtures/native-registry-consumer + cp ../zed-interfaces/Cargo.lock "$fixture/Cargo.lock" + cargo run --offline --manifest-path "$fixture/Cargo.toml" + cargo run --locked --offline --manifest-path "$fixture/Cargo.toml" + cargo test --offline --manifest-path "$fixture/Cargo.toml" + cargo test --locked --offline --manifest-path "$fixture/Cargo.toml" + + - name: Commit only the current-main canary product + working-directory: harness + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + rm .github/workflows/materialize-native-registry-canary-merged-product.yml + git add \ + .github/workflows/native-registry-contract.yml \ + docs/native-registry-contract-canary.md \ + fixtures/native-registry-consumer/Cargo.toml \ + fixtures/native-registry-consumer/src/main.rs \ + fixtures/native-registry-consumer/tests/topology.rs \ + .github/workflows/materialize-native-registry-canary-merged-product.yml + git diff --cached --check + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit -m 'test(DEN-1420): certify merged native-registry product' + git push \ + "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + HEAD:agent/native-registry-canary-merged-product diff --git a/.github/workflows/materialize-zed-mise-slices.yml b/.github/workflows/materialize-zed-mise-slices.yml new file mode 100644 index 00000000..7dd29308 --- /dev/null +++ b/.github/workflows/materialize-zed-mise-slices.yml @@ -0,0 +1,229 @@ +name: materialize blocked mise slices + +on: + pull_request: + paths: + - .github/workflows/materialize-zed-mise-slices.yml + +permissions: + contents: read + +concurrency: + group: materialize-zed-mise-slices-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + DEN1461_REF: 3c2bc7f50c052640e93be4ccd4cf7211bce1db30 + DEN1462_REF: b71b16f3caee7119474df5c72ce9bf2d0530138b + +jobs: + lock-contract: + name: Materialize DEN-1461 + runs-on: ubuntu-24.04 + timeout-minutes: 50 + steps: + - name: Check out exact lock-contract candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.DEN1461_REF }} + path: source + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable lock candidate + run: test "$(git -C source rev-parse HEAD)" = "$DEN1461_REF" + + - name: Install Rust validation toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + + - name: Apply complete reviewed wire-format and semantic hardening + working-directory: source + run: | + set -euo pipefail + python3 scripts/apply_den1461_current_wire.py + python3 scripts/harden_den1461_lock.py + python3 scripts/fix_den1461_enum_size.py + cargo fmt --all + cargo fmt --all --check + git diff --check + cargo test --locked --lib mise_lock::tests -- --nocapture + cargo test --locked --test mise_lock_contract -- --nocapture + cargo clippy --locked --all-targets -- -D warnings + + - name: Remove temporary source-repository materializers + working-directory: source + run: | + set -euo pipefail + rm -f \ + .github/workflows/harden-den-1461-lock-order.yml \ + .github/workflows/apply-den-1461-current-wire.yml \ + scripts/apply_den1461_current_wire.py \ + scripts/harden_den1461_lock.py \ + scripts/fix_den1461_enum_size.py + git diff --check + + - name: Package ordinary source delta + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + import shutil + import subprocess + import tarfile + + source = Path('source') + stage = Path('/tmp/den1461-materialized') + if stage.exists(): + shutil.rmtree(stage) + stage.mkdir(parents=True) + + def lines(*args: str) -> list[str]: + output = subprocess.check_output(['git', '-C', str(source), *args], text=True) + return [line for line in output.splitlines() if line] + + changed = set(lines('diff', '--name-only', '--diff-filter=ACMRTUXB', 'HEAD')) + changed.update(lines('ls-files', '--others', '--exclude-standard')) + deleted = sorted(set(lines('diff', '--name-only', '--diff-filter=D', 'HEAD'))) + changed = sorted(changed) + + for relative in changed: + src = source / relative + if not src.is_file(): + raise SystemExit(f'changed path is not a regular file: {relative}') + dst = stage / 'files' / relative + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + (stage / 'base-sha.txt').write_text( + subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True) + ) + (stage / 'changed-files.txt').write_text('\n'.join(changed) + ('\n' if changed else '')) + (stage / 'deleted-files.txt').write_text('\n'.join(deleted) + ('\n' if deleted else '')) + + with tarfile.open('/tmp/den1461-materialized.tar.gz', 'w:gz') as archive: + archive.add(stage, arcname='den1461-materialized') + PY + + - name: Upload validated DEN-1461 delta + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den1461-materialized + path: /tmp/den1461-materialized.tar.gz + if-no-files-found: error + retention-days: 7 + include-hidden-files: false + + mise-export: + name: Materialize DEN-1462 + runs-on: ubuntu-24.04 + timeout-minutes: 50 + steps: + - name: Check out exact export candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.DEN1462_REF }} + path: source + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify immutable export candidate + run: test "$(git -C source rev-parse HEAD)" = "$DEN1462_REF" + + - name: Install Rust validation toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + + - name: Apply complete typed CLI integration and hardening + working-directory: source + run: | + set -euo pipefail + python3 scripts/apply_den1462_mise_export.py + python3 scripts/harden_den1462_export.py + python3 scripts/fix_den1462_flags_contract.py + python3 scripts/fix_den1462_selector_paths.py + cargo fmt --all + cargo fmt --all --check + git diff --check + cargo test --locked mise_export -- --nocapture + cargo test --locked --test mise_export_cli -- --nocapture + cargo test --locked cli::tests::environment_export_is_typed_and_rejects_ambiguous_write_modes -- --nocapture + cargo test --locked cli::tests::cli_flags_toml_is_in_sync_with_clap -- --nocapture + cargo clippy --locked --all-targets -- -D warnings + + - name: Remove temporary source-repository materializers + working-directory: source + run: | + set -euo pipefail + rm -f \ + .github/workflows/apply-den-1462-mise-export.yml \ + scripts/apply_den1462_mise_export.py \ + scripts/harden_den1462_export.py \ + scripts/fix_den1462_flags_contract.py \ + scripts/fix_den1462_selector_paths.py + git diff --check + + - name: Package ordinary source delta + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + import shutil + import subprocess + import tarfile + + source = Path('source') + stage = Path('/tmp/den1462-materialized') + if stage.exists(): + shutil.rmtree(stage) + stage.mkdir(parents=True) + + def lines(*args: str) -> list[str]: + output = subprocess.check_output(['git', '-C', str(source), *args], text=True) + return [line for line in output.splitlines() if line] + + changed = set(lines('diff', '--name-only', '--diff-filter=ACMRTUXB', 'HEAD')) + changed.update(lines('ls-files', '--others', '--exclude-standard')) + deleted = sorted(set(lines('diff', '--name-only', '--diff-filter=D', 'HEAD'))) + changed = sorted(changed) + + for relative in changed: + src = source / relative + if not src.is_file(): + raise SystemExit(f'changed path is not a regular file: {relative}') + dst = stage / 'files' / relative + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + (stage / 'base-sha.txt').write_text( + subprocess.check_output(['git', '-C', str(source), 'rev-parse', 'HEAD'], text=True) + ) + (stage / 'changed-files.txt').write_text('\n'.join(changed) + ('\n' if changed else '')) + (stage / 'deleted-files.txt').write_text('\n'.join(deleted) + ('\n' if deleted else '')) + + with tarfile.open('/tmp/den1462-materialized.tar.gz', 'w:gz') as archive: + archive.add(stage, arcname='den1462-materialized') + PY + + - name: Upload validated DEN-1462 delta + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den1462-materialized + path: /tmp/den1462-materialized.tar.gz + if-no-files-found: error + retention-days: 7 + include-hidden-files: false diff --git a/.github/workflows/merge-app-audited-ready.yml b/.github/workflows/merge-app-audited-ready.yml new file mode 100644 index 00000000..4564cec9 --- /dev/null +++ b/.github/workflows/merge-app-audited-ready.yml @@ -0,0 +1,90 @@ +name: merge audited ready PRs with per-installation tokens + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-APP-AUDITED-READY" + +permissions: + contents: read + +concurrency: + group: merge-app-audited-ready + cancel-in-progress: false + +jobs: + merge: + if: "${{ github.event.head_commit.message == 'chore: merge audited ready PRs with app tokens' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 6 + matrix: + include: + - owner: discrete-event-systems + repository: discrete-event-systems/des-simulate.rs + pr: 113 + head: e21a25508f7c2c7512f4cf09020dec1d6fbe2299 + - owner: canonical-cloud + repository: canonical-cloud/canonical-cloud-api-server.rs + pr: 46 + head: b72c620e9784eaac3b0ca70630e64de3227a186f + - owner: ORESoftware + repository: ORESoftware/ai-agent-coordinator.rs + pr: 56 + head: 689b2575d3d52cf6b204adde1613efe8de5c5cbc + - owner: opto-sync + repository: opto-sync/syncer.rs + pr: 2 + head: e99cf3c8303739fddd2b28c025b5067d776088bb + - owner: opto-sync + repository: opto-sync/opto-sync-clients + pr: 40 + head: 098a94f040723d19783e8ff6daa3bc265dfa6eec + - owner: opto-sync + repository: opto-sync/indexeddb-schema.rs + pr: 2 + head: 8563baf28a6d44e4eea0c70ab526f8339a719360 + - owner: opter-io + repository: opter-io/opter-sync + pr: 3 + head: 7078d21d8411106e4185b20c8c04d22b854b3119 + - owner: canonical-cloud + repository: canonical-cloud/canonical-cloud-macos-app.swift + pr: 11 + head: fb8cf26f9c92f0f439411b58fc069ceff31b8313 + - owner: benefactor-cc + repository: benefactor-cc/benefactor-dart-client + pr: 18 + head: ceb8ea2f692cc6bb277b9485ab4896f01999c60b + - owner: benefactor-cc + repository: benefactor-cc/benefactor-api-server.rs + pr: 17 + head: ee881e9ada5c4d5516ab2f80c6810e841b4c19e2 + steps: + - name: Mint a short-lived installation token for ${{ matrix.owner }} + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 + with: + client-id: ${{ vars.FLEET_APP_CLIENT_ID }} + private-key: ${{ secrets.FLEET_APP_PRIVATE_KEY }} + owner: ${{ matrix.owner }} + permission-contents: write + permission-metadata: read + permission-pull-requests: write + + - name: Check out trusted exact-head gate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Revalidate, merge, and post-verify the exact audited head + shell: bash + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + TARGET_REPOSITORY: ${{ matrix.repository }} + TARGET_PR: ${{ matrix.pr }} + TARGET_HEAD: ${{ matrix.head }} + run: node .github/scripts/merge-app-audited-ready.mjs diff --git a/.github/workflows/merge-audited-ready-batch.yml b/.github/workflows/merge-audited-ready-batch.yml new file mode 100644 index 00000000..4f61b65a --- /dev/null +++ b/.github/workflows/merge-audited-ready-batch.yml @@ -0,0 +1,38 @@ +name: merge audited ready batch + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-AUDITED-BATCH" + +permissions: + contents: read + +concurrency: + group: merge-audited-ready-batch + cancel-in-progress: false + +jobs: + merge: + if: "${{ github.event.head_commit.message == 'chore: merge audited ready batch' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Check out the exact audited merger + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Revalidate and merge audited exact heads + shell: bash + run: node .github/scripts/merge-audited-ready-batch.mjs diff --git a/.github/workflows/merge-graphql-frozen-ready.yml b/.github/workflows/merge-graphql-frozen-ready.yml new file mode 100644 index 00000000..30214b1a --- /dev/null +++ b/.github/workflows/merge-graphql-frozen-ready.yml @@ -0,0 +1,31 @@ +name: merge frozen ready PRs through GraphQL + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-GRAPHQL-FROZEN-READY" + +permissions: + contents: read + +concurrency: + group: merge-graphql-frozen-ready + cancel-in-progress: false + +jobs: + merge: + if: "${{ github.event.head_commit.message == 'chore: merge frozen ready PRs through GraphQL' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted GraphQL-only merger + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Revalidate, merge, and verify frozen exact heads + shell: bash + run: node .github/scripts/merge-graphql-frozen-ready.mjs diff --git a/.github/workflows/merge-graphql-ready-portfolios-v2.yml b/.github/workflows/merge-graphql-ready-portfolios-v2.yml new file mode 100644 index 00000000..564e13df --- /dev/null +++ b/.github/workflows/merge-graphql-ready-portfolios-v2.yml @@ -0,0 +1,89 @@ +name: merge ready portfolios through GraphQL v2 + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS-V2" + +permissions: + contents: read + +concurrency: + group: merge-graphql-ready-portfolios + cancel-in-progress: false + +jobs: + discover: + if: "${{ github.event.head_commit.message == 'chore: rerun ready portfolios through GraphQL' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted split-query discovery + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Discover exact heads through bounded GraphQL queries and public git + shell: bash + run: node .github/scripts/discover-graphql-ready-portfolios-v2.mjs + - name: Upload immutable exact-head portfolio archives + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: graphql-ready-portfolio-bundle-v2 + path: graphql-portfolio-bundle + if-no-files-found: error + retention-days: 1 + + validate: + needs: discover + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out trusted credential-free validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Download exact archives without a fleet credential + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: graphql-ready-portfolio-bundle-v2 + path: portfolio-bundle + - name: Install only factory contract test dependencies + shell: bash + run: python3 -m pip install 'pytest>=8,<10' 'PyYAML>=6,<7' + - name: Validate exact archives with every GitHub credential removed + shell: bash + env: + GH_TOKEN: "" + GITHUB_TOKEN: "" + FLEET_GH_TOKEN: "" + run: python3 .github/scripts/validate-ready-portfolios.py + - name: Upload credential-free validation result + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: graphql-ready-portfolio-validation-v2 + path: portfolio-validated + if-no-files-found: error + retention-days: 1 + + merge: + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted GraphQL exact-head merger + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Download credential-free validation result + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: graphql-ready-portfolio-validation-v2 + path: portfolio-validated + - name: Revalidate, promote, merge, and exact-head verify through GraphQL + shell: bash + run: node .github/scripts/merge-graphql-validated-portfolios.mjs diff --git a/.github/workflows/merge-graphql-ready-portfolios.yml b/.github/workflows/merge-graphql-ready-portfolios.yml new file mode 100644 index 00000000..cf43f0e4 --- /dev/null +++ b/.github/workflows/merge-graphql-ready-portfolios.yml @@ -0,0 +1,97 @@ +name: merge ready portfolios through GraphQL and credential-free validation + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-GRAPHQL-READY-PORTFOLIOS" + +permissions: + contents: read + +concurrency: + group: merge-graphql-ready-portfolios + cancel-in-progress: false + +jobs: + discover: + if: "${{ github.event.head_commit.message == 'chore: merge ready portfolios through GraphQL' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted GraphQL and git discovery + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Discover exact heads through GraphQL and archive public branches with git + shell: bash + run: node .github/scripts/discover-graphql-ready-portfolios.mjs + + - name: Upload immutable exact-head portfolio archives + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: graphql-ready-portfolio-bundle + path: graphql-portfolio-bundle + if-no-files-found: error + retention-days: 1 + + validate: + needs: discover + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out trusted credential-free validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Download exact archives without a fleet credential + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: graphql-ready-portfolio-bundle + path: portfolio-bundle + + - name: Install only the factory contract test dependencies + shell: bash + run: python3 -m pip install 'pytest>=8,<10' 'PyYAML>=6,<7' + + - name: Validate exact archives with every GitHub credential removed + shell: bash + env: + GH_TOKEN: "" + GITHUB_TOKEN: "" + FLEET_GH_TOKEN: "" + run: python3 .github/scripts/validate-ready-portfolios.py + + - name: Upload credential-free validation result + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: graphql-ready-portfolio-validation + path: portfolio-validated + if-no-files-found: error + retention-days: 1 + + merge: + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted GraphQL exact-head merger + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Download credential-free validation result + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: graphql-ready-portfolio-validation + path: portfolio-validated + + - name: Revalidate, promote, merge, and exact-head verify through GraphQL + shell: bash + run: node .github/scripts/merge-graphql-validated-portfolios.mjs diff --git a/.github/workflows/merge-open-pr-fleet.yml b/.github/workflows/merge-open-pr-fleet.yml new file mode 100644 index 00000000..7ef885af --- /dev/null +++ b/.github/workflows/merge-open-pr-fleet.yml @@ -0,0 +1,122 @@ +name: merge open PR fleet + +on: + pull_request: + paths: + - ".github/workflows/merge-open-pr-fleet.yml" + - ".github/scripts/merge-open-pr-fleet.mjs" + push: + branches: [main] + paths: + - ".github/pr-merge/PREFLIGHT" + - ".github/pr-merge/APPLY" + +permissions: + contents: read + +concurrency: + group: merge-open-pr-fleet-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + validate: + if: github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out the exact workflow and merger implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Validate workflow syntax + uses: docker://rhysd/actionlint@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 + with: + args: .github/workflows/merge-open-pr-fleet.yml + + - name: Validate merger JavaScript syntax + run: node --check .github/scripts/merge-open-pr-fleet.mjs + + - name: Enforce the guarded trigger contract + shell: bash + run: | + set -euo pipefail + workflow=.github/workflows/merge-open-pr-fleet.yml + grep -Fq 'branches: [main]' "$workflow" + grep -Fq '.github/pr-merge/PREFLIGHT' "$workflow" + grep -Fq '.github/pr-merge/APPLY' "$workflow" + grep -Fq "github.event.head_commit.message == 'chore: preflight safe PR fleet merge'" "$workflow" + grep -Fq "github.event.head_commit.message == 'chore: merge safe PR fleet'" "$workflow" + grep -Fq 'permissions:' "$workflow" + grep -Fq 'contents: read' "$workflow" + python3 - <<'PY' + from pathlib import Path + + text = Path('.github/workflows/merge-open-pr-fleet.yml').read_text(encoding='utf-8') + opener = '$' + '{{' + expected_group = ( + 'group: merge-open-pr-fleet-' + + opener + + ' github.event_name }}-' + + opener + + ' github.ref }}' + ) + expected_cancel = ( + 'cancel-in-progress: ' + + opener + + " github.event_name == 'pull_request' }}" + ) + assert expected_group in text, 'event/ref-scoped concurrency group is missing' + assert expected_cancel in text, 'pull-request cancellation contract is missing' + assert not any(ord(character) < 0x20 and character not in '\n\t\r' for character in text) + PY + if grep -Eq -- '(^|[[:space:]])--force(-with-lease)?([[:space:]]|$)|contents:[[:space:]]*write' "$workflow"; then + echo 'merge workflow must not use a force-update flag or grant the workflow token write access' >&2 + exit 1 + fi + + merge: + if: >- + github.event_name == 'push' && + (github.event.head_commit.message == 'chore: preflight safe PR fleet merge' || + github.event.head_commit.message == 'chore: merge safe PR fleet') + runs-on: ubuntu-24.04 + timeout-minutes: 180 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + TARGET: "509" + steps: + - name: Resolve the exact guarded mode + shell: bash + env: + HEAD_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + set -Eeuo pipefail + case "$HEAD_COMMIT_MESSAGE" in + 'chore: preflight safe PR fleet merge') + echo 'APPLY=false' >> "$GITHUB_ENV" + ;; + 'chore: merge safe PR fleet') + echo 'APPLY=true' >> "$GITHUB_ENV" + ;; + *) + echo 'unexpected merge workflow commit message' >&2 + exit 1 + ;; + esac + + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Check out the exact merger implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Audit and optionally merge exact heads + shell: bash + run: node .github/scripts/merge-open-pr-fleet.mjs diff --git a/.github/workflows/merge-ready-portfolios-v2.yml b/.github/workflows/merge-ready-portfolios-v2.yml new file mode 100644 index 00000000..2297ed85 --- /dev/null +++ b/.github/workflows/merge-ready-portfolios-v2.yml @@ -0,0 +1,119 @@ +name: merge independently validated ready portfolios v2 + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-READY-PORTFOLIOS-V2" + +permissions: + contents: read + +concurrency: + group: merge-ready-portfolios + cancel-in-progress: false + +jobs: + discover: + if: "${{ github.event.head_commit.message == 'chore: rerun independently validated ready portfolios' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored fleet credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$FLEET_GH_TOKEN" + GH_TOKEN="$FLEET_GH_TOKEN" gh api user --jq '.login' | grep -Fx ORESoftware + + - name: Check out trusted fleet scripts + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Discover exact ready portfolios and download immutable archives + shell: bash + run: node .github/scripts/ready-portfolio-fleet.mjs discover + + - name: Upload immutable portfolio bundle + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ready-portfolio-bundle-v2 + path: portfolio-bundle + if-no-files-found: error + retention-days: 1 + + validate: + needs: discover + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out trusted credential-free validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Download immutable archives without a fleet credential + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ready-portfolio-bundle-v2 + path: portfolio-bundle + + - name: Normalize GitHub tarball roots without executing portfolio code + shell: bash + env: + GH_TOKEN: "" + GITHUB_TOKEN: "" + FLEET_GH_TOKEN: "" + run: python3 .github/scripts/normalize-portfolio-archives.py + + - name: Install only the factory contract test dependencies + shell: bash + run: python3 -m pip install 'pytest>=8,<10' 'PyYAML>=6,<7' + + - name: Validate exact archives with all GitHub credentials removed + shell: bash + env: + GH_TOKEN: "" + GITHUB_TOKEN: "" + FLEET_GH_TOKEN: "" + run: python3 .github/scripts/validate-ready-portfolios.py + + - name: Upload validation results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ready-portfolio-validation-v2 + path: portfolio-validated + if-no-files-found: error + retention-days: 1 + + merge: + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored fleet credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$FLEET_GH_TOKEN" + GH_TOKEN="$FLEET_GH_TOKEN" gh api user --jq '.login' | grep -Fx ORESoftware + + - name: Check out trusted exact-head merger + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Download credential-free validation result + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ready-portfolio-validation-v2 + path: portfolio-validated + + - name: Revalidate, promote, merge, and post-verify exact heads + shell: bash + run: node .github/scripts/ready-portfolio-fleet.mjs merge diff --git a/.github/workflows/merge-ready-portfolios.yml b/.github/workflows/merge-ready-portfolios.yml new file mode 100644 index 00000000..e2747ef8 --- /dev/null +++ b/.github/workflows/merge-ready-portfolios.yml @@ -0,0 +1,111 @@ +name: merge independently validated ready portfolios + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-READY-PORTFOLIOS" + +permissions: + contents: read + +concurrency: + group: merge-ready-portfolios + cancel-in-progress: false + +jobs: + discover: + if: "${{ github.event.head_commit.message == 'chore: merge independently validated ready portfolios' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored fleet credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$FLEET_GH_TOKEN" + GH_TOKEN="$FLEET_GH_TOKEN" gh api user --jq '.login' | grep -Fx ORESoftware + + - name: Check out trusted fleet scripts + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Discover exact ready portfolios and download immutable archives + shell: bash + run: node .github/scripts/ready-portfolio-fleet.mjs discover + + - name: Upload immutable portfolio bundle + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ready-portfolio-bundle + path: portfolio-bundle + if-no-files-found: error + retention-days: 1 + + validate: + needs: discover + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out trusted credential-free validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Download immutable archives without a fleet credential + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ready-portfolio-bundle + path: portfolio-bundle + + - name: Install only the factory contract test dependencies + shell: bash + run: python3 -m pip install 'pytest>=8,<10' 'PyYAML>=6,<7' + + - name: Validate exact archives with all GitHub credentials removed + shell: bash + env: + GH_TOKEN: "" + GITHUB_TOKEN: "" + FLEET_GH_TOKEN: "" + run: python3 .github/scripts/validate-ready-portfolios.py + + - name: Upload signed-by-workflow validation results + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: ready-portfolio-validation + path: portfolio-validated + if-no-files-found: error + retention-days: 1 + + merge: + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Validate stored fleet credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$FLEET_GH_TOKEN" + GH_TOKEN="$FLEET_GH_TOKEN" gh api user --jq '.login' | grep -Fx ORESoftware + + - name: Check out trusted exact-head merger + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Download credential-free validation result + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: ready-portfolio-validation + path: portfolio-validated + + - name: Revalidate, promote, merge, and post-verify exact heads + shell: bash + run: node .github/scripts/ready-portfolio-fleet.mjs merge diff --git a/.github/workflows/merge-resolved-legacy-harnesses.yml b/.github/workflows/merge-resolved-legacy-harnesses.yml new file mode 100644 index 00000000..49f4f1ae --- /dev/null +++ b/.github/workflows/merge-resolved-legacy-harnesses.yml @@ -0,0 +1,86 @@ +name: merge source-resolved legacy harnesses + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/MERGE-RESOLVED-LEGACY-HARNESSES" + +permissions: + contents: read + +concurrency: + group: merge-resolved-legacy-harnesses + cancel-in-progress: false + +jobs: + discover: + if: "${{ github.event.head_commit.message == 'chore: merge source-resolved legacy harnesses' }}" + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted GraphQL discovery + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Discover exact source-resolved draft heads and archive via public git + shell: bash + run: node .github/scripts/discover-resolved-legacy-harnesses.mjs + - name: Upload exact immutable archives + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: resolved-legacy-harness-bundle + path: legacy-harness-bundle + if-no-files-found: error + retention-days: 1 + + validate: + needs: discover + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Check out trusted credential-free validator + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Download exact archives without a fleet credential + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: resolved-legacy-harness-bundle + path: legacy-harness-bundle + - name: Validate test plans and workflow safety with all GitHub credentials removed + shell: bash + env: + GH_TOKEN: "" + GITHUB_TOKEN: "" + FLEET_GH_TOKEN: "" + run: python3 .github/scripts/validate-resolved-legacy-harnesses.py + - name: Upload validation result + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: resolved-legacy-harness-validation + path: legacy-harness-validated + if-no-files-found: error + retention-days: 1 + + merge: + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + FLEET_GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + steps: + - name: Check out trusted exact-head GraphQL merger + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Download credential-free validation result + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + name: resolved-legacy-harness-validation + path: portfolio-validated + - name: Revalidate, promote, merge, and exact-head verify through GraphQL + shell: bash + run: node .github/scripts/merge-graphql-validated-portfolios.mjs diff --git a/.github/workflows/mise-environment.yml b/.github/workflows/mise-environment.yml new file mode 100644 index 00000000..7c2e38fc --- /dev/null +++ b/.github/workflows/mise-environment.yml @@ -0,0 +1,169 @@ +name: mise environment interoperability + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: ada14c922d439eeb994b85a9b3ae6b9474d9ab44 + schedule: + - cron: "41 4 * * 1" + +permissions: + contents: read + +concurrency: + group: mise-environment-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || 'ada14c922d439eeb994b85a9b3ae6b9474d9ab44' }} + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + +jobs: + certify: + name: mise contract / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - macos-15 + - windows-2025 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out immutable zed CLI + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify the canonical EnvironmentPlan contract pin + run: | + set -euo pipefail + grep -F "$ZED_INTERFACES_REF" zed-cli/Cargo.toml + grep -F "$ZED_INTERFACES_REF" zed-cli/Cargo.lock + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + python --version + + - name: Run the in-repository mise contract tests + run: | + set -euo pipefail + python -m py_compile \ + harness/scripts/mise_environment.py \ + harness/scripts/mise_lock_naming.py \ + harness/scripts/mise_frozen_identity.py + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib environment::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml --test mise_environment_cli + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + + - name: Build the immutable zed executable + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + if [[ "$RUNNER_OS" == "Windows" ]]; then + test -x zed-cli/target/release/zed.exe || test -f zed-cli/target/release/zed.exe + else + test -x zed-cli/target/release/zed + fi + + - name: Certify project-local mise import and frozen identity + run: | + set -euo pipefail + ZED_BIN="zed-cli/target/release/zed" + if [[ "$RUNNER_OS" == "Windows" ]]; then + ZED_BIN="${ZED_BIN}.exe" + fi + python harness/scripts/mise_environment.py \ + --zed "$ZED_BIN" \ + --work-root "$RUNNER_TEMP/zed-mise-environment" + python harness/scripts/mise_lock_naming.py \ + --zed "$ZED_BIN" \ + --work-root "$RUNNER_TEMP/zed-mise-lock-naming" + python harness/scripts/mise_frozen_identity.py \ + --zed "$ZED_BIN" \ + --work-root "$RUNNER_TEMP/zed-mise-frozen-identity" + + - name: Collect failure diagnostics + if: failure() + run: | + set +e + mkdir -p "$RUNNER_TEMP/zed-mise-diagnostics" + cp -R "$RUNNER_TEMP/zed-mise-environment" \ + "$RUNNER_TEMP/zed-mise-diagnostics/environment-work-root" 2>/dev/null || true + cp -R "$RUNNER_TEMP/zed-mise-lock-naming" \ + "$RUNNER_TEMP/zed-mise-diagnostics/lock-naming-work-root" 2>/dev/null || true + cp -R "$RUNNER_TEMP/zed-mise-frozen-identity" \ + "$RUNNER_TEMP/zed-mise-diagnostics/frozen-identity-work-root" 2>/dev/null || true + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$RUNNER_TEMP/zed-mise-diagnostics/zed-cli-status.txt" 2>&1 + git -C harness status --porcelain=v1 --untracked-files=all \ + > "$RUNNER_TEMP/zed-mise-diagnostics/harness-status.txt" 2>&1 + python --version \ + > "$RUNNER_TEMP/zed-mise-diagnostics/python-version.txt" 2>&1 + rustc --version --verbose \ + > "$RUNNER_TEMP/zed-mise-diagnostics/rust-version.txt" 2>&1 + cargo --version --verbose \ + > "$RUNNER_TEMP/zed-mise-diagnostics/cargo-version.txt" 2>&1 + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mise-environment-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-mise-diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: true + + gate: + name: All mise environment platforms passed + if: always() + needs: certify + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Enforce aggregate result + env: + MATRIX_RESULT: ${{ needs.certify.result }} + run: | + set -euo pipefail + printf 'mise-environment=%s\n' "$MATRIX_RESULT" + test "$MATRIX_RESULT" = success diff --git a/.github/workflows/mise-export-boundaries.yml b/.github/workflows/mise-export-boundaries.yml new file mode 100644 index 00000000..28c11ce5 --- /dev/null +++ b/.github/workflows/mise-export-boundaries.yml @@ -0,0 +1,153 @@ +name: mise export boundaries + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: 3434716591e048d0a769bafd47353d2953e89e48 + schedule: + - cron: "17 5 * * 3" + +permissions: + contents: read + +concurrency: + group: mise-export-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '3434716591e048d0a769bafd47353d2953e89e48' }} + +jobs: + certify: + name: export / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - macos-15 + - windows-2025 + steps: + - name: Require an immutable CLI pin + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out immutable zed CLI + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install validation toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + python --version + + - name: Run focused exporter tests and strict lint + run: | + set -euo pipefail + python -m py_compile \ + harness/scripts/mise_export_boundaries.py \ + harness/scripts/mise_export_ownership_state.py + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml mise_export + cargo test --locked --manifest-path zed-cli/Cargo.toml --test mise_export_api_boundaries + cargo test --locked --manifest-path zed-cli/Cargo.toml --test mise_export_cli + cargo test --locked --manifest-path zed-cli/Cargo.toml --test mise_export_reserved_paths_cli + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + + - name: Build immutable release CLI + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + if [[ "$RUNNER_OS" == "Windows" ]]; then + test -f zed-cli/target/release/zed.exe + else + test -x zed-cli/target/release/zed + fi + + - name: Certify ownership, conflict, secret, and path boundaries + run: | + set -euo pipefail + ZED_BIN="$PWD/zed-cli/target/release/zed" + if [[ "$RUNNER_OS" == "Windows" ]]; then + ZED_BIN="${ZED_BIN}.exe" + fi + python harness/scripts/mise_export_boundaries.py \ + --zed "$ZED_BIN" \ + --work-root "$RUNNER_TEMP/zed-mise-export" + python harness/scripts/mise_export_ownership_state.py \ + --zed "$ZED_BIN" \ + --work-root "$RUNNER_TEMP/zed-mise-export-ownership" + + - name: Collect failure diagnostics + if: failure() + run: | + set +e + diagnostics="$RUNNER_TEMP/zed-mise-export-diagnostics" + mkdir -p "$diagnostics" + cp -R "$RUNNER_TEMP/zed-mise-export" "$diagnostics/boundaries-work-root" 2>/dev/null || true + cp -R "$RUNNER_TEMP/zed-mise-export-ownership" "$diagnostics/ownership-work-root" 2>/dev/null || true + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/zed-cli-status.txt" 2>&1 + git -C harness status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/harness-status.txt" 2>&1 + rustc --version --verbose > "$diagnostics/rust-version.txt" 2>&1 + cargo --version --verbose > "$diagnostics/cargo-version.txt" 2>&1 + python --version > "$diagnostics/python-version.txt" 2>&1 + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mise-export-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-mise-export-diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: true + + gate: + name: All mise export platforms passed + if: always() + needs: certify + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Enforce aggregate result + env: + MATRIX_RESULT: ${{ needs.certify.result }} + run: | + set -euo pipefail + printf 'mise-export=%s\n' "$MATRIX_RESULT" + test "$MATRIX_RESULT" = success diff --git a/.github/workflows/mise-lock-contract.yml b/.github/workflows/mise-lock-contract.yml new file mode 100644 index 00000000..7834992c --- /dev/null +++ b/.github/workflows/mise-lock-contract.yml @@ -0,0 +1,121 @@ +name: complete current mise lock contract + +on: + pull_request: + paths: + - .github/workflows/mise-lock-contract.yml + - docs/mise-lock-contract-certification.md + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: 7977b1d43231a68e1ac34903e0f18cef5135add6 + +permissions: + contents: read + +concurrency: + group: mise-lock-contract-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '7977b1d43231a68e1ac34903e0f18cef5135add6' }} + ZED_INTERFACES_REF: a23cdd2ac509a39ca6fd6d21a3774fdd3a0f7660 + +jobs: + certify: + name: lock contract / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - macos-15 + - windows-2025 + + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out immutable zed CLI + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify canonical interface pin + run: | + set -euo pipefail + grep -F "$ZED_INTERFACES_REF" zed-cli/Cargo.toml + grep -F "$ZED_INTERFACES_REF" zed-cli/Cargo.lock + + - name: Install validation toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Certify complete lock identity + run: | + set -euo pipefail + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml mise_lock + cargo test --locked --manifest-path zed-cli/Cargo.toml --test mise_lock_contract + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + + - name: Re-run deterministic round-trip canaries + run: | + set -euo pipefail + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + mise_lock::tests::complete_current_lock_round_trips_and_is_frozen_portable + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + mise_lock::tests::compact_checksum_and_table_forms_have_the_same_identity + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + mise_lock::tests::ordered_additional_artifacts_change_semantic_identity + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --test mise_lock_contract \ + pinned_current_mise_wire_fixture_round_trips_without_platform_key_loss + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --test mise_lock_contract \ + provenance_mutation_in_current_wire_changes_semantic_identity + + gate: + name: All current mise lock platforms passed + if: always() + needs: certify + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Enforce aggregate result + env: + MATRIX_RESULT: ${{ needs.certify.result }} + run: | + set -euo pipefail + printf 'mise-lock-contract=%s\n' "$MATRIX_RESULT" + test "$MATRIX_RESULT" = success diff --git a/.github/workflows/mise-runtime.yml b/.github/workflows/mise-runtime.yml new file mode 100644 index 00000000..207db1e3 --- /dev/null +++ b/.github/workflows/mise-runtime.yml @@ -0,0 +1,134 @@ +name: mise runtime compatibility + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: 854b55c348307d00e2fd2ec8309792c249208192 + schedule: + - cron: "23 4 * * 2" + +permissions: + contents: read + +concurrency: + group: mise-runtime-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '854b55c348307d00e2fd2ec8309792c249208192' }} + +jobs: + certify: + name: Runtime / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - macos-15 + steps: + - name: Require an immutable CLI pin + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out certification harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out immutable zed CLI + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + rustc --version --verbose + cargo --version --verbose + bash --version | head -1 + + - name: Build the immutable CLI and run direct runtime tests + run: | + set -euo pipefail + bash -n harness/scripts/mise_runtime.sh + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --test develop_edge_cases \ + --test develop_shell_edge_cases \ + --test develop_help_contract + cargo test --locked --manifest-path zed-cli/Cargo.toml dev::tests + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + test -x zed-cli/target/release/zed + + - name: Certify consumer-visible mise runtime behavior + run: | + set -euo pipefail + bash harness/scripts/mise_runtime.sh \ + --zed "$PWD/zed-cli/target/release/zed" \ + --work-root "$RUNNER_TEMP/zed-mise-runtime" + + - name: Collect failure diagnostics + if: failure() + run: | + set +e + diagnostics="$RUNNER_TEMP/zed-mise-runtime-diagnostics" + mkdir -p "$diagnostics" + cp -R "$RUNNER_TEMP/zed-mise-runtime" "$diagnostics/work-root" 2>/dev/null || true + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/zed-cli-status.txt" 2>&1 + git -C harness status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/harness-status.txt" 2>&1 + rustc --version --verbose > "$diagnostics/rust-version.txt" 2>&1 + cargo --version --verbose > "$diagnostics/cargo-version.txt" 2>&1 + bash --version > "$diagnostics/bash-version.txt" 2>&1 + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mise-runtime-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-mise-runtime-diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: true + + gate: + name: All mise runtime platforms passed + if: always() + needs: certify + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Enforce aggregate result + env: + MATRIX_RESULT: ${{ needs.certify.result }} + run: | + set -euo pipefail + printf 'mise-runtime=%s\n' "$MATRIX_RESULT" + test "$MATRIX_RESULT" = success diff --git a/.github/workflows/mise-tier1-certification.yml b/.github/workflows/mise-tier1-certification.yml new file mode 100644 index 00000000..4daa196e --- /dev/null +++ b/.github/workflows/mise-tier1-certification.yml @@ -0,0 +1,87 @@ +name: mise tier-1 compatibility + +on: + pull_request: + paths: + - .github/workflows/mise-tier1-certification.yml + - compat/mise/** + - scripts/mise_tier1_certify.py + workflow_dispatch: + inputs: + zed_ref: + description: zed-pkg/zed-cli branch, tag, or commit to certify + required: false + default: feat/den-1420-mise-dev-adapter + +permissions: + contents: read + +concurrency: + group: mise-tier1-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + certify: + name: Tier 1 (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - name: Check out certification repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + show-progress: false + + - name: Check out zed-cli + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: zed-pkg/zed-cli + ref: ${{ inputs.zed_ref || 'feat/den-1420-mise-dev-adapter' }} + path: .zed-cli + persist-credentials: false + show-progress: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + + - name: Install Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + + - name: Build zed-cli + working-directory: .zed-cli + run: cargo build --locked + + - name: Select zed binary + shell: bash + run: | + if [[ "$RUNNER_OS" == "Windows" ]]; then + echo "ZED_BINARY=$GITHUB_WORKSPACE/.zed-cli/target/debug/zed.exe" >> "$GITHUB_ENV" + else + echo "ZED_BINARY=$GITHUB_WORKSPACE/.zed-cli/target/debug/zed" >> "$GITHUB_ENV" + fi + + - name: Run tier-1 certification + shell: bash + run: | + python scripts/mise_tier1_certify.py \ + --zed "$ZED_BINARY" \ + --fixtures compat/mise/fixtures \ + --report "mise-tier1-${RUNNER_OS}-${RUNNER_ARCH}.json" + + - name: Upload compatibility report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: mise-tier1-${{ runner.os }}-${{ runner.arch }} + path: mise-tier1-${{ runner.os }}-${{ runner.arch }}.json + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/native-dependency-range.yml b/.github/workflows/native-dependency-range.yml new file mode 100644 index 00000000..c397ff60 --- /dev/null +++ b/.github/workflows/native-dependency-range.yml @@ -0,0 +1,124 @@ +name: zed native dependency range contract + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_interfaces_ref: + description: Immutable zed-interfaces commit to certify + required: false + default: c872714fdb9a811e4a01f70082dbd61252975f47 + +permissions: + contents: read + +concurrency: + group: zed-native-dependency-range-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_INTERFACES_REF: ${{ github.event.inputs.zed_interfaces_ref || 'c872714fdb9a811e4a01f70082dbd61252975f47' }} + +jobs: + native-dependency-range: + name: Native dependency range and lock + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Require immutable interface pin + run: | + set -euo pipefail + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out canary harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-interfaces candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust with formatter and Clippy + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Verify formatter and deterministic schema + run: | + set -euo pipefail + cd zed-interfaces + cargo fmt --all -- --check + test -s schemas/native-dependency-lock.json + cp schemas/native-dependency-lock.json \ + "$RUNNER_TEMP/native-dependency-lock.checked-in.json" + cargo run --locked --example generate_schemas + git diff --exit-code -- schemas/native-dependency-lock.json + cmp "$RUNNER_TEMP/native-dependency-lock.checked-in.json" \ + schemas/native-dependency-lock.json + sha256sum schemas/native-dependency-lock.json \ + | tee "$RUNNER_TEMP/native-dependency-lock-schema.sha256" + cargo run --locked --example generate_schemas + sha256sum --check "$RUNNER_TEMP/native-dependency-lock-schema.sha256" + git diff --exit-code -- schemas/native-dependency-lock.json + + - name: Run complete interface gates + run: | + set -euo pipefail + cd zed-interfaces + cargo test --locked --all-targets + cargo clippy --locked --all-targets --all-features -- -D warnings + git diff --check + + - name: Compile and run external consumer offline + run: | + set -euo pipefail + fixture=harness/fixtures/native-dependency-consumer + cargo generate-lockfile --offline --manifest-path "$fixture/Cargo.toml" + cargo run --locked --offline --manifest-path "$fixture/Cargo.toml" + + - name: Capture certification evidence + if: always() + run: | + set +e + evidence="$RUNNER_TEMP/native-dependency-range-evidence" + mkdir -p "$evidence/src" "$evidence/examples" "$evidence/schemas" "$evidence/consumer" + cp zed-interfaces/src/native_dependency.rs "$evidence/src/" 2>/dev/null + cp zed-interfaces/src/lib.rs "$evidence/src/" 2>/dev/null + cp zed-interfaces/examples/generate_schemas.rs "$evidence/examples/" 2>/dev/null + cp zed-interfaces/schemas/native-dependency-lock.json "$evidence/schemas/" 2>/dev/null + cp harness/fixtures/native-dependency-consumer/Cargo.lock "$evidence/consumer/" 2>/dev/null + cp "$RUNNER_TEMP/native-dependency-lock-schema.sha256" "$evidence/" 2>/dev/null + git -C zed-interfaces diff > "$evidence/certification.patch" 2>&1 + git -C zed-interfaces status --porcelain=v1 --untracked-files=all \ + > "$evidence/interfaces-status.txt" 2>&1 + + - name: Upload certification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-dependency-range-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/native-dependency-range-evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/native-lifecycle.yml b/.github/workflows/native-lifecycle.yml new file mode 100644 index 00000000..7f2d5ece --- /dev/null +++ b/.github/workflows/native-lifecycle.yml @@ -0,0 +1,54 @@ +name: native dependency lifecycle acceptance + +on: + pull_request: + paths: + - ".github/workflows/native-lifecycle.yml" + - "scripts/native_lifecycle.py" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: native-lifecycle-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + # Immutable zed-cli PR #48 commit validated by its complete Rust suite. + ZED_CLI_REF: 9a3c849f4be3a4a5f7e1af0d180e748df73f0d9c + +jobs: + acceptance: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out acceptance harness + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + + - name: Check out immutable zed-cli candidate + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: candidate + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + + - name: Build immutable candidate + shell: bash + run: cargo build --locked --release --manifest-path candidate/Cargo.toml + + - name: Validate acceptance harness syntax + shell: bash + run: python3 -m py_compile scripts/native_lifecycle.py + + - name: Exercise native dependencies and staged hooks + shell: bash + run: | + set -euo pipefail + python3 scripts/native_lifecycle.py \ + --zed candidate/target/release/zed \ + --work-root "$RUNNER_TEMP/native-lifecycle" diff --git a/.github/workflows/native-lockfile-provenance-current-main.yml b/.github/workflows/native-lockfile-provenance-current-main.yml new file mode 100644 index 00000000..bac9f90a --- /dev/null +++ b/.github/workflows/native-lockfile-provenance-current-main.yml @@ -0,0 +1,171 @@ +name: current-main native lockfile provenance + +on: + pull_request: + push: + branches: + - agent/den-1565-native-lock-canary-current-main-v2 + workflow_dispatch: + inputs: + zed_interfaces_ref: + description: Immutable zed-interfaces commit to certify + required: false + default: a23cdd2ac509a39ca6fd6d21a3774fdd3a0f7660 + +permissions: + contents: read + +concurrency: + group: current-main-native-lock-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_INTERFACES_REF: ${{ github.event.inputs.zed_interfaces_ref || 'a23cdd2ac509a39ca6fd6d21a3774fdd3a0f7660' }} + ZED_CLI_REF: 0f8f19b16b6415827f83f09b124fab60f7165704 + ZED_API_REF: 4eaacda14d5c5e38b81404415631e364ac7bdb2c + ZED_WEB_REF: eb2fabc7a6c22333dcb89e5a629bab7679d76af0 + +jobs: + native-lockfile-provenance: + name: Schema, external consumer, and current Rust applications + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - name: Require immutable pins + run: | + set -euo pipefail + for value in "$ZED_INTERFACES_REF" "$ZED_CLI_REF" "$ZED_API_REF" "$ZED_WEB_REF"; do + [[ "$value" =~ ^[0-9a-f]{40}$ ]] + done + + - name: Check out canary harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + persist-credentials: false + show-progress: false + + - name: Check out exact zed-interfaces candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + persist-credentials: false + show-progress: false + + - name: Check out exact current applications + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + persist-credentials: false + show-progress: false + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-api-server.rs + ref: ${{ env.ZED_API_REF }} + path: zed-api-server.rs + persist-credentials: false + show-progress: false + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-web-server.rs + ref: ${{ env.ZED_WEB_REF }} + path: zed-web-server.rs + persist-credentials: false + show-progress: false + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Verify exact interface contract and schema + run: | + set -euo pipefail + cd zed-interfaces + cargo fmt --all -- --check + test -s schemas/lockfile.json + cp schemas/lockfile.json "$RUNNER_TEMP/lockfile.checked-in.json" + cargo run --locked --example generate_schemas + git diff --exit-code -- schemas/lockfile.json + cmp "$RUNNER_TEMP/lockfile.checked-in.json" schemas/lockfile.json + sha256sum schemas/lockfile.json | tee "$RUNNER_TEMP/lockfile-schema.sha256" + cargo test --locked --all-targets + cargo clippy --locked --all-targets --all-features -- -D warnings + git diff --check + + - name: Compile and execute external consumer offline + run: | + set -euo pipefail + fixture=harness/fixtures/native-lockfile-consumer + cargo generate-lockfile --offline --manifest-path "$fixture/Cargo.toml" + cargo run --locked --offline --manifest-path "$fixture/Cargo.toml" + + - name: Compile current CLI against the exact interface candidate + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + import re + + path = Path('zed-cli/Cargo.toml') + source = path.read_text(encoding='utf-8') + updated, count = re.subn( + r'zed-interfaces\s*=\s*\{[^\n]+\}', + 'zed-interfaces = { path = "../zed-interfaces" }', + source, + count=1, + ) + if count != 1: + raise SystemExit('expected one zed-interfaces dependency declaration') + path.write_text(updated, encoding='utf-8') + PY + cargo check --manifest-path zed-cli/Cargo.toml --all-targets + cargo test --manifest-path zed-cli/Cargo.toml --all-targets --no-run + + - name: Compile current API and web servers + run: | + set -euo pipefail + cargo check --manifest-path zed-api-server.rs/Cargo.toml --workspace --all-targets + cargo check --manifest-path zed-web-server.rs/Cargo.toml --all-targets + + - name: Capture certification evidence + if: always() + run: | + set +e + evidence="$RUNNER_TEMP/native-lockfile-provenance-evidence" + mkdir -p "$evidence/interfaces" "$evidence/consumer" "$evidence/cli" "$evidence/api" "$evidence/web" + cp zed-interfaces/schemas/lockfile.json "$evidence/interfaces/" 2>/dev/null + cp "$RUNNER_TEMP/lockfile-schema.sha256" "$evidence/interfaces/" 2>/dev/null + cp harness/fixtures/native-lockfile-consumer/Cargo.lock "$evidence/consumer/" 2>/dev/null + git -C zed-interfaces status --porcelain=v1 --untracked-files=all > "$evidence/interfaces/status.txt" 2>&1 + git -C zed-cli diff -- Cargo.toml Cargo.lock > "$evidence/cli/path-dependency.patch" 2>&1 + git -C zed-cli status --porcelain=v1 --untracked-files=all > "$evidence/cli/status.txt" 2>&1 + git -C zed-api-server.rs status --porcelain=v1 --untracked-files=all > "$evidence/api/status.txt" 2>&1 + git -C zed-web-server.rs status --porcelain=v1 --untracked-files=all > "$evidence/web/status.txt" 2>&1 + printf '%s\n' "$ZED_INTERFACES_REF" > "$evidence/interfaces/commit.txt" + printf '%s\n' "$ZED_CLI_REF" > "$evidence/cli/commit.txt" + printf '%s\n' "$ZED_API_REF" > "$evidence/api/commit.txt" + printf '%s\n' "$ZED_WEB_REF" > "$evidence/web/commit.txt" + + - name: Upload certification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: current-main-native-lock-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/native-lockfile-provenance-evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/native-lockfile-provenance.yml b/.github/workflows/native-lockfile-provenance.yml new file mode 100644 index 00000000..d9ee4bc6 --- /dev/null +++ b/.github/workflows/native-lockfile-provenance.yml @@ -0,0 +1,198 @@ +name: zed native lockfile provenance + +on: + pull_request: + push: + branches: + - agent/native-lockfile-provenance-canary + workflow_dispatch: + inputs: + zed_interfaces_ref: + description: Immutable zed-interfaces commit to certify + required: false + default: 1f5dd1a644f3ac46cee11cb9612bf94ca0e84786 + zed_cli_ref: + description: Immutable zed-cli commit to compile against the candidate + required: false + default: 7e5ac1897a872223f8316ef1c3342a2fa1982504 + zed_api_ref: + description: Immutable zed-api-server.rs commit to compile against the candidate + required: false + default: 3d2d4c98024e5bf718dc79c2648bd10d7759a31c + zed_web_ref: + description: Immutable zed-web-server.rs commit to compile against the candidate + required: false + default: b669850d317ebc07a8960aea4014706c55ffad22 + +permissions: + contents: read + +concurrency: + group: zed-native-lockfile-provenance-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_INTERFACES_REF: ${{ github.event.inputs.zed_interfaces_ref || '1f5dd1a644f3ac46cee11cb9612bf94ca0e84786' }} + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '7e5ac1897a872223f8316ef1c3342a2fa1982504' }} + ZED_API_REF: ${{ github.event.inputs.zed_api_ref || '3d2d4c98024e5bf718dc79c2648bd10d7759a31c' }} + ZED_WEB_REF: ${{ github.event.inputs.zed_web_ref || 'b669850d317ebc07a8960aea4014706c55ffad22' }} + +jobs: + native-lockfile-provenance: + name: Lockfile schema, consumer, and Rust consumers + runs-on: ubuntu-22.04 + timeout-minutes: 60 + steps: + - name: Require immutable pins + run: | + set -euo pipefail + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_API_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_WEB_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out canary harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-interfaces candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact current zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact current zed-api-server + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-api-server.rs + ref: ${{ env.ZED_API_REF }} + path: zed-api-server.rs + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact current zed-web-server + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-web-server.rs + ref: ${{ env.ZED_WEB_REF }} + path: zed-web-server.rs + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust with formatter and Clippy + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Verify checked-in interface contract + run: | + set -euo pipefail + cd zed-interfaces + cargo fmt --all -- --check + test -s schemas/lockfile.json + cp schemas/lockfile.json "$RUNNER_TEMP/lockfile.checked-in.json" + cargo run --locked --example generate_schemas + git diff --exit-code -- schemas/lockfile.json + cmp "$RUNNER_TEMP/lockfile.checked-in.json" schemas/lockfile.json + sha256sum schemas/lockfile.json | tee "$RUNNER_TEMP/lockfile-schema.sha256" + cargo test --locked --all-targets + cargo clippy --locked --all-targets --all-features -- -D warnings + git diff --check + + - name: Compile and run external lockfile consumer offline + run: | + set -euo pipefail + fixture=harness/fixtures/native-lockfile-consumer + cargo generate-lockfile --offline --manifest-path "$fixture/Cargo.toml" + cargo run --locked --offline --manifest-path "$fixture/Cargo.toml" + + - name: Compile current zed-cli against exact interface candidate + run: | + set -euo pipefail + python3 - <<'PY' + from pathlib import Path + import re + + path = Path("zed-cli/Cargo.toml") + source = path.read_text(encoding="utf-8") + updated, count = re.subn( + r'zed-interfaces\s*=\s*\{[^\n]+\}', + 'zed-interfaces = { path = "../zed-interfaces" }', + source, + count=1, + ) + if count != 1: + raise SystemExit("expected one zed-interfaces dependency declaration") + path.write_text(updated, encoding="utf-8") + PY + cargo check --manifest-path zed-cli/Cargo.toml --all-targets + cargo test --manifest-path zed-cli/Cargo.toml --all-targets --no-run + + - name: Compile current API and web servers against exact interface candidate + run: | + set -euo pipefail + cargo check --manifest-path zed-api-server.rs/Cargo.toml --workspace --all-targets + cargo check --manifest-path zed-web-server.rs/Cargo.toml --all-targets + + - name: Capture certification evidence + if: always() + run: | + set +e + evidence="$RUNNER_TEMP/native-lockfile-provenance-evidence" + mkdir -p "$evidence/interfaces" "$evidence/consumer" "$evidence/cli" \ + "$evidence/api" "$evidence/web" + cp zed-interfaces/schemas/lockfile.json "$evidence/interfaces/" 2>/dev/null + cp "$RUNNER_TEMP/lockfile-schema.sha256" "$evidence/interfaces/" 2>/dev/null + cp harness/fixtures/native-lockfile-consumer/Cargo.lock "$evidence/consumer/" 2>/dev/null + git -C zed-interfaces status --porcelain=v1 --untracked-files=all \ + > "$evidence/interfaces/status.txt" 2>&1 + git -C zed-cli diff -- Cargo.toml Cargo.lock > "$evidence/cli/path-dependency.patch" 2>&1 + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$evidence/cli/status.txt" 2>&1 + git -C zed-api-server.rs status --porcelain=v1 --untracked-files=all \ + > "$evidence/api/status.txt" 2>&1 + git -C zed-web-server.rs status --porcelain=v1 --untracked-files=all \ + > "$evidence/web/status.txt" 2>&1 + printf '%s\n' "$ZED_INTERFACES_REF" > "$evidence/interfaces/commit.txt" + printf '%s\n' "$ZED_CLI_REF" > "$evidence/cli/commit.txt" + printf '%s\n' "$ZED_API_REF" > "$evidence/api/commit.txt" + printf '%s\n' "$ZED_WEB_REF" > "$evidence/web/commit.txt" + + - name: Upload certification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-lockfile-provenance-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/native-lockfile-provenance-evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/native-registry-contract.yml b/.github/workflows/native-registry-contract.yml new file mode 100644 index 00000000..f339bdc5 --- /dev/null +++ b/.github/workflows/native-registry-contract.yml @@ -0,0 +1,151 @@ +name: zed native registry contract canary + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_interfaces_ref: + description: Immutable zed-interfaces commit to certify + required: false + default: ac8b36c0261ad59cf73b3b06c78c13be8370fcfa + +permissions: + contents: read + +concurrency: + group: zed-native-registry-contract-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_INTERFACES_REF: ${{ github.event.inputs.zed_interfaces_ref || 'ac8b36c0261ad59cf73b3b06c78c13be8370fcfa' }} + +jobs: + native-registry-contract: + name: Native registry schema and consumer + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Require immutable interface pin + run: | + set -euo pipefail + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out canary harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed-interfaces candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust with formatter and Clippy + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Generate schema deterministically + run: | + set -euo pipefail + cd zed-interfaces + cargo run --locked --example generate_schemas + test -s schemas/native-registry-adapter-record.json + git diff --exit-code -- schemas/native-registry-adapter-record.json + sha256sum schemas/native-registry-adapter-record.json \ + | tee "$RUNNER_TEMP/native-registry-schema.sha256" + cp schemas/native-registry-adapter-record.json \ + "$RUNNER_TEMP/native-registry-adapter-record.first.json" + cargo run --locked --example generate_schemas + cmp "$RUNNER_TEMP/native-registry-adapter-record.first.json" \ + schemas/native-registry-adapter-record.json + sha256sum --check "$RUNNER_TEMP/native-registry-schema.sha256" + + - name: Verify generated schema surface + run: | + set -euo pipefail + python3 - <<'PY' + import json + from pathlib import Path + + path = Path("zed-interfaces/schemas/native-registry-adapter-record.json") + schema = json.loads(path.read_text(encoding="utf-8")) + text = json.dumps(schema, sort_keys=True) + required = ( + "NativeRegistryAdapterRecord", + "platform_packages", + "sha256", + "portable", + "meta", + "platform", + "npm", + "cargo", + ) + missing = [token for token in required if token not in text] + if missing: + raise SystemExit(f"schema is missing public contract tokens: {missing}") + PY + + - name: Run complete zed-interfaces gates + run: | + set -euo pipefail + cd zed-interfaces + cargo fmt --all -- --check + cargo test --locked --all-targets + cargo clippy --locked --all-targets --all-features -- -D warnings + git diff --check + + - name: Compile and test public contract from an external crate + run: | + set -euo pipefail + fixture=harness/fixtures/native-registry-consumer + cp zed-interfaces/Cargo.lock "$fixture/Cargo.lock" + cargo run --offline --manifest-path "$fixture/Cargo.toml" + cargo run --locked --offline --manifest-path "$fixture/Cargo.toml" + cargo test --offline --manifest-path "$fixture/Cargo.toml" + cargo test --locked --offline --manifest-path "$fixture/Cargo.toml" + + - name: Record evidence + if: always() + run: | + set +e + mkdir -p "$RUNNER_TEMP/native-registry-contract-evidence" + cp zed-interfaces/schemas/native-registry-adapter-record.json \ + "$RUNNER_TEMP/native-registry-contract-evidence/" 2>/dev/null + cp "$RUNNER_TEMP/native-registry-schema.sha256" \ + "$RUNNER_TEMP/native-registry-contract-evidence/" 2>/dev/null + cp harness/fixtures/native-registry-consumer/Cargo.lock \ + "$RUNNER_TEMP/native-registry-contract-evidence/consumer-Cargo.lock" 2>/dev/null + git -C zed-interfaces status --porcelain=v1 --untracked-files=all \ + > "$RUNNER_TEMP/native-registry-contract-evidence/interfaces-status.txt" 2>&1 + printf '%s\n' "$ZED_INTERFACES_REF" \ + > "$RUNNER_TEMP/native-registry-contract-evidence/zed-interfaces-ref.txt" + + - name: Upload contract evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-registry-contract-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/native-registry-contract-evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/nix-bundle-rustfmt-evidence.yml b/.github/workflows/nix-bundle-rustfmt-evidence.yml new file mode 100644 index 00000000..d0cdbb38 --- /dev/null +++ b/.github/workflows/nix-bundle-rustfmt-evidence.yml @@ -0,0 +1,111 @@ +name: external Nix bundle rustfmt evidence + +on: + pull_request: + paths: + - ".github/workflows/nix-bundle-rustfmt-evidence.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: external-nix-bundle-rustfmt-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + evidence: + name: format exact production Nix bundle files / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04, windows-2022, macos-13] + steps: + - name: Check out evidence workflow + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + persist-credentials: false + show-progress: false + + - name: Check out exact production commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: 1b282b94efc994bb9352b572c9652d5a078787f2 + path: zed-cli + persist-credentials: false + show-progress: false + + - name: Install pinned rustfmt + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt + + - name: Produce constrained exact bytes + working-directory: zed-cli + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = 1b282b94efc994bb9352b572c9652d5a078787f2 + cargo fmt --all + + python3 - <<'PY' + import subprocess + + expected = { + 'src/nix_bundle_write.rs', + 'tests/nix_bundle_write_boundaries.rs', + 'tests/nix_bundle_write_cli.rs', + } + changed = { + line.strip() + for line in subprocess.check_output( + ['git', 'diff', '--name-only'], text=True + ).splitlines() + if line.strip() + } + if changed != expected: + raise SystemExit( + 'expected exactly the three reviewed files; got: ' + + ', '.join(sorted(changed)) + ) + PY + + cargo fmt --all --check + cargo test --locked --test nix_bundle_write_cli --no-run + cargo test --locked --test nix_bundle_write_boundaries --no-run + + mkdir -p "$RUNNER_TEMP/nix-bundle-rustfmt/src" \ + "$RUNNER_TEMP/nix-bundle-rustfmt/tests" + cp src/nix_bundle_write.rs \ + "$RUNNER_TEMP/nix-bundle-rustfmt/src/nix_bundle_write.rs" + cp tests/nix_bundle_write_boundaries.rs \ + "$RUNNER_TEMP/nix-bundle-rustfmt/tests/nix_bundle_write_boundaries.rs" + cp tests/nix_bundle_write_cli.rs \ + "$RUNNER_TEMP/nix-bundle-rustfmt/tests/nix_bundle_write_cli.rs" + git diff --binary -- \ + src/nix_bundle_write.rs \ + tests/nix_bundle_write_boundaries.rs \ + tests/nix_bundle_write_cli.rs \ + > "$RUNNER_TEMP/nix-bundle-rustfmt/rustfmt.patch" + test -s "$RUNNER_TEMP/nix-bundle-rustfmt/rustfmt.patch" + sha256sum \ + "$RUNNER_TEMP/nix-bundle-rustfmt/src/nix_bundle_write.rs" \ + "$RUNNER_TEMP/nix-bundle-rustfmt/tests/nix_bundle_write_boundaries.rs" \ + "$RUNNER_TEMP/nix-bundle-rustfmt/tests/nix_bundle_write_cli.rs" \ + "$RUNNER_TEMP/nix-bundle-rustfmt/rustfmt.patch" \ + > "$RUNNER_TEMP/nix-bundle-rustfmt/SHA256SUMS" + git diff --check + + - name: Upload exact formatted files + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nix-bundle-rustfmt-production-${{ runner.os }}-${{ github.run_id }} + path: ${{ runner.temp }}/nix-bundle-rustfmt + if-no-files-found: error + retention-days: 1 + include-hidden-files: false diff --git a/.github/workflows/nix-bundle-write-canary.yml b/.github/workflows/nix-bundle-write-canary.yml new file mode 100644 index 00000000..ce7626e8 --- /dev/null +++ b/.github/workflows/nix-bundle-write-canary.yml @@ -0,0 +1,230 @@ +name: external nix bundle write + +on: + pull_request: + paths: + - ".github/workflows/nix-bundle-write-canary.yml" + - ".zed-nix-bundle-write-cli-ref" + - "scripts/nix_bundle_write_canary.py" + - "scripts/nix_bundle_write_fixture_canary.py" + - "docs/nix-bundle-write-canary.md" + push: + branches: [main] + paths: + - ".github/workflows/nix-bundle-write-canary.yml" + - ".zed-nix-bundle-write-cli-ref" + - "scripts/nix_bundle_write_canary.py" + - "scripts/nix_bundle_write_fixture_canary.py" + - "docs/nix-bundle-write-canary.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Optional immutable zed-cli commit override + required: false + default: "" + +permissions: + contents: read + +concurrency: + group: external-nix-bundle-write-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + canary: + name: Black-box bundle and offline replay / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + + steps: + - name: Check out harness and reviewed pin + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + persist-credentials: false + show-progress: false + + - name: Load immutable CLI pin + id: pin + shell: bash + run: | + set -euo pipefail + override="${{ github.event.inputs.zed_cli_ref || '' }}" + if [[ -n "$override" ]]; then + ref="$override" + else + ref="$(tr -d '[:space:]' < harness/.zed-nix-bundle-write-cli-ref)" + fi + [[ "$ref" =~ ^[0-9a-f]{40}$ ]] + printf 'ref=%s\n' "$ref" >> "$GITHUB_OUTPUT" + printf 'zed-cli=%s\n' "$ref" + + - name: Check out exact zed-cli implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ steps.pin.outputs.ref }} + path: zed-cli + persist-credentials: false + show-progress: false + + - name: Check out immutable real package fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-lib + ref: 222cdf57f48530fce8e6c1f58632d9676203512e + path: node-fixture + persist-credentials: false + show-progress: false + + - name: Install pinned Rust toolchain action + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + components: rustfmt + + - name: Install pinned Nix action + uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 + with: + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = false + + - name: Verify immutable inputs and build exact CLI + shell: bash + run: | + set -euo pipefail + { + test "$(git -C zed-cli rev-parse HEAD)" = "${{ steps.pin.outputs.ref }}" + test "$(git -C node-fixture rev-parse HEAD)" = 222cdf57f48530fce8e6c1f58632d9676203512e + grep -F 'rev = "c2e049006453c26ca8ca291783f681fce75cb01f"' zed-cli/Cargo.toml + grep -F '#c2e049006453c26ca8ca291783f681fce75cb01f' zed-cli/Cargo.lock + python3 - <<'PY' + from pathlib import Path + + for source in ( + Path('harness/scripts/nix_bundle_write_canary.py'), + Path('harness/scripts/nix_bundle_write_fixture_canary.py'), + ): + compile(source.read_text(encoding='utf-8'), str(source), 'exec') + PY + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo build --release --locked --manifest-path zed-cli/Cargo.toml --bin zed + mkdir -p "$RUNNER_TEMP/zed-bundle-write-bin" + install -m 0755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-bundle-write-bin/zed" + python3 - "$RUNNER_TEMP/zed-bundle-write-bin/zed" \ + "$RUNNER_TEMP/zed-bundle-write-bin/SHA256SUMS" <<'PY' + import hashlib + import sys + from pathlib import Path + + binary = Path(sys.argv[1]) + output = Path(sys.argv[2]) + output.write_text( + f"{hashlib.sha256(binary.read_bytes()).hexdigest()} zed\n", + encoding="utf-8", + ) + PY + cat "$RUNNER_TEMP/zed-bundle-write-bin/SHA256SUMS" + "$RUNNER_TEMP/zed-bundle-write-bin/zed" --version + nix --version + } 2>&1 | tee "$RUNNER_TEMP/zed-nix-bundle-write-build.log" + + - name: Run synthetic command and offline Nix canary + env: + WORK_ROOT: ${{ runner.temp }}/zed-nix-bundle-write-canary + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/nix_bundle_write_canary.py \ + --zed "$RUNNER_TEMP/zed-bundle-write-bin/zed" \ + --work-root "$WORK_ROOT" \ + 2>&1 | tee "$RUNNER_TEMP/zed-nix-bundle-write-canary.log" + + - name: Run immutable real-fixture path-independence canary + env: + WORK_ROOT: ${{ runner.temp }}/zed-nix-bundle-write-fixture-canary + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/nix_bundle_write_fixture_canary.py \ + --zed "$RUNNER_TEMP/zed-bundle-write-bin/zed" \ + --fixture node-fixture \ + --work-root "$WORK_ROOT" \ + 2>&1 | tee "$RUNNER_TEMP/zed-nix-bundle-write-fixture-canary.log" + + - name: Prove checkouts and workflow remain immutable and read-only + shell: bash + run: | + set -euo pipefail + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C node-fixture status --porcelain=v1 --untracked-files=all)" + python3 - <<'PY' + import re + from pathlib import Path + + workflow = Path('harness/.github/workflows/nix-bundle-write-canary.yml').read_text() + pin = Path('harness/.zed-nix-bundle-write-cli-ref').read_text().strip() + assert re.fullmatch(r'[0-9a-f]{40}', pin) + assert 'permissions:\n contents: read' in workflow + assert 'persist-credentials: false' in workflow + assert 'ubuntu-24.04' in workflow and 'macos-15' in workflow + for exact in ( + '222cdf57f48530fce8e6c1f58632d9676203512e', + 'c2e049006453c26ca8ca291783f681fce75cb01f', + ): + assert exact in workflow + action_token = 'us' + 'es' + pattern = rf'^\s*{action_token}:\s*([^\s]+)$' + for match in re.finditer(pattern, workflow, re.MULTILINE): + assert re.search(r'@[0-9a-f]{40}$', match.group(1)), match.group(1) + for forbidden in ( + 'contents' + ': write', + 'git' + ' push', + 'pull_request' + '_target:', + 'id-token' + ': write', + 'packages' + ': write', + 'secrets' + ': inherit', + 'ref:' + ' main', + ): + assert forbidden not in workflow, forbidden + PY + + - name: Collect bounded failure diagnostics + if: failure() + env: + SYNTHETIC_ROOT: ${{ runner.temp }}/zed-nix-bundle-write-canary + FIXTURE_ROOT: ${{ runner.temp }}/zed-nix-bundle-write-fixture-canary + shell: bash + run: | + set +e + diagnostics="$RUNNER_TEMP/zed-nix-bundle-write-diagnostics" + mkdir -p "$diagnostics" + cp "$RUNNER_TEMP/zed-nix-bundle-write-build.log" \ + "$diagnostics/build.log" 2>/dev/null || true + cp "$RUNNER_TEMP/zed-nix-bundle-write-canary.log" \ + "$diagnostics/synthetic.log" 2>/dev/null || true + cp "$RUNNER_TEMP/zed-nix-bundle-write-fixture-canary.log" \ + "$diagnostics/fixture.log" 2>/dev/null || true + git -C harness status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/harness-status.txt" 2>&1 + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/zed-cli-status.txt" 2>&1 + git -C node-fixture status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/node-fixture-status.txt" 2>&1 + find "$SYNTHETIC_ROOT" "$FIXTURE_ROOT" -maxdepth 6 -print \ + > "$diagnostics/work-tree.txt" 2>&1 || true + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nix-bundle-write-diagnostics-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-nix-bundle-write-diagnostics + if-no-files-found: warn + retention-days: 3 + include-hidden-files: false diff --git a/.github/workflows/nix-export-plan.yml b/.github/workflows/nix-export-plan.yml new file mode 100644 index 00000000..76a283a9 --- /dev/null +++ b/.github/workflows/nix-export-plan.yml @@ -0,0 +1,215 @@ +name: external nix export plan + +on: + pull_request: + paths: + - ".github/workflows/nix-export-plan.yml" + - ".zed-nix-plan-cli-ref" + - "scripts/nix_export_plan_canary.py" + - "docs/nix-export-plan-canary.md" + push: + branches: [main] + paths: + - ".github/workflows/nix-export-plan.yml" + - ".zed-nix-plan-cli-ref" + - "scripts/nix_export_plan_canary.py" + - "docs/nix-export-plan-canary.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Optional immutable zed-cli commit override + required: false + default: "" + +permissions: + contents: read + +concurrency: + group: external-nix-export-plan-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + canary: + name: Independent frozen plan replay on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + + steps: + - name: Check out harness and reviewed pin + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: harness + persist-credentials: false + show-progress: false + + - name: Load immutable CLI pin + id: pin + shell: bash + run: | + set -euo pipefail + override="${{ github.event.inputs.zed_cli_ref || '' }}" + if [[ -n "$override" ]]; then + ref="$override" + else + ref="$(tr -d '[:space:]' < harness/.zed-nix-plan-cli-ref)" + fi + [[ "$ref" =~ ^[0-9a-f]{40}$ ]] + printf 'ref=%s\n' "$ref" >> "$GITHUB_OUTPUT" + printf 'zed-cli=%s\n' "$ref" + + - name: Check out exact zed-cli implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: zed-pkg/zed-cli + ref: ${{ steps.pin.outputs.ref }} + path: zed-cli + persist-credentials: false + show-progress: false + + - name: Check out immutable single-package fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: zed-pkg-test/node-lib + ref: 222cdf57f48530fce8e6c1f58632d9676203512e + path: node-fixture + persist-credentials: false + show-progress: false + + - name: Check out immutable polyglot fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: zed-pkg-test/polyglot-lib + ref: 998964aa58e49595547650b4a8da407b7a2283a9 + path: polyglot-fixture + persist-credentials: false + show-progress: false + + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + components: rustfmt + + - name: Verify current interface pin and build exact CLI + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "${{ steps.pin.outputs.ref }}" + test "$(git -C node-fixture rev-parse HEAD)" = 222cdf57f48530fce8e6c1f58632d9676203512e + test "$(git -C polyglot-fixture rev-parse HEAD)" = 998964aa58e49595547650b4a8da407b7a2283a9 + grep -F 'rev = "c2e049006453c26ca8ca291783f681fce75cb01f"' zed-cli/Cargo.toml + grep -F '#c2e049006453c26ca8ca291783f681fce75cb01f' zed-cli/Cargo.lock + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo build --release --locked --manifest-path zed-cli/Cargo.toml --bin zed + mkdir -p "$RUNNER_TEMP/zed-plan-bin" + install -m 0755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-plan-bin/zed" + python3 - "$RUNNER_TEMP/zed-plan-bin/zed" "$RUNNER_TEMP/zed-plan-bin/SHA256SUMS" <<'PY' + import hashlib + import sys + from pathlib import Path + + binary = Path(sys.argv[1]) + output = Path(sys.argv[2]) + output.write_text(f"{hashlib.sha256(binary.read_bytes()).hexdigest()} zed\n", encoding="utf-8") + PY + cat "$RUNNER_TEMP/zed-plan-bin/SHA256SUMS" + "$RUNNER_TEMP/zed-plan-bin/zed" --version + + - name: Run single-package, prebuilt-bin, and polyglot canaries + env: + WORK_ROOT: ${{ runner.temp }}/zed-nix-export-plan-canary + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/nix_export_plan_canary.py \ + --node-fixture node-fixture \ + --polyglot-fixture polyglot-fixture \ + --zed "$RUNNER_TEMP/zed-plan-bin/zed" \ + --work-root "$WORK_ROOT" + + - name: Prove sources and workflow are immutable and read-only + shell: bash + run: | + set -euo pipefail + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C node-fixture status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C polyglot-fixture status --porcelain=v1 --untracked-files=all)" + python3 - <<'PY' + import re + from pathlib import Path + + workflow = Path('harness/.github/workflows/nix-export-plan.yml').read_text() + pin = Path('harness/.zed-nix-plan-cli-ref').read_text().strip() + assert re.fullmatch(r'[0-9a-f]{40}', pin) + assert 'permissions:\n contents: read' in workflow + assert 'persist-credentials: false' in workflow + assert 'ubuntu-24.04' in workflow and 'macos-15' in workflow + for exact in ( + '222cdf57f48530fce8e6c1f58632d9676203512e', + '998964aa58e49595547650b4a8da407b7a2283a9', + 'c2e049006453c26ca8ca291783f681fce75cb01f', + ): + assert exact in workflow + for match in re.finditer(r'^\s*uses:\s*([^\s]+)$', workflow, re.MULTILINE): + assert re.search(r'@[0-9a-f]{40}$', match.group(1)), match.group(1) + for forbidden in ( + 'contents' + ': write', + 'git' + ' push', + 'pull_request' + '_target:', + 'id-token' + ': write', + 'packages' + ': write', + 'secrets' + ': inherit', + 'ref:' + ' main', + ): + assert forbidden not in workflow, forbidden + PY + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-nix-export-plan-canary + shell: bash + run: | + set +e + diagnostics="$WORK_ROOT/diagnostics" + mkdir -p "$diagnostics" + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/zed-cli-status.txt" 2>&1 + git -C node-fixture status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/node-fixture-status.txt" 2>&1 + git -C polyglot-fixture status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/polyglot-fixture-status.txt" 2>&1 + python3 - "$WORK_ROOT" "$diagnostics/work-tree.txt" <<'PY' + import os + import sys + from pathlib import Path + + root = Path(sys.argv[1]) + output = Path(sys.argv[2]) + rows = [] + if root.exists(): + for current, dirs, files in os.walk(root, followlinks=False): + current_path = Path(current) + for name in sorted(dirs + files): + path = current_path / name + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + rows.append(f"l {relative} -> {os.readlink(path)}") + elif path.is_dir(): + rows.append(f"d {relative}") + else: + rows.append(f"f {relative}") + output.write_text("\n".join(rows) + ("\n" if rows else ""), encoding="utf-8") + PY + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: nix-export-plan-diagnostics-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-nix-export-plan-canary/diagnostics + if-no-files-found: warn + retention-days: 3 + include-hidden-files: false diff --git a/.github/workflows/nix-flake-bundle.yml b/.github/workflows/nix-flake-bundle.yml new file mode 100644 index 00000000..12f60e13 --- /dev/null +++ b/.github/workflows/nix-flake-bundle.yml @@ -0,0 +1,298 @@ +name: external nix flake bundle + +on: + pull_request: + paths: + - ".github/workflows/nix-flake-bundle.yml" + - ".zed-nix-bundle-cli-ref" + - "scripts/nix_flake_bundle_external.rs" + - "scripts/verify_nix_flake_bundle_canary.py" + - "docs/nix-flake-bundle-canary.md" + push: + branches: [main] + paths: + - ".github/workflows/nix-flake-bundle.yml" + - ".zed-nix-bundle-cli-ref" + - "scripts/nix_flake_bundle_external.rs" + - "scripts/verify_nix_flake_bundle_canary.py" + - "docs/nix-flake-bundle-canary.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Optional immutable zed-cli commit override + required: false + default: "" + +permissions: + contents: read + +concurrency: + group: external-nix-flake-bundle-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + canary: + name: Clean-room flake bundle on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 55 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15] + + steps: + - name: Check out harness and reviewed pin + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + path: harness + persist-credentials: false + show-progress: false + + - name: Load immutable CLI pin + id: pin + shell: bash + run: | + set -euo pipefail + override="${{ github.event.inputs.zed_cli_ref || '' }}" + if [[ -n "$override" ]]; then + ref="$override" + else + ref="$(tr -d '[:space:]' < harness/.zed-nix-bundle-cli-ref)" + fi + [[ "$ref" =~ ^[0-9a-f]{40}$ ]] + printf 'ref=%s\n' "$ref" >> "$GITHUB_OUTPUT" + printf 'zed-cli=%s\n' "$ref" + + - name: Check out exact zed-cli renderer + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: zed-pkg/zed-cli + ref: ${{ steps.pin.outputs.ref }} + path: zed-cli + persist-credentials: false + show-progress: false + + - uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + components: rustfmt + + - uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 + with: + extra_nix_config: | + experimental-features = nix-command flakes + accept-flake-config = false + + - name: Verify exact candidate and shared-contract pins + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "${{ steps.pin.outputs.ref }}" + grep -F 'rev = "c2e049006453c26ca8ca291783f681fce75cb01f"' zed-cli/Cargo.toml + grep -F '#c2e049006453c26ca8ca291783f681fce75cb01f' zed-cli/Cargo.lock + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + python3 -m py_compile harness/scripts/verify_nix_flake_bundle_canary.py + + - name: Exercise the renderer only through its public API + id: render + shell: bash + env: + CANDIDATE_SHA: ${{ steps.pin.outputs.ref }} + FORBIDDEN_CANARY: bundle-secret-${{ github.run_id }}-${{ github.run_attempt }} + FORBIDDEN_PATH: ${{ runner.temp }}/private-workspace-must-not-leak + run: | + set -euo pipefail + bundle_a="$RUNNER_TEMP/zed-nix-bundle-a" + bundle_b="$RUNNER_TEMP/zed-nix-bundle-b" + evidence="$RUNNER_TEMP/zed-nix-bundle-evidence" + report="$evidence/report.json" + test ! -e "$bundle_a" + test ! -e "$bundle_b" + mkdir -p "$evidence" + + system="$(nix eval --impure --raw --expr builtins.currentSystem)" + install -m 0644 \ + harness/scripts/nix_flake_bundle_external.rs \ + zed-cli/tests/external_nix_flake_bundle.rs + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + EXTERNAL_NIX_SYSTEM="$system" \ + BUNDLE_OUT_A="$bundle_a" \ + BUNDLE_OUT_B="$bundle_b" \ + BUNDLE_REPORT="$report" \ + cargo test --locked \ + --manifest-path zed-cli/Cargo.toml \ + --test external_nix_flake_bundle \ + -- --nocapture --test-threads=1 + rm zed-cli/tests/external_nix_flake_bundle.rs + + python3 harness/scripts/verify_nix_flake_bundle_canary.py \ + --bundle-a "$bundle_a" \ + --bundle-b "$bundle_b" \ + --report "$report" \ + --forbidden "$FORBIDDEN_CANARY" \ + --forbidden "$FORBIDDEN_PATH" \ + --exercise-tamper \ + | tee "$evidence/verification.jsonl" + + printf 'bundle_a=%s\n' "$bundle_a" >> "$GITHUB_OUTPUT" + printf 'bundle_b=%s\n' "$bundle_b" >> "$GITHUB_OUTPUT" + printf 'evidence=%s\n' "$evidence" >> "$GITHUB_OUTPUT" + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + + - name: Prime the locked closure, then replay both flakes offline + id: build + shell: bash + env: + BUNDLE_A: ${{ steps.render.outputs.bundle_a }} + BUNDLE_B: ${{ steps.render.outputs.bundle_b }} + EVIDENCE: ${{ steps.render.outputs.evidence }} + run: | + set -euo pipefail + + # Archive both byte-identical sources and realize the exact locked package + # once while the configured binary cache is available. A fresh empty Nix + # store cannot honestly replay stdenv offline without first obtaining its + # immutable closure; attempting that merely bootstraps all of Nixpkgs from + # source and tests runner capacity rather than the generated package. + for bundle in "$BUNDLE_A" "$BUNDLE_B"; do + ( + cd "$bundle" + nix flake archive --no-update-lock-file + ) + done + primed_out="$( + cd "$BUNDLE_A" + nix build \ + --no-update-lock-file \ + --no-link \ + --print-build-logs \ + --print-out-paths \ + .#sample + )" + + for bundle in "$BUNDLE_A" "$BUNDLE_B"; do + ( + cd "$bundle" + nix flake check \ + --offline \ + --no-update-lock-file \ + --print-build-logs + ) + done + + out_a="$( + cd "$BUNDLE_A" + nix build \ + --offline \ + --no-update-lock-file \ + --no-link \ + --print-build-logs \ + --print-out-paths \ + .#sample + )" + out_b="$( + cd "$BUNDLE_B" + nix build \ + --offline \ + --no-update-lock-file \ + --no-link \ + --print-build-logs \ + --print-out-paths \ + .#sample + )" + test "$primed_out" = "$out_a" + test "$out_a" = "$out_b" + + mkdir -p "$RUNNER_TEMP/empty-home" + actual="$(env -i HOME="$RUNNER_TEMP/empty-home" PATH=/usr/bin:/bin "$out_a/bin/sample")" + test "$actual" = external-bundle-ok + if nix-store -q --references "$out_a" | grep -Eqi 'zed-cli|zed-pkg'; then + echo 'generated package unexpectedly retains a Zed runtime dependency' >&2 + exit 1 + fi + + nix path-info --json "$out_a" > "$EVIDENCE/nix-path-info.json" + printf '%s\n' "$out_a" > "$EVIDENCE/output-path.txt" + python3 - "$BUNDLE_A" "$EVIDENCE/bundle-files.sha256" <<'PY' + import hashlib + import sys + from pathlib import Path + + root = Path(sys.argv[1]) + output = Path(sys.argv[2]) + rows = [] + for path in sorted(candidate for candidate in root.rglob('*') if candidate.is_file()): + rows.append( + f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.relative_to(root).as_posix()}" + ) + output.write_text("\n".join(rows) + "\n", encoding="utf-8") + PY + + - name: Prove the workflow remains immutable and credential-free + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import re + from pathlib import Path + + workflow = Path('harness/.github/workflows/nix-flake-bundle.yml').read_text() + pin = Path('harness/.zed-nix-bundle-cli-ref').read_text().strip() + rust = Path('harness/scripts/nix_flake_bundle_external.rs').read_text() + verifier = Path('harness/scripts/verify_nix_flake_bundle_canary.py').read_text() + + assert re.fullmatch(r'[0-9a-f]{40}', pin) + assert 'permissions:\n contents: read' in workflow + assert 'persist-credentials: false' in workflow + assert 'ubuntu-24.04' in workflow and 'macos-15' in workflow + assert '--offline' in workflow and '--no-update-lock-file' in workflow + assert 'primed_out=' in workflow + assert '--exercise-tamper' in workflow + assert 'credential_canaries_retained' in rust + assert 'tamper_cases' in verifier + for match in re.finditer(r'^\s*uses:\s*([^\s]+)$', workflow, re.MULTILINE): + value = match.group(1) + assert re.search(r'@[0-9a-f]{40}$', value), value + for forbidden in ( + 'contents' + ': write', + 'pull_request' + '_target:', + 'secrets' + ': inherit', + 'git' + ' push', + 'id-token' + ': write', + 'packages' + ': write', + 'reqwest' + '::', + 'Command' + '::new', + ): + assert forbidden not in workflow + '\n' + rust, forbidden + PY + + - name: Upload non-secret reproducibility evidence + if: success() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: nix-flake-bundle-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.render.outputs.evidence }} + if-no-files-found: error + retention-days: 7 + include-hidden-files: false + + - name: Collect failure diagnostics + if: failure() + shell: bash + run: | + set +e + diagnostics="$RUNNER_TEMP/nix-flake-bundle-diagnostics" + mkdir -p "$diagnostics" + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$diagnostics/zed-cli-status.txt" 2>&1 + find "$RUNNER_TEMP" -maxdepth 8 \( -type f -o -type l \) 2>/dev/null \ + | LC_ALL=C sort > "$diagnostics/work-tree.txt" 2>&1 + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: nix-flake-bundle-failure-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/nix-flake-bundle-diagnostics + if-no-files-found: warn + retention-days: 3 + include-hidden-files: false diff --git a/.github/workflows/oci-interface-lock-refresh.yml b/.github/workflows/oci-interface-lock-refresh.yml new file mode 100644 index 00000000..a55c0910 --- /dev/null +++ b/.github/workflows/oci-interface-lock-refresh.yml @@ -0,0 +1,75 @@ +name: exact OCI generated-file artifact + +on: + pull_request: + paths: + - ".github/workflows/oci-interface-lock-refresh.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: exact-oci-generated-files-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ZED_CLI_REF: a5ee43a148fe8fb1325c407c26e7d53f09f568de + ZED_INTERFACES_REF: 2082012d1903783a89200c3bd44d2659a6daf872 + +jobs: + generate: + runs-on: ubuntu-22.04 + timeout-minutes: 20 + steps: + - name: Check out exact product candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install formatter and generate exact bytes + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + cargo update --manifest-path zed-cli/Cargo.toml \ + -p zed-interfaces \ + --precise "$ZED_INTERFACES_REF" + cargo fmt --all --manifest-path zed-cli/Cargo.toml + git -C zed-cli diff --check + + mkdir -p evidence/tree + files=( + Cargo.lock + src/cli_oci.rs + src/completion.rs + src/nix_bundle_write.rs + src/nix_export_bundle/archive.rs + src/nix_export_bundle/templates.rs + src/oci_command.rs + tests/nix_bundle_write_boundaries.rs + tests/nix_bundle_write_cli.rs + tests/nix_export_bundle/common.rs + ) + for path in "${files[@]}"; do + mkdir -p "evidence/tree/$(dirname "$path")" + cp "zed-cli/$path" "evidence/tree/$path" + done + git -C zed-cli diff --binary > evidence/generated.patch + git -C zed-cli status --short > evidence/status.txt + find evidence/tree -type f -print0 | sort -z | xargs -0 sha256sum > evidence/SHA256SUMS + + - name: Upload exact generated files + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: exact-oci-generated-files-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + if-no-files-found: error + retention-days: 7 + include-hidden-files: false \ No newline at end of file diff --git a/.github/workflows/oci-interop-consolidated.yml b/.github/workflows/oci-interop-consolidated.yml new file mode 100644 index 00000000..447dea10 --- /dev/null +++ b/.github/workflows/oci-interop-consolidated.yml @@ -0,0 +1,269 @@ +name: consolidated OCI interoperability contract + +on: + pull_request: + paths: + - ".github/workflows/oci-interop-consolidated.yml" + - "scripts/oci_plan_contract.py" + - "scripts/oci_layout_contract.py" + - "scripts/oci_oras_push_contract.py" + - "docs/oci-plan-contract.md" + - "docs/oci-layout-contract.md" + - "docs/oci-oras-push-contract.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli OCI candidate + required: true + default: 172cdbc39a72cbf28fc3170d35bc4d3fd832f65d + type: string + +permissions: + contents: read + +concurrency: + group: oci-interop-consolidated-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + HARNESS_REF: ${{ github.event.pull_request.head.sha || github.sha }} + ZED_CLI_REF: ${{ inputs.zed_cli_ref || '172cdbc39a72cbf28fc3170d35bc4d3fd832f65d' }} + ZED_INTERFACES_REF: 2082012d1903783a89200c3bd44d2659a6daf872 + NODE_LIB_REF: 2d41e658a382fdff27ef87a0db24dbaff280a685 + NODE_APP_REF: ef794674647b3620e711f4f855865f8ad3340498 + POLYGLOT_LIB_REF: e62ceb6337324a294d4fc861692a267ff0db97d0 + REGISTRY_IMAGE: registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 + REGISTRY_HOST: 127.0.0.1:5000 + REGISTRY_USERNAME: zed-contract + REGISTRY_PASSWORD: zed-contract-password-not-a-secret + +jobs: + contract: + name: Exact plan, layout, and authenticated-push candidate + runs-on: ubuntu-22.04 + timeout-minutes: 75 + steps: + - name: Validate immutable inputs + run: | + set -euo pipefail + [[ "$HARNESS_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$NODE_LIB_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$NODE_APP_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$POLYGLOT_LIB_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$REGISTRY_IMAGE" =~ ^registry@sha256:[0-9a-f]{64}$ ]] + + - name: Check out exact harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.HARNESS_REF }} + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed CLI candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact node library fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-lib + ref: ${{ env.NODE_LIB_REF }} + path: fixtures/node-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact node application fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-app + ref: ${{ env.NODE_APP_REF }} + path: fixtures/node-app + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact polyglot fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/polyglot-lib + ref: ${{ env.POLYGLOT_LIB_REF }} + path: fixtures/polyglot-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install exact stable ORAS CLI + uses: oras-project/setup-oras@4fb5323ce5e4debde6f8fe277e461b0815a75992 + with: + version: 1.3.2 + + - name: Install declared Rust tools and registry prerequisites + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + sudo apt-get update + sudo apt-get install --yes --no-install-recommends apache2-utils strace + rustc --version --verbose + cargo --version --verbose + cargo clippy --version + oras version + strace --version | head -1 + docker version + + - name: Verify exact checkouts and harness syntax + run: | + set -euo pipefail + test "$(git -C harness rev-parse HEAD)" = "$HARNESS_REF" + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C fixtures/node-lib rev-parse HEAD)" = "$NODE_LIB_REF" + test "$(git -C fixtures/node-app rev-parse HEAD)" = "$NODE_APP_REF" + test "$(git -C fixtures/polyglot-lib rev-parse HEAD)" = "$POLYGLOT_LIB_REF" + test -z "${ZED_PKG_TOKEN:-}" + PYTHONPYCACHEPREFIX="$RUNNER_TEMP/pycache" python3 -m py_compile \ + harness/scripts/oci_plan_contract.py \ + harness/scripts/oci_layout_contract.py \ + harness/scripts/oci_oras_push_contract.py + + - name: Generate exact combined lock and validate candidate + run: | + set -euo pipefail + grep -F "rev = \"$ZED_INTERFACES_REF\"" zed-cli/Cargo.toml + cargo update --manifest-path zed-cli/Cargo.toml \ + -p zed-interfaces \ + --precise "$ZED_INTERFACES_REF" + grep -F "rev=$ZED_INTERFACES_REF#$ZED_INTERFACES_REF" zed-cli/Cargo.lock + mkdir -p evidence/lock + cp zed-cli/Cargo.lock evidence/lock/Cargo.lock + sha256sum evidence/lock/Cargo.lock > evidence/lock/SHA256SUMS + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets oci + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets cli_oci + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets oci_command + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + zed-cli/target/release/zed --version + + - name: Run immutable publication-plan contract + run: | + set -euo pipefail + rm -rf evidence/plan + python3 harness/scripts/oci_plan_contract.py \ + --zed zed-cli/target/release/zed \ + --strace "$(command -v strace)" \ + --node-lib fixtures/node-lib \ + --node-app fixtures/node-app \ + --polyglot-lib fixtures/polyglot-lib \ + --evidence evidence/plan + + - name: Run local image-layout contract + run: | + set -euo pipefail + rm -rf evidence/layout + python3 harness/scripts/oci_layout_contract.py \ + --zed zed-cli/target/release/zed \ + --strace "$(command -v strace)" \ + --node-lib fixtures/node-lib \ + --node-app fixtures/node-app \ + --polyglot-lib fixtures/polyglot-lib \ + --evidence evidence/layout + + - name: Start isolated basic-auth OCI registry + run: | + set -euo pipefail + mkdir -p evidence registry-auth + htpasswd -Bbn "$REGISTRY_USERNAME" "$REGISTRY_PASSWORD" > registry-auth/htpasswd + chmod 600 registry-auth/htpasswd + docker run --detach \ + --name zed-oci-contract-registry \ + --publish 127.0.0.1:5000:5000 \ + --volume "$PWD/registry-auth:/auth:ro" \ + --env REGISTRY_AUTH=htpasswd \ + --env REGISTRY_AUTH_HTPASSWD_REALM='Zed OCI Contract' \ + --env REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + "$REGISTRY_IMAGE" + + for attempt in $(seq 1 60); do + status=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --user "$REGISTRY_USERNAME:$REGISTRY_PASSWORD" \ + "http://$REGISTRY_HOST/v2/" || true) + if [[ "$status" == 200 ]]; then + exit 0 + fi + sleep 1 + done + docker logs zed-oci-contract-registry >&2 || true + echo 'registry did not become healthy' >&2 + exit 1 + + - name: Run authenticated ORAS push contract + run: | + set -euo pipefail + rm -rf evidence/push + python3 harness/scripts/oci_oras_push_contract.py \ + --zed zed-cli/target/release/zed \ + --oras "$(command -v oras)" \ + --strace "$(command -v strace)" \ + --node-lib fixtures/node-lib \ + --registry "$REGISTRY_HOST" \ + --username "$REGISTRY_USERNAME" \ + --password "$REGISTRY_PASSWORD" \ + --evidence evidence/push + + - name: Capture and remove registry + if: always() + run: | + set +e + mkdir -p evidence + docker inspect zed-oci-contract-registry > evidence/registry-inspect.json 2>&1 + docker logs zed-oci-contract-registry > evidence/registry.log 2>&1 + docker rm --force zed-oci-contract-registry + rm -rf registry-auth + + - name: Reject test-password leakage from evidence + if: always() + run: | + set -euo pipefail + if grep --recursive --fixed-strings --binary-files=without-match \ + "$REGISTRY_PASSWORD" evidence; then + echo 'test registry password leaked into uploaded evidence' >&2 + exit 1 + fi + + - name: Require clean exact checkouts + if: always() + run: | + set -euo pipefail + git -C zed-cli checkout -- Cargo.lock + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C zed-cli status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C fixtures/node-lib status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C fixtures/node-app status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C fixtures/polyglot-lib status --porcelain=v1 --untracked-files=all)" + + - name: Upload consolidated OCI evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: oci-interop-consolidated-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false \ No newline at end of file diff --git a/.github/workflows/oci-layout-contract.yml b/.github/workflows/oci-layout-contract.yml new file mode 100644 index 00000000..bd886061 --- /dev/null +++ b/.github/workflows/oci-layout-contract.yml @@ -0,0 +1,178 @@ +name: local OCI image-layout contract + +on: + pull_request: + paths: + - ".github/workflows/oci-layout-contract.yml" + - "scripts/oci_layout_contract.py" + - "scripts/oci_plan_contract.py" + - "docs/oci-layout-contract.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli layout candidate + required: true + default: c6207598240c073ecce13e344c54c6074e37eb6a + type: string + +permissions: + contents: read + +concurrency: + group: oci-layout-contract-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + HARNESS_REF: ${{ github.event.pull_request.head.sha || github.sha }} + ZED_CLI_REF: ${{ inputs.zed_cli_ref || 'c6207598240c073ecce13e344c54c6074e37eb6a' }} + NODE_LIB_REF: 2d41e658a382fdff27ef87a0db24dbaff280a685 + NODE_APP_REF: ef794674647b3620e711f4f855865f8ad3340498 + POLYGLOT_LIB_REF: e62ceb6337324a294d4fc861692a267ff0db97d0 + +jobs: + contract: + name: Exact local-layout candidate + runs-on: ubuntu-24.04 + timeout-minutes: 55 + steps: + - name: Validate immutable inputs + run: | + set -euo pipefail + [[ "$HARNESS_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$NODE_LIB_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$NODE_APP_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$POLYGLOT_LIB_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out exact harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.HARNESS_REF }} + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed CLI candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact node library fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-lib + ref: ${{ env.NODE_LIB_REF }} + path: fixtures/node-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact node application fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-app + ref: ${{ env.NODE_APP_REF }} + path: fixtures/node-app + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact polyglot fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/polyglot-lib + ref: ${{ env.POLYGLOT_LIB_REF }} + path: fixtures/polyglot-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install declared Rust toolchain and syscall tracer + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + if ! command -v strace >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install --yes --no-install-recommends strace + fi + rustc --version --verbose + cargo --version --verbose + cargo clippy --version + strace --version | head -1 + + - name: Verify exact checkouts and harness syntax + run: | + set -euo pipefail + test "$(git -C harness rev-parse HEAD)" = "$HARNESS_REF" + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C fixtures/node-lib rev-parse HEAD)" = "$NODE_LIB_REF" + test "$(git -C fixtures/node-app rev-parse HEAD)" = "$NODE_APP_REF" + test "$(git -C fixtures/polyglot-lib rev-parse HEAD)" = "$POLYGLOT_LIB_REF" + python3 -m py_compile \ + harness/scripts/oci_plan_contract.py \ + harness/scripts/oci_layout_contract.py + + - name: Capture deterministic rustfmt result + run: | + set -euo pipefail + mkdir -p evidence/formatted-source + cargo fmt --all --manifest-path zed-cli/Cargo.toml + git -C zed-cli diff --binary -- \ + src/oci_layout.rs \ + src/cli_oci.rs \ + src/main.rs \ + src/lib.rs > evidence/rustfmt.patch + cp zed-cli/src/oci_layout.rs evidence/formatted-source/oci_layout.rs + cp zed-cli/src/cli_oci.rs evidence/formatted-source/cli_oci.rs + cp zed-cli/src/main.rs evidence/formatted-source/main.rs + cp zed-cli/src/lib.rs evidence/formatted-source/lib.rs + + - name: Build, test, and lint the formatted candidate + run: | + set -euo pipefail + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets oci + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets oci_layout + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + zed-cli/target/release/zed --version + + - name: Run black-box local-layout contract + run: | + set -euo pipefail + rm -rf evidence/layout + python3 harness/scripts/oci_layout_contract.py \ + --zed zed-cli/target/release/zed \ + --strace "$(command -v strace)" \ + --node-lib fixtures/node-lib \ + --node-app fixtures/node-app \ + --polyglot-lib fixtures/polyglot-lib \ + --evidence evidence/layout + + - name: Require the pinned candidate to be rustfmt-clean + run: | + set -euo pipefail + git -C zed-cli diff --check + git -C zed-cli diff --exit-code + + - name: Upload formatting and contract evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: oci-layout-contract-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + if-no-files-found: warn + retention-days: 7 + include-hidden-files: false \ No newline at end of file diff --git a/.github/workflows/oci-oras-push-contract.yml b/.github/workflows/oci-oras-push-contract.yml new file mode 100644 index 00000000..19c372ac --- /dev/null +++ b/.github/workflows/oci-oras-push-contract.yml @@ -0,0 +1,196 @@ +name: authenticated ORAS OCI push contract + +on: + pull_request: + paths: + - ".github/workflows/oci-oras-push-contract.yml" + - "scripts/oci_oras_push_contract.py" + - "scripts/oci_layout_contract.py" + - "scripts/oci_plan_contract.py" + - "docs/oci-oras-push-contract.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli ORAS push candidate + required: true + default: 1241f1cbe90e0b9cf9d71cd7ba30f5e3c1c1c296 + type: string + +permissions: + contents: read + +concurrency: + group: oci-oras-push-contract-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + HARNESS_REF: ${{ github.event.pull_request.head.sha || github.sha }} + ZED_CLI_REF: ${{ inputs.zed_cli_ref || '1241f1cbe90e0b9cf9d71cd7ba30f5e3c1c1c296' }} + NODE_LIB_REF: 2d41e658a382fdff27ef87a0db24dbaff280a685 + REGISTRY_IMAGE: registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373 + REGISTRY_HOST: 127.0.0.1:5000 + REGISTRY_USERNAME: zed-contract + REGISTRY_PASSWORD: zed-contract-password-not-a-secret + +jobs: + contract: + name: Exact authenticated-push candidate + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Validate immutable inputs + run: | + set -euo pipefail + [[ "$HARNESS_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$NODE_LIB_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$REGISTRY_IMAGE" =~ ^registry@sha256:[0-9a-f]{64}$ ]] + + - name: Check out exact harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.HARNESS_REF }} + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed CLI candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact node library fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-lib + ref: ${{ env.NODE_LIB_REF }} + path: fixtures/node-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install exact stable ORAS CLI + uses: oras-project/setup-oras@4fb5323ce5e4debde6f8fe277e461b0815a75992 + with: + version: 1.3.2 + + - name: Install declared Rust tools and registry prerequisites + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt --component clippy + rustup default stable + sudo apt-get update + sudo apt-get install --yes --no-install-recommends apache2-utils strace + rustc --version --verbose + cargo --version --verbose + cargo clippy --version + oras version + strace --version | head -1 + docker version + + - name: Verify exact checkouts and harness syntax + run: | + set -euo pipefail + test "$(git -C harness rev-parse HEAD)" = "$HARNESS_REF" + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C fixtures/node-lib rev-parse HEAD)" = "$NODE_LIB_REF" + python3 -m py_compile \ + harness/scripts/oci_plan_contract.py \ + harness/scripts/oci_layout_contract.py \ + harness/scripts/oci_oras_push_contract.py + + - name: Build, test, and lint the exact candidate + run: | + set -euo pipefail + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets oci_push + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets cli_oci + cargo test --locked --manifest-path zed-cli/Cargo.toml cli_flags_toml_is_in_sync_with_clap + cargo test --locked --manifest-path zed-cli/Cargo.toml readme_documents_every_command + cargo clippy --locked --manifest-path zed-cli/Cargo.toml --all-targets -- -D warnings + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + zed-cli/target/release/zed --version + + - name: Start isolated basic-auth OCI registry + run: | + set -euo pipefail + mkdir -p evidence registry-auth + htpasswd -Bbn "$REGISTRY_USERNAME" "$REGISTRY_PASSWORD" > registry-auth/htpasswd + chmod 600 registry-auth/htpasswd + docker run --detach \ + --name zed-oci-contract-registry \ + --publish 127.0.0.1:5000:5000 \ + --volume "$PWD/registry-auth:/auth:ro" \ + --env REGISTRY_AUTH=htpasswd \ + --env REGISTRY_AUTH_HTPASSWD_REALM='Zed OCI Contract' \ + --env REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + "$REGISTRY_IMAGE" + + for attempt in $(seq 1 60); do + status=$(curl --silent --output /dev/null --write-out '%{http_code}' \ + --user "$REGISTRY_USERNAME:$REGISTRY_PASSWORD" \ + "http://$REGISTRY_HOST/v2/" || true) + if [[ "$status" == 200 ]]; then + exit 0 + fi + sleep 1 + done + docker logs zed-oci-contract-registry >&2 || true + echo 'registry did not become healthy' >&2 + exit 1 + + - name: Run black-box authenticated ORAS push contract + run: | + set -euo pipefail + rm -rf evidence/contract + python3 harness/scripts/oci_oras_push_contract.py \ + --zed zed-cli/target/release/zed \ + --oras "$(command -v oras)" \ + --strace "$(command -v strace)" \ + --node-lib fixtures/node-lib \ + --registry "$REGISTRY_HOST" \ + --username "$REGISTRY_USERNAME" \ + --password "$REGISTRY_PASSWORD" \ + --evidence evidence/contract + + - name: Capture registry evidence + if: always() + run: | + set +e + mkdir -p evidence + docker inspect zed-oci-contract-registry > evidence/registry-inspect.json 2>&1 + docker logs zed-oci-contract-registry > evidence/registry.log 2>&1 + docker rm --force zed-oci-contract-registry + rm -rf registry-auth + + - name: Reject test-password leakage from evidence + if: always() + run: | + set -euo pipefail + if grep --recursive --fixed-strings --binary-files=without-match \ + "$REGISTRY_PASSWORD" evidence; then + echo 'test registry password leaked into uploaded evidence' >&2 + exit 1 + fi + + - name: Upload authenticated push evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: oci-oras-push-contract-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + if-no-files-found: warn + retention-days: 7 + include-hidden-files: false \ No newline at end of file diff --git a/.github/workflows/oci-plan-contract.yml b/.github/workflows/oci-plan-contract.yml new file mode 100644 index 00000000..d7192e3b --- /dev/null +++ b/.github/workflows/oci-plan-contract.yml @@ -0,0 +1,153 @@ +name: immutable OCI plan contract + +on: + pull_request: + paths: + - ".github/workflows/oci-plan-contract.yml" + - "scripts/oci_plan_contract.py" + - "docs/oci-plan-contract.md" + - "README.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli candidate commit + required: true + default: 8bc7d97711fc44db2f31419e4b188c8916288787 + type: string + +permissions: + contents: read + +concurrency: + group: oci-plan-contract-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + HARNESS_REF: ${{ github.event.pull_request.head.sha || github.sha }} + ZED_CLI_REF: ${{ inputs.zed_cli_ref || '8bc7d97711fc44db2f31419e4b188c8916288787' }} + NODE_LIB_REF: 2d41e658a382fdff27ef87a0db24dbaff280a685 + NODE_APP_REF: ef794674647b3620e711f4f855865f8ad3340498 + POLYGLOT_LIB_REF: e62ceb6337324a294d4fc861692a267ff0db97d0 + +jobs: + contract: + name: Exact OCI plan candidate + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Validate immutable inputs + run: | + set -euo pipefail + [[ "$HARNESS_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$NODE_LIB_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$NODE_APP_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$POLYGLOT_LIB_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out exact harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.HARNESS_REF }} + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact zed CLI candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact node library fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-lib + ref: ${{ env.NODE_LIB_REF }} + path: fixtures/node-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact node application fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/node-app + ref: ${{ env.NODE_APP_REF }} + path: fixtures/node-app + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact polyglot fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/polyglot-lib + ref: ${{ env.POLYGLOT_LIB_REF }} + path: fixtures/polyglot-lib + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install declared Rust toolchain and syscall tracer + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + if ! command -v strace >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install --yes --no-install-recommends strace + fi + rustc --version --verbose + cargo --version --verbose + strace --version | head -1 + + - name: Verify exact checkouts and harness syntax + run: | + set -euo pipefail + test "$(git -C harness rev-parse HEAD)" = "$HARNESS_REF" + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + test "$(git -C fixtures/node-lib rev-parse HEAD)" = "$NODE_LIB_REF" + test "$(git -C fixtures/node-app rev-parse HEAD)" = "$NODE_APP_REF" + test "$(git -C fixtures/polyglot-lib rev-parse HEAD)" = "$POLYGLOT_LIB_REF" + python3 -m py_compile harness/scripts/oci_plan_contract.py + + - name: Build and unit-test the exact candidate + run: | + set -euo pipefail + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --all-targets oci + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + zed-cli/target/release/zed --version + + - name: Run black-box OCI plan contract + run: | + set -euo pipefail + rm -rf evidence + python3 harness/scripts/oci_plan_contract.py \ + --zed zed-cli/target/release/zed \ + --strace "$(command -v strace)" \ + --node-lib fixtures/node-lib \ + --node-app fixtures/node-app \ + --polyglot-lib fixtures/polyglot-lib \ + --evidence evidence + + - name: Upload deterministic evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: oci-plan-contract-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + if-no-files-found: warn + retention-days: 7 + include-hidden-files: false \ No newline at end of file diff --git a/.github/workflows/opto-sync-adoption-fleet.yml b/.github/workflows/opto-sync-adoption-fleet.yml new file mode 100644 index 00000000..c2c41491 --- /dev/null +++ b/.github/workflows/opto-sync-adoption-fleet.yml @@ -0,0 +1,172 @@ +name: Opto Sync adoption fleet canary + +on: + pull_request: + paths: + - .github/workflows/opto-sync-adoption-fleet.yml + - fixtures/opto-sync-adoption/** + push: + branches: [main] + paths: + - .github/workflows/opto-sync-adoption-fleet.yml + - fixtures/opto-sync-adoption/** + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: opto-sync-adoption-fleet-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact harness revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + show-progress: false + + - name: Validate exact source blobs and product boundaries + run: | + set -euo pipefail + python3 - <<'PY' + from __future__ import annotations + + import hashlib + import json + import re + from pathlib import Path + + ROOT = Path("fixtures/opto-sync-adoption") + CLIENT_COMMIT = "068414c8ff7d4262d0a395959b5209d5908f0fcc" + VALIDATOR_COMMIT = "da7459fdf3ce4096ce6b88aeed5fc98066f804d9" + CHECKOUT_COMMIT = "11d5960a326750d5838078e36cf38b85af677262" + + EXPECTED = { + "sonus-auris": { + "repository": "sonus-auris/sonus-auris-clients", + "head": "21c6db6ea05194513b01b64c433ef29a90e3da21", + "manifest_blob": "dc80f025ee9f473dac0bb5ca0f789dd92ad7cbac", + "workflow_blob": "16ba4805c55853ff8d1eea890e7f5f7fc4b7ddb9", + "owner": "sonus-auris", + "scopes": {"recordingMetadata", "retentionPolicies", "schedule", "syncCheckpoints"}, + "transports": {"http", "supabase-realtime", "websocket"}, + "excluded": {"raw audio", "plaintext recordings", "encryption keys", "recovery material"}, + }, + "3fa": { + "repository": "3FA-app/3fa-clients", + "head": "0c0783d9b9b5f4eb9123951399713b59085359d4", + "manifest_blob": "f809b8a2e2e0e0a373c56228ff491be25a7896b0", + "workflow_blob": "30075c0db01859435b36bd0fb1833e5fcc885014", + "owner": "3FA-app", + "scopes": {"deviceDirectory", "encryptedMessageMetadata", "keyBundleMetadata", "syncCheckpoints"}, + "transports": {"http", "supabase-realtime", "websocket"}, + "excluded": {"private keys", "otp seeds", "plaintext", "access or refresh tokens"}, + }, + "scintilla": { + "repository": "scintilla-run/scintilla-clients", + "head": "c4c1dcc179e96eccf2273a00029c62b5f9a1e0d6", + "manifest_blob": "8366532f877636a7e8785643aec1b4295567ef95", + "workflow_blob": "c746d8d3a39699198eb5fd823fea11c2cecc3e65", + "owner": "scintilla-run", + "scopes": {"artifactIndex", "experimentDefinitions", "observationCheckpoints", "runMetadata"}, + "transports": {"http", "websocket"}, + "excluded": {"artifact binaries", "runner credentials", "signing keys", "live high-volume telemetry streams"}, + }, + } + + def git_blob_sha(data: bytes) -> str: + header = f"blob {len(data)}\0".encode() + return hashlib.sha1(header + data).hexdigest() + + evidence = {"schema": "opto-sync.fleet-canary.v1", "projects": {}} + + for fixture, contract in EXPECTED.items(): + fixture_root = ROOT / fixture + manifest_path = fixture_root / ".opto-sync.json" + workflow_path = fixture_root / "opto-sync-adoption.yml" + manifest_bytes = manifest_path.read_bytes() + workflow_bytes = workflow_path.read_bytes() + + assert git_blob_sha(manifest_bytes) == contract["manifest_blob"] + assert git_blob_sha(workflow_bytes) == contract["workflow_blob"] + assert re.fullmatch(r"[0-9a-f]{40}", contract["head"]) + + manifest = json.loads(manifest_bytes) + assert manifest["schemaVersion"] == "opto-sync.adoption.v1" + assert manifest["project"] == contract["repository"] + assert manifest["clientPackage"] == { + "package": "opto-sync/opto-sync-clients", + "repository": "https://github.com/opto-sync/opto-sync-clients", + "version": "0.2.0", + "commit": CLIENT_COMMIT, + } + assert set(manifest["languages"]) == {"dart", "gleam", "rust", "typescript"} + assert set(manifest["dataScopes"]) == contract["scopes"] + assert set(manifest["localStores"]) == {"indexeddb", "sqlite"} + assert set(manifest["transports"]) == contract["transports"] + assert set(manifest["writeStrategies"]) == { + "local-durable", + "local-then-remote", + "remote-confirmed", + } + assert manifest["conflictPolicy"] == { + "arrayStrategy": "mergeByKey", + "resolveByTimestamp": True, + "lwwKeys": ["updatedAt"], + "arrayMatchKeys": ["id"], + "tombstones": True, + } + assert manifest["background"] == { + "webServiceWorker": True, + "mobileWorker": True, + "desktopWorker": True, + } + assert manifest["security"] == { + "authenticatedTransport": True, + "encryptedAtRest": True, + "secretsInManifest": False, + } + assert manifest["rollout"] == { + "phase": "declared", + "owners": [contract["owner"]], + } + notes = manifest["notes"].lower() + assert contract["excluded"] <= {term for term in contract["excluded"] if term in notes} + + workflow = workflow_bytes.decode() + assert "pull_request_target:" not in workflow + assert "contents: write" not in workflow + assert "permissions:\n contents: read" in workflow + assert workflow.count("persist-credentials: false") == 2 + assert "repository: opto-sync/opto-sync-clients" in workflow + assert f"ref: {VALIDATOR_COMMIT}" in workflow + checkout_refs = re.findall(r"uses:\s*actions/checkout@([0-9a-f]{40})", workflow) + assert checkout_refs == [CHECKOUT_COMMIT, CHECKOUT_COMMIT] + + evidence["projects"][fixture] = { + "repository": contract["repository"], + "head": contract["head"], + "manifest_git_blob": contract["manifest_blob"], + "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + "workflow_git_blob": contract["workflow_blob"], + "workflow_sha256": hashlib.sha256(workflow_bytes).hexdigest(), + "rollout": "declared", + } + + evidence_path = Path("opto-sync-adoption-fleet-evidence.json") + evidence_path.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n") + print(evidence_path.read_text()) + PY + + - name: Upload exact-blob evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: opto-sync-adoption-fleet-${{ github.run_id }}-${{ github.run_attempt }} + path: opto-sync-adoption-fleet-evidence.json + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/ores-lint.yml b/.github/workflows/ores-lint.yml new file mode 100644 index 00000000..722a91c9 --- /dev/null +++ b/.github/workflows/ores-lint.yml @@ -0,0 +1,54 @@ +# Managed by .ores-lint/ - regenerated by the rollout script. +name: ores-lint + +on: + pull_request: + workflow_dispatch: + # Add `push:` here once this repo's lint debt is paid down. + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Detect project types + id: detect + run: | + { [ -f package.json ] || git ls-files '*.js' '*.mjs' '*.cjs' '*.ts' '*.tsx' | grep -q .; } \ + && echo "js=true" >> "$GITHUB_OUTPUT" || echo "js=false" >> "$GITHUB_OUTPUT" + { [ -f Cargo.toml ] || git ls-files '*.rs' | grep -q .; } \ + && echo "rs=true" >> "$GITHUB_OUTPUT" || echo "rs=false" >> "$GITHUB_OUTPUT" + { [ -f pubspec.yaml ] || git ls-files '*.dart' | grep -q .; } \ + && echo "dart=true" >> "$GITHUB_OUTPUT" || echo "dart=false" >> "$GITHUB_OUTPUT" + { [ -f gleam.toml ] || git ls-files '*.gleam' | grep -q .; } \ + && echo "gleam=true" >> "$GITHUB_OUTPUT" || echo "gleam=false" >> "$GITHUB_OUTPUT" + + # Node is the runner for ESLint and for the cross-language require-send + # scanner, so it is installed whenever any supported language is present. + - uses: actions/setup-node@v4 + if: steps.detect.outputs.js == 'true' || steps.detect.outputs.rs == 'true' || steps.detect.outputs.dart == 'true' || steps.detect.outputs.gleam == 'true' + with: + node-version: '22' + + # ESLint is a global tool here, not a repo dependency - so this job never + # runs `npm install` and never needs the repo's node_modules. + - name: Install eslint globally + if: steps.detect.outputs.js == 'true' + run: npm i -g eslint typescript-eslint + + - name: Install clippy + if: steps.detect.outputs.rs == 'true' + run: rustup component add clippy || true + + # Dart/Gleam SDKs are optional. When absent, dart.sh / gleam.sh skip + # with an actionable message; require-send.mjs still runs via Node. + - uses: dart-lang/setup-dart@v1 + if: steps.detect.outputs.dart == 'true' + continue-on-error: true + + - name: Run ores-lint + run: sh .ores-lint/lint.sh + env: + # Warn-only. Set to 1 here to make lint findings fail this repo's CI. + ORES_LINT_STRICT: '0' diff --git a/.github/workflows/org-app-installs.yml b/.github/workflows/org-app-installs.yml new file mode 100644 index 00000000..90ac9c13 --- /dev/null +++ b/.github/workflows/org-app-installs.yml @@ -0,0 +1,136 @@ +name: zed-pkg-test app installs + +on: + pull_request: + paths: + - scripts/org_app_installs.py + - .github/workflows/org-app-installs.yml + push: + branches: [main] + paths: + - scripts/org_app_installs.py + - .github/workflows/org-app-installs.yml + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to test + required: false + default: 8cc6745c75fd46dabf2e383cbcfc14db5ff933da + +permissions: + contents: read + +concurrency: + group: zed-pkg-test-app-installs-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '8cc6745c75fd46dabf2e383cbcfc14db5ff933da' }} + ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 + +jobs: + org-app-installs: + name: Real app fixtures, adapters, uninstall/restore, and workspaces + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out e2e harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out pinned zed-interfaces sibling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + rustc --version --verbose + cargo --version --verbose + git --version + python3 --version + + - name: Compile the org install harness and check Rust formatting + run: | + set -euo pipefail + python3 -m py_compile harness/scripts/org_app_installs.py + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + + - name: Build immutable zed binary + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + install -Dm755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Install real applications from zed-pkg-test + env: + WORK_ROOT: ${{ runner.temp }}/zed-pkg-test-app-installs + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/org_app_installs.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-pkg-test-app-installs + run: | + set +e + mkdir -p "$WORK_ROOT/diagnostics" + find "$WORK_ROOT" -maxdepth 12 -printf '%y %P -> %l\n' \ + | LC_ALL=C sort > "$WORK_ROOT/diagnostics/work-tree.txt" 2>&1 + find "$WORK_ROOT/registry" -maxdepth 10 -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/registry-sha256.txt" 2>&1 + find "$WORK_ROOT" -path '*/.zpkg.lock' -type f -print -exec sed -n '1,260p' {} \; \ + > "$WORK_ROOT/diagnostics/lockfiles.txt" 2>&1 + find "$WORK_ROOT" -path '*/.zed/*' -type f -print -exec sed -n '1,200p' {} \; \ + > "$WORK_ROOT/diagnostics/adapter-files.txt" 2>&1 + true + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: zed-pkg-test-app-installs-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-pkg-test-app-installs/diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/org-fixture-inventory.yml b/.github/workflows/org-fixture-inventory.yml new file mode 100644 index 00000000..3dda3232 --- /dev/null +++ b/.github/workflows/org-fixture-inventory.yml @@ -0,0 +1,91 @@ +name: zed-pkg-test organization inventory + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + schedule: + - cron: "31 4 * * *" + +permissions: + contents: read + +concurrency: + group: zed-pkg-test-inventory-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + inventory: + name: Audit core lifecycle and complete public fleet + runs-on: ubuntu-24.04 + timeout-minutes: 12 + steps: + - name: Check out the inventory harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Run all Python contract tests + run: | + set -euo pipefail + python3 -Werror -m unittest discover -s tests -p 'test_*.py' -v + + - name: Validate checked-in core lifecycle coverage + run: | + set -euo pipefail + python3 scripts/org_fixture_inventory.py \ + --output "$RUNNER_TEMP/zed-pkg-test-inventory-static.json" + + - name: Probe the complete live public fleet and all Zed manifests + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + python3 scripts/org_fleet_contract.py \ + --output "$RUNNER_TEMP/zed-pkg-test-inventory-live.json" + python3 - "$RUNNER_TEMP/zed-pkg-test-inventory-live.json" <<'PY' + import json + import sys + from pathlib import Path + + report = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + assert report["schema"] == "zed-pkg-test/public-fleet-inventory-v3", report + assert report["package_organizations"] == ["zed-pkg-test", "zedtest", "zed-pkg"], report + assert report["core_repository_count"] == 22, report + assert report["core_package_count"] == 21, report + assert report["live_repository_count"] >= report["core_repository_count"], report + assert report["supplemental_repository_count"] == ( + report["live_repository_count"] - report["core_repository_count"] + ), report + assert report["missing_core_repositories"] == [], report + assert report["errors"] == [], report + assert report["required_manifest_present_count"] == report["required_manifest_count"], report + assert report["manifest_present_count"] >= report["required_manifest_count"], report + assert all( + row["manifest"]["package_org_allowed"] is not False + for row in report["repositories"] + ), report + print( + "verified", + report["live_repository_count"], + "public repositories,", + report["manifest_present_count"], + "valid package manifests,", + report["supplemental_repository_count"], + "supplemental repositories, and reviewed package namespaces", + ",".join(report["package_organizations"]), + ) + PY + + - name: Upload inventory evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: zed-pkg-test-inventory-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-pkg-test-inventory-*.json + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/preempt-legacy-fleet-recovery.yml b/.github/workflows/preempt-legacy-fleet-recovery.yml new file mode 100644 index 00000000..714d287f --- /dev/null +++ b/.github/workflows/preempt-legacy-fleet-recovery.yml @@ -0,0 +1,25 @@ +# One-time concurrency preemption for the superseded legacy recovery workflow. +name: preempt legacy fleet recovery + +on: + push: + branches: [main] + paths: + - ".github/workflows/preempt-legacy-fleet-recovery.yml" + - ".github/fleet-preempt/START" + +permissions: + contents: read + +concurrency: + group: resume-rate-limited-test-org-fleet + cancel-in-progress: true + +jobs: + preempt: + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' && github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 2 + steps: + - name: Confirm legacy recovery preemption + run: echo "The superseded resume-rate-limited-test-org-fleet run has been preempted by repository concurrency." diff --git a/.github/workflows/preflight-open-pr-fleet.yml b/.github/workflows/preflight-open-pr-fleet.yml new file mode 100644 index 00000000..96f64782 --- /dev/null +++ b/.github/workflows/preflight-open-pr-fleet.yml @@ -0,0 +1,39 @@ +name: preflight open PR fleet merge + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/PREFLIGHT-2" + +permissions: + contents: read + +concurrency: + group: preflight-open-pr-fleet + cancel-in-progress: false + +jobs: + preflight: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + TARGET: "509" + APPLY: "false" + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + + - name: Check out the exact merger implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + - name: Run read-only exact-head preflight + shell: bash + run: node .github/scripts/merge-open-pr-fleet.mjs diff --git a/.github/workflows/probe-macos-paced-runner.yml b/.github/workflows/probe-macos-paced-runner.yml new file mode 100644 index 00000000..c03011eb --- /dev/null +++ b/.github/workflows/probe-macos-paced-runner.yml @@ -0,0 +1,44 @@ +name: probe macOS paced fleet runner + +on: + push: + branches: [agent/probe-macos-paced-runner] + paths: + - ".github/workflows/probe-macos-paced-runner.yml" + +permissions: + contents: read + +concurrency: + group: probe-macos-paced-runner + cancel-in-progress: true + +jobs: + probe: + runs-on: macos-15 + timeout-minutes: 10 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Install a pinned Node runtime + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + cache: '' + + - name: Verify the fleet runtime without secrets + shell: bash + run: | + set -euo pipefail + test "$RUNNER_OS" = macOS + git --version + node --version + bash --version | head -n 1 + node --check scripts/bootstrap-test-org-fleet-worktree-safe.mjs + node scripts/validate-test-org-fleet.mjs + node --test tests/bootstrap-retry-policy.test.mjs tests/paced-fleet-executor.test.mjs + node -e 'fetch("https://api.github.com/rate_limit", {headers:{"user-agent":"zed-pkg-test-macos-probe"}}).then(r=>{if(!r.ok) throw new Error(String(r.status)); console.log("github-api-ok")})' diff --git a/.github/workflows/probe-sonus-ci-arc.yml b/.github/workflows/probe-sonus-ci-arc.yml new file mode 100644 index 00000000..6862f9f4 --- /dev/null +++ b/.github/workflows/probe-sonus-ci-arc.yml @@ -0,0 +1,45 @@ +name: probe sonus-ci ARC runner + +on: + push: + branches: [agent/probe-sonus-ci-arc] + paths: + - ".github/workflows/probe-sonus-ci-arc.yml" + +permissions: + contents: read + +concurrency: + group: probe-sonus-ci-arc + cancel-in-progress: true + +jobs: + probe: + runs-on: [self-hosted, linux, sonus-ci] + timeout-minutes: 10 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Install a pinned Node runtime + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '22' + cache: '' + + - name: Verify the fleet runtime without secrets + shell: bash + run: | + set -euo pipefail + test "$RUNNER_OS" = Linux + test -n "$RUNNER_NAME" + git --version + node --version + python3 --version + node --check scripts/bootstrap-test-org-fleet-worktree-safe.mjs + node scripts/validate-test-org-fleet.mjs + node --test tests/bootstrap-retry-policy.test.mjs tests/paced-fleet-executor.test.mjs + node -e 'fetch("https://api.github.com/rate_limit", {headers:{"user-agent":"zed-pkg-test-arc-probe"}}).then(r=>{if(!r.ok) throw new Error(String(r.status)); console.log("github-api-ok")})' diff --git a/.github/workflows/production-package-inventory.yml b/.github/workflows/production-package-inventory.yml new file mode 100644 index 00000000..fce155de --- /dev/null +++ b/.github/workflows/production-package-inventory.yml @@ -0,0 +1,76 @@ +name: production package inventory + +on: + pull_request: + paths: + - .github/workflows/production-package-inventory.yml + - config/production-packages.json + - scripts/production_package_inventory.py + - docs/production-release-certification.md + push: + branches: [main] + paths: + - config/production-packages.json + - scripts/production_package_inventory.py + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: production-package-inventory-${{ github.ref }} + cancel-in-progress: true + +jobs: + inventory: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + outputs: + count: ${{ steps.inventory.outputs.count }} + matrix: ${{ steps.inventory.outputs.matrix }} + steps: + - name: Check out the exact harness head + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + - name: Validate workflow syntax + uses: docker://rhysd/actionlint@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 + with: + args: .github/workflows/production-package-inventory.yml + - name: Validate inventory logic without network + run: | + set -euo pipefail + python3 -m py_compile scripts/production_package_inventory.py + python3 scripts/production_package_inventory.py --self-test + - name: Resolve exact tag-ready package heads + id: inventory + env: + GH_TOKEN: ${{ github.token }} + run: | + set +e + python3 scripts/production_package_inventory.py \ + --config config/production-packages.json \ + --output evidence/package-inventory + inventory_status=$? + set -e + + if [[ -f evidence/package-inventory/production-package-inventory.md ]]; then + cat evidence/package-inventory/production-package-inventory.md + cat evidence/package-inventory/production-package-inventory.md >>"$GITHUB_STEP_SUMMARY" + fi + if [[ -f evidence/package-inventory/production-package-inventory.json ]]; then + count="$(jq -r .count evidence/package-inventory/production-package-inventory.json)" + printf 'resolved_count=%s\n' "$count" + fi + exit "$inventory_status" + - name: Upload immutable inventory evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: production-package-inventory-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence/package-inventory + if-no-files-found: error + retention-days: 30 + include-hidden-files: false diff --git a/.github/workflows/production-package-roundtrip.yml b/.github/workflows/production-package-roundtrip.yml new file mode 100644 index 00000000..1f7ad602 --- /dev/null +++ b/.github/workflows/production-package-roundtrip.yml @@ -0,0 +1,237 @@ +name: production package publish/install roundtrip + +on: + pull_request: + paths: + - ".github/workflows/production-package-roundtrip.yml" + - "config/production-package-release.json" + - "scripts/production_package_*.py" + - "docs/production-package-roundtrip.md" + push: + branches: [main] + paths: + - ".github/workflows/production-package-roundtrip.yml" + - "config/production-package-release.json" + - "scripts/production_package_*.py" + - "docs/production-package-roundtrip.md" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: production-package-roundtrip-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_API_SHA: 17f2abdcf473dcfee9d24b19f531e26f8513425b + ZED_CLI_SHA: 1bf3d898bc17c4a5d82cf44188b78ffd44c507e4 + ZED_PKG_REGISTRY: http://127.0.0.1:8080 + ZED_PKG_ALLOW_NO_MANIFEST: "1" + ZED_PKG_INSTALL_MODE: copy + +jobs: + real-repository-roundtrip: + name: 19 exact repositories / Rust API / Postgres + runs-on: ubuntu-24.04 + timeout-minutes: 120 + services: + postgres: + image: pgvector/pgvector:0.8.2-pg17-bookworm + env: + POSTGRES_USER: zed + POSTGRES_PASSWORD: zed + POSTGRES_DB: zed + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U zed -d zed" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + + steps: + - name: Check out the exact certification controller + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + fetch-depth: 1 + + - name: Check out the exact registry API + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: zed-pkg/zed-api-server.rs + ref: ${{ env.ZED_API_SHA }} + path: zed-api-server.rs + persist-credentials: false + fetch-depth: 1 + + - name: Check out the exact publishing CLI + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_SHA }} + path: zed-cli + persist-credentials: false + fetch-depth: 1 + + - name: Verify immutable controller pins and dependency closure + shell: bash + run: | + set -euo pipefail + test "$(git -C zed-api-server.rs rev-parse HEAD)" = "$ZED_API_SHA" + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_SHA" + python3 -m py_compile scripts/production_package_*.py + python3 - <<'PY' + import importlib.util + import pathlib + import sys + + path = pathlib.Path("scripts/production_package_roundtrip.py") + sys.path.insert(0, str(path.parent.resolve())) + spec = importlib.util.spec_from_file_location("roundtrip", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + assert spec.loader is not None + spec.loader.exec_module(module) + raw, packages = module.load_ledger(pathlib.Path("config/production-package-release.json")) + assert raw["registry_api_sha"] == "17f2abdcf473dcfee9d24b19f531e26f8513425b" + assert raw["zed_cli_sha"] == "1bf3d898bc17c4a5d82cf44188b78ffd44c507e4" + assert len(packages) == 19 + assert [ + package.package + for package in packages + if package.allow_repo_name_mismatch + ] == ["zed-pkg/zed-api-server", "opto-sync/syncer-rs"] + PY + git diff --check + + - name: Require the stored cross-organization source credential + shell: bash + env: + ZED_PKG_GITHUB_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + if [[ -z "$ZED_PKG_GITHUB_TOKEN" ]]; then + echo "::error::Configure the FLEET_GH_TOKEN Actions secret for private cross-organization source reads" + exit 1 + fi + echo "::add-mask::$ZED_PKG_GITHUB_TOKEN" + + - name: Install the pinned Rust toolchain + uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + + - name: Cache the two Rust builds + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: | + zed-api-server.rs + zed-cli + + - name: Build the exact registry API and Zed CLI + shell: bash + run: | + set -euo pipefail + cargo build --release --locked --manifest-path zed-api-server.rs/Cargo.toml + cargo build --release --locked --manifest-path zed-cli/Cargo.toml --bin zed + test -x zed-api-server.rs/target/release/zed-api-server + test -x zed-cli/target/release/zed + + - name: Start the real Rust registry on Postgres and local artifact storage + shell: bash + env: + DATABASE_URL: postgres://zed:zed@127.0.0.1:5432/zed + AUTO_MIGRATE: "true" + BIND_ADDR: 127.0.0.1:8080 + PUBLIC_BASE_URL: http://127.0.0.1:8080 + STORAGE_BACKEND: local + STORAGE_LOCAL_DIR: ${{ runner.temp }}/zed-artifacts + ZED_VERIFY_TAGS: "off" + ZED_RATE_LIMIT_DISABLED: "1" + DB_CONNECT_MAX_WAIT_SECS: "30" + RUST_LOG: info + run: | + set -euo pipefail + mkdir -p "$STORAGE_LOCAL_DIR" "$RUNNER_TEMP/production-package-evidence" + zed-api-server.rs/target/release/zed-api-server \ + >"$RUNNER_TEMP/zed-api.log" 2>&1 & + echo $! >"$RUNNER_TEMP/zed-api.pid" + for attempt in $(seq 1 90); do + if curl -fsS "$ZED_PKG_REGISTRY/healthz" >/dev/null; then + exit 0 + fi + if ! kill -0 "$(cat "$RUNNER_TEMP/zed-api.pid")" 2>/dev/null; then + cat "$RUNNER_TEMP/zed-api.log" >&2 + exit 1 + fi + sleep 1 + done + cat "$RUNNER_TEMP/zed-api.log" >&2 + exit 1 + + - name: Mint a short-lived registry administrator token + shell: bash + env: + DATABASE_URL: postgres://zed:zed@127.0.0.1:5432/zed + PUBLIC_BASE_URL: http://127.0.0.1:8080 + STORAGE_BACKEND: local + STORAGE_LOCAL_DIR: ${{ runner.temp }}/zed-artifacts + ZED_VERIFY_TAGS: "off" + run: | + set -euo pipefail + output=$(zed-api-server.rs/target/release/zed-api-server create-token \ + --name production-package-roundtrip \ + --expires-in-days 1) + token=$(printf '%s\n' "$output" | tail -n 1) + case "$token" in + zpkg_*) ;; + *) printf '%s\n' "$output" >&2; exit 1 ;; + esac + echo "::add-mask::$token" + printf 'ZED_PKG_TOKEN=%s\n' "$token" >> "$GITHUB_ENV" + + - name: Publish, retry, install, uninstall, and frozen-reinstall all 19 packages + shell: bash + env: + ZED_PKG_GITHUB_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + python3 scripts/production_package_roundtrip.py \ + --ledger config/production-package-release.json \ + --workspace "$RUNNER_TEMP/production-package-work" \ + --evidence "$RUNNER_TEMP/production-package-evidence" \ + --zed "$GITHUB_WORKSPACE/zed-cli/target/release/zed" \ + --registry "$ZED_PKG_REGISTRY" + + - name: Preserve registry diagnostics + if: always() + shell: bash + run: | + set +e + evidence="$RUNNER_TEMP/production-package-evidence" + mkdir -p "$evidence" + cp "$RUNNER_TEMP/zed-api.log" "$evidence/zed-api.log" 2>/dev/null + find "$RUNNER_TEMP/zed-artifacts" -type f -printf '%P\t%s\n' \ + 2>/dev/null | sort > "$evidence/artifact-files.tsv" + curl -fsS "$ZED_PKG_REGISTRY/healthz" \ + > "$evidence/final-health.txt" 2>&1 + + - name: Upload bounded release evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: production-package-roundtrip-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/production-package-evidence + if-no-files-found: error + retention-days: 30 + include-hidden-files: false + + - name: Stop the registry + if: always() + shell: bash + run: | + if test -f "$RUNNER_TEMP/zed-api.pid"; then + kill "$(cat "$RUNNER_TEMP/zed-api.pid")" 2>/dev/null || true + fi diff --git a/.github/workflows/provision-declmig-e2e-once.yml b/.github/workflows/provision-declmig-e2e-once.yml new file mode 100644 index 00000000..0f1e8514 --- /dev/null +++ b/.github/workflows/provision-declmig-e2e-once.yml @@ -0,0 +1,250 @@ +name: provision Declarative Migrations aggregate E2E once + +on: + push: + branches: + - agent/provision-declmig-e2e-once + paths: + - bootstrap/provision-declmig-e2e.once + +permissions: + contents: read + +concurrency: + group: provision-declmig-e2e-once + cancel-in-progress: false + +jobs: + provision: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + BLUEPRINT_REPOSITORY: declarative-migrations/.github + BLUEPRINT_COMMIT: 6ed862e42a2c8ad170fb6840ea2a7b2fd7fba594 + BOOTSTRAP_BRANCH: agent/initial-declmig-e2e-v1 + steps: + - name: Check out the scoped authorization marker + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate the one-time scope and stored credential + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/provision-declmig-e2e.once)" = "PROVISION_DECLMIG_E2E" + test -n "$GH_TOKEN" + test "$(gh api user --jq .login)" = "ORESoftware" + gh api orgs/declarative-migrations --silent + gh api orgs/declarative-migrations-test --silent + + - name: Fetch the exact reviewed repository blueprint + shell: bash + run: | + set -euo pipefail + git clone --quiet --filter=blob:none --no-checkout \ + "https://github.com/${BLUEPRINT_REPOSITORY}.git" blueprint + git -C blueprint fetch --quiet --depth=1 origin "$BLUEPRINT_COMMIT" + git -C blueprint checkout --quiet --detach "$BLUEPRINT_COMMIT" + test "$(git -C blueprint rev-parse HEAD)" = "$BLUEPRINT_COMMIT" + test -f blueprint/blueprints/declmig-e2e/common/.github/workflows/e2e.yml + test -f blueprint/blueprints/declmig-e2e/production/config/repository.json + test -f blueprint/blueprints/declmig-e2e/test/config/repository.json + + - name: Configure authenticated Git transport + shell: bash + run: | + set -euo pipefail + gh auth setup-git + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Create or adopt both aggregate repositories and open bootstrap PRs + id: bootstrap + shell: bash + run: | + set -euo pipefail + + provision_repository() { + local organization="$1" + local mode="$2" + local description="$3" + local full_name="${organization}/declmig-e2e" + local work="work-${organization}" + local created=false + + case "$full_name" in + declarative-migrations/declmig-e2e|declarative-migrations-test/declmig-e2e) ;; + *) echo "::error::Refusing unexpected repository $full_name"; return 1 ;; + esac + + if ! gh api "repos/$full_name" --silent 2>/dev/null; then + gh api --method POST "orgs/$organization/repos" \ + -f name=declmig-e2e \ + -f description="$description" \ + -F private=false \ + -F auto_init=true \ + -F has_issues=true \ + -F has_projects=true \ + -F has_wiki=false \ + --silent + created=true + fi + + gh api --method PATCH "repos/$full_name" \ + -f description="$description" \ + -F private=false \ + -F has_issues=true \ + -F has_projects=true \ + -F has_wiki=false \ + -F delete_branch_on_merge=true \ + --silent + + gh api --method PUT "repos/$full_name/actions/permissions/workflow" \ + --input - --silent <<'JSON' + { + "default_workflow_permissions": "read", + "can_approve_pull_request_reviews": false + } + JSON + + gh api --method PUT "repos/$full_name/topics" \ + --input - --silent <<'JSON' + { + "names": ["declarative-migrations", "database-migrations", "end-to-end-testing", "postgresql", "rust"] + } + JSON + + gh api --method PUT "repos/$full_name/vulnerability-alerts" --silent || \ + echo "::warning::Could not enable vulnerability alerts for $full_name" + gh api --method PUT "repos/$full_name/automated-security-fixes" --silent || \ + echo "::warning::Could not enable automated security fixes for $full_name" + + rm -rf "$work" + git clone --quiet "https://github.com/${full_name}.git" "$work" + + if git -C "$work" ls-remote --exit-code --heads origin "$BOOTSTRAP_BRANCH" >/dev/null 2>&1; then + git -C "$work" fetch --quiet origin "$BOOTSTRAP_BRANCH" + git -C "$work" checkout --quiet -b "$BOOTSTRAP_BRANCH" FETCH_HEAD + else + git -C "$work" checkout --quiet -b "$BOOTSTRAP_BRANCH" + fi + + rm -f "$work/README.md" "$work/SECURITY.md" "$work/.gitignore" + rm -f "$work/.github/workflows/e2e.yml" "$work/.github/dependabot.yml" + rm -rf "$work/config" "$work/pins" "$work/fixtures" "$work/scripts" + + cp -a blueprint/blueprints/declmig-e2e/common/. "$work/" + mkdir -p "$work/config" + cp "blueprint/blueprints/declmig-e2e/${mode}/config/repository.json" \ + "$work/config/repository.json" + + ( + cd "$work" + GITHUB_REPOSITORY="$full_name" python3 scripts/validate_repository_config.py + python3 -m py_compile scripts/validate_repository_config.py + bash -n scripts/run_postgres_smoke.sh + git add -A + if ! git diff --cached --quiet; then + git commit --quiet -m "feat: bootstrap aggregate Declarative Migrations E2E" + git push --quiet --set-upstream origin "$BOOTSTRAP_BRANCH" + fi + ) + + local body_file + body_file="$(mktemp)" + cat > "$body_file" < /tmp/branch-protection.json <<'JSON' + { + "required_status_checks": { + "strict": true, + "contexts": ["contract", "postgres-smoke"] + }, + "enforce_admins": false, + "required_pull_request_reviews": { + "dismiss_stale_reviews": true, + "require_code_owner_reviews": false, + "required_approving_review_count": 1 + }, + "restrictions": null, + "allow_force_pushes": false, + "allow_deletions": false, + "required_conversation_resolution": true + } + JSON + gh api --method PUT "repos/$full_name/branches/main/protection" \ + --input /tmp/branch-protection.json --silent || \ + echo "::warning::Could not apply branch protection for $full_name" + + printf '%s\t%s\t%s\n' "$full_name" "$created" "$pr_url" >> /tmp/declmig-e2e-results.tsv + } + + : > /tmp/declmig-e2e-results.tsv + provision_repository \ + declarative-migrations-test \ + test \ + "Aggregate candidate, destructive, and cross-repository E2E certification for Declarative Migrations." + provision_repository \ + declarative-migrations \ + production \ + "Stable Declarative Migrations promotion orchestration using immutable test-organization evidence." + + while IFS=$'\t' read -r repository created pr_url; do + { + echo "### $repository" + echo + echo "Created now: \`$created\`" + echo + echo "Bootstrap PR: $pr_url" + echo + } >> "$GITHUB_STEP_SUMMARY" + done < /tmp/declmig-e2e-results.tsv + + - name: Verify both repositories and bootstrap pull requests exist + shell: bash + run: | + set -euo pipefail + for full_name in \ + declarative-migrations-test/declmig-e2e \ + declarative-migrations/declmig-e2e + do + gh api "repos/$full_name" --silent + test "$(gh api "repos/$full_name" --jq .visibility)" = "public" + test "$(gh pr list --repo "$full_name" --head "$BOOTSTRAP_BRANCH" --state open --json isDraft --jq '.[0].isDraft')" = "true" + done diff --git a/.github/workflows/publication-immutability.yml b/.github/workflows/publication-immutability.yml new file mode 100644 index 00000000..44a75ae9 --- /dev/null +++ b/.github/workflows/publication-immutability.yml @@ -0,0 +1,196 @@ +name: zed immutable publication canary + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: 8929a58c6591c7a964d7f91412665d7c8a4afdf3 + +permissions: + contents: read + +concurrency: + group: zed-publication-immutability-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '8929a58c6591c7a964d7f91412665d7c8a4afdf3' }} + ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 + +jobs: + build-zed: + name: Build immutable zed CLI + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-interfaces sibling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Build release binary once + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + install -Dm755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + (cd "$RUNNER_TEMP/zed-bin" && sha256sum zed | tee SHA256SUMS) + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Upload immutable zed binary + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: zed-immutability-cli-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-bin + if-no-files-found: error + retention-days: 7 + include-hidden-files: false + + publication-immutability: + name: Immutable publish / ${{ matrix.repo }} + needs: build-zed + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - repo: node-lib + fixture_ref: 222cdf57f48530fce8e6c1f58632d9676203512e + - repo: rust-lib + fixture_ref: e2382ee42b253ec8741b1aa5d0ebb165bb888257 + steps: + - name: Check out canary harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Require immutable fixture pin + env: + FIXTURE_REF: ${{ matrix.fixture_ref }} + run: | + set -euo pipefail + [[ "$FIXTURE_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out fixture + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg-test/${{ matrix.repo }} + ref: ${{ matrix.fixture_ref }} + path: fixture + fetch-depth: 1 + submodules: recursive + persist-credentials: false + show-progress: false + + - name: Download immutable zed binary + uses: actions/download-artifact@974686ed5098c7f9c9289ec946b9058e496a2561 # v8.0.0 + with: + name: zed-immutability-cli-${{ github.run_id }}-${{ github.run_attempt }} + path: tooling + + - name: Verify harness and binary + env: + WORK_ROOT: ${{ runner.temp }}/zed-publication-immutability/${{ matrix.repo }} + run: | + set -euo pipefail + python3 -m py_compile harness/scripts/publication_immutability.py + chmod +x tooling/zed + (cd tooling && sha256sum --check SHA256SUMS) + test ! -e "$WORK_ROOT" + tooling/zed --version + + - name: Certify immutable publication behavior + env: + WORK_ROOT: ${{ runner.temp }}/zed-publication-immutability/${{ matrix.repo }} + run: | + set -euo pipefail + python3 harness/scripts/publication_immutability.py \ + --fixture-dir fixture \ + --zed tooling/zed \ + --work-root "$WORK_ROOT" + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-publication-immutability/${{ matrix.repo }} + run: | + set +e + mkdir -p "$WORK_ROOT/logs" + git -C fixture status --porcelain=v1 --untracked-files=all \ + > "$WORK_ROOT/logs/fixture-status.txt" 2>&1 + find "$WORK_ROOT" -maxdepth 9 -printf '%y %P -> %l\n' \ + | LC_ALL=C sort > "$WORK_ROOT/logs/work-tree.txt" 2>&1 + find "$WORK_ROOT/registry" -maxdepth 9 -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/logs/registry-sha256.txt" 2>&1 + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: publication-immutability-${{ matrix.repo }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-publication-immutability/${{ matrix.repo }} + if-no-files-found: warn + retention-days: 14 + include-hidden-files: true + + publication-immutability-gate: + name: Immutable publication canaries passed + if: always() + needs: [build-zed, publication-immutability] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Enforce aggregate result + env: + BUILD_RESULT: ${{ needs.build-zed.result }} + CANARY_RESULT: ${{ needs.publication-immutability.result }} + run: | + set -euo pipefail + printf 'build-zed=%s\npublication-immutability=%s\n' \ + "$BUILD_RESULT" "$CANARY_RESULT" + test "$BUILD_RESULT" = success + test "$CANARY_RESULT" = success diff --git a/.github/workflows/real-k8s-gitops-contract.yml b/.github/workflows/real-k8s-gitops-contract.yml new file mode 100644 index 00000000..796cf73c --- /dev/null +++ b/.github/workflows/real-k8s-gitops-contract.yml @@ -0,0 +1,146 @@ +name: real k8s-cluster GitOps contract + +on: + pull_request: + paths: + - .github/workflows/real-k8s-gitops-contract.yml + - scripts/real_k8s_gitops_contract.py + - docs/real-k8s-gitops-contract.md + workflow_dispatch: + inputs: + product_commit: + description: Exact zed-pkg/zed-cli commit + required: false + default: 1c94813c6b59c7621c1256d7c55246e7f06f44b7 + type: string + cluster_commit: + description: Exact ORESoftware/k8s-cluster commit + required: false + default: cfe39128e309936a4bcaeca4aaa35e4b9bec8888 + type: string + +permissions: + contents: read + +concurrency: + group: real-k8s-gitops-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + ZED_CLI_COMMIT: ${{ inputs.product_commit || '1c94813c6b59c7621c1256d7c55246e7f06f44b7' }} + K8S_CLUSTER_COMMIT: ${{ inputs.cluster_commit || 'cfe39128e309936a4bcaeca4aaa35e4b9bec8888' }} + +jobs: + certify: + runs-on: ubuntu-24.04 + timeout-minutes: 35 + steps: + - name: Check out independent test-org harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact Zed product candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact merged k8s-cluster contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ORESoftware/k8s-cluster + ref: ${{ env.K8S_CLUSTER_COMMIT }} + path: product/k8s-cluster + fetch-depth: 1 + submodules: false + persist-credentials: false + show-progress: false + + - name: Verify immutable checkout identities + run: | + set -euo pipefail + [[ "$ZED_CLI_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ "$K8S_CLUSTER_COMMIT" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + test "$(git -C product/k8s-cluster rev-parse HEAD)" = "$K8S_CLUSTER_COMMIT" + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt, clippy + + - name: Install Python 3.12 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.12' + + - name: Run focused source contracts + run: | + set -euo pipefail + cargo fmt --manifest-path product/zed-cli/Cargo.toml --all --check + cargo test --manifest-path product/zed-cli/Cargo.toml --locked --test external_gitops_dispatch + python product/k8s-cluster/tools/test_gitops_composition.py -v + + - name: Build exact Zed sibling binaries + run: | + set -euo pipefail + cargo build \ + --manifest-path product/zed-cli/Cargo.toml \ + --locked \ + --release \ + --bin zed \ + --bin zed-gitops + + - name: Certify the real merged contract + run: | + set -euo pipefail + mkdir -p artifacts + python -m py_compile scripts/real_k8s_gitops_contract.py + rm -rf scripts/__pycache__ + python scripts/real_k8s_gitops_contract.py \ + --zed "$GITHUB_WORKSPACE/product/zed-cli/target/release/zed" \ + --zed-gitops "$GITHUB_WORKSPACE/product/zed-cli/target/release/zed-gitops" \ + --product product/zed-cli \ + --cluster product/k8s-cluster \ + --product-commit "$ZED_CLI_COMMIT" \ + --cluster-commit "$K8S_CLUSTER_COMMIT" \ + --evidence artifacts/real-k8s-gitops-contract.json + rm -rf scripts/__pycache__ + + - name: Require clean source checkouts + run: | + set -euo pipefail + harness_status="$(git status --porcelain --untracked-files=all | grep -vE '^\?\? (artifacts|product)/' || true)" + if [[ -n "$harness_status" ]]; then + printf 'unexpected harness drift:\n%s\n' "$harness_status" >&2 + exit 1 + fi + for checkout in product/zed-cli product/k8s-cluster; do + checkout_status="$(git -C "$checkout" status --porcelain)" + if [[ -n "$checkout_status" ]]; then + printf 'unexpected drift in %s:\n%s\n' "$checkout" "$checkout_status" >&2 + exit 1 + fi + done + + - name: Upload exact-contract evidence + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: real-k8s-gitops-${{ env.ZED_CLI_COMMIT }}-${{ env.K8S_CLUSTER_COMMIT }} + path: artifacts/real-k8s-gitops-contract.json + if-no-files-found: error + retention-days: 14 + include-hidden-files: false \ No newline at end of file diff --git a/.github/workflows/recover-test-org-fleet-v2.yml b/.github/workflows/recover-test-org-fleet-v2.yml new file mode 100644 index 00000000..b87420ea --- /dev/null +++ b/.github/workflows/recover-test-org-fleet-v2.yml @@ -0,0 +1,105 @@ +# One-time idempotent recovery for the cross-organization test fleet. +name: recover test organization fleet v3 + +on: + push: + branches: + - main + paths: + - bootstrap/execute-test-org-fleet-v2.once + +permissions: + contents: read + +concurrency: + group: apply-test-org-fleet-all + cancel-in-progress: true + +jobs: + authorize: + name: Validate one-time fleet recovery authorization + runs-on: ubuntu-24.04 + timeout-minutes: 2 + outputs: + organizations: ${{ steps.matrix.outputs.organizations }} + steps: + - name: Check out the authorization marker + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + - id: matrix + name: Validate marker and publish the in-scope matrix + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/execute-test-org-fleet-v2.once)" = "APPLY_TEST_FLEET_V3" + organizations='["zed-pkg-test","3fa-app-test","declarative-migrations-test","cliptown-test","claritas-viz-test","embedded-alerts-test","evento-globolo-test","fiducia-cloud-test","memebank-test","opto-sync-test","quaestor-ledger-test","sonus-auris-test","messaging-intel-test","scintilla-run-test","file-tunnel-test","shared-auth-test","hypesiege-test","streempilot-test"]' + printf 'organizations=%s\n' "$organizations" >> "$GITHUB_OUTPUT" + + recover: + name: Recover ${{ matrix.organization }} + needs: authorize + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + organization: ${{ fromJSON(needs.authorize.outputs.organizations) }} + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate the fleet and worktree-safe launcher + run: | + node --check scripts/bootstrap-test-org-fleet.mjs + node --check scripts/bootstrap-test-org-fleet-worktree-safe.mjs + node scripts/validate-test-org-fleet.mjs + node --test tests/test-org-fleet.test.mjs + + - name: Export the persisted fleet PAT without logging it + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + test -n "$STORED_FLEET_PAT" + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Verify organization visibility + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + run: | + set -euo pipefail + gh api "orgs/$FLEET_ORGANIZATION" --silent + + - name: Cool down between serialized organization writes + run: sleep 15 + + - name: Create or adopt repositories, push commits, and open draft PRs + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + TEST_ORG_FLEET_SKIP_TOPICS: "true" + run: | + set -euo pipefail + node scripts/bootstrap-test-org-fleet-worktree-safe.mjs \ + --apply \ + --org "$FLEET_ORGANIZATION" \ + --concurrency 1 + + - name: Record completion + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + run: | + { + echo "## $FLEET_ORGANIZATION" + echo + echo "Worktree-safe idempotent repository creation/adoption, generated commits, and draft pull requests completed." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/recursive-installs-stress.yml b/.github/workflows/recursive-installs-stress.yml new file mode 100644 index 00000000..20a29b84 --- /dev/null +++ b/.github/workflows/recursive-installs-stress.yml @@ -0,0 +1,131 @@ +name: recursive install stress + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to test + required: false + default: 8cc6745c75fd46dabf2e383cbcfc14db5ff933da + +permissions: + contents: read + +concurrency: + group: recursive-install-stress-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '8cc6745c75fd46dabf2e383cbcfc14db5ff933da' }} + ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 + +jobs: + recursive-install-stress: + name: Five-worker and four-process stress + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out e2e harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-interfaces sibling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Prove five live HTTP artifact transfers + env: + ZED_PKG_INSTALL_CONCURRENCY: "5" + run: | + set -euo pipefail + python3 -m py_compile harness/scripts/recursive_installs_stress.py + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml \ + --test recursive_install_concurrency -- --nocapture + + - name: Build immutable zed binary + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + install -Dm755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Run four-process and frozen stress contract + env: + WORK_ROOT: ${{ runner.temp }}/zed-recursive-stress + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/recursive_installs_stress.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-recursive-stress + run: | + set +e + mkdir -p "$WORK_ROOT/diagnostics" + find "$WORK_ROOT" -maxdepth 10 -printf '%y %P -> %l\n' \ + | LC_ALL=C sort > "$WORK_ROOT/diagnostics/work-tree.txt" 2>&1 + find "$WORK_ROOT/registry" -maxdepth 10 -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/registry-sha256.txt" 2>&1 + find "$WORK_ROOT" -path '*/cache/*.tar.gz' -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/all-cache-sha256.txt" 2>&1 + true + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: recursive-install-stress-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-recursive-stress/diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/recursive-installs.yml b/.github/workflows/recursive-installs.yml new file mode 100644 index 00000000..b3abcd74 --- /dev/null +++ b/.github/workflows/recursive-installs.yml @@ -0,0 +1,131 @@ +name: recursive install graph + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to test + required: false + default: 8cc6745c75fd46dabf2e383cbcfc14db5ff933da + +permissions: + contents: read + +concurrency: + group: recursive-install-graph-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || '8cc6745c75fd46dabf2e383cbcfc14db5ff933da' }} + ZED_INTERFACES_REF: dc0e0a0620b9462817950b552d3d334a184b1cb1 + +jobs: + recursive-install-e2e: + name: Recursive graph and shared-home lock + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Require immutable component pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out e2e harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-cli + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out zed-interfaces sibling + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install the declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Format and unit-test recursive installer and process locks + run: | + set -euo pipefail + python3 -m py_compile harness/scripts/recursive_installs.py + cargo fmt --all --check --manifest-path zed-cli/Cargo.toml + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib install_graph::hardening_tests + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib store::tests + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib config::tests + + - name: Build immutable zed binary + run: | + set -euo pipefail + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + install -Dm755 zed-cli/target/release/zed "$RUNNER_TEMP/zed-bin/zed" + "$RUNNER_TEMP/zed-bin/zed" --version + + - name: Run recursive install and lock contract + env: + WORK_ROOT: ${{ runner.temp }}/zed-recursive-install + run: | + set -euo pipefail + test ! -e "$WORK_ROOT" + python3 harness/scripts/recursive_installs.py \ + --zed "$RUNNER_TEMP/zed-bin/zed" \ + --work-root "$WORK_ROOT" + + - name: Collect failure diagnostics + if: failure() + env: + WORK_ROOT: ${{ runner.temp }}/zed-recursive-install + run: | + set +e + mkdir -p "$WORK_ROOT/diagnostics" + find "$WORK_ROOT" -maxdepth 9 -printf '%y %P -> %l\n' \ + | LC_ALL=C sort > "$WORK_ROOT/diagnostics/work-tree.txt" 2>&1 + find "$WORK_ROOT/registry" -maxdepth 9 -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/registry-sha256.txt" 2>&1 + find "$WORK_ROOT" -path '*/cache/*.tar.gz' -type f -print0 2>/dev/null \ + | sort -z | xargs -0 -r sha256sum \ + > "$WORK_ROOT/diagnostics/all-cache-sha256.txt" 2>&1 + true + + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: recursive-install-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zed-recursive-install/diagnostics + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/registry-default-canary.yml b/.github/workflows/registry-default-canary.yml new file mode 100644 index 00000000..2b8e6f1f --- /dev/null +++ b/.github/workflows/registry-default-canary.yml @@ -0,0 +1,151 @@ +name: DEN-2760 public registry default canary + +on: + pull_request: + paths: + - '.github/workflows/registry-default-canary.yml' + push: + branches: [main] + paths: + - '.github/workflows/registry-default-canary.yml' + workflow_dispatch: + inputs: + zed_cli_ref: + description: Immutable zed-cli commit to certify + required: false + default: d0678568fbcff75697512cc36ae84a356548f26c + +permissions: + contents: read + +concurrency: + group: den-2760-registry-default-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: '1' + ZED_CLI_REF: ${{ github.event.inputs.zed_cli_ref || 'd0678568fbcff75697512cc36ae84a356548f26c' }} + ZED_INTERFACES_SHA: d1bf8ef7c88a75292cbe8d697bfd269f30d62e2b + EXPECTED_REGISTRY: https://registry.zpkg.net + FORBIDDEN_REGISTRY: https://registry.zpkg.tech + +jobs: + exact-default: + name: zpkg.net default / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + + steps: + - name: Require immutable product and interface pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_INTERFACES_SHA" =~ ^[0-9a-f]{40}$ ]] + test "$EXPECTED_REGISTRY" = 'https://registry.zpkg.net' + test "$FORBIDDEN_REGISTRY" = 'https://registry.zpkg.tech' + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Verify exact product and dependency candidate + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + grep -F "rev = \"$ZED_INTERFACES_SHA\"" zed-cli/Cargo.toml + grep -F "#$ZED_INTERFACES_SHA" zed-cli/Cargo.lock + grep -F "ZED_INTERFACES_SHA: $ZED_INTERFACES_SHA" \ + zed-cli/.github/workflows/frozen-lock-integrity.yml + grep -F "ZED_INTERFACES_SHA: $ZED_INTERFACES_SHA" \ + zed-cli/.github/workflows/nix-bundle-write.yml + grep -F "ZED_INTERFACES_SHA: $ZED_INTERFACES_SHA" \ + zed-cli/.github/workflows/nix-export-bundle.yml + + - name: Install pinned Rust toolchain action + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + components: rustfmt,clippy + + - name: Prove every public default surface uses zpkg.net + run: | + set -euo pipefail + surfaces=( + .cli-flags.toml + .dev-cli-flags.toml + .fetch-cli-flags.toml + .nix-interop-cli-flags.toml + README.md + ) + for surface in "${surfaces[@]}"; do + grep -F "$EXPECTED_REGISTRY" "zed-cli/$surface" + if grep -F "$FORBIDDEN_REGISTRY" "zed-cli/$surface"; then + echo "forbidden registry remains in $surface" >&2 + exit 1 + fi + done + if git -C zed-cli grep -nF "$FORBIDDEN_REGISTRY" -- .; then + echo 'forbidden public registry remains in tracked product content' >&2 + exit 1 + fi + + - name: Run formatter, registry tests, and strict lint + working-directory: zed-cli + run: | + set -euo pipefail + cargo fmt --all -- --check + cargo test --locked --lib registry::tests -- --nocapture + cargo clippy --locked --lib --bin zed -- -D warnings + cargo build --locked --bin zed + + - name: Prove compiled help advertises the canonical default + working-directory: zed-cli + run: | + set -euo pipefail + binary=target/debug/zed + if [[ "$RUNNER_OS" == 'Windows' ]]; then + binary=target/debug/zed.exe + fi + "$binary" --help | tee "$RUNNER_TEMP/zed-help.txt" + grep -F "$EXPECTED_REGISTRY" "$RUNNER_TEMP/zed-help.txt" + if grep -F "$FORBIDDEN_REGISTRY" "$RUNNER_TEMP/zed-help.txt"; then + echo 'compiled help advertises the forbidden registry' >&2 + exit 1 + fi + + - name: Record bounded evidence + if: always() + run: | + set +e + evidence="$RUNNER_TEMP/den-2760-registry-default" + mkdir -p "$evidence" + printf '%s\n' "$ZED_CLI_REF" > "$evidence/zed-cli-commit.txt" + printf '%s\n' "$ZED_INTERFACES_SHA" > "$evidence/zed-interfaces-commit.txt" + printf '%s\n' "$EXPECTED_REGISTRY" > "$evidence/expected-registry.txt" + cp "$RUNNER_TEMP/zed-help.txt" "$evidence/zed-help.txt" 2>/dev/null + git -C zed-cli status --porcelain=v1 --untracked-files=all \ + > "$evidence/product-status.txt" 2>&1 + + - name: Upload bounded evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: den-2760-registry-default-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/den-2760-registry-default + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/resolve-shared-auth-pr24-v2.yml b/.github/workflows/resolve-shared-auth-pr24-v2.yml new file mode 100644 index 00000000..e8cfcee6 --- /dev/null +++ b/.github/workflows/resolve-shared-auth-pr24-v2.yml @@ -0,0 +1,178 @@ +name: resolve shared-auth PR 24 semantically v2 + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/RESOLVE-SHARED-AUTH-24-V2" + +permissions: + contents: read + +concurrency: + group: resolve-shared-auth-pr-24 + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + REPOSITORY: shared-auth/shared-auth-server.rs + HEAD_BRANCH: agent/harden-response-headers + EXPECTED_HEAD: a8097ef23fbab7fd8f3487a22319600e466b8d8b + EXPECTED_MAIN: 3547e16770e821fbc0e1e08f20820ac89fe1f8ce + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + gh auth setup-git + + - name: Merge histories and synthesize the conflicted router + shell: bash + run: | + set -Eeuo pipefail + workdir="$(mktemp -d)" + git clone --no-tags "https://github.com/${REPOSITORY}.git" "$workdir/repo" + cd "$workdir/repo" + git fetch origin \ + "+refs/heads/main:refs/remotes/origin/main" \ + "+refs/heads/${HEAD_BRANCH}:refs/remotes/origin/${HEAD_BRANCH}" + + actual_main="$(git rev-parse origin/main)" + actual_head="$(git rev-parse "origin/${HEAD_BRANCH}")" + test "$actual_main" = "$EXPECTED_MAIN" || { + echo "::error::main moved: expected $EXPECTED_MAIN, found $actual_main" + exit 1 + } + test "$actual_head" = "$EXPECTED_HEAD" || { + echo "::error::PR head moved: expected $EXPECTED_HEAD, found $actual_head" + exit 1 + } + + git checkout -B "$HEAD_BRANCH" "origin/$HEAD_BRANCH" + git config user.name "Alexander Mills" + git config user.email "alex@oresoftware.com" + + set +e + git merge --no-commit --no-ff origin/main + merge_status=$? + set -e + + if ((merge_status != 0)); then + mapfile -t conflicts < <(git diff --name-only --diff-filter=U | sort) + ((${#conflicts[@]} > 0)) + for path in "${conflicts[@]}"; do + [[ "$path" == "src/http/mod.rs" ]] || { + echo "::error::unexpected conflicted path: $path" + exit 1 + } + done + fi + + # Git reconciled the integration tests automatically. Prove that the + # result contains both current-main security behavior and every PR 24 + # regression before accepting it. + grep -q 'app_with_introspect_secret' tests/integration.rs + grep -q 'async fn ui_exchange_empty_token_returns_error_fragment' tests/integration.rs + grep -q 'async fn security_headers_present_on_responses' tests/integration.rs + grep -q 'async fn jwks_keeps_public_cache_not_no_store' tests/integration.rs + grep -q 'async fn webhook_fails_closed_without_secret' tests/integration.rs + + # Rebuild the sole conflicted file from current main, preserving its + # passkey/MFA routes, DELETE support, visibility changes, and all other + # intervening work; then add the independent response policy. + git show origin/main:src/http/mod.rs > src/http/mod.rs + node <<'NODE' + const fs = require('node:fs'); + const path = 'src/http/mod.rs'; + let source = fs.readFileSync(path, 'utf8'); + + if (source.includes('HeaderName::from_static("permissions-policy")')) { + throw new Error('response hardening already exists on main'); + } + + const oldImport = ' http::{header, HeaderValue, Method},'; + const newImport = ' http::{header, HeaderName, HeaderValue, Method},'; + if (!source.includes(oldImport)) throw new Error('expected header import not found'); + source = source.replace(oldImport, newImport); + + const boundary = ` .layer(SetResponseHeaderLayer::if_not_present( + header::STRICT_TRANSPORT_SECURITY, + HeaderValue::from_static("max-age=31536000; includeSubDomains"), + )) + .layer(cors)`; + const resolved = ` .layer(SetResponseHeaderLayer::if_not_present( + header::STRICT_TRANSPORT_SECURITY, + HeaderValue::from_static("max-age=31536000; includeSubDomains"), + )) + // Token-bearing responses must not enter shared or browser caches. + // if_not_present preserves the JWKS handler's public cache policy. + .layer(SetResponseHeaderLayer::if_not_present( + header::CACHE_CONTROL, + HeaderValue::from_static("no-store"), + )) + // The script-free UI needs none of these powerful browser features. + .layer(SetResponseHeaderLayer::if_not_present( + HeaderName::from_static("permissions-policy"), + HeaderValue::from_static( + "accelerometer=(), camera=(), display-capture=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=()", + ), + )) + // Prevent cross-origin opener relationships around authentication UI. + .layer(SetResponseHeaderLayer::if_not_present( + HeaderName::from_static("cross-origin-opener-policy"), + HeaderValue::from_static("same-origin"), + )) + .layer(cors)`; + + if (!source.includes(boundary)) throw new Error('HSTS/CORS boundary not found'); + source = source.replace(boundary, resolved); + fs.writeFileSync(path, source); + NODE + + git add src/http/mod.rs tests/integration.rs + test -z "$(git diff --name-only --diff-filter=U)" + git diff --cached --check + + git commit -m "merge: reconcile response hardening with current auth surface" \ + -m "Preserve current main's passkey, factor, DELETE-CORS, and fail-closed introspection work while integrating PR 24's no-store, Permissions-Policy, COOP, JWKS cache exception, UI failure, and webhook regression coverage." + + git merge-base --is-ancestor "$EXPECTED_HEAD" HEAD + git merge-base --is-ancestor "$EXPECTED_MAIN" HEAD + test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3 + + cargo fmt --all -- --check + cargo test --locked --test integration + cargo test --locked + + git diff --check origin/main...HEAD + mapfile -t final_changed < <(git diff --name-only origin/main...HEAD | sort) + expected_changed=(src/http/mod.rs tests/integration.rs) + [[ "${final_changed[*]}" == "${expected_changed[*]}" ]] || { + printf '::error::unexpected final diff paths: %s\n' "${final_changed[*]}" + exit 1 + } + + new_head="$(git rev-parse HEAD)" + git push origin "HEAD:refs/heads/${HEAD_BRANCH}" + remote_head="$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" + test "$remote_head" = "$new_head" + echo "SEMANTIC_HEAD=$new_head" | tee -a "$GITHUB_ENV" + + - name: Record result + shell: bash + run: | + { + echo "## shared-auth PR 24 semantic resolution" + echo + echo "- Previous PR head: \`$EXPECTED_HEAD\`" + echo "- Current main integrated: \`$EXPECTED_MAIN\`" + echo "- Resolved merge head: \`$SEMANTIC_HEAD\`" + echo "- Exact two-parent semantic merge; no ours/theirs shortcut" + echo "- Validation: cargo fmt, integration tests, full locked test suite" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/resolve-shared-auth-pr24-v3.yml b/.github/workflows/resolve-shared-auth-pr24-v3.yml new file mode 100644 index 00000000..3b650766 --- /dev/null +++ b/.github/workflows/resolve-shared-auth-pr24-v3.yml @@ -0,0 +1,159 @@ +name: resolve shared-auth PR 24 semantically v3 + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/RESOLVE-SHARED-AUTH-24-V3" + +permissions: + contents: read + +concurrency: + group: resolve-shared-auth-pr-24 + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + REPOSITORY: shared-auth/shared-auth-server.rs + HEAD_BRANCH: agent/harden-response-headers + EXPECTED_HEAD: a8097ef23fbab7fd8f3487a22319600e466b8d8b + EXPECTED_MAIN: 3547e16770e821fbc0e1e08f20820ac89fe1f8ce + steps: + - name: Validate credential and reconstruct exact histories + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + gh auth setup-git + + workdir="$(mktemp -d)" + echo "WORKDIR=$workdir" >> "$GITHUB_ENV" + git clone --no-tags "https://github.com/${REPOSITORY}.git" "$workdir/repo" + cd "$workdir/repo" + git fetch origin \ + "+refs/heads/main:refs/remotes/origin/main" \ + "+refs/heads/${HEAD_BRANCH}:refs/remotes/origin/${HEAD_BRANCH}" + + test "$(git rev-parse origin/main)" = "$EXPECTED_MAIN" + test "$(git rev-parse "origin/${HEAD_BRANCH}")" = "$EXPECTED_HEAD" + git checkout -B "$HEAD_BRANCH" "origin/$HEAD_BRANCH" + git config user.name "Alexander Mills" + git config user.email "alex@oresoftware.com" + + - name: Perform semantic two-branch reconciliation + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + + set +e + git merge --no-commit --no-ff origin/main + merge_status=$? + set -e + if ((merge_status != 0)); then + mapfile -t conflicts < <(git diff --name-only --diff-filter=U | sort) + [[ "${conflicts[*]}" == "src/http/mod.rs" ]] || { + printf '::error::unexpected conflicts: %s\n' "${conflicts[*]}" + exit 1 + } + fi + + # The integration suite merged cleanly. Require the important behavior + # from both histories before resolving the sole router conflict. + grep -q 'app_with_introspect_secret' tests/integration.rs + grep -q 'async fn ui_exchange_empty_token_returns_error_fragment' tests/integration.rs + grep -q 'async fn security_headers_present_on_responses' tests/integration.rs + grep -q 'async fn jwks_keeps_public_cache_not_no_store' tests/integration.rs + grep -q 'async fn webhook_fails_closed_without_secret' tests/integration.rs + + git show origin/main:src/http/mod.rs > src/http/mod.rs + node <<'NODE' + const fs = require('node:fs'); + const path = 'src/http/mod.rs'; + let source = fs.readFileSync(path, 'utf8'); + + const oldImport = ' http::{header, HeaderValue, Method},'; + const newImport = ' http::{header, HeaderName, HeaderValue, Method},'; + if (!source.includes(oldImport)) throw new Error('expected axum header import not found'); + source = source.replace(oldImport, newImport); + + const marker = ' .layer(cors)\n .with_state(state)'; + if (source.split(marker).length !== 2) throw new Error('expected one CORS/state boundary'); + const layers = [ + ' // Token-bearing responses must not enter shared or browser caches.', + ' // if_not_present preserves the explicit public JWKS cache policy.', + ' .layer(SetResponseHeaderLayer::if_not_present(', + ' header::CACHE_CONTROL,', + ' HeaderValue::from_static("no-store"),', + ' ))', + ' // The script-free UI needs no powerful browser capabilities.', + ' .layer(SetResponseHeaderLayer::if_not_present(', + ' HeaderName::from_static("permissions-policy"),', + ' HeaderValue::from_static(', + ' "accelerometer=(), camera=(), display-capture=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=()",', + ' ),', + ' ))', + ' // Isolate authentication UI from cross-origin opener relationships.', + ' .layer(SetResponseHeaderLayer::if_not_present(', + ' HeaderName::from_static("cross-origin-opener-policy"),', + ' HeaderValue::from_static("same-origin"),', + ' ))', + ].join('\n'); + source = source.replace(marker, layers + '\n' + marker); + fs.writeFileSync(path, source); + NODE + + git add src/http/mod.rs tests/integration.rs + test -z "$(git diff --name-only --diff-filter=U)" + git diff --cached --check + git commit -m "merge: reconcile response hardening with current auth surface" \ + -m "Preserve passkey, factor, DELETE-CORS, visibility, and fail-closed introspection changes from current main while integrating PR 24 response headers and router regressions." + + git merge-base --is-ancestor "$EXPECTED_HEAD" HEAD + git merge-base --is-ancestor "$EXPECTED_MAIN" HEAD + test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3 + + - name: Validate exact resolved head + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + cargo fmt --all -- --check + cargo test --locked --test integration + cargo test --locked + git diff --check origin/main...HEAD + mapfile -t final_changed < <(git diff --name-only origin/main...HEAD | sort) + expected=(src/http/mod.rs tests/integration.rs) + [[ "${final_changed[*]}" == "${expected[*]}" ]] || { + printf '::error::unexpected final diff paths: %s\n' "${final_changed[*]}" + exit 1 + } + + - name: Push exact semantic head + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + new_head="$(git rev-parse HEAD)" + test "$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" = "$EXPECTED_HEAD" + git push origin "HEAD:refs/heads/${HEAD_BRANCH}" + test "$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" = "$new_head" + echo "SEMANTIC_HEAD=$new_head" >> "$GITHUB_ENV" + + - name: Record result + shell: bash + run: | + { + echo "## shared-auth PR 24 semantic resolution" + echo + echo "- Old head: \`$EXPECTED_HEAD\`" + echo "- Integrated main: \`$EXPECTED_MAIN\`" + echo "- Resolved head: \`$SEMANTIC_HEAD\`" + echo "- Validation: cargo fmt, integration suite, full locked suite" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/resolve-shared-auth-pr24.yml b/.github/workflows/resolve-shared-auth-pr24.yml new file mode 100644 index 00000000..73084efe --- /dev/null +++ b/.github/workflows/resolve-shared-auth-pr24.yml @@ -0,0 +1,276 @@ +name: resolve shared-auth PR 24 semantically + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/RESOLVE-SHARED-AUTH-24" + +permissions: + contents: read + +concurrency: + group: resolve-shared-auth-pr-24 + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + REPOSITORY: shared-auth/shared-auth-server.rs + HEAD_BRANCH: agent/harden-response-headers + EXPECTED_HEAD: a8097ef23fbab7fd8f3487a22319600e466b8d8b + EXPECTED_MAIN: 3547e16770e821fbc0e1e08f20820ac89fe1f8ce + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + gh auth setup-git + + - name: Reconstruct both branches and resolve by intent + shell: bash + run: | + set -Eeuo pipefail + workdir="$(mktemp -d)" + git clone --no-tags "https://github.com/${REPOSITORY}.git" "$workdir/repo" + cd "$workdir/repo" + git fetch origin \ + "+refs/heads/main:refs/remotes/origin/main" \ + "+refs/heads/${HEAD_BRANCH}:refs/remotes/origin/${HEAD_BRANCH}" + + actual_main="$(git rev-parse origin/main)" + actual_head="$(git rev-parse "origin/${HEAD_BRANCH}")" + test "$actual_main" = "$EXPECTED_MAIN" || { + echo "::error::main moved: expected $EXPECTED_MAIN, found $actual_main" + exit 1 + } + test "$actual_head" = "$EXPECTED_HEAD" || { + echo "::error::PR head moved: expected $EXPECTED_HEAD, found $actual_head" + exit 1 + } + + git checkout -B "$HEAD_BRANCH" "origin/$HEAD_BRANCH" + git config user.name "Alexander Mills" + git config user.email "alex@oresoftware.com" + + set +e + git merge --no-commit --no-ff origin/main + merge_status=$? + set -e + + if ((merge_status != 0)); then + mapfile -t conflicts < <(git diff --name-only --diff-filter=U | sort) + expected=(src/http/mod.rs tests/integration.rs) + if [[ "${conflicts[*]}" != "${expected[*]}" ]]; then + printf '::error::unexpected conflicts: %s\n' "${conflicts[*]}" + exit 1 + fi + fi + + # Current main is authoritative for all features landed since the old + # merge base (passkeys, MFA factors, DELETE CORS, fail-closed + # introspection, and their tests). Reapply the independent PR intent + # on top of that current contract. + git show origin/main:src/http/mod.rs > src/http/mod.rs + git show origin/main:tests/integration.rs > tests/integration.rs + + node <<'NODE' + const fs = require('node:fs'); + + const httpPath = 'src/http/mod.rs'; + let http = fs.readFileSync(httpPath, 'utf8'); + if (http.includes('HeaderName::from_static("permissions-policy")')) { + throw new Error('main already contains the response-header hardening; refusing duplicate insertion'); + } + + const oldImport = ' http::{header, HeaderValue, Method},'; + const newImport = ' http::{header, HeaderName, HeaderValue, Method},'; + if (!http.includes(oldImport)) throw new Error('expected axum header import was not found'); + http = http.replace(oldImport, newImport); + + const layerAnchor = ` .layer(SetResponseHeaderLayer::if_not_present( + header::STRICT_TRANSPORT_SECURITY, + HeaderValue::from_static("max-age=31536000; includeSubDomains"), + )) + .layer(cors)`; + const hardening = ` .layer(SetResponseHeaderLayer::if_not_present( + header::STRICT_TRANSPORT_SECURITY, + HeaderValue::from_static("max-age=31536000; includeSubDomains"), + )) + // Token-bearing bodies must never enter shared or browser caches. + // if_not_present preserves the JWKS handler's explicit public cache. + .layer(SetResponseHeaderLayer::if_not_present( + header::CACHE_CONTROL, + HeaderValue::from_static("no-store"), + )) + // The script-free UI needs no powerful browser capabilities. + .layer(SetResponseHeaderLayer::if_not_present( + HeaderName::from_static("permissions-policy"), + HeaderValue::from_static( + "accelerometer=(), camera=(), display-capture=(), geolocation=(), gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=()", + ), + )) + // Isolate auth pages from cross-origin opener relationships. + .layer(SetResponseHeaderLayer::if_not_present( + HeaderName::from_static("cross-origin-opener-policy"), + HeaderValue::from_static("same-origin"), + )) + .layer(cors)`; + if (!http.includes(layerAnchor)) throw new Error('expected HSTS/CORS layer boundary was not found'); + http = http.replace(layerAnchor, hardening); + fs.writeFileSync(httpPath, http); + + const testsPath = 'tests/integration.rs'; + let tests = fs.readFileSync(testsPath, 'utf8'); + const nextTest = '#[tokio::test]\nasync fn metrics_records_exchanges() {'; + if (!tests.includes(nextTest)) throw new Error('metrics test insertion boundary was not found'); + if (tests.includes('async fn security_headers_present_on_responses()')) { + throw new Error('main already contains the PR 24 integration tests; refusing duplicate insertion'); + } + + const additions = `#[tokio::test] + async fn ui_exchange_empty_token_returns_error_fragment() { + let resp = app() + .await + .oneshot( + Request::post("/ui/exchange") + .header("content-type", "application/x-www-form-urlencoded") + .body(Body::from("access_token=")) + .unwrap(), + ) + .await + .unwrap(); + let frag = body_string(resp).await; + assert!( + frag.contains("unauthorized"), + "expected error fragment, got: {frag}" + ); + assert!(!frag.contains("exchanged")); + assert!(!frag.contains("access_token")); + } + + #[tokio::test] + async fn security_headers_present_on_responses() { + let resp = app() + .await + .oneshot(Request::get("/").body(Body::empty()).unwrap()) + .await + .unwrap(); + let headers = resp.headers(); + let get = |name: &str| { + headers + .get(name) + .unwrap_or_else(|| panic!("missing security header: {name}")) + .to_str() + .unwrap() + .to_string() + }; + + assert_eq!(get("x-content-type-options"), "nosniff"); + assert_eq!(get("x-frame-options"), "DENY"); + assert_eq!(get("referrer-policy"), "no-referrer"); + assert!(get("content-security-policy").contains("default-src 'none'")); + assert!(get("strict-transport-security").contains("max-age=")); + assert_eq!(get("cache-control"), "no-store"); + assert!(get("permissions-policy").contains("camera=()")); + assert_eq!(get("cross-origin-opener-policy"), "same-origin"); + } + + #[tokio::test] + async fn jwks_keeps_public_cache_not_no_store() { + let resp = app() + .await + .oneshot( + Request::get("/.well-known/jwks.json") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let cache_control = resp + .headers() + .get("cache-control") + .expect("JWKS response must define cache policy") + .to_str() + .unwrap(); + assert!( + cache_control.contains("public"), + "JWKS must stay publicly cacheable, got: {cache_control}" + ); + assert!(cache_control.contains("max-age=300")); + assert!(!cache_control.contains("no-store")); + } + + #[tokio::test] + async fn webhook_fails_closed_without_secret() { + let resp = app() + .await + .oneshot( + Request::post("/internal/webhook/sync") + .header("content-type", "application/json") + .body(Body::from("{}")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); + } + + `; + tests = tests.replace(nextTest, additions + nextTest); + fs.writeFileSync(testsPath, tests); + NODE + + git add src/http/mod.rs tests/integration.rs + test -z "$(git diff --name-only --diff-filter=U)" + git diff --cached --check + + mapfile -t changed < <(git diff --cached --name-only | sort) + expected_changed=(src/http/mod.rs tests/integration.rs) + [[ "${changed[*]}" == "${expected_changed[*]}" ]] || { + printf '::error::unexpected staged paths: %s\n' "${changed[*]}" + exit 1 + } + + git commit -m "merge: reconcile response hardening with current auth surface" \ + -m "Preserve current main's passkey, factor, DELETE-CORS, and fail-closed introspection work while integrating PR 24's no-store, Permissions-Policy, COOP, JWKS cache exception, UI failure, and webhook regression coverage." + + git merge-base --is-ancestor "$EXPECTED_HEAD" HEAD + git merge-base --is-ancestor "$EXPECTED_MAIN" HEAD + test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3 + + cargo fmt --all -- --check + cargo test --locked --test integration + cargo test --locked + + git diff --check origin/main...HEAD + mapfile -t final_changed < <(git diff --name-only origin/main...HEAD | sort) + [[ "${final_changed[*]}" == "${expected_changed[*]}" ]] || { + printf '::error::unexpected final diff paths: %s\n' "${final_changed[*]}" + exit 1 + } + + new_head="$(git rev-parse HEAD)" + git push origin "HEAD:refs/heads/${HEAD_BRANCH}" + remote_head="$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" + test "$remote_head" = "$new_head" + echo "SEMANTIC_HEAD=$new_head" | tee -a "$GITHUB_ENV" + echo "Resolved and pushed exact head: $new_head" + + - name: Record result + shell: bash + run: | + { + echo "## shared-auth PR 24 semantic resolution" + echo + echo "- Previous PR head: \`$EXPECTED_HEAD\`" + echo "- Current main integrated: \`$EXPECTED_MAIN\`" + echo "- Resolved merge head: \`$SEMANTIC_HEAD\`" + echo "- Preserved current main features and added PR 24 hardening/tests" + echo "- Validation: cargo fmt, integration tests, full locked test suite" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/resolve-sp-interfaces-pr14.yml b/.github/workflows/resolve-sp-interfaces-pr14.yml new file mode 100644 index 00000000..ce612907 --- /dev/null +++ b/.github/workflows/resolve-sp-interfaces-pr14.yml @@ -0,0 +1,155 @@ +name: resolve sp-interfaces PR 14 semantically + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/RESOLVE-SP-INTERFACES-14" + +permissions: + contents: read + +concurrency: + group: resolve-sp-interfaces + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + REPOSITORY: StreemPilot/sp-interfaces + HEAD_BRANCH: agent/streamyard-parity-foundation + EXPECTED_HEAD: c30470fd8646d5f3fc37eead57df40131b1542eb + EXPECTED_MAIN: 945675569c52a7b669f610dbc186871023f9f78f + steps: + - name: Validate credential and exact branch heads + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + gh auth setup-git + + workdir="$(mktemp -d)" + echo "WORKDIR=$workdir" >> "$GITHUB_ENV" + git clone --no-tags "https://github.com/${REPOSITORY}.git" "$workdir/repo" + cd "$workdir/repo" + git fetch origin \ + "+refs/heads/main:refs/remotes/origin/main" \ + "+refs/heads/${HEAD_BRANCH}:refs/remotes/origin/${HEAD_BRANCH}" + test "$(git rev-parse origin/main)" = "$EXPECTED_MAIN" + test "$(git rev-parse "origin/${HEAD_BRANCH}")" = "$EXPECTED_HEAD" + git checkout -B "$HEAD_BRANCH" "origin/$HEAD_BRANCH" + git config user.name "Alexander Mills" + git config user.email "alex@oresoftware.com" + + - name: Merge the restored canonical contracts into the WebSocket branch + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + + set +e + git merge --no-commit --no-ff origin/main + status=$? + set -e + if ((status != 0)); then + mapfile -t conflicts < <(git diff --name-only --diff-filter=U | sort) + printf '::error::unexpected conflicts while combining independent contract work: %s\n' "${conflicts[*]}" + exit 1 + fi + + git diff --cached --check + git commit -m "merge: combine control WebSocket contract with restored interfaces" \ + -m "Preserve the restored schemas, SQL invariant, generators, bindings, CI, tests, and Zed package metadata while adding PR 14's bounded authenticated /v1/ws control-plane contract." + + git merge-base --is-ancestor "$EXPECTED_HEAD" HEAD + git merge-base --is-ancestor "$EXPECTED_MAIN" HEAD + test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3 + + required=( + .zpkg.toml + .github/workflows/ci.yml + package.json + scripts/generate.mjs + schema/domain.schema.json + schema/messaging.schema.json + schema/signaling.schema.json + schema/control-websocket.schema.json + sql/schema.sql + sql/migrations/20260803_chat_comment_invariants.sql + tests/chat-sql-invariants.test.mjs + tests/control-websocket-contract.test.mjs + generated/rust/Cargo.toml + generated/typescript/index.ts + generated/dart/streempilot_interfaces.dart + ) + for path in "${required[@]}"; do + test -s "$path" || { echo "::error file=$path::required combined file missing"; exit 1; } + done + + node <<'NODE' + const fs = require('node:fs'); + const pkg = JSON.parse(fs.readFileSync('package.json', 'utf8')); + const schema = JSON.parse(fs.readFileSync('schema/control-websocket.schema.json', 'utf8')); + if (pkg.exports['./schema/control-websocket'] !== './schema/control-websocket.schema.json') { + throw new Error('control WebSocket schema export is missing'); + } + const topics = schema.$defs.topic.enum; + const expected = ['studio.presence', 'broadcast.state', 'destination.health', 'recording.state', 'chat.message']; + if (JSON.stringify(topics) !== JSON.stringify(expected)) throw new Error('unexpected control topics'); + if (schema.properties.maximumClientCommandBytes.const !== 8192) throw new Error('command bound changed'); + const serialized = JSON.stringify(schema).toLowerCase(); + for (const forbidden of ['raw.media', 'sdp_offer', 'stream_key', 'oauth_token']) { + if (serialized.includes(forbidden)) throw new Error(`forbidden control-socket payload: ${forbidden}`); + } + NODE + + - name: Validate the exact combined head + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + npm ci + npm run check:generated + node --test tests/*.test.mjs + npx tsc --strict --noEmit --target es2022 generated/typescript/index.ts + cargo check --locked --manifest-path generated/rust/Cargo.toml + cargo test --locked --manifest-path generated/rust/Cargo.toml + if command -v dart >/dev/null 2>&1; then + dart analyze generated/dart + fi + git diff --check origin/main...HEAD + + mapfile -t changed < <(git diff --name-only origin/main...HEAD | sort) + expected=(package.json schema/control-websocket.schema.json tests/control-websocket-contract.test.mjs) + [[ "${changed[*]}" == "${expected[*]}" ]] || { + printf '::error::unexpected final diff paths: %s\n' "${changed[*]}" + exit 1 + } + + - name: Push exact semantic head + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + new_head="$(git rev-parse HEAD)" + test "$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" = "$EXPECTED_HEAD" + git push origin "HEAD:refs/heads/${HEAD_BRANCH}" + test "$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" = "$new_head" + echo "SEMANTIC_HEAD=$new_head" >> "$GITHUB_ENV" + + - name: Record result + shell: bash + run: | + { + echo "## sp-interfaces PR 14 semantic resolution" + echo + echo "- Old PR head: \`$EXPECTED_HEAD\`" + echo "- Restored main integrated: \`$EXPECTED_MAIN\`" + echo "- Resolved head: \`$SEMANTIC_HEAD\`" + echo "- Independent SQL/Zed restoration and control-WebSocket contract both retained" + echo "- Generator, Node, TypeScript, and Rust validation passed" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/resolve-sp-interfaces-pr3-v2.yml b/.github/workflows/resolve-sp-interfaces-pr3-v2.yml new file mode 100644 index 00000000..8d458623 --- /dev/null +++ b/.github/workflows/resolve-sp-interfaces-pr3-v2.yml @@ -0,0 +1,150 @@ +name: resolve sp-interfaces PR 3 semantically v2 + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/RESOLVE-SP-INTERFACES-3-V2" + +permissions: + contents: read + +concurrency: + group: resolve-sp-interfaces + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + REPOSITORY: StreemPilot/sp-interfaces + HEAD_BRANCH: agent/chat-concurrency-invariants + EXPECTED_HEAD: f3b4a7fbaf051e10c6b7af4f2f1238c54ee859ea + EXPECTED_MAIN: 26fe896bd30f6e35c0de6dbbf28e442d05a1a038 + steps: + - name: Validate credential and exact branch heads + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + gh auth setup-git + + workdir="$(mktemp -d)" + echo "WORKDIR=$workdir" >> "$GITHUB_ENV" + git clone --no-tags "https://github.com/${REPOSITORY}.git" "$workdir/repo" + cd "$workdir/repo" + git fetch origin \ + "+refs/heads/main:refs/remotes/origin/main" \ + "+refs/heads/${HEAD_BRANCH}:refs/remotes/origin/${HEAD_BRANCH}" + test "$(git rev-parse origin/main)" = "$EXPECTED_MAIN" + test "$(git rev-parse "origin/${HEAD_BRANCH}")" = "$EXPECTED_HEAD" + git checkout -B "$HEAD_BRANCH" "$EXPECTED_HEAD" + git config user.name "Alexander Mills" + git config user.email "alex@oresoftware.com" + + - name: Construct the semantic two-parent tree + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + + # Keep the complete interfaces project and PR 3 invariant. Integrate + # current main's useful Zed package metadata and short canonical name, + # but do not propagate its destructive mass deletion of contracts. + git show "$EXPECTED_MAIN:.zpkg.toml" > .zpkg.toml + sed -i '1s/^# streempilot-interfaces$/# sp-interfaces/' README.md + grep -q '^# sp-interfaces$' README.md + if ! grep -q '^## Zed package$' README.md; then + cat >> README.md <<'README' + + ## Zed package + + This repository is the canonical `sp-interfaces` Zed package. Its + `.zpkg.toml` manifest lets consumers install a pinned dependency instead + of copying schemas or generated bindings into downstream repositories. + README + fi + + git add .zpkg.toml README.md + git diff --cached --check + tree="$(git write-tree)" + message_file="$(mktemp)" + cat > "$message_file" <<'MSG' + merge: restore canonical interfaces and retain Zed packaging + + Preserve the full StreemPilot contract source, generators, generated Rust/TypeScript/Dart bindings, SQL, tests, CI, and PR 3's one-featured-comment invariant. Retain current main's short sp-interfaces identity and .zpkg.toml package manifest. Do not propagate the mass deletion that erased the repository's canonical contract responsibilities. + MSG + new_head="$(git commit-tree "$tree" -p "$EXPECTED_HEAD" -p "$EXPECTED_MAIN" < "$message_file")" + git reset --hard "$new_head" + git merge-base --is-ancestor "$EXPECTED_HEAD" HEAD + git merge-base --is-ancestor "$EXPECTED_MAIN" HEAD + test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3 + echo "SEMANTIC_HEAD=$new_head" >> "$GITHUB_ENV" + + - name: Validate restored repository and PR invariant + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + required=( + .zpkg.toml + .github/workflows/ci.yml + package.json + package-lock.json + scripts/generate.mjs + schema/domain.schema.json + schema/messaging.schema.json + schema/signaling.schema.json + sql/schema.sql + sql/migrations/20260803_chat_comment_invariants.sql + tests/contracts.test.mjs + tests/chat-sql-invariants.test.mjs + generated/rust/Cargo.toml + generated/typescript/index.ts + generated/dart/streempilot_interfaces.dart + ) + for path in "${required[@]}"; do + test -s "$path" || { echo "::error file=$path::required restored file missing"; exit 1; } + done + grep -q 'chat_messages_one_on_stream_per_broadcast_idx' sql/schema.sql + grep -q 'ranked_visible_comments' sql/migrations/20260803_chat_comment_invariants.sql + + npm ci + npm run check:generated + node --test tests/*.test.mjs + npx tsc --strict --noEmit --target es2022 generated/typescript/index.ts + cargo check --locked --manifest-path generated/rust/Cargo.toml + cargo test --locked --manifest-path generated/rust/Cargo.toml + git diff --check "$EXPECTED_MAIN"...HEAD + + mapfile -t versus_pr < <(git diff --name-only "$EXPECTED_HEAD"...HEAD | sort) + expected=(.zpkg.toml README.md) + [[ "${versus_pr[*]}" == "${expected[*]}" ]] || { + printf '::error::unexpected additions over PR 3: %s\n' "${versus_pr[*]}" + exit 1 + } + + - name: Push exact semantic head + shell: bash + run: | + set -Eeuo pipefail + cd "$WORKDIR/repo" + test "$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" = "$EXPECTED_HEAD" + git push origin "$SEMANTIC_HEAD:refs/heads/${HEAD_BRANCH}" + test "$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" = "$SEMANTIC_HEAD" + + - name: Record result + shell: bash + run: | + { + echo "## sp-interfaces PR 3 semantic restoration" + echo + echo "- Old PR head: \`$EXPECTED_HEAD\`" + echo "- Integrated main: \`$EXPECTED_MAIN\`" + echo "- Resolved head: \`$SEMANTIC_HEAD\`" + echo "- Full contracts restored; Zed packaging retained" + echo "- Generator, Node, TypeScript, and Rust validation passed" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/resolve-sp-interfaces-pr3.yml b/.github/workflows/resolve-sp-interfaces-pr3.yml new file mode 100644 index 00000000..fc9d8457 --- /dev/null +++ b/.github/workflows/resolve-sp-interfaces-pr3.yml @@ -0,0 +1,162 @@ +name: resolve sp-interfaces PR 3 semantically + +on: + push: + branches: [main] + paths: + - ".github/pr-merge/RESOLVE-SP-INTERFACES-3" + +permissions: + contents: read + +concurrency: + group: resolve-sp-interfaces + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + GH_TOKEN: ${{ secrets.FLEET_GH_TOKEN }} + REPOSITORY: StreemPilot/sp-interfaces + HEAD_BRANCH: agent/chat-concurrency-invariants + EXPECTED_HEAD: f3b4a7fbaf051e10c6b7af4f2f1238c54ee859ea + EXPECTED_MAIN: 26fe896bd30f6e35c0de6dbbf28e442d05a1a038 + steps: + - name: Validate stored credential + shell: bash + run: | + set -Eeuo pipefail + test -n "$GH_TOKEN" + test "$(gh api user --jq '.login')" = "ORESoftware" + gh auth setup-git + + - name: Build a two-parent semantic restoration commit + shell: bash + run: | + set -Eeuo pipefail + workdir="$(mktemp -d)" + git clone --no-tags "https://github.com/${REPOSITORY}.git" "$workdir/repo" + cd "$workdir/repo" + git fetch origin \ + "+refs/heads/main:refs/remotes/origin/main" \ + "+refs/heads/${HEAD_BRANCH}:refs/remotes/origin/${HEAD_BRANCH}" + + actual_main="$(git rev-parse origin/main)" + actual_head="$(git rev-parse "origin/${HEAD_BRANCH}")" + test "$actual_main" = "$EXPECTED_MAIN" || { + echo "::error::main moved: expected $EXPECTED_MAIN, found $actual_main" + exit 1 + } + test "$actual_head" = "$EXPECTED_HEAD" || { + echo "::error::PR head moved: expected $EXPECTED_HEAD, found $actual_head" + exit 1 + } + + git checkout -B "$HEAD_BRANCH" "$EXPECTED_HEAD" + git config user.name "Alexander Mills" + git config user.email "alex@oresoftware.com" + + # PR 3 retains the complete interfaces repository and adds the + # concurrency invariant. Current main's independently valuable intent + # is the short canonical name plus the Zed package manifest; its mass + # deletion of schemas, generators, bindings, CI, SQL, and tests is not + # a valid consolidation result and is deliberately not propagated. + git show "$EXPECTED_MAIN:.zpkg.toml" > .zpkg.toml + + node <<'NODE' + const fs = require('node:fs'); + const path = 'README.md'; + let readme = fs.readFileSync(path, 'utf8'); + readme = readme.replace(/^# streempilot-interfaces\s*$/m, '# sp-interfaces'); + if (!readme.includes('## Zed package')) { + readme += ` + + ## Zed package + + This repository is the canonical `sp-interfaces` Zed package. The package + manifest is `.zpkg.toml`; consumers should install it as a pinned Zed + dependency rather than copying schemas or generated bindings. + `; + } + fs.writeFileSync(path, readme.replace(/\n{3,}/g, '\n\n').trimEnd() + '\n'); + NODE + + git add .zpkg.toml README.md + git diff --cached --check + + # The desired tree is the complete PR head plus current main's useful + # package metadata. Create an explicit two-parent merge commit so both + # histories remain visible without accepting the destructive deletion + # side effects of a textual ours/theirs merge. + tree="$(git write-tree)" + message_file="$(mktemp)" + cat > "$message_file" <<'MSG' + merge: restore canonical interfaces and retain Zed packaging + + Preserve the full StreemPilot contract source, generators, generated Rust/TypeScript/Dart bindings, SQL, tests, CI, and PR 3's one-featured-comment invariant. Retain current main's short sp-interfaces identity and .zpkg.toml package manifest. The mass deletions on main are intentionally not propagated because they erase the repository's canonical responsibility. + MSG + new_head="$(git commit-tree "$tree" -p "$EXPECTED_HEAD" -p "$EXPECTED_MAIN" < "$message_file")" + git reset --hard "$new_head" + + git merge-base --is-ancestor "$EXPECTED_HEAD" HEAD + git merge-base --is-ancestor "$EXPECTED_MAIN" HEAD + test "$(git rev-list --parents -n 1 HEAD | wc -w)" -eq 3 + + required=( + .zpkg.toml + .github/workflows/ci.yml + package.json + package-lock.json + scripts/generate.mjs + schema/domain.schema.json + schema/messaging.schema.json + schema/signaling.schema.json + sql/schema.sql + sql/migrations/20260803_chat_comment_invariants.sql + tests/contracts.test.mjs + tests/chat-sql-invariants.test.mjs + generated/rust/Cargo.toml + generated/typescript/index.ts + generated/dart/streempilot_interfaces.dart + ) + for path in "${required[@]}"; do + test -s "$path" || { echo "::error file=$path::required restored file missing"; exit 1; } + done + + npm ci + npm run check:generated + node --test tests/*.test.mjs + npx tsc --strict --noEmit --target es2022 generated/typescript/index.ts + cargo check --locked --manifest-path generated/rust/Cargo.toml + cargo test --locked --manifest-path generated/rust/Cargo.toml + git diff --check "$EXPECTED_MAIN"...HEAD + + # Against PR 3, the merge adds only the package manifest and README + # reconciliation. Against main, it restores the canonical repository. + mapfile -t versus_pr < <(git diff --name-only "$EXPECTED_HEAD"...HEAD | sort) + expected_vs_pr=(.zpkg.toml README.md) + [[ "${versus_pr[*]}" == "${expected_vs_pr[*]}" ]] || { + printf '::error::unexpected semantic additions over PR 3: %s\n' "${versus_pr[*]}" + exit 1 + } + + git push origin "$new_head:refs/heads/${HEAD_BRANCH}" + remote_head="$(git ls-remote origin "refs/heads/${HEAD_BRANCH}" | cut -f1)" + test "$remote_head" = "$new_head" + echo "SEMANTIC_HEAD=$new_head" | tee -a "$GITHUB_ENV" + + - name: Record result + shell: bash + run: | + { + echo "## sp-interfaces PR 3 semantic resolution" + echo + echo "- Previous PR head: \`$EXPECTED_HEAD\`" + echo "- Current main integrated: \`$EXPECTED_MAIN\`" + echo "- Resolved merge head: \`$SEMANTIC_HEAD\`" + echo "- Preserved full interfaces project and PR 3 invariant" + echo "- Preserved short canonical name and Zed package manifest" + echo "- Validation: generator drift, Node contracts, TypeScript, Rust" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/resume-test-org-fleet.yml b/.github/workflows/resume-test-org-fleet.yml new file mode 100644 index 00000000..17fa8d38 --- /dev/null +++ b/.github/workflows/resume-test-org-fleet.yml @@ -0,0 +1,106 @@ +# One-time idempotent recovery for organizations interrupted by GitHub's secondary content-creation limiter. +name: resume rate-limited test organization fleet + +on: + push: + branches: [main] + paths: + - ".github/workflows/resume-test-org-fleet.yml" + - ".github/fleet-resume/START" + +permissions: + contents: read + +concurrency: + group: resume-rate-limited-test-org-fleet + cancel-in-progress: false + +jobs: + validate: + name: Validate recovery contract + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate fleet source and manifest + run: | + node --check scripts/bootstrap-test-org-fleet.mjs + node --check scripts/validate-test-org-fleet.mjs + node scripts/validate-test-org-fleet.mjs + node --test tests/test-org-fleet.test.mjs tests/git-transport-worktree.test.mjs + + resume: + name: Resume ${{ matrix.organization }} + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + organization: + - evento-globolo-test + - fiducia-cloud-test + - sonus-auris-test + - messaging-intel-test + - quaestor-ledger-test + - opto-sync-test + - memebank-test + - file-tunnel-test + - scintilla-run-test + - streempilot-test + - shared-auth-test + - hypesiege-test + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Export the encrypted fleet PAT without logging it + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + test -n "$STORED_FLEET_PAT" + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Verify target organization visibility + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + run: | + set -euo pipefail + gh api "orgs/$FLEET_ORGANIZATION" --silent + + - name: Adopt completed work and create missing repositories and PRs + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + TEST_ORG_FLEET_SKIP_TOPICS: "true" + run: | + set -euo pipefail + node scripts/bootstrap-test-org-fleet.mjs \ + --apply \ + --org "$FLEET_ORGANIZATION" \ + --concurrency 1 + + - name: Record organization completion + shell: bash + env: + FLEET_ORGANIZATION: ${{ matrix.organization }} + run: | + { + echo "## $FLEET_ORGANIZATION" + echo + echo "Idempotent recovery completed with authenticated Git transport." + echo + echo "Repository topics were deferred to minimize content-creation API pressure." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/reusable-polyglot-sdk-matrix.yml b/.github/workflows/reusable-polyglot-sdk-matrix.yml new file mode 100644 index 00000000..4362b8bb --- /dev/null +++ b/.github/workflows/reusable-polyglot-sdk-matrix.yml @@ -0,0 +1,422 @@ +name: Reusable polyglot SDK consumer matrix + +on: + workflow_call: + inputs: + client_repository: + description: owner/repository containing the client SDK matrix + required: true + type: string + client_ref: + description: Git ref to test + required: false + default: main + type: string + interfaces_repository: + description: sibling interfaces repository required by Rust SDKs + required: false + default: zed-pkg/zed-interfaces + type: string + interfaces_ref: + description: interfaces Git ref to test + required: false + default: main + type: string + secrets: + CLIENT_SDK_READ_TOKEN: + required: false + workflow_dispatch: + inputs: + client_repository: + description: owner/repository containing the client SDK matrix + required: true + default: zed-pkg/zed-clients + type: string + client_ref: + description: Git ref to test + required: true + default: main + type: string + interfaces_repository: + description: sibling interfaces repository required by Rust SDKs + required: true + default: zed-pkg/zed-interfaces + type: string + interfaces_ref: + description: interfaces Git ref to test + required: true + default: main + type: string + +permissions: + contents: read + +concurrency: + group: polyglot-sdk-${{ inputs.client_repository }}-${{ inputs.client_ref }} + cancel-in-progress: true + +jobs: + consumer: + name: ${{ matrix.language }} consumer + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - language: gleam + paths: clients/gleam|clients/gleamlang|gleam|gleamlang + tools: |- + erlang latest + gleam latest + commands: erl|gleam + required: true + - language: erlang + paths: clients/erlang|erlang + tools: erlang latest + commands: erlc + required: true + - language: elixir + paths: clients/elixir|elixir + tools: |- + erlang latest + elixir latest + commands: elixir|mix + required: true + - language: dart + paths: clients/dart|dart + tools: dart latest + commands: dart + required: true + - language: rust + paths: clients/rust|rust + tools: rust stable + commands: cargo|rustc + required: true + needs_interfaces: true + - language: java + paths: clients/java|java + tools: java 21 + commands: java|javac + required: true + - language: go + paths: clients/go|clients/golang|go|golang + tools: go latest + commands: go + required: true + - language: python3 + paths: clients/python3|clients/python|python3|python + tools: python 3.12 + commands: python + required: true + - language: ruby + paths: clients/ruby|ruby + tools: ruby latest + commands: ruby + required: true + - language: php + paths: clients/php|php + tools: |- + php latest + composer latest + commands: php|composer + required: true + - language: typescript-node + paths: clients/typescript/node|clients/typescript/nodejs|clients/typescript|typescript/node|typescript + tools: node 22 + commands: node|npm + required: true + - language: typescript-deno + paths: clients/typescript/deno|typescript/deno|clients/typescript|typescript + tools: |- + node 22 + deno latest + commands: node|npm|deno + required: true + - language: typescript-bun + paths: clients/typescript/bun|typescript/bun|clients/typescript|typescript + tools: |- + node 22 + bun latest + commands: node|npm|bun + required: true + - language: typescript-edge + paths: clients/typescript/edge|clients/typescript/edge-runtime|typescript/edge|clients/typescript|typescript + tools: node 22 + commands: node|npm + required: true + - language: kotlin + paths: clients/kotlin|kotlin + tools: |- + java 21 + kotlin latest + commands: java|kotlinc + required: true + - language: swift + paths: clients/swift|swift + tools: swift latest + commands: swift + required: true + - language: wasm + paths: clients/wasm|wasm + tools: rust stable + commands: cargo|rustc + required: true + needs_interfaces: true + # Discovery lanes stay optional until zed-clients declares them in + # its canonical .zpkg.toml package surface. + - language: zig + paths: clients/zig|zig + tools: zig latest + commands: zig + required: false + - language: cpp + paths: clients/cpp|clients/cxx|clients/c++|cpp|cxx + tools: cmake latest + commands: cmake + required: false + - language: c + paths: clients/c|c + tools: cmake latest + commands: cmake + required: false + - language: csharp + paths: clients/csharp|clients/dotnet|csharp|dotnet + tools: dotnet 8 + commands: dotnet + required: false + + steps: + - name: Check out SDK repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ inputs.client_repository }} + ref: ${{ inputs.client_ref }} + path: sdk + token: ${{ secrets.CLIENT_SDK_READ_TOKEN || github.token }} + persist-credentials: false + + - name: Locate SDK slice + id: locate + shell: bash + env: + LANGUAGE: ${{ matrix.language }} + CANDIDATE_PATHS: ${{ matrix.paths }} + REQUIRED: ${{ matrix.required }} + run: | + set -euo pipefail + IFS='|' read -r -a candidates <<< "$CANDIDATE_PATHS" + sdk_dir="" + for candidate in "${candidates[@]}"; do + if [[ -d "sdk/$candidate" ]]; then + sdk_dir="sdk/$candidate" + break + fi + done + + if [[ -z "$sdk_dir" ]]; then + if [[ "$REQUIRED" == "true" ]]; then + echo "Required $LANGUAGE SDK directory missing. Checked: $CANDIDATE_PATHS" >&2 + exit 1 + fi + echo "Optional $LANGUAGE SDK is not present; skipping." + echo 'present=false' >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Found $LANGUAGE SDK at $sdk_dir" + echo 'present=true' >> "$GITHUB_OUTPUT" + echo "sdk_dir=$sdk_dir" >> "$GITHUB_OUTPUT" + + - name: Check out exact sibling interfaces package + if: steps.locate.outputs.present == 'true' && matrix.needs_interfaces == true + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: ${{ inputs.interfaces_repository }} + ref: ${{ inputs.interfaces_ref }} + path: zed-interfaces + token: ${{ secrets.CLIENT_SDK_READ_TOKEN || github.token }} + persist-credentials: false + + - name: Install language toolchain + if: steps.locate.outputs.present == 'true' + uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + with: + tool_versions: ${{ matrix.tools }} + install: true + cache: true + github_token: ${{ github.token }} + + - name: Verify installed toolchain + if: steps.locate.outputs.present == 'true' + shell: bash + env: + COMMANDS: ${{ matrix.commands }} + run: | + set -euo pipefail + IFS='|' read -r -a commands <<< "$COMMANDS" + for command in "${commands[@]}"; do + command -v "$command" >/dev/null || { + echo "required executable was not installed: $command" >&2 + exit 1 + } + "$command" --version 2>/dev/null || true + done + + - name: Test SDK consumer + if: steps.locate.outputs.present == 'true' + shell: bash + env: + LANGUAGE: ${{ matrix.language }} + SDK_DIR: ${{ steps.locate.outputs.sdk_dir }} + run: | + set -euo pipefail + cd "$SDK_DIR" + echo "Testing $LANGUAGE SDK at $SDK_DIR" + + case "$LANGUAGE" in + gleam) + test -f gleam.toml + gleam deps download + gleam test + ;; + erlang) + mapfile -d '' files < <(find . -name '*.erl' -print0) + test "${#files[@]}" -gt 0 + mkdir -p _build/e2e + erlc -o _build/e2e "${files[@]}" + ;; + elixir) + test -f mix.exs + mix local.hex --force + mix deps.get + mix test + ;; + dart) + test -f pubspec.yaml + dart pub get + if [[ -d test ]]; then dart test; else dart analyze; fi + ;; + rust) + test -f Cargo.toml + cargo test --all-targets --all-features + ;; + java) + if [[ -x ./gradlew ]]; then + ./gradlew test + elif [[ -f pom.xml ]]; then + mvn -B test + else + mapfile -d '' files < <(find . -name '*.java' -print0) + test "${#files[@]}" -gt 0 + mkdir -p build/e2e + javac -d build/e2e "${files[@]}" + fi + ;; + go) + test -f go.mod + go test ./... + ;; + python3) + python -m pip install --upgrade pip pytest + if [[ -f pyproject.toml || -f setup.py || -f setup.cfg ]]; then + python -m pip install -e . + fi + if [[ -d tests || -d test ]]; then + python -m pytest + else + python -m compileall . + fi + ;; + ruby) + if [[ -f Gemfile ]]; then + gem install bundler + bundle install + if bundle exec rake -T 2>/dev/null | grep -q test; then + bundle exec rake test + else + mapfile -d '' files < <(find . -name '*.rb' -print0) + test "${#files[@]}" -gt 0 + for file in "${files[@]}"; do ruby -c "$file"; done + fi + else + mapfile -d '' files < <(find . -name '*.rb' -print0) + test "${#files[@]}" -gt 0 + for file in "${files[@]}"; do ruby -c "$file"; done + fi + ;; + php) + if [[ -f composer.json ]]; then + composer install --no-interaction --prefer-dist + if php -r '$d=json_decode(file_get_contents("composer.json"), true); exit(isset($d["scripts"]["test"]) ? 0 : 1);'; then + composer run-script test --no-interaction + elif [[ -x vendor/bin/phpunit ]]; then + vendor/bin/phpunit + else + mapfile -d '' files < <(find . -name '*.php' -not -path './vendor/*' -print0) + test "${#files[@]}" -gt 0 + for file in "${files[@]}"; do php -l "$file"; done + fi + else + mapfile -d '' files < <(find . -name '*.php' -print0) + test "${#files[@]}" -gt 0 + for file in "${files[@]}"; do php -l "$file"; done + fi + ;; + typescript-node) + npm ci + npm run build + npm test + npx tsc --noEmit + ;; + typescript-deno) + npm ci + npm run build + deno eval 'await import("./dist/runtimes/deno/index.js")' + ;; + typescript-bun) + npm ci + npm run build + bun -e 'await import("./dist/runtimes/bun/index.js")' + ;; + typescript-edge) + npm ci + npm run build + node -e 'import("./dist/runtimes/edge/index.js")' + ;; + kotlin) + if [[ -x ./gradlew ]]; then + ./gradlew test + else + mapfile -d '' files < <(find . -name '*.kt' -print0) + test "${#files[@]}" -gt 0 + mkdir -p build + kotlinc "${files[@]}" -d build/sdk.jar + fi + ;; + swift) + test -f Package.swift + swift test + ;; + zig) + test -f build.zig + zig build test + ;; + cpp|c) + test -f CMakeLists.txt + cmake -S . -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --parallel + ctest --test-dir build --output-on-failure + ;; + csharp) + dotnet test + ;; + wasm) + test -f Cargo.toml + cargo test --all-targets + ;; + *) + echo "No runner registered for $LANGUAGE" >&2 + exit 1 + ;; + esac diff --git a/.github/workflows/run-test-org-fleet-arm64.yml b/.github/workflows/run-test-org-fleet-arm64.yml new file mode 100644 index 00000000..a485a8fe --- /dev/null +++ b/.github/workflows/run-test-org-fleet-arm64.yml @@ -0,0 +1,255 @@ +# Dedicated authoritative ARM64 execution lane for the paced cross-organization test fleet. +name: run authoritative test organization fleet on ARM64 + +on: + push: + branches: + - agent/run-fleet-arm64 + paths: + - .github/workflows/run-test-org-fleet-arm64.yml + - bootstrap/execute-test-org-fleet-paced.once + +permissions: + contents: read + +concurrency: + group: apply-test-org-fleet-arm64-authoritative + cancel-in-progress: true + +jobs: + finish: + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' && github.ref == 'refs/heads/agent/run-fleet-arm64' + runs-on: ubuntu-24.04-arm + timeout-minutes: 360 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate authorization, tools, and fleet contract + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/execute-test-org-fleet-paced.once)" = "APPLY_TEST_FLEET_PACED_ARM64_V2" + command -v gh + command -v git + command -v jq + command -v node + node --check scripts/list-test-org-fleet-repositories.mjs + node --check scripts/bootstrap-test-org-fleet-worktree-safe.mjs + node scripts/validate-test-org-fleet.mjs + node --test \ + tests/test-org-fleet.test.mjs \ + tests/bootstrap-retry-policy.test.mjs \ + tests/paced-fleet-executor.test.mjs + + - name: Export the persisted fleet PAT without logging it + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + test -n "$STORED_FLEET_PAT" + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Cancel superseded fleet writers + shell: bash + env: + GH_REPO: ${{ github.repository }} + CURRENT_RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + + for status in queued in_progress; do + mapfile -t run_ids < <( + gh api --paginate "repos/$GH_REPO/actions/runs?status=$status&per_page=100" \ + --jq '.workflow_runs[] | select( + .path == ".github/workflows/apply-test-org-fleet.yml" or + .path == ".github/workflows/recover-test-org-fleet-v2.yml" or + .path == ".github/workflows/finish-test-org-fleet-single-runner.yml" or + .path == ".github/workflows/finish-test-org-fleet-paced.yml" or + .path == ".github/workflows/resume-test-org-fleet.yml" + ) | .id' + ) + + for run_id in "${run_ids[@]}"; do + [[ "$run_id" == "$CURRENT_RUN_ID" ]] && continue + echo "Cancelling superseded fleet run $run_id ($status)" + gh run cancel "$run_id" --repo "$GH_REPO" || true + done + done + + sleep 5 + + - name: Provision one repository at a time + shell: bash + env: + TEST_ORG_FLEET_SKIP_TOPICS: "true" + GH_REPO: ${{ github.repository }} + CURRENT_RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + + legacy_fleet_runs() { + for status in queued in_progress; do + gh api --paginate "repos/$GH_REPO/actions/runs?status=$status&per_page=100" \ + --jq '.workflow_runs[] | select( + .path == ".github/workflows/apply-test-org-fleet.yml" or + .path == ".github/workflows/recover-test-org-fleet-v2.yml" or + .path == ".github/workflows/finish-test-org-fleet-single-runner.yml" or + .path == ".github/workflows/finish-test-org-fleet-paced.yml" or + .path == ".github/workflows/resume-test-org-fleet.yml" + ) | .id' 2>/dev/null || true + done + } + + cancel_legacy_fleet_runs() { + local run_id + while IFS= read -r run_id; do + [[ -z "$run_id" || "$run_id" == "$CURRENT_RUN_ID" ]] && continue + echo "Guardian cancelling superseded fleet run $run_id" + gh run cancel "$run_id" --repo "$GH_REPO" >/dev/null 2>&1 || true + done < <(legacy_fleet_runs | sort -u) + } + + cancel_legacy_fleet_runs + ( + while sleep 45; do + cancel_legacy_fleet_runs + done + ) & + guardian_pid=$! + cleanup_guardian() { + kill "$guardian_pid" >/dev/null 2>&1 || true + wait "$guardian_pid" >/dev/null 2>&1 || true + } + trap cleanup_guardian EXIT + + organizations=( + fiducia-cloud-test + evento-globolo-test + opto-sync-test + quaestor-ledger-test + memebank-test + scintilla-run-test + file-tunnel-test + shared-auth-test + hypesiege-test + streempilot-test + sonus-auris-test + messaging-intel-test + 3fa-app-test + declarative-migrations-test + cliptown-test + claritas-viz-test + embedded-alerts-test + ) + + failures=() + processed=0 + writes=0 + + { + echo '## Authoritative ARM64 paced test-fleet completion' + echo + echo '| Organization | Specialized repos | Visible repos | Result |' + echo '|---|---:|---:|---:|' + } >> "$GITHUB_STEP_SUMMARY" + + apply_repository() { + local organization="$1" + local repository="$2" + local summary_file="$RUNNER_TEMP/fleet-${organization}-${repository}.json" + + for attempt in 1 2 3; do + rm -f "$summary_file" + if node scripts/bootstrap-test-org-fleet-worktree-safe.mjs \ + --apply \ + --org "$organization" \ + --repo "$repository" \ + --concurrency 1 \ + --summary-json > "$summary_file"; then + local request_writes + request_writes="$(jq '(.created // 0) + (.committed // 0) + (.pullRequests // 0)' "$summary_file")" + writes=$((writes + request_writes)) + if [[ "$request_writes" -gt 0 ]]; then + sleep 15 + else + sleep 2 + fi + return 0 + fi + [[ "$attempt" -lt 3 ]] && sleep $((attempt * 90)) + done + return 1 + } + + finalize_organization() { + local organization="$1" + local summary_file="$RUNNER_TEMP/fleet-${organization}-final.json" + + for attempt in 1 2 3; do + rm -f "$summary_file" + if node scripts/bootstrap-test-org-fleet-worktree-safe.mjs \ + --apply \ + --org "$organization" \ + --concurrency 1 \ + --summary-json > "$summary_file"; then + local request_writes + request_writes="$(jq '(.created // 0) + (.committed // 0) + (.pullRequests // 0)' "$summary_file")" + writes=$((writes + request_writes)) + [[ "$request_writes" -gt 0 ]] && sleep 15 + return 0 + fi + [[ "$attempt" -lt 3 ]] && sleep $((attempt * 90)) + done + return 1 + } + + for organization in "${organizations[@]}"; do + echo "::group::Paced provisioning for $organization" + cancel_legacy_fleet_runs + gh api "orgs/$organization" --silent + mapfile -t repositories < <(node scripts/list-test-org-fleet-repositories.mjs --org "$organization") + organization_failed=false + + for repository in "${repositories[@]}"; do + processed=$((processed + 1)) + echo "[$processed/287] $organization/$repository" + if ! apply_repository "$organization" "$repository"; then + organization_failed=true + echo "::error::Failed to provision $organization/$repository after three idempotent attempts" + fi + done + + if [[ "$organization_failed" == false ]] && ! finalize_organization "$organization"; then + organization_failed=true + echo "::error::Final governance/idempotency pass failed for $organization" + fi + + visible="$(gh api --paginate "orgs/$organization/repos?type=all&per_page=100" --jq '.[].name' | wc -l | tr -d '[:space:]')" + if [[ "$organization_failed" == true ]]; then + failures+=("$organization") + printf '| `%s` | %s | %s | **failure** |\n' "$organization" "${#repositories[@]}" "$visible" >> "$GITHUB_STEP_SUMMARY" + else + printf '| `%s` | %s | %s | success |\n' "$organization" "${#repositories[@]}" "$visible" >> "$GITHUB_STEP_SUMMARY" + fi + echo "::endgroup::" + done + + { + echo + echo "Specialized repository filters processed: $processed" + echo + echo "Observed create/commit/PR write operations: $writes" + } >> "$GITHUB_STEP_SUMMARY" + + test "$processed" -eq 287 + if (( ${#failures[@]} > 0 )); then + printf '::error::Authoritative ARM64 fleet completion failed for: %s\n' "${failures[*]}" + exit 1 + fi + + echo 'All selected organizations completed; `r2g` and `r2g-test` were never selected.' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/secure-fleet-token-handoff.yml b/.github/workflows/secure-fleet-token-handoff.yml new file mode 100644 index 00000000..867ffe9d --- /dev/null +++ b/.github/workflows/secure-fleet-token-handoff.yml @@ -0,0 +1,296 @@ +# One-time encrypted credential handoff. Remove the START file after the selected apply run is dispatched. +name: secure fleet token handoff + +on: + push: + branches: [main] + paths: + - ".github/fleet-handoff/START" + +permissions: + contents: write + +concurrency: + group: secure-fleet-token-handoff + cancel-in-progress: false + +jobs: + handoff: + if: >- + github.repository == 'zed-pkg-test/zed-pkg-e2e' && + github.ref == 'refs/heads/main' && + !contains(github.event.head_commit.removed, '.github/fleet-handoff/START') + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Publish an ephemeral encryption key for the selected organization + id: publish + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_LOGIN: ORESoftware + run: | + set -euo pipefail + umask 077 + + start_response="$(gh api "repos/$GITHUB_REPOSITORY/contents/.github/fleet-handoff/START?ref=$GITHUB_SHA")" + handoff_organization="$( + jq -r '.content' <<< "$start_response" \ + | tr -d '\n' \ + | base64 -d \ + | tr -d '\r' \ + | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' + )" + + allowed_organizations=( + all + zed-pkg-test + 3fa-app-test + declarative-migrations-test + cliptown-test + claritas-viz-test + embedded-alerts-test + evento-globolo-test + fiducia-cloud-test + memebank-test + opto-sync-test + quaestor-ledger-test + sonus-auris-test + messaging-intel-test + scintilla-run-test + file-tunnel-test + shared-auth-test + hypesiege-test + hacker-house-medellin-test + streempilot-test + ) + allowed=false + for candidate in "${allowed_organizations[@]}"; do + if [[ "$candidate" == "$handoff_organization" ]]; then + allowed=true + break + fi + done + if [[ "$allowed" != true ]]; then + echo "::error::START must contain exactly one approved test organization or all" + exit 1 + fi + + handoff_branch="agent/fleet-token-handoff-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + nonce="$(openssl rand -hex 32)" + expires_at="$(date -u -d '+15 minutes' '+%Y-%m-%dT%H:%M:%SZ')" + private_key="$RUNNER_TEMP/fleet-handoff-private.pem" + public_key="$RUNNER_TEMP/fleet-handoff-public.pem" + metadata="$RUNNER_TEMP/fleet-handoff-metadata.json" + + openssl genpkey -algorithm RSA \ + -pkeyopt rsa_keygen_bits:4096 \ + -out "$private_key" >/dev/null 2>&1 + openssl pkey -in "$private_key" -pubout -out "$public_key" + fingerprint="$(openssl pkey -pubin -in "$public_key" -outform DER | sha256sum | awk '{print $1}')" + + jq -n \ + --arg repository "$GITHUB_REPOSITORY" \ + --arg organization "$handoff_organization" \ + --arg branch "$handoff_branch" \ + --arg run_id "$GITHUB_RUN_ID" \ + --arg run_attempt "$GITHUB_RUN_ATTEMPT" \ + --arg source_sha "$GITHUB_SHA" \ + --arg nonce "$nonce" \ + --arg expected_login "$EXPECTED_LOGIN" \ + --arg fingerprint_sha256 "$fingerprint" \ + --arg expires_at "$expires_at" \ + '{ + repository: $repository, + organization: $organization, + branch: $branch, + run_id: $run_id, + run_attempt: $run_attempt, + source_sha: $source_sha, + nonce: $nonce, + expected_login: $expected_login, + fingerprint_sha256: $fingerprint_sha256, + expires_at: $expires_at, + algorithm: "RSA-OAEP-SHA256", + ciphertext_path: ".github/fleet-handoff/ciphertext.b64" + }' > "$metadata" + + gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" \ + -f ref="refs/heads/$handoff_branch" \ + -f sha="$GITHUB_SHA" \ + --silent + + put_file() { + local source_path="$1" + local repository_path="$2" + local message="$3" + local encoded + encoded="$(base64 -w0 "$source_path")" + gh api --method PUT "repos/$GITHUB_REPOSITORY/contents/$repository_path" \ + -f message="$message" \ + -f content="$encoded" \ + -f branch="$handoff_branch" \ + --silent + } + + put_file "$public_key" ".github/fleet-handoff/public.pem" \ + "chore: publish ephemeral fleet handoff key" + put_file "$metadata" ".github/fleet-handoff/metadata.json" \ + "chore: publish fleet handoff metadata" + + printf 'HANDOFF_BRANCH=%s\n' "$handoff_branch" >> "$GITHUB_ENV" + printf 'HANDOFF_NONCE=%s\n' "$nonce" >> "$GITHUB_ENV" + printf 'HANDOFF_ORGANIZATION=%s\n' "$handoff_organization" >> "$GITHUB_ENV" + printf 'HANDOFF_PRIVATE_KEY=%s\n' "$private_key" >> "$GITHUB_ENV" + printf 'HANDOFF_EXPIRES_AT=%s\n' "$expires_at" >> "$GITHUB_ENV" + printf 'branch=%s\n' "$handoff_branch" >> "$GITHUB_OUTPUT" + + { + echo "## Encrypted token handoff ready" + echo + echo "Selected organization: \`$handoff_organization\`" + echo + echo "Temporary branch: \`$handoff_branch\`" + echo + echo "Public-key fingerprint (SHA-256): \`$fingerprint\`" + echo + echo "Expires at: \`$expires_at\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Receive ciphertext, store the secret, and dispatch the selected fleet + shell: bash + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_LOGIN: ORESoftware + run: | + set -euo pipefail + umask 077 + + ciphertext_text="$RUNNER_TEMP/fleet-handoff-ciphertext.b64" + ciphertext_binary="$RUNNER_TEMP/fleet-handoff-ciphertext.bin" + payload_json="$RUNNER_TEMP/fleet-handoff-payload.json" + found=false + + for _ in $(seq 1 180); do + if response="$(gh api "repos/$GITHUB_REPOSITORY/contents/.github/fleet-handoff/ciphertext.b64?ref=$HANDOFF_BRANCH" 2>/dev/null)"; then + jq -r '.content' <<< "$response" | tr -d '\n' | base64 -d > "$ciphertext_text" + found=true + break + fi + sleep 5 + done + + if [[ "$found" != true ]]; then + echo "::error::No encrypted payload arrived before the handoff expired" + exit 1 + fi + + base64 -d "$ciphertext_text" > "$ciphertext_binary" + openssl pkeyutl -decrypt \ + -inkey "$HANDOFF_PRIVATE_KEY" \ + -in "$ciphertext_binary" \ + -out "$payload_json" \ + -pkeyopt rsa_padding_mode:oaep \ + -pkeyopt rsa_oaep_md:sha256 \ + -pkeyopt rsa_mgf1_md:sha256 + + payload_nonce="$(jq -er '.nonce | strings' "$payload_json")" + fleet_token="$(jq -er '.token | strings' "$payload_json")" + + if [[ "$payload_nonce" != "$HANDOFF_NONCE" ]]; then + echo "::error::Encrypted payload nonce mismatch" + exit 1 + fi + if (( ${#fleet_token} < 30 || ${#fleet_token} > 255 )); then + echo "::error::Encrypted payload did not contain a plausible GitHub token" + exit 1 + fi + if [[ "$fleet_token" != ghp_* && "$fleet_token" != github_pat_* ]]; then + echo "::error::Encrypted payload token prefix is not recognized" + exit 1 + fi + + printf '::add-mask::%s\n' "$fleet_token" + + authenticated_login="$(GH_TOKEN="$fleet_token" gh api user --jq '.login')" + if [[ "$authenticated_login" != "$EXPECTED_LOGIN" ]]; then + echo "::error::Token authenticated as an unexpected GitHub account" + exit 1 + fi + + if [[ "$HANDOFF_ORGANIZATION" == all ]]; then + organizations=( + zed-pkg-test + 3fa-app-test + declarative-migrations-test + cliptown-test + claritas-viz-test + embedded-alerts-test + evento-globolo-test + fiducia-cloud-test + memebank-test + opto-sync-test + quaestor-ledger-test + sonus-auris-test + messaging-intel-test + scintilla-run-test + file-tunnel-test + shared-auth-test + hypesiege-test + hacker-house-medellin-test + streempilot-test + ) + else + organizations=("$HANDOFF_ORGANIZATION") + fi + for organization in "${organizations[@]}"; do + GH_TOKEN="$fleet_token" gh api "orgs/$organization" --silent + done + + printf '%s' "$fleet_token" \ + | GH_TOKEN="$fleet_token" gh secret set FLEET_GH_TOKEN \ + --repo "$GITHUB_REPOSITORY" + + if ! GH_TOKEN="$fleet_token" gh secret list --repo "$GITHUB_REPOSITORY" \ + | awk '{print $1}' | grep -Fxq FLEET_GH_TOKEN; then + echo "::error::The encrypted repository secret could not be verified" + exit 1 + fi + + GH_TOKEN="$fleet_token" gh workflow run apply-test-org-fleet.yml \ + --repo "$GITHUB_REPOSITORY" \ + --ref main \ + -f organization="$HANDOFF_ORGANIZATION" \ + -f authentication=pat \ + -f repository= \ + -f confirm=APPLY_TEST_FLEET + + unset fleet_token + rm -f "$ciphertext_text" "$ciphertext_binary" "$payload_json" "$HANDOFF_PRIVATE_KEY" + + { + echo + echo "The encrypted \`FLEET_GH_TOKEN\` repository secret was stored successfully." + echo + echo "The \`$HANDOFF_ORGANIZATION\` apply workflow was dispatched on \`main\`." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Remove the temporary handoff branch and key material + if: always() + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set +e + rm -f \ + "${HANDOFF_PRIVATE_KEY:-}" \ + "$RUNNER_TEMP/fleet-handoff-public.pem" \ + "$RUNNER_TEMP/fleet-handoff-metadata.json" \ + "$RUNNER_TEMP/fleet-handoff-ciphertext.b64" \ + "$RUNNER_TEMP/fleet-handoff-ciphertext.bin" \ + "$RUNNER_TEMP/fleet-handoff-payload.json" + if [[ -n "${HANDOFF_BRANCH:-}" ]]; then + gh api --method DELETE \ + "repos/$GITHUB_REPOSITORY/git/refs/heads/${HANDOFF_BRANCH#refs/heads/}" \ + --silent || true + fi diff --git a/.github/workflows/shared-auth-exact-head-certification.yml b/.github/workflows/shared-auth-exact-head-certification.yml new file mode 100644 index 00000000..eb9c933c --- /dev/null +++ b/.github/workflows/shared-auth-exact-head-certification.yml @@ -0,0 +1,425 @@ +name: Shared Auth exact-head certification + +on: + pull_request: + branches: [main] + types: [opened, reopened, synchronize] + +permissions: + contents: write + +concurrency: + group: shared-auth-exact-head-${{ github.event.pull_request.head.ref }} + cancel-in-progress: false + +jobs: + certify: + if: >- + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.head.ref == 'feat/den-2788-full-stack-live-r2' + runs-on: ubuntu-24.04 + timeout-minutes: 75 + services: + postgres: + image: postgres:17-bookworm + env: + POSTGRES_DB: shared_auth_test + POSTGRES_USER: shared_auth + POSTGRES_PASSWORD: shared_auth_test + ports: ['5432:5432'] + options: >- + --health-cmd="pg_isready -U shared_auth -d shared_auth_test" + --health-interval=5s --health-timeout=5s --health-retries=20 + redis: + image: redis:8.2-alpine + ports: ['6379:6379'] + options: >- + --health-cmd="redis-cli ping" + --health-interval=5s --health-timeout=5s --health-retries=20 + + steps: + - name: Check out the requesting evidence head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: evidence-repo + fetch-depth: 0 + persist-credentials: true + + - name: Read and authorize the exact certification request + id: request + working-directory: evidence-repo + shell: bash + run: | + set -euo pipefail + test "$(tr -d '\r\n' < bootstrap/shared-auth-configurable-issuer.once)" = \ + 'CERTIFY_SHARED_AUTH_CONFIGURABLE_ISSUER_V1' + jq -e '.repository == "shared-auth/shared-auth-server.rs"' \ + config/shared-auth-certification.json >/dev/null + jq -e '.independent_test_repository == "shared-auth-test/server-api-contract-e2e"' \ + config/shared-auth-certification.json >/dev/null + for key in \ + repository \ + branch \ + expected_head \ + independent_test_repository \ + independent_test_head; do + value="$(jq -er --arg key "$key" '.[$key]' config/shared-auth-certification.json)" + printf '%s=%s\n' "$key" "$value" >> "$GITHUB_OUTPUT" + done + + - name: Check out the exact private Shared Auth candidate + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: ${{ steps.request.outputs.repository }} + ref: ${{ steps.request.outputs.expected_head }} + path: shared-auth + fetch-depth: 0 + token: ${{ secrets.FLEET_GH_TOKEN }} + persist-credentials: false + + - name: Check out the independent Shared Auth test contract + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + repository: ${{ steps.request.outputs.independent_test_repository }} + ref: ${{ steps.request.outputs.independent_test_head }} + path: shared-auth-tests + fetch-depth: 1 + token: ${{ secrets.FLEET_GH_TOKEN }} + persist-credentials: false + + - name: Assert immutable source and independent test pins + shell: bash + run: | + set -euo pipefail + test "$(git -C shared-auth rev-parse HEAD)" = \ + "${{ steps.request.outputs.expected_head }}" + test "$(git -C shared-auth-tests rev-parse HEAD)" = \ + "${{ steps.request.outputs.independent_test_head }}" + test -z "$(git -C shared-auth config --local --get-all http.https://github.com/.extraheader || true)" + node --check shared-auth-tests/scripts/verify-zpkg-handoff-certified-head-v2.mjs + + - name: Run the independent exact-source contract + shell: bash + run: | + set -euo pipefail + mkdir -p evidence-repo/evidence/shared-auth + node shared-auth-tests/scripts/verify-zpkg-handoff-certified-head-v2.mjs \ + --source-root shared-auth \ + --lock shared-auth-tests/zpkg-handoff-lock.json \ + --expected-sha "${{ steps.request.outputs.expected_head }}" \ + --json-output evidence-repo/evidence/shared-auth/shared-auth-test-contract-v2.json + jq -e \ + --arg sha "${{ steps.request.outputs.expected_head }}" \ + '.source.actualSha == $sha and .certification.exactHead == true and .certification.currentDependencyApis == true and .certification.boundedCsrfCookie == true' \ + evidence-repo/evidence/shared-auth/shared-auth-test-contract-v2.json >/dev/null + + - name: Install the Rust verification toolchain + uses: dtolnay/rust-toolchain@2c7215f132e9ebf062739d9130488b56d53c060c + with: + toolchain: stable + components: rustfmt, clippy + + - name: Apply declarative schema and validate realms + working-directory: shared-auth + env: + PGPASSWORD: shared_auth_test + shell: bash + run: | + set -euo pipefail + mkdir -p ../evidence-repo/evidence/shared-auth + psql -h 127.0.0.1 -U shared_auth -d shared_auth_test -v ON_ERROR_STOP=1 \ + -f db/schema.sql \ + >../evidence-repo/evidence/shared-auth/schema-apply.log + python3 -m unittest scripts/test_validate_auth_realms.py + python3 scripts/validate_auth_realms.py \ + --contract config/auth-realms.contract.json \ + --schema db/schema.sql \ + --realm-source src/realm.rs + psql -h 127.0.0.1 -U shared_auth -d shared_auth_test -Atc \ + "select to_regclass('shared_auth.browser_authorization_codes') is not null" | grep -Fx t + python3 -c 'import socket; connection = socket.create_connection(("127.0.0.1", 6379), 5); connection.sendall(b"*1\r\n$4\r\nPING\r\n"); response = connection.recv(64); connection.close(); assert response == b"+PONG\r\n", response' + + - name: Prove database first-use, replay, PKCE, redirect, client, and expiry boundaries + env: + PGPASSWORD: shared_auth_test + shell: bash + run: | + set -euo pipefail + psql -h 127.0.0.1 -U shared_auth -d shared_auth_test -v ON_ERROR_STOP=1 <<'SQL' + insert into shared_auth.browser_authorization_codes ( + code_hash, + client_id, + redirect_uri, + return_path, + supabase_project, + code_challenge, + encrypted_tokens, + created_at, + expires_at + ) values + ( + repeat('a', 43), + 'zpkg', + 'https://app.zpkg.net/auth/shared/callback', + '/dashboard', + 'zpkg', + repeat('b', 43), + repeat('e', 64), + now(), + now() + interval '90 seconds' + ), + ( + repeat('c', 43), + 'zpkg', + 'https://app.zpkg.net/auth/shared/callback', + '/dashboard', + 'zpkg', + repeat('d', 43), + repeat('f', 64), + now(), + now() + interval '90 seconds' + ), + ( + repeat('g', 43), + 'zpkg', + 'https://app.zpkg.net/auth/shared/callback', + '/dashboard', + 'zpkg', + repeat('h', 43), + repeat('i', 64), + now() - interval '120 seconds', + now() - interval '60 seconds' + ), + ( + repeat('j', 43), + 'zpkg', + 'https://app.zpkg.net/auth/shared/callback', + '/dashboard', + 'zpkg', + repeat('k', 43), + repeat('l', 64), + now(), + now() + interval '90 seconds' + ); + + do $$ + declare + first_count integer; + replay_count integer; + wrong_pkce_count integer; + wrong_redirect_count integer; + wrong_client_count integer; + expired_count integer; + begin + with redeemed as ( + update shared_auth.browser_authorization_codes + set consumed_at = now() + where code_hash = repeat('a', 43) + and client_id = 'zpkg' + and redirect_uri = 'https://app.zpkg.net/auth/shared/callback' + and consumed_at is null + and expires_at > now() + and code_challenge = repeat('b', 43) + returning encrypted_tokens, supabase_project + ) select count(*) into first_count from redeemed; + + with redeemed as ( + update shared_auth.browser_authorization_codes + set consumed_at = now() + where code_hash = repeat('a', 43) + and client_id = 'zpkg' + and redirect_uri = 'https://app.zpkg.net/auth/shared/callback' + and consumed_at is null + and expires_at > now() + and code_challenge = repeat('b', 43) + returning encrypted_tokens, supabase_project + ) select count(*) into replay_count from redeemed; + + with redeemed as ( + update shared_auth.browser_authorization_codes + set consumed_at = now() + where code_hash = repeat('c', 43) + and client_id = 'zpkg' + and redirect_uri = 'https://app.zpkg.net/auth/shared/callback' + and consumed_at is null + and expires_at > now() + and code_challenge = repeat('x', 43) + returning encrypted_tokens, supabase_project + ) select count(*) into wrong_pkce_count from redeemed; + + with redeemed as ( + update shared_auth.browser_authorization_codes + set consumed_at = now() + where code_hash = repeat('c', 43) + and client_id = 'zpkg' + and redirect_uri = 'https://evil.example/callback' + and consumed_at is null + and expires_at > now() + and code_challenge = repeat('d', 43) + returning encrypted_tokens, supabase_project + ) select count(*) into wrong_redirect_count from redeemed; + + with redeemed as ( + update shared_auth.browser_authorization_codes + set consumed_at = now() + where code_hash = repeat('c', 43) + and client_id = 'other-client' + and redirect_uri = 'https://app.zpkg.net/auth/shared/callback' + and consumed_at is null + and expires_at > now() + and code_challenge = repeat('d', 43) + returning encrypted_tokens, supabase_project + ) select count(*) into wrong_client_count from redeemed; + + with redeemed as ( + update shared_auth.browser_authorization_codes + set consumed_at = now() + where code_hash = repeat('g', 43) + and client_id = 'zpkg' + and redirect_uri = 'https://app.zpkg.net/auth/shared/callback' + and consumed_at is null + and expires_at > now() + and code_challenge = repeat('h', 43) + returning encrypted_tokens, supabase_project + ) select count(*) into expired_count from redeemed; + + if first_count <> 1 then + raise exception 'first redemption count %, expected 1', first_count; + end if; + if replay_count <> 0 then + raise exception 'replay count %, expected 0', replay_count; + end if; + if wrong_pkce_count <> 0 then + raise exception 'wrong PKCE count %, expected 0', wrong_pkce_count; + end if; + if wrong_redirect_count <> 0 then + raise exception 'wrong redirect count %, expected 0', wrong_redirect_count; + end if; + if wrong_client_count <> 0 then + raise exception 'wrong client count %, expected 0', wrong_client_count; + end if; + if expired_count <> 0 then + raise exception 'expired count %, expected 0', expired_count; + end if; + end + $$; + SQL + + - name: Prove concurrent redemption consumes exactly once + env: + PGPASSWORD: shared_auth_test + shell: bash + run: | + set -euo pipefail + redeem_sql="with redeemed as ( + update shared_auth.browser_authorization_codes + set consumed_at = now() + where code_hash = repeat('j', 43) + and client_id = 'zpkg' + and redirect_uri = 'https://app.zpkg.net/auth/shared/callback' + and consumed_at is null + and expires_at > now() + and code_challenge = repeat('k', 43) + returning encrypted_tokens, supabase_project + ) select count(*) from redeemed;" + + psql -h 127.0.0.1 -U shared_auth -d shared_auth_test -Atc "$redeem_sql" \ + >evidence-repo/evidence/shared-auth/concurrent-a.txt & + pid_a=$! + psql -h 127.0.0.1 -U shared_auth -d shared_auth_test -Atc "$redeem_sql" \ + >evidence-repo/evidence/shared-auth/concurrent-b.txt & + pid_b=$! + wait "$pid_a" + wait "$pid_b" + + total=$(awk '{ total += $1 } END { print total + 0 }' \ + evidence-repo/evidence/shared-auth/concurrent-a.txt \ + evidence-repo/evidence/shared-auth/concurrent-b.txt) + test "$total" = 1 + test "$(psql -h 127.0.0.1 -U shared_auth -d shared_auth_test -Atc \ + "select count(*) from shared_auth.browser_authorization_codes where code_hash=repeat('j',43) and consumed_at is not null")" = 1 + + - name: Run exact locked Rust certification + working-directory: shared-auth + env: + AUTH_TEST_DATABASE_URL: postgres://shared_auth:shared_auth_test@127.0.0.1:5432/shared_auth_test + AUTH_TEST_REDIS_URL: redis://127.0.0.1:6379/ + shell: bash + run: | + set -euo pipefail + cargo fmt --all -- --check + cargo check --all-targets --locked + cargo clippy --all-targets --locked -- -D warnings + cargo test --all-targets --locked + cargo build --release --locked + git diff --exit-code + + - name: Persist exact-head certification evidence + working-directory: evidence-repo + shell: bash + env: + CERTIFIED_SHA: ${{ steps.request.outputs.expected_head }} + HEAD_BRANCH: ${{ github.event.pull_request.head.ref }} + INDEPENDENT_TEST_SHA: ${{ steps.request.outputs.independent_test_head }} + run: | + set -euo pipefail + mkdir -p evidence/shared-auth + jq --arg sha "$CERTIFIED_SHA" '.expected_head = $sha | .certified_head = $sha' \ + config/shared-auth-certification.json >config/shared-auth-certification.json.tmp + mv config/shared-auth-certification.json.tmp config/shared-auth-certification.json + jq -n \ + --arg sha "$CERTIFIED_SHA" \ + --arg test_sha "$INDEPENDENT_TEST_SHA" \ + '{ + schema: 2, + repository: "shared-auth/shared-auth-server.rs", + commit: $sha, + postgres_schema: true, + postgres_first_use: 1, + postgres_replay: 0, + postgres_wrong_pkce: 0, + postgres_wrong_redirect: 0, + postgres_wrong_client: 0, + postgres_expired: 0, + postgres_concurrent_successes: 1, + redis_ping: true, + realm_contract: true, + fmt: true, + check: true, + clippy: true, + tests: true, + release_build: true, + configured_supabase_issuer: true, + hosted_endpoint_test: true, + self_hosted_endpoint_test: true, + current_crypto_apis: true, + bounded_csrf_cookie: true, + independent_test_repository: "shared-auth-test/server-api-contract-e2e", + independent_test_commit: $test_sha, + independent_contract_v2: true, + production_certified: false + }' >evidence/shared-auth/exact-head-certification.json + cat >evidence/shared-auth/exact-head-certification.md <&1 | tee "$evidence/build-a.log" + python static-registry-protocol/build_static_registry.py \ + --fixtures static-registry-protocol/fixtures \ + --out "$tree_b" \ + 2>&1 | tee "$evidence/build-b.log" + + python - "$tree_a" "$tree_b" "$evidence/inventory.json" <<'PY' + import hashlib + import json + import sys + from pathlib import Path + + first, second, output = map(Path, sys.argv[1:]) + + def inventory(root: Path): + rows = [] + for path in sorted(p for p in root.rglob("*") if p.is_file()): + data = path.read_bytes() + rows.append({ + "path": path.relative_to(root).as_posix(), + "size": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + }) + return rows + + left = inventory(first) + right = inventory(second) + if left != right: + raise SystemExit( + "independent static-registry builds are not byte-identical\n" + + json.dumps({"first": left, "second": right}, indent=2) + ) + checkpoint = json.loads((first / "checkpoint.json").read_text()) + result = { + "schema": "zed-pkg-test/static-registry-local-evidence/v1", + "source_date_epoch": 0, + "tree_sha256": checkpoint["tree_sha256"], + "checkpoint_sequence": checkpoint["seq"], + "object_count": len(checkpoint["files"]), + "files": left, + } + output.write_text( + json.dumps(result, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + print( + f"byte-identical trees: {len(left)} files, " + f"tree_sha256={checkpoint['tree_sha256']}" + ) + PY + + printf 'tree=%s\n' "$tree_a" >> "$GITHUB_OUTPUT" + printf 'evidence=%s\n' "$evidence" >> "$GITHUB_OUTPUT" + + - name: Run green conformance and every declared red mutation + run: | + set -euo pipefail + tree="${{ steps.build.outputs.tree }}" + evidence="${{ steps.build.outputs.evidence }}" + python static-registry-protocol/check_static_registry.py \ + --base "$tree" \ + 2>&1 | tee "$evidence/conformance.log" + python static-registry-protocol/check_static_registry.py \ + --base "$tree" self-test \ + 2>&1 | tee "$evidence/red-tests.log" + + expected="$(python - <<'PY' + import runpy + namespace = runpy.run_path("static-registry-protocol/check_static_registry.py") + print(len(namespace["MUTATIONS"])) + PY + )" + test "$expected" -gt 0 + grep -F '== PASS' "$evidence/conformance.log" + grep -F "== self-test PASS ($expected mutations)" "$evidence/red-tests.log" + test "$(grep -c 'red as required' "$evidence/red-tests.log")" -eq "$expected" + + - name: Prove source checkout stayed immutable + run: | + set -euo pipefail + git diff --exit-code + status="$(git status --porcelain=v1 --untracked-files=all)" + if [[ -n "$status" ]]; then + printf '%s\n' "$status" >&2 + exit 1 + fi + + - name: Upload bounded local evidence + if: ${{ always() && steps.build.outputs.evidence != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: static-registry-protocol-${{ runner.os }}-${{ github.event.pull_request.head.sha || github.sha }} + path: ${{ steps.build.outputs.evidence }} + if-no-files-found: error + retention-days: 7 + include-hidden-files: false diff --git a/.github/workflows/terminal-context-cross-platform.yml b/.github/workflows/terminal-context-cross-platform.yml new file mode 100644 index 00000000..8da9052d --- /dev/null +++ b/.github/workflows/terminal-context-cross-platform.yml @@ -0,0 +1,105 @@ +name: terminal context cross-platform certification + +on: + pull_request: + paths: + - ".github/workflows/terminal-context-cross-platform.yml" + - "scripts/terminal_context_acceptance.py" + - "docs/terminal-context-certification.md" + push: + branches: [main] + paths: + - ".github/workflows/terminal-context-cross-platform.yml" + - "scripts/terminal_context_acceptance.py" + - "docs/terminal-context-certification.md" + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact 40-character zed-pkg/zed-cli commit to certify + required: false + default: ad6ff369a763f0fbdc3b677894655c73b23e062c + type: string + +permissions: + contents: read + +concurrency: + group: terminal-context-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + ZED_CLI_REF: ${{ inputs.zed_cli_ref || 'ad6ff369a763f0fbdc3b677894655c73b23e062c' }} + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + +jobs: + certify: + name: Terminal context / ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-24.04 + label: Linux + - os: macos-15 + label: macOS + - os: windows-2025 + label: Windows + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + steps: + - name: Check out exact acceptance harness + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact Zed candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Validate immutable candidate ref + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + + - name: Install stable Rust + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Run native terminal and prompt contracts + run: | + set -euo pipefail + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib terminal_context::tests -- --nocapture + cargo test --locked --manifest-path zed-cli/Cargo.toml --lib interactive::tests -- --nocapture + cargo build --locked --manifest-path zed-cli/Cargo.toml --bin zed + + - name: Validate acceptance harness syntax + run: | + set -euo pipefail + py=python3 + if [[ "$RUNNER_OS" == "Windows" ]]; then py=python; fi + "$py" -m py_compile scripts/terminal_context_acceptance.py + + - name: Run black-box terminal-context acceptance + run: | + set -euo pipefail + py=python3 + if [[ "$RUNNER_OS" == "Windows" ]]; then py=python; fi + "$py" scripts/terminal_context_acceptance.py --zed-cli-root zed-cli diff --git a/.github/workflows/test-org-fleet.yml b/.github/workflows/test-org-fleet.yml new file mode 100644 index 00000000..7be1ad32 --- /dev/null +++ b/.github/workflows/test-org-fleet.yml @@ -0,0 +1,140 @@ +# generated by zed-pkg-test/test-org-fleet +name: test organization fleet contract + +on: + pull_request: + paths: + - "bootstrap/test-org-fleet.json.gz.b64.parts/**" + - "bootstrap/test-org-fleet.extensions.json" + - "scripts/bootstrap-test-org-fleet.mjs" + - "scripts/bootstrap-test-org-fleet.parts/**" + - "scripts/audit-test-org-fleet.mjs" + - "scripts/audit-test-org-fleet-live.mjs" + - "scripts/fleet-manifest-extensions.mjs" + - "scripts/test-org-fleet-integration-policy.mjs" + - "scripts/export-test-org-fleet-index.mjs" + - "scripts/read-fleet-manifest.mjs" + - "scripts/validate-test-org-fleet.mjs" + - "tests/test-org-fleet.test.mjs" + - "tests/audit-test-org-fleet.test.mjs" + - "tests/test-org-fleet-source-certification.test.mjs" + - "tests/generated-plan-validator-gitlinks.test.mjs" + - "tests/export-test-org-fleet-index.test.mjs" + - "tests/idempotent-fleet-pr-repair.test.mjs" + - "tests/secure-fleet-handoff-scope.test.mjs" + - "tests/fleet-manifest-extensions.test.mjs" + - "tests/quaestor-fleet-extension.test.mjs" + - "docs/test-org-fleet.md" + - "docs/test-org-fleet-index.md" + - "docs/test-org-fleet-app-auth.md" + - "docs/test-org-source-certification.md" + - "docs/quaestor-ledger-test-extension.md" + - ".github/fleet-handoff/START" + - ".github/fleet-resume/START" + - ".github/workflows/apply-test-org-fleet.yml" + - ".github/workflows/cancel-stale-fleet-recovery.yml" + - ".github/workflows/dispatch-test-org-fleet-once.yml" + - ".github/workflows/resume-test-org-fleet.yml" + - ".github/workflows/secure-fleet-token-handoff.yml" + - ".github/workflows/test-org-fleet.yml" + - "bootstrap/execute-test-org-fleet-v2.once" + push: + branches: [main] + paths: + - "bootstrap/test-org-fleet.json.gz.b64.parts/**" + - "bootstrap/test-org-fleet.extensions.json" + - "scripts/bootstrap-test-org-fleet.mjs" + - "scripts/bootstrap-test-org-fleet.parts/**" + - "scripts/audit-test-org-fleet.mjs" + - "scripts/audit-test-org-fleet-live.mjs" + - "scripts/fleet-manifest-extensions.mjs" + - "scripts/test-org-fleet-integration-policy.mjs" + - "scripts/export-test-org-fleet-index.mjs" + - "scripts/read-fleet-manifest.mjs" + - "scripts/validate-test-org-fleet.mjs" + - "tests/test-org-fleet.test.mjs" + - "tests/audit-test-org-fleet.test.mjs" + - "tests/test-org-fleet-source-certification.test.mjs" + - "tests/generated-plan-validator-gitlinks.test.mjs" + - "tests/export-test-org-fleet-index.test.mjs" + - "tests/idempotent-fleet-pr-repair.test.mjs" + - "tests/secure-fleet-handoff-scope.test.mjs" + - "tests/fleet-manifest-extensions.test.mjs" + - "tests/quaestor-fleet-extension.test.mjs" + - "docs/test-org-fleet.md" + - "docs/test-org-fleet-index.md" + - "docs/test-org-fleet-app-auth.md" + - "docs/test-org-source-certification.md" + - "docs/quaestor-ledger-test-extension.md" + - ".github/fleet-handoff/START" + - ".github/fleet-resume/START" + - ".github/workflows/apply-test-org-fleet.yml" + - ".github/workflows/cancel-stale-fleet-recovery.yml" + - ".github/workflows/dispatch-test-org-fleet-once.yml" + - ".github/workflows/resume-test-org-fleet.yml" + - ".github/workflows/secure-fleet-token-handoff.yml" + - ".github/workflows/test-org-fleet.yml" + - "bootstrap/execute-test-org-fleet-v2.once" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: test-org-fleet-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + validate: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out without persisted credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Parse scripts + run: | + node --check scripts/bootstrap-test-org-fleet.mjs + node --check scripts/audit-test-org-fleet.mjs + node --check scripts/audit-test-org-fleet-live.mjs + node --check scripts/fleet-manifest-extensions.mjs + node --check scripts/test-org-fleet-integration-policy.mjs + node --check scripts/export-test-org-fleet-index.mjs + node --check scripts/validate-test-org-fleet.mjs + + - name: Validate manifest + run: node scripts/validate-test-org-fleet.mjs + + - name: Run fleet contract tests + run: | + node --test \ + tests/test-org-fleet.test.mjs \ + tests/audit-test-org-fleet.test.mjs \ + tests/test-org-fleet-source-certification.test.mjs \ + tests/generated-plan-validator-gitlinks.test.mjs \ + tests/export-test-org-fleet-index.test.mjs \ + tests/idempotent-fleet-pr-repair.test.mjs \ + tests/secure-fleet-handoff-scope.test.mjs \ + tests/fleet-manifest-extensions.test.mjs \ + tests/quaestor-fleet-extension.test.mjs + + - name: Export canonical repository index + run: | + node scripts/export-test-org-fleet-index.mjs > /tmp/canonical-test-fleet-index.json + node -e 'const p=require("/tmp/canonical-test-fleet-index.json"); if(p.expectedRepositoryCount!==363||p.managedRepositoryCount!==341||p.retainedRepositoryCount!==22) process.exit(1)' + { + echo '### Canonical paired test-fleet index' + echo + echo '```json' + cat /tmp/canonical-test-fleet-index.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Prove dry run is write-free + run: | + env -u GH_TOKEN -u GITHUB_TOKEN \ + node scripts/bootstrap-test-org-fleet.mjs --dry-run --summary-json > /tmp/test-org-fleet-plan.json + node -e 'const p=require("/tmp/test-org-fleet-plan.json"); if(p.mode!=="dry-run"||p.planned!==341||p.created||p.committed||p.pullRequests) process.exit(1)' diff --git a/.github/workflows/verify-live-test-org-fleet.yml b/.github/workflows/verify-live-test-org-fleet.yml new file mode 100644 index 00000000..0b90f134 --- /dev/null +++ b/.github/workflows/verify-live-test-org-fleet.yml @@ -0,0 +1,147 @@ +name: verify live test organization fleet + +# `workflow_run.workflows` is resolved by GitHub against the `name:` declared by +# workflows on the *default branch*, and only fires for runs of the default +# branch. This file previously waited on "run authoritative test organization +# fleet on ARM64", which lives solely on the unmerged `agent/run-fleet-arm64` +# branch and only runs on pushes to that branch — so the chain was dead twice +# over and this verification never executed. It now chains from the fleet +# writers that do exist on `main`, and the credential-free contract job runs +# directly on pushes and pull requests so the verifier is exercised on its own +# changes. `tests/workflow-trigger-references.test.mjs` fails the build if any +# `workflow_run` reference in this repository goes dangling again. + +on: + workflow_run: + workflows: + - apply test organization fleet + - finish test organization fleet with paced writes + - finish test organization fleet on one runner + - recover test organization fleet v3 + - resume rate-limited test organization fleet + types: + - completed + pull_request: + paths: + - ".github/workflows/verify-live-test-org-fleet.yml" + - "bootstrap/test-org-fleet.extensions.json" + - "scripts/fleet-manifest-extensions.mjs" + - "scripts/verify-live-test-org-fleet.mjs" + - "scripts/validate-test-org-fleet.mjs" + - "scripts/read-fleet-manifest.mjs" + - "tests/fleet-manifest-extensions.test.mjs" + - "tests/quaestor-fleet-extension.test.mjs" + - "tests/live-fleet-verifier.test.mjs" + - "tests/workflow-trigger-references.test.mjs" + push: + branches: [main] + paths: + - ".github/workflows/verify-live-test-org-fleet.yml" + - "bootstrap/test-org-fleet.extensions.json" + - "scripts/fleet-manifest-extensions.mjs" + - "scripts/verify-live-test-org-fleet.mjs" + - "scripts/validate-test-org-fleet.mjs" + - "scripts/read-fleet-manifest.mjs" + - "tests/fleet-manifest-extensions.test.mjs" + - "tests/quaestor-fleet-extension.test.mjs" + - "tests/live-fleet-verifier.test.mjs" + - "tests/workflow-trigger-references.test.mjs" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: verify-live-test-org-fleet-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + contract: + name: Validate verifier, manifest, and trigger contracts + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out the commit under verification + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + show-progress: false + + - name: Validate verifier and manifest contract + shell: bash + run: | + set -euo pipefail + node --check scripts/verify-live-test-org-fleet.mjs + node --test tests/live-fleet-verifier.test.mjs tests/fleet-manifest-extensions.test.mjs tests/quaestor-fleet-extension.test.mjs + node scripts/validate-test-org-fleet.mjs + + - name: Prove every workflow_run trigger in this repository resolves + shell: bash + run: | + set -euo pipefail + node --test tests/workflow-trigger-references.test.mjs + + verify-live: + name: Verify all repositories, default branches, and open PRs + needs: contract + # The live fleet is the state of 19 external organizations, not a property + # of a pull request diff, and `secrets.FLEET_GH_TOKEN` is not exposed to + # pull_request runs from forks. Live verification therefore runs on the + # authoritative events — a completed fleet writer, a push to main, or a + # manual dispatch — while pull requests still run the contract job above. + if: github.repository == 'zed-pkg-test/zed-pkg-e2e' && github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out the authoritative verifier from main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + show-progress: false + + - name: Export the persisted fleet PAT without logging it + shell: bash + env: + STORED_FLEET_PAT: ${{ secrets.FLEET_GH_TOKEN }} + run: | + set -euo pipefail + if [[ -z "${STORED_FLEET_PAT:-}" ]]; then + echo '::error::secrets.FLEET_GH_TOKEN is not available to this run; live fleet verification cannot be attempted.' + exit 1 + fi + printf 'GH_TOKEN=%s\n' "$STORED_FLEET_PAT" >> "$GITHUB_ENV" + + - name: Verify all repositories, default branches, and open PRs + shell: bash + env: + FLEET_VERIFY_CONCURRENCY: "4" + run: | + set -euo pipefail + report="$RUNNER_TEMP/live-test-org-fleet-report.json" + set +e + node scripts/verify-live-test-org-fleet.mjs > "$report" + status=$? + set -e + + { + echo + echo '## Machine-readable failure summary' + echo + echo '```json' + jq '{expected, verified, failureCounts: { + missingRepositories: (.failures.missingRepositories | length), + missingDefaultBranches: (.failures.missingDefaultBranches | length), + missingOpenPullRequests: (.failures.missingOpenPullRequests | length) + }}' "$report" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + if [[ "$status" -ne 0 ]]; then + echo '::error::The live fleet does not yet satisfy the exact 341-repository/PR contract.' + jq '.failures | with_entries(.value |= .[0:25])' "$report" + exit "$status" + fi + + echo 'All 322 specialized repositories, all 19 governance repositories, their default branches, and open pull requests were verified.' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/version-interop-conformance.yml b/.github/workflows/version-interop-conformance.yml new file mode 100644 index 00000000..167fb58a --- /dev/null +++ b/.github/workflows/version-interop-conformance.yml @@ -0,0 +1,151 @@ +name: version interoperability conformance + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + inputs: + zed_interfaces_ref: + description: Immutable zed-interfaces commit to certify + required: false + default: 032b52f6e335e5696cb793d6a955c8f0658a95eb + +permissions: + contents: read + +concurrency: + group: version-interop-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +defaults: + run: + shell: bash + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: "1" + ZED_INTERFACES_REF: ${{ github.event.inputs.zed_interfaces_ref || '032b52f6e335e5696cb793d6a955c8f0658a95eb' }} + +jobs: + conformance: + name: SemVer, publication topology, and immutable bytes + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Require immutable product pin + run: | + set -euo pipefail + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out exact harness candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: harness + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact interface product + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust with formatter and Clippy + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal --component rustfmt,clippy + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Validate corpus shape and pin agreement + run: | + set -euo pipefail + python3 - <<'PY' + import json + import os + from pathlib import Path + + root = Path("harness/fixtures/version-interop-corpus") + corpus = json.loads((root / "cases.json").read_text(encoding="utf-8")) + expected = os.environ["ZED_INTERFACES_REF"] + assert corpus["schema"] == "zed.version-interop-corpus/v1" + assert corpus["interface_commit"] == expected + assert len(corpus["precedence"]) == 3 + assert len(corpus["collisions"]) == 4 + assert len(corpus["valid_records"]) == 2 + assert len(corpus["invalid_records"]) == 19 + for field in ("precedence", "collisions", "valid_records", "invalid_records"): + ids = [case["id"] for case in corpus[field]] + assert len(ids) == len(set(ids)), f"duplicate {field} ids" + + rust = (root / "src/main.rs").read_text(encoding="utf-8") + assert f'const INTERFACE_COMMIT: &str = "{expected}";' in rust + PY + + - name: Run complete interface gates + run: | + set -euo pipefail + cargo fmt --all --manifest-path zed-interfaces/Cargo.toml -- --check + cargo test --locked --manifest-path zed-interfaces/Cargo.toml --all-targets + cargo clippy --locked --manifest-path zed-interfaces/Cargo.toml \ + --all-targets --all-features -- -D warnings + git -C zed-interfaces diff --check + + - name: Generate locked external consumer + run: | + set -euo pipefail + export CARGO_TARGET_DIR="$RUNNER_TEMP/version-interop-target" + fixture=harness/fixtures/version-interop-corpus + cargo generate-lockfile --offline --manifest-path "$fixture/Cargo.toml" + cargo fmt --all --manifest-path "$fixture/Cargo.toml" -- --check + + - name: Run external corpus offline + run: | + set -euo pipefail + export CARGO_TARGET_DIR="$RUNNER_TEMP/version-interop-target" + fixture=harness/fixtures/version-interop-corpus + cargo clippy --locked --offline --manifest-path "$fixture/Cargo.toml" -- -D warnings + cargo run --locked --offline --manifest-path "$fixture/Cargo.toml" 2>&1 \ + | tee "$RUNNER_TEMP/version-interop.log" + + - name: Capture conformance evidence + if: always() + run: | + set +e + fixture=harness/fixtures/version-interop-corpus + evidence="$RUNNER_TEMP/version-interop-evidence" + mkdir -p "$evidence" + cp "$fixture/cases.json" "$evidence/" 2>/dev/null + cp "$fixture/Cargo.lock" "$evidence/consumer-Cargo.lock" 2>/dev/null + cp "$RUNNER_TEMP/version-interop.log" "$evidence/" 2>/dev/null + sha256sum "$fixture/cases.json" > "$evidence/cases.sha256" 2>/dev/null + printf '%s\n' "$ZED_INTERFACES_REF" > "$evidence/zed-interfaces-commit.txt" + git -C zed-interfaces status --porcelain=v1 --untracked-files=all \ + > "$evidence/interfaces-status.txt" 2>&1 + rm -f "$fixture/Cargo.lock" + git -C harness status --porcelain=v1 --untracked-files=all \ + > "$evidence/harness-status.txt" 2>&1 + + - name: Require clean source checkouts + if: always() + run: | + set -euo pipefail + test -z "$(git -C zed-interfaces status --porcelain=v1 --untracked-files=all)" + test -z "$(git -C harness status --porcelain=v1 --untracked-files=all)" + + - name: Upload conformance evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: version-interop-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/version-interop-evidence + if-no-files-found: warn + retention-days: 14 + include-hidden-files: false diff --git a/.github/workflows/vscode-inspect-v1-interop.yml b/.github/workflows/vscode-inspect-v1-interop.yml new file mode 100644 index 00000000..2fb34aa8 --- /dev/null +++ b/.github/workflows/vscode-inspect-v1-interop.yml @@ -0,0 +1,155 @@ +name: VS Code inspect v1 interop + +on: + pull_request: + paths: + - ".github/workflows/vscode-inspect-v1-interop.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: vscode-inspect-v1-${{ github.event.pull_request.number || github.ref || github.run_id }} + cancel-in-progress: true + +env: + ZED_CLI_REF: b7036935ce3711563f1527472fc64e92bc1341aa + ZED_VSCODE_REF: c30cfe8c483ba8ae45c18abbf714f858f7f4d419 + +jobs: + interop: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + defaults: + run: + shell: bash + steps: + - name: Validate exact product pins + run: | + set -euo pipefail + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_VSCODE_REF" =~ ^[0-9a-f]{40}$ ]] + + - name: Check out exact zed-cli candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact VS Code adapter candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-vscode + ref: ${{ env.ZED_VSCODE_REF }} + path: zed-vscode + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install declared Rust toolchain + run: | + set -euo pipefail + rustup toolchain install stable --profile minimal + rustup default stable + + - name: Build exact CLI candidate + run: | + set -euo pipefail + test "$(git -C zed-cli rev-parse HEAD)" = "$ZED_CLI_REF" + cargo build --locked --release --manifest-path zed-cli/Cargo.toml --bin zed + + - name: Set up Node.js 22 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + cache: npm + cache-dependency-path: zed-vscode/package-lock.json + + - name: Run exact VS Code adapter unit tests + working-directory: zed-vscode + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$ZED_VSCODE_REF" + npm ci --ignore-scripts --no-audit --no-fund + npm test + + - name: Exercise real CLI JSON through real VS Code adapter + env: + ZED_BINARY: ${{ github.workspace }}/zed-cli/target/release/zed + run: | + set -euo pipefail + node <<'NODE' + 'use strict'; + const assert = require('node:assert/strict'); + const crypto = require('node:crypto'); + const fs = require('node:fs'); + const os = require('node:os'); + const path = require('node:path'); + const {inspectRoot} = require('./zed-vscode/src/inspector'); + + function snapshot(root) { + const rows = []; + function walk(directory) { + for (const name of fs.readdirSync(directory).sort()) { + const absolute = path.join(directory, name); + const relative = path.relative(root, absolute); + const stat = fs.lstatSync(absolute); + if (stat.isDirectory()) { + rows.push([relative, 'dir']); + walk(absolute); + } else if (stat.isSymbolicLink()) { + rows.push([relative, 'symlink', fs.readlinkSync(absolute)]); + } else { + rows.push([relative, 'file', crypto.createHash('sha256').update(fs.readFileSync(absolute)).digest('hex')]); + } + } + } + walk(root); + return rows; + } + + (async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'zed-vscode-inspect-v1-')); + fs.writeFileSync(path.join(root, '.zpkg.toml'), '[package]\norg = "acme"\nname = "widget"\nversion = "1.0.0"\n\n[package.repository]\nvcs = "git"\nurl = "https://example.invalid/acme/widget"\n'); + fs.writeFileSync(path.join(root, '.zpkg.lock'), 'version = 1\n'); + fs.mkdirSync(path.join(root, '.zpkg-staging')); + fs.writeFileSync(path.join(root, '.zpkg-staging', 'sentinel'), 'must-survive'); + const before = snapshot(root); + + const report = await inspectRoot(root, { + executable: process.env.ZED_BINARY, + timeoutMs: 10000, + }); + + assert.equal(report.source, 'cli', JSON.stringify(report)); + assert.equal(path.resolve(report.workspaceRoot), path.resolve(root)); + const recovery = report.issues.find((issue) => issue.id === 'RECOVERY_PENDING'); + assert.ok(recovery, JSON.stringify(report)); + assert.equal(recovery.severity, 'warning'); + assert.equal(recovery.actions.length, 1); + assert.deepEqual(recovery.actions[0], { + id: 'recover-frozen', + title: 'Recover with the locked graph', + kind: 'command', + command: 'zed', + arguments: ['install', '--frozen'], + requiresConfirmation: true, + workingDirectory: fs.realpathSync.native(root), + mutatesProject: true, + requiresNetwork: true, + executesPackageCode: true, + }); + assert.deepEqual(snapshot(root), before); + assert.equal(fs.readFileSync(path.join(root, '.zpkg-staging', 'sentinel'), 'utf8'), 'must-survive'); + assert.equal(fs.existsSync(path.join(root, 'zed_modules')), false); + console.log(`certified cli=${process.env.ZED_CLI_REF} vscode=${process.env.ZED_VSCODE_REF}`); + })().catch((error) => { + console.error(error); + process.exitCode = 1; + }); + NODE diff --git a/.github/workflows/windows-develop-clean-room.yml b/.github/workflows/windows-develop-clean-room.yml new file mode 100644 index 00000000..cceb672c --- /dev/null +++ b/.github/workflows/windows-develop-clean-room.yml @@ -0,0 +1,219 @@ +name: Windows develop clean-room cross-org canary + +on: + pull_request: + paths: + - '.github/workflows/windows-develop-clean-room.yml' + - 'docs/windows-develop-clean-room.md' + push: + branches: [main] + paths: + - '.github/workflows/windows-develop-clean-room.yml' + - 'docs/windows-develop-clean-room.md' + schedule: + - cron: '17 9 * * 3' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: windows-develop-clean-room-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ZED_E2E_REF: 144d17e28afa9586bc71ec43f340a0e0328a54b4 + ZED_CLI_REF: fd3b3e487b2bdd129dd67403ad51f7299cfe6828 + ZED_INTERFACES_REF: c2e049006453c26ca8ca291783f681fce75cb01f + FLAGS2ENV_REF: 2f62e40932a0fcb8b9bf1b4c84473e34fa3c51c7 + CARGO_TERM_COLOR: always + PYTHONDONTWRITEBYTECODE: '1' + +jobs: + acceptance: + runs-on: windows-2022 + timeout-minutes: 40 + steps: + - name: Require immutable source pins + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + foreach ($pin in @( + $env:ZED_E2E_REF, + $env:ZED_CLI_REF, + $env:ZED_INTERFACES_REF, + $env:FLAGS2ENV_REF + )) { + if ($pin -notmatch '^[0-9a-f]{40}$') { + throw "non-immutable source pin: $pin" + } + } + + - name: Check out this cross-org canary + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: canary + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact independent E2E contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-e2e + ref: ${{ env.ZED_E2E_REF }} + path: zed-e2e + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the merged CLI implementation + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out the exact interface contract + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-interfaces + ref: ${{ env.ZED_INTERFACES_REF }} + path: zed-interfaces + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + + - name: Install Python 3.13 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: '3.13' + + - name: Verify immutable checkouts and transitive contracts + shell: pwsh + env: + ARTIFACT_DIR: ${{ runner.temp }}\zed-develop-windows-cross-org + run: | + $ErrorActionPreference = 'Stop' + New-Item -ItemType Directory -Force -Path $env:ARTIFACT_DIR | Out-Null + + $e2e = (git -C zed-e2e rev-parse HEAD).Trim() + $cli = (git -C zed-cli rev-parse HEAD).Trim() + $interfaces = (git -C zed-interfaces rev-parse HEAD).Trim() + if ($e2e -ne $env:ZED_E2E_REF) { throw "E2E ref drift: $e2e" } + if ($cli -ne $env:ZED_CLI_REF) { throw "CLI ref drift: $cli" } + if ($interfaces -ne $env:ZED_INTERFACES_REF) { throw "interfaces ref drift: $interfaces" } + + $cargo = Get-Content -Raw zed-cli/Cargo.toml + if (-not $cargo.Contains('rev = "' + $env:ZED_INTERFACES_REF + '"')) { + throw 'zed-interfaces pin drifted in Cargo.toml' + } + if (-not $cargo.Contains('rev = "' + $env:FLAGS2ENV_REF + '"')) { + throw 'flags2env pin drifted in Cargo.toml' + } + + @( + "canary_commit=$((git -C canary rev-parse HEAD).Trim())", + "zed_e2e_commit=$e2e", + "zed_cli_commit=$cli", + "zed_interfaces_commit=$interfaces", + "flags2env_commit=$env:FLAGS2ENV_REF", + "runner_os=${{ runner.os }}", + "runner_arch=${{ runner.arch }}", + "rustc=$(rustc --version)", + "cargo=$(cargo --version)", + "python=$(python --version)", + "pwsh=$((Get-Command pwsh.exe).Source)", + "cmd=$env:COMSPEC" + ) | Set-Content -Encoding utf8 "$env:ARTIFACT_DIR/environment.txt" + + - name: Audit the imported E2E policy without mutating checkouts + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + foreach ($path in @( + 'zed-e2e/tests/cli/zed_develop_windows_clean_room.py', + 'zed-e2e/tests/cli/run_zed_develop_windows_clean_room.py', + 'zed-e2e/tests/cli/test_zed_develop_windows_policy.py' + )) { + python -c "compile(open(r'$path', encoding='utf-8').read(), r'$path', 'exec')" + } + python -m unittest -v zed-e2e/tests/cli/test_zed_develop_windows_policy.py + + - name: Build the exact merged CLI + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + cargo build --locked --manifest-path zed-cli/Cargo.toml --bin zed + if (-not (Test-Path 'zed-cli\target\debug\zed.exe')) { + throw 'zed.exe was not built' + } + + - name: Run the imported native Windows contract + shell: pwsh + env: + ARTIFACT_DIR: ${{ runner.temp }}\zed-develop-windows-cross-org + run: | + $ErrorActionPreference = 'Stop' + python zed-e2e/tests/cli/run_zed_develop_windows_clean_room.py ` + --zed "$env:GITHUB_WORKSPACE\zed-cli\target\debug\zed.exe" ` + --artifact-dir "$env:ARTIFACT_DIR" ` + --candidate-sha "$env:ZED_CLI_REF" ` + --interfaces-sha "$env:ZED_INTERFACES_REF" ` + --flags2env-sha "$env:FLAGS2ENV_REF" + + - name: Scan retained evidence for all credential canaries + shell: pwsh + env: + ARTIFACT_DIR: ${{ runner.temp }}\zed-develop-windows-cross-org + run: | + $ErrorActionPreference = 'Stop' + $canaries = @( + 'windows-clean-room-inherited-secret-canary', + 'windows-clean-room-registry-token-canary', + 'windows-clean-room-dotenv-canary', + 'windows-clean-room-direnv-canary', + 'windows-clean-room-production-env-canary', + 'windows-clean-room-powershell-profile-canary', + 'windows-clean-room-codex-credential-canary', + 'windows-clean-room-aws-credential-canary', + 'windows-clean-room-gcloud-credential-canary', + 'windows-clean-room-gh-credential-canary', + 'windows-clean-room-npm-credential-canary' + ) + foreach ($file in Get-ChildItem -File -Recurse $env:ARTIFACT_DIR) { + $content = Get-Content -Raw $file.FullName + foreach ($canary in $canaries) { + if ($content.Contains($canary)) { + throw "credential canary leaked into evidence: $($file.FullName)" + } + } + } + + - name: Prove every source checkout remained clean + if: always() + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + foreach ($repository in @('canary', 'zed-e2e', 'zed-cli', 'zed-interfaces')) { + git -C $repository status --short + $dirty = git -C $repository status --porcelain=v1 --untracked-files=all + if ($dirty) { throw "$repository checkout was mutated" } + } + + - name: Upload bounded cross-org evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: windows-develop-cross-org-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}\zed-develop-windows-cross-org\ + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/workstation-cloudflare-inventory-v2.yml b/.github/workflows/workstation-cloudflare-inventory-v2.yml new file mode 100644 index 00000000..22b1b11e --- /dev/null +++ b/.github/workflows/workstation-cloudflare-inventory-v2.yml @@ -0,0 +1,215 @@ +name: workstation Cloudflare inventory v2 +run-name: workstation Cloudflare inventory v2 @ ${{ github.sha }} + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/workstation-cloudflare-inventory-v2.yml + push: + branches: [main] + paths: + - .github/workflows/workstation-cloudflare-inventory-v2.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: workstation-cloudflare-inventory-v2 + cancel-in-progress: false + +jobs: + validate: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 2 + + - name: Validate workflow syntax + uses: docker://rhysd/actionlint@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 + with: + args: .github/workflows/workstation-cloudflare-inventory-v2.yml + + - name: Validate immutable safety markers + shell: bash + run: | + set -euo pipefail + workflow=.github/workflows/workstation-cloudflare-inventory-v2.yml + grep -qF 'rsa_padding_mode:oaep' "$workflow" + grep -qF 'rsa_oaep_md:sha256' "$workflow" + grep -qF 'Destroy credential material' "$workflow" + grep -qF 'Authorization: Bearer' "$workflow" + git diff --check HEAD^ HEAD + + inventory: + if: >- + github.event_name != 'pull_request' && + github.repository == 'zed-pkg-test/zed-pkg-e2e' && + github.ref == 'refs/heads/main' + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + issues: write + env: + ISSUE_NUMBER: 158 + CONFIRMATION: inspect-zpkg-workstation-tunnel-v2 + ACCOUNT_ID: 62b833940607839add74bd2379cac303 + TUNNEL_ID: 2eb5c6d9-a2b7-4c5e-9222-2f43e91f90e1 + ZPKG_ZONE_ID: b559136046dcffc550ee8b3ed49cdf09 + steps: + - name: Check out exact trusted main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Receive one run-bound encrypted Cloudflare token + id: handoff + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -Eeuo pipefail + umask 077 + run_id="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(openssl rand -hex 16)" + private_key="$RUNNER_TEMP/cf-private.pem" + public_key="$RUNNER_TEMP/cf-public.pem" + encrypted="$RUNNER_TEMP/cf-token.bin" + payload="$RUNNER_TEMP/cf-payload.json" + token_file="$RUNNER_TEMP/cf-api-token" + matches="$RUNNER_TEMP/cf-matches" + + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out "$private_key" >/dev/null 2>&1 + openssl pkey -in "$private_key" -pubout -out "$public_key" >/dev/null 2>&1 + key_b64="$(base64 -w0 < "$public_key")" + gh api --method POST "repos/${GITHUB_REPOSITORY}/issues/${ISSUE_NUMBER}/comments" \ + -f body="CF_WORKSTATION_V2_PUBLIC_KEY protocol=1 run_id=${run_id} key_b64=${key_b64} confirm=${CONFIRMATION}" >/dev/null + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" + echo "token_file=${token_file}" >> "$GITHUB_OUTPUT" + echo "HANDSHAKE_READY run_id=${run_id} issue=${ISSUE_NUMBER}" + + prefix="CF_WORKSTATION_V2_CIPHERTEXT protocol=1 run_id=${run_id} data=" + ciphertext='' + for _ in $(seq 1 360); do + gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${ISSUE_NUMBER}/comments" \ + --jq ".[] | select(.body | startswith(\"${prefix}\")) | .body" > "$matches" + count="$(awk 'NF { n += 1 } END { print n + 0 }' "$matches")" + if [ "$count" -gt 1 ]; then + echo "duplicate ciphertexts for ${run_id}" >&2 + exit 1 + fi + if [ "$count" -eq 1 ]; then + line="$(cat "$matches")" + ciphertext="${line#"$prefix"}" + break + fi + sleep 5 + done + test -n "$ciphertext" + printf '%s' "$ciphertext" | base64 -d > "$encrypted" + openssl pkeyutl -decrypt -inkey "$private_key" -in "$encrypted" \ + -pkeyopt rsa_padding_mode:oaep \ + -pkeyopt rsa_oaep_md:sha256 \ + -pkeyopt rsa_mgf1_md:sha256 \ + -out "$payload" + jq -e --arg run_id "$run_id" --arg confirm "$CONFIRMATION" ' + .run_id == $run_id and .confirm == $confirm and + (.cf_api_token | type == "string" and length >= 20) + ' "$payload" >/dev/null + token="$(jq -r .cf_api_token "$payload")" + echo "::add-mask::${token}" + printf '%s' "$token" > "$token_file" + chmod 600 "$token_file" + rm -f "$encrypted" "$payload" "$matches" + + - name: Read live tunnel and Access routing + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TOKEN_FILE: ${{ steps.handoff.outputs.token_file }} + RUN_ID: ${{ steps.handoff.outputs.run_id }} + run: | + set -Eeuo pipefail + umask 077 + token="$(cat "$TOKEN_FILE")" + api=https://api.cloudflare.com/client/v4 + raw="$RUNNER_TEMP/cf-raw" + out="$RUNNER_TEMP/workstation-cloudflare-inventory.json" + mkdir -p "$raw" + + get_required() { + curl --fail-with-body --silent --show-error \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + "${api}$2" > "$raw/$1.json" + jq -e '.success == true' "$raw/$1.json" >/dev/null + } + get_optional() { + if ! curl --fail-with-body --silent --show-error \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + "${api}$2" > "$raw/$1.json"; then + jq -n '{success:false,result:null}' > "$raw/$1.json" + fi + } + + get_required verify '/user/tokens/verify' + get_required tunnels "/accounts/${ACCOUNT_ID}/cfd_tunnel?is_deleted=false&per_page=100" + get_required config "/accounts/${ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/configurations" + get_optional connections "/accounts/${ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/connections" + get_optional access "/accounts/${ACCOUNT_ID}/access/apps?per_page=200" + get_optional dns "/zones/${ZPKG_ZONE_ID}/dns_records?per_page=500" + + jq -n \ + --arg tunnel_id "$TUNNEL_ID" \ + --slurpfile verify "$raw/verify.json" \ + --slurpfile tunnels "$raw/tunnels.json" \ + --slurpfile config "$raw/config.json" \ + --slurpfile connections "$raw/connections.json" \ + --slurpfile access "$raw/access.json" \ + --slurpfile dns "$raw/dns.json" ' + { + schema: "cloudflare.workstation-route/v2", + token_status: ($verify[0].result.status // "unknown"), + tunnel: (($tunnels[0].result // []) | map(select(.id == $tunnel_id or .name == "zpkg-registry-local")) | map({id,name,status,remote_config,created_at})), + ingress: (($config[0].result.config.ingress // $config[0].result.ingress // []) | map({hostname,service,originRequest})), + connectors: (($connections[0].result // []) | map({id,colo_name,opened_at,origin_ip,is_pending_reconnect,uuid})), + access_apps: (($access[0].result // []) | map({id,name,domain,type,session_duration})), + tunnel_dns: (($dns[0].result // []) | map(select(((.content // "") | ascii_downcase | contains("cfargotunnel.com")) or ((.content // "") | ascii_downcase | contains(($tunnel_id | ascii_downcase)))) | {id,type,name,content,proxied,ttl})) + } + ' > "$out" + jq -e '.tunnel | length == 1' "$out" >/dev/null + jq . "$out" + + payload_b64="$(base64 -w0 < "$out")" + gh api --method POST "repos/${GITHUB_REPOSITORY}/issues/${ISSUE_NUMBER}/comments" \ + -f body="CF_WORKSTATION_V2_RESULT protocol=1 run_id=${RUN_ID} payload_b64=${payload_b64}" >/dev/null + + - name: Upload non-secret route inventory + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: workstation-cloudflare-route-${{ github.run_id }} + path: ${{ runner.temp }}/workstation-cloudflare-inventory.json + if-no-files-found: error + retention-days: 7 + + - name: Destroy credential material + if: always() + shell: bash + run: | + shred -u \ + "$RUNNER_TEMP/cf-private.pem" \ + "$RUNNER_TEMP/cf-public.pem" \ + "$RUNNER_TEMP/cf-token.bin" \ + "$RUNNER_TEMP/cf-payload.json" \ + "$RUNNER_TEMP/cf-api-token" \ + 2>/dev/null || true + rm -rf "$RUNNER_TEMP/cf-matches" "$RUNNER_TEMP/cf-raw" diff --git a/.github/workflows/workstation-cloudflare-inventory.yml b/.github/workflows/workstation-cloudflare-inventory.yml new file mode 100644 index 00000000..d95081e1 --- /dev/null +++ b/.github/workflows/workstation-cloudflare-inventory.yml @@ -0,0 +1,284 @@ +name: workstation Cloudflare inventory +run-name: workstation Cloudflare inventory @ ${{ github.sha }} + +on: + pull_request: + branches: [main] + paths: + - .github/workflows/workstation-cloudflare-inventory.yml + push: + branches: [main] + paths: + - .github/workflows/workstation-cloudflare-inventory.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: workstation-cloudflare-inventory + cancel-in-progress: false + +jobs: + validate: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 2 + + - name: Validate workflow syntax + uses: docker://rhysd/actionlint@sha256:b1934ee5f1c509618f2508e6eb47ee0d3520686341fec936f3b79331f9315667 + with: + args: .github/workflows/workstation-cloudflare-inventory.yml + + - name: Validate credential boundary + shell: bash + run: | + set -euo pipefail + workflow=.github/workflows/workstation-cloudflare-inventory.yml + grep -qF 'rsa_padding_mode:oaep' "$workflow" + grep -qF 'rsa_oaep_md:sha256' "$workflow" + grep -qF 'Destroy credential material' "$workflow" + credential_pattern='gh''p_|github_''pat_|cfat_|BEGIN [A-Z ]*PRIVATE KEY' + if grep -nE "$credential_pattern" "$workflow"; then + echo 'credential-shaped source material detected' >&2 + exit 1 + fi + git diff --check HEAD^ HEAD + + inventory: + if: >- + github.event_name != 'pull_request' && + github.repository == 'zed-pkg-test/zed-pkg-e2e' && + github.ref == 'refs/heads/main' + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + issues: write + env: + ISSUE_NUMBER: 158 + CONFIRMATION: inspect-zpkg-workstation-tunnel + ACCOUNT_ID: 62b833940607839add74bd2379cac303 + TUNNEL_ID: 2eb5c6d9-a2b7-4c5e-9222-2f43e91f90e1 + ZPKG_ZONE_ID: b559136046dcffc550ee8b3ed49cdf09 + steps: + - name: Check out exact trusted main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + fetch-depth: 1 + + - name: Receive one run-bound encrypted Cloudflare API token + id: handoff + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -Eeuo pipefail + umask 077 + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + run_id="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(openssl rand -hex 16)" + private_key="$RUNNER_TEMP/workstation-cf-private.pem" + public_key="$RUNNER_TEMP/workstation-cf-public.pem" + matches="$RUNNER_TEMP/workstation-cf-ciphertext-matches" + encrypted="$RUNNER_TEMP/workstation-cf-token.bin" + payload="$RUNNER_TEMP/workstation-cf-payload.json" + token_file="$RUNNER_TEMP/workstation-cf-api-token" + + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:4096 -out "$private_key" >/dev/null 2>&1 + openssl pkey -in "$private_key" -pubout -out "$public_key" >/dev/null 2>&1 + key_b64="$(base64 -w0 < "$public_key")" + gh api --method POST "repos/${GITHUB_REPOSITORY}/issues/${ISSUE_NUMBER}/comments" \ + -f body="CF_WORKSTATION_INVENTORY_PUBLIC_KEY protocol=1 run_id=${run_id} key_b64=${key_b64} confirm=${CONFIRMATION}" >/dev/null + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" + echo "token_file=${token_file}" >> "$GITHUB_OUTPUT" + echo "HANDSHAKE_READY run_id=${run_id} issue=${ISSUE_NUMBER}" + + prefix="CF_WORKSTATION_INVENTORY_CIPHERTEXT protocol=1 run_id=${run_id} data=" + query_matches() { + gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${ISSUE_NUMBER}/comments" \ + --jq ".[] | select(.body | startswith(\"${prefix}\")) | .body" > "$matches" + } + reject_duplicates() { + count="$1" + gh api --method POST "repos/${GITHUB_REPOSITORY}/issues/${ISSUE_NUMBER}/comments" \ + -f body="CF_WORKSTATION_INVENTORY_REJECTED protocol=1 run_id=${run_id} reason=duplicate_ciphertexts count=${count}" >/dev/null + echo "expected exactly one ciphertext; observed ${count}" >&2 + exit 1 + } + + ciphertext='' + for _ in $(seq 1 360); do + query_matches + count="$(awk 'NF { count += 1 } END { print count + 0 }' "$matches")" + [ "$count" -le 1 ] || reject_duplicates "$count" + if [ "$count" -eq 1 ]; then + sleep 5 + query_matches + count="$(awk 'NF { count += 1 } END { print count + 0 }' "$matches")" + [ "$count" -eq 1 ] || reject_duplicates "$count" + line="$(cat "$matches")" + ciphertext="${line#"$prefix"}" + break + fi + sleep 5 + done + test -n "$ciphertext" || { echo "encrypted credential was not received for ${run_id}" >&2; exit 1; } + printf '%s' "$ciphertext" | base64 -d > "$encrypted" + openssl pkeyutl -decrypt -inkey "$private_key" -in "$encrypted" \ + -pkeyopt rsa_padding_mode:oaep \ + -pkeyopt rsa_oaep_md:sha256 \ + -pkeyopt rsa_mgf1_md:sha256 \ + -out "$payload" + jq -e --arg run_id "$run_id" --arg confirmation "$CONFIRMATION" ' + (.run_id == $run_id) and (.confirm == $confirmation) and + (.cf_api_token | type == "string" and length >= 20 and (test("\\s") | not)) + ' "$payload" >/dev/null + token="$(jq -r .cf_api_token "$payload")" + echo "::add-mask::${token}" + printf '%s' "$token" > "$token_file" + chmod 600 "$token_file" + rm -f "$matches" "$encrypted" "$payload" + + - name: Query tunnel, DNS, and Access metadata + id: inventory + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TOKEN_FILE: ${{ steps.handoff.outputs.token_file }} + RUN_ID: ${{ steps.handoff.outputs.run_id }} + run: | + set -Eeuo pipefail + umask 077 + token="$(cat "$TOKEN_FILE")" + api=https://api.cloudflare.com/client/v4 + out="$RUNNER_TEMP/workstation-tunnel-inventory.json" + raw="$RUNNER_TEMP/workstation-tunnel-raw" + mkdir -p "$raw" + + cf_get_required() { + name="$1" + path="$2" + destination="$raw/${name}.json" + curl --fail-with-body --silent --show-error \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + "${api}${path}" > "$destination" + jq -e '.success == true' "$destination" >/dev/null + } + cf_get_optional() { + name="$1" + path="$2" + destination="$raw/${name}.json" + if ! curl --fail-with-body --silent --show-error \ + -H "Authorization: Bearer ${token}" \ + -H 'Content-Type: application/json' \ + "${api}${path}" > "$destination"; then + jq -n --arg endpoint "$path" '{success:false,endpoint:$endpoint,result:null,errors:[{message:"request failed or permission denied"}]}' > "$destination" + fi + } + + cf_get_required verify '/user/tokens/verify' + cf_get_required tunnels "/accounts/${ACCOUNT_ID}/cfd_tunnel?is_deleted=false&per_page=100" + cf_get_required tunnel_config "/accounts/${ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/configurations" + cf_get_optional tunnel_connections "/accounts/${ACCOUNT_ID}/cfd_tunnel/${TUNNEL_ID}/connections" + cf_get_optional access_apps "/accounts/${ACCOUNT_ID}/access/apps?per_page=200" + cf_get_optional dns_records "/zones/${ZPKG_ZONE_ID}/dns_records?per_page=500" + + jq -n \ + --arg account_id "$ACCOUNT_ID" \ + --arg tunnel_id "$TUNNEL_ID" \ + --arg zone_id "$ZPKG_ZONE_ID" \ + --slurpfile verify "$raw/verify.json" \ + --slurpfile tunnels "$raw/tunnels.json" \ + --slurpfile config "$raw/tunnel_config.json" \ + --slurpfile connections "$raw/tunnel_connections.json" \ + --slurpfile apps "$raw/access_apps.json" \ + --slurpfile dns "$raw/dns_records.json" ' + def relevant_domain: + ascii_downcase | test("(zpkg|registry|ssh|mac[-.]?mini|workstation|toolbox|credistrata)"); + { + schema: "cloudflare.workstation-tunnel-inventory/v1", + account_id: $account_id, + tunnel_id: $tunnel_id, + zone_id: $zone_id, + token_status: ($verify[0].result.status // "unknown"), + tunnel: ( + ($tunnels[0].result // []) + | map(select(.id == $tunnel_id or .name == "zpkg-registry-local")) + | map({id,name,status,remote_config,created_at,deleted_at}) + ), + ingress: ( + $config[0].result.config.ingress // $config[0].result.ingress // [] + | map({hostname,service,originRequest}) + ), + connections: { + readable: ($connections[0].success // false), + count: (($connections[0].result // []) | length), + connectors: (($connections[0].result // []) | map({id,colo_name,is_pending_reconnect,opened_at,origin_ip,uuid})) + }, + dns: { + readable: ($dns[0].success // false), + records: ( + ($dns[0].result // []) + | map(select( + (((.name // "") | type) == "string" and ((.name // "") | relevant_domain)) or + (((.content // "") | type) == "string" and ((.content // "") | ascii_downcase | contains(($tunnel_id | ascii_downcase)))) or + (((.content // "") | type) == "string" and ((.content // "") | ascii_downcase | contains("cfargotunnel.com"))) + )) + | map({id,type,name,content,proxied,ttl}) + ) + }, + access: { + readable: ($apps[0].success // false), + applications: ( + ($apps[0].result // []) + | map(select( + (((.domain // "") | type) == "string" and ((.domain // "") | relevant_domain)) or + (((.name // "") | type) == "string" and ((.name // "") | relevant_domain)) or + (((.type // "") | type) == "string" and ((.type // "") | ascii_downcase | test("ssh|self_hosted"))) + )) + | map({id,name,domain,type,session_duration,auto_redirect_to_identity,enable_binding_cookie,http_only_cookie_attribute,same_site_cookie_attribute}) + ) + } + } + ' > "$out" + + jq -e '.tunnel | length >= 1' "$out" >/dev/null + jq . "$out" + + tunnel_status="$(jq -r '.tunnel[0].status // "unknown"' "$out")" + connector_count="$(jq -r '.connections.count' "$out")" + ingress_b64="$(jq -c '[.ingress[] | {hostname,service}]' "$out" | base64 -w0)" + dns_b64="$(jq -c '.dns.records' "$out" | base64 -w0)" + access_b64="$(jq -c '.access.applications' "$out" | base64 -w0)" + gh api --method POST "repos/${GITHUB_REPOSITORY}/issues/${ISSUE_NUMBER}/comments" \ + -f body="CF_WORKSTATION_INVENTORY_RESULT protocol=1 run_id=${RUN_ID} tunnel_status=${tunnel_status} connector_count=${connector_count} ingress_b64=${ingress_b64} dns_b64=${dns_b64} access_b64=${access_b64}" >/dev/null + + - name: Upload non-secret inventory + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: workstation-tunnel-inventory-${{ github.run_id }} + path: ${{ runner.temp }}/workstation-tunnel-inventory.json + if-no-files-found: error + retention-days: 7 + + - name: Destroy credential material + if: always() + shell: bash + run: | + shred -u \ + "$RUNNER_TEMP/workstation-cf-private.pem" \ + "$RUNNER_TEMP/workstation-cf-public.pem" \ + "$RUNNER_TEMP/workstation-cf-token.bin" \ + "$RUNNER_TEMP/workstation-cf-payload.json" \ + "$RUNNER_TEMP/workstation-cf-api-token" \ + 2>/dev/null || true + rm -rf "$RUNNER_TEMP/workstation-cf-ciphertext-matches" "$RUNNER_TEMP/workstation-tunnel-raw" diff --git a/.github/workflows/zed-cli-graph-github-production-pinned.yml b/.github/workflows/zed-cli-graph-github-production-pinned.yml new file mode 100644 index 00000000..519b7049 --- /dev/null +++ b/.github/workflows/zed-cli-graph-github-production-pinned.yml @@ -0,0 +1,120 @@ +name: zed-cli graph github production pinned + +on: + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact zed-pkg/zed-cli branch, tag, or commit to certify + required: true + type: string + run_live_smoke: + description: Scan the live zed-pkg-test organization on Linux + required: true + default: false + type: boolean + repository_dispatch: + types: + - zed-cli-graph-github-candidate-pinned + +permissions: + contents: read + +concurrency: + group: zed-cli-graph-github-pinned-${{ inputs.zed_cli_ref || github.event.client_payload.zed_cli_ref || github.run_id }} + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + ZED_CLI_REF: ${{ inputs.zed_cli_ref || github.event.client_payload.zed_cli_ref }} + RUN_LIVE_SMOKE: ${{ inputs.run_live_smoke || github.event.client_payload.run_live_smoke || false }} + +jobs: + production-conformance: + name: ${{ matrix.os }} / Rust production port + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + + steps: + - name: Reject a missing candidate ref + shell: bash + run: | + set -euo pipefail + test -n "${ZED_CLI_REF}" + + - name: Check out test authority + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: e2e + persist-credentials: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + token: ${{ secrets.ZED_PKG_CROSS_ORG_READ_TOKEN || github.token }} + persist-credentials: false + + - name: Record immutable candidate head + id: candidate + shell: bash + working-directory: zed-cli + run: | + set -euo pipefail + sha="$(git rev-parse HEAD)" + [[ "${sha}" =~ ^[0-9a-f]{40}$ ]] + printf 'sha=%s\n' "${sha}" >> "${GITHUB_OUTPUT}" + test -z "$(git status --porcelain=v1)" + + - name: Install stable Rust with rustup + shell: bash + run: | + set -euo pipefail + rustup set profile minimal + rustup toolchain install stable --no-self-update + rustup default stable + rustc --version --verbose + cargo --version --verbose + + - name: Syntax-check bounded verifier + shell: bash + run: | + set -euo pipefail + python -m py_compile e2e/harness/verify_zed_cli_graph_github_production.py + + - name: Build and test complete candidate + shell: bash + run: | + set -euo pipefail + cargo test --manifest-path zed-cli/Cargo.toml --locked --all-targets --all-features + cargo build --manifest-path zed-cli/Cargo.toml --locked + + - name: Verify command contract and deterministic formats + shell: bash + env: + EXPECTED_CANDIDATE_SHA: ${{ steps.candidate.outputs.sha }} + ZED_PKG_GITHUB_TOKEN: ${{ secrets.ZED_PKG_LIVE_SMOKE_TOKEN || github.token }} + run: | + set -euo pipefail + python e2e/harness/verify_zed_cli_graph_github_production.py \ + --workspace zed-cli \ + --run-live-smoke "${RUN_LIVE_SMOKE}" \ + --live-org zed-pkg-test + + - name: Upload bounded failure evidence + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: zed-cli-graph-github-${{ runner.os }}-${{ steps.candidate.outputs.sha }} + path: .graph-github-evidence/** + if-no-files-found: ignore + retention-days: 3 + compression-level: 9 diff --git a/.github/workflows/zed-cli-graph-github-production.yml b/.github/workflows/zed-cli-graph-github-production.yml new file mode 100644 index 00000000..fb7d14bb --- /dev/null +++ b/.github/workflows/zed-cli-graph-github-production.yml @@ -0,0 +1,112 @@ +name: zed-cli graph github production + +on: + workflow_dispatch: + inputs: + zed_cli_ref: + description: Exact zed-pkg/zed-cli branch, tag, or commit to certify + required: true + type: string + run_live_smoke: + description: Scan the live zed-pkg-test organization on Linux + required: true + default: false + type: boolean + repository_dispatch: + types: + - zed-cli-graph-github-candidate + +permissions: + contents: read + +concurrency: + group: zed-cli-graph-github-${{ inputs.zed_cli_ref || github.event.client_payload.zed_cli_ref || github.run_id }} + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + ZED_CLI_REF: ${{ inputs.zed_cli_ref || github.event.client_payload.zed_cli_ref }} + RUN_LIVE_SMOKE: ${{ inputs.run_live_smoke || github.event.client_payload.run_live_smoke || false }} + +jobs: + production-conformance: + name: ${{ matrix.os }} / Rust production port + runs-on: ${{ matrix.os }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + + steps: + - name: Reject a missing candidate ref + shell: bash + run: | + set -euo pipefail + test -n "${ZED_CLI_REF}" + + - name: Check out test authority + uses: actions/checkout@v4 + with: + path: e2e + persist-credentials: false + + - name: Check out exact zed-cli candidate + uses: actions/checkout@v4 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_REF }} + path: zed-cli + token: ${{ secrets.ZED_PKG_CROSS_ORG_READ_TOKEN || github.token }} + persist-credentials: false + + - name: Record immutable candidate head + id: candidate + shell: bash + working-directory: zed-cli + run: | + set -euo pipefail + printf 'sha=%s\n' "$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" + git status --porcelain=v1 + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@stable + + - name: Restore Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: zed-cli + cache-on-failure: false + + - name: Build and test the complete candidate + shell: bash + run: | + set -euo pipefail + cargo test --manifest-path zed-cli/Cargo.toml --locked --all-targets --all-features + cargo build --manifest-path zed-cli/Cargo.toml --locked + + - name: Verify command contract and deterministic formats + shell: bash + env: + EXPECTED_CANDIDATE_SHA: ${{ steps.candidate.outputs.sha }} + ZED_PKG_GITHUB_TOKEN: ${{ secrets.ZED_PKG_LIVE_SMOKE_TOKEN || github.token }} + run: | + set -euo pipefail + python e2e/harness/verify_zed_cli_graph_github_production.py \ + --workspace zed-cli \ + --run-live-smoke "${RUN_LIVE_SMOKE}" \ + --live-org zed-pkg-test + + - name: Upload bounded failure evidence + if: failure() + uses: actions/upload-artifact@v4 + with: + name: zed-cli-graph-github-${{ runner.os }}-${{ steps.candidate.outputs.sha }} + path: | + .graph-github-evidence/** + if-no-files-found: ignore + retention-days: 3 + compression-level: 9 diff --git a/.github/workflows/zed-clients-polyglot.yml b/.github/workflows/zed-clients-polyglot.yml new file mode 100644 index 00000000..ce2cefdf --- /dev/null +++ b/.github/workflows/zed-clients-polyglot.yml @@ -0,0 +1,36 @@ +name: Zed Clients polyglot SDK matrix + +on: + pull_request: + paths: + - .github/workflows/reusable-polyglot-sdk-matrix.yml + - .github/workflows/zed-clients-polyglot.yml + workflow_dispatch: + inputs: + client_ref: + description: zed-clients git ref to validate + required: true + default: main + type: string + repository_dispatch: + types: [client-sdk-updated] + schedule: + - cron: "23 7 * * *" + +permissions: + contents: read + +concurrency: + group: zed-clients-polyglot-sdk-${{ github.event.pull_request.number || inputs.client_ref || github.event.client_payload.client_ref || 'main' }} + cancel-in-progress: true + +jobs: + sdk: + uses: ./.github/workflows/reusable-polyglot-sdk-matrix.yml + with: + client_repository: zed-pkg/zed-clients + client_ref: ${{ inputs.client_ref || github.event.client_payload.client_ref || 'main' }} + interfaces_repository: zed-pkg/zed-interfaces + interfaces_ref: main + secrets: + CLIENT_SDK_READ_TOKEN: ${{ secrets.CLIENT_SDK_READ_TOKEN }} diff --git a/.github/workflows/zedignore-manifest-union.yml b/.github/workflows/zedignore-manifest-union.yml new file mode 100644 index 00000000..6a2cb455 --- /dev/null +++ b/.github/workflows/zedignore-manifest-union.yml @@ -0,0 +1,276 @@ +name: zedignore and manifest publish-exclude union + +on: + pull_request: + paths: + - .github/workflows/zedignore-manifest-union.yml + - docs/zedignore-manifest-union.md + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: zedignore-manifest-union-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ZED_CLI_COMMIT: ca42115744b0d6d1ee18045b0e480e17a699af6f + ZED_CLI_BASE: 1ab18fcb2ff884e82af4cac4513d7b983a23c84a + +jobs: + certify: + name: publish-ignore union / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-15, windows-2025] + + steps: + - name: Check out independent certification repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + show-progress: false + + - name: Check out exact product candidate and history + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: zed-pkg/zed-cli + ref: ${{ env.ZED_CLI_COMMIT }} + path: product/zed-cli + fetch-depth: 0 + persist-credentials: false + show-progress: false + + - name: Require immutable base ancestry and exact two-file delta + shell: bash + run: | + set -euo pipefail + [[ "$ZED_CLI_COMMIT" =~ ^[0-9a-f]{40}$ ]] + [[ "$ZED_CLI_BASE" =~ ^[0-9a-f]{40}$ ]] + test "$(git -C product/zed-cli rev-parse HEAD)" = "$ZED_CLI_COMMIT" + git -C product/zed-cli merge-base --is-ancestor "$ZED_CLI_BASE" "$ZED_CLI_COMMIT" + changed="$(git -C product/zed-cli diff --name-only "$ZED_CLI_BASE" "$ZED_CLI_COMMIT")" + expected=$'docs/publish-ignore.md\nsrc/ops_entry.rs' + test "$changed" = "$expected" + git -C product/zed-cli diff --check "$ZED_CLI_BASE" "$ZED_CLI_COMMIT" + + - name: Install pinned Rust toolchain + uses: dtolnay/rust-toolchain@4be7066ada62dd38de10e7b70166bc74ed198c30 + with: + toolchain: stable + components: rustfmt,clippy + + - name: Install Python 3.12 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v6.0.0 + with: + python-version: "3.12" + check-latest: false + + - name: Run exact-head Rust quality gates + shell: bash + run: | + set -euo pipefail + cargo fmt --manifest-path product/zed-cli/Cargo.toml --all --check + cargo test --manifest-path product/zed-cli/Cargo.toml --locked --lib ops::publish_ignore_tests + cargo clippy --manifest-path product/zed-cli/Cargo.toml --locked --lib --bin zed -- -D warnings + cargo build --manifest-path product/zed-cli/Cargo.toml --locked --release --bin zed + + - name: Run three-source-state archive and warning contract + shell: bash + env: + RUNNER_OS_NAME: ${{ runner.os }} + RUNNER_ARCH_NAME: ${{ runner.arch }} + run: | + set -euo pipefail + mkdir -p artifacts + python - <<'PY' + import hashlib + import json + import os + import pathlib + import subprocess + import tarfile + import tempfile + + candidate = os.environ["ZED_CLI_COMMIT"] + base = os.environ["ZED_CLI_BASE"] + workspace = pathlib.Path.cwd() + binary = workspace / "product" / "zed-cli" / "target" / "release" / ("zed.exe" if os.name == "nt" else "zed") + assert binary.is_file(), binary + + sensitive = ( + "TOKEN", "SECRET", "PASSWORD", "PRIVATE_KEY", "ACCESS_KEY", + "API_KEY", "AUTHORIZATION", "COOKIE", "CREDENTIAL", + ) + + def clean_environment(home): + environment = { + key: value for key, value in os.environ.items() + if not key.upper().startswith("ZED_PKG_") + and not any(marker in key.upper() for marker in sensitive) + } + environment["ZED_PKG_HOME"] = str(home) + environment["ZED_PKG_REGISTRY"] = "http://127.0.0.1:9" + environment["GIT_TERMINAL_PROMPT"] = "0" + return environment + + def write_manifest(project, exclusions): + rendered = ", ".join(json.dumps(rule) for rule in exclusions) + (project / ".zpkg.toml").write_text( + "\n".join([ + "[package]", + 'org = "acme"', + 'name = "ignore-fixture"', + 'version = "1.0.0"', + 'description = "publish ignore conformance fixture"', + 'license = "MIT"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://example.invalid/acme/ignore-fixture"', + "", + "[publish]", + f"exclude = [{rendered}]", + "", + ]), + encoding="utf-8", + ) + + def invoke_pack(root, name, exclusions, zedignore): + project = root / name + project.mkdir() + write_manifest(project, exclusions) + (project / "LICENSE").write_text("MIT\n", encoding="utf-8") + (project / "visible.txt").write_text("visible\n", encoding="utf-8") + (project / ".hidden").write_text("intentional hidden data\n", encoding="utf-8") + (project / ".manifest-only").mkdir() + (project / ".manifest-only" / "private.txt").write_text("manifest excluded\n", encoding="utf-8") + (project / ".zed-only").mkdir() + (project / ".zed-only" / "private.txt").write_text("zedignore excluded\n", encoding="utf-8") + (project / "target").mkdir() + (project / "target" / "keep.txt").write_text("re-included\n", encoding="utf-8") + if zedignore is not None: + (project / ".zedignore").write_text(zedignore, encoding="utf-8") + + out = root / f"{name}-out" + home = root / f"{name}-home" + result = subprocess.run( + [str(binary), "pack", "--out", str(out)], + cwd=project, + env=clean_environment(home), + text=True, + capture_output=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, (result.stdout, result.stderr) + archive = out / "acme-ignore-fixture-1.0.0.tar.gz" + assert archive.is_file(), archive + with tarfile.open(archive, mode="r:gz") as packaged: + names = set(packaged.getnames()) + return result, names, archive + + temp = pathlib.Path(os.environ.get("RUNNER_TEMP") or tempfile.gettempdir()) + root = pathlib.Path(tempfile.mkdtemp(prefix="zedignore-union-", dir=temp)) + + conflict, names, archive = invoke_pack( + root, + "conflict", + [".manifest-only/**", "target/**", ".zpkg.toml", "LICENSE*"], + "# file-level publish policy\n.zed-only/**\n!target\n", + ) + assert conflict.stderr.count("both .zpkg.toml [publish].exclude") == 1, conflict.stderr + assert conflict.stderr.count("conflicting publish-ignore rules") == 1, conflict.stderr + assert "manifest first, .zedignore second" in conflict.stderr + assert "the later .zedignore rule wins" in conflict.stderr + assert "`target`" in conflict.stderr + required = { + "pkg/.zpkg.toml", + "pkg/LICENSE", + "pkg/visible.txt", + "pkg/.hidden", + "pkg/target/keep.txt", + } + assert required <= names, (required - names, sorted(names)) + forbidden = { + "pkg/.zedignore", + "pkg/.manifest-only/private.txt", + "pkg/.zed-only/private.txt", + } + assert names.isdisjoint(forbidden), sorted(names & forbidden) + + duplicate, _, _ = invoke_pack( + root, + "same-polarity", + [".cache/**"], + "**/.CACHE/**\n", + ) + assert duplicate.stderr.count("both .zpkg.toml [publish].exclude") == 1, duplicate.stderr + assert "conflicting publish-ignore rules" not in duplicate.stderr + + manifest_only, _, _ = invoke_pack( + root, + "manifest-only", + [".manifest-only/**"], + None, + ) + assert "both .zpkg.toml [publish].exclude" not in manifest_only.stderr + assert "conflicting publish-ignore rules" not in manifest_only.stderr + + file_only, _, _ = invoke_pack( + root, + "file-only", + [], + ".zed-only/**\n", + ) + assert "both .zpkg.toml [publish].exclude" not in file_only.stderr + assert "conflicting publish-ignore rules" not in file_only.stderr + + evidence = { + "$schema": "zed-pkg-test/zedignore-manifest-union/v1", + "candidate": candidate, + "base": base, + "runner": { + "os": os.environ["RUNNER_OS_NAME"], + "arch": os.environ["RUNNER_ARCH_NAME"], + }, + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "conflict_archive_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(), + "assertions": { + "archive_members_checked": True, + "both_sources_warning_once": True, + "conflict_warning_once": True, + "same_polarity_duplicate_quiet": True, + "single_source_quiet": True, + "hidden_file_not_blanket_excluded": True, + "always_include_preserved": True, + "later_zedignore_negation_wins": True, + }, + "result": "passed", + } + output = workspace / "artifacts" / f"zedignore-union-{os.environ['RUNNER_OS_NAME']}-{os.environ['RUNNER_ARCH_NAME']}.json" + output.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8") + PY + + - name: Require immutable product and harness source + shell: bash + run: | + set -euo pipefail + git -C product/zed-cli diff --exit-code + git -C product/zed-cli diff --cached --exit-code + status="$(git status --porcelain --untracked-files=all | grep -vE '^\?\? (artifacts|product)/' || true)" + test -z "$status" + + - name: Upload commit-addressed evidence + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: zedignore-manifest-union-${{ runner.os }}-${{ runner.arch }}-${{ env.ZED_CLI_COMMIT }} + path: artifacts + if-no-files-found: error + retention-days: 14 + include-hidden-files: false diff --git a/.gitignore b/.gitignore index 5b5bafbd..ecc23bf1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,12 @@ node_modules/ +__pycache__/ +*.py[cod] playwright-report/ test-results/ .stack/ .zed/ +tmp +temp +tmp/worktrees +temp/worktrees +/scripts/.audit-test-org-fleet-live-*.mjs diff --git a/.ores-lint/README.md b/.ores-lint/README.md new file mode 100644 index 00000000..6fe9cd41 --- /dev/null +++ b/.ores-lint/README.md @@ -0,0 +1,218 @@ +# ores-lint + +A vendored, dependency-free lint baseline for every JavaScript/TypeScript, +Rust, Dart/Flutter and Gleam repo in the org fleet. Everything it needs is in +this directory — there is nothing to install from a registry and nothing to +keep in version sync. + +Only universally accepted linters are used as hosts: + +| language | host linter | custom house rules | +|---|---|---| +| TypeScript / JS | ESLint 9+ (flat config) | `ores/require-send`, `ores/semi` | +| Rust | clippy | `implicit_return` house style; `#[must_use]` on log `Event`; require-send scanner | +| Dart / Flutter | `dart analyze` | require-send scanner | +| Gleam | `gleam format --check` + `gleam check` | require-send scanner | + +There is no ESLint-quality plugin host for Gleam, and Dart's `custom_lint` / +Rust's `dylint` would pull registry packages into every repo. The one house +rule that those hosts cannot express — **logger chains must end in `send()`** — +is implemented as a vendored ESLint rule for TS and a small Node scanner for +the other three languages. + +## Running it + +```sh +sh .ores-lint/lint.sh # lints whatever this repo contains +sh .ores-lint/selftest.sh # verifies the toolkit still works after a toolchain upgrade +``` + +For JS repos it is also wired into `npm run lint:ores`, and runs automatically +before `npm run build` (`prebuild`) and before `npm publish` (`prepublishOnly`). + +## What it enforces + +**JavaScript / TypeScript** — via ESLint (flat config): + +| rule | why | +|---|---| +| `semi` | house style: semicolons are required, missing ones warn | +| `ores/require-send` | a logging chain that reaches `.info()`/`.warn()`/… but never calls `.send()` or `.send(boolean)` builds an event that is never delivered | +| correctness set | `eqeqeq`, `no-unreachable`, `no-dupe-keys`, `use-isnan`, `valid-typeof`, `no-async-promise-executor`, and similar low-false-positive checks | + +**Rust** — via clippy, plus rustc `#[must_use]` on ores-otel `Event`: + +| lint | why | +|---|---| +| `clippy::implicit_return` | house style: prefer an explicit `return` at tail position | +| `clippy::correctness`, `clippy::suspicious` | real defects | +| `unwrap_used`, `expect_used`, `panic_in_result_fn`, `todo`, `dbg_macro` | things that should not reach a publish | + +**Dart / Flutter** — via `dart analyze` (or `flutter analyze`). If the repo has +no `analysis_options.yaml`, rollout drops a baseline of analyzer-shipped +correctness lints. Existing files are never overwritten. Prefer +`package:lints` / `package:flutter_lints` when the package already depends on +them — those are the Dart equivalents of `eslint:recommended`. + +**Gleam** — via `gleam format --check` and `gleam check`. Unused values already +catch many forgotten `send` calls; the scanner catches the rest (assigned +events, unfinished pipes). + +**require-send (all four languages).** The ores-otel logger builds an event +through method chaining (TS/Rust/Dart) or pipes (Gleam). Delivery is +`send()`, `send(boolean)`, or `send_with_store(...)`. Forgetting that call +means the event is built and then dropped — unless shutdown recovers it, which +is a fallback, not the API. Tests that deliberately build unsent events are +skipped unless `ORES_LINT_REQUIRE_SEND_INCLUDE_TESTS=1`. + +## Overriding a finding on one line + +This is expected. Some tests, shutdown-recovery fixtures, and rare control +flow should not call `send()`. Use the comment that matches the host linter, +or the unified ores-lint form which works in every language: + +``` +// ores-lint-disable-next-line require-send +// ores-lint-disable-line require-send +// ores-lint-disable-file require-send +``` + +Language-native equivalents, also honoured: + +| language | one-line override | +|---|---| +| TypeScript | `// eslint-disable-next-line ores/require-send` | +| Rust (`#[must_use]`) | `#[allow(unused_must_use)]` on the next statement | +| Dart analyzer | `// ignore: name_of_lint` (analyzer rules only; require-send uses the ores-lint form) | +| Gleam | the ores-lint form above | + +Do not disable the rule for a whole file unless the file is a generated +fixture. Prefer the next-line form. + +## The two things worth knowing + + +**1. Implicit-return findings are capped.** `clippy::implicit_return` fires once +per implicit return, which on a real crate is hundreds of identical lines. The +lint stays fully enabled so nothing is missed, but `rust.sh` collapses it into a +single warning showing at most 5 concrete locations plus `... and N more`. The +same cap applies to every ESLint rule via `eslint/formatter.mjs`. Change it with +`ORES_LINT_MAX_EXAMPLES`. + +**2. `clippy::needless_return` had to be disabled.** It ships enabled in +clippy's default `style` group and warns on exactly the explicit returns this +house style asks for. Enabling `implicit_return` without allowing +`needless_return` makes the two lints contradict each other on every function in +the crate. `selftest.sh` asserts this stays true. + +## Scope: sub-projects and repo boundaries + +The linter does **not** assume the repo root is the only project. + +- **Rust** — `rust.sh` finds every crate in the repo, including ones under + `apps/` or `clients/`. Crates that are workspace members of an already-linted + root are skipped (via `cargo metadata --no-deps`) so nothing is linted twice, + and findings from every crate are aggregated into **one** capped report. +- **JS/TS** — a flat config at the repo root makes `eslint .` reach nested + packages, so the config goes in even when the JS lives in a subdirectory. + +**Nested git repositories are a hard boundary.** A repo checked out inside +another repo gets its own ores-lint install; the parent must not lint it, or the +same findings get reported twice under the wrong repo name and the same +`package.json` gets wired with two conflicting relative paths. `rollout.mjs` +records those boundaries in `.ores-lint/nested-repos.json`, and both halves of +the linter read it. + +To exclude a repo entirely — vendored upstream forks, for instance — drop an +empty `.ores-lint-ignore` file at its root. + +## Legacy config migration + +ESLint 9+ reads flat config **only**. Three older mechanisms are silently +ignored, which means any repo still relying on them has not been linted at all: + +| legacy mechanism | status | +|---|---| +| `.eslintrc*` | ignored entirely; rules are dead | +| `eslintConfig` key in `package.json` | ignored entirely | +| `.eslintignore` | ignored, with a warning | + +`audit.mjs` reports every repo in each category. `.eslintignore` is ported +automatically into flat-config `ignores` by `base.mjs` (gitignore semantics +preserved), so its intent keeps applying. The other two need a human decision +and are migrated per repo — porting the rules that still make sense, and saying +in a comment which ones were dropped and why. + +## Warn-only, by design + +`lint.sh` exits 0 no matter what it finds. It is wired into build and publish +hooks across hundreds of repos, so it is built to be incapable of breaking one +unless a human opts in. + +To make findings blocking for a single repo, create `.ores-lint/local.sh`: + +```sh +ORES_LINT_STRICT=1 +``` + +`local.sh` is yours — the rollout script never overwrites it. Everything else in +this directory is managed and will be replaced on the next rollout. + +## Knobs + +| variable | default | meaning | +|---|---|---| +| `ORES_LINT_MAX_EXAMPLES` | `5` | example locations shown per rule | +| `ORES_LINT_STRICT` | `0` | `1` makes any finding exit non-zero | +| `ORES_LINT_SKIP_JS` / `ORES_LINT_SKIP_RUST` / `ORES_LINT_SKIP_DART` / `ORES_LINT_SKIP_GLEAM` | `0` | skip one language host | +| `ORES_LINT_SKIP_REQUIRE_SEND` | `0` | skip the cross-language send() scanner | +| `ORES_LINT_REQUIRE_SEND_INCLUDE_TESTS` | `0` | `1` also scans `test/` / `*_test.dart` / etc. | +| `ORES_LINT_RUST_ALL_TARGETS` | `0` | `1` also lints tests/benches/examples | +| `ORES_LINT_RUST_EXTRA` | — | extra flags appended to the clippy invocation | + +## Graceful degradation + +Nothing here is allowed to fail loudly for an environmental reason. ESLint not +installed, too old, clippy not installed, dart/flutter not installed, gleam not +installed, no TypeScript parser available, crate deps not fetchable — each is +reported as an actionable skip, not an error. +Repo-specific ESLint config that already existed is never overwritten. +The same is true of `analysis_options.yaml`. + +CI follows the same model: the workflow runs `npm i -g eslint typescript-eslint` +and never runs `npm install` for the repo itself, so linting a PR does not +require the repo's dependency tree to resolve. + +## Per-repo customisation + +`eslint.config.mjs` at the repo root takes options: + +```js +export default await oresConfig({ + requireSend: { loggerNames: ['myLogger'], terminalMethods: ['send', 'flush'] }, + rules: { 'no-console': 'warn' }, + ignores: ['**/generated/**'], +}); +``` + +Once you edit that file the rollout script leaves it alone. + +--- + +## Fleet operations (from the `codes` directory) + +```sh +node .ores-lint-toolkit/audit.mjs # report the fleet's lint posture +node .ores-lint-toolkit/audit.mjs --json out.json # ...as machine-readable data +node .ores-lint-toolkit/rollout.mjs --dry-run # preview +node .ores-lint-toolkit/rollout.mjs # install / re-install everywhere +node .ores-lint-toolkit/rollout.mjs --only ores-otel +node .ores-lint-toolkit/rollout.mjs --shard 0/8 # one slice of a fleet-wide run +node .ores-lint-toolkit/verify.mjs # assert every repo is correctly installed +``` + +A full rollout over ~900 repos takes a few minutes. `--shard k/n` splits it into +bounded chunks, which matters when the runner has a per-command time limit. + +Re-run the rollout after editing anything in `.ores-lint-toolkit/` — it is +idempotent and propagates the change to every repo. diff --git a/.ores-lint/VERSION b/.ores-lint/VERSION new file mode 100644 index 00000000..f0bb29e7 --- /dev/null +++ b/.ores-lint/VERSION @@ -0,0 +1 @@ +1.3.0 diff --git a/.ores-lint/config.sh b/.ores-lint/config.sh new file mode 100644 index 00000000..b9205726 --- /dev/null +++ b/.ores-lint/config.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# ores-lint shared configuration. Sourced by lint.sh, js.sh and rust.sh. +# Every value can be overridden from the environment, or per repo in local.sh. + +# Maximum number of concrete example locations shown for any one rule. +: "${ORES_LINT_MAX_EXAMPLES:=5}" + +# Warn-only by default: lint.sh exits 0 no matter what it finds. +# Flip to 1 (per repo, or in CI) once a repo's debt is paid down. +: "${ORES_LINT_STRICT:=0}" + +: "${ORES_LINT_SKIP_RUST:=0}" +: "${ORES_LINT_SKIP_JS:=0}" +: "${ORES_LINT_SKIP_DART:=0}" +: "${ORES_LINT_SKIP_GLEAM:=0}" +: "${ORES_LINT_SKIP_REQUIRE_SEND:=0}" +: "${ORES_LINT_REQUIRE_SEND_INCLUDE_TESTS:=0}" + +# How deep to search for nested sub-projects (crates and packages). Repos here +# routinely hold crates under apps/ and clients/ that a root-only lint misses. +: "${ORES_LINT_DEPTH:=5}" + +# Include tests/benches/examples in the Rust pass. Off by default so the +# pre-publish signal is about shipped code. +: "${ORES_LINT_RUST_ALL_TARGETS:=0}" + +# Minimum ESLint major version. Flat config needs 9+. ESLint is expected to be +# installed GLOBALLY, once - see required-tools.json. Nothing is ever installed +# into a repo's node_modules. +: "${ORES_LINT_ESLINT_MIN_MAJOR:=9}" + +# The exact clippy diagnostic text for `clippy::implicit_return`. selftest.sh +# verifies this still matches, so a future clippy rewording surfaces as a test +# failure rather than as a silently empty report. +: "${ORES_LINT_IMPLICIT_RETURN_MSG:=missing \`return\` statement}" + +export ORES_LINT_MAX_EXAMPLES ORES_LINT_STRICT ORES_LINT_SKIP_RUST ORES_LINT_SKIP_JS +export ORES_LINT_SKIP_DART ORES_LINT_SKIP_GLEAM ORES_LINT_SKIP_REQUIRE_SEND +export ORES_LINT_REQUIRE_SEND_INCLUDE_TESTS +export ORES_LINT_DEPTH ORES_LINT_RUST_ALL_TARGETS ORES_LINT_ESLINT_MIN_MAJOR +export ORES_LINT_IMPLICIT_RETURN_MSG + +# Repo-local overrides, never overwritten by the rollout script. Sourced last so +# anything set here wins. +ORES_LINT_CFG_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +[ -f "$ORES_LINT_CFG_DIR/local.sh" ] && . "$ORES_LINT_CFG_DIR/local.sh" diff --git a/.ores-lint/dart.sh b/.ores-lint/dart.sh new file mode 100755 index 00000000..bc3c7ec7 --- /dev/null +++ b/.ores-lint/dart.sh @@ -0,0 +1,114 @@ +#!/bin/sh +# ores-lint :: Dart / Flutter +# +# Uses the Dart analyzer (`dart analyze` / `flutter analyze`) - the universally +# accepted linter for the language, equivalent to ESLint for TypeScript. Custom +# house rules that the analyzer cannot express (require-send) live in +# require-send.mjs and run from lint.sh. +# +# Nothing is installed. Missing dart/flutter is an actionable skip. + +set -u +DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$DIR/config.sh" +ROOT=${1:-.} +ROOT=$(CDPATH= cd -- "$ROOT" && pwd) + +[ "${ORES_LINT_SKIP_DART}" = "1" ] && { echo "ores-lint[dart]: skipped (ORES_LINT_SKIP_DART=1)"; exit 0; } + +has_dart=0 +if find "$ROOT" -maxdepth "$ORES_LINT_DEPTH" \ + \( -name node_modules -o -name target -o -name .git -o -name vendor -o -name .vendor -o -name build \) -prune -o \ + \( -type f -name pubspec.yaml -o -type f -name '*.dart' \) -print 2>/dev/null | head -1 | grep -q .; then + has_dart=1 +fi +[ "$has_dart" = "0" ] && exit 0 + +ANALYZE="" +if command -v dart >/dev/null 2>&1; then + ANALYZE="dart analyze" +elif command -v flutter >/dev/null 2>&1; then + ANALYZE="flutter analyze" +else + echo "ores-lint[dart]: dart/flutter not found on PATH - skipping" + echo " install the Dart SDK, or Flutter, then re-run" + exit 0 +fi + +# Nested git repos are someone else's analyzer run. +NESTED_FILE="$DIR/nested-repos.json" +EXCLUDE="" +if [ -f "$NESTED_FILE" ]; then + NESTED=$(grep -o '"[^"]*"' "$NESTED_FILE" 2>/dev/null | tr -d '"') + for nrepo in $NESTED; do + [ -n "$nrepo" ] && EXCLUDE="$EXCLUDE --fatal-infos=false" + done +fi + +OUT=$(mktemp) || exit 0 +RC=0 +( cd "$ROOT" && $ANALYZE --format=machine 2>/dev/null || $ANALYZE ) >"$OUT" 2>&1 || RC=$? + +if grep -q 'No issues found!' "$OUT"; then + echo "ores-lint[dart]: clean" + rm -f "$OUT" + exit 0 +fi + +if [ "$RC" -ne 0 ] && ! grep -qE 'error|warning|info|ERROR|WARNING' "$OUT"; then + echo "ores-lint[dart]: analyzer could not run in $ROOT (exit $RC). First lines:" + sed -n '1,6p' "$OUT" | sed 's/^/ | /' + rm -f "$OUT" + exit 0 +fi + +awk -v MAXEX="$ORES_LINT_MAX_EXAMPLES" ' +BEGIN { max = MAXEX + 0; if (max < 1) max = 1; n = 0; FS = "|" } +# machine format: SEVERITY|TYPE|FILE|LINE|COLUMN|LENGTH|CODE|MESSAGE +NF >= 8 && $1 ~ /^(ERROR|WARNING|INFO)$/ { + loc = $3 ":" $4 ":" $5 + msg = $7 ": " $8 + sev = tolower($1) + if (sev == "info") sev = "warning" + key = loc "|" msg + if (key in seen) next + seen[key] = 1 + if (!(msg in count)) { order[++n] = msg; sev_of[msg] = sev } + count[msg]++ + if (shown[msg] < max) { ex[msg] = ex[msg] (shown[msg]++ ? "\n" : "") " " loc } + next +} +# human format fallback: " warning - path:line:col - message - code" +{ + line = $0 + if (match(line, /(error|warning|info) • /) || match(line, /(error|warning|info) - /)) { + n++ + raw[++human] = line + } +} +END { + if (n == 0 && human == 0) { print "ores-lint[dart]: clean"; exit 0 } + if (n == 0) { + printf "ores-lint[dart]: %d finding(s)\n", human + limit = (human < max ? human : max) + for (i = 1; i <= limit; i++) print " " raw[i] + if (human > max) printf " ... and %d more\n", human - max + print "" + exit 0 + } + total = 0 + for (i = 1; i <= n; i++) total += count[order[i]] + printf "ores-lint[dart]: %d finding(s) across %d rule(s)\n", total, n + for (i = 1; i <= n; i++) { + msg = order[i] + printf "\n %s: %s\n", sev_of[msg], msg + printf " %d instance(s); showing %d:\n", count[msg], (count[msg] < max ? count[msg] : max) + print ex[msg] + if (count[msg] > max) printf " ... and %d more\n", count[msg] - max + } + print "" +} +' "$OUT" + +rm -f "$OUT" +exit 0 diff --git a/.ores-lint/eslint/base.mjs b/.ores-lint/eslint/base.mjs new file mode 100644 index 00000000..596f2f26 --- /dev/null +++ b/.ores-lint/eslint/base.mjs @@ -0,0 +1,179 @@ +/** + * ores-lint :: shared flat-config factory + * + * Everything here degrades gracefully. A repo missing TypeScript tooling gets + * its JS linted rather than an error; an ESLint that has dropped core `semi` + * falls back to the vendored rule. The point is that a lint config rolled out + * to hundreds of heterogeneous repos must never be the thing that breaks them. + */ + +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; +import oresPlugin from './plugin.mjs'; + +const require_ = createRequire(import.meta.url); + +// ESLint and its optional TypeScript parser are expected to live in a single +// global install rather than in every repo's node_modules, so resolution has to +// look outside this file's own tree. js.sh passes the global root in. +const EXTRA_PATHS = [ + process.env.ORES_LINT_GLOBAL_ROOT, + ...(process.env.NODE_PATH ? process.env.NODE_PATH.split(':') : []), +].filter(Boolean); + +function tryResolve(id) { + try { return require_.resolve(id); } catch { /* try the global root next */ } + if (EXTRA_PATHS.length) { + try { return require_.resolve(id, { paths: EXTRA_PATHS }); } catch { /* not installed */ } + } + return null; +} + +/** Is core `semi` still shipped by the installed ESLint? */ +async function coreSemiAvailable() { + for (const id of ['eslint/use-at-your-own-risk']) { + const resolved = tryResolve(id); + if (!resolved) continue; + try { + const { builtinRules } = await import(pathToFileURL(resolved).href); + return builtinRules.has('semi'); + } catch { /* fall through */ } + } + return true; // could not introspect: assume core rules are intact +} + +/** typescript-eslint, if the repo happens to have it. */ +async function loadTsSupport() { + for (const id of ['typescript-eslint', '@typescript-eslint/parser']) { + const resolved = tryResolve(id); + if (!resolved) continue; + try { + // Import by resolved path: a bare specifier would not find a global install. + const mod = await import(pathToFileURL(resolved).href); + const m = mod.default || mod; + if (id === 'typescript-eslint' && m.parser) return { parser: m.parser, source: id }; + if (m.parseForESLint || m.parse) return { parser: m, source: id }; + } catch { /* fall through to the next candidate */ } + } + return null; +} + +const JS_FILES = ['**/*.js', '**/*.mjs', '**/*.cjs', '**/*.jsx']; +const TS_FILES = ['**/*.ts', '**/*.mts', '**/*.cts', '**/*.tsx']; + +/** + * Directories that are separate git repositories nested inside this one. They + * have their own ores-lint install and must not be linted from here, or their + * findings would be reported twice under the wrong repo. + */ +/** + * ESLint 10 dropped support for `.eslintignore` and merely warns that it is + * being ignored. Rather than let a repo's stated intent silently stop applying, + * port it into flat-config `ignores`. + */ +function legacyIgnoreFile() { + try { + const raw = readFileSync(new URL('../../.eslintignore', import.meta.url), 'utf8'); + return raw.split('\n') + .map((l) => l.trim()) + .filter((l) => l && !l.startsWith('#') && !l.startsWith('!')) + // .eslintignore used gitignore semantics: a bare name matched anywhere. + .map((l) => (l.includes('/') ? l.replace(/^\/+/, '') : `**/${l}`)) + .map((l) => (l.endsWith('/') ? `${l}**` : l)); + } catch { return []; } +} + +function nestedRepoIgnores() { + try { + const raw = readFileSync(new URL('../nested-repos.json', import.meta.url), 'utf8'); + const dirs = JSON.parse(raw); + return Array.isArray(dirs) ? dirs.map((d) => `${d}/**`) : []; + } catch { return []; } +} + +const IGNORES = [ + '**/node_modules/**', '**/dist/**', '**/build/**', '**/out/**', '**/target/**', + '**/coverage/**', '**/.next/**', '**/vendor/**', '**/*.min.js', '**/*.bundle.js', + '**/.ores-lint/**', +]; + +/** + * @param {object} [opts] + * @param {object} [opts.requireSend] options forwarded to ores/require-send + * @param {object} [opts.rules] extra rules merged last (repo overrides) + * @param {string[]} [opts.ignores] extra ignore globs + */ +export default async function oresConfig(opts = {}) { + const useCoreSemi = await coreSemiAvailable(); + const ts = await loadTsSupport(); + + const semiRules = useCoreSemi + ? { semi: ['warn', 'always'], 'no-extra-semi': 'warn', 'semi-style': ['warn', 'last'] } + : { 'ores/semi': 'warn' }; + + // Correctness rules chosen for a near-zero false-positive rate, because this + // config lands in repos nobody is going to hand-tune afterwards. + const correctness = { + 'ores/require-send': ['warn', opts.requireSend || {}], + 'no-unused-vars': ['warn', { args: 'after-used', argsIgnorePattern: '^_', varsIgnorePattern: '^_', caughtErrors: 'none' }], + eqeqeq: ['warn', 'smart'], + 'no-fallthrough': 'warn', + 'no-unreachable': 'warn', + 'no-dupe-keys': 'warn', + 'no-dupe-else-if': 'warn', + 'no-duplicate-case': 'warn', + 'no-self-compare': 'warn', + 'no-unsafe-negation': 'warn', + 'no-cond-assign': ['warn', 'always'], + 'no-constant-condition': ['warn', { checkLoops: false }], + 'no-async-promise-executor': 'warn', + 'no-promise-executor-return': 'warn', + 'no-compare-neg-zero': 'warn', + 'no-irregular-whitespace': 'warn', + 'no-template-curly-in-string': 'warn', + 'valid-typeof': 'warn', + 'use-isnan': 'warn', + 'no-debugger': 'warn', + // Deliberately NOT enabled: no-undef. Without a `globals` package it fires + // on console/process/window everywhere, and TypeScript already covers it. + }; + + const configs = [ + { ignores: [...IGNORES, ...nestedRepoIgnores(), ...legacyIgnoreFile(), ...(opts.ignores || [])] }, + { + files: JS_FILES, + plugins: { ores: oresPlugin }, + languageOptions: { ecmaVersion: 'latest', sourceType: 'module' }, + rules: { ...semiRules, ...correctness, ...(opts.rules || {}) }, + }, + ]; + + if (!ts) { + // No TypeScript parser anywhere. Globally ignore TS rather than leaving it + // merely unmatched: a repo-specific config block that happens to match + // `**/*.ts` would otherwise hand TS source to the JS parser and produce a + // wall of bogus "Parsing error" findings. Ignoring is honest; js.sh prints + // a note so the gap stays visible instead of looking like a clean repo. + configs.push({ ignores: TS_FILES }); + } + + if (ts) { + configs.push({ + files: TS_FILES, + plugins: { ores: oresPlugin }, + languageOptions: { parser: ts.parser, ecmaVersion: 'latest', sourceType: 'module' }, + rules: { + ...semiRules, + ...correctness, + // TypeScript's own compiler reports unused symbols with better fidelity. + 'no-unused-vars': 'off', + ...(opts.rules || {}), + }, + }); + } + + return configs; +} + +export const meta = { tsSupport: null }; diff --git a/.ores-lint/eslint/formatter.mjs b/.ores-lint/eslint/formatter.mjs new file mode 100644 index 00000000..e4a9a52b --- /dev/null +++ b/.ores-lint/eslint/formatter.mjs @@ -0,0 +1,79 @@ +/** + * ores-lint :: capped ESLint formatter + * + * Same reporting contract as the Rust side: one block per rule, at most + * ORES_LINT_MAX_EXAMPLES concrete locations, then a count of the remainder. + * A rollout across hundreds of repos surfaces thousands of missing semicolons; + * printing every one of them buries the findings that actually matter. + */ + +const MAX = Math.max(1, Number(process.env.ORES_LINT_MAX_EXAMPLES || 5)); + +const LABELS = { + semi: 'missing semicolon (ores house style)', + 'ores/semi': 'missing semicolon (ores house style)', + 'ores/require-send': 'logging chain never delivered (ores custom rule)', +}; + +export default function oresFormatter(results) { + const byRule = new Map(); + let files = 0; + let errors = 0; + let warnings = 0; + const parseErrors = []; + + for (const result of results) { + if (!result.messages.length) continue; + files++; + const rel = (result.filePath || '').replace(`${process.cwd()}/`, ''); + for (const m of result.messages) { + if (m.severity === 2) errors++; else warnings++; + if (!m.ruleId) { parseErrors.push(`${rel}:${m.line || 0}: ${m.message}`); continue; } + let entry = byRule.get(m.ruleId); + if (!entry) { entry = { count: 0, examples: [], severity: m.severity, message: m.message }; byRule.set(m.ruleId, entry); } + entry.count++; + if (entry.examples.length < MAX) entry.examples.push(`${rel}:${m.line}:${m.column}`); + } + } + + // Report how many files were actually examined. "clean" and "nothing was + // linted" are otherwise indistinguishable, which makes a silent coverage gap + // look like a passing repo - the single most misleading thing a linter can do. + const examined = results.length; + if (!byRule.size && !parseErrors.length) { + return examined === 0 + ? 'ores-lint[js]: no lintable files matched (check ignores / file extensions)\n' + : `ores-lint[js]: clean (${examined} file${examined === 1 ? '' : 's'} linted)\n`; + } + + const out = []; + const total = errors + warnings; + out.push(`ores-lint[js]: ${total} finding(s) across ${byRule.size} rule(s) in ${files} of ${examined} file(s) linted`); + + // House rules first, then the rest by frequency. + const ordered = [...byRule.entries()].sort((a, b) => { + const ah = a[0] in LABELS ? 0 : 1; + const bh = b[0] in LABELS ? 0 : 1; + return ah - bh || b[1].count - a[1].count; + }); + + for (const [ruleId, entry] of ordered) { + const sev = entry.severity === 2 ? 'error' : 'warning'; + const label = LABELS[ruleId] || entry.message; + out.push(''); + out.push(` ${sev}: ${label} [${ruleId}]`); + out.push(` ${entry.count} instance(s); showing ${Math.min(entry.count, MAX)}:`); + for (const ex of entry.examples) out.push(` ${ex}`); + if (entry.count > MAX) out.push(` ... and ${entry.count - MAX} more`); + } + + if (parseErrors.length) { + out.push(''); + out.push(` note: ${parseErrors.length} file(s) could not be parsed (usually a missing parser, not a defect):`); + for (const p of parseErrors.slice(0, MAX)) out.push(` ${p}`); + if (parseErrors.length > MAX) out.push(` ... and ${parseErrors.length - MAX} more`); + } + + out.push(''); + return `${out.join('\n')}\n`; +} diff --git a/.ores-lint/eslint/plugin.mjs b/.ores-lint/eslint/plugin.mjs new file mode 100644 index 00000000..b010e4da --- /dev/null +++ b/.ores-lint/eslint/plugin.mjs @@ -0,0 +1,361 @@ +/** + * ores-lint :: vendored ESLint plugin + * + * Plain ESM. No build step, no registry dependencies. The only thing it needs + * is an `eslint` that already exists in the repo. + * + * Rules + * ores/require-send - a logging chain that reaches a level method must end + * in a terminal call (.send(), .send(true), ...). + * Generalised from the ores-otel next-loggers plugin. + * ores/semi - fallback semicolon rule, used only when core `semi` + * is unavailable. See base.mjs. + */ + +const DEFAULT_LEVEL_METHODS = ['trace', 'debug', 'info', 'log', 'warn', 'error', 'fatal']; +const DEFAULT_TERMINAL_METHODS = ['send', 'send_with_store']; + +const LOGGER_EXPORTS = new Set([ + 'logger', 'browserLogger', 'edgeLogger', 'cloudflareWorkerLogger', + 'nodeLogger', 'bunLogger', 'denoLogger', +]); +const FACTORY_EXPORTS = new Set([ + 'createLogger', 'createBrowserLogger', 'createEdgeLogger', + 'createCloudflareWorkerLogger', 'createNodeLogger', 'createBunLogger', + 'createDenoLogger', +]); +const CLASS_EXPORTS = new Set([ + 'BaseLogger', 'BrowserLogger', 'EdgeLogger', 'CloudflareWorkerLogger', + 'NodeLogger', 'BunLogger', 'DenoLogger', +]); + +function hasType(node, ...types) { + return Boolean(node && types.includes(String(node.type))); +} + +/** Strip wrappers that do not change the identity of the underlying expression. */ +function unwrap(node) { + let current = node; + while ( + current && + (hasType(current, 'ChainExpression', 'AwaitExpression', 'TSAsExpression', 'TSTypeAssertion', 'TSNonNullExpression') || + (current.type === 'UnaryExpression' && current.operator === 'void')) + ) { + current = current.expression || current.argument; + } + return current || undefined; +} + +function getPropertyName(node) { + if (!hasType(node, 'MemberExpression', 'OptionalMemberExpression')) return undefined; + const property = node.property; + if (!property) return undefined; + if (!node.computed && property.type === 'Identifier') return property.name; + if (node.computed && property.type === 'Literal' && typeof property.value === 'string') return property.value; + return undefined; +} + +function getQualifiedName(node) { + const current = unwrap(node); + if (!current) return undefined; + if (current.type === 'Identifier') return current.name; + if (current.type === 'ThisExpression') return 'this'; + if (hasType(current, 'MemberExpression', 'OptionalMemberExpression')) { + const objectName = getQualifiedName(current.object); + const propertyName = getPropertyName(current); + return objectName && propertyName ? `${objectName}.${propertyName}` : undefined; + } + return undefined; +} + +/** Walk a call chain back to its root, collecting method names left-to-right. */ +function collectCallChain(node, methods) { + const current = unwrap(node); + if (!current) return undefined; + if (hasType(current, 'CallExpression', 'OptionalCallExpression')) { + const callee = unwrap(current.callee); + if (callee && hasType(callee, 'MemberExpression', 'OptionalMemberExpression')) { + const root = collectCallChain(callee.object, methods); + const method = getPropertyName(callee); + if (method) methods.push(method); + return root; + } + return getQualifiedName(callee); + } + return getQualifiedName(current); +} + +function isTrackedModule(source, moduleNames) { + if (typeof source !== 'string') return false; + for (const moduleName of moduleNames) { + if (source === moduleName || source.startsWith(`${moduleName}/`)) return true; + } + return false; +} + +function inspectChain(node, knownLoggers, levelMethods, terminalMethods) { + const methods = []; + const root = collectCallChain(node, methods); + const levelIndex = methods.findIndex((method) => levelMethods.has(method)); + const delivered = levelIndex >= 0 && methods.slice(levelIndex + 1).some((method) => terminalMethods.has(method)); + const isEvent = Boolean(root && knownLoggers.has(root) && levelIndex >= 0); + return { methods, root, levelIndex, delivered, isEvent }; +} + +export const requireSendRule = { + meta: { + type: 'problem', + docs: { + description: 'require chainable logger events to call a terminal method such as send() or send(boolean)', + url: 'https://github.com/ores-otel/ores.otel.log', + }, + schema: [{ + type: 'object', + properties: { + loggerNames: { type: 'array', items: { type: 'string' }, uniqueItems: true }, + moduleNames: { type: 'array', items: { type: 'string' }, uniqueItems: true }, + levelMethods: { type: 'array', items: { type: 'string' }, uniqueItems: true }, + terminalMethods: { type: 'array', items: { type: 'string' }, uniqueItems: true }, + }, + additionalProperties: false, + }], + messages: { + missingSend: "Logging chain never calls {{terminal}} - this log event is built but never delivered. Override with // eslint-disable-next-line ores/require-send or // ores-lint-disable-next-line require-send", + }, + }, + + create(context) { + const options = context.options[0] || {}; + const knownLoggers = new Set(['log', 'logger', 'ddlog', ...(options.loggerNames || [])]); + const knownFactories = new Set(); + const knownClasses = new Set(); + const moduleNames = new Set(['@oresoftware/next-loggers', ...(options.moduleNames || [])]); + const levelMethods = new Set(options.levelMethods || DEFAULT_LEVEL_METHODS); + const terminalMethods = new Set(options.terminalMethods || DEFAULT_TERMINAL_METHODS); + const terminalLabel = [...terminalMethods].map((m) => `.${m}()`).join(' or '); + + const scopes = []; + const enterScope = () => scopes.push({ pending: new Map() }); + const exitScope = () => { + const scope = scopes.pop(); + if (!scope) return; + for (const node of scope.pending.values()) { + context.report({ node, messageId: 'missingSend', data: { terminal: terminalLabel } }); + } + }; + const markPending = (name, node) => { + if (!name || !scopes.length) return; + scopes[scopes.length - 1].pending.set(name, node); + }; + const clearPending = (name) => { + if (!name) return; + for (let i = scopes.length - 1; i >= 0; i--) { + if (scopes[i].pending.has(name)) { + scopes[i].pending.delete(name); + return; + } + } + }; + const isPending = (name) => { + if (!name) return false; + for (let i = scopes.length - 1; i >= 0; i--) { + if (scopes[i].pending.has(name)) return true; + } + return false; + }; + + const isLoggerProducer = (node) => { + const current = unwrap(node); + if (!current) return false; + const directName = getQualifiedName(current); + if (directName && knownLoggers.has(directName)) return true; + if (current.type === 'NewExpression') { + const className = getQualifiedName(current.callee); + return Boolean(className && knownClasses.has(className)); + } + if (hasType(current, 'CallExpression', 'OptionalCallExpression')) { + const calleeName = getQualifiedName(current.callee); + if (calleeName && knownFactories.has(calleeName)) return true; + const callee = unwrap(current.callee); + if (callee && hasType(callee, 'MemberExpression', 'OptionalMemberExpression')) { + const method = getPropertyName(callee); + const owner = getQualifiedName(callee.object); + return method === 'anew' && Boolean(owner && knownLoggers.has(owner)); + } + } + return false; + }; + + const consumeTerminalUse = (node) => { + const current = unwrap(node); + if (!current) return; + const chain = inspectChain(current, knownLoggers, levelMethods, terminalMethods); + if (chain.root && chain.methods.some((method) => terminalMethods.has(method))) { + clearPending(chain.root); + } + if (hasType(current, 'CallExpression', 'OptionalCallExpression')) { + for (const arg of current.arguments || []) { + const name = getQualifiedName(unwrap(arg)); + if (isPending(name)) clearPending(name); + } + } + }; + + const functionEnter = () => enterScope(); + const functionExit = () => exitScope(); + + return { + Program() { enterScope(); }, + 'Program:exit'() { exitScope(); }, + FunctionDeclaration: functionEnter, + 'FunctionDeclaration:exit': functionExit, + FunctionExpression: functionEnter, + 'FunctionExpression:exit': functionExit, + ArrowFunctionExpression(node) { + enterScope(); + if (node.body && node.body.type !== 'BlockStatement') { + const name = getQualifiedName(unwrap(node.body)); + if (isPending(name)) clearPending(name); + } + }, + 'ArrowFunctionExpression:exit': functionExit, + + ImportDeclaration(node) { + if (!isTrackedModule(node.source?.value, moduleNames)) return; + for (const specifier of node.specifiers || []) { + const localName = specifier.local?.name; + if (!localName) continue; + if (specifier.type === 'ImportDefaultSpecifier') { knownLoggers.add(localName); continue; } + if (specifier.type === 'ImportNamespaceSpecifier') { + for (const name of LOGGER_EXPORTS) knownLoggers.add(`${localName}.${name}`); + for (const name of FACTORY_EXPORTS) knownFactories.add(`${localName}.${name}`); + for (const name of CLASS_EXPORTS) knownClasses.add(`${localName}.${name}`); + continue; + } + const importedName = specifier.imported?.name || specifier.imported?.value; + if (typeof importedName !== 'string') continue; + if (LOGGER_EXPORTS.has(importedName)) knownLoggers.add(localName); + if (FACTORY_EXPORTS.has(importedName)) knownFactories.add(localName); + if (CLASS_EXPORTS.has(importedName)) knownClasses.add(localName); + } + }, + + VariableDeclarator(node) { + if (node.id?.type === 'Identifier' && node.id.name && isLoggerProducer(node.init)) { + knownLoggers.add(node.id.name); + } + if (node.id?.type !== 'Identifier' || !node.id.name) return; + const chain = inspectChain(node.init, knownLoggers, levelMethods, terminalMethods); + if (!chain.isEvent) return; + if (chain.delivered) return; + markPending(node.id.name, node); + }, + + AssignmentExpression(node) { + const assignedName = getQualifiedName(node.left); + if (assignedName && isLoggerProducer(node.right)) knownLoggers.add(assignedName); + const chain = inspectChain(node.right, knownLoggers, levelMethods, terminalMethods); + if (chain.isEvent && !chain.delivered && assignedName) { + markPending(assignedName, node); + return; + } + consumeTerminalUse(node.right); + }, + + ReturnStatement(node) { + if (!node.argument) return; + const name = getQualifiedName(unwrap(node.argument)); + if (isPending(name)) { + clearPending(name); + return; + } + const chain = inspectChain(node.argument, knownLoggers, levelMethods, terminalMethods); + if (chain.isEvent && !chain.delivered) return; + consumeTerminalUse(node.argument); + }, + + CallExpression(node) { + consumeTerminalUse(node); + }, + + ExpressionStatement(node) { + const chain = inspectChain(node.expression, knownLoggers, levelMethods, terminalMethods); + if (chain.isEvent && !chain.delivered) { + context.report({ node, messageId: 'missingSend', data: { terminal: terminalLabel } }); + return; + } + consumeTerminalUse(node.expression); + }, + }; + }, +}; + +/** + * Fallback semicolon rule. Only wired up when core `semi` is missing, so in + * practice this is dormant - it exists so the house style survives a future + * ESLint that drops its formatting rules. + */ +const NEEDS_SEMI = new Set([ + 'ExpressionStatement', 'ReturnStatement', 'ThrowStatement', 'BreakStatement', + 'ContinueStatement', 'DebuggerStatement', 'DoWhileStatement', 'ImportDeclaration', + 'ExportAllDeclaration', 'PropertyDefinition', 'TSTypeAliasDeclaration', + 'TSDeclareFunction', 'TSImportEqualsDeclaration', +]); + +export const semiRule = { + meta: { + type: 'layout', + fixable: 'code', + schema: [], + docs: { description: 'require semicolons at the end of statements (vendored fallback)' }, + messages: { missingSemi: 'Missing semicolon.' }, + }, + create(context) { + const sourceCode = context.sourceCode || context.getSourceCode(); + + const check = (node) => { + const lastToken = sourceCode.getLastToken(node); + if (!lastToken) return; + if (lastToken.type === 'Punctuator' && lastToken.value === ';') return; + context.report({ + node, + loc: lastToken.loc.end, + messageId: 'missingSemi', + fix: (fixer) => fixer.insertTextAfter(lastToken, ';'), + }); + }; + + const handlers = {}; + for (const type of NEEDS_SEMI) handlers[type] = check; + + // `for (let i = 0 ...)` heads and `for (const x of y)` must not get one. + handlers.VariableDeclaration = (node) => { + const parent = node.parent; + if (parent && ( + (parent.type === 'ForStatement' && parent.init === node) || + ((parent.type === 'ForInStatement' || parent.type === 'ForOfStatement') && parent.left === node) + )) return; + check(node); + }; + + // Only export forms that are expressions/re-exports need a semicolon; + // `export function f() {}` and `export class C {}` do not. + const checkExport = (node) => { + const decl = node.declaration; + if (decl && ['FunctionDeclaration', 'ClassDeclaration', 'TSInterfaceDeclaration', 'TSEnumDeclaration', 'TSModuleDeclaration'].includes(decl.type)) return; + check(node); + }; + handlers.ExportNamedDeclaration = checkExport; + handlers.ExportDefaultDeclaration = checkExport; + + return handlers; + }, +}; + +export const rules = { + 'require-send': requireSendRule, + semi: semiRule, +}; + +const plugin = { meta: { name: 'ores-lint', version: '1.3.0' }, rules }; +export default plugin; diff --git a/.ores-lint/gleam.sh b/.ores-lint/gleam.sh new file mode 100755 index 00000000..f3ec2dfc --- /dev/null +++ b/.ores-lint/gleam.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# ores-lint :: Gleam +# +# Uses the Gleam compiler toolchain - the universally accepted checker for the +# language (there is no ESLint-equivalent plugin host): +# gleam format --check formatting, analogous to rustfmt --check +# gleam check compiler warnings / unused values +# Custom house rules (require-send on logging pipes) live in require-send.mjs +# and run from lint.sh. +# +# Nothing is installed. Missing gleam is an actionable skip. + +set -u +DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$DIR/config.sh" +ROOT=${1:-.} +ROOT=$(CDPATH= cd -- "$ROOT" && pwd) + +[ "${ORES_LINT_SKIP_GLEAM}" = "1" ] && { echo "ores-lint[gleam]: skipped (ORES_LINT_SKIP_GLEAM=1)"; exit 0; } + +TOMLS=$(cd "$ROOT" && find . -maxdepth "$ORES_LINT_DEPTH" \ + \( -name node_modules -o -name target -o -name .git -o -name build -o -name vendor -o -name .vendor -o -name .ores-lint \) -prune -o \ + -type f -name gleam.toml -print 2>/dev/null \ + | sed 's|^\./||; s|gleam\.toml$||; s|/$||; s|^$|.|' | sort) + +[ -z "$TOMLS" ] && exit 0 + +command -v gleam >/dev/null 2>&1 || { + echo "ores-lint[gleam]: gleam not found on PATH - skipping" + echo " install from https://gleam.run/getting-started/installing/" + exit 0 +} + +NESTED_FILE="$DIR/nested-repos.json" +if [ -f "$NESTED_FILE" ]; then + NESTED=$(grep -o '"[^"]*"' "$NESTED_FILE" 2>/dev/null | tr -d '"') + if [ -n "$NESTED" ]; then + KEPT="" + for c in $TOMLS; do + drop=0 + for nrepo in $NESTED; do + case "$c" in + "$nrepo"|"$nrepo"/*) drop=1; break ;; + esac + done + [ "$drop" = "0" ] && KEPT="$KEPT +$c" + done + TOMLS=$(printf '%s' "$KEPT" | sed '/^$/d') + fi +fi + +[ -z "$TOMLS" ] && { echo "ores-lint[gleam]: all packages belong to nested repos - nothing to do here"; exit 0; } + +RAW=$(mktemp) || exit 0 +RAN=0 +FAILED="" + +for c in $TOMLS; do + if [ "$c" = "." ]; then cdir="$ROOT"; else cdir="$ROOT/$c"; fi + OUT=$(mktemp) + RC=0 + FMT_ARGS="src" + [ -d "$cdir/test" ] && FMT_ARGS="$FMT_ARGS test" + ( cd "$cdir" && gleam format --check $FMT_ARGS 2>/dev/null; gleam check ) >"$OUT" 2>&1 || RC=$? + RAN=$((RAN + 1)) + if [ "$RC" -ne 0 ] && ! grep -qE 'error|warning|Which files to format' "$OUT"; then + FAILED="$FAILED + $c (exit $RC): $(sed -n '1,2p' "$OUT" | tr '\n' ' ' | cut -c1-140)" + rm -f "$OUT" + continue + fi + if [ "$c" = "." ]; then cat "$OUT" >> "$RAW"; else sed "s|^|$c/|" "$OUT" >> "$RAW"; fi + rm -f "$OUT" +done + +echo "ores-lint[gleam]: linted $RAN package(s)" +[ -n "$FAILED" ] && printf 'ores-lint[gleam]: gleam could not run in some packages:%s\n' "$FAILED" + +awk -v MAXEX="$ORES_LINT_MAX_EXAMPLES" ' +BEGIN { max = MAXEX + 0; if (max < 1) max = 1; n = 0 } +/error:|warning:/ { + msg = $0 + n++ + if (shown < max) { ex = ex (shown++ ? "\n" : "") " " msg; } + next +} +END { + if (n == 0) { print "ores-lint[gleam]: clean"; exit 0 } + printf "ores-lint[gleam]: %d finding(s)\n", n + print ex + if (n > max) printf " ... and %d more\n", n - max + print "" +} +' "$RAW" + +rm -f "$RAW" +exit 0 diff --git a/.ores-lint/install-git-hooks.sh b/.ores-lint/install-git-hooks.sh new file mode 100755 index 00000000..c4dbce24 --- /dev/null +++ b/.ores-lint/install-git-hooks.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Optional: install a pre-push hook that runs ores-lint. +# Not installed automatically by the rollout - run this yourself per repo. +set -u +ROOT=$(git rev-parse --show-toplevel 2>/dev/null) || { echo "not a git repo"; exit 1; } +HOOK="$ROOT/.git/hooks/pre-push" +if [ -e "$HOOK" ] && ! grep -q 'ores-lint' "$HOOK"; then + echo "refusing to clobber an existing pre-push hook: $HOOK" + exit 1 +fi +cat > "$HOOK" <<'INNER' +#!/bin/sh +# installed by .ores-lint/install-git-hooks.sh +[ -x "$(git rev-parse --show-toplevel)/.ores-lint/lint.sh" ] && \ + sh "$(git rev-parse --show-toplevel)/.ores-lint/lint.sh" +exit 0 +INNER +chmod +x "$HOOK" +echo "installed $HOOK" diff --git a/.ores-lint/js.sh b/.ores-lint/js.sh new file mode 100755 index 00000000..ecbb0763 --- /dev/null +++ b/.ores-lint/js.sh @@ -0,0 +1,108 @@ +#!/bin/sh +# ores-lint :: JavaScript / TypeScript +# +# ESLint is treated as a GLOBALLY INSTALLED TOOL, not a per-repo dependency. +# Nothing here installs anything and nothing reaches the network; hundreds of +# repos sharing one global eslint is the whole point. Resolution order: +# +# 1. a local node_modules/.bin/eslint, walking up for monorepos +# (respected if a repo genuinely pins its own) +# 2. eslint on PATH (npm i -g eslint) +# 3. the global npm root +# +# If none is found this prints an actionable skip and exits 0. + +set -u +DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$DIR/config.sh" +ROOT=${1:-.} + +[ "${ORES_LINT_SKIP_JS}" = "1" ] && { echo "ores-lint[js]: skipped (ORES_LINT_SKIP_JS=1)"; exit 0; } + +find_local_eslint() { + d=$(CDPATH= cd -- "$1" && pwd) + while [ -n "$d" ] && [ "$d" != "/" ]; do + if [ -x "$d/node_modules/.bin/eslint" ]; then printf '%s\n' "$d/node_modules/.bin/eslint"; return 0; fi + d=$(dirname "$d") + done + return 1 +} + +GLOBAL_ROOT="" +for probe in "npm root -g" "pnpm root -g" "yarn global dir"; do + r=$($probe 2>/dev/null | head -1) + [ -n "$r" ] && [ -d "$r" ] && { GLOBAL_ROOT="$r"; break; } +done + +ESLINT="" +ESLINT_KIND="" +if ESLINT=$(find_local_eslint "$ROOT"); then + ESLINT_KIND="local" +elif command -v eslint >/dev/null 2>&1; then + ESLINT=$(command -v eslint); ESLINT_KIND="global (PATH)" +elif [ -n "$GLOBAL_ROOT" ] && [ -x "$GLOBAL_ROOT/.bin/eslint" ]; then + ESLINT="$GLOBAL_ROOT/.bin/eslint"; ESLINT_KIND="global" +else + echo "ores-lint[js]: no eslint found - skipping" + echo " install it once, globally: npm i -g eslint" + echo " (ores-lint never adds eslint to a repo's node_modules)" + exit 0 +fi + +# Version gate. Flat config needs ESLint 9+; an older one would fail confusingly. +VER=$("$ESLINT" --version 2>/dev/null | sed 's/^v//') +MAJOR=$(printf '%s' "$VER" | cut -d. -f1) +case "$MAJOR" in + ''|*[!0-9]*) : ;; # unparseable version: proceed rather than block + *) + if [ "$MAJOR" -lt "${ORES_LINT_ESLINT_MIN_MAJOR}" ]; then + echo "ores-lint[js]: found eslint $VER ($ESLINT_KIND) but flat config needs >=${ORES_LINT_ESLINT_MIN_MAJOR} - skipping" + echo " upgrade with: npm i -g eslint@latest" + exit 0 + fi + ;; +esac + +CONFIG="" +for c in eslint.config.mjs eslint.config.js eslint.config.cjs; do + [ -f "$ROOT/$c" ] && { CONFIG="$ROOT/$c"; break; } +done +[ -z "$CONFIG" ] && { echo "ores-lint[js]: no flat eslint config found - skipping"; exit 0; } + +# Let base.mjs find globally installed optional tooling (typescript-eslint). +[ -n "$GLOBAL_ROOT" ] && export ORES_LINT_GLOBAL_ROOT="$GLOBAL_ROOT" +[ -n "$GLOBAL_ROOT" ] && export NODE_PATH="${NODE_PATH:+$NODE_PATH:}$GLOBAL_ROOT" + +# Make the TypeScript gap visible. A repo with a tsconfig whose .ts files are +# being skipped looks identical to a clean repo otherwise. +if [ -f "$ROOT/tsconfig.json" ] || ls "$ROOT"/src/*.ts >/dev/null 2>&1; then + if ! node -e " + const {createRequire}=require('node:module'); + const r=createRequire('$ROOT/package.json'); + const paths=[process.env.ORES_LINT_GLOBAL_ROOT].filter(Boolean); + for (const id of ['typescript-eslint','@typescript-eslint/parser']) { + try { r.resolve(id); process.exit(0); } catch {} + if (paths.length) { try { r.resolve(id,{paths}); process.exit(0); } catch {} } + } + process.exit(1); + " 2>/dev/null; then + echo "ores-lint[js]: NOTE - this repo has TypeScript but no typescript-eslint parser;" + echo " .ts/.tsx files are being SKIPPED. Fix with: npm i -g typescript-eslint" + fi +fi + +OUT=$(mktemp) || exit 0 +RC=0 +( cd "$ROOT" && "$ESLINT" . \ + --no-error-on-unmatched-pattern \ + --format "$DIR/eslint/formatter.mjs" ) >"$OUT" 2>&1 || RC=$? + +if [ "$RC" -ne 0 ] && ! grep -q 'ores-lint\[js\]' "$OUT"; then + echo "ores-lint[js]: eslint $VER ($ESLINT_KIND) could not run in $ROOT (exit $RC). First lines:" + sed -n '1,6p' "$OUT" | sed 's/^/ | /' + rm -f "$OUT"; exit 0 +fi + +cat "$OUT" +rm -f "$OUT" +exit 0 diff --git a/.ores-lint/lint.sh b/.ores-lint/lint.sh new file mode 100755 index 00000000..af39ccac --- /dev/null +++ b/.ores-lint/lint.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# ores-lint :: entry point +# +# Warn-only by default. This is wired into build and publish hooks across +# hundreds of repos, so it is designed to be incapable of breaking one unless a +# human explicitly sets ORES_LINT_STRICT=1. +# +# Both halves discover sub-projects rather than assuming the repo root is the +# only project: `eslint .` walks nested packages from the root config, and +# rust.sh finds every crate including ones buried under apps/ or clients/. + +set -u +DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +ROOT=$(dirname "$DIR") +. "$DIR/config.sh" + +echo "ores-lint v$(cat "$DIR/VERSION" 2>/dev/null || echo '?') :: $(basename "$ROOT")" + +FOUND=0 +LOG=$(mktemp) || exit 0 + +# JS: an eslint flat config at the repo root is the trigger. The rollout puts +# one there whenever the repo contains any JS/TS at all, so nested packages in +# a Rust-rooted repo still get linted. +for c in eslint.config.mjs eslint.config.js eslint.config.cjs; do + if [ -f "$ROOT/$c" ]; then + sh "$DIR/js.sh" "$ROOT" | tee -a "$LOG" + FOUND=1 + break + fi +done + +# Rust: any Cargo.toml anywhere in the repo, not just at the root. +if find "$ROOT" -maxdepth "$ORES_LINT_DEPTH" \ + \( -name node_modules -o -name target -o -name .git -o -name vendor -o -name .vendor \) -prune -o \ + -type f -name Cargo.toml -print 2>/dev/null | head -1 | grep -q .; then + sh "$DIR/rust.sh" "$ROOT" | tee -a "$LOG" + FOUND=1 +fi + +# Dart / Flutter: pubspec.yaml or any .dart source. +if find "$ROOT" -maxdepth "$ORES_LINT_DEPTH" \ + \( -name node_modules -o -name target -o -name .git -o -name vendor -o -name .vendor -o -name build \) -prune -o \ + \( -type f -name pubspec.yaml -o -type f -name '*.dart' \) -print 2>/dev/null | head -1 | grep -q .; then + sh "$DIR/dart.sh" "$ROOT" | tee -a "$LOG" + FOUND=1 +fi + +# Gleam: gleam.toml or any .gleam source. +if find "$ROOT" -maxdepth "$ORES_LINT_DEPTH" \ + \( -name node_modules -o -name target -o -name .git -o -name vendor -o -name .vendor -o -name build \) -prune -o \ + \( -type f -name gleam.toml -o -type f -name '*.gleam' \) -print 2>/dev/null | head -1 | grep -q .; then + sh "$DIR/gleam.sh" "$ROOT" | tee -a "$LOG" + FOUND=1 +fi + +# House rule shared across Rust / Dart / Gleam (TypeScript is ores/require-send). +if command -v node >/dev/null 2>&1 && [ -f "$DIR/require-send.mjs" ]; then + if find "$ROOT" -maxdepth "$ORES_LINT_DEPTH" \ + \( -name node_modules -o -name target -o -name .git -o -name vendor -o -name .vendor -o -name build \) -prune -o \ + -type f \( -name '*.rs' -o -name '*.dart' -o -name '*.gleam' \) -print 2>/dev/null | head -1 | grep -q .; then + node "$DIR/require-send.mjs" "$ROOT" | tee -a "$LOG" + FOUND=1 + fi +fi + +[ "$FOUND" = "0" ] && echo "ores-lint: no JS, Rust, Dart or Gleam project found in this repo - nothing to do" + +if [ "${ORES_LINT_STRICT}" = "1" ] && grep -q 'finding(s) across' "$LOG"; then + rm -f "$LOG" + echo "ores-lint: FAILING because ORES_LINT_STRICT=1" + exit 1 +fi + +rm -f "$LOG" +exit 0 diff --git a/.ores-lint/nested-repos.json b/.ores-lint/nested-repos.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/.ores-lint/nested-repos.json @@ -0,0 +1 @@ +[] diff --git a/.ores-lint/require-send.mjs b/.ores-lint/require-send.mjs new file mode 100644 index 00000000..7662ddd5 --- /dev/null +++ b/.ores-lint/require-send.mjs @@ -0,0 +1,448 @@ +#!/usr/bin/env node +/** + * ores-lint :: require-send (Rust, Dart, Gleam) + * + * House rule, same contract as ores/require-send in ESLint: a logging chain + * that reaches a level method must be delivered with send() / send(boolean) / + * send_with_store(...). TypeScript stays on ESLint; this file covers the + * languages ESLint cannot parse. + * + * Line-level overrides (any of these, on the finding line or the previous line): + * ores-lint-disable-next-line require-send + * ores-lint-disable-line require-send + * File-level: + * ores-lint-disable-file require-send + * + * Warn-only. Prints the same capped report format as rust.sh / js.sh. + */ +import { execFileSync } from 'node:child_process'; +import { readFileSync, existsSync } from 'node:fs'; +import { join, dirname } from 'node:path'; + +const ROOT = process.argv[2] ? process.argv[2] : process.cwd(); +const DIR = dirname(new URL(import.meta.url).pathname); +const MAX = Math.max(1, Number(process.env.ORES_LINT_MAX_EXAMPLES || 5)); +const INCLUDE_TESTS = process.env.ORES_LINT_REQUIRE_SEND_INCLUDE_TESTS === '1'; +const LEVEL = new Set(['trace', 'debug', 'info', 'log', 'warn', 'error', 'fatal']); +const TERMINAL = new Set(['send', 'send_with_store']); +const SKIP_DIR = new Set([ + 'node_modules', 'target', 'dist', 'build', 'out', 'vendor', 'coverage', + '.git', '.worktrees', '_to_delete', '.next', '.ores-lint', '.vendor', + 'deps', 'third_party', 'thirdparty', 'external', 'submodules', '.r2g', +]); +const TEST_RE = /(?:^|\/)(?:test|tests|spec)\/|_test\.(?:dart|gleam|rs)$|\.test\.|\.spec\./i; + +function nestedRepos() { + try { + const raw = JSON.parse(readFileSync(join(DIR, 'nested-repos.json'), 'utf8')); + return Array.isArray(raw) ? raw : []; + } catch { + return []; + } +} + +function trackedFiles() { + let output; + try { + output = execFileSync('git', ['-C', ROOT, 'ls-files', '-z'], { + encoding: 'buffer', + maxBuffer: 64 * 1024 * 1024, + }).toString('utf8'); + } catch { + return []; + } + const nested = nestedRepos(); + return output.split('\0').filter(Boolean).filter((rel) => { + if (!/\.(?:rs|dart|gleam)$/.test(rel)) return false; + const parts = rel.split('/'); + if (parts.some((p) => SKIP_DIR.has(p))) return false; + if (nested.some((n) => rel === n || rel.startsWith(`${n}/`))) return false; + if (!INCLUDE_TESTS && TEST_RE.test(rel)) return false; + return true; + }); +} + +function looksLikeLogger(name) { + if (!name) return false; + const base = String(name).split('.').pop(); + return ( + /^(?:log|logger|ddlog|telemetry|audit|self|this)$/i.test(base) + || /logger$/i.test(base) + || /(?:^|_)log$/i.test(base) + ); +} + +function isDisabled(suppressions, line) { + if (suppressions.file) return true; + if (suppressions.lines.has(line) || suppressions.lines.has(line - 1)) return true; + if (suppressions.next.has(line - 1)) return true; + return false; +} + +function collectSuppressions(source) { + const file = /ores-lint-disable-file\s+require-send/.test(source); + const lines = new Set(); + const next = new Set(); + const raw = source.split('\n'); + for (let i = 0; i < raw.length; i++) { + const line = raw[i]; + if (/ores-lint-disable-line\s+require-send/.test(line) || /eslint-disable-line\s+ores\/require-send/.test(line)) { + lines.add(i + 1); + } + if (/ores-lint-disable-next-line\s+require-send/.test(line) || /eslint-disable-next-line\s+ores\/require-send/.test(line)) { + next.add(i + 1); + } + } + return { file, lines, next }; +} + +function tokenize(source) { + const tokens = []; + const n = source.length; + let i = 0; + let line = 1; + let col = 1; + const push = (type, value, startLine, startCol) => { + tokens.push({ type, value, line: startLine, col: startCol }); + }; + const bump = (ch) => { + if (ch === '\n') { line += 1; col = 1; } else col += 1; + }; + + while (i < n) { + const ch = source[i]; + const startLine = line; + const startCol = col; + + if (ch === '/' && source[i + 1] === '/') { + while (i < n && source[i] !== '\n') { bump(source[i]); i += 1; } + continue; + } + if (ch === '/' && source[i + 1] === '*') { + i += 2; bump('/'); bump('*'); + while (i < n && !(source[i] === '*' && source[i + 1] === '/')) { bump(source[i]); i += 1; } + if (i < n) { bump('*'); bump('/'); i += 2; } + continue; + } + if (ch === '#') { + // Gleam does not use # comments; rust raw strings / dart interpolations + // are handled as identifiers or other. Treat # as other. + } + if (ch === '"' || ch === "'" || ch === '`') { + const q = ch; + bump(ch); i += 1; + while (i < n && source[i] !== q) { + if (source[i] === '\\' && i + 1 < n) { bump(source[i]); bump(source[i + 1]); i += 2; continue; } + if (source[i] === '\n' && q !== '`') break; + bump(source[i]); i += 1; + } + if (i < n && source[i] === q) { bump(q); i += 1; } + push('string', '', startLine, startCol); + continue; + } + if (/\s/.test(ch)) { + bump(ch); i += 1; + continue; + } + if (ch === '|' && source[i + 1] === '>') { + push('pipe', '|>', startLine, startCol); + bump('|'); bump('>'); i += 2; + continue; + } + if (ch === '=' && source[i + 1] === '>') { + push('arrow', '=>', startLine, startCol); + bump('='); bump('>'); i += 2; + continue; + } + if (/[A-Za-z_]/.test(ch)) { + let value = ''; + while (i < n && /[A-Za-z0-9_]/.test(source[i])) { value += source[i]; bump(source[i]); i += 1; } + push('ident', value, startLine, startCol); + continue; + } + const singles = { + '.': 'dot', '(': 'lparen', ')': 'rparen', '[': 'lbracket', ']': 'rbracket', + '{': 'lbrace', '}': 'rbrace', ';': 'semi', ',': 'comma', '=': 'eq', + }; + if (singles[ch]) { + push(singles[ch], ch, startLine, startCol); + bump(ch); i += 1; + continue; + } + bump(ch); i += 1; + } + return tokens; +} + +function skipBalanced(tokens, start, open, close) { + let depth = 0; + for (let i = start; i < tokens.length; i++) { + if (tokens[i].type === open) depth += 1; + else if (tokens[i].type === close) { + depth -= 1; + if (depth === 0) return i; + } + } + return tokens.length - 1; +} + +function qualifiedName(tokens, index) { + // Walk left across ident.ident + let i = index; + if (!tokens[i] || tokens[i].type !== 'ident') return { name: '', start: index }; + let name = tokens[i].value; + while (i >= 2 && tokens[i - 1].type === 'dot' && tokens[i - 2].type === 'ident') { + name = `${tokens[i - 2].value}.${name}`; + i -= 2; + } + return { name, start: i }; +} + +function countTopLevelArgs(tokens, lparenIndex) { + if (tokens[lparenIndex]?.type !== 'lparen') return 0; + let depth = 0; + let args = 0; + let seen = false; + for (let i = lparenIndex; i < tokens.length; i++) { + const t = tokens[i]; + if (t.type === 'lparen' || t.type === 'lbracket' || t.type === 'lbrace') depth += 1; + else if (t.type === 'rparen' || t.type === 'rbracket' || t.type === 'rbrace') { + depth -= 1; + if (depth === 0) return seen ? args + 1 : 0; + } else if (depth === 1 && t.type === 'comma') args += 1; + else if (depth === 1 && t.type !== 'comma') seen = true; + } + return seen ? args + 1 : 0; +} + +function walkMethodChain(tokens, start) { + // start at the root ident of `root.level(args).more(args)` + const methods = []; + let i = start; + if (!tokens[i] || tokens[i].type !== 'ident') return null; + while (i + 2 < tokens.length && tokens[i + 1].type === 'dot' && tokens[i + 2].type === 'ident' && tokens[i + 3]?.type !== 'lparen') { + if (LEVEL.has(tokens[i + 2].value) || TERMINAL.has(tokens[i + 2].value)) break; + i += 2; + } + const root = qualifiedName(tokens, i).name; + let firstArgCount = 0; + while (i + 2 < tokens.length && tokens[i + 1].type === 'dot' && tokens[i + 2].type === 'ident') { + const method = tokens[i + 2].value; + methods.push(method); + i += 2; + if (tokens[i + 1]?.type === 'lparen') { + if (methods.length === 1) firstArgCount = countTopLevelArgs(tokens, i + 1); + i = skipBalanced(tokens, i + 1, 'lparen', 'rparen'); + } + } + return { root, methods, end: i, line: tokens[start].line, firstArgCount }; +} + +function walkGleamPipe(tokens, start) { + // start at ident of a call: `logging.info(...)` or `info(...)` + if (!tokens[start] || tokens[start].type !== 'ident') return null; + const head = qualifiedName(tokens, start); + let i = start; + while (i + 1 < tokens.length && tokens[i + 1].type === 'dot' && tokens[i + 2]?.type === 'ident') i += 2; + const callee = qualifiedName(tokens, i).name; + const calleeBase = callee.split('.').pop(); + if (tokens[i + 1]?.type !== 'lparen') { + // Bare `|> send` / `|> logging.send` only — not type variables named `error`. + if (TERMINAL.has(calleeBase) && tokens[head.start - 1]?.type === 'pipe') { + return { callee, methods: [calleeBase], end: i, line: tokens[head.start].line }; + } + return null; + } + const methods = [calleeBase]; + i = skipBalanced(tokens, i + 1, 'lparen', 'rparen'); + while (tokens[i + 1]?.type === 'pipe') { + i += 1; + if (tokens[i + 1]?.type !== 'ident') break; + i += 1; + while (i + 1 < tokens.length && tokens[i + 1].type === 'dot' && tokens[i + 2]?.type === 'ident') i += 2; + const step = tokens[i].value; + methods.push(step); + if (tokens[i + 1]?.type === 'lparen') i = skipBalanced(tokens, i + 1, 'lparen', 'rparen'); + } + return { callee, methods, end: i, line: tokens[head.start].line }; +} + +function precedingAssignment(tokens, start) { + // `let name =` or `final name =` or `var name =` immediately before start + let i = start - 1; + if (tokens[i]?.type !== 'eq') return null; + i -= 1; + if (tokens[i]?.type !== 'ident') return null; + const name = tokens[i].value; + const prev = tokens[i - 1]?.value; + if (prev && /^(let|var|final|const|mut)$/.test(prev)) return name; + // dart `LogEvent event =` / rust `let mut event =` already handled via let + if (tokens[i - 1]?.type === 'ident') return name; + return name; +} + +function isReturnish(tokens, start) { + const prev = tokens[start - 1]; + if (!prev) return false; + if (prev.type === 'arrow') return true; + if (prev.type === 'ident' && prev.value === 'return') return true; + return false; +} + +function nextNonChainIsSemi(tokens, end) { + const t = tokens[end + 1]; + return t?.type === 'semi'; +} + +export function analyzeSource(source, language) { + const suppressions = collectSuppressions(source); + const tokens = tokenize(source); + const findings = []; + const pending = new Map(); + const scopeStack = [pending]; + + const current = () => scopeStack[scopeStack.length - 1]; + const mark = (name, finding) => { if (name) current().set(name, finding); }; + const clear = (name) => { + if (!name) return; + for (let i = scopeStack.length - 1; i >= 0; i--) { + if (scopeStack[i].has(name)) { scopeStack[i].delete(name); return; } + } + }; + const report = (finding) => { + if (isDisabled(suppressions, finding.line)) return; + findings.push(finding); + }; + const flushScope = (map) => { + for (const finding of map.values()) report(finding); + }; + + for (let i = 0; i < tokens.length; i++) { + const tok = tokens[i]; + if (tok.type === 'lbrace') { scopeStack.push(new Map()); continue; } + if (tok.type === 'rbrace') { + if (scopeStack.length > 1) flushScope(scopeStack.pop()); + continue; + } + + // `name.send(...)` or `send(name)` / `logging.send(name)` + if (tok.type === 'ident' && TERMINAL.has(tok.value)) { + const prev = tokens[i - 1]; + if (prev?.type === 'dot' && tokens[i - 2]?.type === 'ident') { + clear(qualifiedName(tokens, i - 2).name); + } + if (tokens[i + 1]?.type === 'lparen') { + const inner = tokens[i + 2]; + if (inner?.type === 'ident') clear(inner.value); + } + } + + if (tok.type !== 'ident') continue; + + if (language === 'gleam') { + const calleeBase = tok.value; + const isLevel = LEVEL.has(calleeBase); + if (!isLevel) continue; + if (tokens[i - 1]?.type === 'ident' && tokens[i - 1].value === 'fn') continue; + const qual = qualifiedName(tokens, i); + if (tokens[i + 1]?.type !== 'lparen' && tokens[qual.start - 1]?.type !== 'pipe') continue; + const pipe = walkGleamPipe(tokens, qual.start); + if (!pipe) continue; + if (!pipe.methods.some((m) => LEVEL.has(m))) continue; + if (pipe.methods.some((m) => TERMINAL.has(m))) { + i = pipe.end; + continue; + } + const assigned = precedingAssignment(tokens, qual.start); + const finding = { line: pipe.line, col: tok.col, message: 'logging chain never calls send()' }; + const prev = tokens[qual.start - 1]?.type; + const next = tokens[pipe.end + 1]?.type; + if (assigned) mark(assigned, finding); + else if (isReturnish(tokens, qual.start) || prev === 'lparen' || prev === 'comma' || next === 'rbrace') { /* handoff / tail return */ } + else report(finding); + i = pipe.end; + continue; + } + + // Rust / Dart method chains: look for `.level(` + if (tok.type === 'ident' && LEVEL.has(tok.value) && tokens[i - 1]?.type === 'dot' && tokens[i + 1]?.type === 'lparen') { + let rootIndex = i - 2; + while (rootIndex >= 2 && tokens[rootIndex]?.type === 'ident' && tokens[rootIndex - 1]?.type === 'dot' && tokens[rootIndex - 2]?.type === 'ident') { + rootIndex -= 2; + } + if (tokens[rootIndex]?.type !== 'ident') continue; + const chain = walkMethodChain(tokens, rootIndex); + if (!chain) continue; + if (!looksLikeLogger(chain.root) && chain.root !== 'self' && chain.root !== 'this') continue; + if (!chain.methods.some((m) => LEVEL.has(m))) continue; + if (chain.methods.some((m) => TERMINAL.has(m))) { + i = chain.end; + continue; + } + // Convenience emit: logger.log(level, msg, ctx, fields) already calls send() + // internally. The chainable API is one argument (or two in Dart). + if (chain.methods.length === 1 && (chain.firstArgCount || 0) >= 3) { + i = chain.end; + continue; + } + const assignName = precedingAssignment(tokens, rootIndex); + const finding = { line: chain.line, col: tokens[rootIndex].col, message: 'logging chain never calls send()' }; + const prev = tokens[rootIndex - 1]?.type; + if (assignName) mark(assignName, finding); + else if (isReturnish(tokens, rootIndex) || prev === 'lparen' || prev === 'comma') { /* handoff */ } + else if (language === 'rust' && !nextNonChainIsSemi(tokens, chain.end)) { /* rust tail expression / return */ } + else report(finding); + i = chain.end; + } + } + + while (scopeStack.length) flushScope(scopeStack.pop()); + return findings; +} + +export function formatReport(results) { + const all = []; + for (const { file, findings } of results) { + for (const f of findings) all.push({ ...f, file }); + } + if (!all.length) { + return results.length + ? `ores-lint[require-send]: clean (${results.length} file${results.length === 1 ? '' : 's'} scanned)\n` + : 'ores-lint[require-send]: no Rust/Dart/Gleam source to scan\n'; + } + const lines = [ + `ores-lint[require-send]: ${all.length} finding(s) across 1 rule(s) in ${results.filter((r) => r.findings.length).length} file(s)`, + '', + ' warning: logging chain never delivered (ores custom rule) [require-send]', + ` ${all.length} instance(s); showing ${Math.min(all.length, MAX)}:`, + ]; + for (const f of all.slice(0, MAX)) { + lines.push(` ${f.file}:${f.line}:${f.col}`); + } + if (all.length > MAX) lines.push(` ... and ${all.length - MAX} more`); + lines.push(''); + return `${lines.join('\n')}\n`; +} + +function main() { + if (process.env.ORES_LINT_SKIP_REQUIRE_SEND === '1') { + process.stdout.write('ores-lint[require-send]: skipped (ORES_LINT_SKIP_REQUIRE_SEND=1)\n'); + return; + } + const files = trackedFiles(); + const results = []; + for (const rel of files) { + const abs = join(ROOT, rel); + if (!existsSync(abs)) continue; + let source; + try { source = readFileSync(abs, 'utf8'); } catch { continue; } + const language = rel.endsWith('.gleam') ? 'gleam' : rel.endsWith('.dart') ? 'dart' : 'rust'; + results.push({ file: rel, findings: analyzeSource(source, language) }); + } + process.stdout.write(formatReport(results)); +} + +const isMain = process.argv[1] && import.meta.url.endsWith(process.argv[1].split('/').pop()); +if (import.meta.url === `file://${process.argv[1]}` || process.argv[1]?.endsWith('require-send.mjs')) { + main(); +} diff --git a/.ores-lint/required-tools.json b/.ores-lint/required-tools.json new file mode 100644 index 00000000..65ace40e --- /dev/null +++ b/.ores-lint/required-tools.json @@ -0,0 +1,30 @@ +{ + "_comment": "Tool versions ores-lint expects. These are DECLARATIONS, not dependencies - nothing here is ever installed into a repo's node_modules. Install them once, globally. The rollout script mirrors this block into each package.json under the `oresLint` key so the requirement travels with the repo.", + "eslint": { + "minMajor": 9, + "range": ">=9", + "install": "npm i -g eslint", + "why": "flat config (eslint.config.mjs) requires ESLint 9 or newer" + }, + "typescript-eslint": { + "minMajor": 8, + "range": ">=8", + "install": "npm i -g typescript-eslint", + "why": "optional; without it .ts/.tsx files are skipped rather than linted" + }, + "clippy": { + "range": "any", + "install": "rustup component add clippy", + "why": "the Rust half is a clippy wrapper" + }, + "dart": { + "range": "any", + "install": "install the Dart SDK or Flutter", + "why": "optional; dart analyze is the Dart/Flutter linter. Without it Dart source is skipped." + }, + "gleam": { + "range": "any", + "install": "see https://gleam.run/getting-started/installing/", + "why": "optional; gleam format --check and gleam check. Without it Gleam source is skipped." + } +} diff --git a/.ores-lint/rust.sh b/.ores-lint/rust.sh new file mode 100755 index 00000000..a0dec12b --- /dev/null +++ b/.ores-lint/rust.sh @@ -0,0 +1,171 @@ +#!/bin/sh +# ores-lint :: Rust +# +# Discovers every crate in the repo - not just one at the root - runs clippy on +# each, and aggregates ALL of them into a single report so the example cap +# applies per repo rather than per crate. +# +# Workspace handling: after linting a crate root, `cargo metadata --no-deps` +# tells us exactly which manifests that invocation already covered, so workspace +# members are not linted twice while genuinely independent nested crates still +# get their own run. +# +# The headline custom behaviour: `clippy::implicit_return` fires once per +# implicit return, which across a repo means hundreds of identical warnings. The +# lint stays enabled so nothing is missed, but it is reported as ONE warning +# carrying at most ORES_LINT_MAX_EXAMPLES locations plus a total count. +# +# Critical interaction, handled below: `clippy::needless_return` ships enabled +# in clippy's default `style` group and warns on exactly the explicit returns +# this house style asks for. Enabling implicit_return without allowing +# needless_return makes the two lints contradict each other on every function. + +set -u +DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$DIR/config.sh" +ROOT=${1:-.} +ROOT=$(CDPATH= cd -- "$ROOT" && pwd) + +[ "${ORES_LINT_SKIP_RUST}" = "1" ] && { echo "ores-lint[rust]: skipped (ORES_LINT_SKIP_RUST=1)"; exit 0; } +command -v cargo >/dev/null 2>&1 || { echo "ores-lint[rust]: cargo not found on PATH - skipping"; exit 0; } +cargo clippy --version >/dev/null 2>&1 || { echo "ores-lint[rust]: clippy not installed (rustup component add clippy) - skipping"; exit 0; } + +LINTS=" +-W clippy::implicit_return +-A clippy::needless_return +-A clippy::let_and_return +-W clippy::correctness +-W clippy::suspicious +-W clippy::await_holding_lock +-W clippy::unwrap_used +-W clippy::expect_used +-W clippy::panic_in_result_fn +-W clippy::todo +-W clippy::unimplemented +-W clippy::dbg_macro +-W clippy::mem_forget +-W clippy::float_cmp +-W clippy::lossy_float_literal +" +[ -n "${ORES_LINT_RUST_EXTRA:-}" ] && LINTS="$LINTS $ORES_LINT_RUST_EXTRA" +TARGETS="" +[ "${ORES_LINT_RUST_ALL_TARGETS}" = "1" ] && TARGETS="--all-targets" + +# --- discover crates -------------------------------------------------------- +CRATES=$(cd "$ROOT" && find . -maxdepth "${ORES_LINT_DEPTH}" \ + \( -name node_modules -o -name target -o -name .git -o -name dist -o -name build \ + -o -name vendor -o -name .vendor -o -name .worktrees -o -name _to_delete \ + -o -name .ores-lint \) -prune -o \ + -type f -name Cargo.toml -print 2>/dev/null \ + | sed 's|^\./||; s|Cargo\.toml$||; s|/$||; s|^$|.|' | sort) + +[ -z "$CRATES" ] && { echo "ores-lint[rust]: no Cargo.toml found - nothing to do"; exit 0; } + +# Drop crates that live inside a NESTED git repository. Those belong to a +# different repo with its own ores-lint install; linting them from here would +# report the same findings twice under the wrong repo name. +NESTED_FILE="$DIR/nested-repos.json" +if [ -f "$NESTED_FILE" ]; then + NESTED=$(grep -o '"[^"]*"' "$NESTED_FILE" 2>/dev/null | tr -d '"') + if [ -n "$NESTED" ]; then + KEPT="" + for c in $CRATES; do + drop=0 + for nrepo in $NESTED; do + case "$c" in + "$nrepo"|"$nrepo"/*) drop=1; break ;; + esac + done + [ "$drop" = "0" ] && KEPT="$KEPT +$c" + done + CRATES=$(printf '%s' "$KEPT" | sed '/^$/d') + fi +fi + +[ -z "$CRATES" ] && { echo "ores-lint[rust]: all crates belong to nested repos - nothing to do here"; exit 0; } + +RAW=$(mktemp) || exit 0 +COVERED=$(mktemp) || exit 0 +RAN=0 +SKIPPED_MEMBERS=0 +FAILED="" + +for c in $CRATES; do + if [ "$c" = "." ]; then cdir="$ROOT"; else cdir="$ROOT/$c"; fi + # Already covered by an earlier workspace invocation? + if [ -s "$COVERED" ] && grep -qxF "$cdir" "$COVERED"; then + SKIPPED_MEMBERS=$((SKIPPED_MEMBERS + 1)) + continue + fi + + RC=0 + OUT=$(mktemp) + # shellcheck disable=SC2086 + ( cd "$cdir" && cargo clippy --workspace $TARGETS --message-format=short -- $LINTS ) >"$OUT" 2>&1 || RC=$? + + if [ "$RC" -ne 0 ] && ! grep -q ': warning: \|: error: ' "$OUT"; then + FAILED="$FAILED + $c (exit $RC): $(sed -n '1,2p' "$OUT" | tr '\n' ' ' | cut -c1-140)" + rm -f "$OUT" + continue + fi + RAN=$((RAN + 1)) + + # Re-root diagnostic paths at the repo, so a repo-wide report stays navigable. + if [ "$c" = "." ]; then + cat "$OUT" >> "$RAW" + else + sed "s|^|$c/|" "$OUT" >> "$RAW" + fi + rm -f "$OUT" + + # Record which manifests this invocation covered (workspace members). + ( cd "$cdir" && cargo metadata --no-deps --offline --format-version 1 2>/dev/null ) \ + | grep -o '"manifest_path":"[^"]*"' \ + | sed 's/"manifest_path":"//; s/"$//; s|/Cargo\.toml$||' >> "$COVERED" 2>/dev/null || true +done + +echo "ores-lint[rust]: linted $RAN crate root(s)$([ "$SKIPPED_MEMBERS" -gt 0 ] && echo ", $SKIPPED_MEMBERS workspace member(s) already covered")" +[ -n "$FAILED" ] && printf 'ores-lint[rust]: clippy could not run in some crates:%s\n' "$FAILED" + +awk -v MAXEX="$ORES_LINT_MAX_EXAMPLES" -v TARGETMSG="$ORES_LINT_IMPLICIT_RETURN_MSG" ' +BEGIN { max = MAXEX + 0; if (max < 1) max = 1; n = 0 } +match($0, /: (warning|error): /) { + loc = substr($0, 1, RSTART - 1) + rest = substr($0, RSTART + 2) + ci = index(rest, ": ") + sev = substr(rest, 1, ci - 1) + msg = substr(rest, ci + 2) + key = loc "|" msg + if (key in seen) next # same finding reported by two crate runs + seen[key] = 1 + if (!(msg in count)) { order[++n] = msg; sev_of[msg] = sev } + count[msg]++ + if (shown[msg] < max) { ex[msg] = ex[msg] (shown[msg]++ ? "\n" : "") " " loc } + next +} +END { + if (n == 0) { print "ores-lint[rust]: clean"; exit 0 } + total = 0 + for (i = 1; i <= n; i++) total += count[order[i]] + printf "ores-lint[rust]: %d finding(s) across %d rule(s)\n", total, n + for (pass = 1; pass <= 2; pass++) { + for (i = 1; i <= n; i++) { + msg = order[i] + is_target = (msg == TARGETMSG) + if ((pass == 1) != is_target) continue + label = is_target ? "implicit return (ores house style)" : msg + printf "\n %s: %s\n", sev_of[msg], label + if (is_target) printf " prefer an explicit `return` at tail position\n" + printf " %d instance(s); showing %d:\n", count[msg], (count[msg] < max ? count[msg] : max) + print ex[msg] + if (count[msg] > max) printf " ... and %d more\n", count[msg] - max + } + } + print "" +} +' "$RAW" + +rm -f "$RAW" "$COVERED" +exit 0 diff --git a/.ores-lint/selftest.sh b/.ores-lint/selftest.sh new file mode 100755 index 00000000..b76bc9f7 --- /dev/null +++ b/.ores-lint/selftest.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# ores-lint :: self-test +# +# Guards the two assumptions this toolkit rests on: +# 1. clippy still words `implicit_return` the way config.sh expects, and +# `needless_return` can still be silenced (otherwise the two lints fight). +# 2. the vendored ESLint plugin still loads and its rules still fire. +# +# Run after a toolchain upgrade. Exits non-zero if an assumption has broken - +# a silently empty lint report is far worse than a failing test. + +set -u +DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "$DIR/config.sh" +FAIL=0 +pass() { echo " ok - $1"; } +fail() { echo " FAIL - $1"; FAIL=1; } + +echo "ores-lint self-test" + +# --- Rust ------------------------------------------------------------------- +if command -v cargo >/dev/null 2>&1 && cargo clippy --version >/dev/null 2>&1; then + T=$(mktemp -d) + mkdir -p "$T/src" + cat > "$T/Cargo.toml" <<'EOF' +[package] +name = "ores_lint_selftest" +version = "0.0.0" +edition = "2021" +EOF + cat > "$T/src/lib.rs" <<'EOF' +pub fn implicit(x: i32) -> i32 { x } +pub fn explicit(x: i32) -> i32 { return x; } +EOF + OUT=$( cd "$T" && cargo clippy --message-format=short -- \ + -W clippy::implicit_return -A clippy::needless_return 2>&1 ) + + if printf '%s' "$OUT" | grep -qF "$ORES_LINT_IMPLICIT_RETURN_MSG"; then + pass "clippy implicit_return message matches config.sh" + else + fail "clippy implicit_return wording changed - update ORES_LINT_IMPLICIT_RETURN_MSG in config.sh" + printf '%s\n' "$OUT" | sed -n '1,4p' | sed 's/^/ /' + fi + + if printf '%s' "$OUT" | grep -q 'unneeded `return`'; then + fail "needless_return still fires despite -A; it contradicts the house style" + else + pass "needless_return correctly silenced" + fi + + N=$(printf '%s\n' "$OUT" | grep -cF "$ORES_LINT_IMPLICIT_RETURN_MSG") + [ "$N" = "1" ] && pass "exactly 1 implicit return detected in fixture" \ + || fail "expected 1 implicit return in fixture, saw $N" + rm -rf "$T" +else + echo " skip - cargo/clippy unavailable" +fi + +# --- JavaScript ------------------------------------------------------------- +if command -v node >/dev/null 2>&1; then + if node --input-type=module -e " + const p = await import('$DIR/eslint/plugin.mjs'); + const names = Object.keys(p.default.rules); + if (!names.includes('require-send') || !names.includes('semi')) process.exit(3); + " 2>/dev/null; then + pass "vendored eslint plugin loads with both rules" + else + fail "vendored eslint plugin failed to load" + fi + + if node "$DIR/require-send.test.mjs" >/dev/null 2>&1; then + pass "require-send scanner fixtures" + else + fail "require-send scanner fixtures failed" + node "$DIR/require-send.test.mjs" 2>&1 | sed -n '1,20p' | sed 's/^/ /' + fi +else + echo " skip - node unavailable" +fi + +# --- Dart ------------------------------------------------------------------- +if command -v dart >/dev/null 2>&1 || command -v flutter >/dev/null 2>&1; then + pass "dart/flutter available for analyzer pass" +else + echo " skip - dart/flutter unavailable (dart.sh will no-op)" +fi + +# --- Gleam ------------------------------------------------------------------ +if command -v gleam >/dev/null 2>&1; then + pass "gleam available for format/check pass" +else + echo " skip - gleam unavailable (gleam.sh will no-op)" +fi + +[ "$FAIL" = "0" ] && echo "self-test passed" || echo "self-test FAILED" +exit "$FAIL" diff --git a/.zed-cli-global-git-submodule-mode-ref b/.zed-cli-global-git-submodule-mode-ref new file mode 100644 index 00000000..02ba1205 --- /dev/null +++ b/.zed-cli-global-git-submodule-mode-ref @@ -0,0 +1 @@ +ff166471338e46729b7cf892b44240452a56ace6 diff --git a/.zed-cli-ref b/.zed-cli-ref new file mode 100644 index 00000000..b9795d69 --- /dev/null +++ b/.zed-cli-ref @@ -0,0 +1 @@ +d8763b8dcc3bd5c5b6b4b1d13519065a6a2cb99a diff --git a/.zed-nix-bundle-cli-ref b/.zed-nix-bundle-cli-ref new file mode 100644 index 00000000..59bafff6 --- /dev/null +++ b/.zed-nix-bundle-cli-ref @@ -0,0 +1 @@ +b82da352472e301425c175b78d9d2c1c0cd0f1db diff --git a/.zed-nix-bundle-write-cli-ref b/.zed-nix-bundle-write-cli-ref new file mode 100644 index 00000000..12f95760 --- /dev/null +++ b/.zed-nix-bundle-write-cli-ref @@ -0,0 +1 @@ +1b282b94efc994bb9352b572c9652d5a078787f2 diff --git a/.zed-nix-plan-cli-ref b/.zed-nix-plan-cli-ref new file mode 100644 index 00000000..46ec3c16 --- /dev/null +++ b/.zed-nix-plan-cli-ref @@ -0,0 +1 @@ +19841ece0bf3649a0ed4005c150fb3721d166605 diff --git a/README.md b/README.md index b07567bc..7d03ec45 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,35 @@ # zed-pkg-e2e End-to-end verification for the **zed-pkg-test** fixture organization. This -repository owns two complementary suites: +repository owns three complementary suites: -1. browser automation against a running zed registry stack; and +1. browser automation against a running zed registry stack; 2. a stateless GitHub Actions matrix that exercises package author and consumer - lifecycles across every repository in `zed-pkg-test`. + lifecycles across every repository in `zed-pkg-test`; and +3. pinned cross-repository acceptance canaries for CLI behavior that must be + proven with real package-author and consumer repositories before release. + +## GitHub API fallback (registry down, GitHub up) + +`.github/workflows/github-api-fallback.yml` calls `api.github.com` for real. +`scripts/github_api_fallback.py` publishes a packed tarball to +[`zed-pkg-test/github-api-fallback-canary`](https://github.com/zed-pkg-test/github-api-fallback-canary) +Releases, then downloads the same sha256 from the public +`/releases/download/` URL with no `registry.zpkg.net` hop. Optional `--zed` +proves `zed install --frozen` against a closed loopback registry still +materializes that payload (needs zed-cli#294). Pull requests only compile the +script; `workflow_dispatch` runs the live GitHub APIs. See +`docs/github-api-fallback-canary.md`. + +`scripts/ghcr_fallback.py` is the complementary Packages proof: it publishes +the packed tarball only to +[`ghcr.io/zed-pkg-test/ghcr-fallback-canary`](https://github.com/zed-pkg-test/ghcr-fallback-canary) +(no Release) so `zed install --frozen` must take the GHCR fallback branch. +See `docs/ghcr-fallback-canary.md`. + +Cloudflare Workers in [`zed-pkg/zed-infra`](https://github.com/zed-pkg/zed-infra) +(`workers/registry-proxy`, `workers/cdn-proxy`) reconstruct the same GitHub +URLs at `registry.zpkg.net` / `cdn.zpkg.net` when the k8s origin is 502. ## How this differs from `zed-pkg/zed-e2e` @@ -75,6 +99,120 @@ python3 scripts/lifecycle.py \ The work root is disposable. Reusing the same path is safe because the harness requires a fresh root and GitHub-hosted jobs receive a new runner filesystem. + +## GitHub / R2 fallback (registry down, CDN up) + +`.github/workflows/github-r2-fallback.yml` is a reusable canary for the +`cdn.zpkg.net` guessable-object contract. `scripts/github_r2_fallback.py` +publishes a disposable package to a `file://` registry, copies the tarball onto +`packages/` and `github/` keys, points `--registry` at a closed loopback port, +and proves `zed install --frozen` restores the packed payload from a loopback +CDN. It also round-trips the typed `get_version` RPC frame over TCP NDJSON +(from `github.com/oresoftware/api-docs`). Pull requests only compile the +script; a full product run needs exact `zed-cli` and `zed-interfaces` commits +via `workflow_call` or `workflow_dispatch`. See +`docs/github-r2-fallback-canary.md`. + +## Durable first-install acceptance + +`.github/workflows/durable-first-install.yml` is the cross-organization +acceptance gate for DEN-1413. It builds one immutable `zed-cli` commit on Linux +and macOS, publishes the exact pinned `zed-pkg-test/node-lib` fixture to a fresh +`file://` registry, removes Zed state from the pinned `node-app` fixture, and +installs the real package from a nested consumer directory. + +`scripts/durable-first-install.sh` verifies that: + +- a dependency-bearing first install creates `.zpkg.toml` at the inferred + native-project root, not in the nested invocation directory; +- two independently created consumers with the same basename and inputs receive + byte-identical manifests and lockfiles; +- the generated manifest records direct dependency, target, adapter, marker, + and deterministic local identity metadata; +- copy-mode package and Node adapter outputs contain no symlinks and the real + Node application executes successfully; +- an inferred local consumer cannot be published, even with dry-run and VCS + checks skipped; +- `--do-not-write-new-manifest` is an informational no-op for generated and + authored existing manifests; +- the canonical flag and `ZED_PKG_DO_NOT_WRITE_NEW_MANIFEST` preserve explicit + ephemeral installs, while `--skip-manifest` remains a deprecated compatibility + spelling; +- failed resolution removes the exact generated manifest and leaves no lock, + package tree, adapter output, or transaction debris; and +- lock-only frozen restoration fails by default and succeeds byte-for-byte only + when no-new-manifest intent is explicit. + +All component refs and actions are immutable. During implementation review, the +workflow pins the exact head of the corresponding `zed-cli` pull request. The +pin must move to that PR's final reviewed head after every implementation +change, then to the merge commit when the CLI work lands. The workflow has +read-only repository permissions and uploads only bounded local-registry +diagnostics on failure. + +## GitHub / R2 fallback (registry down, CDN up) + +`.github/workflows/github-r2-fallback.yml` is a reusable canary for the +`cdn.zpkg.net` guessable-object contract. `scripts/github_r2_fallback.py` +publishes a disposable package to a `file://` registry, copies the tarball onto +`packages/` and `github/` keys, points `--registry` at a closed loopback port, +and proves `zed install --frozen` restores the packed payload from a loopback +CDN. It also round-trips the typed `get_version` RPC frame over TCP NDJSON +(from `github.com/oresoftware/api-docs`). Pull requests only compile the +script; a full product run needs exact `zed-cli` and `zed-interfaces` commits +via `workflow_call` or `workflow_dispatch`. See +`docs/github-r2-fallback-canary.md`. + +Run the acceptance harness locally with sibling fixture checkouts: + +```bash +cargo build --release --manifest-path ../zed-cli/Cargo.toml --bin zed +bash scripts/durable-first-install.sh \ + ../zed-cli/target/release/zed \ + ../node-lib \ + ../node-app \ + /tmp/zed-durable-first-install +``` + +### Polyglot durable first-install acceptance + +`.github/workflows/durable-first-install-polyglot.yml` verifies the same default +manifest-creation contract against immutable real Go, Python, and Rust +app/library pairs on Linux and macOS. It is intentionally separate from the full +lifecycle matrix because it removes each consumer's authored `.zpkg.toml` before +installation and treats the resulting generated manifest as the artifact under +test. + +For every ecosystem, `scripts/durable-first-install-polyglot.sh`: + +- publishes the pinned library to a fresh credential-free `file://` registry; +- invokes `zed install` from a nested directory below the native project root; +- verifies that one deterministic, non-publishable `.zpkg.toml` is created only + at the inferred root; +- checks inferred target and adapter metadata plus the direct dependency; +- compares an explicit `@^1.0.0` install with an unversioned latest-resolution + install and requires byte-identical manifest and lockfile output; +- executes the real native application through Go, Python, or Cargo; +- proves copy-mode output contains no symlinks; +- uninstalls materialized content while retaining managed state; and +- performs a frozen nested-directory reinstall without changing manifest or + lockfile bytes. + +All source repositories, the CLI implementation, the shared interface contract, +and third-party actions are pinned to immutable commit SHAs. Each matrix entry +owns isolated registry, home, build, consumer, and diagnostics directories. + +Run one ecosystem locally after building the selected CLI: + +```bash +bash scripts/durable-first-install-polyglot.sh \ + ../zed-cli/target/release/zed \ + go \ + ../go-lib \ + ../go-app \ + /tmp/zed-durable-go +``` + ## Browser suites | Suite | What it covers | @@ -82,6 +220,21 @@ requires a fresh root and GitHub-hosted jobs receive a new runner filesystem. | `suites/fixture-package-pages.spec.ts` | Each single-language fixture (`node-lib`, `rust-lib`, `go-lib`, `python-lib`) renders the version, description, repository URL, and install snippet persisted from its `.zpkg.toml`; each is findable through HTMX search; a missing package is a real 404. | | `suites/polyglot-fan-out.spec.ts` | `polyglot-lib` becomes four separately addressable packages with distinct content hashes, and the unsuffixed repository name is not itself a package. Ecosystem-specific install enforcement remains in the lifecycle matrix, which reads each derived artifact manifest and exercises the native toolchain. | +## Managed application lifecycle certification + +`.github/workflows/formal-app-lifecycle.yml` pins the exact production +`zed-pkg/zed-sync` commit for DEN-4181. It verifies immutable commit and artifact +provenance, cross-checks the registered Quint model against the canonical trace +and JSON Schema, requires Rust, JavaScript, and Dart to consume the same fixture, +and executes the production repository's complete pinned verification matrix. + +Run the cross-repository structural check against a sibling checkout: + +```bash +python3 scripts/verify_formal_app_lifecycle_contract.py ../zed-sync +python3 -m unittest -v tests/test_formal_app_lifecycle_contract.py +``` + The browser workflow pins the stack, CLI, interfaces, Rust servers, and fixture inputs to immutable commit SHAs. Both Node workspaces install from their checked-in lockfiles with `npm ci`. The fixture publisher forces the local API URL, uses a @@ -130,3 +283,15 @@ environment variables after the sibling stack is running. ## License MIT + +## Archived DEN-2725 replay evidence + +The preserved `evidence/den-2725-final-export/` namespace contains the verified +final replay export from source run `31243116442`: + +- root-dispatch commit: `1c94813c6b59c7621c1256d7c55246e7f06f44b7`; +- current-main commit: `0246a3af901b65ca6724d936c7f3232c9f72fcd7`; and +- verified product tree: `99ab08e3142c2a52d2c97482134e99fae3811b43`. + +The original export identified eight root product files for import. They remain +evidence only here and do not replace this repository's E2E control-plane tree. diff --git a/bootstrap/audit-test-org-fleet.once b/bootstrap/audit-test-org-fleet.once new file mode 100644 index 00000000..18592dde --- /dev/null +++ b/bootstrap/audit-test-org-fleet.once @@ -0,0 +1 @@ +AUDIT_TEST_FLEET_V1 diff --git a/bootstrap/execute-test-org-fleet-paced.once b/bootstrap/execute-test-org-fleet-paced.once new file mode 100644 index 00000000..5132b46c --- /dev/null +++ b/bootstrap/execute-test-org-fleet-paced.once @@ -0,0 +1 @@ +APPLY_TEST_FLEET_PACED_ARM64_V2 diff --git a/bootstrap/execute-test-org-fleet-single-runner.once b/bootstrap/execute-test-org-fleet-single-runner.once new file mode 100644 index 00000000..153b9550 --- /dev/null +++ b/bootstrap/execute-test-org-fleet-single-runner.once @@ -0,0 +1 @@ +APPLY_TEST_FLEET_SINGLE_RUNNER_V3 diff --git a/bootstrap/execute-test-org-fleet-v2.once b/bootstrap/execute-test-org-fleet-v2.once new file mode 100644 index 00000000..57406c6e --- /dev/null +++ b/bootstrap/execute-test-org-fleet-v2.once @@ -0,0 +1 @@ +APPLY_TEST_FLEET_V3 diff --git a/bootstrap/execute-test-org-fleet.once b/bootstrap/execute-test-org-fleet.once new file mode 100644 index 00000000..d91807a7 --- /dev/null +++ b/bootstrap/execute-test-org-fleet.once @@ -0,0 +1 @@ +APPLY_TEST_FLEET diff --git a/bootstrap/led-dynamo.parts/000.b64 b/bootstrap/led-dynamo.parts/000.b64 new file mode 100644 index 00000000..027ed9e5 --- /dev/null +++ b/bootstrap/led-dynamo.parts/000.b64 @@ -0,0 +1 @@ +H4sIAHhFc2oC/+19a2PbNrLo9/srEGcbS1mJkuzESenYXcdWGm+d2MdymvZGOQolUTZridSSlB/r+P72Ow8ABEjKVtK023uu9uGIJDAABoPBvDD48L+EuIH/C7ESehN/xRUrzmmQns36KzV+PfSTQRxM0yAK8ethfOqFwb89fK5fBkNfTONoFIz9mhhEYRoH/Rl+EtNoHAyua8ILh/BhMpmFQXotznxvnJ4JLJ+IURSLsT+sD6+h6chRDabRNBgk0NYHeoY3WGjiAewrWQa7FUA3vXE9CU5D79TPPgRRmj34k74/HPrD7E08S9IVevgo26POQHM3qshxe2fvTduZDHG8D40uisgYPA+iG3bDzpkX+0P7YzlSTmfB0AsH8DrxB7MYMWLiKUiSmd8NU38yHXupxFB65osDGMQ1ggr9QQpNHbT3aOAiuU6gtNMNswEe7O+233ba2Pc3+yfiIBj4YeJjP3ej6XUcnJ6lojKoirXm2oY5Nt3RKKZRHfnxBDqE3Q4SmLjY71+L09iD7g1rYhT7vohGYgBjP4XxpBGMAEbjxwlUiPqpF4RBeCo8gDu97oZQND0DOEk0Si8BXTReL0miQeDhiIbRYDbxw9RArajg0LsrHVmlu1KldoZAQ10ATphRH8UlEG00S0XsJzCMAYKpiSAcjGdD7If6PA4mgWwEqxM6YLgAdpbQbE1hLibRMBjhvz6NbTrrj4PkrCaGQcIowgnEl4RamrsGzFTij6FjACKAztOAs/7xBEMzU0RrKhGV4JvLs2hijyWAHo1mcQiN+lRpGAHiqM3fgADwDZYfReNxdImjg7kbBjioxMWpO4GPXj+68Gk8POVhlEJ3uRc4EdNseuWn5Mwbj0Xfl1iDlgHHnjmkGHuQpEACgTcGyo2pyfxQHerC67boHL46eb9z3Bb7HXF0fPjz/l57D+ZzpwMvuis18X7/5PXhuxMBZY533p78Kg5fiZ23v4qf9t/u1UT7l6PjdqcjDo+74f6bo4P9Nrzcf7t78G5v/+2P4iVUfHsI9L0PVA5wTw4Ftilh7bc7CO1N+3j3NTzuvNw/2D/5tdYNX+2fvEWorw6PxY442jk+2d99d7BzLI7eHR8ddtrQgT2A+3b/7atjaKb9pv32xIFm4Z1o/wwPovN65+AA2+qGO+9gAMfYRbF7ePTr8f6Pr0/E68ODvTa8fNmGvu28PGhzWzCu3YOd/Tc1sbfzZufHNtU6BDAwPCzHHRTvX7fxHba4A//bPdk/fIsj2T18e3IMjzUY6PGJrvt+v9OuiZ3j/Q7i5NXx4RsYI+IUqhwSFKj4ts1gEN/Cmhgogs/vOm0NUey1dw4AWAcr0yhVaYvL4CYBDCiKfWaSL2fBGFjgLJ3OUlhOyBTSRjfEFQP/9PFrAwnpwo+BWzeQRB6K/w00NvUG5/BGAH/3YyAryUC7oXPhh8Mobjj/9rGq8U8d60gQB9EASBHofxSczmJV1Q8v+K/zuBs+oB/+lQdcFfggcN7YTxPnDHkIEXrDeIUw27CWgAnSWolgnQBUWGPMamFlOnudXgcKACwH9j8Pe3WRDKIhDuuxk1xOVddOE3wxjk5NzNFM7gP5ApL1HrOrd4nwFGszwweWlIhPE2Cln4BX0iKD1dcf+6IPfHhwRj38NPQv9OcAuPMpo0GWgVnbjX1ArhjB31ms6uLuEgPjoeqS1QEniCeATlXyMorPEWSkSkF73RC2tkkE8Lhh/Mg9TM/iaHYKnQIefBH4lzC5sT/2vQREhBlwltj/1wyYM7GHlz5sbT61CiOuiXgm+bE/jRLE/vVqovcEADTCjqUplR1Dj+gH8TKAiHQ1mcBT4ohjP4nGxPjCEfDnFHYcf4IMC8hkfO3CvuwnfgwFGJlxFJ4CBJRQpoC1PhJI3z/zLgJgdoSffgTySgLznAAmgAK8IfK7iQ8bX8hAYQ+MoNNAIlHoCyBwZKo0ylcAxIMNNQSEzZI6kHmY4o6Jbdao64mYgDwCiyCBvu8Au6UiE+9aToin5gLQbuDQmBRoFLrA3P77753Wd7wYhj6JGYgihJafs24oJ82u/cys7YjXM0CdnE0x8HA3uPSuE9wjsCNBzDNDa4qkmXoCGyIwhgtoeogb8WzADyAlDHFvqE+jSz+u4VYCXRrOaJ+uD/3pOLrGqYapZPQQ/vb0e3uFQwfGMHE4I+LTGJdKPQhHsfcJcDidwrRzKU1MAcppchcXAciivIg82L9GHtDIpvgxSA+nCdSAdgYkfADPCIA6qHFYdLh+Yt6nu6E3hc5PY5RczJWIzG3sXVt8stPefXcMnF2v9I4t+NEwI9yCSUoGyeB9UH8VCGBIMAm4zwKlvD/z0gRGZr/dOdqHmTwHjCOyL2ALp7V5ESD9EcditiaxHVxgd8/9a6C0d7AmzQFeeDAYQMkcSHKpwYrJZkpCB+pC8Tum+Tr2USYQF7NxCFyzH4yBFACVsm0gNMUiAN2vZ/0MFztDaIvmSYm83XASJSkxkAtYvgZjgN2Y2bbwRiMWh+Wag94HsIwHaQ2La/Lq4rKdJjwID0SYU1zyUG2CWxiTyuWZD9R94QVjRIM1g1KSb+S0AhDBxR5Lzt1Zs9l/xkJ6xrxBxvIysu2jLoDcOQ79tG5J8rDUkwRwWO9HsEoEiv7I7nhlkdozALwAmSGrZCoMvTiOLgVwtXM/RrEUKl2TMEkwQFyfhUOgFZLNpsGVP4YFgMqaI95wYwmtZ1jIINEBHrAranJwni/9fgPpC8DicoWV7IN2OGQxHwSfeDgLwqjR7hytryF5HXvJtO/H8bU4CmgdiPd+vxNB91KJ+Bh2cx8ZBuxUqGoE4Qx44hgQBuolCLI4ttS/ArqKFEJAOwlPgfsCJnl9wETGwLUACKgictQg2Q/TMyK/hw/Fnpd6wE1T2sk/ffqEELthtn4Y79DhBg5RvNuHH7sH+91Q5P7Tna09ba6Vv+8P1PtjZN4I6O+0GmlnQe2pHJrIkCLUjiU3sbE/8dP4+s727kT6V4zApE2mQ1KxYSzZjMhZiwmZiFONSpiUlOQKmlHPQC3waNAWgYxwwyDOj2wDAYWk6CqlhVZ5AmQlRSsirsHAn6aKAKSI4gGTjS4TJESYqxptqROPNbw3/3VyAp+DISpquGAiapwBYCtqOSkKOdaMBMZLktpn8x0+JFOgtIDYF7yAAvV6nf6PZfV+A0sZNg8/+QR1dr0wImmAKSK9niIrPYRBA1kAp06uwwH9wtn+Zwdk8Q5IYBOPlW7cgoQJfBz0Eaq0K8hpgf1t4vdnwPMADyOox9CKc2WBGowDYNbUyaNofH06hoXU2ftJjLxJAAwZxA2Ub3ESYfFLJl/eKSDzOhO4ExO8lzwpuIgySTkyVj7wFkKRBcabBjaYPdpwGmrNE5rk6oBehnXQKWr2KkE+6p37+XEisEPVCeL1mk4UQMAsiGiME7tXvLYQhL3MRkE80fYKLcUMYUeDkfcRZV58bYGKYVXWp9QbiytKfiWlwKHnT6BfZkV/zTfQWkMBKY0G0VjtyfUkmMzGODo583LVkBhp0ybKQgDqp1mf9hskRpZwuOZwnlxlQoHuRbjrIqBXY5+3+mCE0ifb9KJ4GIRsvPEvQCCbEkDUGZKpnHU2RQz06sj4AKADhK1UWpey1YTsqfX9Ggj5IFjF1w1Jv+q15rA8jAsyaZyaJYBDUP+gnXoa1X2pI4DQY2ibiaFu+mKG6wY4nKlxftpESxKqEnE9WyRjoGu2TSFJsMCG71mKHgFI3QJ0MfaV6QjXF+2BCLEgYewe7rV7h696oBnuvds9MZTCIdvYohBFGdabEMgUMDeaAWHgr2AUDJSZNdQitwPrqo8CXwqaikL+gAS6RBnDoItpCt0lNulHoB2DyO8B/SY4kzUUCIDBJ2T3GsTBBGebmC4hH96NowTVxGiE4iLLl4aESmInmbmgcaBJUuFwwhExuKEzsy+gQ9qfG0fvDg56x+3/etfunPRO2m+ODnZOtPz1UBzNYuDavmTru6w2yKefYWKH0hjAXD9IznmvRW7Zh0ErgSGjSNZJaMlJObKsV/udzru27k5DqmfO9WSM/UIbuiteSf1Zamvd0LChu+IIdAfoOGxr \ No newline at end of file diff --git a/bootstrap/led-dynamo.parts/001.b64 b/bootstrap/led-dynamo.parts/001.b64 new file mode 100644 index 00000000..7b5d1548 --- /dev/null +++ b/bootstrap/led-dynamo.parts/001.b64 @@ -0,0 +1 @@ +0qzrTT252UAXoGWQvH1WhtIgHQO87go2g/92QxBQQZpzxQc/PENTMhb8CEprNLx2cbOv0+7jkiwFiPZYAAiGqPpGgOoJv4CZZ0tm4ioRgSBT97JiQlhdx90frRKx4kmkW0gbCc2zbEQkQGDjIQmYuIiGPzC8Cz0xWbtKlYQ+x2j+vmcMgDu0/947CMQxdoCV+W/TvKTYcODf14EdXRKWREC8BqiJiOkW/t7Win6XvGAxxwGTyRoJywfZdo26E9OUEo4t9q731aSmJaxatq3yuob9N7nDFUPuk8zXku8svANqAAU5yF54KP1Yb34DnlBPSALKXirDHjpzvtQ/Y+GtuO0oDLE6Q2iCHQu2ZeVXmYDEB+sIVEvaEdF5kUjrEGooSA5DFu0mwJOBtU3HswSghaczVNxCfwbwxwsIfLgrSKODIyUfLWLU1O5WQwktyexbiVpNvH0K9l0g68um3rBNIQcesfdFCVVAKVNpUzkhKxsSAvI92BHPrhNCEiqY6HoQrWYT9tJma3292ZTKoyBHH/bnKX9a019SkOdraOFNSXbUHBTQFPRp24StMZlN0TKAKItPYQbQkzCBiiipSwXXkRpbAqrbAEpF0rQn9Y4T9FlkRgDWro0d3RGfnH9Pz08d3KI/KZkiZK5EJq4EpZMZvBkLmAZY1mO9w5P5EfVhrJzt2TVjw9Z2amVrWPrclj63pc9t6XNb+tyWPrelz23pc1v63JY+t6XP7Rv53HZRAXDSiI0qH+R+9bEboq4stkCgKSh9K91Qdom+N52W08SXPgtp9BLE7ydkQ2Ehkt6BJIOvDCWbXu8WlMcFovGUrwEBGroKwjtL02niNhpsRXKA7hqZHtAoG003/KDNpTBJH3HFx0Ps9I0wR9pCsU5yrATefOiuDH20T3dXPopbWauHKrcqbmI6iQcNUNScOKGl8uAD9L8fDCuzMPFGfg932OpH7Azsi4JAue7NHvJzNp6Crqp+3m7KUunQdUeTdBOrAQdgu6Q4bu8evgHRDgTS3pv9t733+3snr10xa21Av0Dn3JxbeOcXu/D6XYUB8us2CoWq9NM7Adtl15rU6YcfGIOVPb8/g5W4C5wPRrpLisoRsjBv3P4XiMv/MoYPQmaGFsTZww+Er0rsI9X2UNDHCUhC7xwQC1s0KFcfuW9+OJuII1SoD0FzAA57w+akk2h64I/Smn46Rr1CPr6MUlghxnd+oYrc3jGShceAneMtReyxSWmXNgXVQfxODljCYC17eeZjN8y3Ch1Df+TNxinhQv7u9UlfCv0kYZwoKNl7gPS8HJBZHj5MkVeHvgtCRDReoOmIEG43y+9cc0YUQpmKRqEodr1SFfVt6KbCzfcb8+owfC5fMuvGK9eVJCBBAa8dz58JaEJaOP3KI9CMR9QCiGfQ6IsKaPKZXbwdx1G8rSqTfXOEyvTIoekUW1uiaX5lSynwmFBA1UoOUAXwabrjWcICkeWUxGDpDP63H0eA6OpmBva22D5Tztd3QNb/HT3gIWyLJ83vN8Tnz1a3tsXT1tpX9WsYTFBQQ+Xdvxr4qIYpWxmyWtiRyU6SzO/f4XmlUq3yM1GDMe9kjQN2PQtTY+pnCfq3jN7SC+DOIFZVsrFWxePiFx5w1tztf44zdsjHvac9hnJAButbhOn9wZ0EQQwo7Q367GT/DlEd4J+glaDFWT7t0jTN1teqi/PphXi0DO5ph+SP9U3egO6LDui7IFlmL9HDkXt9F69MpkC2NqukVz22BfdAb+8lKM5DW6P1tQUgai+wDVW/dvNzvwDMmCbCBsjvXGOSDCzQsYthz0t7szC46k1ws9l4Mpfj05CZecMgFY7XnoC8OaeGHg7XmkPOudeua/D9AkgeUEUy+DzlZW9cV5KeuYHcQSbfZAsJhg6Q1KRSdYKk50+m6TV09Et4pvJbBEPm46iqAS8nUPfzb6Tqb9QBikj7si48oD7MWRfYH/TNAHaremuZU1a8gC3Q+bJNkKOBeFXqLdATpIoAexHctgB+1QeSWAyT6G4A4cbhzaUKW2Bro7f+/MlXIZS3vsT2FWELoGHBLk1b4MI74O9mnYqNpN4psRB09KE6hTof6tv4bupdjyNviK8X3QrYsbcrg5huFNNnawp01JTfqpIRdc6iy0puYapvu2PfU3vHEVoZboAcYXNRrFvc1v7qyGhfIAiJitf+eByZ5MMGkF5hj8L/KPdoTyq9JUV0BAXoya7CvvFOFr2V/+4Mzs3GpSGz2Loqf6I85BUGrZ/V/BxF4akJ0ZqcHDBaEHbzQ79kTHLBlED55opdAV/mhjDxrnqlGh5+KGp5rGmPo7g39KfpWa8PEq2hu1Gz0hncM7U7qa19S2EoN1nmoObRG36bTVOQ1CUP1qKA3h9j3+9N/EkUX/f61xj4kSuAhzfRfQELvTcAhh7MoMghGZZegLiwXTMV51HQi5Mk6A37E10oaG2YhQazGFZa2pP0QH2WJbnr218u9uYQlefW+OnRKhpggwGWqW5qyWE0SV1X8i+yh+XqKiSDGAFFWYKoKU+CH7vi0WSWMpRX6uWL1d42S1P4mkUNc4Ho2s4lRtL0oEesoTRzOwH1kKxPPvbFdblLpd2UGBuMTivoE0CMTCIVbiEbJ2PWDGbTdR9vKn3r4Qcs81EP04il6FHwfQ+2eBKhbJFj7CuDOrDLUg1e/UeutvVms2Z/UKttLf/BXEittee5r6ZRBEOccp9LjB3a6mAUvTV2ZC8BmGnP/9eDCg/JMdVPEBI3es1ms1qooYtrEZMEtOi8ojf823l49mJgJRgS2uNTAYhmir8FvMMM9+BbmCBf6bE2XcS+EkG25sm/WvoZYqidOpXR6q446PqpVHOIY+Wpu6Ki+EIZiy1GXgxi13Uk7fc6wsYykcyDOlehQgUjV3a+msTKQy0voOWVIK0d5EqWaETNO2nhQUWi6+6pvbXc2RSvU2Lkj+JTZeOXBnK0WX+56X++Td8KktNhcVYs3JxYriQXtz/HnXDuX19GeBCH7PFZ/gIUnLorZt/xWYbM8YMKl5NFIxCHP2LYJ8eaUSMYhCddBCr8KXM3QOGLQULlTgMqNovHX++DkC68j9pzD6qnN6Sp4HhJEBd7zKKpkYsbORe35H25YrcRoYFCBGhUMkyAf8uYg8fqGYMO1IMMalCPjx832Le/8tHsnIOByRdIOrF/ivFC7HMZYPRZUg+IfFTMwzwKAmjoawZxFsB4Q2+K6kgO2UxO6IpB3sRt6FA1Ua9D5Tr3OMl5WYhqEnXIweFnB50ySPvEf7orf+PX3RXXmCwjVtKBhdEYxt4obaw115r11lpD1ahJCMHQrm2k15CzHkSNO3qjIVHkMcPixaIVOMk0s5KkJmDBiCKl9AdvyK43b4zRuMAk0CUMBUfeOPFlGeXSxvcfmE0AzQ8lDAKPJ7iyxwJnQkRjrCjDm1ot3ZgQ3YzNm51OSIjSLeDHSRAe0B6DJVq2+K469LXQzA/elfFhAzTqfFNz9oK5rSu13miEFmACi+MNaP6T2QQLNo3v0p6Er9cK7RvWOatF1OuMGeOXY7RWrdTMVySW8ARxQG9pu6pqvnVlx4Omb++a/znIoLCcHDYmBhL0plTcmSQ/bugTS861x7uU/ALSGW0yIR5roNBvGanPawV2C3ynFVbekcJcMP0bFZyLJzApKlBFAtDhCoxsuNZnLejoEO060RjhO+TYxRZk1D8weVfgssdVjwFaZ1GSus+bz5vI+9IzLtfgBDwyWh0YlY5b10H8+yBrcKksQp5OxBlB7oKc6bDJuqbcYI0OVDQaCsWxIjQ63Ni4aCm2o6BNsZ+lvZDcXeLJjNcHdvtSHXiYF8WvtGsyXlj99LJ4FWKvbk4CI3bo5k9O/i32R0hXDxs6XjzRrDQnUWY0Pw93a3fhTlEGHzfggB9FGvJInELmAE1D92KSSkmlI/ldXSNTlD6fpzqoesN9TO4jL5jVlLXz5HfR2E9hdBlKxCQUkSbnharIuZHVczOkgfJRD7l1FY6EfAhIKrxKa0WZ+KMqnm06RldRibiR0HlXqAm9FQC/N82arEncVbom9G4h9wqz/lydQYHkjaEm8rsB7gXKpYAbgAnU \ No newline at end of file diff --git a/bootstrap/led-dynamo.parts/002.b64 b/bootstrap/led-dynamo.parts/002.b64 new file mode 100644 index 00000000..71f22b90 --- /dev/null +++ b/bootstrap/led-dynamo.parts/002.b64 @@ -0,0 +1 @@ +UC7yPUP+D7hBtl3jcO6PBhh8bQJSaseNaWAuKhiqEcm3afyyj8LizkpAluyZp19zaPUVhuNszGfRcjVlZ1WVMXoBxr2rzm7TkQKKjicK5HOHF3QakY3h8qQ6f86xbGLSkmCIedtcG5fTZdIYmstE9dEVlwlH7YV4Fow7iOVcbTdCxlm++GBWfaAAwyyriimTYzZNpTxP8e+GCUIyPIxVB0T1/QXalmj8fa0rICuF5Z/bZqziuhFpzzYaLeMHFkfAAh+zL6WLX4GZs3BUYCUeaTiLLmvMn2tiCoU+mksn66BePQbS/8OjOEMrPggog3NLcYY5wDJoZrtrLMWTlRj5PBpHQPSDwD5POQhQ/ZIbNcbk9qQQIF9oUldh1tA7DLUleYo0xezkBQ9QigUJkqQ3xBK/RX3+hPqeBBfPwqSOK37Wn4XprI4mGCkkU3ijxlWdwsbxfCCdzmhAHwbnsEv/Y324Pniy5j/1vMHTvve8+fT52tOh92zjSas5aD1vrT1b73vfN/st8VBcPHOA3eQhDtNoHHrXDexVPY2iMaz6IPwHh6RnyMWjNtbMGYuBRjSapEhmwXR6nW+ic4nB9v6E2xh40Pl/XKxlpQALMAWk5wIUVnPhL/yPhllakFuyVWL5pEMVEcSeuPRiDHdNSsEU9eockMXOVI6DedksZYoFOtpXkmABxCWdfkAnWqiTU1hnWygculz8+GRZXksFN3tj9Mu0KJi9+WanKAFTRkZL7gsdD8fj32hj4ShYmSOhPhh7CWyDnJeAo4GtnAd8NL+mNz7jTKMjTujsREDnmjQ2KQQaA/89gQkIBmdxNPHtqfGY1eN5zqeYemf47JmcrszSLbzhEMgjyQKpg4lf73t4JDjLmEEBvTiZeIJdG5hl2pnffSBxn8SAQg4B65ziZUDHv/g0RSGXQEmuETxFtkhOgeWJxOWJxOWJxOWJxOWJxOWJxOWJxOWJxOWJxOWJxOWJxD/+RCIqUN/4KOKrTPlZTBH9yuOHsuvFc4d5LYSOIAKkrwoqsG25Cx8/xOAsVPKeXj3bVMcRCXgvA44nE83Aqlo+KuZ2c9HQyVyEHE3BS54C6WxUMZF4bEZ6GHU0pPGOjfKu+NkfvJg917F6FKpWAlWG4If+pQyVcsUjO1yYDhD445EZtJR/zvdPHuuRwVd85GdObFlJeXkQKFdBDezCHzz40NwUZYFgH2uFKO78uSXqzF0nlrJTSuUAuHf3QuBi5SDI+lqhyESGkq/MY3VAHx1XmoVINQkl8dMMRk1cuSJYX6uJa/nvhTee+RgJW82dmLgSL0QTzwJc04/SoPryaHgMaqtcQRNVWImVK5T+mPQAlPptBuHJxowzbo8eUavmAbebfPCgHv+Ha/HYrPx3cfURGqaBbd470ae+Dge9cnU/5a/cyckFGi5vJI4uexPvtyg26OHRh9nzjybkRwboeVOp7EgGHLmCC1GFMOOs8AAuoIzropGhhyHpA9jUKiYBjf2wYh1tQJaN6UxF03HumARsB/M9b1l9z6HFcSrXgJpW1Xr7cTMX2TeCBr8Ta6XnK8lfQ2MBRSmFLaCHfKCX4CZVgQ5Uc8BuBXTDvxcKVnUwB16l6oAYSUdJpgGeoSoALKVzpU+asb5q6bIdnyPUK+xHfYQhywVOgDg8HV9Pz3D7KjvXIrvCZZwE5W1S+XqT2biyUTXfJLN+pVW1e8K7ci8ajZARMCx/7E0Tf2icKLvTW5sdmbOGZW8o0shc/umOU3OEkSA7r4b4SGPvwsfYQKs9c6TecFixWqw6E+8KBr+ZQZmAxDtEDlTJxov8Z7TxRDTwZH/TaSJRwrM6X1qOgWpVsy0D/PTMI/mIm/lOdloWAHUQxO5h/ijd3ON0YmtbAoSWEBt1G6Hyde0OMMd8PHnbRloGzwRfKyValth6SIQ9kGcqZN2WofGGTFATGT3rUziSwGhT0TsJAXB4GzNn5lqeNKfJ4UJyu6xm/X1WhVla43ltVnWcO7KmCi0GkLCGPmwz+pRWFTmWtYTQI0jm7WLUNYGADkjhzXXpRSUDtllSvncFNYzGVW8fiw3osY2KHDsFqWU8m4S60/REPSZ4ig8pDjSv6wpcP0glf35WxueCkWxBPBKVlnjxAitUxYM5zDWbLGQSaqh/F2afFelks/d3BFoTa0+f5vlljlGWs9DsoNyXnHZQr0qEbDMtQrmILWP2WSbkQ6pzjzwschaiNfcsROuusxDfb/wxRyHuOaqgjuoMvLDX95E/4Go/gyfJa4qrhDQ5VKfQ3Hv8cv/keOd4/+BXcXD49kfML7P3q3jT7nR2fmx3V8qPVeS2QTymWZTlgd6Zi9973EJplT06c5MoZi0dKD00cozY7VBy0GImnakwIoOfuS6qNY9UT0xaznPERxoERn3T+NFVUHKkhA6h8JLKhL8qS1o10Zp3DqVYQ/IFL7yufKbBfhaP6V9azNU7WzbkxfKm56JZSg0BbYNBNAwGRXTKhBw5Wae78rp9cHDYXckz0FEQU2y6LZA0gX/APlxjaDVaUeVbZB7e4Bo2FtzScRNRAjhUpy0ED8A3CDLwZrXV4ya+8SQHRp5pzvdLQf+q7pkTgaOuyVbmnThBa4Pch6TFAXbkCh0TqJKOYm9NLm15xL1Ah9gUT7UawWKHLuikUc9LBkHQm01hHvEQrD2NqzurKDF8aF49A4puXrVa+b/PfFNZXn2py4/w65Pv83/XN6zyu6r8OsF/0sr/XVuzyu/l4FslqVcDq3z7vv48aVnlX9nlm98X/trlfyztvwH/mWeVf52D/zz/99nIKr+vyiNdKfjm2JtNq/w/Vfk1Lm/UWh+V9P+nsv60nmT4zOHnIIfPZvGvVf5NDj7BbA6y37nxvs2Vf2L0qllS/vA++lm36fPo3vm16fO/yuA/ZXrj3zb84zL4Lfq7xlRhw++o8k825qwXG/8nmh549g16UG+s8u90/+fM17qNz59V+RaVN6mIf7fs8u/L4K8/nwv/F1V+Yz2jNJPqNtat8r/q8T4z1kjTWDXPrPL/W8M3Zkri8yn9teE37fktls/RTyu3HteM9dgsWY9ren6pZEmvbHpY1+vXpGTueb+EHp7o+TJw2DJ61bL781TDf2ZANv6uf2+V39D4oTX7xCvQpw3/mU2fz1rGyiL4TRv/zzX8jYX2i+81/EJ5Xl8te74e2PPFf5+Ost+5+fpBl1/L1tTT1lz+UNfln5fz8+Zzq7xj92fD+FvaH9cuz1jiv6XlhSyPm77xvle2a35v7igfa3/EYde7vEp5d5GM3Vs0mnGO/+iLz7Tqm8LoyWidX9hOK1mFTlb+iYdbMw/X/9xTrdkY/7jjrMuI7WXE9h8bsS2v/ZoTtX04GgUDDj6UV9rpc5ciuxhMH5eUwO4Iy06G5xl1221jebzOj9o3wrStQO7TCFlY9jz04m8Xky37gySvB25c4YdsW3IM48qXRB4qwVd8J2hNwHrBQ5nwRoV3UEgLbwg6ZgzDdfXFeRylTeHx8ta8AC30FDgTTCb+MKCQjk0O1oj9iQz45bmXhnkO6fBpBWDsDYd4cUyYvpoHh2HcRoghZmfwF81PsO71HHIEs6yTRLN4QAHIwLTRTvjpjtCDT87csO3JbJwGkl7tGO63kUiuQ+gwpsLJ3TBjBG1v8s0xHCFEMd0DDlun2yvtu2asa+J45jhSaRm/vYzfXsZvL+O3l/Hby/jtZfz2Mn57Gb+9jN9exm9/o/jt32d/UyrYoja4ooKq7rHOTgvjRauGtrpYJjndEbKpDc/5RyG/XCQtbKp5bVn6xgY1Ay+/26gGAKTFwO4e6JBU2skVCmGr/i0xCsjONDJ1ff5M1rk2FjCNY/g21wyr9iXNnN5BKHWulQfPVcyBzigYIA9a2ejm2O3M2vVLL5mUgMDXd/RPfb5n8GjHKAGOr/O11TsTdSA2lPWN3hdbD/OtIx8uRT1/WADCb8AMSurj63xt9c6onVwGo7Lh03vbIpo7HoIiVwKLjY23UvFHEOtYbeJjNh57RTe0MbxgiKJPjUUPoHCluiKXb3kQxTQD2XdhYcKzrzt0YnOQP/rgCcg7l2zqtu/QAsysIceUOYzqxnVanK0w \ No newline at end of file diff --git a/bootstrap/led-dynamo.parts/003.b64 b/bootstrap/led-dynamo.parts/003.b64 new file mode 100644 index 00000000..8647ce3f --- /dev/null +++ b/bootstrap/led-dynamo.parts/003.b64 @@ -0,0 +1 @@ +d8GWTJUol+U4qcP/zeu2vviSrjmT/kW3cxUj9XI5r+T9XBIJrnuzS23VRAcko1myC9ByJ2ToaEzuLAzNONdUIS6YV6FHOYysbNqUjU7IRsyDLyUgjIMvGTQqvA9Cskp1vfDplwyG+iWz7dK9ED0MbacYHj+prDZWqxjAE12GGBCajzM0xsABbNV7DrVQAio+mYJp78pv0dBzULxEg88BQLN2Rxw8QMGb54NKd+XmVqbyQ8KiKmqg6n4EXTHBEPz8S+/SC9Ifci8paQ9lU06IIirVHzYXuH5Ij1gKACpXeSV35qIsu/2jHI3KMia+Muq8E290SEJmkcMwszloxCR5Nh6NlIT3IxNXvsp1/AciWg3EUd/vxLs8wJQntD8BcdTyn0qCd2CmlI9m0mBDSb+5tLu4gXLK0H/M2Z50SlvJ0rm03E515llSj/CTjiw205FOQJEZZ1lz/Su6A4G/OcToKfrbYaH0vqGY5XEoDE5wPp4Ce6Xbg5B5RzFx13ck4q+WZgtd1YGLaGB2suLyFwrnY9heKo1ut/G3Rk2srnK4pSRNJko7dadeMAo0X2dkEB3RAmyQwJIrn/52YzZtLdBPtYxyJ356Fg1dsXp02DlZ1cwF2OIQc+zBDrwqncZ1xNwqlMxn/lzV6WZhy8CEouKfncO3Dic6C0bXuuMqqkNFlgYjUXmgSTE6r5JSfIn7F9/wUfmUuSR1LsC/3eSo9/aTAigvz9Hfic+UIFYe8vhqRFJ9xGIOfX/Y0Oaty9MI/ufAqkDy5cUhFhAUG4b6BWg5jUTLwcTFpQ3wT1DhKNRXmqgJnBRFcM1UVGphus5DJTIEwRxp5yrNXoBYGqEXqsHSn3o9mhhlQj9tEO/Ub5iWCG4VW0VSlPKEkqjkXL7EaTo+kGn25E05JydH+O9jBOpwNSlJjWZADm+lvJTVq4rHtmQlKesRv72RrahsfolzAgIRnWlSgFCsbZCbDlvnpMLOHkvGDOTW7EJloFqsiiN72Q/SKyHR6OzyvzWdSNwLr6ucJlCLkbACKXOgcLcEItl548Xoz8ptuEChWOjBlgiDMVCyHCG+u1Vj/pcGRN0HPB2z0fU90JjsC/avxt/f0GI4ijCGfOBIHP0d8JATDohGGBpymQp2ufol3XJeU0Wng4cEV3YlfzqRVy11V/IcCuZB1eXlpcc1cHB6nL0ItsR/LdwF0HBAFdVrFbMoA1lFGLCuQeivmQAhtuna3QwkUL1D3GBUkZqnzQ++S/iOKAtSNUMEAYFOzuEOaE1owCaSTP2BTu3KoUWs5siNObQSs+558XwF1dHKsM7rmumrTOWLKqtA+obdlZMND89xJ9/eWneeOk3x4onTZF0bu5gptVSWdYr/BsaFnZgzeFD7GuZQyRaDWJBsaxWfXVhdMKh0dVOzs1XJ51xshXgR1VxF3xI+kYI3R04wXlTMnYM5+SgIMfKEbxyTX+Rpr1czVGxfXETBcDu/97/xplJ9q4lD8nz/sC20QMC7mjrHSC0UdjMaBImfaq9+FwfOFHiDX1n9201OMNkZjyvH/mn7alqJVxt/W62SiGJJEavVEmnhfmEhLyvgyzZuCn5eTjD30ty+aKwm44gIb7G4SnzeY1cX2WNJY181TrvM327R4NfIhRe99/s7SeJP+uj9kuupG77f6bxp9KE7Cd6aq27VkbkGZahoH2RP3Dw41mWMNwtiaIlTug+jtS/f9I/40mgUYx34HUY2yUicBYCzJTAPvU1vc+Dlyy+Dj1bBPPR/wrscbHr1ZZDJiJgH3cGXOdj8bjHgMqKzATwBpNc6B/w6E9YRJIO4oaztsPxv2Re2inZfd5Q00LsYJLA6NrMbR7WDcIsve1jN/Cgsca/eo5zkSuWsWrmvpsiW+zSfJ+YK2oSeb8CmxdzXPDHlPtu0kPuYm078+pHwyCzsiHHreMCcFE6diTetVPBuhiqGlfOs8DMdN8QZiMY+Rh5UPtGMBpIJyPqcp/RW7XhKwE1mMuANgfxlg3aZNuvAVurL+N2y+F1clmLugv5Gwb7aUVGI2V000ncOhMXCfL1pUOfUwGjiLg/2Ja+DiuzN8vWrKyhgzw99w62axb+qdMw1w8uaJSpfOEmzdzWbZE+Xfj+hDhgFpkH2EETl+Z2/Iu7XQg455pF57kB3hIpszgavtCojnlRjQEcCUxaJkLztKJbmr5pRKJVXjKjQHHV/ZhCm3rlfTIsMNIGBUjhFeLGHj7GEmDGB/ks2Jb4EgS/8UfmTOY+9zBCt+kVxSPbNB582ZfQWDFI50VUQiJprDoVBO4Z1wcynZUTtMqJ2GVG7jKhdRtQuI2qXEbXLiNplRO0yonYZUfsnZ0TO1JjfG4/2FeFltg5VGmSG+l1JQNjzYsTWpYrxGpG5O6nDtjSWxSmw75vHq31dMBnV6qF1WhVHQf48iBaBNPEGcSQDjENfndRP6/KM5BkZcjj8GOQc0Dj493U4kK2nyMnqONoSrG4UGxygXsXH4mMYuYICv3GtSjqhIfCrur5jLi5pYL3YACzoOihOKRLgx2L6aeT3iwXYIaW4rrTZ/zNBReoYJDt9e7p/RcdSMUOeh0msOny6Vtss3k1BBMFgnEvMoif9I8b321szcM41wndsF6DrYkzcsXIIyo8RCUkAGCTMGt0FiuBuN+fnyjZvxKuZF8vV8rdlKihEWdAzKZGqiMIkHQK8AahdnKkLoL/2krM33hQvi7uo4RnagevuxIMaXcQEhTskNZ4EmOHt3dv9X3rto8Pd17eqISJX1+V6N8eXIM6e12BPibzhwIOR6WJAaz1G1w0Skuvuwt8D75rSZyI9ue4J/kOv5kU3Sj887CQ0ZVlwBUtPbtYwjh19IC8s5G2rBKzy+lMBI33BnX4hEaFdYFyxA3tlchal29s6JfictOQa10ZHbRiqu9x6Dy8etcIwR0E8QRmjp92f1ucxDKuX+H6YXcWZ5aqFfSjGnHMqpSECPyTTnArJLHS/rMOvKVRRdZSdV654tIq/ggHKmxkYOfW4LJE7qxg3fM7iXiQv6GW8wHVHk9SMMXMoDzOQX48Xf6WsDhD4K/rKaRJ70kqF1SpmFJuDF5VVsuyolHpb0UdN9DADt0Ek8mrOytrTDTMfK8tLWwVCM4nNCJI0yYljT5mm+LckLBmUWtUhSlYfQSKD9phLyZLGoJBn+JXuihFKilGm/FQtLZmLhyCvbC7sc25FHSxItehJZcRM5lbSp2O4b2hh7Ml399Vp3Oglcavqw/9l9bm1jVtXZSUJhg2/Vr0xspWK5jmuq8xHsBBKCmasiCaD4hyRe9lliXCJVir01yY7ebMdTCsQqeuCiFqRKTJ7IPn2dvb2jrsrVWcWXsbetAcNYI7uyufeZ9ofM6Nsd0WGRJsUygZc2lnlOgQZAFjoYHogv7guOn8rj2Q3qhzG6fhXU+D90BP8yuZ6LpAlqJTrz3XxRt4Hle9kAcrwqZ3dLKzJfqDSr6vz7kvfK6qbNSTwKiPsFHE7mo17ydksHUaXYUX96LGkAiPNd5ZVXjq9gM0QB8rHb1NELe72L5iL6dhZfFdRnE0zte5KdI7CWzUPrjw4umNMs8tPLxR7UNvKP82YY5f7ktugt1WGbxn6qyPntZQgKmY4MDPvqh6LhO5cAOceUjJk9E72/DiufKYYrc9mfdd9ubPXO27/17t250Te9Yoh9BxShszoB4Om5KW/dOCFBuposwTFBds387qds+hSjdYZ4CaIADN6zm5gwIB0s1M7u7vtoxM0hxLGMjHYdfHvg8qNebd4tqVhEK5qMBjWzELaX9LTo8DS2YMMC61WC/Nt87fKnRPNydnzc2amav9CJNI17aVI+0KMLYADGn5+9CafXmDsRNN4w0JePLIWGw9cXaeNmkhFLWm6hYKy+SPFDCnDN4mjlWLfsj3gCxYhivQVvZ9AMXxhCEHm6qPR5EZinrLZNiWhOQOi+zKy9rJxYdQANlB1onOcWXM+3x6e9F4dvnu7VxiztYXJYWtfplvUURZDzN10q+E7EQiXDLiCafzFZ37/mdm9mgx+WWOsFKfNKksXreQ7L6u6WsCqmksIq6hFU7KKlFSYZdCPoqkpoSmdxEeyepA/ \ No newline at end of file diff --git a/bootstrap/led-dynamo.parts/004.b64 b/bootstrap/led-dynamo.parts/004.b64 new file mode 100644 index 00000000..f79c4f2b --- /dev/null +++ b/bootstrap/led-dynamo.parts/004.b64 @@ -0,0 +1 @@ +caScp1u6DZjPAUiRGNtRkgpephCWtebkigcOJ0vMA2Me4oDSuGqrdIgjW8kZY36kgfG9HeTbDMKZT1LjPNDBSGKaOY3ceUAowBBaCtOVcoQi3SChjQOQDzKx751v5nPUz0ldr/7ThsqGME3bi+seAzbb/PPAOz2FtdCrVtU9DDiI2ldCo/BXRjB1t/YlOfYJQWhnZ2OFRNQCM68rzZnUTjTxKzCjNroptXv1PmKAGZP04F9QXLZNEHypSwpDf2Eo/duVR5w4/g7AbIUARXvYI8iVRzKDGLfD838HKX0ZHRRQcIRELG+RXwgLpXR7FJlQvhHNFvrKQdU97OZn8RY9IJ+5ENKjpNx55KZTjd7c/o77HjTjtGZMcshHikXKuXNN+4/FNkO6eAj+KvuAZpBMyFTdutAhA+S6r/3xOAKk6p2sVrBH1ITjiNuSybR3R7qIQG+PAVo502xDVpJibY5xxPxP1pfit0LnikVKDCaAnZKCZcYTpIRc0dtq+fVeFh5PVBhORQfklNG/2uck0vQ2V4rEzWJdaIus66o8PWdNOgV8gxQS9+RkfAHmXXEHzAWmBfWsWXgOeA+1HvtnTxWZfBE/TqEZXjClZYst0X1UChnFz+XEYbEHfR4ntNcpXYK18UTNRGZzBbUelKyqM5SOQ9CQQxCqMkOsaT+QNjGkmqQ3CcYg2lfUJQ8FGS2vcpucpJdZFfiz6w7SeNwbGARpmef3kH3HGPSE1nmM4uBcr616P4rOL6N4InY6goIouuH7w+Of9vaPBYYXd0OMOxGOcLrh8bu3MmKRCop6XXrfyRONMId+P/BCV8GsJ+NgwvVmQNfeEOtwlAP8mMGE4UUXLXWQjFqq13FT3eIWKMBZhnrItgruKNGYJTEfvWz0g7DwvRu+67SPuaVu2P6FYnDIbBO2354c/3p0uP/2ROZhzjnZPi5T9S5T9f55qXpBy1sshhcKinf7RvRQFBvxvDJqPZ/OlkJ8OdICH+0Y328Xw4uDmAWlkbtZUt8A1pI3rhPv+rpUvhaucGHsiGSCM56F9mrUmAghgkvM9L6MRp3QlyID6EwIyPW+N3HEPkYvsDEvMWqdehitk0WH6Oy4BBBptht+0pnEKYwp5ugFefhnyDe/yluJ+hyvpPUrivZB9qYndm68MAw0FGVH0FsyRngZursM3V2G7i5Dd5ehu8vQ3WXo7jJ0dxm6uwzdXYbu/pmhu5m68h8I3bV1pW8VuvvNQ3S1rvTFkOimpv9fonz/58T2/s6o3hszDIezj9wW4ntvXqeTcc1yqt9+bZxveRLQuyJ5AallYbhayS8Jvi2E2n5ZpK2yT5RF2pZEOt0fcLozGPhTZIh2A8XgWGXV4FuV5TXXLl+dXcuFMWTxL1mB/+eCVrPz8fOCVltrz82QwAVDSFWsJCX+qC4QZfr5s0r+Mz9ca4Wj6rorLofV3VarGG1UBtybBl8ZmXqp4zz1TH+L0M2MxjPz2+3CoZzv2y8XDuVs/ZGhnGi1/fJQTqz1VaGcpQGabDkui9EkaqvcHYZU+VBhPuu6qKi23570Tn49atOJE2DrjTPgtZtk9Uv8dGuWjuogQFQ/1gQy4cr+2732L73XJ28Oqv/ZkE4rfpMqKCa3Xf0PRneq2+l54wGZxnXti+ofqdYpuuTOuFBt16eAjXwM6FeEgOa3gfxWUBr1OWdL4JvpDa9Q2bagf9ayyNA81eiKkl5kZNwfFIQnod8Xgid7ZIXgGfORBcfNi8uTAObE5eHr0k3+WCKsQP1617w8A+VVxTPpvHFbmROIg62Yc+TzGt8fEqch3hUS9ztC4HJ5ySm2aG7IDifdyliOKx5BP6Hz8cPuyosHw2hA6UORY213wxf4r8CcVVsoCXdX6B2wum2E+mLip55ma90VydiMb6jtwQc0haCSC8qIXLnwkqh9S16kyaSP0kSA/q+BN/a3WgpUGqRjf5vZvspLI31XLxr8kcol6bX8yfnzbvBS4jpfYumyJbI+C2qJFyao9wWjzYl3xU27z9ea0yt4RtOaux77E7JBbU5h38FtqCla8G6zD3rsKWzr4dB92Gq1NgfROIrdh6On+F+JabS63cigbPc0Doabp97Uxeq3yJ88mB8Y6HSW1vqzNI1C6iVsc2fQo1Q36Dw3K9xMgrB+5qNPwqUP3NRD6be8QSMK+t7ds2AIyusmUHTq1ylpEsZhAE/b7EcxblCt6RUQGjBp8fDp06e6ORodj76eRlN3DZ/l6L4ffW8NvLnWpPZfNBTCXzQkTbxAtPNknLV4xuBbi99Mt2XyVoHDUv4YcilmLknpW5S5e4Y+fgaVXtmuZKXsHIfzojFl8Ih3EQyBsvCXpB2kHolD/sYJd7Ulbvt1++DgkHwAgsQiIChZXtUfe9D49hHxaHQAoQ0nArHlBU0iQ02mvg/arMC1A4/hDK/fgGeYti3SjgVFj8NvtF9sv2gwUNkC04HGjuQXLxryPeMaB8U/h8EFtypnHwAiR0B918jTlHRn6Kx80YDicnmQv3zbTOssIWwpo7Hzrxlgu0Pxx5giUhGYzgDJ1ZhTbWG+Zs2BMRc0OlnpntFpHKURkM/W1tYqGypWf1i9TJJVF/6u3rqNhlEYvbK3ICPrTNU6pFsOZUsGmW5t38j+0E4vU9tuUUJtztVJ5RwQQLwqFVEcdu74EKurwFKHQ4qCU3JrZRX2o0mQrtZoD8raV3yVG8LewL97KnxJs12JXY4A3boBQWAQX0/TyIm9cBhN3r3b36tUa9hDd27X8Ct0jQinRgQmBYUeEGGPidB9S5RWmQuE6iko1RrflovRbatjfwTDi/2p76XuKjoWYDWt1oIkmUFLXqrj2fZwn6Zgrtvc+JQ1YctMCb5q6UmrtRuZBVzmUFfZUPPJUMtyoVIO1Fy6dBVVa8TJBaNcNnHYPOK0wp3SHygds1ZfZHVgYWpRwHJj3vWiwZvfysPN/9GRYpn1szxSzDDNzokUa5VHipk23WWk2DJS7M+91P2u+DDp8ayPAQtmFu8sGqxmZDIMQswNnnlN1Z3lOkhMnbxbOEgs6x5GjOkr0O8OCsN8j193pzvd564kK3MEUSh2D/ZLEpiLNklbEd/IlcymdIUI1LXSkdeV50ugFAVbCyV0kE5iGSsmEnT0jMbeaQLUXDdqd0O1/pEsoVPoJ2BHPGltmBNakX9paBfSEypJorvC0htw24M9kLGAM+J2J9ae50rzsXD7nTQVqmSRHVDlPmVnnN8dH5CXXiZE9sII+hcrmXMZObaMHFtGji0jx5aRY8vIsWXk2DJybBk5towcW0aO/bmRY6jf/PkhY7LVYqDYYOwV45SezA+wqtF1bxdWzJhxsfIXh3oZ17arLX7Ojc1/1Ri1r4osKwSU/a4QLZxEyq0YJ8g1O7P+QOUu/BkNqO1wNrHDoBizPTIlucblVvNzIuZvo4UtFE2wb9itDiMbj/eUlfbWjJm6O62hHQ3FI8CBPfwgh1Cx1w+iT5+Ohw1pxjO2K91rLBCBPqyNHBiwkAU/xackKiTK+/jwAyzbCu4SlI+RYFnatHH3dY+M0ZpACveDcksUxTENSi59zoaU6AlSNeSjK1TSyUIol1XFhwlVRfUZZt/OlkJWersHd4+4s3t8eHDQ6xy123uFYfcQz2heVD0mGyOaLVwxWl+b0wBXxd6WgdP0giQ4Sk3ImcE/K6UCB+S/lMhJ/uZcYwWkyUSRu6St65WgEaghK7RhLyREuvdx0Sg2+wbrl9HVC2AVMmKQM6nw3cLb5plvT9PiVkaXrssOITMaRBr8tsxL6DjcTINwJMVlQVScgSIrUJJOJ8srRpQj+BJKmtNaNgGl+ScomYK8V2UrnxK1LL0BhpRkNyTDiOo3lHjQOpRflqeAOlV8Pc+vJLtfkl8hIyhGTTbAmzwh4oBzHI0/1MySRCGlRfnLbUkvpNvK4J2u+4pdWCWliz4tG1/5FAy5rArF2KYfSvJafF2UkgmBIxExvRNfPZaP/dKBJPJe6xwIEPvCdBwiXXg6KOlWVOD/9sHahK9dhY9VJB0jQklGIMmezEkaYmeAw4kzWiZLq9DtV8yG5LDk/cJyECbyNWSZ8hDXi6qlUiWqsZuN \ No newline at end of file diff --git a/bootstrap/led-dynamo.parts/005.b64 b/bootstrap/led-dynamo.parts/005.b64 new file mode 100644 index 00000000..eda42631 --- /dev/null +++ b/bootstrap/led-dynamo.parts/005.b64 @@ -0,0 +1 @@ +YtxmddOIdDHvkv9Ppq1Yur6Wrq8/y/UVe8m0Pr3T/XUMRfqwkV6Lo0Df3MUmkJGVKSFLO8ycg3RSnUGGNTtlYhDe0IO1Hi/uBYtVN+rmtWYlKRPmecP0gz/8uiQJElekQ5s4QXiMEMpuINNS6qijLIesnYCA8ERBhgk6mWARo6VK3QeXR6aRK0Hp50Y6BEaufwWaKSZT6IbalDNEUSwWfQzC8lAvR7dDP4pSkMlB98TwTD9z6RB7RJsjEEl/Bpp7jDYGMi3p8aTIMPA+3CQSKmKHmk9oJ6ZLeJVxDs1RfTRIsGwC7ES5eDwxPbtOaMQ8WabTjiXivfbP+7vt3v7e1jSog8pfbwnrw/sOqghbl0lBF7CzJQsjvcMygcPSDbd0wy3dcEs33NINt3TDLd1wSzfc0g23dMP96W44pUr9+a44o+WiO+5Puz/tWyVn+BZpFkqzK6DbSGdUAKD1dAY8BM/Lpr7EzNozI0HCXypngpzGHk4j+r6C8LyNxvUOHRNqX6V3XD7GxhZ5a4U31QtT5W8uv5Zsj80iu8R4a4KO/RyCLhGEhRQIZOO9eYUa9kvSsGvCsvgqhb+HBt/e06tnNalY96LRKPFTy7t3Q7eYsY9vPyRtI3dTmfT/7cndAICNfX9qFeoZc0vXlZH9okduFgBufpS+BuU3/MZ+mb9CHgKd9Lns8Lu2SMw9/T4N6tLYQTt02fn3WTy+AzYbNebCv9fYUdqk8jIg3matDd3mmx2Qrn/pvd/fO3mN67HVbJrV+Ojg/Hqv26jEcUXLZUbLgFyMxrIAFiAJnOHitXn4bwg05YrvN2oYJz3FHTv0XaCFmY+bMa4h11xQrnsSTclPdJudLgPwOUdL+VUZ7HwCAboH2gJyo8ojmI6aeGSkgZfg9OnuwvndzE+AvgadcgDNtehXkAa8RBpF6RIHNMfYADDbPq0CGwi9elD5jv5FRqygcXfFCHbdPDjDy0KLu6IWu7pRwR8klbWqnVy9mIzfQAp5z/GEb83MjM4vGD2uPbdf6YTVx7op5YfFeLAPOZcVpwmZATljsviYD2/HdDKsml1xAb0K0opNxpjqf6s0/3/+UjsaaJb4PY160SVdb2Pgu5jzHdYHzvzuzvGPh72jn37s/dw+7uwfvsVlXMgBPzB2Fbdkp8mT2sS74p3BVaSZP/GvSsmjvqqYXGb562DG5IOaAhvtB+gDaeVKqPMbPXN50nI0SM641I+cLDQjZYfQ55xupzmp/pA7np4t3OxwPd1NYRyv5wl3QjqYV364vuRqkuxc/g93HqgfZ34M197x59xVIi8nMd2qd96gc8elYlX2ipIGmblvB1KmyPlxa3eCLXOyfri5vbq5/Sj0bYcWNdlEU70HvFQw/UrIyN0W+MNkwXcDOKILZkCTA9W5NMRBzcU0ItnSWrp4T0pWueRWly+mRmyllBjvvdCk4DPOcrDkJlLxzUc5WVF+hw/3psgq3BpFfhpAjyFNcmgKN5a74jNGPW5LSDlR+qyNIioBiZIWFghEQAW0h5aFpuOsNfPrkAVWXDimAFspXKaCHXP8sTdN6CKxolu8Vh5aMScMpbx0r4xrEgJ5BRQCOlQrw1wEFP7HDHLIieuAKDUz2aJ1OLqHEVDN8ZsxqX/clTgCivF+izDUAYkY/pGC7mca52fx+DHnOX+wJZp4mdwsNI+QWwEd6iD8za1sGX9BaxJlWxwEZEZycDE0G1q9LBUr5Bw9bc6RLGANKLmRYxVYfhh54zHmhXAFfOGQitaGohwtFWMNvEgOySEc9tIzP6x8pti1zxzWJo/ucxkzBY+CX13GUixjKf7UWAovHs6CMJoTS7HDX8mw1u4cra9pOVLepCAvs8yCaGUUg77zaH6khN0yhjgk0/W17FG19FVxE18RKSH7QxdJ8M9GbsgqHoFDSMiKaxtRS8JKuiHvI2jv0IZW7rEKczBcOhzpAA3C3MrzuEEiPhFm0LCPvouBh0YQ9DBj2+i2CP2UHDlyOoILtLhSbEVIxtiJP4lkr7WVnZYI9QNnSbxW5ns8Bx6MggEFRI8D4BpktGWndTQmV37fP8Nkdp/e0DD2aBSfrPPLU5H3vEnvnHMmSnxyDwXaTm3jN3JPWLbTQAZByB9A5BIxsylmx9Cnmj8V4H5C3LELc8hoR09M6FNoiS+t8biLL4MrlsEVy+CKZXDFMrjirxpcAWWA/zeWURbLKItllMUyymIZZfH/XpTFFHQHZD9BhB5MjrTQrz7CXpf5OymHt1R5KOIBLSzqBRRV9bgY5soORuuglfcjWB5WXbJPEfvdElq/m8BkAAZ6nEFpS7RaT9eaTYzV6KOPAVpHhbvfHyNFRmFDaoOYA/kf/42WiCZ+B/Z5nkC9J40sOP4f/73mbOBn2rF7lBKKoYE2z17IV/vHb1AcUN6WrW5XhZB0MQ7BUGPnaVC0AKYgCEw8geZk2i8BfZjpS+Y733+13+t09lFehCkcewO/jqERpeWOdjqd94fH95bt9F4fdk4wNOP7Nae18Rz63GqWljtCGYuvVy1+2wEpqruS9//mS2qnNZ4dx+m0PdRleDK10TlIkkXFCzmlztk2JQoew74hzPpk2iNONyDTESzxdAY7+v8xC+Gd9OqI6qZZ6iIKUEmGLbkCraStjV6qvMj6WbosAEKzWFeeGyv/OIy9S/QLkP28wukp+cjuI3keE9pYX+spU/YcMLSDoxlWfubwCEYFXwJhIWREpydciZS7kfUFw0deH8M+uakrqrEXv3z5wIswslGb3yRnpd7bXe5p5OW6nn3gLlAPevTC7kXPwK8dG2QR7GA6JZJVJNpdsemZY79oDMXZcd0F0U0Waz0w+kFdzsbEv9TVAHPbU7NEAGnoFCLVXSFwuZHfCenLZ9VqE//Ntcm/7m04IwUCyEUc8kSM0PMoDZnfza6+m31UvojvxkNqcuu7BPg1wF+RJ0l7KkKkx2dKg0EPc7K/QAFruyI7hxYY7LczIC9stZq/o1uHjOXJweJYhbe4NeW+ZJvSrjxVan8OXgX2qzKCMz5mSh4RIhqRKec2Io8mzb+axhnBnb+X/rhWE0mgrMRrFajTskuMxpEHn2Wm4Y7cpNG/8IpYVH5kMoZCJ6MoYV48lfiN16q8F/6EaYcblO9Ue1b7GHimqFB6njrsAdxJJZHr73h9PdGy/iRpEATy4Vh5k2mpPu+lj1XmZKCa4N+Y42Dsh6fpmSRKnNg9qb5pPW5TemlgoaBWxHdK0FaUqMt76L4GVb6WtSGBM5saCRXMo9NB2EsgCMk3rqMCMD+xKwzK59ssDGJm1yIoWiE93HJkhLmoKW84dlZ2Dm+zgjcYPPk5YyDYNy6IXCU5iy67K7qbxuwBIAZbMeDJ0QLIDyoJugKuc3vbkz2v8hxfLcPLU4jhfjWJQ3mFGSEcx2EPT8Y2zBmfxolQZw4l77Vfam6mEJ8xvyjk9UKRCJX3HWwZCA3/4S3ifjpMLgMMD6FOZ/1Ek4QEJ+1k7T0386dbxEuxM9o7S0/G1FMgMb7TY83KWDOio5y4Vk5YvLtuPgTKBJEXy++GZEZD8Rsd78RAmf19DRDeRCSU12oz/iIw+WgpBtb6UjAlIVUMCcOqNDC5qjE6Rb+z2RA1V6MSmXve5NkU83Lyy0nFLmLe7nFbpLiT9i8n2q1r89ZSdpcHKQV3t/jNWjwcKHESDM6VoMDcTPP7MLrMr3Jc3vi6bm0GL8T6uvj82VzeTpC0J9P0GqMyDbZpbyEAyeD4vFnJuJMOsSN15Y8SOajItuxBniNVRYOym/NuqqCqwUjfv9q8TaiyyHbF7D8jGDDzWGy8q5YATGMP1LYsLoch/91cIHaFLPymrO3cuB9bjLwqvsOIwkrroEwE454Aoutm45J75oTQbIR2+EsJr0ViEaLREFoi0qQD1WbTnHjJcjpbHCyY/IF7VlMrn0uQrMbfc+p9rUyP50oci1ihupzjBHryYEu8P8h4dRUvHfGuK2tPZV+sVWm1yMp/zdbxa7ZaXwIjCnnTsbagknKAqmNfuq/3MZYMBI8KoKhZtcR4jIxWCLXq8weCay7Yzb92OE1m//orBNVwbMkXeu6tytNgekXWffSEZoMr \ No newline at end of file diff --git a/bootstrap/led-dynamo.parts/006.b64 b/bootstrap/led-dynamo.parts/006.b64 new file mode 100644 index 00000000..45032c63 --- /dev/null +++ b/bootstrap/led-dynamo.parts/006.b64 @@ -0,0 +1 @@ +lJIe/UXCU/w1f05oSjsccrb1IeZSv6QsdiqzgzK/1pNgMhtjTni2WhpmV/YaoJWUfER3xKlkXWAr5vUl7YfGO9go/dT3Y0OP84EzBrPJnbErQZTWsRuwf35NvAr0i8ISMKV83ThUJuNZKRBAZr0foc+DfWFTD2OyJcbQoo3Z8ekGGKLZWjdEezhnNfLI95EFtagrihR6hUQvH9EpJNKQfi7pEzBN3pPo3OcZcJTDXfVIat0B+stPvRCEiKGY4YoWn6hCQxZs3GRTUdMzUFOIv8V4FDtrCAVMJTM8C0YOQXLZvI2G/mpCV5SkdQyKwEJAnXjVYxIJulUF1zeGFHiocDEV8fm76zrGcZiBL+F0Il1py5wdy7CSZVjJMqxkGVayzNmxjCZZRpMso0mW0STLaJJvF03CG5SDJkOcQjIJdEl57MJzN1MfQabhb9rq62ZJO+QniQv8pM+MKuM0lgbRczbOILEemuA3aexGPwMmnMfCIew5GKVOLzJ7onmw4YoOaDTQr1GXXDI/kqxx9n6oniqbrdF0MOSCMNh6S5YUQrk+8MsOyFydDsg34uAQxCeUL3ZIjhBHr3/t7O/uHIiXhzvHe0blua4PV6w90aX0MTOF9FFqwODUxfxJ3rppfC2kKsaCzTKEseKnlHsHH53Jb5RUA0gLiRs4tR+nzAxWcQJcftNgtWN1E8QKLnlDZphXaBy7NcuPqAEQvv3EKE3qoFEKn1c3s8ulB/qYrXEzq7qNk5up4NWx744PKquO05g/9au0TKBNB2+TdvAgeZWTPGCblVWsoJsbeDGfyoDlEfcDUG/j6zrbYpXBF+BVqvqMKmPDge3BG1dUs+wAIsjy0ltZLDrXZSS90VFAae0V26Cl3ld+DvlAXTbpmUNrNZtXTzUXWoentaagWK5Eat+5jQYvva0p7jzF3AJsRAaWlNjjVrNk1Oa8LUSDMreDiwZx5b53xVN9UFx/Xze/Q+cKBVrfr2UFWhuywMfNEtybXXFwSVxXKrkkE+KWRsBHWgFj4tEj9QXxVyMmpdGYXyfKQJLZRxo5+9GR/kJSn7S8yOvG6/K6cfFbBMoubEzEoWEnwMghsjMmfJE1Gh+cO5pXNplC6+oDWT/OcNHNJvpa7DrdlqhUEmopua8pZfTJt9SR7zMzWV+PlSVL3AyvF2yN9+B8G3t5Sxg2xufGEG9Wzl3FwRq2+Qy1mrswyjpOYXB8My4VwTZ/mkE7ISA2MU12Iy+sg/5XYyEKzeu846KkVlfp+JPIO5eCb3k//mJ2c2LFfxWLeWbyW8SeTRLxHIu2OYM/gQICC+PfwKB38LTn7h7PW9GYarPGRQza57qdbIZhgqOp8YxHTAfD7Plc9ceMu4ShsDYxM09pfuXxSwKH01xiLwZONPCzscm7SZkzSTc1i9YxHzo1FQ4j4RpumXxVdKZuIJkBHWJSbzp7ybqY1B8MzYPl7AugcdYnNpWCbZTJpFuQglNgM4bSzbqe0kGluRhnYpCOhcauOH+eNGQLxHGyMnhp+LWojwRPTcO4RNy59iZjfRBSciR9ajVIaKtkfKJuiOJMRmubyphvaSlkU0CDPppnUSLBPRd2JLZFoiBPLlAfvZwg9bAiQZDlPdvIKlASSJanK5dm8KUZfGkGX5rBl2bwpRl8aQZfmsGXZvClGfwbmcFRXegDE2vo6EPSBXA2vWnws0r1eAFK63mA2QHfZlGKSqpnlZq0eL4uswy+RzliFX5LGwHCSxpZS3vGupnXVN3Dm4lEFjupu4BpaDjllE9rBjR3SruEpp4BYNo1riw88GDJQYEb7IIBmaOGYV/EUDRlMLC6QsnM7qxN1wYOdOFMWzQyStXlkORg9FV8E8qQd3o2iJ0gKibFRpSysuRi/3CryFU+Aq56RCvRFfujt1F6xFqXWZBCtF07CVwdEw1SlzA5dy3r9REUdulkppkYUJCFJcAA76M46vs5aAjjRz9FDHHwWIMvqKtR29yGDQ6NSN8OGoyZ9P/8KOV2Q1M3mM5c8bQ5qclcR67YePImsOEI1q/M8maFtacbXKOE+i/9/h9B/QD2D6J+gPw7qF/X/hLql4NZlPoRpf8p6m/9odS/ML2uPTXIb33tXnpdswi8tfZ8Lr1qQ9jdm0GHi/0u/qxJcg4DNWcoPzNTzY5qMtvZkYXTer2O8szXdf3+xTWn65r6F+p6q7zrJZOijHyZyS4/M9oM6Mi8tlAXF89Fqw+j1GP/yQSE5k2T3urCFgX4Xcn2zR9KOBt/sEgoPx7TQvmnjctxGvA/RCUfP+3MRqPgyhV1MpMSyzBRsNDOy5Mb+pcn3qnL9t0F6me8q6T+HEyhyPcfQdW3QEyWUvOrcaNA5F0MBTt2YWOFYoC83xgL3nh6lqHBMvPPl2ctLsDNGmwAgQMfcNWJMD5cguhUPiYQ798dH/Divv/SFND10AAlb4cg3nDss7ri0i0T/IV3kwKN4MchHQrgtLVyG+adUvcgc+E4stdOcjGQGC9wPKh/HQ7kXiqvmwfNH61aQ71JTeOZusIgO8c3HuH9xOZbhHQ4TbWbjpkjW3G0crHF5f+6eWzpjNBfJ5NtMIFBBQmMZ9ygU2N1zQeK6WYzZxH7svIMWWw30slUkiM8a6a+KAwkQwsIviiDwiloE1HSHoYslHw2IC3iLZ1EaMSdzstOuxtF8RAXik5Ai/RFFFgTo7HPFoBgBP2oGac20FE0PWOVe5A/PaNufY0F0E58PY0C2Bbn+1JzPcSDPqoL2StlFgZGbHlc8bIhDi1Lfrf7VHWE0tcC7WeYMRLnGGMyfFKY84dO5pONVuFJuV1t/6kj2h45RjXC5PkYkV0GjHhVpsxEgAQrostQnAUJWUi6IRp4yEVkzRAaazM7u28ZyLRtxjDKk8vnR+SzGRZrQM4qwyv2CuV+MZzxJuErPzJ7jJQRTxEFIIxYAvlRKXSucIMWfJytPW0N8G+zyX9FdsPV/K98sXt5fdOXLT2xZGhVpZ+Uw+MojuwgmMqCSzbUYJ7BXXutA8tir0zh2iq/9NguPbZLj+3SY7v02C49tkuP7dJju/TYLj22S4/tt/DYktKTP+wjUx145kkaJUnroz8GM+4xmXBRtDrqk0lEitZnsjip7xYNox6rDuMUb/yt2Z/oCt/cO6XW5N9nZkQnLn7NLIRlXwFm4Z2+/jjfjszom3+fnfoyh4bhxfm3Wmumg1ofJZb+jUdXpJb5DbFkNpFZI3qkeP41J2MegudNSNnk3TEZhJB8Bkw+W9eAHoHSVOfl8gcf9lKnudggsfBZLr2S553dsk7+cHFzpdfEarbMV+dUKK56qAZLvlge+9bxU1nPXOlVJ6GTBMUvTpYhDTmlTL8qaR/NuSidfFjNE+Qqd50pMXuQJLj6sWqdOovOZZ/MdeVIaTmpGK2phKjYjQgYJ8i6lU9ECQEw2L/dzB3ArbU7f7KPRaFyIQ1ESm6XYjJKWj5uHyKzIbEMQxwcNo5ruvmIpDvYucbahoRCm61iOEoFyb2XhxQ0eL6ECDZoEgXRMkbQEdfoEkAwCOmDrqLYdsmt7B+xJLsS6JLR0lJUhO9v/rJb39mL8Zc14hMx/HWOAeGp33nMy7J2/6+P/xdWeCQJJ6gBAA== \ No newline at end of file diff --git a/bootstrap/provision-declmig-e2e.once b/bootstrap/provision-declmig-e2e.once new file mode 100644 index 00000000..2df6f105 --- /dev/null +++ b/bootstrap/provision-declmig-e2e.once @@ -0,0 +1 @@ +PROVISION_DECLMIG_E2E diff --git a/bootstrap/test-org-fleet.extensions.json b/bootstrap/test-org-fleet.extensions.json new file mode 100644 index 00000000..61973a2e --- /dev/null +++ b/bootstrap/test-org-fleet.extensions.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": 1, + "description": "Auditable additive repositories that extend the deterministic compressed fleet manifest.", + "pairs": [ + { + "sourceOrg": "quaestor-ledger", + "testOrg": "quaestor-ledger-test", + "repositories": [ + { + "name": "tenant-isolation-e2e", + "state": "create", + "profile": "security-e2e", + "sources": [ + "quaestor-ledger/quaestor-ledger-server.rs", + "quaestor-ledger/quaestor-web-server.rs", + "shared-auth/shared-auth-server.rs" + ], + "focus": [ + "cross-tenant read rejection", + "cross-tenant write rejection", + "JWT and Supabase tenant-claim mismatch", + "tenant-scoped idempotency namespaces", + "audit-event ownership and redaction" + ], + "imports": [] + }, + { + "name": "webhook-auth-replay-e2e", + "state": "create", + "profile": "security-e2e", + "sources": [ + "quaestor-ledger/quaestor-ledger-server.rs", + "quaestor-ledger/quaestor-executor" + ], + "focus": [ + "HMAC signature verification", + "timestamp skew rejection", + "nonce and event replay rejection", + "payload tamper rejection", + "redacted webhook audit evidence" + ], + "imports": [] + }, + { + "name": "migration-recovery-e2e", + "state": "create", + "profile": "database-e2e", + "sources": [ + "quaestor-ledger/quaestor-ledger-server.rs", + "quaestor-ledger/quaestor-infra" + ], + "focus": [ + "forward and backward migration safety", + "backup restore verification", + "point-in-time recovery", + "closed-period durability", + "tenant and audit integrity after recovery" + ], + "imports": [] + } + ] + } + ] +} diff --git a/bootstrap/test-org-fleet.json.gz.b64.parts/000.b64.part b/bootstrap/test-org-fleet.json.gz.b64.parts/000.b64.part new file mode 100644 index 00000000..34799394 --- /dev/null +++ b/bootstrap/test-org-fleet.json.gz.b64.parts/000.b64.part @@ -0,0 +1 @@ +H4sIAAAAAAACA+29XZPbtrIufJ9fgfL1MJNkfZy91tVxPIm3d+x44nGS/dauVacgskUhAgEGAKWRV+3//lYDJEUJoD7JGY3Im8QjQiIJdDca3U8//e+vCHml4xlk9DdQmknx6p/k2xv8NAUBihpI7qiBV/8kr7775ru/R9/8R/TNX1/ZAbQwM6mYWX2CXGpmpFrhsC+QRPk8jQxoc1v9Ad+B+1IuOYtx3L+/IoSQV/AY8yKB5KNKqWBfqGFS6Ff/JP9jLxPySn2X2m9Wf9jffWU/+Jf7/NVEURHP8N40BWFumWCGUW5HRjOqBGgdLb4tf+ZVKhegBBUxfL/ri+thje/mBeef4M8CtMGnfJUoOjXVxQSmtOBm/asZZaK6qGWhYrhn9u1esSwrDJ1wiGKZZcxEekbXP5ODSEDEqw8ygY3JSHFkMclkUnBYT8sXSNZ/CGrYAqKcxnOawuZM0bia31c5E5GR0bTgPPAMOaiMaV0N5kC1iXLFFoxDCoHZeKMgAYGzZ78gpKhH1YL0I+Pwgao5KBxSf0wmK7IhNHb6pUqjKQewa/2/TnaUnDJuZ6SUHp3Mo1gKXWT2N/9dy0khhLtLMSmEKaLv/vp1Kbb2+pyJxE5COUlrAYM/C6YgaUz6xopEuZILsFJRf8mb84gJbSjnzRH4goGP68WM8lpU3ErlLNKZnJcLSMi/7P//t1rInOGLG0Vjc8qL05zte+kZUG5mGw9r7cTGUxZmFoFSUunmxyyBLJcGJ6z5Ma52xFnGzMboJUy0jOdgouUMRETznLMYdSP86hMllxqUNSknvHn59X1vn3O6WiqWzkzzUfMiz8FA4+s4LcBBsCLbmJg4Bq3ZhHFmNucAdC6FZosN+ZHTKWcCDpuAKS+MwQnICk6NbBH8jMZSR9/+zZ+BBBYs3ivz1V2ooHz1ZeNplyxJwVhd3VhIKhIlWbJ+sKZISB1pFrjQtDSBGVGgDVUmPBMJ6LmRebsoZNQo9hiaAvvFfXPAmSgem09lJ3VzKkQil5vC77Q8KvJU0QQOeNeYAxWRnhUmkUvR8qrU0AnVcKrYV9/fu+xSLalKNkRWcj6h8XxTjI3akOspZbxQEFEjMxZvCX0sRVwotW0P8JmiXIEGtbAbf/jV4xmV+tT3tl/eq+xUGWYfoPF0nHoGDO07mIhLrbd02spo88tAE1ARzgq6ERtvXSha2oXg62qIC/SqTn3j6vv7XtrgNmYipiWn2++uIVZgIgWJ8xia1wzNclCRgj/Au6YA7WaUwBSEhq0J2fAggq/OhAG1S593v3n59X0vHiupdRRzBsJs6rKy7lhOTcBwrz0258nhhq1DGqAgxiVvWd0c1FSqzLqVJ75m4yf2GjAnw9GkQIu9sYozJYt0lhcmcDGDTKrQt7SkG2YA9WZKY5SUXLZZ6Zjv81XazHTMD/BSeN58okV5imkuDKepjr6LQCyaH8MjM1FsPezmG87AOcQ5B7NtqCvT7vw8JtLwC2dxfpZzlsV7Zbg8qbDNjdlIyXXEmTbbnwactLA3t/FFRYXGdd29C8OjAYGTfqpA1z+w760zKtgUNt8unimZbXlfU6ZgKh+dO6WLHN8BkkN2YikMCBPpWLG8aRvDTniRMHm882G/ttdI0dwUCjYfLoF4+2kpE5vbi2EZLlbmRJCJYsuOgYjVKvcNN75365suYaJMfOr6um/ve2PNUkE5KlVT4+K1C9g0UAU3LMq3HfFYAS74jG5OHFpkISA22x+qpF2HWars0p/60vUPHHDS2Dr55Xy1z/+KFdWz9VbjqXWUKDY1AcMYlVIZdj2MVHh+PdXzcF/f98JFziXd0EZ0e7c/U4Cn+k3rayBVB4kzntIZOmA5Va2OFhNTRU/3NaaK7nexlKK4V0cLylmCcbQNt18WyZRTBdFS2qhI4+K8mIASYDZ3pwRyLleZNVE2OBASk7DfoaSRMW4EJzod5ff3arG3q7R6VBnlODc2MJIXZq/GSpWA2rIPCSSFPS0HbNc6bkSZWofR6ld3ftxHlTailuvXw9Pt5jUXeawHJOB2CRczfXVfxn4yKmgK6oYoSJk2anVDaEJzgx/hMuucxnBDKi/9hlCRkEIw1E/KCYgFU1LgEpPSn4XqvPB18+Y21Pgbq2MMuEbFhLOGnYXHHGIDyYN90TpKy1xM8V+NcUwbJtLtEY34lkwwcjPZiHnhZzTf8L5Uoc32OPvZ1rhc8lXKpTe2/jyVrV8J3TeV278U+IGVmUnh3dF9ujV2O2xdH1iV98j2s+3H4UAz74nsh1sj6XKOx+3tsXZ/qyz21rVaiKJMComO96b/ShUkka+G60hjHTDdvGoUNK5VqtRQ9xbh+Hf9LxsIzWyu4A+6oFsPvg7g2r2iGbndGOTUcvMezUWxOYXmySBw2WrOlOLPNAb9a+M+UxkXgbvgk1caS3Il/VOuHfaBLkCQ9UFg6/JUyS8giDsPtz8Dy9A19Z/i3xt/1ZPSMEbbr3mz/Y1MJnZ8OGXQCGkYm6pYgEik2jF59VbSwzNi/qKDW7kj/Y77bKVG/HHOdjvnfQFi66FalxEDWCzeMqvl2StncUDKNg1LveFU2artaxva0v4YpszUxQrwX18FprNdWzct01Nqa7Vux6pq9UTET6yUDhbVGgWcLJkKKGkZYya1sqIDC71p617xPF5VAzPXm/KMSvq8SjqXhjPxAtX0raIJhx17aenokJwJEVDTWOYrYqVj1MyXopmpXfOhqGaMXnf+AlXz5+ItmB2aWe2Q1dY4auaL10xRpGCGoph6yabmBerlAz73/YcdmpnR+OMDmRSMJ9uX6kwlca+A++qooC9GQa3M5tkLV9GNi2XO4Z9b0/lK+k9HGsiljQv/+io0G/sMAChORfoCLYCCCVV/2WEAPn6+J1sZxlH/r0H/Z/A4lO1Zxiyyoe6cU2PzZNvP11TUdb6uh3gxZvOO1VGaJX//q83fUJX9/a8E54HFZMIEtVHybY19844wkcAjQcRoUKnvEAer7E/eyySj3oBkJWjG4ogzMSchCJYd9fn9A3nzmkwKgQfvEph0kAl4YvnaM+8nC1apTlETUNYuVi1jDhasY+XmRy49o30Hi4n/aca0N59UJ9Ptz34O7QIWuQOP9n/uLwJ/FrSJPLwsWXDovR5OARWikiY2y6qwxmOXSGxAMDs3NhbaD2oB6mt1vGNg49lGUZcv9gz2KrOmAXRMc/APDZzFTBaazKSceyvgkJ2QEC5j/6oCmkRS8BVRUprw8qkix4IOBwImS8XMrgzc88lajZHtYU8rgesxjWdwYYYnljxpS5Z4yZCWcS7BTOzrEQWFDkSFrBzUQzyMUvkuShIBBtPZx+RJr8YqrfP4haiApZclLusntGpfP+b2HDnntnFWIbaiochbs+IWKRbyXKxUVMgZgueYJrBnGIJRgg8il7SMyrm8MNnAbUh//YduX+xUfm11u+WyA97Y3axlRExVKom9UeuPFBOSQN56fZ0BrgYMSpIqBJGWfFFWzLVJ0AY+8JLwOLh5cJrnTKSkghJyIIqK1D9iPUD2G+IBbCmqWRGcgEKTXEGM5amxty42gEoyMBSLkQjNJizdQkyXeDuDiHGBQLmYxFJMOYvNRfo2NVKzB8vUqDrdKU0b5anHS9Omh9xLdA3rkYBwKecBq1JFI+xpacrQoy0LXXR7sG0tFJ4/VFa/GWIRmnq2PaKugCVCGjYt4ax6DNodGrR7vgjZhqSfoHEhsHYYofyXH19vxLGbCOW/TC1+azdC+YMtmnDVvwRrfdBIxliPe0PKgtgbUpZHk7om54bolYhvyMPdTw6rXDn0BG/GRLoXlazYopkROBuWfComNZMTxqGuTt5pwLwa68OMWLlGt2amAKY0hNDdGIfrFhb3jSG4tLgKR9u5164i264b+/hAQGCVQLZZ+OhChphPKCkZAn75L5+IjqkgU/aIZUl6F1Bgs1anUa9Tip5Thi5t2/7pPNm2NX/yRNt21NNdkm3zlKB7j6Kq4EcJ36WPzUr/41QR/19+O+BMhNbmWCV7jywBxCXkf/f4AEhZSbkinE0hXsXcD5hwlk8kVQnB4j1ZmGC4ChVns76qrqdyRAOjRl28RlW8Ez1A0Nw0RLbaJpzOPi/n3YWmfCq0QUeCvL5/R8rSvK0ZsnXKJHMHPm9q5dyGjypWglHiu05qH3WfI5LaNpbjj9DFJGGqIb+3KL+DgY2WOmtWeempX6zmfl7l8GAfcY/+VucH9Dh/lok/3QqA6BmdNyqvR/29eP0VeXaI9noTdPW6a8s/L1Vr76jap68/Oh+fOKoL0G2uJyhLN0LHffdF6W1eTA7R26RB7nLtmttnELu5jMjFACLZc+hcwqQ91N2FDWgP+cQzyjmIFDBNUkedfdgHpubbsWZlUEfBYpOPglSMj6RkfHzxRuPlBpn3IRFqAk4NuiZy6jBTeLxanCrt5QsQjz2GtDCIOBkuOUSAIFmUcr/hK8JCOhUhUyoCURp8M/vlCwVb9pgYrEBPGKTuGKgQjINvr2tupL1wW//rZAl6hwhdSO6+93LLv7xnBm7vtoiV7LV7qU2qQN8+FPkmzeY6B2iTgwRx/f7lMmxOfG7ITszlrvnp25tpWbOz7PN2LuRZjXN/8A1LbQfRJr9em2btGHWwbm3fsA8LTQuDz8xJiHi0tNIsZQJh9EnAVq+Z+wjiMbLqTLI1DCGskJS5ppYYxRar4SXZ6vUS9OAAN1O/UTvW+TxA9JF7/maW5vht3/LVUsdU6edXZDwneg5LXxIKA9FUYpUUDgrs6qV/a2bgm30u0yOj0deDc65g diff --git a/bootstrap/test-org-fleet.json.gz.b64.parts/001.b64.part b/bootstrap/test-org-fleet.json.gz.b64.parts/001.b64.part new file mode 100644 index 00000000..0aaec6fa --- /dev/null +++ b/bootstrap/test-org-fleet.json.gz.b64.parts/001.b64.part @@ -0,0 +1 @@ +ABHCYvZk03sTn5LtCiX6tvHvM+D1ruuDpV97d0fKbhXbeVGK4sIWgFPeMmYhUWrc3LQMwTNu4FiWUVFQTqbUY32sbo4/ufZGByZ3TXLdnUK3wcJ7fM4Yv366GCHnLvFI2LbqKEq4jH/cK8yMTGQhEurj5mvWRKKqPiRhZ/IS5WJjTXpFSyUQc6pcoKymYtVB8FR46B62R+fwP/zy/vaNjOdK0nh29z1p/BSpf4pgQQTEBifkhlQ0nQ5MZQWAGMkBm8bAUzM8ngqlysvzTlS9zW7wRrN3wWGaGF6T2+bH9TO463CKmq6XkXz71+jb//CMreuG4HBvMmyQLUu2cwEqpQ0Uf1p4cajXQScHvBOnq6MjWYuunXh47PpdTsR/bJ06yju2r1pXCJGqT0cPx8nKUCWTl663TZurHfwaVdRmsIy/Hxq1cnwIVRYL//AY8UkL2TZZ8xqTP+REj5r7kjS3IfUvWnlt05JNEvcXqrwlXtFqLDK/u7/CXCbOAycVQ+AUKMKPSRpKVdtT/ibb+HkucMsWcHOdAlZXqUSbvQ/aBGzdlumipMssZel6B6w7TRZMS7UKV7i7miB4zJl/7LINF8gEzBJAEG0g11cnXa5XVveihQQkyEluIF/3B9t9dm+XwOeWL70Shj4678ErhC7L2bcahFVvhy0ByZwFCqixzFVhQ5+6FNEzhHSKSXajrs+orTul9ED90eiG8lIlrgwI2ncgWIjagsPQRhWxvWs8w+rYlrCjApycQ0tdbXeaS4wk9Sg0dftS29cMG9q9UMmxrYSIWTK/8Lk657RcDcIn1hEtDkkKCkteNdOuTeLAZMTrXflCnXFOVQrEHqEDmW2EQoTqmbVRlAWKFsugJXEGSIcQMpj7vci6+T59a86iQyheNjolXpSYOC6OFv6XXAHyKJhW5rtwB8YqMm3IdhdG0iwptR0eL1BeNhare5mZyQwmCpZPLDj1bQ3Nfe1+ElnD9FbBKcE+iT7IThrDoU0S/Z7L7uOauWh4cjQtvnzBrvU5qO5BHk8kEXd370mqaJZRRfB9AoQdgqz91sy7TgWLyVQBJDLzJaoQCSQkYTQVEp1ePabUO0upHyAhjZzreUl3KwaYeffx0+gHY1pdqha8WO0P7xxVZgHKbo4XCr54uiQ7Fq4bbFkfSqtXF3cn0t/YuHYJsqoL4W+a/COv79+V9COVS+AYSTB+/AEymFAxb/RKxF9+KXl06vg5ovrFe2UnqVbktl6a8odChASbA09Fexp4NIRlNAWiWDwj+HfoVJAqtMLtFAkN1GcCgvnEpwomUppOU3Pts9BR9m3X7B6TdzvhQa+DZuSgJgPOtoQaDVTa990//Imqrv2lkzYETOqGjms2PCXPqTbgSE4sIVS8Qr7QLDf6IHMw6vWo1wfrNWrbt//HnyT7+X90oc8VHVFj3+6UvOg8Je5P0wNUR2unLUR6tP3ZTJo5rPzMq41C5nSFXeD1qO0vUNv7ozbCEul1ddNhKndKNZS/AK2FUN1pVFnstPZvdbDJnt0+Q+AYPDZhzrlRzuSH643tJl/XWw2s3ikrz4iRdVF66glysMWunuY2m8yjfEl3XsclRTbco8WqPhe7HgzOO9u+1RvO8s9yKYic2KBLy7A1O7JIyGSFP1cmviDxrUBS+H0n1uyQw6P5752YrTtbVNGztfMTwgItiTYKaLabuW3cwk/dwvu829GMbSMT22WqKlKrOXtc1jIH1PZ3mDw4VnGs1RQCYvNkbGyjyj6DyooX3zX0cujXulPVioStjXmtEe9qI2GzuIrVeD6+Gk1FWraRbu1sujV/kayX3c4X0J1Wzxiyca/Im0+/3rWoa5GjtvqB8JSJYHK53q9D9eBXpfNXy6RWVE0ODqnQKl23JwkLf50yMysmXzN5Qq8cnSPOdwHk13fbN5rDyp0MBV2wtAUzocCydtPEf1DKOTEY/DVEsy+X2SyyXKeXRZt2sKh0yZ1WxXkgYcE2SI4CrY2FL0CC5vG/ZxNtpAjR1DZDoYOLMFWJ5Z64nM5IR21vyMczOwnNrF/FDGRtDTLWJEzeigi6EftuZVMJhj7RJh3DX3o9mMAmKrVr6HE4aXl8M9t8FUQZeDlH56ptf/xfDx9/JrIwuS9OTbz6FSKGjwH4IWk41dGCfWkB+a0H7Ab6/cZ0UTOIOzifBfaVx+DbJUzIr+9uiALhtgcyZQngLnBDaIylkyV+zwH/dEw5dNx67KvGDK9f67b+Y2tb7A8W+AQ5g13vd3PI4Lq1WVdZhhJbu1nAvz3GZh5IkSf00M7ih5+JDp+SM85FYVE6Pjpx0sOeFQ856Y5jwuGZEw7Pp+ebLWMupk/MqOfPrOcDzFK4XudXp+H39rVC2p1TkVBNmMgL/yn6BAqMuv2sup2vNrFKYwryhar2vqTlZj+o0UsfjoaPmctuMpeNhbJcIhhAaW0EdVA84mDlxvtpMCH+L2WIziHWYZUOBO3h0enwNv35RVfI9pwFrApfCtZPgVenlr5qEY/t4SsMfyh1EY6vVxmftr7wG5G7SxSFJ+hxjvULe0ThlFTwTik4yH4s2Jcz8sL3nK6WiqUzQ+6LPAcDCEMEDoIVWSC3VyaRvej8KlFBL8JZlmElhK0FjlLJExCdJ4Q7NRs5ewRO3JN6E/Tw21vi2OiwFZrHqS5Ma3eLu/t3LUywTXqPQaV3N0zo01qRY4Xi9zev35IKErITB0I4nQAPYQSMoiEK76SIISGZvFSXoj+bYGsjo9Jni3JQmLDGZhG7uzTuGtevW3qy+Hz7geQyRCtXe59ko8eiXzkVSPO6fKKlhr9QmsLGUvXb3LOaxa7be+6TmuNrIlsBQgFWyl0iUeYUYSF5ESRNZbqqjBhWw07nXEVW7kyfnsaTG5H7n98S9D/u734kbx5+aytmDJDHU592BuFGX6SAYXkcffKBtZ5EzuT+KvdHS/+lW/aA4DWf0su3OvZlLhgH9nSMX5BNIEmwzR6HZtiniQnaGrMbFvSDHew6uOBwiwF6uPtJl/CgdREbgoGw4eANEscsWAJKOyhQFYP4s4ACdG+YoK3XugXKsTQ7aNiCg9tavQcHV3EJmuf7hvrh3eAway+wdeHekc32uv2BnPqNkh0+p3vn9ejSEKkNcT8YcGLMDDI/TtaIwrWF2Kx4nx9fOzSVcJiknZVnaZnkYxIeJz3ldZAkPW/88RgDt/MLz6qMekYTuSR3Hz943BgP977mIiapnCq9I8I5rJBl31jVLqWhDXta+RaExnMhl7a9QRZKsyNeL6lax43m9kRz2+/9Rszps2JOu9TX8zCk64LaUVNfuqaO3BaXqaH7gGJtlBabp5pRP1+6fo64sE5wYccEmTrXZTcRJW6TJEwjJeXh/BWWpmL7VIS/VAZOt+PXVZe1y8zK9YwRW6flqrNH11m5/aJ0dGMeKgK118dl67YOWDuTecNKyVVR7YgmNDegOs7KdS8OkFHGSV7oGVnCZCblnDx8eCDl4+tQG/ZAb7C797+0V/MPKu1WsYFYl+gJVt+G+XtkA7E8HrwlcN1CVly1wd0Xh6nJRBTs7tl3jYJSLyYGN3viMz4zXNtMtRwdpAMaG/JbAeRhAdxARRFA8HV9pHnZGl7PYRnMtVw28fXLxA3uko5TQINttFIeSgNZqBSkIZ70IWME0c2OrJsd2R7huxlbsM37s3oOf/3uH2RGRcKDbiM+P50G2IWaJwbfL7POExK64FH1EllbcNpfGFontPTn4XQcyKMFpUOTSAq+IiVwowWrczI94vWicpCWXUYplxPJZRiUszFkNybnLZcTykuydwuRdLgbqVIq2BcLvDEMD69MpMjYQ3mESLmq4vOGlC6jw+dkBTcsmiqaAdKzkV/f9QjS2XjNW1ikPMqonrUAbwKjOeRG6sPHJ0w+FkeMb4cMBQaHoD2BYS3QnsDILZ+/5demih7wZP1DhOzK2fYv3XovxwjJ3rU5ujUHLRLy+rHIyAPQj58+kIcit43HyX9+/vDfx4EO6gxXx8nog0TxvBB6yxQeFdM+5TEvCfzTY9GIs2JPozs7TGbn2vPw8Im0lgwqWZgg5G6XDl18iWp/UlLuXU8jJTs2ylFKLllKes13HeEhdS4lzsENMbhroCr2klnUGBCJrdPyciCVZ3xkumzIiS87/VF5Zug8+7VXsE6TlorUZmvSPj38dk/C+bFwHqwtazbgBFj/SNIObUcbkrTskBa0Hygko59+sp/e7/1G1OjzokY71M3DUKM7EaPjefoK9HTEjF6mfu7BjLZBRcft8zrUckBQ0TpLEsUziOesa1KnTo7P52A13AsSpnUROhf/8onYF4+YT71R5JzFCA9V8AfsJJfTMRUCSUcbdcnDwHAobPCiAEl8KiqKyDFVPP15+QBJauShjt4THj5v3+Hd659fky+hbk/raSHwGINNXOowowdxFCehgB9eFglV5HKZ5no8bbsjalSnd3tgk+o+BpOCbDlbTwvsJAePfi8fGgeoThUV84BIWK4tElNDuUyHxxWV05VFW0Ylnlc/fbj36JoF3F6CUbdpIRJv/iqg8g4okYKc0xXRLBXUcZoOL0hbx7ORbw6rQNDAdt1VrnthUJL7m8W+FnCJolNDFmv8yTaMRybgEkmEFgkzA2sEZyHgFTS9f++1b1B65VPaVQ+H8YN49Qpwrg/oWjkoFxV7HPfMStmJb7oF8Dm+DyV6HcTMlCzSWaBn4JqoKpwGsqI1KZRPborUZRb/k8tdnt61Oh0WbhbFXBbJlFMFkevhusfW4HfOkyAHcztWDtzDEcQQ+FtNTOMZHFECKRVL diff --git a/bootstrap/test-org-fleet.json.gz.b64.parts/002.b64.part b/bootstrap/test-org-fleet.json.gz.b64.parts/002.b64.part new file mode 100644 index 00000000..01108ef7 --- /dev/null +++ b/bootstrap/test-org-fleet.json.gz.b64.parts/002.b64.part @@ -0,0 +1 @@ +mSBTyjj64D5QoSRJvkiDsnP2LrLv6B4c43GZYH04BmDsNLofwjxlSRGz0g4EEcwbI3YDmO8lX6VcmgqHTHJeaIJxPhC60DekhADdkAy0RoRGekMQj3FD5AQnlVatRm2bUQdjtvMgeZRzKoDg/ffDmIsJZ/GhKOYeqfbKqWvm8OzM7Hbpj88SbKzR7fqvYMui8OAzYkjr1yM2HPD6/h3JqQr49FzGc000ZDSfSQW6eR4kDYNNYiUF+ek3AtyFKNdw+ED5EVNgKxKnhaY8WNdCNJ2C/zhY9DjFojcXPu+6Ldoxq3JypmP7Z09MPpz4rGekO068476ER3Xhn+TV/y1/9Nb96N6cpXddF5OEqUae79ab4JeWQtm4WHZ8+OfWMr3iVKSFm8T/2Z6TtbJvzsS/vgrNyt6WPeWyV50ar9A6ulcbLeNoGS/HMm7+5v7WkgdYRifmQ7eOoVk4zzKm8iqtYipHizhaxMuxiGU/tlhmtztvdlv9P5U7zWbosm80Xzyc9WyDuT0D5xlLB2G/QnOJLzYazNFgvkgX0uH2DzCHKOVDN4j+HJxnEv+gC3qVJhFfbDSJo0m8HJNof/nr8pf/eYSBzOgCxCEGEmV+6AbSn4PzDGSsZ1TlV2ki3auNRnI0kpdjJH90v/n1mwNyMkUK5hCz6OR86IYxNAvnmUZX1XaFhhFfbDSLo1m8uOP0/zskI1NMDjGKKONDN4n+HJyZjOFAs+vMx+CbjSZxNIkv0iTO4PGgdAsK+eAzLv4knGcU59Jwdp3YHfdqo1kczeLLiDKWurg7M61oEloB31yWPzZwexmahfMMpl6y6XWeq+2bjeZyNJcXF288INxohfcwGLgdOnTDGJiE0+xin0S3YQETMoFA/epuaTy+5QYnMi9r2PXXf2gpCIgklyxgijPKsdYTEqLgzwICDZdCbR6FNBEHmuzu2TI8HgUsMo/qurOopfPOGb15OhCriaJMnFIS/Td7H5IqWeQ++46VhnrLDPZ5C7IxMD23BbHIf7stm9iORukZy30yqUsuvu+r8491YFDvNOhoCiJGorGOeaGe0W59/vze95+E3+VNgZ0C72HchBAj5+CzQ2mD5FCllLYykOEMRxkVq8GRQ1VOcdRoLXY9ohXTnMaBI8GUsuBUw2POfG/fShaZFMj64BENMZHIJZnIAinG2GWyCPUoQY19P3KMSl3TB3UgPthetGdB45RlJJZZzsGA3dg8FySbsLSQhSYtHXLjQhuZgSJzWBHbElWFOav2nkaHxVyEx/ao4ta4NK+rZCA4qRHBI40NX0US+Q8xSOK30S73tYyl4RYoSOmFLn4hApQPMp4Hu6lW3Gk2iDEkP6tcq/KEE1U+7ROIVENKDpIqS8Y0oRypeg7+UiW3x8rhF1CynBMyYdRX7RlVPgPf+qhYClLYbQvszFaPCacXe5jsS/7KB29wdS6piWdX5I9hS19trEgIWBLrbgWOf4YiVxaZMh5qMP/w8AMJN1upA7d23gbmidUUN5GgyBDP/+yYxi8sCvVte7dDP7/+/EAmigWcubv3v3gzLRkGwNzjBdrSJ4BNpn279AQNe7YMJd2lSBfPC7ju86T381WfRAvYgb3a3i+PFT3LMU2+/WZO0u1QtnXUDCiBAz4Qw0JeeyyFQAddG6myQLhLqhXJQZFyXMCTK8kGB8ca6CYnUqDZlwsVrtMDq99+94F9T4wk3711/3ef+JykWU7DcSsapDZ2nZhj6e+tQhK7tXKp9fCECVMakeMJjmqe4KeJV+Cte98hSwrkOawCLdVlHiLYX8g4eHKMldS6nCuSgGB+knuILMkGOGR4nImkAf4kDlZ9y/7FRyEIg+DNWKwJl6l/9JFKQZhfO6YqYYKGqLU1xAowPJWEzdh7OWfkXskMzAwKTd4qOqWCDotgGQN+UZ0ve4Kgw+FRURSoSLMEYnoSqXJFxh26o2XhTshSMQOa5JKzEFV/6RiFszsKaKLbiLwHFUCwjOlSLEClsI+juyv79FRM7kG6dqsvlBNkT74JdIyw4IdWvuWtz//z/ZvwKRISgm15hmWPNnhwn8hJYhFNEaS4ce/DbNQa+XLKtxt3lnYbM/I0+ngmUo6swbt7Udh7EZfR3742Z5wTvWQm0JjaelyeoaufeM0wPiyvjNNsktCoip8+hdXbvOVJHW0kT4jtKRHYusq+V34qmmUgCx+staPPDcbGBmW46u5Gzdz0k8D8qjufIg5V1q3Rj+jwNXZdkEg4PewZo+Hh81wfCRsWcbTyXTadaEO1n9J24vXvD+Ttm3vy+kuhgFBBQ4ewn4oJKAGBxhRv6u4aXuAHDLa0IJAV4S1pbELRbROK1sTZifHnuS8GvwWCNqItJ+Zvx3bR2mscrrxlxb64c5wfJhrNgecl0uL8jL5FRkpOdDyDjB6E5G5zJPAsHUnBV61HceeyXKBEbKxEr01MMshgQsU82L+kurindcnvr2/LPtgkg4RRgt+5IZ9+ePh8+/HNpxubK6Ip3JSNMG8IiFit7A/ckAnj3HUzEQl5w1lu5FKQ0qfa265EsUUzjbWnX8lXzUyVfbXb+h39ypnAoLLxcDXV/fU/yZeYjZFLvQcH1jqmVWXXLzWZR/mSBnJ46+uoI9i47GglZgIDWpxOOJD731+39UkOtp8qckxy+lY+1ETVzBQAKWfhIvNA5bP1YNirLtgJ7D0rVkOdzyTV0bKyrQA3+waeGiN7LRIlWULYxweSgJ4bmfs3QyOTYx9v1SJYIcgDVn2IhJTSdYGS4i1SH925nT73VjQWth67TMwp1sVJwJtPv975UdQUA0ghL5GlGP389dP7g7yKYVeBxVLpyPX6i1yBY9dh0262GdvUMNI5xGzKYuKeOABVgyln6cwPTyF4SmDYPeA9/mEdibbftDsXWVDFLjYS0WOYUsYqylkOPbTY3RAMvM8+A3K6j5LRFMjHN598LIor3SVT9hiMX7UdOdalqPa3Q513C25IruQCI1nx0PIwpXcf2dmJWmWjO/kJ+LfVQ9zWT9Pi0/gDTy6pkfnKSQQio/DnJpKqxA95sfwzHn5st04/qlWIJDKK5WSyCgTMKqyxV2gKSZEPTNCq5t/WSLuc7dMKWZdZ43cigUdI7r5vi3u5Br9t4Q0PliYSrDF1brBubxo+KIEpIxQRWmaGFTOH2KbqS+c6Pbs2NxdpOV5oHv5CPn1H3kqJOdw7xRZAmA2pk48C7N9BpGUekJhwgQJOUKoutHy9XJleQLwJ4o3q+NXTuMU3e6zP8TWeaIoaYbiQaQFlszYrzdrATFjdqaQJHrfMrMgmgjKO2IAA1cwrLtMGjG5Y7jOyQiRohnXkwlo9wMGPl6TK2T66TWe6gHg7qISTq8Ofz1ZY+kIUFfNAdYoChhue75OrFEgsVV4MEO5dBskjbfbsSeeEbOyPl3fqPGSji0mdNcCEQN5eqxvK/KcI6MXCEiaTMVTTFA3nyEVul352f+WU3cipNj6lbsv1W3dMFcG9KuyfJEXOWYyUePGsEH7KOWEpaDMs56U8gUTrRtQtHInnESl2lwz4vMrhwT4oebj7qSWzQn5GartgDgkjLSlWOnbKS3hI7rCaipM4CXfN2zEUgSc86BmEhCfcbR8Z4VaX+ZfNGni0mroeIhetoHfYDSSgmj+GQ2oTJiiyiRoFNBuV8sUrJfbTuBalPBwzU4f0gqCZdcBvJ2rmYxminDKl8SSeMgH6hpSrU2NmCGauQSR4qQrN3bgCEjsa0XQOOlOuV4WcgYpecC+EpphwFh+KoOkP8RJHbhI6jtSuI7H4H1Bfx0cbuTfk9ffv/NQh1TOC5VwhxOHDL++ZgZby+JJMuUvrF3jLjsye+70oPtUAHfNkvp17PnPSX6zZdtp+Glk/AX35Cdtlv77vRN5za4fMihjLyturwCvdscQr3ZnIq3729qPuc8Su7ppOX78iNo/BvXQU6DIL2DgK04TmAef60DxhP2rov2FH6rhz6k5TlyOetRP1POJ+R56IDyDc9ybsas/NPfbc7FKT7Zm5RYfdPooJ26nZgWF1BXCgR12+El0eTmPKg7W5z4aRXarzW9v+sUWfl0xBWcvkVbMpJRXJaJ533TdnVOTnU+QBtVM8HO2gEFD7AlT5B/ugbbr88fM9aQHVNdV81OTr0GQFE6r+cogyO/keijYzd8xMJlEZmu64PvHwdQ0OxRsdHeD99OsdUWAzxglg2wOf2kv6yGqjqNAOv0boRPpmoapydD89rOJE/SdnBqIETzIHSQpi1ydUQ9+icnpotDycYbLFHtBamjT+/vr9ccHTuufBRQLbqnXpg95GaoPwjIuSj/NMyX35SoTmOV8dSte2gybp0/uHTjm6qznvnae7R7nRRe5E4Xrk5qF8JRKqRUUZ8MktuCXya91yUKZwCoZlUurKn96oI59BOFxpBgfSdh+e4Gw7KAH6rX5rOSOziTZSQLD4WSpmrGdMtKC5nkkzOIJIC62I3OR13aakyzOqNRIKFkxbh+GmRVJkbliGAI2YQMJ8kXFurm0+olTZaqS9gtCAyphgOhtYIxKLs4ms8x9V3lundMZdSkYsC2FI8KSCBaPhK3WZe8VarMDW/uzlqBVyaRu7Zbuf9fqYiddkvFHlzl2uSFAhBcOGH+0Uwki0ConzV0mbg1rGvUija+121KHcOEjJ/DMkkVhn5qIlnffCxNOlVOAzEjpFvi67nuhYyCxjuzKOLf2Ng/3QJhTfsKZdGybTjoVmRo50e1+T4xPqRLuUh/98/4ZkUkiDtiKg2goyiZ1AC+TCcP1hW1pUtXYlvmCuvR7LPDES diff --git a/bootstrap/test-org-fleet.json.gz.b64.parts/003.b64.part b/bootstrap/test-org-fleet.json.gz.b64.parts/003.b64.part new file mode 100644 index 00000000..225cea4f --- /dev/null +++ b/bootstrap/test-org-fleet.json.gz.b64.parts/003.b64.part @@ -0,0 +1 @@ +60LNeyXgeRMfFgOsZwS39ymNA6TMzqgTJD1hCRCjwK8Iz7LCWGuiYBpookgw81BXA49pkrPSJNdb/GK79VQMPBFm1p7wAH8SWvbzA7GYHocEcElEn6vfMgm1hIxd6pDAQvIiZD4Lgf62IFMGPBkYNccXSNZ62hNn0BExHXycfJ7e4v9jfnwddMoMqV+HuAUL3IPgWdPQeYA2IZ6TnPld9ZT8ArgDX7D/fYaQHF6d82dBAYtnoq2+480ana0xuyt17mQxwW6QwqiqmXmDtRYeIS6MVI3SnV/f3RAUoRsbzHXVOVU81rHFdUxr219RTjk/fTJGbq3F7fbarImkbw784qnGnMY2hKLJH7LAJpia4Joz0KTst6kPZ5scKSVbk33lNO9ziU/I2xwuSien9Ro5/xB3ljbYQTPARX5A4m/M8m3BuRt2N2LCUm1iWXTHwfn+hCaBCTOawJ8FBuBUMA6/PjThkiEjkCcipXxgaDa2Uf8A9QyTCRaDahhYgL42qSYq56/rQEuP4lEzlCj4s7AVctsWOZuwtJCFJsE4227GnJfQK6dPiq3SL4vs1HUaoW8VieqeR4vCH3LiWqgdFF8NtIjfDTCCRxobvoqkiF19uBhUL8mKT8MW3vbBp9G5J2rDCgFajdLG2RPFnwVYH357glc5JMQWF3RbHrT3JTuKtO2avGMCbac/7xmg5NNv+mLrc5+IDKdnzqrOVfh06qpyWx+19zq1dySyumCtPZLPqkJbJopOzaiw16mw10RydWBHqYL1gWFpXa2WBgyda/cBvaUqDxu5fP3DmNPilpZTw8S5LGGyT1yOrwFrXXm82/EJAQS34OId34pys7vcjZcvAF2nbQ4TFpsXGFYhmE2m9gbH3xufa/L09YAA2HIBWtpp7GlB2g7WH1Sqv2I5d70l6N7+Cj1IS5tt6b5LOhMLyYI5Rfeb4cboui0IfOHJxf6EBi1q9JcpPVhqek0CbPtrM6ogifAZbxv/bv/CX358HdE8v8UXKqvbThEubSCPijxYTVYmBnQs81CHsoWMWzLaCRsagHMTtfF86eqbvhMQ8GgAIQ/EThXanxZoUkZNPAv0h0jYdAqYpwyJlCrEwMoQrePYUwPnfWJzUtkyp6ulwhaZ5L7IczAAijwAB8EK7xz0Fru3zCTWGsrCBFu46FwKjU2mBuXqNtu271z0jf7u3aw6/uTpuWgFNCFGSr8Pap1ycmWiX8K9dYMMVxe7Y2xMf69gRC1FoSNaKKaDQMTG9d0gxNd8SVc6kuIW5xojewnB+ZU26asSxCSu23ndIGAAG9viP8tIx20mJ4zDDZL71vhEkRDAhnfYA1etSN3U7cUAFKkL6UTVLNShjj4CWI3Vum2uXMG+TgLHuLbhpx41MxYrmc+kAMLZFOJV7MV3X02lgrJkq1SjkK4GIsPIdySl9w7ffvNNNJOFIprZuToUSHJoBHffHHXF6Wx/+swQ98nPeknlG08QEmRSr/WxlJurVMgcy+I1QvLaFbJRQxmOAbXpY7PlFZlRkfCueSJH3bs+3Ss3+8Z+uOvMWg4+/OzRXIby21/TPA9FPloWDCP41TeTnSWhLUr3nonikWQ0/vhAfmcikcsAuYXlO8dqdL2keYC3beUfqZkhS2ZmsjCobWmrml7iQbbMZfUQvK+czSimORaw7a7+QIf0RGk60nxvCd+xQvR3mkXffptnZBmWoLuHz97dOUBOws0GcReAxLXeWVDud5LHoq0YxeoyQ/p24fqo+ag3yKjeILuOzT6fEKUK46gJiBUS8sh5qP9Jlhti4NEEyqFLjSITQKcd2ztp8Ok5hhh2dXYklgnYzKGhTOzZybqyPI2Q+81xYnZ0xO3NB/IxLzR5/foNKauDfTAvyzBDmOXelZT6n7XxN7mS6pxTg6gGkgDO7JDMUKNd+xxWUdWy/Kks0fFCZTeSKY3hhHy0iEr354fvfvDaS7BUUB5ps+JA5rAK1KmFu7lbXrHqh8PGDsUUVIMCZGipojLgFuWq2M/4cnQr5o6OumfkFL/95psZyTnWN8Kjwe3LqzNxegbJOvzo54n03JKM6QCbEDbGI+X0BTbMOawIl1oPq2mzi8/GENXB2QvyoDoSrRlWL8Uz6tMa1LxEpH0vhEdcdZJRwabNGPqmYUIWwyA3kfPO6wrdYZmtii+15ibtDmvTJid9I7JcO/kMEkZ3sxi6HuEHYrLaCfQqjNfAKFP7LqrrOljbVlPnmlFjijMsD7VXPYZezw699n/PsYzuWcvoulba06voarUe9fbK9HYsoLtcfd1TP0fW2VBPl5lABIpTWz0q7ZUp7YCK6PokS2tbKvypdhB6x0rucsvoMbdFnF0bArQFrWesQBYtBFb/HSYPMp6DIbmSyGilB0idhjlzFFi5J3t2PJZ4V5b+aHk6O7W/u9xuSRcQKsvUQFU8CweFdpfsDQqcXIE99KwwiVyK60OGOAzHjHEgtYkKQHD0zMYSZWEC+VoUDwKPLJiwlRUea1ggkErxohQbDYknChN64tBzQtYxLLsUZIvlIc1OONucvVJ6n93dvxtWUJBLkUaqEJGWdHdXlBwUzigNFgRfdhLM4aFxJ+GgqM12FX7eAjLpE7FNrW1qybHWbTOUfPS+6fwoe37iMr1Ilsf1ir6w0po2MTm3pGbbSw4X2AhJFF1Grpij6o5wgGd8wS01nq6+JgOtXXcNNGo8WGOzNWZ3nc0noEkkBV+R+muIzBIQG6n0DbGFL/HqVkHiWICxiIbylWGxviGuxOYWqymFhYX/ulFqU8UrFWiGZ68YXkydTQVddhCQPrD8W8t0m+kUe2xVv0bz3LexwW90wA4T6LZZSsWCwbKNAYbENJ4FQIBWYohVCNT/LkNbe6ago8hW+8QeE2Y69VkHhuCvD/GVEdmpbDtGHatp8UzJDBr3PULlagt5UuWpUy14tBtGmOC6qrNB5ztrqUK9+/ihBFk3LKzfPyPFk98F7pr1xPdxhJtRo2meR2Uoya1Wx8GkHoWjrZ1O1bLh8MYMKCS2KX3AD9MgElKR+w8qOGSbxqSKZi9ROMjdhy4EAM9yhLOMmYEtPgKNJ1LOI1wjEMgmcAUm4gQJ0OA2GXjMmRpaeJja7gmaGWgsx8Uv/2cmElDkjeQymzAq3hQ5S8j3NJGSvMdXep2BYnF14ePc/b+t4GJtU8LFukIiEYbMbNinfpthSUqf+c0WAdmZ3uz4GBjmPcLUFE1a7u3LUR0TOCSS07BUw8tt9o4j7Vg82mCka8vx+v6df7GMFo2H/7MP/73fcoSQPiuEtGN93Y0gXUdqQxjSUW2vU21HBOnFquuJDRhc5H3U0mvS0gFBRm29W1WK1y28pmWh+i7CW7vD7smIFjTXM9naPoFQIUpghD60JK8svBsYS3qZUIxq76TreuAOD+Hrc/DR/Bjv3pG1/+WFapdl9AXWeVU/0dNSKazHVpmbAZ1qkSIFuZJIbe8QCF232u1cSNZ4PMNiMgUaDOotQKGLjxE7mYAvKTa3KFISQrJPGNUWFGgIVg1P6YWamx577NamPHIx0ScyN42g8M3p+etjJWoq1YQlCQibDLzFns23CXAw3moIMEup5oRyLpec+bwDv76rbROhk5HbaRPeUJPwrFP1nTbp7T7VkFDGV+SvNGtFcFb5IwUClj4NnAZub02mNNwNfkoZx6z2RW9EfXXp7ZMyv0UYdla5tHynd3b9Bnn+y2k09jI59VuW+HwqfXfyCfPpO+/9kLxQyao/bAL9mAnDOKdRE2S/QaHfHLEb3PtfcnLLaTZJKEL2BXLmayqSiXxkIr0h5Vn3hpTBptslTCyCF7uwM9D4j5zTVRPS++HN/YvB8NoUG4LA9vTrO6H4q7kGt+u/8GYnqNABFL9/yAnhtBB+4V/o3BdjVQL3aypSRZOBMfr22+KzTRDaCnRahnfR4HMJk4DQHBQmOGajPbjIfs+rdkVCsGsOjyIFOPV5BwbX7rXav2UNdhCKdqxQKHjM2K5ghSbOjBJUmUY05ABP5sKbE/YMcUk50CxyjkfkHI+ua2g3lj1wv6MXPpY8IcGoe05XthQx3POppbK69KC8iS+FaKTO65bVp2MrsJs6j+zwuMYdtJMdtP97jqCn5+XN61hjd6OeEOtUFaPu0ulRe69Pe0fs0+Vq7YngJ0dyPCrr9SnrgCBQlbK6w8tL0Na3+KQhdf3+h9cfvJld5ZAQUGpn5nFU1RerqjN4HIqq/iEnB7bzOgEjdEDM63heOCgcSxM2FiG6iGOABBKC+XZIyrAWh8T3hbEk2CgqNAsCE2EBwhDbaPFCywB7xQUZhShETIl1Ct3oQQTss7bUdDaDk9vEb1LNQZFYUR3gX8IXJ1MlM9JoDjEgxEaZO41quErnjWk6i39japiJqaLHA8IYB73SBrIWAFgAO6XAD4u/uf+VhEnhQMc0B4Lsb1l+mejCHvFgZcA4Ko3wfjm6AGui5dS0cYjOqErInPFAE1CgovAYdThuMciHzM1l907rzYr0V/vQtvR9Fz9UB3ObPbOQwFDaubFrjBUPzwYN3LVj9I71e6tYQpQsTICpuIEDHBgV0VTREn/HQes9NgEHd7Pap/kHJZ4sgZzLld9OMwz0XROa+WZ2nXi9SB3fOUenR38kTyL71tFOZuDzGYT3Zspvencf8r99Q/J//I3k//hHW0KcaPYFDuMUjqWIC6UuFVvxgmmBW1b+PHhwEFETRArvgASPVMA+UhhXOjKow2Ea4Mb1fRTAusjohANZ97O1xN02EDRF1HCNFEZ0cFUIZKG/ugkOrju27oUIFxPO4udHCPcJ diff --git a/bootstrap/test-org-fleet.json.gz.b64.parts/004.b64.part b/bootstrap/test-org-fleet.json.gz.b64.parts/004.b64.part new file mode 100644 index 00000000..61b00f22 --- /dev/null +++ b/bootstrap/test-org-fleet.json.gz.b64.parts/004.b64.part @@ -0,0 +1 @@ +ZmvM/+3UCF5ZUKwrDZzkveEnQ9jKopxStLxTTSHmR3Wpwd4ZodEj7m1bmrC8Zg/M+Hh/3ZOLnUU83ugC3zjLpThJlnY3qEkUTUmCRyA/COtaFx3WBmm4PWo2VqdvwTlPFOqv2q3chzaaGWSBU90cVhNJlWc9FkwXlBPXYWVg/KO94yE720v29hBu0fAuE76tL9NRprdtgo5JuB7/jGdkd4+/2Qh2fFawY2fqeG53YD0q5otVzBHHeIEKeSSAcd0NuD5bjxr5YjVyiGBFedn6+FYe5a1iLAAeDWnm4keFfLEKORzHVVUh28jG8yIXz9sNRzJS0RTOCSA8UQAT0TFyGlmUIbHv50eeipyzGJEk4evuGEqMnIMIcOIxP6uUsBRj5JeIRXIL10cLjDKyG9Vx+yuRIFX2xPqzAO1jTcql9vYC2+zYCZRfA6wMo9wSo6x8RJNKwWZM9LDkp0wXIYBtDqs1w1nHqMhzpcjidLDnzwlBTyuAkWZJnRwLsXO20KMpWMg4eMXQLPePB3Xf1Kor6rCQkVaPIrt8/WIinsssffuWfU+4jCknhvnL/+033+AA5HNUomVMGBJhZkoW6Sz30Zn4ZjmmYgoFw0NLlJnqyGagOwXZPpcIIXo2gNvnMkCpxHRJ/ujrMGYPSQvm8ogQ/pUgcpcw0TKeg4mqxGXXRT7P5kpXlToHJWI3jEUItq2LCYbJJ6CGxgncI2jbW+ynQmu7MuoWTzcA4tyD0R4Yp42SWpdLEC2Z6l0qzkrhfn4gNlT9Vnp7Ac2auK+aoCh8oEYQADUVVGP7YH14se/ViIHDY8dcFsmU071ScBx4O3CSOQWxXRb1tcDuP31HsoIblgdUu2wO2wbyHhBeu0/crbfM5wFuq8xSCx0vNTOi6RTMSMl7INBWz6iCJNqYng1C3vX13UDbj68LM7v9+O7uze3vMME/xA0p4ZqWbDeWC1CrG7Ldn6MBwP359eeH27v3vzRht5ZWn4n0xRDzOuGOeqWSXC/KbXOBWpGTbV84dd8tEDFZry7BTVUq7ULSulzhIAS+Is14Wd26e4bWSpbEeDpbsGRPRdwpFAx7ZeXo1gl49kZdDi2wVOyLVWsSB5A6QgrsHmUC2O3/+v2nh0BdJRblDus0Ju3y5PM90cFTKvA7F4WH7/72d3L/05sfAs02mYLYkF8/vSPwiEVQGTU+v3dQFFBySnZ2b03QwhB4jGeYBRlYDHkJE1wvEeVUa0xKdFw827l45JwajIDajVxJC1PAluszEAYznKGdADAfIQyZwypQTlmIEHWaYikTZMJEEgwH1YmKQQGxjTR5VLldl29KPn/8fE/0HJahwF2R2+3EVzYmUg6k0IFuv+7FSSwlx1zw2K4J+5/JDLmb47Vc9NMFrHvxSPFxRYaW4d0diWmOzeFaOIAwHUo4WwBOb8uYhWQx2FNJKiwdVMs4xLkEmn1lVBQ2Zb5gvsTaNCc2sstKX2hgYlaeC3rLmXcvXVziBtKW6sajJeOA7Smd2xF0U7F3XGs5t6kPS35zkIWTxIWcD82hUTBVoGeR9eoiBYV+AT5vOyCi0ECwiWWwy+mUZthxrlxn7/wbw7hBkVd/LOf6cLjNRZyGcV/iNCdL29rIc0154qIiZEE5S4KCI2BJNEtFCMOLndDJDGgSqFC95FB1j+djF13qkaeuq+AaZ5OTAmv2tOQic43mkV7sWua+P/zp/cNoRp6iJ0fH8dS2UlQbVCfORbi1YZEpD/RP64OJeM8bdsVEvHvyjuIFPvWJz+EiPvWeY9Hq83bo6Fh/T69dtcHTUXGvUXHHotbLVdg9ta3EOieYqXLlBaPSDkVpx7rXS1XZlvJXtwgkhO8pK2BHTb1KTR2OU9wnqXZbsOOpSLVLPr+qOiyQyjFSlXH3Q4m11xlfjMoNDLQrKPph/M8nEhR7u4liSQpHhMZOD7faJi0BQUgAE34+Junu/S9ebFIybUVOaxoQqo4awGzFeOkuJbl4obJBxIjLNFKQ0Pjqwq4lcLHs9kHqPLkm7n39bkJYUQ0uNEve3elwDSwBhPTtLP+7zjYgfVL8tyx77/T+Lkdc5nor/MRI9/k07N0ti35eMYECmrSwdScZE2QDzxruDERqezgWF/jFBTOM4jJoOPLN0oL66u7CggcZY82eZdXWM9vzLZ4BnqIUWTIzIxVymtCE5gaUviEZJIxi/QDlK8NifUPKwM7tEiaW5/vh7id9Q968f+eqDGp/dCXiF1Np0GeJQb04t+tlwp9p3Y5DXzj1dJBLbXS9zNq6/wK4duu6XlZiLO/jyN39HNzdoQXfpO8+iYU50MU8phxEQr3PazkIUEbsYPYe8B49dVYwchnlnQJRDYWs4IjUPkMqqp+ied6n1XgtEiVZQtjHB7KEie+xh6WrMv2JolP/ZCmkYVPmogkXGUnw1qmXQPFh+0xz4BnyEnN2fGOIcrOo/ATCmTb1xuEVGz18/NmfSJrq6LsIxOIi+UF4r1tE75CZLjW9DS6z7TesN4guUwC73qSj8P/OKTomEH/Ss54R+D/pfiMY5lnBMF1q5ulAmCVMZlLOR1W9FlUd4S+XqaJH0rpvEiqNynkdyjkgmEtNrFCGBztOjO4PlB2roD/SGCZSzsk7oQ1NFc3IeybmkLwT5L/J/yeLz8UEyGc2/yznZMoesRLSPy7QnNoIB4MWIo4gU7RFkpOM5vn5GdAXluOsjg1JtA41d0qreXRIdcqSImYlA9dt9ZeQCZwUXVMBql6WwRcpwIddMI75LZ9enMaGryJL44FL4VfGWi7FcgaHxLNp2bMjJLKJLJHNE4vO0SH2Kmny1+/+4aMp8FXotKXJi5xOd4XXAwCMAYmBzVD007KgeyFgGQJgCAqCx51omdZCHD7YzJq0kOcFmVdrUVtQxagwAyOSr2xsVKUxuq5p7V4u7h4+b8/Pu9c/vyb4Gn62ouBzpFsut8+25A06HFINjcypjkBGOd3bg+Ii1r7W1hIB5SdEVEZ5CyjCbv5FnoR6qNI0VZBeLB9GrxXLNEPW/4w5CoiuoXPdC4GSLnyeEJqjQFAe2kAxtUJcNtWf6U2Sx7EquVcYd0gCngrCHc6bOqRGkIW74tRuhXMP6tjZJ1YyJBa94yTfKpa08TEr0LkUmi1gWLiLXpm0Q6t8Dpm2Ny9Wi32uie9aePjaggcjtXa3oNjQup8HiK1BFHbnX6PsghDZ0jFo4U0b+bbDkNgHowCye8alCfNt2+s5Xt8Ni/2+3A2mTGlj2e6INkXCpAPF/g6TT5/f3JDUdolz/NsK6TFvyKfPH+5JAghFdbCmGh0bQsaOiNjyQmPlbpurtDOE2/alk5v/SZnpclE31nC9wBYua0ZQ7F4yzFKBnNZ07Hm1rfuZAFmagUJqyZjoWAEIYoshjoO/zqQ2ToJGYOwTAmPbJGIHOLZr49EAyG7f6g9p+S9l1kJPqZcswN6NNsc1YRuBsV0DY3cs/om7Rst2MWJjnx0b27Wit+FjnRwg1yTlWEjVEIIugT37XqcrjozdM3UM3ObkJz4D4nPyPUe47LPCZbtW1tMhs+6Mt9bnUYmvUolHIO3lKu8+Hjmnouf5zaPGvjSNHRC6dgkTZeKo3oN2HoPKweeHUU6KuZ0Xe5HTqaXm1ktQ5N2bHwiiI+YcDuSkamtWbeNyF3ikckvVR0YE+7FGNv4U4XnkaQTmvLW3IbOcF1Xk1Ye9rLBP3XZSoTAB6cjkAsLhNwuewUDMoOTBBTIjG8h8CbKQMI2N6YgGHm70gdAWGbAKkheuL2Kwh4w2Mg9HcxvAoUEJRh2c6BhGe0F7it0pkGTYC/2+vv2NhAA8GlLbA6uNfwo7nSH0OADpxp57F9qKqjfsrTJZHjXinh3Drg4TpWOloqrwqeuA4kIbmbm8bVu9z/pIsXVhDqt2kiq3GTcmaGBcmWAQ8hLVk9dp0UY/NoNLrQm6CiJeEduKN7js6KG2uKIV52oYre+AexhNZ6K40DxgX8UbZfZvveN2jdQ9TCRujs8cHt1OUSE8NwHR1pGszPvNpCF6SXMfBV73YJzAVCogVaPEgXUamlETuRZgq5cBIZhRQxiSKfvnFZmACqK1yxckCkT4FHvh/dv7ww30iOluW/++cd0Wz010DIFan2YK+cDAxxrtPcK6e7YMvUO7FVBediMjzWPpLgT4AHHe8//QnQK829b7NJD3Ov3PBPmpmIASECjX+vzrp59JjMTqwrBA/Q+iufFHEsi5XI1Yb/yiBRLurfzNQU2xiE7ELzRs0ehqbBElfoVwJWHlOcWnFxCheClkUl3keaOxYi+sRqBNAM4kzpYyC1cF1EG78OV9pNmXmg15QspsGs9BRTOJvbAzbGvBmQjTZ4dGejUDh6H1u0XhZ1TPrInpuJot9MK3s1kWVTf0oM6t3whne3d8wXqKU4oPdqy2fKBFQl4/Fhl5APrx0wfyUOR0QjWQ//z84b8DoJ4HGc/BtNVRv5kpmbEiIz8yBVP5iBiDn9huMLet5FgX3xGXsAoSA+F2XzbPwS+9efj0I5nIAkv4GXTMznfk8pxOA9b4yVMJuU5/Vh+s8HxogP5cXw65kfpJ1X59y8tU/IeHT1aHZqskHFdRoLE/jHEtCHKMvkjVVjt7it63kfk5c1CGjskEZnTBdiHkO1Xuxox2qN/hdepGxQ954mFoecLkY/G0Wr6+5QvV8lhmWSFwF8Z/UZFEMeDtRkUfFf1iFb3X3jKt809zdpLOHm0YjlVymue86gJBMsgmoPSM5ZpoQ1ea1AVWc8eaE+5SmNO0DJ7bQQvKWULbjsGtjad2cO1Z9k6E diff --git a/bootstrap/test-org-fleet.json.gz.b64.parts/005.b64.part b/bootstrap/test-org-fleet.json.gz.b64.parts/005.b64.part new file mode 100644 index 00000000..8dc3dce7 --- /dev/null +++ b/bootstrap/test-org-fleet.json.gz.b64.parts/005.b64.part @@ -0,0 +1 @@ +UuWYj8tyao4p5hwVvlOF91sTpAr6bwLZfyshbY+nh1mHUyjNXp51wIqjUukJiAVwmYdZd2ulrrLmRZ4qmkCYSJHEhdLSj+omhTNGQNq6nmoul6RCfY9OwHU7AT0y1/Xa+WeXGj6J596oaia5ghhcr9ZAPbRz8IuMCiILk/u5CyGFexKMay9sO3qZndQ38j0TxSPJaPzxgfzORCKXmlCVFhb26UgkR829Es3tuW69T8LB1jUIwVNO2aXXkBb8z4k1O9VvkAxUit3mMypCDQOr43JWGLc9K0CQfxuWhazh/N4QRfWsFZ69Du7H1MSzqMhHXe5Ulw++cUi8uiqQtD8YKf0kz/ayTif9Aa5KikXX5LFbfsUdMopIkptezi91TvPokmQ0MIq8cTNBLCivyJ0HA5T7NGBvaJKsqmCkHdfMQdrziZ8QZ6kLdpK2wkZE/5S3RWSg9Aub1uCiaIqEVXnBeaTgT6yNawZHzsMDPJug9wMyql4g0mUSeaeoJ9S0DOr/+H2Uapzhxd+XM0JqmfRhUNVcVUExDkTJZcRhAZzU8PAwh7WVYMkdwt0oygKRvQru5hgTAZI9zNcRlylJCkU74Vp7JhGvhKsPbj4p51j5p+geMrajG6l0I90nO7knIa0mkFjBwvgyYQLDTFKtiAq5zYksJhyicgLxWLsoC/+2q40BESjhLlA0m7C0kAXShBkV6CDlvNwJmCWSECpAaVwHtnO6ys4vBXkmqe6rtshZk6i2C503Aeg8bHq0rFZohhrJYGENSwGq3RzGSmrtHoTIyR8YoHSJTaJgWmjKfVO7z26vvRddTGrO3D0WmQB2vcC2VkifKwtDSpgcSjNCevXLFOcea59KNY8aCanrE+rEIR/R7Eose0CpFEkVUgdk7bITiBG7GQuUzDRmh8xhpVv63rZEHNaWGNsIoVi2hRb0SpgZGBZX5pfg/kyk4KtRcreT29bCuNCb5BFnU4hXMYeLFd+n8qUtLwmpDbk2dDp1uWob2iD1THmLNZPKRJiMSir7bUtMAwDxhWzAS+MZxPNALYDTBpdLty5K23awFvtyUV1la1UmP8p/sO5dGjYt4QwHFr8/XyL3yXzt5qy0hjXWFfdVaCNQQ79O0Ooiz5Wrt9/ZWBm76DARiqHYrxcKBtYzq1kkpTM5hx5KqrqO1B1p0I+m775/VxM/TIokBd+6rl3fKRVRwK5W50Lc+sLnQoQzlpUC0HYbWw8AtjdEaXxlHBc5Pb+RwDPZ2pdb4LVDLDdKvJ7CgGLJF3FNXtB1CCNTMe5gIjxcMQ4pEJcFP7qBXK5klmMJ7B+OhStCp0UbKlxyDmeLzKhIeMCGr8X2eDdhTMVdflr93Gq956aDvwxce0UiT3DfKYEnN0HcG6JO9VbiKIyA3YSlG7Vao9G8pJRKJXFkq11TWw+27uwp7rqPJre6gHepf+8AEntvhC4mCVMNhbxFhXzhvLsbFzNqFHt89c+tlXrFqUgLN4f/sz0r/hz866vQfFwImf5lmLpNCv4dBq/Jxk/uQEjyfeECGZCkULa41mebSZFno+m7atP3f/fdZm9HgAPs4Vp3h24V22aixTZu0jV8hf/636/+fwO/dzurigMA diff --git a/compat/mise/README.md b/compat/mise/README.md new file mode 100644 index 00000000..04724a4b --- /dev/null +++ b/compat/mise/README.md @@ -0,0 +1,47 @@ +# mise compatibility certification + +This directory independently certifies the compatibility claims made by +`zed-pkg`. It does not grant feature credit for command names or for preserving +configuration bytes alone: each tier has observable acceptance criteria and +negative canaries. + +## Tier 1: lossless project adapter + +The initial suite covers project-local mise configuration interoperability: + +- `.mise.toml` and `mise.toml` discovery; +- deterministic import to `zed-dev.toml` plus `.zed/mise.lock` provenance; +- frozen verification before and after a lossless export; +- multi-version order preservation; +- typed `env` and `vars` plus task/settings preservation; +- unknown top-level hooks/plugins preserved but never executed; +- conflicting project config rejection; +- `.tool-versions` reported as unsupported until its dedicated adapter lands; +- source and generated-plan tamper rejection. + +The machine-readable claim surface is `tier1-cases.json`. A release must not +claim a higher tier than the latest passing certification report. + +## Running locally + +Build `zed-cli`, then run: + +```sh +python3 scripts/mise_tier1_certify.py \ + --zed /path/to/zed-cli/target/debug/zed \ + --fixtures compat/mise/fixtures \ + --report mise-tier1-report.json +``` + +The runner copies every fixture into a temporary directory. It never executes +commands from imported mise hooks, plugins, tasks, or templates. + +## Compatibility vocabulary + +- **supported**: Zed interprets the field and the suite checks semantics. +- **preserved-inert**: Zed round-trips the field losslessly but does not execute + it without a separately certified native implementation and trust policy. +- **unsupported-explicit**: Zed rejects or diagnoses the input; it is never + silently discarded. +- **negative-canary**: malformed, mutable, conflicting, or tampered state that + must fail closed. diff --git a/compat/mise/fixtures/negative/conflicting-config/.mise.toml b/compat/mise/fixtures/negative/conflicting-config/.mise.toml new file mode 100644 index 00000000..ca1c5f58 --- /dev/null +++ b/compat/mise/fixtures/negative/conflicting-config/.mise.toml @@ -0,0 +1,2 @@ +[tools] +node = "22.4.0" diff --git a/compat/mise/fixtures/negative/conflicting-config/mise.toml b/compat/mise/fixtures/negative/conflicting-config/mise.toml new file mode 100644 index 00000000..864ab3dc --- /dev/null +++ b/compat/mise/fixtures/negative/conflicting-config/mise.toml @@ -0,0 +1,2 @@ +[tools] +node = "20.15.1" diff --git a/compat/mise/fixtures/negative/tool-versions-only/.tool-versions b/compat/mise/fixtures/negative/tool-versions-only/.tool-versions new file mode 100644 index 00000000..8fe36ee3 --- /dev/null +++ b/compat/mise/fixtures/negative/tool-versions-only/.tool-versions @@ -0,0 +1,2 @@ +nodejs 22.4.0 +python 3.12.4 diff --git a/compat/mise/fixtures/preserved/inert-hooks-plugins/.mise.toml b/compat/mise/fixtures/preserved/inert-hooks-plugins/.mise.toml new file mode 100644 index 00000000..e87085d8 --- /dev/null +++ b/compat/mise/fixtures/preserved/inert-hooks-plugins/.mise.toml @@ -0,0 +1,11 @@ +[tools] +node = "22.4.0" + +[hooks] +postinstall = "touch ADAPTER_EXECUTED_HOOK" + +[plugins] +custom = "https://example.invalid/never-fetch-or-execute.git" + +[tasks.never-run] +run = "touch ADAPTER_EXECUTED_TASK" diff --git a/compat/mise/fixtures/supported/basic/.mise.toml b/compat/mise/fixtures/supported/basic/.mise.toml new file mode 100644 index 00000000..88c91569 --- /dev/null +++ b/compat/mise/fixtures/supported/basic/.mise.toml @@ -0,0 +1,15 @@ +[tools] +node = "22.4.0" +python = { version = "3.12.4", virtualenv = ".venv" } + +[env] +APP_ENV = "test" +RETRIES = 3 +FEATURE_ENABLED = true + +[vars] +release_channel = "stable" + +[tasks.setup] +description = "Restore Zed package dependencies" +run = "zed install --frozen" diff --git a/compat/mise/fixtures/supported/multi-version/.mise.toml b/compat/mise/fixtures/supported/multi-version/.mise.toml new file mode 100644 index 00000000..c9824d30 --- /dev/null +++ b/compat/mise/fixtures/supported/multi-version/.mise.toml @@ -0,0 +1,3 @@ +[tools] +node = ["22.4.0", "20.15.1", "22.4.0"] +python = ["3.12.4", "3.11.9"] diff --git a/compat/mise/fixtures/supported/typed-state/.mise.toml b/compat/mise/fixtures/supported/typed-state/.mise.toml new file mode 100644 index 00000000..64f4e97c --- /dev/null +++ b/compat/mise/fixtures/supported/typed-state/.mise.toml @@ -0,0 +1,18 @@ +[tools] +node = { version = "22.4.0", postinstall = "never-run-by-adapter" } + +[env] +STRING_VALUE = "alpha" +INTEGER_VALUE = 7 +FLOAT_VALUE = 2.5 +BOOLEAN_VALUE = true +ARRAY_VALUE = ["first", "second"] +TABLE_VALUE = { nested = "value", enabled = true } + +[vars] +channel = "stable" +retries = 3 + +[tasks.never-run] +description = "Adapter operations must preserve but never execute this task" +run = "touch ADAPTER_EXECUTED_TASK" diff --git a/compat/mise/tier1-cases.json b/compat/mise/tier1-cases.json new file mode 100644 index 00000000..4f717a4f --- /dev/null +++ b/compat/mise/tier1-cases.json @@ -0,0 +1,95 @@ +{ + "schema_version": 1, + "compatibility_tier": 1, + "name": "lossless-project-adapter", + "implementation_issue": "DEN-1420", + "program_issue": "DEN-1435", + "claims": [ + { + "id": "project-config-basic", + "status": "supported", + "fixture": "supported/basic", + "assertions": [ + "import_succeeds", + "provenance_lock_written", + "normalized_plan_written", + "frozen_verify_succeeds", + "export_preserves_source_bytes" + ] + }, + { + "id": "multi-version-order", + "status": "supported", + "fixture": "supported/multi-version", + "assertions": [ + "version_order_preserved", + "duplicate_versions_stably_deduplicated", + "round_trip_stable" + ] + }, + { + "id": "typed-env-vars-tasks", + "status": "preserved-inert", + "fixture": "supported/typed-state", + "assertions": [ + "typed_values_preserved", + "tasks_not_executed_during_adapter_operations", + "round_trip_stable" + ] + }, + { + "id": "unknown-hooks-plugins", + "status": "preserved-inert", + "fixture": "preserved/inert-hooks-plugins", + "assertions": [ + "unknown_sections_preserved", + "hook_not_executed", + "plugin_not_executed", + "round_trip_stable" + ] + }, + { + "id": "conflicting-config-files", + "status": "negative-canary", + "fixture": "negative/conflicting-config", + "assertions": [ + "import_fails_closed", + "diagnostic_mentions_conflict" + ] + }, + { + "id": "tool-versions-only", + "status": "unsupported-explicit", + "fixture": "negative/tool-versions-only", + "assertions": [ + "import_fails_closed", + "no_silent_empty_plan" + ] + }, + { + "id": "source-tamper", + "status": "negative-canary", + "fixture": "supported/basic", + "assertions": [ + "verify_rejects_source_hash_drift" + ] + }, + { + "id": "plan-tamper", + "status": "negative-canary", + "fixture": "supported/basic", + "assertions": [ + "verify_rejects_normalized_plan_drift" + ] + } + ], + "not_yet_claimed": [ + ".tool-versions import", + "official mise.lock semantic replay", + "task execution parity", + "native runtime installation", + "shell activation parity", + "hook or plugin execution", + "global mise configuration merge" + ] +} diff --git a/compat/tasks/README.md b/compat/tasks/README.md new file mode 100644 index 00000000..f00f8591 --- /dev/null +++ b/compat/tasks/README.md @@ -0,0 +1,65 @@ +# Canonical task CLI certification + +This suite independently certifies the product candidate that exposes the +native schema-v2 task runtime through both: + +```console +zed task ... +zed-task ... +``` + +The two executable surfaces must route through one implementation and produce +identical observable results for the certified subset. The test repository does +not import Rust helpers from the product and does not execute mise, asdf, +Devbox, Flox, Nix, or manager plugins. + +## Exact-head contract + +The workflow checks out a full 40-character `zed-pkg/zed-cli` commit, verifies +the checkout identity, builds `zed` and `zed-task`, and runs fresh-process tests +on Ubuntu, macOS, and Windows. JSON evidence records the candidate SHA, +platform, binary versions, and each case result. + +A commit after the recorded candidate invalidates the evidence until the +workflow is repinned or manually dispatched with the new SHA. + +## Certified behavior + +- Byte-identical canonical/staged JSON and diagnostic streams for `list`, + `info`, `graph`, and dry-run execution. +- Alias resolution and deterministic task ordering. +- Real dependency execution with scalar environment propagation. +- Argument transport through `ZED_TASK_ARGC`, `ZED_TASK_ARGS_JSON`, and + `ZED_TASK_ARG_` without argument interpolation into the task command. +- Content-verified source/output caching, exact replay skipping, and rerun after + source drift. +- Cache records limited to schema, task name, SHA-256 identities, and output + paths rather than environment values. +- Rejection of zero concurrency before execution. +- Rejection of live child output mixed into JSON reports. +- Confirmation required before a protected task mutates the project. +- Explicit `--yes` approval executes the same task afterward. + +## Non-claims + +This canary does not certify task-local tool installation, secret providers, +template evaluation, sandboxing, network policy, process-tree cancellation, +watch mode, or manager-native task parsing. Those remain explicit product and +certification slices. + +Runtime execution itself requires no network and uses only temporary project, +home, cache, and evidence paths. Cargo may access its normal dependency sources +while building the reviewed candidate; that build activity is separate from the +runtime claim. + +## Evidence schema + +Each platform uploads one JSON artifact using: + +```text +zed-pkg-test/canonical-task-canary/v1 +``` + +The report contains no environment values or credentials. Failure details are +bounded to command diagnostics and assertion context generated inside the +temporary fixture. diff --git a/config/production-package-release.json b/config/production-package-release.json new file mode 100644 index 00000000..449b0037 --- /dev/null +++ b/config/production-package-release.json @@ -0,0 +1,236 @@ +{ + "count": 19, + "packages": [ + { + "dependencies": [], + "github_owner": "shared-auth", + "name": "shared-auth-interfaces", + "org": "shared-auth", + "package": "shared-auth/shared-auth-interfaces", + "repo": "shared-auth-interfaces", + "sha": "24a5289f25a42d23bf075e9c57f5ee0d6de3da5d", + "tag": null, + "version": "0.1.1" + }, + { + "dependencies": [ + "shared-auth/shared-auth-interfaces" + ], + "github_owner": "shared-auth", + "name": "shared-auth-lib", + "org": "shared-auth", + "package": "shared-auth/shared-auth-lib", + "repo": "shared-auth-lib", + "sha": "a038046932c9cab4dfeba9526d1247fdf094a5dc", + "tag": null, + "version": "0.1.1" + }, + { + "dependencies": [ + "shared-auth/shared-auth-interfaces", + "shared-auth/shared-auth-lib" + ], + "github_owner": "ORESoftware", + "name": "k8s-libs-and-shared-defs", + "org": "oresoftware", + "package": "oresoftware/k8s-libs-and-shared-defs", + "repo": "k8s-libs-and-shared-defs", + "sha": "fb9bf8f85393352dd3ec7444da8735c8af8a2f52", + "tag": null, + "version": "0.1.1" + }, + { + "allow_repo_name_mismatch": true, + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-api-server", + "org": "zed-pkg", + "package": "zed-pkg/zed-api-server", + "repo": "zed-api-server.rs", + "sha": "17f2abdcf473dcfee9d24b19f531e26f8513425b", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-e2e", + "org": "zed-pkg", + "package": "zed-pkg/zed-e2e", + "repo": "zed-e2e", + "sha": "f636118e83a8a3edde7285431651850eddf53a14", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-infra", + "org": "zed-pkg", + "package": "zed-pkg/zed-infra", + "repo": "zed-infra", + "sha": "807ce6eb427270cb9df2f0af676faab310e00b05", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-intellij", + "org": "zed-pkg", + "package": "zed-pkg/zed-intellij", + "repo": "zed-intellij", + "sha": "7282fdbc7046bb2ba58369b59ef627211033e145", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-interfaces", + "org": "zed-pkg", + "package": "zed-pkg/zed-interfaces", + "repo": "zed-interfaces", + "sha": "2687d46b2049669b5d1183d9d2cbc69c63a78f61", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-jetbrains-air", + "org": "zed-pkg", + "package": "zed-pkg/zed-jetbrains-air", + "repo": "zed-jetbrains-air", + "sha": "be357c41573e4afb936faea2b410272b32bf5649", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-lock", + "org": "zed-pkg", + "package": "zed-pkg/zed-lock", + "repo": "zed-lock", + "sha": "803e7050ef0a7baa36aecb215d4530b7c28094ce", + "tag": null, + "version": "0.1.1" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-monorepo", + "org": "zed-pkg", + "package": "zed-pkg/zed-monorepo", + "repo": "zed-monorepo", + "sha": "ac9ecd49b13799f387111eb16c11aa0ea7db1d5a", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-sublimetext", + "org": "zed-pkg", + "package": "zed-pkg/zed-sublimetext", + "repo": "zed-sublimetext", + "sha": "c3153867dff0b560946c9b7e287ed50366b87e6c", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-sync", + "org": "zed-pkg", + "package": "zed-pkg/zed-sync", + "repo": "zed-sync", + "sha": "0dcaee9d9a8185870f7d06f9a077ecf5161d9e7f", + "tag": null, + "version": "0.1.0" + }, + { + "allow_repo_name_mismatch": true, + "dependencies": [], + "github_owner": "opto-sync", + "name": "syncer-rs", + "org": "opto-sync", + "package": "opto-sync/syncer-rs", + "repo": "syncer.rs", + "sha": "27692fb7c7423999eaff45afa5bd64b1bf86ad57", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [ + "zed-pkg/zed-interfaces" + ], + "github_owner": "zed-pkg", + "name": "zed-clients", + "org": "zed-pkg", + "package": "zed-pkg/zed-clients", + "repo": "zed-clients", + "sha": "e79723981b88f8b6fbb2f49e7d3d954e2832cdc6", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [ + "oresoftware/k8s-libs-and-shared-defs" + ], + "github_owner": "zed-pkg", + "name": "zed-orm-core", + "org": "zed-pkg", + "package": "zed-pkg/zed-orm-core", + "repo": "zed-orm-core", + "sha": "a78baf355e8ba3b98fa09780af9f67e8b7c12b85", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [], + "github_owner": "zed-pkg", + "name": "zed-docs", + "org": "zed-pkg", + "package": "zed-pkg/zed-docs", + "repo": "zed-docs", + "sha": "a5e9c713c6e6b6c5120a89e9a0233c7eeb420720", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [ + "oresoftware/k8s-libs-and-shared-defs", + "zed-pkg/zed-interfaces" + ], + "github_owner": "zed-pkg", + "name": "zed-lib-core", + "org": "zed-pkg", + "package": "zed-pkg/zed-lib-core", + "repo": "zed-lib-core", + "sha": "700f1f9578c6633a20693a5b1f52970ab845a740", + "tag": null, + "version": "0.1.0" + }, + { + "dependencies": [ + "zed-pkg/zed-clients", + "zed-pkg/zed-interfaces", + "zed-pkg/zed-lock" + ], + "github_owner": "zed-pkg", + "name": "zed-cli", + "org": "zed-pkg", + "package": "zed-pkg/zed-cli", + "repo": "zed-cli", + "sha": "1bf3d898bc17c4a5d82cf44188b78ffd44c507e4", + "tag": null, + "version": "0.1.0" + } + ], + "purpose": "DEN-2788 exact-source API-only publish/install certification across GitHub organizations", + "registry_api_sha": "17f2abdcf473dcfee9d24b19f531e26f8513425b", + "schema": 1, + "zed_cli_sha": "1bf3d898bc17c4a5d82cf44188b78ffd44c507e4" +} diff --git a/config/production-packages.json b/config/production-packages.json new file mode 100644 index 00000000..e76f0f2e --- /dev/null +++ b/config/production-packages.json @@ -0,0 +1,23 @@ +{ + "organization": "zed-pkg", + "minimum_packages": 16, + "maximum_packages": 25, + "repositories": [ + "zed-api-server.rs", + "zed-cli", + "zed-clients", + "zed-e2e", + "zed-infra", + "zed-interfaces", + "zed-intellij", + "zed-jetbrains-air", + "zed-lib", + "zed-lib-core", + "zed-lock", + "zed-monorepo", + "zed-orm-core", + "zed-sublimetext", + "zed-sync", + "zed-web-server.rs" + ] +} diff --git a/config/shared-auth-certification.json b/config/shared-auth-certification.json new file mode 100644 index 00000000..ee854692 --- /dev/null +++ b/config/shared-auth-certification.json @@ -0,0 +1,9 @@ +{ + "schema": 1, + "repository": "shared-auth/shared-auth-server.rs", + "branch": "feat/zpkg-browser-handoff-20260809", + "expected_head": "a471b20f8f81094f57393dc797b3cad0868c2f40", + "independent_test_repository": "shared-auth-test/server-api-contract-e2e", + "independent_test_head": "f41cdf668bbedab9ed75d01266a2086fa87b9ed1", + "certified_head": "a471b20f8f81094f57393dc797b3cad0868c2f40" +} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..6d9066f5 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,53 @@ +# zed-pkg E2E documentation + +This directory records the cross-repository acceptance contracts maintained by +`zed-pkg-test/zed-pkg-e2e`. Each canary is intended to prove behavior through a +real compiled `zed` binary and disposable Git/package fixtures rather than only +through unit-level implementation tests. + +## Git submodule certification + +The Git-submodule surface is split into complementary contracts so failures are +attributable and the security boundaries remain explicit: + +- [`global-git-submodule-mode-canary.md`](global-git-submodule-mode-canary.md) + certifies the shared `--git-submodules[=true|false]` and + `ZED_PKG_GIT_SUBMODULES` contract for both `zed install` and + `zed overtake`, including recursive initialization, explicit CLI precedence, + migration into `.zpkg.toml` and `.zpkg.lock`, and idempotent takeover. +- [`git-submodule-interop-canary.md`](git-submodule-interop-canary.md) certifies + baseline interoperability between Zed packages and Git-managed submodules, + including repositories where Git and Zed retain authority over different + paths. +- [`git-submodule-pack-canary.md`](git-submodule-pack-canary.md) certifies the + archive, pack, and publication boundary so submodule metadata or unchecked + worktree state cannot silently enter a package artifact. + +Together these contracts cover transport, ownership migration, lock provenance, +replay, and publication safety without treating Git submodules and Zed packages +as mutually exclusive project models. + +## Canary requirements + +A permanent cross-repository canary should: + +1. pin every product, interface, fixture, and third-party Action input to an + immutable commit SHA; +2. use read-only GitHub permissions unless a narrower write is explicitly part + of the behavior under test; +3. disable persisted checkout credentials and clear inherited product tokens + before invoking `zed`; +4. create disposable repositories, registries, homes, consumers, and worktrees + per job; +5. exercise the release binary through its public CLI rather than importing + private implementation modules; +6. verify the source checkout remains clean after the run; and +7. retain bounded diagnostics or evidence sufficient to reproduce a failed + invariant. + +## Change process + +Behavioral changes should update the implementation repository and the matching +acceptance contract in separate pull requests. During review, the canary pins +the exact implementation PR head. After merge, that pin moves to the shipped +merge commit so `main` continues to certify the artifact users actually receive. diff --git a/docs/candidate-validation.md b/docs/candidate-validation.md index 4c0ff45f..3fde30e9 100644 --- a/docs/candidate-validation.md +++ b/docs/candidate-validation.md @@ -84,6 +84,18 @@ semantics. The smoke workflow is now repinned to the formatted candidate above; a successful matrix rerun is still required before the candidate is treated as passing. +## Current full-lifecycle baseline + +The full fixture lifecycle uses reviewed `zed-pkg/zed-cli` commit +`d8763b8dcc3bd5c5b6b4b1d13519065a6a2cb99a`. Its locked interface dependency +and the explicitly checked-out sibling both use +`8428bc574111fa148e590c8350c7855035ce2046`, which contains the canonical +whole-repository target identity contract. Keeping those two interface pins +identical prevents the test harness from compiling one contract while +reporting evidence for another. The candidate smoke workflow uses the same +product pair and rejects drift from `.zed-cli-ref` before it builds or runs any +fixture. + Record the smoke workflow run and any later lifecycle, browser, and install-boundary runs on the owning Linear issue. A smoke failure must be classified as a product regression, fixture drift, harness defect, or diff --git a/docs/complete-constraint-solver-canary.md b/docs/complete-constraint-solver-canary.md new file mode 100644 index 00000000..dd476eb2 --- /dev/null +++ b/docs/complete-constraint-solver-canary.md @@ -0,0 +1,132 @@ +# Complete constraint-solver black-box canary + +Linear: [DEN-1553](https://linear.app/denman/issue/DEN-1553/zed-cli-solve-overlapping-compatible-transitive-constraints-before) + +Product candidate: `zed-pkg/zed-cli#89` at immutable commit +`ddf0487f560c19000e2844b62797f1cd6299c26e`. + +The selected-only overlap assertions were restored and proven on certification +head `05d8eeb4d42957ca6e01e3829798ff2eb285dd95`. That exact head built the final +product and passed both public-CLI canaries before publishing the permanent +script and deleting its temporary helper. This document-only evidence commit +then triggers the final read-only repository matrix under the connected GitHub +app. + +## Purpose + +This canary verifies the complete one-version dependency solver through the +public `zed` executable, immutable package artifacts, an isolated `file://` +registry, real lockfile publication, and normal symlink materialization. It is +separate from the solver's in-crate tests so a graph-selection result cannot +pass only because an internal helper and its test share the same mistake. + +## Certified graphs + +### Overlapping compatible ranges + +```text +consumer +├── overlap-left@1.0.0 -> overlap-shared ^1 +└── overlap-right@1.0.0 -> overlap-shared <=1.5.0 + +published overlap-shared: 1.5.0, 1.9.0 +``` + +The only common solution is `overlap-shared@1.5.0`. The canary installs the +same consumer with opposite TOML dependency order and requires byte-identical +lockfiles. Each cold solve must report exactly +`(resolved=3, workers=5, downloaded=3)`: both root packages plus the selected +shared package, with no speculative acquisition of rejected +`overlap-shared@1.9.0`. It then publishes `overlap-shared@1.10.0` and proves a +fresh frozen replay still consumes the exact locked `1.5.0` graph without +re-solving. + +### Multi-coordinate backtracking + +```text +backtrack-a@1.1.0 -> backtrack-x =2.0.0 +backtrack-a@1.0.0 -> backtrack-x =1.0.0 +backtrack-b@1.1.0 -> backtrack-x =1.0.0 +backtrack-b@1.0.0 -> backtrack-x =2.0.0 +consumer -> backtrack-a ^1, backtrack-b ^1 +``` + +Newest-first selection initially chooses incompatible `a@1.1.0` and +`b@1.1.0`. The stable solution is `a@1.1.0`, `b@1.0.0`, and `x@2.0.0`. + +### Rejected-candidate constraint removal + +The newest `stale-chooser` candidate contributes both a conflicting shared +version and a `stale-obsolete` dependency. A peer forces the other shared +version. The canary requires the solver to backtrack to the older chooser, +materialize `stale-replacement`, and omit `stale-obsolete` from both the lock +and project tree. + +A candidate required by an active branch may enter the immutable global store +before later backtracking rejects that branch. It must never appear in +`.zpkg.lock` or the consumer project. The overlap graph has a stricter ordering +contract: unresolved shallower parents must contribute their constraints before +the deeper shared coordinate is acquired, so the rejected `1.9.0` archive does +not enter either cold home. Yanked candidates have the strictest boundary: +immutable metadata must reject them before archive acquisition. + +### Deterministic unsatisfiable provenance + +Two package paths require mutually exclusive exact versions of one shared +package. The canary runs opposite root declaration orders and requires: + +- failure before project transaction or lockfile publication; +- both complete provenance paths in the diagnostic; and +- byte-identical normalized error output. + +### Dependency cycles + +A two-package cycle must terminate with exactly one selected version for each +coordinate and materialize both packages once. + +## Acquisition, policy, and replay boundaries + +The canary retains the existing recursive-installer contracts while exercising +the new solver: + +- the reported worker bound remains five; +- equal-depth cold frontiers can reach the bounded acquisition pool instead of + serializing candidate downloads; +- unresolved shallower parents are selected before deeper candidate acquisition; +- the canonical overlap graph downloads exactly its three selected artifacts; +- selected artifacts use the existing per-SHA acquisition and global store; +- project output remains symlink-first; +- warm frozen replay downloads zero artifacts; +- fresh frozen replay uses the exact lock graph; +- failure does not create `.zpkg.lock`, `zed_modules`, or transaction staging; +- a freshly solved graph whose only common version is yanked fails with + `--frozen` guidance; +- the yanked archive SHA never enters a fresh home before that failure; and +- an existing exact lock remains authoritative after the selected version is + subsequently marked yanked. + +The product's ordinary integration suite separately retains the opaque-version +compatibility boundary: opaque package identifiers are exact-only and a semver +range such as `^1` cannot match an opaque tag such as `legacy-api`. + +The workflow uses one hosted Ubuntu job by default to conserve Actions usage. +A manual `run_arc_smoke` input repeats the same executable canary on the +registered `sonus-ci` self-hosted ARC scale set, providing a direct parity lane +for the AWS/Hetzner continuity infrastructure without making PR checks wait for +an unregistered runner. + +## Promotion rule + +Keep this pull request draft until all of the following are true on the exact +product commit above: + +1. the hosted black-box canary passes without weakened assertions; +2. every inherited lifecycle, recursive, browser, durable-manifest, and mise + workflow in this repository passes on the certification head; +3. every product workflow passes on the same immutable `zed-cli` SHA; +4. no unresolved review thread or change request remains; and +5. this certification PR merges before `zed-pkg/zed-cli#89`. + +Whenever the product head advances, update the immutable `ZED_CLI_REF` and this +document together. Branch names and abbreviated SHAs are not valid promotion +evidence. diff --git a/docs/den-2995-rust-enum-wire-canary.md b/docs/den-2995-rust-enum-wire-canary.md new file mode 100644 index 00000000..adea71fd --- /dev/null +++ b/docs/den-2995-rust-enum-wire-canary.md @@ -0,0 +1,47 @@ +# DEN-2995 — generated Rust enum wire-value canary + +This independent `zed-pkg-test` lane observes one immutable +`ORESoftware/k8s-libs-and-shared-defs` source commit and exercises the generated +Rust crate as an external consumer. + +## Contract + +SQL enum labels are wire values, not formatting hints. Generated Rust Serde must +round-trip the exact source label, including separators and leading digits. +`ssh-ed25519` must never become `sshed25519`; `in_progress` must never become +`inprogress`. + +The workflow: + +1. reads the exact source repository and SHA from + `fixtures/den-2995-rust-enum-wire-source.json`; +2. checks out that immutable commit with read-only permissions and no persisted + credentials; +3. runs the authoritative generator drift check; +4. inspects the generator policy; +5. injects a black-box integration test into the generated Rust crate and runs + it with Cargo; +6. records whether the pinned source is deliberately pre-fix or expected to be + repaired. + +## Promotion sequence + +The initial pin has `expect_fixed: false`. It is retained only to prove that the +canary detects the already-merged defect rather than silently passing a broken +implementation. + +Before promotion: + +- repin `sha` to the exact repaired source PR head; +- change `expect_fixed` to `true`; +- require the exact-head workflow to pass; +- preserve the immutable source SHA and resulting run as Linear/GitHub evidence. + +The source fix belongs in the pg-defs generator. Hand-editing generated Rust is +not acceptable evidence and will fail `generate.mjs --check`. + +## Non-goals + +This repository does not publish packages, modify the source branch, apply +Postgres DDL, use a personal access token, or carry provider, Linear, Cloudflare, +or R2 credentials. diff --git a/docs/direct-source-render.md b/docs/direct-source-render.md new file mode 100644 index 00000000..74f5dc18 --- /dev/null +++ b/docs/direct-source-render.md @@ -0,0 +1,72 @@ +# Exact direct-source snapshot Kustomize render certification + +This `zed-pkg-test` lane proves that the submodule-backed inventory record and the Argo CD direct-source declaration resolve to the same exact private child commit, that the declared endpoint ports are permitted by the app NetworkPolicy, and that checksum-pinned manifest bytes from that commit render deterministically without materializing the submodule in the cluster superproject. + +## Private-repository and billing boundaries + +A credential-free cross-organization checkout of `daedalus-fab/fabrication-server.rs` failed with `Repository not found`. That is the expected GitHub permission boundary: a test-repository `GITHUB_TOKEN` cannot read a private repository in another organization. + +The owner repository's Actions are also currently blocked before job execution by its billing/spending-limit state. Neither limitation is bypassed with a user PAT. + +Instead, the five deployment files were exported through the connected repository reader and committed as an immutable test fixture. Every snapshot file retains the exact Git blob SHA from the source commit, and every runner recomputes `git hash-object` before rendering. This proves source-byte provenance and renderability without claiming private-repository reachability. + +Private source reachability remains a separate, owner-scoped GitHub App credential contract. + +## Immutable inputs + +```text +ORESoftware/k8s-cluster@0c71eed05f8be50ac22ebe73715ded4a871234a8 +daedalus-fab/fabrication-server.rs@cea4a3c772012e1f2f87050bac24e911b8e2e577 +Kustomize v5.8.1 +``` + +The source commit is owner PR `daedalus-fab/fabrication-server.rs#11`. It changes OTLP egress from gRPC port 4317 to the Deployment's HTTP/protobuf endpoint port 4318 and adds a source-level parity regression. + +The cluster commit is stacked PR `ORESoftware/k8s-cluster#1211`. It advances the mode-160000 gitlink and both GitOps catalog pins to the exact source repair while retaining the trigger hardening from `#1209`. + +## Three-platform proof + +Ubuntu 24.04, macOS 15, and Windows Server 2025 independently: + +1. check out the exact cluster commit with `submodules: false`; +2. load the five-file source snapshot and its revision/repository/blob manifest; +3. recompute every snapshot file's Git blob SHA and require an exact match with the private source commit; +4. verify `.gitmodules`, the mode-160000 Git index entry, catalog inventory, catalog direct source, and snapshot metadata identify the same repository and commit; +5. prove the deployment source is not materialized beneath the superproject checkout; +6. run unit tests proving an OTLP 4318 endpoint with only 4317 egress fails closed; +7. extract the real snapshot's NATS and OTLP endpoint ports and require the NetworkPolicy to permit both; +8. download the official Kustomize v5.8.1 asset selected for the runner platform; +9. verify the release asset against a hard-coded official SHA-256 digest before safe extraction; +10. render the checksum-pinned `k8s` snapshot twice and require byte-identical output; +11. require exactly one `Deployment`, `Service`, `NetworkPolicy`, and `ExternalSecret` in namespace `daedalus`; +12. reject cluster-scoped resources, a rendered Kubernetes `Secret`, or plaintext `stringData`; and +13. upload endpoint-policy, source-blob, binary, and render digests plus the rendered YAML. + +The pinned release assets and digests come from the official `kubernetes-sigs/kustomize` v5.8.1 GitHub release. Archive extraction rejects path traversal and links. + +## Layered evidence + +The layers remain intentionally distinct: + +- static catalog validation works during a private-repository credential outage; +- snapshot rendering proves the exact exported deployment bytes still render; +- endpoint-policy validation proves checked-in egress permits the declared runtime URLs; +- an authenticated GitHub App checkout must separately prove live private-repository reachability. + +A passing static catalog check is not treated as proof of source renderability, and a passing source snapshot does not grant repository or cluster credentials. + +## Isolation + +All repository checkouts disable persisted credentials. The canary uses no user PAT, GitHub App private key, registry credential, Kubernetes credential, Cloudflare token, DNS permission, R2 key, or database access. It runs `kustomize build` only; it never invokes `kubectl`, `argocd`, Helm installation, a registry push, or an apply operation. + +Passing this canary does not activate the inert ApplicationSet and does not authorize production reconciliation. + +## Coordination + +- Owner source repair: `daedalus-fab/fabrication-server.rs#11` +- Stacked exact-pin update: `ORESoftware/k8s-cluster#1211` +- Baseline catalog/trigger repair: `ORESoftware/k8s-cluster#1209` +- Current-main adversarial matrix: `zed-pkg-test/security-adversarial-e2e#6` +- Root dispatcher merge: `zed-pkg/zed-cli#242` +- GitOps roadmap: `ORESoftware/k8s-cluster#1097` +- Linear: `DEN-2724`, `DEN-2725`, `DEN-630` diff --git a/docs/federated-test-org-scenarios.md b/docs/federated-test-org-scenarios.md new file mode 100644 index 00000000..7a62ac7b --- /dev/null +++ b/docs/federated-test-org-scenarios.md @@ -0,0 +1,54 @@ +# Federated `zed-pkg-test` scenario acceptance + +This harness turns four independent repositories in the `zed-pkg-test` organization into one executable acceptance layer for a checksum-verified `zed` release-candidate binary. + +## Covered repositories + +| Test repository | Executable acceptance surface | +| --- | --- | +| `zed-pkg-test/offline-cache-e2e` | cold install, offline frozen replay, shared cache reuse, corrupt-cache rejection, and immutable-registry recovery | +| `zed-pkg-test/version-solver-e2e` | compatible-range selection, exact pins, deterministic conflict diagnostics, and stable SemVer build-metadata outcomes | +| `zed-pkg-test/security-adversarial-e2e` | symlink containment, target traversal rejection, explicit build-hook consent, and tampered-lock fail-closed behavior | +| `zed-pkg-test/manager-interop-e2e` | parser-only/read-only mise and asdf verification, ambiguity rejection, moving-selector rejection, and typed unsupported-manager boundaries | + +The scenario runner creates isolated registries, homes, consumers, and package graphs under a disposable root. It does not mutate any of the four source repositories. + +## Binary provenance + +The workflow queries the public `zed-pkg/zed-cli` Actions artifact inventory for the newest non-expired `zed-x86_64-unknown-linux-gnu` artifact produced from a `release/**` branch. Before execution it verifies: + +1. the downloaded Actions artifact ZIP against GitHub's recorded `sha256:` digest; +2. the inner `zed-x86_64-unknown-linux-gnu.tar.gz` against the checksum packaged by the release workflow; +3. the source commit recorded on the selected workflow artifact. + +The JSON report records the CLI version, source commit, inner archive SHA-256, suite inventory, and every individual check. + +## August 6, 2026 evidence + +The first execution used: + +- CLI: `zed 0.1.0` +- source commit: `2f98df7b0f3f20bd8eaec6abbe768566833589bc` +- inner archive SHA-256: `ed55e39811cac6303162536397562ed004c96c4cd7e09b531f0d323f59142aff` +- result: **4 suites and 19 checks passed** + +The machine-readable receipt is retained at `evidence/2026-08-06-zed-0.1.0-rc3-test-org.json`. + +## Running locally + +```bash +ZED_BIN=/absolute/path/to/zed \ +ZED_BINARY_ARCHIVE=/absolute/path/to/zed-x86_64-unknown-linux-gnu.tar.gz \ +ZED_SOURCE_COMMIT=<40-character-commit> \ +ZED_E2E_ROOT=/tmp/zed-test-org-federation \ +ZED_E2E_REPORT=/tmp/federated-test-org-report.json \ +python3 scripts/run-federated-test-org-scenarios.py +``` + +The runner exits nonzero on the first failed invariant and still prints the partial JSON report to stderr. + +## Scheduled execution + +`.github/workflows/federated-test-org-scenarios.yml` runs daily in the test organization and supports manual dispatch. The workflow has read-only repository and Actions permissions. Cross-repository artifact access uses `TEST_FLEET_READ_TOKEN` when configured and otherwise attempts the ephemeral GitHub token. + +A passing workflow must report exactly these four suites and exactly 19 passing checks. Adding or removing checks therefore requires an intentional harness update rather than silently changing the acceptance surface. diff --git a/docs/frozen-fetch-canary.md b/docs/frozen-fetch-canary.md new file mode 100644 index 00000000..9669e070 --- /dev/null +++ b/docs/frozen-fetch-canary.md @@ -0,0 +1,38 @@ +# External frozen-fetch canary + +This repository validates `zed fetch --frozen` outside the `zed-cli` +implementation repository. The lifecycle workflow pins one immutable zed-cli +commit, builds it once, and reuses the checksummed binary for both the existing +22-repository matrix and a dedicated resolver-only fetch job. + +The canary uses the real `zed-pkg-test/node-lib` package fixture and a disposable +`file://` registry. It performs this sequence: + +1. publish the fixture through normal `zed publish`; +2. create a consumer lock through normal copy installation; +3. uninstall all project materialization while retaining exact lock bytes; +4. remove the install store; +5. run two independent frozen fetches with a separate home path; +6. compare every output file by relative path, byte length, and SHA-256; +7. verify `zed.fetch/v1` identity, artifact, provenance, source-classification, + and lock-digest metadata; +8. prove the consumer tree and fixture checkout were unchanged; +9. prove the configured fetch home was never created; +10. reject a tampered artifact digest before final output publication; +11. reject an existing caller-owned destination without changing it; +12. reject a missing lock; and +13. export a deterministic dependency-free lock. + +Literal registry paths and URLs must not appear in the portable index. Temporary +bundle and isolated-store directories must not survive success or failure. + +The workflow uses read-only repository permissions, commit-pinned checkout and +artifact actions, an exact 40-character zed-cli revision, disabled persisted +checkout credentials, and failure-only diagnostics. The aggregate lifecycle +gate requires the external fetch canary and every fixture lifecycle to pass. + +During development, `.zed-cli-ref` contains the exact reviewed commit of the +stacked zed-cli pull request. A manual workflow dispatch may override it only +with another full 40-character commit. Before this canary merges to `main`, +update `.zed-cli-ref` to the final merged zed-cli commit so scheduled and +repository-dispatch runs never depend on an unmerged branch head. diff --git a/docs/ghcr-fallback-canary.md b/docs/ghcr-fallback-canary.md new file mode 100644 index 00000000..e956be3d --- /dev/null +++ b/docs/ghcr-fallback-canary.md @@ -0,0 +1,36 @@ +# Live GHCR fallback canary + +This proves GitHub Packages (GHCR) can host a packed Zed artifact when +`registry.zpkg.net` is down. Unlike the Releases canary +(`docs/github-api-fallback-canary.md`) this fixture has **no GitHub Release**. +`zed install` source fallback tries a Release sidecar first, then GHCR; a +package that exists only as `ghcr.io/zed-pkg-test/ghcr-fallback-canary:v0.0.1` +forces the Packages branch. + +The fixture repository is +[`zed-pkg-test/ghcr-fallback-canary`](https://github.com/zed-pkg-test/ghcr-fallback-canary) +(created by the script if missing). The OCI artifact uses Zed media types +(`application/vnd.zed.package.v1.tar+gzip` layer). + +## What is asserted + +1. GHCR HTTP: blob upload, manifest PUT for tag `v0.0.1`, GET of the same + digest. +2. Optional: `zed install` / `zed install --frozen` against a **closed** + `127.0.0.1` registry restores the payload from GHCR. Needs a zed-cli build + with `source_fallback::ghcr_version` (zed-pkg/zed-cli#298). + +Token needs `write:packages` (and `repo` to create the public fixture). Read +from `GITHUB_TOKEN` / `GH_TOKEN` / `ZED_PKG_GITHUB_TOKEN`; never printed. + +## Local run + +```bash +python3 scripts/ghcr_fallback.py --work-root /tmp/zed-ghcr-contract --skip-network + +python3 scripts/ghcr_fallback.py --work-root /tmp/zed-ghcr-live + +python3 scripts/ghcr_fallback.py \ + --work-root /tmp/zed-ghcr-zed \ + --zed /path/to/zed +``` diff --git a/docs/git-submodule-interop-canary.md b/docs/git-submodule-interop-canary.md new file mode 100644 index 00000000..910c7650 --- /dev/null +++ b/docs/git-submodule-interop-canary.md @@ -0,0 +1,201 @@ +# Git-submodule interoperability canary + +This repository independently certifies the Git-submodule implementation merged +by `zed-pkg/zed-cli#108` rather than relying only on product-repository tests. +The workflow pins these immutable shipped inputs: + +```text +zed-pkg/zed-cli@d8763b8dcc3bd5c5b6b4b1d13519065a6a2cb99a +zed-pkg/zed-interfaces@8428bc574111fa148e590c8350c7855035ce2046 +``` + +The same merged CLI commit is recorded in `.zed-cli-ref`, so this PR also +exercises the shipped implementation through the repository's full lifecycle +suite across all maintained fixture repositories. Before promotion, the +certification branch is merged with current E2E `main` and revalidated at the +resulting head; independent acceptance additions are preserved rather than +resolved by selecting one side of a divergent history. + +## Contract + +`scripts/git_submodule_interop.py` creates disposable local Git repositories and +certifies twelve black-box behaviors through the compiled public `zed` binary: + +1. `--git-submodules=false` overrides an inherited truthy environment setting. +2. Cooperative install initializes top-level and nested submodules recursively. +3. Takeover records exact workspace, direct dependency, commit, digest, and size + provenance. +4. A fresh clone replays frozen state without changing one lockfile byte. +5. Dirty adopted content fails before lock mutation. +6. Committed branch-provenance drift fails before lock mutation. +7. A mixed repository adopts only the explicit Zed package and leaves the + ordinary submodule Git-managed. +8. Mixed fresh-clone frozen replay initializes both Git transports, preserves + manifest and lock bytes, and materializes only the adopted package through + `zed_modules`. +9. An all-Git-only takeover may initialize requested transport but publishes no + Zed lock, modules, or recovery state and preserves the authored manifest. +10. Malformed package intent fails closed before any Zed mutation. +11. A pre-commit resolution failure restores exact authored manifest bytes. +12. The same failure removes a takeover-generated manifest from a previously + manifestless root. + +The product's four dedicated integration binaries add direct coverage for CLI +and environment precedence, nested synchronization, idempotent takeover, +manifestless rollback, true clone-without-submodules restoration, clean recursive +status, and branch metadata refresh without replacing the immutable gitlink pin. +Unit and Clippy coverage also enforce that only true `NotFound` is skippable: +directory-valued, dangling, symlinked, and invalid `.zpkg.toml` entries fail +closed. + +## Evidence and safety + +Each Ubuntu 24.04 and macOS 15 job retains: + +- the complete command transcript; +- adopted and mixed-repository lock evidence; +- the release binary SHA-256; +- the exact CLI version; and +- the twelve certified contract checks. + +The workflow has only `contents: read`, disables persisted checkout credentials, +pins third-party Actions to full commit SHAs, verifies the CLI/interface/source +pins, keeps Python bytecode outside checked-out sources, denies Clippy warnings, +and requires clean product, interface, and harness checkouts afterward. + +No account, public package registry, credential, Docker daemon, or persistent +namespace participates. The canary explicitly removes inherited Zed and GitHub +tokens from product subprocesses. Local file transport is enabled only through +the acceptance process's explicit Git configuration. + +A manually dispatched optional job repeats the same black-box contract on the +registered `[self-hosted, linux, sonus-ci]` ARC scale set when hosted runner +capacity is constrained. + +## Local execution + +From sibling checkouts: +This canary independently certifies the consolidated Git/Zed submodule behavior +in `zed-pkg/zed-cli#108`. It runs outside the product repository against one +exact immutable CLI commit: + +```text +zed-pkg/zed-cli@b2ec50cd1d7c182bd9219795cd2918387c9c4cd8 +``` + +The product feature builds on the merged cooperative-install and takeover work +in `zed-pkg/zed-cli#96` plus the failure-atomic migration hardening in #101. + +## Contract under test + +A superproject may use Git submodules for different purposes at the same time: + +- a submodule containing a valid regular-file `.zpkg.toml` can be adopted into + the root Zed workspace, direct dependency intent, deterministic + materialization, and additive `[[git-submodule]]` lock provenance; +- a submodule without `.zpkg.toml` remains Git-managed and never appears in the + Zed manifest or lock; and +- ordinary `zed install --git-submodules` still initializes every configured + Git submodule, whether Zed adopted it or not. + +Missing package intent is the only skip condition. A present but invalid, +directory-valued, dangling, or symlinked package manifest remains an error. + +## Product certification included at the pin + +The consolidated product candidate also includes focused real-process suites for: + +- opt-in defaults and CLI/environment boolean precedence; +- top-level and nested recursive initialization; +- takeover idempotence and preservation of `.gitmodules` and gitlinks; +- rollback for failed takeover, including a manifestless root; +- fresh-clone frozen restoration with clean recursive Git status; and +- configured branch provenance that never replaces the immutable gitlink pin. + +The workflow runs all four dedicated integration binaries before the independent +Python canary. + +## Black-box scenarios + +`scripts/git_submodule_interop.py` creates real local Git repositories and runs +the compiled public `zed` executable. It does not call Rust library internals. + +### Mixed takeover and frozen replay + +The harness creates one Zed package submodule and one ordinary Git-only +submodule, then verifies: + +1. `zed overtake --git-submodules` adopts exactly one package; +2. the Git-only path is reported and absent from `.zpkg.toml` and `.zpkg.lock`; +3. the adopted package has exact workspace/dependency intent and immutable Git + lock provenance; +4. the resulting Zed manifest and lock can be committed normally; +5. a fresh `git clone --no-recurse-submodules` starts with neither submodule + initialized; +6. `ZED_PKG_GIT_SUBMODULES=yes zed install --frozen` initializes both + submodules; +7. frozen replay preserves committed manifest and lock bytes exactly; and +8. the adopted package is materialized through `zed_modules`, while the ordinary + submodule remains solely Git-managed. + +### Git-only migration boundary + +A repository whose only submodule has no `.zpkg.toml` is synchronized, but +`overtake` returns an actionable failure and does not create or change: + +- `.zpkg.toml`; +- `.zpkg.lock`; +- `zed_modules/`; or +- `.zpkg-staging/`. + +This distinguishes intentional Git transport side effects from Zed authority +migration. + +### Invalid package boundary + +A submodule containing malformed `.zpkg.toml` is not silently treated as +Git-only. Takeover fails before changing the root manifest, producing a lock, or +materializing a package. Product-focused tests at the same pinned commit also +prove that directory and symlink manifest entries fail the regular-file gate. + +## Workflow gates + +`.github/workflows/git-submodule-interop.yml`: + +- requires a full 40-character CLI pin; +- uses read-only repository permissions; +- pins checkout and artifact Actions to immutable commits; +- disables persisted checkout credentials; +- keeps Python bytecode outside checked-out sources; +- compiles the Python canary; +- checks both repositories with `git diff --check`; +- runs product `rustfmt`, all four dedicated Git-submodule integration suites, + and all-target Clippy with warnings denied; +- builds the exact release executable; +- runs the independent black-box harness from a fresh work root; +- requires clean harness and product source trees afterward; and +- uploads bounded result or failure evidence for 14 days. + +The normal gate uses Ubuntu 24.04. A manually dispatched optional job can run the +same canary on the registered `[self-hosted, linux, sonus-ci]` ARC scale set when +hosted Actions capacity is constrained. + +No registry account, package namespace, GitHub token, cloud credential, Docker +daemon, or persistent runner state participates. All package and submodule +transports are disposable local filesystem repositories. + +## Local execution + +Build the pinned CLI and run the canary from sibling checkouts: + +```bash +cargo build --locked --release \ + --manifest-path ../zed-cli/Cargo.toml \ + --bin zed + +python3 scripts/git_submodule_interop.py \ + --zed ../zed-cli/target/release/zed \ + --work-root /tmp/zed-git-submodule-interop +``` + +The work root must not already exist. It is retained on failure for inspection. diff --git a/docs/git-submodule-overtake-lock.md b/docs/git-submodule-overtake-lock.md new file mode 100644 index 00000000..5f92dee4 --- /dev/null +++ b/docs/git-submodule-overtake-lock.md @@ -0,0 +1,149 @@ +# Git-submodule and project-operation lock certification + +This test-org lane certifies the complete remaining DEN-2038 surface in +`zed-pkg/zed-cli#243` at its current-main-integrated, nested-invocation-hardened, +thread-affine, shared-ownership, recovery-ordered, and cooperative-install-locked +exact candidate commit: + +```text +591b40846da73f7540bd36add33645d11cd33ab0 +``` + +The candidate was first merged with current product `main` through integration +PR `zed-pkg/zed-cli#245`, preserving the independent external GitOps dispatcher +and its tests. When `main` later advanced with DEN-3018 publish-ignore +diagnostics, the branch was reconciled through a true two-parent merge. The +locking/recovery files and DEN-3018 files are disjoint, and both complete feature +sets remain present. The final nested-install root correction is committed +directly in `src/main.rs`; its temporary branch-writing finalizer removed itself, +leaving exactly the three reviewed implementation files in the product PR. A +small ownership-contract clarification was then committed by `ORESoftware` so +GitHub executes the final matrix normally instead of classifying a bot-authored +head as `action_required`. + +The product now routes all of these operations through the same canonical +checkout-local `.zed/operation.lock` boundary: + +- ordinary install and frozen restore; +- add, remove, uninstall, and initialization through the existing facades; +- `zed overtake --git-submodules`; +- eager `.zpkg-staging` recovery; +- modular-takeover recovery before Git work; and +- `zed install --git-submodules`, including Git synchronization, recursive + submodule checkout, dependency resolution, materialization, adapter wiring, + and lock finalization under one descriptor lifetime. + +## Reentrant ownership invariants + +The reentrancy cache is thread-local. An owned operation guard must therefore be +acquired, used, and dropped on one thread. The final candidate stores the actual +kernel `LockGuard` inside an `Rc`-owned object and keeps only a weak reference in +the thread-local lookup table. This provides three properties: + +1. `OperationGuard` is naturally neither `Send` nor `Sync`; +2. every same-thread nested acquisition shares the same descriptor ownership; +3. dropping outer and inner handles in any order cannot release the operating- + system lock before the final live handle is dropped. + +Two `compile_fail` rustdoc contracts prove the public type cannot satisfy either +`Send` or `Sync`. A focused unit regression drops the outer guard first, +confirms a queued same-process OS-lock probe remains contended, drops the final +inner guard, then confirms acquisition succeeds. + +## Why separate black-box tests + +Product unit tests prove nested facade reuse, thread affinity, shared descriptor +lifetime, and superproject path selection. They do not by themselves prove that +independent operating-system processes contend on the same checkout identity, +that path aliases cannot escape ownership, that recovery coordinates different +Zed homes, or that Git synchronization and installation share one uninterrupted +ownership window. + +Four test-org scripts exercise those boundaries using the real release +executable and disposable local Git repositories. A temporary `git` shim pauses +at `git submodule sync` after the CLI must already own the checkout. After the +test releases it, the shim `exec`s the exact host Git binary with the original +arguments. + +## Nineteen certified process checks + +### Takeover normal completion + +1. Takeover owns `.zed/operation.lock` before Git transport begins. +2. A frozen install through a symlink alias remains blocked while takeover owns + the checkout. +3. After release, takeover publishes complete adopted state and the frozen + waiter succeeds. + +### Owner termination + +4. An independent install remains blocked behind a paused takeover. +5. Terminating the takeover process group releases the descriptor lock without + deleting the lockfile or running stale-lock reclamation. +6. Pre-mutation termination preserves exact manifest bytes and leaves no + adoption or recovery state. + +### Nested takeover invocation + +7. Takeover launched from `packages/client/src` owns the superproject root lock. +8. It never creates a nested `packages/client/src/.zed/operation.lock` identity. +9. A root-level frozen install blocks and succeeds behind nested takeover. + +### Nested cooperative `install --git-submodules` + +10. Cooperative install launched from `packages/client/src` owns the + superproject operation lock before `git submodule sync` begins. +11. The nested invocation creates no second operation lock, manifest, lockfile, + or staging identity below the authority root. +12. A different-home root frozen install remains blocked across synchronization, + recursive checkout, and the full install/finalizer boundary. +13. The frozen waiter succeeds only after the cooperative install publishes a + complete root lockfile and the pinned child checkout. + +The frozen waiter starts before `.zpkg.lock` exists. Against the previous CLI +path it would race and fail because sync ran before the installer facade acquired +ownership. The nested holder also proves that superproject discovery governs +both Git synchronization and installation ownership. + +### Transaction recovery versus another owner + +14. Different-home eager recovery remains blocked behind active checkout + ownership. +15. Pending destination and backup bytes stay exact while blocked. +16. After release, recovery restores exact bytes and removes staging. +17. The recovered process completes frozen installation against adopted state. + +### Recovery ordering inside modular takeover + +18. A pending transaction is recovered before takeover reaches + `git submodule sync`. +19. Exact backup bytes are restored before `.gitmodules` verification, + synchronization, or adoption begins. + +## Matrix and promotion boundary + +The workflow runs on Ubuntu 24.04 and macOS 15. It: + +- pins the CLI, `zed-interfaces`, and `zed-lock` to full commit IDs; +- uses read-only repository permissions and commit-pinned Actions; +- disables persisted checkout credentials; +- prevents Python bytecode writes and compiles all four harnesses through a + runner-temporary cache; +- runs source diff checks and Rust formatting; +- runs focused `project_lock` and `git_submodules` library tests; +- runs both thread-affinity compile-fail rustdoc contracts; +- runs strict Clippy on Linux; +- builds the exact release executable; +- executes six contention, termination, recovery, and ordering scenarios; and +- prints and rejects any dirty product or harness checkout without cleanup or + reset. + +Evidence contains binary identity, process IDs, measured blocking intervals, CLI +version, and all nineteen named checks. No account, public registry, Cloudflare +resource, credential, Docker daemon, or persistent namespace participates. + +Merge this test-org PR only after both platform jobs pass on the exact head. +Then product PR `zed-pkg/zed-cli#243` can be promoted if its complete matrix and +review state remain clean on the same product commit. + +Linear: DEN-2038. diff --git a/docs/git-submodule-pack-canary.md b/docs/git-submodule-pack-canary.md new file mode 100644 index 00000000..d6985d49 --- /dev/null +++ b/docs/git-submodule-pack-canary.md @@ -0,0 +1,124 @@ +# Git-submodule package-boundary canary + +This workflow independently certifies the archive, publication, and Git-metadata +boundaries in `zed-pkg/zed-cli#110`. It invokes only the compiled public +executable and inspects resulting `tar.gz` bytes with Python's standard library. + +The workflow pins exact CLI candidate +`0786987fe3a173659463940f4f905dc3b5bd19a8` and exact interface contract +`c2e049006453c26ca8ca291783f681fce75cb01f`. The candidate is the formatter-clean, +locally tested product head merged with current `main`, including mixed-submodule +takeover, Dart wiring, mise frozen identity, Windows test gating, atomic Nix +bundle behavior, conservative package exclusions, and indirect `.gitmodules` +metadata hardening. Ubuntu 24.04 and macOS 15 build that exact source, run the +focused product tests, and execute disposable local Git fixtures. Linux also +runs all-target Clippy with warnings denied. + +## Certified behavior + +The existing twelve-check archive contract uses a superproject containing a +Zed-package submodule, which itself contains a nested ordinary Git submodule. It +proves: +This contract independently certifies the archive and publication boundary in +`zed-pkg/zed-cli#110`. It invokes only the compiled public executable and +inspects the resulting `tar.gz` bytes with Python's standard library. + +The workflow pins exact CLI candidate +`82c9d0e0b9e96402afa76f3b85c4b9e16c3cb101` and exact interface contract +`c2e049006453c26ca8ca291783f681fce75cb01f`. The candidate is the reviewed +semantic merge of the package guard with current `main`, including the latest +mixed-submodule, Dart-wiring, mise-identity, Windows test-gating, and atomic Nix +bundle changes, plus the contextual-error, conservative exclusion, and indirect +`.gitmodules` metadata hardening. Ubuntu 24.04 and macOS 15 build that source, +run the focused product tests, and execute disposable local Git fixtures. Linux +additionally runs all-target Clippy with warnings denied. + +## Certified behavior + +The fixture graph is a superproject containing a Zed-package submodule, which +itself contains a nested ordinary Git submodule. The black-box contract proves: + +1. an included but uninitialized submodule rejects `zed pack` before an archive + is created; +2. the same checkout rejects `zed publish --dry-run` before publication + planning; +3. `zed install --git-submodules` initializes both submodule levels recursively; +4. a materialized checkout packages root, child, and nested runtime files; +5. no root or nested `.git`, `.gitmodules`, `.hg`, or `.svn` control data enters + the archive; +6. the initialized package reaches a credential-free publication dry run; +7. a dirty included submodule fails before archive creation; +8. a dirty nested submodule also fails before archive creation; +9. an authored `publish.exclude = ["vendor/client/**"]` permits that entire + subtree to remain uninitialized and omitted; +10. a project `.zedignore` with the same recursive boundary behaves identically; +11. a misleading `./vendor/client/**` pattern cannot bypass initialization; and +12. a misleading `/vendor/client/**` pattern cannot bypass initialization. + +A separate three-check metadata contract proves: + +13. a symlinked worktree `.gitmodules` rejects cooperative install before sync, + lockfile, or materialized-module mutation; +14. a directory-backed worktree `.gitmodules` rejects pack before archive + creation; and +15. a `120000` symlink entry in the Git index rejects pack even when the + worktree `.gitmodules` is still a regular file. + +Modern Git porcelain intentionally refuses to create the third hostile state. +The canary therefore rewrites one entry in a valid version-2 index, supports +both SHA-1 and SHA-256 repository formats, recomputes the index checksum, and +then proves `git ls-files --stage` accepts the resulting `120000` entry before +invoking Zed. This keeps the contract at the actual Git/product boundary instead +of mocking command output. +3. a symlinked `.gitmodules` rejects cooperative install before sync, lockfile, + or materialized-module mutation; +4. a directory-backed `.gitmodules` rejects pack before archive creation; +5. `zed install --git-submodules` initializes both submodule levels recursively; +6. a materialized checkout packages root, child, and nested runtime files; +7. no root or nested `.git`, `.gitmodules`, `.hg`, or `.svn` control data enters + the archive; +8. the initialized package reaches a credential-free publication dry run; +9. a dirty included submodule fails before archive creation; +10. a dirty nested submodule also fails before archive creation; +11. an authored `publish.exclude = ["vendor/client/**"]` permits that entire + subtree to remain uninitialized and omitted; +12. a project `.zedignore` with the same recursive boundary behaves identically; +13. a misleading `./vendor/client/**` pattern cannot bypass initialization; and +14. a misleading `/vendor/client/**` pattern cannot bypass initialization. + +An uninitialized submodule is optional only when one authored canonical +recursive rule conclusively excludes its complete subtree under the exact glob +syntax used by the packer. Sampling filenames, stripping path prefixes, changing +path separators, or otherwise normalizing an authored pattern is not treated as +proof that an unknown runtime file cannot enter the package. + +Before any Git-submodule parser or mutating command runs, the worktree +`.gitmodules` entry must be a regular file. When tracked, its Git index entry +must also be a stage-zero regular blob; symlink, gitlink, conflict-stage, +directory, and other indirect representations fail closed. + +## Isolation and evidence + +Every Git repository, Zed home, registry directory, output directory, and +credential home is created below the runner's temporary work root. Both +contracts remove inherited GitHub and Zed token variables, disable interactive +Git authentication and system Git configuration, and enable local file +transport only through process-local Git configuration for their own fixtures. + +No public registry, package namespace, account, token, container daemon, or +persistent runner state is used. Each operating-system job retains bounded +command transcripts and JSON evidence containing the exact binary version, +binary SHA-256, and all fifteen completed checks. +credential home is created below the runner's temporary work root. The contract +removes inherited GitHub and Zed token variables, disables interactive Git +authentication and system Git configuration, and enables local file transport +only through process-local Git configuration for its own fixtures. + +No public registry, package namespace, account, token, container daemon, or +persistent runner state is used. Each operating-system job retains only the +bounded command transcript and JSON evidence containing the exact binary +version, binary SHA-256, and fourteen completed checks. + +The workflow itself has `contents: read`, disables persisted checkout +credentials, and pins all third-party Actions and product inputs to full commit +SHAs. diff --git a/docs/github-api-fallback-canary.md b/docs/github-api-fallback-canary.md new file mode 100644 index 00000000..98bf565e --- /dev/null +++ b/docs/github-api-fallback-canary.md @@ -0,0 +1,45 @@ +# Live GitHub API fallback canary + +This proves GitHub can host a packed Zed artifact **without** `registry.zpkg.net` +or `cdn.zpkg.net`. It talks to `api.github.com` and `uploads.github.com` in +the sibling test org [`zed-pkg-test`](https://github.com/zed-pkg-test), then +downloads the same bytes from the public Release URL. + +The fixture repository is +[`zed-pkg-test/github-api-fallback-canary`](https://github.com/zed-pkg-test/github-api-fallback-canary) +(created by the script if missing; always public so unauthenticated `GET` +works when the registry is down). + +## What is asserted + +1. GitHub REST: repo, git ref, tags, Releases, asset upload, asset download. +2. Unauthenticated `https://github.com/{org}/{repo}/releases/download/{tag}/{asset}` + returns the exact sha256 that was uploaded. +3. Optional: `zed install` / `zed install --frozen` against a **closed** + `127.0.0.1` registry, with `ZED_PKG_SOURCE_FALLBACK_ALLOW_LOOPBACK=true`, + restores the payload from GitHub. That needs a zed-cli build that contains + `source_fallback.rs` (zed-pkg/zed-cli#294). + +This is complementary to the loopback CDN mock (`scripts/github_r2_fallback.py` +on `feat/github-r2-fallback-canary`): that test never leaves 127.0.0.1. This +one is the GitHub-API proof. + +## Local run + +```bash +# Contract only (no GitHub): +python3 scripts/github_api_fallback.py --work-root /tmp/zed-gh-api-contract --skip-network + +# Live GitHub APIs (creates/updates the public canary repo + release): +python3 scripts/github_api_fallback.py --work-root /tmp/zed-gh-api-live + +# Plus zed install through GitHub fallback: +cargo build --release --manifest-path ../zed-pkg/zed-cli/Cargo.toml --bin zed +python3 scripts/github_api_fallback.py \ + --work-root /tmp/zed-gh-api-zed \ + --zed ../zed-pkg/zed-cli/target/release/zed +``` + +The token is read from `GITHUB_TOKEN` / `GH_TOKEN` / `ZED_PKG_GITHUB_TOKEN` +and is never printed. The canary repo is public; do not put secrets in the +sidecar JSON. diff --git a/docs/github-dependency-inventory-conformance.md b/docs/github-dependency-inventory-conformance.md new file mode 100644 index 00000000..9a80befc --- /dev/null +++ b/docs/github-dependency-inventory-conformance.md @@ -0,0 +1,96 @@ +# GitHub dependency-inventory conformance (DEN-2957, DEN-2996) + +This directory certifies the repository and GitHub-organization ingestion behavior planned for `zed graph github` before the production Rust CLI adopts it. + +## Resource boundary + +The emitted resource is `zpkg/github-dependency-inventory/v1`. It is a source inventory pinned to exact repository commits. It is **not** a universal resolved package graph and always reports: + +```json +{ + "completeness": { + "inventory": "complete", + "resolution": "not-claimed" + } +} +``` + +A flat `.zpkg.lock` contributes exact package selections to the top-level `pins` evidence collection. A flat closure does not encode parent-child relationships, so pins are marked `topological: false` and never become graph nodes or edges. When one unique pin matches an already-proven direct `.zpkg.toml` declaration, that declaration edge may carry the selected version, artifact digest, optional VCS commit, and lock provenance. Unmatched/transitive pins remain evidence only. + +`.zpkg.toml`, `.gitmodules`, `flake.nix`, `flake.lock`, and governed Nix `sources.json` files contribute typed source edges where the relationship is actually present in source. Only DEN-2865's shared resolver builder may claim an exact `zpkg/dependency-graph/v1` resolution. + +## Reference command + +The dependency-free test-org reference implementation is split by trust boundary under `harness/github_inventory_*.py`; the CLI facade mirrors the intended production surface: + +```console +python harness/github_dependency_inventory.py \ + --org zed-pkg \ + --repo zed-pkg/zed-cli \ + --include zed,git-submodule,nix \ + --format json \ + --output inventory.json +``` + +`--format` also accepts `dot` and `mermaid`. Repeated `--repo`, `--org`, and `--include` values are normalized, deduplicated, and sorted. Credentials are read only from `ZED_PKG_GITHUB_TOKEN` or `GITHUB_TOKEN`; there is intentionally no token flag. A token is sent automatically only to `api.github.com` or a loopback fake server. GitHub Enterprise or another custom HTTPS origin additionally requires `ZED_PKG_GITHUB_ALLOW_TOKEN_TO_API_BASE=1`, preventing an API-base override from silently exfiltrating the default GitHub token. + +Offline certification uses the compact fixture API: + +```console +python harness/github_dependency_inventory.py \ + --fixture fixtures/github-dependency-inventory/fixture.json \ + --org acme \ + --repo acme/app \ + --include zed,git-submodule,nix \ + --format json +``` + +## Deterministic envelope + +The canonical JSON envelope records: + +- normalized repository and organization inputs; +- every discovered repository, including archived repositories and explicit acquisition failures; +- exact default-branch commit SHA before tree or blob acquisition; +- source manifests with exact blob SHA and byte count; +- repository/package nodes and typed, proven topology edges; +- exact lock-pin evidence with package coordinate, selected version, artifact SHA-256, optional VCS commit, source package/repository, and immutable provenance; +- provenance containing repository commit, manifest kind, path, blob SHA, and line or JSON pointer where available; +- SCCs, canonical cycle sets, and dependency-first waves over topology only; +- contradiction diagnostics such as conflicting lock pins, missing gitlinks, and unmapped gitlinks; +- hard configured limits and bounded request/byte usage. + +The JSON writer sorts keys and all semantically unordered collections. DOT and Mermaid use hash-derived identifiers and format-specific label escaping. Lock pins render as disconnected, explicitly labeled evidence nodes; they never receive or contribute dependency arrows. Golden outputs are regenerated independently on Linux, macOS, and Windows and compared byte-for-byte. + +## Hardening invariants + +1. Organization pagination follows only same-origin `rel="next"` links and rejects pagination cycles. +2. HTTP redirects are not followed, preventing authorization forwarding to another origin. +3. Plain HTTP API origins are accepted only on loopback for the fake-server suite. +4. Retry count and `Retry-After` sleep are bounded. Response bodies from failed HTTP requests are never retained or surfaced. +5. Repository, node, edge/pin-evidence, request, per-response byte, total-response byte, manifest byte, individual field byte, repository-tree entry, JSON-depth, and wall-clock limits fail closed. Successful HTTP bodies are bounded while they are read, not after unbounded buffering. A limit error never becomes a partial inventory. +6. GitHub recursive-tree truncation is an explicit repository failure; it is never interpreted as a complete scan. +7. Blob content is fetched by exact tree SHA, size-checked, bounded, and decoded from GitHub's declared base64 representation. +8. Output replacement uses a same-directory temporary file, file `fsync`, atomic replacement, and directory `fsync` where supported. Failed runs preserve the prior output and remove temporary files. +9. Tokens are absent from argv, output, filenames, request paths, retained errors, graph labels, and golden evidence. +10. Inventory failures may produce a versioned partial artifact and exit status `1`; invalid input exits `2`; hard limits exit `3`. A complete artifact exits `0`. +11. Flat lock entries are immutable evidence, not topology. They are excluded from nodes, edges, SCCs, cycles, and dependency-first waves. Multiple distinct selections for one coordinate remain explicit contradictions rather than being collapsed. + +## Fixture coverage + +The checked-in fixture, split semantic/transport suites, and local fake server cover: + +- multi-page organization discovery and duplicate input normalization; +- a private repository and bearer-authenticated requests; +- archived and manifest-free repositories; +- Zed declaration diamonds and an explicit cycle; +- a flat lock closure whose transitive `shared` selection must not fabricate `app -> shared`; +- direct declared-edge selection annotation, optional VCS commit evidence, conflicting/duplicate pins, and invalid artifact digests; +- Git submodule gitlinks, flake declarations/locks, and Nix source pins; +- hostile requirement text that attempts DOT/Mermaid label injection; +- HTTP 403/429/5xx rate-limit retry behavior, redirects, truncated trees, bounded response reads, byte/field/depth limits, wrapped base64 blobs, custom-origin token policy, and token-bearing error bodies; +- failure-atomic output, a 10,000-node non-recursive SCC stress graph, and byte-identical JSON/DOT/Mermaid goldens. + +## Production handoff + +After the test-org conformance PRs are green, the production `zed-pkg/zed-cli` integration should add the modular `zed graph github` command and port this behavior without editing the merged DEN-2864 contract paths or introducing a second package resolver. The test fixture and goldens remain the cross-language conformance authority until the shared Rust builder supersedes them explicitly. Workspace/member manifest discovery is tracked independently in DEN-2997. diff --git a/docs/github-r2-fallback-canary.md b/docs/github-r2-fallback-canary.md new file mode 100644 index 00000000..b0000334 --- /dev/null +++ b/docs/github-r2-fallback-canary.md @@ -0,0 +1,41 @@ +# GitHub / R2 registry-fallback canary + +This repository proves that `zed install --frozen` still materializes a +package when the HTTP registry is down, as long as the packed tarball is at a +guessable public CDN key (`packages/{org}/{name}/{version}/…` or +`github/{owner}/{repo}/{tag}/…`). That is the `cdn.zpkg.net` contract. + +The canary is credential-free and uses only loopback HTTP: + +1. publish a disposable package to a `file://` registry; +2. copy the tarball onto the guessable CDN layout; +3. install once so a lock exists, then uninstall and drop the store; +4. point `--registry` at a closed `127.0.0.1` port; +5. set `ZED_PKG_SOURCE_FALLBACK_ALLOW_LOOPBACK=true` (production loopback stays + hermetic) and `--r2-public-base http://127.0.0.1:`; +6. `zed install --frozen` must restore the packed payload from the CDN mock. + +It also round-trips the `get_version` JSON call/receipt frame over TCP NDJSON, +using the same key as `zed-interfaces/route-maps/zed-api.route-map.json` +(generated by `github.com/oresoftware/api-docs`). zed-cli itself still speaks +HTTP; the TCP check is the typed RPC wire, not a second CLI transport. + +When the TypeScript copies are passed in, the generated `zed-clients` routes +file must match `zed-interfaces/generated/typescript/zed_api.ts` (banner +comments excluded). + +## Local run + +From a sibling checkout of `zed-pkg`: + +```bash +cargo build --release --manifest-path ../zed-pkg/zed-cli/Cargo.toml --bin zed +python3 scripts/github_r2_fallback.py \ + --zed ../zed-pkg/zed-cli/target/release/zed \ + --work-root /tmp/zed-github-r2-fallback \ + --route-map ../zed-pkg/zed-interfaces/route-maps/zed-api.route-map.json \ + --interfaces-generated-ts ../zed-pkg/zed-interfaces/generated/typescript/zed_api.ts \ + --clients-generated-ts ../zed-pkg/zed-clients/clients/typescript/src/generated/zed-api.routes.ts +``` + +The work root must not already exist. diff --git a/docs/gitmodules-metadata-canary.md b/docs/gitmodules-metadata-canary.md new file mode 100644 index 00000000..c98bf9b6 --- /dev/null +++ b/docs/gitmodules-metadata-canary.md @@ -0,0 +1,47 @@ +# `.gitmodules` metadata-boundary canary + +This canary independently certifies the metadata hardening added to +`zed-pkg/zed-cli#110`. It pins exact CLI candidate +`0786987fe3a173659463940f4f905dc3b5bd19a8` and exact interface contract +`c2e049006453c26ca8ca291783f681fce75cb01f`. + +The earlier package-boundary canary proves that initialized and clean submodule +source is packaged while nested Git control data is omitted. This follow-up +focuses on the trustworthiness of `.gitmodules` itself before any parser, +checkout, lock refresh, archive, or publication path consumes it. The candidate +also includes the exact rustfmt repair found by the first three-platform run. + +## Unix black-box contract + +Ubuntu 24.04 and macOS 15 build the exact locked release binary and exercise +real disposable Git repositories. The contract proves: + +1. a committed stage-zero regular `.gitmodules` blob permits recursive + `zed install --git-submodules`; +2. a symlinked `.gitmodules` rejects `zed pack` before archive creation; +3. the same symlink representation rejects install before submodule checkout or + `.zpkg.lock` mutation; +4. a directory at `.gitmodules` rejects pack before Git parsing; +5. a stage-zero index entry with Gitlink mode `160000` rejects pack; and +6. an index containing conflict stages 1, 2, and 3 rejects install before + checkout or lock mutation. + +Each negative case requires a specific metadata-boundary diagnostic, verifies +that no archive or lockfile is left behind, and retains only a bounded transcript +and JSON record. + +## Windows exact-head gate + +Windows Server 2025 runs the focused metadata unit suite and all-target Clippy +with warnings denied against the same immutable CLI and interface commits. The +filesystem symlink attacks remain in the Unix black-box matrix because ordinary +Windows hosted runners do not provide the same unprivileged symlink semantics; +the platform-independent Git index-mode and stage validation is compiled and +executed by the Windows unit suite. + +## Isolation + +The workflow has read-only repository permissions, disables persisted checkout +credentials, removes inherited GitHub and Zed token variables from product +subprocesses, uses only disposable local file transport, and never contacts or +mutates a public registry. diff --git a/docs/gitops-dispatch-canary.md b/docs/gitops-dispatch-canary.md new file mode 100644 index 00000000..67195c41 --- /dev/null +++ b/docs/gitops-dispatch-canary.md @@ -0,0 +1,60 @@ +# Independent `zed gitops` external-dispatch canary + +This canary certifies the root-command dispatch layer introduced for DEN-2725 against one exact immutable `zed-pkg/zed-cli` commit. It runs in the `zed-pkg-test` organization so the evidence is independent of the product repository and its branch-local tests. + +The current product authority is `zed-pkg/zed-cli#233`. The earlier #230 branch was intentionally closed as a duplicate after #233 incorporated the same safe sibling-dispatch model plus reviewed trailing-root-option, literal-double-dash, and malformed-boolean behavior. + +## Certified boundary + +On Ubuntu 24.04, macOS 15, and Windows Server 2025, the workflow: + +1. checks out the exact 40-character product commit without persisted credentials; +2. runs rustfmt, the focused external-subcommand unit and integration suites, and strict Clippy; +3. builds exact release `zed` and `zed-gitops` binaries; +4. compiles a standard-library-only probe executable used as an untrusted external command; and +5. runs independent black-box harnesses against copied binaries and disposable Git repositories. + +The harnesses prove 18 behavior and security contracts: + +- the Zed package installs both sibling executables and its smoke test exercises root dispatch; +- root help and Bash completion expose `gitops validate`; +- `zed gitops ...` and `zed help gitops` reach the sibling validator; +- direct and root-dispatched validation return identical structured reports for a valid exact-gitlink catalog; +- online-mode omission fails explicitly until online evidence exists; +- pin drift propagates policy exit code 2 and the `source.pin-drift` diagnostic; +- a missing documented validator fails closed with an actionable message; +- the current working directory and relative `PATH` entries are never searched; +- the sibling `zed-gitops` executable wins over a same-named executable in an absolute `PATH` directory; +- arbitrary arguments are not rewritten by a shell; +- explicitly supplied root options before the external command become the expected child environment variables; +- child exit status is preserved; +- a `zed-install` executable cannot shadow the built-in `install` command; +- recognized root options after the external command are lifted from child argv into canonical child environment; +- a literal `--` terminates root-option extraction and preserves all following child arguments; and +- malformed trailing root booleans fail closed without executing the extension. + +Each platform emits two commit-addressed JSON evidence records under schemas: + +```text +zed-pkg-test/gitops-dispatch-canary/v1 +zed-pkg-test/gitops-dispatch-trailing-options/v1 +``` + +The evidence contains only the immutable candidate, platform identity where applicable, binary SHA-256 values, and passed check names. It does not record command environments or credentials. + +## Isolation + +The workflow has read-only repository permission and all third-party Actions are pinned to immutable commits. Product and harness checkouts use `persist-credentials: false`. The black-box subprocess environment removes token, secret, password, private-key, access-key, API-key, authorization, cookie, and inherited Zed credential variables before invoking either binary or the probe. + +The fixture uses only a disposable local Git index and a synthetic mode-160000 gitlink. It performs no clone, push, registry access, Kubernetes mutation, Argo CD reconciliation, Cloudflare API call, DNS change, R2 access, or persistent namespace write. + +## Coordination + +- Product implementation: `zed-pkg/zed-cli#233` +- Closed superseded product lane: `zed-pkg/zed-cli#230` +- Standalone validator baseline: `zed-pkg/zed-cli#224` +- Real merged cluster-contract certification: `zed-pkg-test/zed-pkg-e2e#121` +- Linear: `DEN-2725` +- Portfolio roadmap: `ORESoftware/k8s-cluster#1097` + +This canary certifies command discovery and process boundaries. It does not activate the inert ApplicationSet pilot and does not claim online repository reachability, cluster access, or deployment authority. \ No newline at end of file diff --git a/docs/gitops-dispatch-current-main.md b/docs/gitops-dispatch-current-main.md new file mode 100644 index 00000000..ccff0b78 --- /dev/null +++ b/docs/gitops-dispatch-current-main.md @@ -0,0 +1,50 @@ +# Current-main root `zed gitops` certification + +This canary independently certifies the current-main DEN-2725 candidate after the frozen offline tool-profile slice landed: + +```text +product PR: zed-pkg/zed-cli#242 +candidate: 8eb8e31fe719ca598961a98bd139ef99ac905e8e +exact parent: de774cc9fc5b786d85e9b5ff19251675f0e9391b +``` + +The earlier immutable candidates and parents were independently certified and merged into this test repository by PRs #130 and #131. Product `main` then advanced through the separately certified offline tool-profile feature, which legitimately overlapped the GitOps branch at `src/lib.rs`. The current product candidate preserves `pub mod tool_profile;` and adds only the reviewed `external_subcommands` export. + +This pin update changes no certification logic. The workflow does not merge another branch, apply a correction script, or rewrite product source. It checks out the immutable product commit with its parent and fails before compilation unless: + +1. the candidate SHA is exact; +2. its direct parent is exact; and +3. its complete delta is exactly these eight reviewed paths: + +```text +.zpkg.toml +docs/gitops-validator.md +scripts/validate-zed-package-graph.sh +src/completion.rs +src/external_subcommands.rs +src/lib.rs +src/main.rs +tests/external_gitops_dispatch.rs +``` + +## Platforms and gates + +The test-org workflow runs on Ubuntu 24.04, macOS 15, and Windows Server 2025. Every platform runs: + +- Rust formatting; +- focused external-dispatch unit and integration tests; +- strict Clippy with warnings denied; +- the Zed package-graph validator; +- locked release builds of `zed` and `zed-gitops`; +- a standard-library-only hostile external-command probe; and +- the existing black-box dispatch and trailing-root-option suites. + +The black-box checks cover sibling-command selection, no working-directory or relative-PATH search, sibling precedence over absolute PATH, argument and exit-code propagation, structured validator parity, missing-extension behavior, literal `--`, recognized root-option lifting, malformed boolean rejection, help/completion visibility, and installed package outputs. + +## Credential and mutation boundary + +The canary strips inherited token, secret, password, private-key, access-key, API-key, authorization, cookie, and Zed credential variables from subprocesses. It uses only disposable local directories and a local Git index fixture. + +It performs no package publication, registry write, GitHub mutation, Kubernetes/Argo operation, Cloudflare request, external agent task, SOPS decryption, tool download, or persistent namespace write. Product source must remain byte-for-byte unchanged throughout the run. + +Evidence is commit-addressed and records only the immutable candidate/parent, runner OS/architecture, binary SHA-256 values, and passed check names. A green result certifies this exact product commit only; any successor requires another immutable pin. diff --git a/docs/global-git-submodule-mode-canary.md b/docs/global-git-submodule-mode-canary.md new file mode 100644 index 00000000..2e38261e --- /dev/null +++ b/docs/global-git-submodule-mode-canary.md @@ -0,0 +1,48 @@ +# Global Git-submodule compatibility canary + +This repository independently certifies the shared Git-submodule compatibility +switch merged from [`zed-pkg/zed-cli#150`](https://github.com/zed-pkg/zed-cli/pull/150). +The workflow pins the exact shipped merge commit: + +```text +ff166471338e46729b7cf892b44240452a56ace6 +``` + +and the exact public interface contract used by that commit: + +```text +c2e049006453c26ca8ca291783f681fce75cb01f +``` + +## Contract boundary + +The black-box harness builds disposable local Git repositories and invokes only +the compiled `zed` executable. It proves that one typed control surface works +consistently for cooperative installation and takeover: + +- Git transport remains disabled by default; +- `zed --git-submodules install` initializes top-level and nested submodules; +- `zed install --git-submodules=false` overrides an inherited true environment; +- `ZED_PKG_GIT_SUBMODULES=yes zed install` enables recursive initialization; +- `zed --git-submodules overtake` adopts eligible `.gitmodules` entries into + `.zpkg.toml` and `.zpkg.lock` with exact package and commit provenance; +- `zed overtake --git-submodules` and environment-only takeover are idempotent; +- explicit false disables takeover without mutating the manifest or lock; and +- root and takeover help expose the same option and environment key. + +The merged implementation's focused Rust tests and real-process interoperability +test also run before the black-box contract. Linux additionally runs all-target +Clippy with warnings denied. + +## Isolation + +The Ubuntu 24.04 and macOS 15 jobs use process-local `file` Git transport and an +empty disposable `file://` package registry. They receive no Zed token, GitHub +token, persisted checkout credential, publication credential, or public +registry mutation capability. Action, product, and interface inputs are pinned +by immutable full commit SHA. + +The existing twelve-check Git-submodule canary remains the broader certification +for takeover, mixed repositories, frozen replay, drift rejection, and rollback. +This focused lane is additive: it certifies the shared global CLI/environment +routing on the exact commit now present on `zed-cli` `main`. diff --git a/docs/inspect-current-main.md b/docs/inspect-current-main.md new file mode 100644 index 00000000..b2f1d51c --- /dev/null +++ b/docs/inspect-current-main.md @@ -0,0 +1,55 @@ +# Current-main `zed inspect` certification + +This canary independently certifies the canonical read-only IDE inspection contract: + +```text +product PR: zed-pkg/zed-cli#244 +candidate: b7036935ce3711563f1527472fc64e92bc1341aa +exact base: 1ab18fcb2ff884e82af4cac4513d7b983a23c84a +``` + +The workflow uses one immutable product commit and one immutable comparison base. It proves the base is an ancestor of the candidate and rejects the candidate unless the complete base-to-head delta is exactly: + +```text +schemas/inspect-v1.schema.json +src/cli_model.rs +src/inspect.rs +src/lib.rs +tests/inspect_cli.rs +``` + +The candidate contains a formatting follow-up commit, so its direct parent is intentionally not treated as the feature base. The canary does not poll a mutable branch, merge another candidate, apply a source correction, or couple CLI promotion to any IDE adapter release. + +## Three-platform quality and black-box contract + +Ubuntu 24.04, macOS 15, and Windows Server 2025 each run: + +- Rust formatting; +- focused inspect and CLI-model unit tests; +- compiled inspect CLI integration tests; +- strict focused Clippy with warnings denied; +- a locked release build of the exact candidate; and +- a standard-library-only adversarial black-box suite. + +The suite provides an unreachable registry, malformed saved credentials, fake environment and CLI token values, and a pending transaction sentinel. It requires: + +- exactly one JSON document and no successful stderr; +- `schema_version=1.0`; +- `offline=true`, `mutates_project=false`, and `loads_credentials=false`; +- stable recovery diagnostics and explicit suggested-action mutation, network, and package-code risk metadata; +- no token or malformed-manifest secret disclosure; +- byte-identical project and credential-home snapshots; +- no transaction recovery, materialization, or lockfile rewrite; +- help that succeeds without loading malformed credentials; +- global option values named `inspect` not triggering the command; and +- relative or missing roots failing without creating project or home state. + +## Credential and mutation boundary + +Every subprocess receives a scrubbed environment with token, secret, password, private-key, access-key, API-key, authorization, cookie, credential, and inherited Zed variables removed. Only disposable local project/home directories and an unreachable loopback registry are used. + +The workflow performs no package publication, registry write, package installation, package-code execution, transaction recovery, GitHub or Linear mutation, provider invocation, Slack task execution, Kubernetes/Argo operation, Cloudflare request, SOPS decryption, or persistent namespace write. + +Product and harness source trees must remain unchanged. Evidence contains only the immutable candidate/base, runner OS and architecture, release-binary SHA-256, and result. + +A green result certifies this exact product commit only. Any successor requires a new immutable pin. Downstream VS Code interoperability is intentionally separate and must be repinned after the CLI contract itself merges. diff --git a/docs/lifecycle-source-map.md b/docs/lifecycle-source-map.md new file mode 100644 index 00000000..27ef070c --- /dev/null +++ b/docs/lifecycle-source-map.md @@ -0,0 +1,68 @@ +# Lifecycle fixture source-map contract + +The stateless lifecycle harness must know where every declared fixture +dependency can be published from. `scripts/lifecycle.py` owns two disjoint +registries: + +- `PACKAGE_SOURCES`: native Zed package name to fixture repository and manifest + path; and +- `NON_PACKAGE_REPOS`: repositories that deliberately exercise the negative + no-manifest contract. + +A repository may not appear in both classifications. When a fixture is promoted +from a schema or support repository into a published Zed package, the source map +and non-package classification must change in the same reviewed commit. + +## Shared-schema promotion + +`zed-pkg-test/shared-schema` now publishes +`zedtest/shared-schema@1.0.0`. The source map therefore contains: + +```python +"zedtest/shared-schema": ("shared-schema", ".") +``` + +and `shared-schema` is no longer a non-package repository. This allows +`python-app` and future heterogeneous consumers to seed the exact schema package +before their own lifecycle begins. It also makes the `shared-schema` matrix row +run release, deterministic pack, r2g, publish, copy install, frozen replay, +yank, and restore checks instead of expecting all package commands to fail. + +## Source-cleanliness invariant + +`zed publish` writes its deterministic archive to `.zed/pack/` even when a +fixture repository does not ignore that path. The lifecycle harness removes +only the exact untracked archives expected from the manifest outputs before it +asserts that the source checkout is clean. + +This cleanup is deliberately narrow: + +- the expected filename is derived from the exact package name and version; +- only `?? .zed/pack/.tar.gz` status entries may be removed; +- tracked or modified files are rejected; +- unexpected files are rejected; +- symlinks, directories, and non-regular files are rejected; and +- all other source mutations still fail the lifecycle job. + +Repositories that already ignore `.zed/pack` produce no status entries and need +no cleanup. The harness does not treat arbitrary `.zed` content as disposable. + +## Drift gate + +`.github/workflows/lifecycle-source-map.yml` checks out immutable revisions of +`python-app`, `shared-schema`, and `polyglot-lib`. The standard-library contract +test proves that: + +1. every dependency declared by the Python consumer has a source-map entry; +2. each mapped path contains a `.zpkg.toml`; +3. the manifest's published outputs include the dependency name being mapped; +4. package source repositories are disjoint from `NON_PACKAGE_REPOS`; +5. repository names, package names, and relative paths are normalized; +6. the exact transient publish archive can be removed without weakening source + cleanliness; +7. unexpected or tracked pack files are never deleted; and +8. the checks leave all harness and fixture repositories clean. + +The workflow has read-only permissions, bounded execution, immutable Action and +fixture pins, no Python bytecode side effects, and no registry or publication +credentials. diff --git a/docs/live-test-org-fleet-audit.md b/docs/live-test-org-fleet-audit.md new file mode 100644 index 00000000..c17ae38e --- /dev/null +++ b/docs/live-test-org-fleet-audit.md @@ -0,0 +1,61 @@ +# Live paired test-fleet audit + +The exact repository denominator is exported from the current compressed manifest by: + +```sh +node scripts/export-test-org-fleet-index.mjs > canonical-test-fleet-index.json +``` + +The live auditor intentionally does **not** hard-code a pair count or repository count. It reads the same manifest and compares each production/test pair with GitHub's current organization inventories. The workflow then requires the auditor's pair and expected-repository totals to match the canonical exporter for the exact commit being tested. + +This matters because the fleet evolves. At the time this document was updated, current `main` exported 19 production/test pairs and 360 expected repository identities; future reviewed manifest changes may legitimately change those totals without requiring a second source-of-truth edit in the auditor. + +For every pair the auditor reports: + +- canonical repositories present and missing; +- independently added repositories, which are preserved as extras; +- archived, disabled, or default-branch-less canonical repositories; +- production and test organization repository counts; and +- production-side hygiene findings as warnings. + +`r2g` and `r2g-test` remain excluded by the canonical manifest and exporter. + +## Visibility boundary + +An exact audit requires metadata visibility into private repositories across every paired test organization. The repository-scoped `GITHUB_TOKEN` cannot provide that visibility and is deliberately rejected for the live job because permission gaps would otherwise become false missing-repository findings. + +Scheduled and manually dispatched live audits require `FLEET_AUDIT_TOKEN`, which must be either: + +- a read-only cross-organization token limited to repository metadata; or +- an approved short-lived GitHub App installation token supplied under the same secret name. + +The workflow refuses to substitute the normal `GITHUB_TOKEN` and fails with an explicit configuration message when the cross-organization credential is absent. Pull-request and push jobs remain secret-free and run only syntax, unit, and canonical-exporter contracts. + +## Local use + +```sh +GH_TOKEN="$(gh auth token)" node scripts/audit-live-test-org-fleet.mjs +GH_TOKEN="$(gh auth token)" node scripts/audit-live-test-org-fleet.mjs --json > live-audit.json +GH_TOKEN="$(gh auth token)" node scripts/audit-live-test-org-fleet.mjs --org shared-auth-test +``` + +The token needs repository metadata read access only. The auditor never writes GitHub state or prints the token. + +## Failure semantics + +The live job fails independently when: + +- the auditor cannot run or produce valid JSON; +- its pair count differs from the canonical exporter; +- its expected repository count differs from the canonical exporter; +- a canonical repository is missing; +- a canonical repository is archived, disabled, or lacks a default branch; or +- cross-organization credentials are absent. + +Extras are reported but preserved. Use `--strict-extras` only for an intentional exact-equality investigation; later product-specific repositories are not deletion targets. + +## Automation and evidence + +`.github/workflows/live-test-org-fleet-audit.yml` runs the offline auditor and canonical-exporter contracts on relevant pull requests and manifest changes. The exact cross-organization comparison runs only on the daily schedule or manual dispatch, where `FLEET_AUDIT_TOKEN` is required. + +The live job writes the pair-by-pair table, exact missing repository names, and canonical hygiene findings to the GitHub step summary. It retains the exact canonical index plus sanitized JSON and stderr evidence, making portfolio drift visible without relying on a stale hand-maintained count table or confusing permission gaps with repository deletion. diff --git a/docs/mise-environment.md b/docs/mise-environment.md new file mode 100644 index 00000000..b5121729 --- /dev/null +++ b/docs/mise-environment.md @@ -0,0 +1,85 @@ +# mise environment certification + +This suite certifies the project-local mise interoperability surface implemented by `zed-pkg/zed-cli`. + +`zed-pkg` is an independent multi-language package manager and has no relationship with the Zed editor. Here, `zed` means the `zed-pkg` CLI. + +## Scope + +The workflow `.github/workflows/mise-environment.yml` builds immutable `zed-cli` commit `93ff1cce4a0ed1a29a8172e1ba01c1cc12fddd01` against immutable `zed-interfaces` commit `c2e049006453c26ca8ca291783f681fce75cb01f`, then runs the same black-box suites on: + +- Ubuntu 24.04 x64; +- macOS 15 arm64; and +- Windows Server 2025 x64. + +Both revisions are full 40-character pins. A pull request that updates the certified CLI must update the pin explicitly and prove the new commit across the complete matrix. + +The CLI candidate is the clean static adapter in `zed-pkg/zed-cli#91`. It semantically includes the two Windows portability prerequisites already merged to `main`: + +- `zed-pkg/zed-cli#92`, which keeps the auth permission helper warning-free on Windows; +- `zed-pkg/zed-cli#94`, which gates Unix-only shell/executable test fixtures without changing production behavior. + +Relative to updated `main`, PR #91 still contains only the static adapter, typed CLI/dispatcher integration, tests, flags metadata, README, and documentation. Runtime `zed dev --mise` composition is already merged independently through `zed-pkg/zed-cli#70` and is documented alongside this read-only surface. + +## Certified behavior + +`scripts/mise_environment.py` exercises the real compiled `zed` executable. It verifies: + +- `zed env import mise` and `zed env verify mise --frozen`; +- exact config/lock coverage and stable machine-readable output; +- semantic digest stability across TOML key/table ordering and presentation changes; +- digest drift when locked artifact provenance changes; +- SHA-256 lock metadata, malformed-checksum rejection, and missing-checksum rejection; +- no parent or user-global mise configuration leakage; +- no dependency on an ambient `mise` executable—the child process receives an empty `PATH`; +- no mutation of `mise.toml`, `mise.lock`, or the project tree; +- fail-closed ambiguous-config, missing-lock, and config/lock-drift behavior; +- `.tool-versions` authoring import and frozen-lock rejection; and +- project-root escape rejection through symlink canonicalization where the host permits symlink creation. + +`scripts/mise_lock_naming.py` independently certifies mise's current normalized adjacent-lock convention through implicit and explicit real-CLI calls: + +```text +mise.toml -> mise.lock +.mise.toml -> mise.lock +mise.test.toml -> mise.test.lock +``` + +It proves that `.mise.toml` discovers `mise.lock`, that the JSON result reports the normalized lock path, and that a legacy `.mise.lock`-only project fails closed rather than being silently accepted. + +The harnesses intentionally do not install runtimes or execute mise tasks, hooks, templates, plugins, or environment expressions. Those are separate compatibility and trust surfaces. + +## Platform defects found by this suite + +The Windows matrix passed the actual static-mise unit and real CLI tests, then exposed two existing current-main portability defects under strict all-target Clippy: + +1. an auth helper parameter used only on Unix; +2. Unix-only develop-shell helpers and shell-script executable fixtures compiled on Windows. + +PRs #92 and #94 isolate and merge those repairs without weakening `-D warnings` or changing product behavior. This final candidate contains both prerequisite trees. + +## Local execution + +Build the exact `zed-cli` revision being certified, then run: + +```bash +python scripts/mise_environment.py \ + --zed ../zed-cli/target/release/zed \ + --work-root /tmp/zed-mise-environment + +python scripts/mise_lock_naming.py \ + --zed ../zed-cli/target/release/zed \ + --work-root /tmp/zed-mise-lock-naming +``` + +Each work root must not already exist. On success, each harness emits a JSON certification summary. On failure, it reports the exact failing command or invariant; GitHub Actions also uploads the disposable work roots and toolchain diagnostics. + +## Promotion rule + +Promote and merge this certification only after: + +1. all three operating-system jobs and the aggregate gate pass on `93ff1cce4a0ed1a29a8172e1ba01c1cc12fddd01`; +2. ordinary `zed-cli` CI, publish/R2G, development-shell, policy, Nix, OCI, and repository-hardening checks are green at that same head; +3. ordinary lifecycle, source-map, recursive-install, browser, install-boundary, and runtime E2E checks are green on this PR head; +4. the CLI and test pull requests cross-link their Linear issues and exact commit SHAs; and +5. legacy static adapter PR #38 is closed as superseded by current-main PR #91. diff --git a/docs/mise-export-boundaries.md b/docs/mise-export-boundaries.md new file mode 100644 index 00000000..deea84ae --- /dev/null +++ b/docs/mise-export-boundaries.md @@ -0,0 +1,79 @@ +# mise export boundary certification + +This suite independently certifies deterministic, conflict-safe projection from a Zed `EnvironmentPlanV2` into project-local mise TOML. + +`zed-pkg` is an independent multi-language package manager and is unrelated to the Zed editor. Here, `zed` refers only to the `zed-pkg` CLI. + +## Immutable implementation candidate + +Canonical product PR: `zed-pkg/zed-cli#131`. + +Pinned product head: + +```text +3434716591e048d0a769bafd47353d2953e89e48 +``` + +The product branch was semantically merged with current `main` head `751358fe130e64b646844bc7d857c4cff215ca2d`. Current submodule CLI and project-lock behavior were adopted verbatim, including the independently merged Windows lock-contention normalization; `src/main.rs` retains only the already-tested exporter import and dispatch arms inside the current main-line recovery and locking flow. The exporter remains a ten-file source, test, contract, and documentation delta with no materializer or synchronization workflow. + +The public Rust API enforces the reserved exporter-state source boundary before delegating to the implementation. This protects the CLI, external Rust callers, portable case aliases, and in-project symlink aliases through one shared guard. + +The certification workflow pins the full immutable product head above and never follows a mutable branch during a run. + +## Certified behavior + +The harnesses run the real CLI with an empty `PATH` and prove: + +- print mode is deterministic and read-only; +- `--write` creates generated TOML plus typed ownership state; +- `--check` verifies exact generated bytes; +- repeated writes are idempotent; +- hand-edited owned files are not overwritten; +- differing unowned files are not adopted or overwritten; +- an existing generated output cannot silently transfer ownership to a different plan, even when both plans currently render byte-identical TOML; +- a stale or corrupted recorded output digest is rejected rather than silently repaired; +- ownership-state failures leave both the generated output and state file unchanged; +- the public Rust API and real CLI reject the reserved exporter-state file as the plan source; +- the public API rejects a symlink to the reserved exporter-state file on Unix; +- safe missing output parent directories can be created transactionally; +- output cannot alias its input plan, including case-only aliases relevant to Windows; +- output and input cannot target the reserved export-state path; +- output and input cannot target transaction staging; +- nested secret-like keys in vars and tool options fail with exact field paths; +- symlinked export-state parents cannot redirect state outside the project; and +- no ambient mise executable, global mise configuration, or credentials are required. + +Both successful and failed commands are checked for unintended project mutation. + +## Three-platform gate + +The permanent workflow runs on: + +- Ubuntu 24.04; +- macOS 15; and +- Windows Server 2025. + +Each platform requires formatting, focused exporter unit tests, the public-API boundary target, both real CLI test targets, all-target Clippy with warnings denied, a locked release build, and both black-box harnesses. An aggregate gate requires every platform to succeed. All ordinary repository-wide E2E workflows remain merge gates on the same certification head. + +## Deliberate boundary + +This certifies deterministic manager-configuration export and ownership safety. It does not claim: + +- complete current `mise.lock` export; +- native `EnvironmentLock` translation; +- a general semantic three-way merge for arbitrary hand-authored mise files; +- execution of mise tasks, hooks, templates, plugins, or environment expressions; or +- installation of runtimes. + +The complete current manager-lock identity contract is separately merged and certified through DEN-1461. More sophisticated manager merge planning and lock-aware round trips remain separate follow-up work under DEN-1462. + +## Promotion gate + +Promotion requires: + +1. the workflow pin matching final immutable head `3434716591e048d0a769bafd47353d2953e89e48` of `zed-pkg/zed-cli#131`; +2. Ubuntu 24.04, macOS 15, and Windows Server 2025 success; +3. formatting, focused unit tests, the public API target, real CLI tests, a locked release build, and all-target Clippy with warnings denied; +4. both black-box harnesses passing on all platforms, with symlink assertions skipped only where the host cannot create symlinks; +5. ordinary lifecycle, recursive-install, browser, install-boundary, and runtime-mise workflows remaining green; and +6. exact implementation, certification, and merge commits recorded in DEN-1462 and the canonical mise interoperability policy. diff --git a/docs/mise-frozen-identity.md b/docs/mise-frozen-identity.md new file mode 100644 index 00000000..3f35c8e2 --- /dev/null +++ b/docs/mise-frozen-identity.md @@ -0,0 +1,64 @@ +# mise frozen identity certification + +This suite certifies the next read-only mise parity slice after the merged Tier-1 import/verify contract. + +`zed-pkg` is an independent multi-language package manager and is unrelated to the Zed editor. Here, `zed` means the `zed-pkg` CLI. + +## Immutable candidate + +Implementation PR: `zed-pkg/zed-cli#104`. + +The workflow builds and tests exact CLI commit: + +```text +41b7a228377f39aa07527725ee4439c1d153997e +``` + +The commit is a semantic two-parent composition of the reviewed frozen-identity product tree and then-current `main`. The implementation PR contains only: + +- `src/environment.rs`; +- `tests/mise_environment_cli.rs`; +- `docs/mise-interop.md`; and +- the permanent read-only `.github/workflows/mise-frozen-identity.yml` matrix. + +## Scope + +`scripts/mise_frozen_identity.py` runs the real immutable CLI with an empty `PATH` and verifies that frozen project-local mise state fails closed when its declared identity is incomplete or inconsistent. + +The certified checks are: + +- every applicable platform requested through `settings.lockfile_platforms` exists for every configured tool; +- a tool-level `os` constraint limits which requested platforms apply; +- a backend-qualified configuration such as `aqua:jqlang/jq` equals the lock's backend identity; +- the resolved lock version satisfies the authored exact, boundary-prefix, or supported SemVer-range requirement; +- a plain prefix such as `22` accepts `22.4.0` but rejects `220.0.0`; +- unsupported manager-specific range syntax fails closed rather than being approximated; +- successful and failing verification remain read-only; and +- no ambient mise executable or global configuration is required. + +The existing static-mise baseline was also strengthened: every tool now carries every globally requested platform so semantic-digest, provenance-drift, checksum, boundary, and mutation tests remain valid under the stricter completeness rule. + +## Cross-platform workflow + +`.github/workflows/mise-environment.yml` pins the full candidate SHA above and runs on: + +- Ubuntu 24.04; +- macOS 15; and +- Windows Server 2025. + +Each platform requires formatting, environment unit tests, the complete real CLI integration target, all-target Clippy with warnings denied, a locked release build, the original static import harness, normalized lock-naming certification, and the new frozen-identity harness. Failure diagnostics preserve all three disposable work roots plus toolchain and checkout state. + +## Deliberate boundary + +This suite does not claim complete current `mise.lock` parity. It does not yet certify option-dependent multiple identities, `install`, `url_api`, verified provenance, additional artifacts, shared conda/pkgx dependency sections, deterministic export, or import/export/import round trips. Those remain tracked by DEN-1461 and the remaining DEN-1481 work. + +## Promotion gate + +Promotion requires: + +1. every exact-head implementation workflow on `41b7a228377f39aa07527725ee4439c1d153997e` to pass; +2. this black-box harness and the existing static mise harnesses to pass on all three platforms; +3. strict all-target Clippy with warnings denied; +4. ordinary lifecycle, recursive-install, browser, install-boundary, and runtime-mise workflows to remain green; +5. the E2E workflow pin to match the implementation head exactly; and +6. exact implementation and certification commits to be recorded in DEN-1481, DEN-1461, and the canonical mise policy document. diff --git a/docs/mise-lock-contract-certification.md b/docs/mise-lock-contract-certification.md new file mode 100644 index 00000000..88537c8f --- /dev/null +++ b/docs/mise-lock-contract-certification.md @@ -0,0 +1,80 @@ +# Complete current `mise.lock` certification + +This suite independently certifies the exact current `zed-pkg/zed-cli` product head implementing the complete project-local `mise.lock` identity contract. The workflow pins full commit SHAs and never follows a mutable branch during a run. + +## Immutable product under test + +The reviewed contract merged through `zed-pkg/zed-cli#109`. The later reconstruction PR `zed-pkg/zed-cli#120` closed without merge and is not a product authority. + +Pinned CLI head: + +```text +7977b1d43231a68e1ac34903e0f18cef5135add6 +``` + +Pinned interface dependency declared by that exact CLI head: + +```text +a23cdd2ac509a39ca6fd6d21a3774fdd3a0f7660 +``` + +The workflow verifies both immutable pins against `Cargo.toml` and `Cargo.lock` before running the contract. A newer mutable `main` commit does not silently change an in-flight certification; advancing the baseline requires an ordinary reviewed commit here. + +## Current mise wire-format canary + +The implementation repository includes `tests/fixtures/mise-lock/current-actionlint.lock`, copied from mise commit `72379d0c459808f980a037065ac9c39a60032280`. + +That fixture uses mise's current quoted literal platform keys: + +```toml +[[tools.actionlint]] +version = "1.7.12" +backend = "aqua:rhysd/actionlint" + +[tools.actionlint."platforms.linux-x64"] +checksum = "sha256:..." +url = "https://..." +url_api = "https://api.github.com/..." +provenance = "github-attestations" +``` + +The cross-platform suite names the current-wire tests explicitly. They require parse, deterministic render back to quoted `"platforms."` keys, reparse, normalized equality, semantic-digest equality, and provenance-mutation sensitivity. A lock identity that mixes the current quoted form with the earlier nested compatibility form fails closed. + +## Certified behavior + +Across Ubuntu 24.04, macOS 15, and Windows Server 2025, the suite requires: + +- compilation of the complete lock model; +- strict formatting and all-target Clippy with warnings denied; +- unit and public consumer tests; +- option-dependent and multi-version tool identities; +- compact and detailed platform representations; +- conda/pkgx shared-package references; +- ordered additional release artifacts; +- provenance and verification-state relationships; +- current quoted platform-key wire compatibility; +- deterministic TOML/canonical JSON round trips; +- stable semantic SHA-256 identity; +- credential, fragment, malformed-host, and secret-query URL rejection; +- malformed or missing checksum rejection in frozen-portable mode; and +- explicit rejection of source-only restore state in portable locks. + +The highest-risk deterministic canaries run again after the ordinary focused targets so a passing aggregate gate cannot be produced solely by test-filtering mistakes. + +## Prior independent build evidence + +Before the permanent gate was opened, `zed-pkg-test/zed-pkg-e2e#47`, workflow run `30908244036`, independently executed the complete reviewed DEN-1461 source transformation. Its DEN-1461 job passed and uploaded ordinary-source artifact `den1461-materialized` with digest: + +```text +sha256:c07630605d85522850229b03055ea8f6c4439cb7f9c11e66f9d4eada6c651112 +``` + +That historical artifact is supporting evidence only. Review-ready promotion of this PR requires a green matrix against the current immutable product and dependency pins above. + +## Claim boundary + +This suite certifies the complete current manager-lock data and identity contract. It does not claim that `zed env import/verify mise` consumes every represented field, that native installation/activation supports every identity, or that deterministic export and offline replay are complete. Those remain separate DEN-1461 and DEN-1462 gates. + +Tracking: DEN-1461, DEN-1481, DEN-1462. + +`zed-pkg` is independent of and unrelated to the Zed editor. diff --git a/docs/mise-runtime.md b/docs/mise-runtime.md new file mode 100644 index 00000000..011cade5 --- /dev/null +++ b/docs/mise-runtime.md @@ -0,0 +1,74 @@ +# mise runtime certification + +This suite independently certifies the consumer-visible runtime composition added to `zed-pkg/zed-cli` by DEN-1420. + +`zed-pkg` is an independent multi-language package manager and has no relationship with the Zed editor. In this document, `zed` means the `zed-pkg` CLI. + +## Relationship to the static mise suite + +The draft static suite in `zed-pkg-e2e#7` verifies deterministic import, export, lock parsing, normalized digests, drift detection, and project-boundary behavior **without executing mise**. + +This runtime suite exercises a separate contract: `zed dev --mise auto|never|required` invoking `mise exec`, then composing the activated toolchain with Zed package restoration, `zed_modules/.bin`, and project-local mutable caches. + +Neither suite may substitute for the other. + +## Lifecycle baseline + +The canonical shared-schema lifecycle/source-map correction from `zed-pkg-e2e#24` is merged on `main` as `636fbe87aa26cf25a1c22d8f64383d4b5b8e6da8`. + +That baseline classifies `shared-schema` as a package fixture, maps `zedtest/shared-schema`, removes only the exact untracked publish archive with fail-closed checks, and passed the complete 22-fixture lifecycle matrix. This runtime suite is now based directly on `main`; it does not duplicate or carry a private copy of the lifecycle repair. + +## Immutable candidate + +`.github/workflows/mise-runtime.yml` builds a full 40-character `zed-cli` commit pin and runs the same black-box harness on Ubuntu 24.04 and macOS 15. The workflow does not install mise or download language runtimes. Instead, it supplies a deterministic executable stub whose arguments and relevant environment values are observable. + +The current candidate is `854b55c348307d00e2fd2ec8309792c249208192`, the current-main integration head of `zed-pkg/zed-cli#70`. That head preserves the six reviewed mise runtime paths while incorporating the merged recursive installer, strict frozen lock/fetch, Nix export/fetch/flake, OCI, and related current-main history. Update the pin whenever that pull request changes, and never promote this certification against a branch name or abbreviated SHA. + +## Certified behavior + +`scripts/mise_runtime.sh` proves: + +- `--mise required` enters `mise exec` exactly once; +- mise-selected tools and `zed_modules/.bin` are both available to the child command; +- noninteractive POSIX command execution preserves the composed PATH instead of starting a login shell that can rewrite it, including on macOS; +- `ZED_DEV_MISE` supplies the flags-to-environment fallback while an explicit CLI flag wins; +- `auto` falls back to the native Zed environment when no project configuration exists; +- `required` fails closed when configuration or the mise executable is missing; +- frozen runtime composition requires adjacent `mise.lock` evidence; +- frozen mode enables locked operation, ignores `.tool-versions`, and isolates user/system mise configuration beneath `.zed/dev/mise`; +- frozen integrated mode rejects ambient mise activation because the parent environment's config and lock cannot be proven by Zed; +- `mise.toml` has deterministic precedence when both supported config names exist; +- `.mise.toml` uses the same adjacent `mise.lock` contract; +- configuration discovery cannot escape the owning Git checkout; +- Zed and mise activation markers prevent recursive re-entry in non-frozen mode; and +- the exact child exit status survives both `mise exec` and `zed dev`. + +The workflow also runs all three direct command-shell test targets: `develop_edge_cases`, `develop_shell_edge_cases`, and `develop_help_contract`. This keeps the source-level Bash/Fish/PowerShell/cmd/generic dispatch matrix synchronized with the black-box PATH assertions. + +The first runtime assertion is a regression test for the macOS PATH failure that exposed the former `bash -lc` behavior. It requires both a mise-selected executable and a project-local Zed executable to remain callable in the expected order. + +## Local execution + +Build the exact candidate and run: + +```bash +cargo build --locked --release --manifest-path ../zed-cli/Cargo.toml --bin zed +bash scripts/mise_runtime.sh \ + --zed ../zed-cli/target/release/zed \ + --work-root /tmp/zed-mise-runtime +``` + +The work root must not already exist. On success, the harness emits TAP-style assertions followed by a machine-readable summary. On failure, an ERR trap identifies the failing line and command; GitHub Actions also uploads the disposable project tree and toolchain diagnostics. + +## Promotion gate + +Keep this pull request draft until: + +1. the pull request is based directly on current `main` containing merged PR #24; +2. `zed-pkg/zed-cli#70` has a final immutable current-main integration head; +3. the workflow and documentation pins match that head exactly; +4. Ubuntu and macOS runtime jobs plus the aggregate gate pass on that pin; +5. ordinary lifecycle, install/OCI, browser, and policy checks pass on the retargeted pull request; +6. the static suite in `zed-pkg-e2e#7` remains green for its own declared surface; +7. ordinary `zed-cli` CI, development-shell, policy, hardening, Nix, OCI, and formal-review checks are green; and +8. DEN-1420 and DEN-1449 link both pull requests and their exact candidate SHA. diff --git a/docs/native-dependency-range-canary.md b/docs/native-dependency-range-canary.md new file mode 100644 index 00000000..2d2c3f60 --- /dev/null +++ b/docs/native-dependency-range-canary.md @@ -0,0 +1,90 @@ +# Native dependency range and exact-lock canary + +This canary certifies the public `zed-interfaces` contract that translates npm +and Cargo dependency declarations before freezing one exact artifact identity. +It does not invoke npm, Cargo, crates.io, the npm registry, Nix, or another +resolver. + +## Immutable candidate + +The workflow pins the exact current-main product head: + +```text +zed-pkg/zed-interfaces@c872714fdb9a811e4a01f70082dbd61252975f47 +``` + +That candidate is zero commits behind the merged native-registry foundation and +contains only the six-file source-aware native dependency range and exact-lock +layer. A branch name, abbreviated commit, or moving tag is never accepted as +the product input. + +## Source-aware behavior + +The same declaration is deliberately interpreted through the selected native +registry: + +- npm `1.2.3` becomes exact `=1.2.3`; +- Cargo `1.2.3` becomes default caret `^1.2.3`; +- npm `1.2` becomes x-range `1.2.*`; and +- Cargo `1.2` remains default caret `^1.2`. + +The external consumer also exercises comparator intersections, wildcards, +major-zero caret behavior, explicit prereleases, and highest-satisfying +selection independent of candidate order. + +Npm partial inequality comparators receive their native boundaries: `>1` +canonicalizes to `>=2.0.0`, `>1.2` to `>=1.3.0`, and `<=1.2` to `<1.3.0`. +Whitespace between an operator and its version is covered for npm and Cargo; +Cargo still requires commas between multiple comparator clauses. + +## Frozen lock boundary + +A successful resolution records the original declaration, deterministic +canonical requirement, exact strict-SemVer package version, lowercase nonzero +artifact SHA-256, size, and archive format. The public validator recomputes the +translation and rejects canonical-requirement drift, a resolved version outside +the range, malformed artifact identity, duplicate candidates, and SemVer build +metadata. + +Npm requirements and exact candidate identities are also constrained to numeric +components representable exactly by JavaScript. Leading-zero partials and +components above `Number.MAX_SAFE_INTEGER` are rejected to avoid a Rust/npm +interpretation split. + +## Unsupported syntax + +The v1 canary requires fail-closed rejection for unions, hyphen ranges, npm +dist-tags and aliases, workspace/local/Git/URL sources, npm comma syntax, Cargo +whitespace-only comparator intersections, and npm-style Cargo `x` wildcards. +No rejected syntax may silently degrade to an opaque exact string. + +## Reproducibility and isolation + +The workflow: + +- requires one exact 40-character `zed-interfaces` commit; +- checks out all sources without persisted credentials; +- verifies rustfmt and the checked-in schema; +- generates the schema twice and requires identical bytes; +- runs the complete locked crate tests and Clippy with warnings denied; +- executes a separate Rust consumer locked and offline; and +- retains the schema, SHA-256, source status, exact source files, and consumer + lockfile as bounded evidence. + +It uses read-only repository permissions and immutable third-party Action +commits. No registry credentials or public mutation path are present. + +## Current-main composition + +The native-registry canary is already merged on current E2E `main`. This range +canary therefore carries exactly four files: the focused workflow, this +document, and the two-file external Rust consumer. It does not duplicate the +native-registry, Nix-bundle, lifecycle, browser, install, graph, or stress +harnesses. + +The branch preserves the prior reviewed range-canary history as a second parent +while its tree is rebuilt from current E2E `main`. The complete repository +workflow set remains a promotion gate: focused native-range certification must +pass alongside the current fixture lifecycle, install/OCI boundaries, browser +E2E, recursive graph and stress, mise runtime, Git-submodule, source-map, and +native-registry suites before this canary or the product layer merges. diff --git a/docs/native-lockfile-provenance-canary.md b/docs/native-lockfile-provenance-canary.md new file mode 100644 index 00000000..6ecf5cbe --- /dev/null +++ b/docs/native-lockfile-provenance-canary.md @@ -0,0 +1,77 @@ +# Native dependency provenance lockfile canary + +This canary proves that exact source-aware npm/Cargo resolutions can be stored in +`.zpkg.lock`, read back without reinterpretation, and consumed by the primary +Rust applications that share `zed-interfaces`. + +## Exact lockfile behavior + +The external Rust consumer verifies: + +- legacy lockfile version 1 documents remain readable with no native entries; +- npm and Cargo exact locks serialize as `[[native-dependency]]` tables; +- the original declaration, canonical requirement, exact version, digest, size, + and format round trip; +- insertion order does not change emitted TOML; +- npm and Cargo packages with the same textual name remain separate identities; +- upsert replaces only `(registry, package.name)`; +- duplicate native identities fail during write and parse; +- requirement-receipt drift fails during upsert; and +- malformed artifact identity fails before serialization. + +## Interface gates + +The workflow pins one exact `zed-interfaces` commit and requires: + +- rustfmt check; +- checked-in lockfile JSON Schema drift rejection; +- complete locked interface target tests; +- Clippy with warnings denied; and +- an external consumer built and executed offline after the interface build. + +## Primary Rust consumer compatibility + +The canary pins exact reviewed commits for: + +- `zed-cli` — `7e5ac1897a872223f8316ef1c3342a2fa1982504`; +- `zed-api-server.rs` — `3d2d4c98024e5bf718dc79c2648bd10d7759a31c`; and +- `zed-web-server.rs` — `b669850d317ebc07a8960aea4014706c55ffad22`. + +For zed-cli, a disposable checkout replaces only the Git-based +`zed-interfaces` dependency with the exact candidate path, then runs +`cargo check --all-targets` and `cargo test --all-targets --no-run`. + +The API and web repositories already use sibling path dependencies, so their +exact checkouts are placed next to the candidate and compiled with all targets. +The API workspace includes its migration crate. No consumer source changes are +committed by the canary. + +This makes the additive serialized change and the Rust source-level field +addition explicit across the CLI, API, web server, and an independent external +consumer. + +## Isolation + +- immutable 40-character interface and consumer pins; +- read-only repository permissions; +- no persisted checkout credentials; +- immutable third-party Action pins; +- no registry publication or credentials; and +- evidence containing schema digest, consumer lock, exact commits, CLI path + dependency patch, and source-tree status for every checked-out consumer. + +## Stack + +The native dependency range canary branch contains its reviewed lifecycle repair +as a real merge parent rather than only as pull-request metadata. The exact +inherited base commit is +`61a1216ee6ef2e2c7fc95bed2c6e4160e384cf4d`, whose parents are the range canary +head and lifecycle repair head. + +The lockfile branch then promoted its own previously tested synthetic merge into +real history at `e9166360507d173cc6a434ca8c5fd64cf354b007`. Final workflows run +on descendants of that commit, so source-map, native-range, lockfile, and all +consumer compatibility checks exercise one concrete branch tree. + +After the stacked bases land, retarget this PR to `main` without changing the +exact product commit or consumer behavior. diff --git a/docs/native-lockfile-provenance-current-main.md b/docs/native-lockfile-provenance-current-main.md new file mode 100644 index 00000000..78174d65 --- /dev/null +++ b/docs/native-lockfile-provenance-current-main.md @@ -0,0 +1,22 @@ +# Current-main native lockfile provenance canary + +This canary certifies the exact additive `[[native-dependency]]` contract in +`zed-pkg/zed-interfaces@a23cdd2ac509a39ca6fd6d21a3774fdd3a0f7660`. + +It runs from current E2E `main` and adds only this document, one read-only +workflow, and a two-file external Rust consumer. It does not duplicate the +range, native-registry, lifecycle, browser, install, graph, stress, source-map, +or environment canaries already on `main`. + +The contract preserves legacy lockfile v1 parsing while freezing source-aware +npm/Cargo resolutions as exact versions with immutable artifact SHA-256, size, +and format. It validates entries during parse, write, and upsert; rejects +canonical-requirement drift, duplicate `(registry, package.name)` identities, +malformed artifact provenance, and unsatisfied exact locks; and writes package, +native dependency, and Nix provenance in deterministic order. + +The focused workflow checks the exact candidate with formatting, deterministic +schema regeneration, every locked test target, strict Clippy, an external +consumer executed locked and offline, and current pinned CLI/API/web consumers. +No registry credentials, publication route, moving product pin, or persisted +checkout credentials are present. diff --git a/docs/native-registry-contract-canary.md b/docs/native-registry-contract-canary.md new file mode 100644 index 00000000..5326a417 --- /dev/null +++ b/docs/native-registry-contract-canary.md @@ -0,0 +1,62 @@ +# Native registry contract canary + +This canary consumes the public `zed-interfaces` native-registry API from a +separate repository. It is intentionally independent of npm, crates.io, +credentials, and publication side effects. + +## Certified candidate + +The workflow pins the exact current-main interface candidate: + +```text +a6aaad6911c845be7d8069d67f9bef82c85452b7 +``` + +That candidate preserves the merged native dependency/install-hook interface +surface while adding the finalized immutable native-registry publication +contract and its final review documentation. It contains no branch-writing or +temporary finalizer workflow. + +## Certified boundary + +For the immutable candidate, the workflow proves that: + +1. the complete interface crate compiles with its locked dependency graph; +2. the native-registry JSON Schema can be generated twice with identical bytes; +3. the checked-in Rust tests and schema-contract test pass; +4. rustfmt and strict Clippy pass over every target; +5. an external crate can construct and validate a generic npm wrapper with + Linux/musl and macOS platform packages; +6. a portable Cargo publication validates independently; +7. presentation order does not alter canonical adapter bytes; +8. SemVer versions differing only in build metadata collide at the native + publication boundary; and +9. publication-family topology fails closed for duplicate portable/meta roles, + empty or incomplete meta selections, dangling or mismatched edges, and + duplicate platform identity. + +Platform-only families are also certified when consumers select native packages +directly without a generic wrapper. + +The consumer keeps architecture and libc outside the version. It uses one +`1.4.2` API version for the wrapper and every platform package, with exact +artifact SHA-256, size, and format on each publication. + +## Isolation + +`.github/workflows/native-registry-contract.yml` pins the interface candidate to +a full 40-character commit, checks it out without persisted credentials, and +runs on a clean Ubuntu 24.04 hosted runner. After the full crate build warms the +Cargo cache, the external consumer is compiled and executed offline. The +generated schema, checksum, resolved consumer lock, exact candidate SHA, and +source-tree status are uploaded as a bounded evidence bundle. + +The canary performs no registry mutation and requires no npm, crates.io, OCI, +Zed registry, or other publication credential. + +## Updating the pin + +Change `ZED_INTERFACES_REF` only to a reviewed commit that contains the complete +contract, generated schema registration, topology hardening, and tests. The +pull request must show the exact certified commit and terminal-successful +workflow set before the pin is promoted to `main`. diff --git a/docs/nix-bundle-write-canary.md b/docs/nix-bundle-write-canary.md new file mode 100644 index 00000000..55f2f557 --- /dev/null +++ b/docs/nix-bundle-write-canary.md @@ -0,0 +1,79 @@ +# External Nix bundle-write canary + +Tracking: DEN-1592 + +This repository independently certifies the merged product command: + +```bash +zed interop nix bundle write \ + --frozen \ + --flake-lock ./approved-flake.lock \ + --out ./exports/package-flake +``` + +The workflow pins `zed-pkg/zed-cli@1b282b94efc994bb9352b572c9652d5a078787f2`. +The canaries import no Rust implementation module: they build that exact release +binary and drive only the executable plus Nix. + +## Synthetic full-command contract + +On Ubuntu 24.04 and macOS 15, `scripts/nix_bundle_write_canary.py` proves: + +- invocation below the project root discovers `.zpkg.toml` and `.zpkg.lock`; +- a missing output is created as a deterministic standalone flake; +- the compact receipt is `zed.nix-flake-bundle-write/v1` and reports the + canonical published destination; +- source project bytes remain unchanged and the configured Zed home is never + created; +- an exact second invocation returns `already-current` without rewriting bytes; +- the visible `--output` alias and environment-only flag contract work; +- an unknown option fails before output creation; +- a caller-supplied symlink as the immediate output parent is rejected without + writing through its target; +- tampered caller-owned output fails unchanged; +- generated text does not leak the source path, configured Zed home, or invalid + registry sentinel; and +- after one explicit locked acquisition step, the generated flake passes + `nix flake check --offline` and reproduces the same `nix build --offline` + store output. + +## Immutable real-fixture replay + +`scripts/nix_bundle_write_fixture_canary.py` independently checks out +`zed-pkg-test/node-lib@222cdf57f48530fce8e6c1f58632d9676203512e`, +creates two clean copies below unrelated absolute paths, adds the same explicit +artifact-only Nix intent and empty frozen lock, and requires the command to emit +byte-identical standalone bundles. + +The real-fixture canary independently verifies: + +- public package identity `zed-pkg-test/node-lib@1.0.0`; +- canonical `node_lib` Nix attribute and current runner system; +- exact artifact SHA-256 and size agreement with the retained plan; +- sorted, unique inventory entries whose bytes and sizes are independently + rehashed; +- immutable Nixpkgs revision and NAR hash evidence; +- identical domain-separated bundle identity across absolute project paths; +- no symlinks in either output; +- no source, temporary, home, credential, or invalid-registry value in generated + text; +- unchanged disposable project copies; +- a clean immutable fixture checkout; and +- non-creation of the configured Zed home. + +## Reproducibility and trust + +`.zed-nix-bundle-write-cli-ref` contains the default full merged commit pin. +Manual workflow overrides must also be full 40-character commits. The shared +`zed-interfaces` revision is asserted in both `Cargo.toml` and `Cargo.lock`. +The real fixture and every third-party Action are also full-commit pins. + +The workflow has read-only repository permissions, disables persisted checkout +credentials, owns all package, Nix, output, and Zed state below fresh +runner-temporary directories, and uploads bounded diagnostics only on failure. +It never publishes to a Zed registry, Cachix, Attic, GitHub release, package +namespace, or OCI registry. + +The canaries complement product-repository unit and integration tests. They are +persistent post-merge drift gates for the actual merged binary and immutable +external package inputs. diff --git a/docs/nix-export-plan-canary.md b/docs/nix-export-plan-canary.md new file mode 100644 index 00000000..a28e024b --- /dev/null +++ b/docs/nix-export-plan-canary.md @@ -0,0 +1,58 @@ +# External Zed → Nix plan certification + +Tracking: DEN-1411, DEN-1418, DEN-1422 + +This repository tests deterministic Nix export planning outside the `zed-cli` +implementation repository. `.zed-nix-plan-cli-ref` pins one exact 40-character +CLI commit. The workflow builds that commit and runs it against immutable clean +checkouts of real `zed-pkg-test/node-lib` and `zed-pkg-test/polyglot-lib` +fixtures on Ubuntu 24.04 and macOS 15. + +The reviewed inputs for this candidate are: + +- `zed-pkg/zed-cli@d8a2217edde8fa8c0a51f1db92e6f0b2b77296f5` from current-main planner PR #82; +- `zed-pkg-test/node-lib@222cdf57f48530fce8e6c1f58632d9676203512e`; +- `zed-pkg-test/polyglot-lib@998964aa58e49595547650b4a8da407b7a2283a9`; +- `zed-pkg/zed-interfaces@c2e049006453c26ca8ca291783f681fce75cb01f`. + +The candidate contains the merged install-shaped Zed→Nix bridge, pure Nix +evaluation ratchet, resolver-only `zed fetch`, resolver-only Nix FOD, and the +read-only planner. It is constructed from current `main` by a two-parent semantic +merge and contains only the eight planner-specific paths. It does not restore +obsolete Cargo pins, temporary branch-writing workflows, or stale fetch fixtures. + +The canary proves: + +- identical package bytes at different absolute paths emit identical compact + `zed.nix-export-plan/v1` JSON; +- exact manifest and lock digests are retained; +- packed artifact identity is stable and path-independent; +- systems are normalized deterministically; +- a real prebuilt executable appears in source and packed artifact; +- token, Supabase key, registry credentials, home, and workspace paths do not + enter the plan; +- planning never creates the configured global Zed home; +- fixture checkouts and disposable package copies remain unchanged; +- a manifest comment changes both exact-manifest and artifact identity; +- missing locks and source build hooks fail with actionable diagnostics; +- a polyglot package requires explicit target selection; +- target synonyms resolve to the same canonical target and plan; +- isolated Node and Rust targets are re-rooted to published identities; and +- different target payloads do not collapse to one artifact identity. + +The workflow verifies that the built CLI and both fixtures resolve to exact pins, +that Cargo uses the reviewed immutable interface revision, and that all three +checkouts remain clean after the canary. + +The workflow has top-level `contents: read`, uses only commit-pinned Actions, +disables persisted checkout credentials, accepts only a full immutable manual +CLI override, grants no OIDC or package-write permission, and uploads bounded +failure diagnostics only when an assertion fails. Its policy check scans the +entire workflow while constructing denylist tokens from fragments, avoiding +false self-matches without weakening checks against write permissions, pushes, +`pull_request_target`, secret inheritance, OIDC/package writes, or mutable +fixture refs. + +Before this canary merges to `main`, `.zed-nix-plan-cli-ref` must be advanced to +the final merged planner commit. Scheduled or dispatch-driven certification must +never depend on a mutable branch or pull-request merge ref. diff --git a/docs/nix-flake-bundle-canary.md b/docs/nix-flake-bundle-canary.md new file mode 100644 index 00000000..02d4edca --- /dev/null +++ b/docs/nix-flake-bundle-canary.md @@ -0,0 +1,125 @@ +# External standalone Nix flake-bundle canary + +Tracking: DEN-1422, DEN-1508, DEN-1418, DEN-1411 + +This repository independently certifies the public standalone-flake renderer from +`zed-pkg/zed-cli`. Here, **Zed** means the independent `zed-pkg` package manager; +it is unrelated to the Zed text editor. + +The current reviewed stack is: + +- external planner canary pinned to merged planner + `zed-pkg/zed-cli@19841ece0bf3649a0ed4005c150fb3721d166605`; +- this standalone-bundle canary pinned to renderer candidate + `zed-pkg/zed-cli@af4c44173ece59b799e77b1e1ed4e35268712b6e` from PR #83; and +- immutable shared contract + `zed-pkg/zed-interfaces@c2e049006453c26ca8ca291783f681fce75cb01f`. + +The canary is intentionally outside the implementation repository. It checks out +one exact CLI commit with persisted Git credentials disabled and injects one +temporary integration test that uses only the public +`zed_cli::nix_export_bundle` and `zed_cli::nix_export_plan` APIs. The temporary +test is removed before source-checkout cleanliness is asserted. + +## Contract under test + +The external consumer creates a deterministic prebuilt executable artifact and +an explicit strict-v1 export plan, then renders the same bundle twice into two +fresh absolute directories. It requires byte-identical output containing only: + +- `flake.nix`; +- the exact immutable `flake.lock`; +- `package.nix`; +- `README.md`; +- the exact Zed artifact; +- canonical `metadata/plan.json`; and +- canonical `metadata/bundle.json`. + +The independent Python verifier does not call implementation helpers. It checks: + +- the exact strict-v1 file set; +- safe relative paths, regular files, no symlinks, and canonical persisted modes; +- canonical compact JSON; +- sorted, unique inventory paths; +- every recorded file size and SHA-256; +- exact plan and flake-lock digests; +- the domain-separated `zed.nix-flake-bundle/v1` digest; +- byte-identical replay across clean output directories; +- absence of credential and absolute-path canaries; and +- binding between the retained external report and the canonical bundle record. + +## Negative coverage + +The public renderer must reject: + +1. a wrong Zed artifact digest; +2. a mutable Nixpkgs revision; +3. a symbolic-link archive entry; +4. a declared binary without executable mode; and +5. dependency edges that strict bundle v1 does not yet support. + +The in-memory validator and the independent persisted verifier separately reject: + +1. changed `package.nix`; +2. a missing artifact; +3. an unrecorded extra file; +4. changed `flake.lock`; and +5. noncanonical or modified bundle metadata. + +Assertions are not converted to warnings or skipped unsupported behavior. + +## Real Nix replay + +On Ubuntu 24.04 and macOS 15, the workflow archives both byte-identical flake +sources and performs one explicit online realization of the exact locked package +while the configured binary cache is available. It then runs `nix flake check` +and `nix build` for both clean-room bundles with `--offline` and +`--no-update-lock-file`. The primed output and both offline output paths must be +identical. + +This split is deliberate. An empty Nix store does not contain the immutable +Nixpkgs/stdenv closure. Asking it to build offline immediately tests whether a +hosted runner can bootstrap all of Nixpkgs from source, not whether the generated +Zed package is reproducible or network-independent after its declared closure is +available. The one online realization is therefore a visible input-acquisition +boundary; every replay assertion after that boundary is offline and lock-preserving. + +The resulting executable runs with no `zed` command on `PATH`, and the output +closure must not retain a `zed-cli` or `zed-pkg` runtime dependency. + +## CI and evidence policy + +The workflow has top-level `contents: read`, uses only commit-pinned Actions, +disables persisted checkout credentials, consumes no repository secret, grants +no OIDC or package permission, and never commits or pushes. Successful jobs +retain only non-secret hashes, systems, selected output evidence, and Nix path +information for seven days. Failure diagnostics are retained for three days. + +The complete workflow and external Rust consumer are scanned for prohibited write, +secret, OIDC/package-publish, network-client, and process-spawning capabilities. +The denylist tokens are assembled from fragments so the policy code does not +falsely match its own rule declarations. + +The candidate pin may be overridden manually only with another exact lowercase +40-character commit. Promotion to a durable baseline requires review and a +follow-up pin to the exact post-merge renderer commit. + +## Stack and merge order + +This PR is stacked on the external plan canary because a complete Zed → Nix +acceptance path is ordered as: + +1. immutable frozen plan; +2. pure deterministic bundle rendering; +3. independently verified persistence and replay; +4. explicit immutable closure acquisition followed by offline Nix replay; and +5. later adapter attestation, overlay/cache publication, and registry storage. + +After the external planner canary merges, this branch must be rebuilt on `main` +with exactly five bundle-specific files. After renderer PR #83 merges, the +candidate pin must be advanced to its exact merge commit before this canary can +become a durable baseline. + +The canary does not claim that locked dependency-graph assembly, source-builder +inference, overlay publication, signed binary caches, or upstream Nixpkgs +submission are complete. Those remain separately reviewed contracts. diff --git a/docs/oci-layout-contract.md b/docs/oci-layout-contract.md new file mode 100644 index 00000000..df7333de --- /dev/null +++ b/docs/oci-layout-contract.md @@ -0,0 +1,107 @@ +# Local OCI image-layout certification + +This suite certifies the credential-free local layout boundary added by +`zed-pkg/zed-cli#56`. It treats the Rust implementation as an untrusted producer: +the black-box harness walks the generated OCI image-layout, hashes every blob +independently, and compares the result with a separate plan-only invocation. + +The suite is stacked on the publication-plan contract in +`docs/oci-plan-contract.md`. Planning remains the authoritative identity model; +this follow-up proves that `--out` materializes those exact bytes without adding +network, authentication, or hidden project mutations. + +## Exact inputs + +`.github/workflows/oci-layout-contract.yml` pins: + +- one exact 40-character `zed-pkg/zed-cli` candidate commit; +- immutable commits for `node-lib`, `node-app`, and `polyglot-lib` fixtures; and +- immutable commits for every third-party GitHub Action. + +The job grants only `contents: read`, disables persisted checkout credentials, +and builds the candidate with its checked-in Cargo lockfile. + +## Certified output + +For each successful layout, the harness requires: + +1. `oci-layout` contains image-layout version `1.0.0`; +2. `index.json` is an OCI image index with exactly one manifest descriptor; +3. the index descriptor digest equals the resolved publication reference and + the plan-only manifest descriptor; +4. `org.opencontainers.image.ref.name` preserves the requested version tag; +5. every descriptor points to `blobs/sha256/`; +6. every blob filename equals the SHA-256 of its bytes; +7. every descriptor size equals the actual byte count; +8. the OCI manifest carries the expected Zed config and typed package, + source-manifest, and optional lockfile layers; +9. no undeclared or duplicate blob exists; +10. the reported blob count and total byte count match the independently walked + layout; and +11. two separate materializations of the same package produce byte-identical + trees apart from their requested output path. + +## Fixture coverage + +| Fixture | Boundary exercised | +| --- | --- | +| `node-lib` | deterministic layout, plan parity, full descriptor walk, syscall audit | +| `polyglot-lib` | one selected Rust target re-rooted from a four-language source repository | +| `node-app` | exact `.zpkg.lock` bytes preserved as a typed OCI layer | + +The suite also creates a pre-existing output directory and proves the command +fails without changing its sentinel file. + +## Source-tree and security boundaries + +Before and after every command, the harness fingerprints the source fixture, +including relative paths, file bytes, symlink targets, and permission bits. It +also installs an intentionally unrecoverable `.zpkg-staging` sentinel. A passing +candidate must therefore leave the source tree unchanged, leave no `.zed/pack` +output, and avoid transaction recovery. + +The representative `node-lib` materialization runs under `strace`. Any runtime +network socket syscall fails the job. Opening either poisoned saved-credentials +file also fails. Closed loopback proxies and nonexistent registry/auth endpoints +provide an additional guardrail. + +## Formatting and executable validation + +The workflow deliberately cannot push fixes back to the product branch. It: + +1. runs `cargo fmt` on a disposable exact checkout; +2. records a patch and formatted source files as evidence; +3. runs focused OCI tests, Clippy with warnings denied, and a locked release + build against the formatted tree; +4. runs the black-box contract; and +5. finally requires the originally pinned candidate to have produced no + rustfmt diff. + +A formatting-only failure therefore exposes the exact deterministic correction +without granting repository write permission. Product changes are reviewed and +committed in `zed-pkg/zed-cli`, then this canary is repinned to the new exact +commit. + +## Evidence + +Every run uploads formatting evidence, structured command results, layout +fingerprints, and the syscall trace for seven days. The evidence schema is +`zed-pkg-test.oci-layout-evidence/v1` and contains no registry token or account +secret. + +## Local execution + +Build the exact CLI candidate and place fixture checkouts beside this repository, +then run: + +```bash +python3 scripts/oci_layout_contract.py \ + --zed ../zed-cli/target/release/zed \ + --strace "$(command -v strace)" \ + --node-lib ../node-lib \ + --node-app ../node-app \ + --polyglot-lib ../polyglot-lib \ + --evidence /tmp/zed-oci-layout-evidence +``` + +Linux with `strace` is required for the credential/network assertion. diff --git a/docs/oci-oras-push-contract.md b/docs/oci-oras-push-contract.md new file mode 100644 index 00000000..c83f5199 --- /dev/null +++ b/docs/oci-oras-push-contract.md @@ -0,0 +1,95 @@ +# Authenticated ORAS push certification + +This suite certifies the first mutating OCI transport boundary in `zed-cli`. +It starts from a Zed-generated OCI image layout, pushes that exact layout to a +throwaway basic-auth registry through ORAS, and independently resolves the +remote tag back to the expected SHA-256 manifest digest. + +The test deliberately does **not** publish to GHCR or any persistent company +registry. The registry, username, password, repository, and tags exist only for +the duration of one GitHub Actions job. + +## Security contract + +`zed oci push` must select exactly one authentication source: + +1. `--username` with `--password-stdin`; +2. an explicit `--registry-config`; or +3. `--anonymous`. + +The command is not permitted to discover credentials implicitly from +`~/.docker/config.json`, `$DOCKER_CONFIG/config.json`, Zed registry credentials, +or shared-auth session state. The password-stdin path creates a temporary +Docker-compatible registry config with mode `0600`, passes that exact file to +ORAS, and removes it before returning. + +The representative authenticated push runs under `strace`. The suite fails if +it observes an open of an implicit Docker or Zed credential file. Password text +is never included in command arguments or structured evidence, and the harness +fails if it appears in stdout, stderr, syscall evidence, or the evidence JSON. + +## Registry and transport contract + +The workflow starts an isolated Docker Distribution registry on loopback with +bcrypt-backed HTTP Basic Authentication. `--plain-http` is accepted only because +the destination host is `127.0.0.1`; a non-loopback plain-HTTP destination must +fail before ORAS transport begins. + +The candidate must: + +1. verify every local OCI descriptor, blob size, SHA-256, media type, and exact + blob-set membership; +2. require the layout tag to match the destination tag; +3. resolve the existing remote tag through ORAS; +4. copy the layout with `oras cp --from-oci-layout` and an explicit destination + registry config; +5. resolve the remote tag again; and +6. return success only when the remote digest equals the verified local + manifest digest. + +## Mutation semantics + +The contract exercises three remote states: + +| State | Required result | +| --- | --- | +| tag missing | `pushed` | +| tag already resolves to the same digest | `already-present`, without another copy | +| tag resolves to a different digest | fail unless `--allow-tag-replacement` is explicit | + +After an explicitly permitted replacement, independent ORAS resolution must +return the second layout's digest. A refused replacement must leave the first +digest untouched. + +## Negative coverage + +The black-box harness also proves that: + +- a tampered package blob is rejected before ORAS is executed; +- omitting all authentication modes fails closed; +- plain HTTP is rejected for a non-loopback registry; +- source fixtures and unrecoverable transaction sentinels remain unchanged; +- no persistent `.zed/pack` directory is produced; and +- no `zed-oci-auth-*` temporary directory remains after success or failure. + +## Reproducibility + +The workflow pins the exact `zed-cli` candidate and fixture commits, pins every +GitHub Action to an immutable commit, requests a specific stable ORAS release, +builds with the candidate's checked-in Cargo lockfile, and grants only +`contents: read`. + +Structured command evidence, syscall evidence, the registry logs, and the exact +resolved final digest are uploaded for seven days. No real registry secret is +used or retained. + +## Deliberately deferred + +This contract does not yet cover: + +- digest-verified pull and restore; +- recursive referrer copy; +- SPDX, CycloneDX, in-toto, or signature attachment; +- cloud-registry-specific identity federation; +- client certificates or identity-token authentication; or +- multi-platform index publication across several Zed targets. diff --git a/docs/oci-plan-contract.md b/docs/oci-plan-contract.md new file mode 100644 index 00000000..8a1ec24e --- /dev/null +++ b/docs/oci-plan-contract.md @@ -0,0 +1,94 @@ +# Immutable OCI publication-plan contract + +This suite certifies the first OCI interoperability boundary for **zed-pkg**: +`zed oci plan` computes the exact immutable artifact identity without logging +in, contacting a registry, recovering a pending project mutation, or writing a +persistent package output. + +It is deliberately narrower than OCI publication. Authentication, blob upload, +manifest push, pull, signature/referrer handling, and registry-specific retries +remain separate follow-up contracts. A planner that passes this suite has +proved what bytes and descriptors a later transport is allowed to publish; it +has not proved transport behavior. + +## Immutable inputs + +`.github/workflows/oci-plan-contract.yml` accepts one exact 40-character +`zed-pkg/zed-cli` commit. The workflow also pins the three fixture repositories +and every third-party Action to immutable commits. It builds the candidate with +its checked-in lockfile and runs the real release binary. + +The initial fixture set is intentionally small but structurally complete: + +| Fixture | Contract exercised | +| --- | --- | +| `node-lib` | single-package deterministic planning, tag/digest rejection, and syscall audit | +| `polyglot-lib` | required target selection and independent Node.js, Python, Go, and Rust artifacts | +| `node-app` | required frozen lock provenance, lock layer inclusion, version drift, and digest canonicalization | + +## Certified invariants + +For every successful plan, the harness asserts: + +1. two independent invocations produce byte-identical JSON; +2. the public package identity matches the selected source or polyglot target; +3. the requested mutable tag is preserved while the resolved reference gains + exactly one canonical `sha256:` digest; +4. that digest equals the OCI manifest descriptor and planned manifest blob; +5. config, package, source-manifest, optional lockfile, and OCI-manifest blobs + have unique typed descriptors and positive sizes; +6. each blob digest agrees with the corresponding adapter layer; +7. the project tree is byte- and mode-identical before and after planning; +8. no `.zed/pack` directory is left behind; and +9. deliberately unrecoverable `.zpkg-staging` state remains untouched. + +The representative `node-lib` run executes under `strace`. Any runtime network +socket syscall fails the test. Opening either the poisoned saved-credentials +file or poisoned refresh-session file also fails. Closed loopback proxies and a +nonexistent registry endpoint provide an additional defense against an +accidental transport call. + +## Fail-closed cases + +The same executable must reject, without changing the project tree: + +- a destination tag that differs from the package version; +- a caller-selected digest instead of a digest derived from the planned bytes; +- a polyglot source without `--target`; +- a dependency-bearing package without `.zpkg.lock`; +- a locked dependency version outside the manifest requirement; and +- a noncanonical uppercase lock digest. + +Assertions are not weakened when a candidate changes. A behavior change must +update the shared OCI contract, unit tests, this black-box suite, and its design +rationale together. + +## Evidence + +Every completed harness invocation writes `evidence/evidence.json`, containing +all commands, exit statuses, stdout/stderr, plan fingerprints, and negative-case +coverage. The syscall trace for the audited run is stored beside it. GitHub +Actions uploads the directory for seven days. + +The evidence contains no registry token or account secret: the job grants only +`contents: read`, clears Zed token/password variables, disables persisted +checkout credentials, and runs the planner with unreadable invalid credential +files. + +## Local execution + +Build the exact candidate and place fixture checkouts beside this repository, +then run: + +```bash +python3 scripts/oci_plan_contract.py \ + --zed ../zed-cli/target/release/zed \ + --strace "$(command -v strace)" \ + --node-lib ../node-lib \ + --node-app ../node-app \ + --polyglot-lib ../polyglot-lib \ + --evidence /tmp/zed-oci-plan-evidence +``` + +Linux with `strace` is required because the credential/network claim is a +syscall-level assertion, not merely a mock-server assumption. diff --git a/docs/org-fixture-inventory.md b/docs/org-fixture-inventory.md new file mode 100644 index 00000000..708c2aff --- /dev/null +++ b/docs/org-fixture-inventory.md @@ -0,0 +1,47 @@ +# Organization fixture inventory + +`zed-pkg-test` is an executable compatibility surface for `zed-pkg`. A fixture +repository is not considered covered merely because it exists: it must be +classified, included in the full lifecycle matrix, and probeable as a package. + +## Checked contracts + +`fixtures/org-repositories.json` is the reviewed inventory. The inventory gate +fails when: + +- a live public repository is missing from the inventory or the lifecycle matrix; +- a classified repository disappears, is archived, disabled, forked, private, or + changes its default branch away from `main`; +- a package fixture loses its root `.zpkg.toml`, emits invalid TOML, or lacks the + required `[package]` identity fields; +- `scripts/lifecycle.py` classifies a package source outside the reviewed + inventory or changes the non-package boundary. + +The one manifestless repository is `zed-pkg-e2e`, the orchestrator itself. + +## Local commands + +```bash +python3 -m unittest discover -s tests -p 'test_*.py' -v +python3 scripts/org_fixture_inventory.py +``` + +The live probe uses read-only GitHub API access and needs no personal access +token: + +```bash +GITHUB_TOKEN=... python3 scripts/org_fixture_inventory.py \ + --live \ + --output /tmp/zed-pkg-test-inventory.json +``` + +GitHub Actions supplies its ephemeral read-only repository token. The script +also works without a token against the public API, subject to anonymous rate +limits. + +## Adding a fixture + +Add the repository to `fixtures/org-repositories.json`, add it to the +`strategy.matrix.repo` list in `.github/workflows/lifecycle.yml`, and make sure +the root manifest is valid. The unit tests and live E2E gate intentionally fail +until all three surfaces agree. diff --git a/docs/production-package-roundtrip.md b/docs/production-package-roundtrip.md new file mode 100644 index 00000000..d9b9877e --- /dev/null +++ b/docs/production-package-roundtrip.md @@ -0,0 +1,27 @@ +# Production package roundtrip + +`production-package-roundtrip.yml` is the fail-closed acceptance lane for +DEN-2788. It uses one immutable `zed-cli` revision against the real Rust +`zed-api-server`, PostgreSQL, and the server's artifact-storage contract. + +The release ledger is dependency-closed and ordered. It contains 19 exact +repository commits across `zed-pkg`, `shared-auth`, and `ORESoftware`, including +`zed-lib-core`, its shared schema dependency, and the Shared Auth packages that +close that dependency graph. + +For every package the lane: + +1. fetches the exact source commit and verifies any declared release tag; +2. re-parses `.zpkg.toml` and verifies identity, version, repository, and + dependencies against the ledger; +3. publishes through `zed publish` and verifies API metadata plus downloaded + artifact SHA-256 and archive readability; +4. repeats the publish and requires immutable idempotency; +5. installs the complete graph with `zed install`, verifies all 19 package + manifests, uninstalls it, and restores the same graph with `--frozen`. + +The PR lane uses local artifact storage so it is deterministic and does not +write production. The same script is suitable for a protected Kubernetes/R2 +lane by pointing `--registry` at the deployed API and supplying a short-lived +registry token. Production publication must retain the exact ledger, CLI, API, +and evidence artifact; it may not substitute mutable branch heads. diff --git a/docs/production-release-certification.md b/docs/production-release-certification.md new file mode 100644 index 00000000..2087c923 --- /dev/null +++ b/docs/production-release-certification.md @@ -0,0 +1,46 @@ +# Production registry release certification + +The production release is evidence-driven. A workflow run is not successful merely because a deployment or a publish command was attempted; every immutable source, image, database migration, package version, and R2 object must be tied to one ledger. + +## Immutable component set + +The release ledger records exact commits and image digests for: + +- `zed-lib-core`, including the canonical shared-definition revision +- `zed-api-server.rs` runtime and migration image +- `zed-web-server.rs` +- `shared-auth-server.rs` +- `zed-cli` +- `zed-infra` and the Argo CD application revision + +The migration job uses the same API image digest as the runtime and completes before either API or web replicas are promoted. Long-running replicas run with automatic migration disabled. + +## Full-stack certification + +The test stack uses one PostgreSQL instance for both machine publication and the canonical `zed_*` projection. It proves: + +- Shared Auth authorization code + S256 PKCE + exact callback +- single-use code redemption and replay rejection +- origin-scoped signed application session +- delegated `zed-pkg` audience and `zpkg:account` scope +- account pages and API mutations observe the same canonical rows +- private packages remain invisible outside their memberships +- private-to-public promotion succeeds at exactly 10 days and 50 downloads, and fails above either limit +- machine publication creates the package, immutable version, verified R2 upload ledger, and audit fact before returning success +- identical publish retries are idempotent and divergent same-version facts are rejected + +## Package and R2 evidence + +The inventory resolves 16–25 root `.zpkg.toml` repositories, including `zed-lib-core`, and requires each manifest's release tag to peel to the exact recorded commit. Publication is dependency ordered and uses one pinned Zed CLI binary. + +For every package version the ledger contains: + +- repository, commit, tag, manifest identity, version, and archive format +- API package and version metadata +- artifact SHA-256, byte length, and download URL +- downloaded artifact SHA-256 and successful archive listing +- direct R2 `HEAD` for `artifacts/.` in the configured bucket +- clean install, lockfile digest, uninstall, frozen reinstall, and post-reinstall digest +- first-publish or already-identical result plus a verified idempotent retry conflict + +At least one tar-family artifact and one ZIP artifact must be certified across the production and format-fixture ledgers. Secrets are available only to a protected production environment; pull-request workflows validate logic and immutable source pins without receiving credentials. diff --git a/docs/publication-immutability-canary.md b/docs/publication-immutability-canary.md new file mode 100644 index 00000000..45967874 --- /dev/null +++ b/docs/publication-immutability-canary.md @@ -0,0 +1,64 @@ +# Publication immutability canary + +This canary certifies the registry behavior required by deterministic builders +such as Nix and by every native-registry export that treats a public version as +an immutable identity. + +## Contract + +For one `{org, name, version}` publication: + +1. the first upload creates content-addressed artifact and version metadata; +2. an exact retry is idempotent and changes no registry byte; +3. a retry with changed artifact bytes and the same package identity fails; +4. the failed retry changes no metadata, artifact, yank state, or index object; +5. the artifact referenced by the original metadata still matches its SHA-256. + +The test does not accept “last writer wins.” Replacing bytes behind a version +would invalidate `.zpkg.lock`, Nix fixed-output derivations, remote caches, and +native-registry attestations. + +## Isolation and reproducibility + +`.github/workflows/publication-immutability.yml` builds one exact `zed-cli` +commit, verifies the binary checksum, and tests immutable commits of both a Node +and Rust fixture. Each matrix job owns a fresh directory-backed `file://` +registry and fresh Zed homes. + +The canary clears `ZED_PKG_TOKEN`, disables interactive Git authentication, +never contacts the public Zed registry, and checks that the fixture checkout +remains clean. Third-party Actions, the CLI, interfaces crate, and fixtures are +all pinned to full commit SHAs. + +## Test sequence + +`scripts/publication_immutability.py` performs the following black-box sequence: + +- publish the fixture; +- verify metadata and content-addressed artifact SHA-256; +- snapshot every registry file; +- republish the identical source and require the same complete snapshot; +- locate a source file that is actually present in the emitted tarball; +- mutate that file while preserving package name and version; +- pack again and prove that the candidate artifact digest changed; +- attempt the same-version publication and require failure; +- require the complete registry snapshot to remain unchanged; +- re-verify the original metadata and artifact digest. + +A JSON evidence summary and command log remain under the job work root. On +failure, the workflow uploads the source status, registry checksums, tree +listing, logs, and summary for inspection. + +## Local invocation + +Build the selected CLI, then run against a clean fixture checkout: + +```bash +python3 scripts/publication_immutability.py \ + --fixture-dir ../node-lib \ + --zed ../zed-cli/target/release/zed \ + --work-root /tmp/zed-publication-immutability +``` + +The work root must not already exist. It is intentionally retained after the +run so the registry and evidence can be audited. diff --git a/docs/quaestor-ledger-test-extension.md b/docs/quaestor-ledger-test-extension.md new file mode 100644 index 00000000..761e8160 --- /dev/null +++ b/docs/quaestor-ledger-test-extension.md @@ -0,0 +1,21 @@ +# Quaestor Ledger test-fleet extension + +The additive fleet overlay provisions three focused repositories in `quaestor-ledger-test` without rewriting the retained compressed base manifest: + +| Repository | Profile | Required risk coverage | +|---|---|---| +| `tenant-isolation-e2e` | `security-e2e` | Cross-tenant read/write rejection, tenant-claim mismatch, idempotency namespace isolation, and audit ownership. | +| `webhook-auth-replay-e2e` | `security-e2e` | HMAC verification, timestamp skew, replay rejection, payload tamper rejection, and redacted evidence. | +| `migration-recovery-e2e` | `database-e2e` | Forward/backward migration safety, backup restore, point-in-time recovery, closed-period durability, and post-recovery integrity. | + +`readFleetManifest()` and the bootstrap launcher both apply the same extension file. They reject duplicate repository names, duplicate pair patches, unknown test organizations, and source/test organization mismatches before any GitHub write. + +Validate the composed fleet before applying it: + +```sh +node scripts/validate-test-org-fleet.mjs +node --test tests/fleet-manifest-extensions.test.mjs tests/quaestor-fleet-extension.test.mjs tests/test-org-fleet.test.mjs +node scripts/bootstrap-test-org-fleet.mjs --dry-run --summary-json --org quaestor-ledger-test +``` + +Repository creation remains credential-gated. The apply workflow must use a short-lived GitHub App token or organization-managed `FLEET_GH_TOKEN`; credentials must never be committed to the manifest, overlay, generated test plans, or workflow logs. diff --git a/docs/real-k8s-gitops-contract.md b/docs/real-k8s-gitops-contract.md new file mode 100644 index 00000000..e5062d86 --- /dev/null +++ b/docs/real-k8s-gitops-contract.md @@ -0,0 +1,49 @@ +# Real `k8s-cluster` GitOps contract certification + +This test-org lane validates the actual merged ORESoftware GitOps contract rather than only a synthetic fixture. + +## Immutable inputs + +The default certification pins both repositories by full commit: + +- `zed-pkg/zed-cli@1c94813c6b59c7621c1256d7c55246e7f06f44b7` +- `ORESoftware/k8s-cluster@cfe39128e309936a4bcaeca4aaa35e4b9bec8888` + +The Zed commit is the coordinated product PR `zed-pkg/zed-cli#233` and contains the standalone validator plus reviewed root `zed gitops ...` dispatch. The earlier #230 candidate was intentionally closed as a duplicate. The cluster commit contains the merged v1alpha1 catalog, exact-pin validator, deterministic preview renderer, and inert ApplicationSet pilot from DEN-2724. + +## What the workflow proves + +The workflow checks out both public repositories without persisted credentials or submodule materialization, verifies the exact commit identities, and then: + +1. runs the focused Zed root-dispatch integration test; +2. runs the native `k8s-cluster` GitOps validator test suite; +3. builds exact release `zed` and `zed-gitops` siblings; +4. runs the native cluster validator and deterministic preview renderer against the real Git index and catalog; +5. runs both `zed-gitops validate` and root-dispatched `zed gitops validate` against that same checkout; +6. requires identical root/standalone reports and agreement with the native validator's record count; +7. verifies the real catalog retains exact gitlink-to-source pin parity and `pilot-inert` migration posture; +8. verifies the ApplicationSet template retains `missingkey=error`, exact child target revisions, collision-free names, and its explicit inert annotation; and +9. proves no other Argo YAML/JSON file references the pilot ApplicationSet by name or filename. + +The resulting artifact uses schema: + +```text +zed-pkg-test/real-k8s-gitops-contract/v1 +``` + +It contains only the two immutable commits, report and preview SHA-256 values, record counts, and passed check names. + +## Isolation and non-goals + +The workflow has read-only repository permission. All Actions are pinned to immutable commits, all checkouts disable persisted credentials, and subprocesses remove credential-shaped environment variables. + +It performs no submodule clone, private repository read, registry write, Kubernetes mutation, Argo CD reconciliation, Cloudflare request, DNS change, R2 access, or persistent namespace operation. Passing this contract does **not** authorize activation of the pilot ApplicationSet. Activation still requires a separate production PR with ownership parity, prune/deletion behavior, and rollback evidence. + +## Coordination + +- Cluster implementation: `ORESoftware/k8s-cluster#1109` +- Root dispatch: `zed-pkg/zed-cli#233` +- Closed superseded product lane: `zed-pkg/zed-cli#230` +- Three-platform dispatch canary: `zed-pkg-test/zed-pkg-e2e#118` +- GitOps roadmap: `ORESoftware/k8s-cluster#1097` +- Linear: `DEN-2724`, `DEN-2725`, and `DEN-630` \ No newline at end of file diff --git a/docs/terminal-context-certification.md b/docs/terminal-context-certification.md new file mode 100644 index 00000000..77b7a9e3 --- /dev/null +++ b/docs/terminal-context-certification.md @@ -0,0 +1,52 @@ +# Terminal-context certification + +This repository independently certifies the terminal/TTY/shell context behavior owned by `zed-pkg/zed-cli`. + +Canonical production ownership: + +- Linear: `DEN-2591` +- implementation: `zed-pkg/zed-cli#219` +- shipped merge: `ad6ff369a763f0fbdc3b677894655c73b23e062c` +- companion flags2env API: `DEN-2581` / `ORESoftware/flags-2-env#24` +- policy documentation: `zed-pkg/zed-docs#55` +- test-org acceptance: `DEN-2925` / `zed-pkg-test/zed-pkg-e2e#123` + +## Why this lives in the test organization + +The production repository already has unit, Docker, macOS, and Windows coverage. This test-org gate is deliberately independent: it checks out an immutable shipped candidate, builds that candidate on native GitHub runners, and drives the resulting executable as an external consumer would. + +A failure here is not fixed by adding a second terminal detector to the test harness. The defect is filed against the owning production repository, fixed there, and then re-certified here. + +## Platform matrix + +The workflow runs on: + +- Ubuntu 24.04; +- macOS 15; +- Windows Server 2025. + +Every lane checks the exact product Git commit before testing. + +## Acceptance contract + +The workflow runs the production `terminal_context` and `interactive` Rust unit contracts and then exercises the built CLI through the real `zed init` mutation checkpoint in a disposable test-org-owned working directory. + +`zed init` is intentionally used instead of registry installation: terminal-context certification should not depend on registry transport, artifact paths, network access, or package materialization. The first Windows run exposed an unrelated `file:///D:/...` registry-path defect before any prompt assertion ran; that transport defect is tracked separately rather than weakening or conflating this certification. + +The black-box assertions are: + +1. Piped `yes` is not human consent in ordinary CI/non-TTY execution. +2. Forced terminal stdin and stderr do not bypass CI when CI is forced true. +3. `TERM=dumb` remains fail-closed even with terminal state forced true and CI cleared. +4. The documented `ZED_PKG_FORCE_*` controls can explicitly simulate a safe terminal for deterministic testing. +5. The shared `F2E_FORCE_*` spellings produce the same accepted behavior without Zed-specific force variables present. +6. stdout may remain redirected while stderr carries the interactive checkpoint. +7. A rejected checkpoint leaves `.zpkg.toml` absent. +8. An accepted checkpoint creates the requested manifest with the expected package identity. +9. Linux and macOS additionally exercise the production `forkpty()` harness with a real pseudo-terminal; Windows uses the deterministic override path because that upstream helper is Unix-specific. + +## Security and permissions + +The certification workflow uses only `contents: read`. It does not require GitHub PATs, Linear tokens, Cloudflare credentials, registry credentials, or any other secret. The mutation target is a temporary runner directory and never a checked-out repository. + +The workflow must never copy user-supplied credentials into source, logs, artifacts, issue bodies, or environment variables. \ No newline at end of file diff --git a/docs/test-org-fleet-app-auth.md b/docs/test-org-fleet-app-auth.md new file mode 100644 index 00000000..d39125c4 --- /dev/null +++ b/docs/test-org-fleet-app-auth.md @@ -0,0 +1,123 @@ + +# Applying the test fleet with secret-backed credentials + +The fleet supports two credential sources: + +1. a personal access token stored as the encrypted repository Actions secret `FLEET_GH_TOKEN`; or +2. short-lived GitHub App installation tokens minted from `FLEET_APP_CLIENT_ID` and `FLEET_APP_PRIVATE_KEY`. + +The workflow input `authentication=auto` prefers `FLEET_GH_TOKEN` when it exists and otherwise uses the GitHub App. Explicit `pat` and `github-app` modes fail closed when their required secret-backed configuration is missing. + +No credential value belongs in the repository, workflow source, fleet manifest, generated repositories, commit messages, issue bodies, pull-request descriptions, workflow-dispatch inputs, or logs. + +## Live installation audit + +Audit date: 2026-08-08. + +The GitHub connector is installed on every in-scope test organization. `zed-pkg-test` exposes the 22 retained package-manager fixtures. Each of the other 18 test organizations currently exposes zero repositories, so repository creation is the first required write in those organizations. + +| Test organization | Repositories visible to the installation | +|---|---:| +| `zed-pkg-test` | 22 | +| `3fa-app-test` | 0 | +| `declarative-migrations-test` | 0 | +| `cliptown-test` | 0 | +| `claritas-viz-test` | 0 | +| `embedded-alerts-test` | 0 | +| `evento-globolo-test` | 0 | +| `fiducia-cloud-test` | 0 | +| `memebank-test` | 0 | +| `opto-sync-test` | 0 | +| `quaestor-ledger-test` | 0 | +| `sonus-auris-test` | 0 | +| `messaging-intel-test` | 0 | +| `scintilla-run-test` | 0 | +| `file-tunnel-test` | 0 | +| `shared-auth-test` | 0 | +| `hypesiege-test` | 0 | +| `hacker-house-medellin-test` | 0 | +| `streempilot-test` | 0 | + +`r2g` and `r2g-test` remain explicitly excluded. + +## Personal access token contract + +Store the PAT in `zed-pkg-test/zed-pkg-e2e` as the repository Actions secret named exactly `FLEET_GH_TOKEN`. + +The token owner must be allowed to create repositories in every selected test organization. For a classic PAT that creates private repositories, use the `repo` scope. Include the `workflow` scope because generated harness commits may write files under `.github/workflows/`. Authorize the token for each organization when SAML SSO enforcement requires it. + +The workflow exposes the stored secret only to the selected job runner, writes it to that runner's `GH_TOKEN` environment through `GITHUB_ENV` without printing it, and discards the runner after the job. GitHub's encrypted Actions-secret store is the only persistent location expected by this path. + +Do not use an Actions variable for a token. Variables are not secret storage. Do not pass a token as a `workflow_dispatch` input. + +## GitHub App contract + +Use a dedicated GitHub App whose credentials are controlled by the fleet operator. Install it on all 19 test organizations with access to **all repositories**, including repositories created after installation. Grant only these repository permissions: + +- Administration: write — create/adopt repositories and configure repository settings. +- Contents: write — create branches, trees, commits, and generated files. +- Metadata: read — inspect repositories and refs. +- Pull requests: write — open or reuse generated draft pull requests. +- Workflows: write — write generated files under `.github/workflows/`. + +The workflow mints one installation token per target organization. Tokens are not shared between organizations and are revoked by the token action when each job finishes. + +## Repository configuration + +Configure one of the following in `zed-pkg-test/zed-pkg-e2e`: + +### PAT path + +- Actions secret `FLEET_GH_TOKEN` containing the PAT. + +### GitHub App path + +- Actions variable `FLEET_APP_CLIENT_ID` containing the GitHub App client ID. +- Actions secret `FLEET_APP_PRIVATE_KEY` containing the GitHub App private key. + +`authentication=auto` works when either path is complete. If both are configured, it selects the PAT. Use the explicit workflow input to force one path. + +## Normal execution + +Open **Actions → apply test organization fleet → Run workflow** on the default branch. + +Choose either `all` or one test organization. An optional repository glob can constrain the run. Choose `auto`, `pat`, or `github-app`, then type `APPLY_TEST_FLEET` exactly in the confirmation field. + +For every selected organization, the workflow: + +1. validates the manifest and contract tests; +2. selects a configured secret-backed credential without printing it; +3. verifies visibility of the target organization; +4. creates or adopts the `.github` governance repository and specialized test repositories; +5. writes generated harnesses and immutable source pins; +6. pushes generated commits to agent branches; and +7. opens or reuses draft pull requests. + +The matrix serializes organizations with `max-parallel: 1`. Each organization also invokes the bootstrap with concurrency one, limiting rate pressure and keeping failures isolated. + +## One-time encrypted PAT handoff + +Use the encrypted handoff only when the repository secret is absent and an already-authorized PAT must be installed without placing it in source, workflow inputs, comments, or logs. + +1. Add or update `.github/fleet-handoff/START` on `main` so its complete trimmed content is exactly one approved test organization or `all`. +2. Review the scope in the commit or pull request before merging. Updating the handoff workflow itself does **not** start a handoff. +3. The workflow publishes a 4096-bit ephemeral RSA public key and metadata on a temporary branch. Metadata includes the selected organization, nonce, fingerprint, source SHA, and expiry. +4. Encrypt a JSON payload containing the matching nonce and token with RSA-OAEP-SHA256, then publish only the base64 ciphertext at the metadata path on that temporary branch. +5. The runner verifies the nonce, expected GitHub login, token shape, and visibility of only the selected organization unless `all` was explicitly reviewed. +6. The runner stores `FLEET_GH_TOKEN`, dispatches `apply-test-org-fleet.yml` for the exact START selection, deletes local plaintext/key material, and removes the temporary branch. +7. Remove `.github/fleet-handoff/START` after the apply run is dispatched. A deletion is explicitly skipped and does not start another handoff. + +The handoff expires after 15 minutes and persists only ciphertext during the exchange. Never publish the private key or decrypted payload. + +## Failure modes + +- **`authentication=pat` reports a missing secret:** create the repository Actions secret `FLEET_GH_TOKEN` or use the reviewed encrypted handoff. +- **PAT receives 401:** verify that the stored value is current and copied without surrounding whitespace. +- **PAT receives 403 creating a repository:** verify scopes, organization repository-creation policy, membership, and SAML SSO authorization. +- **GitHub App installation not found:** install the fleet App on the named test organization. +- **GitHub App repository creation returns 403:** grant repository Administration write and verify the organization permits the App to create repositories. +- **Workflow-file write returns 403:** grant the classic PAT's `workflow` scope or the App's Workflows write permission. +- **A newly created repository is inaccessible to the App:** configure the installation for all repositories rather than selected repositories. +- **Source repository cannot be resolved:** the generated repository remains source-gated; rerun after the source repository has a default-branch commit or after read access is granted. +- **Encrypted handoff rejects START:** use exactly one allowlisted test organization or the literal `all`; do not include prose or JSON. +- **Encrypted handoff expires:** start a new reviewed handoff and encrypt against the new nonce and public key. diff --git a/docs/test-org-fleet-index.md b/docs/test-org-fleet-index.md new file mode 100644 index 00000000..89d87610 --- /dev/null +++ b/docs/test-org-fleet-index.md @@ -0,0 +1,29 @@ +# Canonical paired test-fleet index + +The live test-fleet manifest is composed from deterministic compressed/base64 parts under `bootstrap/test-org-fleet.json.gz.b64.parts/` plus the audited additive overlay at `bootstrap/test-org-fleet.extensions.json`. + +Use the deterministic exporter instead of copied spreadsheets, archived reports, or manually maintained counts: + +```bash +node scripts/export-test-org-fleet-index.mjs > /tmp/canonical-test-fleet-index.json +``` + +The exporter includes, for every paired test organization: + +- retained baseline repositories; +- the public `.github` governance repository; +- every specialized managed repository; +- managed and total expected counts; and +- the exact expected repository-name set. + +It fails unless the composed live manifest still describes: + +- 19 production/test pairs; +- 22 retained repositories; +- 322 specialized repositories; +- 19 governance repositories; +- 341 managed repositories; +- 363 total expected repositories; and +- explicit exclusion of `r2g` and `r2g-test`. + +The fleet contract workflow publishes the full JSON index to its GitHub Actions job summary, making manifest changes visible in review before provisioning occurs. diff --git a/docs/test-org-fleet.md b/docs/test-org-fleet.md new file mode 100644 index 00000000..8f10cd6a --- /dev/null +++ b/docs/test-org-fleet.md @@ -0,0 +1,102 @@ + +# Cross-organization test repository fleet + +This repository is the control plane for the paired `*-test` organizations. The retained base manifest is stored as deterministic base64 parts under `bootstrap/test-org-fleet.json.gz.b64.parts/` and reconstructed automatically by the tooling (inspect with `cat bootstrap/test-org-fleet.json.gz.b64.parts/* | base64 -d | gzip -dc`). Audited additive repositories are declared in plain JSON at `bootstrap/test-org-fleet.extensions.json`; the validator, exporter, and bootstrap compose both sources and reject duplicate or mismatched patches. The fleet is intentionally product-specific: it creates focused consumer, API, UI, emulator, database, protocol, chaos, security, interoperability, performance, CLI, MCP, storage, audio, WebRTC, and infrastructure repositories instead of cloning one generic template everywhere. + +## Scope + +- 19 source/test organization pairs. +- 322 specialized repositories to create or adopt. +- 19 organization `.github` governance repositories. +- 22 existing `zed-pkg-test` fixtures retained as the baseline. +- 234 declared dependency lanes across real git submodules, Zed dependencies, and native package managers. +- `r2g` and `r2g-test` are hard-excluded by both the manifest and validator. + +## Fleet matrix + +| Source organization | Test organization | Specialized repos | Existing baseline | Primary scope | +|---|---|---:|---:|---| +| `zed-pkg` | `zed-pkg-test` | 14 | 22 | Package-manager, registry, adapter, workspace, security, and universal environment interoperability. | +| `3FA-app` | `3fa-app-test` | 12 | 0 | Multi-device authenticator, desktop, browser extension, sync, SDK, and recovery testing. | +| `declarative-migrations` | `declarative-migrations-test` | 12 | 0 | PostgreSQL/CockroachDB declarative migration correctness, rollback, and fault tolerance. | +| `cliptown` | `cliptown-test` | 13 | 0 | Cross-device clipboard, extension, API, SDK, offline sync, and Memebank interoperation. | +| `claritas-viz` | `claritas-viz-test` | 13 | 0 | Visualization SDK, API, Flutter/web UI, rendering fidelity, accessibility, and scale. | +| `embedded-alerts` | `embedded-alerts-test` | 13 | 0 | Embeddable alert UI, SDKs, API, WebSocket delivery, providers, and offline queues. | +| `evento-globolo` | `evento-globolo-test` | 17 | 0 | Global event discovery, organizers, ticketing, real-time updates, clients, and multi-framework UIs. | +| `fiducia-cloud` | `fiducia-cloud-test` | 31 | 0 | Polyglot clients plus consensus, routing, messaging, auth, observability, scale, and control-plane tests. | +| `memebank` | `memebank-test` | 14 | 0 | PWA/Flutter media bank, REST/OCR, storage, search, encryption, billing, and Cliptown interop. | +| `opto-sync` | `opto-sync-test` | 19 | 0 | Offline-first engines, clients, storage backends, conflicts, queues, clocks, and package interoperability. | +| `quaestor-ledger` | `quaestor-ledger-test` | 19 | 0 | Double-entry ledger, billing, executor, clients, UI, sync, auth, reconciliation, tenant isolation, webhook replay defense, and disaster recovery. | +| `sonus-auris` | `sonus-auris-test` | 19 | 0 | Always-on/scheduled audio recorder, encryption, retention, desktop/mobile, API, sync, and evidentiary integrity. | +| `messaging-intel` | `messaging-intel-test` | 17 | 0 | Read-only messaging connectors, privacy/redaction, analytics, mobile/extension UI, sync, and browser resilience. | +| `scintilla-run` | `scintilla-run-test` | 17 | 0 | Job/lambda runner, sandboxing, clients, Flutter/web UI, retries, replay, sync, and MCP. | +| `file-tunnel` | `file-tunnel-test` | 17 | 0 | Resumable encrypted file transfer, clients, UI, network faults, sync, and integrity. | +| `shared-auth` | `shared-auth-test` | 19 | 0 | OAuth/OIDC/WebAuthn, sessions, recovery, tenant isolation, clients, NATS/DLQ, sync, and auditing. | +| `hypesiege` | `hypesiege-test` | 18 | 0 | Social publishing scheduler with provider adapters, media, analytics, Flutter/web UI, SDKs, CLI, and offline sync. | +| `StreemPilot` | `streempilot-test` | 20 | 0 | Browser-first live studio with WebRTC, guests, recording, RTMP destinations, Flutter UI, SDKs, CLI, and sync. | +| `hacker-house-medellin` | `hacker-house-medellin-test` | 18 | 0 | Booking, tenant safety, access lifecycle, payments, offline sync, notification recovery, UI, clients, CLI, and MCP. | + +## Dependency strategy + +Every SDK-oriented repository declares one or more real dependency lanes: + +1. **Git submodule.** The bootstrap resolves the source default branch to a 40-character commit, writes `.gitmodules`, and adds a real `160000` gitlink to that commit. +2. **Zed.** The bootstrap writes `.zpkg.toml` using the same `owner/repository` naming convention already exercised by `zed-pkg-test`, plus `source-pins.json` for immutable provenance. Frozen lock replay is the required integration gate. +3. **Native package manager.** The manifest records the package manager, source subtree, expected coordinate when known, and source commit. Consumers must install the built artifact outside the source tree; source-path-only success is not a release pass. + +Fiducia receives exactly ten language-specific repositories: TypeScript/npm, Python/PyPI, Go modules, Rust/Cargo, Java/Maven, C#/NuGet, Dart/pub, Gleam/Hex, Kotlin/Gradle, and Swift/SwiftPM. Each also carries submodule and Zed lanes. + +## Apply workflow + +Validation and dry-run require no credential: + +```sh +node scripts/validate-test-org-fleet.mjs +node --test tests/test-org-fleet.test.mjs tests/fleet-manifest-extensions.test.mjs tests/quaestor-fleet-extension.test.mjs +node scripts/bootstrap-test-org-fleet.mjs --dry-run --summary-json +``` + +Creating repositories requires a short-lived credential authorized to create repositories in every selected test organization and to write contents and pull requests: + +```sh +GH_TOKEN="" node scripts/bootstrap-test-org-fleet.mjs --apply +``` + +Useful constrained runs: + +```sh +# One organization +GH_TOKEN="" node scripts/bootstrap-test-org-fleet.mjs --apply --org quaestor-ledger-test + +# One family of repositories +GH_TOKEN="" node scripts/bootstrap-test-org-fleet.mjs --apply --org 3fa-app-test --repo "clients-*-consumer" +``` + +## What `--apply` performs + +For every selected test organization it creates or adopts the public `.github` repository, seeds organization profile/community-health files and a reusable credential-free contract workflow, creates each missing test repository with the declared visibility, resolves all accessible source repositories to immutable commits, writes real submodule gitlinks, Zed metadata, native-package metadata, test plans, source gates, and two workflows, pushes a generated commit to `agent/initial-test-harness-v1` (or the governance branch), and opens a draft pull request. Re-running the command updates generated paths and reuses open branches/PRs rather than replacing repositories. + +## Source-gated projects + +`embedded-alerts` and `evento-globolo` were empty when this matrix was designed. Their expected source repository names are encoded explicitly. The bootstrap still creates useful test repositories and pull requests, but it writes `SOURCE-GATE.md` until those source repositories expose a default-branch commit. The same mechanism handles planned clients such as `claritas-viz/claritas-clients` and `memebank/memebank-clients`. Re-running the bootstrap converts the gate into an immutable source pin and, where requested, a real gitlink. + +## CI and security rules + +- Pull-request jobs use `permissions: contents: read`, a full immutable `actions/checkout` commit, `persist-credentials: false`, and no cross-organization secret. +- Full private-source integration is gated by `ENABLE_TEST_FLEET_INTEGRATION=true` and the organization-managed `TEST_FLEET_READ_TOKEN`. +- Generated validators reject credential-shaped values, moving source pins for existing repos, missing gitlinks, invalid dependency modes, duplicate repositories, unknown extension targets, source-pair mismatches, and any reintroduction of `r2g`. +- Synthetic fixtures are required for biometric recovery, private messages, authentication secrets, and raw audio/media. Logs must remain redacted. +- Scale suites separate a safe CI tier from explicit external tiers, such as Fiducia’s million-WebSocket target and File Tunnel’s very-large-file target. + +## Product-specific depth + +- **3FA:** Android/iOS emulators, desktop lifecycle, extension isolation, SDK consumers, WebSockets, offline sync, authenticator security, and recovery-factor policy. +- **Declarative Migrations:** forward/rollback across PostgreSQL and CockroachDB, concurrency, crash atomicity, drift, replay, data preservation, packaging, and fuzz properties. +- **ClipTown/Memebank:** every clipboard target, extension behavior, binary/image round trips between apps, OCR, storage providers, encryption, offline queues, and CORS/origin matrices. +- **Claritas:** multi-language SDKs, API, Flutter/web UI, rendering goldens, exports, accessibility, streaming, and million-point performance fixtures. +- **Fiducia:** polyglot clients plus Raft, locks/leases/fencing, semaphores, idempotency, cron failover, routing, discovery, NATS/DLQ, WebSockets, memory resize, auth, OTEL, edge partitions, sync, and control planes. +- **Quaestor Ledger:** double-entry and idempotency oracles plus dedicated tenant-isolation, webhook-auth/replay, migration, restore, and point-in-time-recovery repositories. +- **Sonus Auris:** device/desktop recording, permission lifecycle, schedules, codecs, end-to-end encryption, retention, evidentiary integrity, long-run soak, API, clients, sync, and desktop shutdown. +- **Messaging Intel:** read-only connector enforcement, browser-extension resilience, supported messaging/dating fixtures, privacy/redaction, analytics reproducibility, sync, and browser rotation. +- **Scintilla/File Tunnel/Shared Auth:** sandboxed job lifecycle; resumable encrypted transfer under faults; and OIDC/OAuth/WebAuthn/session/recovery/NATS security matrices. +- **Hypesiege/StreemPilot:** social publishing provider/rate-limit/media/calendar/analytics parity and live-studio WebRTC/guest/screen-share/recording/RTMP/reconnect coverage. diff --git a/docs/test-org-source-certification.md b/docs/test-org-source-certification.md new file mode 100644 index 00000000..07fee3f0 --- /dev/null +++ b/docs/test-org-source-certification.md @@ -0,0 +1,65 @@ +# Generated test-org source certification + +The paired `*-test` repositories have three different evidence meanings. They must never be reported as equivalent. + +## Credential-free contract lane + +Every generated repository has a pull-request workflow that validates the committed test plan, immutable source pins, declared dependency lanes, and any product-specific tests added outside the generated file set. + +This lane deliberately receives no cross-organization credential. It can prove that the committed snapshot is internally coherent, but it cannot prove that a private source repository still matches the snapshot or compiles at the pinned commit. + +Product-specific overlays are preserved. The fleet generator owns only its explicit managed paths, including `README.md`, `test-plan.json`, `source-pins.json`, generated workflows, dependency metadata, and source gates. Files such as executable fixtures, browser suites, contract parsers, and product-specific workflows remain repository-owned. A fleet rerun must update managed files without deleting or replacing those overlays. + +## Generic protected source-access lane + +The generic generated integration workflow runs only when the organization variable `ENABLE_TEST_FLEET_INTEGRATION` is exactly `true` and the protected read credential is configured. + +The credential is passed only to the pinned checkout action. Generated workflows must not fall back to `github.token`, because the repository token does not establish the intended cross-organization read authority and can turn a missing credential into a misleading partial run. Persisted checkout credentials remain disabled. + +The generic generated lane validates protected source access and the generated plan. It does **not** know how to build or test every product and therefore never certifies a source pin by itself. A successful generic lane reports `sourceAccessPassed: true` and `certified: false`, with a reason stating that product certification is still required. + +A skipped lane is **not** source certification. + +## Product-specific executable certification + +A repository may preserve a product-specific executable overlay that checks out the exact reviewed source, verifies its commit identity, and executes the relevant build, lint, unit, integration, browser, restart, protocol, or security tests. + +Only that executable product lane may report `certified: true`, and only after all required checks pass. A plan-only run, credential check, source checkout, static snapshot comparison, or skipped job is insufficient. + +## Durable status artifact + +Every generic integration workflow includes a separate status job with `if: always()`. It records: + +- whether the protected lane was enabled; +- whether it executed; +- whether source access and the generated plan passed; +- `certified: false` because the generic lane is not product-aware; +- the raw dependency-job result; +- a bounded reason string. + +Generic expected meanings: + +| Integration result | `executed` | `sourceAccessPassed` | `certified` | Meaning | +|---|---:|---:|---:|---| +| `success` | true | true | false | Protected source access passed; product certification remains required. | +| `failure` / `cancelled` | true | false | false | Protected access or plan validation did not pass. | +| `skipped` | false | false | false | The protected lane was not enabled. | + +A product-specific certification overlay may publish a separate status artifact whose `certified` field becomes true only after its exact source tests pass. + +Status artifacts contain no source credential and are safe to retain as metadata-only evidence. + +## Update protocol + +1. Reconcile source changes in the owning repository first. +2. Record an immutable source commit in the generated plan. +3. Preserve all product-specific overlay files. +4. Run the credential-free contract lane on the harness pull request. +5. Run the protected source-access lane when cross-organization access must be proven. +6. Run the product-specific executable lane before claiming source compilation or integration certification. +7. Link the status artifacts and product evidence from the source pull request and Linear issue. +8. When conflicts occur, preserve runtime behavior, documentation, client compatibility, and security policy as separate meanings. Never choose an entire conflict side solely because it is newer. + +## Security boundary + +Credentials belong only in protected organization or repository secrets. Do not place a PAT, GitHub App private key, cloud credential, or source token in generated plans, repository files, workflow inputs, command-line arguments, summaries, artifacts, issues, or Linear documents. diff --git a/docs/version-interop-conformance.md b/docs/version-interop-conformance.md new file mode 100644 index 00000000..2a7a5e69 --- /dev/null +++ b/docs/version-interop-conformance.md @@ -0,0 +1,79 @@ +# Version and native-publication interoperability corpus + +This contract converts the strict native-registry rules in +`zed-pkg/zed-interfaces` into reusable data plus an external Rust consumer. It +is stacked on the independent current-main native-registry contract canary so +future CLI, server, SDK, lockfile, and publication-plan work can reuse one +reviewed set of edge cases. + +## Immutable product pin + +The corpus certifies exactly: + +```text +zed-pkg/zed-interfaces@032b52f6e335e5696cb793d6a955c8f0658a95eb +``` + +The same 40-character commit appears in `cases.json`, the Rust consumer, and the +workflow environment. CI rejects disagreement among those pins. + +## Data sets + +`fixtures/version-interop-corpus/cases.json` contains: + +- three SemVer precedence projections with build metadata removed; +- four collision and non-collision pairs; +- a valid npm generic-wrapper family with Linux/ARM64/musl and Darwin/ARM64 + packages; +- a valid portable Cargo package; and +- nineteen fail-closed mutations. + +The negative corpus covers schema/version drift, malformed native names, +platform-selector topology, meta-to-platform references, duplicate identities, +artifact digest shape, and zero-byte artifacts. + +## Assertions + +The external consumer proves that: + +1. build metadata never differentiates precedence identity; +2. prerelease identity still participates in precedence; +3. valid npm and Cargo records survive canonical JSON round trips; +4. publication and platform-selector ordering do not change canonical bytes; +5. each negative mutation returns the exact expected public error class; and +6. the corpus cannot shrink without an explicit source change. + +## Isolation + +The workflow checks out the product commit without persisted credentials, builds +and tests the complete interface crate, generates a fixture lockfile, and then +runs the external consumer offline. Repository permissions are read-only and all +third-party Actions use immutable commits. No npm, crates.io, OCI, or Zed +registry is contacted or mutated. + +Evidence includes the checked corpus, its SHA-256, the generated consumer lock, +the exact product commit, command output, and clean-tree status. + +## Current-main relationship + +The current replacement is based on the same native-registry consumer branch as +`zed-pkg-test/zed-pkg-e2e#51`. The original reviewed corpus history remains a +merge parent, while the five-file review delta is carried onto the latest E2E +baseline and repinned to the current product integration head. The complete +browser, fixture lifecycle, recursive, mise, Git-submodule, and source-map +workflow set remains a promotion gate in addition to this focused corpus job. + +## Local shape + +With `zed-interfaces` checked out as a sibling of the harness repository: + +```sh +cargo test --locked --manifest-path ../zed-interfaces/Cargo.toml --all-targets +cargo generate-lockfile --offline \ + --manifest-path fixtures/version-interop-corpus/Cargo.toml +cargo run --locked --offline \ + --manifest-path fixtures/version-interop-corpus/Cargo.toml +``` + +The fixture is not a publisher. It validates the shared planning and identity +boundary that a later credential-bearing `apply` implementation must recheck. diff --git a/docs/windows-develop-clean-room.md b/docs/windows-develop-clean-room.md new file mode 100644 index 00000000..3a33fee7 --- /dev/null +++ b/docs/windows-develop-clean-room.md @@ -0,0 +1,65 @@ +# Cross-organization Windows `zed develop` canary + +This repository is the fixture-backed release acceptance boundary for Zed. The +workflow `.github/workflows/windows-develop-clean-room.yml` independently runs +the Windows shell contract owned by `zed-pkg/zed-e2e` against one immutable +merged `zed-cli` commit. + +## Why this is separate + +`zed-pkg/zed-cli` owns implementation tests. `zed-pkg/zed-e2e` owns the primary +external clean-room contract. This repository adds a third release perspective: +it checks out that exact external contract from another organization and proves +it can build and exercise the merged CLI without branch-local helpers, private +credentials, a registry service, or state retained from an earlier job. + +The canary is additive. It does not replace the primary E2E workflow or permit a +failed primary check to be ignored. + +## Immutable inputs + +```text +zed-e2e 144d17e28afa9586bc71ec43f340a0e0328a54b4 +zed-cli fd3b3e487b2bdd129dd67403ad51f7299cfe6828 +zed-interfaces c2e049006453c26ca8ca291783f681fce75cb01f +flags-2-env 2f62e40932a0fcb8b9bf1b4c84473e34fa3c51c7 +``` + +Every source and third-party Action is pinned to a full commit. Pin changes are +reviewed changes and require a complete replay. + +## Contract + +On `windows-2022`, the canary builds the real `zed.exe` and imports the exact +policy and functional harness from the pinned E2E commit. It verifies: + +- canonical `develop` / `dev` equivalence; +- PowerShell profile suppression, selected project cwd, and child exit + propagation; +- cmd.exe `/D /S /C`, selected cwd, managed environment, and `ERRORLEVEL`; +- default `COMSPEC` behavior; +- project-local Python virtual environments through `Scripts` and `sys.prefix`; +- isolated HOME and USERPROFILE without credential copying; +- no implicit dotenv, PowerShell profile, registry-token, or provider-secret + loading; +- bounded `--no-install` writes; +- actionable failure diagnostics; +- read-only workflow permissions and immutable source pins; and +- clean source checkouts without reset or cleanup-based masking. + +The cmd.exe fixture discovers the nearest manifest-owning project root and +creates two fixed relative batch names there. One batch performs native +statements; the launcher captures and returns its `ERRORLEVEL`. Zed executes the +launcher through relative `CALL`, avoiding an incidental absolute-path quoting +contract while preserving the product behavior under test. + +## Evidence + +The workflow stores only the bounded environment identity file and the +machine-readable clean-room report for seven days. It scans every retained file +for all fake credential canaries before upload. Temporary homes, virtual +environments, source checkouts, and batch fixtures are not retained. + +The owning Linear issues are DEN-1614, DEN-1616, and DEN-1634. The implementation +merged through `zed-pkg/zed-cli#100`; the primary external contract is +`zed-pkg/zed-e2e#16`. diff --git a/docs/zed-cli-graph-github-production-matrix.md b/docs/zed-cli-graph-github-production-matrix.md new file mode 100644 index 00000000..895104cd --- /dev/null +++ b/docs/zed-cli-graph-github-production-matrix.md @@ -0,0 +1,44 @@ +# `zed graph github` production certification matrix + +This test-org workflow certifies an exact `zed-pkg/zed-cli` branch, tag, or commit without consuming production-org Actions minutes. + +## Dispatch + +Run **zed-cli graph github production** with an exact candidate ref. Prefer a full 40-character commit SHA for promotion evidence. + +- `zed_cli_ref`: exact `zed-pkg/zed-cli` candidate ref. +- `run_live_smoke`: optional. The live `zed-pkg-test` organization scan runs only on Linux; macOS and Windows record an intentional skip while still running the deterministic fake-server suite. + +No GitHub token may be passed in command arguments. The candidate reads `ZED_PKG_GITHUB_TOKEN` from the job environment. The workflow does not persist checkout credentials. + +## Required matrix + +Every candidate must pass on Linux, macOS, and Windows: + +1. locked full-target/full-feature Cargo tests and build; +2. `zed graph github --help` contract, including repository/org inputs, source filters, JSON/DOT/Mermaid output, output path, and loopback/GitHub Enterprise API-base support; +3. absence of token/password/secret command-line options; +4. repeated deterministic fake GitHub scans with duplicate/mixed-case repository, organization, and include inputs; +5. byte-identical JSON, DOT, and Mermaid outputs; +6. exact default-branch commit, tree, blob, and gitlink evidence; +7. same-origin pagination acceptance and cross-origin pagination rejection; +8. authorization-bearing redirect rejection; +9. explicit partial inventory for a truncated recursive tree; +10. non-topological flat lock pins, with a transitive pin unable to fabricate a package edge; +11. iterative SCC/cycle/wave results over proven topology only; +12. renderer escaping for hostile requirements; +13. credential absence from stdout, stderr, graph output, request URLs, and bounded evidence. + +The fake server is loopback-only and requires the sentinel token on every request. It emits two same-origin organization pages, archived/private repository metadata, exact commit/tree/blob responses, Zed declarations and lock pins, a validated gitlink, Nix inputs, a two-node cycle, a cross-origin pagination trap, a redirect trap, and a truncated-tree partial case. + +## Live test-org smoke + +The optional Linux live smoke scans `zed-pkg-test` twice for each format and compares bytes. Exit `0` means complete inventory; exit `1` is accepted only when the artifact explicitly reports partial inventory. Evidence stores hashes, byte counts, immutable candidate identity, and aggregate counts—not raw inventories, response bodies, or credentials. + +A changing test-org head can make repeated live bytes differ. In that case, rerun after the test repositories are quiescent or add an exact-head fixture; do not weaken deterministic fake-server checks. + +## Promotion rule + +A production PR may reference this workflow only after the run is tied to the PR's exact head SHA. Branch-name-only runs are exploratory and cannot authorize merge. The production PR must also retain `resolution=not-claimed`; exact package resolution remains owned by DEN-2865. + +The durable AI review queue is `ORESoftware/ai-agent-bridge.rs#104` / DEN-2871. Queueing a review does not count as approval while the provider bridge is unavailable. \ No newline at end of file diff --git a/docs/zedignore-manifest-union.md b/docs/zedignore-manifest-union.md new file mode 100644 index 00000000..d1f87f6f --- /dev/null +++ b/docs/zedignore-manifest-union.md @@ -0,0 +1,67 @@ +# DEN-3018 `.zedignore` and manifest union certification + +This independent canary certifies the exact `zed-pkg/zed-cli` candidate that +makes the existing two-source publish-ignore contract visible to users. + +## Immutable product identity + +```text +comparison base: 1ab18fcb2ff884e82af4cac4513d7b983a23c84a +candidate: ca42115744b0d6d1ee18045b0e480e17a699af6f +product PR: zed-pkg/zed-cli#247 +``` + +This candidate supersedes `bc9d5336c1ccda55d9f98b3d9cabd5a25316c78a` +with a documentation-only correction from “six” to “five” rule families in an +example. That prior candidate had superseded +`4b27a48976686e2fd309926fafd0a2411de9fb87` with only the exact `cargo fmt` +output reported by the product repository's otherwise-green +check/test/Clippy job. + +The workflow requires the comparison base to be an ancestor of the candidate +and the complete base-to-candidate delta to contain exactly: + +```text +docs/publish-ignore.md +src/ops_entry.rs +``` + +It never follows a mutable product branch. + +## Certified contract + +Ubuntu 24.04, macOS 15, and Windows Server 2025 each run formatting, the focused +Rust unit tests, strict Clippy, and a locked release build. The resulting binary +then packs disposable local fixtures and inspects the real `tar.gz` members. + +The adversarial fixture proves that: + +- `[publish].exclude` is evaluated before `.zedignore`; +- both active sources produce exactly one union warning; +- opposite-polarity rules for the normalized `target` family produce one + conflict warning and the later `.zedignore` negation wins; +- same-polarity duplicates with case and `**/` spelling differences do not + produce a contradiction warning; +- manifest-only and `.zedignore`-only projects do not produce a dual-source + warning; +- manifest exclusions, `.zedignore` exclusions, and the `.zedignore` control + file stay out of the artifact; +- a deliberately retained hidden file is not removed by a blanket-dotfile + policy; and +- `.zpkg.toml` plus `LICENSE` remain present even when explicit exclusions match + them. + +The process receives a disposable Zed home and an unreachable loopback registry. +It performs no publication, registry write, package install, provider execution, +Cloudflare request, credential read, or persistent namespace mutation. Uploaded +evidence contains only immutable commit identities, runner identity, binary and +fixture-archive digests, boolean assertion results, and the final result. + +## Promotion order + +Keep both pull requests draft until exact-head product and independent test-org +checks are terminal-green and review threads are clear. Merge the independent +certification first; promote `zed-pkg/zed-cli#247` only while its final candidate +still matches the certified SHA and its comparison-base delta remains exact. + +Linear: DEN-3018 diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 00000000..ee5719da --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,11 @@ +// ores-lint house ESLint config. +// Managed by .ores-lint/ - see .ores-lint/README.md before editing. +// Repo-specific tweaks go in the options object below; the rollout script will +// not overwrite this file once you have changed it. +import oresConfig from './.ores-lint/eslint/base.mjs'; + +export default await oresConfig({ + // requireSend: { loggerNames: ['myLogger'], terminalMethods: ['send', 'flush'] }, + // rules: { 'no-console': 'warn' }, + // ignores: ['**/generated/**'], +}); diff --git a/evidence/2026-08-06-zed-0.1.0-rc3-test-org.json b/evidence/2026-08-06-zed-0.1.0-rc3-test-org.json new file mode 100644 index 00000000..3e927acd --- /dev/null +++ b/evidence/2026-08-06-zed-0.1.0-rc3-test-org.json @@ -0,0 +1,113 @@ +{ + "binary": { + "version": "zed 0.1.0", + "archive_sha256": "ed55e39811cac6303162536397562ed004c96c4cd7e09b531f0d323f59142aff", + "source_commit": "2f98df7b0f3f20bd8eaec6abbe768566833589bc" + }, + "suites": { + "zed-pkg-test/offline-cache-e2e": { + "checks": [ + { + "name": "cold publish/install and Node consumer import", + "status": "pass", + "detail": "2a1d43172e751afd41564bd47ba549cafed2f6a0c9750a3dd6517437534ea4d7" + }, + { + "name": "offline frozen reinstall with registry absent and proxies dead", + "status": "pass" + }, + { + "name": "shared-home cache reuse in second checkout", + "status": "pass" + }, + { + "name": "corrupt cache + offline registry rejected without partial materialization", + "status": "pass" + }, + { + "name": "corrupt cache recovers from immutable registry artifact", + "status": "pass" + } + ], + "status": "pass" + }, + "zed-pkg-test/version-solver-e2e": { + "checks": [ + { + "name": "compatible range selects highest matching version", + "status": "pass", + "detail": "1.1.0" + }, + { + "name": "exact version remains exact", + "status": "pass" + }, + { + "name": "incompatible transitive constraints fail deterministically", + "status": "pass", + "detail": " available versions: 2.0.0, 1.1.0, 1.0.0" + }, + { + "name": "SemVer build-metadata ambiguity has a stable outcome", + "status": "pass", + "detail": "('resolved', '1.0.0+bbb')" + } + ], + "status": "pass" + }, + "zed-pkg-test/security-adversarial-e2e": { + "checks": [ + { + "name": "absolute symlink omitted from deterministic artifact without external bytes", + "status": "pass" + }, + { + "name": "target directory traversal rejected", + "status": "pass" + }, + { + "name": "package build hook does not execute without explicit consent", + "status": "pass" + }, + { + "name": "explicit --allow-build executes declared hook", + "status": "pass", + "detail": "zed_modules/zed-pkg-test/hook-fixture/build-marker.txt" + }, + { + "name": "tampered frozen lock rejected without partial install", + "status": "pass" + } + ], + "status": "pass" + }, + "zed-pkg-test/manager-interop-e2e": { + "checks": [ + { + "name": "mise frozen import/verify is parser-only, deterministic, and read-only", + "status": "pass", + "detail": "024d7c10747805264e831a8f1de9e288367b6c23201f2ae18c5967212ff74781" + }, + { + "name": "mise ambiguity and incomplete frozen state fail closed", + "status": "pass" + }, + { + "name": "asdf frozen import/verify validates immutable plugin/artifact provenance read-only", + "status": "pass", + "detail": "88a61a024d7fc0a7c32c7813076c23643bf8f6835fd8f3dbed1b1a1cdee632e8" + }, + { + "name": "asdf moving selector rejected in frozen mode", + "status": "pass" + }, + { + "name": "unsupported manager labels fail at typed CLI boundary", + "status": "pass", + "detail": "flox,devbox,nix" + } + ], + "status": "pass" + } + } +} diff --git a/evidence/den-2725-final-export/.zpkg.toml b/evidence/den-2725-final-export/.zpkg.toml new file mode 100644 index 00000000..66ae9761 --- /dev/null +++ b/evidence/den-2725-final-export/.zpkg.toml @@ -0,0 +1,54 @@ +[package] +org = "zed-pkg" +name = "zed-cli" +version = "0.1.0" +description = "The zed universal package-manager CLI" +license = "MIT" +keywords = ["package-manager", "cli", "polyglot", "vcs", "global-tools"] +language = "rust" + +[package.repository] +vcs = "git" +url = "https://github.com/zed-pkg/zed-cli" + +[build] +command = "cargo build --release --locked --bins" +outputs = ["target/release/zed", "target/release/zed-gitops"] + +[bin] +zed = "target/release/zed" +"zed-gitops" = "target/release/zed-gitops" + +[install] +adapter = "none" +dir = ".vendor/.zed" + +[dependencies] +"zed-pkg/zed-clients" = "^0.1.0" +"zed-pkg/zed-interfaces" = "^0.1.0" +"zed-pkg/zed-lock" = "^0.1.1" + +[publish] +include_readme = true +tag_format = "v{version}" +smoke_test = "test -x \"$ZED_PKG_TEST_TARGET/target/release/zed\" && test -x \"$ZED_PKG_TEST_TARGET/target/release/zed-gitops\" && \"$ZED_PKG_TEST_TARGET/target/release/zed\" --version && \"$ZED_PKG_TEST_TARGET/target/release/zed\" gitops validate --help" +exclude = [ + ".env", + ".env.*", + ".direnv/**", + ".zed/**", + ".zed-pack/**", + ".vendor/.zed/**", + "target/**", + "tmp/**", + "**/*.log", + ".DS_Store", +] + +[publish.native] +registry = "crates-io" +package = "zed-cli" + +[scripts] +test = "bash scripts/validate-zed-package-graph.sh && cargo test --all-targets --locked" +validate-package-graph = "bash scripts/validate-zed-package-graph.sh" diff --git a/evidence/den-2725-final-export/docs/gitops-validator.md b/evidence/den-2725-final-export/docs/gitops-validator.md new file mode 100644 index 00000000..e3147e07 --- /dev/null +++ b/evidence/den-2725-final-export/docs/gitops-validator.md @@ -0,0 +1,67 @@ +# `zed gitops` external validator + +`zed gitops` is the read-only GitOps validation lane tracked by DEN-2725. The +root `zed` binary now securely dispatches this command to the separately built +`zed-gitops` executable, while root help and shell completions expose the same +public command contract. + +Install or build both binaries into the same bin directory: + +```console +cargo install --path . --bins +zed gitops validate --root . --offline --strict +zed gitops validate --root . --offline --strict --format json +zed gitops validate --root . --offline --strict --format sarif +``` + +The standalone spelling remains supported for automation that deliberately +pins the validator executable: + +```console +zed-gitops validate --root . --offline --strict +``` + +The dispatcher resolves `zed-gitops` beside the running `zed` executable first, +then searches only absolute `PATH` entries. It never invokes a shell, never +searches the current working directory implicitly, never permits an extension +to shadow a built-in command or alias, and preserves the child process exit +code. Root options placed before or after `gitops` are passed as their canonical +`ZED_PKG_*` environment variables rather than being exposed on the child +command line. A literal `--` ends global-option extraction and passes every +remaining argument to the external command unchanged. + +## Evidence checked + +- catalog JSON is regular UTF-8 data beneath the selected repository root; +- unknown fields fail under `--strict`; +- `.gitmodules` provides exactly the declared inventory path and repository; +- the Git index contains that path as a mode-160000 gitlink; +- catalog inventory revision equals the indexed gitlink SHA; +- Argo source repository canonicalizes to the same upstream repository; +- Argo `targetRevision` is an exact lowercase 40-hex commit equal to the + gitlink; +- the source is the direct app repository, not a path inside + `ORESoftware/k8s-cluster`; +- application names and inventory paths are unique; +- `*-infra` repositories cannot be app records; +- AppProject and destination namespace cannot be `default`; +- `pilot-inert` records cannot enable automated sync, prune, or self-heal; +- the retained static Application is a regular parent-owned file. + +The command does not read Kubernetes credentials, clone private repositories, +resolve remote branch tips, or apply manifests. Online validation is not +implemented yet, so invocations must pass `--offline`; omitting it fails +explicitly instead of misreporting a local-only run as online evidence. Policy +failures exit with code 2; tool/configuration failures exit with code 1. + +## Ownership boundary + +The root CLI owns extension discovery, built-in collision prevention, help, +completion, TTY inheritance, and exit-code propagation. `zed-gitops` owns the +current validation implementation. Follow-up work should expose the existing +`git_submodules` repository-identity and index primitives as a stable +`zed-pkg` library surface so the validator does not maintain parallel generic +Git parsing. + +The deployment-specific schema and policy remain versioned in `k8s-cluster`; +Zed remains the validator UX rather than the deployment controller. diff --git a/evidence/den-2725-final-export/evidence/SHA256SUMS b/evidence/den-2725-final-export/evidence/SHA256SUMS new file mode 100644 index 00000000..940aa67d --- /dev/null +++ b/evidence/den-2725-final-export/evidence/SHA256SUMS @@ -0,0 +1,10 @@ +17ea028f56b5e008a5fc698af4969aa65cdc80666f32865df1658ac1c525960c corrected/.zpkg.toml +deff169c5e53f448792b0414ae870d33ae492ae8ca86201834fc6497987d2714 corrected/den-2725-final-replay.patch +3cb8ade82fd5c4c316e8890d4dd8334e4dfa15b51bd136f31f366014e3509d7e corrected/docs/gitops-validator.md +949c852aed7209cd84121422154228f9c2474287b3cd9c37412b5641038e435d corrected/replay-metadata.json +552d42db805474d2509a6a7972cdeeda811145d80a31c398d01453335599b363 corrected/scripts/validate-zed-package-graph.sh +b2bb33fce89d5b24565f53d193daf534cb2d047883cba496b832b9b82e772407 corrected/src/completion.rs +db796f56d7077e13fd70b8814b772be0a7c8656fa333f38a957b90e6a3c8e61d corrected/src/external_subcommands.rs +aca432247c8b999ea02acded9c2445c286d94dac6de486b4524db21086cc58f5 corrected/src/lib.rs +54ce57b60b76915a8b246a4e8e07c70fd50050c360c81ef3d283df39b7b51314 corrected/src/main.rs +fb4d0cb70a9048b332bdac6427dabfd5d1eefec5313297906db34b4cf554916d corrected/tests/external_gitops_dispatch.rs diff --git a/evidence/den-2725-final-export/evidence/den-2725-final-replay.patch b/evidence/den-2725-final-export/evidence/den-2725-final-replay.patch new file mode 100644 index 00000000..a18c5f69 --- /dev/null +++ b/evidence/den-2725-final-export/evidence/den-2725-final-replay.patch @@ -0,0 +1,893 @@ +diff --git a/.zpkg.toml b/.zpkg.toml +index 5f74350..66ae976 100644 +--- a/.zpkg.toml ++++ b/.zpkg.toml +@@ -12,11 +12,12 @@ vcs = "git" + url = "https://github.com/zed-pkg/zed-cli" + + [build] +-command = "cargo build --release --locked" +-outputs = ["target/release/zed"] ++command = "cargo build --release --locked --bins" ++outputs = ["target/release/zed", "target/release/zed-gitops"] + + [bin] + zed = "target/release/zed" ++"zed-gitops" = "target/release/zed-gitops" + + [install] + adapter = "none" +@@ -30,7 +31,7 @@ dir = ".vendor/.zed" + [publish] + include_readme = true + tag_format = "v{version}" +-smoke_test = "test -x \"$ZED_PKG_TEST_TARGET/target/release/zed\" && \"$ZED_PKG_TEST_TARGET/target/release/zed\" --version" ++smoke_test = "test -x \"$ZED_PKG_TEST_TARGET/target/release/zed\" && test -x \"$ZED_PKG_TEST_TARGET/target/release/zed-gitops\" && \"$ZED_PKG_TEST_TARGET/target/release/zed\" --version && \"$ZED_PKG_TEST_TARGET/target/release/zed\" gitops validate --help" + exclude = [ + ".env", + ".env.*", +diff --git a/docs/gitops-validator.md b/docs/gitops-validator.md +index cdfa53c..e3147e0 100644 +--- a/docs/gitops-validator.md ++++ b/docs/gitops-validator.md +@@ -1,24 +1,35 @@ +-# `zed-gitops` external validator ++# `zed gitops` external validator + +-`zed-gitops` is the first read-only implementation slice for DEN-2725. It +-validates the `GitOpsApplication` records owned by +-`ORESoftware/k8s-cluster` without initializing or executing child repositories. ++`zed gitops` is the read-only GitOps validation lane tracked by DEN-2725. The ++root `zed` binary now securely dispatches this command to the separately built ++`zed-gitops` executable, while root help and shell completions expose the same ++public command contract. + +-The executable is intentionally separate from the core package-manager command +-graph. Once generic external-command discovery lands, the public spelling is: ++Install or build both binaries into the same bin directory: + + ```console ++cargo install --path . --bins + zed gitops validate --root . --offline --strict ++zed gitops validate --root . --offline --strict --format json ++zed gitops validate --root . --offline --strict --format sarif + ``` + +-The executable available in this PR is directly invokable as: ++The standalone spelling remains supported for automation that deliberately ++pins the validator executable: + + ```console + zed-gitops validate --root . --offline --strict +-zed-gitops validate --root . --offline --strict --format json +-zed-gitops validate --root . --offline --strict --format sarif + ``` + ++The dispatcher resolves `zed-gitops` beside the running `zed` executable first, ++then searches only absolute `PATH` entries. It never invokes a shell, never ++searches the current working directory implicitly, never permits an extension ++to shadow a built-in command or alias, and preserves the child process exit ++code. Root options placed before or after `gitops` are passed as their canonical ++`ZED_PKG_*` environment variables rather than being exposed on the child ++command line. A literal `--` ends global-option extraction and passes every ++remaining argument to the external command unchanged. ++ + ## Evidence checked + + - catalog JSON is regular UTF-8 data beneath the selected repository root; +@@ -38,20 +49,19 @@ zed-gitops validate --root . --offline --strict --format sarif + - the retained static Application is a regular parent-owned file. + + The command does not read Kubernetes credentials, clone private repositories, +-resolve remote branch tips, or apply manifests. Online validation is not implemented +-yet, so invocations must pass `--offline`; omitting it fails explicitly instead of +-misreporting a local-only run as online evidence. Policy failures exit with code 2; +-tool/configuration failures exit with code 1. ++resolve remote branch tips, or apply manifests. Online validation is not ++implemented yet, so invocations must pass `--offline`; omitting it fails ++explicitly instead of misreporting a local-only run as online evidence. Policy ++failures exit with code 2; tool/configuration failures exit with code 1. + + ## Ownership boundary + +-This first slice keeps parsing local to the external executable so it can land +-without changing ordinary install/update behavior or contending with unrelated +-root-command branches. Follow-up work should expose the existing +-`git_submodules` repository-identity and index primitives as a stable library +-surface, then make core `zed` discover `zed-*` executables and route +-`zed gitops ...` with the same help, completion, TTY, and exit-code behavior. ++The root CLI owns extension discovery, built-in collision prevention, help, ++completion, TTY inheritance, and exit-code propagation. `zed-gitops` owns the ++current validation implementation. Follow-up work should expose the existing ++`git_submodules` repository-identity and index primitives as a stable ++`zed-pkg` library surface so the validator does not maintain parallel generic ++Git parsing. + +-The deployment-specific schema and policy remain versioned in +-`k8s-cluster`; Zed remains the validator UX rather than the deployment +-controller. ++The deployment-specific schema and policy remain versioned in `k8s-cluster`; ++Zed remains the validator UX rather than the deployment controller. +diff --git a/scripts/validate-zed-package-graph.sh b/scripts/validate-zed-package-graph.sh +index 1ae5b26..c3378a4 100755 +--- a/scripts/validate-zed-package-graph.sh ++++ b/scripts/validate-zed-package-graph.sh +@@ -16,6 +16,8 @@ for dependency in \ + done + + grep -Fq 'dir = ".vendor/.zed"' .zpkg.toml || { echo 'Zed install directory must be .vendor/.zed' >&2; exit 1; } ++grep -Fq 'outputs = ["target/release/zed", "target/release/zed-gitops"]' .zpkg.toml || { echo 'Zed package must publish both root and GitOps executables' >&2; exit 1; } ++grep -Fq '"zed-gitops" = "target/release/zed-gitops"' .zpkg.toml || { echo 'Zed package must install the sibling zed-gitops executable' >&2; exit 1; } + grep -Fq '".vendor/.zed/**"' .zpkg.toml || { echo 'publish exclusions must omit materialized Zed dependencies' >&2; exit 1; } + + if [[ -f .zpkg.lock ]] && [[ "$(wc -c < .zpkg.lock)" -le 12 ]]; then +diff --git a/src/completion.rs b/src/completion.rs +index 34525dd..2c68bb2 100644 +--- a/src/completion.rs ++++ b/src/completion.rs +@@ -6,12 +6,14 @@ use anyhow::{Context, Result}; + use clap_complete::{Shell, generate}; + + use crate::cli_model; +-use crate::{dev, fetch, git_submodules, global, nix_bundle_write, nix_export_plan}; ++use crate::{ ++ dev, external_subcommands, fetch, git_submodules, global, nix_bundle_write, nix_export_plan, ++}; + +-/// Build the complete public command tree shared by root help and completion +-/// generation. Every modular command must compose here rather than maintaining +-/// a second, partial root-help model. +-pub fn root_command() -> clap::Command { ++/// Build the complete built-in command tree without external extensions. ++/// The external dispatcher uses this model to guarantee that a `zed-*` ++/// executable can never shadow a built-in name or alias. ++pub(crate) fn built_in_root_command() -> clap::Command { + global::augment_root_command(git_submodules::augment_root_command( + nix_bundle_write::augment_root_command(nix_export_plan::augment_root_command( + fetch::augment_root_command(dev::augment_root_command(cli_model::command())), +@@ -19,6 +21,13 @@ pub fn root_command() -> clap::Command { + )) + } + ++/// Build the complete public command tree shared by root help and completion ++/// generation. Every modular or external command must compose here rather than ++/// maintaining a second, partial root-help model. ++pub fn root_command() -> clap::Command { ++ external_subcommands::augment_root_command(built_in_root_command()) ++} ++ + /// Print the complete top-level help tree. + pub fn print_root_help() -> Result<()> { + let mut command = root_command(); +@@ -76,6 +85,8 @@ mod tests { + "completions", + "self-update", + "r2g", ++ "gitops", ++ "validate", + ] { + assert!(script.contains(command), "missing command {command:?}"); + } +@@ -94,6 +105,8 @@ mod tests { + "--output", + "--python-venv", + "--isolated-home", ++ "--catalog", ++ "--offline", + ] { + assert!(script.contains(option), "missing option {option:?}"); + } +@@ -125,6 +138,8 @@ mod tests { + "completions", + "self-update", + "r2g", ++ "gitops", ++ "validate", + ] { + assert!(script.contains(command), "missing command {command:?}"); + } +@@ -143,6 +158,8 @@ mod tests { + "--output", + "--python-venv", + "--isolated-home", ++ "--catalog", ++ "--offline", + ] { + assert!(script.contains(option), "missing option {option:?}"); + } +diff --git a/src/external_subcommands.rs b/src/external_subcommands.rs +new file mode 100644 +index 0000000..55a2914 +--- /dev/null ++++ b/src/external_subcommands.rs +@@ -0,0 +1,592 @@ ++//! Safe dispatch for separately installed `zed-*` command extensions. ++//! ++//! Built-in commands always win. External commands are resolved beside the ++//! running `zed` executable first and then from absolute `PATH` entries. The ++//! dispatcher never invokes a shell or searches the current working directory ++//! implicitly. ++ ++use std::env; ++use std::ffi::{OsStr, OsString}; ++use std::fs; ++use std::path::{Path, PathBuf}; ++use std::process::{Command as ProcessCommand, ExitStatus}; ++ ++use anyhow::{Context, Result, anyhow}; ++use clap::{Arg, ArgAction, Command as ClapCommand}; ++ ++const EXTERNAL_PREFIX: &str = "zed-"; ++const KNOWN_EXTERNAL_COMMAND: &str = "gitops"; ++const EXTERNAL_COMMAND_ENV: &str = "ZED_EXTERNAL_SUBCOMMAND"; ++ ++const ROOT_VALUE_OPTIONS: &[(&str, &str)] = &[ ++ ("--registry", "ZED_PKG_REGISTRY"), ++ ("--home", "ZED_PKG_HOME"), ++ ("--token", "ZED_PKG_TOKEN"), ++ ("--auth-url", "ZED_PKG_AUTH_URL"), ++ ("--supabase-url", "ZED_PKG_SUPABASE_URL"), ++ ("--supabase-key", "ZED_PKG_SUPABASE_KEY"), ++ ("--global-bin-dir", "ZED_PKG_GLOBAL_BIN_DIR"), ++]; ++ ++const ROOT_BOOLEAN_OPTIONS: &[(&str, &str)] = &[ ++ ("--interactive", "ZED_PKG_INTERACTIVE"), ++ ("--git-submodules", "ZED_PKG_GIT_SUBMODULES"), ++]; ++ ++type ExternalEnvironment = Vec<(OsString, OsString)>; ++type ParsedExternalArguments = (Vec, ExternalEnvironment); ++ ++#[derive(Debug, Clone, PartialEq, Eq)] ++struct ExternalRoute { ++ name: String, ++ arguments: Vec, ++ environment: ExternalEnvironment, ++} ++ ++/// Dispatch a root command to `zed-` when it is not built in. ++/// ++/// Returning `None` leaves the established typed root parser in control. ++/// Returning `Some` means an external command was selected, including a ++/// fail-closed error when the documented `gitops` executable is absent. ++pub fn dispatch(args: Vec) -> Option> { ++ let route = external_route(&args)?; ++ if is_builtin_name(&route.name) { ++ return None; ++ } ++ ++ let executable = resolve_external(&route.name); ++ match executable { ++ Some(executable) => Some(run_external(&executable, &route)), ++ None if route.name == KNOWN_EXTERNAL_COMMAND => Some(Err(anyhow!( ++ "external subcommand `gitops` requires `zed-gitops` beside the `zed` executable or in an absolute PATH directory" ++ ))), ++ None => None, ++ } ++} ++ ++/// Add the supported external GitOps contract to root help and generated ++/// completions. Runtime execution still resolves the separately installed ++/// `zed-gitops` executable. ++pub fn augment_root_command(command: ClapCommand) -> ClapCommand { ++ if command.find_subcommand(KNOWN_EXTERNAL_COMMAND).is_some() { ++ return command; ++ } ++ ++ let validate = ClapCommand::new("validate") ++ .about("Validate a repository-owned exact-pin GitOps application catalog") ++ .arg( ++ Arg::new("root") ++ .long("root") ++ .value_name("PATH") ++ .default_value(".") ++ .help("Superproject root containing .gitmodules and the Git index"), ++ ) ++ .arg( ++ Arg::new("catalog") ++ .long("catalog") ++ .value_name("PATH") ++ .default_value("catalog/gitops/apps") ++ .help("Repository-relative GitOpsApplication catalog directory"), ++ ) ++ .arg( ++ Arg::new("format") ++ .long("format") ++ .value_name("FORMAT") ++ .value_parser(["human", "json", "sarif"]) ++ .default_value("human") ++ .help("Diagnostic output format"), ++ ) ++ .arg( ++ Arg::new("strict") ++ .long("strict") ++ .action(ArgAction::SetTrue) ++ .help("Reject unknown fields in catalog objects"), ++ ) ++ .arg( ++ Arg::new("offline") ++ .long("offline") ++ .action(ArgAction::SetTrue) ++ .help("Use only local evidence; required until online checks are implemented"), ++ ); ++ ++ command.subcommand( ++ ClapCommand::new(KNOWN_EXTERNAL_COMMAND) ++ .about("Validate GitOps composition through the external zed-gitops executable") ++ .after_help("Install `zed` and `zed-gitops` into the same bin directory, or place `zed-gitops` in an absolute PATH directory.") ++ .subcommand_required(true) ++ .subcommand(validate), ++ ) ++} ++ ++fn external_route(args: &[OsString]) -> Option { ++ let mut index = 1; ++ let mut environment = Vec::new(); ++ ++ while index < args.len() { ++ let token = args[index].to_str()?; ++ if matches!(token, "--" | "--help" | "-h" | "--version" | "-V") { ++ return None; ++ } ++ ++ if let Some((key, inline)) = root_value_option(token) { ++ let (value, consumed) = match inline { ++ Some(value) if !value.is_empty() => (OsString::from(value), 1), ++ Some(_) => return None, ++ None => { ++ let value = args.get(index + 1)?.clone(); ++ if value.is_empty() { ++ return None; ++ } ++ (value, 2) ++ } ++ }; ++ environment.push((OsString::from(key), value)); ++ index += consumed; ++ continue; ++ } ++ ++ if let Some((key, value)) = root_boolean_option(token) { ++ environment.push((OsString::from(key), OsString::from(value))); ++ index += 1; ++ continue; ++ } ++ ++ if token.starts_with('-') { ++ return None; ++ } ++ ++ if token == "help" { ++ let name = args.get(index + 1)?.to_str()?; ++ if !valid_external_name(name) { ++ return None; ++ } ++ let (mut arguments, trailing_environment) = extract_root_options(&args[index + 2..])?; ++ environment.extend(trailing_environment); ++ ensure_help_argument(&mut arguments); ++ return Some(ExternalRoute { ++ name: name.to_owned(), ++ arguments, ++ environment, ++ }); ++ } ++ ++ if !valid_external_name(token) { ++ return None; ++ } ++ let (arguments, trailing_environment) = extract_root_options(&args[index + 1..])?; ++ environment.extend(trailing_environment); ++ return Some(ExternalRoute { ++ name: token.to_owned(), ++ arguments, ++ environment, ++ }); ++ } ++ ++ None ++} ++ ++fn ensure_help_argument(arguments: &mut Vec) { ++ if arguments ++ .iter() ++ .any(|argument| argument == OsStr::new("--help") || argument == OsStr::new("-h")) ++ { ++ return; ++ } ++ let insertion = arguments ++ .iter() ++ .position(|argument| argument == OsStr::new("--")) ++ .unwrap_or(arguments.len()); ++ arguments.insert(insertion, OsString::from("--help")); ++} ++ ++fn extract_root_options(args: &[OsString]) -> Option { ++ let mut arguments = Vec::new(); ++ let mut environment = Vec::new(); ++ let mut index = 0; ++ ++ while index < args.len() { ++ let Some(token) = args[index].to_str() else { ++ arguments.push(args[index].clone()); ++ index += 1; ++ continue; ++ }; ++ if token == "--" { ++ arguments.extend_from_slice(&args[index..]); ++ break; ++ } ++ ++ if let Some((key, inline)) = root_value_option(token) { ++ let (value, consumed) = match inline { ++ Some(value) if !value.is_empty() => (OsString::from(value), 1), ++ Some(_) => return None, ++ None => { ++ let value = args.get(index + 1)?.clone(); ++ if value.is_empty() { ++ return None; ++ } ++ (value, 2) ++ } ++ }; ++ environment.push((OsString::from(key), value)); ++ index += consumed; ++ continue; ++ } ++ ++ if let Some((key, value)) = root_boolean_option(token) { ++ environment.push((OsString::from(key), OsString::from(value))); ++ index += 1; ++ continue; ++ } ++ if is_root_boolean_spelling(token) { ++ return None; ++ } ++ ++ arguments.push(args[index].clone()); ++ index += 1; ++ } ++ ++ Some((arguments, environment)) ++} ++ ++fn is_root_boolean_spelling(token: &str) -> bool { ++ ROOT_BOOLEAN_OPTIONS.iter().any(|(option, _)| { ++ token == *option ++ || token == format!("--no-{}", option.trim_start_matches('-')) ++ || token ++ .strip_prefix(option) ++ .is_some_and(|tail| tail.starts_with('=')) ++ }) ++} ++ ++fn root_value_option(token: &str) -> Option<(&'static str, Option<&str>)> { ++ ROOT_VALUE_OPTIONS.iter().find_map(|(option, key)| { ++ if token == *option { ++ Some((*key, None)) ++ } else { ++ token ++ .strip_prefix(option) ++ .and_then(|tail| tail.strip_prefix('=')) ++ .map(|value| (*key, Some(value))) ++ } ++ }) ++} ++ ++fn root_boolean_option(token: &str) -> Option<(&'static str, &'static str)> { ++ ROOT_BOOLEAN_OPTIONS.iter().find_map(|(option, key)| { ++ if token == *option { ++ return Some((*key, "true")); ++ } ++ let negative = format!("--no-{}", option.trim_start_matches('-')); ++ if token == negative { ++ return Some((*key, "false")); ++ } ++ token ++ .strip_prefix(option) ++ .and_then(|tail| tail.strip_prefix('=')) ++ .and_then(normalize_boolean) ++ .map(|value| (*key, value)) ++ }) ++} ++ ++fn normalize_boolean(value: &str) -> Option<&'static str> { ++ match value.trim().to_ascii_lowercase().as_str() { ++ "true" | "1" | "yes" | "on" => Some("true"), ++ "false" | "0" | "no" | "off" => Some("false"), ++ _ => None, ++ } ++} ++ ++fn valid_external_name(name: &str) -> bool { ++ let mut characters = name.chars(); ++ let Some(first) = characters.next() else { ++ return false; ++ }; ++ name.len() <= 64 ++ && first.is_ascii_alphanumeric() ++ && characters ++ .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_')) ++} ++ ++fn is_builtin_name(name: &str) -> bool { ++ crate::completion::built_in_root_command() ++ .find_subcommand(name) ++ .is_some() ++} ++ ++fn resolve_external(name: &str) -> Option { ++ let sibling = env::current_exe() ++ .ok() ++ .and_then(|executable| executable.parent().map(Path::to_path_buf)); ++ let path = env::var_os("PATH"); ++ resolve_in_locations(name, sibling.as_deref(), path.as_deref()) ++} ++ ++fn resolve_in_locations( ++ name: &str, ++ sibling: Option<&Path>, ++ path: Option<&OsStr>, ++) -> Option { ++ let stem = format!("{EXTERNAL_PREFIX}{name}"); ++ ++ if let Some(executable) = sibling ++ .filter(|directory| directory.is_absolute()) ++ .and_then(|directory| executable_in(directory, &stem)) ++ { ++ return Some(executable); ++ } ++ ++ let path = path?; ++ for directory in env::split_paths(path) { ++ if !directory.is_absolute() { ++ continue; ++ } ++ if let Some(executable) = executable_in(&directory, &stem) { ++ return Some(executable); ++ } ++ } ++ None ++} ++ ++fn executable_in(directory: &Path, stem: &str) -> Option { ++ executable_candidates(directory, stem) ++ .into_iter() ++ .find(|candidate| is_executable_file(candidate)) ++} ++ ++#[cfg(windows)] ++fn executable_candidates(directory: &Path, stem: &str) -> Vec { ++ vec![directory.join(format!("{stem}.exe")), directory.join(stem)] ++} ++ ++#[cfg(not(windows))] ++fn executable_candidates(directory: &Path, stem: &str) -> Vec { ++ vec![directory.join(stem)] ++} ++ ++fn is_executable_file(path: &Path) -> bool { ++ let Ok(metadata) = fs::metadata(path) else { ++ return false; ++ }; ++ if !metadata.is_file() { ++ return false; ++ } ++ ++ #[cfg(unix)] ++ { ++ use std::os::unix::fs::PermissionsExt; ++ metadata.permissions().mode() & 0o111 != 0 ++ } ++ #[cfg(not(unix))] ++ { ++ true ++ } ++} ++ ++fn run_external(executable: &Path, route: &ExternalRoute) -> Result { ++ let mut command = ProcessCommand::new(executable); ++ command.args(&route.arguments); ++ for (key, value) in &route.environment { ++ command.env(key, value); ++ } ++ command.env(EXTERNAL_COMMAND_ENV, &route.name); ++ ++ let status = command.status().with_context(|| { ++ format!( ++ "running external subcommand `{}` through {}", ++ route.name, ++ executable.display() ++ ) ++ })?; ++ Ok(status_code(status)) ++} ++ ++fn status_code(status: ExitStatus) -> i32 { ++ if let Some(code) = status.code() { ++ return code; ++ } ++ ++ #[cfg(unix)] ++ { ++ use std::os::unix::process::ExitStatusExt; ++ status.signal().map(|signal| 128 + signal).unwrap_or(1) ++ } ++ #[cfg(not(unix))] ++ { ++ 1 ++ } ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ ++ fn os_args(values: &[&str]) -> Vec { ++ values.iter().map(OsString::from).collect() ++ } ++ ++ #[test] ++ fn routes_plugin_arguments_without_shell_rewriting() { ++ let route = external_route(&os_args(&[ ++ "zed", ++ "gitops", ++ "validate", ++ "--root", ++ "workspace with spaces", ++ "--offline", ++ ])) ++ .expect("external route"); ++ assert_eq!(route.name, "gitops"); ++ assert_eq!( ++ route.arguments, ++ os_args(&["validate", "--root", "workspace with spaces", "--offline"]) ++ ); ++ } ++ ++ #[test] ++ fn root_options_before_plugin_become_child_environment() { ++ let route = external_route(&os_args(&[ ++ "zed", ++ "--home", ++ "/tmp/zed-home", ++ "--git-submodules=false", ++ "gitops", ++ "validate", ++ ])) ++ .expect("external route"); ++ assert_eq!( ++ route.environment, ++ vec![ ++ ( ++ OsString::from("ZED_PKG_HOME"), ++ OsString::from("/tmp/zed-home") ++ ), ++ ( ++ OsString::from("ZED_PKG_GIT_SUBMODULES"), ++ OsString::from("false") ++ ), ++ ] ++ ); ++ } ++ ++ #[test] ++ fn root_options_after_plugin_are_lifted_until_double_dash() { ++ let route = external_route(&os_args(&[ ++ "zed", ++ "gitops", ++ "validate", ++ "--token", ++ "fixture-value", ++ "--offline", ++ "--", ++ "--home", ++ "child-owned-value", ++ ])) ++ .expect("external route"); ++ assert_eq!( ++ route.arguments, ++ os_args(&["validate", "--offline", "--", "--home", "child-owned-value"]) ++ ); ++ assert_eq!( ++ route.environment, ++ vec![( ++ OsString::from("ZED_PKG_TOKEN"), ++ OsString::from("fixture-value") ++ )] ++ ); ++ } ++ ++ #[test] ++ fn malformed_trailing_root_boolean_fails_closed() { ++ assert!( ++ external_route(&os_args(&[ ++ "zed", ++ "gitops", ++ "validate", ++ "--git-submodules=maybe" ++ ])) ++ .is_none() ++ ); ++ } ++ ++ #[test] ++ fn help_spelling_routes_to_external_help() { ++ let route = external_route(&os_args(&["zed", "help", "gitops", "validate"])) ++ .expect("external help route"); ++ assert_eq!(route.name, "gitops"); ++ assert_eq!(route.arguments, os_args(&["validate", "--help"])); ++ } ++ ++ #[test] ++ fn external_help_is_inserted_before_literal_double_dash() { ++ let route = external_route(&os_args(&[ ++ "zed", ++ "help", ++ "gitops", ++ "validate", ++ "--", ++ "--root", ++ "child-owned", ++ ])) ++ .expect("external help route"); ++ assert_eq!( ++ route.arguments, ++ os_args(&["validate", "--help", "--", "--root", "child-owned"]) ++ ); ++ } ++ ++ #[test] ++ fn builtins_and_unsafe_names_are_never_external() { ++ assert!(is_builtin_name("install")); ++ assert!(is_builtin_name("dev")); ++ assert!(!is_builtin_name("gitops")); ++ for invalid in ["../gitops", ".gitops", "git/ops", "-gitops", "gitops.exe"] { ++ assert!(!valid_external_name(invalid), "accepted {invalid:?}"); ++ } ++ } ++ ++ #[test] ++ fn root_help_model_contains_the_known_external_contract() { ++ let command = augment_root_command(crate::completion::built_in_root_command()); ++ let gitops = command.find_subcommand("gitops").expect("gitops command"); ++ assert!(gitops.find_subcommand("validate").is_some()); ++ } ++ ++ #[test] ++ fn relative_path_entries_are_not_searched() { ++ assert!(resolve_in_locations("demo", None, Some(OsStr::new("relative/bin"))).is_none()); ++ } ++ ++ #[cfg(unix)] ++ fn write_executable(path: &Path, exit_code: i32) { ++ use std::os::unix::fs::PermissionsExt; ++ ++ fs::write(path, format!("#!/bin/sh\nexit {exit_code}\n")).expect("write executable"); ++ let mut permissions = fs::metadata(path).expect("metadata").permissions(); ++ permissions.set_mode(0o755); ++ fs::set_permissions(path, permissions).expect("set executable permissions"); ++ } ++ ++ #[cfg(unix)] ++ #[test] ++ fn sibling_precedes_absolute_path_and_exit_code_is_preserved() { ++ let sibling = tempfile::tempdir().expect("sibling directory"); ++ let path_directory = tempfile::tempdir().expect("PATH directory"); ++ let sibling_executable = sibling.path().join("zed-demo"); ++ let path_executable = path_directory.path().join("zed-demo"); ++ write_executable(&sibling_executable, 23); ++ write_executable(&path_executable, 41); ++ let joined_path = env::join_paths([path_directory.path()]).expect("join PATH"); ++ ++ let resolved = ++ resolve_in_locations("demo", Some(sibling.path()), Some(joined_path.as_os_str())) ++ .expect("resolved executable"); ++ assert_eq!(resolved, sibling_executable); ++ ++ let route = ExternalRoute { ++ name: "demo".to_owned(), ++ arguments: Vec::new(), ++ environment: Vec::new(), ++ }; ++ assert_eq!(run_external(&resolved, &route).expect("run external"), 23); ++ } ++} +diff --git a/src/lib.rs b/src/lib.rs +index e44b2ac..5f39c53 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -10,6 +10,7 @@ pub mod config; + mod dart_wiring; + pub mod dev; + pub mod environment; ++pub mod external_subcommands; + pub mod fetch; + pub mod flags; + pub mod git_submodules; +diff --git a/src/main.rs b/src/main.rs +index ae0dbe5..67f2514 100644 +--- a/src/main.rs ++++ b/src/main.rs +@@ -92,7 +92,17 @@ fn main() { + } + } + } +- if let Some(result) = dev::dispatch(args) { ++ if let Some(result) = dev::dispatch(args.clone()) { ++ match result { ++ Ok(0) => return, ++ Ok(code) => std::process::exit(code), ++ Err(error) => { ++ eprintln!("error: {error:#}"); ++ std::process::exit(1); ++ } ++ } ++ } ++ if let Some(result) = zed_cli::external_subcommands::dispatch(args) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), +diff --git a/tests/external_gitops_dispatch.rs b/tests/external_gitops_dispatch.rs +new file mode 100644 +index 0000000..5a3c57b +--- /dev/null ++++ b/tests/external_gitops_dispatch.rs +@@ -0,0 +1,57 @@ ++use std::process::{Command, Output}; ++ ++fn text(output: &Output) -> String { ++ format!( ++ "{}{}", ++ String::from_utf8_lossy(&output.stdout), ++ String::from_utf8_lossy(&output.stderr) ++ ) ++} ++ ++#[test] ++fn root_help_advertises_gitops_validate() { ++ let output = Command::new(env!("CARGO_BIN_EXE_zed")) ++ .arg("--help") ++ .output() ++ .expect("run zed help"); ++ assert!(output.status.success(), "{}", text(&output)); ++ let text = text(&output); ++ assert!(text.contains("gitops"), "{text}"); ++ assert!(text.contains("Validate GitOps composition"), "{text}"); ++} ++ ++#[test] ++fn root_dispatches_to_the_sibling_gitops_binary() { ++ let output = Command::new(env!("CARGO_BIN_EXE_zed")) ++ .args(["gitops", "validate", "--help"]) ++ .output() ++ .expect("run zed gitops help"); ++ assert!(output.status.success(), "{}", text(&output)); ++ let text = text(&output); ++ assert!(text.contains("Usage: zed-gitops"), "{text}"); ++ assert!(text.contains("validate [OPTIONS]"), "{text}"); ++ assert!(text.contains("--offline"), "{text}"); ++} ++ ++#[test] ++fn root_help_alias_keeps_help_before_literal_double_dash() { ++ let output = Command::new(env!("CARGO_BIN_EXE_zed")) ++ .args(["help", "gitops", "validate", "--", "--root", "child-owned"]) ++ .output() ++ .expect("run zed help gitops with literal delimiter"); ++ assert!(output.status.success(), "{}", text(&output)); ++ let text = text(&output); ++ assert!(text.contains("Usage: zed-gitops"), "{text}"); ++ assert!(text.contains("validate [OPTIONS]"), "{text}"); ++ assert!(text.contains("--offline"), "{text}"); ++} ++ ++#[test] ++fn root_help_alias_reaches_the_external_binary() { ++ let output = Command::new(env!("CARGO_BIN_EXE_zed")) ++ .args(["help", "gitops"]) ++ .output() ++ .expect("run zed help gitops"); ++ assert!(output.status.success(), "{}", text(&output)); ++ assert!(text(&output).contains("Usage: zed-gitops")); ++} diff --git a/evidence/den-2725-final-export/evidence/gitops-dispatch-Linux-X64.json b/evidence/den-2725-final-export/evidence/gitops-dispatch-Linux-X64.json new file mode 100644 index 00000000..ffacec72 --- /dev/null +++ b/evidence/den-2725-final-export/evidence/gitops-dispatch-Linux-X64.json @@ -0,0 +1,33 @@ +{ + "$schema": "zed-pkg-test/gitops-dispatch-canary/v1", + "binaries": { + "probeSha256": "701c875d5ed69cd1042f4726e54eacc1c96fd7f1133ed75680dae150d08ab658", + "zedGitopsSha256": "71cf25ac61f3012c017721f457fdb2a61e171413aad27d1f29acf2e0fdbb456e", + "zedSha256": "8b08ffddfa05c4ec4d391d94531f9f9bf16abf84d0af4ebe4b55f8fde6acd677" + }, + "candidate": "1c94813c6b59c7621c1256d7c55246e7f06f44b7", + "checkCount": 15, + "checks": [ + "package-installs-both-sibling-binaries", + "standalone-validator-help", + "root-help-advertises-gitops", + "root-dispatches-to-sibling-validator", + "help-alias-reaches-validator", + "completion-exposes-gitops-contract", + "root-and-standalone-validation-parity", + "online-mode-fails-explicitly", + "policy-failure-exit-code-and-diagnostic", + "missing-known-extension-fails-closed", + "working-directory-is-never-searched", + "relative-path-entry-is-rejected", + "sibling-validator-precedes-path", + "arguments-environment-and-exit-code-are-preserved", + "built-in-command-cannot-be-shadowed" + ], + "platform": { + "machine": "x86_64", + "python": "3.12.13", + "system": "Linux" + }, + "result": "passed" +} diff --git a/evidence/den-2725-final-export/evidence/gitops-dispatch-trailing-Linux-X64.json b/evidence/den-2725-final-export/evidence/gitops-dispatch-trailing-Linux-X64.json new file mode 100644 index 00000000..f98e00c1 --- /dev/null +++ b/evidence/den-2725-final-export/evidence/gitops-dispatch-trailing-Linux-X64.json @@ -0,0 +1,11 @@ +{ + "$schema": "zed-pkg-test/gitops-dispatch-trailing-options/v1", + "candidate": "1c94813c6b59c7621c1256d7c55246e7f06f44b7", + "checkCount": 3, + "checks": [ + "recognized-trailing-root-options-are-lifted", + "literal-double-dash-terminates-root-option-extraction", + "malformed-trailing-root-boolean-fails-closed" + ], + "result": "passed" +} diff --git a/evidence/den-2725-final-export/evidence/replay-metadata.json b/evidence/den-2725-final-export/evidence/replay-metadata.json new file mode 100644 index 00000000..23cbe617 --- /dev/null +++ b/evidence/den-2725-final-export/evidence/replay-metadata.json @@ -0,0 +1,6 @@ +{ + "$schema": "zed-pkg-test/den-2725-final-replay/v1", + "mainCommit": "0246a3af901b65ca6724d936c7f3232c9f72fcd7", + "rootDispatchCommit": "1c94813c6b59c7621c1256d7c55246e7f06f44b7", + "treeSha": "99ab08e3142c2a52d2c97482134e99fae3811b43" +} diff --git a/evidence/den-2725-final-export/scripts/validate-zed-package-graph.sh b/evidence/den-2725-final-export/scripts/validate-zed-package-graph.sh new file mode 100755 index 00000000..c3378a42 --- /dev/null +++ b/evidence/den-2725-final-export/scripts/validate-zed-package-graph.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" + +interfaces_manifest="${1:-}" +clients_manifest="${2:-}" +lock_manifest="${3:-}" + +[[ -f .zpkg.toml ]] || { echo 'missing .zpkg.toml' >&2; exit 1; } +for dependency in \ + '"zed-pkg/zed-clients" = "^0.1.0"' \ + '"zed-pkg/zed-interfaces" = "^0.1.0"' \ + '"zed-pkg/zed-lock" = "^0.1.1"'; do + grep -Fq "$dependency" .zpkg.toml || { printf 'missing canonical Zed dependency: %s\n' "$dependency" >&2; exit 1; } +done + +grep -Fq 'dir = ".vendor/.zed"' .zpkg.toml || { echo 'Zed install directory must be .vendor/.zed' >&2; exit 1; } +grep -Fq 'outputs = ["target/release/zed", "target/release/zed-gitops"]' .zpkg.toml || { echo 'Zed package must publish both root and GitOps executables' >&2; exit 1; } +grep -Fq '"zed-gitops" = "target/release/zed-gitops"' .zpkg.toml || { echo 'Zed package must install the sibling zed-gitops executable' >&2; exit 1; } +grep -Fq '".vendor/.zed/**"' .zpkg.toml || { echo 'publish exclusions must omit materialized Zed dependencies' >&2; exit 1; } + +if [[ -f .zpkg.lock ]] && [[ "$(wc -c < .zpkg.lock)" -le 12 ]]; then + echo '.zpkg.lock is an empty placeholder; regenerate it with the resolver or remove it' >&2 + exit 1 +fi + +if [[ -d crates/zed-lock ]]; then + echo 'the extracted lock crate must remain independently owned; do not restore crates/zed-lock' >&2 + exit 1 +fi + +if grep -Fq '"zed-pkg/zed-lib"' .zpkg.toml || grep -Fq '"zed-pkg/zed-libs"' .zpkg.toml; then + echo 'do not reference a canonical lib coordinate until that repository and package exist' >&2 + exit 1 +fi + +python3 - "$interfaces_manifest" "$clients_manifest" "$lock_manifest" <<'PY' +from __future__ import annotations + +import pathlib +import sys +import tomllib + +root = pathlib.Path.cwd() +manifest = tomllib.loads((root / ".zpkg.toml").read_text(encoding="utf-8")) +cargo = tomllib.loads((root / "Cargo.toml").read_text(encoding="utf-8")) +cargo_lock = (root / "Cargo.lock").read_text(encoding="utf-8") +errors: list[str] = [] +expected_lock_revision = "a0dc78d385bc3ab553d3027b427f5f1428239c9c" +expected_lock_source = ( + "git+https://github.com/zed-pkg/zed-lock.git?" + f"rev={expected_lock_revision}#{expected_lock_revision}" +) + +repository = manifest.get("package", {}).get("repository", {}) +if repository.get("url") != "https://github.com/zed-pkg/zed-cli": + errors.append("package.repository.url must point at zed-pkg/zed-cli") + +cargo_dependencies = cargo.get("dependencies", {}) +native_interfaces = cargo_dependencies.get("zed-interfaces") +if native_interfaces is None: + errors.append("Cargo.toml must retain the native zed-interfaces dependency") + +native_lock = cargo_dependencies.get("zed-lock") +if not isinstance(native_lock, dict): + errors.append("Cargo.toml must use the independently owned zed-lock Git dependency") +else: + if native_lock.get("git") != "https://github.com/zed-pkg/zed-lock.git": + errors.append("zed-lock Cargo dependency must use the canonical repository") + if native_lock.get("rev") != expected_lock_revision: + errors.append("zed-lock Cargo dependency must pin the hardened v0.1.1 merge commit") + +if 'name = "zed-lock"\nversion = "0.1.1"' not in cargo_lock: + errors.append("Cargo.lock must resolve zed-lock version 0.1.1") +if expected_lock_source not in cargo_lock: + errors.append("Cargo.lock must resolve the exact hardened zed-lock merge commit") + +for name in manifest.get("dependencies", {}): + package = name.lower().split("/", 1)[-1] + if package.endswith("-infra"): + errors.append(f"CLI must not import infrastructure package: {name}") + +interfaces_path = pathlib.Path(sys.argv[1]) if sys.argv[1] else None +clients_path = pathlib.Path(sys.argv[2]) if sys.argv[2] else None +lock_path = pathlib.Path(sys.argv[3]) if sys.argv[3] else None + +if interfaces_path: + interfaces = tomllib.loads(interfaces_path.read_text(encoding="utf-8")) + if interfaces.get("package", {}).get("name") != "zed-interfaces": + errors.append("sibling interfaces manifest does not provide zed-interfaces") + +if clients_path: + clients = tomllib.loads(clients_path.read_text(encoding="utf-8")) + if clients.get("package", {}).get("name") != "zed-clients": + errors.append("sibling clients manifest does not provide zed-clients") + client_dependencies = clients.get("dependencies", {}) + if "zed-pkg/zed-interfaces" not in client_dependencies: + errors.append("zed-clients must itself depend on zed-interfaces") + if "rust" not in clients.get("targets", {}): + errors.append("zed-clients must retain its Rust SDK target") + +if lock_path: + lock_package = tomllib.loads(lock_path.read_text(encoding="utf-8")) + package = lock_package.get("package", {}) + if package.get("org") != "zed-pkg" or package.get("name") != "zed-lock": + errors.append("sibling lock manifest must provide zed-pkg/zed-lock") + if package.get("version") != "0.1.1": + errors.append("sibling lock package must be hardened version 0.1.1") + if package.get("repository", {}).get("url") != "https://github.com/zed-pkg/zed-lock": + errors.append("sibling lock package must declare the canonical repository") + rust_target = lock_package.get("targets", {}).get("rust", {}) + if rust_target.get("dir") != ".": + errors.append("zed-lock must publish its root Rust target") + native = rust_target.get("native", {}) + if native.get("registry") != "crates-io" or native.get("package") != "zed-lock": + errors.append("zed-lock must retain its crates-io native publication route") + sibling_placeholder = lock_path.parent / ".zpkg.lock" + if sibling_placeholder.exists(): + errors.append("dependency-free zed-lock must not restore a placeholder .zpkg.lock") + +if errors: + for error in errors: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(1) +PY + +printf 'zed-cli package graph validated with canonical zed-lock v0.1.1 ownership\n' diff --git a/evidence/den-2725-final-export/src/completion.rs b/evidence/den-2725-final-export/src/completion.rs new file mode 100644 index 00000000..2c68bb24 --- /dev/null +++ b/evidence/den-2725-final-export/src/completion.rs @@ -0,0 +1,167 @@ +//! Shell-completion generation from the same clap command model used at runtime. + +use std::io; + +use anyhow::{Context, Result}; +use clap_complete::{Shell, generate}; + +use crate::cli_model; +use crate::{ + dev, external_subcommands, fetch, git_submodules, global, nix_bundle_write, nix_export_plan, +}; + +/// Build the complete built-in command tree without external extensions. +/// The external dispatcher uses this model to guarantee that a `zed-*` +/// executable can never shadow a built-in name or alias. +pub(crate) fn built_in_root_command() -> clap::Command { + global::augment_root_command(git_submodules::augment_root_command( + nix_bundle_write::augment_root_command(nix_export_plan::augment_root_command( + fetch::augment_root_command(dev::augment_root_command(cli_model::command())), + )), + )) +} + +/// Build the complete public command tree shared by root help and completion +/// generation. Every modular or external command must compose here rather than +/// maintaining a second, partial root-help model. +pub fn root_command() -> clap::Command { + external_subcommands::augment_root_command(built_in_root_command()) +} + +/// Print the complete top-level help tree. +pub fn print_root_help() -> Result<()> { + let mut command = root_command(); + command.print_help().context("printing zed help")?; + println!(); + Ok(()) +} + +/// Write a static completion script for `zed` to stdout. +pub fn print(shell: Shell) { + let mut command = root_command(); + generate(shell, &mut command, "zed", &mut io::stdout()); +} + +#[cfg(test)] +fn render(shell: Shell) -> String { + let mut command = root_command(); + let mut output = Vec::new(); + generate(shell, &mut command, "zed", &mut output); + String::from_utf8(output).expect("completion output must be UTF-8") +} + +#[cfg(test)] +mod tests { + use clap_complete::Shell; + + use super::render; + + #[test] + fn bash_completion_contains_commands_aliases_and_manifest_flags() { + let script = render(Shell::Bash); + assert!( + script.contains("_zed"), + "missing generated completion function" + ); + assert!( + script.contains("complete"), + "missing Bash completion registration" + ); + for command in [ + "install", + "init", + "fetch", + "interop", + "nix", + "plan", + "export", + "bundle", + "write", + "develop", + "dev", + "overtake", + "global", + "bin-dir", + "completions", + "self-update", + "r2g", + "gitops", + "validate", + ] { + assert!(script.contains(command), "missing command {command:?}"); + } + for option in [ + "--do-not-write-new-manifest", + "--allow-no-manifest", + "--skip-manifest", + "--git-submodules", + "--install-mode", + "--global-bin-dir", + "--frozen", + "--json", + "--target", + "--flake-lock", + "--out", + "--output", + "--python-venv", + "--isolated-home", + "--catalog", + "--offline", + ] { + assert!(script.contains(option), "missing option {option:?}"); + } + } + + #[test] + fn zsh_completion_contains_registration_commands_and_manifest_flags() { + let script = render(Shell::Zsh); + assert!( + script.contains("#compdef zed"), + "missing zsh compdef header" + ); + assert!(script.contains("_zed"), "missing zsh completion function"); + for command in [ + "install", + "init", + "fetch", + "interop", + "nix", + "plan", + "export", + "bundle", + "write", + "develop", + "dev", + "overtake", + "global", + "bin-dir", + "completions", + "self-update", + "r2g", + "gitops", + "validate", + ] { + assert!(script.contains(command), "missing command {command:?}"); + } + for option in [ + "--do-not-write-new-manifest", + "--allow-no-manifest", + "--skip-manifest", + "--git-submodules", + "--install-mode", + "--global-bin-dir", + "--frozen", + "--json", + "--target", + "--flake-lock", + "--out", + "--output", + "--python-venv", + "--isolated-home", + "--catalog", + "--offline", + ] { + assert!(script.contains(option), "missing option {option:?}"); + } + } +} diff --git a/evidence/den-2725-final-export/src/external_subcommands.rs b/evidence/den-2725-final-export/src/external_subcommands.rs new file mode 100644 index 00000000..55a29149 --- /dev/null +++ b/evidence/den-2725-final-export/src/external_subcommands.rs @@ -0,0 +1,592 @@ +//! Safe dispatch for separately installed `zed-*` command extensions. +//! +//! Built-in commands always win. External commands are resolved beside the +//! running `zed` executable first and then from absolute `PATH` entries. The +//! dispatcher never invokes a shell or searches the current working directory +//! implicitly. + +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command as ProcessCommand, ExitStatus}; + +use anyhow::{Context, Result, anyhow}; +use clap::{Arg, ArgAction, Command as ClapCommand}; + +const EXTERNAL_PREFIX: &str = "zed-"; +const KNOWN_EXTERNAL_COMMAND: &str = "gitops"; +const EXTERNAL_COMMAND_ENV: &str = "ZED_EXTERNAL_SUBCOMMAND"; + +const ROOT_VALUE_OPTIONS: &[(&str, &str)] = &[ + ("--registry", "ZED_PKG_REGISTRY"), + ("--home", "ZED_PKG_HOME"), + ("--token", "ZED_PKG_TOKEN"), + ("--auth-url", "ZED_PKG_AUTH_URL"), + ("--supabase-url", "ZED_PKG_SUPABASE_URL"), + ("--supabase-key", "ZED_PKG_SUPABASE_KEY"), + ("--global-bin-dir", "ZED_PKG_GLOBAL_BIN_DIR"), +]; + +const ROOT_BOOLEAN_OPTIONS: &[(&str, &str)] = &[ + ("--interactive", "ZED_PKG_INTERACTIVE"), + ("--git-submodules", "ZED_PKG_GIT_SUBMODULES"), +]; + +type ExternalEnvironment = Vec<(OsString, OsString)>; +type ParsedExternalArguments = (Vec, ExternalEnvironment); + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ExternalRoute { + name: String, + arguments: Vec, + environment: ExternalEnvironment, +} + +/// Dispatch a root command to `zed-` when it is not built in. +/// +/// Returning `None` leaves the established typed root parser in control. +/// Returning `Some` means an external command was selected, including a +/// fail-closed error when the documented `gitops` executable is absent. +pub fn dispatch(args: Vec) -> Option> { + let route = external_route(&args)?; + if is_builtin_name(&route.name) { + return None; + } + + let executable = resolve_external(&route.name); + match executable { + Some(executable) => Some(run_external(&executable, &route)), + None if route.name == KNOWN_EXTERNAL_COMMAND => Some(Err(anyhow!( + "external subcommand `gitops` requires `zed-gitops` beside the `zed` executable or in an absolute PATH directory" + ))), + None => None, + } +} + +/// Add the supported external GitOps contract to root help and generated +/// completions. Runtime execution still resolves the separately installed +/// `zed-gitops` executable. +pub fn augment_root_command(command: ClapCommand) -> ClapCommand { + if command.find_subcommand(KNOWN_EXTERNAL_COMMAND).is_some() { + return command; + } + + let validate = ClapCommand::new("validate") + .about("Validate a repository-owned exact-pin GitOps application catalog") + .arg( + Arg::new("root") + .long("root") + .value_name("PATH") + .default_value(".") + .help("Superproject root containing .gitmodules and the Git index"), + ) + .arg( + Arg::new("catalog") + .long("catalog") + .value_name("PATH") + .default_value("catalog/gitops/apps") + .help("Repository-relative GitOpsApplication catalog directory"), + ) + .arg( + Arg::new("format") + .long("format") + .value_name("FORMAT") + .value_parser(["human", "json", "sarif"]) + .default_value("human") + .help("Diagnostic output format"), + ) + .arg( + Arg::new("strict") + .long("strict") + .action(ArgAction::SetTrue) + .help("Reject unknown fields in catalog objects"), + ) + .arg( + Arg::new("offline") + .long("offline") + .action(ArgAction::SetTrue) + .help("Use only local evidence; required until online checks are implemented"), + ); + + command.subcommand( + ClapCommand::new(KNOWN_EXTERNAL_COMMAND) + .about("Validate GitOps composition through the external zed-gitops executable") + .after_help("Install `zed` and `zed-gitops` into the same bin directory, or place `zed-gitops` in an absolute PATH directory.") + .subcommand_required(true) + .subcommand(validate), + ) +} + +fn external_route(args: &[OsString]) -> Option { + let mut index = 1; + let mut environment = Vec::new(); + + while index < args.len() { + let token = args[index].to_str()?; + if matches!(token, "--" | "--help" | "-h" | "--version" | "-V") { + return None; + } + + if let Some((key, inline)) = root_value_option(token) { + let (value, consumed) = match inline { + Some(value) if !value.is_empty() => (OsString::from(value), 1), + Some(_) => return None, + None => { + let value = args.get(index + 1)?.clone(); + if value.is_empty() { + return None; + } + (value, 2) + } + }; + environment.push((OsString::from(key), value)); + index += consumed; + continue; + } + + if let Some((key, value)) = root_boolean_option(token) { + environment.push((OsString::from(key), OsString::from(value))); + index += 1; + continue; + } + + if token.starts_with('-') { + return None; + } + + if token == "help" { + let name = args.get(index + 1)?.to_str()?; + if !valid_external_name(name) { + return None; + } + let (mut arguments, trailing_environment) = extract_root_options(&args[index + 2..])?; + environment.extend(trailing_environment); + ensure_help_argument(&mut arguments); + return Some(ExternalRoute { + name: name.to_owned(), + arguments, + environment, + }); + } + + if !valid_external_name(token) { + return None; + } + let (arguments, trailing_environment) = extract_root_options(&args[index + 1..])?; + environment.extend(trailing_environment); + return Some(ExternalRoute { + name: token.to_owned(), + arguments, + environment, + }); + } + + None +} + +fn ensure_help_argument(arguments: &mut Vec) { + if arguments + .iter() + .any(|argument| argument == OsStr::new("--help") || argument == OsStr::new("-h")) + { + return; + } + let insertion = arguments + .iter() + .position(|argument| argument == OsStr::new("--")) + .unwrap_or(arguments.len()); + arguments.insert(insertion, OsString::from("--help")); +} + +fn extract_root_options(args: &[OsString]) -> Option { + let mut arguments = Vec::new(); + let mut environment = Vec::new(); + let mut index = 0; + + while index < args.len() { + let Some(token) = args[index].to_str() else { + arguments.push(args[index].clone()); + index += 1; + continue; + }; + if token == "--" { + arguments.extend_from_slice(&args[index..]); + break; + } + + if let Some((key, inline)) = root_value_option(token) { + let (value, consumed) = match inline { + Some(value) if !value.is_empty() => (OsString::from(value), 1), + Some(_) => return None, + None => { + let value = args.get(index + 1)?.clone(); + if value.is_empty() { + return None; + } + (value, 2) + } + }; + environment.push((OsString::from(key), value)); + index += consumed; + continue; + } + + if let Some((key, value)) = root_boolean_option(token) { + environment.push((OsString::from(key), OsString::from(value))); + index += 1; + continue; + } + if is_root_boolean_spelling(token) { + return None; + } + + arguments.push(args[index].clone()); + index += 1; + } + + Some((arguments, environment)) +} + +fn is_root_boolean_spelling(token: &str) -> bool { + ROOT_BOOLEAN_OPTIONS.iter().any(|(option, _)| { + token == *option + || token == format!("--no-{}", option.trim_start_matches('-')) + || token + .strip_prefix(option) + .is_some_and(|tail| tail.starts_with('=')) + }) +} + +fn root_value_option(token: &str) -> Option<(&'static str, Option<&str>)> { + ROOT_VALUE_OPTIONS.iter().find_map(|(option, key)| { + if token == *option { + Some((*key, None)) + } else { + token + .strip_prefix(option) + .and_then(|tail| tail.strip_prefix('=')) + .map(|value| (*key, Some(value))) + } + }) +} + +fn root_boolean_option(token: &str) -> Option<(&'static str, &'static str)> { + ROOT_BOOLEAN_OPTIONS.iter().find_map(|(option, key)| { + if token == *option { + return Some((*key, "true")); + } + let negative = format!("--no-{}", option.trim_start_matches('-')); + if token == negative { + return Some((*key, "false")); + } + token + .strip_prefix(option) + .and_then(|tail| tail.strip_prefix('=')) + .and_then(normalize_boolean) + .map(|value| (*key, value)) + }) +} + +fn normalize_boolean(value: &str) -> Option<&'static str> { + match value.trim().to_ascii_lowercase().as_str() { + "true" | "1" | "yes" | "on" => Some("true"), + "false" | "0" | "no" | "off" => Some("false"), + _ => None, + } +} + +fn valid_external_name(name: &str) -> bool { + let mut characters = name.chars(); + let Some(first) = characters.next() else { + return false; + }; + name.len() <= 64 + && first.is_ascii_alphanumeric() + && characters + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_')) +} + +fn is_builtin_name(name: &str) -> bool { + crate::completion::built_in_root_command() + .find_subcommand(name) + .is_some() +} + +fn resolve_external(name: &str) -> Option { + let sibling = env::current_exe() + .ok() + .and_then(|executable| executable.parent().map(Path::to_path_buf)); + let path = env::var_os("PATH"); + resolve_in_locations(name, sibling.as_deref(), path.as_deref()) +} + +fn resolve_in_locations( + name: &str, + sibling: Option<&Path>, + path: Option<&OsStr>, +) -> Option { + let stem = format!("{EXTERNAL_PREFIX}{name}"); + + if let Some(executable) = sibling + .filter(|directory| directory.is_absolute()) + .and_then(|directory| executable_in(directory, &stem)) + { + return Some(executable); + } + + let path = path?; + for directory in env::split_paths(path) { + if !directory.is_absolute() { + continue; + } + if let Some(executable) = executable_in(&directory, &stem) { + return Some(executable); + } + } + None +} + +fn executable_in(directory: &Path, stem: &str) -> Option { + executable_candidates(directory, stem) + .into_iter() + .find(|candidate| is_executable_file(candidate)) +} + +#[cfg(windows)] +fn executable_candidates(directory: &Path, stem: &str) -> Vec { + vec![directory.join(format!("{stem}.exe")), directory.join(stem)] +} + +#[cfg(not(windows))] +fn executable_candidates(directory: &Path, stem: &str) -> Vec { + vec![directory.join(stem)] +} + +fn is_executable_file(path: &Path) -> bool { + let Ok(metadata) = fs::metadata(path) else { + return false; + }; + if !metadata.is_file() { + return false; + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + metadata.permissions().mode() & 0o111 != 0 + } + #[cfg(not(unix))] + { + true + } +} + +fn run_external(executable: &Path, route: &ExternalRoute) -> Result { + let mut command = ProcessCommand::new(executable); + command.args(&route.arguments); + for (key, value) in &route.environment { + command.env(key, value); + } + command.env(EXTERNAL_COMMAND_ENV, &route.name); + + let status = command.status().with_context(|| { + format!( + "running external subcommand `{}` through {}", + route.name, + executable.display() + ) + })?; + Ok(status_code(status)) +} + +fn status_code(status: ExitStatus) -> i32 { + if let Some(code) = status.code() { + return code; + } + + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + status.signal().map(|signal| 128 + signal).unwrap_or(1) + } + #[cfg(not(unix))] + { + 1 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn os_args(values: &[&str]) -> Vec { + values.iter().map(OsString::from).collect() + } + + #[test] + fn routes_plugin_arguments_without_shell_rewriting() { + let route = external_route(&os_args(&[ + "zed", + "gitops", + "validate", + "--root", + "workspace with spaces", + "--offline", + ])) + .expect("external route"); + assert_eq!(route.name, "gitops"); + assert_eq!( + route.arguments, + os_args(&["validate", "--root", "workspace with spaces", "--offline"]) + ); + } + + #[test] + fn root_options_before_plugin_become_child_environment() { + let route = external_route(&os_args(&[ + "zed", + "--home", + "/tmp/zed-home", + "--git-submodules=false", + "gitops", + "validate", + ])) + .expect("external route"); + assert_eq!( + route.environment, + vec![ + ( + OsString::from("ZED_PKG_HOME"), + OsString::from("/tmp/zed-home") + ), + ( + OsString::from("ZED_PKG_GIT_SUBMODULES"), + OsString::from("false") + ), + ] + ); + } + + #[test] + fn root_options_after_plugin_are_lifted_until_double_dash() { + let route = external_route(&os_args(&[ + "zed", + "gitops", + "validate", + "--token", + "fixture-value", + "--offline", + "--", + "--home", + "child-owned-value", + ])) + .expect("external route"); + assert_eq!( + route.arguments, + os_args(&["validate", "--offline", "--", "--home", "child-owned-value"]) + ); + assert_eq!( + route.environment, + vec![( + OsString::from("ZED_PKG_TOKEN"), + OsString::from("fixture-value") + )] + ); + } + + #[test] + fn malformed_trailing_root_boolean_fails_closed() { + assert!( + external_route(&os_args(&[ + "zed", + "gitops", + "validate", + "--git-submodules=maybe" + ])) + .is_none() + ); + } + + #[test] + fn help_spelling_routes_to_external_help() { + let route = external_route(&os_args(&["zed", "help", "gitops", "validate"])) + .expect("external help route"); + assert_eq!(route.name, "gitops"); + assert_eq!(route.arguments, os_args(&["validate", "--help"])); + } + + #[test] + fn external_help_is_inserted_before_literal_double_dash() { + let route = external_route(&os_args(&[ + "zed", + "help", + "gitops", + "validate", + "--", + "--root", + "child-owned", + ])) + .expect("external help route"); + assert_eq!( + route.arguments, + os_args(&["validate", "--help", "--", "--root", "child-owned"]) + ); + } + + #[test] + fn builtins_and_unsafe_names_are_never_external() { + assert!(is_builtin_name("install")); + assert!(is_builtin_name("dev")); + assert!(!is_builtin_name("gitops")); + for invalid in ["../gitops", ".gitops", "git/ops", "-gitops", "gitops.exe"] { + assert!(!valid_external_name(invalid), "accepted {invalid:?}"); + } + } + + #[test] + fn root_help_model_contains_the_known_external_contract() { + let command = augment_root_command(crate::completion::built_in_root_command()); + let gitops = command.find_subcommand("gitops").expect("gitops command"); + assert!(gitops.find_subcommand("validate").is_some()); + } + + #[test] + fn relative_path_entries_are_not_searched() { + assert!(resolve_in_locations("demo", None, Some(OsStr::new("relative/bin"))).is_none()); + } + + #[cfg(unix)] + fn write_executable(path: &Path, exit_code: i32) { + use std::os::unix::fs::PermissionsExt; + + fs::write(path, format!("#!/bin/sh\nexit {exit_code}\n")).expect("write executable"); + let mut permissions = fs::metadata(path).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(path, permissions).expect("set executable permissions"); + } + + #[cfg(unix)] + #[test] + fn sibling_precedes_absolute_path_and_exit_code_is_preserved() { + let sibling = tempfile::tempdir().expect("sibling directory"); + let path_directory = tempfile::tempdir().expect("PATH directory"); + let sibling_executable = sibling.path().join("zed-demo"); + let path_executable = path_directory.path().join("zed-demo"); + write_executable(&sibling_executable, 23); + write_executable(&path_executable, 41); + let joined_path = env::join_paths([path_directory.path()]).expect("join PATH"); + + let resolved = + resolve_in_locations("demo", Some(sibling.path()), Some(joined_path.as_os_str())) + .expect("resolved executable"); + assert_eq!(resolved, sibling_executable); + + let route = ExternalRoute { + name: "demo".to_owned(), + arguments: Vec::new(), + environment: Vec::new(), + }; + assert_eq!(run_external(&resolved, &route).expect("run external"), 23); + } +} diff --git a/evidence/den-2725-final-export/src/lib.rs b/evidence/den-2725-final-export/src/lib.rs new file mode 100644 index 00000000..5f39c53a --- /dev/null +++ b/evidence/den-2725-final-export/src/lib.rs @@ -0,0 +1,45 @@ +//! Library core of the `zed` CLI. The binary in `main.rs` is a thin +//! dispatcher over these modules; integration tests drive them directly. + +pub mod asdf_environment; +pub mod auth; +pub mod cli; +pub mod cli_model; +pub mod completion; +pub mod config; +mod dart_wiring; +pub mod dev; +pub mod environment; +pub mod external_subcommands; +pub mod fetch; +pub mod flags; +pub mod git_submodules; +pub mod global; +pub mod install_graph; +pub mod interactive; +pub mod lock_waiter; +pub mod managed_install; +pub mod manifestless; +pub mod mise_lock; +pub mod nix_bundle_write; +pub mod nix_export_bundle; +pub mod nix_export_plan; +#[path = "ops_entry.rs"] +pub mod ops; +pub mod pack; +pub(crate) mod pack_guard; +pub(crate) mod pack_inputs; +pub mod preflight; +pub mod project_lock; +pub mod r2g; +pub mod registry; +pub mod release; +pub mod store; +pub mod task_cli; +pub mod task_runtime; +pub mod terminal_context; +pub mod transaction; +pub mod update; +pub mod vcs; + +pub mod tool_versions; diff --git a/evidence/den-2725-final-export/src/main.rs b/evidence/den-2725-final-export/src/main.rs new file mode 100644 index 00000000..67f25141 --- /dev/null +++ b/evidence/den-2725-final-export/src/main.rs @@ -0,0 +1,392 @@ +use std::ffi::{OsStr, OsString}; + +use zed_cli::asdf_environment; +use zed_cli::auth; +use zed_cli::cli::EnvCmd; +use zed_cli::cli::{ + AuthCmd, CacheCmd, Cli, Cmd, EnvironmentManagerArg, OrgCmd, ReleaseCmd, StoreCmd, TaskCmd, +}; +use zed_cli::completion; +use zed_cli::config::Config; +use zed_cli::dev; +use zed_cli::environment; +use zed_cli::fetch; +use zed_cli::git_submodules as submodules; +use zed_cli::global; +use zed_cli::managed_install; +use zed_cli::nix_bundle_write; +use zed_cli::nix_export_plan; +use zed_cli::ops; +use zed_cli::preflight; +use zed_cli::r2g::{self, R2gOptions}; +use zed_cli::release; +use zed_cli::store::Store; +use zed_cli::task_cli::{self, TaskAction}; +use zed_cli::update; + +fn main() { + let args = std::env::args_os().collect::>(); + zed_cli::cli_model::prepare_environment(&args); + if let Err(error) = zed_cli::flags::normalize_global_boolean_environment(&args) { + eprintln!("error: {error:#}"); + std::process::exit(2); + } + if root_help_requested(&args) { + if let Err(error) = completion::print_root_help() { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + return; + } + let global_requested = args.iter().skip(1).any(|argument| { + let argument = argument.as_os_str(); + argument == OsStr::new("global") || argument == OsStr::new("--global") + }); + if global_requested && let Some(result) = global::dispatch(args.clone()) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } + if let Some(result) = submodules::dispatch(args.clone()) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } + if let Some(result) = nix_bundle_write::dispatch(args.clone()) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } + if let Some(result) = nix_export_plan::dispatch(args.clone()) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } + if let Some(result) = fetch::dispatch(args.clone()) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } + if let Some(result) = dev::dispatch(args.clone()) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } + if let Some(result) = zed_cli::external_subcommands::dispatch(args) { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } + + if let Err(error) = zed_cli::flags::apply_cli_flags() { + eprintln!("error: {error:#}"); + std::process::exit(2); + } + let cli = zed_cli::cli_model::parse(); + if let Err(error) = run(cli) { + eprintln!("error: {error:#}"); + std::process::exit(1); + } +} + +fn root_help_requested(args: &[OsString]) -> bool { + let mut index = 1; + while index < args.len() { + let token = args[index].to_string_lossy(); + if token == "--help" || token == "-h" { + return true; + } + if token == "help" { + return args + .iter() + .skip(index + 1) + .all(|argument| argument.to_string_lossy().starts_with('-')); + } + if root_global_option_takes_value(&token) { + index += if token.contains('=') { 1 } else { 2 }; + continue; + } + if token.starts_with('-') { + index += 1; + continue; + } + return false; + } + false +} + +fn root_global_option_takes_value(token: &str) -> bool { + const OPTIONS: &[&str] = &[ + "--registry", + "--home", + "--token", + "--auth-url", + "--supabase-url", + "--supabase-key", + "--global-bin-dir", + ]; + OPTIONS.iter().any(|option| { + token == *option + || token + .strip_prefix(option) + .is_some_and(|remainder| remainder.starts_with('=')) + }) +} + +fn run(cli: Cli) -> anyhow::Result<()> { + let cfg = Config::from_globals(&cli.globals)?; + let git_submodules = cli.globals.git_submodules; + let cwd = std::env::current_dir()?; + if cwd.join(zed_cli::transaction::STAGING_DIR).is_dir() { + // Every live project transaction already owns this kernel-backed + // install lock. Recover under the same lock so a concurrent process + // cannot mistake an in-flight rollback journal for an abandoned one. + let store = Store::new(&cfg.home); + let _recovery_lock = store.install_lock()?; + zed_cli::transaction::recover_pending(&cwd)?; + } + match cli.cmd { + Cmd::Init { org, name } => ops::init(&cwd, org, name, cfg.interactive), + Cmd::Add { spec } => ops::add(&cwd, &cfg, &spec), + Cmd::Remove { spec } => ops::remove(&cwd, &cfg, &spec), + Cmd::Install { + specs, + frozen, + install_mode, + adapter, + allow_build, + target, + allow_no_manifest, + allow_ecosystem_mismatch, + } => { + if git_submodules { + submodules::sync(&cwd)?; + } + managed_install::install( + &cwd, + &cfg, + &specs, + frozen, + install_mode, + adapter, + allow_build, + target.as_deref(), + allow_no_manifest, + allow_ecosystem_mismatch, + ) + .map(|_| ()) + } + Cmd::Uninstall { specs } => ops::uninstall(&cwd, &cfg, &specs), + Cmd::Env { cmd } => match cmd { + EnvCmd::Import { + manager, + config, + lock, + frozen, + json, + } => match manager { + EnvironmentManagerArg::Mise => { + let imported = + environment::import_mise(&cwd, config.as_deref(), lock.as_deref(), frozen)?; + environment::print_import(&imported, json) + } + EnvironmentManagerArg::Asdf => { + let imported = asdf_environment::import_asdf( + &cwd, + config.as_deref(), + lock.as_deref(), + frozen, + )?; + asdf_environment::print_import(&imported, json) + } + }, + EnvCmd::Verify { + manager, + config, + lock, + frozen, + json, + } => match manager { + EnvironmentManagerArg::Mise => { + let imported = + environment::import_mise(&cwd, config.as_deref(), lock.as_deref(), frozen)?; + environment::print_verification(&imported, json) + } + EnvironmentManagerArg::Asdf => { + let imported = asdf_environment::import_asdf( + &cwd, + config.as_deref(), + lock.as_deref(), + frozen, + )?; + asdf_environment::print_verification(&imported, json) + } + }, + }, + Cmd::Task { plan, json, cmd } => { + let action = match cmd { + TaskCmd::List { all } => TaskAction::List { all }, + TaskCmd::Info { task } => TaskAction::Info { task }, + TaskCmd::Graph { task } => TaskAction::Graph { task }, + TaskCmd::Run { + task, + dry_run, + yes, + jobs, + no_cache, + args, + } => TaskAction::Run { + task, + dry_run, + yes, + jobs, + no_cache, + args, + }, + }; + task_cli::execute(&cwd, plan.as_deref(), json, action) + } + Cmd::Completions { shell } => { + completion::print(shell.into()); + Ok(()) + } + Cmd::Build { force } => ops::build_cmd(&cwd, &cfg, force), + Cmd::Run { command, args } => match ops::run(&cwd, &command, &args) { + Ok(code) => std::process::exit(code), + Err(error) => Err(error), + }, + Cmd::Gc { + older_than, + dry_run, + } => ops::gc(&cfg, &older_than, dry_run), + Cmd::Find { query } => ops::find(&cfg, &query), + Cmd::Pack { out } => ops::pack_cmd(&cwd, out.as_deref()).map(|_| ()), + Cmd::Release { cmd } => match cmd { + ReleaseCmd::Plan { json } => release::plan(&cwd, json), + ReleaseCmd::Preflight => preflight::preflight(&cwd), + }, + Cmd::Publish { + dry_run, + allow_dirty, + skip_vcs_checks, + } => { + managed_install::ensure_publishable(&cwd)?; + ops::publish(&cwd, &cfg, dry_run, allow_dirty, skip_vcs_checks) + } + Cmd::Yank { spec, undo } => ops::yank(&cfg, &spec, undo), + Cmd::R2g { + docker, + image, + runtime, + root, + clean, + } => r2g::run( + &cwd, + &cfg, + &R2gOptions { + docker, + image, + runtime, + root, + clean, + }, + ), + Cmd::SelfUpdate { + check, + force, + skip_checksum, + } => update::self_update(env!("CARGO_PKG_VERSION"), check, force, skip_checksum), + Cmd::Login { + email, + provider, + password_stdin, + } => auth::login(&cfg, email.as_deref(), provider, password_stdin), + Cmd::Signup { + email, + provider, + display_name, + password_stdin, + } => auth::signup( + &cfg, + email.as_deref(), + provider, + display_name.as_deref(), + password_stdin, + ), + Cmd::Logout => auth::signout(&cfg), + Cmd::Auth { cmd } => match cmd { + AuthCmd::Login { + email, + provider, + password_stdin, + } => auth::login(&cfg, email.as_deref(), provider, password_stdin), + AuthCmd::Signup { + email, + provider, + display_name, + password_stdin, + } => auth::signup( + &cfg, + email.as_deref(), + provider, + display_name.as_deref(), + password_stdin, + ), + AuthCmd::Signout => auth::signout(&cfg), + AuthCmd::ImportToken => ops::login(&cfg), + AuthCmd::Status => auth::status(&cfg), + AuthCmd::Refresh => auth::refresh(&cfg), + AuthCmd::Token => auth::print_token(&cfg), + }, + Cmd::Org { cmd } => match cmd { + OrgCmd::Claim { slug } => ops::org_claim(&cfg, &slug), + OrgCmd::Audit { slug, limit } => ops::org_audit(&cfg, &slug, limit), + }, + Cmd::Store { cmd } => match cmd { + StoreCmd::Status => ops::store_status(&cfg), + StoreCmd::Path => { + println!("{}", Store::new(&cfg.home).root().display()); + Ok(()) + } + StoreCmd::Prune => ops::store_prune(&cfg), + }, + Cmd::Cache { cmd } => match cmd { + CacheCmd::Clean => ops::cache_clean(&cfg), + }, + } +} diff --git a/evidence/den-2725-final-export/tests/external_gitops_dispatch.rs b/evidence/den-2725-final-export/tests/external_gitops_dispatch.rs new file mode 100644 index 00000000..5a3c57bf --- /dev/null +++ b/evidence/den-2725-final-export/tests/external_gitops_dispatch.rs @@ -0,0 +1,57 @@ +use std::process::{Command, Output}; + +fn text(output: &Output) -> String { + format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +#[test] +fn root_help_advertises_gitops_validate() { + let output = Command::new(env!("CARGO_BIN_EXE_zed")) + .arg("--help") + .output() + .expect("run zed help"); + assert!(output.status.success(), "{}", text(&output)); + let text = text(&output); + assert!(text.contains("gitops"), "{text}"); + assert!(text.contains("Validate GitOps composition"), "{text}"); +} + +#[test] +fn root_dispatches_to_the_sibling_gitops_binary() { + let output = Command::new(env!("CARGO_BIN_EXE_zed")) + .args(["gitops", "validate", "--help"]) + .output() + .expect("run zed gitops help"); + assert!(output.status.success(), "{}", text(&output)); + let text = text(&output); + assert!(text.contains("Usage: zed-gitops"), "{text}"); + assert!(text.contains("validate [OPTIONS]"), "{text}"); + assert!(text.contains("--offline"), "{text}"); +} + +#[test] +fn root_help_alias_keeps_help_before_literal_double_dash() { + let output = Command::new(env!("CARGO_BIN_EXE_zed")) + .args(["help", "gitops", "validate", "--", "--root", "child-owned"]) + .output() + .expect("run zed help gitops with literal delimiter"); + assert!(output.status.success(), "{}", text(&output)); + let text = text(&output); + assert!(text.contains("Usage: zed-gitops"), "{text}"); + assert!(text.contains("validate [OPTIONS]"), "{text}"); + assert!(text.contains("--offline"), "{text}"); +} + +#[test] +fn root_help_alias_reaches_the_external_binary() { + let output = Command::new(env!("CARGO_BIN_EXE_zed")) + .args(["help", "gitops"]) + .output() + .expect("run zed help gitops"); + assert!(output.status.success(), "{}", text(&output)); + assert!(text(&output).contains("Usage: zed-gitops")); +} diff --git a/evidence/package-inventory/production-package-inventory.json b/evidence/package-inventory/production-package-inventory.json new file mode 100644 index 00000000..44559358 --- /dev/null +++ b/evidence/package-inventory/production-package-inventory.json @@ -0,0 +1,312 @@ +{ + "count": 22, + "packages": [ + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-api-server.rs", + "org": "zed-pkg", + "package": "zed-pkg/zed-api-server.rs", + "repo": "zed-api-server.rs", + "repository_url": "https://github.com/zed-pkg/zed-api-server.rs", + "sha": "949b0872cf715b85ffc7e80a2a9e9cb9eb99efcf", + "tag": "v0.1.0", + "tag_sha": "949b0872cf715b85ffc7e80a2a9e9cb9eb99efcf", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-docs", + "org": "zed-pkg", + "package": "zed-pkg/zed-docs", + "repo": "zed-docs", + "repository_url": "https://github.com/zed-pkg/zed-docs", + "sha": "a5e9c713c6e6b6c5120a89e9a0233c7eeb420720", + "tag": "v0.1.0", + "tag_sha": "a5e9c713c6e6b6c5120a89e9a0233c7eeb420720", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-e2e", + "org": "zed-pkg", + "package": "zed-pkg/zed-e2e", + "repo": "zed-e2e", + "repository_url": "https://github.com/zed-pkg/zed-e2e", + "sha": "f636118e83a8a3edde7285431651850eddf53a14", + "tag": "v0.1.0", + "tag_sha": "f636118e83a8a3edde7285431651850eddf53a14", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-eclipse", + "org": "zed-pkg", + "package": "zed-pkg/zed-eclipse", + "repo": "zed-eclipse", + "repository_url": "https://github.com/zed-pkg/zed-eclipse", + "sha": "91f16c272b3221d0a70880b530ab670ade8bacb2", + "tag": "v0.1.0", + "tag_sha": "91f16c272b3221d0a70880b530ab670ade8bacb2", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-infra", + "org": "zed-pkg", + "package": "zed-pkg/zed-infra", + "repo": "zed-infra", + "repository_url": "https://github.com/zed-pkg/zed-infra", + "sha": "807ce6eb427270cb9df2f0af676faab310e00b05", + "tag": "v0.1.0", + "tag_sha": "807ce6eb427270cb9df2f0af676faab310e00b05", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-intellij", + "org": "zed-pkg", + "package": "zed-pkg/zed-intellij", + "repo": "zed-intellij", + "repository_url": "https://github.com/zed-pkg/zed-intellij", + "sha": "7282fdbc7046bb2ba58369b59ef627211033e145", + "tag": "v0.1.0", + "tag_sha": "7282fdbc7046bb2ba58369b59ef627211033e145", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-interfaces", + "org": "zed-pkg", + "package": "zed-pkg/zed-interfaces", + "repo": "zed-interfaces", + "repository_url": "https://github.com/zed-pkg/zed-interfaces", + "sha": "5394b2e7b070354ee79a4c6ac79c26d6264970cd", + "tag": "v0.1.0", + "tag_sha": "5394b2e7b070354ee79a4c6ac79c26d6264970cd", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-jetbrains-air", + "org": "zed-pkg", + "package": "zed-pkg/zed-jetbrains-air", + "repo": "zed-jetbrains-air", + "repository_url": "https://github.com/zed-pkg/zed-jetbrains-air", + "sha": "be357c41573e4afb936faea2b410272b32bf5649", + "tag": "v0.1.0", + "tag_sha": "be357c41573e4afb936faea2b410272b32bf5649", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-monorepo", + "org": "zed-pkg", + "package": "zed-pkg/zed-monorepo", + "repo": "zed-monorepo", + "repository_url": "https://github.com/zed-pkg/zed-monorepo", + "sha": "d130f0af41ddc261d35fbe6c456dc9fed656e190", + "tag": "v0.1.0", + "tag_sha": "d130f0af41ddc261d35fbe6c456dc9fed656e190", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [ + "oresoftware/k8s-libs-and-shared-defs" + ], + "name": "zed-orm-core", + "org": "zed-pkg", + "package": "zed-pkg/zed-orm-core", + "repo": "zed-orm-core", + "repository_url": "https://github.com/zed-pkg/zed-orm-core", + "sha": "a78baf355e8ba3b98fa09780af9f67e8b7c12b85", + "tag": "v0.1.0", + "tag_sha": "a78baf355e8ba3b98fa09780af9f67e8b7c12b85", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-pkg.github.io", + "org": "zed-pkg", + "package": "zed-pkg/zed-pkg.github.io", + "repo": "zed-pkg.github.io", + "repository_url": "https://github.com/zed-pkg/zed-pkg.github.io", + "sha": "ec5c9fcbc09ce2b9b8f2798a75b9740b8cefd76a", + "tag": "v0.1.0", + "tag_sha": "ec5c9fcbc09ce2b9b8f2798a75b9740b8cefd76a", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-qtcreator", + "org": "zed-pkg", + "package": "zed-pkg/zed-qtcreator", + "repo": "zed-qtcreator", + "repository_url": "https://github.com/zed-pkg/zed-qtcreator", + "sha": "6dcc1f350b7301ea71b2e90450217fc5e10a8519", + "tag": "v0.1.0", + "tag_sha": "6dcc1f350b7301ea71b2e90450217fc5e10a8519", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-sublimetext", + "org": "zed-pkg", + "package": "zed-pkg/zed-sublimetext", + "repo": "zed-sublimetext", + "repository_url": "https://github.com/zed-pkg/zed-sublimetext", + "sha": "c3153867dff0b560946c9b7e287ed50366b87e6c", + "tag": "v0.1.0", + "tag_sha": "c3153867dff0b560946c9b7e287ed50366b87e6c", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-sync", + "org": "zed-pkg", + "package": "zed-pkg/zed-sync", + "repo": "zed-sync", + "repository_url": "https://github.com/zed-pkg/zed-sync", + "sha": "0dcaee9d9a8185870f7d06f9a077ecf5161d9e7f", + "tag": "v0.1.0", + "tag_sha": "0dcaee9d9a8185870f7d06f9a077ecf5161d9e7f", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-visual-studio", + "org": "zed-pkg", + "package": "zed-pkg/zed-visual-studio", + "repo": "zed-visual-studio", + "repository_url": "https://github.com/zed-pkg/zed-visual-studio", + "sha": "5966b53d4951f02ffc07939175103764b9a2423f", + "tag": "v0.1.0", + "tag_sha": "5966b53d4951f02ffc07939175103764b9a2423f", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-vscode", + "org": "zed-pkg", + "package": "zed-pkg/zed-vscode", + "repo": "zed-vscode", + "repository_url": "https://github.com/zed-pkg/zed-vscode", + "sha": "1949d987a1c367771730b3c44bee2459364e0e56", + "tag": "v0.1.0", + "tag_sha": "1949d987a1c367771730b3c44bee2459364e0e56", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-web-server.rs", + "org": "zed-pkg", + "package": "zed-pkg/zed-web-server.rs", + "repo": "zed-web-server.rs", + "repository_url": "https://github.com/zed-pkg/zed-web-server.rs", + "sha": "98d94515eca2d5cd99849475d5e7b0cb6c2a93fe", + "tag": "v0.1.0", + "tag_sha": "98d94515eca2d5cd99849475d5e7b0cb6c2a93fe", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [], + "name": "zed-xcode", + "org": "zed-pkg", + "package": "zed-pkg/zed-xcode", + "repo": "zed-xcode", + "repository_url": "https://github.com/zed-pkg/zed-xcode", + "sha": "36309fa60cc7efa661e415752422fee8e1d5672f", + "tag": "v0.1.0", + "tag_sha": "36309fa60cc7efa661e415752422fee8e1d5672f", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [ + "zed-pkg/zed-interfaces" + ], + "name": "zed-clients", + "org": "zed-pkg", + "package": "zed-pkg/zed-clients", + "repo": "zed-clients", + "repository_url": "https://github.com/zed-pkg/zed-clients", + "sha": "e79723981b88f8b6fbb2f49e7d3d954e2832cdc6", + "tag": "v0.1.0", + "tag_sha": "e79723981b88f8b6fbb2f49e7d3d954e2832cdc6", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [ + "oresoftware/k8s-libs-and-shared-defs", + "zed-pkg/zed-interfaces" + ], + "name": "zed-lib", + "org": "zed-pkg", + "package": "zed-pkg/zed-lib", + "repo": "zed-lib", + "repository_url": "https://github.com/zed-pkg/zed-lib", + "sha": "12071278e728100cc80b8ba60297ebff8e0914cb", + "tag": "v0.1.0", + "tag_sha": "12071278e728100cc80b8ba60297ebff8e0914cb", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [ + "oresoftware/k8s-libs-and-shared-defs", + "zed-pkg/zed-interfaces" + ], + "name": "zed-lib-core", + "org": "zed-pkg", + "package": "zed-pkg/zed-lib-core", + "repo": "zed-lib-core", + "repository_url": "https://github.com/zed-pkg/zed-lib-core", + "sha": "700f1f9578c6633a20693a5b1f52970ab845a740", + "tag": "v0.1.0", + "tag_sha": "700f1f9578c6633a20693a5b1f52970ab845a740", + "version": "0.1.0" + }, + { + "archive_format": "tar.gz", + "dependencies": [ + "zed-pkg/zed-clients", + "zed-pkg/zed-interfaces", + "zed-pkg/zed-lock" + ], + "name": "zed-cli", + "org": "zed-pkg", + "package": "zed-pkg/zed-cli", + "repo": "zed-cli", + "repository_url": "https://github.com/zed-pkg/zed-cli", + "sha": "751358fe130e64b646844bc7d857c4cff215ca2d", + "tag": "v0.1.0", + "tag_sha": "751358fe130e64b646844bc7d857c4cff215ca2d", + "version": "0.1.0" + } + ], + "rejected": [ + { + "reason": "zed-pkg/zed-lock: tag 'v0.1.1' peels to a0dc78d385bc3ab553d3027b427f5f1428239c9c, default-branch head is 1db0da00d30fcf2e0762f50eedb1f88458020b52", + "repo": "zed-lock" + } + ], + "schema": 1 +} diff --git a/evidence/package-inventory/production-package-inventory.md b/evidence/package-inventory/production-package-inventory.md new file mode 100644 index 00000000..6b36296f --- /dev/null +++ b/evidence/package-inventory/production-package-inventory.md @@ -0,0 +1,32 @@ +# Production package inventory + +Selected immutable packages: **22** + +| Order | Package | Version | Repository | Commit | Tag | Format | +| ---: | --- | --- | --- | --- | --- | --- | +| 1 | `zed-pkg/zed-api-server.rs` | `0.1.0` | `zed-api-server.rs` | `949b0872cf715b85ffc7e80a2a9e9cb9eb99efcf` | `v0.1.0` | `tar.gz` | +| 2 | `zed-pkg/zed-docs` | `0.1.0` | `zed-docs` | `a5e9c713c6e6b6c5120a89e9a0233c7eeb420720` | `v0.1.0` | `tar.gz` | +| 3 | `zed-pkg/zed-e2e` | `0.1.0` | `zed-e2e` | `f636118e83a8a3edde7285431651850eddf53a14` | `v0.1.0` | `tar.gz` | +| 4 | `zed-pkg/zed-eclipse` | `0.1.0` | `zed-eclipse` | `91f16c272b3221d0a70880b530ab670ade8bacb2` | `v0.1.0` | `tar.gz` | +| 5 | `zed-pkg/zed-infra` | `0.1.0` | `zed-infra` | `807ce6eb427270cb9df2f0af676faab310e00b05` | `v0.1.0` | `tar.gz` | +| 6 | `zed-pkg/zed-intellij` | `0.1.0` | `zed-intellij` | `7282fdbc7046bb2ba58369b59ef627211033e145` | `v0.1.0` | `tar.gz` | +| 7 | `zed-pkg/zed-interfaces` | `0.1.0` | `zed-interfaces` | `5394b2e7b070354ee79a4c6ac79c26d6264970cd` | `v0.1.0` | `tar.gz` | +| 8 | `zed-pkg/zed-jetbrains-air` | `0.1.0` | `zed-jetbrains-air` | `be357c41573e4afb936faea2b410272b32bf5649` | `v0.1.0` | `tar.gz` | +| 9 | `zed-pkg/zed-monorepo` | `0.1.0` | `zed-monorepo` | `d130f0af41ddc261d35fbe6c456dc9fed656e190` | `v0.1.0` | `tar.gz` | +| 10 | `zed-pkg/zed-orm-core` | `0.1.0` | `zed-orm-core` | `a78baf355e8ba3b98fa09780af9f67e8b7c12b85` | `v0.1.0` | `tar.gz` | +| 11 | `zed-pkg/zed-pkg.github.io` | `0.1.0` | `zed-pkg.github.io` | `ec5c9fcbc09ce2b9b8f2798a75b9740b8cefd76a` | `v0.1.0` | `tar.gz` | +| 12 | `zed-pkg/zed-qtcreator` | `0.1.0` | `zed-qtcreator` | `6dcc1f350b7301ea71b2e90450217fc5e10a8519` | `v0.1.0` | `tar.gz` | +| 13 | `zed-pkg/zed-sublimetext` | `0.1.0` | `zed-sublimetext` | `c3153867dff0b560946c9b7e287ed50366b87e6c` | `v0.1.0` | `tar.gz` | +| 14 | `zed-pkg/zed-sync` | `0.1.0` | `zed-sync` | `0dcaee9d9a8185870f7d06f9a077ecf5161d9e7f` | `v0.1.0` | `tar.gz` | +| 15 | `zed-pkg/zed-visual-studio` | `0.1.0` | `zed-visual-studio` | `5966b53d4951f02ffc07939175103764b9a2423f` | `v0.1.0` | `tar.gz` | +| 16 | `zed-pkg/zed-vscode` | `0.1.0` | `zed-vscode` | `1949d987a1c367771730b3c44bee2459364e0e56` | `v0.1.0` | `tar.gz` | +| 17 | `zed-pkg/zed-web-server.rs` | `0.1.0` | `zed-web-server.rs` | `98d94515eca2d5cd99849475d5e7b0cb6c2a93fe` | `v0.1.0` | `tar.gz` | +| 18 | `zed-pkg/zed-xcode` | `0.1.0` | `zed-xcode` | `36309fa60cc7efa661e415752422fee8e1d5672f` | `v0.1.0` | `tar.gz` | +| 19 | `zed-pkg/zed-clients` | `0.1.0` | `zed-clients` | `e79723981b88f8b6fbb2f49e7d3d954e2832cdc6` | `v0.1.0` | `tar.gz` | +| 20 | `zed-pkg/zed-lib` | `0.1.0` | `zed-lib` | `12071278e728100cc80b8ba60297ebff8e0914cb` | `v0.1.0` | `tar.gz` | +| 21 | `zed-pkg/zed-lib-core` | `0.1.0` | `zed-lib-core` | `700f1f9578c6633a20693a5b1f52970ab845a740` | `v0.1.0` | `tar.gz` | +| 22 | `zed-pkg/zed-cli` | `0.1.0` | `zed-cli` | `751358fe130e64b646844bc7d857c4cff215ca2d` | `v0.1.0` | `tar.gz` | + +## Rejected candidates + +- `zed-lock` — zed-pkg/zed-lock: tag 'v0.1.1' peels to a0dc78d385bc3ab553d3027b427f5f1428239c9c, default-branch head is 1db0da00d30fcf2e0762f50eedb1f88458020b52 diff --git a/evidence/release-preparation/release-preparation.json b/evidence/release-preparation/release-preparation.json new file mode 100644 index 00000000..c7f910a5 --- /dev/null +++ b/evidence/release-preparation/release-preparation.json @@ -0,0 +1,177 @@ +{ + "failed": 4, + "failures": [ + { + "error": "quote_from_bytes() expected bytes", + "repository": "zed-pkg/zed-eclipse" + }, + { + "error": "quote_from_bytes() expected bytes", + "repository": "zed-pkg/zed-monorepo" + }, + { + "error": "quote_from_bytes() expected bytes", + "repository": "zed-pkg/zed-lib" + }, + { + "error": "quote_from_bytes() expected bytes", + "repository": "zed-pkg/zed-web-server.rs" + } + ], + "packages": [ + { + "manifest_created": false, + "repository": "zed-pkg/zed-sync", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "0dcaee9d9a8185870f7d06f9a077ecf5161d9e7f", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-sublimetext", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "c3153867dff0b560946c9b7e287ed50366b87e6c", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-xcode", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "36309fa60cc7efa661e415752422fee8e1d5672f", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-interfaces", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "5394b2e7b070354ee79a4c6ac79c26d6264970cd", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-lib-core", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "700f1f9578c6633a20693a5b1f52970ab845a740", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-docs", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "a5e9c713c6e6b6c5120a89e9a0233c7eeb420720", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-vscode", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "1949d987a1c367771730b3c44bee2459364e0e56", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-clients", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "e79723981b88f8b6fbb2f49e7d3d954e2832cdc6", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-cli", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "751358fe130e64b646844bc7d857c4cff215ca2d", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-pkg.github.io", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "ec5c9fcbc09ce2b9b8f2798a75b9740b8cefd76a", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-infra", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "807ce6eb427270cb9df2f0af676faab310e00b05", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-intellij", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "7282fdbc7046bb2ba58369b59ef627211033e145", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-jetbrains-air", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "be357c41573e4afb936faea2b410272b32bf5649", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-visual-studio", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "5966b53d4951f02ffc07939175103764b9a2423f", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-orm-core", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "a78baf355e8ba3b98fa09780af9f67e8b7c12b85", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-e2e", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "f636118e83a8a3edde7285431651850eddf53a14", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-qtcreator", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "6dcc1f350b7301ea71b2e90450217fc5e10a8519", + "version": "0.1.0" + }, + { + "manifest_created": true, + "repository": "zed-pkg/zed-api-server.rs", + "tag": "v0.1.0", + "tag_created": true, + "tag_sha": "949b0872cf715b85ffc7e80a2a9e9cb9eb99efcf", + "version": "0.1.0" + }, + { + "manifest_created": false, + "repository": "zed-pkg/zed-lock", + "tag": "v0.1.1", + "tag_created": false, + "tag_sha": "a0dc78d385bc3ab553d3027b427f5f1428239c9c", + "version": "0.1.1" + } + ], + "prepared": 19, + "schema": 1 +} diff --git a/evidence/release-preparation/release-preparation.md b/evidence/release-preparation/release-preparation.md new file mode 100644 index 00000000..8fe10901 --- /dev/null +++ b/evidence/release-preparation/release-preparation.md @@ -0,0 +1,33 @@ +# Production package release preparation + +Prepared: **19** +Failed: **4** + +| Package | Version | Tag | Commit | +| --- | --- | --- | --- | +| `zed-pkg/zed-sync` | `0.1.0` | `v0.1.0` | `0dcaee9d9a8185870f7d06f9a077ecf5161d9e7f` | +| `zed-pkg/zed-sublimetext` | `0.1.0` | `v0.1.0` | `c3153867dff0b560946c9b7e287ed50366b87e6c` | +| `zed-pkg/zed-xcode` | `0.1.0` | `v0.1.0` | `36309fa60cc7efa661e415752422fee8e1d5672f` | +| `zed-pkg/zed-interfaces` | `0.1.0` | `v0.1.0` | `5394b2e7b070354ee79a4c6ac79c26d6264970cd` | +| `zed-pkg/zed-lib-core` | `0.1.0` | `v0.1.0` | `700f1f9578c6633a20693a5b1f52970ab845a740` | +| `zed-pkg/zed-docs` | `0.1.0` | `v0.1.0` | `a5e9c713c6e6b6c5120a89e9a0233c7eeb420720` | +| `zed-pkg/zed-vscode` | `0.1.0` | `v0.1.0` | `1949d987a1c367771730b3c44bee2459364e0e56` | +| `zed-pkg/zed-clients` | `0.1.0` | `v0.1.0` | `e79723981b88f8b6fbb2f49e7d3d954e2832cdc6` | +| `zed-pkg/zed-cli` | `0.1.0` | `v0.1.0` | `751358fe130e64b646844bc7d857c4cff215ca2d` | +| `zed-pkg/zed-pkg.github.io` | `0.1.0` | `v0.1.0` | `ec5c9fcbc09ce2b9b8f2798a75b9740b8cefd76a` | +| `zed-pkg/zed-infra` | `0.1.0` | `v0.1.0` | `807ce6eb427270cb9df2f0af676faab310e00b05` | +| `zed-pkg/zed-intellij` | `0.1.0` | `v0.1.0` | `7282fdbc7046bb2ba58369b59ef627211033e145` | +| `zed-pkg/zed-jetbrains-air` | `0.1.0` | `v0.1.0` | `be357c41573e4afb936faea2b410272b32bf5649` | +| `zed-pkg/zed-visual-studio` | `0.1.0` | `v0.1.0` | `5966b53d4951f02ffc07939175103764b9a2423f` | +| `zed-pkg/zed-orm-core` | `0.1.0` | `v0.1.0` | `a78baf355e8ba3b98fa09780af9f67e8b7c12b85` | +| `zed-pkg/zed-e2e` | `0.1.0` | `v0.1.0` | `f636118e83a8a3edde7285431651850eddf53a14` | +| `zed-pkg/zed-qtcreator` | `0.1.0` | `v0.1.0` | `6dcc1f350b7301ea71b2e90450217fc5e10a8519` | +| `zed-pkg/zed-api-server.rs` | `0.1.0` | `v0.1.0` | `949b0872cf715b85ffc7e80a2a9e9cb9eb99efcf` | +| `zed-pkg/zed-lock` | `0.1.1` | `v0.1.1` | `a0dc78d385bc3ab553d3027b427f5f1428239c9c` | + +## Failures + +- `zed-pkg/zed-eclipse` — quote_from_bytes() expected bytes +- `zed-pkg/zed-monorepo` — quote_from_bytes() expected bytes +- `zed-pkg/zed-lib` — quote_from_bytes() expected bytes +- `zed-pkg/zed-web-server.rs` — quote_from_bytes() expected bytes diff --git a/evidence/shared-auth/concurrent-a.txt b/evidence/shared-auth/concurrent-a.txt new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/evidence/shared-auth/concurrent-a.txt @@ -0,0 +1 @@ +1 diff --git a/evidence/shared-auth/concurrent-b.txt b/evidence/shared-auth/concurrent-b.txt new file mode 100644 index 00000000..573541ac --- /dev/null +++ b/evidence/shared-auth/concurrent-b.txt @@ -0,0 +1 @@ +0 diff --git a/evidence/shared-auth/exact-head-certification.json b/evidence/shared-auth/exact-head-certification.json new file mode 100644 index 00000000..ed33ec6f --- /dev/null +++ b/evidence/shared-auth/exact-head-certification.json @@ -0,0 +1,29 @@ +{ + "schema": 2, + "repository": "shared-auth/shared-auth-server.rs", + "commit": "a471b20f8f81094f57393dc797b3cad0868c2f40", + "postgres_schema": true, + "postgres_first_use": 1, + "postgres_replay": 0, + "postgres_wrong_pkce": 0, + "postgres_wrong_redirect": 0, + "postgres_wrong_client": 0, + "postgres_expired": 0, + "postgres_concurrent_successes": 1, + "redis_ping": true, + "realm_contract": true, + "fmt": true, + "check": true, + "clippy": true, + "tests": true, + "release_build": true, + "configured_supabase_issuer": true, + "hosted_endpoint_test": true, + "self_hosted_endpoint_test": true, + "current_crypto_apis": true, + "bounded_csrf_cookie": true, + "independent_test_repository": "shared-auth-test/server-api-contract-e2e", + "independent_test_commit": "f41cdf668bbedab9ed75d01266a2086fa87b9ed1", + "independent_contract_v2": true, + "production_certified": false +} diff --git a/evidence/shared-auth/exact-head-certification.md b/evidence/shared-auth/exact-head-certification.md new file mode 100644 index 00000000..3f7798ae --- /dev/null +++ b/evidence/shared-auth/exact-head-certification.md @@ -0,0 +1,14 @@ +# Shared Auth exact-head certification + +Certified commit: `a471b20f8f81094f57393dc797b3cad0868c2f40` + +The Zpkg browser-handoff source passed the independent Shared Auth test contract, +declarative PostgreSQL schema application, realm validation, Redis reachability, +first-use/replay/PKCE/redirect/client/expiry database boundaries, concurrent +single-consumer redemption, locked formatting, compile, Clippy with warnings denied, +all-target tests, and a release build. + +Independent contract commit: `f41cdf668bbedab9ed75d01266a2086fa87b9ed1`. + +This is disposable test-environment evidence, not production Supabase, RDS, +Kubernetes, browser, or traffic certification. diff --git a/evidence/shared-auth/schema-apply.log b/evidence/shared-auth/schema-apply.log new file mode 100644 index 00000000..1b3e88fd --- /dev/null +++ b/evidence/shared-auth/schema-apply.log @@ -0,0 +1,37 @@ +CREATE SCHEMA +CREATE TABLE +CREATE INDEX +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE TABLE +CREATE INDEX +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE INDEX +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE INDEX +CREATE TABLE +CREATE INDEX +CREATE TABLE +CREATE INDEX diff --git a/evidence/shared-auth/shared-auth-test-contract-v2.json b/evidence/shared-auth/shared-auth-test-contract-v2.json new file mode 100644 index 00000000..376d24b1 --- /dev/null +++ b/evidence/shared-auth/shared-auth-test-contract-v2.json @@ -0,0 +1,321 @@ +{ + "schemaVersion": 2, + "generatedAt": "2026-08-09T19:55:20.246Z", + "source": { + "repository": "shared-auth/shared-auth-server.rs", + "pullRequest": 69, + "expectedSha": "a471b20f8f81094f57393dc797b3cad0868c2f40", + "actualSha": "a471b20f8f81094f57393dc797b3cad0868c2f40" + }, + "contract": { + "authorizePath": "/authorize", + "redeemPath": "/auth/handoff/redeem", + "clientId": "zpkg", + "pkceMethod": "S256", + "defaultCodeTtlSeconds": 90, + "codeTable": "shared_auth.browser_authorization_codes" + }, + "security": { + "immutableSourcePin": true, + "exactRedirectAllowlist": true, + "exactReturnPathAllowlist": true, + "sameOriginPasswordPost": true, + "browserReceivesOnlyCodeAndState": true, + "backendRedeemRequiresClientSecret": true, + "singleUseAtomicConsume": true, + "encryptedAtRest": true, + "providerTokensNeverEnterUrls": true, + "configuredSupabaseIssuer": true, + "currentCryptoApis": true, + "hostScopedCsrfCookie": true + }, + "checks": [ + { + "id": "dependencies.current_crypto_apis", + "detail": "the exact candidate must remain pinned to the dependency family certified by this contract" + }, + { + "id": "routes.authorize", + "detail": "GET and POST /authorize must remain registered" + }, + { + "id": "routes.redeem", + "detail": "backend code redemption must remain registered" + }, + { + "id": "headers.no_store", + "detail": "authorization and redemption responses must remain non-cacheable" + }, + { + "id": "headers.no_referrer", + "detail": "authorization responses must retain no-referrer policy" + }, + { + "id": "headers.csp", + "detail": "authorization UI must retain its restrictive CSP" + }, + { + "id": "handoff.default_ttl", + "detail": "authorization-code TTL must default to 90 seconds" + }, + { + "id": "handoff.required_secrets", + "detail": "browser handoff must fail closed without client, encryption, and database configuration" + }, + { + "id": "handoff.exact_redirect_allowlist", + "detail": "redirect URI must be exact-match allowlisted" + }, + { + "id": "handoff.exact_return_allowlist", + "detail": "return path must be exact-match allowlisted" + }, + { + "id": "handoff.pkce_s256", + "detail": "authorization must require a 43-character base64url S256 challenge" + }, + { + "id": "handoff.opaque_state", + "detail": "state must remain bounded opaque base64url" + }, + { + "id": "handoff.configured_issuer_endpoint", + "detail": "password sign-in must derive its endpoint from the configured issuer" + }, + { + "id": "handoff.endpoint_regressions", + "detail": "hosted and explicit self-hosted issuer behavior must have regression tests" + }, + { + "id": "handoff.encrypted_code_issue", + "detail": "browser handoff must store encrypted tokens behind an opaque hashed code" + }, + { + "id": "handoff.browser_only_code_and_state", + "detail": "browser redirect must contain exactly code and state, never provider tokens" + }, + { + "id": "handoff.backend_client_secret", + "detail": "backend redemption must authenticate the registered client secret" + }, + { + "id": "handoff.pkce_verifier", + "detail": "redemption must validate and derive the PKCE verifier challenge" + }, + { + "id": "handoff.atomic_single_use", + "detail": "single use, client, redirect, expiry, and PKCE checks must remain one atomic update" + }, + { + "id": "handoff.decrypt_after_consume", + "detail": "provider tokens may be decrypted only after successful atomic consume" + }, + { + "id": "client_config.strict_json", + "detail": "registered browser-client JSON must reject unknown fields" + }, + { + "id": "client_config.secret_indirection", + "detail": "client metadata must reference an environment secret instead of embedding one" + }, + { + "id": "client_config.secret_entropy", + "detail": "backend client secret must retain its minimum entropy floor" + }, + { + "id": "client_config.safe_redirects", + "detail": "redirect registration must reject unsafe URL forms" + }, + { + "id": "client_config.origin_local_returns", + "detail": "return paths must remain local to the product origin" + }, + { + "id": "http.same_origin_password_post", + "detail": "password submission must enforce the Shared Auth origin" + }, + { + "id": "http.csrf_binding", + "detail": "authorization form must bind an HTTP-only, SameSite, ten-minute CSRF cookie" + }, + { + "id": "http.host_scoped_cookie_tests", + "detail": "host scope, secure behavior, and lifetime must have regression tests" + }, + { + "id": "http.cookie_removal", + "detail": "successful authorization must remove the CSRF cookie immediately" + }, + { + "id": "http.no_transitive_time_import", + "detail": "cookie handling must not depend on a transitive time-module re-export" + }, + { + "id": "http.hidden_s256", + "detail": "authorization form must submit S256 explicitly" + }, + { + "id": "http.backend_bearer", + "detail": "redeem endpoint must require backend bearer authentication" + }, + { + "id": "http.tokens_backend_only", + "detail": "provider tokens may appear only in the backend redemption response" + }, + { + "id": "crypto.aes256gcm", + "detail": "token bundles must remain AES-256-GCM encrypted" + }, + { + "id": "crypto.modern_rand", + "detail": "nonce generation must use the current rand 0.10 API" + }, + { + "id": "crypto.fresh_nonce", + "detail": "every ciphertext must use a fresh 96-bit nonce" + }, + { + "id": "crypto.no_deprecated_nonce_slice", + "detail": "AES-GCM nonce construction must not use the deprecated generic-array slice API" + }, + { + "id": "crypto.authenticated_context", + "detail": "ciphertext must authenticate the handoff context as AAD" + }, + { + "id": "crypto.hashed_codes", + "detail": "plaintext authorization codes must not be stored" + }, + { + "id": "crypto.pkce_hash", + "detail": "PKCE challenge derivation must remain SHA-256/base64url" + }, + { + "id": "crypto.current_hmac_keyinit", + "detail": "client-secret comparison must use current HMAC KeyInit and constant-time verification" + }, + { + "id": "schema.authorization_codes", + "detail": "declarative schema must own the browser authorization-code table" + }, + { + "id": "schema.column.code_hash", + "detail": "authorization-code table must retain code_hash" + }, + { + "id": "schema.column.client_id", + "detail": "authorization-code table must retain client_id" + }, + { + "id": "schema.column.redirect_uri", + "detail": "authorization-code table must retain redirect_uri" + }, + { + "id": "schema.column.return_path", + "detail": "authorization-code table must retain return_path" + }, + { + "id": "schema.column.supabase_project", + "detail": "authorization-code table must retain supabase_project" + }, + { + "id": "schema.column.code_challenge", + "detail": "authorization-code table must retain code_challenge" + }, + { + "id": "schema.column.encrypted_tokens", + "detail": "authorization-code table must retain encrypted_tokens" + }, + { + "id": "schema.column.expires_at", + "detail": "authorization-code table must retain expires_at" + }, + { + "id": "schema.column.consumed_at", + "detail": "authorization-code table must retain consumed_at" + }, + { + "id": "schema.active_expiry_index", + "detail": "unconsumed codes must retain their expiry index" + }, + { + "id": "schema.declarative_ownership", + "detail": "schema ownership must remain declarative and migration-controlled" + }, + { + "id": "external_secret.AUTH_BROWSER_CLIENTS", + "detail": "Kubernetes ExternalSecret must project AUTH_BROWSER_CLIENTS" + }, + { + "id": "external_secret.AUTH_HANDOFF_ENCRYPTION_KEY", + "detail": "Kubernetes ExternalSecret must project AUTH_HANDOFF_ENCRYPTION_KEY" + }, + { + "id": "external_secret.AUTH_BROWSER_ZPKG_SECRET", + "detail": "Kubernetes ExternalSecret must project AUTH_BROWSER_ZPKG_SECRET" + }, + { + "id": "external_secret.AUTH_DATABASE_URL", + "detail": "Kubernetes ExternalSecret must project AUTH_DATABASE_URL" + }, + { + "id": "external_secret.no_values", + "detail": "deployment source must reference rather than embed credentials" + } + ], + "certification": { + "exactHead": true, + "independentTestRepository": "shared-auth-test/server-api-contract-e2e", + "configuredIssuerEndpoint": true, + "hostedEndpointRegression": true, + "selfHostedEndpointRegression": true, + "currentDependencyApis": true, + "boundedCsrfCookie": true + }, + "files": { + "cargo": { + "path": "Cargo.toml", + "sha256": "bbff84b41971cd47e3b1eeed0482a62e9991886c2d873b7cb3e93d57ca5a87f7", + "bytes": 3865 + }, + "router": { + "path": "src/http/mod.rs", + "sha256": "b6236968b7aeed9c96e6586fbcf584397ceadc3ee48839ab68b57067d361110c", + "bytes": 8854 + }, + "handoffHttp": { + "path": "src/http/handoff.rs", + "sha256": "b8f040d682e9f40563433ba68d8d6573bc1f2ef455900564f7fff62768d49860", + "bytes": 8456 + }, + "handoff": { + "path": "src/handoff/mod.rs", + "sha256": "e7bd4c21a93ced99c14566d2c2e54052afe398b871e910f690516747d1ada848", + "bytes": 15228 + }, + "handoffConfig": { + "path": "src/handoff/config.rs", + "sha256": "5562546c4e3cc07c5ce3409d31af82f7df86a16444fb21e3d9d906eeb58f68ad", + "bytes": 5802 + }, + "handoffCrypto": { + "path": "src/handoff/crypto.rs", + "sha256": "d840ee5c023078632f0275f414926df13d9536e3238d2973e8c3da237846883a", + "bytes": 4691 + }, + "schema": { + "path": "db/schema.sql", + "sha256": "375530f902f39a9a778f61b6f7966e7f6fd4d133f817176227f9ddcd277274d4", + "bytes": 17844 + }, + "externalSecret": { + "path": "deploy/k8s/externalsecret.yaml", + "sha256": "c5b9bb120798020a3894fc29395f754fb714b12915cfd27fdd169a06629c7202", + "bytes": 3428 + } + }, + "limitations": [ + "This report certifies exact source and the enumerated source/deployment contract assertions.", + "Runtime PostgreSQL, Redis, compilation, browser, Supabase, RDS, Kubernetes, and production traffic evidence must be supplied by the invoking workflow." + ] +} diff --git a/fixtures/den-2995-rust-enum-wire-source.json b/fixtures/den-2995-rust-enum-wire-source.json new file mode 100644 index 00000000..4bee3f97 --- /dev/null +++ b/fixtures/den-2995-rust-enum-wire-source.json @@ -0,0 +1,8 @@ +{ + "repository": "ORESoftware/k8s-libs-and-shared-defs", + "sha": "c1319a39c202568c2680b8221663a2093ad24a87", + "branch": "agent/den-2768-rust-enum-wire-values", + "linear": "DEN-2995", + "expect_fixed": false, + "note": "Pre-fix observation pin. Flip expect_fixed to true and repin to the repaired source head before promotion." +} diff --git a/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/deployment.yaml b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/deployment.yaml new file mode 100644 index 00000000..52875e09 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/deployment.yaml @@ -0,0 +1,101 @@ +# dd-fabrication-server — NATS-driven fabrication planner. +# +# Layer-2 (app-owned) manifest: namespace-scoped ONLY. The `daedalus` Namespace, its +# ResourceQuota/LimitRange and the default-deny NetworkPolicy are Layer-1, owned by +# ORESoftware/k8s-cluster (remote/argocd/projects/daedalus.tenant.yaml). Do NOT add a +# Namespace/ClusterRole/CRD here — the `daedalus` AppProject enforces +# clusterResourceWhitelist: [] and the sync will fail. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dd-fabrication-server + namespace: daedalus + labels: + app: dd-fabrication-server + app.kubernetes.io/name: dd-fabrication-server + app.kubernetes.io/part-of: daedalus +spec: + replicas: 1 + selector: + matchLabels: + app: dd-fabrication-server + template: + metadata: + labels: + app: dd-fabrication-server + app.kubernetes.io/name: dd-fabrication-server + app.kubernetes.io/part-of: daedalus + annotations: + # Declared for discoverability; this cluster's Prometheus still scrapes + # from static_configs in k8s-cluster, so adding a target needs a PR to + # remote/argocd/observability/{prometheus,otel-collector}.configmap.yaml. + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "8113" + prometheus.io/scheme: http + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: { type: RuntimeDefault } + containers: + - name: fabrication-server + image: ghcr.io/daedalus-fab/fabrication-server.rs:main + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8113 + env: + - { name: HOST, value: "0.0.0.0" } + - { name: PORT, value: "8113" } + # Cluster NATS (Layer-1, messaging namespace). Subjects use the + # dd_nats_subject_defs defaults; override only if the contract changes. + - { name: NATS_URL, value: "nats://dd-nats.messaging.svc.cluster.local:4222" } + # 4318 (HTTP), not 4317 (gRPC): dd-telemetry exports OTLP over + # HTTP/protobuf (the gRPC/tonic stack is deliberately excluded) and + # appends /v1/traces. Aimed at 4317 every export is an HTTP POST at + # a gRPC listener and is dropped silently — which is why this + # service has never produced traces despite the NetworkPolicy + # already allowing the egress. + - { name: OTEL_EXPORTER_OTLP_ENDPOINT, value: "http://dd-otel-collector.observability.svc.cluster.local:4318" } + - { name: OTEL_SERVICE_NAME, value: "dd-fabrication-server" } + - { name: RUST_LOG, value: "info" } + # The DSN provisioned by k8s/externalsecret.yaml. Without this the + # ExternalSecret materialized a Secret that nothing read, and + # src/persistence.rs — which reads FABRICATION_DATABASE_URL — would + # start with persistence silently disabled. + - name: FABRICATION_DATABASE_URL + valueFrom: + secretKeyRef: { name: dd-daedalus-fabrication-secrets, key: FABRICATION_DATABASE_URL } + - name: POD_NAME + valueFrom: { fieldRef: { fieldPath: metadata.name } } + - name: POD_NAMESPACE + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - name: NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } + readinessProbe: + httpGet: { path: /readyz, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /healthz, port: http } + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: { cpu: 100m, memory: 128Mi } + limits: { cpu: 1, memory: 512Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: tmp, mountPath: /tmp } + lifecycle: + preStop: + # Drain: let endpoint removal propagate before shutdown begins. + exec: { command: ["sleep", "5"] } + volumes: + - name: tmp + emptyDir: {} diff --git a/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/externalsecret.yaml b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/externalsecret.yaml new file mode 100644 index 00000000..446a8779 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/externalsecret.yaml @@ -0,0 +1,41 @@ +# Secrets for dd-fabrication-server, materialized by External Secrets Operator. +# +# Layer-2 (app-owned), namespace-scoped. The ClusterSecretStore itself is +# Layer-1 and lives in ORESoftware/k8s-cluster. +# +# WHAT BELONGS HERE: only true secrets. The Supabase audience, issuer, and JWKS +# URL are public values and are set inline in deployment.yaml — routing them +# through ESO would imply a confidentiality they do not have and make the auth +# configuration harder to read. +# +# Preferring the JWKS URL (RS256/ES256) over the legacy HS256 shared secret is +# what keeps this Secret down to a single key: with JWKS there is no symmetric +# secret to distribute at all. DAEDALUS_API_SUPABASE_JWT_SECRET is therefore +# intentionally absent; add it only if the project is pinned to HS256. +# NAME IS PER-SERVICE ON PURPOSE. All three daedalus services previously +# declared this ExternalSecret as `dd-daedalus-secrets` with the same +# target.name in the same namespace, while each Argo Application syncs with +# prune+selfHeal. Three controllers claiming one Secret with three different +# key sets fight forever: whichever syncs last wins and the other two services +# lose their credentials. One Secret per owning service avoids that entirely. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: dd-daedalus-fabrication-secrets + namespace: daedalus + labels: + app.kubernetes.io/part-of: daedalus +spec: + refreshInterval: 15m + secretStoreRef: + name: dd-cluster-secrets + kind: ClusterSecretStore + target: + name: dd-daedalus-fabrication-secrets + creationPolicy: Owner + data: + # The daedalus Postgres DSN. Carries a password, so it is never inline. + - secretKey: FABRICATION_DATABASE_URL + remoteRef: + key: dd/remote-dev/daedalus-secrets + property: FABRICATION_DATABASE_URL diff --git a/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/kustomization.yaml b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/kustomization.yaml new file mode 100644 index 00000000..9d235707 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Namespace-scoped resources only — the `daedalus` Namespace itself is Layer-1 +# (owned by ORESoftware/k8s-cluster). See the org AGENTS.md / k8s-cluster +# docs/app-deploy-contract.md. +namespace: daedalus + +resources: + - deployment.yaml + - service.yaml + - networkpolicy.yaml + - externalsecret.yaml diff --git a/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/networkpolicy.yaml b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/networkpolicy.yaml new file mode 100644 index 00000000..1be79509 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/networkpolicy.yaml @@ -0,0 +1,51 @@ +# App-owned allow-list. Pairs with the Layer-1 default-deny in the daedalus tenant +# scaffold: default-deny drops everything, these rules re-open exactly what this app needs. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dd-fabrication-server + namespace: daedalus +spec: + podSelector: + matchLabels: + app: dd-fabrication-server + policyTypes: [Ingress, Egress] + ingress: + # Prometheus/OTel scrape + in-cluster callers from observability. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + ports: + - { protocol: TCP, port: 8113 } + # Sibling daedalus services. + - from: + - podSelector: + matchLabels: + app.kubernetes.io/part-of: daedalus + ports: + - { protocol: TCP, port: 8113 } + egress: + # DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - { protocol: UDP, port: 53 } + - { protocol: TCP, port: 53 } + # NATS (messaging namespace) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: messaging + ports: + - { protocol: TCP, port: 4222 } + # OTLP telemetry + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + ports: + - { protocol: TCP, port: 4317 } diff --git a/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/service.yaml b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/service.yaml new file mode 100644 index 00000000..1f4fd233 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: dd-fabrication-server + namespace: daedalus + labels: + app: dd-fabrication-server + app.kubernetes.io/part-of: daedalus +spec: + type: ClusterIP + selector: + app: dd-fabrication-server + ports: + - name: http + port: 8113 + targetPort: http diff --git a/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/source.json b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/source.json new file mode 100644 index 00000000..7f13707a --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/source.json @@ -0,0 +1,29 @@ +{ + "$schema": "zed-pkg-test/direct-source-snapshot/v1", + "exportMode": "connector-read-no-product-credentials", + "repository": "github.com/daedalus-fab/fabrication-server.rs", + "revision": "5e305b9bf40d28139bdabf9342c304f759cbfee5", + "sourcePath": "k8s", + "files": { + "k8s/deployment.yaml": { + "gitBlobSha1": "52875e0935950ba845e1fb04c9675deef795549b" + }, + "k8s/externalsecret.yaml": { + "gitBlobSha1": "446a87798b16537de92ed7fbef5fa4de94a87d75" + }, + "k8s/kustomization.yaml": { + "gitBlobSha1": "9d23570713e6760f97b41220add5afacf4403753" + }, + "k8s/networkpolicy.yaml": { + "gitBlobSha1": "1be79509a0000a9ae33a5d9437fe781a6a44c79b" + }, + "k8s/service.yaml": { + "gitBlobSha1": "1f4fd233b99ce2dd3260c9c7d76a96960bbe60d2" + } + }, + "limitations": [ + "This snapshot proves the exported manifest bytes and renderability.", + "It does not prove that a test-org GitHub token can read the private source repository.", + "Private repository reachability remains a separately provisioned GitHub App credential contract." + ] +} diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/deployment.yaml b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/deployment.yaml new file mode 100644 index 00000000..52875e09 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/deployment.yaml @@ -0,0 +1,101 @@ +# dd-fabrication-server — NATS-driven fabrication planner. +# +# Layer-2 (app-owned) manifest: namespace-scoped ONLY. The `daedalus` Namespace, its +# ResourceQuota/LimitRange and the default-deny NetworkPolicy are Layer-1, owned by +# ORESoftware/k8s-cluster (remote/argocd/projects/daedalus.tenant.yaml). Do NOT add a +# Namespace/ClusterRole/CRD here — the `daedalus` AppProject enforces +# clusterResourceWhitelist: [] and the sync will fail. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dd-fabrication-server + namespace: daedalus + labels: + app: dd-fabrication-server + app.kubernetes.io/name: dd-fabrication-server + app.kubernetes.io/part-of: daedalus +spec: + replicas: 1 + selector: + matchLabels: + app: dd-fabrication-server + template: + metadata: + labels: + app: dd-fabrication-server + app.kubernetes.io/name: dd-fabrication-server + app.kubernetes.io/part-of: daedalus + annotations: + # Declared for discoverability; this cluster's Prometheus still scrapes + # from static_configs in k8s-cluster, so adding a target needs a PR to + # remote/argocd/observability/{prometheus,otel-collector}.configmap.yaml. + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "8113" + prometheus.io/scheme: http + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: { type: RuntimeDefault } + containers: + - name: fabrication-server + image: ghcr.io/daedalus-fab/fabrication-server.rs:main + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8113 + env: + - { name: HOST, value: "0.0.0.0" } + - { name: PORT, value: "8113" } + # Cluster NATS (Layer-1, messaging namespace). Subjects use the + # dd_nats_subject_defs defaults; override only if the contract changes. + - { name: NATS_URL, value: "nats://dd-nats.messaging.svc.cluster.local:4222" } + # 4318 (HTTP), not 4317 (gRPC): dd-telemetry exports OTLP over + # HTTP/protobuf (the gRPC/tonic stack is deliberately excluded) and + # appends /v1/traces. Aimed at 4317 every export is an HTTP POST at + # a gRPC listener and is dropped silently — which is why this + # service has never produced traces despite the NetworkPolicy + # already allowing the egress. + - { name: OTEL_EXPORTER_OTLP_ENDPOINT, value: "http://dd-otel-collector.observability.svc.cluster.local:4318" } + - { name: OTEL_SERVICE_NAME, value: "dd-fabrication-server" } + - { name: RUST_LOG, value: "info" } + # The DSN provisioned by k8s/externalsecret.yaml. Without this the + # ExternalSecret materialized a Secret that nothing read, and + # src/persistence.rs — which reads FABRICATION_DATABASE_URL — would + # start with persistence silently disabled. + - name: FABRICATION_DATABASE_URL + valueFrom: + secretKeyRef: { name: dd-daedalus-fabrication-secrets, key: FABRICATION_DATABASE_URL } + - name: POD_NAME + valueFrom: { fieldRef: { fieldPath: metadata.name } } + - name: POD_NAMESPACE + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - name: NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } + readinessProbe: + httpGet: { path: /readyz, port: http } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /healthz, port: http } + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: { cpu: 100m, memory: 128Mi } + limits: { cpu: 1, memory: 512Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: tmp, mountPath: /tmp } + lifecycle: + preStop: + # Drain: let endpoint removal propagate before shutdown begins. + exec: { command: ["sleep", "5"] } + volumes: + - name: tmp + emptyDir: {} diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/externalsecret.yaml b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/externalsecret.yaml new file mode 100644 index 00000000..446a8779 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/externalsecret.yaml @@ -0,0 +1,41 @@ +# Secrets for dd-fabrication-server, materialized by External Secrets Operator. +# +# Layer-2 (app-owned), namespace-scoped. The ClusterSecretStore itself is +# Layer-1 and lives in ORESoftware/k8s-cluster. +# +# WHAT BELONGS HERE: only true secrets. The Supabase audience, issuer, and JWKS +# URL are public values and are set inline in deployment.yaml — routing them +# through ESO would imply a confidentiality they do not have and make the auth +# configuration harder to read. +# +# Preferring the JWKS URL (RS256/ES256) over the legacy HS256 shared secret is +# what keeps this Secret down to a single key: with JWKS there is no symmetric +# secret to distribute at all. DAEDALUS_API_SUPABASE_JWT_SECRET is therefore +# intentionally absent; add it only if the project is pinned to HS256. +# NAME IS PER-SERVICE ON PURPOSE. All three daedalus services previously +# declared this ExternalSecret as `dd-daedalus-secrets` with the same +# target.name in the same namespace, while each Argo Application syncs with +# prune+selfHeal. Three controllers claiming one Secret with three different +# key sets fight forever: whichever syncs last wins and the other two services +# lose their credentials. One Secret per owning service avoids that entirely. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: dd-daedalus-fabrication-secrets + namespace: daedalus + labels: + app.kubernetes.io/part-of: daedalus +spec: + refreshInterval: 15m + secretStoreRef: + name: dd-cluster-secrets + kind: ClusterSecretStore + target: + name: dd-daedalus-fabrication-secrets + creationPolicy: Owner + data: + # The daedalus Postgres DSN. Carries a password, so it is never inline. + - secretKey: FABRICATION_DATABASE_URL + remoteRef: + key: dd/remote-dev/daedalus-secrets + property: FABRICATION_DATABASE_URL diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/kustomization.yaml b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/kustomization.yaml new file mode 100644 index 00000000..9d235707 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Namespace-scoped resources only — the `daedalus` Namespace itself is Layer-1 +# (owned by ORESoftware/k8s-cluster). See the org AGENTS.md / k8s-cluster +# docs/app-deploy-contract.md. +namespace: daedalus + +resources: + - deployment.yaml + - service.yaml + - networkpolicy.yaml + - externalsecret.yaml diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/networkpolicy.yaml b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/networkpolicy.yaml new file mode 100644 index 00000000..db738e9e --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/networkpolicy.yaml @@ -0,0 +1,51 @@ +# App-owned allow-list. Pairs with the Layer-1 default-deny in the daedalus tenant +# scaffold: default-deny drops everything, these rules re-open exactly what this app needs. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dd-fabrication-server + namespace: daedalus +spec: + podSelector: + matchLabels: + app: dd-fabrication-server + policyTypes: [Ingress, Egress] + ingress: + # Prometheus/OTel scrape + in-cluster callers from observability. + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + ports: + - { protocol: TCP, port: 8113 } + # Sibling daedalus services. + - from: + - podSelector: + matchLabels: + app.kubernetes.io/part-of: daedalus + ports: + - { protocol: TCP, port: 8113 } + egress: + # DNS + - to: + - namespaceSelector: {} + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - { protocol: UDP, port: 53 } + - { protocol: TCP, port: 53 } + # NATS (messaging namespace) + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: messaging + ports: + - { protocol: TCP, port: 4222 } + # OTLP over HTTP/protobuf, matching OTEL_EXPORTER_OTLP_ENDPOINT in the Deployment. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + ports: + - { protocol: TCP, port: 4318 } diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/service.yaml b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/service.yaml new file mode 100644 index 00000000..1f4fd233 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/k8s/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: dd-fabrication-server + namespace: daedalus + labels: + app: dd-fabrication-server + app.kubernetes.io/part-of: daedalus +spec: + type: ClusterIP + selector: + app: dd-fabrication-server + ports: + - name: http + port: 8113 + targetPort: http diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/scripts/ci/check_k8s_network_contract.py b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/scripts/ci/check_k8s_network_contract.py new file mode 100644 index 00000000..ab99bb1e --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/scripts/ci/check_k8s_network_contract.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Validate app endpoint ports against the namespace-scoped NetworkPolicy. + +The check intentionally uses only the Python standard library so it can run in +an isolated source checkout without Kubernetes, Helm, or private path +dependencies. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from urllib.parse import urlparse + + +class ContractError(ValueError): + """Raised when the checked-in Kubernetes source violates the app contract.""" + + +def _value_for_environment_variable(source: str, name: str) -> str: + lines = source.splitlines() + name_pattern = re.compile(rf"\bname:\s*{re.escape(name)}\b") + value_pattern = re.compile(r"\bvalue:\s*[\"']?([^\"'\s,}}]+)") + + for index, line in enumerate(lines): + if not name_pattern.search(line): + continue + for candidate in lines[index : index + 4]: + match = value_pattern.search(candidate) + if match: + return match.group(1) + raise ContractError(f"{name} is declared without a literal value") + raise ContractError(f"deployment does not declare {name}") + + +def _url_port(source: str, name: str) -> int: + value = _value_for_environment_variable(source, name) + parsed = urlparse(value) + if not parsed.scheme or not parsed.hostname or parsed.port is None: + raise ContractError(f"{name} must be an absolute URL with an explicit port: {value}") + return parsed.port + + +def _network_policy_ports(source: str) -> set[int]: + ports = {int(value) for value in re.findall(r"(?m)^\s*-?\s*\{?\s*protocol:\s*TCP,\s*port:\s*(\d+)", source)} + ports.update(int(value) for value in re.findall(r"(?m)^\s*port:\s*(\d+)\s*$", source)) + if not ports: + raise ContractError("NetworkPolicy does not declare any TCP egress ports") + return ports + + +def _metadata_namespace(source: str, resource: str) -> str: + metadata = re.search(r"(?ms)^metadata:\s*$\n(?P(?:^[ \t].*\n?)*)", source) + if metadata is None: + raise ContractError(f"{resource} does not contain metadata") + namespace = re.search(r"(?m)^\s+namespace:\s*([^\s#]+)\s*$", metadata.group("body")) + if namespace is None: + raise ContractError(f"{resource} metadata does not declare a namespace") + return namespace.group(1) + + +def validate(deployment: str, network_policy: str) -> dict[str, object]: + deployment_namespace = _metadata_namespace(deployment, "Deployment") + policy_namespace = _metadata_namespace(network_policy, "NetworkPolicy") + if deployment_namespace != policy_namespace: + raise ContractError( + "Deployment and NetworkPolicy namespaces differ: " + f"{deployment_namespace} != {policy_namespace}" + ) + + endpoint_ports = { + "NATS_URL": _url_port(deployment, "NATS_URL"), + "OTEL_EXPORTER_OTLP_ENDPOINT": _url_port( + deployment, "OTEL_EXPORTER_OTLP_ENDPOINT" + ), + } + allowed_ports = _network_policy_ports(network_policy) + missing = { + name: port for name, port in endpoint_ports.items() if port not in allowed_ports + } + if missing: + rendered = ", ".join(f"{name}={port}" for name, port in sorted(missing.items())) + raise ContractError( + f"NetworkPolicy egress does not permit declared endpoint port(s): {rendered}; " + f"allowed TCP ports are {sorted(allowed_ports)}" + ) + + return { + "namespace": deployment_namespace, + "endpointPorts": endpoint_ports, + "allowedTcpPorts": sorted(allowed_ports), + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--deployment", type=Path, default=Path("k8s/deployment.yaml")) + parser.add_argument( + "--network-policy", type=Path, default=Path("k8s/networkpolicy.yaml") + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + evidence = validate( + args.deployment.read_text(encoding="utf-8"), + args.network_policy.read_text(encoding="utf-8"), + ) + except (ContractError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + endpoints = ", ".join( + f"{name}={port}" for name, port in evidence["endpointPorts"].items() + ) + print( + f"Kubernetes endpoint/egress contract valid in {evidence['namespace']}: " + f"{endpoints}; allowed TCP ports={evidence['allowedTcpPorts']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/scripts/ci/test_check_k8s_network_contract.py b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/scripts/ci/test_check_k8s_network_contract.py new file mode 100644 index 00000000..2e6c6056 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/scripts/ci/test_check_k8s_network_contract.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import unittest + +from check_k8s_network_contract import ContractError, validate + + +DEPLOYMENT = """\ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example + namespace: daedalus +spec: + template: + spec: + containers: + - name: app + env: + - { name: NATS_URL, value: "nats://nats.messaging.svc:4222" } + - { name: OTEL_EXPORTER_OTLP_ENDPOINT, value: "http://otel.observability.svc:4318" } +""" + +NETWORK_POLICY = """\ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: example + namespace: daedalus +spec: + egress: + - to: [] + ports: + - { protocol: TCP, port: 4222 } + - { protocol: TCP, port: 4318 } +""" + + +class KubernetesNetworkContractTests(unittest.TestCase): + def test_declared_endpoint_ports_are_permitted(self) -> None: + result = validate(DEPLOYMENT, NETWORK_POLICY) + self.assertEqual("daedalus", result["namespace"]) + self.assertEqual( + {"NATS_URL": 4222, "OTEL_EXPORTER_OTLP_ENDPOINT": 4318}, + result["endpointPorts"], + ) + + def test_otlp_port_mismatch_fails_closed(self) -> None: + with self.assertRaisesRegex( + ContractError, + r"OTEL_EXPORTER_OTLP_ENDPOINT=4318", + ): + validate(DEPLOYMENT, NETWORK_POLICY.replace("port: 4318", "port: 4317")) + + def test_namespace_mismatch_fails_closed(self) -> None: + with self.assertRaisesRegex(ContractError, r"namespaces differ"): + validate(DEPLOYMENT, NETWORK_POLICY.replace("namespace: daedalus", "namespace: other")) + + def test_endpoint_requires_an_explicit_port(self) -> None: + deployment = DEPLOYMENT.replace( + "http://otel.observability.svc:4318", + "http://otel.observability.svc", + ) + with self.assertRaisesRegex(ContractError, r"explicit port"): + validate(deployment, NETWORK_POLICY) + + +if __name__ == "__main__": + unittest.main() diff --git a/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/source.json b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/source.json new file mode 100644 index 00000000..b925f633 --- /dev/null +++ b/fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577/source.json @@ -0,0 +1,36 @@ +{ + "$schema": "zed-pkg-test/direct-source-snapshot/v1", + "exportMode": "connector-read-no-product-credentials", + "repository": "github.com/daedalus-fab/fabrication-server.rs", + "revision": "cea4a3c772012e1f2f87050bac24e911b8e2e577", + "sourcePath": "k8s", + "files": { + "k8s/deployment.yaml": { + "gitBlobSha1": "52875e0935950ba845e1fb04c9675deef795549b" + }, + "k8s/externalsecret.yaml": { + "gitBlobSha1": "446a87798b16537de92ed7fbef5fa4de94a87d75" + }, + "k8s/kustomization.yaml": { + "gitBlobSha1": "9d23570713e6760f97b41220add5afacf4403753" + }, + "k8s/networkpolicy.yaml": { + "gitBlobSha1": "db738e9efd15707c61db573e296e38a0dd62ae6e" + }, + "k8s/service.yaml": { + "gitBlobSha1": "1f4fd233b99ce2dd3260c9c7d76a96960bbe60d2" + }, + "scripts/ci/check_k8s_network_contract.py": { + "gitBlobSha1": "ab99bb1eeec553fd119a4c68bcc13fc744491122" + }, + "scripts/ci/test_check_k8s_network_contract.py": { + "gitBlobSha1": "2e6c605609c51b79125c187bfccc22ce6c9c974d" + } + }, + "sourcePullRequest": "daedalus-fab/fabrication-server.rs#11", + "limitations": [ + "This snapshot proves the exported manifest and regression-script bytes plus renderability.", + "It does not prove that a test-org GitHub token can read the private source repository.", + "Private repository reachability remains a separately provisioned GitHub App credential contract." + ] +} diff --git a/fixtures/external_dispatch_probe.rs b/fixtures/external_dispatch_probe.rs new file mode 100644 index 00000000..76cb934c --- /dev/null +++ b/fixtures/external_dispatch_probe.rs @@ -0,0 +1,66 @@ +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process; + +fn json_string(value: &str) -> String { + let mut output = String::with_capacity(value.len() + 2); + output.push('"'); + for character in value.chars() { + match character { + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '\t' => output.push_str("\\t"), + character if character <= '\u{001f}' => { + use std::fmt::Write as _; + write!(&mut output, "\\u{:04x}", character as u32) + .expect("writing to a String cannot fail"); + } + character => output.push(character), + } + } + output.push('"'); + output +} + +fn main() { + let output = env::var_os("ZED_PROBE_OUTPUT") + .map(PathBuf::from) + .expect("ZED_PROBE_OUTPUT must name the evidence file"); + let arguments = env::args_os() + .skip(1) + .map(|argument| argument.to_string_lossy().into_owned()) + .collect::>(); + let environment = [ + "ZED_EXTERNAL_SUBCOMMAND", + "ZED_PKG_HOME", + "ZED_PKG_GIT_SUBMODULES", + ]; + + let mut payload = String::from("{\"args\":["); + for (index, argument) in arguments.iter().enumerate() { + if index != 0 { + payload.push(','); + } + payload.push_str(&json_string(argument)); + } + payload.push_str("],\"env\":{"); + for (index, key) in environment.iter().enumerate() { + if index != 0 { + payload.push(','); + } + payload.push_str(&json_string(key)); + payload.push(':'); + payload.push_str(&json_string(&env::var(key).unwrap_or_default())); + } + payload.push_str("}}\n"); + + fs::write(output, payload).expect("write probe evidence"); + let exit_code = env::var("ZED_PROBE_EXIT") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + process::exit(exit_code); +} diff --git a/fixtures/github-dependency-inventory/fixture.json b/fixtures/github-dependency-inventory/fixture.json new file mode 100644 index 00000000..a2d38f5e --- /dev/null +++ b/fixtures/github-dependency-inventory/fixture.json @@ -0,0 +1,76 @@ +{ + "organizations": { + "acme": [ + "acme/lib-b", + "acme/app", + "acme/shared", + "acme/cycle-b", + "acme/lib-a", + "acme/archived", + "acme/cycle-a", + "acme/app" + ] + }, + "page_size": 3, + "repositories": { + "acme/app": { + "commit_sha": "1111111111111111111111111111111111111111", + "default_branch": "main", + "files": { + ".gitmodules": "[submodule \"shared\"]\n\tpath = vendor/shared\n\turl = git@github.com:acme/shared.git\n", + ".zpkg.lock": "version = 1\n\n[[package]]\norg = \"acme\"\nname = \"lib-a\"\nversion = \"1.0.0\"\nsha256 = \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"\n\n[[package]]\norg = \"acme\"\nname = \"lib-b\"\nversion = \"1.0.0\"\nsha256 = \"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"\n\n[[package]]\norg = \"acme\"\nname = \"shared\"\nversion = \"2.0.0\"\nsha256 = \"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc\"\n", + ".zpkg.toml": "[package]\norg = \"acme\"\nname = \"app\"\nversion = \"0.1.0\"\n\n[dependencies]\n\"acme/lib-b\" = \"^1.0\"\n\"acme/lib-a\" = \"^1.0\\\" ]\\nmalicious\"\n", + "flake.lock": "{\n \"nodes\": {\n \"cycle-a\": {\n \"locked\": {\n \"owner\": \"acme\",\n \"repo\": \"cycle-a\",\n \"rev\": \"5555555555555555555555555555555555555555\",\n \"type\": \"github\"\n }\n },\n \"root\": {\n \"inputs\": {\n \"cycle-a\": \"cycle-a\"\n }\n }\n },\n \"root\": \"root\",\n \"version\": 7\n}\n", + "flake.nix": "{\n inputs.lib-a.url = \"github:acme/lib-a/release-1\";\n outputs = { self, ... }: {};\n}\n", + "nix/sources.json": "{\n \"shared\": {\n \"owner\": \"acme\",\n \"repo\": \"shared\",\n \"rev\": \"4444444444444444444444444444444444444444\"\n }\n}\n" + }, + "gitlinks": { + "vendor/shared": "4444444444444444444444444444444444444444" + }, + "private": true + }, + "acme/archived": { + "archived": true, + "commit_sha": "7777777777777777777777777777777777777777", + "default_branch": "trunk", + "files": {} + }, + "acme/cycle-a": { + "commit_sha": "5555555555555555555555555555555555555555", + "default_branch": "main", + "files": { + ".zpkg.toml": "[package]\norg = \"acme\"\nname = \"cycle-a\"\nversion = \"0.1.0\"\n\n[dependencies]\n\"acme/cycle-b\" = \"^0.1\"\n" + } + }, + "acme/cycle-b": { + "commit_sha": "6666666666666666666666666666666666666666", + "default_branch": "main", + "files": { + ".zpkg.toml": "[package]\norg = \"acme\"\nname = \"cycle-b\"\nversion = \"0.1.0\"\n\n[dependencies]\n\"acme/cycle-a\" = \"^0.1\"\n" + } + }, + "acme/lib-a": { + "commit_sha": "2222222222222222222222222222222222222222", + "default_branch": "main", + "files": { + ".zpkg.toml": "[package]\norg = \"acme\"\nname = \"lib-a\"\nversion = \"0.1.0\"\n\n[dependencies]\n\"acme/shared\" = \"^2.0\"\n" + } + }, + "acme/lib-b": { + "commit_sha": "3333333333333333333333333333333333333333", + "default_branch": "main", + "files": { + ".zpkg.toml": "[package]\norg = \"acme\"\nname = \"lib-b\"\nversion = \"0.1.0\"\n\n[dependencies]\n\"acme/shared\" = \"^2.0\"\n" + } + }, + "acme/shared": { + "commit_sha": "4444444444444444444444444444444444444444", + "default_branch": "main", + "files": { + ".zpkg.toml": "[package]\norg = \"acme\"\nname = \"shared\"\nversion = \"0.1.0\"\n" + } + } + }, + "require_token": false, + "schema": "zpkg/github-api-fixture/v1" +} diff --git a/fixtures/github-dependency-inventory/golden/inventory.dot b/fixtures/github-dependency-inventory/golden/inventory.dot new file mode 100644 index 00000000..265aebb0 --- /dev/null +++ b/fixtures/github-dependency-inventory/golden/inventory.dot @@ -0,0 +1,36 @@ +digraph zpkg_github_dependency_inventory { + graph [rankdir="LR"]; + node [shape="box"]; + n_7c974afcce1aea9376ae [label="acme/app", tooltip="repository"]; + n_c5d913dfdf91ec28e2d4 [label="acme/archived", tooltip="repository"]; + n_fb7cd9a2d983b4042332 [label="acme/cycle-a", tooltip="repository"]; + n_0c6884588964c77c72f9 [label="acme/cycle-b", tooltip="repository"]; + n_118763adadfdd7ab90b3 [label="acme/lib-a", tooltip="repository"]; + n_09ea16b5257c3f10ad4f [label="acme/lib-b", tooltip="repository"]; + n_3248a465db81578fdcb9 [label="acme/shared", tooltip="repository"]; + n_623011f81f79be3f23b8 [label="acme/app", tooltip="package"]; + n_1b2839d86de0d19b88d1 [label="acme/cycle-a", tooltip="package"]; + n_15a9926fb052dc7358a8 [label="acme/cycle-b", tooltip="package"]; + n_04944cea1bb2592e6626 [label="acme/lib-a", tooltip="package"]; + n_dced539eaf368baa2f03 [label="acme/lib-b", tooltip="package"]; + n_d361611f054f0a6e62a3 [label="acme/shared", tooltip="package"]; + n_3b00ca6717e90cc5942f [shape="note", style="dashed", label="lock pin (evidence only) | acme/lib-a@1.0.0 | sha256:aaaaaaaaaaaa", tooltip="non-topological lock evidence"]; + n_f88134d82fe5095ced40 [shape="note", style="dashed", label="lock pin (evidence only) | acme/lib-b@1.0.0 | sha256:bbbbbbbbbbbb", tooltip="non-topological lock evidence"]; + n_c466c1b288ce01b88d02 [shape="note", style="dashed", label="lock pin (evidence only) | acme/shared@2.0.0 | sha256:cccccccccccc", tooltip="non-topological lock evidence"]; + n_7c974afcce1aea9376ae -> n_fb7cd9a2d983b4042332 [label="nix-flake-lock | 5555555555555555555555555555555555555555"]; + n_7c974afcce1aea9376ae -> n_118763adadfdd7ab90b3 [label="nix-flake-declared | release-1"]; + n_7c974afcce1aea9376ae -> n_3248a465db81578fdcb9 [label="git-submodule | 4444444444444444444444444444444444444444"]; + n_7c974afcce1aea9376ae -> n_3248a465db81578fdcb9 [label="nix-source-pin | 4444444444444444444444444444444444444444"]; + n_7c974afcce1aea9376ae -> n_623011f81f79be3f23b8 [label="contains-zed-package"]; + n_fb7cd9a2d983b4042332 -> n_1b2839d86de0d19b88d1 [label="contains-zed-package"]; + n_0c6884588964c77c72f9 -> n_15a9926fb052dc7358a8 [label="contains-zed-package"]; + n_118763adadfdd7ab90b3 -> n_04944cea1bb2592e6626 [label="contains-zed-package"]; + n_09ea16b5257c3f10ad4f -> n_dced539eaf368baa2f03 [label="contains-zed-package"]; + n_3248a465db81578fdcb9 -> n_d361611f054f0a6e62a3 [label="contains-zed-package"]; + n_623011f81f79be3f23b8 -> n_04944cea1bb2592e6626 [label="zed-declared | ^1.0\" ]\nmalicious | 1.0.0"]; + n_623011f81f79be3f23b8 -> n_dced539eaf368baa2f03 [label="zed-declared | ^1.0 | 1.0.0"]; + n_1b2839d86de0d19b88d1 -> n_15a9926fb052dc7358a8 [label="zed-declared | ^0.1"]; + n_15a9926fb052dc7358a8 -> n_1b2839d86de0d19b88d1 [label="zed-declared | ^0.1"]; + n_04944cea1bb2592e6626 -> n_d361611f054f0a6e62a3 [label="zed-declared | ^2.0"]; + n_dced539eaf368baa2f03 -> n_d361611f054f0a6e62a3 [label="zed-declared | ^2.0"]; +} diff --git a/fixtures/github-dependency-inventory/golden/inventory.json b/fixtures/github-dependency-inventory/golden/inventory.json new file mode 100644 index 00000000..dcce96f3 --- /dev/null +++ b/fixtures/github-dependency-inventory/golden/inventory.json @@ -0,0 +1 @@ +{"completeness":{"inventory":"complete","resolution":"not-claimed"},"contradictions":[],"cycles":[["zpkg-package:acme/cycle-a","zpkg-package:acme/cycle-b"]],"edges":[{"input_name":"cycle-a","kind":"nix-flake-lock","provenance":[{"blob_sha":"377e360e7a219b9bf80292bd07310379677a2427","json_pointer":"/nodes/cycle-a/locked","line_end":3,"line_start":3,"manifest_kind":"nix-flake-lock","path":"flake.lock","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"selected_commit":"5555555555555555555555555555555555555555","source":"github-repo:acme/app","target":"github-repo:acme/cycle-a"},{"kind":"nix-flake-declared","provenance":[{"blob_sha":"42e041b17f09e39896bd8a97b3c40d9118bf7dc2","line_end":2,"line_start":2,"manifest_kind":"nix-flake","path":"flake.nix","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"requirement":"release-1","source":"github-repo:acme/app","target":"github-repo:acme/lib-a"},{"kind":"git-submodule","provenance":[{"blob_sha":"8effca003731ec6ccee1373a7c67a7aa28d37d5a","line_end":3,"line_start":3,"manifest_kind":"gitmodules","path":".gitmodules","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"selected_commit":"4444444444444444444444444444444444444444","source":"github-repo:acme/app","source_path":"vendor/shared","target":"github-repo:acme/shared"},{"kind":"nix-source-pin","provenance":[{"blob_sha":"e260fcb2e57e28a7f57319f946045ed17215289a","json_pointer":"/shared","line_end":2,"line_start":2,"manifest_kind":"nix-sources","path":"nix/sources.json","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"selected_commit":"4444444444444444444444444444444444444444","source":"github-repo:acme/app","target":"github-repo:acme/shared"},{"kind":"contains-zed-package","provenance":[{"blob_sha":"e4ce6878d54fb8c07548e4437d605faa22014e3a","line_end":3,"line_start":3,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"source":"github-repo:acme/app","target":"zpkg-package:acme/app"},{"kind":"contains-zed-package","provenance":[{"blob_sha":"d26a52b28f1d34f908967671c93235bafc928eb5","line_end":3,"line_start":3,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/cycle-a","repository_commit":"5555555555555555555555555555555555555555"}],"source":"github-repo:acme/cycle-a","target":"zpkg-package:acme/cycle-a"},{"kind":"contains-zed-package","provenance":[{"blob_sha":"80fad3d8c01a5a114fa1cc88b0b43bfed3f67b2f","line_end":3,"line_start":3,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/cycle-b","repository_commit":"6666666666666666666666666666666666666666"}],"source":"github-repo:acme/cycle-b","target":"zpkg-package:acme/cycle-b"},{"kind":"contains-zed-package","provenance":[{"blob_sha":"4523df3535d805089f9b5b78a2fd78eab9c3347a","line_end":3,"line_start":3,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/lib-a","repository_commit":"2222222222222222222222222222222222222222"}],"source":"github-repo:acme/lib-a","target":"zpkg-package:acme/lib-a"},{"kind":"contains-zed-package","provenance":[{"blob_sha":"9e82a473b5a1e5828211a448969f16e0a1c7d22a","line_end":3,"line_start":3,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/lib-b","repository_commit":"3333333333333333333333333333333333333333"}],"source":"github-repo:acme/lib-b","target":"zpkg-package:acme/lib-b"},{"kind":"contains-zed-package","provenance":[{"blob_sha":"47a4ce4d4ab0d0626e79f9d9cec581a16cce518b","line_end":3,"line_start":3,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/shared","repository_commit":"4444444444444444444444444444444444444444"}],"source":"github-repo:acme/shared","target":"zpkg-package:acme/shared"},{"artifact_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","kind":"zed-declared","provenance":[{"blob_sha":"e4ce6878d54fb8c07548e4437d605faa22014e3a","line_end":8,"line_start":8,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"requirement":"^1.0\" ]\nmalicious","selected_version":"1.0.0","selection_provenance":[{"blob_sha":"6ade1ad94a214adddaeb4d7bcfc270a314c3c20e","manifest_kind":"zed-lock","path":".zpkg.lock","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"source":"zpkg-package:acme/app","target":"zpkg-package:acme/lib-a"},{"artifact_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","kind":"zed-declared","provenance":[{"blob_sha":"e4ce6878d54fb8c07548e4437d605faa22014e3a","line_end":7,"line_start":7,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"requirement":"^1.0","selected_version":"1.0.0","selection_provenance":[{"blob_sha":"6ade1ad94a214adddaeb4d7bcfc270a314c3c20e","manifest_kind":"zed-lock","path":".zpkg.lock","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"source":"zpkg-package:acme/app","target":"zpkg-package:acme/lib-b"},{"kind":"zed-declared","provenance":[{"blob_sha":"d26a52b28f1d34f908967671c93235bafc928eb5","line_end":7,"line_start":7,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/cycle-a","repository_commit":"5555555555555555555555555555555555555555"}],"requirement":"^0.1","source":"zpkg-package:acme/cycle-a","target":"zpkg-package:acme/cycle-b"},{"kind":"zed-declared","provenance":[{"blob_sha":"80fad3d8c01a5a114fa1cc88b0b43bfed3f67b2f","line_end":7,"line_start":7,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/cycle-b","repository_commit":"6666666666666666666666666666666666666666"}],"requirement":"^0.1","source":"zpkg-package:acme/cycle-b","target":"zpkg-package:acme/cycle-a"},{"kind":"zed-declared","provenance":[{"blob_sha":"4523df3535d805089f9b5b78a2fd78eab9c3347a","line_end":7,"line_start":7,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/lib-a","repository_commit":"2222222222222222222222222222222222222222"}],"requirement":"^2.0","source":"zpkg-package:acme/lib-a","target":"zpkg-package:acme/shared"},{"kind":"zed-declared","provenance":[{"blob_sha":"9e82a473b5a1e5828211a448969f16e0a1c7d22a","line_end":7,"line_start":7,"manifest_kind":"zed-manifest","path":".zpkg.toml","repository":"acme/lib-b","repository_commit":"3333333333333333333333333333333333333333"}],"requirement":"^2.0","source":"zpkg-package:acme/lib-b","target":"zpkg-package:acme/shared"}],"failures":[],"inputs":{"includes":["git-submodule","nix","zed"],"organizations":["acme"],"repositories":["acme/app"]},"limits":{"max_edges":40000,"max_field_bytes":16384,"max_json_depth":64,"max_manifest_bytes":2097152,"max_nodes":20000,"max_repositories":1000,"max_requests":20000,"max_response_bytes":4194304,"max_retries":2,"max_seconds":120.0,"max_total_response_bytes":134217728,"max_tree_entries":50000},"nodes":[{"id":"github-repo:acme/app","kind":"repository","label":"acme/app","repository":"acme/app","scanned":true},{"id":"github-repo:acme/archived","kind":"repository","label":"acme/archived","repository":"acme/archived","scanned":true},{"id":"github-repo:acme/cycle-a","kind":"repository","label":"acme/cycle-a","repository":"acme/cycle-a","scanned":true},{"id":"github-repo:acme/cycle-b","kind":"repository","label":"acme/cycle-b","repository":"acme/cycle-b","scanned":true},{"id":"github-repo:acme/lib-a","kind":"repository","label":"acme/lib-a","repository":"acme/lib-a","scanned":true},{"id":"github-repo:acme/lib-b","kind":"repository","label":"acme/lib-b","repository":"acme/lib-b","scanned":true},{"id":"github-repo:acme/shared","kind":"repository","label":"acme/shared","repository":"acme/shared","scanned":true},{"id":"zpkg-package:acme/app","kind":"package","label":"acme/app","package":"acme/app"},{"id":"zpkg-package:acme/cycle-a","kind":"package","label":"acme/cycle-a","package":"acme/cycle-a"},{"id":"zpkg-package:acme/cycle-b","kind":"package","label":"acme/cycle-b","package":"acme/cycle-b"},{"id":"zpkg-package:acme/lib-a","kind":"package","label":"acme/lib-a","package":"acme/lib-a"},{"id":"zpkg-package:acme/lib-b","kind":"package","label":"acme/lib-b","package":"acme/lib-b"},{"id":"zpkg-package:acme/shared","kind":"package","label":"acme/shared","package":"acme/shared"}],"pins":[{"artifact_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","kind":"zed-lock-pin","package":"acme/lib-a","provenance":[{"blob_sha":"6ade1ad94a214adddaeb4d7bcfc270a314c3c20e","manifest_kind":"zed-lock","path":".zpkg.lock","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"repository":"acme/app","selected_version":"1.0.0","source":"zpkg-package:acme/app","topological":false},{"artifact_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","kind":"zed-lock-pin","package":"acme/lib-b","provenance":[{"blob_sha":"6ade1ad94a214adddaeb4d7bcfc270a314c3c20e","manifest_kind":"zed-lock","path":".zpkg.lock","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"repository":"acme/app","selected_version":"1.0.0","source":"zpkg-package:acme/app","topological":false},{"artifact_sha256":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc","kind":"zed-lock-pin","package":"acme/shared","provenance":[{"blob_sha":"6ade1ad94a214adddaeb4d7bcfc270a314c3c20e","manifest_kind":"zed-lock","path":".zpkg.lock","repository":"acme/app","repository_commit":"1111111111111111111111111111111111111111"}],"repository":"acme/app","selected_version":"2.0.0","source":"zpkg-package:acme/app","topological":false}],"repositories":[{"archived":false,"commit_sha":"1111111111111111111111111111111111111111","default_branch":"main","disabled":false,"full_name":"acme/app","manifests":[{"blob_sha":"8effca003731ec6ccee1373a7c67a7aa28d37d5a","bytes":81,"kind":"gitmodules","path":".gitmodules"},{"blob_sha":"42e041b17f09e39896bd8a97b3c40d9118bf7dc2","bytes":87,"kind":"nix-flake","path":"flake.nix"},{"blob_sha":"377e360e7a219b9bf80292bd07310379677a2427","bytes":316,"kind":"nix-flake-lock","path":"flake.lock"},{"blob_sha":"e260fcb2e57e28a7f57319f946045ed17215289a","bytes":119,"kind":"nix-sources","path":"nix/sources.json"},{"blob_sha":"6ade1ad94a214adddaeb4d7bcfc270a314c3c20e","bytes":418,"kind":"zed-lock","path":".zpkg.lock"},{"blob_sha":"e4ce6878d54fb8c07548e4437d605faa22014e3a","bytes":129,"kind":"zed-manifest","path":".zpkg.toml"}],"missing_manifests":["npins/sources.json"],"private":true,"status":"scanned"},{"archived":true,"commit_sha":"7777777777777777777777777777777777777777","default_branch":"trunk","disabled":false,"full_name":"acme/archived","manifests":[],"missing_manifests":[".gitmodules",".zpkg.lock",".zpkg.toml","flake.lock","flake.nix","nix/sources.json","npins/sources.json"],"private":false,"status":"scanned"},{"archived":false,"commit_sha":"5555555555555555555555555555555555555555","default_branch":"main","disabled":false,"full_name":"acme/cycle-a","manifests":[{"blob_sha":"d26a52b28f1d34f908967671c93235bafc928eb5","bytes":98,"kind":"zed-manifest","path":".zpkg.toml"}],"missing_manifests":[".gitmodules",".zpkg.lock","flake.lock","flake.nix","nix/sources.json","npins/sources.json"],"private":false,"status":"scanned"},{"archived":false,"commit_sha":"6666666666666666666666666666666666666666","default_branch":"main","disabled":false,"full_name":"acme/cycle-b","manifests":[{"blob_sha":"80fad3d8c01a5a114fa1cc88b0b43bfed3f67b2f","bytes":98,"kind":"zed-manifest","path":".zpkg.toml"}],"missing_manifests":[".gitmodules",".zpkg.lock","flake.lock","flake.nix","nix/sources.json","npins/sources.json"],"private":false,"status":"scanned"},{"archived":false,"commit_sha":"2222222222222222222222222222222222222222","default_branch":"main","disabled":false,"full_name":"acme/lib-a","manifests":[{"blob_sha":"4523df3535d805089f9b5b78a2fd78eab9c3347a","bytes":95,"kind":"zed-manifest","path":".zpkg.toml"}],"missing_manifests":[".gitmodules",".zpkg.lock","flake.lock","flake.nix","nix/sources.json","npins/sources.json"],"private":false,"status":"scanned"},{"archived":false,"commit_sha":"3333333333333333333333333333333333333333","default_branch":"main","disabled":false,"full_name":"acme/lib-b","manifests":[{"blob_sha":"9e82a473b5a1e5828211a448969f16e0a1c7d22a","bytes":95,"kind":"zed-manifest","path":".zpkg.toml"}],"missing_manifests":[".gitmodules",".zpkg.lock","flake.lock","flake.nix","nix/sources.json","npins/sources.json"],"private":false,"status":"scanned"},{"archived":false,"commit_sha":"4444444444444444444444444444444444444444","default_branch":"main","disabled":false,"full_name":"acme/shared","manifests":[{"blob_sha":"47a4ce4d4ab0d0626e79f9d9cec581a16cce518b","bytes":57,"kind":"zed-manifest","path":".zpkg.toml"}],"missing_manifests":[".gitmodules",".zpkg.lock","flake.lock","flake.nix","nix/sources.json","npins/sources.json"],"private":false,"status":"scanned"}],"schema":"zpkg/github-dependency-inventory/v1","strongly_connected_components":[{"cyclic":false,"id":"scc-0000","nodes":["github-repo:acme/app"]},{"cyclic":false,"id":"scc-0001","nodes":["github-repo:acme/archived"]},{"cyclic":false,"id":"scc-0002","nodes":["github-repo:acme/cycle-a"]},{"cyclic":false,"id":"scc-0003","nodes":["github-repo:acme/cycle-b"]},{"cyclic":false,"id":"scc-0004","nodes":["github-repo:acme/lib-a"]},{"cyclic":false,"id":"scc-0005","nodes":["github-repo:acme/lib-b"]},{"cyclic":false,"id":"scc-0006","nodes":["github-repo:acme/shared"]},{"cyclic":false,"id":"scc-0007","nodes":["zpkg-package:acme/app"]},{"cyclic":true,"id":"scc-0008","nodes":["zpkg-package:acme/cycle-a","zpkg-package:acme/cycle-b"]},{"cyclic":false,"id":"scc-0009","nodes":["zpkg-package:acme/lib-a"]},{"cyclic":false,"id":"scc-0010","nodes":["zpkg-package:acme/lib-b"]},{"cyclic":false,"id":"scc-0011","nodes":["zpkg-package:acme/shared"]}],"topological_waves":[["scc-0001","scc-0008","scc-0011"],["scc-0002","scc-0003","scc-0006","scc-0009","scc-0010"],["scc-0004","scc-0005","scc-0007"],["scc-0000"]],"usage":{"requests":29,"response_bytes":6662}} diff --git a/fixtures/github-dependency-inventory/golden/inventory.mmd b/fixtures/github-dependency-inventory/golden/inventory.mmd new file mode 100644 index 00000000..31358af2 --- /dev/null +++ b/fixtures/github-dependency-inventory/golden/inventory.mmd @@ -0,0 +1,35 @@ +flowchart LR + n_7c974afcce1aea9376ae["acme/app"] + n_c5d913dfdf91ec28e2d4["acme/archived"] + n_fb7cd9a2d983b4042332["acme/cycle-a"] + n_0c6884588964c77c72f9["acme/cycle-b"] + n_118763adadfdd7ab90b3["acme/lib-a"] + n_09ea16b5257c3f10ad4f["acme/lib-b"] + n_3248a465db81578fdcb9["acme/shared"] + n_623011f81f79be3f23b8["acme/app"] + n_1b2839d86de0d19b88d1["acme/cycle-a"] + n_15a9926fb052dc7358a8["acme/cycle-b"] + n_04944cea1bb2592e6626["acme/lib-a"] + n_dced539eaf368baa2f03["acme/lib-b"] + n_d361611f054f0a6e62a3["acme/shared"] + n_3b00ca6717e90cc5942f["lock pin (evidence only) | acme/lib-a@1.0.0 | sha256:aaaaaaaaaaaa"] + n_f88134d82fe5095ced40["lock pin (evidence only) | acme/lib-b@1.0.0 | sha256:bbbbbbbbbbbb"] + n_c466c1b288ce01b88d02["lock pin (evidence only) | acme/shared@2.0.0 | sha256:cccccccccccc"] + n_7c974afcce1aea9376ae -->|"nix-flake-lock | 5555555555555555555555555555555555555555"| n_fb7cd9a2d983b4042332 + n_7c974afcce1aea9376ae -->|"nix-flake-declared | release-1"| n_118763adadfdd7ab90b3 + n_7c974afcce1aea9376ae -->|"git-submodule | 4444444444444444444444444444444444444444"| n_3248a465db81578fdcb9 + n_7c974afcce1aea9376ae -->|"nix-source-pin | 4444444444444444444444444444444444444444"| n_3248a465db81578fdcb9 + n_7c974afcce1aea9376ae -->|"contains-zed-package"| n_623011f81f79be3f23b8 + n_fb7cd9a2d983b4042332 -->|"contains-zed-package"| n_1b2839d86de0d19b88d1 + n_0c6884588964c77c72f9 -->|"contains-zed-package"| n_15a9926fb052dc7358a8 + n_118763adadfdd7ab90b3 -->|"contains-zed-package"| n_04944cea1bb2592e6626 + n_09ea16b5257c3f10ad4f -->|"contains-zed-package"| n_dced539eaf368baa2f03 + n_3248a465db81578fdcb9 -->|"contains-zed-package"| n_d361611f054f0a6e62a3 + n_623011f81f79be3f23b8 -->|"zed-declared | ^1.0" ] malicious | 1.0.0"| n_04944cea1bb2592e6626 + n_623011f81f79be3f23b8 -->|"zed-declared | ^1.0 | 1.0.0"| n_dced539eaf368baa2f03 + n_1b2839d86de0d19b88d1 -->|"zed-declared | ^0.1"| n_15a9926fb052dc7358a8 + n_15a9926fb052dc7358a8 -->|"zed-declared | ^0.1"| n_1b2839d86de0d19b88d1 + n_04944cea1bb2592e6626 -->|"zed-declared | ^2.0"| n_d361611f054f0a6e62a3 + n_dced539eaf368baa2f03 -->|"zed-declared | ^2.0"| n_d361611f054f0a6e62a3 + classDef lockEvidence stroke-dasharray: 5 5 + class n_3b00ca6717e90cc5942f,n_f88134d82fe5095ced40,n_c466c1b288ce01b88d02 lockEvidence diff --git a/fixtures/native-dependency-consumer/Cargo.toml b/fixtures/native-dependency-consumer/Cargo.toml new file mode 100644 index 00000000..538bb72e --- /dev/null +++ b/fixtures/native-dependency-consumer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "zed-native-dependency-contract-consumer" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +zed-interfaces = { path = "../../../zed-interfaces" } diff --git a/fixtures/native-dependency-consumer/src/main.rs b/fixtures/native-dependency-consumer/src/main.rs new file mode 100644 index 00000000..99678141 --- /dev/null +++ b/fixtures/native-dependency-consumer/src/main.rs @@ -0,0 +1,290 @@ +use zed_interfaces::{ + ArtifactFormat, NATIVE_DEPENDENCY_LOCK_SCHEMA_V1, NativeArtifact, NativeDependencyError, + NativeDependencyLock, NativeRegistry, NativeVersionCandidate, NativeVersionRequirement, +}; + +fn artifact(digit: char) -> NativeArtifact { + NativeArtifact { + sha256: std::iter::repeat_n(digit, 64).collect(), + size: 256, + format: ArtifactFormat::TarGz, + } +} + +fn candidate(version: &str, digit: char) -> NativeVersionCandidate { + NativeVersionCandidate { + version: version.to_string(), + artifact: artifact(digit), + } +} + +fn assert_source_aware_translation() { + let npm_exact = NativeVersionRequirement::parse(NativeRegistry::Npm, "1.2.3").unwrap(); + let cargo_caret = + NativeVersionRequirement::parse(NativeRegistry::Cargo, "1.2.3").unwrap(); + assert_eq!(npm_exact.canonical, "=1.2.3"); + assert_eq!(cargo_caret.canonical, "^1.2.3"); + assert!(npm_exact.matches("1.2.3").unwrap()); + assert!(!npm_exact.matches("1.2.4").unwrap()); + assert!(cargo_caret.matches("1.9.9").unwrap()); + assert!(!cargo_caret.matches("2.0.0").unwrap()); + + let npm_minor = NativeVersionRequirement::parse(NativeRegistry::Npm, "1.2").unwrap(); + let cargo_minor = NativeVersionRequirement::parse(NativeRegistry::Cargo, "1.2").unwrap(); + assert_eq!(npm_minor.canonical, "1.2.*"); + assert_eq!(cargo_minor.canonical, "^1.2"); + assert!(!npm_minor.matches("1.3.0").unwrap()); + assert!(cargo_minor.matches("1.3.0").unwrap()); + + let npm_comparators = + NativeVersionRequirement::parse(NativeRegistry::Npm, ">=1.2.3 <2.0.0").unwrap(); + let cargo_comparators = + NativeVersionRequirement::parse(NativeRegistry::Cargo, ">=1.2.3, <2.0.0").unwrap(); + assert_eq!(npm_comparators.canonical, ">=1.2.3, <2.0.0"); + assert_eq!(cargo_comparators.canonical, ">=1.2.3, <2.0.0"); + + let npm_spaced = + NativeVersionRequirement::parse(NativeRegistry::Npm, ">= 1.2.3 < 2.0.0").unwrap(); + assert_eq!(npm_spaced.canonical, ">=1.2.3, <2.0.0"); + assert!(npm_spaced.matches("1.9.9").unwrap()); + assert!(!npm_spaced.matches("2.0.0").unwrap()); + + let cargo_spaced = + NativeVersionRequirement::parse(NativeRegistry::Cargo, ">= 1.2, < 1.5").unwrap(); + assert_eq!(cargo_spaced.canonical, ">=1.2, <1.5"); + assert!(cargo_spaced.matches("1.4.99").unwrap()); + assert!(!cargo_spaced.matches("1.5.0").unwrap()); + + let npm_x = NativeVersionRequirement::parse(NativeRegistry::Npm, "v1.2.X").unwrap(); + assert_eq!(npm_x.canonical, "1.2.*"); + + let npm_gt_major = NativeVersionRequirement::parse(NativeRegistry::Npm, ">1").unwrap(); + let npm_gt_minor = NativeVersionRequirement::parse(NativeRegistry::Npm, ">1.2").unwrap(); + let npm_lte_minor = NativeVersionRequirement::parse(NativeRegistry::Npm, "<=1.2").unwrap(); + assert_eq!(npm_gt_major.canonical, ">=2.0.0"); + assert_eq!(npm_gt_minor.canonical, ">=1.3.0"); + assert_eq!(npm_lte_minor.canonical, "<1.3.0"); + assert!(!npm_gt_major.matches("1.99.99").unwrap()); + assert!(npm_gt_major.matches("2.0.0").unwrap()); + assert!(!npm_gt_minor.matches("1.2.999").unwrap()); + assert!(npm_gt_minor.matches("1.3.0").unwrap()); + assert!(npm_lte_minor.matches("1.2.999").unwrap()); + assert!(!npm_lte_minor.matches("1.3.0").unwrap()); + + let cargo_zero = NativeVersionRequirement::parse(NativeRegistry::Cargo, "0.2.3").unwrap(); + assert!(cargo_zero.matches("0.2.9").unwrap()); + assert!(!cargo_zero.matches("0.3.0").unwrap()); + + let ordinary = NativeVersionRequirement::parse(NativeRegistry::Npm, "^1.2.3").unwrap(); + assert!(!ordinary.matches("1.3.0-beta.1").unwrap()); + let prerelease = + NativeVersionRequirement::parse(NativeRegistry::Npm, "1.3.0-beta.1").unwrap(); + assert!(prerelease.matches("1.3.0-beta.1").unwrap()); +} + +fn assert_exact_lock_resolution() { + let candidates = vec![ + candidate("1.2.3", 'a'), + candidate("1.9.0", 'b'), + candidate("1.10.0", 'c'), + candidate("2.0.0", 'd'), + ]; + let cargo = NativeDependencyLock::resolve( + NativeRegistry::Cargo, + "fiducia_core", + "1.2.3", + &candidates, + ) + .unwrap(); + assert_eq!(cargo.schema, NATIVE_DEPENDENCY_LOCK_SCHEMA_V1); + assert_eq!(cargo.package.version, "1.10.0"); + assert_eq!(cargo.artifact.sha256, "c".repeat(64)); + cargo.validate().unwrap(); + assert!(!cargo.canonical_json_bytes().unwrap().is_empty()); + + let mut reversed = candidates; + reversed.reverse(); + let reordered = NativeDependencyLock::resolve( + NativeRegistry::Cargo, + "fiducia_core", + "1.2.3", + &reversed, + ) + .unwrap(); + assert_eq!(cargo, reordered); + assert_eq!( + cargo.canonical_json_bytes().unwrap(), + reordered.canonical_json_bytes().unwrap() + ); + + let npm = NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@fiducia/core", + "1.2.3", + &[candidate("1.2.3", 'a'), candidate("1.2.4", 'b')], + ) + .unwrap(); + assert_eq!(npm.requirement.canonical, "=1.2.3"); + assert_eq!(npm.package.version, "1.2.3"); +} + +fn assert_requirement_rejections() { + for declared in [ + "", + " latest", + "latest", + "1.0.0 || 2.0.0", + "1.0.0 - 2.0.0", + "workspace:^1.0.0", + "file:../core", + "npm:@fiducia/core@1.0.0", + "git+https://example.invalid/core", + ">=1.0.0, <2.0.0", + "1.2.3+linux", + "01.2", + "1.02", + ">1.02", + "^01.2", + "9007199254740992.0.0", + ">9007199254740991", + ] { + assert!(NativeVersionRequirement::parse(NativeRegistry::Npm, declared).is_err()); + } + + for declared in [ + "", + "1.0.0 || 2.0.0", + "1.0.0 - 2.0.0", + ">=1.0.0 <2.0.0", + ">= 1.0 < 2.0", + "1.x", + "file:../core", + "1.2.3+linux", + ] { + assert!(NativeVersionRequirement::parse(NativeRegistry::Cargo, declared).is_err()); + } +} + +fn assert_lock_rejections() { + let mut canonical_drift = NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@fiducia/core", + "^1.2.3", + &[candidate("1.9.0", 'a')], + ) + .unwrap(); + canonical_drift.requirement.canonical = "^1.3.0".to_string(); + assert!(matches!( + canonical_drift.validate(), + Err(NativeDependencyError::CanonicalRequirementDrift { .. }) + )); + + let mut version_drift = NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@fiducia/core", + "^1.2.3", + &[candidate("1.9.0", 'a')], + ) + .unwrap(); + version_drift.package.version = "2.0.0".to_string(); + assert!(matches!( + version_drift.validate(), + Err(NativeDependencyError::ResolvedVersionDoesNotMatch { .. }) + )); + + let mut malformed_digest = NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@fiducia/core", + "^1.2.3", + &[candidate("1.9.0", 'a')], + ) + .unwrap(); + malformed_digest.artifact.sha256 = "A".repeat(64); + assert!(matches!( + malformed_digest.validate(), + Err(NativeDependencyError::InvalidSha256 { .. }) + )); + + let mut zero_digest = NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@fiducia/core", + "^1.2.3", + &[candidate("1.9.0", 'a')], + ) + .unwrap(); + zero_digest.artifact.sha256 = "0".repeat(64); + assert!(matches!( + zero_digest.validate(), + Err(NativeDependencyError::InvalidSha256 { .. }) + )); + + let mut empty_artifact = NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@fiducia/core", + "^1.2.3", + &[candidate("1.9.0", 'a')], + ) + .unwrap(); + empty_artifact.artifact.size = 0; + assert!(matches!( + empty_artifact.validate(), + Err(NativeDependencyError::EmptyArtifact { .. }) + )); + + assert!(matches!( + NativeDependencyLock::resolve( + NativeRegistry::Cargo, + "fiducia_core", + "1.2.3", + &[candidate("1.2.3", 'a'), candidate("1.2.3", 'b')], + ), + Err(NativeDependencyError::DuplicateCandidateVersion { .. }) + )); + + assert!(matches!( + NativeDependencyLock::resolve( + NativeRegistry::Cargo, + "fiducia_core", + "1.2.3", + &[candidate("1.2.3+linux", 'a')], + ), + Err(NativeDependencyError::BuildMetadataNotAllowed { .. }) + )); + + assert!(matches!( + NativeDependencyLock::resolve( + NativeRegistry::Cargo, + "fiducia_core", + "^2.0.0", + &[candidate("1.9.0", 'a')], + ), + Err(NativeDependencyError::NoMatchingVersion { .. }) + )); + + assert!(matches!( + NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@Fiducia/core", + "1.2.3", + &[candidate("1.2.3", 'a')], + ), + Err(NativeDependencyError::InvalidPackageName { .. }) + )); + + assert!(matches!( + NativeDependencyLock::resolve( + NativeRegistry::Npm, + "@fiducia/core", + "*", + &[candidate("9007199254740992.0.0", 'a')], + ), + Err(NativeDependencyError::InvalidVersion { .. }) + )); +} + +fn main() { + assert_source_aware_translation(); + assert_exact_lock_resolution(); + assert_requirement_rejections(); + assert_lock_rejections(); +} diff --git a/fixtures/native-lockfile-consumer/Cargo.toml b/fixtures/native-lockfile-consumer/Cargo.toml new file mode 100644 index 00000000..bded9269 --- /dev/null +++ b/fixtures/native-lockfile-consumer/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "zed-native-lockfile-consumer" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +toml = "1" +zed-interfaces = { path = "../../../zed-interfaces" } diff --git a/fixtures/native-lockfile-consumer/src/main.rs b/fixtures/native-lockfile-consumer/src/main.rs new file mode 100644 index 00000000..b5b09366 --- /dev/null +++ b/fixtures/native-lockfile-consumer/src/main.rs @@ -0,0 +1,151 @@ +use zed_interfaces::{ + ArtifactFormat, Lockfile, LockfileError, NativeArtifact, NativeDependencyLock, NativeRegistry, + NativeVersionCandidate, +}; + +fn artifact(digit: char) -> NativeArtifact { + NativeArtifact { + sha256: std::iter::repeat_n(digit, 64).collect(), + size: 1024, + format: ArtifactFormat::TarGz, + } +} + +fn exact_lock( + registry: NativeRegistry, + package: &str, + declared: &str, + version: &str, + digit: char, +) -> NativeDependencyLock { + NativeDependencyLock::resolve( + registry, + package, + declared, + &[NativeVersionCandidate { + version: version.to_string(), + artifact: artifact(digit), + }], + ) + .expect("fixture must resolve") +} + +fn assert_legacy_compatibility() { + let legacy = r#" +version = 1 + +[[package]] +org = "zedtest" +name = "core" +version = "1.0.0" +sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +size = 128 +format = "tar.gz" +vcs_tag = "v1.0.0" +vcs_commit = "0123456789abcdef0123456789abcdef01234567" +source = "file:///tmp/registry" +"#; + + let lockfile = Lockfile::parse(legacy).expect("legacy lockfile must parse"); + assert!(lockfile.native_dependencies.is_empty()); + assert!(!lockfile.to_toml_string().unwrap().contains("native-dependency")); +} + +fn assert_round_trip_and_ordering() { + let npm = exact_lock(NativeRegistry::Npm, "core", "^1.2.3", "1.9.0", 'a'); + let cargo = exact_lock(NativeRegistry::Cargo, "core", "1.2.3", "1.9.0", 'b'); + + let mut forward = Lockfile::default(); + forward.upsert_native_dependency(cargo.clone()).unwrap(); + forward.upsert_native_dependency(npm.clone()).unwrap(); + + let mut reverse = Lockfile::default(); + reverse.upsert_native_dependency(npm).unwrap(); + reverse.upsert_native_dependency(cargo).unwrap(); + + let forward_toml = forward.to_toml_string().unwrap(); + let reverse_toml = reverse.to_toml_string().unwrap(); + assert_eq!(forward_toml, reverse_toml); + assert_eq!(forward_toml.matches("[[native-dependency]]").count(), 2); + + let parsed = Lockfile::parse(&forward_toml).unwrap(); + assert_eq!(parsed, forward); + assert_eq!( + parsed + .find_native_dependency(NativeRegistry::Npm, "core") + .unwrap() + .artifact + .sha256, + "a".repeat(64) + ); + assert_eq!( + parsed + .find_native_dependency(NativeRegistry::Cargo, "core") + .unwrap() + .artifact + .sha256, + "b".repeat(64) + ); +} + +fn assert_upsert_replacement() { + let old = exact_lock(NativeRegistry::Npm, "core", "1.2.3", "1.2.3", 'a'); + let new = exact_lock(NativeRegistry::Npm, "core", "1.2.4", "1.2.4", 'b'); + + let mut lockfile = Lockfile::default(); + lockfile.upsert_native_dependency(old).unwrap(); + lockfile.upsert_native_dependency(new).unwrap(); + + assert_eq!(lockfile.native_dependencies.len(), 1); + assert_eq!( + lockfile + .find_native_dependency(NativeRegistry::Npm, "core") + .unwrap() + .package + .version, + "1.2.4" + ); +} + +fn assert_fail_closed_validation() { + let first = exact_lock(NativeRegistry::Npm, "core", "1.2.3", "1.2.3", 'a'); + let second = exact_lock(NativeRegistry::Npm, "core", "1.2.4", "1.2.4", 'b'); + let duplicate = Lockfile { + version: Lockfile::CURRENT_VERSION, + packages: Vec::new(), + native_dependencies: vec![first, second], + nix_adapters: Vec::new(), + }; + assert!(matches!( + duplicate.to_toml_string(), + Err(LockfileError::DuplicateNativeDependency(_)) + )); + let raw_duplicate = toml::to_string_pretty(&duplicate).unwrap(); + assert!(matches!( + Lockfile::parse(&raw_duplicate), + Err(LockfileError::DuplicateNativeDependency(_)) + )); + + let mut drift = exact_lock(NativeRegistry::Npm, "core", "^1.2.3", "1.9.0", 'c'); + drift.requirement.canonical = "^1.3.0".to_string(); + let mut lockfile = Lockfile::default(); + assert!(matches!( + lockfile.upsert_native_dependency(drift), + Err(LockfileError::InvalidNativeDependency(_)) + )); + + let mut invalid_artifact = exact_lock(NativeRegistry::Cargo, "core", "1.2.3", "1.9.0", 'd'); + invalid_artifact.artifact.sha256 = "0".repeat(64); + lockfile.native_dependencies.push(invalid_artifact); + assert!(matches!( + lockfile.to_toml_string(), + Err(LockfileError::InvalidNativeDependency(_)) + )); +} + +fn main() { + assert_legacy_compatibility(); + assert_round_trip_and_ordering(); + assert_upsert_replacement(); + assert_fail_closed_validation(); +} diff --git a/fixtures/native-registry-consumer/Cargo.toml b/fixtures/native-registry-consumer/Cargo.toml new file mode 100644 index 00000000..81b943c3 --- /dev/null +++ b/fixtures/native-registry-consumer/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "zed-native-registry-contract-consumer" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +zed-interfaces = { path = "../../../zed-interfaces" } diff --git a/fixtures/native-registry-consumer/src/main.rs b/fixtures/native-registry-consumer/src/main.rs new file mode 100644 index 00000000..8aa92fa8 --- /dev/null +++ b/fixtures/native-registry-consumer/src/main.rs @@ -0,0 +1,337 @@ +use zed_interfaces::{ + ArtifactFormat, NATIVE_REGISTRY_ADAPTER_SCHEMA_V1, NativeArtifact, NativePackageIdentity, + NativePlatform, NativePlatformPackage, NativePublication, NativePublicationKind, + NativeRegistry, NativeRegistryAdapterRecord, NativeRegistryError, ZedNativePackageIdentity, + native_versions_collide, semver_precedence_identity, +}; + +const VERSION: &str = "1.4.2"; +const WRAPPER: &str = "@fiducia/core"; +const LINUX_PACKAGE: &str = "@fiducia/core-linux-arm64-musl"; +const DARWIN_PACKAGE: &str = "@fiducia/core-darwin-arm64"; + +fn platform(os: &str, arch: &str, libc: Option<&str>) -> NativePlatform { + NativePlatform { + os: os.to_string(), + arch: arch.to_string(), + libc: libc.map(str::to_string), + } +} + +fn artifact(digit: char) -> NativeArtifact { + NativeArtifact { + sha256: std::iter::repeat_n(digit, 64).collect(), + size: 1024, + format: ArtifactFormat::TarGz, + } +} + +fn package(name: &str, kind: NativePublicationKind, digest_digit: char) -> NativePublication { + NativePublication { + package: NativePackageIdentity { + name: name.to_string(), + version: VERSION.to_string(), + }, + kind, + platform: None, + platform_packages: Vec::new(), + artifact: artifact(digest_digit), + } +} + +fn valid_npm_record() -> NativeRegistryAdapterRecord { + let linux_arm64_musl = platform("linux", "arm64", Some("musl")); + let darwin_arm64 = platform("darwin", "arm64", None); + + let mut wrapper = package(WRAPPER, NativePublicationKind::Meta, 'a'); + wrapper.platform_packages = vec![ + NativePlatformPackage { + platform: linux_arm64_musl.clone(), + package: LINUX_PACKAGE.to_string(), + }, + NativePlatformPackage { + platform: darwin_arm64.clone(), + package: DARWIN_PACKAGE.to_string(), + }, + ]; + + let mut linux = package(LINUX_PACKAGE, NativePublicationKind::Platform, 'b'); + linux.platform = Some(linux_arm64_musl); + + let mut darwin = package(DARWIN_PACKAGE, NativePublicationKind::Platform, 'c'); + darwin.platform = Some(darwin_arm64); + + NativeRegistryAdapterRecord { + schema: NATIVE_REGISTRY_ADAPTER_SCHEMA_V1.to_string(), + registry: NativeRegistry::Npm, + source: ZedNativePackageIdentity { + org: "fiducia".to_string(), + name: "core".to_string(), + version: VERSION.to_string(), + target: Some("node".to_string()), + }, + publications: vec![darwin, wrapper, linux], + } +} + +fn meta_mut(record: &mut NativeRegistryAdapterRecord) -> &mut NativePublication { + record + .publications + .iter_mut() + .find(|publication| publication.kind == NativePublicationKind::Meta) + .expect("fixture contains one meta publication") +} + +fn platform_mut<'a>( + record: &'a mut NativeRegistryAdapterRecord, + package_name: &str, +) -> &'a mut NativePublication { + record + .publications + .iter_mut() + .find(|publication| publication.package.name == package_name) + .expect("fixture contains requested platform publication") +} + +fn assert_positive_contract() { + let record = valid_npm_record(); + record.validate().expect("valid npm publication family"); + + let canonical = record + .canonical_json_bytes() + .expect("canonical native-registry adapter JSON"); + assert!(!canonical.is_empty()); + + let mut reordered = record.clone(); + reordered.publications.reverse(); + for publication in &mut reordered.publications { + publication.platform_packages.reverse(); + } + assert_eq!( + canonical, + reordered + .canonical_json_bytes() + .expect("presentation order must not affect canonical bytes") + ); + + assert_eq!( + semver_precedence_identity("1.4.2+linux-arm64").unwrap(), + VERSION + ); + assert!(native_versions_collide("1.4.2+linux", "1.4.2+darwin").unwrap()); + assert!(!native_versions_collide("1.4.2-rc.1", VERSION).unwrap()); + + let cargo_record = NativeRegistryAdapterRecord { + schema: NATIVE_REGISTRY_ADAPTER_SCHEMA_V1.to_string(), + registry: NativeRegistry::Cargo, + source: ZedNativePackageIdentity { + org: "fiducia".to_string(), + name: "core".to_string(), + version: VERSION.to_string(), + target: Some("rust".to_string()), + }, + publications: vec![package( + "fiducia_core", + NativePublicationKind::Portable, + 'd', + )], + }; + cargo_record + .validate() + .expect("portable Cargo publication must validate"); +} + +fn assert_identity_rejections() { + let mut unsupported_schema = valid_npm_record(); + unsupported_schema.schema = "zed.native-registry-adapter/v2".to_string(); + assert!(matches!( + unsupported_schema.validate(), + Err(NativeRegistryError::UnsupportedSchema { .. }) + )); + + let mut no_publications = valid_npm_record(); + no_publications.publications.clear(); + assert!(matches!( + no_publications.validate(), + Err(NativeRegistryError::NoPublications) + )); + + let mut invalid_semver = valid_npm_record(); + invalid_semver.source.version = "1.4".to_string(); + assert!(matches!( + invalid_semver.validate(), + Err(NativeRegistryError::InvalidSemver { .. }) + )); + + let mut build_metadata = valid_npm_record(); + build_metadata.source.version = "1.4.2+linux".to_string(); + for publication in &mut build_metadata.publications { + publication.package.version = "1.4.2+linux".to_string(); + } + assert!(matches!( + build_metadata.validate(), + Err(NativeRegistryError::BuildMetadataNotAllowed { .. }) + )); + + let mut version_drift = valid_npm_record(); + version_drift.publications[0].package.version = "1.4.3".to_string(); + assert!(matches!( + version_drift.validate(), + Err(NativeRegistryError::VersionDrift { .. }) + )); + + let mut invalid_npm_name = valid_npm_record(); + invalid_npm_name.publications[0].package.name = "@Fiducia/core".to_string(); + assert!(matches!( + invalid_npm_name.validate(), + Err(NativeRegistryError::InvalidPackageName { + registry: NativeRegistry::Npm, + .. + }) + )); + + let invalid_cargo_name = NativeRegistryAdapterRecord { + schema: NATIVE_REGISTRY_ADAPTER_SCHEMA_V1.to_string(), + registry: NativeRegistry::Cargo, + source: ZedNativePackageIdentity { + org: "fiducia".to_string(), + name: "core".to_string(), + version: VERSION.to_string(), + target: Some("rust".to_string()), + }, + publications: vec![package( + "bad/name", + NativePublicationKind::Portable, + 'd', + )], + }; + assert!(matches!( + invalid_cargo_name.validate(), + Err(NativeRegistryError::InvalidPackageName { + registry: NativeRegistry::Cargo, + .. + }) + )); +} + +fn assert_platform_rejections() { + let mut missing_selector = valid_npm_record(); + platform_mut(&mut missing_selector, LINUX_PACKAGE).platform = None; + assert!(matches!( + missing_selector.validate(), + Err(NativeRegistryError::PlatformRequired { .. }) + )); + + let mut unexpected_selector = valid_npm_record(); + meta_mut(&mut unexpected_selector).platform = Some(platform("linux", "x64", Some("gnu"))); + assert!(matches!( + unexpected_selector.validate(), + Err(NativeRegistryError::UnexpectedPlatform { .. }) + )); + + let mut invalid_platform_token = valid_npm_record(); + platform_mut(&mut invalid_platform_token, LINUX_PACKAGE) + .platform + .as_mut() + .expect("platform selector") + .arch = "ARM64".to_string(); + assert!(matches!( + invalid_platform_token.validate(), + Err(NativeRegistryError::InvalidPlatformToken { .. }) + )); + + let mut duplicate_meta_edge = valid_npm_record(); + let duplicate = meta_mut(&mut duplicate_meta_edge).platform_packages[0].clone(); + meta_mut(&mut duplicate_meta_edge) + .platform_packages + .push(duplicate); + assert!(matches!( + duplicate_meta_edge.validate(), + Err(NativeRegistryError::DuplicateMetaPlatform { .. }) + )); + + let mut mismatched_edge = valid_npm_record(); + meta_mut(&mut mismatched_edge).platform_packages[0].package = DARWIN_PACKAGE.to_string(); + assert!(matches!( + mismatched_edge.validate(), + Err(NativeRegistryError::PlatformPackageMismatch { .. }) + )); + + let mut dangling_edge = valid_npm_record(); + dangling_edge + .publications + .retain(|publication| publication.package.name != LINUX_PACKAGE); + assert!(matches!( + dangling_edge.validate(), + Err(NativeRegistryError::MissingPlatformPublication { .. }) + )); + + let mut duplicate_platform = valid_npm_record(); + let linux_platform = platform_mut(&mut duplicate_platform, LINUX_PACKAGE) + .platform + .clone(); + let mut extra = package( + "@fiducia/core-linux-arm64-alias", + NativePublicationKind::Platform, + 'd', + ); + extra.platform = linux_platform; + duplicate_platform.publications.push(extra); + assert!(matches!( + duplicate_platform.validate(), + Err(NativeRegistryError::DuplicatePlatformPublication { .. }) + )); + + let mut multiple_meta = valid_npm_record(); + multiple_meta.publications.push(package( + "@fiducia/core-wrapper", + NativePublicationKind::Meta, + 'd', + )); + assert!(matches!( + multiple_meta.validate(), + Err(NativeRegistryError::MultipleMetaPackages) + )); + + let mut platform_selects_platform = valid_npm_record(); + let selection = meta_mut(&mut platform_selects_platform).platform_packages[0].clone(); + platform_mut(&mut platform_selects_platform, LINUX_PACKAGE) + .platform_packages + .push(selection); + assert!(matches!( + platform_selects_platform.validate(), + Err(NativeRegistryError::PlatformPackagesNotAllowed { .. }) + )); +} + +fn assert_artifact_and_duplicate_rejections() { + let mut malformed_digest = valid_npm_record(); + malformed_digest.publications[0].artifact.sha256 = "A".repeat(64); + assert!(matches!( + malformed_digest.validate(), + Err(NativeRegistryError::InvalidSha256 { .. }) + )); + + let mut empty_artifact = valid_npm_record(); + empty_artifact.publications[0].artifact.size = 0; + assert!(matches!( + empty_artifact.validate(), + Err(NativeRegistryError::EmptyArtifact { .. }) + )); + + let mut duplicate_package = valid_npm_record(); + duplicate_package + .publications + .push(duplicate_package.publications[0].clone()); + assert!(matches!( + duplicate_package.validate(), + Err(NativeRegistryError::DuplicatePackageVersion { .. }) + )); +} + +fn main() { + assert_positive_contract(); + assert_identity_rejections(); + assert_platform_rejections(); + assert_artifact_and_duplicate_rejections(); +} diff --git a/fixtures/native-registry-consumer/tests/topology.rs b/fixtures/native-registry-consumer/tests/topology.rs new file mode 100644 index 00000000..868bb206 --- /dev/null +++ b/fixtures/native-registry-consumer/tests/topology.rs @@ -0,0 +1,137 @@ +use zed_interfaces::{ + ArtifactFormat, NATIVE_REGISTRY_ADAPTER_SCHEMA_V1, NativeArtifact, NativePackageIdentity, + NativePlatform, NativePlatformPackage, NativePublication, NativePublicationKind, + NativeRegistry, NativeRegistryAdapterRecord, NativeRegistryError, ZedNativePackageIdentity, +}; + +const VERSION: &str = "1.4.2"; +const WRAPPER: &str = "@fiducia/core"; +const LINUX_PACKAGE: &str = "@fiducia/core-linux-arm64-musl"; +const DARWIN_PACKAGE: &str = "@fiducia/core-darwin-arm64"; + +fn platform(os: &str, arch: &str, libc: Option<&str>) -> NativePlatform { + NativePlatform { + os: os.to_string(), + arch: arch.to_string(), + libc: libc.map(str::to_string), + } +} + +fn artifact(digit: char) -> NativeArtifact { + NativeArtifact { + sha256: std::iter::repeat_n(digit, 64).collect(), + size: 1024, + format: ArtifactFormat::TarGz, + } +} + +fn publication(name: &str, kind: NativePublicationKind, digit: char) -> NativePublication { + NativePublication { + package: NativePackageIdentity { + name: name.to_string(), + version: VERSION.to_string(), + }, + kind, + platform: None, + platform_packages: Vec::new(), + artifact: artifact(digit), + } +} + +fn record() -> NativeRegistryAdapterRecord { + let linux = platform("linux", "arm64", Some("musl")); + let darwin = platform("darwin", "arm64", None); + + let mut wrapper = publication(WRAPPER, NativePublicationKind::Meta, 'a'); + wrapper.platform_packages = vec![ + NativePlatformPackage { + platform: linux.clone(), + package: LINUX_PACKAGE.to_string(), + }, + NativePlatformPackage { + platform: darwin.clone(), + package: DARWIN_PACKAGE.to_string(), + }, + ]; + + let mut linux_package = publication(LINUX_PACKAGE, NativePublicationKind::Platform, 'b'); + linux_package.platform = Some(linux); + let mut darwin_package = publication(DARWIN_PACKAGE, NativePublicationKind::Platform, 'c'); + darwin_package.platform = Some(darwin); + + NativeRegistryAdapterRecord { + schema: NATIVE_REGISTRY_ADAPTER_SCHEMA_V1.to_string(), + registry: NativeRegistry::Npm, + source: ZedNativePackageIdentity { + org: "fiducia".to_string(), + name: "core".to_string(), + version: VERSION.to_string(), + target: Some("node".to_string()), + }, + publications: vec![wrapper, linux_package, darwin_package], + } +} + +fn meta(record: &mut NativeRegistryAdapterRecord) -> &mut NativePublication { + record + .publications + .iter_mut() + .find(|publication| publication.kind == NativePublicationKind::Meta) + .expect("fixture contains one meta package") +} + +#[test] +fn rejects_more_than_one_portable_publication() { + let mut candidate = record(); + candidate.publications.extend([ + publication( + "@fiducia/core-portable", + NativePublicationKind::Portable, + 'd', + ), + publication( + "@fiducia/core-portable-extra", + NativePublicationKind::Portable, + 'e', + ), + ]); + + assert!(matches!( + candidate.validate(), + Err(NativeRegistryError::MultiplePortablePackages) + )); +} + +#[test] +fn rejects_empty_meta_platform_selection() { + let mut candidate = record(); + meta(&mut candidate).platform_packages.clear(); + + assert!(matches!( + candidate.validate(), + Err(NativeRegistryError::MetaPackageRequiresPlatformSelections { .. }) + )); +} + +#[test] +fn rejects_unselected_platform_publication() { + let mut candidate = record(); + meta(&mut candidate).platform_packages.pop(); + + assert!(matches!( + candidate.validate(), + Err(NativeRegistryError::UnselectedPlatformPublication { .. }) + )); +} + +#[test] +fn accepts_platform_only_publication_family() { + let mut candidate = record(); + candidate + .publications + .retain(|publication| publication.kind == NativePublicationKind::Platform); + + candidate + .validate() + .expect("platform-only publication family remains valid"); +} diff --git a/fixtures/opto-sync-adoption/3fa/.opto-sync.json b/fixtures/opto-sync-adoption/3fa/.opto-sync.json new file mode 100644 index 00000000..f809b8a2 --- /dev/null +++ b/fixtures/opto-sync-adoption/3fa/.opto-sync.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": "opto-sync.adoption.v1", + "project": "3FA-app/3fa-clients", + "clientPackage": { + "package": "opto-sync/opto-sync-clients", + "repository": "https://github.com/opto-sync/opto-sync-clients", + "version": "0.2.0", + "commit": "068414c8ff7d4262d0a395959b5209d5908f0fcc" + }, + "languages": [ + "dart", + "gleam", + "rust", + "typescript" + ], + "dataScopes": [ + "deviceDirectory", + "encryptedMessageMetadata", + "keyBundleMetadata", + "syncCheckpoints" + ], + "localStores": [ + "indexeddb", + "sqlite" + ], + "transports": [ + "http", + "supabase-realtime", + "websocket" + ], + "writeStrategies": [ + "local-durable", + "local-then-remote", + "remote-confirmed" + ], + "conflictPolicy": { + "arrayStrategy": "mergeByKey", + "resolveByTimestamp": true, + "lwwKeys": [ + "updatedAt" + ], + "arrayMatchKeys": [ + "id" + ], + "tombstones": true + }, + "background": { + "webServiceWorker": true, + "mobileWorker": true, + "desktopWorker": true + }, + "security": { + "authenticatedTransport": true, + "encryptedAtRest": true, + "secretsInManifest": false + }, + "rollout": { + "phase": "declared", + "owners": [ + "3FA-app" + ] + }, + "notes": "Sync device, encrypted-message, key-bundle metadata, and causal checkpoints only. Private keys, OTP seeds, plaintext message content, passwords, and access or refresh tokens remain outside Opto Sync." +} diff --git a/fixtures/opto-sync-adoption/3fa/opto-sync-adoption.yml b/fixtures/opto-sync-adoption/3fa/opto-sync-adoption.yml new file mode 100644 index 00000000..30075c0d --- /dev/null +++ b/fixtures/opto-sync-adoption/3fa/opto-sync-adoption.yml @@ -0,0 +1,47 @@ +name: Opto Sync adoption + +on: + push: + branches: [main] + paths: ['.opto-sync.json', '.github/workflows/opto-sync-adoption.yml'] + pull_request: + paths: ['.opto-sync.json', '.github/workflows/opto-sync-adoption.yml'] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout 3FA clients + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - name: Checkout pinned Opto Sync validator + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: opto-sync/opto-sync-clients + ref: da7459fdf3ce4096ce6b88aeed5fc98066f804d9 + path: .opto-sync-tools + persist-credentials: false + - name: Validate manifest and secret boundary + run: | + set -euo pipefail + python3 .opto-sync-tools/scripts/check-adoption-manifest.py .opto-sync.json + python3 - <<'PY' + import json + manifest = json.load(open('.opto-sync.json', encoding='utf-8')) + assert manifest['project'] == '3FA-app/3fa-clients' + assert manifest['rollout']['phase'] == 'declared' + assert {'deviceDirectory', 'keyBundleMetadata', 'syncCheckpoints'} <= set(manifest['dataScopes']) + notes = manifest['notes'].lower() + for excluded in ('private keys', 'otp seeds', 'plaintext', 'access or refresh tokens'): + assert excluded in notes + PY diff --git a/fixtures/opto-sync-adoption/scintilla/.opto-sync.json b/fixtures/opto-sync-adoption/scintilla/.opto-sync.json new file mode 100644 index 00000000..8366532f --- /dev/null +++ b/fixtures/opto-sync-adoption/scintilla/.opto-sync.json @@ -0,0 +1,63 @@ +{ + "schemaVersion": "opto-sync.adoption.v1", + "project": "scintilla-run/scintilla-clients", + "clientPackage": { + "package": "opto-sync/opto-sync-clients", + "repository": "https://github.com/opto-sync/opto-sync-clients", + "version": "0.2.0", + "commit": "068414c8ff7d4262d0a395959b5209d5908f0fcc" + }, + "languages": [ + "dart", + "gleam", + "rust", + "typescript" + ], + "dataScopes": [ + "artifactIndex", + "experimentDefinitions", + "observationCheckpoints", + "runMetadata" + ], + "localStores": [ + "indexeddb", + "sqlite" + ], + "transports": [ + "http", + "websocket" + ], + "writeStrategies": [ + "local-durable", + "local-then-remote", + "remote-confirmed" + ], + "conflictPolicy": { + "arrayStrategy": "mergeByKey", + "resolveByTimestamp": true, + "lwwKeys": [ + "updatedAt" + ], + "arrayMatchKeys": [ + "id" + ], + "tombstones": true + }, + "background": { + "webServiceWorker": true, + "mobileWorker": true, + "desktopWorker": true + }, + "security": { + "authenticatedTransport": true, + "encryptedAtRest": true, + "secretsInManifest": false + }, + "rollout": { + "phase": "declared", + "owners": [ + "scintilla-run" + ] + }, + "notes": "Sync experiment definitions, run metadata, artifact indexes, observation checkpoints, and annotations. Artifact binaries, runner credentials, signing keys, and live high-volume telemetry streams remain outside Opto Sync." +} diff --git a/fixtures/opto-sync-adoption/scintilla/opto-sync-adoption.yml b/fixtures/opto-sync-adoption/scintilla/opto-sync-adoption.yml new file mode 100644 index 00000000..c746d8d3 --- /dev/null +++ b/fixtures/opto-sync-adoption/scintilla/opto-sync-adoption.yml @@ -0,0 +1,47 @@ +name: Opto Sync adoption + +on: + push: + branches: [main] + paths: ['.opto-sync.json', '.github/workflows/opto-sync-adoption.yml'] + pull_request: + paths: ['.opto-sync.json', '.github/workflows/opto-sync-adoption.yml'] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout Scintilla clients + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - name: Checkout pinned Opto Sync validator + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: opto-sync/opto-sync-clients + ref: da7459fdf3ce4096ce6b88aeed5fc98066f804d9 + path: .opto-sync-tools + persist-credentials: false + - name: Validate manifest and experiment boundary + run: | + set -euo pipefail + python3 .opto-sync-tools/scripts/check-adoption-manifest.py .opto-sync.json + python3 - <<'PY' + import json + manifest = json.load(open('.opto-sync.json', encoding='utf-8')) + assert manifest['project'] == 'scintilla-run/scintilla-clients' + assert manifest['rollout']['phase'] == 'declared' + assert {'experimentDefinitions', 'runMetadata', 'observationCheckpoints'} <= set(manifest['dataScopes']) + notes = manifest['notes'].lower() + for excluded in ('artifact binaries', 'runner credentials', 'signing keys', 'live high-volume telemetry streams'): + assert excluded in notes + PY diff --git a/fixtures/opto-sync-adoption/sonus-auris/.opto-sync.json b/fixtures/opto-sync-adoption/sonus-auris/.opto-sync.json new file mode 100644 index 00000000..dc80f025 --- /dev/null +++ b/fixtures/opto-sync-adoption/sonus-auris/.opto-sync.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": "opto-sync.adoption.v1", + "project": "sonus-auris/sonus-auris-clients", + "clientPackage": { + "package": "opto-sync/opto-sync-clients", + "repository": "https://github.com/opto-sync/opto-sync-clients", + "version": "0.2.0", + "commit": "068414c8ff7d4262d0a395959b5209d5908f0fcc" + }, + "languages": [ + "dart", + "gleam", + "rust", + "typescript" + ], + "dataScopes": [ + "recordingMetadata", + "retentionPolicies", + "schedule", + "syncCheckpoints" + ], + "localStores": [ + "indexeddb", + "sqlite" + ], + "transports": [ + "http", + "supabase-realtime", + "websocket" + ], + "writeStrategies": [ + "local-durable", + "local-then-remote", + "remote-confirmed" + ], + "conflictPolicy": { + "arrayStrategy": "mergeByKey", + "resolveByTimestamp": true, + "lwwKeys": [ + "updatedAt" + ], + "arrayMatchKeys": [ + "id" + ], + "tombstones": true + }, + "background": { + "webServiceWorker": true, + "mobileWorker": true, + "desktopWorker": true + }, + "security": { + "authenticatedTransport": true, + "encryptedAtRest": true, + "secretsInManifest": false + }, + "rollout": { + "phase": "declared", + "owners": [ + "sonus-auris" + ] + }, + "notes": "Sync recording metadata, schedules, retention policy, upload state, and causal checkpoints only. Raw audio, plaintext recordings, encryption keys, recovery material, device capabilities, and access or refresh tokens remain outside Opto Sync." +} diff --git a/fixtures/opto-sync-adoption/sonus-auris/opto-sync-adoption.yml b/fixtures/opto-sync-adoption/sonus-auris/opto-sync-adoption.yml new file mode 100644 index 00000000..16ba4805 --- /dev/null +++ b/fixtures/opto-sync-adoption/sonus-auris/opto-sync-adoption.yml @@ -0,0 +1,47 @@ +name: Opto Sync adoption + +on: + push: + branches: [main] + paths: ['.opto-sync.json', '.github/workflows/opto-sync-adoption.yml'] + pull_request: + paths: ['.opto-sync.json', '.github/workflows/opto-sync-adoption.yml'] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout Sonus Auris clients + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - name: Checkout pinned Opto Sync validator + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + repository: opto-sync/opto-sync-clients + ref: da7459fdf3ce4096ce6b88aeed5fc98066f804d9 + path: .opto-sync-tools + persist-credentials: false + - name: Validate manifest and audio-security boundary + run: | + set -euo pipefail + python3 .opto-sync-tools/scripts/check-adoption-manifest.py .opto-sync.json + python3 - <<'PY' + import json + manifest = json.load(open('.opto-sync.json', encoding='utf-8')) + assert manifest['project'] == 'sonus-auris/sonus-auris-clients' + assert manifest['rollout']['phase'] == 'declared' + assert {'recordingMetadata', 'retentionPolicies', 'schedule', 'syncCheckpoints'} <= set(manifest['dataScopes']) + notes = manifest['notes'].lower() + for excluded in ('raw audio', 'plaintext recordings', 'encryption keys', 'recovery material'): + assert excluded in notes + PY diff --git a/fixtures/org-repositories.json b/fixtures/org-repositories.json new file mode 100644 index 00000000..f60885cc --- /dev/null +++ b/fixtures/org-repositories.json @@ -0,0 +1,122 @@ +{ + "schema": "zed-pkg-test/repository-inventory-v1", + "organization": "zed-pkg-test", + "package_organizations": [ + "zed-pkg-test", + "zedtest", + "zed-pkg" + ], + "orchestrator": "zed-pkg-e2e", + "repositories": [ + { + "name": "node-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "node-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "rust-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "rust-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "go-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "go-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "python-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "python-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "dart-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "dart-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "gleam-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "gleam-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "polyglot-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "polyglot-node-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "polyglot-go-app", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "multiversion-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "awkward-lib", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "subtree-consumer", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "submodule-consumer", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "workspace-monorepo", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "shared-schema", + "kind": "package", + "manifest": ".zpkg.toml" + }, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": null + } + ] +} diff --git a/fixtures/version-interop-corpus/Cargo.toml b/fixtures/version-interop-corpus/Cargo.toml new file mode 100644 index 00000000..3355c3c7 --- /dev/null +++ b/fixtures/version-interop-corpus/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "zed-version-interop-corpus" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +zed-interfaces = { path = "../../../zed-interfaces" } diff --git a/fixtures/version-interop-corpus/cases.json b/fixtures/version-interop-corpus/cases.json new file mode 100644 index 00000000..740f19c9 --- /dev/null +++ b/fixtures/version-interop-corpus/cases.json @@ -0,0 +1,173 @@ +{ + "schema": "zed.version-interop-corpus/v1", + "interface_commit": "032b52f6e335e5696cb793d6a955c8f0658a95eb", + "precedence": [ + { + "id": "release-build-metadata", + "input": "1.4.2+linux-arm64", + "expected": "1.4.2" + }, + { + "id": "alternate-release-build-metadata", + "input": "1.4.2+darwin-arm64", + "expected": "1.4.2" + }, + { + "id": "prerelease-build-metadata", + "input": "1.4.2-rc.1+linux-arm64", + "expected": "1.4.2-rc.1" + } + ], + "collisions": [ + { + "id": "same-release-different-platform-metadata", + "left": "1.4.2+linux-arm64", + "right": "1.4.2+darwin-arm64", + "expected": true + }, + { + "id": "plain-and-build-metadata", + "left": "1.4.2", + "right": "1.4.2+build.7", + "expected": true + }, + { + "id": "prerelease-versus-release", + "left": "1.4.2-rc.1", + "right": "1.4.2", + "expected": false + }, + { + "id": "different-patch-versions", + "left": "1.4.3+linux-arm64", + "right": "1.4.2+darwin-arm64", + "expected": false + } + ], + "valid_records": [ + { + "id": "npm-multiplatform", + "kind": "npm-multiplatform" + }, + { + "id": "cargo-portable", + "kind": "cargo-portable" + } + ], + "invalid_records": [ + { + "id": "unsupported-schema", + "base": "npm-multiplatform", + "mutation": "unsupported-schema", + "expected_error": "UnsupportedSchema" + }, + { + "id": "empty-publication-set", + "base": "npm-multiplatform", + "mutation": "no-publications", + "expected_error": "NoPublications" + }, + { + "id": "invalid-source-semver", + "base": "npm-multiplatform", + "mutation": "invalid-source-semver", + "expected_error": "InvalidSemver" + }, + { + "id": "build-metadata-at-publication-boundary", + "base": "npm-multiplatform", + "mutation": "build-metadata", + "expected_error": "BuildMetadataNotAllowed" + }, + { + "id": "source-publication-version-drift", + "base": "npm-multiplatform", + "mutation": "version-drift", + "expected_error": "VersionDrift" + }, + { + "id": "invalid-npm-name", + "base": "npm-multiplatform", + "mutation": "invalid-npm-name", + "expected_error": "InvalidPackageName" + }, + { + "id": "invalid-cargo-name", + "base": "cargo-portable", + "mutation": "invalid-cargo-name", + "expected_error": "InvalidPackageName" + }, + { + "id": "platform-selector-required", + "base": "npm-multiplatform", + "mutation": "platform-required", + "expected_error": "PlatformRequired" + }, + { + "id": "meta-package-cannot-have-platform", + "base": "npm-multiplatform", + "mutation": "unexpected-platform", + "expected_error": "UnexpectedPlatform" + }, + { + "id": "uppercase-platform-token", + "base": "npm-multiplatform", + "mutation": "invalid-platform-token", + "expected_error": "InvalidPlatformToken" + }, + { + "id": "duplicate-meta-selector", + "base": "npm-multiplatform", + "mutation": "duplicate-meta-platform", + "expected_error": "DuplicateMetaPlatform" + }, + { + "id": "meta-selector-package-mismatch", + "base": "npm-multiplatform", + "mutation": "platform-package-mismatch", + "expected_error": "PlatformPackageMismatch" + }, + { + "id": "missing-platform-publication", + "base": "npm-multiplatform", + "mutation": "missing-platform-publication", + "expected_error": "MissingPlatformPublication" + }, + { + "id": "duplicate-platform-publication", + "base": "npm-multiplatform", + "mutation": "duplicate-platform-publication", + "expected_error": "DuplicatePlatformPublication" + }, + { + "id": "multiple-meta-packages", + "base": "npm-multiplatform", + "mutation": "multiple-meta-packages", + "expected_error": "MultipleMetaPackages" + }, + { + "id": "platform-package-selects-platform-package", + "base": "npm-multiplatform", + "mutation": "platform-packages-not-allowed", + "expected_error": "PlatformPackagesNotAllowed" + }, + { + "id": "uppercase-artifact-digest", + "base": "npm-multiplatform", + "mutation": "invalid-sha256", + "expected_error": "InvalidSha256" + }, + { + "id": "zero-byte-artifact", + "base": "npm-multiplatform", + "mutation": "empty-artifact", + "expected_error": "EmptyArtifact" + }, + { + "id": "duplicate-package-version-identity", + "base": "npm-multiplatform", + "mutation": "duplicate-package-version", + "expected_error": "DuplicatePackageVersion" + } + ] +} diff --git a/fixtures/version-interop-corpus/src/main.rs b/fixtures/version-interop-corpus/src/main.rs new file mode 100644 index 00000000..c11437d1 --- /dev/null +++ b/fixtures/version-interop-corpus/src/main.rs @@ -0,0 +1,413 @@ +use std::collections::BTreeSet; + +use serde::Deserialize; +use zed_interfaces::{ + ArtifactFormat, NATIVE_REGISTRY_ADAPTER_SCHEMA_V1, NativeArtifact, NativePackageIdentity, + NativePlatform, NativePlatformPackage, NativePublication, NativePublicationKind, + NativeRegistry, NativeRegistryAdapterRecord, NativeRegistryError, ZedNativePackageIdentity, + native_versions_collide, semver_precedence_identity, +}; + +const INTERFACE_COMMIT: &str = "032b52f6e335e5696cb793d6a955c8f0658a95eb"; +const VERSION: &str = "1.4.2"; +const WRAPPER: &str = "@fiducia/core"; +const LINUX_PACKAGE: &str = "@fiducia/core-linux-arm64-musl"; +const DARWIN_PACKAGE: &str = "@fiducia/core-darwin-arm64"; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Corpus { + schema: String, + interface_commit: String, + precedence: Vec, + collisions: Vec, + valid_records: Vec, + invalid_records: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PrecedenceCase { + id: String, + input: String, + expected: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct CollisionCase { + id: String, + left: String, + right: String, + expected: bool, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ValidRecordCase { + id: String, + kind: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct InvalidRecordCase { + id: String, + base: String, + mutation: String, + expected_error: String, +} + +fn platform(os: &str, arch: &str, libc: Option<&str>) -> NativePlatform { + NativePlatform { + os: os.to_string(), + arch: arch.to_string(), + libc: libc.map(str::to_string), + } +} + +fn artifact(digit: char) -> NativeArtifact { + NativeArtifact { + sha256: std::iter::repeat_n(digit, 64).collect(), + size: 1024, + format: ArtifactFormat::TarGz, + } +} + +fn publication( + name: &str, + kind: NativePublicationKind, + selected_platform: Option, + digest_digit: char, +) -> NativePublication { + NativePublication { + package: NativePackageIdentity { + name: name.to_string(), + version: VERSION.to_string(), + }, + kind, + platform: selected_platform, + platform_packages: Vec::new(), + artifact: artifact(digest_digit), + } +} + +fn npm_multiplatform_record() -> NativeRegistryAdapterRecord { + let linux = platform("linux", "arm64", Some("musl")); + let darwin = platform("darwin", "arm64", None); + + let mut wrapper = publication(WRAPPER, NativePublicationKind::Meta, None, 'a'); + wrapper.platform_packages = vec![ + NativePlatformPackage { + platform: linux.clone(), + package: LINUX_PACKAGE.to_string(), + }, + NativePlatformPackage { + platform: darwin.clone(), + package: DARWIN_PACKAGE.to_string(), + }, + ]; + + NativeRegistryAdapterRecord { + schema: NATIVE_REGISTRY_ADAPTER_SCHEMA_V1.to_string(), + registry: NativeRegistry::Npm, + source: ZedNativePackageIdentity { + org: "fiducia".to_string(), + name: "core".to_string(), + version: VERSION.to_string(), + target: Some("node".to_string()), + }, + publications: vec![ + publication( + LINUX_PACKAGE, + NativePublicationKind::Platform, + Some(linux), + 'b', + ), + wrapper, + publication( + DARWIN_PACKAGE, + NativePublicationKind::Platform, + Some(darwin), + 'c', + ), + ], + } +} + +fn cargo_portable_record() -> NativeRegistryAdapterRecord { + NativeRegistryAdapterRecord { + schema: NATIVE_REGISTRY_ADAPTER_SCHEMA_V1.to_string(), + registry: NativeRegistry::Cargo, + source: ZedNativePackageIdentity { + org: "fiducia".to_string(), + name: "core".to_string(), + version: VERSION.to_string(), + target: Some("rust".to_string()), + }, + publications: vec![publication( + "fiducia_core", + NativePublicationKind::Portable, + None, + 'd', + )], + } +} + +fn record_for_kind(kind: &str) -> NativeRegistryAdapterRecord { + match kind { + "npm-multiplatform" => npm_multiplatform_record(), + "cargo-portable" => cargo_portable_record(), + other => panic!("unsupported corpus record kind: {other}"), + } +} + +fn meta_mut(record: &mut NativeRegistryAdapterRecord) -> &mut NativePublication { + record + .publications + .iter_mut() + .find(|item| item.kind == NativePublicationKind::Meta) + .expect("npm corpus contains one meta publication") +} + +fn publication_mut<'a>( + record: &'a mut NativeRegistryAdapterRecord, + package_name: &str, +) -> &'a mut NativePublication { + record + .publications + .iter_mut() + .find(|item| item.package.name == package_name) + .unwrap_or_else(|| panic!("missing corpus publication: {package_name}")) +} + +fn apply_mutation(record: &mut NativeRegistryAdapterRecord, mutation: &str) { + match mutation { + "unsupported-schema" => { + record.schema = "zed.native-registry-adapter/v2".to_string(); + } + "no-publications" => record.publications.clear(), + "invalid-source-semver" => record.source.version = "1.4".to_string(), + "build-metadata" => { + record.source.version = "1.4.2+linux".to_string(); + for item in &mut record.publications { + item.package.version = "1.4.2+linux".to_string(); + } + } + "version-drift" => { + record.publications[0].package.version = "1.4.3".to_string(); + } + "invalid-npm-name" => { + record.publications[0].package.name = "@Fiducia/core".to_string(); + } + "invalid-cargo-name" => { + record.publications[0].package.name = "bad/name".to_string(); + } + "platform-required" => { + publication_mut(record, LINUX_PACKAGE).platform = None; + } + "unexpected-platform" => { + meta_mut(record).platform = Some(platform("linux", "x64", Some("gnu"))); + } + "invalid-platform-token" => { + publication_mut(record, LINUX_PACKAGE) + .platform + .as_mut() + .expect("platform selector") + .arch = "ARM64".to_string(); + } + "duplicate-meta-platform" => { + let duplicate = meta_mut(record).platform_packages[0].clone(); + meta_mut(record).platform_packages.push(duplicate); + } + "platform-package-mismatch" => { + meta_mut(record).platform_packages[0].package = DARWIN_PACKAGE.to_string(); + } + "missing-platform-publication" => { + record + .publications + .retain(|item| item.package.name != LINUX_PACKAGE); + } + "duplicate-platform-publication" => { + let duplicate_platform = publication_mut(record, LINUX_PACKAGE).platform.clone(); + record.publications.push(publication( + "@fiducia/core-linux-arm64-alias", + NativePublicationKind::Platform, + duplicate_platform, + 'e', + )); + } + "multiple-meta-packages" => { + record.publications.push(publication( + "@fiducia/core-wrapper", + NativePublicationKind::Meta, + None, + 'e', + )); + } + "platform-packages-not-allowed" => { + let selection = meta_mut(record).platform_packages[0].clone(); + publication_mut(record, LINUX_PACKAGE) + .platform_packages + .push(selection); + } + "invalid-sha256" => { + record.publications[0].artifact.sha256 = "A".repeat(64); + } + "empty-artifact" => { + record.publications[0].artifact.size = 0; + } + "duplicate-package-version" => { + record.publications.push(record.publications[0].clone()); + } + other => panic!("unsupported corpus mutation: {other}"), + } +} + +fn error_name(error: &NativeRegistryError) -> &'static str { + match error { + NativeRegistryError::UnsupportedSchema { .. } => "UnsupportedSchema", + NativeRegistryError::NoPublications => "NoPublications", + NativeRegistryError::InvalidPackageName { .. } => "InvalidPackageName", + NativeRegistryError::InvalidSemver { .. } => "InvalidSemver", + NativeRegistryError::BuildMetadataNotAllowed { .. } => "BuildMetadataNotAllowed", + NativeRegistryError::VersionDrift { .. } => "VersionDrift", + NativeRegistryError::InvalidIdentityComponent { .. } => "InvalidIdentityComponent", + NativeRegistryError::InvalidPlatformToken { .. } => "InvalidPlatformToken", + NativeRegistryError::InvalidSha256 { .. } => "InvalidSha256", + NativeRegistryError::EmptyArtifact { .. } => "EmptyArtifact", + NativeRegistryError::PlatformRequired { .. } => "PlatformRequired", + NativeRegistryError::UnexpectedPlatform { .. } => "UnexpectedPlatform", + NativeRegistryError::PlatformPackagesNotAllowed { .. } => "PlatformPackagesNotAllowed", + NativeRegistryError::DuplicateMetaPlatform { .. } => "DuplicateMetaPlatform", + NativeRegistryError::DuplicatePackageVersion { .. } => "DuplicatePackageVersion", + NativeRegistryError::MultipleMetaPackages => "MultipleMetaPackages", + NativeRegistryError::DuplicatePlatformPublication { .. } => "DuplicatePlatformPublication", + NativeRegistryError::PlatformPackageMismatch { .. } => "PlatformPackageMismatch", + NativeRegistryError::MissingPlatformPublication { .. } => "MissingPlatformPublication", + NativeRegistryError::Serialization(_) => "Serialization", + } +} + +fn assert_unique_ids<'a>(category: &str, ids: impl IntoIterator) { + let mut seen = BTreeSet::new(); + for id in ids { + assert!(seen.insert(id), "duplicate {category} case id: {id}"); + } +} + +fn validate_corpus_shape(corpus: &Corpus) { + assert_eq!(corpus.schema, "zed.version-interop-corpus/v1"); + assert_eq!(corpus.interface_commit, INTERFACE_COMMIT); + assert_eq!(corpus.precedence.len(), 3, "precedence corpus shrank"); + assert_eq!(corpus.collisions.len(), 4, "collision corpus shrank"); + assert_eq!( + corpus.valid_records.len(), + 2, + "positive record corpus shrank" + ); + assert_eq!( + corpus.invalid_records.len(), + 19, + "negative record corpus shrank" + ); + + assert_unique_ids( + "precedence", + corpus.precedence.iter().map(|case| case.id.as_str()), + ); + assert_unique_ids( + "collision", + corpus.collisions.iter().map(|case| case.id.as_str()), + ); + assert_unique_ids( + "valid record", + corpus.valid_records.iter().map(|case| case.id.as_str()), + ); + assert_unique_ids( + "invalid record", + corpus.invalid_records.iter().map(|case| case.id.as_str()), + ); +} + +fn run_precedence_cases(corpus: &Corpus) { + for case in &corpus.precedence { + let actual = semver_precedence_identity(&case.input) + .unwrap_or_else(|error| panic!("precedence case {} failed: {error}", case.id)); + assert_eq!(actual, case.expected, "precedence case {}", case.id); + } +} + +fn run_collision_cases(corpus: &Corpus) { + for case in &corpus.collisions { + let actual = native_versions_collide(&case.left, &case.right) + .unwrap_or_else(|error| panic!("collision case {} failed: {error}", case.id)); + assert_eq!(actual, case.expected, "collision case {}", case.id); + } +} + +fn run_valid_records(corpus: &Corpus) { + for case in &corpus.valid_records { + let record = record_for_kind(&case.kind); + record + .validate() + .unwrap_or_else(|error| panic!("valid record {} failed: {error}", case.id)); + let canonical = record + .canonical_json_bytes() + .unwrap_or_else(|error| panic!("valid record {} was not canonical: {error}", case.id)); + + let round_trip: NativeRegistryAdapterRecord = serde_json::from_slice(&canonical) + .unwrap_or_else(|error| panic!("valid record {} did not round trip: {error}", case.id)); + round_trip + .validate() + .unwrap_or_else(|error| panic!("round-tripped record {} failed: {error}", case.id)); + + let mut reordered = record.clone(); + reordered.publications.reverse(); + for publication in &mut reordered.publications { + publication.platform_packages.reverse(); + } + assert_eq!( + canonical, + reordered.canonical_json_bytes().unwrap(), + "canonical ordering drifted for {}", + case.id + ); + } +} + +fn run_invalid_records(corpus: &Corpus) { + for case in &corpus.invalid_records { + let mut record = record_for_kind(&case.base); + apply_mutation(&mut record, &case.mutation); + let error = match record.validate() { + Ok(()) => panic!("invalid record {} unexpectedly validated", case.id), + Err(error) => error, + }; + assert_eq!( + error_name(&error), + case.expected_error, + "invalid record {} returned {error}", + case.id + ); + } +} + +fn main() { + let corpus: Corpus = + serde_json::from_str(include_str!("../cases.json")).expect("corpus JSON must parse"); + validate_corpus_shape(&corpus); + run_precedence_cases(&corpus); + run_collision_cases(&corpus); + run_valid_records(&corpus); + run_invalid_records(&corpus); + + println!( + "certified {} precedence, {} collision, {} valid, and {} invalid cases against {}", + corpus.precedence.len(), + corpus.collisions.len(), + corpus.valid_records.len(), + corpus.invalid_records.len(), + corpus.interface_commit + ); +} diff --git a/harness/github_dependency_inventory.py b/harness/github_dependency_inventory.py new file mode 100644 index 00000000..b66099d5 --- /dev/null +++ b/harness/github_dependency_inventory.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Deterministic GitHub repository dependency inventory reference CLI. + +This is the first-certification implementation for Linear DEN-2957. It emits +repository/source inventory pinned to exact commits, never a universal current +resolution. The implementation uses only the Python standard library. +""" + +from __future__ import annotations + +from github_inventory_core import * # noqa: F401,F403 - public conformance facade +from github_inventory_builder import * # noqa: F401,F403 - public conformance facade +from github_inventory_graph import * # noqa: F401,F403 - public conformance facade + +def run_inventory( + *, + repositories: Sequence[str], + organizations: Sequence[str], + includes: Sequence[str], + limits: Limits, + transport: Transport, + token: str | None, + sleeper: Callable[[float], None] = time.sleep, +) -> dict[str, Any]: + normalized_repositories = sorted({normalize_repo(value) for value in repositories}) + normalized_organizations = sorted({normalize_org(value) for value in organizations}) + normalized_includes = normalize_includes(includes) + if not normalized_repositories and not normalized_organizations: + raise InputError("at least one --repo or --org is required") + budget = Budget(limits) + client = GitHubClient(transport, budget, token, sleeper=sleeper) + builder = InventoryBuilder( + client, + limits, + normalized_repositories, + normalized_organizations, + normalized_includes, + token, + ) + return builder.build() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="github_dependency_inventory.py", + description=( + "Emit a deterministic, provenance-rich GitHub repository dependency inventory. " + "Credentials are environment-only." + ), + ) + parser.add_argument("--repo", action="append", default=[], metavar="OWNER/NAME") + parser.add_argument("--org", action="append", default=[], metavar="OWNER") + parser.add_argument( + "--include", + action="append", + default=[], + metavar="KINDS", + help="comma-separated subset of zed,git-submodule,nix; default is all", + ) + parser.add_argument("--format", choices=("json", "dot", "mermaid"), default="json") + parser.add_argument("--output", type=Path) + parser.add_argument("--fixture", type=Path) + parser.add_argument( + "--api-base", + default=os.environ.get("ZED_PKG_GITHUB_API_BASE", DEFAULT_API_BASE), + help="GitHub API origin; plain HTTP is accepted only on loopback", + ) + parser.add_argument("--max-repositories", type=int, default=Limits.max_repositories) + parser.add_argument("--max-nodes", type=int, default=Limits.max_nodes) + parser.add_argument("--max-edges", type=int, default=Limits.max_edges) + parser.add_argument("--max-requests", type=int, default=Limits.max_requests) + parser.add_argument("--max-response-bytes", type=int, default=Limits.max_response_bytes) + parser.add_argument( + "--max-total-response-bytes", type=int, default=Limits.max_total_response_bytes + ) + parser.add_argument("--max-manifest-bytes", type=int, default=Limits.max_manifest_bytes) + parser.add_argument("--max-field-bytes", type=int, default=Limits.max_field_bytes) + parser.add_argument("--max-tree-entries", type=int, default=Limits.max_tree_entries) + parser.add_argument("--max-json-depth", type=int, default=Limits.max_json_depth) + parser.add_argument("--max-seconds", type=float, default=Limits.max_seconds) + parser.add_argument("--max-retries", type=int, default=Limits.max_retries) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + token = os.environ.get("ZED_PKG_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN") + try: + limits = Limits( + max_repositories=args.max_repositories, + max_nodes=args.max_nodes, + max_edges=args.max_edges, + max_requests=args.max_requests, + max_response_bytes=args.max_response_bytes, + max_total_response_bytes=args.max_total_response_bytes, + max_manifest_bytes=args.max_manifest_bytes, + max_field_bytes=args.max_field_bytes, + max_tree_entries=args.max_tree_entries, + max_json_depth=args.max_json_depth, + max_seconds=args.max_seconds, + max_retries=args.max_retries, + ) + if args.fixture: + transport: Transport = FixtureTransport(FixtureBackend.from_path(args.fixture)) + else: + allow_custom_token = os.environ.get( + "ZED_PKG_GITHUB_ALLOW_TOKEN_TO_API_BASE", "" + ).strip().lower() in {"1", "true", "yes", "on"} + transport = HttpTransport( + args.api_base, + allow_token_to_custom_origin=allow_custom_token, + ) + inventory = run_inventory( + repositories=args.repo, + organizations=args.org, + includes=args.include, + limits=limits, + transport=transport, + token=token, + ) + rendered = render_inventory(inventory, args.format) + if args.output: + write_atomic(args.output, rendered) + else: + sys.stdout.write(rendered) + return 1 if inventory["completeness"]["inventory"] == "partial" else 0 + except LimitError as error: + print(f"error: {redact_text(str(error), token)}", file=sys.stderr) + return 3 + except InventoryError as error: + print(f"error: {redact_text(str(error), token)}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/github_inventory_acquisition.py b/harness/github_inventory_acquisition.py new file mode 100644 index 00000000..f75aa8e1 --- /dev/null +++ b/harness/github_inventory_acquisition.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +"""Immutable GitHub repository selection and manifest acquisition for DEN-2957.""" + +from __future__ import annotations + +from github_inventory_core import * # noqa: F401,F403 +import github_inventory_core as _core +from github_inventory_zed_workspace import ( + WorkspaceMemberDeclaration, + workspace_member_declarations, +) + +_metadata_full_name = _core._metadata_full_name +_quote = _core._quote +_validate_repo_path = _core._validate_repo_path + + +class RepositoryAcquisitionMixin: + def _resolve_repositories( + self, + repositories: Sequence[str], + organizations: Sequence[str], + ) -> list[RepoInfo]: + selected: dict[str, RepoInfo] = {} + for repository in normalize_repositories(repositories): + owner, name = repository.split("/", 1) + metadata = self.client.get_json( + f"/repos/{_quote(owner)}/{_quote(name)}", + expected_type=dict, + ) + full_name = _metadata_full_name(metadata) + selected[full_name] = RepoInfo( + full_name=full_name, + private=bool(metadata.get("private", False)), + archived=bool(metadata.get("archived", False)), + default_branch=_bounded_text( + str(metadata.get("default_branch", "main")), + "default branch", + self.limits.max_field_bytes, + ), + ) + + for organization in normalize_organizations(organizations): + items = self.client.paginate(f"/orgs/{_quote(organization)}/repos?type=all&per_page=100") + for metadata in items: + full_name = _metadata_full_name(metadata) + if not full_name.startswith(organization + "/"): + raise ParseFailure( + f"organization listing for {organization} returned {full_name}" + ) + selected.setdefault( + full_name, + RepoInfo( + full_name=full_name, + private=bool(metadata.get("private", False)), + archived=bool(metadata.get("archived", False)), + default_branch=_bounded_text( + str(metadata.get("default_branch", "main")), + "default branch", + self.limits.max_field_bytes, + ), + ), + ) + if len(selected) > self.limits.max_repositories: + raise LimitError( + f"repository limit exceeded ({len(selected)} > {self.limits.max_repositories})" + ) + return [selected[key] for key in sorted(selected)] + + def _read_manifest_blob( + self, + *, + full_name: str, + commit_sha: str, + path: str, + entry: Mapping[str, Any], + ) -> SourceBlob: + _validate_repo_path(path) + if entry.get("type") != "blob": + raise ParseFailure(f"manifest path {path} is not a blob") + declared_size = entry.get("size") + if not isinstance(declared_size, int) or declared_size < 0: + raise ParseFailure(f"tree entry for {path} is missing a valid size") + if declared_size > self.limits.max_manifest_bytes: + raise LimitError( + f"manifest {path} exceeds {self.limits.max_manifest_bytes} bytes" + ) + blob_sha = entry.get("sha") + if not isinstance(blob_sha, str): + raise ParseFailure(f"manifest {path} is missing a blob SHA") + data = self.client.get_blob(full_name, blob_sha) + if len(data) != declared_size: + raise ParseFailure( + f"manifest {path} size mismatch: tree={declared_size} blob={len(data)}" + ) + if len(data) > self.limits.max_manifest_bytes: + raise LimitError( + f"manifest {path} exceeds {self.limits.max_manifest_bytes} bytes" + ) + return SourceBlob( + repository=full_name, + repository_commit=commit_sha, + kind=manifest_kind(path), + path=path, + blob_sha=blob_sha, + data=data, + ) + + def _expand_zed_workspace_manifests( + self, + *, + full_name: str, + commit_sha: str, + entries: Mapping[str, Mapping[str, Any]], + blobs: dict[str, SourceBlob], + ) -> tuple[WorkspaceMemberDeclaration, ...]: + root = blobs.get(".zpkg.toml") + if root is None: + return () + + queue: list[tuple[str, int]] = [(root.path, 0)] + scanned: set[str] = set() + declared_by: dict[str, str] = {} + declarations: list[WorkspaceMemberDeclaration] = [] + + while queue: + manifest_path, depth = queue.pop(0) + if manifest_path in scanned: + continue + scanned.add(manifest_path) + if depth > self.limits.max_json_depth: + raise LimitError( + "Zed workspace nesting exceeded " + f"{self.limits.max_json_depth} levels" + ) + blob = blobs[manifest_path] + for declaration in workspace_member_declarations( + blob, + max_field_bytes=self.limits.max_field_bytes, + ): + previous_parent = declared_by.get(declaration.member_manifest_path) + if previous_parent is not None: + raise ParseFailure( + "ambiguous workspace member manifest " + f"{declaration.member_manifest_path!r} is declared by " + f"{previous_parent!r} and {declaration.parent_manifest_path!r}" + ) + declared_by[declaration.member_manifest_path] = declaration.parent_manifest_path + declarations.append(declaration) + if len(declarations) > self.limits.max_tree_entries: + raise LimitError( + "Zed workspace member limit exceeded " + f"({len(declarations)} > {self.limits.max_tree_entries})" + ) + entry = entries.get(declaration.member_manifest_path) + if entry is None: + raise ParseFailure( + f"workspace member {declaration.member_dir!r} is missing " + f"{declaration.member_manifest_path} at commit {commit_sha}" + ) + if declaration.member_manifest_path not in blobs: + blobs[declaration.member_manifest_path] = self._read_manifest_blob( + full_name=full_name, + commit_sha=commit_sha, + path=declaration.member_manifest_path, + entry=entry, + ) + queue.append((declaration.member_manifest_path, depth + 1)) + + return tuple(declarations) + + def _scan_repository(self, repo: RepoInfo) -> None: + full_name = repo.full_name + repo_node = self._ensure_repo_node(full_name, scanned=True) + record: dict[str, Any] = { + "full_name": full_name, + "private": repo.private, + "archived": repo.archived, + "default_branch": repo.default_branch, + "status": "scanned", + "manifests": [], + "missing_manifests": [], + } + try: + owner, name = full_name.split("/", 1) + commit = self.client.get_json( + f"/repos/{_quote(owner)}/{_quote(name)}/commits/{_quote(repo.default_branch)}", + expected_type=dict, + ) + commit_sha = normalize_commit_sha(commit.get("sha"), "repository commit") + record["commit_sha"] = commit_sha + self.nodes[repo_node]["commit_sha"] = commit_sha + tree = self.client.get_json( + f"/repos/{_quote(owner)}/{_quote(name)}/git/trees/{commit_sha}?recursive=1", + expected_type=dict, + ) + if tree.get("truncated") is True: + raise ApiError(200, f"tree:{full_name}", code="github_tree_truncated") + tree_items = tree.get("tree") + if not isinstance(tree_items, list): + raise ParseFailure("recursive tree response is missing tree array") + if len(tree_items) > self.limits.max_tree_entries: + raise LimitError( + f"tree entry limit exceeded ({len(tree_items)} > {self.limits.max_tree_entries})" + ) + entries: dict[str, Mapping[str, Any]] = {} + for entry in tree_items: + if not isinstance(entry, dict): + raise ParseFailure("recursive tree entry must be an object") + path = entry.get("path") + if not isinstance(path, str): + raise ParseFailure("recursive tree entry is missing path") + _validate_repo_path(path) + if path in entries: + raise ParseFailure(f"duplicate recursive tree path {path}") + entries[path] = entry + + requested_paths = sorted( + { + path + for include in self.includes + for path in EXPECTED_MANIFESTS[include] + } + ) + blobs: dict[str, SourceBlob] = {} + for path in requested_paths: + entry = entries.get(path) + if entry is None: + record["missing_manifests"].append(path) + continue + blobs[path] = self._read_manifest_blob( + full_name=full_name, + commit_sha=commit_sha, + path=path, + entry=entry, + ) + + workspace_declarations: tuple[WorkspaceMemberDeclaration, ...] = () + if "zed" in self.includes: + workspace_declarations = self._expand_zed_workspace_manifests( + full_name=full_name, + commit_sha=commit_sha, + entries=entries, + blobs=blobs, + ) + + for path in sorted(blobs): + blob = blobs[path] + record["manifests"].append( + { + "kind": blob.kind, + "path": blob.path, + "blob_sha": blob.blob_sha, + "size": len(blob.data), + } + ) + if workspace_declarations: + record["zed_workspace_members"] = [ + { + "parent_manifest": item.parent_manifest_path, + "member_dir": item.member_dir, + "member_manifest": item.member_manifest_path, + } + for item in workspace_declarations + ] + + if ".zpkg.toml" in blobs: + self._parse_zed_manifests(blobs, workspace_declarations) + if ".zpkg.lock" in blobs: + self._parse_zed_lock(blobs[".zpkg.lock"]) + if ".gitmodules" in blobs: + self._parse_gitmodules(blobs[".gitmodules"], full_name) + if "flake.lock" in blobs: + self._parse_flake_lock(blobs["flake.lock"], full_name) + if "flake.nix" in blobs: + self._parse_flake_nix(blobs["flake.nix"], full_name) + if "nix/sources.json" in blobs: + self._parse_nix_sources(blobs["nix/sources.json"], full_name) + if "npins/sources.json" in blobs: + self._parse_npins(blobs["npins/sources.json"], full_name) + for path, entry in sorted(entries.items()): + if path == "Dockerfile" or path.endswith("/Dockerfile"): + self._parse_container_path(full_name, path, entry, commit_sha) + self.repo_records.append(record) + except Exception as error: # controlled per-repository failure + record["status"] = "failed" + record["failure_code"] = _error_code(error) + record["failure_message"] = redact_text(str(error), self.token) + self.repo_records.append(record) + self._failure(full_name, "repository-scan", error) + + +# Preserve the public builder contract while keeping the narrower implementation name. +InventoryAcquisitionMixin = RepositoryAcquisitionMixin diff --git a/harness/github_inventory_builder.py b/harness/github_inventory_builder.py new file mode 100644 index 00000000..a6790c51 --- /dev/null +++ b/harness/github_inventory_builder.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Repository/source acquisition and typed inventory construction for DEN-2957.""" + +from __future__ import annotations + +from github_inventory_core import * # noqa: F401,F403 - internal reference modules share one contract +import github_inventory_core as _core +from github_inventory_graph import analyze_graph + +_bounded_text = _core._bounded_text +_edge_sort_key = _core._edge_sort_key +_error_code = _core._error_code +_find_json_line = _core._find_json_line +_find_toml_line = _core._find_toml_line +_json_pointer = _core._json_pointer +_metadata_full_name = _core._metadata_full_name +_provenance_sort_key = _core._provenance_sort_key +_utf8 = _core._utf8 +_validate_repo_path = _core._validate_repo_path +_walk_json = _core._walk_json + +from github_inventory_parsers import InventoryParsingMixin + +from github_inventory_acquisition import InventoryAcquisitionMixin +from github_inventory_mutation import InventoryMutationMixin + + +def _pin_sort_key(pin: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + pin.get("repository", ""), + pin.get("package", ""), + pin.get("selected_version", ""), + pin.get("artifact_sha256", ""), + pin.get("vcs_commit", ""), + json.dumps(pin.get("provenance", []), sort_keys=True, separators=(",", ":")), + ) + + +class InventoryBuilder( + InventoryAcquisitionMixin, InventoryParsingMixin, InventoryMutationMixin +): + def __init__( + self, + client: GitHubClient, + limits: Limits, + repositories: Sequence[str], + organizations: Sequence[str], + includes: Sequence[str], + token: str | None, + ) -> None: + self.client = client + self.limits = limits + self.requested_repositories = tuple(sorted(set(repositories))) + self.requested_organizations = tuple(sorted(set(organizations))) + self.includes = tuple(sorted(set(includes))) + self.token = token + self.repository_records: dict[str, dict[str, Any]] = {} + self.nodes: dict[str, dict[str, Any]] = {} + self.edges: dict[tuple[Any, ...], dict[str, Any]] = {} + self.pins: list[dict[str, Any]] = [] + self._pin_keys: set[tuple[Any, ...]] = set() + self.failures: list[dict[str, Any]] = [] + self.contradictions: list[dict[str, Any]] = [] + self.package_roots: dict[str, str] = {} + + def build(self) -> dict[str, Any]: + metadata = self._discover_repositories() + if len(metadata) > self.limits.max_repositories: + raise LimitError( + f"repository limit exceeded ({len(metadata)} > {self.limits.max_repositories})" + ) + for full_name in sorted(metadata): + self.client.budget.check_time() + self._scan_repository(full_name, metadata[full_name]) + self._check_graph_limits() + analysis = analyze_graph(self.nodes, self.edges.values()) + inventory = { + "schema": INVENTORY_SCHEMA, + "completeness": { + "inventory": "partial" if self.failures else "complete", + "resolution": "not-claimed", + }, + "inputs": { + "repositories": list(self.requested_repositories), + "organizations": list(self.requested_organizations), + "includes": list(self.includes), + }, + "limits": self.limits.as_dict(), + "usage": { + "requests": self.client.budget.requests, + "response_bytes": self.client.budget.response_bytes, + }, + "repositories": sorted( + self.repository_records.values(), key=lambda item: item["full_name"] + ), + "nodes": sorted(self.nodes.values(), key=lambda item: item["id"]), + "edges": sorted(self.edges.values(), key=_edge_sort_key), + "pins": sorted(self.pins, key=_pin_sort_key), + "strongly_connected_components": analysis["components"], + "cycles": analysis["cycles"], + "topological_waves": analysis["waves"], + "contradictions": sorted( + self.contradictions, + key=lambda item: ( + item.get("code", ""), + item.get("source", ""), + item.get("target", ""), + json.dumps(item, sort_keys=True, separators=(",", ":")), + ), + ), + "failures": sorted( + self.failures, + key=lambda item: ( + item.get("repository", ""), + item.get("stage", ""), + item.get("path", ""), + item.get("code", ""), + ), + ), + } + return inventory diff --git a/harness/github_inventory_client.py b/harness/github_inventory_client.py new file mode 100644 index 00000000..cd22d5b5 --- /dev/null +++ b/harness/github_inventory_client.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Bounded GitHub API client and retry policy for DEN-2957.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any, Callable, Mapping + +from github_inventory_support import * # noqa: F401,F403 +from github_inventory_fixture import FixtureBackend +import github_inventory_support as _support + +_bounded_retry_after = _support._bounded_retry_after +_is_transient_response = _support._is_transient_response +_metadata_full_name = _support._metadata_full_name +_next_link = _support._next_link +_quote = _support._quote + + +class FixtureTransport(Transport): + def __init__(self, backend: FixtureBackend) -> None: + self.backend = backend + self.base_url = backend.base_url + self._base_path = urllib.parse.urlsplit(self.base_url).path.rstrip("/") + self.requests: list[str] = [] + + def request( + self, + path: str, + headers: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> ApiResponse: + del timeout, max_bytes + relative = self.normalize_request_path(path) + self.requests.append(relative) + return self.backend.handle(relative, headers.get("Authorization")) + + +class GitHubClient: + def __init__( + self, + transport: Transport, + budget: Budget, + token: str | None, + sleeper: Callable[[float], None] = time.sleep, + ) -> None: + self.transport = transport + self.budget = budget + if token and not transport.token_destination_allowed(): + raise InputError( + "refusing to send a GitHub token to a custom API origin without " + "ZED_PKG_GITHUB_ALLOW_TOKEN_TO_API_BASE=1" + ) + self.token = token + self.sleeper = sleeper + + def list_org_repositories(self, org: str) -> list[dict[str, Any]]: + encoded = urllib.parse.quote(org, safe="") + path = ( + f"/orgs/{encoded}/repos?direction=asc&page=1&per_page=100&sort=full_name&type=all" + ) + repositories: list[dict[str, Any]] = [] + seen_pages: set[str] = set() + while path: + if path in seen_pages: + raise ApiError(400, path, "pagination_cycle") + seen_pages.add(path) + current_path = path + response, value = self._get_json(current_path) + if not isinstance(value, list): + raise ApiError(502, current_path, "invalid_github_json_shape") + for item in value: + if not isinstance(item, dict): + raise ApiError(502, current_path, "invalid_github_json_shape") + repositories.append(item) + next_link = _next_link(response.header("Link")) + path = ( + self.transport.relative_from_link(next_link, current_path=current_path) + if next_link + else "" + ) + return repositories + + def get_repository(self, full_name: str) -> dict[str, Any]: + owner, name = full_name.split("/", 1) + path = f"/repos/{_quote(owner)}/{_quote(name)}" + _, value = self._get_json(path) + if not isinstance(value, dict): + raise ApiError(502, path, "invalid_github_json_shape") + return value + + def pin_default_branch(self, full_name: str, default_branch: str) -> str: + owner, name = full_name.split("/", 1) + path = f"/repos/{_quote(owner)}/{_quote(name)}/commits/{_quote(default_branch)}" + _, value = self._get_json(path) + if not isinstance(value, dict) or not isinstance(value.get("sha"), str): + raise ApiError(502, path, "invalid_github_json_shape") + sha = value["sha"].lower() + if not GIT_SHA_RE.fullmatch(sha): + raise ApiError(502, path, "invalid_git_sha") + return sha + + def get_tree(self, full_name: str, commit_sha: str) -> list[dict[str, Any]]: + owner, name = full_name.split("/", 1) + path = f"/repos/{_quote(owner)}/{_quote(name)}/git/trees/{commit_sha}?recursive=1" + _, value = self._get_json(path) + if not isinstance(value, dict) or not isinstance(value.get("tree"), list): + raise ApiError(502, path, "invalid_github_json_shape") + if value.get("truncated") is True: + raise ApiError(422, path, "github_tree_truncated") + tree = value["tree"] + if len(tree) > self.budget.limits.max_tree_entries: + raise LimitError( + f"repository tree exceeded {self.budget.limits.max_tree_entries} entries" + ) + result: list[dict[str, Any]] = [] + for entry in tree: + if not isinstance(entry, dict): + raise ApiError(502, path, "invalid_github_json_shape") + result.append(entry) + return result + + def get_blob(self, full_name: str, blob_sha: str, declared_size: int | None) -> bytes: + owner, name = full_name.split("/", 1) + path = f"/repos/{_quote(owner)}/{_quote(name)}/git/blobs/{blob_sha}" + _, value = self._get_json(path) + if not isinstance(value, dict): + raise ApiError(502, path, "invalid_github_json_shape") + if value.get("encoding") != "base64" or not isinstance(value.get("content"), str): + raise ApiError(502, path, "unsupported_blob_encoding") + try: + encoded = "".join(value["content"].split()) + data = base64.b64decode(encoded, validate=True) + except (ValueError, TypeError) as error: + raise ApiError(502, path, "invalid_blob_base64") from error + + response_size = value.get("size") + if not isinstance(response_size, int) or response_size < 0 or len(data) != response_size: + raise ApiError(502, path, "blob_size_mismatch") + if declared_size is not None and len(data) != declared_size: + raise ApiError(502, path, "blob_size_mismatch") + if len(data) > self.budget.limits.max_manifest_bytes: + raise LimitError( + f"manifest blob exceeded {self.budget.limits.max_manifest_bytes} bytes" + ) + + response_sha = value.get("sha") + if not isinstance(response_sha, str): + raise ApiError(502, path, "blob_sha_mismatch") + response_sha = response_sha.lower() + if response_sha != blob_sha or not GIT_SHA_RE.fullmatch(response_sha): + raise ApiError(502, path, "blob_sha_mismatch") + + header = f"blob {len(data)}\0".encode("ascii") + if len(blob_sha) == 40: + computed_sha = hashlib.sha1(header + data, usedforsecurity=False).hexdigest() + elif len(blob_sha) == 64: + computed_sha = hashlib.sha256(header + data).hexdigest() + else: + raise ApiError(502, path, "invalid_git_sha") + if computed_sha != blob_sha: + raise ApiError(502, path, "blob_object_id_mismatch") + return data + + def _get_json(self, path: str) -> tuple[ApiResponse, Any]: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "zed-pkg-den-2957-conformance/1", + "X-GitHub-Api-Version": "2022-11-28", + } + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + attempts = self.budget.limits.max_retries + 1 + response: ApiResponse | None = None + for attempt in range(attempts): + self.budget.begin_request() + timeout = min(30.0, self.budget.remaining_seconds()) + response = self.transport.request( + path, + headers, + timeout, + self.budget.limits.max_response_bytes, + ) + self.budget.consume_response(len(response.body)) + if not _is_transient_response(response) or attempt + 1 >= attempts: + break + retry_after = _bounded_retry_after(response.header("Retry-After")) + self.sleeper(retry_after) + assert response is not None + if response.status != 200: + code = ( + "github_rate_limited" + if response.status in {429, 403} + and response.header("X-RateLimit-Remaining") == "0" + else "github_http_error" + ) + raise ApiError(response.status, path, code) + try: + value = json.loads(response.body.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as error: + raise ApiError(502, path, "invalid_github_json") from error + return response, value diff --git a/harness/github_inventory_core.py b/harness/github_inventory_core.py new file mode 100644 index 00000000..de0e84a5 --- /dev/null +++ b/harness/github_inventory_core.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Public internal facade for the DEN-2957 conformance implementation.""" + +from github_inventory_support import * # noqa: F401,F403 +from github_inventory_fixture import * # noqa: F401,F403 +from github_inventory_client import * # noqa: F401,F403 +import github_inventory_support as _support + +_bounded_retry_after = _support._bounded_retry_after +_bounded_text = _support._bounded_text +_edge_sort_key = _support._edge_sort_key +_error_code = _support._error_code +_find_json_line = _support._find_json_line +_find_toml_line = _support._find_toml_line +_fixture_bytes = _support._fixture_bytes +_fsync_directory = _support._fsync_directory +_is_loopback_host = _support._is_loopback_host +_is_transient_response = _support._is_transient_response +_json_pointer = _support._json_pointer +_json_response = _support._json_response +_metadata_full_name = _support._metadata_full_name +_next_link = _support._next_link +_positive_int = _support._positive_int +_provenance_sort_key = _support._provenance_sort_key +_query_positive_int = _support._query_positive_int +_quote = _support._quote +_utf8 = _support._utf8 +_valid_bearer_header = _support._valid_bearer_header +_validate_repo_path = _support._validate_repo_path +_walk_json = _support._walk_json diff --git a/harness/github_inventory_fixture.py b/harness/github_inventory_fixture.py new file mode 100644 index 00000000..4f0424d2 --- /dev/null +++ b/harness/github_inventory_fixture.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Compact fake-GitHub fixture backend for DEN-2957.""" + +from __future__ import annotations + +import base64 +import json +import urllib.parse +from pathlib import Path +from typing import Any, Mapping + +from github_inventory_support import * # noqa: F401,F403 +import github_inventory_support as _support + +_fixture_bytes = _support._fixture_bytes +_json_response = _support._json_response +_positive_int = _support._positive_int +_query_positive_int = _support._query_positive_int +_quote = _support._quote +_valid_bearer_header = _support._valid_bearer_header +_validate_repo_path = _support._validate_repo_path + +class FixtureBackend: + """Synthesizes bounded GitHub REST responses from a compact checked-in fixture.""" + + def __init__(self, document: Mapping[str, Any], base_url: str = "https://api.github.test") -> None: + if document.get("schema") != FIXTURE_SCHEMA: + raise InputError(f"fixture schema must be {FIXTURE_SCHEMA}") + self.document = json.loads(json.dumps(document)) + self.base_url = base_url.rstrip("/") + self.page_size = _positive_int(document.get("page_size", 100), "fixture page_size") + self.require_token = bool(document.get("require_token", False)) + organizations = document.get("organizations", {}) + repositories = document.get("repositories", {}) + if not isinstance(organizations, dict) or not isinstance(repositories, dict): + raise InputError("fixture organizations and repositories must be objects") + self.organizations: dict[str, list[str]] = {} + for org, values in organizations.items(): + normalized_org = normalize_org(str(org)) + if not isinstance(values, list): + raise InputError(f"fixture organization {org!r} must be a list") + self.organizations[normalized_org] = [normalize_repo(str(value)) for value in values] + self.repositories: dict[str, dict[str, Any]] = {} + for full_name, value in repositories.items(): + normalized = normalize_repo(str(full_name)) + if not isinstance(value, dict): + raise InputError(f"fixture repository {full_name!r} must be an object") + repository = dict(value) + repository.setdefault("default_branch", "main") + repository.setdefault("archived", False) + repository.setdefault("private", False) + repository.setdefault("disabled", False) + repository.setdefault("files", {}) + repository.setdefault("gitlinks", {}) + repository.setdefault("failures", {}) + sha = str(repository.get("commit_sha", "")) + if not GIT_SHA_RE.fullmatch(sha): + raise InputError(f"fixture repository {normalized} has invalid commit_sha") + if not isinstance(repository["files"], dict) or not isinstance(repository["gitlinks"], dict): + raise InputError(f"fixture repository {normalized} files/gitlinks must be objects") + self.repositories[normalized] = repository + self._blob_index: dict[tuple[str, str], bytes] = {} + for full_name, repository in self.repositories.items(): + for path, content in repository["files"].items(): + _validate_repo_path(str(path)) + data = _fixture_bytes(content) + self._blob_index[(full_name, git_blob_sha(data))] = data + for path, sha in repository["gitlinks"].items(): + _validate_repo_path(str(path)) + if not GIT_SHA_RE.fullmatch(str(sha)): + raise InputError(f"fixture gitlink {full_name}:{path} has invalid SHA") + + @classmethod + def from_path(cls, path: Path) -> "FixtureBackend": + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise InputError(f"could not read fixture {path}") from error + if not isinstance(document, dict): + raise InputError("fixture root must be an object") + return cls(document) + + def handle(self, raw_path: str, authorization: str | None) -> ApiResponse: + if self.require_token and not _valid_bearer_header(authorization): + return _json_response(401, {"message": "fixture authorization required"}) + + parsed = urllib.parse.urlsplit(raw_path) + query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True) + path = parsed.path + + org_match = re.fullmatch(r"/orgs/([^/]+)/repos", path) + if org_match: + org = normalize_org(urllib.parse.unquote(org_match.group(1))) + if org not in self.organizations: + return _json_response(404, {"message": "organization not found"}) + page = _query_positive_int(query, "page", 1) + requested_page_size = _query_positive_int(query, "per_page", 100) + page_size = min(requested_page_size, self.page_size) + values = self.organizations[org] + start = (page - 1) * page_size + selected = values[start : start + page_size] + body = [self._metadata(full_name) for full_name in selected] + headers: dict[str, str] = {} + if start + page_size < len(values): + next_query = dict(query) + next_query["page"] = [str(page + 1)] + next_query["per_page"] = [str(requested_page_size)] + encoded = urllib.parse.urlencode( + [(key, value) for key in sorted(next_query) for value in next_query[key]] + ) + next_url = f"{self.base_url}{path}?{encoded}" + headers["Link"] = f'<{next_url}>; rel="next"' + return _json_response(200, body, headers) + + repo_match = re.fullmatch(r"/repos/([^/]+)/([^/]+)", path) + if repo_match: + full_name = normalize_repo( + f"{urllib.parse.unquote(repo_match.group(1))}/{urllib.parse.unquote(repo_match.group(2))}" + ) + failure = self._failure(full_name, "metadata") + if failure: + return failure + if full_name not in self.repositories: + return _json_response(404, {"message": "repository not found"}) + return _json_response(200, self._metadata(full_name)) + + commit_match = re.fullmatch(r"/repos/([^/]+)/([^/]+)/commits/(.+)", path) + if commit_match: + full_name = normalize_repo( + f"{urllib.parse.unquote(commit_match.group(1))}/{urllib.parse.unquote(commit_match.group(2))}" + ) + failure = self._failure(full_name, "commit") + if failure: + return failure + repository = self.repositories.get(full_name) + if repository is None: + return _json_response(404, {"message": "repository not found"}) + ref = urllib.parse.unquote(commit_match.group(3)) + if ref != repository["default_branch"] and ref != repository["commit_sha"]: + return _json_response(404, {"message": "commit not found"}) + return _json_response(200, {"sha": repository["commit_sha"]}) + + tree_match = re.fullmatch(r"/repos/([^/]+)/([^/]+)/git/trees/([0-9a-f]+)", path) + if tree_match: + full_name = normalize_repo( + f"{urllib.parse.unquote(tree_match.group(1))}/{urllib.parse.unquote(tree_match.group(2))}" + ) + failure = self._failure(full_name, "tree") + if failure: + return failure + repository = self.repositories.get(full_name) + if repository is None or tree_match.group(3) != repository["commit_sha"]: + return _json_response(404, {"message": "tree not found"}) + entries: list[dict[str, Any]] = [] + for file_path, content in sorted(repository["files"].items()): + data = _fixture_bytes(content) + entries.append( + { + "path": file_path, + "mode": "100644", + "type": "blob", + "sha": git_blob_sha(data), + "size": len(data), + } + ) + for link_path, sha in sorted(repository["gitlinks"].items()): + entries.append( + { + "path": link_path, + "mode": "160000", + "type": "commit", + "sha": sha, + } + ) + return _json_response( + 200, + { + "sha": repository["commit_sha"], + "truncated": bool(repository.get("tree_truncated", False)), + "tree": entries, + }, + ) + + blob_match = re.fullmatch(r"/repos/([^/]+)/([^/]+)/git/blobs/([0-9a-f]+)", path) + if blob_match: + full_name = normalize_repo( + f"{urllib.parse.unquote(blob_match.group(1))}/{urllib.parse.unquote(blob_match.group(2))}" + ) + failure = self._failure(full_name, "blob") + if failure: + return failure + sha = blob_match.group(3) + data = self._blob_index.get((full_name, sha)) + if data is None: + return _json_response(404, {"message": "blob not found"}) + return _json_response( + 200, + { + "sha": sha, + "size": len(data), + "encoding": "base64", + "content": base64.b64encode(data).decode("ascii"), + }, + ) + + return _json_response(404, {"message": "fixture route not found"}) + + def _metadata(self, full_name: str) -> dict[str, Any]: + repository = self.repositories.get(full_name) + if repository is None: + owner, name = full_name.split("/", 1) + return { + "full_name": full_name, + "name": name, + "owner": {"login": owner}, + "default_branch": "main", + "archived": False, + "private": False, + "disabled": False, + } + owner, name = full_name.split("/", 1) + return { + "full_name": full_name, + "name": name, + "owner": {"login": owner}, + "default_branch": repository["default_branch"], + "archived": bool(repository["archived"]), + "private": bool(repository["private"]), + "disabled": bool(repository["disabled"]), + } + + def _failure(self, full_name: str, stage: str) -> ApiResponse | None: + repository = self.repositories.get(full_name) + if repository is None: + return None + failures = repository.get("failures", {}) + value = failures.get(stage) if isinstance(failures, dict) else None + if value is None: + return None + if isinstance(value, int): + return _json_response(value, {"message": "synthetic failure"}) + if not isinstance(value, dict): + raise InputError(f"fixture failure {full_name}:{stage} must be an integer or object") + status = int(value.get("status", 500)) + body = value.get("body", {"message": "synthetic failure"}) + return _json_response(status, body) diff --git a/harness/github_inventory_graph.py b/harness/github_inventory_graph.py new file mode 100644 index 00000000..39566638 --- /dev/null +++ b/harness/github_inventory_graph.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Deterministic SCC analysis, renderers, and atomic publication for DEN-2957.""" + +from __future__ import annotations + +from github_inventory_core import * # noqa: F401,F403 - internal reference modules share one contract +import github_inventory_core as _core + +_fsync_directory = _core._fsync_directory + + +def analyze_graph( + nodes: Mapping[str, Mapping[str, Any]], + edges: Iterable[Mapping[str, Any]], +) -> dict[str, Any]: + """Return deterministic SCCs and dependency-first condensation waves. + + Kosaraju's algorithm is implemented iteratively so adversarial chains up to + the configured graph limit cannot exhaust Python's call stack. Callers pass + only proven topology; lock-pin evidence is deliberately excluded. + """ + + adjacency: dict[str, set[str]] = {node_id: set() for node_id in nodes} + reverse: dict[str, set[str]] = {node_id: set() for node_id in nodes} + self_loops: set[str] = set() + for edge in edges: + source = str(edge["source"]) + target = str(edge["target"]) + adjacency.setdefault(source, set()).add(target) + adjacency.setdefault(target, set()) + reverse.setdefault(target, set()).add(source) + reverse.setdefault(source, set()) + if source == target: + self_loops.add(source) + + visited: set[str] = set() + finish_order: list[str] = [] + for start_node in sorted(adjacency): + if start_node in visited: + continue + visited.add(start_node) + stack: list[tuple[str, Any]] = [ + (start_node, iter(sorted(adjacency[start_node]))) + ] + while stack: + node, iterator = stack[-1] + try: + target = next(iterator) + except StopIteration: + finish_order.append(node) + stack.pop() + continue + if target in visited: + continue + visited.add(target) + stack.append((target, iter(sorted(adjacency[target])))) + + assigned: set[str] = set() + raw_components: list[tuple[str, ...]] = [] + for start_node in reversed(finish_order): + if start_node in assigned: + continue + assigned.add(start_node) + stack = [start_node] + members: list[str] = [] + while stack: + node = stack.pop() + members.append(node) + for predecessor in reversed(sorted(reverse[node])): + if predecessor in assigned: + continue + assigned.add(predecessor) + stack.append(predecessor) + raw_components.append(tuple(sorted(members))) + + ordered_components = sorted(raw_components) + component_id: dict[str, str] = {} + components: list[dict[str, Any]] = [] + for number, members in enumerate(ordered_components): + identifier = f"scc-{number:04d}" + cyclic = len(members) > 1 or any(member in self_loops for member in members) + components.append({"id": identifier, "nodes": list(members), "cyclic": cyclic}) + for member in members: + component_id[member] = identifier + + dependencies: dict[str, set[str]] = {item["id"]: set() for item in components} + dependents: dict[str, set[str]] = {item["id"]: set() for item in components} + for source, targets in adjacency.items(): + source_component = component_id[source] + for target in targets: + target_component = component_id[target] + if source_component == target_component: + continue + dependencies[source_component].add(target_component) + dependents[target_component].add(source_component) + + unresolved_count = { + component: len(required) for component, required in dependencies.items() + } + current_wave = sorted( + component for component, count in unresolved_count.items() if count == 0 + ) + waves: list[list[str]] = [] + emitted: set[str] = set() + while current_wave: + waves.append(current_wave) + emitted.update(current_wave) + next_wave: set[str] = set() + for resolved in current_wave: + for dependent in sorted(dependents[resolved]): + unresolved_count[dependent] -= 1 + if unresolved_count[dependent] == 0: + next_wave.add(dependent) + current_wave = sorted(next_wave) + + if len(emitted) != len(components): + raise InventoryError("SCC condensation graph unexpectedly contains a cycle") + + cycles = [item["nodes"] for item in components if item["cyclic"]] + return {"components": components, "cycles": cycles, "waves": waves} + + +def render_inventory(inventory: Mapping[str, Any], output_format: str) -> str: + if output_format == "json": + return json.dumps(inventory, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n" + if output_format == "dot": + return render_dot(inventory) + if output_format == "mermaid": + return render_mermaid(inventory) + raise InputError(f"unsupported output format {output_format!r}") + + +def _pin_render_id(pin: Mapping[str, Any]) -> str: + identity = json.dumps(pin, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return render_node_id("lock-pin-evidence:" + identity) + + +def _pin_label(pin: Mapping[str, Any]) -> str: + package = str(pin.get("package", "unknown")) + version = str(pin.get("selected_version", "unknown")) + parts = ["lock pin (evidence only)", f"{package}@{version}"] + artifact_sha256 = pin.get("artifact_sha256") + if artifact_sha256 is not None: + parts.append("sha256:" + str(artifact_sha256)[:12]) + vcs_commit = pin.get("vcs_commit") + if vcs_commit is not None: + parts.append("commit:" + str(vcs_commit)[:12]) + return " | ".join(parts) + + +def render_dot(inventory: Mapping[str, Any]) -> str: + lines = [ + "digraph zpkg_github_dependency_inventory {", + ' graph [rankdir="LR"];', + ' node [shape="box"];', + ] + for node in inventory.get("nodes", []): + identifier = render_node_id(str(node["id"])) + label = dot_escape(str(node.get("label", node["id"]))) + kind = dot_escape(str(node.get("kind", "node"))) + lines.append(f' {identifier} [label="{label}", tooltip="{kind}"];') + for pin in inventory.get("pins", []): + identifier = _pin_render_id(pin) + label = dot_escape(_pin_label(pin)) + lines.append( + f' {identifier} [shape="note", style="dashed", label="{label}", ' + 'tooltip="non-topological lock evidence"];' + ) + for edge in inventory.get("edges", []): + source = render_node_id(str(edge["source"])) + target = render_node_id(str(edge["target"])) + label_parts = [str(edge.get("kind", "dependency"))] + for key in ("requirement", "selected_version", "selected_commit"): + if edge.get(key) is not None: + label_parts.append(str(edge[key])) + label = dot_escape(" | ".join(label_parts)) + lines.append(f' {source} -> {target} [label="{label}"];') + lines.append("}") + return "\n".join(lines) + "\n" + + +def render_mermaid(inventory: Mapping[str, Any]) -> str: + lines = ["flowchart LR"] + for node in inventory.get("nodes", []): + identifier = render_node_id(str(node["id"])) + label = mermaid_escape(str(node.get("label", node["id"]))) + lines.append(f' {identifier}["{label}"]') + pin_ids: list[str] = [] + for pin in inventory.get("pins", []): + identifier = _pin_render_id(pin) + pin_ids.append(identifier) + label = mermaid_escape(_pin_label(pin)) + lines.append(f' {identifier}["{label}"]') + for edge in inventory.get("edges", []): + source = render_node_id(str(edge["source"])) + target = render_node_id(str(edge["target"])) + label_parts = [str(edge.get("kind", "dependency"))] + for key in ("requirement", "selected_version", "selected_commit"): + if edge.get(key) is not None: + label_parts.append(str(edge[key])) + label = mermaid_escape(" | ".join(label_parts)) + lines.append(f' {source} -->|"{label}"| {target}') + if pin_ids: + lines.append(" classDef lockEvidence stroke-dasharray: 5 5") + lines.append(f" class {','.join(pin_ids)} lockEvidence") + return "\n".join(lines) + "\n" + + +def write_atomic(path: Path, content: str) -> None: + path = path.expanduser() + parent = path.parent + parent.mkdir(parents=True, exist_ok=True) + if path.exists() and not path.is_file(): + raise InputError(f"output path {path} is not a regular file") + descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=parent) + temporary_path = Path(temporary) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + _fsync_directory(parent) + except BaseException: + try: + temporary_path.unlink(missing_ok=True) + finally: + raise diff --git a/harness/github_inventory_identity.py b/harness/github_inventory_identity.py new file mode 100644 index 00000000..c22d7023 --- /dev/null +++ b/harness/github_inventory_identity.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Identity normalization, provenance, and graph-label escaping for DEN-2957.""" + +from __future__ import annotations + +import configparser +import hashlib +import html +import io +import json +import re +import urllib.parse +from typing import Any + +from github_inventory_types import * # noqa: F401,F403 + +SEGMENT_RE = re.compile(r"^[A-Za-z0-9_.-]{1,100}$") +GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}(?:[0-9a-f]{24})?$") + +def _validate_repo_path(path: str) -> None: + if len(path.encode("utf-8")) > MAX_REPOSITORY_PATH_BYTES: + raise InputError(f"repository path exceeds {MAX_REPOSITORY_PATH_BYTES} bytes") + if not path or path.startswith(("/", "\\")) or "\\" in path or "\x00" in path: + raise InputError(f"unsafe repository path {path!r}") + parts = path.split("/") + if any(part in {"", ".", ".."} for part in parts): + raise InputError(f"unsafe repository path {path!r}") + + +def normalize_org(value: str) -> str: + value = value.strip() + if not SEGMENT_RE.fullmatch(value) or value in {".", ".."}: + raise InputError(f"invalid GitHub organization {value!r}") + return value.lower() + + +def normalize_repo(value: str) -> str: + value = value.strip() + if value.count("/") != 1: + raise InputError(f"invalid GitHub repository {value!r}; expected owner/name") + owner, name = value.split("/", 1) + if ( + not SEGMENT_RE.fullmatch(owner) + or not SEGMENT_RE.fullmatch(name) + or owner in {".", ".."} + or name in {".", ".."} + ): + raise InputError(f"invalid GitHub repository {value!r}") + return f"{owner.lower()}/{name.lower()}" + + +def normalize_includes(values: Sequence[str]) -> list[str]: + selected: set[str] = set() + for value in values: + for part in value.split(","): + part = part.strip().lower() + if not part: + continue + if part not in ALLOWED_INCLUDES: + raise InputError( + f"unsupported include {part!r}; expected {','.join(ALLOWED_INCLUDES)}" + ) + selected.add(part) + return sorted(selected or set(ALLOWED_INCLUDES)) + + +def normalize_github_url(url: str) -> str: + value = url.strip() + patterns = ( + re.compile(r"^https://github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$", re.IGNORECASE), + re.compile(r"^git@github\.com:([^/]+)/([^/]+?)(?:\.git)?$", re.IGNORECASE), + re.compile(r"^ssh://git@github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$", re.IGNORECASE), + re.compile(r"^git://github\.com/([^/]+)/([^/]+?)(?:\.git)?/?$", re.IGNORECASE), + ) + for pattern in patterns: + match = pattern.fullmatch(value) + if match: + return normalize_repo(f"{match.group(1)}/{match.group(2)}") + raise InputError("submodule/source URL is not an explicit github.com repository URL") + + +def repo_node_id(full_name: str) -> str: + return "github-repo:" + normalize_repo(full_name) + + +def package_node_id(coordinate: str, version: str | None = None) -> str: + base = "zpkg-package:" + normalize_repo(coordinate) + return f"{base}@{version}" if version else base + + +def manifest_kind(path: str) -> str: + if path == ".zpkg.toml": + return "zed-manifest" + if path == ".zpkg.lock": + return "zed-lock" + if path == ".gitmodules": + return "gitmodules" + if path == "flake.nix": + return "nix-flake" + if path == "flake.lock": + return "nix-flake-lock" + if path.endswith("sources.json"): + return "nix-sources" + return "source" + + +def provenance( + blob: SourceBlob, + line: int | None = None, + *, + json_pointer: str | None = None, +) -> dict[str, Any]: + item: dict[str, Any] = { + "repository": blob.repository, + "repository_commit": blob.repository_commit, + "manifest_kind": blob.kind, + "path": blob.path, + "blob_sha": blob.blob_sha, + } + if line is not None: + item["line_start"] = line + item["line_end"] = line + if json_pointer is not None: + item["json_pointer"] = json_pointer + return item + + +def parse_gitmodules(text: str) -> list[dict[str, Any]]: + parser = configparser.RawConfigParser(interpolation=None, strict=True) + parser.optionxform = str.lower + try: + parser.read_file(io.StringIO(text)) + except configparser.Error as error: + raise ParseFailure("invalid .gitmodules syntax") from error + result: list[dict[str, Any]] = [] + lines = text.splitlines() + for section in parser.sections(): + if not re.fullmatch(r'submodule "[^"\r\n]+"', section): + raise ParseFailure(".gitmodules contains a non-submodule section") + if not parser.has_option(section, "path") or not parser.has_option(section, "url"): + raise ParseFailure(f"{section} must contain path and url") + path = parser.get(section, "path").strip() + url = parser.get(section, "url").strip() + _validate_repo_path(path) + line = next( + ( + index + for index, value in enumerate(lines, start=1) + if value.strip().lower().startswith("url") and url in value + ), + None, + ) + result.append({"section": section, "path": path, "url": url, "line": line}) + result.sort(key=lambda item: (item["path"], item["url"], item["section"])) + return result + + +def git_blob_sha(data: bytes) -> str: + header = f"blob {len(data)}\0".encode("ascii") + return hashlib.sha1(header + data, usedforsecurity=False).hexdigest() + + +def render_node_id(node_id: str) -> str: + return "n_" + hashlib.sha256(node_id.encode("utf-8")).hexdigest()[:20] + + +def dot_escape(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\r", "\\r") + .replace("\n", "\\n") + .replace("\t", "\\t") + ) + + +def mermaid_escape(value: str) -> str: + # html.escape handles ampersands first and therefore cannot accidentally + # produce a raw quote or tag delimiter in a Mermaid label. + escaped = html.escape(value, quote=True) + return ( + escaped.replace("\r", " ") + .replace("\n", " ") + .replace("[", "[") + .replace("]", "]") + .replace("{", "{") + .replace("}", "}") + .replace("|", "|") + ) + + +def redact_text(value: str, token: str | None) -> str: + result = value + if token: + result = result.replace(token, "[REDACTED]") + result = re.sub(r"(?i)(authorization:\s*bearer\s+)[^\s]+", r"\1[REDACTED]", result) + result = re.sub(r"(?i)(access_token=)[^&\s]+", r"\1[REDACTED]", result) + return result diff --git a/harness/github_inventory_mutation.py b/harness/github_inventory_mutation.py new file mode 100644 index 00000000..ea200e0b --- /dev/null +++ b/harness/github_inventory_mutation.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Bounded node/edge mutation and diagnostics mixin for DEN-2957.""" + +from __future__ import annotations + +from github_inventory_core import * # noqa: F401,F403 +import github_inventory_core as _core + +_error_code = _core._error_code +_provenance_sort_key = _core._provenance_sort_key + + +class InventoryMutationMixin: + def _ensure_repo_node(self, full_name: str, scanned: bool) -> str: + full_name = normalize_repo(full_name) + node_id = repo_node_id(full_name) + existing = self.nodes.get(node_id) + if existing is None: + if len(self.nodes) >= self.limits.max_nodes: + raise LimitError( + f"node limit exceeded ({len(self.nodes) + 1} > {self.limits.max_nodes})" + ) + self.nodes[node_id] = { + "id": node_id, + "kind": "repository", + "label": full_name, + "repository": full_name, + "scanned": bool(scanned), + } + elif scanned: + existing["scanned"] = True + return node_id + + def _ensure_package_node(self, coordinate: str, exact_version: str | None) -> str: + coordinate = normalize_repo(coordinate) + node_id = package_node_id(coordinate, exact_version) + if node_id not in self.nodes: + if len(self.nodes) >= self.limits.max_nodes: + raise LimitError( + f"node limit exceeded ({len(self.nodes) + 1} > {self.limits.max_nodes})" + ) + self.nodes[node_id] = { + "id": node_id, + "kind": "package", + "label": f"{coordinate}@{exact_version}" if exact_version else coordinate, + "package": coordinate, + **({"version": exact_version} if exact_version else {}), + } + return node_id + + def _add_edge( + self, + source: str, + target: str, + kind: str, + *, + requirement: str | None = None, + selected_version: str | None = None, + selected_commit: str | None = None, + artifact_sha256: str | None = None, + source_path: str | None = None, + input_name: str | None = None, + provenance: Sequence[Mapping[str, Any]] = (), + ) -> None: + if source not in self.nodes: + raise ParseFailure(f"edge source {source} is missing") + if target not in self.nodes: + raise ParseFailure(f"edge target {target} is missing") + key = ( + source, + target, + kind, + requirement, + selected_version, + selected_commit, + artifact_sha256, + source_path, + input_name, + ) + normalized_provenance = [dict(item) for item in provenance] + if key in self.edges: + existing = self.edges[key] + combined = { + json.dumps(item, sort_keys=True, separators=(",", ":")): item + for item in existing["provenance"] + normalized_provenance + } + existing["provenance"] = [combined[key] for key in sorted(combined)] + return + if len(self.edges) >= self.limits.max_edges: + raise LimitError( + f"edge limit exceeded ({len(self.edges) + 1} > {self.limits.max_edges})" + ) + edge: dict[str, Any] = { + "source": source, + "target": target, + "kind": kind, + "provenance": sorted(normalized_provenance, key=_provenance_sort_key), + } + for name, value in ( + ("requirement", requirement), + ("selected_version", selected_version), + ("selected_commit", selected_commit), + ("artifact_sha256", artifact_sha256), + ("source_path", source_path), + ("input_name", input_name), + ): + if value is not None: + edge[name] = value + self.edges[key] = edge + + def _failure(self, repository: str, stage: str, error: BaseException, path: str | None = None) -> None: + code = _error_code(error) + message = redact_text(str(error), self.token) + item: dict[str, Any] = { + "repository": repository, + "stage": stage, + "code": code, + "message": message, + } + if path: + item["path"] = path + self.failures.append(item) + + def _source_failure( + self, + blob: SourceBlob, + stage: str, + code: str, + error: BaseException, + ) -> None: + message = redact_text(str(error), self.token) + self.failures.append( + { + "repository": blob.repository, + "stage": stage, + "path": blob.path, + "code": code, + "message": message, + "repository_commit": blob.repository_commit, + "blob_sha": blob.blob_sha, + } + ) + + def _check_graph_limits(self) -> None: + if len(self.nodes) > self.limits.max_nodes: + raise LimitError(f"node limit exceeded ({len(self.nodes)} > {self.limits.max_nodes})") + if len(self.edges) > self.limits.max_edges: + raise LimitError(f"edge limit exceeded ({len(self.edges)} > {self.limits.max_edges})") diff --git a/harness/github_inventory_parsers.py b/harness/github_inventory_parsers.py new file mode 100644 index 00000000..3f2088ab --- /dev/null +++ b/harness/github_inventory_parsers.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Combined source parser mixin for DEN-2957 and DEN-2997.""" + +import github_inventory_core as _core +import github_inventory_zed_parsers as _zed + + +def _parse_requirement(value: str, label: str, limits: _core.Limits) -> str: + """Preserve the existing Zed parser's bounded opaque requirement semantics.""" + + _core._bounded_text(value, label, limits.max_field_bytes) + return value + + +# The historical parser keeps this operation inline. Expose the same behavior +# before importing the workspace extension so stacked code can reuse one contract. +_zed._parse_requirement = _parse_requirement + +from github_inventory_zed_workspace import WorkspaceZedParsingMixin # noqa: E402 +from github_inventory_source_parsers import SourceParsingMixin # noqa: E402 + + +class InventoryParsingMixin(WorkspaceZedParsingMixin, SourceParsingMixin): + pass diff --git a/harness/github_inventory_source_parsers.py b/harness/github_inventory_source_parsers.py new file mode 100644 index 00000000..799ec7cf --- /dev/null +++ b/harness/github_inventory_source_parsers.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Git-submodule and Nix provenance parser mixin for DEN-2957.""" + +from __future__ import annotations + +from github_inventory_core import * # noqa: F401,F403 +import github_inventory_core as _core + +_bounded_text = _core._bounded_text +_find_json_line = _core._find_json_line +_find_toml_line = _core._find_toml_line +_json_pointer = _core._json_pointer +_utf8 = _core._utf8 +_validate_repo_path = _core._validate_repo_path +_walk_json = _core._walk_json + +class SourceParsingMixin: + def _parse_git_submodules( + self, + blob: SourceBlob | None, + full_name: str, + commit_sha: str, + gitlinks: Mapping[str, str], + ) -> None: + source = repo_node_id(full_name) + mapped_paths: set[str] = set() + if blob is not None: + text = _utf8(blob) + try: + modules = parse_gitmodules(text) + except ParseFailure as error: + self._source_failure(blob, "parse-gitmodules", "invalid_gitmodules", error) + modules = [] + for module in modules: + path = module["path"] + url = module["url"] + line = module["line"] + mapped_paths.add(path) + try: + _bounded_text(url, "submodule URL", self.limits.max_field_bytes) + target_repo = normalize_github_url(url) + except InputError as error: + self._source_failure(blob, "parse-gitmodules", "unsupported_submodule_url", error) + continue + selected_commit = gitlinks.get(path) + target = repo_node_id(target_repo) + self._ensure_repo_node(target_repo, scanned=target_repo in self.repository_records) + self._add_edge( + source, + target, + "git-submodule", + selected_commit=selected_commit, + source_path=path, + provenance=[provenance(blob, line)], + ) + if selected_commit is None: + self.contradictions.append( + { + "code": "submodule-missing-gitlink", + "source": source, + "target": target, + "path": path, + } + ) + for path, sha in sorted(gitlinks.items()): + if path not in mapped_paths: + self.contradictions.append( + { + "code": "unmapped-gitlink", + "source": source, + "path": path, + "selected_commit": sha, + "repository_commit": commit_sha, + } + ) + + def _parse_flake_lock(self, blob: SourceBlob) -> None: + text = _utf8(blob) + try: + document = json.loads(text) + except json.JSONDecodeError as error: + self._source_failure(blob, "parse-flake-lock", "invalid_json", error) + return + if not isinstance(document, dict) or not isinstance(document.get("nodes"), dict): + self._source_failure( + blob, + "parse-flake-lock", + "invalid_flake_lock", + ParseFailure("flake.lock must contain a nodes object"), + ) + return + source = repo_node_id(blob.repository) + for node_name, node in sorted(document["nodes"].items(), key=lambda item: str(item[0])): + if not isinstance(node, dict) or not isinstance(node.get("locked"), dict): + continue + locked = node["locked"] + if locked.get("type") != "github": + continue + owner, repo, rev = locked.get("owner"), locked.get("repo"), locked.get("rev") + if not all(isinstance(value, str) and value for value in (owner, repo, rev)): + self._source_failure( + blob, + "parse-flake-lock", + "invalid_github_lock_node", + ParseFailure(f"flake node {node_name!r} is missing owner/repo/rev"), + ) + continue + try: + _bounded_text(str(node_name), "flake input name", self.limits.max_field_bytes) + if not GIT_SHA_RE.fullmatch(rev.lower()): + raise InputError(f"flake node {node_name!r} has invalid Git revision") + rev = rev.lower() + target_repo = normalize_repo(f"{owner}/{repo}") + except InputError as error: + self._source_failure(blob, "parse-flake-lock", "invalid_github_coordinate", error) + continue + target = repo_node_id(target_repo) + self._ensure_repo_node(target_repo, scanned=target_repo in self.repository_records) + self._add_edge( + source, + target, + "nix-flake-lock", + selected_commit=rev, + input_name=str(node_name), + provenance=[ + provenance( + blob, + _find_json_line(text, str(node_name)), + json_pointer=f"/nodes/{_json_pointer(str(node_name))}/locked", + ) + ], + ) + + def _parse_flake_nix(self, blob: SourceBlob) -> None: + text = _utf8(blob) + source = repo_node_id(blob.repository) + pattern = re.compile( + r"github:([A-Za-z0-9_.-]{1,100})/([A-Za-z0-9_.-]{1,100})(?:/([^\"'\s;)}]+))?" + ) + for match in pattern.finditer(text): + target_repo = normalize_repo(f"{match.group(1)}/{match.group(2)}") + requirement = match.group(3) + if requirement is not None: + try: + _bounded_text(requirement, "flake input reference", self.limits.max_field_bytes) + except InputError as error: + self._source_failure( + blob, + "parse-flake-nix", + "flake_reference_too_large", + error, + ) + continue + target = repo_node_id(target_repo) + self._ensure_repo_node(target_repo, scanned=target_repo in self.repository_records) + line = text.count("\n", 0, match.start()) + 1 + self._add_edge( + source, + target, + "nix-flake-declared", + requirement=requirement, + provenance=[provenance(blob, line)], + ) + + def _parse_nix_sources(self, blob: SourceBlob) -> None: + text = _utf8(blob) + try: + document = json.loads(text) + except json.JSONDecodeError as error: + self._source_failure(blob, "parse-nix-sources", "invalid_json", error) + return + source = repo_node_id(blob.repository) + for pointer, item in _walk_json(document, self.limits.max_json_depth): + if not isinstance(item, dict): + continue + owner, repo = item.get("owner"), item.get("repo") + url = item.get("url") + target_repo: str | None = None + if isinstance(owner, str) and isinstance(repo, str): + try: + target_repo = normalize_repo(f"{owner}/{repo}") + except InputError: + target_repo = None + elif isinstance(url, str): + try: + _bounded_text(url, "Nix source URL", self.limits.max_field_bytes) + target_repo = normalize_github_url(url) + except InputError: + target_repo = None + if target_repo is None: + continue + selected_commit = item.get("rev") if isinstance(item.get("rev"), str) else None + if selected_commit is not None: + if not GIT_SHA_RE.fullmatch(selected_commit.lower()): + self._source_failure( + blob, + "parse-nix-sources", + "invalid_nix_source_revision", + ParseFailure(f"Nix source {target_repo} has invalid Git revision"), + ) + continue + selected_commit = selected_commit.lower() + target = repo_node_id(target_repo) + self._ensure_repo_node(target_repo, scanned=target_repo in self.repository_records) + self._add_edge( + source, + target, + "nix-source-pin" if selected_commit else "nix-source-declared", + selected_commit=selected_commit, + provenance=[ + provenance( + blob, + _find_json_line(text, target_repo.split("/", 1)[1]), + json_pointer=pointer, + ) + ], + ) diff --git a/harness/github_inventory_support.py b/harness/github_inventory_support.py new file mode 100644 index 00000000..d3664b95 --- /dev/null +++ b/harness/github_inventory_support.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Public support facade for the DEN-2957 conformance implementation.""" + +from github_inventory_types import * # noqa: F401,F403 +from github_inventory_identity import * # noqa: F401,F403 +from github_inventory_transport import * # noqa: F401,F403 +from github_inventory_util import * # noqa: F401,F403 + +import github_inventory_util as _util + +_bounded_retry_after = _util._bounded_retry_after +_bounded_text = _util._bounded_text +_edge_sort_key = _util._edge_sort_key +_error_code = _util._error_code +_find_json_line = _util._find_json_line +_find_toml_line = _util._find_toml_line +_fixture_bytes = _util._fixture_bytes +_fsync_directory = _util._fsync_directory +_is_loopback_host = _util._is_loopback_host +_is_transient_response = _util._is_transient_response +_json_pointer = _util._json_pointer +_json_response = _util._json_response +_metadata_full_name = _util._metadata_full_name +_next_link = _util._next_link +_positive_int = _util._positive_int +_provenance_sort_key = _util._provenance_sort_key +_query_positive_int = _util._query_positive_int +_quote = _util._quote +_utf8 = _util._utf8 +_valid_bearer_header = _util._valid_bearer_header +_validate_repo_path = _util._validate_repo_path +_walk_json = _util._walk_json diff --git a/harness/github_inventory_test_support.py b/harness/github_inventory_test_support.py new file mode 100644 index 00000000..a2508c10 --- /dev/null +++ b/harness/github_inventory_test_support.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import copy +import contextlib +import io +import json +import sys +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +FIXTURE_PATH = ROOT / "fixtures" / "github-dependency-inventory" / "fixture.json" +GOLDEN = ROOT / "fixtures" / "github-dependency-inventory" / "golden" +sys.path.insert(0, str(HERE)) + +import github_dependency_inventory as inventory # noqa: E402 + + +class QuietHandler(BaseHTTPRequestHandler): + backend: inventory.FixtureBackend + seen_authorization: list[str | None] + transient_remaining: int = 0 + transient_status: int = 503 + redirect_org: bool = False + oversized_org_bytes: int = 0 + + def do_GET(self) -> None: # noqa: N802 - stdlib callback name + type(self).seen_authorization.append(self.headers.get("Authorization")) + if type(self).redirect_org and self.path.startswith("/orgs/"): + self.send_response(302) + self.send_header("Location", "https://attacker.invalid/steal") + self.end_headers() + return + if type(self).oversized_org_bytes and self.path.startswith("/orgs/"): + body = b"x" * type(self).oversized_org_bytes + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + if type(self).transient_remaining > 0 and self.path.startswith("/orgs/"): + type(self).transient_remaining -= 1 + self.send_response(type(self).transient_status) + self.send_header("Retry-After", "0") + if type(self).transient_status == 403: + self.send_header("X-RateLimit-Remaining", "0") + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"message":"try later"}') + return + response = type(self).backend.handle(self.path, self.headers.get("Authorization")) + self.send_response(response.status) + for key, value in response.headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(response.body) + + def log_message(self, fmt: str, *args: object) -> None: + del fmt, args + + +class FakeServer: + def __init__( + self, + document: dict[str, Any], + *, + transient: int = 0, + transient_status: int = 503, + redirect_org: bool = False, + oversized_org_bytes: int = 0, + ) -> None: + handler = type("BoundQuietHandler", (QuietHandler,), {}) + handler.seen_authorization = [] + handler.transient_remaining = transient + handler.transient_status = transient_status + handler.redirect_org = redirect_org + handler.oversized_org_bytes = oversized_org_bytes + self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + host, port = self.server.server_address + self.base_url = f"http://{host}:{port}" + handler.backend = inventory.FixtureBackend(document, base_url=self.base_url) + self.handler = handler + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def __enter__(self) -> "FakeServer": + self.thread.start() + return self + + def __exit__(self, exc_type: object, exc: object, traceback: object) -> None: + del exc_type, exc, traceback + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + + +class InventoryTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.fixture = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + + def build( + self, + document: dict[str, Any] | None = None, + *, + repositories: list[str] | None = None, + organizations: list[str] | None = None, + includes: list[str] | None = None, + limits: inventory.Limits | None = None, + token: str | None = None, + ) -> tuple[dict[str, Any], inventory.FixtureTransport]: + backend = inventory.FixtureBackend(copy.deepcopy(document or self.fixture)) + transport = inventory.FixtureTransport(backend) + result = inventory.run_inventory( + repositories=repositories or ["ACME/App", "acme/app"], + organizations=organizations or ["ACME", "acme"], + includes=includes or ["nix,zed", "git-submodule", "zed"], + limits=limits or inventory.Limits(), + transport=transport, + token=token, + sleeper=lambda _: None, + ) + return result, transport diff --git a/harness/github_inventory_transport.py b/harness/github_inventory_transport.py new file mode 100644 index 00000000..8af5e422 --- /dev/null +++ b/harness/github_inventory_transport.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Redirect-safe, bounded GitHub HTTP transport for DEN-2957.""" + +from __future__ import annotations + +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Mapping + +from github_inventory_types import * # noqa: F401,F403 +from github_inventory_util import _is_loopback_host + + +def _unsafe_url_path(path: str) -> bool: + """Reject path syntax whose routing meaning can change after decoding.""" + + if not path.startswith("/") or "\\" in path or "\x00" in path: + return True + if any(ord(character) < 0x20 or ord(character) == 0x7F for character in path): + return True + decoded = urllib.parse.unquote(path) + if "\\" in decoded or "\x00" in decoded or "//" in decoded: + return True + return any(segment in {".", ".."} for segment in decoded.split("/")) + + +class Transport: + base_url: str + _base_path: str + + def request( + self, + path: str, + headers: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> ApiResponse: + raise NotImplementedError + + def token_destination_allowed(self) -> bool: + return True + + def normalize_request_path(self, path: str) -> str: + parsed = urllib.parse.urlsplit(path) + if parsed.scheme or parsed.netloc or parsed.username or parsed.password or parsed.fragment: + raise ApiError(400, "request-path", "unsafe_github_api_path") + request_path = parsed.path or "/" + if _unsafe_url_path(request_path): + raise ApiError(400, "request-path", "unsafe_github_api_path") + if parsed.query: + request_path += "?" + parsed.query + return request_path + + def absolute_request_url(self, path: str) -> str: + return self.base_url + self.normalize_request_path(path) + + def relative_from_link(self, link: str, *, current_path: str = "/") -> str: + base = urllib.parse.urlsplit(self.base_url) + current_url = self.absolute_request_url(current_path) + parsed = urllib.parse.urlsplit(urllib.parse.urljoin(current_url, link)) + if (parsed.scheme.lower(), parsed.netloc.lower()) != ( + base.scheme.lower(), + base.netloc.lower(), + ): + raise ApiError(400, "pagination-link", "cross_origin_pagination_link") + if parsed.username or parsed.password or parsed.fragment: + raise ApiError(400, "pagination-link", "unsafe_pagination_link") + path = parsed.path or "/" + if _unsafe_url_path(path): + raise ApiError(400, "pagination-link", "unsafe_pagination_link") + base_path = getattr(self, "_base_path", base.path.rstrip("/")) + if base_path: + if path == base_path: + relative = "/" + elif path.startswith(base_path + "/"): + relative = path[len(base_path) :] + else: + raise ApiError(400, "pagination-link", "pagination_link_outside_api_base") + else: + relative = path + if parsed.query: + relative += "?" + parsed.query + return self.normalize_request_path(relative) + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request( + self, + req: urllib.request.Request, + fp: Any, + code: int, + msg: str, + headers: Any, + newurl: str, + ) -> None: + return None + + +class HttpTransport(Transport): + def __init__( + self, + base_url: str = DEFAULT_API_BASE, + *, + allow_token_to_custom_origin: bool = False, + ) -> None: + parsed = urllib.parse.urlsplit(base_url) + if parsed.scheme not in {"https", "http"} or not parsed.netloc: + raise InputError("GitHub API base must be an absolute http(s) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise InputError("GitHub API base must not contain credentials, query, or fragment") + raw_path = parsed.path or "" + candidate_path = raw_path.rstrip("/") + if candidate_path == "/": + candidate_path = "" + if candidate_path and _unsafe_url_path(candidate_path): + raise InputError("GitHub API base contains ambiguous path syntax") + if parsed.scheme == "http" and not _is_loopback_host(parsed.hostname): + raise InputError("plain HTTP GitHub API bases are allowed only on loopback") + self._base_path = candidate_path + self.base_url = urllib.parse.urlunsplit( + (parsed.scheme.lower(), parsed.netloc, self._base_path, "", "") + ) + self._host = parsed.hostname.lower() if parsed.hostname else "" + self._allow_token_to_custom_origin = allow_token_to_custom_origin + self._opener = urllib.request.build_opener(_NoRedirect()) + + def token_destination_allowed(self) -> bool: + return ( + self._host == "api.github.com" + or _is_loopback_host(self._host) + or self._allow_token_to_custom_origin + ) + + def request( + self, + path: str, + headers: Mapping[str, str], + timeout: float, + max_bytes: int, + ) -> ApiResponse: + relative = self.normalize_request_path(path) + url = self.base_url + relative + request = urllib.request.Request(url, method="GET", headers=dict(headers)) + try: + with self._opener.open(request, timeout=timeout) as response: + body = response.read(max_bytes + 1) + if len(body) > max_bytes: + raise LimitError(f"one GitHub response exceeded {max_bytes} bytes") + return ApiResponse( + status=int(response.status), + headers={str(k): str(v) for k, v in response.headers.items()}, + body=body, + ) + except urllib.error.HTTPError as error: + # Read only a bounded diagnostic body. It is never surfaced in an + # exception, inventory, or retained evidence. + read_limit = min(max_bytes, HTTP_MAX_ERROR_BODY) + body = error.read(read_limit + 1)[:read_limit] + return ApiResponse( + status=int(error.code), + headers={str(k): str(v) for k, v in error.headers.items()}, + body=body, + ) + except (OSError, urllib.error.URLError) as error: + # Avoid retaining raw transport text because it can include proxy + # URLs or injected credential-bearing diagnostics. + raise ApiError(599, relative, "github_transport_error") from error diff --git a/harness/github_inventory_types.py b/harness/github_inventory_types.py new file mode 100644 index 00000000..114be928 --- /dev/null +++ b/harness/github_inventory_types.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Bounded types, limits, and error taxonomy for DEN-2957.""" + +from __future__ import annotations + +import argparse +import base64 +import configparser +import dataclasses +import hashlib +import html +import io +import ipaddress +import json +import os +import re +import sys +import tempfile +import time +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Sequence + +INVENTORY_SCHEMA = "zpkg/github-dependency-inventory/v1" +FIXTURE_SCHEMA = "zpkg/github-api-fixture/v1" +DEFAULT_API_BASE = "https://api.github.com" +ALLOWED_INCLUDES = ("git-submodule", "nix", "zed") +EXPECTED_MANIFESTS = { + "zed": (".zpkg.toml", ".zpkg.lock"), + "git-submodule": (".gitmodules",), + "nix": ("flake.nix", "flake.lock", "nix/sources.json", "npins/sources.json"), +} +TRANSIENT_STATUSES = frozenset({429, 502, 503, 504}) +HTTP_MAX_ERROR_BODY = 4096 +MAX_REPOSITORY_PATH_BYTES = 4096 + +class InventoryError(Exception): + """Base class for controlled command failures.""" + + +class InputError(InventoryError): + """Invalid caller or fixture input.""" + + +class LimitError(InventoryError): + """A hard resource limit was exceeded; no partial output may be published.""" + + +class ApiError(InventoryError): + def __init__(self, status: int, path: str, code: str = "github_http_error") -> None: + self.status = status + self.path = path + self.code = code + super().__init__(f"GitHub API request failed with HTTP {status} at {path}") + + +class ParseFailure(InventoryError): + """One repository source could not be parsed.""" + + +@dataclasses.dataclass(frozen=True) +class Limits: + max_repositories: int = 1_000 + max_nodes: int = 20_000 + max_edges: int = 40_000 + max_requests: int = 20_000 + max_response_bytes: int = 4 * 1024 * 1024 + max_total_response_bytes: int = 128 * 1024 * 1024 + max_manifest_bytes: int = 2 * 1024 * 1024 + max_field_bytes: int = 16 * 1024 + max_tree_entries: int = 50_000 + max_json_depth: int = 64 + max_seconds: float = 120.0 + max_retries: int = 2 + + def validate(self) -> None: + for field in dataclasses.fields(self): + value = getattr(self, field.name) + if isinstance(value, bool): + raise InputError(f"{field.name} must be numeric") + if field.name == "max_seconds": + if not isinstance(value, (int, float)) or value <= 0: + raise InputError("max_seconds must be greater than zero") + continue + if not isinstance(value, int): + raise InputError(f"{field.name} must be an integer") + if field.name == "max_retries": + if value < 0: + raise InputError("max_retries must not be negative") + elif value <= 0: + raise InputError(f"{field.name} must be greater than zero") + if self.max_retries > 8: + raise InputError("max_retries must not exceed 8") + + def as_dict(self) -> dict[str, int | float]: + return dataclasses.asdict(self) + + +class Budget: + def __init__(self, limits: Limits, clock: Callable[[], float] = time.monotonic) -> None: + limits.validate() + self.limits = limits + self.clock = clock + self.started = clock() + self.requests = 0 + self.response_bytes = 0 + + def check_time(self) -> None: + elapsed = self.clock() - self.started + if elapsed > self.limits.max_seconds: + raise LimitError( + f"inventory time limit exceeded ({elapsed:.3f}s > {self.limits.max_seconds:.3f}s)" + ) + + def remaining_seconds(self) -> float: + self.check_time() + remaining = self.limits.max_seconds - (self.clock() - self.started) + return max(0.1, remaining) + + def begin_request(self) -> None: + self.check_time() + self.requests += 1 + if self.requests > self.limits.max_requests: + raise LimitError( + f"GitHub request limit exceeded ({self.requests} > {self.limits.max_requests})" + ) + + def consume_response(self, size: int) -> None: + self.check_time() + if size > self.limits.max_response_bytes: + raise LimitError( + f"one GitHub response exceeded {self.limits.max_response_bytes} bytes" + ) + self.response_bytes += size + if self.response_bytes > self.limits.max_total_response_bytes: + raise LimitError( + "total GitHub response byte limit exceeded " + f"({self.response_bytes} > {self.limits.max_total_response_bytes})" + ) + + +@dataclasses.dataclass(frozen=True) +class SourceBlob: + repository: str + repository_commit: str + kind: str + path: str + blob_sha: str + data: bytes + + +@dataclasses.dataclass(frozen=True) +class ApiResponse: + status: int + headers: Mapping[str, str] + body: bytes + + def header(self, name: str) -> str | None: + wanted = name.lower() + for key, value in self.headers.items(): + if key.lower() == wanted: + return value + return None diff --git a/harness/github_inventory_util.py b/harness/github_inventory_util.py new file mode 100644 index 00000000..cdccd9f9 --- /dev/null +++ b/harness/github_inventory_util.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Low-level bounded parsing, retry, and atomic-file helpers for DEN-2957.""" + +from __future__ import annotations + +import base64 +import ipaddress +import json +import os +import re +import urllib.parse +from pathlib import Path +from typing import Any, Iterable, Mapping + +from github_inventory_types import * # noqa: F401,F403 +from github_inventory_identity import normalize_repo + +def _metadata_full_name(item: Mapping[str, Any]) -> str: + full_name = item.get("full_name") + if not isinstance(full_name, str): + owner = item.get("owner") + name = item.get("name") + if isinstance(owner, dict) and isinstance(owner.get("login"), str) and isinstance(name, str): + full_name = f"{owner['login']}/{name}" + else: + raise ParseFailure("repository metadata is missing full_name") + return normalize_repo(full_name) + + +def _fixture_bytes(value: Any) -> bytes: + if isinstance(value, str): + return value.encode("utf-8") + if isinstance(value, dict) and isinstance(value.get("base64"), str): + try: + return base64.b64decode(value["base64"], validate=True) + except ValueError as error: + raise InputError("fixture base64 content is invalid") from error + raise InputError("fixture file content must be a UTF-8 string or {base64: ...}") + + +def _json_response( + status: int, + value: Any, + headers: Mapping[str, str] | None = None, +) -> ApiResponse: + body = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + response_headers = {"Content-Type": "application/json", **dict(headers or {})} + return ApiResponse(status=status, headers=response_headers, body=body) + + +def _query_positive_int(query: Mapping[str, list[str]], key: str, default: int) -> int: + values = query.get(key) + if not values: + return default + return _positive_int(values[-1], key) + + +def _positive_int(value: Any, label: str) -> int: + try: + result = int(value) + except (TypeError, ValueError) as error: + raise InputError(f"{label} must be an integer") from error + if result <= 0: + raise InputError(f"{label} must be greater than zero") + return result + + +def _valid_bearer_header(value: str | None) -> bool: + if not value or not value.startswith("Bearer "): + return False + return bool(value.removeprefix("Bearer ").strip()) + + +def _quote(value: str) -> str: + return urllib.parse.quote(value, safe="") + + +def _is_loopback_host(host: str | None) -> bool: + if host is None: + return False + if host.lower() == "localhost": + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def _next_link(header: str | None) -> str | None: + if not header: + return None + for part in header.split(","): + match = re.fullmatch(r'\s*<([^>]+)>\s*;\s*rel="([^"]+)"\s*', part) + if match and "next" in match.group(2).split(): + return match.group(1) + return None + + +def _bounded_retry_after(value: str | None) -> float: + if value is None: + return 0.0 + try: + number = float(value) + except ValueError: + return 0.0 + return max(0.0, min(number, 1.0)) + + +def _is_transient_response(response: ApiResponse) -> bool: + if response.status in TRANSIENT_STATUSES: + return True + return ( + response.status == 403 + and response.header("X-RateLimit-Remaining") == "0" + ) + + +def _bounded_text(value: str, label: str, max_bytes: int) -> str: + if "\x00" in value: + raise InputError(f"{label} contains a NUL byte") + size = len(value.encode("utf-8")) + if size > max_bytes: + raise LimitError(f"{label} exceeds {max_bytes} bytes") + return value + + +def _validate_repo_path(path: str) -> None: + if len(path.encode("utf-8")) > MAX_REPOSITORY_PATH_BYTES: + raise InputError(f"repository path exceeds {MAX_REPOSITORY_PATH_BYTES} bytes") + if not path or path.startswith(("/", "\\")) or "\\" in path or "\x00" in path: + raise InputError(f"unsafe repository path {path!r}") + parts = path.split("/") + if any(part in {"", ".", ".."} for part in parts): + raise InputError(f"unsafe repository path {path!r}") + + +def _utf8(blob: SourceBlob) -> str: + try: + return blob.data.decode("utf-8") + except UnicodeError as error: + raise ParseFailure(f"{blob.path} is not UTF-8") from error + + +def _find_toml_line(text: str, key: str) -> int | None: + quoted = json.dumps(key) + for index, line in enumerate(text.splitlines(), start=1): + stripped = line.strip() + if stripped.startswith(f"{key} ") or stripped.startswith(f"{quoted} "): + return index + if stripped.startswith(f"{key}=") or stripped.startswith(f"{quoted}="): + return index + return None + + +def _find_json_line(text: str, value: str) -> int | None: + needle = json.dumps(value) + for index, line in enumerate(text.splitlines(), start=1): + if needle in line: + return index + return None + + +def _json_pointer(value: str) -> str: + return value.replace("~", "~0").replace("/", "~1") + + +def _walk_json(value: Any, max_depth: int) -> Iterable[tuple[str, Any]]: + stack: list[tuple[str, Any, int]] = [("", value, 0)] + while stack: + pointer, current, depth = stack.pop() + if depth > max_depth: + raise LimitError(f"JSON source nesting exceeded {max_depth}") + yield pointer or "/", current + if isinstance(current, dict): + for key in sorted(current, reverse=True): + child = current[key] + stack.append((pointer + "/" + _json_pointer(str(key)), child, depth + 1)) + elif isinstance(current, list): + for index in range(len(current) - 1, -1, -1): + stack.append((pointer + f"/{index}", current[index], depth + 1)) + + +def _edge_sort_key(edge: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + edge.get("source", ""), + edge.get("target", ""), + edge.get("kind", ""), + edge.get("requirement", ""), + edge.get("selected_version", ""), + edge.get("selected_commit", ""), + edge.get("artifact_sha256", ""), + edge.get("source_path", ""), + edge.get("input_name", ""), + ) + + +def _provenance_sort_key(item: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + item.get("repository", ""), + item.get("repository_commit", ""), + item.get("path", ""), + item.get("line_start", -1), + item.get("json_pointer", ""), + item.get("blob_sha", ""), + ) + + +def _error_code(error: BaseException) -> str: + if isinstance(error, ApiError): + return error.code + if isinstance(error, LimitError): + return "limit_exceeded" + if isinstance(error, InputError): + return "invalid_input" + if isinstance(error, ParseFailure): + return "parse_failure" + return "inventory_failure" + + +def _fsync_directory(path: Path) -> None: + if os.name == "nt": + return + flags = os.O_RDONLY + if hasattr(os, "O_DIRECTORY"): + flags |= os.O_DIRECTORY + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/harness/github_inventory_zed_parsers.py b/harness/github_inventory_zed_parsers.py new file mode 100644 index 00000000..0410e2fc --- /dev/null +++ b/harness/github_inventory_zed_parsers.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Zed manifest and lock provenance parser mixin for DEN-2957 and DEN-2996.""" + +from __future__ import annotations + +from github_inventory_core import * # noqa: F401,F403 +import github_inventory_core as _core + +_bounded_text = _core._bounded_text +_find_json_line = _core._find_json_line +_find_toml_line = _core._find_toml_line +_json_pointer = _core._json_pointer +_utf8 = _core._utf8 +_validate_repo_path = _core._validate_repo_path +_walk_json = _core._walk_json + + +class ZedParsingMixin: + def _parse_zed_manifest(self, blob: SourceBlob) -> None: + text = _utf8(blob) + try: + document = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + self._source_failure(blob, "parse-zed-manifest", "invalid_toml", error) + return + package = document.get("package") + source_id = repo_node_id(blob.repository) + if isinstance(package, dict): + org = package.get("org") + name = package.get("name") + if isinstance(org, str) and isinstance(name, str): + try: + coordinate = normalize_repo(f"{org}/{name}") + except InputError as error: + self._source_failure(blob, "parse-zed-manifest", "invalid_package_coordinate", error) + return + package_id = package_node_id(coordinate) + self._ensure_package_node(coordinate, exact_version=None) + self.package_roots[blob.repository] = package_id + self._add_edge( + source_id, + package_id, + "contains-zed-package", + provenance=[provenance(blob, _find_toml_line(text, "name"))], + ) + source_id = package_id + + dependency_tables = ( + ("dependencies", "zed-declared"), + ("build-dependencies", "zed-build-declared"), + ("build_dependencies", "zed-build-declared"), + ) + for table_name, edge_kind in dependency_tables: + dependencies = document.get(table_name, {}) + if dependencies is None: + dependencies = {} + if not isinstance(dependencies, dict): + self._source_failure( + blob, + "parse-zed-manifest", + "invalid_dependencies_table", + ParseFailure(f"[{table_name}] must be a table"), + ) + continue + for raw_coordinate, raw_requirement in sorted( + dependencies.items(), key=lambda item: str(item[0]) + ): + try: + coordinate = normalize_repo(str(raw_coordinate)) + except InputError as error: + self._source_failure( + blob, + "parse-zed-manifest", + "invalid_dependency_coordinate", + error, + ) + continue + if not isinstance(raw_requirement, str): + self._source_failure( + blob, + "parse-zed-manifest", + "invalid_dependency_requirement", + ParseFailure(f"dependency {coordinate} requirement must be a string"), + ) + continue + try: + _bounded_text( + raw_requirement, + f"dependency {coordinate} requirement", + self.limits.max_field_bytes, + ) + except InputError as error: + self._source_failure( + blob, + "parse-zed-manifest", + "dependency_requirement_too_large", + error, + ) + continue + target = package_node_id(coordinate) + self._ensure_package_node(coordinate, exact_version=None) + self._add_edge( + source_id, + target, + edge_kind, + requirement=raw_requirement, + provenance=[ + provenance(blob, _find_toml_line(text, str(raw_coordinate))) + ], + ) + + def _parse_zed_lock(self, blob: SourceBlob) -> None: + """Record exact selections without inventing dependency topology. + + A flat lock closure proves exact selected package/version identities and + may include immutable artifact or VCS identity. It does not prove + parent-child relationships among those selections. Pins therefore live + in a dedicated evidence collection. Only a proven direct declaration + edge may be annotated with a matching unique pin. + """ + + text = _utf8(blob) + try: + document = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + self._source_failure(blob, "parse-zed-lock", "invalid_toml", error) + return + packages = document.get("package", []) + if not isinstance(packages, list): + self._source_failure( + blob, + "parse-zed-lock", + "invalid_lock_packages", + ParseFailure("lock package entries must be an array"), + ) + return + source = self.package_roots.get(blob.repository, repo_node_id(blob.repository)) + selected_by_coordinate: dict[str, list[dict[str, Any]]] = defaultdict(list) + for index, item in enumerate(packages): + if not isinstance(item, dict): + self._source_failure( + blob, + "parse-zed-lock", + "invalid_lock_package", + ParseFailure(f"lock package {index} must be a table"), + ) + continue + org, name, version = item.get("org"), item.get("name"), item.get("version") + if not all(isinstance(value, str) and value for value in (org, name, version)): + self._source_failure( + blob, + "parse-zed-lock", + "invalid_lock_package", + ParseFailure(f"lock package {index} is missing org/name/version"), + ) + continue + try: + coordinate = normalize_repo(f"{org}/{name}") + except InputError as error: + self._source_failure(blob, "parse-zed-lock", "invalid_lock_coordinate", error) + continue + try: + _bounded_text(version, f"lock version for {coordinate}", self.limits.max_field_bytes) + except InputError as error: + self._source_failure(blob, "parse-zed-lock", "invalid_lock_version", error) + continue + + sha256 = item.get("sha256") + if sha256 is not None: + if not isinstance(sha256, str) or not re.fullmatch(r"[0-9a-f]{64}", sha256): + self._source_failure( + blob, + "parse-zed-lock", + "invalid_lock_artifact_sha256", + ParseFailure(f"lock package {coordinate} has invalid sha256"), + ) + continue + + vcs_commit = item.get("vcs_commit") + if vcs_commit is not None: + if not isinstance(vcs_commit, str) or not GIT_SHA_RE.fullmatch(vcs_commit.lower()): + self._source_failure( + blob, + "parse-zed-lock", + "invalid_lock_vcs_commit", + ParseFailure(f"lock package {coordinate} has invalid vcs_commit"), + ) + continue + vcs_commit = vcs_commit.lower() + + pin: dict[str, Any] = { + "kind": "zed-lock-pin", + "topological": False, + "repository": blob.repository, + "source": source, + "package": coordinate, + "selected_version": version, + "provenance": [provenance(blob, _find_toml_line(text, str(name)))], + } + if sha256 is not None: + pin["artifact_sha256"] = sha256 + if vcs_commit is not None: + pin["vcs_commit"] = vcs_commit + + pin_key = ( + blob.repository, + source, + coordinate, + version, + sha256, + vcs_commit, + blob.repository_commit, + blob.blob_sha, + ) + if pin_key not in self._pin_keys: + if len(self.pins) >= self.limits.max_edges: + raise LimitError( + f"pin evidence limit exceeded ({len(self.pins) + 1} > {self.limits.max_edges})" + ) + self._pin_keys.add(pin_key) + self.pins.append(pin) + selected_by_coordinate[coordinate].append(pin) + + for coordinate, pins in sorted(selected_by_coordinate.items()): + selections = { + ( + str(pin["selected_version"]), + pin.get("artifact_sha256"), + pin.get("vcs_commit"), + ) + for pin in pins + } + if len(selections) > 1: + rendered_selections: list[dict[str, Any]] = [] + for version, artifact_sha256, vcs_commit in sorted( + selections, + key=lambda value: (value[0], value[1] or "", value[2] or ""), + ): + selection: dict[str, Any] = {"version": version} + if artifact_sha256 is not None: + selection["artifact_sha256"] = artifact_sha256 + if vcs_commit is not None: + selection["vcs_commit"] = vcs_commit + rendered_selections.append(selection) + self.contradictions.append( + { + "code": "conflicting-zed-lock-pins", + "source": source, + "target": package_node_id(coordinate), + "selections": rendered_selections, + } + ) + continue + + selected_version, artifact_sha256, vcs_commit = next(iter(selections)) + target = package_node_id(coordinate) + matching_pin = min( + pins, + key=lambda pin: json.dumps(pin, sort_keys=True, separators=(",", ":")), + ) + for edge in self.edges.values(): + if ( + edge.get("source") == source + and edge.get("target") == target + and edge.get("kind") in {"zed-declared", "zed-build-declared"} + ): + edge["selected_version"] = selected_version + if artifact_sha256 is not None: + edge["artifact_sha256"] = artifact_sha256 + if vcs_commit is not None: + edge["selected_commit"] = vcs_commit + edge["selection_provenance"] = list(matching_pin["provenance"]) + + pinned_coordinates = set(selected_by_coordinate) + for edge in sorted( + self.edges.values(), + key=lambda value: ( + str(value.get("source", "")), + str(value.get("target", "")), + str(value.get("kind", "")), + str(value.get("requirement", "")), + ), + ): + if edge.get("source") != source or edge.get("kind") not in { + "zed-declared", + "zed-build-declared", + }: + continue + target = str(edge.get("target", "")) + prefix = "zpkg-package:" + if not target.startswith(prefix): + continue + coordinate = target.removeprefix(prefix) + if coordinate in pinned_coordinates: + continue + contradiction: dict[str, Any] = { + "code": "declared-dependency-missing-lock-pin", + "source": source, + "target": target, + "dependency_kind": edge.get("kind"), + "declared_provenance": list(edge.get("provenance", [])), + "lock_provenance": [provenance(blob)], + } + requirement = edge.get("requirement") + if requirement is not None: + contradiction["requirement"] = requirement + self.contradictions.append(contradiction) diff --git a/harness/github_inventory_zed_workspace.py b/harness/github_inventory_zed_workspace.py new file mode 100644 index 00000000..ccadeb67 --- /dev/null +++ b/harness/github_inventory_zed_workspace.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +"""Workspace-aware Zed manifest discovery and parsing for DEN-2997.""" + +from __future__ import annotations + +from dataclasses import dataclass +import tomllib +from typing import Any, Mapping, Sequence + +from github_inventory_core import * # noqa: F401,F403 +import github_inventory_core as _core +import github_inventory_zed_parsers as _zed + +_bounded_text = _core._bounded_text +_find_toml_line = _core._find_toml_line +_utf8 = _core._utf8 +_validate_repo_path = _core._validate_repo_path +_parse_requirement = _zed._parse_requirement + + +@dataclass(frozen=True) +class WorkspaceMemberDeclaration: + parent_manifest_path: str + member_dir: str + member_manifest_path: str + + +def _manifest_dir(path: str) -> str: + return path.rsplit("/", 1)[0] if "/" in path else "" + + +def _manifest_path(directory: str) -> str: + return f"{directory}/.zpkg.toml" if directory else ".zpkg.toml" + + +def parse_zed_document(blob: SourceBlob) -> tuple[str, dict[str, Any]]: + text = _utf8(blob) + try: + document = tomllib.loads(text) + except tomllib.TOMLDecodeError as error: + raise ParseFailure(f"invalid {blob.path}: {error}") from error + if not isinstance(document, dict): + raise ParseFailure(f"{blob.path} must contain a TOML table") + return text, document + + +def _strict_relative_member_dir( + parent_manifest_path: str, + raw_member: object, + max_field_bytes: int, +) -> str: + if not isinstance(raw_member, str) or not raw_member: + raise ParseFailure("workspace member paths must be non-empty strings") + _bounded_text(raw_member, "workspace member path", max_field_bytes) + if raw_member != raw_member.strip(): + raise ParseFailure(f"workspace member path has surrounding whitespace: {raw_member!r}") + if raw_member.startswith(("/", "\\")) or "\\" in raw_member or "\x00" in raw_member: + raise ParseFailure(f"unsafe workspace member path {raw_member!r}") + parts = raw_member.split("/") + if any(part in {"", ".", ".."} for part in parts): + raise ParseFailure(f"unsafe workspace member path {raw_member!r}") + if parts[-1] == ".zpkg.toml": + raise ParseFailure("workspace members must name package directories, not manifest files") + parent_dir = _manifest_dir(parent_manifest_path) + directory = "/".join(part for part in (parent_dir, raw_member) if part) + _validate_repo_path(directory) + return directory + + +def workspace_member_declarations( + blob: SourceBlob, + *, + max_field_bytes: int, +) -> tuple[WorkspaceMemberDeclaration, ...]: + _, document = parse_zed_document(blob) + workspace = document.get("workspace") + if workspace is None: + return () + if not isinstance(workspace, Mapping): + raise ParseFailure(f"{blob.path} [workspace] must be a table") + members = workspace.get("members") + if not isinstance(members, list) or not members: + raise ParseFailure(f"{blob.path} [workspace].members must be a non-empty array") + declarations: list[WorkspaceMemberDeclaration] = [] + seen_dirs: set[str] = set() + for raw_member in members: + member_dir = _strict_relative_member_dir( + blob.path, + raw_member, + max_field_bytes, + ) + if member_dir in seen_dirs: + raise ParseFailure( + f"{blob.path} declares duplicate workspace member {member_dir!r}" + ) + seen_dirs.add(member_dir) + declarations.append( + WorkspaceMemberDeclaration( + parent_manifest_path=blob.path, + member_dir=member_dir, + member_manifest_path=_manifest_path(member_dir), + ) + ) + return tuple(declarations) + + +def normalize_local_dependency_dir( + source_manifest_path: str, + raw_path: object, + max_field_bytes: int, +) -> str: + if not isinstance(raw_path, str) or not raw_path: + raise ParseFailure("local dependency paths must be non-empty strings") + _bounded_text(raw_path, "local dependency path", max_field_bytes) + if raw_path != raw_path.strip(): + raise ParseFailure(f"local dependency path has surrounding whitespace: {raw_path!r}") + if raw_path.startswith(("/", "\\")) or "\\" in raw_path or "\x00" in raw_path: + raise ParseFailure(f"unsafe local dependency path {raw_path!r}") + stack = [part for part in _manifest_dir(source_manifest_path).split("/") if part] + for part in raw_path.split("/"): + if part in {"", "."}: + raise ParseFailure(f"unsafe local dependency path {raw_path!r}") + if part == "..": + if not stack: + raise ParseFailure(f"local dependency path escapes repository: {raw_path!r}") + stack.pop() + else: + stack.append(part) + if not stack: + raise ParseFailure(f"local dependency path resolves to repository root: {raw_path!r}") + directory = "/".join(stack) + _validate_repo_path(directory) + return directory + + +class WorkspaceZedParsingMixin(_zed.ZedParsingMixin): + """Adds workspace/member identity and local-path resolution to Zed parsing.""" + + def _parse_zed_manifests( + self, + blobs: Mapping[str, SourceBlob], + workspace_declarations: Sequence[WorkspaceMemberDeclaration], + ) -> None: + zed_blobs = { + path: blob + for path, blob in blobs.items() + if path == ".zpkg.toml" or path.endswith("/.zpkg.toml") + } + if ".zpkg.toml" not in zed_blobs: + return + + self._zed_manifest_packages: dict[str, dict[str, Any]] = {} + self._zed_path_dependencies: list[dict[str, Any]] = [] + self._zed_workspace_enabled = bool(workspace_declarations) + + for path in sorted(zed_blobs): + info = self._parse_workspace_zed_manifest(zed_blobs[path]) + self._zed_manifest_packages[path] = info + + root = self._zed_manifest_packages[".zpkg.toml"] + self.package_roots[root["repository"]] = root["node_id"] + + declared_member_paths: set[str] = set() + for declaration in workspace_declarations: + parent = self._zed_manifest_packages.get(declaration.parent_manifest_path) + member = self._zed_manifest_packages.get(declaration.member_manifest_path) + if parent is None or member is None: + raise ParseFailure( + "workspace declaration references an unparsed manifest: " + f"{declaration.parent_manifest_path} -> {declaration.member_manifest_path}" + ) + declared_member_paths.add(declaration.member_manifest_path) + parent_node = self.nodes[parent["node_id"]] + parent_metadata = parent_node.setdefault("zed_workspace", {}) + parent_members = parent_metadata.setdefault("member_manifests", []) + if declaration.member_manifest_path not in parent_members: + parent_members.append(declaration.member_manifest_path) + parent_members.sort() + member_node = self.nodes[member["node_id"]] + memberships = member_node.setdefault("zed_workspace_memberships", []) + membership = { + "parent_manifest": declaration.parent_manifest_path, + "member_manifest": declaration.member_manifest_path, + "member_dir": declaration.member_dir, + } + if membership not in memberships: + memberships.append(membership) + memberships.sort( + key=lambda item: ( + item["parent_manifest"], + item["member_manifest"], + ) + ) + self._add_edge( + parent["node_id"], + member["node_id"], + "zed-workspace-member", + source_path=declaration.member_dir, + input_name=member["name"], + provenance=[ + provenance(parent["blob"], _find_toml_line(parent["text"], "members")), + provenance(member["blob"], _find_toml_line(member["text"], "name")), + ], + ) + + if self._zed_workspace_enabled: + unexpected = sorted(set(self._zed_manifest_packages) - {".zpkg.toml"} - declared_member_paths) + if unexpected: + raise ParseFailure( + "workspace contains fetched but undeclared member manifests: " + + ", ".join(unexpected) + ) + + self._resolve_zed_path_dependencies() + + def _parse_workspace_zed_manifest(self, blob: SourceBlob) -> dict[str, Any]: + text, document = parse_zed_document(blob) + package = document.get("package") + if not isinstance(package, Mapping): + raise ParseFailure(f"{blob.path} [package] must be a table") + + repository_owner = blob.repository.split("/", 1)[0] + raw_org = package.get("org", repository_owner) + org = normalize_package_org(str(raw_org)) + name = normalize_package_name(str(package.get("name", ""))) + coordinate = f"{org}/{name}" + version = package.get("version") + exact_version = ( + normalize_exact_version(version, f"{blob.path} package.version") + if isinstance(version, str) + else None + ) + package_id = package_node_id(coordinate, exact_version) + existing = self.nodes.get(package_id) + if existing is not None: + previous_path = existing.get("zed_manifest_path") + if previous_path is not None and previous_path != blob.path: + raise ParseFailure( + f"ambiguous Zed package identity {package_id}: " + f"{previous_path} and {blob.path}" + ) + self._ensure_package_node(coordinate, exact_version) + node = self.nodes[package_id] + node["zed_manifest_path"] = blob.path + node["repository"] = blob.repository + node["repository_commit"] = blob.repository_commit + node["blob_sha"] = blob.blob_sha + + targets = document.get("targets") + if targets is not None and not isinstance(targets, Mapping): + raise ParseFailure(f"{blob.path} targets must be a table") + if isinstance(targets, Mapping): + runtime_targets: list[str] = [] + target_metadata: dict[str, Any] = {} + for target_name in sorted(targets, key=str): + raw_target = targets[target_name] + _bounded_text(str(target_name), "Zed target name", self.limits.max_field_bytes) + if not isinstance(raw_target, Mapping): + raise ParseFailure(f"Zed target {target_name!r} must be a table") + normalized_target: dict[str, Any] = {} + for key in ("kind", "runtime", "language"): + value = raw_target.get(key) + if isinstance(value, str): + _bounded_text(value, f"Zed target {target_name}.{key}", self.limits.max_field_bytes) + normalized_target[key] = value + if raw_target.get("host_only") is True: + normalized_target["host_only"] = True + if raw_target.get("dev_only") is True: + normalized_target["dev_only"] = True + target_metadata[str(target_name)] = normalized_target + runtime = normalized_target.get("runtime") or normalized_target.get("kind") + if isinstance(runtime, str): + runtime_targets.append(runtime) + node["zed_targets"] = target_metadata + if runtime_targets: + node["runtime_targets"] = sorted(set(runtime_targets)) + + containers = document.get("containers") + if containers is not None: + if not isinstance(containers, Mapping): + raise ParseFailure(f"{blob.path} containers must be a table") + normalized_containers: dict[str, Any] = {} + for container_name in sorted(containers, key=str): + raw_container = containers[container_name] + if not isinstance(raw_container, Mapping): + raise ParseFailure(f"container {container_name!r} must be a table") + normalized_container: dict[str, Any] = {} + for key in ("file", "context", "target", "platform"): + value = raw_container.get(key) + if isinstance(value, str): + _bounded_text(value, f"container {container_name}.{key}", self.limits.max_field_bytes) + normalized_container[key] = value + normalized_containers[str(container_name)] = normalized_container + node["zed_containers"] = normalized_containers + + for table_name, scope in ( + ("dependencies", "runtime"), + ("dev-dependencies", "dev"), + ("build-dependencies", "build"), + ("build_dependencies", "build"), + ): + dependencies = document.get(table_name, {}) + if not isinstance(dependencies, Mapping): + raise ParseFailure(f"{blob.path} {table_name} must be a table") + for dependency_name in sorted(dependencies, key=str): + detail = dependencies[dependency_name] + dep_name = normalize_package_name(str(dependency_name)) + dep_org = org + requirement: str | None = None + source_path: str | None = None + provenance_items = [ + provenance(blob, _find_toml_line(text, str(dependency_name))) + ] + if isinstance(detail, str): + requirement = _parse_requirement( + detail, + f"{blob.path} dependency {dependency_name}", + self.limits, + ) + elif isinstance(detail, Mapping): + detail_org = detail.get("org") + if isinstance(detail_org, str): + dep_org = normalize_package_org(detail_org) + detail_name = detail.get("package") + if isinstance(detail_name, str): + dep_name = normalize_package_name(detail_name) + detail_requirement = detail.get("version") + if isinstance(detail_requirement, str): + requirement = _parse_requirement( + detail_requirement, + f"{blob.path} dependency {dependency_name}.version", + self.limits, + ) + path_value = detail.get("path") + if isinstance(path_value, str): + source_path = path_value + else: + raise ParseFailure( + f"{blob.path} dependency {dependency_name!r} must be a string or table" + ) + + relationship = ( + "zed-dev-declared" + if scope == "dev" + else "zed-build-declared" + if scope == "build" + else "zed-declared" + ) + if source_path is not None: + self._zed_path_dependencies.append( + { + "source": package_id, + "source_manifest": blob.path, + "dependency_key": str(dependency_name), + "dependency_name": dep_name, + "dependency_org": dep_org, + "requirement": requirement, + "relationship": relationship, + "raw_path": source_path, + "provenance": provenance_items, + } + ) + continue + + target_coordinate = f"{dep_org}/{dep_name}" + target = self._ensure_package_node(target_coordinate, None) + self._add_edge( + package_id, + target, + relationship, + requirement=requirement, + provenance=provenance_items, + ) + + return { + "blob": blob, + "text": text, + "document": document, + "repository": blob.repository, + "node_id": package_id, + "org": org, + "name": name, + "coordinate": coordinate, + "version": exact_version, + "manifest_path": blob.path, + "directory": _manifest_dir(blob.path), + } + + def _resolve_zed_path_dependencies(self) -> None: + by_directory = { + info["directory"]: info for info in self._zed_manifest_packages.values() + } + for dependency in self._zed_path_dependencies: + target_dir = normalize_local_dependency_dir( + dependency["source_manifest"], + dependency["raw_path"], + self.limits.max_field_bytes, + ) + target_info = by_directory.get(target_dir) + if target_info is None: + if self._zed_workspace_enabled: + raise ParseFailure( + f"workspace dependency {dependency['source_manifest']}:" + f"{dependency['dependency_key']} targets missing member {target_dir!r}" + ) + target_coordinate = ( + f"{dependency['dependency_org']}/{dependency['dependency_name']}" + ) + target = self._ensure_package_node(target_coordinate, None) + target_manifest_path = None + target_provenance: list[Mapping[str, Any]] = [] + else: + if target_info["name"] != dependency["dependency_name"]: + raise ParseFailure( + f"local dependency {dependency['dependency_key']!r} in " + f"{dependency['source_manifest']} resolves to package " + f"{target_info['name']!r}, not {dependency['dependency_name']!r}" + ) + if target_info["org"] != dependency["dependency_org"]: + raise ParseFailure( + f"local dependency {dependency['dependency_key']!r} in " + f"{dependency['source_manifest']} resolves to org " + f"{target_info['org']!r}, not {dependency['dependency_org']!r}" + ) + target = target_info["node_id"] + target_manifest_path = target_info["manifest_path"] + target_provenance = [ + provenance( + target_info["blob"], + _find_toml_line(target_info["text"], "name"), + ) + ] + self._add_edge( + dependency["source"], + target, + dependency["relationship"], + requirement=dependency["requirement"], + source_path=dependency["raw_path"], + input_name=target_manifest_path, + provenance=[*dependency["provenance"], *target_provenance], + ) diff --git a/harness/test_github_inventory_integrity.py b/harness/test_github_inventory_integrity.py new file mode 100644 index 00000000..8c433fa5 --- /dev/null +++ b/harness/test_github_inventory_integrity.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from github_inventory_test_support import * # noqa: F401,F403 + + +class InventoryIntegrityTests(InventoryTestCase): + def test_build_dependency_aliases_emit_typed_edges_and_receive_lock_pins(self) -> None: + document = copy.deepcopy(self.fixture) + document["repositories"]["acme/app"]["files"][".zpkg.toml"] += ''' + +[build-dependencies] +"acme/lib-a" = "^1.0" + +[build_dependencies] +"acme/shared" = "^2.0" +''' + result, _ = self.build(document) + build_edges = { + edge["target"]: edge + for edge in result["edges"] + if edge["source"] == "zpkg-package:acme/app" + and edge["kind"] == "zed-build-declared" + } + self.assertEqual( + set(build_edges), + {"zpkg-package:acme/lib-a", "zpkg-package:acme/shared"}, + ) + self.assertEqual(build_edges["zpkg-package:acme/lib-a"]["selected_version"], "1.0.0") + self.assertEqual(build_edges["zpkg-package:acme/shared"]["selected_version"], "2.0.0") + self.assertEqual( + build_edges["zpkg-package:acme/lib-a"]["artifact_sha256"], "a" * 64 + ) + self.assertEqual( + build_edges["zpkg-package:acme/shared"]["artifact_sha256"], "c" * 64 + ) + + def test_optional_lock_identity_conflicts_sort_stably_without_type_errors(self) -> None: + document = copy.deepcopy(self.fixture) + document["repositories"]["acme/app"]["files"][".zpkg.lock"] += ''' + +[[package]] +org = "acme" +name = "lib-a" +version = "1.0.0" + +[[package]] +org = "acme" +name = "lib-a" +version = "1.0.0" +sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +vcs_commit = "2222222222222222222222222222222222222222" +''' + first, _ = self.build(document) + second_document = copy.deepcopy(document) + second_document["organizations"]["acme"].reverse() + second, _ = self.build(second_document) + self.assertEqual(first["contradictions"], second["contradictions"]) + conflict = next( + item + for item in first["contradictions"] + if item["code"] == "conflicting-zed-lock-pins" + and item["target"] == "zpkg-package:acme/lib-a" + ) + self.assertEqual( + conflict["selections"], + [ + {"version": "1.0.0"}, + {"version": "1.0.0", "artifact_sha256": "a" * 64}, + { + "version": "1.0.0", + "artifact_sha256": "a" * 64, + "vcs_commit": "2" * 40, + }, + ], + ) + + def test_exact_gitlinks_require_matching_type_mode_and_object_id(self) -> None: + backend = inventory.FixtureBackend(copy.deepcopy(self.fixture)) + client = inventory.GitHubClient( + inventory.FixtureTransport(backend), + inventory.Budget(inventory.Limits()), + None, + sleeper=lambda _: None, + ) + builder = inventory.InventoryBuilder( + client, + inventory.Limits(), + ["acme/app"], + [], + ["git-submodule"], + None, + ) + valid = { + "vendor/shared": { + "path": "vendor/shared", + "type": "commit", + "mode": "160000", + "sha": "4" * 40, + } + } + self.assertEqual(builder._exact_gitlinks(valid), {"vendor/shared": "4" * 40}) + + invalid_entries = [ + {"path": "vendor/shared", "type": "commit", "mode": "040000", "sha": "4" * 40}, + {"path": "vendor/shared", "type": "blob", "mode": "160000", "sha": "4" * 40}, + {"path": "vendor/shared", "type": "commit", "mode": "160000", "sha": "not-an-oid"}, + ] + for entry in invalid_entries: + with self.subTest(entry=entry), self.assertRaises(inventory.ParseFailure): + builder._exact_gitlinks({"vendor/shared": entry}) + + with self.assertRaises(inventory.ParseFailure): + builder._tree_entries( + [ + {"path": "vendor/shared", "type": "commit", "mode": "160000", "sha": "4" * 40}, + {"path": "vendor/shared", "type": "blob", "mode": "100644", "sha": "5" * 40}, + ] + ) diff --git a/harness/test_github_inventory_lock_evidence.py b/harness/test_github_inventory_lock_evidence.py new file mode 100644 index 00000000..b27c07ce --- /dev/null +++ b/harness/test_github_inventory_lock_evidence.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import copy + +from github_inventory_test_support import InventoryTestCase, inventory + + +class LockEvidenceTests(InventoryTestCase): + def test_flat_lock_closure_is_evidence_not_topology(self) -> None: + result, _ = self.build() + + self.assertEqual( + [(pin["package"], pin["selected_version"]) for pin in result["pins"]], + [ + ("acme/lib-a", "1.0.0"), + ("acme/lib-b", "1.0.0"), + ("acme/shared", "2.0.0"), + ], + ) + self.assertTrue(all(pin["topological"] is False for pin in result["pins"])) + self.assertFalse(any(edge["kind"] == "zed-lock-pin" for edge in result["edges"])) + + root = "zpkg-package:acme/app" + shared = "zpkg-package:acme/shared" + self.assertFalse( + any( + edge["source"] == root + and edge["target"] == shared + and edge["kind"].startswith("zed-") + for edge in result["edges"] + ), + "the transitive shared pin must not become a fabricated root -> shared edge", + ) + + direct_edges = { + edge["target"]: edge + for edge in result["edges"] + if edge["source"] == root and edge["kind"] == "zed-declared" + } + self.assertEqual(direct_edges["zpkg-package:acme/lib-a"]["selected_version"], "1.0.0") + self.assertEqual(direct_edges["zpkg-package:acme/lib-b"]["selected_version"], "1.0.0") + self.assertEqual( + direct_edges["zpkg-package:acme/lib-a"]["artifact_sha256"], + "a" * 64, + ) + self.assertEqual( + direct_edges["zpkg-package:acme/lib-b"]["artifact_sha256"], + "b" * 64, + ) + self.assertTrue(direct_edges["zpkg-package:acme/lib-a"]["selection_provenance"]) + + node_ids = {node["id"] for node in result["nodes"]} + self.assertFalse(any("@1.0.0" in node_id or "@2.0.0" in node_id for node_id in node_ids)) + analyzed_nodes = { + node_id + for component in result["strongly_connected_components"] + for node_id in component["nodes"] + } + self.assertEqual(analyzed_nodes, node_ids) + + def test_renderers_mark_pins_as_disconnected_evidence(self) -> None: + result, _ = self.build() + dot = inventory.render_dot(result) + mermaid = inventory.render_mermaid(result) + + self.assertEqual(dot.count("lock pin (evidence only)"), 3) + self.assertIn('tooltip="non-topological lock evidence"', dot) + self.assertNotIn('label="zed-lock-pin', dot) + self.assertEqual(mermaid.count("lock pin (evidence only)"), 3) + self.assertIn("classDef lockEvidence stroke-dasharray: 5 5", mermaid) + self.assertNotIn('-->|"zed-lock-pin', mermaid) + + def test_optional_vcs_commit_is_preserved_as_pin_evidence(self) -> None: + document = copy.deepcopy(self.fixture) + lock = document["repositories"]["acme/app"]["files"][".zpkg.lock"] + needle = ( + 'name = "shared"\n' + 'version = "2.0.0"\n' + f'sha256 = "{"c" * 64}"\n' + ) + document["repositories"]["acme/app"]["files"][".zpkg.lock"] = lock.replace( + needle, + needle + f'vcs_commit = "{"4" * 40}"\n', + ) + + result, _ = self.build(document) + shared_pin = next(pin for pin in result["pins"] if pin["package"] == "acme/shared") + self.assertEqual(shared_pin["vcs_commit"], "4" * 40) + self.assertFalse( + any( + edge["source"] == "zpkg-package:acme/app" + and edge["target"] == "zpkg-package:acme/shared" + for edge in result["edges"] + ) + ) + + def test_declared_dependency_without_lock_pin_is_explicit_contradiction(self) -> None: + document = copy.deepcopy(self.fixture) + lock = document["repositories"]["acme/app"]["files"][".zpkg.lock"] + lib_b_block = f'''\n[[package]] +org = "acme" +name = "lib-b" +version = "1.0.0" +sha256 = "{"b" * 64}" +''' + self.assertIn(lib_b_block, lock) + document["repositories"]["acme/app"]["files"][".zpkg.lock"] = lock.replace( + lib_b_block, + "", + ) + + result, _ = self.build(document) + contradiction = next( + item + for item in result["contradictions"] + if item["code"] == "declared-dependency-missing-lock-pin" + ) + self.assertEqual(contradiction["source"], "zpkg-package:acme/app") + self.assertEqual(contradiction["target"], "zpkg-package:acme/lib-b") + self.assertEqual(contradiction["requirement"], "^1.0") + self.assertTrue(contradiction["declared_provenance"]) + self.assertTrue(contradiction["lock_provenance"]) + self.assertFalse(any(pin["package"] == "acme/lib-b" for pin in result["pins"])) + lib_b_edge = next( + edge + for edge in result["edges"] + if edge["source"] == "zpkg-package:acme/app" + and edge["target"] == "zpkg-package:acme/lib-b" + and edge["kind"] == "zed-declared" + ) + self.assertNotIn("selected_version", lib_b_edge) + + def test_invalid_artifact_digest_is_explicit_and_not_annotated(self) -> None: + document = copy.deepcopy(self.fixture) + lock = document["repositories"]["acme/app"]["files"][".zpkg.lock"] + document["repositories"]["acme/app"]["files"][".zpkg.lock"] = lock.replace( + f'sha256 = "{"a" * 64}"', + 'sha256 = "not-a-sha256"', + ) + + result, _ = self.build(document) + self.assertEqual(result["completeness"]["inventory"], "partial") + self.assertTrue( + any(failure["code"] == "invalid_lock_artifact_sha256" for failure in result["failures"]) + ) + self.assertTrue( + any( + item["code"] == "declared-dependency-missing-lock-pin" + and item["target"] == "zpkg-package:acme/lib-a" + for item in result["contradictions"] + ) + ) + self.assertFalse(any(pin["package"] == "acme/lib-a" for pin in result["pins"])) + lib_a_edge = next( + edge + for edge in result["edges"] + if edge["source"] == "zpkg-package:acme/app" + and edge["target"] == "zpkg-package:acme/lib-a" + and edge["kind"] == "zed-declared" + ) + self.assertNotIn("selected_version", lib_a_edge) + + def test_duplicate_identical_lock_entries_are_deduplicated(self) -> None: + document = copy.deepcopy(self.fixture) + document["repositories"]["acme/app"]["files"][".zpkg.lock"] += f'''\n[[package]] +org = "acme" +name = "lib-a" +version = "1.0.0" +sha256 = "{"a" * 64}" +''' + result, _ = self.build(document) + self.assertEqual( + sum(1 for pin in result["pins"] if pin["package"] == "acme/lib-a"), + 1, + ) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/harness/test_github_inventory_semantics.py b/harness/test_github_inventory_semantics.py new file mode 100644 index 00000000..28e14ddb --- /dev/null +++ b/harness/test_github_inventory_semantics.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from github_inventory_test_support import * # noqa: F401,F403 + + +class InventorySemanticsTests(InventoryTestCase): + def test_golden_json_dot_and_mermaid_are_byte_identical(self) -> None: + result, _ = self.build() + for output_format, filename in ( + ("json", "inventory.json"), + ("dot", "inventory.dot"), + ("mermaid", "inventory.mmd"), + ): + actual = inventory.render_inventory(result, output_format) + expected = (GOLDEN / filename).read_bytes() + self.assertEqual(actual.encode("utf-8"), expected, output_format) + + def test_org_pagination_is_complete_deduplicated_and_archived_is_explicit(self) -> None: + result, transport = self.build(repositories=[]) + pages = [path for path in transport.requests if path.startswith("/orgs/acme/repos?")] + self.assertEqual(len(pages), 3) + records = {item["full_name"]: item for item in result["repositories"]} + self.assertEqual(len(records), 7) + self.assertTrue(records["acme/archived"]["archived"]) + self.assertEqual(records["acme/archived"]["status"], "scanned") + self.assertEqual( + records["acme/archived"]["missing_manifests"], + [".gitmodules", ".zpkg.lock", ".zpkg.toml", "flake.lock", "flake.nix", "nix/sources.json", "npins/sources.json"], + ) + self.assertTrue(records["acme/app"]["private"]) + self.assertEqual(result["completeness"]["inventory"], "complete") + self.assertEqual(result["completeness"]["resolution"], "not-claimed") + + def test_cycle_scc_and_dependency_first_waves_are_deterministic(self) -> None: + result, _ = self.build() + self.assertIn( + ["zpkg-package:acme/cycle-a", "zpkg-package:acme/cycle-b"], + result["cycles"], + ) + component_for: dict[str, str] = {} + for component in result["strongly_connected_components"]: + for node in component["nodes"]: + component_for[node] = component["id"] + wave_for = { + component: index + for index, wave in enumerate(result["topological_waves"]) + for component in wave + } + shared = component_for["zpkg-package:acme/shared"] + lib_a = component_for["zpkg-package:acme/lib-a"] + lib_b = component_for["zpkg-package:acme/lib-b"] + self.assertLess(wave_for[shared], wave_for[lib_a]) + self.assertLess(wave_for[shared], wave_for[lib_b]) + incoming = { + edge["source"] + for edge in result["edges"] + if edge["target"] == "zpkg-package:acme/shared" and edge["kind"] == "zed-declared" + } + self.assertEqual( + incoming, + {"zpkg-package:acme/lib-a", "zpkg-package:acme/lib-b"}, + ) + + def test_hostile_labels_are_escaped_in_dot_and_mermaid(self) -> None: + result, _ = self.build() + dot = inventory.render_dot(result) + mermaid = inventory.render_mermaid(result) + self.assertIn(r'^1.0\" ]\nmalicious', dot) + self.assertNotIn("\nmalicious\"]", dot) + self.assertIn("^1.0" ] malicious", mermaid) + self.assertNotIn('^1.0" ]\nmalicious', mermaid) + for node in result["nodes"]: + self.assertRegex(inventory.render_node_id(node["id"]), r"^n_[0-9a-f]{20}$") + + def test_duplicate_edges_merge_provenance_and_sort_stably(self) -> None: + document = copy.deepcopy(self.fixture) + document["repositories"]["acme/lib-a"]["files"]["flake.nix"] = ( + '{ inputs.shared.url = "github:acme/shared"; }\n' + ) + first, _ = self.build(document) + reversed_document = copy.deepcopy(document) + reversed_document["organizations"]["acme"].reverse() + reversed_document["repositories"] = dict( + reversed(list(reversed_document["repositories"].items())) + ) + second, _ = self.build(reversed_document) + self.assertEqual(first, second) + self.assertEqual( + inventory.render_inventory(first, "json"), + inventory.render_inventory(second, "json"), + ) + + def test_conflicting_lock_pins_are_diagnostic_not_silently_collapsed(self) -> None: + document = copy.deepcopy(self.fixture) + document["repositories"]["acme/app"]["files"][".zpkg.lock"] += ''' +[[package]] +org = "acme" +name = "lib-a" +version = "2.0.0" +sha256 = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" +''' + result, _ = self.build(document) + conflicts = [ + item for item in result["contradictions"] if item["code"] == "conflicting-zed-lock-pins" + ] + self.assertEqual(len(conflicts), 1) + self.assertEqual( + [selection["version"] for selection in conflicts[0]["selections"]], + ["1.0.0", "2.0.0"], + ) + + def test_tree_truncation_is_explicit_and_never_claims_complete(self) -> None: + document = copy.deepcopy(self.fixture) + document["repositories"]["acme/lib-b"]["tree_truncated"] = True + result, _ = self.build(document) + self.assertEqual(result["completeness"]["inventory"], "partial") + failure = next(item for item in result["failures"] if item["repository"] == "acme/lib-b") + self.assertEqual(failure["code"], "github_tree_truncated") + record = next(item for item in result["repositories"] if item["full_name"] == "acme/lib-b") + self.assertEqual(record["status"], "failed") + + def test_token_is_environment_only_and_redacted_from_failures(self) -> None: + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as captured: + inventory.build_parser().parse_args(["--org", "acme", "--token", "secret"]) + self.assertEqual(captured.exception.code, 2) + + token = "fixture-super-secret-token" + document = copy.deepcopy(self.fixture) + document["require_token"] = True + document["repositories"]["acme/app"]["failures"] = { + "tree": {"status": 500, "body": {"message": token}} + } + result, transport = self.build(document, token=token) + self.assertEqual(result["completeness"]["inventory"], "partial") + serialized = inventory.render_inventory(result, "json") + self.assertNotIn(token, serialized) + self.assertTrue(transport.requests) + self.assertNotIn(token, "\n".join(transport.requests)) + + def test_iterative_scc_handles_ten_thousand_node_chain(self) -> None: + count = 10_000 + nodes = { + f"node-{index:05d}": {"id": f"node-{index:05d}"} + for index in range(count) + } + edges = [ + { + "source": f"node-{index:05d}", + "target": f"node-{index + 1:05d}", + } + for index in range(count - 1) + ] + analysis = inventory.analyze_graph(nodes, edges) + self.assertEqual(len(analysis["components"]), count) + self.assertEqual(len(analysis["waves"]), count) + self.assertFalse(analysis["cycles"]) + + def test_no_token_or_authorization_material_is_accepted_in_output_paths(self) -> None: + result, _ = self.build(token="secret-token") + serialized = inventory.render_inventory(result, "json") + self.assertNotIn("secret-token", serialized) + self.assertNotIn("Authorization", serialized) + self.assertNotIn("Bearer", serialized) diff --git a/harness/test_github_inventory_transport.py b/harness/test_github_inventory_transport.py new file mode 100644 index 00000000..8daaedcb --- /dev/null +++ b/harness/test_github_inventory_transport.py @@ -0,0 +1,291 @@ +from __future__ import annotations + +import base64 +import hashlib + +from github_inventory_test_support import * # noqa: F401,F403 + + +class InventoryTransportTests(InventoryTestCase): + def test_fake_http_server_covers_private_auth_and_pagination(self) -> None: + document = copy.deepcopy(self.fixture) + document["require_token"] = True + token = "http-fixture-token" + with FakeServer(document) as server: + result = inventory.run_inventory( + repositories=[], + organizations=["acme"], + includes=["zed,git-submodule,nix"], + limits=inventory.Limits(max_seconds=10), + transport=inventory.HttpTransport(server.base_url), + token=token, + sleeper=lambda _: None, + ) + self.assertEqual(result["completeness"]["inventory"], "complete") + self.assertGreaterEqual(len(server.handler.seen_authorization), 3) + self.assertTrue( + all(value == f"Bearer {token}" for value in server.handler.seen_authorization) + ) + + def test_transient_retries_are_bounded(self) -> None: + with FakeServer(copy.deepcopy(self.fixture), transient=2) as server: + result = inventory.run_inventory( + repositories=[], + organizations=["acme"], + includes=["zed"], + limits=inventory.Limits(max_seconds=10, max_retries=2), + transport=inventory.HttpTransport(server.base_url), + token=None, + sleeper=lambda _: None, + ) + self.assertEqual(result["completeness"]["inventory"], "complete") + org_requests = sum(1 for _ in server.handler.seen_authorization) + # 3 pagination requests plus exactly 2 retry attempts, followed by + # repository commit/tree/blob requests. + self.assertGreaterEqual(org_requests, 5) + + def test_redirects_are_not_followed_and_cross_origin_links_are_rejected(self) -> None: + token = "redirect-secret" + with FakeServer(copy.deepcopy(self.fixture), redirect_org=True) as server: + result = inventory.run_inventory( + repositories=[], + organizations=["acme"], + includes=["zed"], + limits=inventory.Limits(max_seconds=10), + transport=inventory.HttpTransport(server.base_url), + token=token, + sleeper=lambda _: None, + ) + self.assertEqual(result["completeness"]["inventory"], "partial") + self.assertEqual(len(server.handler.seen_authorization), 1) + self.assertNotIn(token, inventory.render_inventory(result, "json")) + transport = inventory.HttpTransport("https://api.github.com") + with self.assertRaises(inventory.ApiError) as captured: + transport.relative_from_link("https://attacker.invalid/page=2") + self.assertEqual(captured.exception.code, "cross_origin_pagination_link") + + def test_ghe_api_prefix_pagination_is_normalized_without_duplication(self) -> None: + transport = inventory.HttpTransport("https://github.example/api/v3/") + current = "/orgs/acme/repos?direction=asc&page=1&per_page=100" + expected = "/orgs/acme/repos?direction=asc&page=2&per_page=100" + + self.assertEqual( + transport.relative_from_link( + "https://github.example/api/v3/orgs/acme/repos?direction=asc&page=2&per_page=100", + current_path=current, + ), + expected, + ) + self.assertEqual( + transport.relative_from_link( + "/api/v3/orgs/acme/repos?direction=asc&page=2&per_page=100", + current_path=current, + ), + expected, + ) + self.assertEqual( + transport.relative_from_link("?direction=asc&page=2&per_page=100", current_path=current), + expected, + ) + self.assertEqual( + transport.absolute_request_url(expected), + "https://github.example/api/v3" + expected, + ) + with self.assertRaises(inventory.ApiError) as captured: + transport.relative_from_link( + "https://github.example/outside?page=2", current_path=current + ) + self.assertEqual(captured.exception.code, "pagination_link_outside_api_base") + + def test_api_base_rejects_credentials_queries_fragments_and_dot_segments(self) -> None: + unsafe = [ + "https://user:pass@github.example/api/v3", + "https://github.example/api/v3?token=secret", + "https://github.example/api/v3#fragment", + "https://github.example/api/../v3", + "https://github.example/api/%2e%2e/v3", + "https://github.example/api//v3", + "https://github.example/api/%5c/v3", + ] + for value in unsafe: + with self.subTest(value=value), self.assertRaises(inventory.InputError): + inventory.HttpTransport(value) + + def test_hard_limits_fail_closed_and_preserve_existing_atomic_output(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "inventory.json" + output.write_text("sentinel\n", encoding="utf-8") + stderr = io.StringIO() + argv = [ + "--fixture", + str(FIXTURE_PATH), + "--org", + "acme", + "--max-repositories", + "1", + "--output", + str(output), + ] + with contextlib.redirect_stderr(stderr): + code = inventory.main(argv) + self.assertEqual(code, 3) + self.assertEqual(output.read_text(encoding="utf-8"), "sentinel\n") + self.assertFalse(list(output.parent.glob(f".{output.name}.*.tmp"))) + self.assertIn("repository limit exceeded", stderr.getvalue()) + + def test_response_and_manifest_byte_limits_fail_closed(self) -> None: + with self.assertRaises(inventory.LimitError): + self.build(limits=inventory.Limits(max_response_bytes=10, max_seconds=10)) + with self.assertRaises(inventory.LimitError): + self.build(limits=inventory.Limits(max_manifest_bytes=16, max_seconds=10)) + + def test_json_depth_limit_fails_closed(self) -> None: + document = copy.deepcopy(self.fixture) + nested: dict[str, Any] = {"owner": "acme", "repo": "shared", "rev": "4" * 40} + for index in range(8): + nested = {f"level-{index}": nested} + document["repositories"]["acme/app"]["files"]["nix/sources.json"] = json.dumps(nested) + with self.assertRaises(inventory.LimitError): + self.build(document, limits=inventory.Limits(max_json_depth=3, max_seconds=10)) + + def test_atomic_write_replaces_content_without_leaving_temporary_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "graph.dot" + output.write_text("old\n", encoding="utf-8") + inventory.write_atomic(output, "new\n") + self.assertEqual(output.read_text(encoding="utf-8"), "new\n") + self.assertFalse(list(output.parent.glob(f".{output.name}.*.tmp"))) + + def test_plain_http_non_loopback_and_unsafe_fixture_paths_are_rejected(self) -> None: + with self.assertRaises(inventory.InputError): + inventory.HttpTransport("http://example.com") + document = copy.deepcopy(self.fixture) + document["repositories"]["acme/app"]["files"]["../escape"] = "x" + with self.assertRaises(inventory.InputError): + inventory.FixtureBackend(document) + + def test_rate_limit_403_retries_are_bounded(self) -> None: + with FakeServer(copy.deepcopy(self.fixture), transient=2, transient_status=403) as server: + result = inventory.run_inventory( + repositories=[], + organizations=["acme"], + includes=["zed"], + limits=inventory.Limits(max_seconds=10, max_retries=2), + transport=inventory.HttpTransport(server.base_url), + token=None, + sleeper=lambda _: None, + ) + self.assertEqual(result["completeness"]["inventory"], "complete") + self.assertGreaterEqual(len(server.handler.seen_authorization), 5) + + def test_success_response_body_is_bounded_during_http_read(self) -> None: + with FakeServer(copy.deepcopy(self.fixture), oversized_org_bytes=4096) as server: + with self.assertRaises(inventory.LimitError): + inventory.run_inventory( + repositories=[], + organizations=["acme"], + includes=["zed"], + limits=inventory.Limits(max_seconds=10, max_response_bytes=128), + transport=inventory.HttpTransport(server.base_url), + token=None, + sleeper=lambda _: None, + ) + + def test_custom_api_origin_requires_explicit_token_opt_in(self) -> None: + transport = inventory.HttpTransport("https://github.example") + with self.assertRaises(inventory.InputError): + inventory.GitHubClient( + transport, + inventory.Budget(inventory.Limits()), + "secret-token", + sleeper=lambda _: None, + ) + allowed = inventory.HttpTransport( + "https://github.example", + allow_token_to_custom_origin=True, + ) + client = inventory.GitHubClient( + allowed, + inventory.Budget(inventory.Limits()), + "secret-token", + sleeper=lambda _: None, + ) + self.assertEqual(client.token, "secret-token") + + def test_github_blob_base64_line_wrapping_is_supported(self) -> None: + data = b"abc" + sha = inventory.git_blob_sha(data) + + class WrappedBlobTransport(inventory.Transport): + base_url = "https://api.github.com" + _base_path = "" + + def request(self, path, headers, timeout, max_bytes): + del path, headers, timeout, max_bytes + body = json.dumps( + { + "sha": sha, + "size": len(data), + "encoding": "base64", + "content": "Y\nWJ\nj", + } + ).encode("utf-8") + return inventory.ApiResponse(200, {}, body) + + client = inventory.GitHubClient( + WrappedBlobTransport(), + inventory.Budget(inventory.Limits()), + None, + sleeper=lambda _: None, + ) + self.assertEqual(client.get_blob("acme/app", sha, len(data)), data) + + def test_blob_object_ids_are_recomputed_for_sha1_and_sha256(self) -> None: + data = b"object identity" + header = f"blob {len(data)}\0".encode("ascii") + sha1 = hashlib.sha1(header + data, usedforsecurity=False).hexdigest() + sha256 = hashlib.sha256(header + data).hexdigest() + + class BlobTransport(inventory.Transport): + base_url = "https://api.github.com" + _base_path = "" + + def __init__(self, response_sha: str, response_data: bytes) -> None: + self.response_sha = response_sha + self.response_data = response_data + + def request(self, path, headers, timeout, max_bytes): + del path, headers, timeout, max_bytes + body = json.dumps( + { + "sha": self.response_sha, + "size": len(self.response_data), + "encoding": "base64", + "content": base64.b64encode(self.response_data).decode("ascii"), + } + ).encode("utf-8") + return inventory.ApiResponse(200, {}, body) + + for object_id in (sha1, sha256): + with self.subTest(object_id=object_id): + client = inventory.GitHubClient( + BlobTransport(object_id, data), + inventory.Budget(inventory.Limits()), + None, + sleeper=lambda _: None, + ) + self.assertEqual(client.get_blob("acme/app", object_id, len(data)), data) + + client = inventory.GitHubClient( + BlobTransport(sha1, b"tampered"), + inventory.Budget(inventory.Limits()), + None, + sleeper=lambda _: None, + ) + with self.assertRaises(inventory.ApiError) as captured: + client.get_blob("acme/app", sha1, None) + self.assertEqual(captured.exception.code, "blob_object_id_mismatch") + + def test_field_size_limit_is_fail_closed(self) -> None: + with self.assertRaises(inventory.LimitError): + self.build(limits=inventory.Limits(max_field_bytes=4, max_seconds=10)) diff --git a/harness/test_github_inventory_workspace.py b/harness/test_github_inventory_workspace.py new file mode 100644 index 00000000..1984e023 --- /dev/null +++ b/harness/test_github_inventory_workspace.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +from github_inventory_test_support import * # noqa: F401,F403 + + +ROOT_MANIFEST = ''' +[package] +name = "workspace-monorepo" +version = "0.1.0" + +[workspace] +members = ["packages/core", "packages/utils", "apps/cli"] + +[targets.repository] +dir = "." +''' + +CORE_MANIFEST = ''' +[package] +name = "workspace-core" +version = "0.1.0" + +[dependencies] +lodash = "^4.17.21" +''' + +UTILS_MANIFEST = ''' +[package] +name = "workspace-utils" +version = "0.1.0" + +[dependencies] +workspace-core = { path = "../core" } +''' + +CLI_MANIFEST = ''' +[package] +name = "workspace-cli" +version = "0.1.0" + +[dependencies] +workspace-core = { path = "../../packages/core" } +workspace-utils = { path = "../../packages/utils" } +''' + + +class WorkspaceInventoryTests(InventoryTestCase): + def workspace_fixture(self) -> dict[str, object]: + document = copy.deepcopy(self.fixture) + document["organizations"] = {"acme": ["acme/app"]} + repository = document["repositories"]["acme/app"] + repository["files"] = { + ".zpkg.toml": ROOT_MANIFEST, + "packages/core/.zpkg.toml": CORE_MANIFEST, + "packages/utils/.zpkg.toml": UTILS_MANIFEST, + "apps/cli/.zpkg.toml": CLI_MANIFEST, + } + repository["gitlinks"] = {} + document["repositories"] = {"acme/app": repository} + return document + + def build_workspace(self, document: dict[str, object] | None = None, *, limits=None): + return self.build( + document or self.workspace_fixture(), + repositories=["acme/app"], + organizations=["acme"], + includes=["zed"], + limits=limits, + ) + + def test_workspace_root_members_and_local_dependencies_are_complete(self) -> None: + result, transport = self.build_workspace() + record = result["repositories"][0] + self.assertEqual(record["status"], "scanned") + self.assertEqual( + [item["path"] for item in record["manifests"]], + [ + ".zpkg.toml", + "apps/cli/.zpkg.toml", + "packages/core/.zpkg.toml", + "packages/utils/.zpkg.toml", + ], + ) + self.assertEqual(len(record["zed_workspace_members"]), 3) + node_ids = {node["id"] for node in result["nodes"]} + + def package_id(name: str) -> str: + prefix = f"zpkg-package:acme/{name}" + matches = sorted( + node_id + for node_id in node_ids + if node_id == prefix or node_id.startswith(prefix + "@") + ) + self.assertEqual(len(matches), 1, matches) + return matches[0] + + root_id = package_id("workspace-monorepo") + core_id = package_id("workspace-core") + utils_id = package_id("workspace-utils") + cli_id = package_id("workspace-cli") + self.assertEqual(result["package_roots"]["acme/app"], root_id) + + memberships = [ + edge for edge in result["edges"] if edge["kind"] == "zed-workspace-member" + ] + self.assertEqual(len(memberships), 3) + self.assertEqual( + {edge["target"] for edge in memberships}, + {core_id, utils_id, cli_id}, + ) + local_edges = { + (edge["source"], edge["target"]) + for edge in result["edges"] + if edge["kind"] == "zed-declared" and edge.get("source_path") + } + self.assertEqual( + local_edges, + { + (utils_id, core_id), + (cli_id, core_id), + (cli_id, utils_id), + }, + ) + manifest_blob_requests = [ + request for request in transport.requests if "/git/blobs/" in request + ] + self.assertEqual(len(manifest_blob_requests), 4) + for membership in memberships: + self.assertEqual(len(membership["provenance"]), 2) + self.assertEqual( + {item["repository_commit"] for item in membership["provenance"]}, + {"1111111111111111111111111111111111111111"}, + ) + + def test_workspace_rendering_is_deterministic_in_all_formats(self) -> None: + first, _ = self.build_workspace() + document = self.workspace_fixture() + files = document["repositories"]["acme/app"]["files"] + document["repositories"]["acme/app"]["files"] = dict( + reversed(list(files.items())) + ) + second, _ = self.build_workspace(document) + self.assertEqual(first, second) + for output_format in ("json", "dot", "mermaid"): + self.assertEqual( + inventory.render_inventory(first, output_format), + inventory.render_inventory(second, output_format), + ) + + def assert_workspace_failure(self, document: dict[str, object], text: str) -> None: + result, _ = self.build_workspace(document) + record = result["repositories"][0] + self.assertEqual(record["status"], "failed") + self.assertIn(text, record["failure_message"]) + self.assertEqual(result["completeness"]["inventory"], "partial") + + def test_workspace_member_path_escape_is_rejected(self) -> None: + document = self.workspace_fixture() + document["repositories"]["acme/app"]["files"][".zpkg.toml"] = ( + ROOT_MANIFEST.replace( + 'members = ["packages/core", "packages/utils", "apps/cli"]', + 'members = ["../outside"]', + ) + ) + self.assert_workspace_failure(document, "unsafe workspace member path") + + def test_duplicate_workspace_member_is_rejected(self) -> None: + document = self.workspace_fixture() + document["repositories"]["acme/app"]["files"][".zpkg.toml"] = ( + ROOT_MANIFEST.replace( + 'members = ["packages/core", "packages/utils", "apps/cli"]', + 'members = ["packages/core", "packages/core"]', + ) + ) + self.assert_workspace_failure(document, "duplicate workspace member") + + def test_missing_declared_member_is_rejected(self) -> None: + document = self.workspace_fixture() + del document["repositories"]["acme/app"]["files"]["packages/core/.zpkg.toml"] + self.assert_workspace_failure(document, "is missing packages/core/.zpkg.toml") + + def test_ambiguous_member_identity_is_rejected(self) -> None: + document = self.workspace_fixture() + document["repositories"]["acme/app"]["files"]["packages/utils/.zpkg.toml"] = ( + CORE_MANIFEST + ) + self.assert_workspace_failure(document, "ambiguous Zed package identity") + + def test_missing_local_member_dependency_is_rejected(self) -> None: + document = self.workspace_fixture() + document["repositories"]["acme/app"]["files"]["packages/utils/.zpkg.toml"] = ( + UTILS_MANIFEST.replace('../core', '../missing') + ) + self.assert_workspace_failure(document, "targets missing member") + + def test_local_dependency_repo_escape_is_rejected(self) -> None: + document = self.workspace_fixture() + document["repositories"]["acme/app"]["files"]["packages/utils/.zpkg.toml"] = ( + UTILS_MANIFEST.replace('../core', '../../../outside') + ) + self.assert_workspace_failure(document, "escapes repository") + + def test_workspace_depth_limit_is_fail_closed(self) -> None: + document = self.workspace_fixture() + limits = inventory.Limits(max_json_depth=0) + result, _ = self.build_workspace(document, limits=limits) + record = result["repositories"][0] + self.assertEqual(record["status"], "failed") + self.assertIn("workspace nesting exceeded", record["failure_message"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/harness/verify_zed_cli_graph_github_production.py b/harness/verify_zed_cli_graph_github_production.py new file mode 100644 index 00000000..4e6e0ccd --- /dev/null +++ b/harness/verify_zed_cli_graph_github_production.py @@ -0,0 +1,1077 @@ +#!/usr/bin/env python3 +"""Cross-platform production verifier for ``zed graph github``. + +No third-party packages are used. The verifier runs the candidate against a +bounded in-process GitHub fake, checks deterministic JSON/DOT/Mermaid bytes, +exercises pagination/redirect/truncated-tree failures, and optionally performs +a redacted Linux-only live smoke against ``zed-pkg-test``. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import urllib.parse +from dataclasses import dataclass +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +MAX_CAPTURE_BYTES = 1_048_576 +MAX_EVIDENCE_BYTES = 262_144 +DEFAULT_TIMEOUT_SECONDS = 45.0 +LIVE_TIMEOUT_SECONDS = 180.0 +TOKEN_SENTINEL = "zpkg-test-token-7d8e1af1-redaction-sentinel" +SCHEMA = "zpkg/github-dependency-inventory/v1" +REQUIRED_HELP_FLAGS = ( + "--repo", + "--org", + "--include", + "--format", + "--output", + "--api-base-url", +) +FORBIDDEN_SECRET_FLAG = re.compile( + r"--[a-z0-9][a-z0-9-]*(?:token|password|passwd|secret|credential|private-key)[a-z0-9-]*", + re.IGNORECASE, +) + + +class VerificationError(RuntimeError): + """A bounded, actionable conformance failure.""" + + +@dataclass(frozen=True) +class CommandResult: + argv: tuple[str, ...] + returncode: int + stdout: bytes + stderr: bytes + elapsed_seconds: float + + +@dataclass(frozen=True) +class RepoFixture: + full_name: str + default_branch: str + commit_sha: str + tree_sha: str + archived: bool + private: bool + files: Mapping[str, bytes] + gitlinks: Mapping[str, str] + + +class _BoundedReader(threading.Thread): + def __init__( + self, + stream: Any, + limit: int, + process: subprocess.Popen[bytes], + exceeded: threading.Event, + ) -> None: + super().__init__(daemon=True) + self._stream = stream + self._limit = limit + self._process = process + self._exceeded = exceeded + self.data = bytearray() + self.error: BaseException | None = None + + def run(self) -> None: + try: + while True: + chunk = self._stream.read(8192) + if not chunk: + return + remaining = self._limit - len(self.data) + if remaining > 0: + self.data.extend(chunk[:remaining]) + if len(chunk) > remaining: + self._exceeded.set() + self._process.kill() + return + except BaseException as error: # pragma: no cover - defensive thread path + self.error = error + self._exceeded.set() + self._process.kill() + + +def _redact(data: bytes, secrets: Iterable[str]) -> bytes: + rendered = data + for secret in secrets: + if secret: + rendered = rendered.replace(secret.encode("utf-8"), b"[REDACTED]") + return rendered + + +def run_bounded( + argv: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str], + timeout_seconds: float, + secrets: Iterable[str] = (), + max_capture_bytes: int = MAX_CAPTURE_BYTES, +) -> CommandResult: + started = time.monotonic() + process = subprocess.Popen( + [str(item) for item in argv], + cwd=cwd, + env=dict(env), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert process.stdout is not None + assert process.stderr is not None + exceeded = threading.Event() + stdout_reader = _BoundedReader(process.stdout, max_capture_bytes, process, exceeded) + stderr_reader = _BoundedReader(process.stderr, max_capture_bytes, process, exceeded) + stdout_reader.start() + stderr_reader.start() + try: + process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired as error: + process.kill() + process.wait(timeout=10) + raise VerificationError( + f"candidate exceeded {timeout_seconds:.0f}s: {' '.join(argv[:24])}" + ) from error + finally: + stdout_reader.join(timeout=10) + stderr_reader.join(timeout=10) + + if stdout_reader.error is not None: + raise VerificationError(f"failed reading stdout: {stdout_reader.error}") + if stderr_reader.error is not None: + raise VerificationError(f"failed reading stderr: {stderr_reader.error}") + if exceeded.is_set(): + raise VerificationError( + f"candidate output exceeded {max_capture_bytes} bytes: {' '.join(argv[:24])}" + ) + + secret_values = tuple(secret for secret in secrets if secret) + return CommandResult( + argv=tuple(str(item) for item in argv), + returncode=process.returncode, + stdout=_redact(bytes(stdout_reader.data), secret_values), + stderr=_redact(bytes(stderr_reader.data), secret_values), + elapsed_seconds=time.monotonic() - started, + ) + + +def _git_blob_sha(content: bytes) -> str: + return hashlib.sha1(f"blob {len(content)}\0".encode("ascii") + content).hexdigest() + + +def _sha40(label: str) -> str: + return hashlib.sha1(label.encode("utf-8")).hexdigest() + + +def _compact_json(value: Any) -> bytes: + return ( + json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + "\n" + ).encode("utf-8") + + +def _fixture_repositories() -> dict[str, RepoFixture]: + hostile_requirement = '^1.0"]\nmalicious --> injected' + root_manifest = f'''[package] +org = "acme" +name = "root" +version = "0.1.0" + +[dependencies] +"acme/lib-a" = {json.dumps(hostile_requirement)} +"acme/lib-b" = "^1.0" +'''.encode("utf-8") + root_lock = f'''version = 1 + +[[package]] +org = "acme" +name = "lib-a" +version = "1.0.0" +sha256 = "{'a' * 64}" + +[[package]] +org = "acme" +name = "lib-b" +version = "1.0.0" +sha256 = "{'b' * 64}" + +[[package]] +org = "acme" +name = "shared" +version = "2.0.0" +sha256 = "{'c' * 64}" +vcs_commit = "{_sha40('acme/shared-commit')}" +'''.encode("utf-8") + gitmodules = b'''[submodule "vendor/shared"] +\tpath = vendor/shared +\turl = https://github.com/acme/shared.git +''' + flake_nix = b'''{ + inputs.lib-a.url = "github:acme/lib-a"; + outputs = { self, lib-a }: {}; +} +''' + flake_lock = _compact_json( + { + "version": 7, + "root": "root", + "nodes": { + "root": {"inputs": {"lib-a": "lib-a"}}, + "lib-a": { + "locked": { + "type": "github", + "owner": "acme", + "repo": "lib-a", + "rev": _sha40("acme/lib-a-commit"), + "narHash": "sha256-" + ("d" * 43), + }, + "original": {"type": "github", "owner": "acme", "repo": "lib-a"}, + }, + }, + } + ) + manifests = { + "acme/lib-a": b'''[package] +org = "acme" +name = "lib-a" +version = "1.0.0" + +[dependencies] +"acme/shared" = "^2.0" +''', + "acme/lib-b": b'''[package] +org = "acme" +name = "lib-b" +version = "1.0.0" + +[dependencies] +"acme/shared" = "^2.0" +''', + "acme/shared": b'''[package] +org = "acme" +name = "shared" +version = "2.0.0" +''', + "acme/cycle-a": b'''[package] +org = "acme" +name = "cycle-a" +version = "0.1.0" + +[dependencies] +"acme/cycle-b" = "^0.1" +''', + "acme/cycle-b": b'''[package] +org = "acme" +name = "cycle-b" +version = "0.1.0" + +[dependencies] +"acme/cycle-a" = "^0.1" +''', + } + fixtures: dict[str, RepoFixture] = { + "acme/root": RepoFixture( + full_name="acme/root", + default_branch="main", + commit_sha=_sha40("acme/root-commit"), + tree_sha=_sha40("acme/root-tree"), + archived=False, + private=True, + files={ + ".zpkg.toml": root_manifest, + ".zpkg.lock": root_lock, + ".gitmodules": gitmodules, + "flake.nix": flake_nix, + "flake.lock": flake_lock, + }, + gitlinks={"vendor/shared": _sha40("acme/shared-commit")}, + ) + } + for full_name, manifest in manifests.items(): + fixtures[full_name] = RepoFixture( + full_name=full_name, + default_branch="main", + commit_sha=_sha40(full_name + "-commit"), + tree_sha=_sha40(full_name + "-tree"), + archived=False, + private=False, + files={".zpkg.toml": manifest}, + gitlinks={}, + ) + fixtures["acme/archived"] = RepoFixture( + full_name="acme/archived", + default_branch="main", + commit_sha=_sha40("acme/archived-commit"), + tree_sha=_sha40("acme/archived-tree"), + archived=True, + private=False, + files={"README.md": b"# archived fixture\n"}, + gitlinks={}, + ) + fixtures["acme/truncated"] = RepoFixture( + full_name="acme/truncated", + default_branch="main", + commit_sha=_sha40("acme/truncated-commit"), + tree_sha=_sha40("acme/truncated-tree"), + archived=False, + private=False, + files={".zpkg.toml": manifests["acme/shared"]}, + gitlinks={}, + ) + return fixtures + + +class FakeGitHub: + def __init__(self, token: str) -> None: + self.token = token + self.fixtures = _fixture_repositories() + self.requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + self.server = ThreadingHTTPServer(("127.0.0.1", 0), self._handler_type()) + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + @property + def base_url(self) -> str: + host, port = self.server.server_address[:2] + return f"http://{host}:{port}" + + def _handler_type(self) -> type[BaseHTTPRequestHandler]: + outer = self + + class Handler(BaseHTTPRequestHandler): + server_version = "ZedPkgFakeGitHub/1" + protocol_version = "HTTP/1.1" + + def log_message(self, _format: str, *_args: object) -> None: + return + + def do_GET(self) -> None: # noqa: N802 - stdlib handler API + outer.handle(self) + + return Handler + + def __enter__(self) -> "FakeGitHub": + self.thread.start() + return self + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=10) + + def _record(self, handler: BaseHTTPRequestHandler) -> None: + with self._lock: + self.requests.append( + { + "path": handler.path, + "authorization": handler.headers.get("Authorization"), + } + ) + + def _send_json( + self, + handler: BaseHTTPRequestHandler, + status: int, + payload: Any, + *, + headers: Mapping[str, str] | None = None, + ) -> None: + body = _compact_json(payload) + handler.send_response(status) + handler.send_header("Content-Type", "application/json") + handler.send_header("Content-Length", str(len(body))) + handler.send_header("Connection", "close") + for name, value in (headers or {}).items(): + handler.send_header(name, value) + handler.end_headers() + handler.wfile.write(body) + + def _require_auth(self, handler: BaseHTTPRequestHandler) -> bool: + authorization = handler.headers.get("Authorization", "") + if authorization in {f"Bearer {self.token}", f"token {self.token}"}: + return True + self._send_json( + handler, + HTTPStatus.UNAUTHORIZED, + {"message": "authentication required; credential is never echoed"}, + ) + return False + + def handle(self, handler: BaseHTTPRequestHandler) -> None: + parsed = urllib.parse.urlsplit(handler.path) + path = parsed.path + query = urllib.parse.parse_qs(parsed.query) + self._record(handler) + if not self._require_auth(handler): + return + + if path == "/orgs/acme/repos": + page = int(query.get("page", ["1"])[0]) + names = sorted( + name for name in self.fixtures if name != "acme/truncated" + ) + midpoint = max(1, len(names) // 2) + if page == 1: + selected = names[:midpoint] + headers = { + "Link": ( + f'<{self.base_url}/orgs/acme/repos?per_page=100&page=2>; ' + 'rel="next"' + ) + } + else: + selected = names[midpoint:] + headers = {} + self._send_json( + handler, + HTTPStatus.OK, + [self._metadata(self.fixtures[name]) for name in selected], + headers=headers, + ) + return + + if path == "/orgs/cross-origin/repos": + self._send_json( + handler, + HTTPStatus.OK, + [self._metadata(self.fixtures["acme/root"])], + headers={"Link": '; rel="next"'}, + ) + return + + match = re.fullmatch(r"/repos/([^/]+/[^/]+)", path) + if match: + full_name = urllib.parse.unquote(match.group(1)).lower() + if full_name == "acme/redirect": + handler.send_response(HTTPStatus.FOUND) + handler.send_header("Location", "https://example.invalid/secret") + handler.send_header("Content-Length", "0") + handler.send_header("Connection", "close") + handler.end_headers() + return + fixture = self.fixtures.get(full_name) + if fixture is None: + self._send_json(handler, HTTPStatus.NOT_FOUND, {"message": "not found"}) + return + self._send_json(handler, HTTPStatus.OK, self._metadata(fixture)) + return + + match = re.fullmatch(r"/repos/([^/]+/[^/]+)/branches/([^/]+)", path) + if match: + fixture = self._repo_or_404(handler, match.group(1)) + if fixture is not None: + self._send_json( + handler, + HTTPStatus.OK, + {"name": urllib.parse.unquote(match.group(2)), "commit": {"sha": fixture.commit_sha}}, + ) + return + + match = re.fullmatch(r"/repos/([^/]+/[^/]+)/commits/([^/]+)", path) + if match: + fixture = self._repo_or_404(handler, match.group(1)) + if fixture is not None: + self._send_json( + handler, + HTTPStatus.OK, + {"sha": fixture.commit_sha, "commit": {"tree": {"sha": fixture.tree_sha}}}, + ) + return + + match = re.fullmatch(r"/repos/([^/]+/[^/]+)/git/commits/([0-9a-fA-F]{40})", path) + if match: + fixture = self._repo_or_404(handler, match.group(1)) + if fixture is None: + return + if match.group(2).lower() != fixture.commit_sha: + self._send_json(handler, HTTPStatus.NOT_FOUND, {"message": "unknown commit"}) + else: + self._send_json( + handler, + HTTPStatus.OK, + {"sha": fixture.commit_sha, "tree": {"sha": fixture.tree_sha}}, + ) + return + + match = re.fullmatch(r"/repos/([^/]+/[^/]+)/git/trees/([0-9a-fA-F]{40})", path) + if match: + fixture = self._repo_or_404(handler, match.group(1)) + if fixture is None: + return + if match.group(2).lower() != fixture.tree_sha: + self._send_json(handler, HTTPStatus.NOT_FOUND, {"message": "unknown tree"}) + return + entries: list[dict[str, Any]] = [] + for file_path, content in sorted(fixture.files.items()): + entries.append( + { + "path": file_path, + "mode": "100644", + "type": "blob", + "sha": _git_blob_sha(content), + "size": len(content), + } + ) + for gitlink_path, commit_sha in sorted(fixture.gitlinks.items()): + entries.append( + { + "path": gitlink_path, + "mode": "160000", + "type": "commit", + "sha": commit_sha, + } + ) + self._send_json( + handler, + HTTPStatus.OK, + { + "sha": fixture.tree_sha, + "truncated": fixture.full_name == "acme/truncated", + "tree": entries, + }, + ) + return + + match = re.fullmatch(r"/repos/([^/]+/[^/]+)/git/blobs/([0-9a-fA-F]{40})", path) + if match: + fixture = self._repo_or_404(handler, match.group(1)) + if fixture is None: + return + requested = match.group(2).lower() + for content in fixture.files.values(): + if _git_blob_sha(content) == requested: + self._send_json( + handler, + HTTPStatus.OK, + { + "sha": requested, + "encoding": "base64", + "size": len(content), + "content": base64.b64encode(content).decode("ascii"), + }, + ) + return + self._send_json(handler, HTTPStatus.NOT_FOUND, {"message": "unknown blob"}) + return + + self._send_json(handler, HTTPStatus.NOT_FOUND, {"message": "unhandled endpoint"}) + + def _repo_or_404( + self, handler: BaseHTTPRequestHandler, encoded_full_name: str + ) -> RepoFixture | None: + fixture = self.fixtures.get(urllib.parse.unquote(encoded_full_name).lower()) + if fixture is None: + self._send_json(handler, HTTPStatus.NOT_FOUND, {"message": "not found"}) + return fixture + + @staticmethod + def _metadata(fixture: RepoFixture) -> dict[str, Any]: + owner, name = fixture.full_name.split("/", 1) + return { + "id": int(hashlib.sha1(fixture.full_name.encode()).hexdigest()[:12], 16), + "name": name, + "full_name": fixture.full_name, + "private": fixture.private, + "archived": fixture.archived, + "default_branch": fixture.default_branch, + "owner": {"login": owner}, + } + + +def find_binary(workspace: Path) -> Path: + executable = "zed.exe" if os.name == "nt" else "zed" + for candidate in ( + workspace / "target" / "debug" / executable, + workspace / "target" / "release" / executable, + ): + if candidate.is_file(): + return candidate.resolve() + found = shutil.which("zed") + if found: + return Path(found).resolve() + raise VerificationError("built zed binary not found under target/debug or target/release") + + +def _assert_no_secret(data: bytes, secret: str, context: str) -> None: + if secret and secret.encode("utf-8") in data: + raise VerificationError(f"credential leaked into {context}") + + +def verify_help(binary: Path, workspace: Path, env: Mapping[str, str]) -> CommandResult: + result = run_bounded( + [str(binary), "graph", "github", "--help"], + cwd=workspace, + env=env, + timeout_seconds=20, + secrets=(TOKEN_SENTINEL,), + ) + if result.returncode != 0: + raise VerificationError( + "`zed graph github --help` failed: " + + result.stderr.decode("utf-8", "replace")[:600] + ) + help_text = (result.stdout + b"\n" + result.stderr).decode("utf-8", "replace") + missing = [flag for flag in REQUIRED_HELP_FLAGS if flag not in help_text] + if missing: + raise VerificationError("production help is missing: " + ", ".join(missing)) + forbidden = sorted(set(FORBIDDEN_SECRET_FLAG.findall(help_text))) + if forbidden: + raise VerificationError( + "secret-bearing command-line flags are forbidden: " + ", ".join(forbidden) + ) + return result + + +def _run_inventory( + binary: Path, + workspace: Path, + env: Mapping[str, str], + *, + output_format: str, + output_path: Path, + api_base_url: str | None, + repositories: Sequence[str], + organizations: Sequence[str], + timeout_seconds: float, + expected_codes: set[int] | None = None, +) -> CommandResult: + argv = [str(binary), "graph", "github"] + for repository in repositories: + argv.extend(["--repo", repository]) + for organization in organizations: + argv.extend(["--org", organization]) + argv.extend( + [ + "--include", + "nix,zed,git-submodule", + "--include", + "zed", + "--format", + output_format, + "--output", + str(output_path), + ] + ) + if api_base_url is not None: + argv.extend(["--api-base-url", api_base_url]) + result = run_bounded( + argv, + cwd=workspace, + env=env, + timeout_seconds=timeout_seconds, + secrets=(TOKEN_SENTINEL, env.get("ZED_PKG_GITHUB_TOKEN", "")), + ) + allowed = expected_codes if expected_codes is not None else {0} + if result.returncode not in allowed: + raise VerificationError( + f"candidate returned {result.returncode}, expected {sorted(allowed)} for " + f"{output_format}: {result.stderr.decode('utf-8', 'replace')[:800]}" + ) + _assert_no_secret(result.stdout, TOKEN_SENTINEL, "stdout") + _assert_no_secret(result.stderr, TOKEN_SENTINEL, "stderr") + return result + + +def _list(value: Mapping[str, Any], *names: str) -> list[Any]: + for name in names: + found = value.get(name) + if isinstance(found, list): + return found + return [] + + +def _edge_endpoint(edge: Mapping[str, Any], source: bool) -> str: + names = ("source", "from", "source_id") if source else ("target", "to", "target_id") + for name in names: + value = edge.get(name) + if value is not None: + return str(value) + return "" + + +def _node_coordinate(node: Mapping[str, Any]) -> str: + package = node.get("package") + if isinstance(package, str) and "/" in package: + return package.lower() + org = node.get("org", node.get("organization")) + name = node.get("name", node.get("package_name")) + if isinstance(org, str) and isinstance(name, str): + return f"{org}/{name}".lower() + node_id = str(node.get("id", "")).lower() + match = re.search(r"(?:^|[:@])([a-z0-9_.-]+/[a-z0-9_.-]+)(?:$|[:@#])", node_id) + return match.group(1) if match else "" + + +def _validate_json(raw: bytes, *, require_complete: bool) -> dict[str, Any]: + try: + text = raw.decode("utf-8") + inventory = json.loads(text) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise VerificationError(f"invalid UTF-8 inventory JSON: {error}") from error + if "\r" in text or not text.endswith("\n"): + raise VerificationError("inventory JSON must use stable LF line endings") + if not isinstance(inventory, dict): + raise VerificationError("inventory root must be an object") + if inventory.get("schema", inventory.get("resource")) != SCHEMA: + raise VerificationError("unexpected inventory resource/schema") + completeness = inventory.get("completeness") + if not isinstance(completeness, dict) or completeness.get("resolution") != "not-claimed": + raise VerificationError("inventory must report resolution=not-claimed") + inventory_state = completeness.get("inventory") + if require_complete and inventory_state not in {"complete", True}: + raise VerificationError("deterministic fake inventory must be complete") + + nodes = _list(inventory, "nodes") + edges = _list(inventory, "edges") + pins = _list(inventory, "pins", "lock_pins") + for pin in pins: + if not isinstance(pin, dict) or pin.get("topological") is not False: + raise VerificationError("every flat lock pin must set topological=false") + artifact = pin.get("artifact_sha256", pin.get("sha256")) + if artifact is not None and ( + not isinstance(artifact, str) or re.fullmatch(r"[0-9a-f]{64}", artifact) is None + ): + raise VerificationError("lock artifact SHA must be exact lowercase hex") + for edge in edges: + if isinstance(edge, dict) and "lock-pin" in str(edge.get("kind", "")): + raise VerificationError("flat lock pins must never become topology edges") + + ids_by_coordinate = { + _node_coordinate(node): str(node.get("id", "")) + for node in nodes + if isinstance(node, dict) and _node_coordinate(node) + } + root = ids_by_coordinate.get("acme/root") + shared = ids_by_coordinate.get("acme/shared") + if root and shared: + for edge in edges: + if not isinstance(edge, dict): + continue + if ( + _edge_endpoint(edge, True) == root + and _edge_endpoint(edge, False) == shared + and str(edge.get("kind", "")).lower().startswith("zed") + ): + raise VerificationError( + "transitive shared selection fabricated root -> shared Zed topology" + ) + + components = _list(inventory, "strongly_connected_components", "sccs") + analyzed: set[str] = set() + for component in components: + members = component.get("nodes", []) if isinstance(component, dict) else component + if isinstance(members, list): + analyzed.update(str(member) for member in members) + node_ids = {str(node.get("id")) for node in nodes if isinstance(node, dict)} + if components and analyzed != node_ids: + raise VerificationError("SCC analysis must cover graph nodes only") + + cycle_a = ids_by_coordinate.get("acme/cycle-a") + cycle_b = ids_by_coordinate.get("acme/cycle-b") + if cycle_a and cycle_b: + cycles = _list(inventory, "cycles") + cycle_sets = [] + for cycle in cycles: + members = cycle.get("nodes", []) if isinstance(cycle, dict) else cycle + if isinstance(members, list): + cycle_sets.append({str(member) for member in members}) + if {cycle_a, cycle_b} not in cycle_sets: + raise VerificationError("cycle-a/cycle-b is missing from canonical cycles") + + return inventory + + +def _validate_visual(raw: bytes, output_format: str) -> None: + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise VerificationError(f"{output_format} is not UTF-8") from error + if "\r" in text or not text.endswith("\n"): + raise VerificationError(f"{output_format} must use stable LF line endings") + lower = text.lower() + if "lock pin" not in lower or ("evidence" not in lower and "topological=false" not in lower): + raise VerificationError(f"{output_format} must render disconnected lock evidence") + for line in text.splitlines(): + if "lock-pin" in line.lower() and ("->" in line or "-->" in line): + raise VerificationError(f"{output_format} rendered a lock pin as an edge") + if line.strip().startswith("malicious"): + raise VerificationError(f"{output_format} label escaping allowed injection") + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def verify_fake_server( + binary: Path, + workspace: Path, + base_env: Mapping[str, str], + evidence: dict[str, Any], +) -> None: + with FakeGitHub(TOKEN_SENTINEL) as fake: + env = dict(base_env) + env["ZED_PKG_GITHUB_TOKEN"] = TOKEN_SENTINEL + env.pop("GITHUB_TOKEN", None) + env.pop("ZED_PKG_GITHUB_ALLOW_TOKEN_TO_API_BASE", None) + with tempfile.TemporaryDirectory(prefix="zpkg-graph-github-") as directory: + root = Path(directory) + summary: dict[str, Any] = {} + for output_format in ("json", "dot", "mermaid"): + first = root / f"first.{output_format}" + second = root / f"second.{output_format}" + common = { + "binary": binary, + "workspace": workspace, + "env": env, + "output_format": output_format, + "api_base_url": fake.base_url, + "repositories": ("ACME/ROOT", "acme/root"), + "organizations": ("ACME", "acme"), + "timeout_seconds": DEFAULT_TIMEOUT_SECONDS, + } + _run_inventory(output_path=first, **common) + _run_inventory(output_path=second, **common) + first_bytes = first.read_bytes() + second_bytes = second.read_bytes() + if first_bytes != second_bytes: + raise VerificationError(f"{output_format} changed across identical scans") + if len(first_bytes) > MAX_EVIDENCE_BYTES: + raise VerificationError(f"{output_format} fixture output is unbounded") + _assert_no_secret(first_bytes, TOKEN_SENTINEL, f"{output_format} file") + if output_format == "json": + inventory = _validate_json(first_bytes, require_complete=True) + summary["counts"] = { + name: len(_list(inventory, name)) + for name in ("repositories", "nodes", "edges", "pins", "failures", "contradictions", "cycles") + } + else: + _validate_visual(first_bytes, output_format) + summary[output_format] = { + "bytes": len(first_bytes), + "sha256": _sha256(first_bytes), + } + + result = _run_inventory( + binary, + workspace, + env, + output_format="json", + output_path=root / "cross-origin.json", + api_base_url=fake.base_url, + repositories=(), + organizations=("cross-origin",), + timeout_seconds=20, + expected_codes={1, 2, 3}, + ) + if result.returncode == 0: + raise VerificationError("cross-origin pagination was accepted") + + result = _run_inventory( + binary, + workspace, + env, + output_format="json", + output_path=root / "redirect.json", + api_base_url=fake.base_url, + repositories=("acme/redirect",), + organizations=(), + timeout_seconds=20, + expected_codes={1, 2, 3}, + ) + if result.returncode == 0: + raise VerificationError("authorization-bearing redirect was accepted") + + truncated = root / "truncated.json" + result = _run_inventory( + binary, + workspace, + env, + output_format="json", + output_path=truncated, + api_base_url=fake.base_url, + repositories=("acme/truncated",), + organizations=(), + timeout_seconds=20, + expected_codes={1}, + ) + if result.returncode != 1 or not truncated.is_file(): + raise VerificationError("truncated tree must emit an explicit partial artifact") + partial = _validate_json(truncated.read_bytes(), require_complete=False) + if partial.get("completeness", {}).get("inventory") not in {"partial", False}: + raise VerificationError("truncated tree was not marked partial") + failures = _list(partial, "failures") + if not failures: + raise VerificationError("truncated tree partial artifact has no failure") + summary["truncated"] = { + "bytes": truncated.stat().st_size, + "sha256": _sha256(truncated.read_bytes()), + "failure_count": len(failures), + } + + requests = list(fake.requests) + expected_auth = {f"Bearer {TOKEN_SENTINEL}", f"token {TOKEN_SENTINEL}"} + if not requests or not all(item.get("authorization") in expected_auth for item in requests): + raise VerificationError("loopback token routing did not follow the contract") + if any(TOKEN_SENTINEL in str(item.get("path", "")) for item in requests): + raise VerificationError("credential appeared in a request URL") + summary["requests"] = { + "count": len(requests), + "unique_paths": len({str(item["path"]) for item in requests}), + } + evidence["fake_server"] = summary + + +def _bool_argument(value: str) -> bool: + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off", ""}: + return False + raise argparse.ArgumentTypeError(f"invalid boolean value: {value!r}") + + +def verify_live_smoke( + binary: Path, + workspace: Path, + base_env: Mapping[str, str], + live_org: str, + evidence: dict[str, Any], +) -> None: + token = base_env.get("ZED_PKG_GITHUB_TOKEN", "") + if not token: + raise VerificationError("live smoke requested without ZED_PKG_GITHUB_TOKEN") + with tempfile.TemporaryDirectory(prefix="zpkg-live-smoke-") as directory: + root = Path(directory) + summary: dict[str, Any] = {} + for output_format in ("json", "dot", "mermaid"): + first = root / f"live-first.{output_format}" + second = root / f"live-second.{output_format}" + common = { + "binary": binary, + "workspace": workspace, + "env": base_env, + "output_format": output_format, + "api_base_url": None, + "repositories": (), + "organizations": (live_org,), + "timeout_seconds": LIVE_TIMEOUT_SECONDS, + "expected_codes": {0, 1}, + } + _run_inventory(output_path=first, **common) + _run_inventory(output_path=second, **common) + first_bytes = first.read_bytes() + if first_bytes != second.read_bytes(): + raise VerificationError(f"live {output_format} changed across repeated scans") + _assert_no_secret(first_bytes, token, f"live {output_format} artifact") + if output_format == "json": + inventory = _validate_json(first_bytes, require_complete=False) + summary["counts"] = { + name: len(_list(inventory, name)) + for name in ("repositories", "nodes", "edges", "pins", "failures", "contradictions") + } + summary["inventory_completeness"] = inventory.get("completeness", {}).get("inventory") + else: + text = first_bytes.decode("utf-8") + if "\r" in text or not text.endswith("\n"): + raise VerificationError(f"live {output_format} is not canonical UTF-8/LF") + summary[output_format] = { + "bytes": len(first_bytes), + "sha256": _sha256(first_bytes), + } + evidence["live_smoke"] = summary + + +def write_evidence(path: Path, evidence: Mapping[str, Any]) -> None: + path.mkdir(parents=True, exist_ok=True) + payload = _compact_json(evidence) + if len(payload) > MAX_EVIDENCE_BYTES: + raise VerificationError("redacted evidence summary exceeded bounded size") + temporary = path / ".summary.json.tmp" + temporary.write_bytes(payload) + os.replace(temporary, path / "summary.json") + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--run-live-smoke", type=_bool_argument, default=False) + parser.add_argument("--live-org", default="zed-pkg-test") + parser.add_argument("--evidence-dir", type=Path, default=Path(".graph-github-evidence")) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + workspace = args.workspace.resolve() + if not (workspace / "Cargo.toml").is_file(): + raise VerificationError(f"workspace is not a Rust package: {workspace}") + binary = find_binary(workspace) + env = os.environ.copy() + env.setdefault("NO_COLOR", "1") + env.setdefault("RUST_BACKTRACE", "0") + env.setdefault("PYTHONUTF8", "1") + env.setdefault("SOURCE_DATE_EPOCH", "0") + + help_result = verify_help(binary, workspace, env) + evidence: dict[str, Any] = { + "schema": "zpkg/graph-github-production-evidence/v1", + "candidate": { + "expected_sha": env.get("EXPECTED_CANDIDATE_SHA", ""), + "binary_sha256": _sha256(binary.read_bytes()), + }, + "help": { + "bytes": len(help_result.stdout) + len(help_result.stderr), + "sha256": _sha256(help_result.stdout + b"\n" + help_result.stderr), + }, + "platform": { + "os_name": os.name, + "sys_platform": sys.platform, + "python": sys.version.split()[0], + }, + } + verify_fake_server(binary, workspace, env, evidence) + runner_os = env.get("RUNNER_OS", "") + if args.run_live_smoke and runner_os.lower() == "linux": + verify_live_smoke(binary, workspace, env, args.live_org, evidence) + elif args.run_live_smoke: + evidence["live_smoke"] = { + "skipped": True, + "reason": "live smoke is intentionally Linux-only", + } + write_evidence(args.evidence_dir.resolve(), evidence) + print( + "zed graph github production verification passed: " + + _compact_json( + { + "candidate": evidence["candidate"]["expected_sha"], + "fake_json": evidence["fake_server"]["json"]["sha256"], + "live": "live_smoke" in evidence, + } + ).decode("utf-8").strip() + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except VerificationError as error: + print(f"verification error: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/ide-integrations/README.md b/ide-integrations/README.md new file mode 100644 index 00000000..f341e354 --- /dev/null +++ b/ide-integrations/README.md @@ -0,0 +1,5 @@ +# IDE integration sandbox + +This directory certifies the shared Zed Package Manager IDE contract without using a developer's real package home, credentials, or registry. + +The workflow checks the two live integrations and the five publication candidates using their native toolchains. Full GUI smoke tests remain a promotion gate for dedicated repositories and clean editor instances. diff --git a/ide-integrations/dedicated-repo-certification.md b/ide-integrations/dedicated-repo-certification.md new file mode 100644 index 00000000..aa8ad2bb --- /dev/null +++ b/ide-integrations/dedicated-repo-certification.md @@ -0,0 +1,15 @@ +# Dedicated IDE repository certification + +Tracking: Linear `DEN-2508`. + +This lane independently certifies the first reviewable follow-up in each promoted IDE repository. Product repositories remain the implementation source; `zed-pkg-test/zed-pkg-e2e` pins immutable full commit identities and does not mutate product state. + +| Integration | Product PR | Immutable head | Independent gate | +| --- | --- | --- | --- | +| VS Code | `zed-pkg/zed-vscode#1` | `9754f10e44235828547cbeeb05e43c5786673af9` | Node tests, repository contract, VSIX package | +| Qt Creator | `zed-pkg/zed-qtcreator#1` | `0372ccd4100e369d9e4593d3df66d8b2b507886a` | CMake build + CTest | +| Xcode | `zed-pkg/zed-xcode#1` | `654f96f9c1d3ee80afc3034a883cb9083caefe00` | Swift tests + release build on macOS | +| Eclipse | `zed-pkg/zed-eclipse#1` | `e65e8086f328a20adf3f941e1be988d0d757dc0f` | Java 21 Maven/JUnit | +| Visual Studio | `zed-pkg/zed-visual-studio#1` | `a3520ea8aa19ecbdaa4c71e77d01af6407a7e05f` | .NET 8 tests on Windows | + +The gate proves repository-local code and controller/model contracts. It does not overclaim vendor GUI-instance, signing, or marketplace evidence. Those remain tracked by DEN-2508 and the per-repository conformance records. diff --git a/ide-integrations/parity-matrix.json b/ide-integrations/parity-matrix.json new file mode 100644 index 00000000..d05afc99 --- /dev/null +++ b/ide-integrations/parity-matrix.json @@ -0,0 +1,15 @@ +{ + "schema": "zed-pkg/ide-parity/v1", + "contract_version": 1, + "linear_issue": "DEN-2508", + "required_core_capabilities": ["multi_root_discovery", "manifest_diagnostics", "lock_diagnostics", "materialization_diagnostics", "staging_recovery", "cli_validation", "argv_execution", "bounded_execution", "output_redaction", "explicit_mutation_confirmation", "versioned_inspect_adapter", "deterministic_fallback", "native_unit_tests", "retained_artifact"], + "integrations": { + "sublimetext": {"repository": "zed-pkg/zed-sublimetext", "state": "live", "platforms": ["linux", "macos", "windows"], "native_language": "python", "artifact": ".sublime-package"}, + "jetbrains": {"repository": "zed-pkg/zed-intellij", "state": "live", "platforms": ["linux", "macos", "windows"], "native_language": "kotlin", "artifact": "plugin-zip"}, + "vscode": {"repository": "zed-pkg/zed-vscode", "state": "dedicated-repo-candidate", "candidate_sha": "c1d3318a2cf8cd717f9b0af67117ddbac3e28d2e", "pull_request": 1, "platforms": ["linux", "macos", "windows"], "native_language": "javascript", "artifact": "vsix"}, + "qtcreator": {"repository": "zed-pkg/zed-qtcreator", "state": "dedicated-repo-candidate", "candidate_sha": "8e7958c7c85a305b59accfe82c28e7ff42bfecba", "pull_request": 1, "platforms": ["linux", "macos", "windows"], "native_language": "cpp", "artifact": "qtcreator-plugin"}, + "xcode": {"repository": "zed-pkg/zed-xcode", "state": "dedicated-repo-candidate", "candidate_sha": "9b9304fee2ff236fc59407ddd97cc2c01225a69d", "pull_request": 1, "platforms": ["macos"], "native_language": "swift", "artifact": "appex"}, + "eclipse": {"repository": "zed-pkg/zed-eclipse", "state": "dedicated-repo-candidate", "candidate_sha": "246b46048cced1d1072afd3bedc9849efe3a8a8f", "pull_request": 1, "platforms": ["linux", "macos", "windows"], "native_language": "java", "artifact": "p2-site"}, + "visual-studio": {"repository": "zed-pkg/zed-visual-studio", "state": "dedicated-repo-candidate", "candidate_sha": "d85a9c827140426a8d9401b5c269ff36f591a299", "pull_request": 1, "platforms": ["windows"], "native_language": "csharp", "artifact": "vsix"} + } +} diff --git a/ide-integrations/sandbox_sublime.py b/ide-integrations/sandbox_sublime.py new file mode 100755 index 00000000..a1429502 --- /dev/null +++ b/ide-integrations/sandbox_sublime.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import hashlib, os, sys, tempfile, time +from pathlib import Path +REPO = Path(sys.argv[1]).resolve(); sys.path.insert(0, str(REPO)) +from zed_pkg_insights.analyzer import Analyzer +from zed_pkg_insights.project import discover_zed_roots + +def digest_tree(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if path.is_file(): digest.update(path.relative_to(root).as_posix().encode()); digest.update(path.read_bytes()) + return digest.hexdigest() + +def write_cli(path: Path, correct: bool = True) -> None: + marker = "Zed universal package manager" if correct else "Zed collaborative code editor" + path.write_text("#!/bin/sh\ncase \"$1\" in\n --version) echo 'zed 0.1.0' ;;\n --help) echo '" + marker + "' ;;\n *) echo 'unexpected mutation attempt' >&2; exit 91 ;;\nesac\n"); path.chmod(0o755) + +def analyze(root: Path, zed: Path): + before = digest_tree(root); snapshot = Analyzer().analyze(root, zed_binary=str(zed), probe_cli=True); after = digest_tree(root) + assert before == after, f"analysis mutated {root}"; return {item.code for item in snapshot.diagnostics} + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="zed-ide-sandbox-") as directory: + sandbox = Path(directory); home = sandbox / "home"; home.mkdir(); os.environ["HOME"] = str(home) + cli = sandbox / "zed"; wrong_cli = sandbox / "zed-editor"; write_cli(cli); write_cli(wrong_cli, False) + unmanaged = sandbox / "unmanaged"; unmanaged.mkdir(); assert "ZED001" in analyze(unmanaged, cli) + lock_only = sandbox / "lock-only"; lock_only.mkdir(); (lock_only / ".zpkg.lock").write_text("version = 1\n"); assert "ZED002" in analyze(lock_only, cli) + manifest_only = sandbox / "manifest-only"; manifest_only.mkdir(); (manifest_only / ".zpkg.toml").write_text('[package]\norg = "acme"\nname = "widget"\nversion = "1.0.0"\n[dependencies]\n"zed-pkg/zed-interfaces" = "^0.1.0"\n'); assert "ZED003" in analyze(manifest_only, cli) + stale = sandbox / "stale"; stale.mkdir(); manifest = stale / ".zpkg.toml"; lock = stale / ".zpkg.lock"; lock.write_text("version = 1\n"); time.sleep(1.1); manifest.write_text('[package]\norg = "acme"\nname = "widget"\nversion = "1.0.0"\n[dependencies]\n"zed-pkg/zed-interfaces" = "^0.1.0"\n'); assert {"ZED004", "ZED006"}.issubset(analyze(stale, cli)) + staging = sandbox / "staging"; staging.mkdir(); (staging / ".zpkg.toml").write_text('[package]\norg = "acme"\nname = "widget"\nversion = "1.0.0"\n'); (staging / ".zpkg.lock").write_text("version = 1\n"); transaction = staging / ".zpkg-staging"; transaction.mkdir(); (transaction / "journal.json").write_text("{}"); assert "ZED007" in analyze(staging, cli) + invalid = sandbox / "invalid"; invalid.mkdir(); (invalid / ".zpkg.toml").write_text("[package\n"); assert "ZED014" in analyze(invalid, cli) + wrong = sandbox / "wrong-cli"; wrong.mkdir(); assert "ZED011" in analyze(wrong, wrong_cli) + assert {manifest_only.resolve(), stale.resolve(), staging.resolve()} == set(discover_zed_roots([manifest_only, stale, staging])) + assert not any(home.iterdir()), "sandbox analysis wrote to HOME" + print("sandboxed Sublime diagnostics and multi-root discovery passed"); return 0 +if __name__ == "__main__": raise SystemExit(main()) diff --git a/ide-integrations/verify_matrix.py b/ide-integrations/verify_matrix.py new file mode 100755 index 00000000..b329e6bb --- /dev/null +++ b/ide-integrations/verify_matrix.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import json +import re +import sys +from pathlib import Path + +REQUIRED = {"multi_root_discovery", "manifest_diagnostics", "lock_diagnostics", "materialization_diagnostics", "staging_recovery", "cli_validation", "argv_execution", "bounded_execution", "output_redaction", "explicit_mutation_confirmation", "versioned_inspect_adapter", "deterministic_fallback", "native_unit_tests", "retained_artifact"} +EXPECTED = {"sublimetext", "jetbrains", "vscode", "qtcreator", "xcode", "eclipse", "visual-studio"} +SHA = re.compile(r"^[0-9a-f]{40}$") + +def main(path: str) -> int: + data = json.loads(Path(path).read_text()) + assert data["schema"] == "zed-pkg/ide-parity/v1" and data["contract_version"] == 1 + assert set(data["required_core_capabilities"]) == REQUIRED + assert set(data["integrations"]) == EXPECTED + repositories = set() + for name, item in data["integrations"].items(): + assert item["repository"].startswith("zed-pkg/zed-") + assert item["repository"] not in repositories + repositories.add(item["repository"]) + assert item["state"] in {"live", "publish-ready-candidate", "buildable-candidate", "dedicated-repo-candidate"} + assert item["platforms"] and item["native_language"] and item["artifact"] + if item["state"] == "dedicated-repo-candidate": + assert SHA.fullmatch(item["candidate_sha"]), name + assert item["pull_request"] > 0 + elif item["state"] != "live": + assert item.get("source") + print(f"validated {len(EXPECTED)} IDE integrations and {len(REQUIRED)} core capabilities") + return 0 + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1])) diff --git a/ide-integrations/verify_workflow_supply_chain.py b/ide-integrations/verify_workflow_supply_chain.py new file mode 100644 index 00000000..786aa959 --- /dev/null +++ b/ide-integrations/verify_workflow_supply_chain.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import json +import re +import sys +from pathlib import Path + +SHA = re.compile(r"^[0-9a-f]{40}$") +USES = re.compile(r"(?m)^\s*-?\s*uses:\s*([^\s#]+)") +REPO_REF = re.compile( + r"repository:\s*(zed-pkg/zed-(?:vscode|qtcreator|xcode|eclipse|visual-studio))\s*\n\s*ref:\s*([0-9a-f]{40})" +) + + +def main(workflow_path: str, matrix_path: str) -> int: + workflow = Path(workflow_path).read_text() + matrix = json.loads(Path(matrix_path).read_text()) + errors = [] + + for uses in USES.findall(workflow): + if uses.startswith("./"): + continue + if "@" not in uses: + errors.append(f"action has no immutable ref: {uses}") + continue + ref = uses.rsplit("@", 1)[1] + if not SHA.fullmatch(ref): + errors.append(f"action ref is not a full commit SHA: {uses}") + + for forbidden in ("ubuntu-latest", "macos-latest", "windows-latest"): + if forbidden in workflow: + errors.append(f"mutable runner label is forbidden: {forbidden}") + + for forbidden in ("npm install ", "npx --yes", "persist-credentials: true"): + if forbidden in workflow: + errors.append(f"forbidden mutable/credential-bearing workflow fragment: {forbidden}") + + expected = { + item["repository"]: item["candidate_sha"] + for item in matrix["integrations"].values() + if item.get("state") == "dedicated-repo-candidate" + } + observed = dict(REPO_REF.findall(workflow)) + if observed != expected: + errors.append(f"workflow candidate refs differ from parity matrix: observed={observed!r} expected={expected!r}") + + if workflow.count("persist-credentials: false") < 8: + errors.append("every checkout path should explicitly disable persisted credentials") + + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + + print( + f"verified {len(USES.findall(workflow))} immutable action uses, " + f"{len(expected)} exact candidate refs, fixed runners, npm-ci discipline, and no persisted credentials" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1], sys.argv[2])) diff --git a/org_fleet_contract.py b/org_fleet_contract.py new file mode 100644 index 00000000..3444a4c1 --- /dev/null +++ b/org_fleet_contract.py @@ -0,0 +1,3 @@ +"""Repository-root import surface for the reviewed fleet contract.""" + +from scripts.org_fleet_contract import * # noqa: F401,F403 diff --git a/org_fleet_inventory.py b/org_fleet_inventory.py new file mode 100644 index 00000000..5092bc34 --- /dev/null +++ b/org_fleet_inventory.py @@ -0,0 +1,3 @@ +"""Repository-root import surface for the public fleet inventory auditor.""" + +from scripts.org_fleet_inventory import * # noqa: F401,F403 diff --git a/package.json b/package.json index 86ab43a7..32d6c7db 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,9 @@ "e2e:fixtures": "playwright test suites/fixture-package-pages.spec.ts", "e2e:polyglot": "playwright test suites/polyglot-fan-out.spec.ts", "typecheck": "tsc --noEmit", - "test": "npm run e2e" + "test": "npm run e2e", + "lint:ores": "if [ -f .ores-lint/lint.sh ]; then sh .ores-lint/lint.sh; else echo 'ores-lint not installed'; fi", + "prepublishOnly": "npm run lint:ores" }, "license": "MIT", "devDependencies": { @@ -17,5 +19,13 @@ "@types/node": "^26.1.1", "playwright": "^1.61.1", "typescript": "^5.9.3" + }, + "oresLint": { + "note": "Declared, not installed. ores-lint resolves these from a global install - see .ores-lint/README.md", + "requires": { + "eslint": ">=9", + "typescript-eslint": ">=8 (optional; without it .ts files are skipped)" + }, + "install": "npm i -g eslint typescript-eslint" } } diff --git a/proofs/den-2788-orm-core.json b/proofs/den-2788-orm-core.json new file mode 100644 index 00000000..6c3a38ad --- /dev/null +++ b/proofs/den-2788-orm-core.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "linear_issue": "DEN-2788", + "source_repository": "zed-pkg/zed-orm-core", + "source_pull_request": 2, + "source_commit": "8d01cdc86ab655d6e82f25773d10dda86604da4b", + "expected_package": "zed-pkg/zed-orm-core", + "expected_schema": "zed_pkg", + "expected_shared_defs_revision": "c8bdc06d74746acc6439f9527ebd02697fdf028b", + "release_policy": "private-git-zed-only" +} diff --git a/proofs/den-2788-web-readonly.json b/proofs/den-2788-web-readonly.json new file mode 100644 index 00000000..864c1cbb --- /dev/null +++ b/proofs/den-2788-web-readonly.json @@ -0,0 +1,13 @@ +{ + "schema_version": 1, + "linear_issue": "DEN-2788", + "source_repository": "zed-pkg/zed-web-server.rs", + "source_pull_request": 6, + "source_commit": "3325dc16c51bea5924815bba0f37b0742ba985b4", + "interfaces_repository": "zed-pkg/zed-interfaces", + "interfaces_commit": "07d01604461d00e237c7d86ad3855464167574ec", + "expected_package": "zed-pkg/zed-web-server", + "expected_schema": "zed_pkg", + "expected_database_role": "zed_pkg__web_ro", + "expected_secret": "zed-web-server-db-ro" +} diff --git a/scripts/apply_den2725_help_delimiter_fix.py b/scripts/apply_den2725_help_delimiter_fix.py new file mode 100644 index 00000000..94fb8839 --- /dev/null +++ b/scripts/apply_den2725_help_delimiter_fix.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from pathlib import Path + + +def replace_once(path: Path, old: str, new: str) -> None: + text = path.read_text(encoding="utf-8") + count = text.count(old) + if count != 1: + raise SystemExit(f"{path}: expected one replacement target, found {count}") + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + +def apply(root: Path) -> None: + source = root / "src/external_subcommands.rs" + replace_once( + source, + ''' if !arguments + .iter() + .any(|argument| argument == OsStr::new("--help") || argument == OsStr::new("-h")) + { + arguments.push(OsString::from("--help")); + } +''', + ''' ensure_help_argument(&mut arguments); +''', + ) + replace_once( + source, + '''fn extract_root_options(args: &[OsString]) -> Option { +''', + '''fn ensure_help_argument(arguments: &mut Vec) { + if arguments + .iter() + .any(|argument| argument == OsStr::new("--help") || argument == OsStr::new("-h")) + { + return; + } + let insertion = arguments + .iter() + .position(|argument| argument == OsStr::new("--")) + .unwrap_or(arguments.len()); + arguments.insert(insertion, OsString::from("--help")); +} + +fn extract_root_options(args: &[OsString]) -> Option { +''', + ) + replace_once( + source, + ''' #[test] + fn builtins_and_unsafe_names_are_never_external() { +''', + ''' #[test] + fn external_help_is_inserted_before_literal_double_dash() { + let route = external_route(&os_args(&[ + "zed", + "help", + "gitops", + "validate", + "--", + "--root", + "child-owned", + ])) + .expect("external help route"); + assert_eq!( + route.arguments, + os_args(&["validate", "--help", "--", "--root", "child-owned"]) + ); + } + + #[test] + fn builtins_and_unsafe_names_are_never_external() { +''', + ) + + integration = root / "tests/external_gitops_dispatch.rs" + replace_once( + integration, + '''#[test] +fn root_help_alias_reaches_the_external_binary() { +''', + '''#[test] +fn root_help_alias_keeps_help_before_literal_double_dash() { + let output = Command::new(env!("CARGO_BIN_EXE_zed")) + .args(["help", "gitops", "validate", "--", "--root", "child-owned"]) + .output() + .expect("run zed help gitops with literal delimiter"); + assert!(output.status.success(), "{}", text(&output)); + let text = text(&output); + assert!(text.contains("Usage: zed-gitops"), "{text}"); + assert!(text.contains("validate [OPTIONS]"), "{text}"); + assert!(text.contains("--offline"), "{text}"); +} + +#[test] +fn root_help_alias_reaches_the_external_binary() { +''', + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + apply(args.root.resolve(strict=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit-live-test-org-fleet.mjs b/scripts/audit-live-test-org-fleet.mjs new file mode 100755 index 00000000..a59e344d --- /dev/null +++ b/scripts/audit-live-test-org-fleet.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; +import { readFleetManifest } from './read-fleet-manifest.mjs'; + +const DEFAULT_API_BASE = 'https://api.github.com'; + +export function expectedRepositoryNames(pair) { + return new Set([ + '.github', + ...(pair.existingRepositories ?? []), + ...pair.repositories.map((repository) => repository.name), + ]); +} + +export function diffRepositoryNames(expectedNames, actualNames) { + const expected = new Set(expectedNames); + const actual = new Set(actualNames); + return { + missing: [...expected].filter((name) => !actual.has(name)).sort(), + extras: [...actual].filter((name) => !expected.has(name)).sort(), + present: [...expected].filter((name) => actual.has(name)).sort(), + }; +} + +export function parseArguments(argv) { + const options = { json: false, strictExtras: false, organizations: [] }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === '--json') options.json = true; + else if (argument === '--strict-extras') options.strictExtras = true; + else if (argument === '--org') options.organizations.push(argv[++index]); + else if (argument === '--help' || argument === '-h') options.help = true; + else throw new Error(`unknown argument: ${argument}`); + } + if (options.organizations.some((value) => !value)) throw new Error('--org requires a value'); + return options; +} + +function sleep(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +export function createGitHubClient({ + token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN || '', + apiBase = process.env.GITHUB_API_URL || DEFAULT_API_BASE, + fetchImplementation = globalThis.fetch, +} = {}) { + if (typeof fetchImplementation !== 'function') throw new Error('fetch implementation is required'); + + async function request(endpoint) { + const url = `${apiBase}${endpoint}`; + for (let attempt = 0; attempt < 5; attempt += 1) { + const response = await fetchImplementation(url, { + headers: { + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'zed-pkg-test-live-fleet-audit/1', + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + }); + if (response.ok) return response.json(); + const body = await response.text(); + if ((response.status === 429 || response.status >= 500) && attempt < 4) { + const retryAfterSeconds = Number(response.headers.get('retry-after') ?? 0); + await sleep(Math.max(retryAfterSeconds * 1000, 1000 * (2 ** attempt))); + continue; + } + throw new Error(`GET ${endpoint} -> ${response.status}: ${body.slice(0, 500)}`); + } + throw new Error(`GET ${endpoint} exhausted retries`); + } + + async function listOrganizationRepositories(organization) { + const repositories = []; + for (let page = 1; page <= 20; page += 1) { + const batch = await request(`/orgs/${encodeURIComponent(organization)}/repos?type=all&sort=full_name&direction=asc&per_page=100&page=${page}`); + if (!Array.isArray(batch)) throw new Error(`unexpected repository response for ${organization}`); + repositories.push(...batch); + if (batch.length < 100) return repositories; + } + throw new Error(`repository pagination exceeded 2000 entries for ${organization}`); + } + + return { listOrganizationRepositories }; +} + +function repositoryHygiene(repository) { + const findings = []; + if (repository.archived) findings.push('archived'); + if (repository.disabled) findings.push('disabled'); + if (!repository.default_branch) findings.push('missing-default-branch'); + return findings; +} + +export async function auditFleet(manifest, client, { organizations = [] } = {}) { + const selected = new Set(organizations.map((value) => value.toLowerCase())); + const excluded = new Set((manifest.policy.excludedOrganizations ?? []).map((value) => value.toLowerCase())); + const pairs = manifest.pairs.filter((pair) => selected.size === 0 || selected.has(pair.testOrg.toLowerCase())); + if (selected.size > 0 && pairs.length !== selected.size) { + const known = new Set(pairs.map((pair) => pair.testOrg.toLowerCase())); + const unknown = [...selected].filter((name) => !known.has(name)); + throw new Error(`unknown or excluded test organizations: ${unknown.join(', ')}`); + } + + const reports = []; + for (const pair of pairs) { + if (excluded.has(pair.sourceOrg.toLowerCase()) || excluded.has(pair.testOrg.toLowerCase())) { + throw new Error(`excluded organization entered audit scope: ${pair.testOrg}`); + } + const [sourceRepositories, testRepositories] = await Promise.all([ + client.listOrganizationRepositories(pair.sourceOrg), + client.listOrganizationRepositories(pair.testOrg), + ]); + const expected = expectedRepositoryNames(pair); + const diff = diffRepositoryNames(expected, testRepositories.map((repository) => repository.name)); + const expectedHygiene = testRepositories + .filter((repository) => expected.has(repository.name)) + .flatMap((repository) => repositoryHygiene(repository).map((finding) => ({ repository: repository.name, finding }))); + const sourceHygiene = sourceRepositories + .flatMap((repository) => repositoryHygiene(repository).map((finding) => ({ repository: repository.name, finding }))); + reports.push({ + sourceOrganization: pair.sourceOrg, + testOrganization: pair.testOrg, + expectedRepositories: expected.size, + visibleTestRepositories: testRepositories.length, + visibleSourceRepositories: sourceRepositories.length, + present: diff.present, + missing: diff.missing, + extras: diff.extras, + expectedHygiene, + sourceHygiene, + }); + } + + return { + generatedAt: new Date().toISOString(), + schemaVersion: 1, + pairCount: reports.length, + expectedRepositories: reports.reduce((sum, report) => sum + report.expectedRepositories, 0), + presentRepositories: reports.reduce((sum, report) => sum + report.present.length, 0), + missingRepositories: reports.reduce((sum, report) => sum + report.missing.length, 0), + extraRepositories: reports.reduce((sum, report) => sum + report.extras.length, 0), + expectedHygieneFindings: reports.reduce((sum, report) => sum + report.expectedHygiene.length, 0), + sourceHygieneFindings: reports.reduce((sum, report) => sum + report.sourceHygiene.length, 0), + reports, + }; +} + +export function renderMarkdown(report) { + const lines = [ + '# Live paired test-fleet audit', + '', + `Generated: ${report.generatedAt}`, + '', + `Managed pairs: **${report.pairCount}**`, + '', + `Canonical repositories present: **${report.presentRepositories}/${report.expectedRepositories}**`, + '', + `Missing: **${report.missingRepositories}** · Extras preserved: **${report.extraRepositories}** · Canonical hygiene findings: **${report.expectedHygieneFindings}**`, + '', + '| Production org | Test org | Canonical | Present | Missing | Extras | Hygiene |', + '|---|---|---:|---:|---:|---:|---:|', + ]; + for (const item of report.reports) { + lines.push(`| \`${item.sourceOrganization}\` | \`${item.testOrganization}\` | ${item.expectedRepositories} | ${item.present.length} | ${item.missing.length} | ${item.extras.length} | ${item.expectedHygiene.length} |`); + } + const missing = report.reports.filter((item) => item.missing.length > 0); + if (missing.length) { + lines.push('', '## Missing canonical repositories', ''); + for (const item of missing) lines.push(`- **${item.testOrganization}:** ${item.missing.map((name) => `\`${name}\``).join(', ')}`); + } + const hygiene = report.reports.filter((item) => item.expectedHygiene.length > 0); + if (hygiene.length) { + lines.push('', '## Canonical repository hygiene findings', ''); + for (const item of hygiene) { + for (const finding of item.expectedHygiene) lines.push(`- **${item.testOrganization}/${finding.repository}:** ${finding.finding}`); + } + } + lines.push('', 'Extras are reported but preserved; use `--strict-extras` only for an intentional exact-equality audit.', ''); + return lines.join('\n'); +} + +async function main() { + const options = parseArguments(process.argv.slice(2)); + if (options.help) { + console.log('Usage: node scripts/audit-live-test-org-fleet.mjs [--json] [--strict-extras] [--org ]'); + return; + } + const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); + const manifest = readFleetManifest(path.join(root, 'bootstrap', 'test-org-fleet.json.gz')); + const report = await auditFleet(manifest, createGitHubClient(), options); + const markdown = renderMarkdown(report); + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${markdown}\n`); + if (options.json) process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + else process.stdout.write(`${markdown}\n`); + const failure = report.missingRepositories > 0 + || report.expectedHygieneFindings > 0 + || (options.strictExtras && report.extraRepositories > 0); + if (failure) process.exitCode = 1; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await main(); diff --git a/scripts/audit-test-org-fleet-live.mjs b/scripts/audit-test-org-fleet-live.mjs new file mode 100644 index 00000000..8b962dbc --- /dev/null +++ b/scripts/audit-test-org-fleet-live.mjs @@ -0,0 +1,68 @@ +#!/usr/bin/env node +// Preserve branch path separators for GitHub's git-ref endpoint while keeping +// every individual segment URL-encoded. Harden live audit retry behavior at +// stable seams without changing the deterministic core report implementation. +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const scriptsDirectory = path.join(root, 'scripts'); +const auditorPath = path.join(scriptsDirectory, 'audit-test-org-fleet.mjs'); +let source = fs.readFileSync(auditorPath, 'utf8').replace(/^#![^\n]*\n/, ''); + +const originalBranchEncoding = 'const encodedBranch = route(branch);'; +const hardenedBranchEncoding = "const encodedBranch = branch.split('/').map(route).join('/');"; +if (!source.includes(originalBranchEncoding)) throw new Error('fleet audit branch-path seam changed'); +source = source.replace(originalBranchEncoding, hardenedBranchEncoding); + +const originalRateLimitBlock = ` const secondary = response.status === 403 + && /secondary rate limit|abuse detection|temporarily blocked/i.test(text); + if ((secondary || response.status === 429 || response.status >= 500) && attempt < attempts - 1) { + const retryAfter = Number(response.headers.get('retry-after') ?? 0) * 1000; + const reset = Number(response.headers.get('x-ratelimit-reset') ?? 0) * 1000; + const resetDelay = reset > Date.now() ? reset - Date.now() : 0; + const backoff = secondary + ? Math.min(120000, 5000 * (2 ** attempt)) + : Math.min(30000, 1000 * (2 ** attempt)); + await sleep(Math.max(retryAfter, resetDelay, backoff)); + continue; + }`; +const hardenedRateLimitBlock = ` const secondary = response.status === 403 + && /secondary rate limit|abuse detection|temporarily blocked/i.test(text); + const primary = response.status === 403 + && /API rate limit exceeded/i.test(text); + if ((secondary || primary || response.status === 429 || response.status >= 500) && attempt < attempts - 1) { + const retryAfter = Number(response.headers.get('retry-after') ?? 0) * 1000; + const reset = Number(response.headers.get('x-ratelimit-reset') ?? 0) * 1000; + const resetDelay = primary && reset > Date.now() + ? Math.max(0, reset - Date.now() + 1000) + : 0; + const backoff = primary + ? Math.min(60000, 5000 * (2 ** attempt)) + : secondary + ? Math.min(120000, 5000 * (2 ** attempt)) + : Math.min(30000, 1000 * (2 ** attempt)); + await sleep(Math.max(retryAfter, resetDelay, backoff)); + continue; + }`; +if (!source.includes(originalRateLimitBlock)) throw new Error('fleet audit rate-limit seam changed'); +source = source.replace(originalRateLimitBlock, hardenedRateLimitBlock); + +const originalConcurrency = 'mapConcurrent(generatedWork, 6,'; +const hardenedConcurrency = 'mapConcurrent(generatedWork, 2,'; +if (!source.includes(originalConcurrency)) throw new Error('fleet audit concurrency seam changed'); +source = source.replace(originalConcurrency, hardenedConcurrency); + +// Import the patched auditor from the scripts directory rather than a data URL. +// This preserves its relative `./read-fleet-manifest.mjs` import under Node 22. +const temporaryAuditorPath = path.join( + scriptsDirectory, + `.audit-test-org-fleet-live-${process.pid}-${Date.now()}.mjs`, +); +try { + fs.writeFileSync(temporaryAuditorPath, source, { encoding: 'utf8', mode: 0o600 }); + await import(pathToFileURL(temporaryAuditorPath).href); +} finally { + fs.rmSync(temporaryAuditorPath, { force: true }); +} diff --git a/scripts/audit-test-org-fleet.mjs b/scripts/audit-test-org-fleet.mjs new file mode 100644 index 00000000..02c3cdec --- /dev/null +++ b/scripts/audit-test-org-fleet.mjs @@ -0,0 +1,360 @@ +#!/usr/bin/env node +// generated by zed-pkg-test/test-org-fleet +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { readFleetManifest } from './read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const manifestPath = path.join(root, 'bootstrap', 'test-org-fleet.json.gz'); +const manifest = readFleetManifest(manifestPath); +const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN || ''; +const apiBase = 'https://api.github.com'; +const args = new Map(); +for (let index = 2; index < process.argv.length; index += 1) { + const argument = process.argv[index]; + if (!argument.startsWith('--')) continue; + const next = process.argv[index + 1]; + if (next && !next.startsWith('--')) { + args.set(argument, next); + index += 1; + } else { + args.set(argument, true); + } +} + +const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)); +const route = (value) => encodeURIComponent(String(value)); +const excluded = new Set(manifest.policy?.excludedOrganizations ?? []); + +function expectedFleet() { + if (!excluded.has('r2g') || !excluded.has('r2g-test')) { + throw new Error('r2g and r2g-test must remain explicitly excluded'); + } + + const organizations = []; + for (const pair of manifest.pairs) { + if (excluded.has(pair.sourceOrg) || excluded.has(pair.testOrg)) { + throw new Error(`excluded organization leaked into ${pair.sourceOrg}=>${pair.testOrg}`); + } + + const retained = (pair.existingRepositories ?? []).map((name) => ({ + name, + generated: false, + visibility: null, + branch: null, + kind: 'retained', + })); + const generated = [ + { + name: '.github', + generated: true, + visibility: 'public', + branch: manifest.policy.governanceBranch, + kind: 'governance', + }, + ...pair.repositories.map((repository) => ({ + name: repository.name, + generated: true, + visibility: repository.visibility ?? pair.defaultVisibility, + branch: manifest.policy.branch, + kind: 'harness', + })), + ]; + + const repositories = [...retained, ...generated]; + if (new Set(repositories.map(({ name }) => name)).size !== repositories.length) { + throw new Error(`duplicate expected repository in ${pair.testOrg}`); + } + organizations.push({ + sourceOrg: pair.sourceOrg, + testOrg: pair.testOrg, + repositories, + expectedRepositories: repositories.length, + expectedGenerated: generated.length, + expectedRetained: retained.length, + }); + } + return organizations; +} + +const expected = expectedFleet(); +const plan = { + organizations: expected.length, + repositories: expected.reduce((sum, organization) => sum + organization.expectedRepositories, 0), + generatedRepositories: expected.reduce((sum, organization) => sum + organization.expectedGenerated, 0), + retainedRepositories: expected.reduce((sum, organization) => sum + organization.expectedRetained, 0), + expectedByOrganization: Object.fromEntries(expected.map((organization) => [organization.testOrg, organization.expectedRepositories])), +}; + +if (args.has('--plan-json')) { + process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`); + process.exit(0); +} + +if (!token) throw new Error('GH_TOKEN is required for a live fleet audit'); + +async function api(endpoint, { allow404 = false, attempts = 7 } = {}) { + for (let attempt = 0; attempt < attempts; attempt += 1) { + const response = await fetch(`${apiBase}${endpoint}`, { + headers: { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'x-github-api-version': '2022-11-28', + 'user-agent': 'zed-pkg-test-fleet-audit', + }, + }); + const text = await response.text(); + if (response.ok) return text ? JSON.parse(text) : null; + if (response.status === 404 && allow404) return null; + + const secondary = response.status === 403 + && /secondary rate limit|abuse detection|temporarily blocked/i.test(text); + if ((secondary || response.status === 429 || response.status >= 500) && attempt < attempts - 1) { + const retryAfter = Number(response.headers.get('retry-after') ?? 0) * 1000; + const reset = Number(response.headers.get('x-ratelimit-reset') ?? 0) * 1000; + const resetDelay = reset > Date.now() ? reset - Date.now() : 0; + const backoff = secondary + ? Math.min(120000, 5000 * (2 ** attempt)) + : Math.min(30000, 1000 * (2 ** attempt)); + await sleep(Math.max(retryAfter, resetDelay, backoff)); + continue; + } + throw new Error(`GET ${endpoint} -> ${response.status}: ${text.slice(0, 800)}`); + } + throw new Error(`GET ${endpoint} exhausted retries`); +} + +async function listOrganizationRepositories(organization) { + const repositories = []; + for (let page = 1; ; page += 1) { + const values = await api(`/orgs/${route(organization)}/repos?type=all&per_page=100&page=${page}`); + repositories.push(...values); + if (values.length < 100) return repositories; + } +} + +async function mapConcurrent(items, limit, function_) { + const queue = [...items]; + const results = new Array(items.length); + let cursor = 0; + const workers = Array.from({ length: Math.min(limit, queue.length || 1) }, async () => { + while (true) { + const current = cursor; + cursor += 1; + if (current >= queue.length) return; + results[current] = await function_(queue[current], current); + } + }); + await Promise.all(workers); + return results; +} + +async function auditGeneratedRepository(organization, expectedRepository, actualRepository) { + const branch = expectedRepository.branch; + const encodedBranch = route(branch); + const ref = await api( + `/repos/${route(organization.testOrg)}/${route(expectedRepository.name)}/git/ref/heads/${encodedBranch}`, + { allow404: true }, + ); + if (!ref) { + return { + name: expectedRepository.name, + kind: expectedRepository.kind, + status: 'error', + errors: [`missing generated branch ${branch}`], + }; + } + + const query = new URLSearchParams({ + state: 'open', + head: `${organization.testOrg}:${branch}`, + base: actualRepository.default_branch, + per_page: '100', + }); + const pulls = await api( + `/repos/${route(organization.testOrg)}/${route(expectedRepository.name)}/pulls?${query.toString()}`, + ); + const exact = pulls.filter((pull) => pull.head?.ref === branch && pull.base?.ref === actualRepository.default_branch); + const errors = []; + if (exact.length !== 1) errors.push(`expected one open generated PR, found ${exact.length}`); + + const pull = exact[0] ?? null; + if (pull && pull.draft !== true) errors.push(`generated PR #${pull.number} is not draft`); + if (pull && pull.head?.sha !== ref.object?.sha) { + errors.push(`generated PR #${pull.number} head ${pull.head?.sha} differs from branch ${ref.object?.sha}`); + } + + return { + name: expectedRepository.name, + kind: expectedRepository.kind, + status: errors.length ? 'error' : 'ok', + visibility: actualRepository.visibility, + defaultBranch: actualRepository.default_branch, + generatedBranch: branch, + generatedCommit: ref.object?.sha ?? null, + pullRequest: pull ? { + number: pull.number, + draft: pull.draft, + url: pull.html_url, + head: pull.head?.sha ?? null, + base: pull.base?.ref ?? null, + } : null, + errors, + }; +} + +async function auditOrganization(organization) { + const actual = await listOrganizationRepositories(organization.testOrg); + const actualByName = new Map(actual.map((repository) => [repository.name, repository])); + const expectedNames = new Set(organization.repositories.map((repository) => repository.name)); + const unexpected = actual.map((repository) => repository.name).filter((name) => !expectedNames.has(name)).sort(); + const records = []; + + const generatedWork = []; + for (const expectedRepository of organization.repositories) { + const repository = actualByName.get(expectedRepository.name); + if (!repository) { + records.push({ + name: expectedRepository.name, + kind: expectedRepository.kind, + status: 'error', + errors: ['repository is missing'], + }); + continue; + } + + const errors = []; + if (expectedRepository.visibility && repository.visibility !== expectedRepository.visibility) { + errors.push(`visibility is ${repository.visibility}; expected ${expectedRepository.visibility}`); + } + if (!expectedRepository.generated) { + records.push({ + name: expectedRepository.name, + kind: expectedRepository.kind, + status: errors.length ? 'error' : 'ok', + visibility: repository.visibility, + defaultBranch: repository.default_branch, + errors, + }); + continue; + } + generatedWork.push({ expectedRepository, repository, visibilityErrors: errors }); + } + + const generatedRecords = await mapConcurrent(generatedWork, 6, async ({ expectedRepository, repository, visibilityErrors }) => { + try { + const result = await auditGeneratedRepository(organization, expectedRepository, repository); + result.errors.unshift(...visibilityErrors); + if (result.errors.length) result.status = 'error'; + return result; + } catch (error) { + return { + name: expectedRepository.name, + kind: expectedRepository.kind, + status: 'error', + errors: [...visibilityErrors, error.message], + }; + } + }); + records.push(...generatedRecords); + records.sort((left, right) => left.name.localeCompare(right.name)); + + const errors = records.flatMap((record) => record.errors.map((message) => `${record.name}: ${message}`)); + return { + sourceOrg: organization.sourceOrg, + testOrg: organization.testOrg, + expectedRepositories: organization.expectedRepositories, + visibleRepositories: actual.length, + expectedGenerated: organization.expectedGenerated, + expectedRetained: organization.expectedRetained, + verifiedRepositories: records.filter((record) => record.status === 'ok').length, + verifiedGenerated: records.filter((record) => record.status === 'ok' && record.kind !== 'retained').length, + openGeneratedPullRequests: records.filter((record) => record.pullRequest).length, + unexpectedRepositories: unexpected, + errors, + repositories: records, + }; +} + +const startedAt = new Date().toISOString(); +const organizations = []; +for (const organization of expected) { + process.stderr.write(`auditing ${organization.testOrg}\n`); + try { + organizations.push(await auditOrganization(organization)); + } catch (error) { + organizations.push({ + sourceOrg: organization.sourceOrg, + testOrg: organization.testOrg, + expectedRepositories: organization.expectedRepositories, + visibleRepositories: 0, + expectedGenerated: organization.expectedGenerated, + expectedRetained: organization.expectedRetained, + verifiedRepositories: 0, + verifiedGenerated: 0, + openGeneratedPullRequests: 0, + unexpectedRepositories: [], + errors: [error.message], + repositories: [], + }); + } +} + +const summary = { + auditedAt: new Date().toISOString(), + startedAt, + organizations: organizations.length, + expectedRepositories: plan.repositories, + expectedGeneratedRepositories: plan.generatedRepositories, + expectedRetainedRepositories: plan.retainedRepositories, + verifiedRepositories: organizations.reduce((sum, organization) => sum + organization.verifiedRepositories, 0), + verifiedGeneratedRepositories: organizations.reduce((sum, organization) => sum + organization.verifiedGenerated, 0), + openGeneratedPullRequests: organizations.reduce((sum, organization) => sum + organization.openGeneratedPullRequests, 0), + errors: organizations.reduce((sum, organization) => sum + organization.errors.length, 0), + unexpectedRepositories: organizations.reduce((sum, organization) => sum + organization.unexpectedRepositories.length, 0), +}; + +const report = { + schemaVersion: 1, + generatedBy: 'zed-pkg-test/test-org-fleet live audit', + summary, + organizations, +}; + +const markdown = [ + '# Live test organization fleet audit', + '', + `Audited at: \`${summary.auditedAt}\``, + '', + '| Test organization | Expected repos | Visible repos | Verified generated | Open draft PRs | Errors |', + '|---|---:|---:|---:|---:|---:|', + ...organizations.map((organization) => `| \`${organization.testOrg}\` | ${organization.expectedRepositories} | ${organization.visibleRepositories} | ${organization.verifiedGenerated}/${organization.expectedGenerated} | ${organization.openGeneratedPullRequests}/${organization.expectedGenerated} | ${organization.errors.length} |`), + '', + `- Expected repositories: **${summary.expectedRepositories}**`, + `- Verified expected repositories: **${summary.verifiedRepositories}**`, + `- Expected generated repositories: **${summary.expectedGeneratedRepositories}**`, + `- Verified generated branches: **${summary.verifiedGeneratedRepositories}**`, + `- Open generated draft PRs: **${summary.openGeneratedPullRequests}**`, + `- Retained fixtures expected: **${summary.expectedRetainedRepositories}**`, + `- Unexpected repositories observed: **${summary.unexpectedRepositories}**`, + `- Audit errors: **${summary.errors}**`, +]; +for (const organization of organizations.filter((value) => value.errors.length)) { + markdown.push('', `## ${organization.testOrg} errors`, '', ...organization.errors.map((error) => `- ${error}`)); +} +markdown.push(''); + +if (args.get('--output')) fs.writeFileSync(args.get('--output'), `${JSON.stringify(report, null, 2)}\n`); +if (args.get('--markdown')) fs.writeFileSync(args.get('--markdown'), `${markdown.join('\n')}\n`); +process.stdout.write(`AUDIT_SUMMARY ${JSON.stringify(summary)}\n`); +process.stdout.write(`${markdown.join('\n')}\n`); + +if ( + summary.verifiedRepositories !== summary.expectedRepositories + || summary.verifiedGeneratedRepositories !== summary.expectedGeneratedRepositories + || summary.openGeneratedPullRequests !== summary.expectedGeneratedRepositories + || summary.errors !== 0 +) { + process.exit(1); +} diff --git a/scripts/bootstrap-led-dynamo.mjs b/scripts/bootstrap-led-dynamo.mjs new file mode 100644 index 00000000..196edfbb --- /dev/null +++ b/scripts/bootstrap-led-dynamo.mjs @@ -0,0 +1,170 @@ +import process from 'node:process'; +import { readdir, readFile } from 'node:fs/promises'; +import { gunzipSync } from 'node:zlib'; + +const OWNER = 'led-dynamo'; +const CONFIRMATION = process.env.LEDDY_BOOTSTRAP_CONFIRM; +const TOKEN = process.env.GH_TOKEN; +const PARTS_DIR = new URL('../bootstrap/led-dynamo.parts/', import.meta.url); + +if (CONFIRMATION !== 'CREATE_LEDDY_REPOSITORIES') { + throw new Error('LEDDY_BOOTSTRAP_CONFIRM must be CREATE_LEDDY_REPOSITORIES'); +} +if (!TOKEN) throw new Error('GH_TOKEN is required'); + +const partNames = (await readdir(PARTS_DIR)).filter((name) => name.endsWith('.b64')).sort(); +if (partNames.length === 0) throw new Error('bootstrap archive parts are missing'); +const archive = (await Promise.all(partNames.map((name) => readFile(new URL(name, PARTS_DIR), 'utf8')))).join('').replace(/\s/g, ''); +const repositories = JSON.parse(gunzipSync(Buffer.from(archive, 'base64')).toString('utf8')); + +const headers = { + accept: 'application/vnd.github+json', + authorization: `Bearer ${TOKEN}`, + 'content-type': 'application/json', + 'user-agent': 'leddy-repository-bootstrap', + 'x-github-api-version': '2022-11-28', +}; + +async function api(method, path, body, allowed = []) { + const response = await fetch(`https://api.github.com${path}`, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (response.status === 204) return null; + const text = await response.text(); + const data = text ? JSON.parse(text) : null; + if (!response.ok && !allowed.includes(response.status)) { + const message = data?.message ?? text ?? response.statusText; + throw new Error(`${method} ${path} -> ${response.status}: ${message}`); + } + return { status: response.status, data }; +} + +function encodedPath(path) { + return path.split('/').map(encodeURIComponent).join('/'); +} + +async function ensureRepository(spec) { + const path = `/repos/${OWNER}/${encodeURIComponent(spec.name)}`; + const current = await api('GET', path, undefined, [404]); + if (current.status === 404) { + await api('POST', `/orgs/${OWNER}/repos`, { + name: spec.name, + description: spec.description, + homepage: '', + private: false, + has_issues: true, + has_projects: true, + has_wiki: false, + has_discussions: true, + auto_init: false, + }); + console.log(`created ${OWNER}/${spec.name}`); + } else { + console.log(`adopting ${OWNER}/${spec.name}`); + } + + await api('PATCH', path, { + description: spec.description, + has_issues: true, + has_projects: true, + has_wiki: false, + has_discussions: true, + allow_squash_merge: true, + allow_merge_commit: true, + allow_rebase_merge: true, + delete_branch_on_merge: true, + }); + await api('PUT', `${path}/topics`, { names: spec.topics.slice(0, 20) }); +} + +async function branchHead(repo, branch) { + const result = await api( + 'GET', + `/repos/${OWNER}/${encodeURIComponent(repo)}/git/ref/heads/${encodeURIComponent(branch)}`, + undefined, + [404, 409], + ); + return [404, 409].includes(result.status) ? null : result.data.object.sha; +} + +async function createInitialCommit(spec) { + const repoPath = `/repos/${OWNER}/${encodeURIComponent(spec.name)}`; + const fileEntries = Object.entries(spec.files); + const [firstPath, firstContent] = fileEntries.find(([path]) => path === 'README.md') ?? fileEntries[0] ?? []; + if (!firstPath) throw new Error(`${spec.name} has no bootstrap files`); + + const initial = await api('PUT', `${repoPath}/contents/${encodedPath(firstPath)}`, { + message: 'chore: initialize Leddy repository', + content: Buffer.from(firstContent, 'utf8').toString('base64'), + }); + + const entries = []; + for (const [path, content] of fileEntries) { + const blob = await api('POST', `${repoPath}/git/blobs`, { content, encoding: 'utf-8' }); + entries.push({ path, mode: '100644', type: 'blob', sha: blob.data.sha }); + } + const tree = await api('POST', `${repoPath}/git/trees`, { tree: entries }); + const commit = await api('POST', `${repoPath}/git/commits`, { + message: 'chore: bootstrap Leddy repository', + tree: tree.data.sha, + parents: [initial.data.commit.sha], + }); + await api('PATCH', `${repoPath}/git/refs/heads/main`, { sha: commit.data.sha, force: false }); + return commit.data.sha; +} + +async function ensureFile(repo, path, content) { + const repoPath = `/repos/${OWNER}/${encodeURIComponent(repo)}`; + const filePath = `${repoPath}/contents/${encodedPath(path)}`; + const current = await api('GET', `${filePath}?ref=main`, undefined, [404]); + if (current.status !== 404) { + const existing = Buffer.from(current.data.content.replace(/\n/g, ''), 'base64').toString('utf8'); + if (existing === content) return false; + } + await api('PUT', filePath, { + message: `chore: reconcile ${path}`, + content: Buffer.from(content, 'utf8').toString('base64'), + branch: 'main', + ...(current.status === 404 ? {} : { sha: current.data.sha }), + }); + return true; +} + +async function seedRepository(spec) { + let mainSha = await branchHead(spec.name, 'main'); + if (!mainSha) { + mainSha = await createInitialCommit(spec); + console.log(`seeded ${OWNER}/${spec.name} at ${mainSha.slice(0, 12)}`); + } else { + let changed = 0; + for (const [path, content] of Object.entries(spec.files)) { + if (await ensureFile(spec.name, path, content)) changed += 1; + } + mainSha = await branchHead(spec.name, 'main'); + console.log(`reconciled ${OWNER}/${spec.name} (${changed} files changed)`); + } + + if (!(await branchHead(spec.name, 'dev'))) { + await api('POST', `/repos/${OWNER}/${encodeURIComponent(spec.name)}/git/refs`, { + ref: 'refs/heads/dev', + sha: mainSha, + }); + console.log(`created ${OWNER}/${spec.name} dev branch`); + } +} + +await api('GET', `/orgs/${OWNER}`); +console.log(`verified organization ${OWNER}`); + +for (const spec of repositories) await ensureRepository(spec); +for (const spec of repositories) await seedRepository(spec); + +console.log(JSON.stringify({ + organization: OWNER, + repositories: repositories.map((repository) => repository.name), + count: repositories.length, + zedPackages: ['leddy-interfaces', 'leddy-lib', 'leddy-clients'], + lockfiles: 'resolver-generated only; none fabricated', +}, null, 2)); diff --git a/scripts/bootstrap-test-org-fleet-worktree-safe.mjs b/scripts/bootstrap-test-org-fleet-worktree-safe.mjs new file mode 100644 index 00000000..279e9c29 --- /dev/null +++ b/scripts/bootstrap-test-org-fleet-worktree-safe.mjs @@ -0,0 +1,44 @@ +#!/usr/bin/env node +// Compatibility wrapper for idempotent reruns of generated branches containing gitlinks. +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; + +const scriptsDirectory = path.dirname(new URL(import.meta.url).pathname); +const launcherPath = path.join(scriptsDirectory, 'bootstrap-test-org-fleet.mjs'); +const launcher = fs.readFileSync(launcherPath, 'utf8'); + +const gitlinkIndexSeam = [ + ' runGit([', + " 'update-index',", + " '--add',", + " '--cacheinfo',", + ' `${gitlink.mode},${gitlink.sha},${gitlink.path}`,', + ' ]);', +].join('\n'); +const gitlinkWorktreePlaceholder = + " fs.mkdirSync(gitlinkPath, { recursive: true });"; + +let patchedLauncher = launcher; +if (!launcher.includes(gitlinkWorktreePlaceholder)) { + if (!launcher.includes(gitlinkIndexSeam)) { + throw new Error('bootstrap gitlink index seam changed; refusing an unsafe runtime patch'); + } + patchedLauncher = launcher.replace( + gitlinkIndexSeam, + `${gitlinkIndexSeam}\n${gitlinkWorktreePlaceholder}`, + ); +} + +const temporaryLauncher = path.join( + scriptsDirectory, + `.bootstrap-test-org-fleet-worktree-safe-${process.pid}-${Date.now()}.mjs`, +); + +fs.writeFileSync(temporaryLauncher, patchedLauncher, 'utf8'); +try { + await import(`${pathToFileURL(temporaryLauncher).href}?run=${Date.now()}`); +} finally { + fs.rmSync(temporaryLauncher, { force: true }); +} diff --git a/scripts/bootstrap-test-org-fleet.mjs b/scripts/bootstrap-test-org-fleet.mjs new file mode 100755 index 00000000..f10a0acc --- /dev/null +++ b/scripts/bootstrap-test-org-fleet.mjs @@ -0,0 +1,200 @@ +#!/usr/bin/env node +// generated by zed-pkg-test/test-org-fleet +import fs from 'node:fs'; +import path from 'node:path'; + +import { applyFleetManifestExtensions } from './fleet-manifest-extensions.mjs'; +import { hardenGeneratedIntegrationPolicy } from './test-org-fleet-integration-policy.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const partsDir = path.join(root, 'scripts', 'bootstrap-test-org-fleet.parts'); +let source = fs.readdirSync(partsDir).sort().map((name) => fs.readFileSync(path.join(partsDir, name), 'utf8')).join(''); +source = source.replace(/^#![^\n]*\n/, ''); + +// Harden the generated source at stable seams. The retained source is split +// into parts for connector-size limits; keeping transformations in this small +// launcher makes retry and transport policy auditable. +const originalAttemptLoop = 'for (let attempt = 0; attempt < 5; attempt += 1) {'; +const hardenedAttemptLoop = 'for (let attempt = 0; attempt < 14; attempt += 1) {'; +const originalRetryBlock = ` if ((response.status === 429 || response.status >= 500) && attempt < 4) { + const retryAfter = Number(response.headers.get('retry-after') ?? 0); + await new Promise((resolve) => setTimeout(resolve, Math.max(retryAfter * 1000, 500 * (2 ** attempt)))); + continue; + }`; +const hardenedRetryBlock = ` const primaryRemaining = Number(response.headers.get('x-ratelimit-remaining') ?? -1); + const primaryRateLimit = response.status === 403 && primaryRemaining === 0; + const secondaryRateLimit = response.status === 403 + && /secondary rate limit|temporarily blocked from content creation|abuse detection/i.test(text); + if ((primaryRateLimit || secondaryRateLimit || response.status === 429 || response.status >= 500) && attempt < 13) { + const retryAfterSeconds = Number(response.headers.get('retry-after') ?? 0); + const primaryRemaining = Number(response.headers.get('x-ratelimit-remaining') ?? -1); + const primaryRateLimit = primaryRemaining === 0; + const resetEpochSeconds = Number(response.headers.get('x-ratelimit-reset') ?? 0); + const resetDelayMs = primaryRateLimit && resetEpochSeconds > 0 + ? Math.max(0, (resetEpochSeconds * 1000) - Date.now() + 5000) + : 0; + const exponentialDelayMs = secondaryRateLimit + ? Math.min(180000, 15000 * (2 ** attempt)) + : Math.min(60000, 1000 * (2 ** attempt)); + const retryDelayMs = Math.max(retryAfterSeconds * 1000, resetDelayMs, exponentialDelayMs); + log('retrying GitHub API request after rate limit', { + method, + endpoint, + status: response.status, + attempt: attempt + 1, + primaryRateLimit, + primaryRemaining, + retryDelayMs, + }); + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + continue; + }`; + +const importSeam = "import zlib from 'node:zlib';"; +const extensionFunctionSource = applyFleetManifestExtensions.toString(); +const hardenedImports = `${importSeam} +import childProcess from 'node:child_process'; +import os from 'node:os'; + +${extensionFunctionSource}`; + +const originalManifestLoad = "const manifest = JSON.parse((options.manifest.endsWith('.gz') ? zlib.gunzipSync(manifestBytes) : manifestBytes).toString('utf8'));"; +const hardenedManifestLoad = "const manifest = applyFleetManifestExtensions(JSON.parse((options.manifest.endsWith('.gz') ? zlib.gunzipSync(manifestBytes) : manifestBytes).toString('utf8')), options.manifest);"; + +const originalTopicsGuard = `async function ensureTopics(owner, repository, topics) { + if (!options.apply || repository._planned) return;`; +const hardenedTopicsGuard = `async function ensureTopics(owner, repository, topics) { + if (process.env.TEST_ORG_FLEET_SKIP_TOPICS === 'true') { + log('skipping repository topics during rate-limit recovery', { + repository: \`\${owner}/\${repository.name}\`, + }); + return; + } + if (!options.apply || repository._planned) return;`; + +async function gitPushWriteTree(owner, repository, branchName, files, gitlinks, commitMessage) { + if (!options.apply || repository._planned) return false; + + const defaultBranch = repository.default_branch || manifest.policy.defaultBranch; + const worktree = fs.mkdtempSync(path.join(os.tmpdir(), 'test-org-fleet-worktree-')); + const credentialDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'test-org-fleet-credentials-')); + const askpass = path.join(credentialDirectory, 'git-askpass.sh'); + const remote = `https://github.com/${owner}/${repository.name}.git`; + const gitEnvironment = { + ...process.env, + GIT_ASKPASS: askpass, + GIT_ASKPASS_REQUIRE: 'force', + GIT_TERMINAL_PROMPT: '0', + }; + + fs.writeFileSync(askpass, `#!/bin/sh +case "$1" in + *Username*) printf '%s\\n' x-access-token ;; + *) printf '%s\\n' "$GH_TOKEN" ;; +esac +`, { mode: 0o700 }); + + function runGit(arguments_, { allowFailure = false } = {}) { + const result = childProcess.spawnSync('git', arguments_, { + cwd: worktree, + env: gitEnvironment, + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + }); + if (result.error) throw result.error; + if (result.status !== 0 && !allowFailure) { + const raw = [result.stderr, result.stdout].filter(Boolean).join('\n').trim(); + const redacted = token ? raw.replaceAll(token, '***') : raw; + throw new Error(`git ${arguments_[0]} failed with status ${result.status}: ${redacted.slice(0, 4000)}`); + } + return result; + } + + try { + runGit(['init', '--quiet']); + runGit(['config', 'user.name', 'zed-pkg-test-fleet']); + runGit(['config', 'user.email', '41898282+github-actions[bot]@users.noreply.github.com']); + runGit(['remote', 'add', 'origin', remote]); + + const existingBranch = runGit( + ['fetch', '--quiet', '--depth=1', 'origin', `refs/heads/${branchName}`], + { allowFailure: true }, + ); + if (existingBranch.status === 0) { + runGit(['checkout', '--quiet', '-B', branchName, 'FETCH_HEAD']); + } else { + runGit(['fetch', '--quiet', '--depth=1', 'origin', `refs/heads/${defaultBranch}`]); + runGit(['checkout', '--quiet', '-B', branchName, 'FETCH_HEAD']); + } + + const filePaths = []; + for (const [filePath, content] of files.entries()) { + const destination = path.join(worktree, filePath); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.writeFileSync(destination, content, 'utf8'); + filePaths.push(filePath); + } + if (filePaths.length) runGit(['add', '--', ...filePaths]); + + for (const gitlink of gitlinks) { + const gitlinkPath = path.join(worktree, gitlink.path); + fs.rmSync(gitlinkPath, { recursive: true, force: true }); + runGit([ + 'update-index', + '--add', + '--cacheinfo', + `${gitlink.mode},${gitlink.sha},${gitlink.path}`, + ]); + // Git does not materialize a gitlink worktree directory when a branch is + // checked out without submodules. Keep an empty placeholder so an + // idempotent rerun does not report the valid gitlink as a deletion. + fs.mkdirSync(gitlinkPath, { recursive: true }); + } + + if (!runGit(['status', '--porcelain=v1', '--untracked-files=no']).stdout.trim()) { + stats.noChange += 1; + log('no generated changes', { repository: `${owner}/${repository.name}` }); + return { changed: false, defaultBranch }; + } + + runGit(['commit', '--quiet', '-m', commitMessage]); + const commitSha = runGit(['rev-parse', 'HEAD']).stdout.trim(); + runGit(['push', '--quiet', 'origin', `HEAD:refs/heads/${branchName}`]); + stats.committed += 1; + log('pushed generated commit with git transport', { + repository: `${owner}/${repository.name}`, + branch: branchName, + commit: commitSha, + }); + return { changed: true, defaultBranch }; + } finally { + fs.rmSync(worktree, { recursive: true, force: true }); + fs.rmSync(credentialDirectory, { recursive: true, force: true }); + } +} + +const writeTreePattern = /async function writeTree\(owner, repository, branchName, files, gitlinks, commitMessage\) \{[\s\S]*?\n\}\n\nasync function ensurePullRequest/; +const gitPushWriteTreeSource = gitPushWriteTree + .toString() + .replace('gitPushWriteTree', 'writeTree'); + +if (!source.includes(originalAttemptLoop)) throw new Error('generated bootstrap attempt-loop seam changed'); +if (!source.includes(originalRetryBlock)) throw new Error('generated bootstrap retry seam changed'); +if (!source.includes(importSeam)) throw new Error('generated bootstrap import seam changed'); +if (!source.includes(originalManifestLoad)) throw new Error('generated bootstrap manifest-load seam changed'); +if (!source.includes(originalTopicsGuard)) throw new Error('generated bootstrap topics seam changed'); +if (!writeTreePattern.test(source)) throw new Error('generated bootstrap write-tree seam changed'); + +source = source.replace(originalAttemptLoop, hardenedAttemptLoop); +source = source.replace(originalRetryBlock, hardenedRetryBlock); +source = source.replace(importSeam, hardenedImports); +source = source.replace(originalManifestLoad, hardenedManifestLoad); +source = source.replace(originalTopicsGuard, hardenedTopicsGuard); +source = source.replace( + writeTreePattern, + `${gitPushWriteTreeSource}\n\nasync function ensurePullRequest`, +); +source = hardenGeneratedIntegrationPolicy(source); + +process.env.TEST_ORG_FLEET_REPO_ROOT = root; +await import(`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`); diff --git a/scripts/bootstrap-test-org-fleet.parts/000.mjs.part b/scripts/bootstrap-test-org-fleet.parts/000.mjs.part new file mode 100644 index 00000000..d3526c06 --- /dev/null +++ b/scripts/bootstrap-test-org-fleet.parts/000.mjs.part @@ -0,0 +1,92 @@ +#!/usr/bin/env node +// generated by zed-pkg-test/test-org-fleet +// Idempotently creates the test-repository fleet, seeds generated harnesses, +// pins source repositories, creates real gitlinks for submodule lanes, and opens draft PRs. +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import zlib from 'node:zlib'; + +const CHECKOUT_SHA = '3d3c42e5aac5ba805825da76410c181273ba90b1'; // actions/checkout v7.0.1 +const REPO_ROOT = path.resolve(process.env.TEST_ORG_FLEET_REPO_ROOT || process.cwd()); + +function parseArgs(argv) { + const result = { apply: false, orgs: [], repos: [], concurrency: 4, summaryJson: false, manifest: path.join(REPO_ROOT, 'bootstrap', 'test-org-fleet.json.gz') }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--apply') result.apply = true; + else if (arg === '--dry-run') result.apply = false; + else if (arg === '--summary-json') result.summaryJson = true; + else if (arg === '--org') result.orgs.push(argv[++index]); + else if (arg === '--repo') result.repos.push(argv[++index]); + else if (arg === '--concurrency') result.concurrency = Number(argv[++index]); + else if (arg === '--manifest') result.manifest = path.resolve(argv[++index]); + else if (arg === '--help' || arg === '-h') { + console.log(`Usage: node scripts/bootstrap-test-org-fleet.mjs [options]\n\n` + + ` --dry-run Print the plan (default)\n` + + ` --apply Create/update repos, commits, and draft PRs\n` + + ` --org Limit to a test org (repeatable)\n` + + ` --repo Limit repository names (repeatable)\n` + + ` --concurrency Parallel org workers (default: 4)\n` + + ` --manifest Alternate manifest path\n` + + ` --summary-json Emit machine-readable summary\n`); + process.exit(0); + } else throw new Error(`unknown argument: ${arg}`); + } + if (!Number.isInteger(result.concurrency) || result.concurrency < 1 || result.concurrency > 12) throw new Error('--concurrency must be an integer from 1 to 12'); + return result; +} + +const options = parseArgs(process.argv.slice(2)); +function readManifestBytes(manifestPath) { + if (fs.existsSync(manifestPath)) return fs.readFileSync(manifestPath); + const partsDir = `${manifestPath}.b64.parts`; + if (!fs.existsSync(partsDir)) throw new Error(`manifest not found: ${manifestPath}`); + const encoded = fs.readdirSync(partsDir).sort().map((name) => fs.readFileSync(path.join(partsDir, name), 'utf8').trim()).join(''); + return Buffer.from(encoded, 'base64'); +} +const manifestBytes = readManifestBytes(options.manifest); +const manifest = JSON.parse((options.manifest.endsWith('.gz') ? zlib.gunzipSync(manifestBytes) : manifestBytes).toString('utf8')); +const excluded = new Set(manifest.policy.excludedOrganizations ?? []); +const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; +const apiBase = process.env.GITHUB_API_URL || 'https://api.github.com'; +const sourceCache = new Map(); +const stats = { planned: 0, created: 0, existing: 0, gated: 0, committed: 0, noChange: 0, pullRequests: 0, errors: 0 }; + +if (options.apply && !token) throw new Error('GH_TOKEN (or GITHUB_TOKEN) is required with --apply'); +if (options.apply && manifest.schemaVersion !== 1) throw new Error(`unsupported schemaVersion ${manifest.schemaVersion}`); + +function log(message, context = {}) { + if (options.summaryJson) return; + const prefix = options.apply ? 'apply' : 'plan'; + const suffix = Object.keys(context).length ? ` ${JSON.stringify(context)}` : ''; + console.log(`[${prefix}] ${message}${suffix}`); +} + +function route(value) { + return value.split('/').map(encodeURIComponent).join('/'); +} + +async function api(method, endpoint, body, { allow404 = false } = {}) { + const url = `${apiBase}${endpoint}`; + for (let attempt = 0; attempt < 5; attempt += 1) { + const response = await fetch(url, { + method, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'zed-pkg-test-fleet-bootstrap/1', + ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + if (allow404 && response.status === 404) return null; + if (response.ok) { + if (response.status === 204) return null; + return response.json(); + } + const text = await response.text(); + if ((response.status === 429 || response.status >= 500) && attempt < 4) { + const retryAfter = Number(response.headers.get('retry-after') ?? 0); + await new Promise((resolve) => setTimeout(resolve, \ No newline at end of file diff --git a/scripts/bootstrap-test-org-fleet.parts/001.mjs.part b/scripts/bootstrap-test-org-fleet.parts/001.mjs.part new file mode 100644 index 00000000..851aa43a --- /dev/null +++ b/scripts/bootstrap-test-org-fleet.parts/001.mjs.part @@ -0,0 +1,113 @@ + Math.max(retryAfter * 1000, 500 * (2 ** attempt)))); + continue; + } + throw new Error(`${method} ${endpoint} -> ${response.status}: ${text.slice(0, 1000)}`); + } + throw new Error(`${method} ${endpoint} exhausted retries`); +} + +function globMatch(value, pattern) { + const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&').replaceAll('*', '.*').replaceAll('?', '.'); + return new RegExp(`^${escaped}$`).test(value); +} + +function selectedPair(pair) { + if (excluded.has(pair.sourceOrg) || excluded.has(pair.testOrg)) return false; + if (!pair.testOrg.toLowerCase().endsWith('-test')) throw new Error(`refusing non-test organization ${pair.testOrg}`); + return options.orgs.length === 0 || options.orgs.some((org) => org.toLowerCase() === pair.testOrg.toLowerCase()); +} + +function selectedRepo(repository) { + return options.repos.length === 0 || options.repos.some((pattern) => globMatch(repository.name, pattern)); +} + +async function getRepository(owner, name) { + return api('GET', `/repos/${route(owner)}/${route(name)}`, undefined, { allow404: true }); +} + +async function ensureRepository(owner, spec, defaultVisibility) { + let repository = await getRepository(owner, spec.name); + if (repository) { + stats.existing += 1; + log('repository exists', { repository: `${owner}/${spec.name}` }); + return repository; + } + if (!options.apply) { + stats.planned += 1; + log('would create repository', { repository: `${owner}/${spec.name}`, visibility: spec.visibility ?? defaultVisibility }); + return { owner: { login: owner }, name: spec.name, full_name: `${owner}/${spec.name}`, default_branch: manifest.policy.defaultBranch, visibility: spec.visibility ?? defaultVisibility, _planned: true }; + } + const visibility = spec.visibility ?? defaultVisibility; + repository = await api('POST', `/orgs/${route(owner)}/repos`, { + name: spec.name, + description: spec.description ?? `Generated ${spec.profile ?? 'governance'} test harness for ${owner}.`, + visibility, + private: visibility !== 'public', + auto_init: true, + has_issues: true, + has_projects: false, + has_wiki: false, + has_discussions: false, + is_template: false, + delete_branch_on_merge: true, + }); + stats.created += 1; + log('created repository', { repository: repository.full_name, visibility }); + return repository; +} + +async function resolveSource(fullName) { + if (sourceCache.has(fullName)) return sourceCache.get(fullName); + const promise = (async () => { + const [owner, name] = fullName.split('/'); + const repository = await getRepository(owner, name); + if (!repository) return { fullName, exists: false }; + const branch = repository.default_branch || 'main'; + const ref = await api('GET', `/repos/${route(owner)}/${route(name)}/git/ref/heads/${route(branch)}`, undefined, { allow404: true }); + if (!ref) return { fullName, exists: true, branch, sha: null }; + return { fullName, exists: true, branch, sha: ref.object.sha, visibility: repository.visibility }; + })(); + sourceCache.set(fullName, promise); + return promise; +} + +function generatedHeader(comment = '#') { + return `${comment} ${manifest.policy.generatedFileMarker}; edit the fleet manifest or bootstrapper, not this file.`; +} + +function renderZpkg(pair, spec, resolved) { + const deps = spec.imports.filter((dependency) => dependency.mode === 'zed'); + if (deps.length === 0) return null; + const lines = [generatedHeader('#'), '[package]', `org = ${JSON.stringify(pair.testOrg)}`, `name = ${JSON.stringify(spec.name)}`, 'version = "0.1.0"', `description = ${JSON.stringify(spec.description ?? spec.focus.join('; '))}`, 'license = "MIT"', '', '[package.repository]', 'vcs = "git"', `url = ${JSON.stringify(`https://github.com/${pair.testOrg}/${spec.name}`)}`, '', '[install]', 'dir = ".vendor/.zed"', '', '[dependencies]']; + for (const dependency of deps) lines.push(`${JSON.stringify(dependency.source)} = "^0.1.0"`); + lines.push(''); + return lines.join('\n'); +} + +function renderPlan(pair, spec, resolved, missing) { + return JSON.stringify({ + generatedBy: manifest.policy.generatedFileMarker, + schemaVersion: 1, + sourceOrganization: pair.sourceOrg, + testOrganization: pair.testOrg, + repository: spec.name, + profile: spec.profile, + description: spec.description ?? spec.focus.join('; '), + focus: spec.focus, + services: spec.services ?? [], + matrix: spec.matrix ?? {}, + requiredChecks: manifest.profiles[spec.profile]?.required ?? [], + sources: resolved, + sourceGates: missing, + imports: spec.imports ?? [], + security: { + immutableSourcePins: true, + pullRequestCredentials: false, + leastPrivilege: true, + redactSecretsAndRawMedia: true, + }, + }, null, 2) + '\n'; +} + +function renderReadme(pair, spec, resolved, missing) { + \ No newline at end of file diff --git a/scripts/bootstrap-test-org-fleet.parts/002.mjs.part b/scripts/bootstrap-test-org-fleet.parts/002.mjs.part new file mode 100644 index 00000000..dac7e804 --- /dev/null +++ b/scripts/bootstrap-test-org-fleet.parts/002.mjs.part @@ -0,0 +1,15 @@ +const sourceRows = resolved.map((source) => `| \`${source.fullName}\` | ${source.exists ? `\`${source.sha ?? 'unresolved'}\`` : '**missing / gated**'} | \`${source.branch ?? '-'}\` |`).join('\n'); + const importRows = (spec.imports ?? []).map((dependency) => `| \`${dependency.mode}\` | \`${dependency.source}\` | ${dependency.manager ? `\`${dependency.manager}\`` : '-'} | ${dependency.path ? `\`${dependency.path}\`` : dependency.subdir ? `\`${dependency.subdir}\`` : '-'} |`).join('\n') || '| _none_ | - | - | - |'; + return `${generatedHeader('\n# ${pair.testOrg}/${spec.name}\n\n${spec.description ?? `Generated ${spec.profile} harness for ${pair.sourceOrg}.`}\n\n## Contract\n\nProfile: \`${spec.profile}\` \nSource organization: \`${pair.sourceOrg}\`\n\n${spec.focus.map((item) => `- ${item}`).join('\n')}\n\n## Immutable source pins\n\n| Source | Commit | Branch observed |\n|---|---:|---|\n${sourceRows}\n\n## Dependency lanes\n\n| Mode | Source | Manager | Path/subdir |\n|---|---|---|---|\n${importRows}\n\n## Running\n\nThe pull-request workflow validates the generated contract without cross-organization credentials. Full integration is intentionally release-gated until required source repositories and organization read credentials are present. Run the profile-specific checks recorded in \`test-plan.json\` after materializing the submodule, Zed, or native-package lane.\n\n${missing.length ? `## Source gates\n\n${missing.map((source) => `- \`${source}\` does not exist or has no resolvable default-branch commit.`).join('\n')}\n` : ''}`; +} + +function renderPlanValidator() { + return `${generatedHeader('//')}\nimport fs from 'node:fs';\nimport childProcess from 'node:child_process';\n\nconst plan = JSON.parse(fs.readFileSync(new URL('../test-plan.json', import.meta.url), 'utf8'));\nconst errors = [];\nif (plan.schemaVersion !== 1) errors.push('schemaVersion must be 1');\nif (!plan.repository || !plan.profile) errors.push('repository and profile are required');\nif (!Array.isArray(plan.focus) || plan.focus.length < 2) errors.push('focus must contain at least two assertions');\nif (!Array.isArray(plan.sources) || plan.sources.length === 0) errors.push('at least one source is required');\nfor (const source of plan.sources ?? []) {\n if (source.exists && !/^[0-9a-f]{40}$/.test(source.sha ?? '')) errors.push('existing source must be pinned to a 40-character SHA: ' + source.fullName);\n}\nfor (const dependency of plan.imports ?? []) {\n if (!['git-submodule', 'zed', 'native-package'].includes(dependency.mode)) errors.push('unsupported dependency mode: ' + dependency.mode);\n}\nif (fs.existsSync(new URL('../.gitmodules', import.meta.url))) {\n // Nested gitlinks are omitted by a non-recursive tree listing.\n const tree = childProcess.execFileSync('git', ['ls-tree', '-r', 'HEAD'], { encoding: 'utf8' });\n for (const dependency of plan.imports.filter((item) => item.mode === 'git-submodule')) {\n if (!tree.includes('160000 commit') || !tree.includes(dependency.path)) errors.push('missing gitlink for ' + dependency.path);\n }\n}\nconst serialized = JSON.stringify(plan);\nif (/ghp_[A-Za-z0-9]+|github_pat_[A-Za-z0-9_]+/.test(serialized)) errors.push('credential-shaped value found');\nif (errors.length) { for (const error of errors) console.error('error: ' + error); process.exit(1); }\nconsole.log('validated generated test plan for ' + plan.testOrganization + '/' + plan.repository);\n`; +} + +function renderContractWorkflow(spec) { + return `${generatedHeader('#')}\nname: generated test harness contract\n\non:\n pull_request:\n push:\n branches: [main]\n workflow_dispatch:\n\npermissions:\n contents: read\n\nconcurrency:\n group: generated-contract-\${{ github.event.pull_request.number || github.ref }}\n cancel-in-progress: true\n\njobs:\n contract:\n runs-on: ubuntu-24.04\n timeout-minutes: 10\n steps:\n - name: Check out without credentials\n uses: actions/checkout@${CHECKOUT_SHA} # v7.0.1\n with:\n persist-credentials: false\n show-progress: false\n submodules: false\n - name: Validate generated plan, source pins, and gitlinks\n run: node scripts/validate-plan.mjs\n - name: Record profile\n run: printf '%s\\n' ${JSON.stringify(spec.profile)} > generated-profile.txt\n`; +} + +function renderIntegrationWorkflow(spec) { + return `${generatedHeader('#')}\nname: gated integration\n\non:\n workflow_dispatch:\n schedule:\n - cron: '17 8 * * *'\n\npermissions:\n contents: read\n\njobs:\n integration:\n if: \${{ vars.ENABLE_TEST_FLEET_INTEGRATION == 'true' }}\n runs-on: ${manifest.profiles[spec.profile]?.runner === 'matrix' ? 'ubuntu-24.04' : manifest.profiles[spec.profile]?.runner ?? 'ubuntu-24.04'}\n timeout-minutes: 45\n steps:\n - name: Check out source pins and submodules\n \ No newline at end of file diff --git a/scripts/bootstrap-test-org-fleet.parts/003.mjs.part b/scripts/bootstrap-test-org-fleet.parts/003.mjs.part new file mode 100644 index 00000000..f41e3451 --- /dev/null +++ b/scripts/bootstrap-test-org-fleet.parts/003.mjs.part @@ -0,0 +1,64 @@ + uses: actions/checkout@${CHECKOUT_SHA} # v7.0.1\n with:\n persist-credentials: false\n show-progress: false\n submodules: recursive\n token: \${{ secrets.TEST_FLEET_READ_TOKEN || github.token }}\n - name: Validate contract\n run: node scripts/validate-plan.mjs\n - name: Explain profile gate\n run: |\n echo "Profile: ${spec.profile}"\n echo "Required checks are defined in test-plan.json."\n echo "Add product-specific executable fixtures here without weakening the generated contract."\n`; +} + +function renderGitmodules(spec, resolvedByName) { + const lines = [generatedHeader('#')]; + const gitlinks = []; + for (const dependency of spec.imports.filter((item) => item.mode === 'git-submodule')) { + const source = resolvedByName.get(dependency.source); + if (!source?.exists || !source.sha) continue; + lines.push(`[submodule ${JSON.stringify(dependency.path)}]`, `\tpath = ${dependency.path}`, `\turl = https://github.com/${dependency.source}.git`, ''); + gitlinks.push({ path: dependency.path, mode: '160000', type: 'commit', sha: source.sha }); + } + return { content: gitlinks.length ? `${lines.join('\n')}\n` : null, gitlinks }; +} + +function renderPackageConsumers(spec, resolvedByName) { + const consumers = spec.imports.filter((dependency) => dependency.mode === 'native-package').map((dependency) => ({ + manager: dependency.manager, + package: dependency.package ?? null, + subdir: dependency.subdir ?? null, + source: dependency.source, + sourceSha: resolvedByName.get(dependency.source)?.sha ?? null, + status: dependency.package ? 'coordinate-declared' : 'discover-from-source-artifact', + })); + return consumers.length ? `${JSON.stringify({ generatedBy: manifest.policy.generatedFileMarker, consumers }, null, 2)}\n` : null; +} + +function renderSourcePins(resolved) { + return `${JSON.stringify({ generatedBy: manifest.policy.generatedFileMarker, sources: Object.fromEntries(resolved.map((source) => [source.fullName, { branch: source.branch ?? null, sha: source.sha ?? null, exists: source.exists }])) }, null, 2)}\n`; +} + +async function generateRepositoryFiles(pair, spec) { + const sourceNames = [...new Set([...(spec.sources ?? []), ...(spec.imports ?? []).map((dependency) => dependency.source)])]; + const resolved = await Promise.all(sourceNames.map(resolveSource)); + const missing = resolved.filter((source) => !source.exists || !source.sha).map((source) => source.fullName); + stats.gated += missing.length ? 1 : 0; + const resolvedByName = new Map(resolved.map((source) => [source.fullName, source])); + const gitmodules = renderGitmodules(spec, resolvedByName); + const files = new Map([ + ['README.md', renderReadme(pair, spec, resolved, missing)], + ['test-plan.json', renderPlan(pair, spec, resolved, missing)], + ['source-pins.json', renderSourcePins(resolved)], + ['scripts/validate-plan.mjs', renderPlanValidator()], + ['.github/workflows/test-harness-contract.yml', renderContractWorkflow(spec)], + ['.github/workflows/integration.yml', renderIntegrationWorkflow(spec)], + ['.gitignore', `${generatedHeader('#')}\n.env\n.env.*\n!.env.example\nnode_modules/\n.dart_tool/\ntarget/\nbuild/\n.zed/\n.vendor/\n`], + ]); + const zpkg = renderZpkg(pair, spec, resolved); + if (zpkg) files.set('.zpkg.toml', zpkg); + const packages = renderPackageConsumers(spec, resolvedByName); + if (packages) files.set('package-consumers.json', packages); + if (gitmodules.content) files.set('.gitmodules', gitmodules.content); + if (missing.length) files.set('SOURCE-GATE.md', `${generatedHeader('\n# Source gates\n\n${missing.map((source) => `- [ ] Create or expose \`${source}\`, then rerun the fleet bootstrap to pin its default-branch commit.`).join('\n')}\n`); + return { files, gitlinks: gitmodules.gitlinks, resolved, missing }; +} + +function renderGovernanceFiles(pair) { + const repoCount = pair.repositories.length; + const profileCounts = {}; + for (const repository of pair.repositories) profileCounts[repository.profile] = (profileCounts[repository.profile] ?? 0) + 1; + const profileList = Object.entries(profileCounts).sort(([a], [b]) => a.localeCompare(b)).map(([profile, count]) => `- \`${profile}\`: ${count}`).join('\n'); + return new Map([ + ['profile/README.md', `${generatedHeader('\n# ${pair.testOrg}\n\nTest organization for \`${pair.sourceOrg}\`. The fleet contains ${repoCount} specialized repositories plus this governance repository.\n\n${pair.description}\n\n## Profiles\n\n${profileList}\n\nAll generated pull-request workflows use least privilege, immutable action pins, and no persisted checkout credential. Integration workflows are gated by organization variables and credentials.\n`], + ['README.md', \ No newline at end of file diff --git a/scripts/bootstrap-test-org-fleet.parts/004.mjs.part b/scripts/bootstrap-test-org-fleet.parts/004.mjs.part new file mode 100644 index 00000000..08600bbc --- /dev/null +++ b/scripts/bootstrap-test-org-fleet.parts/004.mjs.part @@ -0,0 +1,72 @@ +`${generatedHeader('\n# ${pair.testOrg} governance\n\nCommunity health, reusable workflow policy, and fleet conventions for ${pair.testOrg}.\n`], + ['SECURITY.md', `${generatedHeader('\n# Security policy\n\nDo not commit access tokens, production credentials, raw biometric material, unencrypted private media, or unredacted message content. Reproduce security failures with synthetic fixtures. Pull-request jobs must be credential-free; private cross-organization integration uses the organization-managed \`TEST_FLEET_READ_TOKEN\` only in gated workflows.\n`], + ['CONTRIBUTING.md', `${generatedHeader('\n# Contributing\n\nKeep source dependencies pinned to immutable 40-character commits. Exercise at least one real dependency lane (submodule, Zed, or native package) and preserve the generated contract. Add product assertions rather than replacing contract checks with no-op smoke tests.\n`], + ['pull_request_template.md', `${generatedHeader('\n## Test surface\n- [ ] Source commits are immutable\n- [ ] Product assertions execute\n- [ ] Failure and recovery paths execute\n- [ ] Logs contain no secrets or raw private media\n- [ ] Emulator/browser/database matrices are justified\n`], + ['.github/workflows/reusable-plan-contract.yml', `${generatedHeader('#')}\nname: reusable generated plan contract\n\non:\n workflow_call:\n\npermissions:\n contents: read\n\njobs:\n validate:\n runs-on: ubuntu-24.04\n timeout-minutes: 10\n steps:\n - uses: actions/checkout@${CHECKOUT_SHA} # v7.0.1\n with:\n persist-credentials: false\n show-progress: false\n submodules: false\n - run: node scripts/validate-plan.mjs\n`], + ]); +} + +async function ensureBranch(owner, repository, branchName) { + const defaultBranch = repository.default_branch || manifest.policy.defaultBranch; + const baseRef = await api('GET', `/repos/${route(owner)}/${route(repository.name)}/git/ref/heads/${route(defaultBranch)}`); + let branchRef = await api('GET', `/repos/${route(owner)}/${route(repository.name)}/git/ref/heads/${route(branchName)}`, undefined, { allow404: true }); + if (!branchRef) { + branchRef = await api('POST', `/repos/${route(owner)}/${route(repository.name)}/git/refs`, { ref: `refs/heads/${branchName}`, sha: baseRef.object.sha }); + log('created branch', { repository: `${owner}/${repository.name}`, branch: branchName }); + } + return { branchRef, defaultBranch }; +} + +async function writeTree(owner, repository, branchName, files, gitlinks, commitMessage) { + if (!options.apply || repository._planned) return false; + const { branchRef, defaultBranch } = await ensureBranch(owner, repository, branchName); + const headSha = branchRef.object.sha; + const headCommit = await api('GET', `/repos/${route(owner)}/${route(repository.name)}/git/commits/${headSha}`); + const treeEntries = []; + for (const [filePath, content] of files.entries()) { + const blob = await api('POST', `/repos/${route(owner)}/${route(repository.name)}/git/blobs`, { content, encoding: 'utf-8' }); + treeEntries.push({ path: filePath, mode: '100644', type: 'blob', sha: blob.sha }); + } + treeEntries.push(...gitlinks); + const tree = await api('POST', `/repos/${route(owner)}/${route(repository.name)}/git/trees`, { base_tree: headCommit.tree.sha, tree: treeEntries }); + if (tree.sha === headCommit.tree.sha) { + stats.noChange += 1; + log('no generated changes', { repository: `${owner}/${repository.name}` }); + return { changed: false, defaultBranch }; + } + const commit = await api('POST', `/repos/${route(owner)}/${route(repository.name)}/git/commits`, { message: commitMessage, tree: tree.sha, parents: [headSha] }); + await api('PATCH', `/repos/${route(owner)}/${route(repository.name)}/git/refs/heads/${route(branchName)}`, { sha: commit.sha, force: false }); + stats.committed += 1; + log('pushed generated commit', { repository: `${owner}/${repository.name}`, branch: branchName, commit: commit.sha }); + return { changed: true, defaultBranch }; +} + +async function ensurePullRequest(owner, repository, branchName, base, title, body) { + if (!options.apply || repository._planned) return; + const pulls = await api('GET', `/repos/${route(owner)}/${route(repository.name)}/pulls?state=open&head=${encodeURIComponent(`${owner}:${branchName}`)}`); + const matching = pulls + .filter((pull) => pull.head?.ref === branchName && pull.base?.ref === base) + .sort((left, right) => left.number - right.number); + const drafts = matching.filter((pull) => pull.draft === true); + + if (drafts.length) { + const keeper = drafts[0]; + for (const duplicate of matching.filter((pull) => pull.number !== keeper.number)) { + await api('PATCH', `/repos/${route(owner)}/${route(repository.name)}/pulls/${duplicate.number}`, { state: 'closed' }); + log('closed duplicate generated pull request', { repository: `${owner}/${repository.name}`, number: duplicate.number, keeper: keeper.number }); + } + log('draft pull request already open', { repository: `${owner}/${repository.name}`, number: keeper.number }); + return; + } + + for (const nonDraft of matching) { + await api('PATCH', `/repos/${route(owner)}/${route(repository.name)}/pulls/${nonDraft.number}`, { state: 'closed' }); + log('closed non-draft generated pull request', { repository: `${owner}/${repository.name}`, number: nonDraft.number }); + } + + const pull = await api('POST', `/repos/${route(owner)}/${route(repository.name)}/pulls`, { title, head: branchName, base, body, draft: true }); + stats.pullRequests += 1; + log('opened draft pull request', { repository: `${owner}/${repository.name}`, number: pull.number }); +} + +async function ens \ No newline at end of file diff --git a/scripts/bootstrap-test-org-fleet.parts/005.mjs.part b/scripts/bootstrap-test-org-fleet.parts/005.mjs.part new file mode 100644 index 00000000..971385ec --- /dev/null +++ b/scripts/bootstrap-test-org-fleet.parts/005.mjs.part @@ -0,0 +1,64 @@ +ureTopics(owner, repository, topics) { + if (!options.apply || repository._planned) return; + const normalized = [...new Set(topics.map((topic) => topic.toLowerCase().replace(/[^a-z0-9-]/g, '-')).filter(Boolean))].slice(0, 20); + await api('PUT', `/repos/${route(owner)}/${route(repository.name)}/topics`, { names: normalized }); +} + +async function applyGovernance(pair) { + const spec = { name: '.github', profile: 'governance', description: `Governance and reusable test workflow policy for ${pair.testOrg}.`, visibility: 'public' }; + const repository = await ensureRepository(pair.testOrg, spec, 'public'); + if (!options.apply || repository._planned) return; + const result = await writeTree(pair.testOrg, repository, manifest.policy.governanceBranch, renderGovernanceFiles(pair), [], 'chore: bootstrap test org governance'); + await ensurePullRequest(pair.testOrg, repository, manifest.policy.governanceBranch, result.defaultBranch, 'chore: bootstrap test organization governance', `## Changes\n- add organization profile and community health files\n- add a credential-free reusable contract workflow\n- document immutable source-pin and privacy requirements\n\n## Why\nStandardize governance for the ${pair.sourceOrg} test fleet.\n\n## Checks\n- generated files are deterministic\n- checkout credentials are not persisted\n- actions are pinned to immutable commits`); + await ensureTopics(pair.testOrg, repository, ['test-fleet', 'governance', 'github-actions']); +} + +async function applyTestRepository(pair, spec) { + stats.planned += options.apply ? 0 : 1; + const repository = await ensureRepository(pair.testOrg, spec, pair.defaultVisibility); + if (!options.apply || repository._planned) return; + const generated = await generateRepositoryFiles(pair, spec); + const result = await writeTree(pair.testOrg, repository, manifest.policy.branch, generated.files, generated.gitlinks, `test: bootstrap ${spec.profile} harness`); + await ensurePullRequest(pair.testOrg, repository, manifest.policy.branch, result.defaultBranch, `test: bootstrap ${spec.profile} harness`, `## Changes\n- add the generated ${spec.profile} contract\n- pin source repositories to immutable commits\n- add submodule, Zed, and native-package lanes where declared\n- add credential-free PR validation and gated integration workflow\n\n## Why\nCreate the specialized ${pair.sourceOrg} test surface for: ${spec.focus.join('; ')}.\n\n## Impact\n${generated.missing.length ? `Source-gated until: ${generated.missing.join(', ')}.` : 'All declared source repositories resolved at bootstrap time.'}\n\n## Checks\n- node scripts/validate-plan.mjs\n- profile-specific checks are recorded in test-plan.json`); + await ensureTopics(pair.testOrg, repository, [...(spec.topics ?? []), pair.sourceOrg.toLowerCase(), spec.profile]); +} + +async function mapConcurrent(items, limit, fn) { + const queue = [...items]; + const workers = Array.from({ length: Math.min(limit, queue.length) }, async () => { + while (queue.length) { + const item = queue.shift(); + try { await fn(item); } + catch (error) { stats.errors += 1; console.error(`error: ${error.stack || error.message}`); } + } + }); + await Promise.all(workers); +} + +const pairs = manifest.pairs.filter(selectedPair); +for (const pair of pairs) { + const repos = pair.repositories.filter(selectedRepo); + log('selected test organization', { source: pair.sourceOrg, test: pair.testOrg, repositories: repos.length, governance: options.repos.length === 0 }); +} + +if (!options.apply) { + const selectedRepos = pairs.flatMap((pair) => pair.repositories.filter(selectedRepo).map((repository) => ({ pair, repository }))); + stats.planned = selectedRepos.length + (options.repos.length === 0 ? pairs.length : 0); + for (const { pair, repository } of selectedRepos) log('would create or update harness', { repository: `${pair.testOrg}/${repository.name}`, profile: repository.profile, sourceCount: repository.sources.length, importLanes: repository.imports.length }); +} else { + await mapConcurrent(pairs, options.concurrency, async (pair) => { + if (options.repos.length === 0) await applyGovernance(pair); + for (const spec of pair.repositories.filter(selectedRepo)) await applyTestRepository(pair, spec); + }); +} + +const summary = { + mode: options.apply ? 'apply' : 'dry-run', + selectedOrganizations: pairs.length, + selectedRepositories: pairs.reduce((sum, pair) => sum + pair.repositories.filter(selectedRepo).length, 0), + governanceRepositories: options.repos.length === 0 ? pairs.length : 0, + ...stats, +}; +if (options.summaryJson) process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); +else console.log(`summary ${JSON.stringify(summary)}`); +if (stats.errors) process.exit(1); diff --git a/scripts/canonical_task_canary.py b/scripts/canonical_task_canary.py new file mode 100644 index 00000000..87fd01b4 --- /dev/null +++ b/scripts/canonical_task_canary.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 +"""Independent black-box certification for `zed task` and `zed-task` parity.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import shutil +import subprocess +import sys +import textwrap +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Callable, Sequence + + +@dataclass +class CaseResult: + name: str + status: str + duration_ms: int + detail: str = "" + + +class CertificationFailure(RuntimeError): + pass + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--zed-task", required=True, type=Path) + parser.add_argument("--work", required=True, type=Path) + parser.add_argument("--evidence", required=True, type=Path) + parser.add_argument("--candidate", required=True) + return parser.parse_args() + + +def runtime_env(work: Path) -> dict[str, str]: + keep = [ + "PATH", + "SystemRoot", + "SYSTEMROOT", + "ComSpec", + "COMSPEC", + "PATHEXT", + "WINDIR", + "TEMP", + "TMP", + "LANG", + "LC_ALL", + ] + env = {name: os.environ[name] for name in keep if name in os.environ} + home = work / "home" + home.mkdir(parents=True, exist_ok=True) + env.update( + { + "HOME": str(home), + "USERPROFILE": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "ZED_PKG_HOME": str(home / ".zed-pkg"), + "ZED_PKG_UPDATE_CHECK": "false", + "NO_COLOR": "1", + "CLICOLOR": "0", + "RUST_BACKTRACE": "0", + } + ) + return env + + +def run( + binary: Path, + args: Sequence[str], + *, + cwd: Path, + env: dict[str, str], + expect_success: bool = True, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [str(binary), *args], + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + succeeded = result.returncode == 0 + if succeeded != expect_success: + expectation = "success" if expect_success else "failure" + raise CertificationFailure( + f"expected {expectation} from {binary.name} {list(args)!r}; " + f"exit={result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + +def main_args(args: Sequence[str]) -> list[str]: + return ["task", *args] + + +def write_plan(root: Path) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "input.txt").write_text("one\n", encoding="utf-8") + (root / "zed-env.toml").write_text( + textwrap.dedent( + r''' + schema = 2 + + [env] + MESSAGE = "hello" + + [tasks.prepare] + description = "prepare inputs" + run = ["printf 'prepare\\n' > order.txt"] + run_windows = ["echo prepare>order.txt"] + + [tasks.build] + description = "build project" + aliases = ["b"] + depends = ["prepare"] + run = ["printf '%s\\n' \"$MESSAGE\" >> order.txt"] + run_windows = ["echo %MESSAGE%>>order.txt"] + + [tasks.args] + run = ["printf '%s' \"$ZED_TASK_ARG_0\" > arg.txt"] + run_windows = ["echo %ZED_TASK_ARG_0%>arg.txt"] + + [tasks.copy] + cache = true + sources = ["input.txt"] + outputs = ["output.txt"] + run = ["cat input.txt > output.txt"] + run_windows = ["type input.txt > output.txt"] + + [tasks.release] + confirm = "publish release?" + run = ["printf release > release.txt"] + run_windows = ["echo release>release.txt"] + ''' + ).lstrip(), + encoding="utf-8", + ) + + +def assert_parity( + zed: Path, + staged: Path, + root: Path, + env: dict[str, str], + args: Sequence[str], +) -> None: + canonical = run(zed, main_args(args), cwd=root, env=env) + compatibility = run(staged, args, cwd=root, env=env) + if canonical.stdout != compatibility.stdout: + raise CertificationFailure( + f"stdout differs for {list(args)!r}\ncanonical:\n{canonical.stdout}\n" + f"staged:\n{compatibility.stdout}" + ) + if canonical.stderr != compatibility.stderr: + raise CertificationFailure( + f"stderr differs for {list(args)!r}\ncanonical:\n{canonical.stderr}\n" + f"staged:\n{compatibility.stderr}" + ) + + +def case_json_parity(zed: Path, staged: Path, root: Path, env: dict[str, str]) -> None: + for args in [ + ["--json", "list"], + ["--json", "info", "b"], + ["--json", "graph", "b"], + ["--json", "run", "b", "--dry-run"], + ]: + assert_parity(zed, staged, root, env, args) + + listed = run(zed, main_args(["--json", "list"]), cwd=root, env=env) + tasks = json.loads(listed.stdout) + names = [task["name"] for task in tasks] + if names != sorted(names) or "build" not in names: + raise CertificationFailure(f"unexpected deterministic task list: {names!r}") + + +def case_execution(zed: Path, _staged: Path, root: Path, env: dict[str, str]) -> None: + output = run(zed, main_args(["run", "b", "--jobs", "2"]), cwd=root, env=env) + if "[done] build" not in output.stderr: + raise CertificationFailure(f"missing completion event: {output.stderr}") + lines = [line.strip() for line in (root / "order.txt").read_text().splitlines()] + if lines != ["prepare", "hello"]: + raise CertificationFailure(f"dependency/environment order drift: {lines!r}") + + +def case_arguments(zed: Path, staged: Path, root: Path, env: dict[str, str]) -> None: + for binary, args in [ + (zed, main_args(["run", "args", "--", "hello world"])), + (staged, ["run", "args", "--", "hello world"]), + ]: + (root / "arg.txt").unlink(missing_ok=True) + run(binary, args, cwd=root, env=env) + value = (root / "arg.txt").read_text(encoding="utf-8").strip() + if value != "hello world": + raise CertificationFailure(f"argument transport drift for {binary.name}: {value!r}") + + +def case_content_cache(zed: Path, _staged: Path, root: Path, env: dict[str, str]) -> None: + first = run(zed, main_args(["run", "copy"]), cwd=root, env=env) + if "[done] copy" not in first.stderr: + raise CertificationFailure("first cacheable execution did not run") + if (root / "output.txt").read_text(encoding="utf-8") != "one\n": + raise CertificationFailure("first cached output content is wrong") + + second = run(zed, main_args(["run", "copy"]), cwd=root, env=env) + if "incremental cache hit" not in second.stderr: + raise CertificationFailure(f"second execution did not hit cache: {second.stderr}") + + (root / "input.txt").write_text("two\n", encoding="utf-8") + third = run(zed, main_args(["run", "copy"]), cwd=root, env=env) + if "[done] copy" not in third.stderr: + raise CertificationFailure("source drift did not rerun task") + if (root / "output.txt").read_text(encoding="utf-8") != "two\n": + raise CertificationFailure("source drift output content is wrong") + + records = sorted((root / ".zed" / "task-cache" / "v1").glob("*.json")) + if not records: + raise CertificationFailure("task cache did not persist a content-identity record") + for record in records: + data = json.loads(record.read_text(encoding="utf-8")) + if set(data) != {"schema", "task", "input_sha256", "output_sha256", "outputs"}: + raise CertificationFailure(f"unexpected cache record fields: {sorted(data)}") + + +def case_fail_closed(zed: Path, staged: Path, root: Path, env: dict[str, str]) -> None: + for binary, prefix in [(zed, ["task"]), (staged, [])]: + zero = run( + binary, + [*prefix, "run", "build", "--jobs", "0"], + cwd=root, + env=env, + expect_success=False, + ) + if "at least one" not in zero.stderr: + raise CertificationFailure(f"zero-jobs diagnostic drift for {binary.name}: {zero.stderr}") + + live_json = run( + binary, + [*prefix, "--json", "run", "build"], + cwd=root, + env=env, + expect_success=False, + ) + if "requires `--dry-run`" not in live_json.stderr: + raise CertificationFailure( + f"live JSON boundary drift for {binary.name}: {live_json.stderr}" + ) + + confirmation = run( + binary, + [*prefix, "run", "release"], + cwd=root, + env=env, + expect_success=False, + ) + if "requires confirmation" not in confirmation.stderr: + raise CertificationFailure( + f"confirmation boundary drift for {binary.name}: {confirmation.stderr}" + ) + + if (root / "release.txt").exists(): + raise CertificationFailure("rejected confirmation mutated the project") + run(zed, main_args(["run", "release", "--yes"]), cwd=root, env=env) + if not (root / "release.txt").is_file(): + raise CertificationFailure("approved confirmation did not execute") + + +def main() -> int: + args = parse_args() + candidate = args.candidate.lower() + if len(candidate) != 40 or any(character not in "0123456789abcdef" for character in candidate): + raise SystemExit("--candidate must be a full 40-character hexadecimal commit") + + zed = args.zed.resolve() + staged = args.zed_task.resolve() + if not zed.is_file() or not staged.is_file(): + raise SystemExit(f"missing binaries: zed={zed}, zed-task={staged}") + + work = args.work.resolve() + if work.exists(): + shutil.rmtree(work) + project = work / "project" + write_plan(project) + env = runtime_env(work) + + versions = { + "zed": run(zed, ["--version"], cwd=project, env=env).stdout.strip(), + "zed_task": run(staged, ["--version"], cwd=project, env=env).stdout.strip(), + } + + cases: list[tuple[str, Callable[[Path, Path, Path, dict[str, str]], None]]] = [ + ("json-parity", case_json_parity), + ("dependency-execution", case_execution), + ("argument-isolation", case_arguments), + ("content-cache", case_content_cache), + ("fail-closed-boundaries", case_fail_closed), + ] + results: list[CaseResult] = [] + failed = False + for name, case in cases: + started = time.monotonic() + try: + case(zed, staged, project, env) + results.append( + CaseResult( + name=name, + status="passed", + duration_ms=round((time.monotonic() - started) * 1000), + ) + ) + print(f"PASS {name}") + except Exception as error: # produce complete evidence after a failure + failed = True + results.append( + CaseResult( + name=name, + status="failed", + duration_ms=round((time.monotonic() - started) * 1000), + detail=str(error), + ) + ) + print(f"FAIL {name}: {error}", file=sys.stderr) + + evidence = { + "schema": "zed-pkg-test/canonical-task-canary/v1", + "candidate": candidate, + "status": "failed" if failed else "passed", + "platform": { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + "versions": versions, + "claims": { + "manager_execution": False, + "network_required_at_runtime": False, + "canonical_and_staged_shared_semantics": True, + }, + "results": [asdict(result) for result in results], + } + evidence_path = args.evidence.resolve() + evidence_path.parent.mkdir(parents=True, exist_ok=True) + evidence_path.write_text(json.dumps(evidence, indent=2, sort_keys=True) + "\n") + print(f"wrote {evidence_path}") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_snapshot_endpoint_policy.py b/scripts/check_snapshot_endpoint_policy.py new file mode 100644 index 00000000..3fd5fa0c --- /dev/null +++ b/scripts/check_snapshot_endpoint_policy.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from urllib.parse import urlparse + + +class ContractError(ValueError): + pass + + +def literal_environment_value(source: str, name: str) -> str: + lines = source.splitlines() + name_pattern = re.compile(rf"\bname:\s*{re.escape(name)}\b") + value_pattern = re.compile(r"\bvalue:\s*[\"']?([^\"'\s,}}]+)") + for index, line in enumerate(lines): + if not name_pattern.search(line): + continue + for candidate in lines[index : index + 4]: + match = value_pattern.search(candidate) + if match: + return match.group(1) + raise ContractError(f"{name} is declared without a literal value") + raise ContractError(f"Deployment does not declare {name}") + + +def endpoint_port(source: str, name: str) -> int: + value = literal_environment_value(source, name) + parsed = urlparse(value) + if not parsed.scheme or not parsed.hostname or parsed.port is None: + raise ContractError(f"{name} must be an absolute URL with an explicit port: {value}") + return parsed.port + + +def network_policy_tcp_ports(source: str) -> set[int]: + ports = { + int(value) + for value in re.findall( + r"(?m)^\s*-?\s*\{?\s*protocol:\s*TCP,\s*port:\s*(\d+)", + source, + ) + } + ports.update( + int(value) + for value in re.findall(r"(?m)^\s*port:\s*(\d+)\s*$", source) + ) + if not ports: + raise ContractError("NetworkPolicy does not declare any TCP ports") + return ports + + +def validate(deployment: str, network_policy: str) -> dict[str, object]: + endpoints = { + "NATS_URL": endpoint_port(deployment, "NATS_URL"), + "OTEL_EXPORTER_OTLP_ENDPOINT": endpoint_port( + deployment, "OTEL_EXPORTER_OTLP_ENDPOINT" + ), + } + allowed = network_policy_tcp_ports(network_policy) + missing = {name: port for name, port in endpoints.items() if port not in allowed} + if missing: + rendered = ", ".join(f"{name}={port}" for name, port in sorted(missing.items())) + raise ContractError( + f"NetworkPolicy does not permit declared endpoint port(s): {rendered}; " + f"allowed TCP ports are {sorted(allowed)}" + ) + return {"endpointPorts": endpoints, "allowedTcpPorts": sorted(allowed)} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--deployment", type=Path, required=True) + parser.add_argument("--network-policy", type=Path, required=True) + parser.add_argument("--evidence", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + result = validate( + args.deployment.read_text(encoding="utf-8"), + args.network_policy.read_text(encoding="utf-8"), + ) + except (ContractError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + if args.evidence is not None: + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text( + json.dumps( + { + "$schema": "zed-pkg-test/snapshot-endpoint-policy/v1", + "result": "passed", + **result, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + print( + "snapshot endpoint/egress contract valid: " + f"endpoints={result['endpointPorts']}; allowed={result['allowedTcpPorts']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/complete-constraint-solver.sh b/scripts/complete-constraint-solver.sh new file mode 100644 index 00000000..4fca132c --- /dev/null +++ b/scripts/complete-constraint-solver.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: complete-constraint-solver.sh ZED_BIN WORK_ROOT +EOF + exit 64 +} + +[[ $# -eq 2 ]] || usage +zed="$1" +work_root="$2" + +[[ -x "$zed" ]] || { echo "zed binary is not executable: $zed" >&2; exit 66; } +[[ ! -e "$work_root" ]] || { echo "work root must be fresh: $work_root" >&2; exit 73; } + +zed="$(cd "$(dirname "$zed")" && pwd)/$(basename "$zed")" +mkdir -p "$work_root" +work_root="$(cd "$work_root" && pwd)" +registry="$work_root/registry" +publisher_home="$work_root/publisher-home" +mkdir -p "$registry" "$publisher_home" +file_registry="file://$registry" +org="solver-yank-e2e" + +unset ZED_PKG_TOKEN ZED_PKG_REGISTRY ZED_PKG_HOME ZED_PKG_TARGET \ + ZED_PKG_ADAPTER ZED_PKG_INSTALL_CONCURRENCY ZED_PKG_ALLOW_NO_MANIFEST \ + ZED_PKG_DO_NOT_WRITE_NEW_MANIFEST ZED_PKG_INTERACTIVE +export CI=true GIT_TERMINAL_PROMPT=0 RUST_BACKTRACE=1 + +publish_package() { + local name="$1" + local version="$2" + shift 2 + local source="$work_root/publish/${name}-${version//[^A-Za-z0-9._-]/-}" + mkdir -p "$source" + { + cat <"$source/.zpkg.toml" + printf '%s/%s@%s\n' "$org" "$name" "$version" >"$source/payload.txt" + ( + cd "$source" + "$zed" \ + --registry "$file_registry" \ + --home "$publisher_home" \ + publish --skip-vcs-checks + ) +} + +write_consumer() { + local root="$1" + mkdir -p "$root" + cat >"$root/.zpkg.toml" <"$work_root/fresh.log" 2>&1; then + echo "fresh resolution unexpectedly selected a yanked-only common version" >&2 + exit 1 +fi +grep -i 'yanked' "$work_root/fresh.log" +grep -F -- '--frozen' "$work_root/fresh.log" +test ! -e "$fresh/.zpkg.lock" +test ! -e "$fresh/zed_modules" +test ! -e "$fresh/.zed" +test ! -e "$fresh/.zpkg-staging" + +find "$fresh_home" -print >"$work_root/fresh-home-paths.txt" 2>/dev/null || true +if grep -Fq "$shared_sha" "$work_root/fresh-home-paths.txt"; then + echo "fresh resolution acquired yanked $org/shared@1.5.0 before rejecting it" >&2 + grep -F "$shared_sha" "$work_root/fresh-home-paths.txt" >&2 + exit 1 +fi + +printf 'DEN-1553 yanked-candidate hardening acceptance passed: %s\n' "$work_root" diff --git a/scripts/complete_constraint_solver.py b/scripts/complete_constraint_solver.py new file mode 100644 index 00000000..6012ef0d --- /dev/null +++ b/scripts/complete_constraint_solver.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python3 +"""Black-box certification for zed's complete one-version dependency solver.""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Sequence + +ORG = "zed-solver-e2e" +PREFETCH_RE = re.compile( + r"recursive install prefetch: (?P\d+) package\(s\), " + r"up to (?P\d+) concurrent, (?P\d+) downloaded" +) + + +@dataclass(frozen=True) +class Completed: + command: tuple[str, ...] + returncode: int + output: str + + +@dataclass(frozen=True) +class Package: + name: str + version: str + dependencies: Mapping[str, str] + + +def shell_quote(value: str) -> str: + if re.fullmatch(r"[A-Za-z0-9_./:@%+=,-]+", value): + return value + return "'" + value.replace("'", "'\"'\"'") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, + timeout: int = 300, +) -> Completed: + argv = tuple(str(value) for value in command) + shown = " ".join(shell_quote(value) for value in argv) + print(f"\n$ (cd {cwd or Path.cwd()} && {shown})", flush=True) + completed = subprocess.run( + argv, + cwd=cwd, + env=dict(env) if env is not None else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + timeout=timeout, + ) + if completed.stdout: + print(completed.stdout, end="", flush=True) + return Completed(argv, completed.returncode, completed.stdout) + + +def require_success(completed: Completed) -> None: + if completed.returncode != 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {shown}\n" + f"{completed.output}" + ) + + +def require_failure(completed: Completed) -> None: + if completed.returncode == 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise AssertionError(f"command unexpectedly succeeded: {shown}") + + +def zed_command(zed: Path, registry: Path, home: Path, *args: str) -> list[str | Path]: + return [ + zed, + "--registry", + f"file://{registry}", + "--home", + home, + *args, + ] + + +def package_key(name: str) -> str: + return f"{ORG}/{name}" + + +def write_package(root: Path, package: Package) -> None: + root.mkdir(parents=True, exist_ok=True) + lines = [ + "[package]", + f'org = "{ORG}"', + f'name = "{package.name}"', + f'version = "{package.version}"', + 'description = "complete solver black-box fixture"', + 'license = "MIT"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://github.com/zed-pkg-test/zed-pkg-e2e"', + ] + if package.dependencies: + lines.extend(("", "[dependencies]")) + for dependency, requirement in package.dependencies.items(): + lines.append(f'"{package_key(dependency)}" = "{requirement}"') + (root / ".zpkg.toml").write_text("\n".join(lines) + "\n", encoding="utf-8") + (root / "payload.txt").write_text( + f"{package_key(package.name)}@{package.version}\n", + encoding="utf-8", + ) + + +def write_consumer( + root: Path, + *, + identity: str, + dependencies: Sequence[tuple[str, str]], +) -> None: + root.mkdir(parents=True, exist_ok=True) + lines = [ + "[package]", + 'org = "solver-consumer"', + f'name = "{identity}"', + 'version = "0.1.0"', + 'description = "black-box solver consumer"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://example.invalid/solver-consumer"', + "", + "[dependencies]", + ] + for name, requirement in dependencies: + lines.append(f'"{package_key(name)}" = "{requirement}"') + lines.append("") + (root / ".zpkg.toml").write_text("\n".join(lines), encoding="utf-8") + + +def publish( + zed: Path, + registry: Path, + publish_home: Path, + sources: Path, + package: Package, + environment: Mapping[str, str], +) -> None: + safe_version = package.version.replace(".", "-").replace("+", "-") + source = sources / f"{package.name}-{safe_version}" + write_package(source, package) + completed = run( + zed_command( + zed, + registry, + publish_home, + "publish", + "--skip-vcs-checks", + ), + cwd=source, + env=environment, + ) + require_success(completed) + + +def parse_prefetch(output: str) -> tuple[int, int, int]: + matches = list(PREFETCH_RE.finditer(output)) + if len(matches) != 1: + raise AssertionError( + f"expected exactly one prefetch summary, got {len(matches)}:\n{output}" + ) + match = matches[0] + return tuple( + int(match.group(field)) + for field in ("resolved", "concurrency", "downloaded") + ) + + +def lock_versions(project: Path) -> dict[str, str]: + with (project / ".zpkg.lock").open("rb") as handle: + document = tomllib.load(handle) + packages = document.get("package") or [] + return { + f"{package['org']}/{package['name']}": str(package["version"]) + for package in packages + } + + +def assert_lock(project: Path, expected: Mapping[str, str]) -> None: + actual = lock_versions(project) + wanted = {package_key(name): version for name, version in expected.items()} + if actual != wanted: + raise AssertionError( + f"lock mismatch for {project}: expected {wanted}, got {actual}" + ) + + +def assert_symlink_graph( + project: Path, + home: Path, + expected: Mapping[str, str], +) -> dict[str, Path]: + store = (home / "store").resolve() + targets: dict[str, Path] = {} + for name, version in expected.items(): + installed = project / "zed_modules" / ORG / name + if not installed.is_symlink(): + raise AssertionError(f"expected symlink materialization: {installed}") + target = installed.resolve(strict=True) + try: + target.relative_to(store) + except ValueError as error: + raise AssertionError(f"{installed} resolves outside {store}: {target}") from error + payload = (target / "payload.txt").read_text(encoding="utf-8") + wanted = f"{package_key(name)}@{version}\n" + if payload != wanted: + raise AssertionError( + f"unexpected payload through {installed}: {payload!r} != {wanted!r}" + ) + targets[name] = target + return targets + + +def assert_absent(project: Path, names: Sequence[str]) -> None: + for name in names: + path = project / "zed_modules" / ORG / name + if path.exists() or path.is_symlink(): + raise AssertionError(f"rejected candidate leaked into project: {path}") + + +def assert_no_project_mutation(project: Path) -> None: + if (project / ".zpkg.lock").exists(): + raise AssertionError(f"failed solve wrote a lockfile: {project}") + modules = project / "zed_modules" + if modules.exists(): + raise AssertionError(f"failed solve materialized modules: {modules}") + staging = project / ".zpkg-staging" + if staging.exists(): + raise AssertionError(f"failed solve leaked transaction staging: {staging}") + + +def normalized_error(output: str, root: Path) -> str: + normalized = output.replace(str(root), "") + return "\n".join(line.rstrip() for line in normalized.splitlines() if line.strip()) + + +def install( + zed: Path, + registry: Path, + home: Path, + project: Path, + environment: Mapping[str, str], + *extra: str, +) -> Completed: + return run( + zed_command( + zed, + registry, + home, + "install", + "--adapter", + "none", + *extra, + ), + cwd=project, + env=environment, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + + zed = args.zed.resolve() + root = args.work_root.resolve() + if not zed.is_file(): + raise RuntimeError(f"zed binary does not exist: {zed}") + if root.exists(): + raise RuntimeError(f"work root must be fresh: {root}") + + registry = root / "registry" + publish_home = root / "publish-home" + sources = root / "sources" + registry.mkdir(parents=True) + + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "RUST_BACKTRACE": "1", + "ZED_PKG_INSTALL_CONCURRENCY": "5", + "ZED_PKG_TOKEN": "", + } + ) + + packages = [ + # Overlapping compatible ranges: 1.5.0 is the only common solution. + Package("overlap-shared", "1.5.0", {}), + Package("overlap-shared", "1.9.0", {}), + Package("overlap-left", "1.0.0", {"overlap-shared": "^1"}), + Package("overlap-right", "1.0.0", {"overlap-shared": "<=1.5.0"}), + # A two-coordinate backtracking graph. Latest a + latest b conflict. + Package("backtrack-x", "1.0.0", {}), + Package("backtrack-x", "2.0.0", {}), + Package("backtrack-a", "1.0.0", {"backtrack-x": "=1.0.0"}), + Package("backtrack-a", "1.1.0", {"backtrack-x": "=2.0.0"}), + Package("backtrack-b", "1.0.0", {"backtrack-x": "=2.0.0"}), + Package("backtrack-b", "1.1.0", {"backtrack-x": "=1.0.0"}), + # Rejected chooser@1.1 contributes obsolete + shared@1. Those + # constraints must disappear when chooser@1.0 is selected. + Package("stale-shared", "1.0.0", {}), + Package("stale-shared", "2.0.0", {}), + Package("stale-obsolete", "1.0.0", {}), + Package("stale-replacement", "1.0.0", {}), + Package( + "stale-chooser", + "1.0.0", + { + "stale-replacement": "=1.0.0", + "stale-shared": "=2.0.0", + }, + ), + Package( + "stale-chooser", + "1.1.0", + { + "stale-obsolete": "=1.0.0", + "stale-shared": "=1.0.0", + }, + ), + Package("stale-force", "1.0.0", {"stale-shared": "=2.0.0"}), + # Deterministic unsatisfiable provenance. + Package("unsat-shared", "1.0.0", {}), + Package("unsat-shared", "2.0.0", {}), + Package("unsat-left", "1.0.0", {"unsat-shared": "=1.0.0"}), + Package("unsat-right", "1.0.0", {"unsat-shared": "=2.0.0"}), + # Cycle termination. + Package("cycle-a", "1.0.0", {"cycle-b": "=1.0.0"}), + Package("cycle-b", "1.0.0", {"cycle-a": "=1.0.0"}), + ] + for package in packages: + publish(zed, registry, publish_home, sources, package, environment) + + # 1. Overlapping ranges resolve to the common older version, regardless of + # root declaration order, and emit byte-identical lockfiles. Unresolved + # root packages must contribute their constraints before the deeper shared + # coordinate is acquired, so each cold home downloads only the three + # selected artifacts and never speculates into rejected shared@1.9.0. + overlap_expected = { + "overlap-left": "1.0.0", + "overlap-right": "1.0.0", + "overlap-shared": "1.5.0", + } + overlap_a = root / "overlap-a" + overlap_b = root / "overlap-b" + write_consumer( + overlap_a, + identity="overlap", + dependencies=( + ("overlap-left", "=1.0.0"), + ("overlap-right", "=1.0.0"), + ), + ) + write_consumer( + overlap_b, + identity="overlap", + dependencies=( + ("overlap-right", "=1.0.0"), + ("overlap-left", "=1.0.0"), + ), + ) + overlap_home_a = root / "overlap-home-a" + overlap_home_b = root / "overlap-home-b" + overlap_first = install( + zed, registry, overlap_home_a, overlap_a, environment + ) + overlap_second = install( + zed, registry, overlap_home_b, overlap_b, environment + ) + require_success(overlap_first) + require_success(overlap_second) + if parse_prefetch(overlap_first.output) != (3, 5, 3): + raise AssertionError(f"unexpected first overlap summary: {overlap_first.output}") + if parse_prefetch(overlap_second.output) != (3, 5, 3): + raise AssertionError(f"unexpected second overlap summary: {overlap_second.output}") + assert_lock(overlap_a, overlap_expected) + assert_lock(overlap_b, overlap_expected) + targets = assert_symlink_graph(overlap_a, overlap_home_a, overlap_expected) + assert_symlink_graph(overlap_b, overlap_home_b, overlap_expected) + lock_bytes = (overlap_a / ".zpkg.lock").read_bytes() + if (overlap_b / ".zpkg.lock").read_bytes() != lock_bytes: + raise AssertionError("root declaration order changed lockfile bytes") + + warm = install( + zed, + registry, + overlap_home_a, + overlap_a, + environment, + "--frozen", + ) + require_success(warm) + if parse_prefetch(warm.output) != (3, 5, 0): + raise AssertionError(f"warm frozen overlap replay downloaded bytes: {warm.output}") + if (overlap_a / ".zpkg.lock").read_bytes() != lock_bytes: + raise AssertionError("warm frozen replay changed overlap lock bytes") + if assert_symlink_graph(overlap_a, overlap_home_a, overlap_expected) != targets: + raise AssertionError("warm frozen replay changed immutable store targets") + + # Add a newer compatible version after the lock exists. Frozen replay in a + # fresh home must consume the exact solved graph without re-solving. + publish( + zed, + registry, + publish_home, + sources, + Package("overlap-shared", "1.10.0", {}), + environment, + ) + frozen_overlap = root / "overlap-frozen" + frozen_overlap.mkdir() + shutil.copy2(overlap_a / ".zpkg.toml", frozen_overlap / ".zpkg.toml") + shutil.copy2(overlap_a / ".zpkg.lock", frozen_overlap / ".zpkg.lock") + frozen_home = root / "overlap-frozen-home" + frozen = install( + zed, + registry, + frozen_home, + frozen_overlap, + environment, + "--frozen", + ) + require_success(frozen) + if parse_prefetch(frozen.output) != (3, 5, 3): + raise AssertionError(f"fresh frozen replay summary mismatch: {frozen.output}") + assert_lock(frozen_overlap, overlap_expected) + assert_symlink_graph(frozen_overlap, frozen_home, overlap_expected) + if (frozen_overlap / ".zpkg.lock").read_bytes() != lock_bytes: + raise AssertionError("fresh frozen replay changed solved lock bytes") + + # 2. Latest-first search must backtrack across package b after selecting + # package a, producing the stable a@1.1 + b@1.0 + x@2 solution. + backtrack_project = root / "backtrack" + write_consumer( + backtrack_project, + identity="backtrack", + dependencies=(("backtrack-a", "^1"), ("backtrack-b", "^1")), + ) + backtrack_home = root / "backtrack-home" + backtrack = install( + zed, registry, backtrack_home, backtrack_project, environment + ) + require_success(backtrack) + backtrack_summary = parse_prefetch(backtrack.output) + if backtrack_summary[:2] != (3, 5) or backtrack_summary[2] < 3: + raise AssertionError(f"backtracking did not produce a valid summary: {backtrack_summary}") + backtrack_expected = { + "backtrack-a": "1.1.0", + "backtrack-b": "1.0.0", + "backtrack-x": "2.0.0", + } + assert_lock(backtrack_project, backtrack_expected) + assert_symlink_graph(backtrack_project, backtrack_home, backtrack_expected) + + # 3. Constraints and dependencies contributed by a rejected candidate must + # not leak into the final graph. + stale_project = root / "stale" + write_consumer( + stale_project, + identity="stale", + dependencies=(("stale-chooser", "^1"), ("stale-force", "=1.0.0")), + ) + stale_home = root / "stale-home" + stale = install(zed, registry, stale_home, stale_project, environment) + require_success(stale) + stale_expected = { + "stale-chooser": "1.0.0", + "stale-force": "1.0.0", + "stale-replacement": "1.0.0", + "stale-shared": "2.0.0", + } + assert_lock(stale_project, stale_expected) + assert_symlink_graph(stale_project, stale_home, stale_expected) + assert_absent(stale_project, ("stale-obsolete",)) + + # 4. An actually unsatisfiable graph reports both provenance paths in a + # byte-stable order independent of declaration order. + unsat_outputs = [] + for suffix, dependencies in ( + ( + "a", + (("unsat-left", "=1.0.0"), ("unsat-right", "=1.0.0")), + ), + ( + "b", + (("unsat-right", "=1.0.0"), ("unsat-left", "=1.0.0")), + ), + ): + project = root / f"unsat-{suffix}" + write_consumer( + project, + identity="unsatisfiable", + dependencies=dependencies, + ) + completed = install( + zed, + registry, + root / f"unsat-home-{suffix}", + project, + environment, + ) + require_failure(completed) + assert_no_project_mutation(project) + normalized = normalized_error(completed.output, root) + for fragment in ( + f"version conflict for {package_key('unsat-shared')}", + f"`=1.0.0` via solver-consumer/unsatisfiable@0.1.0 -> " + f"{package_key('unsat-left')}@1.0.0 -> {package_key('unsat-shared')}", + f"`=2.0.0` via solver-consumer/unsatisfiable@0.1.0 -> " + f"{package_key('unsat-right')}@1.0.0 -> {package_key('unsat-shared')}", + ): + if fragment not in normalized: + raise AssertionError( + f"unsatisfiable diagnostic omitted {fragment!r}:\n{normalized}" + ) + unsat_outputs.append(normalized) + if unsat_outputs[0] != unsat_outputs[1]: + raise AssertionError( + "unsatisfiable diagnostic changed with root declaration order:\n" + f"--- first ---\n{unsat_outputs[0]}\n" + f"--- second ---\n{unsat_outputs[1]}" + ) + + # 5. A dependency cycle terminates with one selected version per package. + cycle_project = root / "cycle" + write_consumer( + cycle_project, + identity="cycle", + dependencies=(("cycle-a", "=1.0.0"),), + ) + cycle_home = root / "cycle-home" + cycle = install(zed, registry, cycle_home, cycle_project, environment) + require_success(cycle) + cycle_expected = {"cycle-a": "1.0.0", "cycle-b": "1.0.0"} + assert_lock(cycle_project, cycle_expected) + assert_symlink_graph(cycle_project, cycle_home, cycle_expected) + + print("\ncomplete constraint solver black-box canary passed", flush=True) + print(f"overlap summary: {parse_prefetch(overlap_first.output)}", flush=True) + print(f"backtrack summary: {backtrack_summary}", flush=True) + print(f"stale-candidate summary: {parse_prefetch(stale.output)}", flush=True) + print(f"cycle summary: {parse_prefetch(cycle.output)}", flush=True) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 - CI boundary must print context. + print(f"error: {error}", file=sys.stderr, flush=True) + raise diff --git a/scripts/den2725_root_dispatch_hardening.py b/scripts/den2725_root_dispatch_hardening.py new file mode 100644 index 00000000..2ac9d7b5 --- /dev/null +++ b/scripts/den2725_root_dispatch_hardening.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +from pathlib import Path + + +def replace_once(path: str, old: str, new: str) -> None: + target = Path(path) + text = target.read_text(encoding="utf-8") + count = text.count(old) + if count != 1: + raise SystemExit(f"{path}: expected one replacement target, found {count}") + target.write_text(text.replace(old, new, 1), encoding="utf-8") + + +replace_once( + "src/external_subcommands.rs", + ''' let mut arguments = args[index + 2..].to_vec(); + if !arguments +''', + ''' let (mut arguments, trailing_environment) = + extract_root_options(&args[index + 2..])?; + environment.extend(trailing_environment); + if !arguments +''', +) + +replace_once( + "src/external_subcommands.rs", + ''' return Some(ExternalRoute { + name: token.to_owned(), + arguments: args[index + 1..].to_vec(), + environment, + }); +''', + ''' let (arguments, trailing_environment) = extract_root_options(&args[index + 1..])?; + environment.extend(trailing_environment); + return Some(ExternalRoute { + name: token.to_owned(), + arguments, + environment, + }); +''', +) + +replace_once( + "src/external_subcommands.rs", + '''fn root_value_option(token: &str) -> Option<(&'static str, Option<&str>)> { +''', + '''fn extract_root_options( + args: &[OsString], +) -> Option<(Vec, Vec<(OsString, OsString)>)> { + let mut arguments = Vec::new(); + let mut environment = Vec::new(); + let mut index = 0; + + while index < args.len() { + let Some(token) = args[index].to_str() else { + arguments.push(args[index].clone()); + index += 1; + continue; + }; + if token == "--" { + arguments.extend_from_slice(&args[index..]); + break; + } + + if let Some((key, inline)) = root_value_option(token) { + let (value, consumed) = match inline { + Some(value) if !value.is_empty() => (OsString::from(value), 1), + Some(_) => return None, + None => { + let value = args.get(index + 1)?.clone(); + if value.is_empty() { + return None; + } + (value, 2) + } + }; + environment.push((OsString::from(key), value)); + index += consumed; + continue; + } + + if let Some((key, value)) = root_boolean_option(token) { + environment.push((OsString::from(key), OsString::from(value))); + index += 1; + continue; + } + if is_root_boolean_spelling(token) { + return None; + } + + arguments.push(args[index].clone()); + index += 1; + } + + Some((arguments, environment)) +} + +fn is_root_boolean_spelling(token: &str) -> bool { + ROOT_BOOLEAN_OPTIONS.iter().any(|(option, _)| { + token == *option + || token == format!("--no-{}", option.trim_start_matches('-')) + || token + .strip_prefix(option) + .is_some_and(|tail| tail.starts_with('=')) + }) +} + +fn root_value_option(token: &str) -> Option<(&'static str, Option<&str>)> { +''', +) + +replace_once( + "src/external_subcommands.rs", + ''' #[test] + fn help_spelling_routes_to_external_help() { +''', + ''' #[test] + fn root_options_after_plugin_are_lifted_until_double_dash() { + let route = external_route(&os_args(&[ + "zed", + "gitops", + "validate", + "--token", + "fixture-value", + "--offline", + "--", + "--home", + "child-owned-value", + ])) + .expect("external route"); + assert_eq!( + route.arguments, + os_args(&[ + "validate", + "--offline", + "--", + "--home", + "child-owned-value" + ]) + ); + assert_eq!( + route.environment, + vec![( + OsString::from("ZED_PKG_TOKEN"), + OsString::from("fixture-value") + )] + ); + } + + #[test] + fn malformed_trailing_root_boolean_fails_closed() { + assert!(external_route(&os_args(&[ + "zed", + "gitops", + "validate", + "--git-submodules=maybe" + ])) + .is_none()); + } + + #[test] + fn help_spelling_routes_to_external_help() { +''', +) + +replace_once( + "docs/gitops-validator.md", + '''Root options placed before `gitops` are passed as their canonical +`ZED_PKG_*` environment variables rather than being exposed on the child +command line. +''', + '''Root options placed before or after `gitops` are passed as their canonical +`ZED_PKG_*` environment variables rather than being exposed on the child +command line. A literal `--` ends global-option extraction and passes every +remaining argument to the external command unchanged. +''', +) + +replace_once( + "tests/external_gitops_dispatch.rs", + ''' assert!(text.contains("Usage: zed-gitops validate"), "{text}"); + assert!(text.contains("--offline"), "{text}"); +''', + ''' assert!(text.contains("Usage: zed-gitops"), "{text}"); + assert!(text.contains("validate [OPTIONS]"), "{text}"); + assert!(text.contains("--offline"), "{text}"); +''', +) diff --git a/scripts/den2725_root_dispatch_lint_fix.py b/scripts/den2725_root_dispatch_lint_fix.py new file mode 100644 index 00000000..bbe5178b --- /dev/null +++ b/scripts/den2725_root_dispatch_lint_fix.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +from pathlib import Path + + +def replace_once(path: str, old: str, new: str) -> None: + target = Path(path) + text = target.read_text(encoding="utf-8") + count = text.count(old) + if count != 1: + raise SystemExit(f"{path}: expected one replacement target, found {count}") + target.write_text(text.replace(old, new, 1), encoding="utf-8") + + +replace_once( + "src/external_subcommands.rs", + '''const ROOT_BOOLEAN_OPTIONS: &[(&str, &str)] = &[ + ("--interactive", "ZED_PKG_INTERACTIVE"), + ("--git-submodules", "ZED_PKG_GIT_SUBMODULES"), +]; + +#[derive(Debug, Clone, PartialEq, Eq)] +''', + '''const ROOT_BOOLEAN_OPTIONS: &[(&str, &str)] = &[ + ("--interactive", "ZED_PKG_INTERACTIVE"), + ("--git-submodules", "ZED_PKG_GIT_SUBMODULES"), +]; + +type ExternalEnvironment = Vec<(OsString, OsString)>; +type ParsedExternalArguments = (Vec, ExternalEnvironment); + +#[derive(Debug, Clone, PartialEq, Eq)] +''', +) + +replace_once( + "src/external_subcommands.rs", + " environment: Vec<(OsString, OsString)>,\n", + " environment: ExternalEnvironment,\n", +) + +replace_once( + "src/external_subcommands.rs", + '''fn extract_root_options( + args: &[OsString], +) -> Option<(Vec, Vec<(OsString, OsString)>)> { +''', + '''fn extract_root_options(args: &[OsString]) -> Option { +''', +) diff --git a/scripts/direct_source_render_canary.py b/scripts/direct_source_render_canary.py new file mode 100644 index 00000000..f730ef41 --- /dev/null +++ b/scripts/direct_source_render_canary.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Iterable, Sequence + + +SENSITIVE_ENV_FRAGMENTS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PRIVATE_KEY", + "ACCESS_KEY", + "API_KEY", + "AUTHORIZATION", + "COOKIE", +) +EXPECTED_REPOSITORY = "github.com/daedalus-fab/fabrication-server.rs" +EXPECTED_KINDS = {"Deployment", "ExternalSecret", "NetworkPolicy", "Service"} +CLUSTER_SCOPED_KINDS = { + "ClusterRole", + "ClusterRoleBinding", + "CustomResourceDefinition", + "MutatingWebhookConfiguration", + "Namespace", + "Node", + "PersistentVolume", + "PriorityClass", + "StorageClass", + "ValidatingWebhookConfiguration", +} + + +class CertificationError(RuntimeError): + pass + + +def require(condition: bool, message: str) -> None: + if not condition: + raise CertificationError(message) + + +def sanitized_environment() -> dict[str, str]: + environment = dict(os.environ) + for key in list(environment): + upper = key.upper() + if any(fragment in upper for fragment in SENSITIVE_ENV_FRAGMENTS): + environment.pop(key, None) + elif upper.startswith(("ZED_PKG_", "ZED_EXTERNAL_", "ZED_PROBE_")): + environment.pop(key, None) + environment["CI"] = "true" + environment["PYTHONDONTWRITEBYTECODE"] = "1" + return environment + + +def run( + command: Sequence[object], + *, + environment: dict[str, str], + cwd: Path | None = None, + expected: int | Iterable[int] = 0, + timeout: int = 120, +) -> subprocess.CompletedProcess[bytes]: + expected_codes = {expected} if isinstance(expected, int) else set(expected) + result = subprocess.run( + [str(item) for item in command], + cwd=cwd, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + if result.returncode not in expected_codes: + rendered = " ".join(repr(str(item)) for item in command) + raise CertificationError( + f"command returned {result.returncode}, expected {sorted(expected_codes)}: {rendered}\n" + f"stdout:\n{result.stdout.decode(errors='replace')}\n" + f"stderr:\n{result.stderr.decode(errors='replace')}" + ) + return result + + +def text(result: subprocess.CompletedProcess[bytes]) -> str: + return result.stdout.decode("utf-8", errors="strict").strip() + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def canonical_repository(value: str) -> str: + normalized = value.strip().replace("\\", "/") + if normalized.startswith("git@github.com:"): + normalized = "github.com/" + normalized.removeprefix("git@github.com:") + elif normalized.startswith("ssh://git@github.com/"): + normalized = "github.com/" + normalized.removeprefix("ssh://git@github.com/") + elif normalized.startswith("https://github.com/"): + normalized = "github.com/" + normalized.removeprefix("https://github.com/") + elif normalized.startswith("http://github.com/"): + normalized = "github.com/" + normalized.removeprefix("http://github.com/") + return normalized.removesuffix("/").removesuffix(".git").lower() + + +def git_head(root: Path, environment: dict[str, str]) -> str: + return text(run(["git", "-C", root, "rev-parse", "HEAD"], environment=environment)) + + +def gitmodule_for_path( + cluster: Path, + path: str, + environment: dict[str, str], +) -> tuple[str, str]: + result = run( + [ + "git", + "-C", + cluster, + "config", + "-f", + ".gitmodules", + "--get-regexp", + r"^submodule\..*\.path$", + ], + environment=environment, + ) + selected_key: str | None = None + for raw_line in text(result).splitlines(): + key, _, value = raw_line.partition(" ") + if value.strip() == path: + selected_key = key.removesuffix(".path") + break + require(selected_key is not None, f".gitmodules has no entry for {path}") + url = text( + run( + [ + "git", + "-C", + cluster, + "config", + "-f", + ".gitmodules", + "--get", + f"{selected_key}.url", + ], + environment=environment, + ) + ) + return selected_key, url + + +def indexed_gitlink( + cluster: Path, + path: str, + environment: dict[str, str], +) -> tuple[str, str, str, str]: + output = text( + run( + ["git", "-C", cluster, "ls-tree", "HEAD", "--", path], + environment=environment, + ) + ) + require(output, f"Git index has no entry for {path}") + left, separator, indexed_path = output.partition("\t") + require(separator == "\t", f"unexpected ls-tree output: {output}") + mode, object_type, revision = left.split() + return mode, object_type, revision, indexed_path + + +def verify_snapshot( + snapshot: Path, + metadata_path: Path, + child_commit: str, + environment: dict[str, str], +) -> tuple[dict[str, object], list[dict[str, str]]]: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + require(isinstance(metadata, dict), "source snapshot metadata must be an object") + require( + metadata.get("$schema") == "zed-pkg-test/direct-source-snapshot/v1", + "source snapshot schema drift", + ) + require( + canonical_repository(str(metadata.get("repository", ""))) == EXPECTED_REPOSITORY, + "source snapshot repository identity drift", + ) + require(metadata.get("revision") == child_commit, "source snapshot revision drift") + require(metadata.get("sourcePath") == "k8s", "source snapshot path drift") + files = metadata.get("files") + require(isinstance(files, dict) and files, "source snapshot file map is empty") + + verified: list[dict[str, str]] = [] + for relative, record in sorted(files.items()): + require(isinstance(relative, str), "snapshot file key must be text") + require(isinstance(record, dict), f"snapshot file metadata must be an object: {relative}") + expected_blob = record.get("gitBlobSha1") + require( + isinstance(expected_blob, str) and re.fullmatch(r"[0-9a-f]{40}", expected_blob), + f"snapshot file lacks an exact Git blob SHA: {relative}", + ) + candidate = snapshot / relative + require(candidate.is_file(), f"snapshot file is missing: {relative}") + actual_blob = text(run(["git", "hash-object", candidate], environment=environment)) + require( + actual_blob == expected_blob, + f"snapshot bytes differ from source Git blob for {relative}: {actual_blob} != {expected_blob}", + ) + verified.append( + { + "path": relative, + "gitBlobSha1": actual_blob, + "sha256": sha256_bytes(candidate.read_bytes()), + } + ) + return metadata, verified + + +def parse_rendered_documents(rendered: bytes) -> list[dict[str, str]]: + source = rendered.decode("utf-8", errors="strict") + require(source.strip(), "Kustomize emitted empty output") + require("\r\n" not in source, "Kustomize output contains platform-specific CRLF") + documents: list[dict[str, str]] = [] + for index, document in enumerate(re.split(r"(?m)^---\s*$", source), start=1): + if not document.strip(): + continue + api_match = re.search(r"(?m)^apiVersion:\s*([^\s#]+)\s*$", document) + kind_match = re.search(r"(?m)^kind:\s*([^\s#]+)\s*$", document) + metadata_match = re.search(r"(?m)^metadata:\s*$", document) + name_match = re.search(r"(?m)^ name:\s*([^\s#]+)\s*$", document) + namespace_match = re.search(r"(?m)^ namespace:\s*([^\s#]+)\s*$", document) + require(api_match is not None, f"rendered document {index} lacks apiVersion") + require(kind_match is not None, f"rendered document {index} lacks kind") + require(metadata_match is not None, f"rendered document {index} lacks metadata") + require(name_match is not None, f"rendered document {index} lacks metadata.name") + require( + namespace_match is not None and namespace_match.group(1) == "daedalus", + f"rendered document {index} is not explicitly namespaced to daedalus", + ) + kind = kind_match.group(1) + require(kind not in CLUSTER_SCOPED_KINDS, f"cluster-scoped resource rendered: {kind}") + documents.append( + { + "apiVersion": api_match.group(1), + "kind": kind, + "name": name_match.group(1), + "namespace": namespace_match.group(1), + } + ) + require(documents, "Kustomize emitted no YAML documents") + return documents + + +def certify(args: argparse.Namespace) -> dict[str, object]: + environment = sanitized_environment() + cluster = args.cluster.resolve(strict=True) + snapshot = args.snapshot.resolve(strict=True) + kustomize = args.kustomize.resolve(strict=True) + install_metadata = json.loads(args.kustomize_metadata.read_text(encoding="utf-8")) + + require(git_head(cluster, environment) == args.cluster_commit, "cluster checkout SHA drift") + snapshot_metadata, snapshot_files = verify_snapshot( + snapshot, + args.source_metadata.resolve(strict=True), + args.child_commit, + environment, + ) + + record_path = cluster / "catalog/gitops/apps/dd-fabrication-server.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + require(isinstance(record, dict), "catalog record must be an object") + require(record.get("metadata", {}).get("name") == "dd-fabrication-server", "catalog app name drift") + spec = record.get("spec") + require(isinstance(spec, dict), "catalog spec must be an object") + inventory = spec.get("inventory") + source = spec.get("source") + migration = spec.get("migration") + require(isinstance(inventory, dict), "catalog inventory must be an object") + require(isinstance(source, dict), "catalog source must be an object") + require(isinstance(migration, dict), "catalog migration must be an object") + + inventory_path = str(inventory.get("path", "")) + source_path = str(source.get("path", "")) + require(inventory.get("mode") == "git-submodule", "inventory mode drift") + require(inventory_path == "remote/deployments/fabrication-server-rs", "inventory path drift") + require(inventory.get("revision") == args.child_commit, "inventory revision differs from snapshot") + require(source.get("mode") == "direct-repository", "source is not direct-repository") + require(source.get("renderer") == "kustomize", "source renderer is not kustomize") + require(source_path == "k8s", "source path drift") + require(source.get("targetRevision") == args.child_commit, "source targetRevision differs from snapshot") + require(migration.get("phase") == "pilot-inert", "migration phase is no longer pilot-inert") + + inventory_repository = canonical_repository(str(inventory.get("repository", ""))) + source_repository = canonical_repository(str(source.get("repository", ""))) + require(inventory_repository == EXPECTED_REPOSITORY, "inventory repository identity drift") + require(source_repository == EXPECTED_REPOSITORY, "source repository identity drift") + require(inventory_repository == source_repository, "inventory/source repository mismatch") + require( + source_repository == canonical_repository(str(snapshot_metadata.get("repository", ""))), + "catalog source differs from snapshot source identity", + ) + + module_key, module_url = gitmodule_for_path(cluster, inventory_path, environment) + require(canonical_repository(module_url) == EXPECTED_REPOSITORY, ".gitmodules repository identity drift") + mode, object_type, indexed_revision, indexed_path = indexed_gitlink( + cluster, inventory_path, environment + ) + require(mode == "160000", f"inventory entry mode is {mode}, not 160000") + require(object_type == "commit", f"inventory entry type is {object_type}, not commit") + require(indexed_path == inventory_path, "indexed gitlink path drift") + require(indexed_revision == args.child_commit, "indexed gitlink differs from snapshot revision") + require(indexed_revision == inventory.get("revision"), "indexed gitlink differs from catalog inventory") + require(indexed_revision == source.get("targetRevision"), "indexed gitlink differs from Argo source") + + require( + not (cluster / inventory_path / source_path / "kustomization.yaml").exists(), + "cluster checkout unexpectedly materialized the child source; snapshot proof is ambiguous", + ) + snapshot_source = snapshot / source_path + require((snapshot_source / "kustomization.yaml").is_file(), "snapshot lacks kustomization.yaml") + + version_output = text(run([kustomize, "version"], environment=environment)) + require("v5.8.1" in version_output, f"unexpected Kustomize version: {version_output}") + render_one = run([kustomize, "build", snapshot_source], environment=environment, timeout=180).stdout + render_two = run([kustomize, "build", snapshot_source], environment=environment, timeout=180).stdout + require(render_one == render_two, "repeated Kustomize renders differ") + + documents = parse_rendered_documents(render_one) + kinds = {document["kind"] for document in documents} + require(kinds == EXPECTED_KINDS, f"unexpected rendered resource kinds: {sorted(kinds)}") + require(len(documents) == len(EXPECTED_KINDS), "rendered duplicate or missing resource kinds") + rendered_text = render_one.decode("utf-8") + require(not re.search(r"(?m)^kind:\s*Secret\s*$", rendered_text), "plaintext Secret resource rendered") + require(not re.search(r"(?m)^stringData:\s*$", rendered_text), "plaintext stringData rendered") + + args.rendered.parent.mkdir(parents=True, exist_ok=True) + args.rendered.write_bytes(render_one) + + return { + "$schema": "zed-pkg-test/direct-source-snapshot-render/v1", + "clusterCommit": args.cluster_commit, + "sourceCommit": args.child_commit, + "catalog": { + "name": record["metadata"]["name"], + "inventoryPath": inventory_path, + "sourcePath": source_path, + "repository": source_repository, + "moduleKey": module_key, + "gitlinkRevision": indexed_revision, + "migrationPhase": migration["phase"], + }, + "snapshot": { + "exportMode": snapshot_metadata.get("exportMode"), + "fileCount": len(snapshot_files), + "files": snapshot_files, + "limitation": "Private repository reachability is not claimed by this snapshot render.", + }, + "kustomize": { + "versionOutput": version_output, + "asset": install_metadata.get("asset"), + "assetSha256": install_metadata.get("assetSha256"), + "executableSha256": install_metadata.get("executableSha256"), + }, + "render": { + "sha256": sha256_bytes(render_one), + "byteCount": len(render_one), + "documentCount": len(documents), + "resources": documents, + }, + "result": "passed", + "checks": [ + "exact-cluster-checkout-identity", + "snapshot-git-blob-provenance", + "catalog-inventory-source-snapshot-parity", + "gitmodules-and-indexed-gitlink-parity", + "source-not-materialized-in-superproject", + "checksum-pinned-kustomize-version", + "deterministic-repeat-render", + "namespace-scoped-resource-set", + "no-plaintext-secret-resource", + ], + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--cluster", type=Path, required=True) + parser.add_argument("--snapshot", type=Path, required=True) + parser.add_argument("--source-metadata", type=Path, required=True) + parser.add_argument("--cluster-commit", required=True) + parser.add_argument("--child-commit", required=True) + parser.add_argument("--kustomize", type=Path, required=True) + parser.add_argument("--kustomize-metadata", type=Path, required=True) + parser.add_argument("--rendered", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + evidence = certify(args) + except (CertificationError, OSError, subprocess.SubprocessError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + f"rendered {evidence['render']['documentCount']} checksum-pinned source resources " + f"at {evidence['render']['sha256']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/durable-first-install-dart.sh b/scripts/durable-first-install-dart.sh new file mode 100755 index 00000000..c89fdf52 --- /dev/null +++ b/scripts/durable-first-install-dart.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: durable-first-install-dart.sh ZED_BIN DART_LIB_SOURCE DART_APP_SOURCE WORK_ROOT +EOF + exit 64 +} + +[[ $# -eq 4 ]] || usage +zed="$1" +lib_source="$2" +app_source="$3" +work_root="$4" + +[[ -x "$zed" ]] || { echo "zed binary is not executable: $zed" >&2; exit 66; } +[[ -f "$lib_source/.zpkg.toml" ]] || { echo "missing Dart library fixture" >&2; exit 66; } +[[ -f "$lib_source/pubspec.yaml" ]] || { echo "missing Dart library pubspec" >&2; exit 66; } +[[ -f "$app_source/.zpkg.toml" ]] || { echo "missing Dart application fixture" >&2; exit 66; } +[[ -f "$app_source/pubspec.yaml" ]] || { echo "missing Dart application pubspec" >&2; exit 66; } +[[ ! -e "$work_root" ]] || { echo "work root must be fresh: $work_root" >&2; exit 73; } + +zed="$(cd "$(dirname "$zed")" && pwd)/$(basename "$zed")" +lib_source="$(cd "$lib_source" && pwd)" +app_source="$(cd "$app_source" && pwd)" +mkdir -p "$work_root" +work_root="$(cd "$work_root" && pwd)" + +registry="$work_root/registry" +home="$work_root/zed-home" +file_registry="file://$registry" +package="zed-pkg-test/dart-lib" +native_name="zed_pkg_test_dart_lib" +installed_relative="zed_modules/zed-pkg-test/dart-lib" +mkdir -p "$registry" "$home" + +# Keep the test credential-free and independent of developer or runner state. +unset ZED_PKG_TOKEN ZED_PKG_REGISTRY ZED_PKG_HOME ZED_PKG_INTERACTIVE \ + ZED_PKG_ALLOW_NO_MANIFEST ZED_PKG_DO_NOT_WRITE_NEW_MANIFEST \ + PUB_CACHE FLUTTER_ROOT + +copy_tree() { + local source="$1" + local destination="$2" + mkdir -p "$destination" + cp -R "$source/." "$destination/" + rm -rf "$destination/.git" +} + +prepare_consumer() { + local destination="$1" + copy_tree "$app_source" "$destination" + rm -rf \ + "$destination/.zpkg.toml" \ + "$destination/.zpkg.lock" \ + "$destination/.vendor" \ + "$destination/.zed" \ + "$destination/.zpkg-staging" \ + "$destination/zed_modules" \ + "$destination/.dart_tool" \ + "$destination/pubspec.lock" \ + "$destination/build" + + # The authored fixture deliberately uses [install].dir = `.vendor/.zed`. + # A generated consumer manifest intentionally defaults to `zed_modules`, so + # only the disposable native consumer copy is adjusted to that path. + perl -0pi -e \ + 's@\.vendor/\.zed/zed-pkg-test/dart-lib@zed_modules/zed-pkg-test/dart-lib@g' \ + "$destination/pubspec.yaml" + mkdir -p "$destination/tool/deep" +} + +assert_no_symlinks() { + local root="$1" + if find "$root/zed_modules" -type l -print -quit | grep -q .; then + echo "copy-mode Dart install contains a symlink: $root" >&2 + find "$root/zed_modules" -type l -print >&2 + exit 1 + fi +} + +assert_dart_wiring() { + local root="$1" + local wiring="$root/.zed/pub-deps.yaml" + local expected="$work_root/expected-pub-deps.yaml" + + test -f "$root/.zed/paths.json" + grep -Fq "$installed_relative" "$root/.zed/paths.json" + test -f "$wiring" + cat >"$expected" <&2 + exit 1 + fi +} + +run_native_consumer() { + local root="$1" + local label="$2" + local log="$work_root/${label}-dart.log" + + grep -Fq "$native_name:" "$root/pubspec.yaml" + grep -Fq "path: $installed_relative" "$root/pubspec.yaml" + ( + cd "$root" + dart pub get --offline + dart run bin/dart_app.dart + ) 2>&1 | tee "$log" + grep -Fq 'OK: zed-sourced dep resolved alongside pub' "$log" +} + +assert_installed() { + local root="$1" + local installed="$root/$installed_relative" + test -f "$installed/.zpkg.toml" + test -f "$installed/pubspec.yaml" + test -f "$installed/lib/dart_lib.dart" + grep -Eq '^name:[[:space:]]+zed_pkg_test_dart_lib([[:space:]]|$)' \ + "$installed/pubspec.yaml" + assert_no_symlinks "$root" + assert_dart_wiring "$root" +} + +publish_source="$work_root/publish/dart-lib" +copy_tree "$lib_source" "$publish_source" +( + cd "$publish_source" + "$zed" \ + --registry "$file_registry" \ + --home "$home/publisher" \ + publish --skip-vcs-checks +) + +test -f "$registry/packages/zed-pkg-test/dart-lib/package.json" +test -f "$registry/packages/zed-pkg-test/dart-lib/versions/1.0.0.json" +test -n "$(find "$registry/artifacts" -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" + +install_consumer() { + local root="$1" + local local_home="$2" + local spec="$3" + local label="$4" + prepare_consumer "$root" + + # Invoke below the Dart project root to certify conservative pubspec-based + # root inference and root-only manifest creation. + ( + cd "$root/tool/deep" + "$zed" \ + --registry "$file_registry" \ + --home "$local_home" \ + install "$spec" \ + --install-mode copy + ) 2>&1 | tee "$work_root/${label}-install.log" + + test -f "$root/.zpkg.toml" + test -f "$root/.zpkg.lock" + test ! -e "$root/tool/.zpkg.toml" + test ! -e "$root/tool/deep/.zpkg.toml" + grep -F 'org = "zed-local"' "$root/.zpkg.toml" + grep -F 'name = "consumer"' "$root/.zpkg.toml" + grep -F 'version = "0.0.0"' "$root/.zpkg.toml" + grep -F 'zed-generated-consumer' "$root/.zpkg.toml" + grep -F '"zed-pkg-test/dart-lib" = "^1.0.0"' "$root/.zpkg.toml" + grep -F 'adapter = "dart"' "$root/.zpkg.toml" + grep -F 'target = "dart"' "$root/.zpkg.toml" + + if grep -Fq "$work_root" "$root/.zpkg.toml"; then + echo "generated Dart manifest leaked an absolute working path" >&2 + exit 1 + fi + grep -Fq "$file_registry" "$root/.zpkg.lock" + + assert_installed "$root" + run_native_consumer "$root" "$label" +} + +explicit="$work_root/explicit/consumer" +latest="$work_root/latest/consumer" +install_consumer "$explicit" "$home/explicit" "$package@^1.0.0" explicit +install_consumer "$latest" "$home/latest" "$package" latest + +# Explicit semver and registry-latest resolution must converge on the same +# durable direct requirement and immutable lock bytes. +cmp "$explicit/.zpkg.toml" "$latest/.zpkg.toml" +cmp "$explicit/.zpkg.lock" "$latest/.zpkg.lock" + +# Uninstall materialized content from the managed root, then prove a frozen +# nested-directory reinstall restores package and native wiring without +# changing durable project state. +cp "$explicit/.zpkg.toml" "$work_root/before-reinstall.toml" +cp "$explicit/.zpkg.lock" "$work_root/before-reinstall.lock" +( + cd "$explicit" + "$zed" --home "$home/explicit" uninstall +) +test -f "$explicit/.zpkg.toml" +test -f "$explicit/.zpkg.lock" +test ! -e "$explicit/$installed_relative" +( + cd "$explicit/tool/deep" + "$zed" \ + --registry "$file_registry" \ + --home "$home/explicit" \ + install --frozen --install-mode copy +) +cmp "$work_root/before-reinstall.toml" "$explicit/.zpkg.toml" +cmp "$work_root/before-reinstall.lock" "$explicit/.zpkg.lock" +assert_installed "$explicit" +run_native_consumer "$explicit" frozen + +printf 'DEN-1579 Dart durable first-install acceptance passed: %s\n' "$work_root" diff --git a/scripts/durable-first-install-polyglot.sh b/scripts/durable-first-install-polyglot.sh new file mode 100755 index 00000000..9fe60466 --- /dev/null +++ b/scripts/durable-first-install-polyglot.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: durable-first-install-polyglot.sh ZED_BIN ECOSYSTEM LIB_SOURCE APP_SOURCE WORK_ROOT + +ECOSYSTEM must be one of: go, python, rust. +EOF + exit 64 +} + +[[ $# -eq 5 ]] || usage +zed="$1" +ecosystem="$2" +lib_source="$3" +app_source="$4" +work_root="$5" + +case "$ecosystem" in + go|python|rust) ;; + *) usage ;; +esac + +[[ -x "$zed" ]] || { echo "zed binary is not executable: $zed" >&2; exit 66; } +[[ -f "$lib_source/.zpkg.toml" ]] || { echo "missing library fixture: $lib_source" >&2; exit 66; } +[[ -f "$app_source/.zpkg.toml" ]] || { echo "missing application fixture: $app_source" >&2; exit 66; } +[[ ! -e "$work_root" ]] || { echo "work root must be fresh: $work_root" >&2; exit 73; } + +zed="$(cd "$(dirname "$zed")" && pwd)/$(basename "$zed")" +lib_source="$(cd "$lib_source" && pwd)" +app_source="$(cd "$app_source" && pwd)" +mkdir -p "$work_root" +work_root="$(cd "$work_root" && pwd)" + +registry="$work_root/registry" +home="$work_root/zed-home" +file_registry="file://$registry" +package="zed-pkg-test/${ecosystem}-lib" +mkdir -p "$registry" "$home" + +# Do not inherit a developer/runner registry, token, compatibility switch, or +# interactive preference. The canary is credential-free and fully local. +unset ZED_PKG_TOKEN ZED_PKG_REGISTRY ZED_PKG_HOME ZED_PKG_INTERACTIVE \ + ZED_PKG_ALLOW_NO_MANIFEST ZED_PKG_DO_NOT_WRITE_NEW_MANIFEST + +copy_tree() { + local source="$1" + local destination="$2" + mkdir -p "$destination" + cp -R "$source/." "$destination/" + rm -rf "$destination/.git" +} + +prepare_consumer() { + local destination="$1" + copy_tree "$app_source" "$destination" + rm -rf \ + "$destination/.zpkg.toml" \ + "$destination/.zpkg.lock" \ + "$destination/.vendor" \ + "$destination/.zed" \ + "$destination/.zpkg-staging" \ + "$destination/zed_modules" \ + "$destination/node_modules" \ + "$destination/target" \ + "$destination/__pycache__" + + # The authored fixture manifests deliberately override [install].dir to + # `.vendor/.zed`. Once the Zed manifest is removed, the generated contract + # correctly defaults to `zed_modules`. Adapt only these disposable native + # manifests so their real applications consume that generated default. + case "$ecosystem" in + go) + perl -0pi -e \ + 's@\.vendor/\.zed/zed-pkg-test/go-lib@zed_modules/zed-pkg-test/go-lib@g' \ + "$destination/go.mod" + ;; + rust) + perl -0pi -e \ + 's@\.vendor/\.zed/zed-pkg-test/rust-lib@zed_modules/zed-pkg-test/rust-lib@g' \ + "$destination/Cargo.toml" + ;; + python) ;; + esac + + mkdir -p "$destination/deep/nested" +} + +python_adapter_path() { + local root="$1" + local candidate + for candidate in "$root/.zed/pythonpath" "$root/.zed/python_path"; do + if [[ -f "$candidate" ]]; then + cat "$candidate" + return 0 + fi + done + echo "python adapter did not write .zed/pythonpath or .zed/python_path" >&2 + return 1 +} + +run_native_consumer() { + local root="$1" + local log="$2" + case "$ecosystem" in + go) + (cd "$root" && go run .) | tee "$log" + grep -Fq 'OK: zed-sourced module resolved' "$log" + ;; + python) + local python_path + python_path="$(python_adapter_path "$root")" + (cd "$root" && PYTHONPATH="$python_path" python main.py) | tee "$log" + grep -Fq 'OK: zed-sourced dep resolved' "$log" + ;; + rust) + (cd "$root" && CARGO_TARGET_DIR="$work_root/cargo-target" cargo run --quiet) \ + | tee "$log" + ;; + esac +} + +assert_package_shape() { + local root="$1" + local installed="$root/zed_modules/$package" + [[ -d "$installed" ]] + [[ -f "$root/.zed/paths.json" ]] + grep -Fq "zed_modules/$package" "$root/.zed/paths.json" + + case "$ecosystem" in + go) + [[ -f "$installed/go.mod" ]] + [[ -f "$installed/greet.go" ]] + [[ -f "$root/.zed/go.work" ]] + grep -Fq "zed_modules/$package" "$root/.zed/go.work" + ;; + python) + [[ -f "$installed/pyproject.toml" ]] + [[ -f "$installed/python_lib/__init__.py" ]] + python_adapter_path "$root" | grep -Fq "zed_modules/$package" + ;; + rust) + [[ -f "$installed/Cargo.toml" ]] + [[ -f "$installed/src/lib.rs" ]] + [[ -f "$root/.zed/cargo-paths.toml" ]] + grep -Fq "zed_modules/$package" "$root/.zed/cargo-paths.toml" + ;; + esac + + if find "$root/zed_modules" -type l -print -quit | grep -q .; then + echo "copy-mode install contains a symlink: $root" >&2 + find "$root/zed_modules" -type l -print >&2 + exit 1 + fi +} + +publish_source="$work_root/publish/${ecosystem}-lib" +copy_tree "$lib_source" "$publish_source" +( + cd "$publish_source" + "$zed" \ + --registry "$file_registry" \ + --home "$home/publisher" \ + publish --skip-vcs-checks +) + +test -f "$registry/packages/zed-pkg-test/${ecosystem}-lib/package.json" +test -f "$registry/packages/zed-pkg-test/${ecosystem}-lib/versions/1.0.0.json" +test -n "$(find "$registry/artifacts" -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" + +install_consumer() { + local root="$1" + local local_home="$2" + local spec="$3" + local label="$4" + prepare_consumer "$root" + + # The invocation is deliberately below the native project root. Zed must + # climb to go.mod, main.py/project structure, or Cargo.toml and write only at + # that root. + ( + cd "$root/deep/nested" + "$zed" \ + --registry "$file_registry" \ + --home "$local_home" \ + install "$spec" \ + --install-mode copy + ) 2>&1 | tee "$work_root/${label}-install.log" + + test -f "$root/.zpkg.toml" + test -f "$root/.zpkg.lock" + test ! -e "$root/deep/.zpkg.toml" + test ! -e "$root/deep/nested/.zpkg.toml" + grep -F 'org = "zed-local"' "$root/.zpkg.toml" + grep -F 'name = "consumer"' "$root/.zpkg.toml" + grep -F 'version = "0.0.0"' "$root/.zpkg.toml" + grep -F 'zed-generated-consumer' "$root/.zpkg.toml" + grep -F "\"$package\" = \"^1.0.0\"" "$root/.zpkg.toml" + grep -F "adapter = \"$ecosystem\"" "$root/.zpkg.toml" + grep -F "target = \"$ecosystem\"" "$root/.zpkg.toml" + + # The generated manifest is portable and must never capture the disposable + # worktree. The lockfile has a different responsibility: it intentionally + # pins the exact immutable artifact URL, which is an absolute file:// URL in + # this isolated local-registry canary. + if grep -Fq "$work_root" "$root/.zpkg.toml"; then + echo "generated manifest leaked an absolute working path" >&2 + exit 1 + fi + grep -Fq "$file_registry" "$root/.zpkg.lock" + + assert_package_shape "$root" + run_native_consumer "$root" "$work_root/${label}-native.log" +} + +explicit="$work_root/explicit/consumer" +latest="$work_root/latest/consumer" +install_consumer "$explicit" "$home/explicit" "$package@^1.0.0" explicit +install_consumer "$latest" "$home/latest" "$package" latest + +# Resolving an unversioned request through the pinned local registry must record +# the same durable direct requirement and produce identical project state as an +# explicit semver request. +cmp "$explicit/.zpkg.toml" "$latest/.zpkg.toml" +cmp "$explicit/.zpkg.lock" "$latest/.zpkg.lock" + +# Managed state remains stable across uninstall and a frozen reinstall. Install +# and frozen-install stay nested-root tests; uninstall currently consumes the +# managed root directly. +cp "$explicit/.zpkg.toml" "$work_root/before-reinstall.toml" +cp "$explicit/.zpkg.lock" "$work_root/before-reinstall.lock" +( + cd "$explicit" + "$zed" --home "$home/explicit" uninstall +) +test -f "$explicit/.zpkg.toml" +test -f "$explicit/.zpkg.lock" +test ! -e "$explicit/zed_modules/$package" +( + cd "$explicit/deep/nested" + "$zed" \ + --registry "$file_registry" \ + --home "$home/explicit" \ + install --frozen --install-mode copy +) +cmp "$work_root/before-reinstall.toml" "$explicit/.zpkg.toml" +cmp "$work_root/before-reinstall.lock" "$explicit/.zpkg.lock" +assert_package_shape "$explicit" +run_native_consumer "$explicit" "$work_root/frozen-native.log" + +printf 'DEN-1413 %s durable first-install acceptance passed: %s\n' \ + "$ecosystem" "$work_root" diff --git a/scripts/durable-first-install.sh b/scripts/durable-first-install.sh new file mode 100755 index 00000000..0648b21f --- /dev/null +++ b/scripts/durable-first-install.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: durable-first-install.sh ZED_BIN NODE_LIB_DIR NODE_APP_DIR WORK_ROOT +EOF + exit 64 +} + +[[ $# -eq 4 ]] || usage +zed="$1" +node_lib_source="$2" +node_app_source="$3" +work_root="$4" + +[[ -x "$zed" ]] || { echo "zed binary is not executable: $zed" >&2; exit 66; } +[[ -f "$node_lib_source/.zpkg.toml" ]] || { echo "missing node-lib fixture" >&2; exit 66; } +[[ -f "$node_app_source/package.json" ]] || { echo "missing node-app fixture" >&2; exit 66; } +[[ ! -e "$work_root" ]] || { echo "work root must be fresh: $work_root" >&2; exit 73; } + +zed="$(cd "$(dirname "$zed")" && pwd)/$(basename "$zed")" +node_lib_source="$(cd "$node_lib_source" && pwd)" +node_app_source="$(cd "$node_app_source" && pwd)" +mkdir -p "$work_root" +work_root="$(cd "$work_root" && pwd)" + +registry="$work_root/registry" +home="$work_root/zed-home" +mkdir -p "$registry" "$home" + +# Never let a developer or runner credential, registry, or compatibility flag +# affect this credential-free local-registry acceptance test. +unset ZED_PKG_TOKEN ZED_PKG_REGISTRY ZED_PKG_HOME ZED_PKG_ALLOW_NO_MANIFEST \ + ZED_PKG_DO_NOT_WRITE_NEW_MANIFEST ZED_PKG_INTERACTIVE + +file_registry="file://$registry" + +copy_tree() { + local source="$1" + local destination="$2" + mkdir -p "$destination" + cp -R "$source/." "$destination/" + rm -rf "$destination/.git" +} + +prepare_consumer() { + local destination="$1" + copy_tree "$node_app_source" "$destination" + rm -rf \ + "$destination/.zpkg.toml" \ + "$destination/.zpkg.lock" \ + "$destination/zed_modules" \ + "$destination/.vendor" \ + "$destination/.zed" \ + "$destination/node_modules" \ + "$destination/.zpkg-staging" + mkdir -p "$destination/src/nested" +} + +assert_no_symlinks() { + local root="$1" + if find "$root/zed_modules" "$root/node_modules" -type l -print -quit \ + | grep -q .; then + echo "copy-mode consumer contains a symlink: $root" >&2 + find "$root/zed_modules" "$root/node_modules" -type l -print >&2 + exit 1 + fi +} + +run_consumer() { + local root="$1" + ( + cd "$root" + npm run check + npm start + ) +} + +publish_source="$work_root/publish/node-lib" +copy_tree "$node_lib_source" "$publish_source" +( + cd "$publish_source" + "$zed" \ + --registry "$file_registry" \ + --home "$home/publisher" \ + publish --skip-vcs-checks +) + +test -f "$registry/packages/zed-pkg-test/node-lib/package.json" +test -f "$registry/packages/zed-pkg-test/node-lib/versions/1.0.0.json" +test -n "$(find "$registry/artifacts" -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" + +install_durable() { + local root="$1" + local local_home="$2" + prepare_consumer "$root" + ( + cd "$root/src/nested" + "$zed" \ + --registry "$file_registry" \ + --home "$local_home" \ + install zed-pkg-test/node-lib@^1.0.0 \ + --install-mode copy + ) + + test -f "$root/.zpkg.toml" + test -f "$root/.zpkg.lock" + test ! -e "$root/src/.zpkg.toml" + test ! -e "$root/src/nested/.zpkg.toml" + grep -F 'org = "zed-local"' "$root/.zpkg.toml" + grep -F 'name = "consumer"' "$root/.zpkg.toml" + grep -F 'version = "0.0.0"' "$root/.zpkg.toml" + grep -F 'zed-generated-consumer' "$root/.zpkg.toml" + grep -F '"zed-pkg-test/node-lib" = "^1.0.0"' "$root/.zpkg.toml" + grep -F 'adapter = "node"' "$root/.zpkg.toml" + grep -F 'target = "node"' "$root/.zpkg.toml" + test -d "$root/zed_modules/zed-pkg-test/node-lib" + test -d "$root/node_modules/@zed-pkg-test/node-lib" + assert_no_symlinks "$root" + run_consumer "$root" +} + +# Two clean projects with the same basename and inputs must receive identical +# manifest and lock bytes; timestamps, absolute project paths, and random IDs +# are not permitted in durable first-install state. +durable_a="$work_root/durable-a/consumer" +durable_b="$work_root/durable-b/consumer" +install_durable "$durable_a" "$home/durable-a" +install_durable "$durable_b" "$home/durable-b" +cmp "$durable_a/.zpkg.toml" "$durable_b/.zpkg.toml" +cmp "$durable_a/.zpkg.lock" "$durable_b/.zpkg.lock" + +# Generated local identity must fail closed at publication before VCS policy or +# upload behavior can make the inferred placeholder look authoritative. +if ( + cd "$durable_a" + "$zed" \ + --registry "$file_registry" \ + --home "$home/generated-publish" \ + publish --dry-run --skip-vcs-checks +) >"$work_root/generated-publish.log" 2>&1; then + echo "generated consumer manifest unexpectedly passed publication" >&2 + exit 1 +fi +grep -F 'auto-generated local consumer manifest' "$work_root/generated-publish.log" + +# The canonical flag is only a no-new-file escape hatch. On an already managed +# project it is an informational no-op, and frozen installation remains normal. +cp "$durable_a/.zpkg.toml" "$work_root/generated-before.toml" +cp "$durable_a/.zpkg.lock" "$work_root/generated-before.lock" +( + cd "$durable_a/src" + "$zed" \ + --registry "$file_registry" \ + --home "$home/durable-a" \ + install --frozen --do-not-write-new-manifest --install-mode copy +) >"$work_root/existing-generated.log" 2>&1 +cmp "$work_root/generated-before.toml" "$durable_a/.zpkg.toml" +cmp "$work_root/generated-before.lock" "$durable_a/.zpkg.lock" +grep -F 'has no effect because .zpkg.toml already exists' \ + "$work_root/existing-generated.log" +run_consumer "$durable_a" + +# An authored manifest is also never converted into ephemeral mode. The +# existing bytes remain untouched while the ordinary dependency graph installs. +authored="$work_root/authored/node-app" +copy_tree "$node_app_source" "$authored" +rm -rf "$authored/.zpkg.lock" "$authored/.vendor" "$authored/.zed" \ + "$authored/node_modules" "$authored/.zpkg-staging" +cp "$authored/.zpkg.toml" "$work_root/authored-before.toml" +( + cd "$authored/src" + "$zed" \ + --registry "$file_registry" \ + --home "$home/authored" \ + install --do-not-write-new-manifest --install-mode copy +) >"$work_root/authored.log" 2>&1 +cmp "$work_root/authored-before.toml" "$authored/.zpkg.toml" +test -f "$authored/.zpkg.lock" +grep -F 'has no effect because .zpkg.toml already exists' "$work_root/authored.log" +run_consumer "$authored" + +# The canonical CLI escape hatch preserves the established synthetic consumer +# path: lock and package outputs are allowed, but no .zpkg.toml is written. +ephemeral="$work_root/ephemeral/consumer" +prepare_consumer "$ephemeral" +( + cd "$ephemeral/src/nested" + "$zed" \ + --registry "$file_registry" \ + --home "$home/ephemeral" \ + install zed-pkg-test/node-lib@^1.0.0 \ + --do-not-write-new-manifest \ + --install-mode copy +) +test ! -e "$ephemeral/.zpkg.toml" +test -f "$ephemeral/.zpkg.lock" +test -d "$ephemeral/zed_modules/zed-pkg-test/node-lib" +test -d "$ephemeral/node_modules/@zed-pkg-test/node-lib" +assert_no_symlinks "$ephemeral" +run_consumer "$ephemeral" + +# The canonical environment variable is equivalent to the canonical flag. +env_ephemeral="$work_root/env-ephemeral/consumer" +prepare_consumer "$env_ephemeral" +( + cd "$env_ephemeral" + ZED_PKG_DO_NOT_WRITE_NEW_MANIFEST=1 \ + "$zed" \ + --registry "$file_registry" \ + --home "$home/env-ephemeral" \ + install zed-pkg-test/node-lib@^1.0.0 \ + --install-mode copy +) +test ! -e "$env_ephemeral/.zpkg.toml" +test -f "$env_ephemeral/.zpkg.lock" +run_consumer "$env_ephemeral" + +# Legacy spellings remain functional during the compatibility window but make +# the migration path explicit in diagnostics. +legacy="$work_root/legacy/consumer" +prepare_consumer "$legacy" +( + cd "$legacy" + "$zed" \ + --registry "$file_registry" \ + --home "$home/legacy" \ + install zed-pkg-test/node-lib@^1.0.0 \ + --skip-manifest \ + --install-mode copy +) >"$work_root/legacy.log" 2>&1 +test ! -e "$legacy/.zpkg.toml" +test -f "$legacy/.zpkg.lock" +grep -F -- '--skip-manifest is deprecated' "$work_root/legacy.log" +run_consumer "$legacy" + +# A failed graph resolution must remove the exact generated manifest and leave +# no lockfile, materialized dependency, adapter output, or transaction debris. +failed="$work_root/failed/consumer" +prepare_consumer "$failed" +if ( + cd "$failed" + "$zed" \ + --registry "$file_registry" \ + --home "$home/failed" \ + install zed-pkg-test/node-lib@=9.9.9 \ + --install-mode copy +) >"$work_root/failed.log" 2>&1; then + echo "an unpublished dependency version unexpectedly installed" >&2 + exit 1 +fi +test ! -e "$failed/.zpkg.toml" +test ! -e "$failed/.zpkg.lock" +test ! -e "$failed/zed_modules" +test ! -e "$failed/node_modules/@zed-pkg-test/node-lib" +test ! -e "$failed/.zed" +test ! -e "$failed/.zpkg-staging" + +# A lockfile alone cannot truthfully reconstruct direct dependency intent. The +# default fails rather than treating every transitive package as direct; the +# explicit ephemeral mode replays the lock byte-for-byte. +restore="$work_root/restore/consumer" +prepare_consumer "$restore" +cp "$ephemeral/.zpkg.lock" "$restore/.zpkg.lock" +if ( + cd "$restore" + "$zed" \ + --registry "$file_registry" \ + --home "$home/ephemeral" \ + install --frozen --install-mode copy +) >"$work_root/implicit-restore.log" 2>&1; then + echo "manifestless lock-only restore unexpectedly inferred a manifest" >&2 + exit 1 +fi +test ! -e "$restore/.zpkg.toml" +grep -F 'lockfile cannot identify which packages were direct dependencies' \ + "$work_root/implicit-restore.log" +( + cd "$restore" + "$zed" \ + --registry "$file_registry" \ + --home "$home/ephemeral" \ + install --frozen --do-not-write-new-manifest --install-mode copy +) +test ! -e "$restore/.zpkg.toml" +cmp "$ephemeral/.zpkg.lock" "$restore/.zpkg.lock" +run_consumer "$restore" + +printf 'DEN-1413 durable first-install acceptance passed: %s\n' "$work_root" diff --git a/scripts/export-test-org-fleet-index.mjs b/scripts/export-test-org-fleet-index.mjs new file mode 100755 index 00000000..d1300788 --- /dev/null +++ b/scripts/export-test-org-fleet-index.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node +import path from 'node:path'; +import { readFleetManifest } from './read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const manifest = readFleetManifest(path.join(root, 'bootstrap', 'test-org-fleet.json.gz')); +const excluded = new Set(manifest.policy.excludedOrganizations ?? []); + +const organizations = manifest.pairs.map((pair) => { + if (excluded.has(pair.sourceOrg) || excluded.has(pair.testOrg)) { + throw new Error(`excluded organization leaked into the fleet: ${pair.testOrg}`); + } + const retainedRepositories = [...(pair.existingRepositories ?? [])]; + const specializedRepositories = pair.repositories.map((repository) => repository.name); + const managedRepositories = ['.github', ...specializedRepositories]; + const expectedRepositories = [...retainedRepositories, ...managedRepositories]; + if (new Set(expectedRepositories).size !== expectedRepositories.length) { + throw new Error(`duplicate repository names for ${pair.testOrg}`); + } + return { + sourceOrganization: pair.sourceOrg, + testOrganization: pair.testOrg, + retainedRepositories, + governanceRepository: '.github', + specializedRepositories, + managedRepositories, + expectedRepositories, + retainedCount: retainedRepositories.length, + specializedCount: specializedRepositories.length, + managedCount: managedRepositories.length, + expectedCount: expectedRepositories.length, + }; +}); + +const result = { + schemaVersion: 1, + generatedFrom: [ + 'bootstrap/test-org-fleet.json.gz.b64.parts', + 'bootstrap/test-org-fleet.extensions.json', + ], + excludedOrganizations: [...excluded].sort(), + pairCount: organizations.length, + retainedRepositoryCount: organizations.reduce((sum, item) => sum + item.retainedCount, 0), + specializedRepositoryCount: organizations.reduce((sum, item) => sum + item.specializedCount, 0), + governanceRepositoryCount: organizations.length, + managedRepositoryCount: organizations.reduce((sum, item) => sum + item.managedCount, 0), + expectedRepositoryCount: organizations.reduce((sum, item) => sum + item.expectedCount, 0), + organizations, +}; + +if (result.pairCount !== 19) throw new Error(`expected 19 pairs, got ${result.pairCount}`); +if (result.retainedRepositoryCount !== 22) throw new Error(`expected 22 retained repositories, got ${result.retainedRepositoryCount}`); +if (result.specializedRepositoryCount !== 322) throw new Error(`expected 322 specialized repositories, got ${result.specializedRepositoryCount}`); +if (result.governanceRepositoryCount !== 19) throw new Error(`expected 19 governance repositories, got ${result.governanceRepositoryCount}`); +if (result.managedRepositoryCount !== 341) throw new Error(`expected 341 managed repositories, got ${result.managedRepositoryCount}`); +if (result.expectedRepositoryCount !== 363) throw new Error(`expected 363 total repositories, got ${result.expectedRepositoryCount}`); +if (!result.excludedOrganizations.map((item) => item.toLowerCase()).includes('r2g')) throw new Error('r2g must remain excluded'); +if (!result.excludedOrganizations.map((item) => item.toLowerCase()).includes('r2g-test')) throw new Error('r2g-test must remain excluded'); + +process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); diff --git a/scripts/fix_global_packages_materialization.py b/scripts/fix_global_packages_materialization.py new file mode 100644 index 00000000..949cd499 --- /dev/null +++ b/scripts/fix_global_packages_materialization.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Narrow compatibility repair for the current-main global package assembly. + +The main materializer intentionally copies the reviewed feature files before +composing them with current main. This verifier preserves the current lockfile +import surface, normalizes the root-help adapter to the `Result` contract +expected by the modular dispatcher, boxes the large install payload so the +public command enum remains Clippy-clean, and removes a redundant Zed package +dependency while Cargo still owns that dependency through an immutable Git pin. +It is idempotent and fails on unknown source shapes rather than guessing. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + + +def patch_ops(product: Path) -> None: + path = product / "src/ops.rs" + text = path.read_text() + complete = "use zed_interfaces::lockfile::{LockedPackage, Lockfile};" + legacy = "use zed_interfaces::lockfile::Lockfile;" + if complete in text: + return + if legacy not in text: + raise SystemExit("src/ops.rs has an unknown lockfile import shape") + path.write_text(text.replace(legacy, complete, 1)) + + +def patch_global(product: Path) -> None: + path = product / "src/global.rs" + text = path.read_text() + + mapped_arm = "Route::RootHelp => Some(print_root_help().map(|()| 0))," + direct_arm = "Route::RootHelp => Some(print_root_help())," + if mapped_arm in text: + text = text.replace(mapped_arm, direct_arm, 1) + elif direct_arm not in text: + raise SystemExit("src/global.rs has an unknown RootHelp dispatch shape") + + variants = ( + '''fn print_root_help() -> Result<()> { + crate::completion::print_root_help() +} +''', + '''fn print_root_help() -> Result { + crate::completion::print_root_help() +} +''', + '''fn print_root_help() -> Result<()> { + let mut command = dev::augment_root_command(augment_root_command(crate::cli::Cli::command())); + command.print_help().context("printing zed help")?; + println!(); + Ok(()) +} +''', + ) + replacement = '''fn print_root_help() -> Result { + crate::completion::print_root_help()?; + Ok(0) +} +''' + if replacement not in text: + for variant in variants: + if variant in text: + text = text.replace(variant, replacement, 1) + break + else: + raise SystemExit("src/global.rs has an unknown print_root_help shape") + + boxed_variant = " Install(Box)," + plain_variant = " Install(GlobalInstallArgs)," + if boxed_variant not in text: + if plain_variant not in text: + raise SystemExit("src/global.rs has an unknown GlobalAction::Install shape") + text = text.replace(plain_variant, boxed_variant, 1) + + boxed_match = ( + " GlobalAction::Install(options) => install(&cfg, &bin_dir, *options)," + ) + plain_match = ( + " GlobalAction::Install(options) => install(&cfg, &bin_dir, options)," + ) + if boxed_match not in text: + if plain_match not in text: + raise SystemExit("src/global.rs has an unknown install dispatch shape") + text = text.replace(plain_match, boxed_match, 1) + + path.write_text(text) + + +def patch_package_manifest(product: Path) -> None: + path = product / ".zpkg.toml" + text = path.read_text() + redundant = '''[dependencies] +"zed-pkg/zed-interfaces" = "^0.1.0" + +''' + if redundant in text: + text = text.replace(redundant, "", 1) + elif "zed-pkg/zed-interfaces" in text: + raise SystemExit(".zpkg.toml has an unknown zed-interfaces dependency shape") + path.write_text(text) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--product", type=Path, required=True) + args = parser.parse_args() + product = args.product.resolve() + patch_ops(product) + patch_global(product) + patch_package_manifest(product) + print( + "preserved current-main imports, normalized root help, boxed the install payload, and aligned package metadata with Cargo" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/fleet-manifest-extensions.mjs b/scripts/fleet-manifest-extensions.mjs new file mode 100644 index 00000000..af2ef743 --- /dev/null +++ b/scripts/fleet-manifest-extensions.mjs @@ -0,0 +1,52 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export function applyFleetManifestExtensions(manifest, manifestPath) { + const extensionPath = path.join(path.dirname(manifestPath), 'test-org-fleet.extensions.json'); + if (!fs.existsSync(extensionPath)) return manifest; + + const extension = JSON.parse(fs.readFileSync(extensionPath, 'utf8')); + if (extension.schemaVersion !== 1) { + throw new Error(`fleet extension schemaVersion must be 1: ${extensionPath}`); + } + if (!Array.isArray(extension.pairs) || extension.pairs.length === 0) { + throw new Error(`fleet extension pairs must be a non-empty array: ${extensionPath}`); + } + + const patchedOrganizations = new Set(); + for (const pairExtension of extension.pairs) { + if (typeof pairExtension.testOrg !== 'string' || pairExtension.testOrg.length === 0) { + throw new Error(`fleet extension testOrg must be a non-empty string: ${extensionPath}`); + } + if (patchedOrganizations.has(pairExtension.testOrg)) { + throw new Error(`duplicate fleet extension pair: ${pairExtension.testOrg}`); + } + patchedOrganizations.add(pairExtension.testOrg); + + const pair = manifest.pairs?.find((candidate) => candidate.testOrg === pairExtension.testOrg); + if (!pair) throw new Error(`fleet extension targets unknown test organization: ${pairExtension.testOrg}`); + if (pairExtension.sourceOrg && pairExtension.sourceOrg !== pair.sourceOrg) { + throw new Error(`fleet extension source organization mismatch for ${pairExtension.testOrg}`); + } + if (!Array.isArray(pairExtension.repositories) || pairExtension.repositories.length === 0) { + throw new Error(`fleet extension repositories must be non-empty for ${pairExtension.testOrg}`); + } + + const names = new Set([ + ...(pair.existingRepositories ?? []), + ...(pair.repositories ?? []).map((repository) => repository.name), + ]); + for (const repository of pairExtension.repositories) { + if (!repository || typeof repository.name !== 'string' || repository.name.length === 0) { + throw new Error(`fleet extension repository name is required for ${pairExtension.testOrg}`); + } + if (names.has(repository.name)) { + throw new Error(`fleet extension duplicates repository ${pairExtension.testOrg}/${repository.name}`); + } + names.add(repository.name); + pair.repositories.push(JSON.parse(JSON.stringify(repository))); + } + } + + return manifest; +} diff --git a/scripts/frozen_fetch_canary.py b/scripts/frozen_fetch_canary.py new file mode 100644 index 00000000..f66b96af --- /dev/null +++ b/scripts/frozen_fetch_canary.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +"""Black-box canary for `zed fetch --frozen`. + +The test publishes one real zed-pkg-test fixture to a private file registry, +creates a consumer lock through normal installation, uninstalls the project +materialization, and then proves the resolver-only bundle contract from the +built CLI under test. All state is disposable and no credentials are used. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tomllib +from pathlib import Path +from typing import Mapping, Sequence + + +def shell_quote(value: str) -> str: + if not value: + return "''" + if re.fullmatch(r"[A-Za-z0-9_@%+=:,./-]+", value): + return value + return "'" + value.replace("'", "'\\''") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path, + extra_env: Mapping[str, str] | None = None, + should_fail: bool = False, +) -> str: + argv = [str(value) for value in command] + print(f"\n$ (cd {cwd} && {' '.join(shell_quote(value) for value in argv)})", flush=True) + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "ZED_PKG_TOKEN": "", + "ZED_PKG_INTERACTIVE": "false", + } + ) + if extra_env: + environment.update(extra_env) + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(completed.stdout, end="", flush=True) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {argv!r}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {argv!r}\n{completed.stdout}" + ) + return completed.stdout + + +def tree_digest(root: Path) -> tuple[tuple[str, int, str], ...]: + if not root.exists(): + return () + rows: list[tuple[str, int, str]] = [] + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + rows.append((relative, -1, f"symlink:{os.readlink(path)}")) + elif path.is_file(): + data = path.read_bytes() + rows.append((relative, len(data), hashlib.sha256(data).hexdigest())) + return tuple(rows) + + +def assert_git_clean(root: Path) -> None: + output = run( + ["git", "status", "--porcelain=v1", "--untracked-files=all"], + cwd=root, + ) + if output.strip(): + raise AssertionError(f"fixture checkout was mutated:\n{output}") + + +def assert_no_fetch_temporaries(parent: Path) -> None: + escaped = sorted( + path + for path in parent.iterdir() + if path.name.startswith(".zed-fetch-") + ) + if escaped: + raise AssertionError(f"temporary fetch directories escaped cleanup: {escaped}") + + +def write_consumer_manifest(directory: Path, org: str, name: str, version: str) -> None: + directory.mkdir(parents=True, exist_ok=True) + content = f'''[package]\norg = "zed-fetch-canary"\nname = "consumer"\nversion = "0.0.0"\ndescription = "External resolver-only fetch canary"\n\n[package.repository]\nvcs = "git"\nurl = "https://localhost/zed-fetch-canary/consumer"\n\n[install]\ndir = "zed_modules"\n\n[dependencies]\n"{org}/{name}" = "={version}"\n''' + (directory / ".zpkg.toml").write_text(content, encoding="utf-8") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fixture-dir", type=Path, required=True) + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + fixture = args.fixture_dir.resolve() + zed = args.zed.resolve() + root = args.work_root.resolve() + if root.exists(): + raise AssertionError(f"work root must be fresh: {root}") + if not zed.is_file(): + raise AssertionError(f"zed binary not found: {zed}") + if not (fixture / ".zpkg.toml").is_file(): + raise AssertionError(f"fixture has no .zpkg.toml: {fixture}") + + root.mkdir(parents=True) + registry = root / "registry" + registry.mkdir() + registry_url = f"file://{registry}" + publish_home = root / "publish-home" + install_home = root / "install-home" + fetch_home = root / "fetch-home-must-remain-absent" + outputs = root / "fetch-outputs" + outputs.mkdir() + + with (fixture / ".zpkg.toml").open("rb") as handle: + manifest = tomllib.load(handle) + package = manifest["package"] + org = str(package["org"]) + name = str(package["name"]) + version = str(package["version"]) + + def zed_cmd( + *command: str | Path, + cwd: Path, + home: Path, + should_fail: bool = False, + extra_env: Mapping[str, str] | None = None, + ) -> str: + return run( + [ + zed, + "--registry", + registry_url, + "--home", + home, + *command, + ], + cwd=cwd, + should_fail=should_fail, + extra_env=extra_env, + ) + + # Strict frozen fetch requires the registry record and lock to agree on + # real VCS provenance. The external checkout deliberately does not fetch + # tags, so create the manifest-derived release tag locally at the exact + # checked-out commit instead of using the publish escape hatch. + assert_git_clean(fixture) + release_tag = f"v{version}" + fixture_commit = run(["git", "rev-parse", "HEAD"], cwd=fixture).strip() + run(["git", "tag", release_tag, fixture_commit], cwd=fixture) + zed_cmd("publish", cwd=fixture, home=publish_home) + assert_git_clean(fixture) + + registry_metadata_path = ( + registry / "packages" / org / name / "versions" / f"{version}.json" + ) + registry_metadata = json.loads(registry_metadata_path.read_text(encoding="utf-8")) + if registry_metadata.get("vcs_tag") != release_tag: + raise AssertionError("registry did not retain the exact release tag") + if registry_metadata.get("vcs_commit") != fixture_commit: + raise AssertionError("registry did not retain the exact source commit") + + consumer = root / "consumer" + write_consumer_manifest(consumer, org, name, version) + zed_cmd( + "install", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + cwd=consumer, + home=install_home, + ) + lock = consumer / ".zpkg.lock" + lock_bytes = lock.read_bytes() + with lock.open("rb") as handle: + lock_data = tomllib.load(handle) + locked_packages = lock_data.get("package", []) + if len(locked_packages) != 1: + raise AssertionError(f"expected one canonical lock entry: {locked_packages!r}") + locked = locked_packages[0] + if locked.get("vcs_tag") != release_tag: + raise AssertionError("lock did not preserve the registry release tag") + if locked.get("vcs_commit") != fixture_commit: + raise AssertionError("lock did not preserve the registry source commit") + if not (consumer / "zed_modules" / org / name / ".zpkg.toml").is_file(): + raise AssertionError("normal install did not materialize the published fixture") + + # Start the fetch boundary from an ordinary manifest + lock, with no + # installed package tree or global store available to the fetch command. + zed_cmd("uninstall", cwd=consumer, home=install_home) + if lock.read_bytes() != lock_bytes: + raise AssertionError("uninstall changed the frozen lock input") + shutil.rmtree(install_home) + if (consumer / "zed_modules").exists(): + raise AssertionError("uninstall left project materialization behind") + project_before = tree_digest(consumer) + + first = outputs / "first" + second = outputs / "second" + zed_cmd( + "fetch", + "--frozen", + "--output", + first, + cwd=consumer, + home=fetch_home, + ) + zed_cmd( + "fetch", + cwd=consumer, + home=fetch_home, + extra_env={ + "ZED_PKG_FROZEN": "yes", + "ZED_PKG_FETCH_OUTPUT": str(second), + }, + ) + + if tree_digest(first) != tree_digest(second): + raise AssertionError("two frozen fetches produced different bundle bytes") + if tree_digest(consumer) != project_before: + raise AssertionError("resolver-only fetch mutated the consumer project") + if fetch_home.exists(): + raise AssertionError("resolver-only fetch wrote the configured global Zed home") + + index_path = first / "metadata" / "index.json" + index_text = index_path.read_text(encoding="utf-8") + if str(registry) in index_text or registry_url in index_text: + raise AssertionError("fetch metadata leaked its literal registry source") + index = json.loads(index_text) + if index.get("schema") != "zed.fetch/v1": + raise AssertionError(f"unexpected fetch schema: {index.get('schema')!r}") + packages = index.get("packages") + if not isinstance(packages, list) or len(packages) != 1: + raise AssertionError(f"expected one fetched package: {packages!r}") + fetched = packages[0] + if (fetched.get("org"), fetched.get("name"), fetched.get("version")) != ( + org, + name, + version, + ): + raise AssertionError(f"wrong package identity in fetch index: {fetched!r}") + digest = str(fetched.get("sha256") or "") + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise AssertionError(f"invalid artifact digest in fetch index: {digest!r}") + if fetched.get("vcs_tag") != release_tag: + raise AssertionError("fetch index did not preserve the release tag") + if fetched.get("vcs_commit") != fixture_commit: + raise AssertionError("fetch index did not preserve the source commit") + if fetched.get("source_kind") != "file": + raise AssertionError(f"wrong source classification: {fetched!r}") + payload = first / "packages" / digest / "pkg" + if not (payload / ".zpkg.toml").is_file(): + raise AssertionError(f"fetch bundle omitted verified package payload: {payload}") + + expected_lock_digest = hashlib.sha256(lock_bytes).hexdigest() + lock_digest_text = (first / "metadata" / "lock.sha256").read_text( + encoding="utf-8" + ) + if lock_digest_text != f"{expected_lock_digest} .zpkg.lock\n": + raise AssertionError("fetch bundle retained the wrong lock digest") + + # Artifact provenance drift must fail before a final output appears. Use a + # syntactically valid nonzero digest so the parser accepts the lock and the + # test reaches the registry-versus-lock verification boundary. + tampered = root / "tampered-consumer" + shutil.copytree(consumer, tampered) + tampered_lock = tampered / ".zpkg.lock" + wrong_digest = "1" * 64 if digest != "1" * 64 else "2" * 64 + tampered_text, replacements = re.subn( + r'^sha256 = "[0-9a-f]{64}"$', + f'sha256 = "{wrong_digest}"', + tampered_lock.read_text(encoding="utf-8"), + count=1, + flags=re.MULTILINE, + ) + if replacements != 1: + raise AssertionError("could not identify one lock digest to tamper") + tampered_lock.write_text(tampered_text, encoding="utf-8") + tampered_output = outputs / "tampered" + failure = zed_cmd( + "fetch", + "--frozen", + "--output", + tampered_output, + cwd=tampered, + home=fetch_home, + should_fail=True, + ) + if "digest changed" not in failure: + raise AssertionError(f"tampered fetch failed for the wrong reason:\n{failure}") + if tampered_output.exists(): + raise AssertionError("tampered fetch published a final output") + + # Existing outputs are immutable caller-owned state. + existing = outputs / "existing" + existing.mkdir() + sentinel = existing / "owned.txt" + sentinel.write_text("caller-owned\n", encoding="utf-8") + zed_cmd( + "fetch", + "--frozen", + "--output", + existing, + cwd=consumer, + home=fetch_home, + should_fail=True, + ) + if sentinel.read_text(encoding="utf-8") != "caller-owned\n": + raise AssertionError("failed fetch changed a pre-existing destination") + + # Missing and dependency-free locks are both explicit, deterministic cases. + missing = root / "missing-lock" + missing.mkdir() + missing_output = outputs / "missing" + zed_cmd( + "fetch", + "--frozen", + "--output", + missing_output, + cwd=missing, + home=fetch_home, + should_fail=True, + ) + if missing_output.exists(): + raise AssertionError("missing-lock fetch published an output") + + empty = root / "empty-lock" + empty.mkdir() + (empty / ".zpkg.lock").write_text("version = 1\n", encoding="utf-8") + empty_output = outputs / "empty" + zed_cmd( + "fetch", + "--frozen", + "--output", + empty_output, + cwd=empty, + home=fetch_home, + ) + empty_index = json.loads( + (empty_output / "metadata" / "index.json").read_text(encoding="utf-8") + ) + if empty_index.get("packages") != []: + raise AssertionError(f"empty lock exported packages: {empty_index!r}") + + assert_no_fetch_temporaries(outputs) + assert_git_clean(fixture) + print( + json.dumps( + { + "result": "PASS", + "package": f"{org}/{name}@{version}", + "vcs_tag": release_tag, + "vcs_commit": fixture_commit, + "artifact_sha256": digest, + "bundle_files": len(tree_digest(first)), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/frozen_fetch_output_boundaries.py b/scripts/frozen_fetch_output_boundaries.py new file mode 100644 index 00000000..36f06d4b --- /dev/null +++ b/scripts/frozen_fetch_output_boundaries.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""External canonical-output checks for `zed fetch --frozen`.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import subprocess +import sys +from pathlib import Path +from typing import Sequence + + +def digest_tree(root: Path) -> tuple[tuple[str, str], ...]: + rows: list[tuple[str, str]] = [] + for path in sorted(root.rglob("*")): + if path.is_symlink(): + rows.append( + ( + path.relative_to(root).as_posix(), + f"symlink:{os.readlink(path)}", + ) + ) + elif path.is_file(): + rows.append( + ( + path.relative_to(root).as_posix(), + hashlib.sha256(path.read_bytes()).hexdigest(), + ) + ) + return tuple(rows) + + +def run_failure(command: Sequence[str | Path], *, cwd: Path) -> str: + argv = [str(item) for item in command] + completed = subprocess.run( + argv, + cwd=cwd, + env={ + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "PATH": os.environ.get("PATH", ""), + "ZED_PKG_INTERACTIVE": "false", + "ZED_PKG_TOKEN": "", + }, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(f"\n$ {' '.join(argv)}", flush=True) + print(completed.stdout, end="", flush=True) + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {argv!r}") + return completed.stdout + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--work-root", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + zed = args.zed.resolve() + root = args.work_root.resolve() + consumer = root / "consumer" + registry = root / "registry" + home = root / "output-boundary-home-must-remain-absent" + + if not zed.is_file(): + raise AssertionError(f"zed binary not found: {zed}") + if not (consumer / ".zpkg.lock").is_file(): + raise AssertionError(f"external canary consumer lock not found: {consumer}") + if not registry.is_dir(): + raise AssertionError(f"external canary registry not found: {registry}") + + command_prefix = [ + zed, + "--registry", + f"file://{registry}", + "--home", + home, + "fetch", + "--frozen", + "--output", + ] + before = digest_tree(consumer) + + missing_parent = root / "missing-output-parent" + missing_output = missing_parent / "bundle" + failure = run_failure([*command_prefix, missing_output], cwd=consumer) + if "parent must already exist" not in failure: + raise AssertionError(f"missing-parent failure was not actionable:\n{failure}") + if missing_parent.exists() or missing_output.exists(): + raise AssertionError("failed fetch created its missing output parent") + + caller_file = root / "caller-owned-parent-file" + caller_file.write_text("caller-owned\n", encoding="utf-8") + failure = run_failure([*command_prefix, caller_file / "bundle"], cwd=consumer) + if "parent must already exist" not in failure: + raise AssertionError(f"non-directory-parent failure was not actionable:\n{failure}") + if caller_file.read_text(encoding="utf-8") != "caller-owned\n": + raise AssertionError("failed fetch changed a caller-owned parent file") + + redirect = root / "output-parent-redirect" + os.symlink(consumer, redirect, target_is_directory=True) + redirected_output = redirect / "generated" + failure = run_failure([*command_prefix, redirected_output], cwd=consumer) + if "canonical fetch output" not in failure: + raise AssertionError(f"symlink-redirection failure was not actionable:\n{failure}") + if (consumer / "generated").exists(): + raise AssertionError("symlinked output parent redirected state into the project") + + if digest_tree(consumer) != before: + raise AssertionError("output-boundary failures mutated the consumer project") + if home.exists(): + raise AssertionError("output-boundary failures wrote the configured Zed home") + + escaped = sorted( + path.name for path in root.iterdir() if path.name.startswith(".zed-fetch-") + ) + if escaped: + raise AssertionError(f"output-boundary failures leaked temporary state: {escaped}") + + print( + "PASS: missing, non-directory, and symlinked output parents fail without mutation" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ghcr_fallback.py b/scripts/ghcr_fallback.py new file mode 100644 index 00000000..c3695e75 --- /dev/null +++ b/scripts/ghcr_fallback.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Live GHCR canary: prove GitHub Packages can host a Zed artifact. + +Unlike github_api_fallback.py this does *not* create a Release. The CLI +source-fallback path tries a Release sidecar first, then GHCR. A package +that exists only on ghcr.io forces the GHCR branch. + +Token (never printed): GITHUB_TOKEN / GH_TOKEN / ZED_PKG_GITHUB_TOKEN +with `write:packages` on zed-pkg-test. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import shutil +import socket +import subprocess +import sys +import tarfile +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +ORG = "zed-pkg-test" +REPO = "ghcr-fallback-canary" +PACKAGE_NAME = "ghcr-fallback-canary" +VERSION = "0.0.1" +TAG = f"v{VERSION}" +API = "https://api.github.com" +GHCR = "https://ghcr.io" +USER_AGENT = "zed-pkg-test-ghcr-fallback/1.0" +PAYLOAD = "ghcr-fallback\n" +LAYER_MEDIA = "application/vnd.zed.package.v1.tar+gzip" +CONFIG_MEDIA = "application/vnd.zed.package.config.v1+json" +MANIFEST_MEDIA = "application/vnd.oci.image.manifest.v1+json" + + +def token_from_env() -> str: + for key in ("ZED_PKG_GITHUB_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"): + value = os.environ.get(key, "").strip() + if value and value != "from-env": + return value + raise RuntimeError("set GITHUB_TOKEN / GH_TOKEN / ZED_PKG_GITHUB_TOKEN") + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def api_request( + method: str, + url: str, + token: str, + *, + body: bytes | None = None, + content_type: str = "application/json", + accept: str = "application/vnd.github+json", +) -> tuple[int, dict[str, str], bytes]: + request = urllib.request.Request( + url, + data=body, + method=method, + headers={ + "Accept": accept, + "Authorization": f"Bearer {token}", + "User-Agent": USER_AGENT, + }, + ) + if body is not None: + request.add_header("Content-Type", content_type) + request.add_header("Content-Length", str(len(body))) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return response.status, dict(response.headers), response.read() + except urllib.error.HTTPError as error: + return error.code, dict(error.headers), error.read() + + +def api_json(method: str, url: str, token: str, payload: dict[str, Any] | None = None) -> Any: + body = None if payload is None else json.dumps(payload).encode("utf-8") + status, _headers, raw = api_request(method, url, token, body=body) + if status >= 300: + raise RuntimeError(f"{method} {url} returned {status}: {raw[:400]!r}") + if not raw: + return {} + return json.loads(raw.decode("utf-8")) + + +def pack_canary(work: Path) -> tuple[bytes, Path]: + pkg = work / "package" / "pkg" + (pkg / "src").mkdir(parents=True) + (pkg / "src" / "payload.txt").write_text(PAYLOAD, encoding="utf-8") + (pkg / ".zpkg.toml").write_text( + f'''[package] +org = "{ORG}" +name = "{PACKAGE_NAME}" +version = "{VERSION}" +description = "Live GHCR fallback canary" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://github.com/{ORG}/{REPO}" + +[install] +dir = "zed_modules" +adapter = "none" +''', + encoding="utf-8", + ) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + archive.add(pkg, arcname="pkg") + tarball = buffer.getvalue() + path = work / f"{PACKAGE_NAME}-{VERSION}.tar.gz" + path.write_bytes(tarball) + return tarball, path + + +def ensure_repo(token: str) -> dict[str, Any]: + status, _headers, raw = api_request("GET", f"{API}/repos/{ORG}/{REPO}", token) + if status == 200: + repo = json.loads(raw.decode("utf-8")) + print(f"repo exists: {repo['html_url']}", flush=True) + return repo + if status != 404: + raise RuntimeError(f"GET repo returned {status}: {raw[:400]!r}") + repo = api_json( + "POST", + f"{API}/orgs/{ORG}/repos", + token, + { + "name": REPO, + "description": "Live GHCR canary: packed Zed artifact on GitHub Packages, no Release.", + "private": False, + "auto_init": True, + "has_issues": False, + "has_projects": False, + "has_wiki": False, + }, + ) + print(f"created {repo['html_url']}", flush=True) + return repo + + +def ensure_manifest_on_default_branch(token: str, repo: dict[str, Any]) -> None: + branch = repo.get("default_branch") or "main" + status, _headers, raw = api_request( + "GET", + f"{API}/repos/{ORG}/{REPO}/contents/.zpkg.toml?ref={branch}", + token, + ) + if status == 200: + return + import base64 + + content = f'''[package] +org = "{ORG}" +name = "{PACKAGE_NAME}" +version = "{VERSION}" +description = "Live GHCR fallback canary" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://github.com/{ORG}/{REPO}" + +[install] +dir = "zed_modules" +adapter = "none" +''' + api_json( + "PUT", + f"{API}/repos/{ORG}/{REPO}/contents/.zpkg.toml", + token, + { + "message": "chore: add canary .zpkg.toml for GHCR fallback", + "content": base64.b64encode(content.encode("utf-8")).decode("ascii"), + "branch": branch, + }, + ) + print(f"wrote .zpkg.toml on {branch}", flush=True) + + +def ghcr_bearer(token: str, scope: str) -> str: + query = urllib.parse.urlencode({"service": "ghcr.io", "scope": scope}) + status, _headers, raw = api_request( + "GET", + f"{GHCR}/token?{query}", + token, + accept="application/json", + ) + if status != 200: + raise RuntimeError(f"GHCR token returned {status}: {raw[:200]!r}") + bearer = json.loads(raw.decode("utf-8")).get("token") + if not bearer: + raise RuntimeError("GHCR token response missing token") + return bearer + + +def ghcr_request( + method: str, + url: str, + bearer: str, + *, + body: bytes | None = None, + content_type: str | None = None, + accept: str = "application/vnd.oci.image.manifest.v1+json", +) -> tuple[int, dict[str, str], bytes]: + request = urllib.request.Request( + url, + data=body, + method=method, + headers={ + "Authorization": f"Bearer {bearer}", + "User-Agent": USER_AGENT, + "Accept": accept, + }, + ) + if body is not None and content_type: + request.add_header("Content-Type", content_type) + request.add_header("Content-Length", str(len(body))) + try: + with urllib.request.urlopen(request, timeout=120) as response: + return response.status, dict(response.headers), response.read() + except urllib.error.HTTPError as error: + return error.code, dict(error.headers), error.read() + + +def put_blob(repo: str, bearer: str, digest: str, media: str, data: bytes) -> None: + status, _headers, _raw = ghcr_request( + "HEAD", f"{GHCR}/v2/{repo}/blobs/{digest}", bearer + ) + if status == 200: + return + status, headers, raw = ghcr_request( + "POST", f"{GHCR}/v2/{repo}/blobs/uploads/", bearer + ) + if status not in {202, 201}: + raise RuntimeError(f"start blob upload returned {status}: {raw[:200]!r}") + location = headers.get("Location") or headers.get("location") + if not location: + raise RuntimeError("blob upload missing Location") + if location.startswith("/"): + location = f"{GHCR}{location}" + sep = "&" if "?" in location else "?" + status, _headers, raw = ghcr_request( + "PUT", + f"{location}{sep}digest={digest}", + bearer, + body=data, + content_type=media, + ) + if status not in {201, 202}: + raise RuntimeError(f"put blob {digest} returned {status}: {raw[:200]!r}") + + +def push_ghcr(token: str, tarball: bytes) -> dict[str, Any]: + repo = f"{ORG}/{REPO}".lower() + bearer = ghcr_bearer(token, f"repository:{repo}:push,pull") + layer_digest = f"sha256:{sha256_hex(tarball)}" + config = { + "schema": "zpkg.oci-config/v1", + "package": {"org": ORG, "name": PACKAGE_NAME, "version": VERSION}, + "repository": f"https://github.com/{ORG}/{REPO}", + "vcs_tag": TAG, + "artifact": { + "format": "tar.gz", + "digest": layer_digest, + "size": len(tarball), + }, + } + config_bytes = json.dumps(config, separators=(",", ":")).encode("utf-8") + config_digest = f"sha256:{sha256_hex(config_bytes)}" + put_blob(repo, bearer, config_digest, CONFIG_MEDIA, config_bytes) + put_blob(repo, bearer, layer_digest, LAYER_MEDIA, tarball) + manifest = { + "schemaVersion": 2, + "mediaType": MANIFEST_MEDIA, + "config": { + "mediaType": CONFIG_MEDIA, + "digest": config_digest, + "size": len(config_bytes), + }, + "layers": [ + { + "mediaType": LAYER_MEDIA, + "digest": layer_digest, + "size": len(tarball), + } + ], + "annotations": { + "org.opencontainers.image.source": f"https://github.com/{ORG}/{REPO}", + "dev.zed-pkg.package": f"{ORG}/{PACKAGE_NAME}", + "dev.zed-pkg.vcs-tag": TAG, + }, + } + manifest_bytes = json.dumps(manifest, separators=(",", ":")).encode("utf-8") + status, _headers, raw = ghcr_request( + "PUT", + f"{GHCR}/v2/{repo}/manifests/{TAG}", + bearer, + body=manifest_bytes, + content_type=MANIFEST_MEDIA, + ) + if status not in {201, 202}: + raise RuntimeError(f"put manifest returned {status}: {raw[:300]!r}") + # Public visibility so later anonymous/org pulls can work. Tokened canary + # still succeeds if this PATCH is denied on a first-publish race. + status, _headers, raw = api_request( + "PATCH", + f"{API}/orgs/{ORG}/packages/container/{REPO}", + token, + body=json.dumps({"visibility": "public"}).encode("utf-8"), + ) + if status >= 300: + print(f"note: package visibility PATCH returned {status}: {raw[:160]!r}", flush=True) + return { + "reference": f"ghcr.io/{repo}:{TAG}", + "sha256": layer_digest.removeprefix("sha256:"), + "size": len(tarball), + "web": f"https://github.com/orgs/{ORG}/packages/container/package/{REPO}", + } + + +def prove_ghcr_get(token: str, digest: str) -> None: + repo = f"{ORG}/{REPO}".lower() + bearer = ghcr_bearer(token, f"repository:{repo}:pull") + status, _headers, raw = ghcr_request( + "GET", f"{GHCR}/v2/{repo}/manifests/{TAG}", bearer + ) + if status != 200: + raise RuntimeError(f"GET GHCR manifest returned {status}: {raw[:200]!r}") + manifest = json.loads(raw.decode("utf-8")) + layer = manifest["layers"][0] + if layer["digest"] != f"sha256:{digest}": + raise RuntimeError(f"GHCR layer digest mismatch: {layer['digest']}") + blob_status, _headers, blob = ghcr_request( + "GET", f"{GHCR}/v2/{repo}/blobs/sha256:{digest}", bearer, accept="*/*" + ) + if blob_status != 200: + raise RuntimeError(f"GET GHCR blob returned {blob_status}") + if sha256_hex(blob) != digest: + raise RuntimeError("GHCR blob bytes do not match digest") + + +def run(command: list[str], cwd: Path, extra_env: dict[str, str] | None = None) -> str: + print(f"$ {' '.join(command)}", flush=True) + environment = os.environ.copy() + environment.update({"CI": "true", "ZED_PKG_INTERACTIVE": "false"}) + if extra_env: + environment.update(extra_env) + completed = subprocess.run( + command, + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(completed.stdout, end="", flush=True) + if completed.returncode != 0: + raise RuntimeError(f"command failed ({completed.returncode})") + return completed.stdout + + +def zed_install(zed: Path, work: Path, digest: str, token: str) -> None: + consumer = work / "consumer" + consumer.mkdir() + (consumer / ".zpkg.toml").write_text( + f'''[package] +org = "zed-pkg-test" +name = "ghcr-fallback-consumer" +version = "0.0.0" + +[package.repository] +vcs = "git" +url = "https://github.com/zed-pkg-test/ghcr-fallback-consumer" + +[install] +dir = "zed_modules" +adapter = "none" + +[dependencies] +"{ORG}/{PACKAGE_NAME}" = "={VERSION}" +''', + encoding="utf-8", + ) + dead = f"http://127.0.0.1:{free_port()}" + extra = { + "ZED_PKG_SOURCE_FALLBACK": "true", + "ZED_PKG_SOURCE_FALLBACK_ALLOW_LOOPBACK": "true", + "ZED_PKG_R2_PUBLIC_BASE": "http://127.0.0.1:1", + "ZED_PKG_GITHUB_TOKEN": token, + } + home = work / "zed-home" + run( + [ + str(zed), + "--registry", + dead, + "--home", + str(home), + "install", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + ], + cwd=consumer, + extra_env=extra, + ) + matches = list(consumer.rglob("payload.txt")) + if not matches or matches[0].read_text(encoding="utf-8") != PAYLOAD: + raise AssertionError("zed install did not restore the GHCR payload") + lock_path = consumer / ".zpkg.lock" + if not lock_path.is_file(): + raise AssertionError("install did not write a lockfile") + lock_bytes = lock_path.read_bytes() + shutil.rmtree(home) + if (consumer / "zed_modules").exists(): + shutil.rmtree(consumer / "zed_modules") + run( + [ + str(zed), + "--registry", + dead, + "--home", + str(work / "zed-home-frozen"), + "install", + "--frozen", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + ], + cwd=consumer, + extra_env=extra, + ) + if lock_path.read_bytes() != lock_bytes: + raise AssertionError("frozen GHCR install mutated the lockfile") + restored = list(consumer.rglob("payload.txt")) + if not restored or restored[0].read_text(encoding="utf-8") != PAYLOAD: + raise AssertionError("frozen GHCR install did not restore the payload") + print( + json.dumps( + {"ok": True, "phase": "zed-install", "registry": dead, "sha256": digest}, + indent=2, + ), + flush=True, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--work-root", type=Path, required=True) + parser.add_argument("--zed", type=Path) + parser.add_argument("--skip-network", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.skip_network: + print(json.dumps({"ok": True, "phase": "contract"}), flush=True) + return 0 + work = args.work_root.resolve() + if work.exists(): + raise AssertionError(f"work root must be fresh: {work}") + work.mkdir(parents=True) + token = token_from_env() + repo = ensure_repo(token) + ensure_manifest_on_default_branch(token, repo) + tarball, _path = pack_canary(work) + digest = sha256_hex(tarball) + published = push_ghcr(token, tarball) + if published["sha256"] != digest: + raise AssertionError("published digest drifted from packed tarball") + prove_ghcr_get(token, digest) + print(json.dumps({"ok": True, "phase": "ghcr-api", **published}, indent=2), flush=True) + if args.zed: + zed_install(args.zed, work, digest, token) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 + print(f"ghcr fallback canary failed: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/scripts/git_submodule_global_mode.py b/scripts/git_submodule_global_mode.py new file mode 100644 index 00000000..cc1485bf --- /dev/null +++ b/scripts/git_submodule_global_mode.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +"""Black-box acceptance for the shared Zed/Git-submodule compatibility switch.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import tomllib +from pathlib import Path +from typing import Mapping, Sequence + +MANIFEST = ".zpkg.toml" +LOCKFILE = ".zpkg.lock" +SUBMODULE_PATH = Path("vendor/client") +NESTED_PATH = Path("vendor/client/vendor/leaf/leaf.txt") + + +class Contract: + def __init__(self, zed: Path, work_root: Path) -> None: + self.zed = zed.resolve() + self.root = work_root.resolve() + self.registry = self.root / "registry" + self.homes = self.root / "homes" + self.repos = self.root / "repos" + self.runs = self.root / "runs" + self.evidence = self.root / "evidence" + self.log_path = self.evidence / "contract.log" + self.checks: list[str] = [] + + if not self.zed.is_file(): + raise RuntimeError(f"zed binary not found: {self.zed}") + if self.root.exists(): + raise RuntimeError(f"work root must not already exist: {self.root}") + for directory in ( + self.registry, + self.homes, + self.repos, + self.runs, + self.evidence, + ): + directory.mkdir(parents=True, exist_ok=True) + self.log_path.write_text("", encoding="utf-8") + + def environment(self, extra: Mapping[str, str] | None = None) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "ZED_PKG_GIT_SUBMODULES", + "ZED_PKG_TOKEN", + "ZED_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + ): + env.pop(key, None) + env.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "true", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "protocol.file.allow", + "GIT_CONFIG_VALUE_0": "always", + "GIT_AUTHOR_NAME": "Zed Contract", + "GIT_AUTHOR_EMAIL": "zed-contract@example.invalid", + "GIT_COMMITTER_NAME": "Zed Contract", + "GIT_COMMITTER_EMAIL": "zed-contract@example.invalid", + } + ) + if extra: + env.update(extra) + return env + + def log(self, text: str) -> None: + print(text, flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + def run( + self, + argv: Sequence[str | Path], + *, + cwd: Path | None = None, + extra_env: Mapping[str, str] | None = None, + should_fail: bool = False, + ) -> str: + command = [str(value) for value in argv] + rendered = " ".join(json.dumps(value) for value in command) + self.log(f"\n$ (cd {cwd or Path.cwd()} && {rendered})") + completed = subprocess.run( + command, + cwd=cwd, + env=self.environment(extra_env), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.stdout: + self.log(completed.stdout) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {rendered}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {rendered}\n" + f"{completed.stdout}" + ) + return completed.stdout + + def git(self, project: Path, *args: str) -> str: + return self.run(["git", "-C", project, *args], cwd=project) + + def zed_cmd( + self, + project: Path, + home_name: str, + *args: str, + extra_env: Mapping[str, str] | None = None, + should_fail: bool = False, + ) -> str: + home = self.homes / home_name + home.mkdir(parents=True, exist_ok=True) + return self.run( + [ + self.zed, + "--registry", + f"file://{self.registry}", + "--home", + home, + *args, + ], + cwd=project, + extra_env=extra_env, + should_fail=should_fail, + ) + + def init_repo(self, path: Path) -> None: + path.mkdir(parents=True) + self.run(["git", "init", "-b", "main", path], cwd=self.root) + self.git(path, "config", "user.name", "Zed Contract") + self.git(path, "config", "user.email", "zed-contract@example.invalid") + + def commit_all(self, path: Path, message: str) -> str: + self.git(path, "add", "-A") + self.git(path, "commit", "-m", message) + return self.git(path, "rev-parse", "HEAD").strip() + + @staticmethod + def write_package(path: Path, org: str, name: str) -> None: + (path / MANIFEST).write_text( + f'''[package] +org = "{org}" +name = "{name}" +version = "1.2.3" + +[package.repository] +vcs = "git" +url = "https://example.invalid/{org}/{name}.git" +''', + encoding="utf-8", + ) + + def add_submodule(self, project: Path, source: Path, destination: str) -> None: + self.run( + [ + "git", + "-C", + project, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + source, + destination, + ], + cwd=project, + ) + + def clone_without_submodules(self, source: Path, destination: Path) -> None: + self.run( + ["git", "clone", "--no-recurse-submodules", source, destination], + cwd=self.root, + ) + self.git(destination, "config", "user.name", "Zed Contract") + self.git(destination, "config", "user.email", "zed-contract@example.invalid") + + def build_fixture(self) -> tuple[Path, str]: + leaf = self.repos / "leaf" + self.init_repo(leaf) + (leaf / "leaf.txt").write_text("nested leaf\n", encoding="utf-8") + self.commit_all(leaf, "leaf") + + child = self.repos / "client" + self.init_repo(child) + self.write_package(child, "acme", "client") + (child / "payload.txt").write_text("client payload\n", encoding="utf-8") + self.add_submodule(child, leaf, "vendor/leaf") + child_commit = self.commit_all(child, "client with nested leaf") + + root = self.repos / "root" + self.init_repo(root) + self.write_package(root, "acme", "root") + self.add_submodule(root, child, SUBMODULE_PATH.as_posix()) + self.commit_all(root, "root with package submodule") + return root, child_commit + + @staticmethod + def assert_uninitialized(project: Path) -> None: + assert not (project / SUBMODULE_PATH / "payload.txt").exists() + assert not (project / NESTED_PATH).exists() + + @staticmethod + def assert_initialized(project: Path) -> None: + assert (project / SUBMODULE_PATH / "payload.txt").is_file() + assert (project / NESTED_PATH).is_file() + + def certify_install_switch(self, root_source: Path) -> None: + default_off = self.runs / "install-default-off" + self.clone_without_submodules(root_source, default_off) + self.zed_cmd(default_off, "install-default-off", "install") + self.assert_uninitialized(default_off) + self.checks.append("install leaves Git transport disabled by default") + + prefix_on = self.runs / "install-prefix-on" + self.clone_without_submodules(root_source, prefix_on) + self.zed_cmd( + prefix_on, + "install-prefix-on", + "--git-submodules", + "install", + extra_env={"ZED_PKG_GIT_SUBMODULES": "definitely-invalid"}, + ) + self.assert_initialized(prefix_on) + self.checks.append( + "global prefix flag enables recursive install and overrides invalid environment" + ) + + suffix_off = self.runs / "install-suffix-off" + self.clone_without_submodules(root_source, suffix_off) + self.zed_cmd( + suffix_off, + "install-suffix-off", + "install", + "--git-submodules=false", + extra_env={"ZED_PKG_GIT_SUBMODULES": "ON"}, + ) + self.assert_uninitialized(suffix_off) + self.checks.append("explicit false overrides inherited install mode") + + environment_on = self.runs / "install-environment-on" + self.clone_without_submodules(root_source, environment_on) + self.zed_cmd( + environment_on, + "install-environment-on", + "install", + extra_env={"ZED_PKG_GIT_SUBMODULES": "yes"}, + ) + self.assert_initialized(environment_on) + self.checks.append("environment-only install mode initializes recursively") + + def certify_overtake_switch(self, root_source: Path, child_commit: str) -> None: + project = self.runs / "overtake" + self.clone_without_submodules(root_source, project) + + output = self.zed_cmd( + project, + "overtake", + "--git-submodules", + "overtake", + ) + assert "overtook 1 Git submodule package(s)" in output + self.assert_initialized(project) + + manifest_before = (project / MANIFEST).read_bytes() + lock_before = (project / LOCKFILE).read_bytes() + manifest = tomllib.loads(manifest_before.decode("utf-8")) + assert manifest["dependencies"]["acme/client"] == "=1.2.3" + assert SUBMODULE_PATH.as_posix() in manifest["workspace"]["members"] + + lock = tomllib.loads(lock_before.decode("utf-8")) + entries = lock.get("git-submodule") + assert isinstance(entries, list) and len(entries) == 1 + entry = entries[0] + assert entry["path"] == SUBMODULE_PATH.as_posix() + assert entry["package"] == "acme/client" + assert entry["version"] == "1.2.3" + assert entry["commit"] == child_commit + assert len(entry["sha256"]) == 64 + int(entry["sha256"], 16) + assert entry["size"] > 0 + self.checks.append("global prefix takeover adopts .gitmodules into manifest and lock") + + self.zed_cmd( + project, + "overtake", + "overtake", + "--git-submodules", + ) + assert (project / MANIFEST).read_bytes() == manifest_before + assert (project / LOCKFILE).read_bytes() == lock_before + self.checks.append("command suffix takeover is idempotent") + + self.zed_cmd( + project, + "overtake", + "overtake", + extra_env={"ZED_PKG_GIT_SUBMODULES": "on"}, + ) + assert (project / MANIFEST).read_bytes() == manifest_before + assert (project / LOCKFILE).read_bytes() == lock_before + self.checks.append("environment-only takeover is idempotent") + + failure = self.zed_cmd( + project, + "overtake", + "overtake", + "--git-submodules=false", + extra_env={"ZED_PKG_GIT_SUBMODULES": "true"}, + should_fail=True, + ) + assert "no takeover source selected" in failure + assert (project / MANIFEST).read_bytes() == manifest_before + assert (project / LOCKFILE).read_bytes() == lock_before + self.checks.append("explicit false disables takeover without mutating Zed state") + + (self.evidence / "adopted-manifest.toml").write_bytes(manifest_before) + (self.evidence / "adopted-lock.toml").write_bytes(lock_before) + + def certify_help_surface(self) -> None: + root_help = self.run([self.zed, "--help"]) + assert "--git-submodules" in root_help + assert "overtake" in root_help + + takeover_help = self.run([self.zed, "overtake", "--help"]) + assert "--git-submodules" in takeover_help + assert "ZED_PKG_GIT_SUBMODULES" in takeover_help + self.checks.append("root and takeover help expose the shared compatibility switch") + + def write_evidence(self) -> None: + evidence = { + "schema": "zed.git-submodule-global-mode-evidence/v1", + "checks": self.checks, + "network_credentials": False, + "public_registry_mutation": False, + } + (self.evidence / "evidence.json").write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + self.log(f"\ncompleted {len(self.checks)} checks") + + def certify(self) -> None: + root_source, child_commit = self.build_fixture() + self.certify_install_switch(root_source) + self.certify_overtake_switch(root_source, child_commit) + self.certify_help_surface() + self.write_evidence() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--work-root", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + Contract(args.zed, args.work_root).certify() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/git_submodule_install_operation_lock.py b/scripts/git_submodule_install_operation_lock.py new file mode 100644 index 00000000..4ba58f3b --- /dev/null +++ b/scripts/git_submodule_install_operation_lock.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Certify `zed install --git-submodules` owns one complete mutation boundary.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path +from typing import Sequence + +from git_submodule_overtake_lock import Contract, ContractError + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args(argv) + + +def require_no_nested_state(nested: Path) -> None: + forbidden = ( + nested / ".zed/operation.lock", + nested / ".zpkg.toml", + nested / ".zpkg.lock", + nested / ".zpkg-staging", + ) + present = [str(path) for path in forbidden if path.exists()] + if present: + raise ContractError( + "nested cooperative install published a second project identity: " + + ", ".join(present) + ) + + +def main(argv: Sequence[str]) -> int: + args = parse_args(argv) + contract = Contract(args.zed, args.work_root) + _, root = contract.fixture("cooperative-install") + nested = root / "packages/client/src" + nested.mkdir(parents=True) + + ready = contract.runs / "install.ready" + release = contract.runs / "install.release" + path = f"{contract.shim_dir}{os.pathsep}{os.environ.get('PATH', '')}" + holder_env = { + "PATH": path, + "ZED_TEST_BLOCK_SUBMODULE_SYNC": "1", + "ZED_TEST_READY": str(ready), + "ZED_TEST_RELEASE": str(release), + } + holder = contract.start( + contract.zed_command("install-holder", "install", "--git-submodules"), + cwd=nested, + env=holder_env, + ) + contract.wait_ready(ready, holder) + + lock_path = root / ".zed/operation.lock" + if not lock_path.is_file(): + raise ContractError( + "nested cooperative install reached Git sync before the superproject owned operation.lock" + ) + require_no_nested_state(nested) + if (root / "vendor/child/.zpkg.toml").exists(): + raise ContractError("Git submodule update ran before the test release") + if (root / ".zpkg.lock").exists(): + raise ContractError("install published a lockfile before Git sync was released") + + started = time.monotonic() + waiter = contract.start( + contract.zed_command("install-waiter", "install", "--frozen"), + cwd=root, + ) + contract.require_blocked(waiter) + blocked_for = time.monotonic() - started + if (root / ".zpkg.lock").exists(): + raise ContractError("frozen waiter raced ahead of cooperative install") + require_no_nested_state(nested) + + release.write_text("release\n", encoding="utf-8") + holder_output = contract.finish(holder, expected=0) + waiter_output = contract.finish(waiter, expected=0) + if "synchronized 1 Git submodule(s)" not in holder_output: + raise ContractError("cooperative install did not report submodule synchronization") + if "error:" in holder_output.lower() or "error:" in waiter_output.lower(): + raise ContractError("serialized install process reported an error") + if not (root / ".zpkg.lock").is_file(): + raise ContractError("cooperative install did not publish a root lockfile") + if not (root / "vendor/child/.zpkg.toml").is_file(): + raise ContractError("cooperative install did not initialize the child submodule") + if ( + root / "vendor/child/payload.txt" + ).read_text(encoding="utf-8") != "payload for cooperative-install\n": + raise ContractError("initialized child payload did not match the pinned gitlink") + require_no_nested_state(nested) + + checks = [ + "nested cooperative install owns the superproject lock before Git sync", + "nested invocation creates no second lock, manifest, lockfile, or staging identity", + "different-home frozen install blocks across Git sync and installation", + "blocked frozen install succeeds only after complete root state publication", + ] + evidence = { + "schema": "zed.git-submodule-install-operation-lock/v1", + "checks": checks, + "holder_pid": holder.pid, + "waiter_pid": waiter.pid, + "holder_cwd": str(nested), + "authority_root": str(root), + "observed_block_seconds": round(blocked_for, 3), + "network_credentials": False, + "public_registry_mutation": False, + } + (contract.evidence / "install-evidence.json").write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + contract.log("\ncertified 4 cooperative-install ownership checks") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/git_submodule_interop.py b/scripts/git_submodule_interop.py new file mode 100644 index 00000000..a4b40fb9 --- /dev/null +++ b/scripts/git_submodule_interop.py @@ -0,0 +1,594 @@ +#!/usr/bin/env python3 +"""Credential-free black-box certification for Zed/Git-submodule interop.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tomllib +from pathlib import Path +from typing import Mapping, Sequence + +MANIFEST = ".zpkg.toml" +LOCKFILE = ".zpkg.lock" +STAGING = ".zpkg-staging" + + +class Contract: + def __init__(self, zed: Path, work_root: Path) -> None: + self.zed = zed.resolve() + self.root = work_root.resolve() + self.registry = self.root / "registry" + self.homes = self.root / "homes" + self.repos = self.root / "repos" + self.runs = self.root / "runs" + self.evidence = self.root / "evidence" + self.log_path = self.evidence / "contract.log" + self.checks: list[str] = [] + + if not self.zed.is_file(): + raise RuntimeError(f"zed binary not found: {self.zed}") + if self.root.exists(): + raise RuntimeError(f"work root must not already exist: {self.root}") + for directory in ( + self.registry, + self.homes, + self.repos, + self.runs, + self.evidence, + ): + directory.mkdir(parents=True, exist_ok=True) + self.log_path.write_text("", encoding="utf-8") + + def env(self, extra: Mapping[str, str] | None = None) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "ZED_PKG_GIT_SUBMODULES", + "ZED_PKG_TOKEN", + "ZED_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + ): + env.pop(key, None) + env.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "true", + "GIT_AUTHOR_NAME": "Zed Contract", + "GIT_AUTHOR_EMAIL": "zed-contract@example.invalid", + "GIT_COMMITTER_NAME": "Zed Contract", + "GIT_COMMITTER_EMAIL": "zed-contract@example.invalid", + # Git rejects local-path submodule transports by default. This + # one-process override keeps every fixture local and explicit. + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "protocol.file.allow", + "GIT_CONFIG_VALUE_0": "always", + } + ) + if extra: + env.update(extra) + return env + + def log(self, text: str) -> None: + print(text, flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + def run( + self, + argv: Sequence[str | Path], + *, + cwd: Path | None = None, + should_fail: bool = False, + extra_env: Mapping[str, str] | None = None, + ) -> str: + command = [str(value) for value in argv] + shown = " ".join(json.dumps(value) for value in command) + self.log(f"\n$ (cd {cwd or Path.cwd()} && {shown})") + completed = subprocess.run( + command, + cwd=cwd, + env=self.env(extra_env), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.stdout: + self.log(completed.stdout) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {shown}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {shown}\n" + f"{completed.stdout}" + ) + return completed.stdout + + def git(self, cwd: Path, *args: str, should_fail: bool = False) -> str: + return self.run(["git", "-C", cwd, *args], cwd=cwd, should_fail=should_fail) + + def zed_cmd( + self, + cwd: Path, + home_name: str, + *args: str, + should_fail: bool = False, + extra_env: Mapping[str, str] | None = None, + ) -> str: + home = self.homes / home_name + home.mkdir(parents=True, exist_ok=True) + return self.run( + [ + self.zed, + "--registry", + f"file://{self.registry}", + "--home", + home, + *args, + ], + cwd=cwd, + should_fail=should_fail, + extra_env=extra_env, + ) + + def init_repo(self, path: Path) -> None: + path.mkdir(parents=True) + self.run(["git", "init", "-b", "main", path], cwd=self.root) + self.git(path, "config", "user.name", "Zed Contract") + self.git(path, "config", "user.email", "zed-contract@example.invalid") + + def commit_all(self, path: Path, message: str) -> str: + self.git(path, "add", "-A") + self.git(path, "commit", "-m", message) + return self.git(path, "rev-parse", "HEAD").strip() + + @staticmethod + def write_package( + path: Path, + *, + org: str, + name: str, + version: str = "1.2.3", + dependency: str | None = None, + ) -> bytes: + text = ( + f'[package]\norg = "{org}"\nname = "{name}"\n' + f'version = "{version}"\n\n' + "[package.repository]\n" + 'vcs = "git"\n' + f'url = "https://example.invalid/{org}/{name}.git"\n' + ) + if dependency: + text += f'\n[dependencies]\n"{dependency}" = "=1.0.0"\n' + data = text.encode("utf-8") + (path / MANIFEST).write_bytes(data) + return data + + def add_submodule(self, root: Path, source: Path, destination: str) -> None: + self.run( + [ + "git", + "-C", + root, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + str(source), + destination, + ], + cwd=root, + ) + + def clone_no_submodules(self, source: Path, destination: Path) -> None: + self.run(["git", "clone", "--no-recurse-submodules", source, destination]) + self.git(destination, "config", "user.name", "Zed Contract") + self.git(destination, "config", "user.email", "zed-contract@example.invalid") + + @staticmethod + def assert_no_install_state(project: Path) -> None: + assert not (project / LOCKFILE).exists(), "failed takeover left a lockfile" + assert not (project / "zed_modules").exists(), "failed takeover left modules" + assert not (project / STAGING).exists(), "failed takeover left staging state" + + def build_fixture_graph(self) -> tuple[Path, Path, Path, str]: + schema = self.repos / "schema" + self.init_repo(schema) + (schema / "schema.txt").write_text("schema-v1\n", encoding="utf-8") + self.commit_all(schema, "schema") + + child = self.repos / "client" + self.init_repo(child) + self.write_package(child, org="acme", name="client") + (child / "lib.txt").write_text("hello from client\n", encoding="utf-8") + self.add_submodule(child, schema, "vendor/schema") + child_commit = self.commit_all(child, "client with nested schema") + + root = self.repos / "root" + self.init_repo(root) + self.write_package(root, org="acme", name="root") + self.add_submodule(root, child, "vendor/client") + self.commit_all(root, "root with client") + return schema, child, root, child_commit + + def certify_cooperative_install(self, root_source: Path) -> None: + disabled = self.runs / "disabled" + self.clone_no_submodules(root_source, disabled) + assert not (disabled / "vendor/client" / MANIFEST).exists() + self.zed_cmd( + disabled, + "disabled", + "install", + "--git-submodules=false", + extra_env={"ZED_PKG_GIT_SUBMODULES": "1"}, + ) + assert not (disabled / "vendor/client" / MANIFEST).exists(), ( + "explicit false did not override inherited submodule mode" + ) + self.checks.append("explicit false overrides inherited submodule mode") + + cooperative = self.runs / "cooperative" + self.clone_no_submodules(root_source, cooperative) + self.zed_cmd(cooperative, "cooperative", "install", "--git-submodules") + assert (cooperative / "vendor/client" / "lib.txt").is_file() + assert ( + cooperative / "vendor/client/vendor/schema/schema.txt" + ).read_text(encoding="utf-8") == "schema-v1\n" + self.checks.append("cooperative install initializes recursive submodules") + + def certify_takeover_and_frozen_replay( + self, root_source: Path, child_commit: str + ) -> None: + adopted = self.runs / "adopted" + self.clone_no_submodules(root_source, adopted) + self.zed_cmd(adopted, "adopted", "overtake", "--git-submodules") + + manifest = tomllib.loads((adopted / MANIFEST).read_text(encoding="utf-8")) + assert manifest["dependencies"]["acme/client"] == "=1.2.3" + assert "vendor/client" in manifest["workspace"]["members"] + + lock_bytes = (adopted / LOCKFILE).read_bytes() + lock = tomllib.loads(lock_bytes.decode("utf-8")) + entries = lock.get("git-submodule") + assert isinstance(entries, list) and len(entries) == 1 + entry = entries[0] + assert entry["path"] == "vendor/client" + assert entry["package"] == "acme/client" + assert entry["version"] == "1.2.3" + assert entry["commit"] == child_commit + assert len(entry["sha256"]) == 64 + int(entry["sha256"], 16) + assert entry["size"] > 0 + assert (adopted / "zed_modules/acme/client/lib.txt").is_file() + self.checks.append("takeover writes workspace, dependency, and Git provenance") + + self.git(adopted, "add", MANIFEST, LOCKFILE) + self.git(adopted, "commit", "-m", "adopt submodule into Zed authority") + + frozen = self.runs / "frozen" + self.clone_no_submodules(adopted, frozen) + before = (frozen / LOCKFILE).read_bytes() + self.zed_cmd( + frozen, + "frozen", + "install", + "--git-submodules", + "--frozen", + ) + assert (frozen / LOCKFILE).read_bytes() == before + assert (frozen / "vendor/client/vendor/schema/schema.txt").is_file() + assert (frozen / "zed_modules/acme/client/lib.txt").is_file() + self.checks.append("fresh-clone frozen replay is lockfile-byte-stable") + + # Dirty content must fail before rewriting the immutable lock. + (frozen / "vendor/client/lib.txt").write_text( + "locally modified\n", encoding="utf-8" + ) + self.zed_cmd( + frozen, + "frozen", + "install", + "--git-submodules", + "--frozen", + should_fail=True, + ) + assert (frozen / LOCKFILE).read_bytes() == before + self.git(frozen / "vendor/client", "reset", "--hard", "HEAD") + self.checks.append("dirty submodule drift is rejected without lock mutation") + + # A committed .gitmodules branch change is transport-provenance drift. + self.git( + frozen, + "config", + "-f", + ".gitmodules", + "submodule.vendor/client.branch", + "alternate", + ) + self.git(frozen, "add", ".gitmodules") + self.git(frozen, "commit", "-m", "change submodule branch metadata") + self.zed_cmd( + frozen, + "frozen", + "install", + "--git-submodules", + "--frozen", + should_fail=True, + ) + assert (frozen / LOCKFILE).read_bytes() == before + self.checks.append("committed branch drift is rejected without lock mutation") + + (self.evidence / "adopted-lock.toml").write_bytes(lock_bytes) + + def build_mixed_fixture(self) -> Path: + zed_child = self.repos / "mixed-client" + self.init_repo(zed_child) + self.write_package(zed_child, org="acme", name="mixed-client") + (zed_child / "lib.txt").write_text("mixed Zed package\n", encoding="utf-8") + self.commit_all(zed_child, "mixed Zed package") + + legacy_child = self.repos / "legacy-docs" + self.init_repo(legacy_child) + (legacy_child / "README.md").write_text( + "ordinary Git submodule\n", encoding="utf-8" + ) + self.commit_all(legacy_child, "legacy documentation") + + root = self.repos / "mixed-root" + self.init_repo(root) + self.write_package(root, org="acme", name="mixed-root") + self.add_submodule(root, zed_child, "vendor/client") + self.add_submodule(root, legacy_child, "vendor/legacy") + self.commit_all(root, "mixed Git and Zed submodules") + return root + + def certify_mixed_repository_contract(self) -> None: + root_source = self.build_mixed_fixture() + adopted = self.runs / "mixed-adopted" + self.clone_no_submodules(root_source, adopted) + output = self.zed_cmd( + adopted, + "mixed-adopted", + "overtake", + "--git-submodules", + ) + assert "overtook 1 Git submodule package(s)" in output + assert "left 1 non-Zed submodule(s) under Git authority" in output + assert "vendor/legacy" in output + + manifest_before_commit = (adopted / MANIFEST).read_bytes() + manifest = tomllib.loads(manifest_before_commit.decode("utf-8")) + assert manifest["dependencies"]["acme/mixed-client"] == "=1.2.3" + assert "vendor/client" in manifest["workspace"]["members"] + assert "vendor/legacy" not in manifest["workspace"]["members"] + + lock_before_commit = (adopted / LOCKFILE).read_bytes() + lock = tomllib.loads(lock_before_commit.decode("utf-8")) + entries = lock.get("git-submodule") + assert isinstance(entries, list) and len(entries) == 1 + assert entries[0]["package"] == "acme/mixed-client" + assert entries[0]["path"] == "vendor/client" + assert "vendor/legacy" not in lock_before_commit.decode("utf-8") + assert (adopted / "zed_modules/acme/mixed-client/lib.txt").is_file() + assert (adopted / "vendor/legacy/README.md").is_file() + self.checks.append("mixed takeover adopts only explicit Zed packages") + + self.git(adopted, "add", MANIFEST, LOCKFILE) + self.git(adopted, "commit", "-m", "adopt only the Zed submodule") + + frozen = self.runs / "mixed-frozen" + self.clone_no_submodules(adopted, frozen) + manifest_before = (frozen / MANIFEST).read_bytes() + lock_before = (frozen / LOCKFILE).read_bytes() + assert not (frozen / "vendor/client/lib.txt").exists() + assert not (frozen / "vendor/legacy/README.md").exists() + + self.zed_cmd( + frozen, + "mixed-frozen", + "install", + "--frozen", + extra_env={"ZED_PKG_GIT_SUBMODULES": "yes"}, + ) + assert (frozen / MANIFEST).read_bytes() == manifest_before + assert (frozen / LOCKFILE).read_bytes() == lock_before + assert (frozen / "vendor/client/lib.txt").read_text( + encoding="utf-8" + ) == "mixed Zed package\n" + assert (frozen / "vendor/legacy/README.md").read_text( + encoding="utf-8" + ) == "ordinary Git submodule\n" + assert (frozen / "zed_modules/acme/mixed-client/lib.txt").read_text( + encoding="utf-8" + ) == "mixed Zed package\n" + recursive_status = self.git(frozen, "submodule", "status", "--recursive") + for line in recursive_status.splitlines(): + if line: + assert line[0] not in "-+U", recursive_status + self.checks.append( + "mixed frozen replay restores adopted and Git-only transports byte-exactly" + ) + + (self.evidence / "mixed-manifest.toml").write_bytes(manifest_before_commit) + (self.evidence / "mixed-lock.toml").write_bytes(lock_before_commit) + + def certify_git_only_boundary(self) -> None: + legacy_child = self.repos / "git-only-child" + self.init_repo(legacy_child) + (legacy_child / "README.md").write_text( + "ordinary Git submodule\n", encoding="utf-8" + ) + self.commit_all(legacy_child, "Git-only child") + + root = self.runs / "git-only-root" + self.init_repo(root) + original = self.write_package(root, org="acme", name="git-only-root") + self.add_submodule(root, legacy_child, "vendor/legacy") + self.commit_all(root, "Git-only submodule root") + self.git( + root, + "submodule", + "deinit", + "--force", + "--", + "vendor/legacy", + ) + assert not (root / "vendor/legacy/README.md").exists() + + output = self.zed_cmd( + root, + "git-only", + "overtake", + "--git-submodules", + should_fail=True, + ) + assert "no overtake-compatible Zed submodules" in output + assert "vendor/legacy" in output + assert (root / MANIFEST).read_bytes() == original + self.assert_no_install_state(root) + assert (root / "vendor/legacy/README.md").read_text( + encoding="utf-8" + ) == "ordinary Git submodule\n" + self.checks.append( + "Git-only takeover synchronizes transport without publishing Zed state" + ) + + def certify_invalid_manifest_boundary(self) -> None: + invalid_child = self.repos / "invalid-manifest-child" + self.init_repo(invalid_child) + (invalid_child / MANIFEST).write_text( + "[package]\nname = [this is not valid TOML\n", + encoding="utf-8", + ) + self.commit_all(invalid_child, "invalid package intent") + + root = self.runs / "invalid-manifest-root" + self.init_repo(root) + original = self.write_package(root, org="acme", name="invalid-root") + self.add_submodule(root, invalid_child, "vendor/invalid") + self.commit_all(root, "invalid package submodule root") + + output = self.zed_cmd( + root, + "invalid-manifest", + "overtake", + "--git-submodules", + should_fail=True, + ) + assert "contains an invalid .zpkg.toml" in output + assert (root / MANIFEST).read_bytes() == original + self.assert_no_install_state(root) + self.checks.append("invalid package intent fails closed before Zed mutation") + + def build_broken_child(self) -> Path: + broken = self.repos / "broken-client" + self.init_repo(broken) + self.write_package( + broken, + org="acme", + name="broken-client", + dependency="acme/missing", + ) + (broken / "lib.txt").write_text("broken dependency graph\n", encoding="utf-8") + self.commit_all(broken, "client with unresolved dependency") + return broken + + def certify_failure_atomic_takeover(self, broken_child: Path) -> None: + existing = self.runs / "rollback-existing" + self.init_repo(existing) + original = ( + b"# preserve these exact bytes on failure\n" + b"[package]\n" + b'org = "acme"\n' + b'name = "rollback-root"\n' + b'version = "1.2.3"\n\n' + b"[package.repository]\n" + b'vcs = "git"\n' + b'url = "https://example.invalid/acme/rollback-root.git"\n' + ) + (existing / MANIFEST).write_bytes(original) + self.add_submodule(existing, broken_child, "vendor/client") + self.commit_all(existing, "rollback root") + output = self.zed_cmd( + existing, + "rollback-existing", + "overtake", + "--git-submodules", + should_fail=True, + ) + assert "restored the prior root manifest" in output + assert (existing / MANIFEST).read_bytes() == original + self.assert_no_install_state(existing) + self.checks.append("failed takeover restores exact authored manifest bytes") + + generated = self.runs / "rollback-generated" + self.init_repo(generated) + self.add_submodule(generated, broken_child, "vendor/client") + self.commit_all(generated, "manifestless rollback root") + output = self.zed_cmd( + generated, + "rollback-generated", + "overtake", + "--git-submodules", + should_fail=True, + ) + assert "restored the prior root manifest" in output + assert not (generated / MANIFEST).exists() + self.assert_no_install_state(generated) + self.checks.append("failed takeover removes a generated root manifest") + + def finish(self) -> None: + version = self.run([self.zed, "--version"]).strip() + zed_sha256 = hashlib.sha256(self.zed.read_bytes()).hexdigest() + record = { + "schema": "zed.git-submodule-interop-evidence/v2", + "zed_version": version, + "zed_sha256": zed_sha256, + "checks": self.checks, + "network_credentials": False, + "public_registry_mutation": False, + } + (self.evidence / "evidence.json").write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + assert len(self.checks) == 12, self.checks + self.log(f"\ncertified {len(self.checks)} Git-submodule interoperability checks") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + contract = Contract(args.zed, args.work_root) + _, _, root, child_commit = contract.build_fixture_graph() + contract.certify_cooperative_install(root) + contract.certify_takeover_and_frozen_replay(root, child_commit) + contract.certify_mixed_repository_contract() + contract.certify_git_only_boundary() + contract.certify_invalid_manifest_boundary() + broken = contract.build_broken_child() + contract.certify_failure_atomic_takeover(broken) + contract.finish() + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error: + print(f"git-submodule interoperability contract failed: {error}", file=sys.stderr) + raise diff --git a/scripts/git_submodule_overtake_lock.py b/scripts/git_submodule_overtake_lock.py new file mode 100644 index 00000000..cd14b9e9 --- /dev/null +++ b/scripts/git_submodule_overtake_lock.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +"""Black-box certification for checkout-local Git-submodule takeover locking.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import signal +import subprocess +import sys +import time +import tomllib +from pathlib import Path +from typing import Mapping, Sequence + + +class ContractError(RuntimeError): + """Raised when an observable process-lock contract is violated.""" + + +class Contract: + def __init__(self, zed: Path, work_root: Path) -> None: + if os.name == "nt": + raise ContractError("this process-group canary requires a POSIX host") + + self.zed = zed.resolve() + self.root = work_root.resolve() + self.real_git = Path(shutil.which("git") or "").resolve() + if not self.zed.is_file(): + raise ContractError(f"zed executable not found: {self.zed}") + if not self.real_git.is_file(): + raise ContractError("git executable not found") + if self.root.exists(): + raise ContractError(f"work root must not already exist: {self.root}") + + self.registry = self.root / "registry" + self.homes = self.root / "homes" + self.repos = self.root / "repos" + self.runs = self.root / "runs" + self.shim_dir = self.root / "git-shim" + self.evidence = self.root / "evidence" + for directory in ( + self.registry, + self.homes, + self.repos, + self.runs, + self.shim_dir, + self.evidence, + ): + directory.mkdir(parents=True, exist_ok=True) + + self.log_path = self.evidence / "contract.log" + self.log_path.write_text("", encoding="utf-8") + self.checks: list[str] = [] + self._write_git_shim() + + def _write_git_shim(self) -> None: + shim = self.shim_dir / "git" + shim.write_text( + f"""#!{sys.executable} +import os +import pathlib +import sys +import time + +args = sys.argv[1:] +if ( + os.environ.get("ZED_TEST_BLOCK_SUBMODULE_SYNC") == "1" + and "submodule" in args + and "sync" in args +): + ready = pathlib.Path(os.environ["ZED_TEST_READY"]) + release = pathlib.Path(os.environ["ZED_TEST_RELEASE"]) + ready.write_text("ready\\n", encoding="utf-8") + deadline = time.monotonic() + 120.0 + while not release.exists(): + if time.monotonic() >= deadline: + print("timed out waiting for test release", file=sys.stderr) + raise SystemExit(98) + time.sleep(0.02) + +os.execv({str(self.real_git)!r}, [{str(self.real_git)!r}, *args]) +""", + encoding="utf-8", + ) + shim.chmod(0o755) + + def log(self, text: str) -> None: + print(text, flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + def base_env(self, home: Path) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "ZED_PKG_GIT_SUBMODULES", + "ZED_PKG_TOKEN", + "ZED_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + "CLOUDFLARE_API_TOKEN", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + ): + env.pop(key, None) + home.mkdir(parents=True, exist_ok=True) + env.update( + { + "HOME": str(home), + "USERPROFILE": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "ZED_PKG_HOME": str(home / ".zed-pkg"), + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "true", + "GIT_AUTHOR_NAME": "Zed Lock Contract", + "GIT_AUTHOR_EMAIL": "zed-lock-contract@example.invalid", + "GIT_COMMITTER_NAME": "Zed Lock Contract", + "GIT_COMMITTER_EMAIL": "zed-lock-contract@example.invalid", + } + ) + return env + + def run( + self, + argv: Sequence[str | Path], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, + expect: int = 0, + ) -> subprocess.CompletedProcess[str]: + command = [str(value) for value in argv] + merged = self.base_env(self.homes / "setup") + if env: + merged.update(env) + self.log(f"\n$ (cd {cwd or Path.cwd()} && {' '.join(command)})") + completed = subprocess.run( + command, + cwd=cwd, + env=merged, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.stdout: + self.log(completed.stdout) + if completed.returncode != expect: + raise ContractError( + f"command returned {completed.returncode}, expected {expect}: " + f"{' '.join(command)}\n{completed.stdout}" + ) + return completed + + def git(self, project: Path, *args: str) -> str: + return self.run([self.real_git, "-C", project, *args], cwd=project).stdout + + def init_repo(self, project: Path) -> None: + project.mkdir(parents=True) + self.run([self.real_git, "init", "-b", "main", project], cwd=self.root) + self.git(project, "config", "user.name", "Zed Lock Contract") + self.git(project, "config", "user.email", "zed-lock-contract@example.invalid") + + def commit_all(self, project: Path, message: str) -> str: + self.git(project, "add", "-A") + self.git(project, "commit", "-m", message) + return self.git(project, "rev-parse", "HEAD").strip() + + @staticmethod + def write_package(project: Path, name: str) -> bytes: + payload = f"""[package] +org = "acme" +name = "{name}" +version = "1.2.3" + +[package.repository] +vcs = "git" +url = "https://example.invalid/acme/{name}.git" +""".encode() + (project / ".zpkg.toml").write_bytes(payload) + return payload + + def fixture(self, name: str) -> tuple[Path, Path]: + child = self.repos / f"{name}-child" + self.init_repo(child) + self.write_package(child, f"{name}-child") + (child / "payload.txt").write_text( + f"payload for {name}\n", encoding="utf-8" + ) + self.commit_all(child, "child package") + + root = self.repos / f"{name}-root" + self.init_repo(root) + self.write_package(root, f"{name}-root") + self.git(root, "config", "protocol.file.allow", "always") + self.run( + [ + self.real_git, + "-C", + root, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + child, + "vendor/child", + ], + cwd=root, + ) + self.commit_all(root, "root with child submodule") + self.git(root, "submodule", "deinit", "--force", "--", "vendor/child") + return child, root + + def zed_command(self, home_name: str, *args: str) -> list[str]: + home = self.homes / home_name + home.mkdir(parents=True, exist_ok=True) + return [ + str(self.zed), + "--registry", + f"file://{self.registry}", + "--home", + str(home), + *args, + ] + + def start( + self, + command: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str] | None = None, + ) -> subprocess.Popen[str]: + merged = self.base_env(self.homes / "process") + if env: + merged.update(env) + self.log(f"\n$ background (cd {cwd} && {' '.join(command)})") + return subprocess.Popen( + list(command), + cwd=cwd, + env=merged, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + + def wait_ready( + self, ready: Path, process: subprocess.Popen[str], timeout: float = 30.0 + ) -> None: + deadline = time.monotonic() + timeout + while not ready.exists(): + if process.poll() is not None: + output = process.communicate()[0] + raise ContractError( + f"holder exited before reaching blocked Git sync: {process.returncode}\n{output}" + ) + if time.monotonic() >= deadline: + raise ContractError("timed out waiting for holder to reach Git sync") + time.sleep(0.02) + + @staticmethod + def require_blocked(process: subprocess.Popen[str], duration: float = 0.8) -> None: + deadline = time.monotonic() + duration + while time.monotonic() < deadline: + if process.poll() is not None: + output = process.communicate()[0] + raise ContractError( + f"contending mutation exited instead of blocking: " + f"{process.returncode}\n{output}" + ) + time.sleep(0.02) + + def finish( + self, + process: subprocess.Popen[str], + *, + expected: int, + timeout: float = 60.0, + ) -> str: + try: + output, _ = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as error: + os.killpg(process.pid, signal.SIGKILL) + output, _ = process.communicate() + raise ContractError(f"process timed out\n{output}") from error + if output: + self.log(output) + if process.returncode != expected: + raise ContractError( + f"process returned {process.returncode}, expected {expected}\n{output}" + ) + return output + + def blocked_holder( + self, root: Path, scenario: str + ) -> tuple[subprocess.Popen[str], Path, Path]: + ready = self.runs / f"{scenario}.ready" + release = self.runs / f"{scenario}.release" + path = f"{self.shim_dir}{os.pathsep}{os.environ.get('PATH', '')}" + env = { + "PATH": path, + "ZED_TEST_BLOCK_SUBMODULE_SYNC": "1", + "ZED_TEST_READY": str(ready), + "ZED_TEST_RELEASE": str(release), + } + process = self.start( + self.zed_command( + f"{scenario}-holder", "overtake", "--git-submodules" + ), + cwd=root, + env=env, + ) + self.wait_ready(ready, process) + lock_path = root / ".zed/operation.lock" + if not lock_path.is_file(): + raise ContractError( + "overtake reached Git transport before publishing its operation lock" + ) + return process, ready, release + + def certify_normal_serialization(self) -> dict[str, float | int]: + _, root = self.fixture("normal") + alias = self.runs / "normal-root-alias" + alias.symlink_to(root, target_is_directory=True) + + holder, _, release = self.blocked_holder(root, "normal") + started = time.monotonic() + waiter = self.start( + self.zed_command("normal-waiter", "install", "--frozen"), + cwd=alias, + ) + self.require_blocked(waiter) + blocked_for = time.monotonic() - started + if (root / ".zpkg.lock").exists(): + raise ContractError("waiter observed a lockfile before takeover was released") + + release.write_text("release\n", encoding="utf-8") + holder_output = self.finish(holder, expected=0) + waiter_output = self.finish(waiter, expected=0) + if "overtook 1 Git submodule package(s)" not in holder_output: + raise ContractError("successful holder did not report one adopted package") + if not (root / ".zpkg.lock").is_file(): + raise ContractError("takeover did not publish the lockfile") + if not (root / "zed_modules/acme/normal-child/payload.txt").is_file(): + raise ContractError("serialized frozen install did not materialize the package") + if "error:" in waiter_output.lower(): + raise ContractError("serialized waiter reported an error") + + self.checks.extend( + [ + "overtake owns operation.lock before Git transport begins", + "symlink-alias install blocks behind active takeover", + "blocked frozen install succeeds after takeover publishes lock state", + ] + ) + return { + "holder_pid": holder.pid, + "waiter_pid": waiter.pid, + "observed_block_seconds": round(blocked_for, 3), + } + + def certify_owner_termination_release(self) -> dict[str, float | int]: + _, root = self.fixture("terminated") + original_manifest = (root / ".zpkg.toml").read_bytes() + holder, _, _ = self.blocked_holder(root, "terminated") + + waiter = self.start( + self.zed_command("terminated-waiter", "install"), + cwd=root, + ) + started = time.monotonic() + self.require_blocked(waiter) + blocked_for = time.monotonic() - started + + os.killpg(holder.pid, signal.SIGTERM) + holder_output = self.finish(holder, expected=-signal.SIGTERM) + waiter_output = self.finish(waiter, expected=0) + if holder_output: + self.log(holder_output) + if "error:" in waiter_output.lower(): + raise ContractError("waiter reported an error after owner termination") + if (root / ".zpkg.toml").read_bytes() != original_manifest: + raise ContractError("terminated takeover changed root manifest bytes") + if (root / ".zpkg-staging").exists(): + raise ContractError("terminated pre-mutation takeover left recovery state") + manifest = tomllib.loads((root / ".zpkg.toml").read_text(encoding="utf-8")) + if manifest.get("dependencies"): + raise ContractError("terminated takeover left adopted dependency intent") + if (root / "vendor/child/.zpkg.toml").exists(): + raise ContractError("holder passed the intentionally blocked Git sync") + + self.checks.extend( + [ + "independent install blocks behind takeover owner", + "kernel lock releases when takeover owner process terminates", + "terminated pre-mutation takeover leaves authored manifest and recovery state intact", + ] + ) + return { + "holder_pid": holder.pid, + "waiter_pid": waiter.pid, + "observed_block_seconds": round(blocked_for, 3), + } + + def finish_evidence( + self, + normal: Mapping[str, float | int], + terminated: Mapping[str, float | int], + ) -> None: + if len(self.checks) != 6: + raise ContractError(f"unexpected check count: {self.checks}") + binary_sha = hashlib.sha256(self.zed.read_bytes()).hexdigest() + version = self.run([self.zed, "--version"]).stdout.strip() + evidence = { + "schema": "zed.git-submodule-overtake-operation-lock/v1", + "zed_version": version, + "zed_sha256": binary_sha, + "checks": self.checks, + "normal_completion": dict(normal), + "owner_termination": dict(terminated), + "network_credentials": False, + "public_registry_mutation": False, + } + (self.evidence / "evidence.json").write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + self.log(f"\ncertified {len(self.checks)} takeover operation-lock checks") + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args(argv) + + +def main(argv: Sequence[str]) -> int: + args = parse_args(argv) + contract = Contract(args.zed, args.work_root) + normal = contract.certify_normal_serialization() + terminated = contract.certify_owner_termination_release() + contract.finish_evidence(normal, terminated) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/git_submodule_overtake_nested_lock.py b/scripts/git_submodule_overtake_nested_lock.py new file mode 100644 index 00000000..a58ddd6e --- /dev/null +++ b/scripts/git_submodule_overtake_nested_lock.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Certify that nested takeover locks the discovered superproject root.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path +from typing import Sequence + +from git_submodule_overtake_lock import Contract, ContractError + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args(argv) + + +def main(argv: Sequence[str]) -> int: + args = parse_args(argv) + contract = Contract(args.zed, args.work_root) + _, root = contract.fixture("nested") + + nested = root / "packages/client/src" + nested.mkdir(parents=True) + ready = contract.runs / "nested.ready" + release = contract.runs / "nested.release" + path = f"{contract.shim_dir}{os.pathsep}{os.environ.get('PATH', '')}" + holder_env = { + "PATH": path, + "ZED_TEST_BLOCK_SUBMODULE_SYNC": "1", + "ZED_TEST_READY": str(ready), + "ZED_TEST_RELEASE": str(release), + } + + holder = contract.start( + contract.zed_command( + "nested-holder", "overtake", "--git-submodules" + ), + cwd=nested, + env=holder_env, + ) + contract.wait_ready(ready, holder) + + root_lock = root / ".zed/operation.lock" + nested_lock = nested / ".zed/operation.lock" + if not root_lock.is_file(): + raise ContractError( + "nested takeover reached Git sync without owning the superproject lock" + ) + if nested_lock.exists(): + raise ContractError( + "nested takeover created a second invocation-directory operation lock" + ) + + waiter_started = time.monotonic() + waiter = contract.start( + contract.zed_command("nested-waiter", "install", "--frozen"), + cwd=root, + ) + contract.require_blocked(waiter) + blocked_for = time.monotonic() - waiter_started + + release.write_text("release\n", encoding="utf-8") + holder_output = contract.finish(holder, expected=0) + waiter_output = contract.finish(waiter, expected=0) + if "overtook 1 Git submodule package(s)" not in holder_output: + raise ContractError("nested takeover did not report one adopted package") + if "error:" in waiter_output.lower(): + raise ContractError("root-level waiter reported an error") + if not (root / ".zpkg.lock").is_file(): + raise ContractError("nested takeover did not publish a root lockfile") + if not (root / "zed_modules/acme/nested-child/payload.txt").is_file(): + raise ContractError("root-level frozen waiter did not materialize the package") + + checks = [ + "nested takeover owns the discovered superproject operation lock", + "nested takeover does not create an invocation-directory lock identity", + "root-level frozen install blocks and succeeds behind nested takeover", + ] + evidence = { + "schema": "zed.git-submodule-nested-overtake-lock/v1", + "checks": checks, + "holder_pid": holder.pid, + "waiter_pid": waiter.pid, + "observed_block_seconds": round(blocked_for, 3), + "network_credentials": False, + "public_registry_mutation": False, + } + (contract.evidence / "nested-evidence.json").write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + contract.log("\ncertified 3 nested takeover lock-identity checks") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/git_submodule_pack.py b/scripts/git_submodule_pack.py new file mode 100644 index 00000000..a19e8a1c --- /dev/null +++ b/scripts/git_submodule_pack.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 +"""Credential-free archive certification for Zed/Git-submodule packaging.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import tarfile +from pathlib import Path, PurePosixPath +from typing import Mapping, Sequence + +MANIFEST = ".zpkg.toml" + + +class Contract: + def __init__(self, zed: Path, work_root: Path) -> None: + self.zed = zed.resolve() + self.root = work_root.resolve() + self.registry = self.root / "registry" + self.homes = self.root / "zed-homes" + self.git_home = self.root / "git-home" + self.repos = self.root / "repos" + self.runs = self.root / "runs" + self.evidence = self.root / "evidence" + self.log_path = self.evidence / "contract.log" + self.checks: list[str] = [] + + if not self.zed.is_file(): + raise RuntimeError(f"zed binary not found: {self.zed}") + if self.root.exists(): + raise RuntimeError(f"work root must not already exist: {self.root}") + for directory in ( + self.registry, + self.homes, + self.git_home, + self.repos, + self.runs, + self.evidence, + ): + directory.mkdir(parents=True, exist_ok=True) + self.log_path.write_text("", encoding="utf-8") + + def env(self, extra: Mapping[str, str] | None = None) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "GH_TOKEN", + "GITHUB_TOKEN", + "ZED_PKG_TOKEN", + "ZED_PKG_REGISTRY", + "ZED_PKG_HOME", + ): + env.pop(key, None) + env.update( + { + "CI": "true", + "HOME": str(self.git_home), + "XDG_CONFIG_HOME": str(self.git_home / ".config"), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "true", + "GIT_AUTHOR_NAME": "Zed Pack Contract", + "GIT_AUTHOR_EMAIL": "zed-pack-contract@example.invalid", + "GIT_COMMITTER_NAME": "Zed Pack Contract", + "GIT_COMMITTER_EMAIL": "zed-pack-contract@example.invalid", + # Every fixture is local. Permit file transport only inside + # the disposable contract process and never persist it. + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "protocol.file.allow", + "GIT_CONFIG_VALUE_0": "always", + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + if extra: + env.update(extra) + return env + + def log(self, text: str) -> None: + print(text, flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + def run( + self, + argv: Sequence[str | Path], + *, + cwd: Path | None = None, + should_fail: bool = False, + extra_env: Mapping[str, str] | None = None, + ) -> str: + command = [str(value) for value in argv] + shown = " ".join(json.dumps(value) for value in command) + self.log(f"\n$ (cd {cwd or Path.cwd()} && {shown})") + completed = subprocess.run( + command, + cwd=cwd, + env=self.env(extra_env), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.stdout: + self.log(completed.stdout) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {shown}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {shown}\n" + f"{completed.stdout}" + ) + return completed.stdout + + def git(self, cwd: Path, *args: str) -> str: + return self.run(["git", "-C", cwd, *args], cwd=cwd) + + def zed_cmd( + self, + cwd: Path, + home_name: str, + *args: str, + should_fail: bool = False, + ) -> str: + home = self.homes / home_name + home.mkdir(parents=True, exist_ok=True) + return self.run( + [ + self.zed, + "--registry", + f"file://{self.registry}", + "--home", + home, + *args, + ], + cwd=cwd, + should_fail=should_fail, + ) + + def init_repo(self, path: Path) -> None: + path.mkdir(parents=True) + self.run(["git", "init", "-b", "main", path], cwd=self.root) + self.git(path, "config", "user.name", "Zed Pack Contract") + self.git(path, "config", "user.email", "zed-pack-contract@example.invalid") + + def commit_all(self, path: Path, message: str) -> str: + self.git(path, "add", "-A") + self.git(path, "commit", "-m", message) + return self.git(path, "rev-parse", "HEAD").strip() + + @staticmethod + def write_manifest( + path: Path, + *, + name: str, + exclude: Sequence[str] = (), + ) -> None: + document = ( + "[package]\n" + 'org = "acme"\n' + f'name = "{name}"\n' + 'version = "1.2.3"\n\n' + "[package.repository]\n" + 'vcs = "git"\n' + f'url = "https://example.invalid/acme/{name}.git"\n' + ) + if exclude: + quoted = ", ".join(json.dumps(pattern) for pattern in exclude) + document += f"\n[publish]\nexclude = [{quoted}]\n" + (path / MANIFEST).write_text(document, encoding="utf-8") + + def add_submodule(self, root: Path, source: Path, destination: str) -> None: + self.run( + [ + "git", + "-C", + root, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + str(source), + destination, + ], + cwd=root, + ) + + def clone_no_submodules(self, source: Path, destination: Path) -> None: + self.run(["git", "clone", "--no-recurse-submodules", source, destination]) + self.git(destination, "config", "user.name", "Zed Pack Contract") + self.git(destination, "config", "user.email", "zed-pack-contract@example.invalid") + + def build_child_graph(self) -> Path: + schema = self.repos / "schema" + self.init_repo(schema) + (schema / "schema.txt").write_text("schema-v1\n", encoding="utf-8") + self.commit_all(schema, "schema") + + child = self.repos / "client" + self.init_repo(child) + self.write_manifest(child, name="client") + (child / "lib.txt").write_text("client-runtime\n", encoding="utf-8") + self.add_submodule(child, schema, "vendor/schema") + self.commit_all(child, "client with nested schema") + return child + + def build_root( + self, + *, + name: str, + child: Path, + exclude: Sequence[str] = (), + zedignore: Sequence[str] = (), + ) -> Path: + root = self.repos / name + self.init_repo(root) + self.write_manifest(root, name=name, exclude=exclude) + if zedignore: + (root / ".zedignore").write_text( + "\n".join(zedignore) + "\n", encoding="utf-8" + ) + (root / "root.txt").write_text(f"{name}-runtime\n", encoding="utf-8") + self.add_submodule(root, child, "vendor/client") + self.commit_all(root, f"{name} with client") + return root + + @staticmethod + def assert_no_archives(path: Path) -> None: + archives = sorted(path.rglob("*.tar.gz")) if path.exists() else [] + assert not archives, f"failed command left package archives: {archives}" + + @staticmethod + def archive_entries(path: Path) -> set[str]: + with tarfile.open(path, mode="r:gz") as archive: + return {member.name for member in archive.getmembers() if member.isfile()} + + @staticmethod + def assert_no_vcs_metadata(entries: set[str]) -> None: + forbidden = {".git", ".gitmodules", ".hg", ".svn"} + leaked = [] + for entry in entries: + if forbidden.intersection(PurePosixPath(entry).parts): + leaked.append(entry) + assert not leaked, f"archive leaked VCS control data: {sorted(leaked)}" + + def certify_uninitialized_rejection(self, source: Path) -> None: + project = self.runs / "uninitialized" + self.clone_no_submodules(source, project) + assert not (project / "vendor/client/.git").exists() + + pack_out = self.runs / "uninitialized-pack" + output = self.zed_cmd( + project, + "uninitialized-pack", + "pack", + "--out", + pack_out, + should_fail=True, + ) + assert "not initialized" in output.lower(), output + self.assert_no_archives(pack_out) + self.checks.append("uninitialized included submodule rejects pack before output") + + output = self.zed_cmd( + project, + "uninitialized-publish", + "publish", + "--dry-run", + "--skip-vcs-checks", + should_fail=True, + ) + assert "not initialized" in output.lower(), output + self.assert_no_archives(project / ".zed/pack") + self.checks.append("uninitialized included submodule rejects publish dry-run") + + def certify_initialized_archive(self, source: Path) -> Path: + project = self.runs / "initialized" + self.clone_no_submodules(source, project) + self.zed_cmd(project, "initialized", "install", "--git-submodules") + + assert (project / "vendor/client/lib.txt").is_file() + assert (project / "vendor/client/vendor/schema/schema.txt").is_file() + assert (project / "vendor/client/.git").is_file() + assert (project / "vendor/client/vendor/schema/.git").is_file() + self.checks.append("zed install recursively materializes package source") + + pack_out = self.runs / "initialized-pack" + self.zed_cmd(project, "initialized", "pack", "--out", pack_out) + archive = pack_out / "acme-included-root-1.2.3.tar.gz" + assert archive.is_file(), f"expected archive missing: {archive}" + entries = self.archive_entries(archive) + assert "pkg/root.txt" in entries + assert "pkg/vendor/client/lib.txt" in entries + assert "pkg/vendor/client/vendor/schema/schema.txt" in entries + self.checks.append("initialized pack embeds top-level and nested runtime source") + + self.assert_no_vcs_metadata(entries) + self.checks.append("archive strips root and nested VCS control metadata") + + output = self.zed_cmd( + project, + "initialized", + "publish", + "--dry-run", + "--skip-vcs-checks", + ) + assert "dry run: would publish acme/included-root@1.2.3" in output, output + self.checks.append("initialized publish dry-run reaches immutable package plan") + return project + + def certify_dirty_rejection(self, project: Path) -> None: + child_file = project / "vendor/client/lib.txt" + child_file.write_text("dirty-client\n", encoding="utf-8") + output = self.zed_cmd( + project, + "dirty-top", + "pack", + "--out", + self.runs / "dirty-top-pack", + should_fail=True, + ) + assert "dirty" in output.lower(), output + self.assert_no_archives(self.runs / "dirty-top-pack") + self.git(project / "vendor/client", "reset", "--hard", "HEAD") + self.checks.append("dirty included submodule rejects pack before output") + + nested_file = project / "vendor/client/vendor/schema/schema.txt" + nested_file.write_text("dirty-schema\n", encoding="utf-8") + output = self.zed_cmd( + project, + "dirty-nested", + "pack", + "--out", + self.runs / "dirty-nested-pack", + should_fail=True, + ) + assert any(word in output.lower() for word in ("dirty", "drift")), output + self.assert_no_archives(self.runs / "dirty-nested-pack") + self.git(project / "vendor/client/vendor/schema", "reset", "--hard", "HEAD") + self.checks.append("dirty nested submodule rejects pack before output") + + def certify_excluded_uninitialized(self, source: Path, *, home_name: str) -> set[str]: + project = self.runs / home_name + self.clone_no_submodules(source, project) + assert not (project / "vendor/client/.git").exists() + + out = self.runs / f"{home_name}-pack" + self.zed_cmd(project, home_name, "pack", "--out", out) + archive = out / f"acme-{source.name}-1.2.3.tar.gz" + assert archive.is_file(), f"expected archive missing: {archive}" + entries = self.archive_entries(archive) + assert not any(entry.startswith("pkg/vendor/client/") for entry in entries) + self.assert_no_vcs_metadata(entries) + return entries + + def certify_noncanonical_exclusion_rejected( + self, + source: Path, + *, + home_name: str, + check: str, + ) -> None: + project = self.runs / home_name + self.clone_no_submodules(source, project) + assert not (project / "vendor/client/.git").exists() + + out = self.runs / f"{home_name}-pack" + output = self.zed_cmd( + project, + home_name, + "pack", + "--out", + out, + should_fail=True, + ) + assert "not initialized" in output.lower(), output + self.assert_no_archives(out) + self.checks.append(check) + + def finish(self) -> None: + version = self.run([self.zed, "--version"]).strip() + record = { + "schema": "zed.git-submodule-pack-evidence/v1", + "zed_version": version, + "zed_sha256": hashlib.sha256(self.zed.read_bytes()).hexdigest(), + "checks": self.checks, + "network_credentials": False, + "public_registry_mutation": False, + } + (self.evidence / "evidence.json").write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + assert len(self.checks) == 12, self.checks + self.log(f"\ncertified {len(self.checks)} Git-submodule packaging checks") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + contract = Contract(args.zed, args.work_root) + child = contract.build_child_graph() + included = contract.build_root(name="included-root", child=child) + excluded = contract.build_root( + name="excluded-root", + child=child, + exclude=["vendor/client/**"], + ) + ignored = contract.build_root( + name="ignored-root", + child=child, + zedignore=["vendor/client/**"], + ) + dot_excluded = contract.build_root( + name="dot-excluded-root", + child=child, + exclude=["./vendor/client/**"], + ) + absolute_excluded = contract.build_root( + name="absolute-excluded-root", + child=child, + exclude=["/vendor/client/**"], + ) + + contract.certify_uninitialized_rejection(included) + initialized = contract.certify_initialized_archive(included) + contract.certify_dirty_rejection(initialized) + contract.certify_excluded_uninitialized(excluded, home_name="publish-excluded") + contract.checks.append( + "publish.exclude permits an omitted, uninitialized submodule subtree" + ) + contract.certify_excluded_uninitialized(ignored, home_name="zedignore-excluded") + contract.checks.append(".zedignore permits an omitted, uninitialized submodule subtree") + contract.certify_noncanonical_exclusion_rejected( + dot_excluded, + home_name="dot-exclusion-rejected", + check="dot-prefixed recursive exclusion cannot bypass initialization", + ) + contract.certify_noncanonical_exclusion_rejected( + absolute_excluded, + home_name="absolute-exclusion-rejected", + check="leading-slash recursive exclusion cannot bypass initialization", + ) + contract.finish() + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error: + print(f"Git-submodule pack contract failed: {error}", file=sys.stderr) + raise diff --git a/scripts/github_api_fallback.py b/scripts/github_api_fallback.py new file mode 100644 index 00000000..df8c68cf --- /dev/null +++ b/scripts/github_api_fallback.py @@ -0,0 +1,615 @@ +#!/usr/bin/env python3 +"""Live GitHub API canary: prove GitHub can host a Zed artifact when zpkg.net is down. + +This is not the loopback CDN mock. It calls api.github.com and +uploads.github.com against github.com/zed-pkg-test/github-api-fallback-canary, +then downloads the same bytes back over the public Release URL with no +registry in the path. + +Phases: + +1. Ensure the public canary repo exists (create if missing). +2. Pack a disposable tarball (stdlib tarfile; no zed required). +3. Ensure a git tag + GitHub Release via the REST API. +4. Upload the tarball and a VersionMetadata sidecar as Release assets. +5. GET the repo, tags, release, and both download URLs; require byte-identical + sha256. That is the 100% GitHub-API proof. +6. If --zed is passed, point the CLI at a closed loopback registry and require + `zed install --frozen` to restore the payload through source fallback. + +Token (never printed): GITHUB_TOKEN, GH_TOKEN, or ZED_PKG_GITHUB_TOKEN. +Needs `repo` (or fine-grained contents + metadata) on zed-pkg-test. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import re +import shutil +import socket +import subprocess +import sys +import tarfile +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Mapping, Sequence + + +ORG = "zed-pkg-test" +REPO = "github-api-fallback-canary" +PACKAGE_NAME = "github-api-fallback-canary" +VERSION = "0.0.1" +TAG = f"v{VERSION}" +TARBALL_ASSET = f"zpkg-{ORG}-{PACKAGE_NAME}-{VERSION}.tar.gz" +SIDECAR_ASSET = f"zpkg-{ORG}-{PACKAGE_NAME}-{VERSION}.json" +API = "https://api.github.com" +UPLOADS = "https://uploads.github.com" +USER_AGENT = "zed-pkg-test-github-api-fallback/1.0" +PAYLOAD = "github-api-fallback\n" + + +def token_from_env() -> str: + for key in ("ZED_PKG_GITHUB_TOKEN", "GITHUB_TOKEN", "GH_TOKEN"): + value = os.environ.get(key, "").strip() + if value and value != "from-env": + return value + raise RuntimeError( + "set GITHUB_TOKEN / GH_TOKEN / ZED_PKG_GITHUB_TOKEN (do not print it)" + ) + + +def api_request( + method: str, + url: str, + token: str, + *, + body: bytes | None = None, + content_type: str = "application/json", + accept: str = "application/vnd.github+json", +) -> tuple[int, dict[str, str], bytes]: + headers = { + "Accept": accept, + "Authorization": f"Bearer {token}", + "User-Agent": USER_AGENT, + "X-GitHub-Api-Version": "2022-11-28", + } + if body is not None: + headers["Content-Type"] = content_type + headers["Content-Length"] = str(len(body)) + request = urllib.request.Request(url, data=body, method=method, headers=headers) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return response.status, dict(response.headers.items()), response.read() + except urllib.error.HTTPError as error: + payload = error.read() + return error.code, dict(error.headers.items()), payload + + +def api_json(method: str, url: str, token: str, payload: Any | None = None) -> Any: + body = None if payload is None else json.dumps(payload).encode("utf-8") + status, _headers, raw = api_request(method, url, token, body=body) + if status >= 400: + snippet = raw.decode("utf-8", errors="replace")[:800] + raise RuntimeError(f"{method} {url} -> {status}: {snippet}") + if not raw: + return None + return json.loads(raw.decode("utf-8")) + + +def public_get(url: str) -> tuple[int, bytes]: + request = urllib.request.Request( + url, + headers={"User-Agent": USER_AGENT, "Accept": "application/octet-stream"}, + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def asset_names(org: str, name: str, version: str, ext: str = "tar.gz") -> list[str]: + names = [f"zpkg-{org}-{name}-{version}.{ext}"] + short = f"zpkg-{name}-{version}.{ext}" + if short not in names: + names.append(short) + return names + + +def sidecar_names(org: str, name: str, version: str) -> list[str]: + names = [f"zpkg-{org}-{name}-{version}.json"] + short = f"zpkg-{name}-{version}.json" + if short not in names: + names.append(short) + return names + + +def pack_canary(work: Path) -> tuple[bytes, Path]: + """Pack the same `pkg/`-rooted layout `zed pack` writes. + + `zed install` rejects archives whose first directory is not `pkg/`. + """ + root = work / "package" + pkg = root / "pkg" + (pkg / "src").mkdir(parents=True) + (pkg / "src" / "payload.txt").write_text(PAYLOAD, encoding="utf-8") + (pkg / ".zpkg.toml").write_text( + f'''[package] +org = "{ORG}" +name = "{PACKAGE_NAME}" +version = "{VERSION}" +description = "Live GitHub API fallback canary" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://github.com/{ORG}/{REPO}" + +[install] +dir = "zed_modules" +adapter = "none" +''', + encoding="utf-8", + ) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + archive.add(pkg, arcname="pkg") + tarball = buffer.getvalue() + path = work / TARBALL_ASSET + path.write_bytes(tarball) + return tarball, path + + +def packed_member_names(tarball: bytes) -> list[str]: + with tarfile.open(fileobj=io.BytesIO(tarball), mode="r:gz") as archive: + return [member.name for member in archive.getmembers()] + + +def ensure_repo(token: str) -> dict[str, Any]: + status, _headers, raw = api_request( + "GET", f"{API}/repos/{ORG}/{REPO}", token + ) + if status == 200: + repo = json.loads(raw.decode("utf-8")) + print(f"repo exists: {repo['html_url']}", flush=True) + return repo + if status != 404: + raise RuntimeError(f"GET repo returned {status}: {raw[:400]!r}") + print(f"creating public repo {ORG}/{REPO}", flush=True) + repo = api_json( + "POST", + f"{API}/orgs/{ORG}/repos", + token, + { + "name": REPO, + "description": "Live GitHub API canary: packed Zed artifact on Releases so registry.zpkg.net outages still install.", + "private": False, + "auto_init": True, + "has_issues": False, + "has_projects": False, + "has_wiki": False, + }, + ) + print(f"created {repo['html_url']}", flush=True) + return repo + + +def ensure_manifest_on_default_branch(token: str, repo: dict[str, Any]) -> None: + branch = repo.get("default_branch") or "main" + status, _headers, raw = api_request( + "GET", + f"{API}/repos/{ORG}/{REPO}/contents/.zpkg.toml?ref={branch}", + token, + ) + if status == 200: + return + content = f'''[package] +org = "{ORG}" +name = "{PACKAGE_NAME}" +version = "{VERSION}" +description = "Live GitHub API fallback canary" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://github.com/{ORG}/{REPO}" + +[install] +dir = "zed_modules" +adapter = "none" +''' + import base64 + + api_json( + "PUT", + f"{API}/repos/{ORG}/{REPO}/contents/.zpkg.toml", + token, + { + "message": "chore: add canary .zpkg.toml for GitHub fallback", + "content": base64.b64encode(content.encode("utf-8")).decode("ascii"), + "branch": branch, + }, + ) + print(f"wrote .zpkg.toml on {branch}", flush=True) + + +def ensure_release(token: str, commitish: str) -> dict[str, Any]: + status, _headers, raw = api_request( + "GET", f"{API}/repos/{ORG}/{REPO}/releases/tags/{TAG}", token + ) + if status == 200: + release = json.loads(raw.decode("utf-8")) + print(f"release exists: {release['html_url']}", flush=True) + return release + release = api_json( + "POST", + f"{API}/repos/{ORG}/{REPO}/releases", + token, + { + "tag_name": TAG, + "target_commitish": commitish, + "name": TAG, + "body": "Packed Zed canary for registry.zpkg.net outage fallback. Uploaded by scripts/github_api_fallback.py.", + "draft": False, + "prerelease": False, + }, + ) + print(f"created release {release['html_url']}", flush=True) + return release + + +def replace_asset( + token: str, release: dict[str, Any], name: str, content_type: str, data: bytes +) -> None: + for asset in release.get("assets") or []: + if asset.get("name") == name: + status, _headers, _raw = api_request( + "DELETE", f"{API}/repos/{ORG}/{REPO}/releases/assets/{asset['id']}", token + ) + if status not in (204, 200): + raise RuntimeError(f"DELETE asset {name} returned {status}") + print(f"deleted existing asset {name}", flush=True) + encoded = urllib.parse.quote(name) + url = ( + f"{UPLOADS}/repos/{ORG}/{REPO}/releases/{release['id']}/assets?name={encoded}" + ) + status, _headers, raw = api_request( + "POST", + url, + token, + body=data, + content_type=content_type, + accept="application/vnd.github+json", + ) + if status not in (201, 200): + raise RuntimeError(f"upload {name} returned {status}: {raw[:400]!r}") + print(f"uploaded {name} ({len(data)} bytes)", flush=True) + + +def prove_github_roundtrip(token: str, tarball: bytes, digest: str) -> dict[str, Any]: + repo = api_json("GET", f"{API}/repos/{ORG}/{REPO}", token) + if repo.get("private") is True: + raise AssertionError("canary repo must be public so unauthenticated install works") + tags = api_json("GET", f"{API}/repos/{ORG}/{REPO}/tags?per_page=100", token) + tag_names = [item["name"] for item in tags] + if TAG not in tag_names: + raise AssertionError(f"tag {TAG} missing from GitHub tags API: {tag_names!r}") + release = api_json("GET", f"{API}/repos/{ORG}/{REPO}/releases/tags/{TAG}", token) + asset_map = {item["name"]: item for item in release.get("assets") or []} + if TARBALL_ASSET not in asset_map: + raise AssertionError(f"{TARBALL_ASSET} missing from release assets") + if SIDECAR_ASSET not in asset_map: + raise AssertionError(f"{SIDECAR_ASSET} missing from release assets") + + api_status, api_bytes = public_get(asset_map[TARBALL_ASSET]["url"]) + if api_status != 200: + # browser_download_url is the unauthenticated path; the API url needs Accept + api_status, _headers, api_bytes = api_request( + "GET", + asset_map[TARBALL_ASSET]["url"], + token, + accept="application/octet-stream", + ) + if api_status != 200: + raise AssertionError(f"API asset download returned {api_status}") + if sha256_bytes(api_bytes) != digest: + raise AssertionError("API asset sha256 did not match the uploaded tarball") + + public_url = ( + f"https://github.com/{ORG}/{REPO}/releases/download/{TAG}/{TARBALL_ASSET}" + ) + public_status, public_bytes = public_get(public_url) + if public_status != 200: + raise AssertionError(f"public Release download returned {public_status}") + if sha256_bytes(public_bytes) != digest: + raise AssertionError("public Release download sha256 did not match") + if public_bytes != tarball: + raise AssertionError("public Release bytes drifted from the upload") + + sidecar_url = ( + f"https://github.com/{ORG}/{REPO}/releases/download/{TAG}/{SIDECAR_ASSET}" + ) + sidecar_status, sidecar_bytes = public_get(sidecar_url) + if sidecar_status != 200: + raise AssertionError(f"sidecar download returned {sidecar_status}") + sidecar = json.loads(sidecar_bytes.decode("utf-8")) + if sidecar.get("sha256") != digest: + raise AssertionError(f"sidecar sha256 {sidecar.get('sha256')} != {digest}") + + print( + json.dumps( + { + "ok": True, + "phase": "github-api", + "repo": repo["html_url"], + "release": release["html_url"], + "tag": TAG, + "sha256": digest, + "public_tarball": public_url, + "bytes": len(tarball), + }, + indent=2, + ), + flush=True, + ) + return sidecar + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def run( + command: Sequence[str | Path], + *, + cwd: Path, + extra_env: Mapping[str, str] | None = None, +) -> str: + argv = [str(value) for value in command] + print(f"$ {' '.join(argv)}", flush=True) + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "ZED_PKG_INTERACTIVE": "false", + } + ) + if extra_env: + environment.update(extra_env) + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(completed.stdout, end="", flush=True) + if completed.returncode != 0: + raise RuntimeError(f"command failed ({completed.returncode}): {argv!r}") + return completed.stdout + + +def zed_frozen_install(zed: Path, work: Path, sidecar: dict[str, Any]) -> None: + consumer = work / "consumer" + consumer.mkdir() + (consumer / ".zpkg.toml").write_text( + f'''[package] +org = "zed-pkg-test" +name = "github-api-fallback-consumer" +version = "0.0.0" +description = "Consumes the live GitHub API canary" + +[package.repository] +vcs = "git" +url = "https://github.com/zed-pkg-test/github-api-fallback-consumer" + +[install] +dir = "zed_modules" +adapter = "none" + +[dependencies] +"{ORG}/{PACKAGE_NAME}" = "={VERSION}" +''', + encoding="utf-8", + ) + # The real lockfile schema is richer; let `zed install` write one first + # against GitHub fallback, then frozen-restore from a wiped store. + dead = f"http://127.0.0.1:{free_port()}" + home = work / "zed-home" + extra = { + "ZED_PKG_SOURCE_FALLBACK": "true", + "ZED_PKG_SOURCE_FALLBACK_ALLOW_LOOPBACK": "true", + "ZED_PKG_R2_PUBLIC_BASE": "http://127.0.0.1:1", + } + run( + [ + zed, + "--registry", + dead, + "--home", + home, + "install", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + ], + cwd=consumer, + extra_env=extra, + ) + payload = consumer / "zed_modules" / ORG / PACKAGE_NAME / "src" / "payload.txt" + # Packed archive uses arcname PACKAGE-VERSION/src/payload.txt — install + # layout depends on the CLI unpack. Accept either layout. + if not payload.is_file(): + matches = list(consumer.rglob("payload.txt")) + if not matches: + raise AssertionError(f"zed install did not materialize payload under {consumer}") + payload = matches[0] + if payload.read_text(encoding="utf-8") != PAYLOAD: + raise AssertionError("GitHub fallback install restored the wrong payload") + lock_path = consumer / ".zpkg.lock" + if not lock_path.is_file(): + raise AssertionError("install did not write a lockfile") + lock_bytes = lock_path.read_bytes() + shutil.rmtree(home) + if (consumer / "zed_modules").exists(): + shutil.rmtree(consumer / "zed_modules") + home_frozen = work / "zed-home-frozen" + run( + [ + zed, + "--registry", + dead, + "--home", + home_frozen, + "install", + "--frozen", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + ], + cwd=consumer, + extra_env=extra, + ) + if lock_path.read_bytes() != lock_bytes: + raise AssertionError("frozen GitHub fallback mutated the lockfile") + restored = list(consumer.rglob("payload.txt")) + if not restored or restored[0].read_text(encoding="utf-8") != PAYLOAD: + raise AssertionError("frozen GitHub fallback did not restore the payload") + print( + json.dumps( + { + "ok": True, + "phase": "zed-install", + "registry": dead, + "sha256": sidecar["sha256"], + }, + indent=2, + ), + flush=True, + ) + + +def assert_contract_helpers() -> None: + assert asset_names(ORG, PACKAGE_NAME, VERSION)[0] == TARBALL_ASSET + assert sidecar_names(ORG, PACKAGE_NAME, VERSION)[0] == SIDECAR_ASSET + assert TAG == "v0.0.1" + packed, _path = pack_canary(Path(tempfile.mkdtemp(prefix="zpkg-canary-pack-"))) + names = packed_member_names(packed) + assert any(name == "pkg" or name.startswith("pkg/") for name in names), names + assert "pkg/.zpkg.toml" in names, names + assert "pkg/src/payload.txt" in names, names + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--work-root", + type=Path, + required=True, + help="fresh directory for the packed canary and optional zed homes", + ) + parser.add_argument( + "--zed", + type=Path, + help="optional zed binary with source-fallback; skipped if omitted", + ) + parser.add_argument( + "--skip-network", + action="store_true", + help="only check local naming contracts (PR syntax job)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + assert_contract_helpers() + if args.skip_network: + print(json.dumps({"ok": True, "phase": "contract"}), flush=True) + return 0 + + work = args.work_root.resolve() + if work.exists(): + raise AssertionError(f"work root must be fresh: {work}") + work.mkdir(parents=True) + token = token_from_env() + tarball, _path = pack_canary(work) + digest = sha256_bytes(tarball) + repo = ensure_repo(token) + # auto_init races: wait until default branch exists + for _ in range(10): + fresh = api_json("GET", f"{API}/repos/{ORG}/{REPO}", token) + if fresh.get("default_branch") and fresh.get("pushed_at"): + repo = fresh + break + time.sleep(1) + ensure_manifest_on_default_branch(token, repo) + repo = api_json("GET", f"{API}/repos/{ORG}/{REPO}", token) + branch = repo.get("default_branch") or "main" + ref = api_json("GET", f"{API}/repos/{ORG}/{REPO}/git/ref/heads/{branch}", token) + commit = ref["object"]["sha"] + release = ensure_release(token, commit) + sidecar = { + "org": ORG, + "name": PACKAGE_NAME, + "version": VERSION, + "sha256": digest, + "size": len(tarball), + "format": "tar.gz", + "vcs_tag": TAG, + "vcs_commit": commit, + "download_url": f"https://github.com/{ORG}/{REPO}/releases/download/{TAG}/{TARBALL_ASSET}", + "published_at": "1970-01-01T00:00:00Z", + "yanked": False, + "mirrors": [ + { + "kind": "github-release", + "url": f"https://github.com/{ORG}/{REPO}/releases/download/{TAG}/{TARBALL_ASSET}", + } + ], + } + replace_asset(token, release, TARBALL_ASSET, "application/gzip", tarball) + replace_asset( + token, + release, + SIDECAR_ASSET, + "application/json", + json.dumps(sidecar, indent=2).encode("utf-8"), + ) + sidecar = prove_github_roundtrip(token, tarball, digest) + if args.zed: + zed = args.zed.resolve() + if not zed.is_file(): + raise AssertionError(f"zed binary not found: {zed}") + zed_frozen_install(zed, work, sidecar) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 + print(f"github-api-fallback canary failed: {error}", file=sys.stderr) + raise diff --git a/scripts/github_r2_fallback.py b/scripts/github_r2_fallback.py new file mode 100644 index 00000000..80419787 --- /dev/null +++ b/scripts/github_r2_fallback.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Black-box canary: registry down, public CDN still installs. + +Publishes a disposable package through a file:// registry, copies the packed +tarball onto the guessable R2 keys (`packages/…` and `github/…`), takes the +HTTP registry down, and proves `zed install --frozen` fetches from a loopback +stand-in for `https://cdn.zpkg.net`. Also round-trips the oresoftware/api-docs +JSON call/receipt frame for `get_version` over TCP NDJSON. +""" + +from __future__ import annotations + +import argparse +import hashlib +import http.server +import json +import os +import re +import shutil +import socket +import socketserver +import subprocess +import sys +import threading +from pathlib import Path +from typing import Mapping, Sequence + + +ORG = "zed-pkg-test" +NAME = "fallback-canary" +VERSION = "0.0.1" +TAG = f"v{VERSION}" +FILENAME = f"{NAME}-{VERSION}.tar.gz" + + +def shell_quote(value: str) -> str: + if not value: + return "''" + if re.fullmatch(r"[A-Za-z0-9_@%+=:,./-]+", value): + return value + return "'" + value.replace("'", "'\\''") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path, + extra_env: Mapping[str, str] | None = None, +) -> str: + argv = [str(value) for value in command] + print(f"\n$ (cd {cwd} && {' '.join(shell_quote(value) for value in argv)})", flush=True) + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "ZED_PKG_TOKEN": "", + "ZED_PKG_INTERACTIVE": "false", + } + ) + if extra_env: + environment.update(extra_env) + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(completed.stdout, end="", flush=True) + if completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {argv!r}\n{completed.stdout}" + ) + return completed.stdout + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def expand(template: str, **values: str) -> str: + out = template + for key, value in values.items(): + out = out.replace("{" + key + "}", value) + return out + + +def load_route_map(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def write_package(directory: Path) -> None: + directory.mkdir(parents=True) + (directory / "src").mkdir() + (directory / "src" / "payload.txt").write_text("cdn-fallback\n", encoding="utf-8") + (directory / ".zpkg.toml").write_text( + f'''[package] +org = "{ORG}" +name = "{NAME}" +version = "{VERSION}" +description = "Disposable GitHub/R2 fallback canary" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://github.com/{ORG}/{NAME}" + +[install] +dir = "zed_modules" +adapter = "none" +''', + encoding="utf-8", + ) + run(["git", "init", "-b", "main"], cwd=directory) + run(["git", "config", "user.email", "canary@zed-pkg-test.invalid"], cwd=directory) + run(["git", "config", "user.name", "zed-pkg-test canary"], cwd=directory) + run(["git", "add", "-A"], cwd=directory) + run(["git", "commit", "-m", "canary package"], cwd=directory) + commit = run(["git", "rev-parse", "HEAD"], cwd=directory).strip() + run(["git", "tag", TAG, commit], cwd=directory) + + +def write_consumer(directory: Path) -> None: + directory.mkdir(parents=True) + (directory / ".zpkg.toml").write_text( + f'''[package] +org = "zed-pkg-test" +name = "fallback-consumer" +version = "0.0.0" +description = "Consumes the GitHub/R2 fallback canary" + +[package.repository] +vcs = "git" +url = "https://github.com/zed-pkg-test/fallback-consumer" + +[install] +dir = "zed_modules" +adapter = "none" + +[dependencies] +"{ORG}/{NAME}" = "={VERSION}" +''', + encoding="utf-8", + ) + + +def start_cdn(root: Path, port: int) -> http.server.ThreadingHTTPServer: + class BoundHandler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(root), **kwargs) + + def log_message(self, format: str, *args) -> None: # noqa: A003 + print(f"cdn: {format % args}", flush=True) + + server = http.server.ThreadingHTTPServer(("127.0.0.1", port), BoundHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +class VersionRpcHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + raw = self.rfile.readline() + call = json.loads(raw) + if call.get("v") != 1 or call.get("op") != "call" or call.get("key") != "get_version": + receipt = { + "v": 1, + "op": "receipt", + "id": call.get("id", ""), + "key": call.get("key", ""), + "transport": "tcp", + "ok": False, + "status": 400, + "error": {"code": "bad_call"}, + } + else: + path = call.get("path") or {} + receipt = { + "v": 1, + "op": "receipt", + "id": call["id"], + "key": "get_version", + "transport": "tcp", + "ok": True, + "status": 200, + "body": { + "org": path.get("org", ORG), + "name": path.get("name", NAME), + "version": path.get("version", VERSION), + "yanked": False, + }, + } + line = json.dumps(receipt, separators=(",", ":")) + "\n" + self.wfile.write(line.encode("utf-8")) + + +def start_tcp_rpc(port: int) -> socketserver.ThreadingTCPServer: + server = socketserver.ThreadingTCPServer(("127.0.0.1", port), VersionRpcHandler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +def tcp_get_version(port: int) -> dict: + call = { + "v": 1, + "op": "call", + "id": "e2e-get-version", + "key": "get_version", + "transport": "tcp", + "path": {"org": ORG, "name": NAME, "version": VERSION}, + } + payload = json.dumps(call, separators=(",", ":")) + "\n" + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload.encode("utf-8")) + sock.shutdown(socket.SHUT_WR) + raw = b"" + while not raw.endswith(b"\n"): + chunk = sock.recv(4096) + if not chunk: + break + raw += chunk + receipt = json.loads(raw.decode("utf-8")) + if not receipt.get("ok"): + raise AssertionError(f"tcp get_version failed: {receipt!r}") + return receipt + + +def copy_guessable_objects(artifact: Path, cdn_root: Path) -> None: + targets = [ + cdn_root / "packages" / ORG / NAME / VERSION / FILENAME, + cdn_root / "github" / ORG / NAME / TAG / FILENAME, + ] + for target in targets: + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(artifact, target) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + parser.add_argument( + "--route-map", + type=Path, + help="zed-api.route-map.json (defaults to a sibling zed-interfaces checkout)", + ) + parser.add_argument( + "--interfaces-generated-ts", + type=Path, + help="zed-interfaces generated TypeScript to compare with zed-clients", + ) + parser.add_argument( + "--clients-generated-ts", + type=Path, + help="zed-clients copy of the generated TypeScript routes", + ) + return parser.parse_args() + + +def assert_route_map(route_map: dict) -> None: + mapping = route_map["map"] + get_version = mapping["get_version"] + if get_version["path"] != "/v1/packages/{org}/{name}/versions/{version}": + raise AssertionError(get_version["path"]) + if get_version["transports"] != ["http", "tcp"]: + raise AssertionError(get_version["transports"]) + if get_version.get("tcp_framing") != "ndjson": + raise AssertionError("get_version must use ndjson on TCP") + if mapping["registry_events"]["transports"] != ["websocket"]: + raise AssertionError(mapping["registry_events"]) + package_path = expand(mapping["get_package"]["path"], org=ORG, name=NAME) + if package_path != f"/v1/packages/{ORG}/{NAME}": + raise AssertionError(package_path) + cdn_path = expand( + mapping["cdn_package_object"]["path"], + org=ORG, + name=NAME, + version=VERSION, + filename=FILENAME, + ) + if cdn_path != f"/packages/{ORG}/{NAME}/{VERSION}/{FILENAME}": + raise AssertionError(cdn_path) + + +def strip_ts_banner(text: str) -> str: + return re.sub(r"^/\*\*.*?\*/\s*", "", text, count=1, flags=re.S).strip() + "\n" + + +def parse_args_and_compare_generated(args: argparse.Namespace) -> None: + if args.interfaces_generated_ts and args.clients_generated_ts: + left = strip_ts_banner(args.interfaces_generated_ts.read_text(encoding="utf-8")) + right = strip_ts_banner(args.clients_generated_ts.read_text(encoding="utf-8")) + if left != right: + raise AssertionError( + "zed-clients generated routes drifted from zed-interfaces generated TypeScript" + ) + + +def main() -> int: + args = parse_args() + zed = args.zed.resolve() + root = args.work_root.resolve() + if root.exists(): + raise AssertionError(f"work root must be fresh: {root}") + if not zed.is_file(): + raise AssertionError(f"zed binary not found: {zed}") + + default_map = ( + Path(__file__).resolve().parents[3] + / "zed-pkg" + / "zed-interfaces" + / "route-maps" + / "zed-api.route-map.json" + ) + route_map_path = (args.route_map or default_map).resolve() + if not route_map_path.is_file(): + raise AssertionError(f"route map not found: {route_map_path}") + route_map = load_route_map(route_map_path) + assert_route_map(route_map) + parse_args_and_compare_generated(args) + + root.mkdir(parents=True) + registry = root / "file-registry" + registry.mkdir() + registry_url = f"file://{registry}" + package = root / "package" + consumer = root / "consumer" + publish_home = root / "publish-home" + install_home = root / "install-home" + fallback_home = root / "fallback-home" + cdn_root = root / "cdn" + cdn_root.mkdir() + + write_package(package) + write_consumer(consumer) + + def zed_cmd(*command: str | Path, cwd: Path, home: Path, extra_env: Mapping[str, str] | None = None) -> str: + return run( + [zed, "--registry", registry_url, "--home", home, *command], + cwd=cwd, + extra_env=extra_env, + ) + + zed_cmd("publish", cwd=package, home=publish_home) + metadata_path = registry / "packages" / ORG / NAME / "versions" / f"{VERSION}.json" + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + sha256 = str(metadata["sha256"]) + artifact = registry / "artifacts" / f"{sha256}.tar.gz" + if not artifact.is_file(): + raise AssertionError(f"published artifact missing: {artifact}") + copy_guessable_objects(artifact, cdn_root) + content_copy = cdn_root / "artifacts" / f"{sha256}.tar.gz" + content_copy.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(artifact, content_copy) + + zed_cmd( + "install", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + cwd=consumer, + home=install_home, + ) + lock = consumer / ".zpkg.lock" + lock_bytes = lock.read_bytes() + payload = consumer / "zed_modules" / ORG / NAME / "src" / "payload.txt" + if payload.read_text(encoding="utf-8") != "cdn-fallback\n": + raise AssertionError("file-registry install did not materialize the canary") + zed_cmd("uninstall", cwd=consumer, home=install_home) + if lock.read_bytes() != lock_bytes: + raise AssertionError("uninstall mutated the lock") + shutil.rmtree(install_home) + if (consumer / "zed_modules").exists(): + raise AssertionError("uninstall left materialization behind") + + dead_port = free_port() + cdn_port = free_port() + tcp_port = free_port() + cdn = start_cdn(cdn_root, cdn_port) + rpc = start_tcp_rpc(tcp_port) + try: + receipt = tcp_get_version(tcp_port) + if receipt["body"]["version"] != VERSION: + raise AssertionError(receipt) + if receipt["transport"] != "tcp": + raise AssertionError(receipt) + + fallback_registry = f"http://127.0.0.1:{dead_port}" + cdn_base = f"http://127.0.0.1:{cdn_port}" + run( + [ + zed, + "--registry", + fallback_registry, + "--home", + fallback_home, + "--r2-public-base", + cdn_base, + "install", + "--frozen", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-ecosystem-mismatch", + ], + cwd=consumer, + extra_env={ + "ZED_PKG_SOURCE_FALLBACK": "true", + "ZED_PKG_SOURCE_FALLBACK_ALLOW_LOOPBACK": "true", + "ZED_PKG_R2_PUBLIC_BASE": cdn_base, + }, + ) + restored = consumer / "zed_modules" / ORG / NAME / "src" / "payload.txt" + if restored.read_text(encoding="utf-8") != "cdn-fallback\n": + raise AssertionError("CDN fallback did not restore the packed payload") + digest = hashlib.sha256(artifact.read_bytes()).hexdigest() + if digest != sha256: + raise AssertionError("packed artifact digest drifted") + print( + json.dumps( + { + "ok": True, + "registry": fallback_registry, + "cdn": cdn_base, + "tcp_rpc": f"127.0.0.1:{tcp_port}", + "sha256": sha256, + "cdn_keys": [ + f"packages/{ORG}/{NAME}/{VERSION}/{FILENAME}", + f"github/{ORG}/{NAME}/{TAG}/{FILENAME}", + f"artifacts/{sha256}.tar.gz", + ], + }, + indent=2, + ), + flush=True, + ) + finally: + cdn.shutdown() + rpc.shutdown() + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 — canary must print the failure + print(f"github-r2-fallback canary failed: {error}", file=sys.stderr) + raise diff --git a/scripts/gitmodules_metadata.py b/scripts/gitmodules_metadata.py new file mode 100644 index 00000000..b6796aa1 --- /dev/null +++ b/scripts/gitmodules_metadata.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Black-box certification for indirect .gitmodules metadata rejection.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import struct +import subprocess +import sys +from pathlib import Path +from typing import Mapping, Sequence + +MANIFEST = ".zpkg.toml" + + +class Contract: + def __init__(self, zed: Path, work_root: Path) -> None: + self.zed = zed.resolve() + self.root = work_root.resolve() + self.registry = self.root / "registry" + self.homes = self.root / "zed-homes" + self.git_home = self.root / "git-home" + self.repos = self.root / "repos" + self.runs = self.root / "runs" + self.evidence = self.root / "evidence" + self.log_path = self.evidence / "contract.log" + self.checks: list[str] = [] + + if not self.zed.is_file(): + raise RuntimeError(f"zed binary not found: {self.zed}") + if self.root.exists(): + raise RuntimeError(f"work root must not already exist: {self.root}") + for directory in ( + self.registry, + self.homes, + self.git_home, + self.repos, + self.runs, + self.evidence, + ): + directory.mkdir(parents=True, exist_ok=True) + self.log_path.write_text("", encoding="utf-8") + + def env(self, extra: Mapping[str, str] | None = None) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "GH_TOKEN", + "GITHUB_TOKEN", + "ZED_PKG_TOKEN", + "ZED_PKG_REGISTRY", + "ZED_PKG_HOME", + ): + env.pop(key, None) + env.update( + { + "CI": "true", + "HOME": str(self.git_home), + "XDG_CONFIG_HOME": str(self.git_home / ".config"), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "true", + "GIT_AUTHOR_NAME": "Zed Metadata Contract", + "GIT_AUTHOR_EMAIL": "zed-metadata-contract@example.invalid", + "GIT_COMMITTER_NAME": "Zed Metadata Contract", + "GIT_COMMITTER_EMAIL": "zed-metadata-contract@example.invalid", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "protocol.file.allow", + "GIT_CONFIG_VALUE_0": "always", + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + if extra: + env.update(extra) + return env + + def log(self, text: str) -> None: + print(text, flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + def run( + self, + argv: Sequence[str | Path], + *, + cwd: Path | None = None, + should_fail: bool = False, + extra_env: Mapping[str, str] | None = None, + ) -> str: + command = [str(value) for value in argv] + shown = " ".join(json.dumps(value) for value in command) + self.log(f"\n$ (cd {cwd or Path.cwd()} && {shown})") + completed = subprocess.run( + command, + cwd=cwd, + env=self.env(extra_env), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.stdout: + self.log(completed.stdout) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {shown}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {shown}\n" + f"{completed.stdout}" + ) + return completed.stdout + + def git(self, cwd: Path, *args: str) -> str: + return self.run(["git", "-C", cwd, *args], cwd=cwd) + + def zed_cmd( + self, + cwd: Path, + home_name: str, + *args: str, + should_fail: bool = False, + ) -> str: + home = self.homes / home_name + home.mkdir(parents=True, exist_ok=True) + return self.run( + [ + self.zed, + "--registry", + f"file://{self.registry}", + "--home", + home, + *args, + ], + cwd=cwd, + should_fail=should_fail, + ) + + def init_repo(self, path: Path) -> None: + path.mkdir(parents=True) + self.run(["git", "init", "-b", "main", path], cwd=self.root) + self.git(path, "config", "user.name", "Zed Metadata Contract") + self.git(path, "config", "user.email", "zed-metadata-contract@example.invalid") + + def commit_all(self, path: Path, message: str) -> str: + self.git(path, "add", "-A") + self.git(path, "commit", "-m", message) + return self.git(path, "rev-parse", "HEAD").strip() + + def clone(self, source: Path, destination: Path) -> None: + self.run(["git", "clone", "--no-recurse-submodules", source, destination]) + self.git(destination, "config", "user.name", "Zed Metadata Contract") + self.git( + destination, + "config", + "user.email", + "zed-metadata-contract@example.invalid", + ) + + def build_regular_source(self) -> Path: + child = self.repos / "child" + self.init_repo(child) + (child / "lib.txt").write_text("child-runtime\n", encoding="utf-8") + self.commit_all(child, "child") + + root = self.repos / "regular-root" + self.init_repo(root) + (root / MANIFEST).write_text( + "[package]\n" + 'org = "acme"\n' + 'name = "metadata-root"\n' + 'version = "1.2.3"\n\n' + "[package.repository]\n" + 'vcs = "git"\n' + 'url = "https://example.invalid/acme/metadata-root.git"\n', + encoding="utf-8", + ) + (root / "root.txt").write_text("root-runtime\n", encoding="utf-8") + self.run( + [ + "git", + "-C", + root, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + child, + "vendor/child", + ], + cwd=root, + ) + self.commit_all(root, "root with child") + return root + + @staticmethod + def assert_no_install_state(project: Path) -> None: + assert not (project / ".zpkg.lock").exists() + assert not (project / "zed_modules").exists() + assert not (project / ".zpkg-staging").exists() + + @staticmethod + def assert_no_archives(path: Path) -> None: + archives = sorted(path.rglob("*.tar.gz")) if path.exists() else [] + assert not archives, f"failed command left archives: {archives}" + + def certify_worktree_symlink(self, source: Path) -> None: + project = self.runs / "worktree-symlink" + self.clone(source, project) + gitmodules = project / ".gitmodules" + external = self.runs / "external-gitmodules" + external.write_text(gitmodules.read_text(encoding="utf-8"), encoding="utf-8") + gitmodules.unlink() + os.symlink(external, gitmodules) + + output = self.zed_cmd( + project, + "worktree-symlink", + "install", + "--git-submodules", + should_fail=True, + ) + assert "must be a regular file" in output.lower(), output + self.assert_no_install_state(project) + self.checks.append("worktree symlink fails before sync or install mutation") + + def certify_worktree_directory(self, source: Path) -> None: + project = self.runs / "worktree-directory" + self.clone(source, project) + gitmodules = project / ".gitmodules" + gitmodules.unlink() + gitmodules.mkdir() + + out = self.runs / "worktree-directory-pack" + output = self.zed_cmd( + project, + "worktree-directory", + "pack", + "--out", + out, + should_fail=True, + ) + assert "must be a regular file" in output.lower(), output + self.assert_no_archives(out) + self.checks.append("worktree directory fails before archive creation") + + def rewrite_index_mode(self, project: Path, path: str, mode: int) -> None: + """Create a valid but non-porcelain index entry for a security fixture.""" + self.git(project, "update-index", "--index-version", "2") + object_format = self.git( + project, "rev-parse", "--show-object-format" + ).strip() + if object_format not in {"sha1", "sha256"}: + raise AssertionError(f"unsupported Git object format: {object_format}") + hash_size = 20 if object_format == "sha1" else 32 + + index_path = project / ".git/index" + data = bytearray(index_path.read_bytes()) + if len(data) < 12 + hash_size or data[:4] != b"DIRC": + raise AssertionError("unrecognized Git index header") + version, entry_count = struct.unpack(">II", data[4:12]) + if version != 2: + raise AssertionError(f"expected Git index version 2, found {version}") + + checksum_start = len(data) - hash_size + expected_checksum = bytes(data[checksum_start:]) + actual_checksum = hashlib.new(object_format, data[:checksum_start]).digest() + if actual_checksum != expected_checksum: + raise AssertionError("Git index checksum is invalid before fixture mutation") + + position = 12 + matched = 0 + fixed_size = 40 + hash_size + 2 + for _ in range(entry_count): + entry_start = position + if entry_start + fixed_size > checksum_start: + raise AssertionError("Git index entry overruns its checksum boundary") + flags_offset = entry_start + 40 + hash_size + flags = struct.unpack(">H", data[flags_offset : flags_offset + 2])[0] + if flags & 0x4000: + raise AssertionError("unexpected extended Git index entry in version 2") + + name_start = entry_start + fixed_size + try: + name_end = data.index(0, name_start, checksum_start) + except ValueError as error: + raise AssertionError("unterminated Git index pathname") from error + name = bytes(data[name_start:name_end]).decode("utf-8") + if name == path: + data[entry_start + 24 : entry_start + 28] = struct.pack( + ">I", mode + ) + matched += 1 + + unpadded_size = name_end + 1 - entry_start + position = entry_start + ((unpadded_size + 7) // 8) * 8 + + if matched != 1: + raise AssertionError( + f"expected exactly one Git index entry for {path!r}, found {matched}" + ) + data[checksum_start:] = hashlib.new( + object_format, data[:checksum_start] + ).digest() + index_path.write_bytes(data) + + def certify_index_symlink_mode(self, source: Path) -> None: + project = self.runs / "index-symlink" + self.clone(source, project) + gitmodules = project / ".gitmodules" + assert gitmodules.is_file() and not gitmodules.is_symlink() + + # Git porcelain intentionally refuses to create a symlink-mode + # .gitmodules entry. Mutate the documented version-2 index format and + # recompute its checksum so the product sees a valid hostile index. + self.rewrite_index_mode(project, ".gitmodules", 0o120000) + mode = self.git(project, "ls-files", "--stage", "--", ".gitmodules") + assert mode.startswith("120000 "), mode + assert gitmodules.is_file() and not gitmodules.is_symlink() + + out = self.runs / "index-symlink-pack" + output = self.zed_cmd( + project, + "index-symlink", + "pack", + "--out", + out, + should_fail=True, + ) + assert "regular git blob" in output.lower(), output + self.assert_no_archives(out) + self.checks.append("symlink index mode fails with a regular worktree file") + + def finish(self) -> None: + version = self.run([self.zed, "--version"]).strip() + record = { + "schema": "zed.gitmodules-metadata-evidence/v1", + "zed_version": version, + "zed_sha256": hashlib.sha256(self.zed.read_bytes()).hexdigest(), + "checks": self.checks, + "network_credentials": False, + "public_registry_mutation": False, + } + (self.evidence / "evidence.json").write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + assert len(self.checks) == 3, self.checks + self.log("\ncertified 3 indirect Git-submodule metadata checks") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + contract = Contract(args.zed, args.work_root) + regular = contract.build_regular_source() + contract.certify_worktree_symlink(regular) + contract.certify_worktree_directory(regular) + contract.certify_index_symlink_mode(regular) + contract.finish() + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error: + print(f"Git-submodule metadata contract failed: {error}", file=sys.stderr) + raise diff --git a/scripts/gitmodules_metadata_contract.py b/scripts/gitmodules_metadata_contract.py new file mode 100644 index 00000000..d657ea44 --- /dev/null +++ b/scripts/gitmodules_metadata_contract.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +"""Black-box certification for the .gitmodules metadata trust boundary.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Mapping, Sequence + +ZERO_OID = "0" * 40 + + +class Contract: + def __init__(self, zed: Path, work_root: Path) -> None: + self.zed = zed.resolve() + self.root = work_root.resolve() + self.registry = self.root / "registry" + self.homes = self.root / "zed-homes" + self.git_home = self.root / "git-home" + self.repos = self.root / "repos" + self.runs = self.root / "runs" + self.evidence = self.root / "evidence" + self.log_path = self.evidence / "contract.log" + self.checks: list[str] = [] + + if not self.zed.is_file(): + raise RuntimeError(f"zed binary not found: {self.zed}") + if self.root.exists(): + raise RuntimeError(f"work root must not already exist: {self.root}") + for directory in ( + self.registry, + self.homes, + self.git_home, + self.repos, + self.runs, + self.evidence, + ): + directory.mkdir(parents=True, exist_ok=True) + self.log_path.write_text("", encoding="utf-8") + + def env(self, extra: Mapping[str, str] | None = None) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "GH_TOKEN", + "GITHUB_TOKEN", + "ZED_PKG_TOKEN", + "ZED_PKG_REGISTRY", + "ZED_PKG_HOME", + ): + env.pop(key, None) + env.update( + { + "CI": "true", + "HOME": str(self.git_home), + "XDG_CONFIG_HOME": str(self.git_home / ".config"), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "true", + "GIT_AUTHOR_NAME": "Zed Metadata Contract", + "GIT_AUTHOR_EMAIL": "zed-metadata-contract@example.invalid", + "GIT_COMMITTER_NAME": "Zed Metadata Contract", + "GIT_COMMITTER_EMAIL": "zed-metadata-contract@example.invalid", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "protocol.file.allow", + "GIT_CONFIG_VALUE_0": "always", + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + if extra: + env.update(extra) + return env + + def log(self, text: str) -> None: + print(text, flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(text) + if not text.endswith("\n"): + handle.write("\n") + + def run( + self, + argv: Sequence[str | Path], + *, + cwd: Path | None = None, + should_fail: bool = False, + stdin: str | None = None, + ) -> str: + command = [str(value) for value in argv] + shown = " ".join(json.dumps(value) for value in command) + self.log(f"\n$ (cd {cwd or Path.cwd()} && {shown})") + completed = subprocess.run( + command, + cwd=cwd, + env=self.env(), + input=stdin, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.stdout: + self.log(completed.stdout) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {shown}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {shown}\n" + f"{completed.stdout}" + ) + return completed.stdout + + def git(self, cwd: Path, *args: str, stdin: str | None = None) -> str: + return self.run(["git", "-C", cwd, *args], cwd=cwd, stdin=stdin) + + def zed_cmd( + self, + cwd: Path, + home_name: str, + *args: str, + should_fail: bool = False, + ) -> str: + home = self.homes / home_name + home.mkdir(parents=True, exist_ok=True) + return self.run( + [ + self.zed, + "--registry", + f"file://{self.registry}", + "--home", + home, + *args, + ], + cwd=cwd, + should_fail=should_fail, + ) + + def init_repo(self, path: Path) -> None: + path.mkdir(parents=True) + self.run(["git", "init", "-b", "main", path], cwd=self.root) + self.git(path, "config", "user.name", "Zed Metadata Contract") + self.git(path, "config", "user.email", "zed-metadata-contract@example.invalid") + + def commit_all(self, path: Path, message: str) -> None: + self.git(path, "add", "-A") + self.git(path, "commit", "-m", message) + + def build_fixture(self) -> Path: + child = self.repos / "child" + self.init_repo(child) + (child / ".zpkg.toml").write_text( + """[package] +org = "acme" +name = "metadata-child" +version = "1.0.0" + +[package.repository] +vcs = "git" +url = "https://example.invalid/acme/metadata-child.git" +""", + encoding="utf-8", + ) + (child / "child.txt").write_text("child-runtime\n", encoding="utf-8") + self.commit_all(child, "child") + + root = self.repos / "root" + self.init_repo(root) + (root / ".zpkg.toml").write_text( + """[package] +org = "acme" +name = "metadata-root" +version = "1.0.0" + +[package.repository] +vcs = "git" +url = "https://example.invalid/acme/metadata-root.git" +""", + encoding="utf-8", + ) + (root / "root.txt").write_text("root-runtime\n", encoding="utf-8") + self.run( + [ + "git", + "-C", + root, + "-c", + "protocol.file.allow=always", + "submodule", + "add", + child, + "vendor/child", + ], + cwd=root, + ) + self.commit_all(root, "root with child") + return root + + def clone(self, source: Path, name: str) -> Path: + destination = self.runs / name + self.run( + ["git", "clone", "--no-recurse-submodules", source, destination], + cwd=self.root, + ) + self.git(destination, "config", "user.name", "Zed Metadata Contract") + self.git( + destination, + "config", + "user.email", + "zed-metadata-contract@example.invalid", + ) + return destination + + @staticmethod + def assert_no_archives(path: Path) -> None: + archives = sorted(path.rglob("*.tar.gz")) if path.exists() else [] + assert not archives, f"failed command left package archives: {archives}" + + @staticmethod + def assert_metadata_error(output: str) -> None: + lowered = output.lower() + assert any( + phrase in lowered + for phrase in ( + "must be a regular file", + "stage-zero regular git blob", + "indirect git submodule metadata", + ) + ), output + + def certify_regular_baseline(self, source: Path) -> None: + project = self.clone(source, "regular-baseline") + self.zed_cmd(project, "regular-baseline", "install", "--git-submodules") + assert (project / "vendor/child/child.txt").is_file() + stage = self.git(project, "ls-files", "--stage", "--", ".gitmodules") + assert stage.startswith("100644 ") or stage.startswith("100755 "), stage + assert " 0\t.gitmodules" in stage, stage + self.checks.append("regular stage-zero .gitmodules permits recursive install") + + def replace_with_symlink(self, project: Path) -> Path: + metadata = project / ".gitmodules" + target = project / "reviewed-gitmodules" + target.write_bytes(metadata.read_bytes()) + metadata.unlink() + os.symlink(target.name, metadata) + assert metadata.is_symlink() + return target + + def certify_symlink_pack_rejection(self, source: Path) -> None: + project = self.clone(source, "symlink-pack") + target = self.replace_with_symlink(project) + before = target.read_bytes() + output_dir = self.runs / "symlink-pack-output" + output = self.zed_cmd( + project, + "symlink-pack", + "pack", + "--out", + output_dir, + should_fail=True, + ) + self.assert_metadata_error(output) + self.assert_no_archives(output_dir) + assert target.read_bytes() == before + self.checks.append("symlinked .gitmodules rejects pack before archive creation") + + def certify_symlink_install_rejection(self, source: Path) -> None: + project = self.clone(source, "symlink-install") + target = self.replace_with_symlink(project) + before = target.read_bytes() + output = self.zed_cmd( + project, + "symlink-install", + "install", + "--git-submodules", + should_fail=True, + ) + self.assert_metadata_error(output) + assert not (project / "vendor/child/.git").exists() + assert not (project / ".zpkg.lock").exists() + assert target.read_bytes() == before + self.checks.append("symlinked .gitmodules rejects install before checkout or lock mutation") + + def certify_directory_pack_rejection(self, source: Path) -> None: + project = self.clone(source, "directory-pack") + metadata = project / ".gitmodules" + metadata.unlink() + metadata.mkdir() + (metadata / "payload").write_text("indirect\n", encoding="utf-8") + output_dir = self.runs / "directory-pack-output" + output = self.zed_cmd( + project, + "directory-pack", + "pack", + "--out", + output_dir, + should_fail=True, + ) + self.assert_metadata_error(output) + self.assert_no_archives(output_dir) + self.checks.append("directory .gitmodules rejects pack before Git parsing") + + def certify_gitlink_index_rejection(self, source: Path) -> None: + project = self.clone(source, "gitlink-index") + head = self.git(project, "rev-parse", "HEAD").strip() + self.git( + project, + "update-index", + "--add", + "--cacheinfo", + "160000", + head, + ".gitmodules", + ) + stage = self.git(project, "ls-files", "--stage", "--", ".gitmodules") + assert stage.startswith("160000 "), stage + + output_dir = self.runs / "gitlink-index-output" + output = self.zed_cmd( + project, + "gitlink-index", + "pack", + "--out", + output_dir, + should_fail=True, + ) + self.assert_metadata_error(output) + self.assert_no_archives(output_dir) + self.checks.append("gitlink-indexed .gitmodules rejects pack before archive creation") + + def certify_conflict_stage_rejection(self, source: Path) -> None: + project = self.clone(source, "conflict-stage") + blob = self.git(project, "hash-object", ".gitmodules").strip() + records = ( + f"0 {ZERO_OID}\t.gitmodules\n" + f"100644 {blob} 1\t.gitmodules\n" + f"100644 {blob} 2\t.gitmodules\n" + f"100644 {blob} 3\t.gitmodules\n" + ) + self.git(project, "update-index", "--index-info", stdin=records) + unmerged = self.git(project, "ls-files", "--unmerged", "--", ".gitmodules") + assert " 1\t.gitmodules" in unmerged, unmerged + assert " 2\t.gitmodules" in unmerged, unmerged + assert " 3\t.gitmodules" in unmerged, unmerged + + output = self.zed_cmd( + project, + "conflict-stage", + "install", + "--git-submodules", + should_fail=True, + ) + self.assert_metadata_error(output) + assert not (project / "vendor/child/.git").exists() + assert not (project / ".zpkg.lock").exists() + self.checks.append("conflicted .gitmodules index rejects install before mutation") + + def finish(self) -> None: + version = self.run([self.zed, "--version"]).strip() + evidence = { + "schema": "zed.gitmodules-metadata-evidence/v1", + "zed_version": version, + "zed_sha256": hashlib.sha256(self.zed.read_bytes()).hexdigest(), + "checks": self.checks, + "network_credentials": False, + "public_registry_mutation": False, + } + (self.evidence / "evidence.json").write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + assert len(self.checks) == 6, self.checks + self.log(f"\ncertified {len(self.checks)} .gitmodules metadata checks") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + contract = Contract(args.zed, args.work_root) + source = contract.build_fixture() + contract.certify_regular_baseline(source) + contract.certify_symlink_pack_rejection(source) + contract.certify_symlink_install_rejection(source) + contract.certify_directory_pack_rejection(source) + contract.certify_gitlink_index_rejection(source) + contract.certify_conflict_stage_rejection(source) + contract.finish() + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (AssertionError, RuntimeError, OSError, subprocess.SubprocessError) as error: + print(f".gitmodules metadata contract failed: {error}", file=sys.stderr) + raise diff --git a/scripts/gitops_dispatch_canary.py b/scripts/gitops_dispatch_canary.py new file mode 100755 index 00000000..a20531c4 --- /dev/null +++ b/scripts/gitops_dispatch_canary.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import re +import shutil +import stat +import subprocess +import sys +import tomllib +from pathlib import Path +from typing import Iterable, Sequence + + +SENSITIVE_ENV_FRAGMENTS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PRIVATE_KEY", + "ACCESS_KEY", + "API_KEY", + "AUTHORIZATION", + "COOKIE", +) +PIN = "32be546f5ee020c1de3b099a47e6760d00e3f6e4" + + +class CertificationError(RuntimeError): + pass + + +def executable_name(stem: str) -> str: + return f"{stem}.exe" if os.name == "nt" else stem + + +def copy_executable(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + if os.name != "nt": + destination.chmod(destination.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def sanitized_environment(home: Path, path_value: str | None = None) -> dict[str, str]: + environment = dict(os.environ) + for key in list(environment): + upper = key.upper() + if any(fragment in upper for fragment in SENSITIVE_ENV_FRAGMENTS): + environment.pop(key, None) + elif upper.startswith(("ZED_PKG_", "ZED_EXTERNAL_", "ZED_PROBE_")): + environment.pop(key, None) + environment["HOME"] = str(home) + environment["USERPROFILE"] = str(home) + environment["CI"] = "true" + if path_value is not None: + environment["PATH"] = path_value + return environment + + +def run( + command: Sequence[object], + *, + environment: dict[str, str], + cwd: Path | None = None, + expected: int | Iterable[int] = 0, +) -> subprocess.CompletedProcess[str]: + expected_codes = {expected} if isinstance(expected, int) else set(expected) + result = subprocess.run( + [str(item) for item in command], + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=45, + check=False, + ) + if result.returncode not in expected_codes: + rendered = " ".join(repr(str(item)) for item in command) + raise CertificationError( + f"command returned {result.returncode}, expected {sorted(expected_codes)}: {rendered}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + +def require(condition: bool, message: str) -> None: + if not condition: + raise CertificationError(message) + + +def combined(result: subprocess.CompletedProcess[str]) -> str: + return f"{result.stdout}\n{result.stderr}" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def build_catalog_fixture(root: Path, environment: dict[str, str]) -> Path: + catalog = root / "catalog" / "gitops" / "apps" + static_apps = root / "remote" / "argocd" / "apps" + catalog.mkdir(parents=True) + static_apps.mkdir(parents=True) + (static_apps / "daedalus.applications.yaml").write_text( + "kind: Application\n", encoding="utf-8" + ) + (root / ".gitmodules").write_text( + '[submodule "remote/deployments/fabrication-server-rs"]\n' + "\tpath = remote/deployments/fabrication-server-rs\n" + "\turl = git@github.com:daedalus-fab/fabrication-server.rs.git\n", + encoding="utf-8", + ) + run(["git", "-C", root, "init", "--quiet"], environment=environment) + run( + [ + "git", + "-C", + root, + "update-index", + "--add", + "--cacheinfo", + f"160000,{PIN},remote/deployments/fabrication-server-rs", + ], + environment=environment, + ) + record = { + "$schema": "../application.schema.json", + "apiVersion": "oresoftware.dev/v1alpha1", + "kind": "GitOpsApplication", + "metadata": {"name": "dd-fabrication-server"}, + "spec": { + "owner": "daedalus-fab", + "inventory": { + "mode": "git-submodule", + "path": "remote/deployments/fabrication-server-rs", + "repository": "git@github.com:daedalus-fab/fabrication-server.rs.git", + "revision": PIN, + }, + "source": { + "mode": "direct-repository", + "repository": "https://github.com/daedalus-fab/fabrication-server.rs", + "targetRevision": PIN, + "path": "k8s", + "renderer": "kustomize", + }, + "argo": { + "project": "daedalus", + "namespace": "daedalus", + "destinationServer": "https://kubernetes.default.svc", + "automated": False, + "prune": False, + "selfHeal": False, + }, + "migration": { + "phase": "pilot-inert", + "staticApplication": "remote/argocd/apps/daedalus.applications.yaml", + }, + }, + } + record_path = catalog / "dd-fabrication-server.json" + record_path.write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return record_path + + +def validate_package_manifest(product: Path) -> None: + manifest = tomllib.loads((product / ".zpkg.toml").read_text(encoding="utf-8")) + outputs = set(manifest.get("build", {}).get("outputs", [])) + require( + outputs == {"target/release/zed", "target/release/zed-gitops"}, + f"unexpected packaged outputs: {sorted(outputs)}", + ) + binaries = manifest.get("bin", {}) + require( + binaries.get("zed") == "target/release/zed" + and binaries.get("zed-gitops") == "target/release/zed-gitops", + "Zed package must install both sibling executables", + ) + smoke = manifest.get("publish", {}).get("smoke_test", "") + require( + "zed-gitops" in smoke and "gitops validate --help" in smoke, + "publish smoke test must exercise root dispatch and the sibling validator", + ) + + +def certify(args: argparse.Namespace) -> dict[str, object]: + candidate = args.candidate + require( + re.fullmatch(r"[0-9a-f]{40}", candidate) is not None, + "candidate must be an exact lowercase 40-character commit", + ) + + zed = args.zed.resolve(strict=True) + zed_gitops = args.zed_gitops.resolve(strict=True) + probe = args.probe.resolve(strict=True) + product = args.product.resolve(strict=True) + work = args.work.resolve() + if work.exists(): + shutil.rmtree(work) + work.mkdir(parents=True) + home = work / "home" + home.mkdir() + original_path = os.environ.get("PATH", "") + base_environment = sanitized_environment(home, original_path) + checks: list[str] = [] + + validate_package_manifest(product) + checks.append("package-installs-both-sibling-binaries") + + direct_help = run( + [zed_gitops, "validate", "--help"], environment=base_environment + ) + require("Usage: zed-gitops validate" in combined(direct_help), combined(direct_help)) + checks.append("standalone-validator-help") + + sibling = work / "sibling" + sibling_zed = sibling / executable_name("zed") + sibling_gitops = sibling / executable_name("zed-gitops") + copy_executable(zed, sibling_zed) + copy_executable(zed_gitops, sibling_gitops) + + root_help = run([sibling_zed, "--help"], environment=base_environment) + root_help_text = combined(root_help) + require("gitops" in root_help_text, "root help omitted gitops") + require( + "Validate GitOps composition" in root_help_text, + "root help omitted the GitOps contract summary", + ) + checks.append("root-help-advertises-gitops") + + routed_help = run( + [sibling_zed, "gitops", "validate", "--help"], + environment=base_environment, + ) + routed_help_text = combined(routed_help) + require("Usage: zed-gitops validate" in routed_help_text, routed_help_text) + require("--offline" in routed_help_text, routed_help_text) + checks.append("root-dispatches-to-sibling-validator") + + alias_help = run( + [sibling_zed, "help", "gitops"], environment=base_environment + ) + require("Usage: zed-gitops" in combined(alias_help), combined(alias_help)) + checks.append("help-alias-reaches-validator") + + completion = run( + [sibling_zed, "completions", "bash"], environment=base_environment + ) + require( + "gitops" in completion.stdout and "validate" in completion.stdout, + "Bash completion omitted gitops validate", + ) + checks.append("completion-exposes-gitops-contract") + + fixture = work / "fixture" + fixture.mkdir() + record_path = build_catalog_fixture(fixture, base_environment) + validate_args = [ + "validate", + "--root", + fixture, + "--offline", + "--strict", + "--format", + "json", + ] + direct = run([sibling_gitops, *validate_args], environment=base_environment) + routed = run( + [sibling_zed, "gitops", *validate_args], environment=base_environment + ) + direct_report = json.loads(direct.stdout) + routed_report = json.loads(routed.stdout) + require(direct_report == routed_report, "root and standalone reports differ") + require( + routed_report.get("valid") is True + and routed_report.get("records") == 1 + and routed_report.get("errors") == 0, + f"unexpected valid report: {routed_report}", + ) + checks.append("root-and-standalone-validation-parity") + + online = run( + [ + sibling_zed, + "gitops", + "validate", + "--root", + fixture, + "--strict", + "--format", + "json", + ], + environment=base_environment, + expected=1, + ) + require( + "online validation is not implemented" in combined(online), + "online-mode omission did not fail explicitly", + ) + checks.append("online-mode-fails-explicitly") + + record = json.loads(record_path.read_text(encoding="utf-8")) + record["spec"]["source"]["targetRevision"] = "a" * 40 + record_path.write_text( + json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + drift = run( + [sibling_zed, "gitops", *validate_args], + environment=base_environment, + expected=2, + ) + drift_report = json.loads(drift.stdout) + rule_ids = { + diagnostic.get("rule_id") + for diagnostic in drift_report.get("diagnostics", []) + if isinstance(diagnostic, dict) + } + require("source.pin-drift" in rule_ids, f"pin drift was not reported: {drift_report}") + checks.append("policy-failure-exit-code-and-diagnostic") + + missing = work / "missing" + missing_zed = missing / executable_name("zed") + copy_executable(zed, missing_zed) + empty_path = work / "empty-path" + empty_path.mkdir() + missing_environment = sanitized_environment(home, str(empty_path)) + missing_result = run( + [missing_zed, "gitops", "validate", "--help"], + environment=missing_environment, + expected=1, + ) + require( + "requires `zed-gitops`" in combined(missing_result), + "missing known extension did not fail with an actionable diagnostic", + ) + checks.append("missing-known-extension-fails-closed") + + cwd_attack = work / "cwd-attack" + cwd_attack.mkdir() + cwd_probe = cwd_attack / executable_name("zed-gitops") + copy_executable(probe, cwd_probe) + cwd_marker = cwd_attack / "marker.json" + cwd_environment = sanitized_environment(home, str(empty_path)) + cwd_environment["ZED_PROBE_OUTPUT"] = str(cwd_marker) + cwd_environment["ZED_PROBE_EXIT"] = "73" + cwd_result = run( + [missing_zed, "gitops", "validate", "--help"], + environment=cwd_environment, + cwd=cwd_attack, + expected=1, + ) + require("requires `zed-gitops`" in combined(cwd_result), combined(cwd_result)) + require(not cwd_marker.exists(), "dispatcher executed zed-gitops from the working directory") + checks.append("working-directory-is-never-searched") + + relative_root = work / "relative-path" + relative_plugins = relative_root / "relative-bin" + relative_plugins.mkdir(parents=True) + relative_probe = relative_plugins / executable_name("zed-gitops") + copy_executable(probe, relative_probe) + relative_marker = relative_root / "marker.json" + relative_environment = sanitized_environment(home, "relative-bin") + relative_environment["ZED_PROBE_OUTPUT"] = str(relative_marker) + relative_environment["ZED_PROBE_EXIT"] = "74" + relative_result = run( + [missing_zed, "gitops", "validate", "--help"], + environment=relative_environment, + cwd=relative_root, + expected=1, + ) + require("requires `zed-gitops`" in combined(relative_result), combined(relative_result)) + require(not relative_marker.exists(), "dispatcher searched a relative PATH entry") + checks.append("relative-path-entry-is-rejected") + + path_plugins = work / "absolute-plugins" + path_plugins.mkdir() + path_probe = path_plugins / executable_name("zed-gitops") + copy_executable(probe, path_probe) + precedence_marker = work / "precedence-marker.json" + precedence_environment = sanitized_environment( + home, os.pathsep.join([str(path_plugins), original_path]) + ) + precedence_environment["ZED_PROBE_OUTPUT"] = str(precedence_marker) + precedence_environment["ZED_PROBE_EXIT"] = "75" + precedence = run( + [sibling_zed, "gitops", "validate", "--help"], + environment=precedence_environment, + ) + require("Usage: zed-gitops validate" in combined(precedence), combined(precedence)) + require(not precedence_marker.exists(), "PATH extension shadowed the sibling validator") + checks.append("sibling-validator-precedes-path") + + generic_probe = path_plugins / executable_name("zed-probe") + copy_executable(probe, generic_probe) + probe_output = work / "probe-output.json" + probe_home = work / "home with spaces" + probe_home.mkdir() + probe_environment = sanitized_environment( + home, os.pathsep.join([str(path_plugins), original_path]) + ) + probe_environment["ZED_PROBE_OUTPUT"] = str(probe_output) + probe_environment["ZED_PROBE_EXIT"] = "37" + literal_arguments = ["literal;$HOME", "two words", "--flag=value"] + probe_result = run( + [ + missing_zed, + "--home", + probe_home, + "--git-submodules=false", + "probe", + *literal_arguments, + ], + environment=probe_environment, + expected=37, + ) + require(probe_result.stdout == "" and probe_result.stderr == "", combined(probe_result)) + probe_payload = json.loads(probe_output.read_text(encoding="utf-8")) + require(probe_payload.get("args") == literal_arguments, f"arguments changed: {probe_payload}") + expected_environment = { + "ZED_EXTERNAL_SUBCOMMAND": "probe", + "ZED_PKG_HOME": str(probe_home), + "ZED_PKG_GIT_SUBMODULES": "false", + } + require( + probe_payload.get("env") == expected_environment, + f"root-option environment changed: {probe_payload}", + ) + checks.append("arguments-environment-and-exit-code-are-preserved") + + collision_probe = path_plugins / executable_name("zed-install") + copy_executable(probe, collision_probe) + collision_marker = work / "collision-marker.json" + collision_environment = sanitized_environment( + home, os.pathsep.join([str(path_plugins), original_path]) + ) + collision_environment["ZED_PROBE_OUTPUT"] = str(collision_marker) + collision_environment["ZED_PROBE_EXIT"] = "91" + collision = run( + [missing_zed, "install", "--help"], environment=collision_environment + ) + require("Usage:" in combined(collision), combined(collision)) + require(not collision_marker.exists(), "external executable shadowed a built-in command") + checks.append("built-in-command-cannot-be-shadowed") + + return { + "$schema": "zed-pkg-test/gitops-dispatch-canary/v1", + "candidate": candidate, + "platform": { + "system": platform.system(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + "binaries": { + "zedSha256": sha256(zed), + "zedGitopsSha256": sha256(zed_gitops), + "probeSha256": sha256(probe), + }, + "result": "passed", + "checkCount": len(checks), + "checks": checks, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--zed-gitops", type=Path, required=True) + parser.add_argument("--probe", type=Path, required=True) + parser.add_argument("--product", type=Path, required=True) + parser.add_argument("--candidate", required=True) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + evidence = certify(args) + except (CertificationError, OSError, subprocess.SubprocessError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + f"certified {evidence['checkCount']} GitOps dispatch checks " + f"against {evidence['candidate']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gitops_dispatch_canary_entry.py b/scripts/gitops_dispatch_canary_entry.py new file mode 100644 index 00000000..a51c723f --- /dev/null +++ b/scripts/gitops_dispatch_canary_entry.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Cross-platform entrypoint for the independent GitOps dispatch canary.""" + +from __future__ import annotations + +import gitops_dispatch_canary as canary + + +_original_combined = canary.combined + + +def platform_normalized_output(result): + """Normalize only Windows' conventional executable suffix for assertions.""" + return _original_combined(result).replace("zed-gitops.exe", "zed-gitops") + + +canary.combined = platform_normalized_output + + +if __name__ == "__main__": + raise SystemExit(canary.main()) diff --git a/scripts/gitops_dispatch_trailing_options.py b/scripts/gitops_dispatch_trailing_options.py new file mode 100644 index 00000000..9ebe6798 --- /dev/null +++ b/scripts/gitops_dispatch_trailing_options.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import shutil +import stat +import subprocess +import sys +from pathlib import Path +from typing import Sequence + + +SENSITIVE_ENV_FRAGMENTS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PRIVATE_KEY", + "ACCESS_KEY", + "API_KEY", + "AUTHORIZATION", + "COOKIE", +) + + +class CertificationError(RuntimeError): + pass + + +def executable_name(stem: str) -> str: + return f"{stem}.exe" if os.name == "nt" else stem + + +def copy_executable(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + if os.name != "nt": + destination.chmod(destination.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def sanitized_environment(home: Path, path_value: str) -> dict[str, str]: + environment = dict(os.environ) + for key in list(environment): + upper = key.upper() + if any(fragment in upper for fragment in SENSITIVE_ENV_FRAGMENTS): + environment.pop(key, None) + elif upper.startswith(("ZED_PKG_", "ZED_EXTERNAL_", "ZED_PROBE_")): + environment.pop(key, None) + environment["HOME"] = str(home) + environment["USERPROFILE"] = str(home) + environment["PATH"] = path_value + environment["CI"] = "true" + return environment + + +def run( + command: Sequence[object], + *, + environment: dict[str, str], + expected: set[int] | None = None, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [str(item) for item in command], + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=30, + check=False, + ) + if expected is not None and result.returncode not in expected: + rendered = " ".join(repr(str(item)) for item in command) + raise CertificationError( + f"command returned {result.returncode}, expected {sorted(expected)}: {rendered}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + +def require(condition: bool, message: str) -> None: + if not condition: + raise CertificationError(message) + + +def invoke_probe( + zed: Path, + arguments: Sequence[object], + *, + environment: dict[str, str], + marker: Path, +) -> tuple[subprocess.CompletedProcess[str], dict[str, object]]: + marker.unlink(missing_ok=True) + child_environment = dict(environment) + child_environment["ZED_PROBE_OUTPUT"] = str(marker) + child_environment["ZED_PROBE_EXIT"] = "0" + result = run([zed, "probe", *arguments], environment=child_environment, expected={0}) + require(marker.is_file(), "external probe did not produce evidence") + payload = json.loads(marker.read_text(encoding="utf-8")) + require(isinstance(payload, dict), "probe evidence must be a JSON object") + return result, payload + + +def certify(args: argparse.Namespace) -> dict[str, object]: + zed = args.zed.resolve(strict=True) + probe = args.probe.resolve(strict=True) + work = args.work.resolve() + if work.exists(): + shutil.rmtree(work) + work.mkdir(parents=True) + home = work / "home" + home.mkdir() + plugins = work / "plugins" + probe_executable = plugins / executable_name("zed-probe") + copy_executable(probe, probe_executable) + original_path = os.environ.get("PATH", "") + environment = sanitized_environment( + home, os.pathsep.join([str(plugins), original_path]) + ) + checks: list[str] = [] + + trailing_home = work / "trailing home" + trailing_home.mkdir() + _, trailing = invoke_probe( + zed, + [ + "literal-before", + "--home", + trailing_home, + "--git-submodules=false", + "literal-after", + ], + environment=environment, + marker=work / "trailing.json", + ) + require( + trailing.get("args") == ["literal-before", "literal-after"], + f"recognized trailing root options leaked into child argv: {trailing}", + ) + require( + trailing.get("env") + == { + "ZED_EXTERNAL_SUBCOMMAND": "probe", + "ZED_PKG_HOME": str(trailing_home), + "ZED_PKG_GIT_SUBMODULES": "false", + }, + f"recognized trailing root options did not become child environment: {trailing}", + ) + checks.append("recognized-trailing-root-options-are-lifted") + + literal_arguments = [ + "literal-before", + "--", + "--home", + "child-owned-home", + "--git-submodules=false", + ] + _, after_double_dash = invoke_probe( + zed, + literal_arguments, + environment=environment, + marker=work / "double-dash.json", + ) + require( + after_double_dash.get("args") == literal_arguments, + f"literal double dash did not terminate root-option extraction: {after_double_dash}", + ) + require( + after_double_dash.get("env") + == { + "ZED_EXTERNAL_SUBCOMMAND": "probe", + "ZED_PKG_HOME": "", + "ZED_PKG_GIT_SUBMODULES": "", + }, + f"arguments after double dash changed root-option environment: {after_double_dash}", + ) + checks.append("literal-double-dash-terminates-root-option-extraction") + + malformed_marker = work / "malformed.json" + malformed_environment = dict(environment) + malformed_environment["ZED_PROBE_OUTPUT"] = str(malformed_marker) + malformed_environment["ZED_PROBE_EXIT"] = "0" + malformed = run( + [zed, "probe", "--git-submodules=maybe", "payload"], + environment=malformed_environment, + ) + require(malformed.returncode != 0, "malformed trailing boolean was accepted") + require( + not malformed_marker.exists(), + "malformed trailing boolean reached the external executable", + ) + checks.append("malformed-trailing-root-boolean-fails-closed") + + return { + "$schema": "zed-pkg-test/gitops-dispatch-trailing-options/v1", + "candidate": args.candidate, + "result": "passed", + "checkCount": len(checks), + "checks": checks, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--probe", type=Path, required=True) + parser.add_argument("--candidate", required=True) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + evidence = certify(args) + except (CertificationError, OSError, subprocess.SubprocessError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + f"certified {evidence['checkCount']} trailing-option checks " + f"against {evidence['candidate']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/global_package_profile_canary.py b/scripts/global_package_profile_canary.py new file mode 100644 index 00000000..f75035f3 --- /dev/null +++ b/scripts/global_package_profile_canary.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +"""Black-box certification for zed global executable package profiles. + +The harness builds no registry service and uses no credential. It creates two +small immutable package artifacts in a disposable file:// registry, invokes only +the compiled zed CLI, and checks PATH ownership, frozen restoration, collision +rollback, and tamper-preserving uninstall behavior. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import shutil +import stat +import subprocess +import tarfile +from pathlib import Path +from typing import Any + +COMMAND = "zed-global-canary" +VERSION = "1.0.0" +VCS_COMMIT = "0123456789abcdef0123456789abcdef01234567" + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def manifest(name: str) -> bytes: + return f'''[package] +org = "canary" +name = "{name}" +version = "{VERSION}" +description = "Disposable global package canary {name}" +license = "MIT" + +[package.repository] +vcs = "git" +url = "https://example.invalid/canary/{name}" + +[bin] +{COMMAND} = "bin/{COMMAND}" + +[install] +adapter = "none" +'''.encode() + + +def command_bytes(name: str) -> bytes: + return ( + "#!/bin/sh\n" + "set -eu\n" + f"printf '%s\\n' 'canary/{name}:{VERSION}'\n" + ).encode() + + +def add_tar_member(archive: tarfile.TarFile, name: str, data: bytes, mode: int) -> None: + info = tarfile.TarInfo(name) + info.size = len(data) + info.mode = mode + info.mtime = 0 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + archive.addfile(info, io.BytesIO(data)) + + +def artifact(name: str) -> bytes: + output = io.BytesIO() + with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w", format=tarfile.GNU_FORMAT) as archive: + add_tar_member(archive, "pkg/.zpkg.toml", manifest(name), 0o644) + add_tar_member(archive, f"pkg/bin/{COMMAND}", command_bytes(name), 0o755) + add_tar_member(archive, "pkg/LICENSE", b"MIT\n", 0o644) + return output.getvalue() + + +def publish_fixture(registry: Path, name: str) -> dict[str, Any]: + payload = artifact(name) + digest = sha256(payload) + artifact_path = registry / "artifacts" / f"{digest}.tar.gz" + artifact_path.parent.mkdir(parents=True, exist_ok=True) + artifact_path.write_bytes(payload) + + package_dir = registry / "packages" / "canary" / name + version_dir = package_dir / "versions" + version_dir.mkdir(parents=True, exist_ok=True) + package_metadata = { + "org": "canary", + "name": name, + "description": f"Disposable global package canary {name}", + "vcs": "git", + "repo_url": f"https://example.invalid/canary/{name}", + "latest": VERSION, + "tags": ["global-package-canary"], + "versions": [VERSION], + } + version_metadata = { + "org": "canary", + "name": name, + "version": VERSION, + "sha256": digest, + "size": len(payload), + "format": "tar.gz", + "vcs_tag": f"v{VERSION}", + "vcs_commit": VCS_COMMIT, + "download_url": artifact_path.resolve().as_uri(), + "published_at": "2026-08-04T00:00:00Z", + "yanked": False, + } + (package_dir / "package.json").write_text( + json.dumps(package_metadata, sort_keys=True, indent=2) + "\n" + ) + (version_dir / f"{VERSION}.json").write_text( + json.dumps(version_metadata, sort_keys=True, indent=2) + "\n" + ) + return { + "package": f"canary/{name}@{VERSION}", + "sha256": digest, + "size": len(payload), + } + + +def executable_path(bin_dir: Path) -> Path: + return bin_dir / (f"{COMMAND}.exe" if os.name == "nt" else COMMAND) + + +def profile_path(home: Path, name: str) -> Path: + return home / "global" / "profiles" / "canary" / name + + +def run( + zed: Path, + home: Path, + bin_dir: Path, + registry: Path, + args: list[str], + evidence: list[dict[str, Any]], + *, + expected: int = 0, +) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env.update( + { + "ZED_PKG_HOME": str(home), + "ZED_PKG_GLOBAL_BIN_DIR": str(bin_dir), + "ZED_PKG_REGISTRY": registry.resolve().as_uri(), + "ZED_PKG_INTERACTIVE": "0", + "GIT_TERMINAL_PROMPT": "0", + } + ) + env.pop("ZED_PKG_TOKEN", None) + completed = subprocess.run( + [str(zed), *args], + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + evidence.append( + { + "argv": args, + "returncode": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + } + ) + if completed.returncode != expected: + raise AssertionError( + f"zed {' '.join(args)} returned {completed.returncode}, expected {expected}\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + return completed + + +def invoke_command(path: Path) -> str: + if not path.is_file(): + raise AssertionError(f"global command was not materialized: {path}") + if os.name != "nt" and not path.stat().st_mode & stat.S_IXUSR: + raise AssertionError(f"global command is not executable: {path}") + completed = subprocess.run( + [str(path)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + raise AssertionError( + f"global command failed: {completed.returncode}\n{completed.stderr}" + ) + return completed.stdout.strip() + + +def assert_no_credentials(root: Path) -> None: + forbidden = ["token", "password", "authorization", "credential"] + for path in root.rglob("*"): + if not path.is_file(): + continue + try: + text = path.read_text(errors="ignore").lower() + except OSError: + continue + for marker in forbidden: + if marker in text: + raise AssertionError(f"credential marker {marker!r} appeared in {path}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + options = parser.parse_args() + + zed = options.zed.resolve() + if not zed.is_file(): + raise SystemExit(f"zed binary does not exist: {zed}") + + work = options.work.resolve() + if work.exists(): + shutil.rmtree(work) + work.mkdir(parents=True) + registry = work / "registry" + home = work / "home" + bin_dir = work / "bin" + evidence_commands: list[dict[str, Any]] = [] + fixtures = [publish_fixture(registry, "alpha"), publish_fixture(registry, "beta")] + + # First installation and executable invocation. + run( + zed, + home, + bin_dir, + registry, + ["global", "install", f"canary/alpha@{VERSION}"], + evidence_commands, + ) + command = executable_path(bin_dir) + if invoke_command(command) != f"canary/alpha:{VERSION}": + raise AssertionError("the installed command did not come from alpha") + listed = run( + zed, + home, + bin_dir, + registry, + ["global", "list"], + evidence_commands, + ) + if "canary/alpha@1.0.0" not in listed.stdout or f"bins: {COMMAND}" not in listed.stdout: + raise AssertionError(f"global list omitted the installed package or command:\n{listed.stdout}") + + # A second package with the same command must fail before changing PATH and + # must not leave a ghost top-level profile behind. + collision = run( + zed, + home, + bin_dir, + registry, + ["global", "install", f"canary/beta@{VERSION}"], + evidence_commands, + expected=1, + ) + if "collision" not in collision.stderr.lower(): + raise AssertionError(f"collision failure was not explicit:\n{collision.stderr}") + if profile_path(home, "beta").exists(): + raise AssertionError("failed collision install left a ghost beta profile") + if invoke_command(command) != f"canary/alpha:{VERSION}": + raise AssertionError("failed collision install changed the active command") + + # Remove profile materialization and the registry, then restore exactly from + # the existing lock and content-addressed store. + shutil.rmtree(profile_path(home, "alpha") / "zed_modules") + command.unlink() + shutil.rmtree(registry) + run( + zed, + home, + bin_dir, + registry, + ["global", "install", "--frozen", "canary/alpha"], + evidence_commands, + ) + if invoke_command(command) != f"canary/alpha:{VERSION}": + raise AssertionError("frozen store-only restore produced the wrong command") + + # Modified commands are not deleted during uninstall. + command.write_text("#!/bin/sh\nprintf '%s\\n' 'user replacement'\n") + command.chmod(0o755) + uninstalled = run( + zed, + home, + bin_dir, + registry, + ["global", "uninstall", "canary/alpha"], + evidence_commands, + ) + if profile_path(home, "alpha").exists(): + raise AssertionError("uninstall did not remove the alpha profile") + if invoke_command(command) != "user replacement": + raise AssertionError("uninstall deleted or replaced a user-modified command") + if "changed after Zed installed it" not in uninstalled.stderr: + raise AssertionError("tamper-preserving uninstall did not emit its warning") + + # A clean second installation proves unchanged owned commands are removed. + clean_registry = work / "clean-registry" + clean_home = work / "clean-home" + clean_bin = work / "clean-bin" + publish_fixture(clean_registry, "alpha") + run( + zed, + clean_home, + clean_bin, + clean_registry, + ["install", "--global", f"canary/alpha@{VERSION}"], + evidence_commands, + ) + clean_command = executable_path(clean_bin) + if invoke_command(clean_command) != f"canary/alpha:{VERSION}": + raise AssertionError("compatibility install route produced the wrong command") + run( + zed, + clean_home, + clean_bin, + clean_registry, + ["uninstall", "--global", "canary/alpha"], + evidence_commands, + ) + if clean_command.exists() or profile_path(clean_home, "alpha").exists(): + raise AssertionError("clean uninstall retained Zed-owned state") + + assert_no_credentials(work) + result = { + "schema": "zed.global-package-profile-canary/v1", + "zed_sha256": sha256(zed.read_bytes()), + "fixtures": fixtures, + "commands": evidence_commands, + "checks": { + "first_install_and_execute": True, + "offline_profile_listing": True, + "collision_is_explicit": True, + "collision_rolls_back_profile": True, + "collision_preserves_active_command": True, + "frozen_store_only_restore": True, + "tampered_command_preserved": True, + "unchanged_command_removed": True, + "credential_markers_absent": True, + }, + } + options.evidence.parent.mkdir(parents=True, exist_ok=True) + options.evidence.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n") + print("global executable package profile canary passed") + + +if __name__ == "__main__": + main() diff --git a/scripts/install_pinned_kustomize.py b/scripts/install_pinned_kustomize.py new file mode 100644 index 00000000..0090b9ff --- /dev/null +++ b/scripts/install_pinned_kustomize.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import stat +import sys +import tarfile +import tempfile +import urllib.request +import zipfile +from pathlib import Path + + +VERSION = "5.8.1" +BASE_URL = ( + "https://github.com/kubernetes-sigs/kustomize/releases/download/" + f"kustomize/v{VERSION}" +) +ASSETS: dict[tuple[str, str], tuple[str, str]] = { + ("linux", "x86_64"): ( + f"kustomize_v{VERSION}_linux_amd64.tar.gz", + "029a7f0f4e1932c52a0476cf02a0fd855c0bb85694b82c338fc648dcb53a819d", + ), + ("linux", "aarch64"): ( + f"kustomize_v{VERSION}_linux_arm64.tar.gz", + "0953ea3e476f66d6ddfcd911d750f5167b9365aa9491b2326398e289fef2c142", + ), + ("darwin", "x86_64"): ( + f"kustomize_v{VERSION}_darwin_amd64.tar.gz", + "ee7cf0c1e3592aa7bb66ba82b359933a95e7f2e0b36e5f53ed0a4535b017f2f8", + ), + ("darwin", "arm64"): ( + f"kustomize_v{VERSION}_darwin_arm64.tar.gz", + "8886f8a78474e608cc81234f729fda188a9767da23e28925802f00ece2bab288", + ), + ("win32", "amd64"): ( + f"kustomize_v{VERSION}_windows_amd64.zip", + "8ec7f5e815e526d4622c06df0a7793d8cfb6eb1c74f816b46166097fef8b26c6", + ), + ("win32", "x86_64"): ( + f"kustomize_v{VERSION}_windows_amd64.zip", + "8ec7f5e815e526d4622c06df0a7793d8cfb6eb1c74f816b46166097fef8b26c6", + ), + ("win32", "arm64"): ( + f"kustomize_v{VERSION}_windows_arm64.zip", + "f6f5090c373965760a30a59af84c81853b80d2c7dbb3236424f440e47eed3f5a", + ), +} + + +class InstallError(RuntimeError): + pass + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def safe_extract_zip(archive: Path, destination: Path) -> None: + root = destination.resolve() + with zipfile.ZipFile(archive) as bundle: + for member in bundle.infolist(): + target = (destination / member.filename).resolve() + if target != root and root not in target.parents: + raise InstallError(f"unsafe zip member: {member.filename}") + bundle.extractall(destination) + + +def safe_extract_tar(archive: Path, destination: Path) -> None: + root = destination.resolve() + with tarfile.open(archive, "r:gz") as bundle: + for member in bundle.getmembers(): + target = (destination / member.name).resolve() + if target != root and root not in target.parents: + raise InstallError(f"unsafe tar member: {member.name}") + if member.issym() or member.islnk(): + raise InstallError(f"archive links are not allowed: {member.name}") + bundle.extractall(destination, filter="data") + + +def normalized_platform() -> tuple[str, str]: + machine = platform.machine().lower() + aliases = { + "amd64": "amd64", + "x86_64": "x86_64", + "aarch64": "aarch64", + "arm64": "arm64", + } + return sys.platform, aliases.get(machine, machine) + + +def install(destination: Path, metadata_path: Path) -> Path: + key = normalized_platform() + asset = ASSETS.get(key) + if asset is None: + raise InstallError(f"unsupported runner platform: {key[0]}/{key[1]}") + asset_name, expected_digest = asset + url = f"{BASE_URL}/{asset_name}" + + destination.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="kustomize-install-") as temporary: + temporary_root = Path(temporary) + archive = temporary_root / asset_name + request = urllib.request.Request( + url, + headers={"User-Agent": "zed-pkg-test-direct-source-render/1"}, + ) + with urllib.request.urlopen(request, timeout=90) as response, archive.open("wb") as output: + shutil.copyfileobj(response, output) + actual_digest = sha256(archive) + if actual_digest != expected_digest: + raise InstallError( + f"Kustomize asset digest mismatch: expected {expected_digest}, got {actual_digest}" + ) + + extracted = temporary_root / "extracted" + extracted.mkdir() + if asset_name.endswith(".zip"): + safe_extract_zip(archive, extracted) + else: + safe_extract_tar(archive, extracted) + + executable_name = "kustomize.exe" if os.name == "nt" else "kustomize" + matches = [path for path in extracted.rglob(executable_name) if path.is_file()] + if len(matches) != 1: + raise InstallError( + f"expected exactly one {executable_name} in {asset_name}, found {len(matches)}" + ) + target = destination / executable_name + shutil.copy2(matches[0], target) + if os.name != "nt": + target.chmod(target.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + metadata_path.parent.mkdir(parents=True, exist_ok=True) + metadata_path.write_text( + json.dumps( + { + "$schema": "zed-pkg-test/pinned-kustomize-install/v1", + "version": VERSION, + "platform": {"system": key[0], "machine": key[1]}, + "asset": asset_name, + "assetUrl": url, + "assetSha256": expected_digest, + "executable": str(target), + "executableSha256": sha256(target), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return target + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--destination", type=Path, required=True) + parser.add_argument("--metadata", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + executable = install(args.destination.resolve(), args.metadata.resolve()) + except (InstallError, OSError, urllib.error.URLError, tarfile.TarError, zipfile.BadZipFile) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print(executable) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/install_recovery_matrix.py b/scripts/install_recovery_matrix.py new file mode 100644 index 00000000..0512b97f --- /dev/null +++ b/scripts/install_recovery_matrix.py @@ -0,0 +1,659 @@ +#!/usr/bin/env python3 +"""Hermetic install recovery, rollback, and idempotency coverage for zed.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +import tomllib +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from threading import Barrier +from typing import Mapping, Sequence + +ORG = "zed-install-recovery" +VERSION = "1.0.0" +BASE_GRAPH: dict[str, tuple[str, ...]] = { + "leaf-a": (), + "leaf-b": (), + "root": ("leaf-a", "leaf-b"), +} +PREFETCH_RE = re.compile( + r"recursive install prefetch: (?P\d+) package\(s\), " + r"up to (?P\d+) concurrent, (?P\d+) downloaded" +) +PACKAGE_BLOCK_RE = re.compile( + r"(?ms)^\[\[package\]\]\n.*?(?=^\[\[package\]\]|\Z)" +) +SHA_ASSIGNMENT_RE = re.compile(r'(?m)^sha256 = "[0-9a-f]{64}"$') + + +@dataclass(frozen=True) +class Completed: + command: tuple[str, ...] + returncode: int + output: str + + +def shell_quote(value: str) -> str: + if re.fullmatch(r"[A-Za-z0-9_./:@%+=,-]+", value): + return value + return "'" + value.replace("'", "'\"'\"'") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, + timeout: int = 240, +) -> Completed: + argv = tuple(str(value) for value in command) + shown = " ".join(shell_quote(value) for value in argv) + print(f"\n$ (cd {cwd or Path.cwd()} && {shown})", flush=True) + completed = subprocess.run( + argv, + cwd=cwd, + env=dict(env) if env is not None else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + timeout=timeout, + ) + if completed.stdout: + print(completed.stdout, end="", flush=True) + return Completed(argv, completed.returncode, completed.stdout) + + +def require_success(completed: Completed) -> None: + if completed.returncode != 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise RuntimeError(f"command failed with exit code {completed.returncode}: {shown}") + + +def require_failure(completed: Completed, *expected_fragments: str) -> None: + if completed.returncode == 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise AssertionError(f"command unexpectedly succeeded: {shown}") + for fragment in expected_fragments: + if fragment not in completed.output: + raise AssertionError( + f"failed command output did not contain {fragment!r}:\n{completed.output}" + ) + + +def zed_command(zed: Path, registry: Path, home: Path, *args: str) -> list[str | Path]: + return [ + zed, + "--registry", + f"file://{registry}", + "--home", + home, + *args, + ] + + +def write_package(root: Path, name: str, dependencies: Sequence[str]) -> None: + root.mkdir(parents=True, exist_ok=True) + lines = [ + "[package]", + f'org = "{ORG}"', + f'name = "{name}"', + f'version = "{VERSION}"', + 'description = "install recovery fixture"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://github.com/zed-pkg-test/zed-pkg-e2e"', + ] + if dependencies: + lines.extend(("", "[dependencies]")) + for dependency in dependencies: + lines.append(f'"{ORG}/{dependency}" = "={VERSION}"') + (root / ".zpkg.toml").write_text("\n".join(lines) + "\n", encoding="utf-8") + (root / "payload.txt").write_text(f"{ORG}/{name}@{VERSION}\n", encoding="utf-8") + + +def write_consumer(root: Path, dependencies: Sequence[str]) -> None: + root.mkdir(parents=True, exist_ok=True) + lines = [ + "[package]", + 'org = "install-recovery-consumer"', + f'name = "{root.name}"', + 'version = "0.1.0"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://example.invalid/install-recovery-consumer"', + "", + "[dependencies]", + ] + for dependency in dependencies: + lines.append(f'"{ORG}/{dependency}" = "={VERSION}"') + lines.append("") + (root / ".zpkg.toml").write_text("\n".join(lines), encoding="utf-8") + + +def manifest_dependencies(project: Path) -> set[str]: + with (project / ".zpkg.toml").open("rb") as handle: + document = tomllib.load(handle) + return set((document.get("dependencies") or {}).keys()) + + +def lock_packages(project: Path) -> list[dict[str, object]]: + with (project / ".zpkg.lock").open("rb") as handle: + document = tomllib.load(handle) + packages = document.get("package") or [] + if not isinstance(packages, list): + raise AssertionError("lockfile package table is not a list") + return packages + + +def lock_keys(project: Path) -> set[str]: + return { + f"{package['org']}/{package['name']}" + for package in lock_packages(project) + } + + +def expected_keys(names: Sequence[str]) -> set[str]: + return {f"{ORG}/{name}" for name in names} + + +def assert_lock(project: Path, names: Sequence[str]) -> None: + actual = lock_keys(project) + expected = expected_keys(names) + if actual != expected: + raise AssertionError( + f"lock graph mismatch for {project}: expected {sorted(expected)}, got {sorted(actual)}" + ) + + +def assert_symlink_install( + project: Path, + home: Path, + names: Sequence[str], +) -> dict[str, Path]: + store = (home / "store").resolve() + targets: dict[str, Path] = {} + for name in names: + installed = project / "zed_modules" / ORG / name + if not installed.is_symlink(): + raise AssertionError(f"expected symlink materialization: {installed}") + target = installed.resolve(strict=True) + try: + target.relative_to(store) + except ValueError as error: + raise AssertionError(f"{installed} resolves outside {store}: {target}") from error + payload = (target / "payload.txt").read_text(encoding="utf-8") + if payload != f"{ORG}/{name}@{VERSION}\n": + raise AssertionError(f"unexpected payload through {installed}: {payload!r}") + targets[name] = target + return targets + + +def assert_copy_install(project: Path, names: Sequence[str]) -> None: + modules = project / "zed_modules" + for name in names: + installed = modules / ORG / name + if installed.is_symlink() or not installed.is_dir(): + raise AssertionError(f"expected copied package directory: {installed}") + payload = (installed / "payload.txt").read_text(encoding="utf-8") + if payload != f"{ORG}/{name}@{VERSION}\n": + raise AssertionError(f"unexpected copied payload in {installed}: {payload!r}") + leaked = [path for path in modules.rglob("*") if path.is_symlink()] + if leaked: + raise AssertionError(f"copy install leaked symlinks: {leaked}") + + +def assert_not_materialized(project: Path, names: Sequence[str]) -> None: + for name in names: + installed = project / "zed_modules" / ORG / name + if installed.exists() or installed.is_symlink(): + raise AssertionError(f"unexpected materialized package: {installed}") + + +def assert_no_transaction_leaks(project: Path) -> None: + staging = project / ".zpkg-staging" + if staging.exists(): + entries = list(staging.iterdir()) if staging.is_dir() else [staging] + raise AssertionError(f"transaction staging leaked in {project}: {entries}") + + +def assert_no_cache_staging(home: Path) -> None: + cache = home / "cache" + if not cache.exists(): + return + leaked_directories = sorted(path for path in cache.iterdir() if path.is_dir()) + leaked_downloads = sorted(cache.glob("**/artifact.download")) + if leaked_directories or leaked_downloads: + raise AssertionError( + f"cache staging leaked in {home}: dirs={leaked_directories}, downloads={leaked_downloads}" + ) + + +def parse_prefetch(output: str) -> tuple[int, int, int]: + matches = list(PREFETCH_RE.finditer(output)) + if len(matches) != 1: + raise AssertionError( + f"expected one recursive prefetch summary, got {len(matches)}:\n{output}" + ) + match = matches[0] + return tuple( + int(match.group(field)) + for field in ("resolved", "concurrency", "downloaded") + ) + + +def assert_prefetch( + completed: Completed, + *, + resolved: int, + downloaded: int, +) -> tuple[int, int, int]: + actual = parse_prefetch(completed.output) + expected = (resolved, 5, downloaded) + if actual != expected: + raise AssertionError(f"prefetch mismatch: expected {expected}, got {actual}") + return actual + + +def registry_version_path(registry: Path, name: str) -> Path: + return registry / "packages" / ORG / name / "versions" / f"{VERSION}.json" + + +def registry_artifact(registry: Path, name: str) -> tuple[str, Path]: + metadata_path = registry_version_path(registry, name) + payload = json.loads(metadata_path.read_text(encoding="utf-8")) + sha256 = str(payload["sha256"]) + if not re.fullmatch(r"[0-9a-f]{64}", sha256): + raise AssertionError(f"invalid registry SHA for {name}: {sha256!r}") + artifact = registry / "artifacts" / f"{sha256}.tar.gz" + if not artifact.is_file(): + raise AssertionError(f"registry artifact missing before test: {artifact}") + return sha256, artifact + + +def tamper_first_sha(lock_text: str) -> str: + tampered, count = SHA_ASSIGNMENT_RE.subn( + 'sha256 = "' + ("0" * 64) + '"', + lock_text, + count=1, + ) + if count != 1: + raise AssertionError("could not locate first SHA assignment in lockfile") + return tampered + + +def duplicate_first_package(lock_text: str) -> str: + match = PACKAGE_BLOCK_RE.search(lock_text) + if match is None: + raise AssertionError("could not locate a package block in lockfile") + return lock_text.rstrip() + "\n\n" + match.group(0).rstrip() + "\n" + + +def publish( + zed: Path, + registry: Path, + home: Path, + source: Path, + env: Mapping[str, str], +) -> None: + completed = run( + zed_command(zed, registry, home, "publish", "--skip-vcs-checks"), + cwd=source, + env=env, + ) + require_success(completed) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + + zed = args.zed.resolve() + root = args.work_root.resolve() + if not zed.is_file(): + raise RuntimeError(f"zed binary does not exist: {zed}") + if root.exists(): + raise RuntimeError(f"work root must be fresh: {root}") + + registry = root / "registry" + publish_home = root / "publish-home" + shared_home = root / "shared-home" + copy_home = root / "copy-home" + tamper_home = root / "tamper-home" + duplicate_home = root / "duplicate-home" + corrupt_home = root / "corrupt-home" + sources = root / "sources" + project = root / "same-project" + copy_project = root / "copy-project" + tamper_project = root / "tamper-project" + duplicate_project = root / "duplicate-project" + corrupt_project = root / "corrupt-project" + registry.mkdir(parents=True) + + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "RUST_BACKTRACE": "1", + "ZED_PKG_INSTALL_CONCURRENCY": "5", + } + ) + + for name, dependencies in BASE_GRAPH.items(): + source = sources / name + write_package(source, name, dependencies) + publish(zed, registry, publish_home, source, environment) + + write_consumer(project, ("root",)) + start = Barrier(3) + + def concurrent_install() -> Completed: + start.wait() + return run( + zed_command(zed, registry, shared_home, "install", "--adapter", "none"), + cwd=project, + env=environment, + timeout=300, + ) + + with ThreadPoolExecutor(max_workers=2, thread_name_prefix="zed-same-project") as executor: + futures = [executor.submit(concurrent_install) for _ in range(2)] + start.wait() + concurrent = [future.result(timeout=360) for future in futures] + + for completed in concurrent: + require_success(completed) + summaries = [parse_prefetch(completed.output) for completed in concurrent] + if any(summary[:2] != (len(BASE_GRAPH), 5) for summary in summaries): + raise AssertionError(f"same-project resolver summaries are wrong: {summaries}") + if sum(summary[2] for summary in summaries) != len(BASE_GRAPH): + raise AssertionError( + f"same-project processes did not deduplicate downloads: {summaries}" + ) + + all_base = tuple(BASE_GRAPH) + assert_lock(project, all_base) + initial_targets = assert_symlink_install(project, shared_home, all_base) + assert_no_transaction_leaks(project) + assert_no_cache_staging(shared_home) + lock_before = (project / ".zpkg.lock").read_bytes() + manifest_before = (project / ".zpkg.toml").read_bytes() + + warm = run( + zed_command(zed, registry, shared_home, "install", "--adapter", "none"), + cwd=project, + env=environment, + ) + require_success(warm) + assert_prefetch(warm, resolved=len(BASE_GRAPH), downloaded=0) + if (project / ".zpkg.lock").read_bytes() != lock_before: + raise AssertionError("warm ordinary install changed lockfile bytes") + if assert_symlink_install(project, shared_home, all_base) != initial_targets: + raise AssertionError("warm ordinary install changed immutable store targets") + + broken = project / "zed_modules" / ORG / "leaf-a" + broken.unlink() + repair = run( + zed_command( + zed, + registry, + shared_home, + "install", + "--frozen", + "--adapter", + "none", + ), + cwd=project, + env=environment, + ) + require_success(repair) + assert_prefetch(repair, resolved=len(BASE_GRAPH), downloaded=0) + if (project / ".zpkg.lock").read_bytes() != lock_before: + raise AssertionError("frozen repair changed lockfile bytes") + assert_symlink_install(project, shared_home, all_base) + + selective_uninstall = run( + zed_command(zed, registry, shared_home, "uninstall", f"{ORG}/root"), + cwd=project, + env=environment, + ) + require_success(selective_uninstall) + assert_not_materialized(project, ("root",)) + assert_symlink_install(project, shared_home, ("leaf-a", "leaf-b")) + if (project / ".zpkg.lock").read_bytes() != lock_before: + raise AssertionError("selective uninstall changed lockfile bytes") + selective_restore = run( + zed_command( + zed, + registry, + shared_home, + "install", + "--frozen", + "--adapter", + "none", + ), + cwd=project, + env=environment, + ) + require_success(selective_restore) + assert_prefetch(selective_restore, resolved=len(BASE_GRAPH), downloaded=0) + assert_symlink_install(project, shared_home, all_base) + + uninstall_all = run( + zed_command(zed, registry, shared_home, "uninstall"), + cwd=project, + env=environment, + ) + require_success(uninstall_all) + assert_not_materialized(project, all_base) + if (project / ".zpkg.lock").read_bytes() != lock_before: + raise AssertionError("full uninstall changed lockfile bytes") + if (project / ".zpkg.toml").read_bytes() != manifest_before: + raise AssertionError("full uninstall changed manifest bytes") + full_restore = run( + zed_command( + zed, + registry, + shared_home, + "install", + "--frozen", + "--adapter", + "none", + ), + cwd=project, + env=environment, + ) + require_success(full_restore) + assert_prefetch(full_restore, resolved=len(BASE_GRAPH), downloaded=0) + assert_symlink_install(project, shared_home, all_base) + + write_consumer(copy_project, ("root",)) + copy_install = run( + zed_command( + zed, + registry, + copy_home, + "install", + "--install-mode", + "copy", + "--adapter", + "none", + ), + cwd=copy_project, + env=environment, + ) + require_success(copy_install) + assert_prefetch(copy_install, resolved=len(BASE_GRAPH), downloaded=len(BASE_GRAPH)) + assert_lock(copy_project, all_base) + assert_copy_install(copy_project, all_base) + copy_lock = (copy_project / ".zpkg.lock").read_bytes() + copy_warm = run( + zed_command( + zed, + registry, + copy_home, + "install", + "--frozen", + "--install-mode", + "copy", + "--adapter", + "none", + ), + cwd=copy_project, + env=environment, + ) + require_success(copy_warm) + assert_prefetch(copy_warm, resolved=len(BASE_GRAPH), downloaded=0) + if (copy_project / ".zpkg.lock").read_bytes() != copy_lock: + raise AssertionError("warm copy-mode frozen install changed lockfile bytes") + assert_copy_install(copy_project, all_base) + + good_lock_text = (project / ".zpkg.lock").read_text(encoding="utf-8") + write_consumer(tamper_project, ("root",)) + (tamper_project / ".zpkg.lock").write_text( + tamper_first_sha(good_lock_text), encoding="utf-8" + ) + tampered_before = (tamper_project / ".zpkg.lock").read_bytes() + tampered = run( + zed_command( + zed, + registry, + tamper_home, + "install", + "--frozen", + "--adapter", + "none", + ), + cwd=tamper_project, + env=environment, + ) + require_failure(tampered, "registry artifact", "changed") + if (tamper_project / ".zpkg.lock").read_bytes() != tampered_before: + raise AssertionError("failed tampered-lock install changed lockfile bytes") + assert_not_materialized(tamper_project, all_base) + assert_no_transaction_leaks(tamper_project) + assert_no_cache_staging(tamper_home) + + write_consumer(duplicate_project, ("root",)) + (duplicate_project / ".zpkg.lock").write_text( + duplicate_first_package(good_lock_text), encoding="utf-8" + ) + duplicate_before = (duplicate_project / ".zpkg.lock").read_bytes() + duplicate = run( + zed_command( + zed, + registry, + duplicate_home, + "install", + "--frozen", + "--adapter", + "none", + ), + cwd=duplicate_project, + env=environment, + ) + require_failure(duplicate, "duplicate package") + if (duplicate_project / ".zpkg.lock").read_bytes() != duplicate_before: + raise AssertionError("failed duplicate-lock install changed lockfile bytes") + assert_not_materialized(duplicate_project, all_base) + assert_no_transaction_leaks(duplicate_project) + + # A new direct dependency whose registry artifact is unavailable must not + # damage the already-installed graph or rewrite its last known-good lock. + extra_source = sources / "extra" + write_package(extra_source, "extra", ()) + publish(zed, registry, publish_home, extra_source, environment) + extra_sha, extra_artifact = registry_artifact(registry, "extra") + extra_bytes = extra_artifact.read_bytes() + extra_artifact.unlink() + write_consumer(project, ("root", "extra")) + preserved_targets = assert_symlink_install(project, shared_home, all_base) + unavailable = run( + zed_command(zed, registry, shared_home, "install", "--adapter", "none"), + cwd=project, + env=environment, + ) + require_failure(unavailable) + if (project / ".zpkg.lock").read_bytes() != lock_before: + raise AssertionError("missing-artifact failure replaced the good lockfile") + if assert_symlink_install(project, shared_home, all_base) != preserved_targets: + raise AssertionError("missing-artifact failure changed the installed graph") + assert_not_materialized(project, ("extra",)) + assert_no_transaction_leaks(project) + assert_no_cache_staging(shared_home) + if (shared_home / "cache" / f"{extra_sha}.tar.gz").exists(): + raise AssertionError("missing artifact produced a final cache entry") + + extra_artifact.write_bytes(extra_bytes) + recovered = run( + zed_command(zed, registry, shared_home, "install", "--adapter", "none"), + cwd=project, + env=environment, + ) + require_success(recovered) + assert_prefetch(recovered, resolved=len(BASE_GRAPH) + 1, downloaded=1) + extended = (*all_base, "extra") + assert_lock(project, extended) + assert_symlink_install(project, shared_home, extended) + assert_no_cache_staging(shared_home) + + # Corrupt bytes must fail before project mutation, leave no final cache + # entry for the bad hash, then recover by downloading exactly that hash. + write_consumer(corrupt_project, ("root",)) + corrupt_sha, corrupt_artifact = registry_artifact(registry, "leaf-b") + corrupt_bytes = corrupt_artifact.read_bytes() + corrupt_artifact.write_bytes(b"not a valid zed artifact\n") + corrupt = run( + zed_command(zed, registry, corrupt_home, "install", "--adapter", "none"), + cwd=corrupt_project, + env=environment, + ) + require_failure(corrupt, "artifact hash mismatch") + if (corrupt_project / ".zpkg.lock").exists(): + raise AssertionError("corrupt-artifact failure wrote a lockfile") + assert_not_materialized(corrupt_project, all_base) + assert_no_transaction_leaks(corrupt_project) + assert_no_cache_staging(corrupt_home) + if (corrupt_home / "cache" / f"{corrupt_sha}.tar.gz").exists(): + raise AssertionError("corrupt artifact was published to the final cache path") + + corrupt_artifact.write_bytes(corrupt_bytes) + corrupt_recovery = run( + zed_command(zed, registry, corrupt_home, "install", "--adapter", "none"), + cwd=corrupt_project, + env=environment, + ) + require_success(corrupt_recovery) + recovery_summary = parse_prefetch(corrupt_recovery.output) + if recovery_summary[:2] != (len(BASE_GRAPH), 5) or recovery_summary[2] < 1: + raise AssertionError(f"unexpected corrupt-artifact recovery summary: {recovery_summary}") + assert_lock(corrupt_project, all_base) + assert_symlink_install(corrupt_project, corrupt_home, all_base) + assert_no_cache_staging(corrupt_home) + + print("\ninstall recovery matrix passed", flush=True) + print(f"same-project concurrent summaries: {summaries}", flush=True) + print(f"corrupt recovery summary: {recovery_summary}", flush=True) + print(f"extended manifest dependencies: {sorted(manifest_dependencies(project))}", flush=True) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 - CI boundary must print full context. + print(f"error: {error}", file=sys.stderr, flush=True) + raise diff --git a/scripts/leddy-package-graph.sh b/scripts/leddy-package-graph.sh new file mode 100644 index 00000000..1ab3c8c2 --- /dev/null +++ b/scripts/leddy-package-graph.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +usage: leddy-package-graph.sh ZED_BIN INTERFACES_DIR LIB_DIR CLIENTS_DIR MONOREPO_DIR WORK_ROOT +EOF + exit 64 +} + +[[ $# -eq 6 ]] || usage +zed="$1" +interfaces_source="$2" +lib_source="$3" +clients_source="$4" +monorepo_source="$5" +work_root="$6" + +[[ -x "$zed" ]] || { echo "zed binary is not executable: $zed" >&2; exit 66; } +for fixture in "$interfaces_source" "$lib_source" "$clients_source" "$monorepo_source"; do + [[ -f "$fixture/.zpkg.toml" ]] || { echo "missing .zpkg.toml in $fixture" >&2; exit 66; } +done +[[ ! -e "$work_root" ]] || { echo "work root must be fresh: $work_root" >&2; exit 73; } + +zed="$(cd "$(dirname "$zed")" && pwd)/$(basename "$zed")" +mkdir -p "$work_root" +work_root="$(cd "$work_root" && pwd)" +registry="$work_root/registry" +home="$work_root/home" +mkdir -p "$registry" "$home" +file_registry="file://$registry" + +unset ZED_PKG_TOKEN ZED_PKG_REGISTRY ZED_PKG_HOME ZED_PKG_ALLOW_NO_MANIFEST \ + ZED_PKG_DO_NOT_WRITE_NEW_MANIFEST ZED_PKG_INTERACTIVE + +copy_tree() { + local source="$1" + local destination="$2" + mkdir -p "$destination" + cp -R "$source/." "$destination/" + rm -rf "$destination/.git" "$destination/target" "$destination/.vendor" \ + "$destination/.zed" "$destination/.zpkg.lock" "$destination/.zpkg-staging" +} + +assert_manifest_contract() { + local root="$1" + local package="$2" + grep -F 'org = "led-dynamo"' "$root/.zpkg.toml" + grep -F "name = \"$package\"" "$root/.zpkg.toml" + grep -F 'version = "0.1.0"' "$root/.zpkg.toml" + grep -F 'dir = ".vendor/.zed"' "$root/.zpkg.toml" +} + +assert_registry_package() { + local package="$1" + local metadata="$registry/packages/led-dynamo/$package/package.json" + local version="$registry/packages/led-dynamo/$package/versions/0.1.0.json" + if [[ ! -f "$metadata" || ! -f "$version" ]]; then + echo "missing registry package led-dynamo/$package@0.1.0" >&2 + find "$registry/packages/led-dynamo" -maxdepth 4 -type f -print 2>/dev/null | sort >&2 || true + return 1 + fi +} + +publish_fixture() { + local label="$1" + local source="$2" + local package="$3" + local destination="$work_root/publish/$label" + copy_tree "$source" "$destination" + assert_manifest_contract "$destination" "$package" + ( + cd "$destination" + "$zed" --registry "$file_registry" --home "$home/publisher-$label" \ + publish --skip-vcs-checks + ) + + if [[ "$label" == clients ]]; then + # The repository target rooted at `dir = "."` is the canonical whole- + # repository package consumed by the monorepo. Non-root targets retain + # their explicit target identities. + for target_package in \ + leddy-clients \ + leddy-clients-nodejs \ + leddy-clients-rust \ + leddy-clients-wasm \ + leddy-clients-python3 \ + leddy-clients-golang; do + assert_registry_package "$target_package" + done + test ! -e "$registry/packages/led-dynamo/leddy-clients-repository/package.json" + else + assert_registry_package "$package" + fi +} + +# Publish in dependency order into a fresh credential-free registry. This proves +# the authored Leddy graph resolves through the real package manager rather than +# only through grep/static repository CI. +publish_fixture interfaces "$interfaces_source" leddy-interfaces +publish_fixture lib "$lib_source" leddy-lib +publish_fixture clients "$clients_source" leddy-clients +publish_fixture monorepo "$monorepo_source" leddy-monorepo + +# Assert the canonical dependency edges before exercising resolver closure. +grep -F '"led-dynamo/leddy-interfaces" = "^0.1.0"' "$lib_source/.zpkg.toml" +grep -F '"led-dynamo/leddy-interfaces" = "^0.1.0"' "$clients_source/.zpkg.toml" +grep -F '"led-dynamo/leddy-lib" = "^0.1.0"' "$clients_source/.zpkg.toml" +for dependency in leddy-interfaces leddy-lib leddy-clients; do + grep -F "\"led-dynamo/$dependency\" = \"^0.1.0\"" "$monorepo_source/.zpkg.toml" +done + +# The multi-target client package is a deliberately broad stress case. Check a +# representative set spanning native, managed, scripting, mobile and WASM. +for target in c cpp zig gleamlang erlang elixir dart rust java golang python3 ruby php \ + typescript-nodejs typescript-deno typescript-bun typescript-edge kotlin swift rust-wasm; do + grep -F "[targets.$target]" "$clients_source/.zpkg.toml" +done + +install_consumer() { + local name="$1" + local consumer="$work_root/$name" + local consumer_home="$home/$name" + mkdir -p "$consumer" + ( + cd "$consumer" + "$zed" --registry "$file_registry" --home "$consumer_home" \ + install led-dynamo/leddy-monorepo@^0.1.0 \ + --do-not-write-new-manifest \ + --install-mode copy + ) + + test ! -e "$consumer/.zpkg.toml" + test -f "$consumer/.zpkg.lock" + test -d "$consumer/zed_modules/led-dynamo/leddy-monorepo" + test -d "$consumer/zed_modules/led-dynamo/leddy-clients" + test -d "$consumer/zed_modules/led-dynamo/leddy-lib" + test -d "$consumer/zed_modules/led-dynamo/leddy-interfaces" + if find "$consumer/zed_modules" -type l -print -quit | grep -q .; then + echo "copy-mode Leddy graph unexpectedly contains symlinks" >&2 + find "$consumer/zed_modules" -type l -print >&2 + exit 1 + fi +} + +install_consumer consumer-a +install_consumer consumer-b +cmp "$work_root/consumer-a/.zpkg.lock" "$work_root/consumer-b/.zpkg.lock" + +# Frozen reinstall must reconstruct the exact graph from lock state with no +# manifest creation and no registry-selection drift. +rm -rf "$work_root/consumer-a/zed_modules" "$work_root/consumer-a/.vendor" "$work_root/consumer-a/.zed" +( + cd "$work_root/consumer-a" + "$zed" --registry "$file_registry" --home "$home/consumer-a" \ + install --frozen --do-not-write-new-manifest --install-mode copy +) +test -d "$work_root/consumer-a/zed_modules/led-dynamo/leddy-monorepo" +test -d "$work_root/consumer-a/zed_modules/led-dynamo/leddy-clients" +cmp "$work_root/consumer-a/.zpkg.lock" "$work_root/consumer-b/.zpkg.lock" + +# The test registry is the only publication surface used by this canary. +test -d "$registry/packages/led-dynamo" +printf 'Leddy Zed graph canary passed\n' diff --git a/scripts/lifecycle.py b/scripts/lifecycle.py old mode 100644 new mode 100755 index a0597007..8eadda42 --- a/scripts/lifecycle.py +++ b/scripts/lifecycle.py @@ -22,7 +22,7 @@ from typing import Mapping, Sequence ORG = "zed-pkg-test" -NON_PACKAGE_REPOS = {"shared-schema", "zed-pkg-e2e"} +NON_PACKAGE_REPOS = {"zed-pkg-e2e"} # Package -> (fixture repository, path containing .zpkg.toml). This is the # dependency graph boundary. Target packages all map to the one source repo @@ -34,6 +34,7 @@ "zed-pkg-test/python-lib": ("python-lib", "."), "zed-pkg-test/dart-lib": ("dart-lib", "."), "zed-pkg-test/gleam-lib": ("gleam-lib", "."), + "zedtest/shared-schema": ("shared-schema", "."), "zedtest/polyglot-lib-nodejs": ("polyglot-lib", "."), "zedtest/polyglot-lib-python": ("polyglot-lib", "."), "zedtest/polyglot-lib-golang": ("polyglot-lib", "."), @@ -62,6 +63,88 @@ def spec(self) -> str: return f"{self.full_name}@{self.version}" +def publish_archive_name(package: PackageRef) -> str: + """Return the deterministic archive name emitted by `zed publish`.""" + return f"{package.full_name.replace('/', '-')}-{package.version}.tar.gz" + + +def remove_transient_pack_outputs(root: Path, packages: Sequence[PackageRef]) -> None: + """Remove only verified untracked archives produced by lifecycle publishes. + + The source-cleanliness invariant remains fail closed: tracked modifications, + symlinks, directories, and unexpected files under `.zed/pack` are rejected. + Repositories that already ignore these archives produce no status entries and + require no cleanup. + """ + + expected = { + f".zed/pack/{publish_archive_name(package)}" for package in packages + } + if not expected: + return + + completed = subprocess.run( + [ + "git", + "status", + "--porcelain=v1", + "--untracked-files=all", + "--", + ".zed/pack", + ], + cwd=root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + detail = completed.stderr.strip() or completed.stdout.strip() + raise RuntimeError(f"failed to inspect transient pack outputs: {detail}") + + entries: list[str] = [] + for line in completed.stdout.splitlines(): + if len(line) < 4: + raise AssertionError(f"malformed git status entry for pack output: {line!r}") + status = line[:2] + relative = line[3:] + if status != "??": + raise AssertionError( + f"refusing to clean tracked or modified pack output: {line}" + ) + if relative not in expected: + raise AssertionError( + f"refusing to clean unexpected pack output: {relative}" + ) + entries.append(relative) + + if not entries: + return + + zed_dir = root / ".zed" + pack_dir = zed_dir / "pack" + if zed_dir.is_symlink() or pack_dir.is_symlink(): + raise AssertionError("refusing to clean pack outputs through a symlink") + + for relative in entries: + archive = root / relative + if archive.is_symlink() or not archive.is_file(): + raise AssertionError( + f"refusing to clean non-regular pack output: {relative}" + ) + if archive.parent != pack_dir: + raise AssertionError( + f"refusing to clean pack output outside .zed/pack: {relative}" + ) + archive.unlink() + + for directory in (pack_dir, zed_dir): + try: + directory.rmdir() + except OSError: + pass + + class Harness: def __init__(self, args: argparse.Namespace) -> None: self.repo = args.repo @@ -224,8 +307,10 @@ def publish_prerequisite(self, package: str) -> None: home=self.root / "dependency-publish-home", ) self.published_sources.add(key) - for output in expected_packages(manifest): + outputs = expected_packages(manifest) + for output in outputs: assert_registry_version(self.registry, output) + remove_transient_pack_outputs(source, outputs) self.assert_git_clean(self.source_root(repo), f"seed source {repo}") def seed_dependencies(self, manifest: dict) -> None: @@ -334,6 +419,7 @@ def run_unit(self, unit: Path, label: str, index: int) -> None: for output_index, output in enumerate(outputs): self.exercise_consumer(output, index, output_index) + remove_transient_pack_outputs(unit, outputs) self.assert_git_clean(self.fixture, self.repo) def exercise_consumer( diff --git a/scripts/list-test-org-fleet-repositories.mjs b/scripts/list-test-org-fleet-repositories.mjs new file mode 100644 index 00000000..e2a456fc --- /dev/null +++ b/scripts/list-test-org-fleet-repositories.mjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import path from 'node:path'; +import process from 'node:process'; +import { readFleetManifest } from './read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const manifest = readFleetManifest(path.join(root, 'bootstrap', 'test-org-fleet.json.gz')); +const organizationIndex = process.argv.indexOf('--org'); +const organization = organizationIndex >= 0 ? process.argv[organizationIndex + 1] : ''; +const json = process.argv.includes('--json'); + +if (!organization) throw new Error('--org is required'); +if ((manifest.policy.excludedOrganizations ?? []).includes(organization)) { + throw new Error(`refusing excluded organization ${organization}`); +} +const pair = manifest.pairs.find((candidate) => candidate.testOrg === organization); +if (!pair) throw new Error(`organization is not in the fleet manifest: ${organization}`); +const repositories = pair.repositories.map((repository) => repository.name); +if (new Set(repositories).size !== repositories.length) { + throw new Error(`duplicate repository names for ${organization}`); +} + +if (json) { + process.stdout.write(`${JSON.stringify({ + organization, + sourceOrganization: pair.sourceOrg, + governanceRepository: '.github', + repositories, + repositoryCount: repositories.length, + }, null, 2)}\n`); +} else { + process.stdout.write(`${repositories.join('\n')}\n`); +} diff --git a/scripts/materialize_global_packages_current_main.py b/scripts/materialize_global_packages_current_main.py new file mode 100644 index 00000000..2ecd2c5d --- /dev/null +++ b/scripts/materialize_global_packages_current_main.py @@ -0,0 +1,765 @@ +#!/usr/bin/env python3 +"""Assemble and validate the global-package CLI on exact current main. + +This script never writes to the product repository. It overlays only the +reviewed permanent files from the superseded feature head, composes them with +the current modular CLI, applies the collision/profile rollback and offline +frozen-replay repairs found by the external canary, and leaves an ordinary +working-tree delta for independent validation and artifact publication. +""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +from pathlib import Path + + +PERMANENT_FEATURE_FILES = ( + ".github/workflows/global-packages.yml", + ".zpkg.lock", + ".zpkg.toml", + "docs/global-packages.md", + "src/global.rs", + "tests/global_cli.rs", +) + +EXPECTED_CHANGED_FILES = { + *PERMANENT_FEATURE_FILES, + "src/completion.rs", + "src/lib.rs", + "src/main.rs", + "src/ops.rs", + # Current main inherited a small rustfmt-only delta from the standalone Nix + # bundle merge. The complete candidate is required to repair it rather than + # hiding a formatter failure behind the new feature. + "src/nix_bundle_write.rs", + "tests/nix_bundle_write_boundaries.rs", + "tests/nix_bundle_write_cli.rs", +} + + +def replace_once(text: str, old: str, new: str, label: str) -> str: + if old not in text: + raise RuntimeError(f"{label}: expected source block was not found") + return text.replace(old, new, 1) + + +def insert_after(text: str, marker: str, addition: str, label: str) -> str: + if addition in text: + return text + if marker not in text: + raise RuntimeError(f"{label}: insertion point was not found") + return text.replace(marker, marker + addition, 1) + + +def copy_feature_files(product: Path, feature: Path) -> None: + for relative in PERMANENT_FEATURE_FILES: + source = feature / relative + destination = product / relative + if not source.is_file(): + raise RuntimeError(f"reviewed feature file is missing: {source}") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + + +def patch_lib(product: Path) -> None: + path = product / "src/lib.rs" + text = path.read_text() + text = insert_after( + text, + "pub mod git_submodules;\n", + "pub mod global;\n", + "src/lib.rs global module", + ) + path.write_text(text) + + +def patch_completion(product: Path) -> None: + path = product / "src/completion.rs" + text = path.read_text() + text = replace_once( + text, + "use std::io;\n\nuse clap_complete::{Shell, generate};", + "use std::io;\n\nuse anyhow::{Context, Result};\nuse clap_complete::{Shell, generate};", + "completion result imports", + ) + text = replace_once( + text, + "use crate::{dev, fetch, git_submodules, nix_bundle_write, nix_export_plan};", + "use crate::{dev, fetch, git_submodules, global, nix_bundle_write, nix_export_plan};", + "completion global import", + ) + old_root = '''fn root_command() -> clap::Command { + git_submodules::augment_root_command(nix_bundle_write::augment_root_command( + nix_export_plan::augment_root_command(fetch::augment_root_command( + dev::augment_root_command(cli_model::command()), + )), + )) +} +''' + new_root = '''/// Build the complete public command tree shared by root help and completion +/// generation. Every modular command must compose here rather than maintaining +/// a second, partial root-help model. +pub fn root_command() -> clap::Command { + global::augment_root_command(git_submodules::augment_root_command( + nix_bundle_write::augment_root_command(nix_export_plan::augment_root_command( + fetch::augment_root_command(dev::augment_root_command(cli_model::command())), + )), + )) +} + +/// Print the complete top-level help tree. +pub fn print_root_help() -> Result<()> { + let mut command = root_command(); + command.print_help().context("printing zed help")?; + println!(); + Ok(()) +} +''' + text = replace_once(text, old_root, new_root, "completion root command") + for command in (' "global",\n', ' "bin-dir",\n'): + if command not in text: + text = insert_after( + text, + ' "overtake",\n', + command, + "completion command assertion", + ) + if ' "--global-bin-dir",\n' not in text: + text = insert_after( + text, + ' "--install-mode",\n', + ' "--global-bin-dir",\n', + "completion global-bin option assertion", + ) + path.write_text(text) + + +def patch_main(product: Path) -> None: + path = product / "src/main.rs" + text = path.read_text() + if not text.startswith("use std::ffi"): + text = "use std::ffi::{OsStr, OsString};\n\n" + text + text = insert_after( + text, + "use zed_cli::git_submodules as submodules;\n", + "use zed_cli::global;\n", + "main global import", + ) + marker = ''' if let Err(error) = zed_cli::flags::normalize_global_boolean_environment(&args) { + eprintln!("error: {error:#}"); + std::process::exit(2); + } +''' + dispatch = ''' if root_help_requested(&args) { + if let Err(error) = completion::print_root_help() { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + return; + } + let global_requested = args.iter().skip(1).any(|argument| { + let argument = argument.as_os_str(); + argument == OsStr::new("global") || argument == OsStr::new("--global") + }); + if global_requested + && let Some(result) = global::dispatch(args.clone()) + { + match result { + Ok(0) => return, + Ok(code) => std::process::exit(code), + Err(error) => { + eprintln!("error: {error:#}"); + std::process::exit(1); + } + } + } +''' + text = insert_after(text, marker, dispatch, "main global dispatcher") + helper_marker = "\nfn run(cli: Cli) -> anyhow::Result<()> {\n" + helpers = ''' +fn root_help_requested(args: &[OsString]) -> bool { + let mut index = 1; + while index < args.len() { + let token = args[index].to_string_lossy(); + if token == "--help" || token == "-h" { + return true; + } + if token == "help" { + return args + .iter() + .skip(index + 1) + .all(|argument| argument.to_string_lossy().starts_with('-')); + } + if root_global_option_takes_value(&token) { + index += if token.contains('=') { 1 } else { 2 }; + continue; + } + if token.starts_with('-') { + index += 1; + continue; + } + return false; + } + false +} + +fn root_global_option_takes_value(token: &str) -> bool { + const OPTIONS: &[&str] = &[ + "--registry", + "--home", + "--token", + "--auth-url", + "--supabase-url", + "--supabase-key", + "--global-bin-dir", + ]; + OPTIONS.iter().any(|option| { + token == *option + || token + .strip_prefix(option) + .is_some_and(|remainder| remainder.starts_with('=')) + }) +} +''' + if "fn root_help_requested" not in text: + if helper_marker not in text: + raise RuntimeError("main helper insertion point was not found") + text = text.replace(helper_marker, "\n" + helpers + helper_marker, 1) + path.write_text(text) + + +def patch_global(product: Path) -> None: + path = product / "src/global.rs" + text = path.read_text() + text = replace_once( + text, + "use std::collections::BTreeMap;", + "use std::collections::{BTreeMap, BTreeSet};", + "global collection imports", + ) + text = text.replace( + "use clap::{Args, CommandFactory, Parser, Subcommand};", + "use clap::{Args, Parser, Subcommand};", + 1, + ) + text = text.replace( + "use crate::{dev, interactive, manifestless};", + "use crate::{interactive, manifestless};", + 1, + ) + + profile = '''#[derive(Debug, Clone)] +struct Profile { + root: PathBuf, + metadata: ProfileMetadata, +} +''' + replacement = profile + ''' +#[derive(Debug)] +struct ProfileReplacement { + root: PathBuf, + backup: Option, +} +''' + if "struct ProfileReplacement" not in text: + text = replace_once(text, profile, replacement, "profile transaction type") + + old_help = '''fn print_root_help() -> Result<()> { + let mut command = dev::augment_root_command(augment_root_command(crate::cli::Cli::command())); + command.print_help().context("printing zed help")?; + println!(); + Ok(()) +} +''' + if old_help in text: + text = text.replace( + old_help, + '''fn print_root_help() -> Result<()> { + crate::completion::print_root_help() +} +''', + 1, + ) + + start = text.index( + "fn install(cfg: &Config, bin_dir: &Path, options: GlobalInstallArgs) -> Result {" + ) + end = text.index("\nfn uninstall(", start) + install = r'''fn install(cfg: &Config, bin_dir: &Path, options: GlobalInstallArgs) -> Result { + let _lock = acquire_lock(cfg)?; + let mut replacements = Vec::new(); + if options.frozen { + let profiles = selected_profiles(cfg, &options.specs)?; + if profiles.is_empty() { + bail!("no global package profiles are installed"); + } + for profile in &profiles { + manifestless::install( + &profile.root, + cfg, + &[], + true, + options.install_mode, + Adapter::None, + options.allow_build, + options.target.as_deref(), + true, + true, + )?; + } + } else { + if options.specs.is_empty() { + bail!( + "global install needs one or more `org/name[@requirement]` package specs; use --frozen to restore existing profiles" + ); + } + let mut package_keys = BTreeSet::new(); + for requested in &options.specs { + let (key, _) = parse_package_spec(requested)?; + if !package_keys.insert(key.clone()) { + bail!("global install contains duplicate top-level package `{key}`"); + } + } + + let result: Result<()> = (|| { + for requested in &options.specs { + let (key, _) = parse_package_spec(requested)?; + let root = profile_root(cfg, &key)?; + replacements.push(stage_profile_replacement(&root)?); + manifestless::install( + &root, + cfg, + std::slice::from_ref(requested), + false, + options.install_mode, + Adapter::None, + options.allow_build, + options.target.as_deref(), + true, + true, + )?; + write_metadata( + &root, + &ProfileMetadata { + package: key.clone(), + requested: requested.clone(), + }, + )?; + let bin_count = profile_bins(&root)?.len(); + if bin_count == 0 { + eprintln!( + "warning: {key} currently exposes no built [bin] entries in this profile; if it declares a [build] step, reinstall with --allow-build" + ); + } + } + Ok(()) + })(); + if let Err(error) = result { + rollback_profile_replacements(&replacements).with_context(|| { + format!("rolling back failed global profile installation after: {error:#}") + })?; + return Err(error); + } + } + + let sync_result: Result<(Vec, usize)> = (|| { + let profiles = discover_profiles(cfg)?; + let installed = sync_bins(cfg, bin_dir, &profiles)?; + Ok((profiles, installed)) + })(); + let (profiles, installed) = match sync_result { + Ok(result) => result, + Err(error) => { + rollback_profile_replacements(&replacements).with_context(|| { + format!("rolling back global profiles after PATH synchronization failed: {error:#}") + })?; + return Err(error); + } + }; + discard_profile_backups(&replacements); + print_path_guidance(bin_dir); + println!( + "{} global package profile(s); {} executable(s) managed in {}", + profiles.len(), + installed, + bin_dir.display() + ); + Ok(0) +} + +fn stage_profile_replacement(root: &Path) -> Result { + let backup = if root.exists() { + let parent = root.parent().context("global profile has no parent")?; + let name = root + .file_name() + .and_then(|value| value.to_str()) + .context("global profile name is not valid UTF-8")?; + let backup = parent.join(format!( + ".{name}.zed-backup-{}", + uuid::Uuid::new_v4() + )); + fs::rename(root, &backup).with_context(|| { + format!( + "moving existing global profile {} to rollback backup {}", + root.display(), + backup.display() + ) + })?; + Some(backup) + } else { + None + }; + if let Err(error) = fs::create_dir_all(root) { + if let Some(backup) = &backup { + let _ = fs::rename(backup, root); + } + return Err(error) + .with_context(|| format!("creating staged global profile {}", root.display())); + } + Ok(ProfileReplacement { + root: root.to_path_buf(), + backup, + }) +} + +fn rollback_profile_replacements(replacements: &[ProfileReplacement]) -> Result<()> { + for replacement in replacements.iter().rev() { + if replacement.root.exists() { + fs::remove_dir_all(&replacement.root).with_context(|| { + format!( + "removing failed staged global profile {}", + replacement.root.display() + ) + })?; + } + if let Some(backup) = &replacement.backup { + fs::rename(backup, &replacement.root).with_context(|| { + format!( + "restoring global profile backup {} to {}", + backup.display(), + replacement.root.display() + ) + })?; + } + } + Ok(()) +} + +fn discard_profile_backups(replacements: &[ProfileReplacement]) { + for replacement in replacements { + let Some(backup) = &replacement.backup else { + continue; + }; + if let Err(error) = fs::remove_dir_all(backup) + && error.kind() != std::io::ErrorKind::NotFound + { + eprintln!( + "warning: could not remove committed global profile backup {}: {error}", + backup.display() + ); + } + } +} +''' + text = text[:start] + install + text[end:] + + old_sync = '''fn sync_bins(cfg: &Config, bin_dir: &Path, profiles: &[Profile]) -> Result { + let desired = collect_desired_bins(profiles)?; + let previous = load_state(cfg)?; + fs::create_dir_all(bin_dir)?; + + for (name, managed) in &previous.bins { +''' + new_sync = '''fn sync_bins(cfg: &Config, bin_dir: &Path, profiles: &[Profile]) -> Result { + let desired = collect_desired_bins(profiles)?; + let previous = load_state(cfg)?; + fs::create_dir_all(bin_dir)?; + + // Prove every destination is absent or still owned before removing any + // stale command. Rejected installation cannot partially mutate PATH. + for name in desired.keys() { + let destination = bin_dir.join(name); + if !destination.exists() { + continue; + } + let current = hash_file(&destination)?; + let owned = previous + .bins + .get(name) + .is_some_and(|managed| managed.sha256 == current); + if !owned { + bail!( + "refusing to replace unmanaged global executable {}; choose another --global-bin-dir or remove the collision explicitly", + destination.display() + ); + } + } + + for (name, managed) in &previous.bins { +''' + text = replace_once(text, old_sync, new_sync, "PATH mutation preflight") + old_duplicate = ''' if destination.exists() { + let current = hash_file(&destination)?; + let owned = previous + .bins + .get(&name) + .is_some_and(|managed| managed.sha256 == current); + if !owned { + bail!( + "refusing to replace unmanaged global executable {}; choose another --global-bin-dir or remove the collision explicitly", + destination.display() + ); + } + } + atomic_copy(&wanted.source, &destination)?; +''' + text = replace_once( + text, + old_duplicate, + " atomic_copy(&wanted.source, &destination)?;\n", + "remove duplicate PATH preflight", + ) + + test_marker = ''' #[test] + fn explicit_global_bin_directory_wins() { +''' + tests = r''' #[test] + fn profile_replacement_rollback_restores_existing_and_removes_new() { + let home = tempfile::tempdir().unwrap(); + let cfg = config(home.path()); + let existing = profile_root(&cfg, "acme/existing").unwrap(); + fs::create_dir_all(&existing).unwrap(); + fs::write(existing.join("marker"), b"old").unwrap(); + let existing_replacement = stage_profile_replacement(&existing).unwrap(); + fs::write(existing.join("marker"), b"new").unwrap(); + + let new_root = profile_root(&cfg, "acme/new").unwrap(); + let new_replacement = stage_profile_replacement(&new_root).unwrap(); + fs::write(new_root.join("marker"), b"new").unwrap(); + + rollback_profile_replacements(&[existing_replacement, new_replacement]).unwrap(); + assert_eq!(fs::read(existing.join("marker")).unwrap(), b"old"); + assert!(!new_root.exists()); + } + + #[test] + fn unmanaged_collision_is_detected_before_stale_owned_command_removal() { + let home = tempfile::tempdir().unwrap(); + let cfg = config(home.path()); + let bin_dir = home.path().join("path-bin"); + fs::create_dir_all(&bin_dir).unwrap(); + + let stale_name = destination_name("stale-tool"); + let stale_path = bin_dir.join(&stale_name); + fs::write(&stale_path, b"stale-owned").unwrap(); + let mut previous = ManagedState::default(); + previous.bins.insert( + stale_name, + ManagedBin { + package: "acme/stale".to_string(), + sha256: hash_file(&stale_path).unwrap(), + }, + ); + atomic_write( + &state_path(&cfg), + &serde_json::to_vec_pretty(&previous).unwrap(), + ) + .unwrap(); + + let desired = add_profile(home.path(), "acme/new", "new-tool", b"desired"); + let unmanaged_path = bin_dir.join(destination_name("new-tool")); + fs::write(&unmanaged_path, b"unmanaged").unwrap(); + + let error = sync_bins(&cfg, &bin_dir, &[desired]).unwrap_err(); + assert!(error.to_string().contains("unmanaged global executable")); + assert_eq!(fs::read(stale_path).unwrap(), b"stale-owned"); + assert_eq!(fs::read(unmanaged_path).unwrap(), b"unmanaged"); + } + +''' + if "profile_replacement_rollback_restores_existing_and_removes_new" not in text: + text = replace_once(text, test_marker, tests + test_marker, "global rollback tests") + path.write_text(text) + + +def patch_ops(product: Path) -> None: + path = product / "src/ops.rs" + text = path.read_text() + old = ''' for locked in &lock.packages { + if !is_slug(&locked.org) || !is_slug(&locked.name) { + bail!( + "lockfile entry `{}/{}` has an invalid identity; refusing", + locked.org, + locked.name + ); + } + require_sha256(&locked.sha256)?; + let vm = reg.get_version(&locked.org, &locked.name, &locked.version)?; + if vm.sha256 != locked.sha256 { + bail!( + "registry artifact for {}@{} changed (lock {} vs registry {}); refusing", + locked.full_name(), + locked.version, + locked.sha256, + vm.sha256 + ); + } + validate_version_metadata(&vm)?; + resolved.insert(locked.full_name(), vm); + } +''' + new = ''' for locked in &lock.packages { + if !is_slug(&locked.org) || !is_slug(&locked.name) { + bail!( + "lockfile entry `{}/{}` has an invalid identity; refusing", + locked.org, + locked.name + ); + } + require_sha256(&locked.sha256)?; + let vm = if store.has(&locked.sha256) { + // The lock already carries every immutable field needed to + // authenticate the atomically promoted store entry. Do not turn + // an exact frozen replay into a registry-availability check. + locked_version_metadata(locked) + } else { + let vm = reg.get_version(&locked.org, &locked.name, &locked.version)?; + if vm.sha256 != locked.sha256 { + bail!( + "registry artifact for {}@{} changed (lock {} vs registry {}); refusing", + locked.full_name(), + locked.version, + locked.sha256, + vm.sha256 + ); + } + vm + }; + validate_version_metadata(&vm)?; + resolved.insert(locked.full_name(), vm); + } +''' + text = replace_once(text, old, new, "frozen registry loop") + marker = '''/// Install body, called with the store lock already held. Split out so the +/// build-hook path can install `[build-dependencies]` into a staging dir +/// under the same lock without deadlocking on a re-acquire. +''' + helper = '''fn locked_version_metadata(locked: &LockedPackage) -> VersionMetadata { + VersionMetadata { + org: locked.org.clone(), + name: locked.name.clone(), + version: locked.version.clone(), + sha256: locked.sha256.clone(), + size: locked.size, + format: locked.format, + vcs_tag: locked.vcs_tag.clone(), + vcs_commit: locked.vcs_commit.clone(), + // This URL is never consumed while the verified store entry exists. + download_url: String::new(), + published_at: "1970-01-01T00:00:00Z".to_string(), + yanked: false, + } +} + +#[cfg(test)] +#[test] +fn frozen_local_metadata_preserves_lock_identity_and_provenance() { + let locked = LockedPackage { + org: "acme".to_string(), + name: "tool".to_string(), + version: "1.2.3".to_string(), + sha256: "a".repeat(64), + size: 42, + format: zed_interfaces::artifact::ArtifactFormat::TarGz, + vcs_tag: "v1.2.3".to_string(), + vcs_commit: Some("b".repeat(40)), + source: "https://registry.invalid".to_string(), + }; + let metadata = locked_version_metadata(&locked); + assert_eq!(metadata.org, locked.org); + assert_eq!(metadata.name, locked.name); + assert_eq!(metadata.version, locked.version); + assert_eq!(metadata.sha256, locked.sha256); + assert_eq!(metadata.size, locked.size); + assert_eq!(metadata.format, locked.format); + assert_eq!(metadata.vcs_tag, locked.vcs_tag); + assert_eq!(metadata.vcs_commit, locked.vcs_commit); + assert!(metadata.download_url.is_empty()); + assert!(!metadata.yanked); +} + +''' + if "fn locked_version_metadata" not in text: + text = replace_once(text, marker, helper + marker, "locked metadata helper") + path.write_text(text) + + +def changed_files(product: Path) -> set[str]: + output = subprocess.check_output( + ["git", "diff", "--name-only", "-z"], cwd=product + ) + return { + item.decode() + for item in output.split(b"\0") + if item + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--product", type=Path, required=True) + parser.add_argument("--feature", type=Path, required=True) + args = parser.parse_args() + + product = args.product.resolve() + feature = args.feature.resolve() + copy_feature_files(product, feature) + patch_lib(product) + patch_completion(product) + patch_main(product) + patch_global(product) + patch_ops(product) + + # Temporary write-enabled helpers are intentionally never copied into the + # assembled product tree. + for name in ( + "enable-offline-frozen-global-once.yml", + "harden-global-package-transaction-once.yml", + "rebase-global-packages-current-main.yml", + ): + candidate = product / ".github/workflows" / name + if candidate.exists(): + candidate.unlink() + + subprocess.run(["cargo", "fmt", "--all"], cwd=product, check=True) + actual = changed_files(product) + unexpected = actual - EXPECTED_CHANGED_FILES + missing = { + ".github/workflows/global-packages.yml", + ".zpkg.lock", + ".zpkg.toml", + "docs/global-packages.md", + "src/global.rs", + "tests/global_cli.rs", + "src/ops.rs", + } - actual + if unexpected: + raise SystemExit( + "materialization changed files outside the reviewed boundary: " + + ", ".join(sorted(unexpected)) + ) + if missing: + raise SystemExit( + "materialization failed to produce required files: " + + ", ".join(sorted(missing)) + ) + print("materialized files:") + for name in sorted(actual): + print(name) + + +if __name__ == "__main__": + main() diff --git a/scripts/mise_environment.py b/scripts/mise_environment.py new file mode 100644 index 00000000..1a590711 --- /dev/null +++ b/scripts/mise_environment.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +"""Black-box certification for zed-pkg's project-local mise adapter. + +The harness intentionally runs the real `zed` executable with an empty PATH. +It proves that import/verification parses committed manager state directly, +never invokes mise, never walks to parent/global configuration, and does not +mutate the consumer project. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any, Iterable + + +class HarnessFailure(RuntimeError): + pass + + +def sha256_hex(digit: str) -> str: + return "sha256:" + digit * 64 + + +def write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8", newline="\n") + + +def file_digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_snapshot(root: Path) -> dict[str, str]: + snapshot: dict[str, str] = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + snapshot[relative] = f"symlink:{os.readlink(path)}" + elif path.is_file(): + snapshot[relative] = f"file:{file_digest(path)}" + elif path.is_dir(): + snapshot[relative] = "dir" + return snapshot + + +def ensure(condition: bool, message: str) -> None: + if not condition: + raise HarnessFailure(message) + + +def run_zed( + zed: Path, + project: Path, + home: Path, + args: Iterable[str], + *, + expect_success: bool, + contains: str | None = None, +) -> subprocess.CompletedProcess[str]: + empty_path = home / "empty-path" + empty_path.mkdir(parents=True, exist_ok=True) + + env = os.environ.copy() + env.update( + { + "HOME": str(home), + "USERPROFILE": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "ZED_PKG_HOME": str(home / ".zed-pkg"), + "MISE_GLOBAL_CONFIG_FILE": str(home / ".config" / "mise" / "config.toml"), + # The executable is invoked by absolute path. Any attempt to shell + # out to mise or another ambient tool must fail. + "PATH": str(empty_path), + } + ) + for name in ( + "ZED_PKG_ENV_CONFIG", + "ZED_PKG_ENV_LOCK", + "ZED_PKG_ENV_JSON", + "ZED_PKG_FROZEN", + "MISE_CONFIG_FILE", + "MISE_TOML", + ): + env.pop(name, None) + + command = [str(zed), *args] + completed = subprocess.run( + command, + cwd=project, + env=env, + text=True, + capture_output=True, + check=False, + ) + succeeded = completed.returncode == 0 + if succeeded != expect_success: + raise HarnessFailure( + "unexpected command result\n" + f"command: {command!r}\n" + f"cwd: {project}\n" + f"returncode: {completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + if contains is not None: + combined = completed.stdout + "\n" + completed.stderr + ensure( + contains in combined, + f"expected {contains!r} in command output, got:\n{combined}", + ) + return completed + + +def parse_json(completed: subprocess.CompletedProcess[str]) -> Any: + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise HarnessFailure( + f"command did not emit valid JSON: {error}\nstdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) from error + + +def baseline_config(*, reordered: bool = False) -> str: + if reordered: + return """[tools] +python="3.12" +node="22" + +[settings] +lockfile_platforms=["macos-arm64","linux-x64"] +lockfile=true +""" + return """[settings] +lockfile = true +lockfile_platforms = ["linux-x64", "macos-arm64"] + +[tools] +node = "22" +python = "3.12" +""" + + +def baseline_lock(*, reordered: bool = False, node_url: str = "https://example.invalid/node") -> str: + node = f"""[[tools.node]] +version = "22.4.0" +backend = "core:node" + +[tools.node.platforms.linux-x64] +checksum = "{sha256_hex('a')}" +size = 100 +url = "{node_url}/linux-x64" + +[tools.node.platforms.macos-arm64] +url = "{node_url}/macos-arm64" +size = 101 +checksum = "{sha256_hex('c')}" +""" + python = f"""[[tools.python]] +version = "3.12.4" +backend = "core:python" + +[tools.python.platforms.linux-x64] +checksum = "{sha256_hex('d')}" +size = 199 +url = "https://example.invalid/python-linux" + +[tools.python.platforms.macos-arm64] +url = "https://example.invalid/python-macos" +size = 200 +checksum = "{sha256_hex('b')}" +""" + return (python + "\n" + node) if reordered else (node + "\n" + python) + +def write_locked_project(project: Path, *, reordered: bool = False, node_url: str | None = None) -> None: + project.mkdir(parents=True, exist_ok=True) + write(project / "mise.toml", baseline_config(reordered=reordered)) + write( + project / "mise.lock", + baseline_lock( + reordered=reordered, + node_url=node_url or "https://example.invalid/node", + ), + ) + + +def verify_json(zed: Path, project: Path, home: Path) -> dict[str, Any]: + completed = run_zed( + zed, + project, + home, + ( + "env", + "verify", + "mise", + "--config", + "mise.toml", + "--lock", + "mise.lock", + "--frozen", + "--json", + ), + expect_success=True, + ) + result = parse_json(completed) + ensure(result.get("manager") == "mise", f"wrong manager: {result!r}") + ensure(result.get("config") == "mise.toml", f"wrong config: {result!r}") + ensure(result.get("lock") == "mise.lock", f"wrong lock: {result!r}") + ensure(result.get("tools") == 2, f"wrong tool count: {result!r}") + ensure(result.get("verified") is True, f"not verified: {result!r}") + digest = result.get("environment_plan_sha256") + ensure(isinstance(digest, str) and len(digest) == 64, f"invalid digest: {result!r}") + ensure(all(character in "0123456789abcdef" for character in digest), f"non-hex digest: {digest!r}") + return result + + +def certify(zed: Path, work_root: Path) -> None: + ensure(zed.is_file(), f"zed executable does not exist: {zed}") + ensure(not work_root.exists(), f"work root must be fresh: {work_root}") + work_root.mkdir(parents=True) + + home = work_root / "home" + write(home / ".config" / "mise" / "config.toml", '[tools]\nglobal_canary = "latest"\n') + + workspace = work_root / "workspace" + write(workspace / "mise.toml", '[tools]\nparent_canary = "latest"\n') + project = workspace / "project" + write_locked_project(project) + + before = tree_snapshot(project) + verified = verify_json(zed, project, home) + after = tree_snapshot(project) + ensure(before == after, f"verification mutated project state:\nbefore={before!r}\nafter={after!r}") + + imported = parse_json( + run_zed( + zed, + project, + home, + ( + "env", + "import", + "mise", + "--config", + "mise.toml", + "--lock", + "mise.lock", + "--frozen", + "--json", + ), + expect_success=True, + ) + ) + tools = imported.get("tools", {}) + ensure(set(tools) == {"node", "python"}, f"parent/global config leaked: {tools!r}") + ensure(tools["node"].get("requirement") == "22", f"wrong node requirement: {tools!r}") + ensure(tools["node"].get("resolved") == "22.4.0", f"wrong node resolution: {tools!r}") + ensure(tools["node"].get("backend") == "core:node", f"wrong node backend: {tools!r}") + ensure(imported.get("activation") == "frozen-install", f"wrong activation: {imported!r}") + + # Equivalent manager state must have the same semantic identity despite + # TOML key/table ordering and presentation differences. + reordered = work_root / "reordered" + write_locked_project(reordered, reordered=True) + reordered_result = verify_json(zed, reordered, home) + ensure( + reordered_result["environment_plan_sha256"] == verified["environment_plan_sha256"], + "presentation-only TOML changes altered the normalized environment identity", + ) + + # Manager provenance is part of semantic drift identity even when the + # artifact checksum remains unchanged. + mirror = work_root / "mirror" + write_locked_project(mirror, reordered=True, node_url="https://mirror.invalid/node") + mirror_result = verify_json(zed, mirror, home) + ensure( + mirror_result["environment_plan_sha256"] != verified["environment_plan_sha256"], + "changing locked artifact provenance did not alter the environment identity", + ) + + malformed = work_root / "malformed" + write_locked_project(malformed) + malformed_text = (malformed / "mise.lock").read_text(encoding="utf-8").replace( + sha256_hex("a"), "sha256:not-hex" + ) + write(malformed / "mise.lock", malformed_text) + run_zed( + zed, + malformed, + home, + ("env", "verify", "mise", "--frozen", "--json"), + expect_success=False, + contains="invalid checksum", + ) + + no_checksum = work_root / "no-checksum" + write_locked_project(no_checksum) + no_checksum_text = "\n".join( + line + for line in (no_checksum / "mise.lock").read_text(encoding="utf-8").splitlines() + if sha256_hex("a") not in line + ) + "\n" + write(no_checksum / "mise.lock", no_checksum_text) + run_zed( + zed, + no_checksum, + home, + ("env", "verify", "mise", "--frozen", "--json"), + expect_success=False, + contains="no cryptographic checksum", + ) + + drift = work_root / "drift" + write_locked_project(drift) + write(drift / "mise.toml", baseline_config() + '\ngo = "1.24"\n') + run_zed( + zed, + drift, + home, + ("env", "verify", "mise", "--frozen", "--json"), + expect_success=False, + contains="lock/config drift", + ) + + ambiguous = work_root / "ambiguous" + write_locked_project(ambiguous) + write(ambiguous / ".mise.toml", '[tools]\nruby = "3.4"\n') + run_zed( + zed, + ambiguous, + home, + ("env", "verify", "mise", "--frozen", "--json"), + expect_success=False, + contains="multiple project-local", + ) + + child = workspace / "child-without-config" + child.mkdir() + run_zed( + zed, + child, + home, + ("env", "import", "mise", "--json"), + expect_success=False, + contains="no project-local mise configuration", + ) + + tool_versions = work_root / "tool-versions" + tool_versions.mkdir() + write(tool_versions / ".tool-versions", "node 22.4.0\npython 3.12.4\n") + authoring_plan = parse_json( + run_zed( + zed, + tool_versions, + home, + ("env", "import", "mise", "--config", ".tool-versions", "--json"), + expect_success=True, + ) + ) + ensure(set(authoring_plan.get("tools", {})) == {"node", "python"}, "bad .tool-versions import") + run_zed( + zed, + tool_versions, + home, + ( + "env", + "verify", + "mise", + "--config", + ".tool-versions", + "--frozen", + "--json", + ), + expect_success=False, + contains="requires a project-local lockfile", + ) + + outside = work_root / "outside.toml" + write(outside, '[tools]\nnode = "22"\n') + escape = work_root / "symlink-escape" + escape.mkdir() + try: + (escape / "mise.toml").symlink_to(outside) + except (OSError, NotImplementedError): + pass + else: + run_zed( + zed, + escape, + home, + ("env", "import", "mise", "--config", "mise.toml", "--json"), + expect_success=False, + contains="escapes the project root", + ) + + print( + json.dumps( + { + "certified": True, + "manager": "mise", + "environment_plan_sha256": verified["environment_plan_sha256"], + "project_mutation": False, + "ambient_mise_required": False, + "parent_global_leakage": False, + }, + indent=2, + sort_keys=True, + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + + try: + certify(args.zed.resolve(), args.work_root.resolve()) + except HarnessFailure as error: + print(f"mise environment certification failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mise_export_boundaries.py b/scripts/mise_export_boundaries.py new file mode 100644 index 00000000..a7f24007 --- /dev/null +++ b/scripts/mise_export_boundaries.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Black-box certification for deterministic, conflict-safe mise export.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Iterable + + +class HarnessFailure(RuntimeError): + pass + + +def ensure(condition: bool, message: str) -> None: + if not condition: + raise HarnessFailure(message) + + +def write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def snapshot(root: Path) -> dict[str, str]: + return { + path.relative_to(root).as_posix(): sha256(path) + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def snapshot_without_operation_rendezvous(root: Path) -> dict[str, str]: + """Snapshot semantic project state while validating the lock separately.""" + relative_lock = Path(".zed/operation.lock") + return { + relative: digest + for relative, digest in snapshot(root).items() + if Path(relative) != relative_lock + } + + +def assert_regular_operation_rendezvous(root: Path) -> None: + operation_lock = root / ".zed/operation.lock" + ensure(operation_lock.is_file(), "mutation did not retain the operation rendezvous") + ensure(not operation_lock.is_symlink(), "operation rendezvous is a symlink") + unexpected = [ + path.relative_to(root).as_posix() + for path in sorted((root / ".zed").rglob("*")) + if path != operation_lock + ] + ensure( + not unexpected, + f"failed mutation left native adapter state: {unexpected!r}", + ) + +def base_plan() -> dict[str, object]: + return { + "schema": 2, + "tools": { + "node": [ + {"requirement": "22.4.0"}, + {"requirement": "20.15.1"}, + ] + }, + "env": {"APP_ENV": "test", "RETRIES": 3}, + "vars": {"release": {"channel": "stable"}}, + "tasks": { + "prepare": {"run": ["echo prepare"]}, + "setup": { + "description": "Restore dependencies", + "aliases": ["bootstrap"], + "depends": ["prepare"], + "run": ["zed install --frozen", "cargo check"], + }, + }, + "platforms": ["linux-x64", "macos-arm64"], + "activation": "none", + } + + +def run( + zed: Path, + project: Path, + home: Path, + args: Iterable[str], + *, + success: bool, + contains: str | None = None, +) -> subprocess.CompletedProcess[str]: + empty_path = home / "empty-path" + empty_path.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env.update( + { + "HOME": str(home), + "USERPROFILE": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "ZED_PKG_HOME": str(home / ".zed-pkg"), + "PATH": str(empty_path), + } + ) + for key in ( + "ZED_PKG_ENV_PLAN", + "ZED_PKG_ENV_OUTPUT", + "ZED_PKG_ENV_JSON", + "ZED_PKG_ENV_CHECK", + "ZED_PKG_ENV_WRITE", + "MISE_CONFIG_FILE", + "MISE_TOML", + ): + env.pop(key, None) + + completed = subprocess.run( + [str(zed), *args], + cwd=project, + env=env, + text=True, + capture_output=True, + check=False, + ) + if (completed.returncode == 0) != success: + raise HarnessFailure( + f"unexpected status {completed.returncode} for {list(args)!r}\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + if contains is not None: + combined = completed.stdout + "\n" + completed.stderr + ensure(contains in combined, f"missing {contains!r} in:\n{combined}") + return completed + + +def export_args(*extra: str, output: str = ".mise.toml") -> tuple[str, ...]: + return ( + "env", + "export", + "mise", + "--plan", + "zed-env.json", + "--output", + output, + *extra, + ) + + +def certify(zed: Path, root: Path) -> None: + ensure(zed.is_file(), f"missing zed executable: {zed}") + ensure(not root.exists(), f"work root must be fresh: {root}") + root.mkdir(parents=True) + home = root / "home" + home.mkdir() + assertions: list[str] = [] + + project = root / "owned" + project.mkdir() + write_json(project / "zed-env.json", base_plan()) + initial = snapshot(project) + + printed = run(zed, project, home, export_args(), success=True) + ensure("[tools]" in printed.stdout, "print mode did not emit mise TOML") + ensure(snapshot(project) == initial, "print mode mutated project") + assertions.append("print-read-only") + + written = run( + zed, + project, + home, + export_args("--write", "--json"), + success=True, + ) + written_json = json.loads(written.stdout) + ensure(written_json.get("action") == "written", f"unexpected write: {written_json}") + ensure((project / ".mise.toml").is_file(), "write mode omitted output") + ensure( + (project / ".zed/mise-export-state.json").is_file(), + "write mode omitted ownership state", + ) + assertions.append("write-owned") + + checked = run( + zed, + project, + home, + export_args("--check", "--json"), + success=True, + ) + ensure(json.loads(checked.stdout).get("action") == "verified", "check not verified") + assertions.append("check-verified") + + unchanged = run( + zed, + project, + home, + export_args("--write", "--json"), + success=True, + ) + ensure( + json.loads(unchanged.stdout).get("action") == "unchanged", + "idempotent write was not unchanged", + ) + assertions.append("write-idempotent") + + original_output = (project / ".mise.toml").read_bytes() + (project / ".mise.toml").write_text("# hand edit\n", encoding="utf-8") + run(zed, project, home, export_args("--write"), success=False, contains="edited") + ensure( + (project / ".mise.toml").read_text(encoding="utf-8") == "# hand edit\n", + "edited output overwritten", + ) + (project / ".mise.toml").write_bytes(original_output) + assertions.append("edited-output-protected") + + unowned = root / "unowned" + unowned.mkdir() + write_json(unowned / "zed-env.json", base_plan()) + (unowned / ".mise.toml").write_text('[tools]\nnode = "18"\n', encoding="utf-8") + before = snapshot_without_operation_rendezvous(unowned) + run(zed, unowned, home, export_args("--write"), success=False, contains="hand-authored") + assert_regular_operation_rendezvous(unowned) + ensure( + snapshot_without_operation_rendezvous(unowned) == before, + "unowned output failure mutated semantic project state", + ) + assertions.append("unowned-output-protected") + assertions.append("operation-rendezvous-nonsemantic") + + nested = root / "nested-output" + nested.mkdir() + write_json(nested / "zed-env.json", base_plan()) + run( + zed, + nested, + home, + export_args("--write", output="generated/config/.mise.toml"), + success=True, + ) + ensure((nested / "generated/config/.mise.toml").is_file(), "safe missing parents not created") + assertions.append("safe-parent-creation") + + boundaries = root / "boundaries" + boundaries.mkdir() + write_json(boundaries / "zed-env.json", base_plan()) + original_plan = (boundaries / "zed-env.json").read_bytes() + for output, diagnostic in ( + ("zed-env.json", "cannot overwrite"), + ("ZED-ENV.JSON", "cannot overwrite"), + (".zed/mise-export-state.json", "reserved export state"), + (".ZED/MISE-EXPORT-STATE.JSON", "reserved export state"), + (".zpkg-staging/mise.toml", "reserved transaction staging"), + (".ZPKG-STAGING/mise.toml", "reserved transaction staging"), + ): + run( + zed, + boundaries, + home, + export_args("--write", output=output), + success=False, + contains=diagnostic, + ) + ensure( + (boundaries / "zed-env.json").read_bytes() == original_plan, + f"boundary {output} mutated plan", + ) + assertions.append("portable-reserved-paths") + + nested_secret = root / "nested-secret" + nested_secret.mkdir() + plan = base_plan() + plan["vars"] = {"release": {"api_token": "plaintext"}} + write_json(nested_secret / "zed-env.json", plan) + run( + zed, + nested_secret, + home, + export_args(), + success=False, + contains="vars.release.api_token", + ) + ensure(not (nested_secret / ".mise.toml").exists(), "secret rejection wrote output") + assertions.append("nested-secret-rejected") + + tool_secret = root / "tool-secret" + tool_secret.mkdir() + plan = base_plan() + plan["tools"] = { + "node": [ + { + "requirement": "22.4.0", + "options": {"config": {"password": "plaintext"}}, + } + ] + } + write_json(tool_secret / "zed-env.json", plan) + run( + zed, + tool_secret, + home, + export_args(), + success=False, + contains="tools.node.versions[0].options.config.password", + ) + assertions.append("nested-tool-secret-rejected") + + symlink_case = root / "symlink-state" + symlink_case.mkdir() + write_json(symlink_case / "zed-env.json", base_plan()) + outside = root / "outside-state" + outside.mkdir() + try: + os.symlink(outside, symlink_case / ".zed", target_is_directory=True) + except (OSError, NotImplementedError): + assertions.append("symlink-state-skipped") + else: + run( + zed, + symlink_case, + home, + export_args("--write"), + success=False, + contains="mise export state crosses a symlink", + ) + ensure(not (symlink_case / ".mise.toml").exists(), "symlink failure wrote output") + ensure(not (outside / "mise-export-state.json").exists(), "state escaped project") + assertions.append("symlink-state-rejected") + + print( + json.dumps( + { + "schema": "zed-pkg.mise-export-boundaries.v1", + "certified": True, + "assertions": assertions, + "ambient_mise_required": False, + }, + sort_keys=True, + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + try: + certify(args.zed.resolve(), args.work_root.resolve()) + except (HarnessFailure, json.JSONDecodeError) as error: + print(f"mise export boundary certification failed: {error}", file=sys.stderr) + return 1 + finally: + if args.work_root.exists() and os.environ.get("KEEP_MISE_EXPORT_ROOT") != "1": + shutil.rmtree(args.work_root, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mise_export_ownership_state.py b/scripts/mise_export_ownership_state.py new file mode 100644 index 00000000..94c147aa --- /dev/null +++ b/scripts/mise_export_ownership_state.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Black-box regressions for Zed-owned mise export state integrity.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Iterable + + +class HarnessFailure(RuntimeError): + pass + + +def ensure(condition: bool, message: str) -> None: + if not condition: + raise HarnessFailure(message) + + +def write_plan(path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "schema": 2, + "tools": {"node": [{"requirement": "22.4.0"}]}, + "env": {"APP_ENV": "test"}, + "vars": {}, + "tasks": {}, + "platforms": ["linux-x64", "macos-arm64", "windows-x64"], + "activation": "none", + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + newline="\n", + ) + + +def run( + zed: Path, + project: Path, + home: Path, + args: Iterable[str], + *, + success: bool, + contains: str | None = None, +) -> subprocess.CompletedProcess[str]: + empty_path = home / "empty-path" + empty_path.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env.update( + { + "HOME": str(home), + "USERPROFILE": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "ZED_PKG_HOME": str(home / ".zed-pkg"), + "PATH": str(empty_path), + } + ) + for key in tuple(env): + if key.startswith("ZED_PKG_ENV_") or key in {"MISE_CONFIG_FILE", "MISE_TOML"}: + env.pop(key, None) + + completed = subprocess.run( + [str(zed), *args], + cwd=project, + env=env, + text=True, + capture_output=True, + check=False, + ) + if (completed.returncode == 0) != success: + raise HarnessFailure( + f"unexpected status {completed.returncode} for {list(args)!r}\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + if contains is not None: + combined = completed.stdout + "\n" + completed.stderr + ensure(contains in combined, f"missing {contains!r} in:\n{combined}") + return completed + + +def export_args(plan: str) -> tuple[str, ...]: + return ( + "env", + "export", + "mise", + "--plan", + plan, + "--output", + ".mise.toml", + "--write", + "--json", + ) + + +def certify(zed: Path, root: Path) -> None: + ensure(zed.is_file(), f"missing zed executable: {zed}") + ensure(not root.exists(), f"work root must be fresh: {root}") + root.mkdir(parents=True) + home = root / "home" + home.mkdir() + + reserved_project = root / "reserved-source" + reserved_project.mkdir() + reserved_state = reserved_project / ".zed/mise-export-state.json" + write_plan(reserved_state) + reserved_before = reserved_state.read_bytes() + run( + zed, + reserved_project, + home, + export_args(".zed/mise-export-state.json"), + success=False, + contains="environment plan cannot target reserved export state", + ) + ensure(reserved_state.read_bytes() == reserved_before, "reserved source plan was mutated") + ensure(not (reserved_project / ".mise.toml").exists(), "reserved source created output") + + project = root / "project" + project.mkdir() + write_plan(project / "plan-one.json") + write_plan(project / "plan-two.json") + + first = run(zed, project, home, export_args("plan-one.json"), success=True) + ensure(json.loads(first.stdout).get("action") == "written", "initial export was not written") + + output_path = project / ".mise.toml" + state_path = project / ".zed/mise-export-state.json" + output_before = output_path.read_bytes() + state_before = state_path.read_bytes() + + run( + zed, + project, + home, + export_args("plan-two.json"), + success=False, + contains="change ownership", + ) + ensure(output_path.read_bytes() == output_before, "ownership-transfer failure changed output") + ensure(state_path.read_bytes() == state_before, "ownership-transfer failure changed state") + + state = json.loads(state_before) + record = state["outputs"][".mise.toml"] + record["output_sha256"] = "0" * 64 + corrupted = (json.dumps(state, indent=2, sort_keys=True) + "\n").encode() + state_path.write_bytes(corrupted) + + run( + zed, + project, + home, + export_args("plan-one.json"), + success=False, + contains="export state records sha256", + ) + ensure(output_path.read_bytes() == output_before, "state-corruption failure changed output") + ensure(state_path.read_bytes() == corrupted, "state-corruption failure repaired state silently") + + print( + json.dumps( + { + "schema": "zed-pkg.mise-export-ownership-state.v1", + "certified": True, + "reserved_state_source_rejected": True, + "ownership_transfer_rejected": True, + "stale_digest_rejected": True, + "ambient_mise_required": False, + }, + sort_keys=True, + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + try: + certify(args.zed.resolve(), args.work_root.resolve()) + except (HarnessFailure, json.JSONDecodeError, KeyError) as error: + print(f"mise export ownership-state certification failed: {error}", file=sys.stderr) + return 1 + finally: + if args.work_root.exists() and os.environ.get("KEEP_MISE_EXPORT_ROOT") != "1": + shutil.rmtree(args.work_root, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mise_frozen_identity.py b/scripts/mise_frozen_identity.py new file mode 100644 index 00000000..65cf2a29 --- /dev/null +++ b/scripts/mise_frozen_identity.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Black-box negative and positive tests for frozen mise identity parity.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Iterable + + +class HarnessFailure(RuntimeError): + pass + + +def ensure(condition: bool, message: str) -> None: + if not condition: + raise HarnessFailure(message) + + +def checksum(digit: str) -> str: + return "sha256:" + digit * 64 + + +def write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8", newline="\n") + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def snapshot(root: Path) -> dict[str, str]: + return { + path.relative_to(root).as_posix(): digest(path) + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def run( + zed: Path, + project: Path, + home: Path, + args: Iterable[str], + *, + success: bool, + contains: str | None = None, +) -> subprocess.CompletedProcess[str]: + empty_path = home / "empty-path" + empty_path.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env.update( + { + "HOME": str(home), + "USERPROFILE": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "ZED_PKG_HOME": str(home / ".zed-pkg"), + "PATH": str(empty_path), + } + ) + for key in ( + "ZED_PKG_ENV_CONFIG", + "ZED_PKG_ENV_LOCK", + "ZED_PKG_ENV_JSON", + "ZED_PKG_FROZEN", + "MISE_CONFIG_FILE", + "MISE_TOML", + ): + env.pop(key, None) + + completed = subprocess.run( + [str(zed), *args], + cwd=project, + env=env, + text=True, + capture_output=True, + check=False, + ) + if (completed.returncode == 0) != success: + raise HarnessFailure( + f"unexpected status {completed.returncode} for {list(args)!r}\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + if contains is not None: + combined = completed.stdout + "\n" + completed.stderr + ensure(contains in combined, f"missing {contains!r} in:\n{combined}") + return completed + + +def config(requirement: str = "^1.7", *, os_constraint: str | None = None) -> str: + tool = f'"aqua:jqlang/jq" = "{requirement}"' + if os_constraint is not None: + tool = ( + '"aqua:jqlang/jq" = { version = ' + f'"{requirement}", os = "{os_constraint}" }}' + ) + return ( + '[settings]\n' + 'lockfile_platforms = ["linux-x64", "macos-arm64"]\n\n' + '[tools]\n' + f'{tool}\n' + ) + + +def lock(backend: str, version: str, platforms: tuple[str, ...]) -> str: + output = ( + '[[tools."aqua:jqlang/jq"]]\n' + f'version = "{version}"\n' + f'backend = "{backend}"\n\n' + ) + for index, platform in enumerate(platforms): + output += ( + f'[tools."aqua:jqlang/jq".platforms.{platform}]\n' + f'checksum = "{checksum(chr(ord("a") + index))}"\n' + f'url = "https://example.invalid/jq-{platform}"\n\n' + ) + return output + + +def write_case( + root: Path, + *, + requirement: str = "^1.7", + backend: str = "aqua:jqlang/jq", + version: str = "1.7.1", + platforms: tuple[str, ...] = ("linux-x64", "macos-arm64"), + os_constraint: str | None = None, +) -> None: + root.mkdir(parents=True) + write(root / "mise.toml", config(requirement, os_constraint=os_constraint)) + write(root / "mise.lock", lock(backend, version, platforms)) + + +def verify(zed: Path, project: Path, home: Path, *, success: bool, contains: str | None = None): + before = snapshot(project) + completed = run( + zed, + project, + home, + ("env", "verify", "mise", "--frozen", "--json"), + success=success, + contains=contains, + ) + ensure(snapshot(project) == before, f"verification mutated {project}") + return completed + + +def certify(zed: Path, root: Path) -> None: + ensure(zed.is_file(), f"missing zed executable: {zed}") + ensure(not root.exists(), f"work root must be fresh: {root}") + root.mkdir(parents=True) + home = root / "home" + home.mkdir() + + valid = root / "valid" + write_case(valid) + completed = verify(zed, valid, home, success=True) + result = json.loads(completed.stdout) + ensure(result.get("verified") is True, f"valid fixture not verified: {result!r}") + + missing_platform = root / "missing-platform" + write_case(missing_platform, platforms=("linux-x64",)) + verify( + zed, + missing_platform, + home, + success=False, + contains="missing requested locked platform", + ) + + constrained = root / "os-constrained" + write_case( + constrained, + platforms=("linux-x64",), + os_constraint="linux", + ) + verify(zed, constrained, home, success=True) + + backend_drift = root / "backend-drift" + write_case(backend_drift, backend="core:node") + verify(zed, backend_drift, home, success=False, contains="mise backend drift") + + version_drift = root / "version-drift" + write_case(version_drift, version="1.6.0") + verify(zed, version_drift, home, success=False, contains="mise version drift") + + prefix_ok = root / "prefix-ok" + write_case(prefix_ok, requirement="22", version="22.4.0") + verify(zed, prefix_ok, home, success=True) + + prefix_boundary = root / "prefix-boundary" + write_case(prefix_boundary, requirement="22", version="220.0.0") + verify(zed, prefix_boundary, home, success=False, contains="mise version drift") + + unsupported_range = root / "unsupported-range" + write_case(unsupported_range, requirement="^1.7 || banana", version="1.7.1") + verify( + zed, + unsupported_range, + home, + success=False, + contains="unsupported mise SemVer requirement", + ) + + print( + json.dumps( + { + "schema": "zed-pkg.mise-frozen-identity.v1", + "certified": True, + "requested_platforms": True, + "tool_os_constraints": True, + "backend_equality": True, + "requirement_satisfaction": True, + "read_only": True, + "ambient_mise_required": False, + }, + sort_keys=True, + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + try: + certify(args.zed.resolve(), args.work_root.resolve()) + except (HarnessFailure, json.JSONDecodeError) as error: + print(f"mise frozen identity certification failed: {error}", file=sys.stderr) + return 1 + finally: + if args.work_root.exists() and os.environ.get("KEEP_MISE_IDENTITY_ROOT") != "1": + shutil.rmtree(args.work_root, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mise_lock_naming.py b/scripts/mise_lock_naming.py new file mode 100644 index 00000000..51258cfe --- /dev/null +++ b/scripts/mise_lock_naming.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Certify mise's normalized adjacent lockfile naming through the real zed CLI.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +def fail(message: str) -> None: + raise RuntimeError(message) + + +def run( + zed: Path, + project: Path, + home: Path, + *args: str, + expect_success: bool, +) -> subprocess.CompletedProcess[str]: + empty_path = home / "empty-path" + empty_path.mkdir(parents=True, exist_ok=True) + env = os.environ.copy() + env.update( + { + "HOME": str(home), + "USERPROFILE": str(home), + "XDG_CONFIG_HOME": str(home / ".config"), + "ZED_PKG_HOME": str(home / ".zed-pkg"), + "PATH": str(empty_path), + } + ) + completed = subprocess.run( + [str(zed), *args], + cwd=project, + env=env, + text=True, + capture_output=True, + check=False, + ) + if (completed.returncode == 0) != expect_success: + fail( + "unexpected command result\n" + f"args={args!r}\n" + f"returncode={completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}" + ) + return completed + + +def write_fixture(project: Path, lock_name: str) -> None: + project.mkdir(parents=True) + (project / ".mise.toml").write_text('[tools]\nnode = "22"\n', encoding="utf-8") + (project / lock_name).write_text( + "[[tools.node]]\n" + 'version = "22.4.0"\n' + 'backend = "core:node"\n' + "[tools.node.platforms.linux-x64]\n" + f'checksum = "sha256:{"a" * 64}"\n', + encoding="utf-8", + ) + + +def certify(zed: Path, work_root: Path) -> None: + if not zed.is_file(): + fail(f"zed executable does not exist: {zed}") + if work_root.exists(): + fail(f"work root must be fresh: {work_root}") + work_root.mkdir(parents=True) + home = work_root / "home" + home.mkdir() + + normalized = work_root / "normalized" + write_fixture(normalized, "mise.lock") + completed = run( + zed, + normalized, + home, + "env", + "verify", + "mise", + "--frozen", + "--json", + expect_success=True, + ) + result = json.loads(completed.stdout) + if result.get("config") != ".mise.toml": + fail(f"unexpected implicit config: {result!r}") + if result.get("lock") != "mise.lock": + fail(f"unexpected implicit lock: {result!r}") + if (normalized / ".mise.lock").exists(): + fail("verification invented .mise.lock") + + explicit = run( + zed, + normalized, + home, + "env", + "verify", + "mise", + "--config", + ".mise.toml", + "--frozen", + "--json", + expect_success=True, + ) + explicit_result = json.loads(explicit.stdout) + if explicit_result.get("lock") != "mise.lock": + fail(f"explicit .mise.toml did not normalize to mise.lock: {explicit_result!r}") + + legacy = work_root / "legacy-dot-lock" + write_fixture(legacy, ".mise.lock") + rejected = run( + zed, + legacy, + home, + "env", + "verify", + "mise", + "--config", + ".mise.toml", + "--frozen", + "--json", + expect_success=False, + ) + combined = rejected.stdout + "\n" + rejected.stderr + if "requires a project-local lockfile" not in combined: + fail(f"legacy .mise.lock was not rejected clearly:\n{combined}") + + print( + json.dumps( + { + "certified": True, + "config": ".mise.toml", + "lock": "mise.lock", + "legacy_dot_lock_rejected": True, + }, + sort_keys=True, + ) + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + try: + certify(args.zed.resolve(), args.work_root.resolve()) + except (RuntimeError, json.JSONDecodeError) as error: + print(f"mise lock naming certification failed: {error}", file=sys.stderr) + return 1 + finally: + if args.work_root.exists() and os.environ.get("KEEP_MISE_LOCK_NAMING_ROOT") != "1": + shutil.rmtree(args.work_root, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mise_runtime.sh b/scripts/mise_runtime.sh new file mode 100644 index 00000000..74133ad6 --- /dev/null +++ b/scripts/mise_runtime.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# Black-box certification for zed-pkg's runtime mise composition. +set -euo pipefail + +report_error() { + local status=$? + printf 'mise runtime certification aborted at line %s: %s (status %s)\n' \ + "$1" "$2" "$status" >&2 + exit "$status" +} +trap 'report_error "$LINENO" "$BASH_COMMAND"' ERR + +usage() { + echo "usage: $0 --zed /absolute/path/to/zed --work-root /fresh/path" >&2 + exit 64 +} + +zed='' +work_root='' +while (($#)); do + case "$1" in + --zed) + (($# >= 2)) || usage + zed=$2 + shift 2 + ;; + --work-root) + (($# >= 2)) || usage + work_root=$2 + shift 2 + ;; + *) usage ;; + esac +done + +[[ -n "$zed" && -n "$work_root" ]] || usage +[[ -x "$zed" ]] || { echo "zed executable is missing: $zed" >&2; exit 1; } +[[ ! -e "$work_root" ]] || { echo "work root already exists: $work_root" >&2; exit 1; } +mkdir -p "$work_root/logs" "$work_root/stub-bin" "$work_root/mise-tool-bin" +work_root=$(cd "$work_root" && pwd -P) +zed=$(cd "$(dirname "$zed")" && printf '%s/%s\n' "$(pwd -P)" "$(basename "$zed")") +shell_bin=/bin/bash +original_path=${PATH:-} +unset MISE_STUB_ACTIVE ZED_DEV_MISE_ACTIVE __MISE_DIFF || true +passed=0 + +fail() { + echo "mise runtime certification failed: $*" >&2 + exit 1 +} + +pass() { + passed=$((passed + 1)) + printf 'ok %02d - %s\n' "$passed" "$1" +} + +write_project() { + local project=$1 + mkdir -p "$project/.git" "$project/zed_modules/.bin" + printf '%s\n' '{}' > "$project/package.json" + cat > "$project/zed_modules/.bin/zed-tool" <<'TOOL' +#!/bin/sh +printf zed-tool +TOOL + chmod +x "$project/zed_modules/.bin/zed-tool" +} + +cat > "$work_root/mise-tool-bin/mise-tool" <<'TOOL' +#!/bin/sh +printf mise-tool +TOOL +chmod +x "$work_root/mise-tool-bin/mise-tool" + +cat > "$work_root/stub-bin/mise" <<'MISE' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == '--version' ]]; then + printf '%s\n' 'mise 2099.0.0-test' + exit 0 +fi +{ + printf 'argv=%q ' "$@" + printf '\n' + printf 'cwd=%s\n' "$PWD" + printf 'MISE_CEILING_PATHS=%s\n' "${MISE_CEILING_PATHS:-}" + printf 'MISE_CONFIG_DIR=%s\n' "${MISE_CONFIG_DIR:-}" + printf 'MISE_LOCKED=%s\n' "${MISE_LOCKED:-}" + printf 'MISE_OVERRIDE_CONFIG_FILENAMES=%s\n' "${MISE_OVERRIDE_CONFIG_FILENAMES:-}" + printf 'MISE_OVERRIDE_TOOL_VERSIONS_FILENAMES=%s\n' "${MISE_OVERRIDE_TOOL_VERSIONS_FILENAMES:-}" + printf 'MISE_SYSTEM_CONFIG_DIR=%s\n' "${MISE_SYSTEM_CONFIG_DIR:-}" + printf 'ZED_DEV_MISE_ACTIVE=%s\n' "${ZED_DEV_MISE_ACTIVE:-}" + printf '%s\n' '---' +} >> "$MISE_STUB_LOG" +[[ "${1:-}" == exec ]] || exit 64 +shift +[[ "${1:-}" == -- ]] || exit 64 +shift +[[ $# -gt 0 ]] || exit 64 +export MISE_STUB_ACTIVE=1 +export PATH="$MISE_STUB_TOOL_DIR:${PATH:-}" +exec "$@" +MISE +chmod +x "$work_root/stub-bin/mise" + +base_env=( + "PATH=$work_root/stub-bin:$original_path" + "MISE_STUB_TOOL_DIR=$work_root/mise-tool-bin" + "ZED_PKG_HOME=$work_root/zed-home" +) + +run_capture() { + local expected=$1 + local stdout=$2 + local stderr=$3 + shift 3 + set +e + "$@" >"$stdout" 2>"$stderr" + local status=$? + set -e + if [[ $status -ne $expected ]]; then + cat "$stdout" >&2 || true + cat "$stderr" >&2 || true + fail "expected status $expected, got $status: $*" + fi +} + +project="$work_root/required" +write_project "$project" +printf '%s\n' '[tools]' 'node = "22.4.0"' > "$project/mise.toml" +log="$work_root/logs/required.log" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --nix never --mise required --python-venv never \ + --shell "$shell_bin" \ + -c 'test "$MISE_STUB_ACTIVE" = 1; test "$(mise-tool)" = mise-tool; test "$(zed-tool)" = zed-tool; case "$PATH" in "$PWD/zed_modules/.bin":*"$MISE_STUB_TOOL_DIR"*) ;; *) exit 46 ;; esac; printf required-ok' +) | grep -q required-ok +[[ $(grep -c '^argv=' "$log") -eq 1 ]] || fail 'required mode did not enter mise exactly once' +pass 'required activation, non-login PATH preservation, and Zed bin coexistence' + +project="$work_root/env-required" +write_project "$project" +printf '%s\n' '[tools]' 'python = "3.12.4"' > "$project/.mise.toml" +log="$work_root/logs/env-required.log" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" ZED_DEV_MISE=required "$zed" dev \ + --no-install --nix never --python-venv never --shell "$shell_bin" \ + -c 'test "$MISE_STUB_ACTIVE" = 1; printf env-required-ok' +) | grep -q env-required-ok +[[ $(grep -c '^argv=' "$log") -eq 1 ]] || fail 'ZED_DEV_MISE did not select required mode' +pass 'flags-to-environment fallback' + +project="$work_root/cli-never" +write_project "$project" +log="$work_root/logs/cli-never.log" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" ZED_DEV_MISE=required "$zed" dev \ + --no-install --nix never --mise never --python-venv never --shell "$shell_bin" \ + -c 'test -z "${MISE_STUB_ACTIVE+x}"; printf cli-never-ok' +) | grep -q cli-never-ok +[[ ! -s "$log" ]] || fail 'CLI --mise never did not override ZED_DEV_MISE' +pass 'CLI precedence over environment' + +project="$work_root/auto-native" +write_project "$project" +log="$work_root/logs/auto-native.log" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --nix never --mise auto --python-venv never --shell "$shell_bin" \ + -c 'test -z "${MISE_STUB_ACTIVE+x}"; printf auto-native-ok' +) | grep -q auto-native-ok +[[ ! -s "$log" ]] || fail 'auto mode probed mise without project configuration' +pass 'auto fallback without configuration' + +project="$work_root/required-missing-config" +write_project "$project" +log="$work_root/logs/required-missing-config.log" +: > "$log" +( + cd "$project" + run_capture 1 "$work_root/required-missing-config.out" "$work_root/required-missing-config.err" \ + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --nix never --mise required --python-venv never --shell "$shell_bin" -c true +) +grep -q 'no project-local `mise.toml` or `.mise.toml`' "$work_root/required-missing-config.err" +[[ ! -s "$log" ]] || fail 'missing configuration should fail before probing mise' +pass 'required failure without configuration' + +project="$work_root/required-missing-binary" +write_project "$project" +printf '%s\n' '[tools]' 'node = "22.4.0"' > "$project/mise.toml" +( + cd "$project" + run_capture 1 "$work_root/required-missing-binary.out" "$work_root/required-missing-binary.err" \ + env PATH= "ZED_PKG_HOME=$work_root/zed-home-missing-binary" "$zed" dev \ + --no-install --nix never --mise required --python-venv never --shell "$shell_bin" -c true +) +grep -q '`mise` is not available on PATH' "$work_root/required-missing-binary.err" +pass 'required failure without executable' + +project="$work_root/frozen" +write_project "$project" +printf '%s\n' '[tools]' 'node = "22.4.0"' > "$project/mise.toml" +printf '%s\n' '[tools]' 'node = "999.0.0"' > "$project/.mise.toml" +printf '%s\n' 'nodejs latest' > "$project/.tool-versions" +log="$work_root/logs/frozen.log" +: > "$log" +( + cd "$project" + run_capture 1 "$work_root/frozen-missing.out" "$work_root/frozen-missing.err" \ + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --frozen --nix never --mise required --python-venv never --shell "$shell_bin" -c true +) +grep -q 'requires the adjacent mise lockfile' "$work_root/frozen-missing.err" +[[ ! -s "$log" ]] || fail 'frozen missing-lock failure unexpectedly entered mise' +printf '%s\n' '{}' > "$project/mise.lock" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --frozen --nix never --mise required --python-venv never --shell "$shell_bin" \ + -c 'test "$MISE_STUB_ACTIVE" = 1; test "$MISE_LOCKED" = 1; printf frozen-ok' +) | grep -q frozen-ok +grep -qx "MISE_LOCKED=1" "$log" +grep -qx "MISE_OVERRIDE_CONFIG_FILENAMES=mise.toml" "$log" +grep -qx "MISE_OVERRIDE_TOOL_VERSIONS_FILENAMES=none" "$log" +grep -qx "MISE_CONFIG_DIR=$project/.zed/dev/mise/config" "$log" +grep -qx "MISE_SYSTEM_CONFIG_DIR=$project/.zed/dev/mise/system-config" "$log" +grep -qx "MISE_CEILING_PATHS=$work_root" "$log" +grep -qx "ZED_DEV_MISE_ACTIVE=1" "$log" +pass 'frozen lock enforcement, isolation, and config precedence' + +: > "$log" +( + cd "$project" + run_capture 1 "$work_root/frozen-ambient.out" "$work_root/frozen-ambient.err" \ + env "${base_env[@]}" "MISE_STUB_LOG=$log" __MISE_DIFF=ambient "$zed" dev \ + --no-install --frozen --nix never --mise required --python-venv never --shell "$shell_bin" -c true +) +grep -q 'cannot verify an ambient mise activation' "$work_root/frozen-ambient.err" +[[ ! -s "$log" ]] || fail 'frozen ambient-mise failure unexpectedly re-entered mise' +pass 'frozen ambient mise fails closed before re-entry' + +project="$work_root/dot-config" +write_project "$project" +printf '%s\n' '[tools]' 'python = "3.12.4"' > "$project/.mise.toml" +printf '%s\n' '{}' > "$project/mise.lock" +log="$work_root/logs/dot-config.log" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --frozen --nix never --mise required --python-venv never --shell "$shell_bin" -c true +) +grep -qx 'MISE_OVERRIDE_CONFIG_FILENAMES=.mise.toml' "$log" +pass 'adjacent lock support for .mise.toml' + +outer="$work_root/parent-boundary" +project="$outer/checkout" +mkdir -p "$outer" +printf '%s\n' '[tools]' 'node = "22.4.0"' > "$outer/mise.toml" +write_project "$project" +log="$work_root/logs/parent-boundary.log" +: > "$log" +( + cd "$project" + run_capture 1 "$work_root/parent-boundary.out" "$work_root/parent-boundary.err" \ + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --nix never --mise required --python-venv never --shell "$shell_bin" -c true +) +grep -q 'no project-local `mise.toml` or `.mise.toml`' "$work_root/parent-boundary.err" +[[ ! -s "$log" ]] || fail 'parent configuration escaped the owning checkout' +pass 'owning-checkout configuration boundary' + +project="$work_root/zed-recursion-guard" +write_project "$project" +printf '%s\n' '[tools]' 'node = "22.4.0"' > "$project/mise.toml" +log="$work_root/logs/zed-recursion-guard.log" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" ZED_DEV_MISE_ACTIVE=1 "$zed" dev \ + --no-install --nix never --mise required --python-venv never --shell "$shell_bin" \ + -c 'test -z "${MISE_STUB_ACTIVE+x}"; printf zed-guard-ok' +) | grep -q zed-guard-ok +[[ ! -s "$log" ]] || fail 'ZED recursion guard still invoked mise' +pass 'ZED re-entry recursion guard' + +project="$work_root/mise-recursion-guard" +write_project "$project" +printf '%s\n' '[tools]' 'node = "22.4.0"' > "$project/mise.toml" +log="$work_root/logs/mise-recursion-guard.log" +: > "$log" +( + cd "$project" + env "${base_env[@]}" "MISE_STUB_LOG=$log" __MISE_DIFF=test-active "$zed" dev \ + --no-install --nix never --mise required --python-venv never --shell "$shell_bin" \ + -c 'test -z "${MISE_STUB_ACTIVE+x}"; printf mise-guard-ok' +) | grep -q mise-guard-ok +[[ ! -s "$log" ]] || fail 'mise-native recursion guard still invoked mise' +pass 'mise-native recursion guard in non-frozen mode' + +project="$work_root/exit-status" +write_project "$project" +printf '%s\n' '[tools]' 'node = "22.4.0"' > "$project/mise.toml" +log="$work_root/logs/exit-status.log" +: > "$log" +( + cd "$project" + run_capture 37 "$work_root/exit-status.out" "$work_root/exit-status.err" \ + env "${base_env[@]}" "MISE_STUB_LOG=$log" "$zed" dev \ + --no-install --nix never --mise required --python-venv never --shell "$shell_bin" -c 'exit 37' +) +[[ $(grep -c '^argv=' "$log") -eq 1 ]] || fail 'exit-status case did not enter mise exactly once' +pass 'exact child exit status' + +printf '{"schema":"zed-pkg.mise-runtime-certification.v1","passed":%d,"platform":"%s"}\n' \ + "$passed" "$(uname -s)" diff --git a/scripts/mise_tier1_certify.py b/scripts/mise_tier1_certify.py new file mode 100644 index 00000000..a0491694 --- /dev/null +++ b/scripts/mise_tier1_certify.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Certify zed-cli's tier-1 mise project adapter. + +The suite intentionally treats imported hooks, plugins, tasks, and templates as +untrusted data. Adapter operations must preserve those fields without running +project code or fetching plugin sources. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import subprocess +import sys +import tempfile +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterable, Sequence + + +@dataclass +class CaseResult: + case: str + status: str + duration_ms: int + detail: str = "" + + +class CertificationFailure(RuntimeError): + pass + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run( + command: Sequence[str | os.PathLike[str]], + *, + cwd: Path, + expect_success: bool = True, +) -> subprocess.CompletedProcess[str]: + rendered = [os.fspath(part) for part in command] + result = subprocess.run( + rendered, + cwd=cwd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={**os.environ, "NO_COLOR": "1", "CLICOLOR": "0"}, + ) + succeeded = result.returncode == 0 + if succeeded != expect_success: + expectation = "success" if expect_success else "failure" + raise CertificationFailure( + f"expected {expectation} from {rendered!r}, got exit " + f"{result.returncode}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + +def copy_fixture(fixtures: Path, relative: str) -> tuple[tempfile.TemporaryDirectory[str], Path]: + temporary = tempfile.TemporaryDirectory(prefix="zed-mise-tier1-") + root = Path(temporary.name) / "project" + shutil.copytree(fixtures / relative, root) + return temporary, root + + +def assert_no_adapter_execution(root: Path) -> None: + forbidden = [ + "ADAPTER_EXECUTED_HOOK", + "ADAPTER_EXECUTED_TASK", + "ADAPTER_EXECUTED_PLUGIN", + ] + found = [name for name in forbidden if (root / name).exists()] + if found: + raise CertificationFailure( + "mise adapter executed imported project code: " + ", ".join(found) + ) + + +def assert_adapter_outputs(root: Path) -> None: + expected = [root / "zed-dev.toml", root / ".zed" / "mise.lock"] + missing = [path.relative_to(root).as_posix() for path in expected if not path.is_file()] + if missing: + raise CertificationFailure("missing adapter output(s): " + ", ".join(missing)) + + +def import_verify_export(zed: Path, root: Path) -> None: + source = root / ".mise.toml" + if not source.is_file(): + source = root / "mise.toml" + before = sha256(source) + + run([zed, "mise", "import"], cwd=root) + assert_adapter_outputs(root) + assert_no_adapter_execution(root) + run([zed, "mise", "verify"], cwd=root) + assert_no_adapter_execution(root) + + # The adapter owns the exact output convention. Calling export without a + # destination exercises its documented project default while the source + # hash check proves losslessness regardless of whether it rewrites or + # validates the existing file. + run([zed, "mise", "export"], cwd=root) + assert_no_adapter_execution(root) + after = sha256(source) + if before != after: + raise CertificationFailure( + f"lossless export changed {source.name}: {before} -> {after}" + ) + run([zed, "mise", "verify"], cwd=root) + + +def supported_basic(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "supported/basic") + try: + import_verify_export(zed, root) + plan = (root / "zed-dev.toml").read_text(encoding="utf-8") + for marker in ["node", "22.4.0", "python", "3.12.4", "mise.vars"]: + if marker not in plan: + raise CertificationFailure(f"normalized plan omitted {marker!r}") + finally: + temporary.cleanup() + + +def supported_multi_version(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "supported/multi-version") + try: + import_verify_export(zed, root) + plan = (root / "zed-dev.toml").read_text(encoding="utf-8") + first = plan.find("22.4.0") + second = plan.find("20.15.1") + if first < 0 or second < 0 or first >= second: + raise CertificationFailure( + "normalized multi-version state did not preserve declared order" + ) + finally: + temporary.cleanup() + + +def supported_typed_state(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "supported/typed-state") + try: + import_verify_export(zed, root) + plan = (root / "zed-dev.toml").read_text(encoding="utf-8") + for marker in [ + "STRING_VALUE", + "INTEGER_VALUE", + "FLOAT_VALUE", + "BOOLEAN_VALUE", + "ARRAY_VALUE", + "TABLE_VALUE", + "mise.vars", + "mise.tasks", + ]: + if marker not in plan: + raise CertificationFailure(f"typed adapter state omitted {marker!r}") + assert_no_adapter_execution(root) + finally: + temporary.cleanup() + + +def preserved_inert_sections(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "preserved/inert-hooks-plugins") + try: + source_before = (root / ".mise.toml").read_bytes() + import_verify_export(zed, root) + source_after = (root / ".mise.toml").read_bytes() + if source_before != source_after: + raise CertificationFailure("unknown mise sections did not round-trip exactly") + plan = (root / "zed-dev.toml").read_text(encoding="utf-8") + for marker in ["mise.unknown_top_level", "postinstall", "plugins"]: + if marker not in plan: + raise CertificationFailure( + f"preserved-but-inert section omitted marker {marker!r}" + ) + assert_no_adapter_execution(root) + finally: + temporary.cleanup() + + +def rejects_conflicting_config(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "negative/conflicting-config") + try: + result = run([zed, "mise", "import"], cwd=root, expect_success=False) + diagnostic = (result.stdout + "\n" + result.stderr).lower() + if ".mise.toml" not in diagnostic or "mise.toml" not in diagnostic: + raise CertificationFailure( + "conflicting-config diagnostic did not identify both project files" + ) + if (root / "zed-dev.toml").exists() or (root / ".zed" / "mise.lock").exists(): + raise CertificationFailure("conflicting configs produced partial adapter state") + finally: + temporary.cleanup() + + +def rejects_tool_versions_only(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "negative/tool-versions-only") + try: + run([zed, "mise", "import"], cwd=root, expect_success=False) + if (root / "zed-dev.toml").exists() or (root / ".zed" / "mise.lock").exists(): + raise CertificationFailure( + "unsupported .tool-versions input silently produced an empty plan" + ) + finally: + temporary.cleanup() + + +def rejects_source_tamper(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "supported/basic") + try: + run([zed, "mise", "import"], cwd=root) + source = root / ".mise.toml" + source.write_text( + source.read_text(encoding="utf-8") + "\n# certification tamper\n", + encoding="utf-8", + ) + run([zed, "mise", "verify"], cwd=root, expect_success=False) + finally: + temporary.cleanup() + + +def rejects_plan_tamper(zed: Path, fixtures: Path) -> None: + temporary, root = copy_fixture(fixtures, "supported/basic") + try: + run([zed, "mise", "import"], cwd=root) + plan = root / "zed-dev.toml" + plan.write_text( + plan.read_text(encoding="utf-8") + + "\n[certification_tamper]\nenabled = true\n", + encoding="utf-8", + ) + run([zed, "mise", "verify"], cwd=root, expect_success=False) + finally: + temporary.cleanup() + + +def certification_cases() -> Iterable[tuple[str, object]]: + return [ + ("project-config-basic", supported_basic), + ("multi-version-order", supported_multi_version), + ("typed-env-vars-tasks", supported_typed_state), + ("unknown-hooks-plugins", preserved_inert_sections), + ("conflicting-config-files", rejects_conflicting_config), + ("tool-versions-only", rejects_tool_versions_only), + ("source-tamper", rejects_source_tamper), + ("plan-tamper", rejects_plan_tamper), + ] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--fixtures", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + zed = args.zed.resolve() + fixtures = args.fixtures.resolve() + report_path = args.report.resolve() + + if not zed.is_file(): + raise SystemExit(f"zed binary does not exist: {zed}") + if not fixtures.is_dir(): + raise SystemExit(f"fixture directory does not exist: {fixtures}") + + version = run([zed, "--version"], cwd=fixtures).stdout.strip() + results: list[CaseResult] = [] + failed = False + + for name, case in certification_cases(): + started = time.monotonic() + try: + case(zed, fixtures) # type: ignore[misc,operator] + results.append( + CaseResult( + case=name, + status="passed", + duration_ms=round((time.monotonic() - started) * 1000), + ) + ) + print(f"PASS {name}") + except Exception as error: # keep the complete report after one failure + failed = True + results.append( + CaseResult( + case=name, + status="failed", + duration_ms=round((time.monotonic() - started) * 1000), + detail=str(error), + ) + ) + print(f"FAIL {name}: {error}", file=sys.stderr) + + report = { + "schema_version": 1, + "compatibility_tier": 1, + "suite": "mise-lossless-project-adapter", + "status": "failed" if failed else "passed", + "zed_version": version, + "platform": { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + "results": [asdict(result) for result in results], + } + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(f"wrote {report_path}") + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/native_lifecycle.py b/scripts/native_lifecycle.py new file mode 100644 index 00000000..356a202b --- /dev/null +++ b/scripts/native_lifecycle.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""Hermetic acceptance test for native dependencies and install hooks. + +The test publishes two local Zed packages into a file registry, installs their +transitive graph with a fake apt-get binary, and verifies permission ordering, +argv safety, graph-wide de-duplication, writable staging, lifecycle ordering, +cache reuse, immutable source-store behavior, and Nix derivation purity. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import stat +import subprocess +from pathlib import Path +from typing import Mapping, Sequence + +ORG = "zed-pkg-test" + + +def shell_run( + command: Sequence[str | Path], + *, + cwd: Path, + env: Mapping[str, str], + should_fail: bool = False, +) -> str: + argv = [str(value) for value in command] + completed = subprocess.run( + argv, + cwd=cwd, + env=dict(env), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(f"\n$ (cd {cwd} && {' '.join(argv)})", flush=True) + print(completed.stdout, end="", flush=True) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {argv}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {argv}" + ) + return completed.stdout + + +def write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def write_executable(path: Path, content: str) -> None: + write(path, content) + path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def package_manifest(name: str, extra: str = "") -> str: + return f'''[package] +org = "{ORG}" +name = "{name}" +version = "1.0.0" +description = "Native lifecycle acceptance fixture" +license = "MIT" + +[package.repository] +url = "https://github.com/{ORG}/zed-pkg-e2e" + +{extra.strip()} +''' + + +def consumer_manifest(name: str = "native-lifecycle-consumer") -> str: + return f'''[package] +org = "{ORG}" +name = "{name}" +version = "0.0.0" + +[package.repository] +url = "https://github.com/{ORG}/zed-pkg-e2e" + +[dependencies] +"{ORG}/native-a" = "^1" +''' + + +def assert_absent(path: Path, message: str) -> None: + if path.exists() or path.is_symlink(): + raise AssertionError(f"{message}: {path}") + + +def assert_contains(text: str, expected: str) -> None: + if expected not in text: + raise AssertionError(f"expected {expected!r} in output:\n{text}") + + +def copy_consumer(source: Path, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=True) + shutil.copy2(source / ".zpkg.toml", destination / ".zpkg.toml") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + + zed = args.zed.resolve() + root = args.work_root.resolve() + if root.exists(): + shutil.rmtree(root) + root.mkdir(parents=True) + registry = root / "registry" + zed_home = root / "zed-home" + registry.mkdir() + zed_home.mkdir() + + base_env = os.environ.copy() + base_env.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "ZED_PKG_REGISTRY": f"file://{registry}", + "ZED_PKG_HOME": str(zed_home), + "ZED_PKG_TOKEN": "", + } + ) + + package_b = root / "packages" / "native-b" + write(package_b / "src" / "lib.txt", "native b\n") + write( + package_b / ".zpkg.toml", + package_manifest( + "native-b", + '''[native-dependencies] +apt = ["libssl-dev", "zlib1g-dev"] +nix = ["openssl", "zlib"] +''', + ), + ) + + package_a = root / "packages" / "native-a" + write(package_a / "src" / "lib.txt", "native a\n") + write( + package_a / ".zpkg.toml", + package_manifest( + "native-a", + f'''[dependencies] +"{ORG}/native-b" = "^1" + +[native-dependencies] +apt = ["pkg-config", "libssl-dev"] +nix = ["pkg-config", "openssl"] + +[hooks] +pre-install = ['printf "pre\\n" >> "$HOOK_CAPTURE"; printf "pre\\n" > lifecycle.txt; printf "%s\\n" "$ZED_INSTALL_PHASE" > pre-phase.txt; printf "%s\\n" "$ZED_NATIVE_MANAGER" > native-manager.txt; printf "%s\\n" "$ZED_NATIVE_PACKAGES" > native-packages.json'] +post-install = ['printf "post\\n" >> "$HOOK_CAPTURE"; printf "post\\n" >> lifecycle.txt'] + +[build] +command = 'printf "build\\n" >> "$HOOK_CAPTURE"; printf "build\\n" >> lifecycle.txt' +''', + ), + ) + + for package in (package_b, package_a): + shell_run( + [zed, "publish", "--skip-vcs-checks"], + cwd=package, + env=base_env, + ) + + consumer = root / "consumer" + consumer.mkdir() + write(consumer / ".zpkg.toml", consumer_manifest()) + + fake_bin = root / "fake-bin" + native_capture = root / "native.capture" + hook_capture = root / "hooks.capture" + write_executable( + fake_bin / "apt-get", + '''#!/bin/sh +set -eu +{ + printf 'call\\n' + for arg do + printf 'arg=%s\\n' "$arg" + done +} >> "$NATIVE_CAPTURE" +''', + ) + install_env = base_env.copy() + install_env.update( + { + "PATH": f"{fake_bin}{os.pathsep}{base_env.get('PATH', '')}", + "NATIVE_CAPTURE": str(native_capture), + "HOOK_CAPTURE": str(hook_capture), + } + ) + + common = [ + zed, + "install", + "--install-mode", + "copy", + "--adapter", + "none", + "--native-manager", + "apt", + ] + + denied = shell_run(common, cwd=consumer, env=install_env, should_fail=True) + assert_contains(denied, "--allow-native-deps") + assert_absent(native_capture, "native manager ran before native consent") + assert_absent(hook_capture, "hook ran before native consent") + assert_absent(consumer / "zed_modules", "project materialized after denial") + + hooks_denied = shell_run( + [*common, "--allow-native-deps"], + cwd=consumer, + env=install_env, + should_fail=True, + ) + assert_contains(hooks_denied, "--allow-install-hooks") + assert_absent(native_capture, "native manager ran before hook consent") + + build_denied = shell_run( + [*common, "--allow-native-deps", "--allow-install-hooks"], + cwd=consumer, + env=install_env, + should_fail=True, + ) + assert_contains(build_denied, "--allow-build") + assert_absent(native_capture, "native manager ran before build consent") + + allowed = [ + *common, + "--allow-native-deps", + "--allow-install-hooks", + "--allow-build", + ] + shell_run(allowed, cwd=consumer, env=install_env) + + native_lines = native_capture.read_text(encoding="utf-8").splitlines() + if native_lines.count("call") != 1: + raise AssertionError(f"expected one apt invocation, got: {native_lines}") + if "arg=--" not in native_lines: + raise AssertionError(f"apt option terminator missing: {native_lines}") + for package in ("pkg-config", "libssl-dev", "zlib1g-dev"): + if native_lines.count(f"arg={package}") != 1: + raise AssertionError( + f"native package {package!r} was not de-duplicated: {native_lines}" + ) + + installed = consumer / "zed_modules" / ORG / "native-a" + if (installed / "lifecycle.txt").read_text(encoding="utf-8") != "pre\nbuild\npost\n": + raise AssertionError("lifecycle hooks/build ran out of order") + if (installed / "pre-phase.txt").read_text(encoding="utf-8") != "pre-install\n": + raise AssertionError("pre-install phase metadata is incorrect") + if (installed / "native-manager.txt").read_text(encoding="utf-8") != "apt\n": + raise AssertionError("hook did not receive selected native manager") + packages = json.loads( + (installed / "native-packages.json").read_text(encoding="utf-8") + ) + if packages != ["pkg-config", "libssl-dev"]: + raise AssertionError(f"hook received wrong per-package native set: {packages}") + if hook_capture.read_text(encoding="utf-8") != "pre\nbuild\npost\n": + raise AssertionError("external lifecycle capture is incorrect") + + source_lifecycle = list((zed_home / "store").rglob("lifecycle.txt")) + if source_lifecycle: + raise AssertionError(f"immutable source store was mutated: {source_lifecycle}") + cached_lifecycle = list((zed_home / "builds").rglob("lifecycle.txt")) + if not cached_lifecycle: + raise AssertionError("finalized lifecycle artifact was not promoted to cache") + + shell_run(allowed, cwd=consumer, env=install_env) + if hook_capture.read_text(encoding="utf-8") != "pre\nbuild\npost\n": + raise AssertionError("cache hit re-ran package-controlled lifecycle code") + native_lines = native_capture.read_text(encoding="utf-8").splitlines() + if native_lines.count("call") != 2: + raise AssertionError("native prerequisites were not checked once per transaction") + + # A derivation may acknowledge inputs supplied by Nix, but Zed must never + # execute a host package manager in the derivation sandbox. + nix_consumer = root / "nix-consumer" + copy_consumer(consumer, nix_consumer) + nix_hook_capture = root / "nix-hooks.capture" + nix_env = base_env.copy() + nix_env.update( + { + "NIX_BUILD_TOP": str(root / "nix-build-top"), + "HOOK_CAPTURE": str(nix_hook_capture), + "NATIVE_CAPTURE": str(root / "must-not-exist.capture"), + } + ) + nix_common = [ + zed, + "install", + "--install-mode", + "copy", + "--adapter", + "none", + "--allow-native-deps", + "--allow-install-hooks", + "--allow-build", + "--native-manager", + "nix", + ] + nix_denied = shell_run( + nix_common, + cwd=nix_consumer, + env=nix_env, + should_fail=True, + ) + assert_contains(nix_denied, "nativeBuildInputs/buildInputs") + assert_absent(root / "must-not-exist.capture", "manager ran in Nix build") + + nix_env["ZED_PKG_NATIVE_DEPS_PROVIDED"] = "1" + shell_run(nix_common, cwd=nix_consumer, env=nix_env) + assert_absent(root / "must-not-exist.capture", "manager ran in acknowledged Nix build") + if nix_hook_capture.read_text(encoding="utf-8") != "pre\nbuild\npost\n": + raise AssertionError("acknowledged Nix lifecycle did not complete") + + print("native dependency and install-hook acceptance contract passed", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/nix_bundle_write_canary.py b/scripts/nix_bundle_write_canary.py new file mode 100755 index 00000000..59f3bdad --- /dev/null +++ b/scripts/nix_bundle_write_canary.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Independent black-box acceptance for `zed interop nix bundle write`. + +The harness owns a fresh project, approved immutable flake lock, output roots, +and Zed home. It invokes only the compiled CLI and Nix executables; it imports +no implementation module from zed-cli. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Mapping, Sequence + +SCHEMA = "zed.nix-flake-bundle-write/v1" +NIXPKGS_REV = "e73de5be04e0eff4190a1432b946d469c794e7b4" +NIXPKGS_NAR_HASH = "sha256-pGvFkM8N0xEkIIXDe5YYfbEAvHrk4IxBrjB/x8OomhE=" + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def run( + argv: Sequence[str | Path], + *, + cwd: Path, + env: Mapping[str, str] | None = None, + expect_success: bool = True, + log: Path | None = None, +) -> subprocess.CompletedProcess[str]: + command = [str(value) for value in argv] + completed = subprocess.run( + command, + cwd=cwd, + env=dict(env) if env is not None else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + transcript = ( + f"$ (cd {cwd} && {' '.join(command)})\n" + f"exit={completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}\n" + ) + print(transcript, flush=True) + if log is not None: + log.parent.mkdir(parents=True, exist_ok=True) + with log.open("a", encoding="utf-8") as handle: + handle.write(transcript) + if expect_success and completed.returncode != 0: + raise AssertionError(f"command failed: {' '.join(command)}") + if not expect_success and completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {' '.join(command)}") + return completed + + +def snapshot(root: Path) -> dict[str, str]: + result: dict[str, str] = {} + if not root.exists(): + return result + for path in sorted(root.rglob("*")): + if path.is_symlink(): + result[path.relative_to(root).as_posix()] = f"symlink:{os.readlink(path)}" + elif path.is_file(): + result[path.relative_to(root).as_posix()] = sha256(path.read_bytes()) + return result + + +def flake_lock() -> bytes: + return json.dumps( + { + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1782467914, + "narHash": NIXPKGS_NAR_HASH, + "owner": "NixOS", + "repo": "nixpkgs", + "rev": NIXPKGS_REV, + "type": "github", + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": NIXPKGS_REV, + "type": "github", + }, + }, + "root": {"inputs": {"nixpkgs": "nixpkgs"}}, + }, + "root": "root", + "version": 7, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + + +def manifest(system: str) -> str: + return f'''[package] +org = "acme" +name = "dataset" +version = "1.2.3" +description = "immutable data fixture" +license = "MIT" + +[package.repository] +url = "https://github.com/acme/dataset" + +[publish.nix] +attribute = "dataset" +systems = ["{system}"] +outputs = ["out"] +''' + + +def create_project(project: Path, system: str) -> None: + (project / "src" / "deep").mkdir(parents=True) + (project / "data").mkdir() + (project / ".zpkg.toml").write_text(manifest(system), encoding="utf-8") + (project / ".zpkg.lock").write_text("version = 1\n", encoding="utf-8") + (project / "data" / "value.txt").write_bytes(b"same immutable payload\n") + + +def clean_environment() -> dict[str, str]: + environment = os.environ.copy() + for key in list(environment): + if key.startswith("ZED_PKG_"): + environment.pop(key, None) + environment["NIX_CONFIG"] = ( + "experimental-features = nix-command flakes\n" + "accept-flake-config = false\n" + ) + return environment + + +def bundle_command( + zed: Path, + zed_home: Path, + arguments: Sequence[str | Path], +) -> list[str | Path]: + return [ + zed, + "--registry", + "https://registry.invalid.example", + "--home", + zed_home, + "interop", + "nix", + "bundle", + "write", + *arguments, + ] + + +def assert_bundle(bundle: Path, project: Path, zed_home: Path) -> None: + expected = { + "flake.nix", + "flake.lock", + "package.nix", + "README.md", + "metadata/plan.json", + "metadata/bundle.json", + "artifacts/acme-dataset-1.2.3.tar.gz", + } + actual = { + path.relative_to(bundle).as_posix() + for path in bundle.rglob("*") + if path.is_file() + } + if actual != expected: + raise AssertionError(f"unexpected bundle inventory: {sorted(actual)}") + + textual = "\n".join( + (bundle / relative).read_text(encoding="utf-8") + for relative in sorted(expected) + if not relative.endswith(".tar.gz") + ) + for forbidden in ( + str(project), + str(zed_home), + "registry.invalid.example", + ): + if forbidden in textual: + raise AssertionError(f"bundle leaked forbidden value: {forbidden}") + + inventory = json.loads((bundle / "metadata" / "bundle.json").read_text()) + if inventory["schema"] != "zed.nix-flake-bundle/v1": + raise AssertionError(inventory) + if inventory["nixpkgs"]["rev"] != NIXPKGS_REV: + raise AssertionError(inventory) + if inventory["nixpkgs"]["nar_hash"] != NIXPKGS_NAR_HASH: + raise AssertionError(inventory) + if len(inventory["bundle_sha256"]) != 64: + raise AssertionError(inventory) + + +def receipt(output: subprocess.CompletedProcess[str]) -> dict: + try: + parsed = json.loads(output.stdout.strip()) + except json.JSONDecodeError as error: + raise AssertionError(f"invalid receipt JSON: {output.stdout!r}") from error + if parsed.get("schema") != SCHEMA: + raise AssertionError(parsed) + return parsed + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--work-root", required=True, type=Path) + args = parser.parse_args() + + zed = args.zed.resolve() + root = args.work_root.resolve() + if not zed.is_file(): + raise FileNotFoundError(zed) + if root.exists(): + raise AssertionError(f"work root must be fresh: {root}") + root.mkdir(parents=True) + + log = root / "diagnostics" / "canary.log" + environment = clean_environment() + system_result = run( + ["nix", "eval", "--impure", "--raw", "--expr", "builtins.currentSystem"], + cwd=root, + env=environment, + log=log, + ) + system = system_result.stdout.strip() + if not system: + raise AssertionError("Nix returned an empty current system") + + project = root / "project" + approved_lock = root / "approved-flake.lock" + output_parent = root / "exports" + bundle = output_parent / "dataset-flake" + zed_home = root / "must-not-create-zed-home" + create_project(project, system) + approved_lock.write_bytes(flake_lock()) + output_parent.mkdir() + project_before = snapshot(project) + + first = run( + bundle_command( + zed, + zed_home, + [ + "--frozen", + "--flake-lock", + approved_lock, + "--out", + bundle, + "--json", + ], + ), + cwd=project / "src" / "deep", + env=environment, + log=log, + ) + first_receipt = receipt(first) + if first_receipt["outcome"] != "created": + raise AssertionError(first_receipt) + canonical_bundle = bundle.resolve() + if first_receipt["destination"] != str(canonical_bundle): + raise AssertionError(first_receipt) + if first_receipt["package"] != { + "org": "acme", + "name": "dataset", + "version": "1.2.3", + }: + raise AssertionError(first_receipt) + assert_bundle(bundle, project, zed_home) + if snapshot(project) != project_before: + raise AssertionError("bundle command mutated the source project") + if zed_home.exists(): + raise AssertionError("credential-independent command created Zed home") + bundle_before = snapshot(bundle) + + second = run( + bundle_command( + zed, + zed_home, + [ + "--frozen", + "--flake-lock", + approved_lock, + "--output", + bundle, + "--json", + ], + ), + cwd=project, + env=environment, + log=log, + ) + second_receipt = receipt(second) + if second_receipt["outcome"] != "already-current": + raise AssertionError(second_receipt) + if second_receipt["bundle_sha256"] != first_receipt["bundle_sha256"]: + raise AssertionError("idempotent invocation changed bundle identity") + if snapshot(bundle) != bundle_before: + raise AssertionError("idempotent invocation rewrote bundle bytes") + + run( + ["nix", "flake", "archive", "--no-update-lock-file"], + cwd=bundle, + env=environment, + log=log, + ) + online_build = run( + [ + "nix", + "build", + "--no-update-lock-file", + "--no-link", + "--print-out-paths", + ".#dataset", + ], + cwd=bundle, + env=environment, + log=log, + ) + run( + ["nix", "flake", "check", "--offline", "--no-update-lock-file"], + cwd=bundle, + env=environment, + log=log, + ) + offline_build = run( + [ + "nix", + "build", + "--offline", + "--no-update-lock-file", + "--no-link", + "--print-out-paths", + ".#dataset", + ], + cwd=bundle, + env=environment, + log=log, + ) + if online_build.stdout.strip() != offline_build.stdout.strip(): + raise AssertionError("offline replay selected a different store output") + if snapshot(bundle) != bundle_before: + raise AssertionError("Nix verification rewrote the standalone bundle") + + env_bundle = output_parent / "environment-flake" + env_only = environment.copy() + env_only.update( + { + "ZED_PKG_FROZEN": "yes", + "ZED_PKG_NIX_PLAN_JSON": "on", + "ZED_PKG_NIX_FLAKE_LOCK": str(approved_lock), + "ZED_PKG_NIX_BUNDLE_OUT": str(env_bundle), + } + ) + env_result = run( + bundle_command(zed, zed_home, []), + cwd=project, + env=env_only, + log=log, + ) + if receipt(env_result)["outcome"] != "created": + raise AssertionError(env_result.stdout) + assert_bundle(env_bundle, project, zed_home) + + unknown = run( + bundle_command( + zed, + zed_home, + [ + "--frozen", + "--flake-lock", + approved_lock, + "--out", + output_parent / "unknown-flake", + "--definitely-unknown", + ], + ), + cwd=project, + env=environment, + expect_success=False, + log=log, + ) + if "unknown Nix interop option" not in unknown.stderr: + raise AssertionError(unknown.stderr) + if (output_parent / "unknown-flake").exists(): + raise AssertionError("unknown-option invocation wrote output") + + if os.name != "nt": + real_parent = root / "real-exports" + linked_parent = root / "linked-exports" + real_parent.mkdir() + linked_parent.symlink_to(real_parent, target_is_directory=True) + linked = run( + bundle_command( + zed, + zed_home, + [ + "--frozen", + "--flake-lock", + approved_lock, + "--out", + linked_parent / "bundle", + ], + ), + cwd=project, + env=environment, + expect_success=False, + log=log, + ) + if "is not a real directory" not in linked.stderr: + raise AssertionError(linked.stderr) + if (real_parent / "bundle").exists(): + raise AssertionError("symlink-parent invocation escaped output boundary") + + (bundle / "README.md").write_bytes(b"tampered\n") + tampered = run( + bundle_command( + zed, + zed_home, + [ + "--frozen", + "--flake-lock", + approved_lock, + "--out", + bundle, + ], + ), + cwd=project, + env=environment, + expect_success=False, + log=log, + ) + if "differs from rendered bytes" not in tampered.stderr: + raise AssertionError(tampered.stderr) + if (bundle / "README.md").read_bytes() != b"tampered\n": + raise AssertionError("tamper rejection overwrote caller-owned bytes") + + print( + json.dumps( + { + "schema": "zed.nix-bundle-write-canary/v1", + "system": system, + "cli": str(zed), + "bundle_sha256": first_receipt["bundle_sha256"], + "store_output": offline_build.stdout.strip(), + "result": "pass", + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/nix_bundle_write_fixture_canary.py b/scripts/nix_bundle_write_fixture_canary.py new file mode 100644 index 00000000..56ee617c --- /dev/null +++ b/scripts/nix_bundle_write_fixture_canary.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +"""Certify Nix bundle writing against an immutable real package fixture.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Mapping, Sequence + +NIXPKGS_REV = "e73de5be04e0eff4190a1432b946d469c794e7b4" +NIXPKGS_NAR_HASH = "sha256-pGvFkM8N0xEkIIXDe5YYfbEAvHrk4IxBrjB/x8OomhE=" + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def run( + argv: Sequence[str | Path], + *, + cwd: Path, + env: Mapping[str, str], + expect_success: bool = True, +) -> subprocess.CompletedProcess[str]: + command = [str(value) for value in argv] + completed = subprocess.run( + command, + cwd=cwd, + env=dict(env), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + print( + f"$ (cd {cwd} && {' '.join(command)})\n" + f"exit={completed.returncode}\n" + f"stdout:\n{completed.stdout}\n" + f"stderr:\n{completed.stderr}\n", + flush=True, + ) + if expect_success and completed.returncode != 0: + raise AssertionError(f"command failed: {' '.join(command)}") + if not expect_success and completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {' '.join(command)}") + return completed + + +def clean_environment(secret: str, home: Path) -> dict[str, str]: + environment = os.environ.copy() + for key in list(environment): + if key.startswith("ZED_PKG_"): + environment.pop(key, None) + environment.update( + { + "NIX_CONFIG": ( + "experimental-features = nix-command flakes\n" + "accept-flake-config = false\n" + ), + "ZED_PKG_TOKEN": secret, + "ZED_PKG_SUPABASE_KEY": secret, + "ZED_PKG_AUTH_PASSWORD": secret, + "ZED_PKG_HOME": str(home), + "ZED_PKG_REGISTRY": "https://person:secret@example.invalid/private", + } + ) + return environment + + +def git_clean(fixture: Path) -> None: + completed = subprocess.run( + ["git", "status", "--porcelain=v1", "--untracked-files=all"], + cwd=fixture, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + if completed.stdout.strip(): + raise AssertionError(f"fixture checkout is dirty:\n{completed.stdout}") + + +def copy_fixture(fixture: Path, destination: Path, system: str) -> None: + shutil.copytree( + fixture, + destination, + ignore=shutil.ignore_patterns( + ".git", + ".zed", + ".zpkg-staging", + "node_modules", + "zed_modules", + ), + ) + manifest = destination / ".zpkg.toml" + existing = manifest.read_text(encoding="utf-8") + manifest.write_text( + existing + + "\n[publish.nix]\n" + + 'attribute = "node_lib"\n' + + f'systems = ["{system}"]\n' + + 'outputs = ["out"]\n', + encoding="utf-8", + ) + (destination / ".zpkg.lock").write_text("version = 1\n", encoding="utf-8") + (destination / "nested" / "deep").mkdir(parents=True) + + +def flake_lock() -> bytes: + return json.dumps( + { + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1782467914, + "narHash": NIXPKGS_NAR_HASH, + "owner": "NixOS", + "repo": "nixpkgs", + "rev": NIXPKGS_REV, + "type": "github", + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": NIXPKGS_REV, + "type": "github", + }, + }, + "root": {"inputs": {"nixpkgs": "nixpkgs"}}, + }, + "root": "root", + "version": 7, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + + +def snapshot(root: Path) -> dict[str, tuple[str, int]]: + result: dict[str, tuple[str, int]] = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise AssertionError(f"unexpected symlink in snapshot: {relative}") + if path.is_file(): + result[relative] = (sha256(path.read_bytes()), path.stat().st_mode & 0o777) + return result + + +def write_bundle( + zed: Path, + project: Path, + approved_lock: Path, + output: Path, + environment: Mapping[str, str], +) -> dict: + completed = run( + [ + zed, + "interop", + "nix", + "bundle", + "write", + "--frozen", + "--flake-lock", + approved_lock, + "--out", + output, + "--json", + ], + cwd=project / "nested" / "deep", + env=environment, + ) + try: + receipt = json.loads(completed.stdout.strip()) + except json.JSONDecodeError as error: + raise AssertionError(f"invalid bundle receipt: {completed.stdout!r}") from error + if receipt.get("schema") != "zed.nix-flake-bundle-write/v1": + raise AssertionError(receipt) + if receipt.get("outcome") != "created": + raise AssertionError(receipt) + return receipt + + +def validate_bundle( + bundle: Path, + project: Path, + secret: str, + unused_home: Path, +) -> dict: + expected = { + "README.md", + "flake.lock", + "flake.nix", + "package.nix", + "metadata/bundle.json", + "metadata/plan.json", + "artifacts/zed-pkg-test-node-lib-1.0.0.tar.gz", + } + actual = set(snapshot(bundle)) + if actual != expected: + raise AssertionError(f"unexpected bundle files: {sorted(actual)}") + + plan_raw = (bundle / "metadata" / "plan.json").read_text(encoding="utf-8") + inventory_raw = (bundle / "metadata" / "bundle.json").read_text( + encoding="utf-8" + ) + textual = "\n".join( + path.read_text(encoding="utf-8") + for path in sorted(bundle.rglob("*")) + if path.is_file() and path.suffix not in {".gz"} + ) + for forbidden in ( + str(project), + str(project.parent), + str(unused_home), + secret, + "person:secret", + "example.invalid", + ): + if forbidden in textual: + raise AssertionError(f"bundle leaked forbidden value: {forbidden!r}") + + plan = json.loads(plan_raw) + if plan["schema"] != "zed.nix-export-plan/v1": + raise AssertionError(plan) + if plan["package"] != { + "org": "zed-pkg-test", + "name": "node-lib", + "version": "1.0.0", + }: + raise AssertionError(plan["package"]) + if plan["package_class"] != "data": + raise AssertionError(plan) + if plan["intent"]["attribute"] != "node_lib": + raise AssertionError(plan) + if plan["dependencies"] != [] or plan["bins"] != {}: + raise AssertionError(plan) + + artifact = bundle / "artifacts" / plan["source"]["file_name"] + artifact_bytes = artifact.read_bytes() + if sha256(artifact_bytes) != plan["source"]["artifact"]["sha256"]: + raise AssertionError("artifact digest differs from plan") + if len(artifact_bytes) != plan["source"]["artifact"]["size"]: + raise AssertionError("artifact size differs from plan") + + inventory = json.loads(inventory_raw) + if inventory["schema"] != "zed.nix-flake-bundle/v1": + raise AssertionError(inventory) + if inventory["nixpkgs"]["rev"] != NIXPKGS_REV: + raise AssertionError(inventory) + if inventory["nixpkgs"]["nar_hash"] != NIXPKGS_NAR_HASH: + raise AssertionError(inventory) + if len(inventory["bundle_sha256"]) != 64: + raise AssertionError(inventory) + + entries = inventory["entries"] + paths = [entry["path"] for entry in entries] + if paths != sorted(paths) or len(paths) != len(set(paths)): + raise AssertionError("bundle inventory paths are not sorted and unique") + for entry in entries: + path = bundle / entry["path"] + data = path.read_bytes() + if sha256(data) != entry["sha256"] or len(data) != entry["size"]: + raise AssertionError(f"inventory mismatch for {entry['path']}") + return inventory + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--fixture", required=True, type=Path) + parser.add_argument("--work-root", required=True, type=Path) + args = parser.parse_args() + + zed = args.zed.resolve() + fixture = args.fixture.resolve() + root = args.work_root.resolve() + if root.exists(): + raise AssertionError(f"work root must be fresh: {root}") + if not zed.is_file(): + raise FileNotFoundError(zed) + if not (fixture / ".zpkg.toml").is_file(): + raise FileNotFoundError(fixture / ".zpkg.toml") + git_clean(fixture) + root.mkdir(parents=True) + + nix_environment = os.environ.copy() + nix_environment["NIX_CONFIG"] = ( + "experimental-features = nix-command flakes\n" + "accept-flake-config = false\n" + ) + system = run( + ["nix", "eval", "--impure", "--raw", "--expr", "builtins.currentSystem"], + cwd=root, + env=nix_environment, + ).stdout.strip() + if not system: + raise AssertionError("Nix returned an empty current system") + + first_project = root / "different" / "absolute" / "first-project" + second_project = root / "other" / "location" / "second-project" + copy_fixture(fixture, first_project, system) + copy_fixture(fixture, second_project, system) + first_before = snapshot(first_project) + second_before = snapshot(second_project) + + approved_lock = root / "approved-flake.lock" + approved_lock.write_bytes(flake_lock()) + outputs = root / "exports" + outputs.mkdir() + first_bundle = outputs / "first" + second_bundle = outputs / "second" + unused_home = root / "zed-home-must-not-exist" + secret = "real-fixture-bundle-secret-must-not-appear" + environment = clean_environment(secret, unused_home) + + first_receipt = write_bundle( + zed, + first_project, + approved_lock, + first_bundle, + environment, + ) + second_receipt = write_bundle( + zed, + second_project, + approved_lock, + second_bundle, + environment, + ) + first_inventory = validate_bundle( + first_bundle, + first_project, + secret, + unused_home, + ) + second_inventory = validate_bundle( + second_bundle, + second_project, + secret, + unused_home, + ) + + if snapshot(first_bundle) != snapshot(second_bundle): + raise AssertionError("identical real fixtures produced different bundle bytes") + if first_receipt["bundle_sha256"] != second_receipt["bundle_sha256"]: + raise AssertionError("identical real fixtures produced different bundle identities") + if first_inventory["bundle_sha256"] != second_inventory["bundle_sha256"]: + raise AssertionError("bundle inventories differ across absolute project paths") + if snapshot(first_project) != first_before or snapshot(second_project) != second_before: + raise AssertionError("bundle writing mutated a real fixture copy") + if unused_home.exists(): + raise AssertionError("bundle writing created the configured Zed home") + git_clean(fixture) + + print( + json.dumps( + { + "schema": "zed.nix-bundle-write-fixture-canary/v1", + "package": "zed-pkg-test/node-lib@1.0.0", + "system": system, + "bundle_sha256": first_inventory["bundle_sha256"], + "result": "pass", + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/nix_export_plan_canary.py b/scripts/nix_export_plan_canary.py new file mode 100755 index 00000000..4693fefe --- /dev/null +++ b/scripts/nix_export_plan_canary.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Independent black-box canary for frozen Zed → Nix export planning.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Mapping, Sequence + + +def run( + command: Sequence[str | Path], + *, + cwd: Path, + env: Mapping[str, str] | None = None, + should_fail: bool = False, +) -> str: + argv = [str(value) for value in command] + environment = { + "CI": "true", + "PATH": os.environ.get("PATH", ""), + "ZED_PKG_INTERACTIVE": "false", + } + if env: + environment.update(env) + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + print(f"\n$ (cd {cwd} && {' '.join(argv)})", flush=True) + print(completed.stdout, end="", flush=True) + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {argv!r}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {argv!r}\n{completed.stdout}" + ) + return completed.stdout + + +def git_clean(root: Path) -> None: + status = run( + ["git", "status", "--porcelain=v1", "--untracked-files=all"], + cwd=root, + ) + if status.strip(): + raise AssertionError(f"fixture checkout was mutated:\n{status}") + + +def copy_fixture(source: Path, destination: Path) -> None: + shutil.copytree( + source, + destination, + ignore=shutil.ignore_patterns(".git", ".zed", "zed_modules", "node_modules"), + ) + + +def tree_digest(root: Path) -> tuple[tuple[str, str], ...]: + rows: list[tuple[str, str]] = [] + for path in sorted(root.rglob("*")): + if path.is_symlink(): + rows.append( + (path.relative_to(root).as_posix(), f"symlink:{os.readlink(path)}") + ) + elif path.is_file(): + rows.append( + ( + path.relative_to(root).as_posix(), + hashlib.sha256(path.read_bytes()).hexdigest(), + ) + ) + return tuple(rows) + + +def append_text(path: Path, value: str) -> None: + with path.open("a", encoding="utf-8") as handle: + handle.write(value) + + +def plan( + zed: Path, + project: Path, + *, + target: str | None = None, + env: Mapping[str, str] | None = None, + should_fail: bool = False, +) -> str: + command: list[str | Path] = [ + zed, + "interop", + "nix", + "plan", + "export", + "--frozen", + "--json", + ] + if target: + command.extend(["--target", target]) + return run(command, cwd=project, env=env, should_fail=should_fail) + + +def assert_hex_sha256(value: object, label: str) -> str: + if not isinstance(value, str) or len(value) != 64: + raise AssertionError(f"{label} is not a SHA-256 hex digest: {value!r}") + int(value, 16) + return value + + +def single_package_canary(fixture: Path, zed: Path, root: Path) -> None: + first = root / "single-a" / "deep" / "package" + second = root / "single-b" / "different" / "package" + copy_fixture(fixture, first) + copy_fixture(fixture, second) + + nix_intent = """ + +[publish.nix] +attribute = "node-lib" +systems = ["x86_64-linux", "aarch64-linux"] +outputs = ["out"] + +[bin] +node-lib-tool = "bin/node-lib-tool" +""" + for project in (first, second): + append_text(project / ".zpkg.toml", nix_intent) + (project / ".zpkg.lock").write_text("version = 1\n", encoding="utf-8") + tool = project / "bin" / "node-lib-tool" + tool.parent.mkdir(parents=True) + tool.write_text("#!/bin/sh\necho node-lib\n", encoding="utf-8") + tool.chmod(0o755) + + first_before = tree_digest(first) + second_before = tree_digest(second) + forbidden_secret = "planner-secret-must-never-appear" + unused_home = root / "planner-home-must-not-exist" + environment = { + "ZED_PKG_TOKEN": forbidden_secret, + "ZED_PKG_SUPABASE_KEY": forbidden_secret, + "ZED_PKG_REGISTRY": "https://person:secret@example.invalid/registry", + "ZED_PKG_HOME": str(unused_home), + } + first_raw = plan(zed, first, env=environment) + second_raw = plan(zed, second, env=environment) + if first_raw != second_raw: + raise AssertionError("identical packages at different absolute paths produced different plans") + if tree_digest(first) != first_before or tree_digest(second) != second_before: + raise AssertionError("read-only planning mutated a package copy") + if unused_home.exists(): + raise AssertionError("planning created the configured global Zed home") + for forbidden in (forbidden_secret, "person:secret", str(first), str(second), str(root)): + if forbidden in first_raw: + raise AssertionError(f"plan leaked forbidden value: {forbidden!r}") + + parsed = json.loads(first_raw) + if parsed.get("schema") != "zed.nix-export-plan/v1": + raise AssertionError(f"unexpected plan schema: {parsed!r}") + package = parsed["package"] + if (package["org"], package["name"], package["version"]) != ( + "zed-pkg-test", + "node-lib", + "1.0.0", + ): + raise AssertionError(f"wrong package identity: {package!r}") + if parsed["package_class"] != "prebuilt-bin": + raise AssertionError(f"expected prebuilt-bin plan: {parsed!r}") + if parsed["bins"] != {"node-lib-tool": "bin/node-lib-tool"}: + raise AssertionError(f"wrong bin inventory: {parsed['bins']!r}") + if parsed["intent"]["systems"] != ["aarch64-linux", "x86_64-linux"]: + raise AssertionError(f"systems were not canonicalized: {parsed['intent']!r}") + assert_hex_sha256(parsed["source"]["artifact"]["sha256"], "artifact") + assert_hex_sha256(parsed["source"]["manifest_sha256"], "manifest") + assert_hex_sha256(parsed["source"]["lock_sha256"], "lock") + + commented = root / "single-commented" + copy_fixture(first, commented) + original = (commented / ".zpkg.toml").read_text(encoding="utf-8") + (commented / ".zpkg.toml").write_text( + "# exact input comment\n" + original, + encoding="utf-8", + ) + commented_plan = json.loads(plan(zed, commented)) + if commented_plan["source"]["manifest_sha256"] == parsed["source"]["manifest_sha256"]: + raise AssertionError("exact manifest-byte change did not alter manifest identity") + if commented_plan["source"]["artifact"]["sha256"] == parsed["source"]["artifact"]["sha256"]: + raise AssertionError("manifest-byte change did not alter immutable artifact identity") + + missing_lock = root / "single-missing-lock" + copy_fixture(first, missing_lock) + (missing_lock / ".zpkg.lock").unlink() + failure = plan(zed, missing_lock, should_fail=True) + if "requires existing lockfile" not in failure: + raise AssertionError(f"missing-lock failure was not actionable:\n{failure}") + + source_build = root / "single-source-build" + copy_fixture(first, source_build) + append_text(source_build / ".zpkg.toml", "\n[build]\ncommand = \"make\"\n") + failure = plan(zed, source_build, should_fail=True) + if "does not infer source builds" not in failure: + raise AssertionError(f"source-build failure was not actionable:\n{failure}") + + +def polyglot_canary(fixture: Path, zed: Path, root: Path) -> None: + project = root / "polyglot" + copy_fixture(fixture, project) + append_text( + project / ".zpkg.toml", + """ + +[targets.nodejs.nix] +attribute = "polyglot-node" +systems = ["x86_64-linux", "aarch64-linux"] +outputs = ["out"] + +[targets.rust.nix] +attribute = "polyglot-rust" +systems = ["x86_64-linux"] +outputs = ["out"] +""", + ) + (project / ".zpkg.lock").write_text("version = 1\n", encoding="utf-8") + before = tree_digest(project) + + failure = plan(zed, project, should_fail=True) + if "requires --target" not in failure: + raise AssertionError(f"ambiguous polyglot failure was not actionable:\n{failure}") + + node_alias = plan(zed, project, target="node") + node_exact = plan(zed, project, target="nodejs") + if node_alias != node_exact: + raise AssertionError("target synonym and exact target produced different plans") + node = json.loads(node_exact) + rust = json.loads(plan(zed, project, target="rust")) + if node["package"]["name"] != "polyglot-lib-nodejs": + raise AssertionError(f"Node target was not re-rooted: {node!r}") + if node["package"].get("target") != "nodejs": + raise AssertionError(f"Node target provenance missing: {node!r}") + if rust["package"]["name"] != "polyglot-lib-rust": + raise AssertionError(f"Rust target was not re-rooted: {rust!r}") + if node["source"]["artifact"]["sha256"] == rust["source"]["artifact"]["sha256"]: + raise AssertionError("isolated Node and Rust targets produced the same artifact identity") + if tree_digest(project) != before: + raise AssertionError("polyglot planning mutated the fixture copy") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--node-fixture", required=True, type=Path) + parser.add_argument("--polyglot-fixture", required=True, type=Path) + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--work-root", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + node_fixture = args.node_fixture.resolve() + polyglot_fixture = args.polyglot_fixture.resolve() + zed = args.zed.resolve() + root = args.work_root.resolve() + + if root.exists(): + raise AssertionError(f"work root must be fresh: {root}") + if not zed.is_file(): + raise AssertionError(f"zed binary not found: {zed}") + root.mkdir(parents=True) + + git_clean(node_fixture) + git_clean(polyglot_fixture) + single_package_canary(node_fixture, zed, root) + polyglot_canary(polyglot_fixture, zed, root) + git_clean(node_fixture) + git_clean(polyglot_fixture) + + print( + json.dumps( + { + "result": "PASS", + "schema": "zed.nix-export-plan/v1", + "single_fixture": "zed-pkg-test/node-lib", + "polyglot_fixture": "zed-pkg-test/polyglot-lib", + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/nix_flake_bundle_external.rs b/scripts/nix_flake_bundle_external.rs new file mode 100644 index 00000000..d3325c76 --- /dev/null +++ b/scripts/nix_flake_bundle_external.rs @@ -0,0 +1,289 @@ +use std::collections::BTreeMap; +use std::env; +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::os::unix::fs::PermissionsExt; +use std::path::{Component, Path, PathBuf}; + +use flate2::{Compression, GzBuilder}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use zed_cli::nix_export_bundle::{RenderedNixExportBundle, render_nix_export_bundle}; +use zed_cli::nix_export_plan::{ + NIX_EXPORT_PLAN_SCHEMA_V1, NixExportPlan, PlannedDependency, PlannedPackageClass, + PlannedZedArtifact, ResolvedNixIntent, +}; +use zed_interfaces::{ + ArtifactFormat, NixBuilderNetwork, NixExportMode, NixInteropArtifact, NixPackageIdentity, + NixPolicyEvidence, NixPolicyProfile, +}; + +fn sha256(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +fn executable_artifact(mode: u32) -> Vec { + let encoder = GzBuilder::new() + .mtime(0) + .write(Vec::new(), Compression::default()); + let mut builder = tar::Builder::new(encoder); + let payload = b"#!/bin/sh\nprintf 'external-bundle-ok\\n'\n"; + let mut header = tar::Header::new_gnu(); + header.set_size(payload.len() as u64); + header.set_mode(mode); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_cksum(); + builder + .append_data(&mut header, "package/bin/sample", payload.as_slice()) + .unwrap(); + builder.into_inner().unwrap().finish().unwrap() +} + +fn symlink_artifact() -> Vec { + let encoder = GzBuilder::new() + .mtime(0) + .write(Vec::new(), Compression::default()); + let mut builder = tar::Builder::new(encoder); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_link_name("/etc/passwd").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "package/bin/sample", std::io::empty()) + .unwrap(); + builder.into_inner().unwrap().finish().unwrap() +} + +fn flake_lock() -> Vec { + serde_json::to_vec(&json!({ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1782467914_u64, + "narHash": "sha256-pGvFkM8N0xEkIIXDe5YYfbEAvHrk4IxBrjB/x8OomhE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e73de5be04e0eff4190a1432b946d469c794e7b4", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e73de5be04e0eff4190a1432b946d469c794e7b4", + "type": "github" + } + }, + "root": { "inputs": { "nixpkgs": "nixpkgs" } } + }, + "root": "root", + "version": 7 + })) + .unwrap() +} + +fn plan(artifact: &[u8], system: &str) -> NixExportPlan { + NixExportPlan { + schema: NIX_EXPORT_PLAN_SCHEMA_V1, + package: NixPackageIdentity { + org: "example".into(), + name: "sample".into(), + version: "1.2.3".into(), + target: None, + }, + package_class: PlannedPackageClass::PrebuiltBin, + intent: ResolvedNixIntent { + mode: NixExportMode::Artifact, + attribute: "sample".into(), + systems: vec![system.into()], + outputs: vec!["out".into()], + }, + source: PlannedZedArtifact { + file_name: "example-sample-1.2.3.tar.gz".into(), + artifact: NixInteropArtifact { + format: ArtifactFormat::TarGz, + sha256: sha256(artifact), + size: artifact.len() as u64, + }, + manifest_sha256: "1".repeat(64), + lock_sha256: "2".repeat(64), + }, + bins: BTreeMap::from([("sample".into(), "bin/sample".into())]), + dependencies: Vec::new(), + policy: NixPolicyEvidence { + profile: NixPolicyProfile::StrictV1, + pure_evaluation: true, + import_from_derivation: false, + sandbox_required: true, + builder_network: NixBuilderNetwork::Disabled, + dirty_source: false, + publishable: true, + }, + } +} + +fn required_path(name: &str) -> PathBuf { + PathBuf::from(env::var_os(name).unwrap_or_else(|| panic!("{name} must be set"))) +} + +fn write_bundle(root: &Path, rendered: &RenderedNixExportBundle) { + assert!( + !root.exists(), + "bundle output must be fresh: {}", + root.display() + ); + fs::create_dir(root).unwrap(); + fs::set_permissions(root, fs::Permissions::from_mode(0o755)).unwrap(); + + for (relative, bytes) in &rendered.files { + let path = Path::new(relative); + assert!( + path.components() + .all(|component| matches!(component, Component::Normal(_))), + "renderer returned unsafe path: {relative}" + ); + let destination = root.join(path); + fs::create_dir_all(destination.parent().unwrap()).unwrap(); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&destination) + .unwrap(); + file.write_all(bytes).unwrap(); + file.sync_all().unwrap(); + fs::set_permissions(&destination, fs::Permissions::from_mode(0o644)).unwrap(); + } +} + +fn assert_source_redacted(rendered: &RenderedNixExportBundle) { + let canary = env::var("FORBIDDEN_CANARY").expect("FORBIDDEN_CANARY must be set"); + let path = env::var("FORBIDDEN_PATH").expect("FORBIDDEN_PATH must be set"); + for bytes in rendered.files.values() { + let text = String::from_utf8_lossy(bytes); + assert!(!text.contains(&canary), "bundle retained credential canary"); + assert!(!text.contains(&path), "bundle retained absolute work path"); + } +} + +#[test] +fn public_renderer_replays_byte_identically_and_detects_tampering() { + let system = env::var("EXTERNAL_NIX_SYSTEM").expect("EXTERNAL_NIX_SYSTEM must be set"); + let artifact = executable_artifact(0o755); + let lock = flake_lock(); + let plan = plan(&artifact, &system); + + let first = render_nix_export_bundle(&plan, &artifact, &lock).unwrap(); + let second = render_nix_export_bundle(&plan, &artifact, &lock).unwrap(); + assert_eq!(first, second); + first.validate().unwrap(); + second.validate().unwrap(); + assert_source_redacted(&first); + + write_bundle(&required_path("BUNDLE_OUT_A"), &first); + write_bundle(&required_path("BUNDLE_OUT_B"), &second); + + let report_path = required_path("BUNDLE_REPORT"); + fs::create_dir_all(report_path.parent().unwrap()).unwrap(); + let report = json!({ + "schema": "zed-pkg-test.nix-flake-bundle-canary/v1", + "candidate": env::var("CANDIDATE_SHA").unwrap_or_default(), + "system": system, + "artifact_sha256": plan.source.artifact.sha256, + "artifact_size": plan.source.artifact.size, + "bundle_sha256": first.inventory.bundle_sha256, + "plan_sha256": first.inventory.plan_sha256, + "flake_lock_sha256": first.inventory.flake_lock_sha256, + "file_count": first.files.len(), + "credential_canaries_retained": false, + "external_registry_required": false + }); + fs::write(report_path, serde_json::to_vec(&report).unwrap()).unwrap(); + + let mut changed_package = first.clone(); + changed_package + .files + .get_mut("package.nix") + .unwrap() + .extend_from_slice(b"\n# tampered\n"); + assert!(changed_package.validate().is_err()); + + let mut missing_artifact = first.clone(); + missing_artifact + .files + .remove("artifacts/example-sample-1.2.3.tar.gz"); + assert!(missing_artifact.validate().is_err()); + + let mut extra_file = first.clone(); + extra_file + .files + .insert("unexpected.txt".into(), b"unexpected".to_vec()); + assert!(extra_file.validate().is_err()); + + let mut changed_lock = first.clone(); + changed_lock + .files + .get_mut("flake.lock") + .unwrap() + .push(b'\n'); + assert!(changed_lock.validate().is_err()); + + let mut changed_inventory = first.clone(); + changed_inventory + .files + .get_mut("metadata/bundle.json") + .unwrap() + .push(b'\n'); + assert!(changed_inventory.validate().is_err()); +} + +#[test] +fn wrong_artifact_digest_fails_before_bundle_creation() { + let artifact = executable_artifact(0o755); + let mut plan = plan(&artifact, "x86_64-linux"); + plan.source.artifact.sha256 = "f".repeat(64); + assert!(render_nix_export_bundle(&plan, &artifact, &flake_lock()).is_err()); +} + +#[test] +fn mutable_nixpkgs_revision_fails_closed() { + let artifact = executable_artifact(0o755); + let plan = plan(&artifact, "x86_64-linux"); + let mut lock: Value = serde_json::from_slice(&flake_lock()).unwrap(); + lock["nodes"]["nixpkgs"]["locked"]["rev"] = json!("nixos-unstable"); + assert!( + render_nix_export_bundle(&plan, &artifact, &serde_json::to_vec(&lock).unwrap()).is_err() + ); +} + +#[test] +fn archive_links_fail_closed() { + let artifact = symlink_artifact(); + let plan = plan(&artifact, "x86_64-linux"); + assert!(render_nix_export_bundle(&plan, &artifact, &flake_lock()).is_err()); +} + +#[test] +fn non_executable_declared_binary_fails_closed() { + let artifact = executable_artifact(0o644); + let plan = plan(&artifact, "x86_64-linux"); + assert!(render_nix_export_bundle(&plan, &artifact, &flake_lock()).is_err()); +} + +#[test] +fn dependency_edges_remain_outside_strict_bundle_v1() { + let artifact = executable_artifact(0o755); + let mut plan = plan(&artifact, "x86_64-linux"); + plan.dependencies.push(PlannedDependency { + org: "example".into(), + name: "dependency".into(), + version: "1.0.0".into(), + sha256: "3".repeat(64), + }); + assert!(render_nix_export_bundle(&plan, &artifact, &flake_lock()).is_err()); +} diff --git a/scripts/oci_layout_contract.py b/scripts/oci_layout_contract.py new file mode 100644 index 00000000..af277cf2 --- /dev/null +++ b/scripts/oci_layout_contract.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +"""Black-box certification for local `zed oci plan --out` layouts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import tempfile +from pathlib import Path +from typing import Any + +from oci_plan_contract import ( + DIGEST_RE, + LOCK_LAYER, + MANIFEST_LAYER, + OCI_MANIFEST, + PACKAGE_LAYER, + ZED_CONFIG, + Harness, + copy_fixture, + fingerprint, + poisoned_environment, + transaction_sentinel, + write_lock, +) + +OCI_INDEX = "application/vnd.oci.image.index.v1+json" +OCI_LAYOUT_VERSION = "1.0.0" +REF_NAME = "org.opencontainers.image.ref.name" + + +def arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--strace", required=True, type=Path) + parser.add_argument("--node-lib", required=True, type=Path) + parser.add_argument("--node-app", required=True, type=Path) + parser.add_argument("--polyglot-lib", required=True, type=Path) + parser.add_argument("--evidence", required=True, type=Path) + return parser.parse_args() + + +def layout_command( + destination: str, + output: Path, + target: str | None = None, +) -> list[str]: + command = ["oci", "plan", destination] + if target is not None: + command.extend(["--target", target]) + command.extend(["--out", str(output), "--json"]) + return command + + +def plan_command(destination: str, target: str | None = None) -> list[str]: + command = ["oci", "plan", destination] + if target is not None: + command.extend(["--target", target]) + command.append("--json") + return command + + +def parse_json(stdout: str, label: str) -> dict[str, Any]: + try: + payload = json.loads(stdout) + except json.JSONDecodeError as error: + raise AssertionError(f"{label} stdout is not JSON: {stdout}") from error + if not isinstance(payload, dict): + raise AssertionError(f"{label} JSON must be an object") + return payload + + +def canonical_digest(value: Any, label: str) -> str: + if not isinstance(value, str) or DIGEST_RE.fullmatch(value) is None: + raise AssertionError(f"{label} is not a canonical OCI digest: {value!r}") + return value + + +def descriptor_blob(layout: Path, descriptor: dict[str, Any], label: str) -> bytes: + digest = canonical_digest(descriptor.get("digest"), f"{label} digest") + size = descriptor.get("size") + if not isinstance(size, int) or size <= 0: + raise AssertionError(f"{label} size is invalid: {size!r}") + blob = layout / "blobs" / "sha256" / digest.removeprefix("sha256:") + if not blob.is_file(): + raise AssertionError(f"{label} blob is missing: {blob}") + content = blob.read_bytes() + if len(content) != size: + raise AssertionError(f"{label} size drift: descriptor={size} actual={len(content)}") + actual = "sha256:" + hashlib.sha256(content).hexdigest() + if actual != digest: + raise AssertionError(f"{label} digest drift: descriptor={digest} actual={actual}") + return content + + +def validate_layout( + layout: Path, + result: dict[str, Any], + *, + expected_tag: str, + has_lock: bool, +) -> str: + assert result["schema"] == "zed.oci-layout-result/v1" + assert Path(result["path"]) == layout + assert result["requested_destination"]["tag"] == expected_tag + resolved = result["resolved_reference"] + manifest_digest = canonical_digest(resolved["digest"], "resolved reference") + assert result["manifest"]["digest"] == manifest_digest + + layout_version = json.loads((layout / "oci-layout").read_text(encoding="utf-8")) + assert layout_version == {"imageLayoutVersion": OCI_LAYOUT_VERSION} + index = json.loads((layout / "index.json").read_text(encoding="utf-8")) + assert index["schemaVersion"] == 2 + assert index["mediaType"] == OCI_INDEX + assert len(index["manifests"]) == 1 + manifest_descriptor = index["manifests"][0] + assert manifest_descriptor["mediaType"] == OCI_MANIFEST + assert manifest_descriptor["digest"] == manifest_digest + assert manifest_descriptor["annotations"][REF_NAME] == expected_tag + + manifest = json.loads(descriptor_blob(layout, manifest_descriptor, "OCI manifest")) + assert manifest["schemaVersion"] == 2 + assert manifest["mediaType"] == OCI_MANIFEST + assert manifest["artifactType"] == ZED_CONFIG + config = manifest["config"] + assert config["mediaType"] == ZED_CONFIG + descriptor_blob(layout, config, "OCI config") + + layers = manifest["layers"] + media_types = {layer["mediaType"] for layer in layers} + expected_media_types = {PACKAGE_LAYER, MANIFEST_LAYER} + if has_lock: + expected_media_types.add(LOCK_LAYER) + assert media_types == expected_media_types + for layer in layers: + descriptor_blob(layout, layer, f"layer {layer['mediaType']}") + + expected_digests = { + manifest_descriptor["digest"], + config["digest"], + *(layer["digest"] for layer in layers), + } + actual_files = { + "sha256:" + path.name + for path in (layout / "blobs" / "sha256").iterdir() + if path.is_file() + } + assert actual_files == expected_digests + assert result["blob_count"] == len(expected_digests) + expected_bytes = manifest_descriptor["size"] + config["size"] + sum( + layer["size"] for layer in layers + ) + assert result["total_blob_bytes"] == expected_bytes + + digest = hashlib.sha256() + for path in sorted(layout.rglob("*"), key=lambda item: item.as_posix()): + if path.is_file(): + digest.update(path.relative_to(layout).as_posix().encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def successful_layout( + harness: Harness, + project: Path, + destination: str, + output: Path, + env: dict[str, str], + *, + expected_tag: str, + target: str | None = None, + has_lock: bool = False, + audit: bool = False, +) -> tuple[dict[str, Any], str]: + before = fingerprint(project) + result = harness.invoke( + project, + layout_command(destination, output, target), + env, + success=True, + audit=audit, + ) + if before != fingerprint(project): + raise AssertionError(f"OCI layout materialization mutated project tree: {project}") + if (project / ".zed" / "pack").exists(): + raise AssertionError("OCI layout materialization left persistent .zed/pack output") + payload = parse_json(result.stdout, "OCI layout") + return payload, validate_layout( + output, + payload, + expected_tag=expected_tag, + has_lock=has_lock, + ) + + +def compare_plan( + harness: Harness, + project: Path, + destination: str, + layout_result: dict[str, Any], + env: dict[str, str], + *, + target: str | None = None, +) -> None: + result = harness.invoke( + project, + plan_command(destination, target), + env, + success=True, + ) + plan = parse_json(result.stdout, "OCI plan") + assert plan["requested_destination"] == layout_result["requested_destination"] + assert plan["resolved_reference"] == layout_result["resolved_reference"] + assert plan["adapter"]["manifest"] == layout_result["manifest"] + + +def save_evidence(harness: Harness, summary: dict[str, Any]) -> None: + payload = { + "schema": "zed-pkg-test.oci-layout-evidence/v1", + "summary": summary, + "invocations": harness.records, + } + (harness.evidence / "evidence.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + args = arguments() + harness = Harness(args.zed, args.evidence, args.strace) + summary: dict[str, Any] = {"layouts": {}, "negative_cases": []} + try: + with tempfile.TemporaryDirectory(prefix="zed-oci-layout-contract-") as temporary: + work = Path(temporary) + env = poisoned_environment(work) + + node = copy_fixture(args.node_lib, work / "node-lib") + node_marker = transaction_sentinel(node) + destination = "oci://ghcr.io/zed-pkg-test/node-lib:1.0.0" + first_result, first_fingerprint = successful_layout( + harness, + node, + destination, + work / "node-layout-a", + env, + expected_tag="1.0.0", + audit=True, + ) + second_result, second_fingerprint = successful_layout( + harness, + node, + destination, + work / "node-layout-b", + env, + expected_tag="1.0.0", + ) + first_without_path = { + key: value for key, value in first_result.items() if key != "path" + } + second_without_path = { + key: value for key, value in second_result.items() if key != "path" + } + assert first_without_path == second_without_path + assert first_fingerprint == second_fingerprint + compare_plan(harness, node, destination, first_result, env) + assert node_marker.is_file() + summary["layouts"]["node-lib"] = first_fingerprint + + existing = work / "existing-layout" + existing.mkdir() + marker = existing / "keep.txt" + marker.write_text("must remain\n", encoding="utf-8") + before = fingerprint(node) + harness.invoke( + node, + layout_command(destination, existing), + env, + success=False, + diagnostic="refusing to replace existing OCI layout output", + ) + assert marker.read_text(encoding="utf-8") == "must remain\n" + assert before == fingerprint(node) + summary["negative_cases"].append("existing-output") + + poly = copy_fixture(args.polyglot_lib, work / "polyglot-lib") + poly_marker = transaction_sentinel(poly) + poly_destination = "oci://ghcr.io/zedtest/polyglot-lib-rust:0.1.0" + poly_result, poly_fingerprint = successful_layout( + harness, + poly, + poly_destination, + work / "polyglot-rust-layout", + env, + expected_tag="0.1.0", + target="rust", + ) + compare_plan( + harness, + poly, + poly_destination, + poly_result, + env, + target="rust", + ) + assert poly_marker.is_file() + summary["layouts"]["polyglot-lib-rust"] = poly_fingerprint + + app = copy_fixture(args.node_app, work / "node-app") + app_marker = transaction_sentinel(app) + write_lock(app / ".zpkg.lock", "1.0.0", "a" * 64) + app_destination = "oci://ghcr.io/zed-pkg-test/node-app:0.1.0" + app_result, app_fingerprint = successful_layout( + harness, + app, + app_destination, + work / "node-app-layout", + env, + expected_tag="0.1.0", + has_lock=True, + ) + compare_plan(harness, app, app_destination, app_result, env) + assert app_marker.is_file() + summary["layouts"]["node-app"] = app_fingerprint + + summary["layout_count"] = len(summary["layouts"]) + summary["negative_case_count"] = len(summary["negative_cases"]) + summary["invocation_count"] = len(harness.records) + assert summary["layout_count"] == 3 + assert summary["negative_case_count"] == 1 + finally: + save_evidence(harness, summary) + + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/oci_oras_push_contract.py b/scripts/oci_oras_push_contract.py new file mode 100644 index 00000000..186802fd --- /dev/null +++ b/scripts/oci_oras_push_contract.py @@ -0,0 +1,567 @@ +#!/usr/bin/env python3 +"""Black-box certification for `zed oci push` against an authenticated registry.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +from oci_layout_contract import canonical_digest, parse_json +from oci_plan_contract import ( + DIGEST_RE, + copy_fixture, + fingerprint, + poisoned_environment, + transaction_sentinel, +) + +TEMP_AUTH_PREFIX = "zed-oci-auth-" +SECRET_REPLACEMENT = "" + + +def arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--oras", required=True, type=Path) + parser.add_argument("--strace", required=True, type=Path) + parser.add_argument("--node-lib", required=True, type=Path) + parser.add_argument("--registry", required=True) + parser.add_argument("--username", required=True) + parser.add_argument("--password", required=True) + parser.add_argument("--evidence", required=True, type=Path) + return parser.parse_args() + + +class PushHarness: + def __init__( + self, + zed: Path, + oras: Path, + strace: Path, + evidence: Path, + secret: str, + ) -> None: + self.zed = zed.resolve() + self.oras = oras.resolve() + self.strace = strace.resolve() + self.evidence = evidence.resolve() + self.secret = secret + self.records: list[dict[str, Any]] = [] + self.trace_index = 0 + self.evidence.mkdir(parents=True, exist_ok=True) + for label, executable in ( + ("zed", self.zed), + ("oras", self.oras), + ("strace", self.strace), + ): + if not executable.is_file() or not os.access(executable, os.X_OK): + raise AssertionError(f"{label} is not executable: {executable}") + + def invoke( + self, + cwd: Path, + args: list[str], + env: dict[str, str], + *, + success: bool, + stdin: str | None = None, + diagnostic: str | None = None, + audit: bool = False, + timeout: int = 180, + ) -> subprocess.CompletedProcess[str]: + argv = [str(self.zed), *args] + command = argv + trace_file: Path | None = None + if audit: + self.trace_index += 1 + trace_file = self.evidence / f"strace-{self.trace_index:02d}.log" + command = [ + str(self.strace), + "-f", + "-qq", + "-e", + "trace=network,openat", + "-o", + str(trace_file), + *argv, + ] + + result = subprocess.run( + command, + cwd=cwd, + env=env, + input=stdin, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + combined = result.stdout + result.stderr + if self.secret in combined: + raise AssertionError("OCI push leaked the registry password to output") + self.records.append( + { + "argv": argv, + "cwd": str(cwd), + "returncode": result.returncode, + "stdout": redact(result.stdout, self.secret), + "stderr": redact(result.stderr, self.secret), + "stdin_supplied": stdin is not None, + "audited": audit, + "trace_file": str(trace_file) if trace_file else None, + } + ) + + if success != (result.returncode == 0): + expectation = "succeed" if success else "fail" + raise AssertionError( + f"expected command to {expectation}: {' '.join(argv)}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if diagnostic is not None and diagnostic not in combined: + raise AssertionError( + f"missing diagnostic {diagnostic!r}: {' '.join(argv)}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if trace_file is not None: + trace = trace_file.read_text(encoding="utf-8", errors="replace") + if self.secret in trace: + raise AssertionError("OCI push leaked the registry password to syscall evidence") + forbidden = [ + Path(env["HOME"]) / ".docker" / "config.json", + Path(env["DOCKER_CONFIG"]) / "config.json", + Path(env["ZED_PKG_HOME"]) / "credentials.toml", + Path(env["ZED_PKG_HOME"]) / "auth" / "sessions.toml", + ] + opened = [str(path) for path in forbidden if str(path) in trace] + if opened: + raise AssertionError( + "explicit OCI push opened implicit credential state: " + + ", ".join(opened) + ) + return result + + def save(self, summary: dict[str, Any]) -> None: + payload = { + "schema": "zed-pkg-test.oci-oras-push-evidence/v1", + "summary": summary, + "invocations": self.records, + } + text = json.dumps(payload, indent=2, sort_keys=True) + "\n" + if self.secret in text: + raise AssertionError("registry password leaked into structured evidence") + (self.evidence / "evidence.json").write_text(text, encoding="utf-8") + + +def redact(value: str, secret: str) -> str: + return value.replace(secret, SECRET_REPLACEMENT) + + +def registry_environment(work: Path) -> dict[str, str]: + env = poisoned_environment(work) + for key in ( + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ): + env.pop(key, None) + env["NO_PROXY"] = "127.0.0.1,localhost" + env["no_proxy"] = env["NO_PROXY"] + + docker_config = work / "implicit-docker-config" + docker_config.mkdir() + poison = docker_config / "config.json" + poison.write_text("this implicit Docker config must never be opened\n", encoding="utf-8") + poison.chmod(0) + env["DOCKER_CONFIG"] = str(docker_config) + env.pop("REGISTRY_AUTH_FILE", None) + return env + + +def layout_command(destination: str, output: Path) -> list[str]: + return ["oci", "plan", destination, "--out", str(output), "--json"] + + +def push_command( + harness: PushHarness, + layout: Path, + destination: str, + username: str, + *, + allow_replacement: bool = False, +) -> list[str]: + command = [ + "oci", + "push", + str(layout), + destination, + "--oras", + str(harness.oras), + "--username", + username, + "--password-stdin", + "--plain-http", + "--json", + ] + if allow_replacement: + command.append("--allow-tag-replacement") + return command + + +def materialize_layout( + harness: PushHarness, + project: Path, + destination: str, + layout: Path, + env: dict[str, str], +) -> dict[str, Any]: + before = fingerprint(project) + result = harness.invoke( + project, + layout_command(destination, layout), + env, + success=True, + ) + if fingerprint(project) != before: + raise AssertionError("OCI layout materialization mutated the source fixture") + if (project / ".zed" / "pack").exists(): + raise AssertionError("OCI layout materialization left persistent .zed/pack output") + payload = parse_json(result.stdout, "OCI layout") + assert payload["schema"] == "zed.oci-layout-result/v1" + assert Path(payload["path"]) == layout + canonical_digest(payload["manifest"]["digest"], "layout manifest") + return payload + + +def authenticated_push( + harness: PushHarness, + project: Path, + layout: Path, + destination: str, + username: str, + password: str, + env: dict[str, str], + *, + success: bool, + status: str | None = None, + diagnostic: str | None = None, + allow_replacement: bool = False, + audit: bool = False, +) -> dict[str, Any] | None: + before = fingerprint(project) + temporary_before = temporary_auth_directories() + result = harness.invoke( + project, + push_command( + harness, + layout, + destination, + username, + allow_replacement=allow_replacement, + ), + env, + stdin=password + "\n", + success=success, + diagnostic=diagnostic, + audit=audit, + ) + if fingerprint(project) != before: + raise AssertionError("OCI push mutated the source fixture") + leaked_temporary = temporary_auth_directories() - temporary_before + if leaked_temporary: + raise AssertionError( + "OCI push left temporary registry credentials behind: " + + ", ".join(sorted(leaked_temporary)) + ) + if not success: + return None + payload = parse_json(result.stdout, "OCI push") + assert payload["schema"] == "zed.oci-push-result/v1" + assert payload["authentication"] == "password-stdin" + assert payload["transport"] == "oras-cp" + assert payload["plain_http"] is True + assert payload["insecure_tls"] is False + if status is not None: + assert payload["status"] == status + canonical_digest(payload["manifest"]["digest"], "push manifest") + assert payload["destination"]["digest"] == payload["manifest"]["digest"] + return payload + + +def temporary_auth_directories() -> set[str]: + root = Path(tempfile.gettempdir()) + return { + str(path.resolve()) + for path in root.glob(TEMP_AUTH_PREFIX + "*") + if path.exists() + } + + +def independent_registry_config( + work: Path, + registry: str, + username: str, + password: str, +) -> Path: + auth = base64.b64encode(f"{username}:{password}".encode()).decode() + path = work / "independent-registry-config.json" + path.write_text( + json.dumps({"auths": {registry: {"auth": auth}}}, sort_keys=True), + encoding="utf-8", + ) + path.chmod(stat.S_IRUSR | stat.S_IWUSR) + return path + + +def resolve_digest( + oras: Path, + registry_config: Path, + target: str, + env: dict[str, str], + password: str, +) -> str: + result = subprocess.run( + [ + str(oras), + "resolve", + "--registry-config", + str(registry_config), + "--plain-http", + target, + ], + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=120, + check=False, + ) + if password in result.stdout + result.stderr: + raise AssertionError("independent ORAS verification leaked the password") + if result.returncode != 0: + raise AssertionError( + f"independent ORAS resolve failed ({result.returncode})\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + candidates = re.findall(r"sha256:[0-9a-f]{64}", result.stdout + result.stderr) + if not candidates: + raise AssertionError("independent ORAS resolve returned no canonical digest") + return candidates[-1] + + +def tamper_package_blob(layout: Path) -> None: + index = json.loads((layout / "index.json").read_text(encoding="utf-8")) + manifest_digest = canonical_digest( + index["manifests"][0]["digest"], "tamper manifest descriptor" + ) + manifest_path = layout / "blobs" / "sha256" / manifest_digest.removeprefix("sha256:") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + package = manifest["layers"][0] + package_digest = canonical_digest(package["digest"], "tamper package descriptor") + package_path = layout / "blobs" / "sha256" / package_digest.removeprefix("sha256:") + package_path.write_bytes(package_path.read_bytes() + b"tampered") + + +def main() -> int: + args = arguments() + if not args.password or "\n" in args.password or "\r" in args.password: + raise AssertionError("test password must be one non-empty line") + registry = args.registry.rstrip("/") + destination = f"oci://{registry}/zed-pkg-test/node-lib:1.0.0" + target = f"{registry}/zed-pkg-test/node-lib:1.0.0" + harness = PushHarness( + args.zed, + args.oras, + args.strace, + args.evidence, + args.password, + ) + summary: dict[str, Any] = {"successful": [], "negative": []} + + try: + with tempfile.TemporaryDirectory(prefix="zed-oci-oras-contract-") as temporary: + work = Path(temporary) + env = registry_environment(work) + project = copy_fixture(args.node_lib, work / "node-lib") + transaction = transaction_sentinel(project) + registry_config = independent_registry_config( + work, registry, args.username, args.password + ) + + first_layout = work / "layout-a" + first_layout_result = materialize_layout( + harness, project, destination, first_layout, env + ) + first_digest = first_layout_result["manifest"]["digest"] + first_push = authenticated_push( + harness, + project, + first_layout, + destination, + args.username, + args.password, + env, + success=True, + status="pushed", + audit=True, + ) + assert first_push is not None + assert first_push["manifest"]["digest"] == first_digest + assert resolve_digest( + harness.oras, registry_config, target, env, args.password + ) == first_digest + summary["successful"].append("authenticated-first-push") + + idempotent = authenticated_push( + harness, + project, + first_layout, + destination, + args.username, + args.password, + env, + success=True, + status="already-present", + ) + assert idempotent is not None + assert idempotent["manifest"]["digest"] == first_digest + summary["successful"].append("idempotent-same-digest") + + replacement_project = copy_fixture(args.node_lib, work / "node-lib-replacement") + replacement_transaction = transaction_sentinel(replacement_project) + (replacement_project / "oci-replacement.txt").write_text( + "this content deliberately changes the package artifact\n", + encoding="utf-8", + ) + replacement_layout = work / "layout-b" + replacement_layout_result = materialize_layout( + harness, + replacement_project, + destination, + replacement_layout, + env, + ) + replacement_digest = replacement_layout_result["manifest"]["digest"] + assert replacement_digest != first_digest + authenticated_push( + harness, + replacement_project, + replacement_layout, + destination, + args.username, + args.password, + env, + success=False, + diagnostic="refusing to replace OCI tag", + ) + assert resolve_digest( + harness.oras, registry_config, target, env, args.password + ) == first_digest + summary["negative"].append("replacement-without-consent") + + replacement_push = authenticated_push( + harness, + replacement_project, + replacement_layout, + destination, + args.username, + args.password, + env, + success=True, + status="replaced", + allow_replacement=True, + ) + assert replacement_push is not None + assert replacement_push["manifest"]["digest"] == replacement_digest + assert resolve_digest( + harness.oras, registry_config, target, env, args.password + ) == replacement_digest + summary["successful"].append("explicit-tag-replacement") + + tampered_layout = work / "layout-tampered" + shutil.copytree(first_layout, tampered_layout) + tamper_package_blob(tampered_layout) + authenticated_push( + harness, + project, + tampered_layout, + destination, + args.username, + args.password, + env, + success=False, + diagnostic="size drift", + ) + summary["negative"].append("tampered-layout-before-transport") + + before = fingerprint(project) + harness.invoke( + project, + [ + "oci", + "push", + str(first_layout), + destination, + "--oras", + str(harness.oras), + "--plain-http", + "--json", + ], + env, + success=False, + diagnostic="choose exactly one OCI authentication mode", + ) + assert fingerprint(project) == before + summary["negative"].append("implicit-auth-rejected") + + harness.invoke( + project, + [ + "oci", + "push", + str(first_layout), + "oci://ghcr.io/zed-pkg-test/node-lib:1.0.0", + "--oras", + str(harness.oras), + "--anonymous", + "--plain-http", + ], + env, + success=False, + diagnostic="--plain-http is accepted only for loopback registries", + ) + summary["negative"].append("plain-http-non-loopback") + + assert transaction.is_file() + assert replacement_transaction.is_file() + assert not (project / ".zed" / "pack").exists() + assert not (replacement_project / ".zed" / "pack").exists() + summary["final_digest"] = replacement_digest + summary["successful_count"] = len(summary["successful"]) + summary["negative_count"] = len(summary["negative"]) + summary["invocation_count"] = len(harness.records) + assert summary["successful_count"] == 3 + assert summary["negative_count"] == 4 + finally: + harness.save(summary) + + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/oci_plan_contract.py b/scripts/oci_plan_contract.py new file mode 100644 index 00000000..2c6c2f27 --- /dev/null +++ b/scripts/oci_plan_contract.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +"""Black-box certification for `zed oci plan`.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import stat +import subprocess +import tempfile +from pathlib import Path +from typing import Any + +DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +NETWORK_RE = re.compile( + r"\b(socket|socketpair|connect|accept|accept4|bind|listen|sendto|recvfrom|" + r"sendmsg|recvmsg|getpeername|getsockname|shutdown)\(" +) +OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json" +ZED_CONFIG = "application/vnd.zed.package.config.v1+json" +PACKAGE_LAYER = "application/vnd.zed.package.v1.tar+gzip" +MANIFEST_LAYER = "application/vnd.zed.package.manifest.v1+toml" +LOCK_LAYER = "application/vnd.zed.package.lock.v1+toml" + + +class Harness: + def __init__(self, zed: Path, evidence: Path, strace: Path) -> None: + self.zed = zed.resolve() + self.evidence = evidence.resolve() + self.strace = strace.resolve() + self.records: list[dict[str, Any]] = [] + self.trace_index = 0 + self.evidence.mkdir(parents=True, exist_ok=True) + for label, path in (("zed", self.zed), ("strace", self.strace)): + if not path.is_file(): + raise AssertionError(f"{label} executable does not exist: {path}") + if not os.access(path, os.X_OK): + raise AssertionError(f"{label} is not executable: {path}") + + def invoke( + self, + project: Path, + args: list[str], + env: dict[str, str], + *, + success: bool, + diagnostic: str | None = None, + audit: bool = False, + ) -> subprocess.CompletedProcess[str]: + argv = [str(self.zed), *args] + command = argv + trace_file: Path | None = None + if audit: + self.trace_index += 1 + trace_file = self.evidence / f"strace-{self.trace_index:02d}.log" + command = [ + str(self.strace), + "-f", + "-qq", + "-e", + "trace=network,openat", + "-o", + str(trace_file), + *argv, + ] + + result = subprocess.run( + command, + cwd=project, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=120, + check=False, + ) + self.records.append( + { + "argv": argv, + "cwd": str(project), + "returncode": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + "audited": audit, + "trace_file": str(trace_file) if trace_file else None, + } + ) + + if success != (result.returncode == 0): + expectation = "succeed" if success else "fail" + raise AssertionError( + f"expected command to {expectation}: {' '.join(argv)}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if diagnostic is not None and diagnostic not in result.stdout + result.stderr: + raise AssertionError( + f"missing diagnostic {diagnostic!r}: {' '.join(argv)}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if trace_file is not None: + self.assert_credential_free(trace_file, env) + return result + + @staticmethod + def assert_credential_free(trace_file: Path, env: dict[str, str]) -> None: + trace = trace_file.read_text(encoding="utf-8", errors="replace") + network = [line for line in trace.splitlines() if NETWORK_RE.search(line)] + if network: + raise AssertionError( + "OCI planning made runtime network syscalls:\n" + "\n".join(network[:40]) + ) + zed_home = Path(env["ZED_PKG_HOME"]) + forbidden = [zed_home / "credentials.toml", zed_home / "auth" / "sessions.toml"] + opened = [str(path) for path in forbidden if str(path) in trace] + if opened: + raise AssertionError("OCI planning opened credential state: " + ", ".join(opened)) + + def save(self, summary: dict[str, Any]) -> None: + payload = { + "schema": "zed-pkg-test.oci-plan-evidence/v1", + "summary": summary, + "invocations": self.records, + } + (self.evidence / "evidence.json").write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + parser.add_argument("--strace", required=True, type=Path) + parser.add_argument("--node-lib", required=True, type=Path) + parser.add_argument("--node-app", required=True, type=Path) + parser.add_argument("--polyglot-lib", required=True, type=Path) + parser.add_argument("--evidence", required=True, type=Path) + return parser.parse_args() + + +def copy_fixture(source: Path, destination: Path) -> Path: + if not (source / ".zpkg.toml").is_file(): + raise AssertionError(f"fixture has no .zpkg.toml: {source}") + shutil.copytree( + source, + destination, + symlinks=True, + ignore=shutil.ignore_patterns(".git"), + ) + return destination + + +def fingerprint(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*"), key=lambda value: value.as_posix()): + metadata = path.lstat() + digest.update(path.relative_to(root).as_posix().encode()) + digest.update(oct(stat.S_IMODE(metadata.st_mode)).encode()) + if path.is_symlink(): + digest.update(b"symlink") + digest.update(os.readlink(path).encode()) + elif path.is_file(): + digest.update(b"file") + digest.update(path.read_bytes()) + elif path.is_dir(): + digest.update(b"dir") + else: + raise AssertionError(f"unsupported fixture entry: {path}") + digest.update(b"\0") + return digest.hexdigest() + + +def poisoned_environment(work: Path) -> dict[str, str]: + home = work / "home" + zed_home = home / ".zed-pkg" + auth = zed_home / "auth" + auth.mkdir(parents=True) + for path in (zed_home / "credentials.toml", auth / "sessions.toml"): + path.write_text("deliberately invalid TOML = [\n", encoding="utf-8") + path.chmod(0) + + env = os.environ.copy() + env.update( + { + "HOME": str(home), + "ZED_PKG_HOME": str(zed_home), + "ZED_PKG_REGISTRY": "http://127.0.0.1:9", + "ZED_PKG_AUTH_URL": "http://127.0.0.1:9/shared-auth", + "GIT_TERMINAL_PROMPT": "0", + "HTTP_PROXY": "http://127.0.0.1:9", + "HTTPS_PROXY": "http://127.0.0.1:9", + "ALL_PROXY": "http://127.0.0.1:9", + "http_proxy": "http://127.0.0.1:9", + "https_proxy": "http://127.0.0.1:9", + "all_proxy": "http://127.0.0.1:9", + "NO_PROXY": "", + "no_proxy": "", + } + ) + for key in ("ZED_PKG_TOKEN", "ZED_PKG_AUTH_PASSWORD"): + env.pop(key, None) + return env + + +def transaction_sentinel(project: Path) -> Path: + marker = project / ".zpkg-staging" / "unrecoverable-contract-sentinel" / "must-remain.txt" + marker.parent.mkdir(parents=True) + marker.write_text("OCI planning must not recover project transactions\n", encoding="utf-8") + return marker + + +def command(destination: str, target: str | None = None) -> list[str]: + result = ["oci", "plan", destination] + if target is not None: + result.extend(["--target", target]) + result.append("--json") + return result + + +def successful_plan( + harness: Harness, + project: Path, + destination: str, + env: dict[str, str], + *, + target: str | None = None, + audit: bool = False, +) -> tuple[str, dict[str, Any]]: + before = fingerprint(project) + result = harness.invoke( + project, + command(destination, target), + env, + success=True, + audit=audit, + ) + if before != fingerprint(project): + raise AssertionError(f"OCI planning mutated project tree: {project}") + if (project / ".zed" / "pack").exists(): + raise AssertionError("OCI planning left persistent .zed/pack output") + try: + return result.stdout, json.loads(result.stdout) + except json.JSONDecodeError as error: + raise AssertionError(f"OCI plan stdout is not JSON: {result.stdout}") from error + + +def failed_plan( + harness: Harness, + project: Path, + destination: str, + env: dict[str, str], + diagnostic: str, + *, + target: str | None = None, +) -> None: + before = fingerprint(project) + harness.invoke( + project, + command(destination, target), + env, + success=False, + diagnostic=diagnostic, + ) + if before != fingerprint(project): + raise AssertionError(f"failed OCI planning mutated project tree: {project}") + + +def deterministic_plan( + harness: Harness, + project: Path, + destination: str, + env: dict[str, str], + *, + target: str | None = None, + audit_first: bool = False, +) -> dict[str, Any]: + first_text, first = successful_plan( + harness, + project, + destination, + env, + target=target, + audit=audit_first, + ) + second_text, second = successful_plan( + harness, + project, + destination, + env, + target=target, + ) + if first_text != second_text or first != second: + raise AssertionError(f"OCI plan is not deterministic: {destination} target={target}") + return first + + +def digest(value: Any, label: str) -> str: + if not isinstance(value, str) or DIGEST_RE.fullmatch(value) is None: + raise AssertionError(f"{label} is not a canonical OCI SHA-256 digest: {value!r}") + return value + + +def validate( + payload: dict[str, Any], + *, + org: str, + name: str, + version: str, + repository: str, + target: str | None, + has_lock: bool, +) -> str: + expected_package: dict[str, Any] = {"org": org, "name": name, "version": version} + if target is not None: + expected_package["target"] = target + assert payload["schema"] == "zed.oci-publish-plan/v1" + assert payload["package"] == expected_package + assert payload["requested_destination"] == { + "registry": "ghcr.io", + "repository": repository, + "tag": version, + } + + resolved = payload["resolved_reference"] + assert resolved["registry"] == "ghcr.io" + assert resolved["repository"] == repository + assert resolved["tag"] == version + manifest_digest = digest(resolved["digest"], "resolved reference") + + adapter = payload["adapter"] + assert adapter["schema"] == "zed.oci-adapter/v1" + assert adapter["package"] == expected_package + assert adapter["reference"] == resolved + assert adapter["manifest"]["mediaType"] == OCI_MANIFEST + assert digest(adapter["manifest"]["digest"], "manifest descriptor") == manifest_digest + assert adapter["manifest"]["size"] > 0 + assert adapter["config"]["mediaType"] == ZED_CONFIG + digest(adapter["config"]["digest"], "config descriptor") + assert adapter["config"]["size"] > 0 + + layers = {layer["kind"]: layer["descriptor"] for layer in adapter["layers"]} + expected_layers = { + "package-tar-gz": PACKAGE_LAYER, + "manifest": MANIFEST_LAYER, + } + if has_lock: + expected_layers["lockfile"] = LOCK_LAYER + assert set(layers) == set(expected_layers) + layer_digests: set[str] = set() + for kind, media_type in expected_layers.items(): + descriptor = layers[kind] + assert descriptor["mediaType"] == media_type + layer_digest = digest(descriptor["digest"], f"{kind} layer") + assert layer_digest not in layer_digests + layer_digests.add(layer_digest) + assert descriptor["size"] > 0 + + blobs = {blob["kind"]: blob for blob in payload["blobs"]} + expected_blobs = {"config", "package", "manifest", "oci-manifest"} + if has_lock: + expected_blobs.add("lockfile") + assert set(blobs) == expected_blobs + for kind, blob in blobs.items(): + digest(blob["digest"], f"{kind} blob") + assert blob["size"] > 0 + assert blob["source"] + assert blobs["oci-manifest"]["digest"] == manifest_digest + assert blobs["config"]["digest"] == adapter["config"]["digest"] + assert blobs["package"]["digest"] == layers["package-tar-gz"]["digest"] + assert blobs["manifest"]["digest"] == layers["manifest"]["digest"] + if has_lock: + assert blobs["lockfile"]["digest"] == layers["lockfile"]["digest"] + + canonical = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode() + return hashlib.sha256(canonical).hexdigest() + + +def write_lock(path: Path, version: str, sha256: str) -> None: + path.write_text( + "\n".join( + [ + "version = 1", + "", + "[[package]]", + 'org = "zed-pkg-test"', + 'name = "node-lib"', + f'version = "{version}"', + f'sha256 = "{sha256}"', + "size = 10", + 'vcs_tag = "v1.0.0"', + 'vcs_commit = "2d41e658a382fdff27ef87a0db24dbaff280a685"', + 'source = "file:///contract-registry"', + "", + ] + ), + encoding="utf-8", + ) + + +def main() -> int: + args = arguments() + harness = Harness(args.zed, args.evidence, args.strace) + summary: dict[str, Any] = {"plans": {}, "negative_cases": []} + try: + with tempfile.TemporaryDirectory(prefix="zed-oci-plan-contract-") as temporary: + work = Path(temporary) + env = poisoned_environment(work) + + node = copy_fixture(args.node_lib, work / "node-lib") + node_marker = transaction_sentinel(node) + payload = deterministic_plan( + harness, + node, + "oci://ghcr.io/zed-pkg-test/node-lib:1.0.0", + env, + audit_first=True, + ) + summary["plans"]["node-lib"] = validate( + payload, + org="zed-pkg-test", + name="node-lib", + version="1.0.0", + repository="zed-pkg-test/node-lib", + target=None, + has_lock=False, + ) + assert node_marker.is_file() + failed_plan( + harness, + node, + "oci://ghcr.io/zed-pkg-test/node-lib:latest", + env, + "must equal package version", + ) + summary["negative_cases"].append("tag-drift") + failed_plan( + harness, + node, + "oci://ghcr.io/zed-pkg-test/node-lib:1.0.0@sha256:" + "a" * 64, + env, + "preselected digest", + ) + summary["negative_cases"].append("caller-selected-digest") + + poly = copy_fixture(args.polyglot_lib, work / "polyglot-lib") + poly_marker = transaction_sentinel(poly) + failed_plan( + harness, + poly, + "oci://ghcr.io/zedtest/polyglot-lib:0.1.0", + env, + "requires --target", + ) + summary["negative_cases"].append("polyglot-target-required") + for target in ("nodejs", "python", "golang", "rust"): + name = f"polyglot-lib-{target}" + payload = deterministic_plan( + harness, + poly, + f"oci://ghcr.io/zedtest/{name}:0.1.0", + env, + target=target, + ) + summary["plans"][name] = validate( + payload, + org="zedtest", + name=name, + version="0.1.0", + repository=f"zedtest/{name}", + target=target, + has_lock=False, + ) + assert poly_marker.is_file() + + app = copy_fixture(args.node_app, work / "node-app") + app_marker = transaction_sentinel(app) + failed_plan( + harness, + app, + "oci://ghcr.io/zed-pkg-test/node-app:0.1.0", + env, + ".zpkg.lock is required", + ) + summary["negative_cases"].append("dependency-lock-required") + + lock = app / ".zpkg.lock" + write_lock(lock, "1.0.0", "a" * 64) + payload = deterministic_plan( + harness, + app, + "oci://ghcr.io/zed-pkg-test/node-app:0.1.0", + env, + ) + summary["plans"]["node-app"] = validate( + payload, + org="zed-pkg-test", + name="node-app", + version="0.1.0", + repository="zed-pkg-test/node-app", + target=None, + has_lock=True, + ) + + write_lock(lock, "2.0.0", "a" * 64) + failed_plan( + harness, + app, + "oci://ghcr.io/zed-pkg-test/node-app:0.1.0", + env, + "lock drift", + ) + summary["negative_cases"].append("dependency-version-drift") + + write_lock(lock, "1.0.0", "A" * 64) + failed_plan( + harness, + app, + "oci://ghcr.io/zed-pkg-test/node-app:0.1.0", + env, + "expected 64 lowercase hex chars", + ) + summary["negative_cases"].append("noncanonical-lock-digest") + assert app_marker.is_file() + + summary["plan_count"] = len(summary["plans"]) + summary["negative_case_count"] = len(summary["negative_cases"]) + summary["invocation_count"] = len(harness.records) + assert summary["plan_count"] == 6 + assert summary["negative_case_count"] == 6 + finally: + harness.save(summary) + + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/org_app_installs.py b/scripts/org_app_installs.py new file mode 100644 index 00000000..c20709cb --- /dev/null +++ b/scripts/org_app_installs.py @@ -0,0 +1,966 @@ +#!/usr/bin/env python3 +"""Install real zed-pkg-test app fixtures and verify adapter/workspace behavior.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Sequence + +GITHUB_ORG = "zed-pkg-test" +APP_REPOS = ( + "node-app", + "rust-app", + "go-app", + "python-app", + "dart-app", + "gleam-app", + "polyglot-node-app", + "polyglot-go-app", +) +PACKAGE_SOURCES: dict[str, tuple[str, str]] = { + "zed-pkg-test/node-lib": ("node-lib", "."), + "zed-pkg-test/rust-lib": ("rust-lib", "."), + "zed-pkg-test/go-lib": ("go-lib", "."), + "zed-pkg-test/python-lib": ("python-lib", "."), + "zed-pkg-test/dart-lib": ("dart-lib", "."), + "zed-pkg-test/gleam-lib": ("gleam-lib", "."), + "zedtest/polyglot-lib-nodejs": ("polyglot-lib", "."), + "zedtest/polyglot-lib-python": ("polyglot-lib", "."), + "zedtest/polyglot-lib-golang": ("polyglot-lib", "."), + "zedtest/polyglot-lib-rust": ("polyglot-lib", "."), + "zedtest/shared-schema": ("shared-schema", "."), +} +EXPECTED_ADAPTER = { + "node-app": "node", + "rust-app": "rust", + "go-app": "go", + "python-app": "python", + "dart-app": "dart", + "gleam-app": "none", + "polyglot-node-app": "node", + "polyglot-go-app": "go", +} +PREFETCH_RE = re.compile( + r"recursive install prefetch: (?P\d+) package\(s\), " + r"up to (?P\d+) concurrent, (?P\d+) downloaded" +) +GENERATED_WIRING_FILES = ( + "paths.json", + "node_path", + "classpath", + "go.work", + "pythonpath", + "cargo-paths.toml", + "pub-deps.yaml", +) + + +@dataclass(frozen=True) +class PackageRef: + full_name: str + version: str + + @property + def org(self) -> str: + return self.full_name.split("/", 1)[0] + + @property + def name(self) -> str: + return self.full_name.split("/", 1)[1] + + +@dataclass(frozen=True) +class Completed: + command: tuple[str, ...] + returncode: int + output: str + + +def shell_quote(value: str) -> str: + if re.fullmatch(r"[A-Za-z0-9_./:@%+=,-]+", value): + return value + return "'" + value.replace("'", "'\"'\"'") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, + timeout: int = 300, +) -> Completed: + argv = tuple(str(value) for value in command) + shown = " ".join(shell_quote(value) for value in argv) + print(f"\n$ (cd {cwd or Path.cwd()} && {shown})", flush=True) + completed = subprocess.run( + argv, + cwd=cwd, + env=dict(env) if env is not None else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + timeout=timeout, + ) + if completed.stdout: + print(completed.stdout, end="", flush=True) + return Completed(argv, completed.returncode, completed.stdout) + + +def require_success(completed: Completed) -> None: + if completed.returncode != 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise RuntimeError(f"command failed with exit code {completed.returncode}: {shown}") + + +def require_failure(completed: Completed, *fragments: str) -> None: + if completed.returncode == 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise AssertionError(f"command unexpectedly succeeded: {shown}") + for fragment in fragments: + if fragment not in completed.output: + raise AssertionError( + f"failed command output did not contain {fragment!r}:\n{completed.output}" + ) + + +def tree_fingerprint(root: Path) -> tuple[tuple[str, int, str], ...]: + if not root.exists(): + return () + rows: list[tuple[str, int, str]] = [] + for candidate in sorted(root.rglob("*")): + if candidate.is_file() and not candidate.is_symlink(): + data = candidate.read_bytes() + rows.append( + ( + candidate.relative_to(root).as_posix(), + len(data), + hashlib.sha256(data).hexdigest(), + ) + ) + return tuple(rows) + + +def zed_command(zed: Path, registry: Path, home: Path, *args: str) -> list[str | Path]: + return [ + zed, + "--registry", + f"file://{registry}", + "--home", + home, + *args, + ] + + +def read_manifest(directory: Path) -> dict: + with (directory / ".zpkg.toml").open("rb") as handle: + return tomllib.load(handle) + + +def dependencies(manifest: dict) -> dict[str, str]: + section = manifest.get("dependencies") or {} + if not isinstance(section, dict): + raise AssertionError("[dependencies] must be a table") + return {str(key): str(value) for key, value in section.items()} + + +def expected_packages(manifest: dict) -> list[PackageRef]: + package = manifest["package"] + org = str(package["org"]) + base_name = str(package["name"]) + version = str(package["version"]) + targets = manifest.get("targets") or {} + if not targets: + return [PackageRef(f"{org}/{base_name}", version)] + result: list[PackageRef] = [] + for target_name, target in sorted(targets.items()): + target_table = target or {} + name = str(target_table.get("name") or f"{base_name}-{target_name}") + result.append(PackageRef(f"{org}/{name}", version)) + return result + + +def modules_dir(manifest: dict) -> str: + install = manifest.get("install") or {} + return str(install.get("dir") or "zed_modules") + + +def clone_repo(root: Path, repo: str, clones: dict[str, Path]) -> Path: + if repo in clones: + return clones[repo] + destination = root / repo + completed = run( + [ + "git", + "clone", + "--depth", + "1", + "--recurse-submodules", + "--shallow-submodules", + f"https://github.com/{GITHUB_ORG}/{repo}.git", + destination, + ], + timeout=300, + ) + require_success(completed) + clones[repo] = destination + return destination + + +def copy_fixture(source: Path, destination: Path) -> None: + def ignore(directory: str, names: list[str]) -> set[str]: + ignored = {name for name in names if name in {".git", ".zed", ".zpkg-staging"}} + if Path(directory) == source: + ignored.update(name for name in names if name == ".zpkg.lock") + return ignored + + shutil.copytree(source, destination, symlinks=True, ignore=ignore) + + +def registry_metadata_path(registry: Path, package: PackageRef) -> Path: + return ( + registry + / "packages" + / package.org + / package.name + / "versions" + / f"{package.version}.json" + ) + + +def assert_registry_package(registry: Path, package: PackageRef) -> None: + metadata_path = registry_metadata_path(registry, package) + if not metadata_path.is_file(): + raise AssertionError(f"missing registry metadata: {metadata_path}") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + sha256 = str(metadata.get("sha256") or "") + if not re.fullmatch(r"[0-9a-f]{64}", sha256): + raise AssertionError(f"invalid SHA in {metadata_path}: {sha256!r}") + artifact = registry / "artifacts" / f"{sha256}.tar.gz" + if not artifact.is_file(): + raise AssertionError(f"missing registry artifact: {artifact}") + + +def parse_lock(project: Path) -> list[dict[str, object]]: + with (project / ".zpkg.lock").open("rb") as handle: + document = tomllib.load(handle) + packages = document.get("package") or [] + if not isinstance(packages, list): + raise AssertionError(f"invalid lock package table in {project}") + return packages + + +def package_key(package: Mapping[str, object]) -> str: + return f"{package['org']}/{package['name']}" + + +def materialized_path(project: Path, install_dir: str, package: Mapping[str, object]) -> Path: + return project / install_dir / str(package["org"]) / str(package["name"]) + + +def assert_symlink_graph( + project: Path, + home: Path, + install_dir: str, + packages: Sequence[Mapping[str, object]], +) -> dict[str, Path]: + store = (home / "store").resolve() + targets: dict[str, Path] = {} + for package in packages: + key = package_key(package) + installed = materialized_path(project, install_dir, package) + if not installed.is_symlink(): + raise AssertionError(f"{project.name}: expected symlink for {key}: {installed}") + target = installed.resolve(strict=True) + try: + target.relative_to(store) + except ValueError as error: + raise AssertionError(f"{installed} resolves outside {store}: {target}") from error + if not (target / ".zpkg.toml").is_file(): + raise AssertionError(f"installed artifact has no manifest: {target}") + targets[key] = target + return targets + + +def assert_paths_index(project: Path, keys: Sequence[str]) -> None: + path = project / ".zed" / "paths.json" + if not path.is_file(): + raise AssertionError(f"missing adapter-independent path index: {path}") + text = path.read_text(encoding="utf-8") + json.loads(text) + for key in keys: + if key not in text: + raise AssertionError(f"{path} does not mention installed package {key}") + + +def assert_adapter( + project: Path, + adapter: str, + install_dir: str, + packages: Sequence[Mapping[str, object]], +) -> None: + keys = [package_key(package) for package in packages] + assert_paths_index(project, keys) + zed_dir = project / ".zed" + + if adapter == "node": + node_path = zed_dir / "node_path" + if node_path.read_text(encoding="utf-8").strip() != install_dir: + raise AssertionError(f"unexpected node_path in {project}: {node_path.read_text()!r}") + for package in packages: + node_link = ( + project + / "node_modules" + / f"@{package['org']}" + / str(package["name"]) + ) + if not node_link.is_symlink(): + raise AssertionError(f"missing Node adapter symlink: {node_link}") + if node_link.resolve(strict=True) != materialized_path( + project, install_dir, package + ).resolve(strict=True): + raise AssertionError(f"Node adapter target mismatch: {node_link}") + elif adapter == "go": + wiring = zed_dir / "go.work" + if not wiring.is_file(): + raise AssertionError(f"missing Go wiring: {wiring}") + text = wiring.read_text(encoding="utf-8") + for package in packages: + if str(package["name"]) not in text: + raise AssertionError(f"{wiring} omits {package_key(package)}") + elif adapter == "python": + wiring = zed_dir / "pythonpath" + if not wiring.is_file(): + raise AssertionError(f"missing Python wiring: {wiring}") + text = wiring.read_text(encoding="utf-8") + for package in packages: + if str(package["name"]) not in text: + raise AssertionError(f"{wiring} omits {package_key(package)}") + elif adapter == "rust": + wiring = zed_dir / "cargo-paths.toml" + if not wiring.is_file(): + raise AssertionError(f"missing Rust wiring: {wiring}") + text = wiring.read_text(encoding="utf-8") + for package in packages: + if str(package["name"]) not in text: + raise AssertionError(f"{wiring} omits {package_key(package)}") + elif adapter == "dart": + wiring = zed_dir / "pub-deps.yaml" + if not wiring.is_file(): + raise AssertionError(f"missing Dart wiring: {wiring}") + text = wiring.read_text(encoding="utf-8") + for package in packages: + if str(package["name"]) not in text: + raise AssertionError(f"{wiring} omits {package_key(package)}") + elif adapter != "none": + raise AssertionError(f"unknown expected adapter: {adapter}") + + +def assert_not_materialized( + project: Path, + install_dir: str, + packages: Sequence[Mapping[str, object]], +) -> None: + for package in packages: + path = materialized_path(project, install_dir, package) + if path.exists() or path.is_symlink(): + raise AssertionError(f"uninstall left {path}") + node_path = ( + project + / "node_modules" + / f"@{package['org']}" + / str(package["name"]) + ) + if node_path.exists() or node_path.is_symlink(): + raise AssertionError(f"uninstall left Node adapter path {node_path}") + + +def assert_generated_wiring_absent(project: Path) -> None: + leaked = [ + project / ".zed" / name + for name in GENERATED_WIRING_FILES + if (project / ".zed" / name).exists() + or (project / ".zed" / name).is_symlink() + ] + if leaked: + raise AssertionError(f"uninstall left generated adapter wiring: {leaked}") + + +def assert_no_staging(project: Path) -> None: + staging = project / ".zpkg-staging" + if staging.exists(): + raise AssertionError(f"transaction staging leaked in {project}: {list(staging.iterdir())}") + + +def parse_prefetch(output: str) -> tuple[int, int, int]: + matches = list(PREFETCH_RE.finditer(output)) + if len(matches) != 1: + raise AssertionError(f"expected one prefetch summary, got {len(matches)}:\n{output}") + match = matches[0] + return tuple( + int(match.group(field)) + for field in ("resolved", "concurrency", "downloaded") + ) + + +def publish_prerequisite( + package_key_value: str, + *, + zed: Path, + registry: Path, + publish_home: Path, + clone_root: Path, + clones: dict[str, Path], + published_sources: set[tuple[str, str]], + environment: Mapping[str, str], +) -> None: + try: + repo, relative = PACKAGE_SOURCES[package_key_value] + except KeyError as error: + raise AssertionError( + f"no source mapping for dependency {package_key_value!r}" + ) from error + source_root = clone_repo(clone_root, repo, clones) + source = (source_root / relative).resolve() + source_manifest = read_manifest(source) + for dependency in dependencies(source_manifest): + publish_prerequisite( + dependency, + zed=zed, + registry=registry, + publish_home=publish_home, + clone_root=clone_root, + clones=clones, + published_sources=published_sources, + environment=environment, + ) + + source_key = (repo, relative) + if source_key in published_sources: + return + completed = run( + zed_command( + zed, + registry, + publish_home, + "publish", + "--skip-vcs-checks", + ), + cwd=source, + env=environment, + ) + require_success(completed) + published_sources.add(source_key) + for output in expected_packages(source_manifest): + assert_registry_package(registry, output) + + +def exercise_app( + repo: str, + *, + zed: Path, + registry: Path, + shared_home: Path, + clone_root: Path, + app_root: Path, + clones: dict[str, Path], + published_sources: set[tuple[str, str]], + environment: Mapping[str, str], +) -> tuple[int, set[str]]: + source = clone_repo(clone_root, repo, clones) + source_manifest = read_manifest(source) + direct = dependencies(source_manifest) + if not direct: + raise AssertionError(f"app fixture {repo} has no Zed dependencies") + for dependency in direct: + publish_prerequisite( + dependency, + zed=zed, + registry=registry, + publish_home=app_root.parent / "publish-home", + clone_root=clone_root, + clones=clones, + published_sources=published_sources, + environment=environment, + ) + + project = app_root / repo + copy_fixture(source, project) + manifest_bytes = (project / ".zpkg.toml").read_bytes() + install_dir = modules_dir(source_manifest) + + cold = run( + zed_command(zed, registry, shared_home, "install"), + cwd=project, + env=environment, + ) + require_success(cold) + packages = parse_lock(project) + keys = {package_key(package) for package in packages} + if not set(direct).issubset(keys): + raise AssertionError( + f"{repo}: lock omits direct dependencies {sorted(set(direct) - keys)}" + ) + summary = parse_prefetch(cold.output) + if summary[:2] != (len(packages), 5): + raise AssertionError(f"{repo}: unexpected cold prefetch summary {summary}") + + targets = assert_symlink_graph(project, shared_home, install_dir, packages) + assert_adapter(project, EXPECTED_ADAPTER[repo], install_dir, packages) + assert_no_staging(project) + lock_bytes = (project / ".zpkg.lock").read_bytes() + + warm = run( + zed_command(zed, registry, shared_home, "install", "--frozen"), + cwd=project, + env=environment, + ) + require_success(warm) + if parse_prefetch(warm.output) != (len(packages), 5, 0): + raise AssertionError(f"{repo}: warm frozen install downloaded bytes") + if (project / ".zpkg.lock").read_bytes() != lock_bytes: + raise AssertionError(f"{repo}: warm frozen install rewrote the lockfile") + if assert_symlink_graph(project, shared_home, install_dir, packages) != targets: + raise AssertionError(f"{repo}: warm frozen install changed store targets") + + first = packages[0] + materialized_path(project, install_dir, first).unlink() + repaired = run( + zed_command(zed, registry, shared_home, "install", "--frozen"), + cwd=project, + env=environment, + ) + require_success(repaired) + if parse_prefetch(repaired.output) != (len(packages), 5, 0): + raise AssertionError(f"{repo}: repair downloaded bytes") + assert_symlink_graph(project, shared_home, install_dir, packages) + assert_adapter(project, EXPECTED_ADAPTER[repo], install_dir, packages) + + uninstalled = run( + zed_command(zed, registry, shared_home, "uninstall"), + cwd=project, + env=environment, + ) + require_success(uninstalled) + assert_not_materialized(project, install_dir, packages) + assert_generated_wiring_absent(project) + if (project / ".zpkg.lock").read_bytes() != lock_bytes: + raise AssertionError(f"{repo}: uninstall rewrote the retained lockfile") + if (project / ".zpkg.toml").read_bytes() != manifest_bytes: + raise AssertionError(f"{repo}: uninstall rewrote the manifest") + assert_no_staging(project) + + restored = run( + zed_command(zed, registry, shared_home, "install", "--frozen"), + cwd=project, + env=environment, + ) + require_success(restored) + if parse_prefetch(restored.output) != (len(packages), 5, 0): + raise AssertionError(f"{repo}: post-uninstall restore downloaded bytes") + assert_symlink_graph(project, shared_home, install_dir, packages) + assert_adapter(project, EXPECTED_ADAPTER[repo], install_dir, packages) + assert_no_staging(project) + + return summary[2], {str(package["sha256"]) for package in packages} + + +def exercise_workspace( + *, + zed: Path, + registry: Path, + home: Path, + clone_root: Path, + workspace_root: Path, + clones: dict[str, Path], + environment: Mapping[str, str], +) -> None: + source = clone_repo(clone_root, "workspace-monorepo", clones) + + def expected_sources(repo_root: Path) -> dict[str, Path]: + return { + "ws-utils": repo_root / "packages" / "utils", + "ws-core": repo_root / "packages" / "core", + } + + def assert_projection(repo_root: Path, project: Path, mode: str) -> None: + expected = expected_sources(repo_root) + path_index = project / ".zed" / "paths.json" + if not path_index.is_file(): + raise AssertionError(f"workspace install omitted {path_index}") + index_text = path_index.read_text(encoding="utf-8") + json.loads(index_text) + for name, expected_source in expected.items(): + key = f"zedtest/{name}" + installed = project / "zed_modules" / "zedtest" / name + node_link = project / "node_modules" / "@zedtest" / name + for projection in (installed, node_link): + if mode == "symlink": + if not projection.is_symlink(): + raise AssertionError( + f"workspace dependency was not symlinked: {projection}" + ) + if projection.resolve(strict=True) != expected_source.resolve( + strict=True + ): + raise AssertionError( + f"workspace target mismatch: {projection} -> {projection.resolve()}" + ) + else: + if projection.is_symlink() or not projection.is_dir(): + raise AssertionError( + f"workspace dependency was not copied: {projection}" + ) + if not (projection / ".zpkg.toml").is_file(): + raise AssertionError( + f"workspace copied projection has no manifest: {projection}" + ) + if key not in index_text or '"version": "workspace"' not in index_text: + raise AssertionError(f"workspace path index omits {key}: {index_text}") + + node_path = project / ".zed" / "node_path" + if node_path.read_text(encoding="utf-8").strip() != "zed_modules": + raise AssertionError(f"unexpected workspace node_path: {node_path.read_text()!r}") + + symlink_repo = workspace_root / "symlink" + copy_fixture(source, symlink_repo) + symlink_project = symlink_repo / "apps" / "cli" + symlink_install = run( + zed_command(zed, registry, home, "install"), + cwd=symlink_project, + env=environment, + ) + require_success(symlink_install) + if PREFETCH_RE.search(symlink_install.output): + raise AssertionError( + "workspace-only install unexpectedly reported registry artifact prefetch" + ) + if parse_lock(symlink_project): + raise AssertionError("workspace packages must not be written as registry lock entries") + symlink_lock = (symlink_project / ".zpkg.lock").read_bytes() + assert_projection(symlink_repo, symlink_project, "symlink") + + live_sentinel = symlink_repo / "packages" / "core" / "live-sentinel.txt" + live_sentinel.write_text("visible after install\n", encoding="utf-8") + visible = ( + symlink_project + / "zed_modules" + / "zedtest" + / "ws-core" + / "live-sentinel.txt" + ) + if visible.read_text(encoding="utf-8") != "visible after install\n": + raise AssertionError("workspace symlink did not expose a post-install source edit") + + registry_before_frozen = tree_fingerprint(registry) + cache_before_frozen = tree_fingerprint(home / "cache") + for projection in [ + symlink_project / "zed_modules" / "zedtest" / "ws-utils", + symlink_project / "zed_modules" / "zedtest" / "ws-core", + symlink_project / "node_modules" / "@zedtest" / "ws-utils", + symlink_project / "node_modules" / "@zedtest" / "ws-core", + ]: + projection.unlink() + + frozen_repair = run( + zed_command(zed, registry, home, "install", "--frozen"), + cwd=symlink_project, + env=environment, + ) + require_success(frozen_repair) + if PREFETCH_RE.search(frozen_repair.output): + raise AssertionError("frozen workspace repair accessed registry artifacts") + if (symlink_project / ".zpkg.lock").read_bytes() != symlink_lock: + raise AssertionError("frozen workspace repair rewrote the empty artifact lock") + if tree_fingerprint(registry) != registry_before_frozen: + raise AssertionError("frozen workspace repair mutated the registry") + if tree_fingerprint(home / "cache") != cache_before_frozen: + raise AssertionError("frozen workspace repair mutated the artifact cache") + assert_projection(symlink_repo, symlink_project, "symlink") + + manifest_path = symlink_project / ".zpkg.toml" + manifest_before = manifest_path.read_text(encoding="utf-8") + incompatible = re.sub( + r'("zedtest/ws-utils"\s*=\s*)"[^"]+"', + r'\1"^2"', + manifest_before, + count=1, + ) + if incompatible == manifest_before: + raise AssertionError("could not rewrite workspace requirement for drift test") + manifest_path.write_text(incompatible, encoding="utf-8") + version_drift = run( + zed_command(zed, registry, home, "install", "--frozen"), + cwd=symlink_project, + env=environment, + ) + require_failure(version_drift, "ws-utils", "does not satisfy `^2`") + if (symlink_project / ".zpkg.lock").read_bytes() != symlink_lock: + raise AssertionError("workspace version-drift failure rewrote the lock") + assert_projection(symlink_repo, symlink_project, "symlink") + manifest_path.write_text(manifest_before, encoding="utf-8") + + symlink_uninstall = run( + zed_command(zed, registry, home, "uninstall"), + cwd=symlink_project, + env=environment, + ) + require_success(symlink_uninstall) + for projection in [ + symlink_project / "zed_modules" / "zedtest" / "ws-utils", + symlink_project / "zed_modules" / "zedtest" / "ws-core", + symlink_project / "node_modules" / "@zedtest" / "ws-utils", + symlink_project / "node_modules" / "@zedtest" / "ws-core", + ]: + if projection.exists() or projection.is_symlink(): + raise AssertionError(f"workspace uninstall left projection: {projection}") + assert_generated_wiring_absent(symlink_project) + assert_no_staging(symlink_project) + if (symlink_project / ".zpkg.lock").read_bytes() != symlink_lock: + raise AssertionError("workspace uninstall rewrote the retained lock") + + symlink_restore = run( + zed_command(zed, registry, home, "install", "--frozen"), + cwd=symlink_project, + env=environment, + ) + require_success(symlink_restore) + if PREFETCH_RE.search(symlink_restore.output): + raise AssertionError("workspace post-uninstall restore accessed registry artifacts") + if (symlink_project / ".zpkg.lock").read_bytes() != symlink_lock: + raise AssertionError("workspace post-uninstall restore rewrote the lock") + assert_projection(symlink_repo, symlink_project, "symlink") + assert_no_staging(symlink_project) + + copy_repo = workspace_root / "copy" + copy_fixture(source, copy_repo) + copy_source_sentinel = copy_repo / "packages" / "core" / "copy-sentinel.txt" + copy_source_sentinel.write_text("before install\n", encoding="utf-8") + copy_project = copy_repo / "apps" / "cli" + copy_install = run( + zed_command( + zed, + registry, + home, + "install", + "--install-mode", + "copy", + ), + cwd=copy_project, + env=environment, + ) + require_success(copy_install) + if PREFETCH_RE.search(copy_install.output): + raise AssertionError("workspace copy install accessed registry artifacts") + if parse_lock(copy_project): + raise AssertionError("workspace copy install wrote registry lock entries") + copy_lock = (copy_project / ".zpkg.lock").read_bytes() + assert_projection(copy_repo, copy_project, "copy") + + copied_sentinels = [ + copy_project / "zed_modules" / "zedtest" / "ws-core" / "copy-sentinel.txt", + copy_project + / "node_modules" + / "@zedtest" + / "ws-core" + / "copy-sentinel.txt", + ] + for copied_sentinel in copied_sentinels: + if copied_sentinel.read_text(encoding="utf-8") != "before install\n": + raise AssertionError(f"workspace copy omitted source content: {copied_sentinel}") + copy_source_sentinel.write_text("after install\n", encoding="utf-8") + for copied_sentinel in copied_sentinels: + if copied_sentinel.read_text(encoding="utf-8") != "before install\n": + raise AssertionError( + f"workspace copy remained coupled to mutable source: {copied_sentinel}" + ) + + for projection in [ + copy_project / "zed_modules" / "zedtest" / "ws-core", + copy_project / "node_modules" / "@zedtest" / "ws-core", + ]: + shutil.rmtree(projection) + copy_repair = run( + zed_command( + zed, + registry, + home, + "install", + "--frozen", + "--install-mode", + "copy", + ), + cwd=copy_project, + env=environment, + ) + require_success(copy_repair) + if PREFETCH_RE.search(copy_repair.output): + raise AssertionError("frozen workspace copy repair accessed registry artifacts") + if (copy_project / ".zpkg.lock").read_bytes() != copy_lock: + raise AssertionError("frozen workspace copy repair rewrote the lock") + assert_projection(copy_repo, copy_project, "copy") + leaked = [ + candidate + for root in (copy_project / "zed_modules", copy_project / "node_modules") + for candidate in root.rglob("*") + if candidate.is_symlink() + ] + if leaked: + raise AssertionError(f"workspace copy mode leaked symlinks: {leaked}") + + copy_uninstall = run( + zed_command(zed, registry, home, "uninstall"), + cwd=copy_project, + env=environment, + ) + require_success(copy_uninstall) + for projection in [ + copy_project / "zed_modules" / "zedtest" / "ws-utils", + copy_project / "zed_modules" / "zedtest" / "ws-core", + copy_project / "node_modules" / "@zedtest" / "ws-utils", + copy_project / "node_modules" / "@zedtest" / "ws-core", + ]: + if projection.exists() or projection.is_symlink(): + raise AssertionError(f"workspace copy uninstall left projection: {projection}") + assert_generated_wiring_absent(copy_project) + assert_no_staging(copy_project) + if (copy_project / ".zpkg.lock").read_bytes() != copy_lock: + raise AssertionError("workspace copy uninstall rewrote the retained lock") + + copy_restore = run( + zed_command( + zed, + registry, + home, + "install", + "--frozen", + "--install-mode", + "copy", + ), + cwd=copy_project, + env=environment, + ) + require_success(copy_restore) + if PREFETCH_RE.search(copy_restore.output): + raise AssertionError("workspace copy post-uninstall restore accessed registry artifacts") + if (copy_project / ".zpkg.lock").read_bytes() != copy_lock: + raise AssertionError("workspace copy post-uninstall restore rewrote the lock") + assert_projection(copy_repo, copy_project, "copy") + assert_no_staging(copy_project) + leaked = [ + candidate + for root in (copy_project / "zed_modules", copy_project / "node_modules") + for candidate in root.rglob("*") + if candidate.is_symlink() + ] + if leaked: + raise AssertionError(f"workspace restored copy mode leaked symlinks: {leaked}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + + zed = args.zed.resolve() + root = args.work_root.resolve() + if not zed.is_file(): + raise RuntimeError(f"zed binary does not exist: {zed}") + if root.exists(): + raise RuntimeError(f"work root must be fresh: {root}") + + registry = root / "registry" + publish_home = root / "publish-home" + shared_home = root / "shared-home" + clone_root = root / "clones" + app_root = root / "apps" + workspace_root = root / "workspace" + for directory in (registry, clone_root, app_root, workspace_root): + directory.mkdir(parents=True, exist_ok=True) + + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "RUST_BACKTRACE": "1", + "ZED_PKG_INSTALL_CONCURRENCY": "5", + } + ) + + clones: dict[str, Path] = {} + published_sources: set[tuple[str, str]] = set() + total_downloaded = 0 + unique_shas: set[str] = set() + per_app: dict[str, int] = {} + + for repo in APP_REPOS: + downloaded, shas = exercise_app( + repo, + zed=zed, + registry=registry, + shared_home=shared_home, + clone_root=clone_root, + app_root=app_root, + clones=clones, + published_sources=published_sources, + environment=environment, + ) + total_downloaded += downloaded + unique_shas.update(shas) + per_app[repo] = downloaded + + if total_downloaded != len(unique_shas): + raise AssertionError( + "shared-home app installs did not download each immutable hash exactly once: " + f"downloaded={total_downloaded}, unique_shas={len(unique_shas)}, per_app={per_app}" + ) + + exercise_workspace( + zed=zed, + registry=registry, + home=shared_home, + clone_root=clone_root, + workspace_root=workspace_root, + clones=clones, + environment=environment, + ) + + cache_files = sorted((shared_home / "cache").glob("*.tar.gz")) + if len(cache_files) != len(unique_shas): + raise AssertionError( + f"shared cache cardinality mismatch: {len(cache_files)} != {len(unique_shas)}" + ) + temporary = sorted((shared_home / "cache").glob("**/artifact.download")) + cache_directories = sorted( + path for path in (shared_home / "cache").iterdir() if path.is_dir() + ) + if temporary or cache_directories: + raise AssertionError( + f"shared cache leaked staging: downloads={temporary}, dirs={cache_directories}" + ) + + print("\nzed-pkg-test app install matrix passed", flush=True) + print(f"apps: {', '.join(APP_REPOS)}", flush=True) + print(f"cold downloads by app: {per_app}", flush=True) + print(f"unique immutable artifacts: {len(unique_shas)}", flush=True) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 - CI boundary must print full context. + print(f"error: {error}", file=sys.stderr, flush=True) + raise diff --git a/scripts/org_fixture_inventory.py b/scripts/org_fixture_inventory.py new file mode 100644 index 00000000..032a9e05 --- /dev/null +++ b/scripts/org_fixture_inventory.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 +"""Fail-closed inventory checks for every repository in the zed-pkg-test org. + +The static lane proves the checked-in inventory, lifecycle matrix, and lifecycle +package-source map agree. The live lane queries GitHub with read-only access and +proves that every public fixture repository still exists with the expected +shape and a parseable root manifest. +""" + +from __future__ import annotations + +import argparse +import base64 +import importlib.util +import json +import os +import re +import sys +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import asdict, dataclass +from pathlib import Path +from types import ModuleType +from typing import Any, Callable, Iterable, Mapping, Sequence + +INVENTORY_SCHEMA = "zed-pkg-test/repository-inventory-v1" +VALID_KINDS = {"package", "orchestrator"} +REPOSITORY_NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +API_VERSION = "2022-11-28" + + +class InventoryError(ValueError): + """Raised when checked-in or live fixture inventory is inconsistent.""" + + +@dataclass(frozen=True) +class RepositorySpec: + name: str + kind: str + manifest: str | None + + @property + def is_package(self) -> bool: + return self.kind == "package" + + +@dataclass(frozen=True) +class Inventory: + organization: str + orchestrator: str + repositories: tuple[RepositorySpec, ...] + + @property + def names(self) -> tuple[str, ...]: + return tuple(repo.name for repo in self.repositories) + + @property + def package_names(self) -> tuple[str, ...]: + return tuple(repo.name for repo in self.repositories if repo.is_package) + + def by_name(self) -> dict[str, RepositorySpec]: + return {repo.name: repo for repo in self.repositories} + + +@dataclass(frozen=True) +class LiveRepository: + name: str + private: bool + archived: bool + disabled: bool + fork: bool + default_branch: str + + +@dataclass(frozen=True) +class ManifestProbe: + repository: str + path: str + present: bool + package: str | None + version: str | None + + +JsonOpener = Callable[[urllib.request.Request], Any] + + +def _require_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise InventoryError(f"{field} must be a non-empty string") + return value + + +def load_inventory(path: Path) -> Inventory: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise InventoryError(f"cannot read inventory {path}: {error}") from error + if not isinstance(raw, dict): + raise InventoryError("inventory root must be an object") + if raw.get("schema") != INVENTORY_SCHEMA: + raise InventoryError( + f"inventory schema must be {INVENTORY_SCHEMA!r}, got {raw.get('schema')!r}" + ) + + organization = _require_string(raw.get("organization"), "organization") + orchestrator = _require_string(raw.get("orchestrator"), "orchestrator") + entries = raw.get("repositories") + if not isinstance(entries, list) or not entries: + raise InventoryError("repositories must be a non-empty array") + + repositories: list[RepositorySpec] = [] + seen: set[str] = set() + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise InventoryError(f"repositories[{index}] must be an object") + name = _require_string(entry.get("name"), f"repositories[{index}].name") + if not REPOSITORY_NAME_RE.fullmatch(name): + raise InventoryError(f"invalid repository name: {name!r}") + if name in seen: + raise InventoryError(f"duplicate repository: {name}") + seen.add(name) + + kind = _require_string(entry.get("kind"), f"repositories[{index}].kind") + if kind not in VALID_KINDS: + raise InventoryError(f"{name}: invalid kind {kind!r}") + + manifest = entry.get("manifest") + if kind == "package": + manifest = _require_string(manifest, f"{name}.manifest") + manifest_path = Path(manifest) + if manifest_path.is_absolute() or ".." in manifest_path.parts: + raise InventoryError(f"{name}: manifest path escapes repository") + elif manifest is not None: + raise InventoryError(f"{name}: orchestrator manifest must be null") + + repositories.append( + RepositorySpec(name=name, kind=kind, manifest=manifest) + ) + + orchestrators = [repo.name for repo in repositories if repo.kind == "orchestrator"] + if orchestrators != [orchestrator]: + raise InventoryError( + "inventory must contain exactly the declared orchestrator; " + f"declared={orchestrator!r}, classified={orchestrators!r}" + ) + + return Inventory( + organization=organization, + orchestrator=orchestrator, + repositories=tuple(repositories), + ) + + +def extract_lifecycle_repositories(workflow_text: str) -> tuple[str, ...]: + """Extract the `strategy.matrix.repo` sequence from lifecycle.yml. + + This deliberately supports only the narrow checked-in shape. A workflow + refactor that makes coverage ambiguous must update this parser and tests + rather than silently skipping repositories. + """ + + lines = workflow_text.splitlines() + matrix_indent: int | None = None + repo_indent: int | None = None + repositories: list[str] = [] + + for line in lines: + stripped = line.strip() + indent = len(line) - len(line.lstrip(" ")) + if stripped == "matrix:": + matrix_indent = indent + repo_indent = None + repositories = [] + continue + if matrix_indent is None: + continue + if stripped and indent <= matrix_indent: + matrix_indent = None + repo_indent = None + continue + if stripped == "repo:" and indent > matrix_indent: + repo_indent = indent + continue + if repo_indent is None: + continue + if stripped and indent <= repo_indent: + break + match = re.fullmatch(r"-\s+([A-Za-z0-9_.-]+)", stripped) + if match: + repositories.append(match.group(1)) + elif stripped and not stripped.startswith("#"): + raise InventoryError( + f"unsupported lifecycle repo matrix entry: {stripped!r}" + ) + + if not repositories: + raise InventoryError("cannot find strategy.matrix.repo in lifecycle workflow") + if len(repositories) != len(set(repositories)): + raise InventoryError("lifecycle repo matrix contains duplicates") + return tuple(repositories) + + +def load_lifecycle_module(path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location("zed_inventory_lifecycle", path) + if spec is None or spec.loader is None: + raise InventoryError(f"cannot load lifecycle module from {path}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def validate_static_inventory( + inventory: Inventory, + lifecycle_repositories: Sequence[str], + package_sources: Mapping[str, tuple[str, str]], + non_package_repositories: Iterable[str], +) -> list[str]: + errors: list[str] = [] + expected = set(inventory.names) + matrix = set(lifecycle_repositories) + + missing_from_matrix = sorted(expected - matrix) + extra_in_matrix = sorted(matrix - expected) + if missing_from_matrix: + errors.append( + "repositories missing from lifecycle matrix: " + + ", ".join(missing_from_matrix) + ) + if extra_in_matrix: + errors.append( + "unknown repositories in lifecycle matrix: " + ", ".join(extra_in_matrix) + ) + + expected_non_packages = {inventory.orchestrator} + actual_non_packages = set(non_package_repositories) + if actual_non_packages != expected_non_packages: + errors.append( + "NON_PACKAGE_REPOS drift: " + f"expected={sorted(expected_non_packages)!r}, " + f"actual={sorted(actual_non_packages)!r}" + ) + + package_repositories = set(inventory.package_names) + source_repositories = {repo for repo, _relative in package_sources.values()} + unknown_source_repositories = sorted(source_repositories - package_repositories) + if unknown_source_repositories: + errors.append( + "PACKAGE_SOURCES references unclassified repositories: " + + ", ".join(unknown_source_repositories) + ) + + for package, source in package_sources.items(): + if not re.fullmatch(r"[^/\s]+/[^/\s]+", package): + errors.append(f"invalid PACKAGE_SOURCES package identity: {package!r}") + if not isinstance(source, tuple) or len(source) != 2: + errors.append(f"{package}: source must be a (repository, path) tuple") + continue + repo, relative = source + relative_path = Path(relative) + if relative_path.is_absolute() or ".." in relative_path.parts: + errors.append(f"{package}: source path escapes repository: {relative!r}") + if repo == inventory.orchestrator: + errors.append(f"{package}: orchestrator cannot be a package source") + + return errors + + +def _headers(token: str | None) -> dict[str, str]: + result = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": API_VERSION, + "User-Agent": "zed-pkg-test-inventory/1", + } + if token: + result["Authorization"] = f"Bearer {token}" + return result + + +def fetch_json( + url: str, + *, + token: str | None = None, + opener: JsonOpener = urllib.request.urlopen, +) -> Any: + request = urllib.request.Request(url, headers=_headers(token)) + try: + with opener(request) as response: + payload = response.read() + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8", errors="replace") + raise InventoryError( + f"GitHub API request failed ({error.code}) for {url}: {body[:300]}" + ) from error + except urllib.error.URLError as error: + raise InventoryError(f"GitHub API request failed for {url}: {error}") from error + try: + return json.loads(payload) + except json.JSONDecodeError as error: + raise InventoryError(f"GitHub API returned invalid JSON for {url}") from error + + +def fetch_live_repositories( + organization: str, + *, + token: str | None = None, + opener: JsonOpener = urllib.request.urlopen, +) -> tuple[LiveRepository, ...]: + encoded = urllib.parse.quote(organization, safe="") + url = ( + f"https://api.github.com/orgs/{encoded}/repos" + "?type=public&per_page=100&sort=full_name&direction=asc" + ) + payload = fetch_json(url, token=token, opener=opener) + if not isinstance(payload, list): + raise InventoryError("organization repositories response must be an array") + + repositories: list[LiveRepository] = [] + for item in payload: + if not isinstance(item, dict): + raise InventoryError("organization repository entry must be an object") + repositories.append( + LiveRepository( + name=_require_string(item.get("name"), "live repository name"), + private=bool(item.get("private")), + archived=bool(item.get("archived")), + disabled=bool(item.get("disabled")), + fork=bool(item.get("fork")), + default_branch=_require_string( + item.get("default_branch"), "live repository default_branch" + ), + ) + ) + return tuple(repositories) + + +def validate_live_repositories( + inventory: Inventory, live_repositories: Sequence[LiveRepository] +) -> list[str]: + errors: list[str] = [] + expected = set(inventory.names) + live_names = {repo.name for repo in live_repositories} + + missing = sorted(expected - live_names) + unexpected = sorted(live_names - expected) + if missing: + errors.append("live repositories missing from org: " + ", ".join(missing)) + if unexpected: + errors.append( + "unclassified live repositories in org: " + ", ".join(unexpected) + ) + + for repo in live_repositories: + if repo.name not in expected: + continue + if repo.private: + errors.append(f"{repo.name}: fixture repository must be public") + if repo.archived: + errors.append(f"{repo.name}: fixture repository must not be archived") + if repo.disabled: + errors.append(f"{repo.name}: fixture repository must not be disabled") + if repo.fork: + errors.append(f"{repo.name}: fixture repository must not be a fork") + if repo.default_branch != "main": + errors.append( + f"{repo.name}: default branch must be main, got {repo.default_branch!r}" + ) + return errors + + +def decode_manifest_payload(repository: str, path: str, payload: object) -> dict[str, Any]: + if not isinstance(payload, dict): + raise InventoryError(f"{repository}:{path}: contents response must be an object") + if payload.get("type") != "file": + raise InventoryError(f"{repository}:{path}: manifest is not a regular file") + if payload.get("encoding") != "base64": + raise InventoryError( + f"{repository}:{path}: expected base64 contents encoding" + ) + content = payload.get("content") + if not isinstance(content, str): + raise InventoryError(f"{repository}:{path}: manifest content is missing") + try: + decoded = base64.b64decode(content, validate=False) + manifest = tomllib.loads(decoded.decode("utf-8")) + except (ValueError, UnicodeDecodeError, tomllib.TOMLDecodeError) as error: + raise InventoryError(f"{repository}:{path}: invalid TOML: {error}") from error + if not isinstance(manifest, dict): + raise InventoryError(f"{repository}:{path}: manifest root must be a table") + package = manifest.get("package") + if not isinstance(package, dict): + raise InventoryError(f"{repository}:{path}: missing [package] table") + for field in ("org", "name", "version"): + _require_string(package.get(field), f"{repository}:{path}: package.{field}") + return manifest + + +def fetch_manifest_probe( + organization: str, + spec: RepositorySpec, + *, + token: str | None = None, + opener: JsonOpener = urllib.request.urlopen, +) -> ManifestProbe: + if spec.manifest is None: + return ManifestProbe( + repository=spec.name, + path="", + present=False, + package=None, + version=None, + ) + + org = urllib.parse.quote(organization, safe="") + repo = urllib.parse.quote(spec.name, safe="") + path = urllib.parse.quote(spec.manifest, safe="/") + url = f"https://api.github.com/repos/{org}/{repo}/contents/{path}?ref=main" + payload = fetch_json(url, token=token, opener=opener) + manifest = decode_manifest_payload(spec.name, spec.manifest, payload) + package = manifest["package"] + return ManifestProbe( + repository=spec.name, + path=spec.manifest, + present=True, + package=f"{package['org']}/{package['name']}", + version=package["version"], + ) + + +def build_report( + inventory: Inventory, + lifecycle_repositories: Sequence[str], + live_repositories: Sequence[LiveRepository] | None, + manifest_probes: Sequence[ManifestProbe], +) -> dict[str, Any]: + return { + "schema": INVENTORY_SCHEMA, + "organization": inventory.organization, + "orchestrator": inventory.orchestrator, + "inventory_repository_count": len(inventory.repositories), + "package_repository_count": len(inventory.package_names), + "lifecycle_repository_count": len(lifecycle_repositories), + "live_repository_count": ( + len(live_repositories) if live_repositories is not None else None + ), + "repositories": [asdict(repo) for repo in inventory.repositories], + "manifests": [asdict(probe) for probe in manifest_probes], + } + + +def run(args: argparse.Namespace) -> dict[str, Any]: + inventory = load_inventory(args.inventory) + workflow_text = args.workflow.read_text(encoding="utf-8") + lifecycle_repositories = extract_lifecycle_repositories(workflow_text) + lifecycle = load_lifecycle_module(args.lifecycle) + + static_errors = validate_static_inventory( + inventory, + lifecycle_repositories, + lifecycle.PACKAGE_SOURCES, + lifecycle.NON_PACKAGE_REPOS, + ) + if static_errors: + raise InventoryError("\n".join(static_errors)) + + live_repositories: tuple[LiveRepository, ...] | None = None + manifest_probes: list[ManifestProbe] = [] + if args.live: + token = os.environ.get("GITHUB_TOKEN") or None + live_repositories = fetch_live_repositories( + inventory.organization, + token=token, + ) + live_errors = validate_live_repositories(inventory, live_repositories) + if live_errors: + raise InventoryError("\n".join(live_errors)) + + for spec in inventory.repositories: + if spec.is_package: + manifest_probes.append( + fetch_manifest_probe( + inventory.organization, + spec, + token=token, + ) + ) + else: + manifest_probes.append( + ManifestProbe( + repository=spec.name, + path="", + present=False, + package=None, + version=None, + ) + ) + + report = build_report( + inventory, + lifecycle_repositories, + live_repositories, + manifest_probes, + ) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--inventory", + type=Path, + default=Path("fixtures/org-repositories.json"), + ) + parser.add_argument( + "--workflow", + type=Path, + default=Path(".github/workflows/lifecycle.yml"), + ) + parser.add_argument( + "--lifecycle", + type=Path, + default=Path("scripts/lifecycle.py"), + ) + parser.add_argument( + "--live", + action="store_true", + help="query the public GitHub organization and validate root manifests", + ) + parser.add_argument("--output", type=Path) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + try: + report = run(args) + except (InventoryError, OSError, AttributeError) as error: + print(f"inventory check failed: {error}", file=sys.stderr) + return 1 + print( + "inventory check passed: " + f"{report['inventory_repository_count']} repositories, " + f"{report['package_repository_count']} package fixtures, " + f"{report['lifecycle_repository_count']} lifecycle lanes" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/org_fleet_contract.py b/scripts/org_fleet_contract.py new file mode 100644 index 00000000..da005bc6 --- /dev/null +++ b/scripts/org_fleet_contract.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Run the public-fleet audit with reviewed Zed package namespace aliases. + +The GitHub organization and Zed registry package organization are distinct +identities. Legacy fixtures intentionally use the short registry namespace +``zedtest`` while newer fixtures use ``zed-pkg-test``. This wrapper reads the +explicit allowlist from the reviewed inventory, preserves each manifest's +actual package organization in evidence, and rejects any unreviewed namespace. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import replace +from pathlib import Path +from typing import Any, Callable, Sequence + +import org_fleet_inventory as fleet + +REPORT_SCHEMA = "zed-pkg-test/public-fleet-inventory-v3" + + +def load_package_organizations(path: Path, github_organization: str) -> tuple[str, ...]: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise fleet.FleetError(f"cannot read package organizations from {path}: {error}") from error + if not isinstance(raw, dict): + raise fleet.FleetError("inventory root must be an object") + values = raw.get("package_organizations") + if not isinstance(values, list) or not values: + raise fleet.FleetError("package_organizations must be a non-empty array") + + organizations: list[str] = [] + for index, value in enumerate(values): + if not isinstance(value, str) or not value.strip(): + raise fleet.FleetError( + f"package_organizations[{index}] must be a non-empty string" + ) + if value in organizations: + raise fleet.FleetError(f"duplicate package organization: {value}") + organizations.append(value) + + if github_organization not in organizations: + raise fleet.FleetError( + "package_organizations must include the GitHub organization " + f"{github_organization!r}" + ) + return tuple(organizations) + + +def audit_with_package_organizations( + github_organization: str, + package_organizations: Sequence[str], + core: dict[str, fleet.CoreSpec], + live_repositories: Sequence[fleet.LiveRepository], + load_manifest: fleet.ManifestLoader, +) -> dict[str, Any]: + allowed = frozenset(package_organizations) + actual_organizations: dict[tuple[str, str], str | None] = {} + + def reviewed_loader( + repository: fleet.LiveRepository, + path: str, + ) -> fleet.ManifestProbe: + probe = load_manifest(repository, path) + actual_organizations[(repository.name, path)] = probe.package_org + if probe.present and probe.package_org in allowed: + # The base auditor expects the GitHub org as its canonical namespace. + # Normalize only for validation, then restore the actual value below. + return replace(probe, package_org=github_organization) + return probe + + report = fleet.audit_fleet( + github_organization, + core, + live_repositories, + reviewed_loader, + ) + report["schema"] = REPORT_SCHEMA + report["package_organizations"] = list(package_organizations) + + for row in report["repositories"]: + manifest = row["manifest"] + key = (str(row["name"]), str(manifest["path"])) + actual = actual_organizations.get(key) + manifest["package_org"] = actual + manifest["package_org_allowed"] = actual in allowed if manifest["present"] else None + + return report + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--inventory", + type=Path, + default=Path("fixtures/org-repositories.json"), + help="reviewed lifecycle and package-namespace inventory", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--token-env", + default="GITHUB_TOKEN", + help="environment variable containing a read-only GitHub token", + ) + return parser.parse_args(argv) + + +def main( + argv: Sequence[str] | None = None, + *, + repository_fetcher: Callable[..., tuple[fleet.LiveRepository, ...]] = fleet.fetch_public_repositories, + manifest_loader_factory: Callable[..., fleet.ManifestLoader] = fleet.make_github_manifest_loader, +) -> int: + args = parse_args(argv if argv is not None else sys.argv[1:]) + try: + github_organization, core = fleet.load_core_inventory(args.inventory) + package_organizations = load_package_organizations( + args.inventory, github_organization + ) + token = os.environ.get(args.token_env) or None + live = repository_fetcher(github_organization, token=token) + loader = manifest_loader_factory(github_organization, token=token) + report = audit_with_package_organizations( + github_organization, + package_organizations, + core, + live, + loader, + ) + except fleet.FleetError as error: + print(f"fleet contract failed: {error}", file=sys.stderr) + return 2 + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + if report["errors"]: + print("fleet contract failed:", file=sys.stderr) + for error in report["errors"]: + print(f"- {error}", file=sys.stderr) + return 1 + print( + "fleet contract passed: " + f"{report['live_repository_count']} public repositories, " + f"{report['manifest_present_count']} valid manifests, " + f"package organizations={','.join(report['package_organizations'])}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/org_fleet_inventory.py b/scripts/org_fleet_inventory.py new file mode 100644 index 00000000..748346b1 --- /dev/null +++ b/scripts/org_fleet_inventory.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Audit the complete public zed-pkg-test repository fleet. + +The checked-in core inventory remains the reviewed lifecycle contract. This +module adds a second, growth-safe live audit that discovers every public repo, +classifies supplemental package fixtures versus harness/governance repos, and +validates all required or present root Zed package manifests. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import sys +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +CORE_SCHEMA = "zed-pkg-test/repository-inventory-v1" +REPORT_SCHEMA = "zed-pkg-test/public-fleet-inventory-v2" +API_VERSION = "2022-11-28" +NAME_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +VALID_CORE_KINDS = {"package", "orchestrator"} + + +class FleetError(ValueError): + """Raised when the inventory or a GitHub response is malformed.""" + + +@dataclass(frozen=True) +class CoreSpec: + name: str + kind: str + manifest: str | None + + @property + def requires_manifest(self) -> bool: + return self.kind == "package" + + +@dataclass(frozen=True) +class LiveRepository: + name: str + private: bool = False + archived: bool = False + disabled: bool = False + fork: bool = False + default_branch: str = "main" + + +@dataclass(frozen=True) +class ManifestProbe: + repository: str + path: str + present: bool + package_org: str | None = None + package_name: str | None = None + version: str | None = None + + +ManifestLoader = Callable[[LiveRepository, str], ManifestProbe] +UrlOpener = Callable[[urllib.request.Request], Any] + + +def _non_empty_string(value: object, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise FleetError(f"{field} must be a non-empty string") + return value + + +def load_core_inventory(path: Path) -> tuple[str, dict[str, CoreSpec]]: + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise FleetError(f"cannot read core inventory {path}: {error}") from error + if not isinstance(raw, dict): + raise FleetError("core inventory root must be an object") + if raw.get("schema") != CORE_SCHEMA: + raise FleetError( + f"core inventory schema must be {CORE_SCHEMA!r}, got {raw.get('schema')!r}" + ) + organization = _non_empty_string(raw.get("organization"), "organization") + orchestrator = _non_empty_string(raw.get("orchestrator"), "orchestrator") + entries = raw.get("repositories") + if not isinstance(entries, list) or not entries: + raise FleetError("repositories must be a non-empty array") + + result: dict[str, CoreSpec] = {} + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise FleetError(f"repositories[{index}] must be an object") + name = _non_empty_string(entry.get("name"), f"repositories[{index}].name") + if not NAME_RE.fullmatch(name): + raise FleetError(f"invalid repository name: {name!r}") + if name in result: + raise FleetError(f"duplicate core repository: {name}") + kind = _non_empty_string(entry.get("kind"), f"{name}.kind") + if kind not in VALID_CORE_KINDS: + raise FleetError(f"{name}: invalid core kind {kind!r}") + manifest = entry.get("manifest") + if kind == "package": + manifest = _non_empty_string(manifest, f"{name}.manifest") + manifest_path = Path(manifest) + if manifest_path.is_absolute() or ".." in manifest_path.parts: + raise FleetError(f"{name}: manifest path escapes repository") + elif manifest is not None: + raise FleetError(f"{name}: orchestrator manifest must be null") + result[name] = CoreSpec(name=name, kind=kind, manifest=manifest) + + orchestrators = [item.name for item in result.values() if item.kind == "orchestrator"] + if orchestrators != [orchestrator]: + raise FleetError( + "core inventory must contain exactly the declared orchestrator; " + f"declared={orchestrator!r}, classified={orchestrators!r}" + ) + return organization, result + + +def classify_repository(name: str, core: Mapping[str, CoreSpec]) -> str: + spec = core.get(name) + if spec is not None: + return f"core-{spec.kind}" + if name == ".github": + return "governance" + if name.endswith("-e2e") or name.endswith("-contract"): + return "harness" + return "supplemental-package" + + +def manifest_requirement( + repository: LiveRepository, + core: Mapping[str, CoreSpec], +) -> tuple[str, bool]: + spec = core.get(repository.name) + if spec is not None: + return spec.manifest or ".zpkg.toml", spec.requires_manifest + category = classify_repository(repository.name, core) + return ".zpkg.toml", category == "supplemental-package" + + +def audit_fleet( + organization: str, + core: Mapping[str, CoreSpec], + live_repositories: Sequence[LiveRepository], + load_manifest: ManifestLoader, +) -> dict[str, Any]: + errors: list[str] = [] + rows: list[dict[str, Any]] = [] + + live_names = [repo.name for repo in live_repositories] + duplicate_names = sorted({name for name in live_names if live_names.count(name) > 1}) + if duplicate_names: + errors.append("duplicate live repositories: " + ", ".join(duplicate_names)) + + missing_core = sorted(set(core) - set(live_names)) + if missing_core: + errors.append("core repositories missing from public org: " + ", ".join(missing_core)) + + required_manifest_count = 0 + required_manifest_present_count = 0 + manifest_present_count = 0 + + for repo in sorted(live_repositories, key=lambda item: item.name): + category = classify_repository(repo.name, core) + repo_errors: list[str] = [] + if repo.private: + repo_errors.append("repository must be public") + if repo.archived: + repo_errors.append("repository must not be archived") + if repo.disabled: + repo_errors.append("repository must not be disabled") + if repo.fork: + repo_errors.append("repository must not be a fork") + if repo.default_branch != "main": + repo_errors.append( + f"default branch must be main, got {repo.default_branch!r}" + ) + + manifest_path, manifest_required = manifest_requirement(repo, core) + if manifest_required: + required_manifest_count += 1 + try: + probe = load_manifest(repo, manifest_path) + except FleetError as error: + probe = ManifestProbe(repo.name, manifest_path, False) + repo_errors.append(str(error)) + + if manifest_required and not probe.present: + repo_errors.append(f"required manifest is missing: {manifest_path}") + if probe.present: + manifest_present_count += 1 + if manifest_required: + required_manifest_present_count += 1 + if probe.package_org != organization: + repo_errors.append( + "manifest package.org mismatch: " + f"expected {organization!r}, got {probe.package_org!r}" + ) + if probe.package_name != repo.name: + repo_errors.append( + "manifest package.name mismatch: " + f"expected {repo.name!r}, got {probe.package_name!r}" + ) + + for message in repo_errors: + errors.append(f"{repo.name}: {message}") + rows.append( + { + "name": repo.name, + "category": category, + "core": repo.name in core, + "private": repo.private, + "archived": repo.archived, + "disabled": repo.disabled, + "fork": repo.fork, + "default_branch": repo.default_branch, + "manifest_required": manifest_required, + "manifest": asdict(probe), + "errors": repo_errors, + } + ) + + category_counts: dict[str, int] = {} + for row in rows: + category = str(row["category"]) + category_counts[category] = category_counts.get(category, 0) + 1 + + return { + "schema": REPORT_SCHEMA, + "organization": organization, + "core_repository_count": len(core), + "core_package_count": sum(spec.requires_manifest for spec in core.values()), + "live_repository_count": len(live_repositories), + "supplemental_repository_count": len(set(live_names) - set(core)), + "missing_core_repositories": missing_core, + "category_counts": dict(sorted(category_counts.items())), + "required_manifest_count": required_manifest_count, + "required_manifest_present_count": required_manifest_present_count, + "manifest_present_count": manifest_present_count, + "repositories": rows, + "errors": errors, + } + + +def _headers(token: str | None) -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": API_VERSION, + "User-Agent": "zed-pkg-test-fleet-inventory/2", + } + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def _read_json_response( + request: urllib.request.Request, + *, + opener: UrlOpener, + allow_not_found: bool = False, +) -> tuple[Any | None, Mapping[str, str]]: + try: + with opener(request) as response: + payload = response.read() + headers = response.headers + except urllib.error.HTTPError as error: + if allow_not_found and error.code == 404: + return None, {} + body = error.read().decode("utf-8", errors="replace") + raise FleetError( + f"GitHub API request failed ({error.code}) for {request.full_url}: {body[:300]}" + ) from error + except urllib.error.URLError as error: + raise FleetError( + f"GitHub API request failed for {request.full_url}: {error}" + ) from error + try: + return json.loads(payload), headers + except json.JSONDecodeError as error: + raise FleetError( + f"GitHub API returned invalid JSON for {request.full_url}" + ) from error + + +def fetch_public_repositories( + organization: str, + *, + token: str | None = None, + opener: UrlOpener = urllib.request.urlopen, +) -> tuple[LiveRepository, ...]: + encoded = urllib.parse.quote(organization, safe="") + repositories: list[LiveRepository] = [] + page = 1 + while True: + url = ( + f"https://api.github.com/orgs/{encoded}/repos" + f"?type=public&per_page=100&page={page}&sort=full_name&direction=asc" + ) + request = urllib.request.Request(url, headers=_headers(token)) + payload, _headers_map = _read_json_response(request, opener=opener) + if not isinstance(payload, list): + raise FleetError("organization repositories response must be an array") + for item in payload: + if not isinstance(item, dict): + raise FleetError("organization repository entry must be an object") + repositories.append( + LiveRepository( + name=_non_empty_string(item.get("name"), "live repository name"), + private=bool(item.get("private")), + archived=bool(item.get("archived")), + disabled=bool(item.get("disabled")), + fork=bool(item.get("fork")), + default_branch=_non_empty_string( + item.get("default_branch"), "live repository default_branch" + ), + ) + ) + if len(payload) < 100: + break + page += 1 + return tuple(repositories) + + +def decode_manifest( + repository: str, + path: str, + payload: object, +) -> ManifestProbe: + if not isinstance(payload, dict): + raise FleetError(f"{path}: contents response must be an object") + if payload.get("type") != "file": + raise FleetError(f"{path}: manifest is not a regular file") + if payload.get("encoding") != "base64": + raise FleetError(f"{path}: expected base64 contents encoding") + content = payload.get("content") + if not isinstance(content, str): + raise FleetError(f"{path}: manifest content is missing") + try: + decoded = base64.b64decode(content, validate=False).decode("utf-8") + manifest = tomllib.loads(decoded) + except (ValueError, UnicodeDecodeError, tomllib.TOMLDecodeError) as error: + raise FleetError(f"{path}: invalid TOML: {error}") from error + package = manifest.get("package") if isinstance(manifest, dict) else None + if not isinstance(package, dict): + raise FleetError(f"{path}: missing [package] table") + package_org = _non_empty_string(package.get("org"), f"{path}: package.org") + package_name = _non_empty_string(package.get("name"), f"{path}: package.name") + version = _non_empty_string(package.get("version"), f"{path}: package.version") + return ManifestProbe( + repository=repository, + path=path, + present=True, + package_org=package_org, + package_name=package_name, + version=version, + ) + + +def make_github_manifest_loader( + organization: str, + *, + token: str | None = None, + opener: UrlOpener = urllib.request.urlopen, +) -> ManifestLoader: + encoded_org = urllib.parse.quote(organization, safe="") + + def load(repository: LiveRepository, path: str) -> ManifestProbe: + encoded_repo = urllib.parse.quote(repository.name, safe="") + encoded_path = urllib.parse.quote(path, safe="/") + url = ( + f"https://api.github.com/repos/{encoded_org}/{encoded_repo}/contents/" + f"{encoded_path}?ref=main" + ) + request = urllib.request.Request(url, headers=_headers(token)) + payload, _response_headers = _read_json_response( + request, + opener=opener, + allow_not_found=True, + ) + if payload is None: + return ManifestProbe(repository.name, path, False) + return decode_manifest(repository.name, path, payload) + + return load + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--inventory", + type=Path, + default=Path("fixtures/org-repositories.json"), + help="reviewed core lifecycle inventory", + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--token-env", + default="GITHUB_TOKEN", + help="environment variable containing a read-only GitHub token", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv if argv is not None else sys.argv[1:]) + try: + organization, core = load_core_inventory(args.inventory) + token = os.environ.get(args.token_env) or None + live = fetch_public_repositories(organization, token=token) + loader = make_github_manifest_loader(organization, token=token) + report = audit_fleet(organization, core, live, loader) + except FleetError as error: + print(f"fleet inventory failed: {error}", file=sys.stderr) + return 2 + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + if report["errors"]: + print("fleet inventory failed:", file=sys.stderr) + for error in report["errors"]: + print(f"- {error}", file=sys.stderr) + return 1 + print( + "fleet inventory passed: " + f"{report['live_repository_count']} public repositories, " + f"{report['manifest_present_count']} valid manifests, " + f"{report['supplemental_repository_count']} supplemental repositories" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/procedure_contract.py b/scripts/procedure_contract.py new file mode 100644 index 00000000..0d84e9ee --- /dev/null +++ b/scripts/procedure_contract.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Fail-closed policy checks for the reusable zed-pkg-test candidate gate.""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import Iterable + +FULL_SHA = r"[0-9a-f]{40}" +REMOTE_USE = re.compile(r"(?m)^\s*(?:-\s*)?uses:\s*([^\s#]+)") + + +class ContractViolation(AssertionError): + """Raised when certification procedure code drifts from its trust boundary.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ContractViolation(message) + + +def require_all(text: str, needles: Iterable[str], label: str) -> None: + for needle in needles: + require(needle in text, f"{label} is missing required contract text: {needle}") + + +def audit_workflow(text: str) -> None: + """Audit the reusable workflow without evaluating pull-request code.""" + + require("workflow_call:" in text, "candidate workflow is no longer reusable") + require("pull_request_target:" not in text, "pull_request_target is forbidden") + require( + re.search(r"(?m)^permissions:\s*$\n\s{2}contents:\s*read\s*$", text) + is not None, + "workflow must declare top-level contents: read", + ) + require( + re.search(r"(?m)^\s{2}[a-zA-Z0-9_-]+:\s*write\s*$", text) is None, + "workflow may not request any write permission", + ) + require("secrets: inherit" not in text, "reusable caller secrets may not be inherited") + require("${{ secrets." not in text, "candidate workflow may not read repository secrets") + require("persist-credentials: true" not in text, "checkout credentials may not persist") + + uses = REMOTE_USE.findall(text) + require(uses, "candidate workflow has no auditable uses entries") + for target in uses: + if target.startswith("./") or target.startswith("docker://"): + continue + require("@" in target, f"remote action/workflow is unpinned: {target}") + ref = target.rsplit("@", 1)[1] + require( + re.fullmatch(FULL_SHA, ref) is not None, + f"remote action/workflow must use an exact commit: {target}", + ) + + checkout_count = sum("actions/checkout@" in target for target in uses) + require(checkout_count > 0, "candidate workflow must check out reviewed inputs") + require( + text.count("persist-credentials: false") >= checkout_count, + "every checkout must explicitly disable persisted credentials", + ) + + require_all( + text, + ( + '[[ "$ZED_CLI_REF" =~ ^[0-9a-f]{40}$ ]]', + '[[ "$HARNESS_REF" =~ ^[0-9a-f]{40}$ ]]', + '[[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{40}$ ]]', + 're.fullmatch(r"[0-9a-f]{40}", ref)', + "fixture_refs=", + "${{ matrix.fixture.ref }}", + "${{ env.HARNESS_REF }}", + "${{ env.ZED_CLI_REF }}", + "${{ env.ZED_INTERFACES_REF }}", + "scripts/candidate_lifecycle.py", + "--fixture-refs-json", + "sha256sum --check SHA256SUMS", + ), + "candidate workflow", + ) + + +def audit_wrapper(text: str) -> None: + """Audit the exact-ref dependency wrapper used by fixture jobs.""" + + require_all( + text, + ( + 're.fullmatch(r"[0-9a-f]{40}", ref)', + "root fixture", + "dependency fixture", + '"fetch",', + '"--depth",', + '"--no-tags",', + '"checkout", "--detach", "FETCH_HEAD"', + '"rev-parse", "HEAD"', + ), + "candidate lifecycle wrapper", + ) + require( + '"clone",\n "--depth"' not in text, + "candidate dependency fetch must not clone a mutable default branch", + ) + require("ref: main" not in text, "candidate wrapper may not contain a mutable main ref") + + +def audit_documentation(text: str) -> None: + """Keep human procedure text aligned with the executable gate.""" + + require_all( + text, + ( + "Two gates, two purposes", + "exact 40-character `zed-cli` commit", + "exact 40-character `zed-pkg-e2e` harness commit", + "never follows a fixture default branch", + "Full candidate certification", + ".github/workflows/lifecycle.yml", + ".github/workflows/e2e.yml", + ".github/workflows/install-boundaries.yml", + "same immutable candidate and dependency graph", + "zed-pkg-test certification runbook", + "product regression, fixture drift, harness defect, or", + ), + "candidate validation documentation", + ) + + +def audit_repository(root: Path) -> None: + workflow = root / ".github/workflows/candidate-smoke.yml" + wrapper = root / "scripts/candidate_lifecycle.py" + documentation = root / "docs/candidate-validation.md" + for path in (workflow, wrapper, documentation): + require(path.is_file(), f"required procedure file is missing: {path.relative_to(root)}") + + audit_workflow(workflow.read_text(encoding="utf-8")) + audit_wrapper(wrapper.read_text(encoding="utf-8")) + audit_documentation(documentation.read_text(encoding="utf-8")) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + audit_repository(args.root.resolve()) + except ContractViolation as error: + print(f"zed-pkg-test procedure contract failed: {error}", file=sys.stderr) + return 1 + print("zed-pkg-test candidate procedure contract passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/production_package_common.py b/scripts/production_package_common.py new file mode 100644 index 00000000..358b3c28 --- /dev/null +++ b/scripts/production_package_common.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + +SHA_RE_LENGTH = 40 +MIN_PACKAGES = 15 +MAX_PACKAGES = 25 + + +class CertificationError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Package: + github_owner: str + repo: str + package: str + org: str + name: str + version: str + sha: str + tag: str | None + dependencies: tuple[str, ...] + allow_repo_name_mismatch: bool + + @property + def source_url(self) -> str: + return f"https://github.com/{self.github_owner}/{self.repo}.git" + + @property + def spec(self) -> str: + return f"{self.package}@{self.version}" + + +@dataclass(frozen=True) +class RegistryPackage: + logical_package: str + org: str + name: str + version: str + target: str | None + + @property + def package(self) -> str: + return f"{self.org}/{self.name}" + + @property + def spec(self) -> str: + return f"{self.package}@{self.version}" + + +@dataclass +class CommandResult: + command: list[str] + returncode: int + stdout: str + stderr: str + duration_seconds: float + + +def command_display(command: Iterable[str]) -> str: + return " ".join(subprocess.list2cmdline([part]) for part in command) + + +def run( + command: list[str], + *, + cwd: Path | None = None, + env: dict[str, str] | None = None, + check: bool = True, + timeout: int = 1800, +) -> CommandResult: + started = time.monotonic() + completed = subprocess.run( + command, + cwd=cwd, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + check=False, + ) + result = CommandResult( + command=command, + returncode=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + duration_seconds=round(time.monotonic() - started, 3), + ) + print(f"$ {command_display(command)}", flush=True) + if result.stdout: + print( + result.stdout, + end="" if result.stdout.endswith("\n") else "\n", + flush=True, + ) + if result.stderr: + print( + result.stderr, + end="" if result.stderr.endswith("\n") else "\n", + file=sys.stderr, + flush=True, + ) + if check and result.returncode != 0: + raise CertificationError( + f"command failed ({result.returncode}): {command_display(command)}" + ) + return result + + +def load_ledger(path: Path) -> tuple[dict[str, Any], list[Package]]: + raw = json.loads(path.read_text(encoding="utf-8")) + if raw.get("schema") != 1: + raise CertificationError("unsupported release-ledger schema") + rows = raw.get("packages") + if not isinstance(rows, list): + raise CertificationError("release ledger packages must be a list") + + packages: list[Package] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + raise CertificationError(f"package row {index} is not an object") + + allow_mismatch = row.get("allow_repo_name_mismatch", False) + if not isinstance(allow_mismatch, bool): + raise CertificationError( + f"package row {index}: allow_repo_name_mismatch must be a boolean" + ) + + package = Package( + github_owner=str(row["github_owner"]), + repo=str(row["repo"]), + package=str(row["package"]), + org=str(row["org"]), + name=str(row["name"]), + version=str(row["version"]), + sha=str(row["sha"]), + tag=str(row["tag"]) if row.get("tag") else None, + dependencies=tuple(str(item) for item in row.get("dependencies", [])), + allow_repo_name_mismatch=allow_mismatch, + ) + + if package.package != f"{package.org}/{package.name}": + raise CertificationError(f"invalid package coordinate: {package.package}") + + repository_name_mismatch = package.repo != package.name + if repository_name_mismatch and not package.allow_repo_name_mismatch: + raise CertificationError( + f"{package.package}: repository/name mismatch " + f"{package.repo!r} != {package.name!r}; declare the exception explicitly" + ) + if package.allow_repo_name_mismatch and not repository_name_mismatch: + raise CertificationError( + f"{package.package}: unnecessary repository/name mismatch exception" + ) + + if len(package.sha) != SHA_RE_LENGTH or any( + c not in "0123456789abcdef" for c in package.sha + ): + raise CertificationError( + f"{package.package}: invalid source SHA {package.sha!r}" + ) + packages.append(package) + + expected_count = int(raw.get("count", -1)) + if expected_count != len(packages): + raise CertificationError( + f"ledger count {expected_count} does not match {len(packages)} package rows" + ) + if not MIN_PACKAGES <= len(packages) <= MAX_PACKAGES: + raise CertificationError( + f"package count {len(packages)} is outside [{MIN_PACKAGES}, {MAX_PACKAGES}]" + ) + + coordinates = [package.package for package in packages] + if len(coordinates) != len(set(coordinates)): + raise CertificationError("release ledger contains duplicate package coordinates") + if "zed-pkg/zed-lib-core" not in coordinates: + raise CertificationError("zed-pkg/zed-lib-core is mandatory") + + selected = set(coordinates) + seen: set[str] = set() + for package in packages: + missing = sorted(set(package.dependencies) - selected) + if missing: + raise CertificationError( + f"{package.package}: dependency closure is incomplete: {missing}" + ) + late = sorted(set(package.dependencies) - seen) + if late: + raise CertificationError( + f"{package.package}: ledger is not topological; " + f"dependencies appear later: {late}" + ) + seen.add(package.package) + + return raw, packages diff --git a/scripts/production_package_inventory.py b/scripts/production_package_inventory.py new file mode 100755 index 00000000..f9c55cae --- /dev/null +++ b/scripts/production_package_inventory.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Resolve immutable Zed package releases from GitHub. + +The default-branch manifest declares the release version and tag name. The +release ledger itself is built from the manifest at the commit peeled from that +tag, so normal development after a release does not invalidate the immutable +release. Default-head drift is retained as evidence instead of treated as a +failure. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import sys +import tomllib +import urllib.error +import urllib.parse +import urllib.request +from collections import defaultdict, deque +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any + +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +VERSION_RE = re.compile(r"^[0-9A-Za-z][0-9A-Za-z.+_-]{0,127}$") + + +class InventoryError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Candidate: + repo: str + sha: str + package: str + org: str + name: str + version: str + tag: str + tag_sha: str + archive_format: str + repository_url: str + dependencies: tuple[str, ...] + default_sha: str + default_head_matches_tag: bool + + +@dataclass(frozen=True) +class ManifestDeclaration: + package: str + org: str + name: str + version: str + tag: str + archive_format: str + repository_url: str + dependencies: tuple[str, ...] + + +def api_get(path: str, token: str | None) -> dict[str, Any]: + url = path if path.startswith("https://") else f"https://api.github.com{path}" + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "zed-pkg-production-inventory/2", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8", errors="replace") + raise InventoryError(f"GitHub API {error.code} for {url}: {body[:500]}") from error + except urllib.error.URLError as error: + raise InventoryError(f"GitHub API unavailable for {url}: {error}") from error + + +def decode_manifest(payload: dict[str, Any], coordinate: str) -> dict[str, Any]: + if payload.get("type") != "file" or payload.get("encoding") != "base64": + raise InventoryError(f"{coordinate}: root .zpkg.toml is not a base64 file") + try: + raw = base64.b64decode(payload["content"], validate=False) + document = tomllib.loads(raw.decode("utf-8")) + except (KeyError, ValueError, UnicodeDecodeError, tomllib.TOMLDecodeError) as error: + raise InventoryError(f"{coordinate}: invalid .zpkg.toml: {error}") from error + if not isinstance(document, dict): + raise InventoryError(f"{coordinate}: manifest root is not a table") + return document + + +def fetch_manifest(org: str, repo: str, ref: str, token: str | None) -> dict[str, Any]: + encoded_ref = urllib.parse.quote(ref, safe="") + payload = api_get(f"/repos/{org}/{repo}/contents/.zpkg.toml?ref={encoded_ref}", token) + return decode_manifest(payload, f"{org}/{repo}@{ref}") + + +def peel_tag(org: str, repo: str, tag: str, token: str | None) -> str: + encoded = urllib.parse.quote(tag, safe="") + reference = api_get(f"/repos/{org}/{repo}/git/ref/tags/{encoded}", token) + obj = reference.get("object") or {} + sha = obj.get("sha") + kind = obj.get("type") + for _ in range(8): + if kind == "commit" and isinstance(sha, str) and SHA_RE.fullmatch(sha): + return sha + if kind != "tag" or not isinstance(sha, str): + break + annotation = api_get(f"/repos/{org}/{repo}/git/tags/{sha}", token) + obj = annotation.get("object") or {} + sha = obj.get("sha") + kind = obj.get("type") + raise InventoryError(f"{org}/{repo}: tag {tag!r} does not peel to a commit") + + +def resolve_archive_format(document: dict[str, Any]) -> str: + publish = document.get("publish") + value: Any = None + if isinstance(publish, dict): + value = publish.get("format") or publish.get("archive_format") + if value is None: + value = "tar.gz" + aliases = {"tgz": "tar.gz", "tar_gz": "tar.gz", "tar_zst": "tar.zst"} + value = aliases.get(str(value), str(value)) + if value not in {"tar.gz", "tar.zst", "zip"}: + raise InventoryError(f"unsupported archive format {value!r}") + return value + + +def expected_package_names(repo: str) -> set[str]: + """Return package identities permitted for one source repository. + + Rust service repositories retain a `.rs` suffix for source-discovery and + deployment conventions, while their cross-registry Zed coordinates may use + the canonical suffix-free package name. Repository URL validation below + still binds either accepted package identity to the exact source repo. + """ + + names = {repo} + if repo.endswith(".rs"): + names.add(repo.removesuffix(".rs")) + return names + + +def parse_manifest(org: str, repo: str, document: dict[str, Any]) -> ManifestDeclaration: + package = document.get("package") + if not isinstance(package, dict): + raise InventoryError(f"{org}/{repo}: missing [package]") + manifest_org = package.get("org") + name = package.get("name") + version = package.get("version") + if manifest_org != org: + raise InventoryError(f"{org}/{repo}: package.org is {manifest_org!r}, expected {org!r}") + allowed_names = expected_package_names(repo) + if name not in allowed_names: + raise InventoryError( + f"{org}/{repo}: package.name is {name!r}, expected one of {sorted(allowed_names)!r}" + ) + if not isinstance(version, str) or not VERSION_RE.fullmatch(version): + raise InventoryError(f"{org}/{repo}: invalid package.version {version!r}") + repository = package.get("repository") + if not isinstance(repository, dict) or not isinstance(repository.get("url"), str): + raise InventoryError(f"{org}/{repo}: missing [package.repository].url") + expected_url = f"https://github.com/{org}/{repo}" + actual_url = repository["url"].removesuffix(".git").rstrip("/") + if actual_url.casefold() != expected_url.casefold(): + raise InventoryError( + f"{org}/{repo}: repository URL {actual_url!r} does not match {expected_url!r}" + ) + publish = document.get("publish") + tag_format = publish.get("tag_format", "v{version}") if isinstance(publish, dict) else "v{version}" + if not isinstance(tag_format, str) or tag_format.count("{version}") != 1: + raise InventoryError(f"{org}/{repo}: publish.tag_format must contain exactly one {{version}}") + tag = tag_format.replace("{version}", version) + dependencies = document.get("dependencies") + dependency_names: tuple[str, ...] + if dependencies is None: + dependency_names = () + elif isinstance(dependencies, dict) and all(isinstance(key, str) for key in dependencies): + dependency_names = tuple(sorted(dependencies)) + else: + raise InventoryError(f"{org}/{repo}: [dependencies] is not a string-keyed table") + return ManifestDeclaration( + package=f"{manifest_org}/{name}", + org=manifest_org, + name=name, + version=version, + tag=tag, + archive_format=resolve_archive_format(document), + repository_url=expected_url, + dependencies=dependency_names, + ) + + +def resolve_candidate( + org: str, + repo: str, + default_sha: str, + default_document: dict[str, Any], + token: str | None, +) -> Candidate: + requested = parse_manifest(org, repo, default_document) + tag_sha = peel_tag(org, repo, requested.tag, token) + tagged_document = fetch_manifest(org, repo, tag_sha, token) + released = parse_manifest(org, repo, tagged_document) + if (released.package, released.version, released.tag) != ( + requested.package, + requested.version, + requested.tag, + ): + raise InventoryError( + f"{org}/{repo}: default manifest requests {requested.package}@{requested.version} " + f"via {requested.tag!r}, but that tag contains " + f"{released.package}@{released.version} via {released.tag!r}" + ) + return Candidate( + repo=repo, + sha=tag_sha, + package=released.package, + org=released.org, + name=released.name, + version=released.version, + tag=released.tag, + tag_sha=tag_sha, + archive_format=released.archive_format, + repository_url=released.repository_url, + dependencies=released.dependencies, + default_sha=default_sha, + default_head_matches_tag=default_sha == tag_sha, + ) + + +def topological_order(candidates: list[Candidate]) -> list[Candidate]: + by_package = {candidate.package: candidate for candidate in candidates} + incoming = {candidate.package: 0 for candidate in candidates} + outgoing: dict[str, set[str]] = defaultdict(set) + for candidate in candidates: + for dependency in candidate.dependencies: + if dependency not in by_package: + continue + if candidate.package not in outgoing[dependency]: + outgoing[dependency].add(candidate.package) + incoming[candidate.package] += 1 + queue = deque(sorted(package for package, degree in incoming.items() if degree == 0)) + ordered: list[Candidate] = [] + while queue: + package = queue.popleft() + ordered.append(by_package[package]) + for dependent in sorted(outgoing[package]): + incoming[dependent] -= 1 + if incoming[dependent] == 0: + queue.append(dependent) + if len(ordered) != len(candidates): + cycle = sorted(package for package, degree in incoming.items() if degree > 0) + raise InventoryError(f"selected package dependencies contain a cycle: {cycle}") + return ordered + + +def write_outputs(candidates: list[Candidate], rejected: list[dict[str, str]], output: Path) -> None: + output.mkdir(parents=True, exist_ok=True) + ledger = { + "schema": 2, + "count": len(candidates), + "drifted_default_heads": sum(not candidate.default_head_matches_tag for candidate in candidates), + "packages": [asdict(candidate) for candidate in candidates], + "rejected": rejected, + } + (output / "production-package-inventory.json").write_text( + json.dumps(ledger, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + lines = [ + "# Production package inventory", + "", + f"Selected immutable packages: **{len(candidates)}**", + f"Default branches advanced after the immutable tag: **{ledger['drifted_default_heads']}**", + "", + "| Order | Package | Version | Release commit | Default commit | Tag | Format |", + "| ---: | --- | --- | --- | --- | --- | --- |", + ] + for index, candidate in enumerate(candidates, 1): + default = candidate.default_sha if candidate.default_head_matches_tag else f"{candidate.default_sha} (advanced)" + lines.append( + f"| {index} | `{candidate.package}` | `{candidate.version}` | " + f"`{candidate.sha}` | `{default}` | `{candidate.tag}` | " + f"`{candidate.archive_format}` |" + ) + if rejected: + lines.extend(["", "## Rejected candidates", ""]) + for item in rejected: + lines.append(f"- `{item['repo']}` — {item['reason']}") + (output / "production-package-inventory.md").write_text( + "\n".join(lines) + "\n", encoding="utf-8" + ) + github_output = os.environ.get("GITHUB_OUTPUT") + if github_output: + compact = json.dumps([asdict(candidate) for candidate in candidates], separators=(",", ":")) + with open(github_output, "a", encoding="utf-8") as handle: + handle.write(f"matrix={compact}\n") + handle.write(f"count={len(candidates)}\n") + + +def inventory(config_path: Path, output: Path, token: str | None) -> None: + config = json.loads(config_path.read_text(encoding="utf-8")) + org = config["organization"] + minimum = int(config["minimum_packages"]) + maximum = int(config["maximum_packages"]) + repos = config["repositories"] + if not isinstance(repos, list) or not repos or len(repos) != len(set(repos)): + raise InventoryError("repository candidates must be a non-empty unique list") + candidates: list[Candidate] = [] + rejected: list[dict[str, str]] = [] + for repo in repos: + try: + metadata = api_get(f"/repos/{org}/{repo}", token) + if metadata.get("archived"): + raise InventoryError("repository is archived") + default_branch = metadata.get("default_branch") + if not isinstance(default_branch, str): + raise InventoryError("repository has no default branch") + commit = api_get( + f"/repos/{org}/{repo}/commits/{urllib.parse.quote(default_branch, safe='')}", + token, + ) + default_sha = commit.get("sha") + if not isinstance(default_sha, str) or not SHA_RE.fullmatch(default_sha): + raise InventoryError("default branch did not resolve to an exact commit") + default_document = fetch_manifest(org, repo, default_sha, token) + candidates.append(resolve_candidate(org, repo, default_sha, default_document, token)) + except InventoryError as error: + rejected.append({"repo": repo, "reason": str(error)}) + candidates = topological_order(candidates) + if not minimum <= len(candidates) <= maximum: + write_outputs(candidates, rejected, output) + raise InventoryError( + f"immutable root-manifest package count {len(candidates)} is outside [{minimum}, {maximum}]" + ) + if not any(candidate.name == "zed-lib-core" for candidate in candidates): + raise InventoryError("zed-lib-core is mandatory but was not selected") + if len({candidate.package for candidate in candidates}) != len(candidates): + raise InventoryError("selected package coordinates are not unique") + write_outputs(candidates, rejected, output) + + +def self_test() -> None: + document = { + "package": { + "org": "zed-pkg", + "name": "a", + "version": "1.0.0", + "repository": {"url": "https://github.com/zed-pkg/a.git"}, + }, + "publish": {"format": "zip", "tag_format": "release-{version}"}, + "dependencies": {"zed-pkg/b": {"version": "1.0.0"}}, + } + declaration = parse_manifest("zed-pkg", "a", document) + assert declaration.package == "zed-pkg/a" + assert declaration.tag == "release-1.0.0" + assert declaration.archive_format == "zip" + assert declaration.dependencies == ("zed-pkg/b",) + + rust_service_document = { + "package": { + "org": "zed-pkg", + "name": "zed-api-server", + "version": "0.1.0", + "repository": {"url": "https://github.com/zed-pkg/zed-api-server.rs"}, + }, + "publish": {"tag_format": "v{version}"}, + } + rust_declaration = parse_manifest( + "zed-pkg", "zed-api-server.rs", rust_service_document + ) + assert rust_declaration.package == "zed-pkg/zed-api-server" + assert rust_declaration.repository_url == "https://github.com/zed-pkg/zed-api-server.rs" + + invalid_rust_service = json.loads(json.dumps(rust_service_document)) + invalid_rust_service["package"]["name"] = "unrelated-api" + try: + parse_manifest("zed-pkg", "zed-api-server.rs", invalid_rust_service) + except InventoryError: + pass + else: + raise AssertionError("unrelated package name was accepted for a .rs repository") + + sample = [ + Candidate( + "b", "b" * 40, "zed-pkg/b", "zed-pkg", "b", "1.0.0", "v1.0.0", + "b" * 40, "zip", "https://github.com/zed-pkg/b", ("zed-pkg/a",), + "c" * 40, False, + ), + Candidate( + "a", "a" * 40, "zed-pkg/a", "zed-pkg", "a", "1.0.0", "v1.0.0", + "a" * 40, "tar.gz", "https://github.com/zed-pkg/a", (), + "a" * 40, True, + ), + ] + assert [candidate.name for candidate in topological_order(sample)] == ["a", "b"] + assert resolve_archive_format({}) == "tar.gz" + try: + resolve_archive_format({"publish": {"format": "rar"}}) + except InventoryError: + pass + else: + raise AssertionError("unknown archive format was accepted") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, default=Path("config/production-packages.json")) + parser.add_argument("--output", type=Path, default=Path("evidence/package-inventory")) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + try: + if args.self_test: + self_test() + else: + inventory( + args.config, + args.output, + os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN"), + ) + except (InventoryError, OSError, json.JSONDecodeError, KeyError, TypeError) as error: + print(f"inventory failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/production_package_registry.py b/scripts/production_package_registry.py new file mode 100644 index 00000000..71480358 --- /dev/null +++ b/scripts/production_package_registry.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import hashlib +import io +import json +import tarfile +import tomllib +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from pathlib import Path +from typing import Any +from production_package_common import CertificationError, RegistryPackage + +def api_json(url: str, token: str | None) -> dict[str, Any]: + headers = {'Accept': 'application/json', 'User-Agent': 'zed-production-roundtrip/1'} + if token: + headers['Authorization'] = f'Bearer {token}' + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + except urllib.error.HTTPError as error: + body = error.read().decode('utf-8', errors='replace') + raise CertificationError(f'HTTP {error.code} for {url}: {body[:1000]}') from error + +def api_bytes(url: str, token: str | None) -> bytes: + headers = {'Accept': 'application/octet-stream', 'User-Agent': 'zed-production-roundtrip/1'} + if token: + headers['Authorization'] = f'Bearer {token}' + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=180) as response: + return response.read() + except urllib.error.HTTPError as error: + body = error.read().decode('utf-8', errors='replace') + raise CertificationError(f'HTTP {error.code} for {url}: {body[:1000]}') from error + +def _verify_embedded_manifest(raw: bytes, package: RegistryPackage) -> dict[str, Any]: + try: + document = tomllib.loads(raw.decode('utf-8')) + except (UnicodeDecodeError, tomllib.TOMLDecodeError) as error: + raise CertificationError(f'{package.package}: artifact contains an invalid .zpkg.toml') from error + declared = document.get('package') + if not isinstance(declared, dict): + raise CertificationError(f'{package.package}: artifact manifest is missing [package]') + actual = (declared.get('org'), declared.get('name'), declared.get('version')) + expected = (package.org, package.name, package.version) + if actual != expected: + raise CertificationError(f'{package.package}: artifact manifest identity {actual!r} != {expected!r}') + targets = document.get('targets') + if package.target is not None and isinstance(targets, dict) and targets: + raise CertificationError(f'{package.package}: derived target artifact retained nested [targets]') + return {'org': package.org, 'name': package.name, 'version': package.version, 'target': package.target} + +def inspect_archive(data: bytes, expected_format: str, package: RegistryPackage) -> dict[str, Any]: + members: list[str] + manifest_bytes: bytes | None = None + if expected_format == 'zip': + with zipfile.ZipFile(io.BytesIO(data)) as archive: + members = archive.namelist() + if archive.testzip() is not None: + raise CertificationError('ZIP artifact failed CRC verification') + manifest_names = [name for name in members if name.lstrip('./') == '.zpkg.toml' or name.lstrip('./').endswith('/.zpkg.toml')] + if len(manifest_names) != 1: + raise CertificationError(f'{package.package}: expected one artifact manifest, found {manifest_names!r}') + manifest_bytes = archive.read(manifest_names[0]) + else: + with tarfile.open(fileobj=io.BytesIO(data), mode='r:*') as archive: + members = archive.getnames() + manifest_names = [name for name in members if name.lstrip('./') == '.zpkg.toml' or name.lstrip('./').endswith('/.zpkg.toml')] + if len(manifest_names) != 1: + raise CertificationError(f'{package.package}: expected one artifact manifest, found {manifest_names!r}') + member = archive.extractfile(manifest_names[0]) + if member is None: + raise CertificationError(f'{package.package}: artifact manifest is not a regular file') + manifest_bytes = member.read() + assert manifest_bytes is not None + embedded = _verify_embedded_manifest(manifest_bytes, package) + return {'member_count': len(members), 'sample_members': members[:20], 'embedded_manifest': embedded} + +def metadata_for(registry: str, package: RegistryPackage, token: str | None) -> dict[str, Any]: + parts = [urllib.parse.quote(value, safe='') for value in (package.org, package.name, package.version)] + url = f"{registry.rstrip('/')}/v1/packages/{parts[0]}/{parts[1]}/versions/{parts[2]}" + metadata = api_json(url, token) + for field, expected in (('org', package.org), ('name', package.name), ('version', package.version)): + if metadata.get(field) != expected: + raise CertificationError(f'{package.package}: metadata {field}={metadata.get(field)!r}, expected {expected!r}') + digest = metadata.get('sha256') + if not isinstance(digest, str) or len(digest) != 64: + raise CertificationError(f'{package.package}: invalid metadata sha256 {digest!r}') + download_url = metadata.get('download_url') + if not isinstance(download_url, str) or not download_url: + raise CertificationError(f'{package.package}: metadata missing download_url') + resolved_download = urllib.parse.urljoin(f"{registry.rstrip('/')}/", download_url) + artifact = api_bytes(resolved_download, token) + actual_digest = hashlib.sha256(artifact).hexdigest() + if actual_digest != digest: + raise CertificationError(f'{package.package}: downloaded artifact sha256 {actual_digest} != metadata {digest}') + archive = inspect_archive(artifact, str(metadata.get('format', 'tar.gz')), package) + return {'metadata_url': url, 'download_url': resolved_download, 'sha256': digest, 'size': len(artifact), 'format': metadata.get('format'), **archive} + +def installed_coordinates(root: Path) -> set[str]: + coordinates: set[str] = set() + for manifest in root.rglob('.zpkg.toml'): + try: + document = tomllib.loads(manifest.read_text(encoding='utf-8')) + except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError): + continue + package = document.get('package') + if isinstance(package, dict) and isinstance(package.get('org'), str) and isinstance(package.get('name'), str): + coordinates.add(f"{package['org']}/{package['name']}") + return coordinates + +def verify_install(root: Path, expected: set[str], phase: str) -> list[str]: + actual = installed_coordinates(root) + missing = sorted(expected - actual) + if missing: + manifests = sorted((str(path.relative_to(root)) for path in root.rglob('.zpkg.toml'))) + raise CertificationError(f'{phase}: installed tree is missing {missing}; discovered manifests={manifests[:100]}') + return sorted(actual & expected) + +def write_evidence(evidence_dir: Path, report: dict[str, Any]) -> None: + evidence_dir.mkdir(parents=True, exist_ok=True) + (evidence_dir / 'production-package-roundtrip.json').write_text(json.dumps(report, indent=2, sort_keys=True) + '\n', encoding='utf-8') + lines = [ + '# Production package publish/install roundtrip', + '', + f"Status: **{report.get('status', 'unknown')}**", + f"Logical source repositories: **{report.get('logical_package_count', report.get('package_count', 0))}**", + f"Concrete registry packages: **{report.get('registry_package_count', 0)}**", + f"Registry: `{report.get('registry', '')}`", + '', + '| Order | Source repository | Registry package | Target | Version | Source SHA | Artifact SHA-256 | Installed | Frozen reinstall |', + '| ---: | --- | --- | --- | --- | --- | --- | --- | --- |', + ] + installed = set(report.get('installed', [])) + frozen = set(report.get('frozen_reinstalled', [])) + order = 0 + for item in report.get('packages', []): + for concrete in item.get('registry_packages', []): + order += 1 + coordinate = concrete['package'] + lines.append( + f"| {order} | `{item['package']}` | `{coordinate}` | `{concrete.get('target') or 'single'}` | `{concrete['version']}` | `{item['source_sha']}` | `{concrete.get('artifact', {}).get('sha256', '')}` | {('yes' if coordinate in installed else 'no')} | {('yes' if coordinate in frozen else 'no')} |" + ) + if report.get('error'): + lines.extend(['', '## Failure', '', '```text', str(report['error']), '```']) + (evidence_dir / 'production-package-roundtrip.md').write_text('\n'.join(lines) + '\n', encoding='utf-8') diff --git a/scripts/production_package_roundtrip.py b/scripts/production_package_roundtrip.py new file mode 100644 index 00000000..cf7e6bd2 --- /dev/null +++ b/scripts/production_package_roundtrip.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse +import os +import shutil +import traceback +from pathlib import Path +from typing import Any +from production_package_common import CertificationError, RegistryPackage, load_ledger, run +from production_package_source import checkout_source, parse_manifest +from production_package_registry import metadata_for, verify_install, write_evidence + +def _concrete_row(package: RegistryPackage) -> dict[str, Any]: + return { + 'package': package.package, + 'version': package.version, + 'target': package.target, + 'spec': package.spec, + 'artifact': {}, + 'idempotent_artifact': {}, + } + +def certify(args: argparse.Namespace) -> None: + ledger_raw, packages = load_ledger(args.ledger) + workspace = args.workspace.resolve() + sources = workspace / 'sources' + consumer = workspace / 'consumer' + home = workspace / 'zed-home' + for path in (sources, consumer, home, args.evidence): + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + cli_env = os.environ.copy() + cli_env.update({'ZED_PKG_REGISTRY': args.registry.rstrip('/'), 'ZED_PKG_HOME': str(home), 'ZED_PKG_ALLOW_NO_MANIFEST': '1', 'ZED_PKG_INSTALL_MODE': 'copy'}) + if args.token: + cli_env['ZED_PKG_TOKEN'] = args.token + report: dict[str, Any] = { + 'schema': 2, + 'status': 'running', + 'registry': args.registry.rstrip('/'), + 'package_count': len(packages), + 'logical_package_count': len(packages), + 'registry_package_count': 0, + 'registry_api_sha': ledger_raw.get('registry_api_sha'), + 'zed_cli_sha': ledger_raw.get('zed_cli_sha'), + 'packages': [], + 'installed': [], + 'frozen_reinstalled': [], + } + try: + run([str(args.zed), '--help'], env=cli_env, timeout=120) + for org in sorted({package.org for package in packages}): + run([str(args.zed), 'org', 'claim', org], env=cli_env, timeout=120) + checkouts: dict[str, Path] = {} + concrete_by_logical: dict[str, tuple[RegistryPackage, ...]] = {} + all_concrete: list[RegistryPackage] = [] + for package in packages: + source = checkout_source(package, sources) + _, concrete = parse_manifest(source, package) + checkouts[package.package] = source + concrete_by_logical[package.package] = concrete + all_concrete.extend(concrete) + report['packages'].append({ + 'package': package.package, + 'version': package.version, + 'source_sha': package.sha, + 'tag': package.tag, + 'dependencies': list(package.dependencies), + 'source_url': package.source_url, + 'publish': {}, + 'idempotent_retry': {}, + 'registry_packages': [_concrete_row(item) for item in concrete], + }) + report['registry_package_count'] = len(all_concrete) + write_evidence(args.evidence, report) + coordinates = [item.package for item in all_concrete] + if len(coordinates) != len(set(coordinates)): + raise CertificationError('release graph contains duplicate concrete registry coordinates') + if len(all_concrete) < len(packages): + raise CertificationError('concrete registry package count cannot be smaller than logical source count') + rows_by_package = {row['package']: row for row in report['packages']} + concrete_rows = { + row['package']: {item['package']: item for item in row['registry_packages']} + for row in report['packages'] + } + for package in packages: + source = checkouts[package.package] + before = run(['git', 'status', '--porcelain=v1', '--untracked-files=all'], cwd=source).stdout + if before: + raise CertificationError(f'{package.package}: dirty before publish:\n{before}') + result = run([str(args.zed), 'publish', '--allow-dirty', '--skip-vcs-checks'], cwd=source, env=cli_env, timeout=1800) + rows_by_package[package.package]['publish'] = {'returncode': result.returncode, 'duration_seconds': result.duration_seconds, 'stdout': result.stdout[-12000:], 'stderr': result.stderr[-4000:]} + for concrete in concrete_by_logical[package.package]: + concrete_rows[package.package][concrete.package]['artifact'] = metadata_for(args.registry, concrete, args.token) + write_evidence(args.evidence, report) + for package in packages: + source = checkouts[package.package] + for generated in (source / '.zed', source / '.zed-pack'): + if generated.exists(): + shutil.rmtree(generated) + result = run([str(args.zed), 'publish', '--allow-dirty', '--skip-vcs-checks'], cwd=source, env=cli_env, timeout=1800) + rows_by_package[package.package]['idempotent_retry'] = {'returncode': result.returncode, 'duration_seconds': result.duration_seconds, 'stdout': result.stdout[-12000:], 'stderr': result.stderr[-4000:]} + for concrete in concrete_by_logical[package.package]: + row = concrete_rows[package.package][concrete.package] + retry_artifact = metadata_for(args.registry, concrete, args.token) + row['idempotent_artifact'] = retry_artifact + if retry_artifact['sha256'] != row['artifact']['sha256']: + raise CertificationError(f'{concrete.package}: idempotent retry changed artifact digest') + write_evidence(args.evidence, report) + specs = [package.spec for package in all_concrete] + run([str(args.zed), 'install', *specs, '--skip-manifest', '--allow-ecosystem-mismatch'], cwd=consumer, env=cli_env, timeout=3600) + expected = {package.package for package in all_concrete} + report['installed'] = verify_install(consumer, expected, 'initial install') + write_evidence(args.evidence, report) + run([str(args.zed), 'uninstall'], cwd=consumer, env=cli_env, timeout=1800) + run([str(args.zed), 'install', '--frozen', '--skip-manifest', '--allow-ecosystem-mismatch'], cwd=consumer, env=cli_env, timeout=3600) + report['frozen_reinstalled'] = verify_install(consumer, expected, 'frozen reinstall') + report['status'] = 'success' + write_evidence(args.evidence, report) + except Exception as error: + report['status'] = 'failure' + report['error'] = f'{type(error).__name__}: {error}\n{traceback.format_exc()}' + write_evidence(args.evidence, report) + raise + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument('--ledger', type=Path, required=True) + parser.add_argument('--workspace', type=Path, required=True) + parser.add_argument('--evidence', type=Path, required=True) + parser.add_argument('--zed', type=Path, required=True) + parser.add_argument('--registry', required=True) + parser.add_argument('--token', default=os.environ.get('ZED_PKG_TOKEN', '')) + return parser.parse_args() + +if __name__ == '__main__': + certify(parse_args()) diff --git a/scripts/production_package_source.py b/scripts/production_package_source.py new file mode 100644 index 00000000..4358b113 --- /dev/null +++ b/scripts/production_package_source.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import os +import shutil +import tomllib +from pathlib import Path +from typing import Any +from urllib.parse import quote +from production_package_common import CertificationError, Package, RegistryPackage, run + +def _credential_file(root: Path) -> Path | None: + token = os.environ.get('ZED_PKG_GITHUB_TOKEN', '').strip() + if not token: + return None + path = root / '.zpkg-git-credentials' + path.write_text( + f'https://x-access-token:{quote(token, safe="")}@github.com\n', + encoding='utf-8', + ) + path.chmod(0o600) + return path + +def _git( + arguments: list[str], + *, + cwd: Path, + credential_file: Path | None, + check: bool = True, + timeout: int = 1800, +): + command = ['git'] + if credential_file is not None: + command.extend(['-c', f'credential.helper=store --file={credential_file}']) + command.extend(arguments) + env = os.environ.copy() + env['GIT_TERMINAL_PROMPT'] = '0' + return run(command, cwd=cwd, env=env, check=check, timeout=timeout) + +def checkout_source(package: Package, root: Path) -> Path: + target = root / package.github_owner / package.repo + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + shutil.rmtree(target) + target.mkdir() + credential_file = _credential_file(root) + try: + _git(['init', '--quiet'], cwd=target, credential_file=credential_file) + _git(['config', 'core.autocrlf', 'false'], cwd=target, credential_file=credential_file) + _git(['remote', 'add', 'origin', package.source_url], cwd=target, credential_file=credential_file) + fetch = _git( + ['fetch', '--quiet', '--no-tags', '--depth=1', 'origin', package.sha], + cwd=target, + credential_file=credential_file, + check=False, + timeout=900, + ) + if fetch.returncode != 0: + ref = f'refs/tags/{package.tag}' if package.tag else 'refs/heads/main' + _git( + ['fetch', '--quiet', '--depth=1', 'origin', ref], + cwd=target, + credential_file=credential_file, + timeout=900, + ) + fetched = _git( + ['rev-parse', 'FETCH_HEAD^{commit}'], + cwd=target, + credential_file=credential_file, + ).stdout.strip() + if fetched != package.sha: + raise CertificationError( + f'{package.package}: fallback ref {ref} resolved to {fetched}, expected {package.sha}' + ) + _git( + ['checkout', '--quiet', '--detach', package.sha], + cwd=target, + credential_file=credential_file, + ) + actual = _git( + ['rev-parse', 'HEAD'], + cwd=target, + credential_file=credential_file, + ).stdout.strip() + if actual != package.sha: + raise CertificationError( + f'{package.package}: checkout resolved to {actual}, expected {package.sha}' + ) + if package.tag: + _git( + [ + 'fetch', + '--quiet', + '--depth=1', + 'origin', + f'refs/tags/{package.tag}:refs/tags/{package.tag}', + ], + cwd=target, + credential_file=credential_file, + timeout=900, + ) + tagged = _git( + ['rev-parse', f'{package.tag}^{{commit}}'], + cwd=target, + credential_file=credential_file, + ).stdout.strip() + if tagged != package.sha: + raise CertificationError( + f'{package.package}: tag {package.tag} resolves to {tagged}, expected {package.sha}' + ) + status = _git( + ['status', '--porcelain=v1', '--untracked-files=all'], + cwd=target, + credential_file=credential_file, + ).stdout + if status: + raise CertificationError(f'{package.package}: checkout is not clean:\n{status}') + return target + finally: + if credential_file is not None: + credential_file.unlink(missing_ok=True) + +def _registry_packages(document: dict[str, Any], package: Package) -> tuple[RegistryPackage, ...]: + targets = document.get('targets') + if targets is None or targets == {}: + return ( + RegistryPackage( + logical_package=package.package, + org=package.org, + name=package.name, + version=package.version, + target=None, + ), + ) + if not isinstance(targets, dict): + raise CertificationError(f'{package.package}: [targets] must be a table') + concrete: list[RegistryPackage] = [] + for target, section in sorted(targets.items()): + if not isinstance(target, str) or not target or '/' in target: + raise CertificationError(f'{package.package}: invalid target key {target!r}') + if not isinstance(section, dict): + raise CertificationError(f'{package.package}: target {target!r} must be a table') + explicit_name = section.get('name') + if explicit_name is not None and (not isinstance(explicit_name, str) or not explicit_name): + raise CertificationError(f'{package.package}: target {target!r} has invalid name') + # The CLI canonicalizes any target spanning the repository root (`dir = "."`) + # to the logical package coordinate, regardless of the target key. This covers + # both `[targets.repository]` packages and language-native roots such as + # zed-lock's `[targets.rust]` without inventing a `-rust` registry package. + is_root_spanning_target = section.get('dir') == '.' + if is_root_spanning_target: + if explicit_name is not None and explicit_name != package.name: + raise CertificationError( + f'{package.package}: repository-root target must use canonical package name ' + f'{package.name!r}, not {explicit_name!r}' + ) + name = package.name + else: + name = explicit_name or f'{package.name}-{target}' + if '/' in name: + raise CertificationError(f'{package.package}: target {target!r} has invalid package name {name!r}') + concrete.append( + RegistryPackage( + logical_package=package.package, + org=package.org, + name=name, + version=package.version, + target=target, + ) + ) + coordinates = [item.package for item in concrete] + if len(coordinates) != len(set(coordinates)): + raise CertificationError(f'{package.package}: targets publish duplicate package identities {coordinates!r}') + return tuple(concrete) + +def parse_manifest(path: Path, package: Package) -> tuple[dict[str, Any], tuple[RegistryPackage, ...]]: + manifest_path = path / '.zpkg.toml' + if not manifest_path.is_file(): + raise CertificationError(f'{package.package}: missing root .zpkg.toml') + document = tomllib.loads(manifest_path.read_text(encoding='utf-8')) + declared = document.get('package') + if not isinstance(declared, dict): + raise CertificationError(f'{package.package}: missing [package]') + values = (declared.get('org'), declared.get('name'), declared.get('version')) + expected = (package.org, package.name, package.version) + if values != expected: + raise CertificationError(f'{package.package}: manifest identity {values!r} does not match {expected!r}') + repository = declared.get('repository') + expected_url = f'https://github.com/{package.github_owner}/{package.repo}'.casefold() + if not isinstance(repository, dict) or not isinstance(repository.get('url'), str): + raise CertificationError(f'{package.package}: missing package.repository.url') + actual_url = repository['url'].removesuffix('.git').rstrip('/').casefold() + if actual_url != expected_url: + raise CertificationError(f'{package.package}: repository URL {actual_url!r} does not match {expected_url!r}') + dependencies = document.get('dependencies') + manifest_dependencies = tuple(sorted(dependencies)) if isinstance(dependencies, dict) else () + if manifest_dependencies != tuple(sorted(package.dependencies)): + raise CertificationError(f'{package.package}: dependency ledger mismatch: manifest={manifest_dependencies!r}, ledger={tuple(sorted(package.dependencies))!r}') + return document, _registry_packages(document, package) diff --git a/scripts/project_recovery_operation_lock.py b/scripts/project_recovery_operation_lock.py new file mode 100644 index 00000000..321fe94f --- /dev/null +++ b/scripts/project_recovery_operation_lock.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Certify pending transaction recovery shares checkout-local ownership.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path +from typing import Sequence + +from git_submodule_overtake_lock import Contract, ContractError + +TRANSACTION_ID = "11111111-2222-4333-8444-555555555555" +BACKUP_RELATIVE = "backups/0000-66666666-7777-4888-8999-aaaaaaaaaaaa" +MARKER_RELATIVE = "recovery-marker.txt" +ORIGINAL = b"authoritative pre-crash bytes\n" +INTERRUPTED = b"interrupted replacement bytes\n" + + +def create_pending_recovery(project: Path) -> tuple[Path, Path, Path]: + staging = project / ".zpkg-staging" / TRANSACTION_ID + backup = staging / BACKUP_RELATIVE + marker = project / MARKER_RELATIVE + backup.parent.mkdir(parents=True) + marker.write_bytes(INTERRUPTED) + backup.write_bytes(ORIGINAL) + metadata = { + "id": TRANSACTION_ID, + "state": "active", + "entries": [ + { + "relative": MARKER_RELATIVE, + "backup": BACKUP_RELATIVE, + "existed": True, + } + ], + } + (staging / "transaction.json").write_text( + json.dumps(metadata, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return staging, backup, marker + + +def certify_different_home_waiter(contract: Contract) -> tuple[list[str], dict[str, float | int]]: + _, root = contract.fixture("recovery") + + # The holder reaches Git synchronization only after acquiring the canonical + # checkout-local operation lock. Create the interrupted journal afterward so + # its own pre-Git recovery cannot consume this fixture. + holder, _, release = contract.blocked_holder(root, "recovery") + staging, backup, marker = create_pending_recovery(root) + + started = time.monotonic() + waiter = contract.start( + contract.zed_command("recovery-waiter", "install", "--frozen"), + cwd=root, + ) + contract.require_blocked(waiter) + blocked_for = time.monotonic() - started + + # A waiter using a different Zed home must block before eager recovery. The + # old Store-home lock allowed it to restore this journal concurrently with + # the active takeover. + if not staging.is_dir(): + raise ContractError("pending recovery ran while another mutation owned the checkout") + if marker.read_bytes() != INTERRUPTED: + raise ContractError("recovery marker changed before checkout ownership was released") + if backup.read_bytes() != ORIGINAL: + raise ContractError("recovery backup changed before checkout ownership was released") + + release.write_text("release\n", encoding="utf-8") + holder_output = contract.finish(holder, expected=0) + waiter_output = contract.finish(waiter, expected=0) + if "overtook 1 Git submodule package(s)" not in holder_output: + raise ContractError("takeover holder did not complete adoption") + if "error:" in waiter_output.lower(): + raise ContractError("waiting frozen install reported an error") + + if marker.read_bytes() != ORIGINAL: + raise ContractError("pending recovery did not restore the exact backup bytes") + if (root / ".zpkg-staging").exists(): + raise ContractError("successful recovery left project staging state") + if not (root / ".zpkg.lock").is_file(): + raise ContractError("takeover did not publish the lockfile after recovery") + if not (root / "zed_modules/acme/recovery-child/payload.txt").is_file(): + raise ContractError("waiting frozen install did not materialize adopted state") + + checks = [ + "different-home eager recovery blocks behind active checkout mutation", + "pending journal and backup remain byte-exact while ownership is held", + "recovery restores exact bytes and removes staging after release", + "recovered process completes frozen install against adopted state", + ] + process = { + "holder_pid": holder.pid, + "waiter_pid": waiter.pid, + "observed_block_seconds": round(blocked_for, 3), + } + return checks, process + + +def certify_overtake_recovers_before_git(contract: Contract) -> tuple[list[str], int]: + _, root = contract.fixture("preflight-recovery") + staging, _, marker = create_pending_recovery(root) + + # Modular takeover dispatch bypasses main's ordinary run() entrypoint. By + # the time the Git shim announces `submodule sync`, recovery must already + # have completed under the same project operation lock. + holder, _, release = contract.blocked_holder(root, "preflight-recovery") + if staging.exists(): + raise ContractError("takeover reached Git sync before recovering pending state") + if marker.read_bytes() != ORIGINAL: + raise ContractError("takeover did not restore exact bytes before Git sync") + + release.write_text("release\n", encoding="utf-8") + output = contract.finish(holder, expected=0) + if "overtook 1 Git submodule package(s)" not in output: + raise ContractError("recovered takeover did not complete adoption") + if not (root / ".zpkg.lock").is_file(): + raise ContractError("recovered takeover did not publish a lockfile") + + checks = [ + "modular takeover recovers pending transaction before Git transport", + "pre-Git recovery restores exact bytes before adoption begins", + ] + return checks, holder.pid + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args(argv) + + +def main(argv: Sequence[str]) -> int: + args = parse_args(argv) + contract = Contract(args.zed, args.work_root) + blocked_checks, blocked_process = certify_different_home_waiter(contract) + pre_git_checks, pre_git_holder_pid = certify_overtake_recovers_before_git(contract) + checks = [*blocked_checks, *pre_git_checks] + if len(checks) != 6: + raise ContractError(f"unexpected recovery check count: {checks}") + + evidence = { + "schema": "zed.project-recovery-operation-lock/v1", + "checks": checks, + "different_home_waiter": blocked_process, + "pre_git_holder_pid": pre_git_holder_pid, + "network_credentials": False, + "public_registry_mutation": False, + } + (contract.evidence / "recovery-evidence.json").write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + contract.log("\ncertified 6 project-recovery ownership checks") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/publication_immutability.py b/scripts/publication_immutability.py new file mode 100644 index 00000000..af91b216 --- /dev/null +++ b/scripts/publication_immutability.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python3 +"""Fail-closed publication immutability certification for zed-pkg. + +The canary publishes a fixture to a fresh file:// registry, proves that an +identical retry is byte-idempotent, then changes bytes without changing the +package identity and proves that the retry is rejected without mutating any +registry object. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tomllib +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Mapping, Sequence + +IGNORED_SOURCE_NAMES = { + ".git", + ".zed-pack", + "build", + "node_modules", + "target", + "zed_modules", +} + + +@dataclass(frozen=True) +class PackageRef: + org: str + name: str + version: str + + @property + def spec(self) -> str: + return f"{self.org}/{self.name}@{self.version}" + + +@dataclass(frozen=True) +class PublishedArtifact: + package: PackageRef + metadata_path: Path + artifact_path: Path + sha256: str + + +class Harness: + def __init__(self, args: argparse.Namespace) -> None: + self.fixture = args.fixture_dir.resolve() + self.zed = args.zed.resolve() + self.root = args.work_root.resolve() + self.registry = self.root / "registry" + self.logs = self.root / "logs" + self.log_path = self.logs / "publication-immutability.log" + + if self.root.exists(): + raise RuntimeError(f"work root must not already exist: {self.root}") + if not self.fixture.is_dir(): + raise RuntimeError(f"fixture directory not found: {self.fixture}") + if not (self.fixture / ".zpkg.toml").is_file(): + raise RuntimeError(f"fixture has no .zpkg.toml: {self.fixture}") + if not self.zed.is_file(): + raise RuntimeError(f"zed binary not found: {self.zed}") + + self.registry.mkdir(parents=True) + self.logs.mkdir(parents=True) + self.log_path.write_text("", encoding="utf-8") + + @property + def registry_url(self) -> str: + return f"file://{self.registry}" + + def environment(self, home: Path) -> dict[str, str]: + result = os.environ.copy() + result.update( + { + "CI": "true", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "ZED_PKG_HOME": str(home), + "ZED_PKG_REGISTRY": self.registry_url, + "ZED_PKG_TOKEN": "", + } + ) + return result + + def log(self, message: str) -> None: + print(message, flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(message + "\n") + + def run( + self, + command: Sequence[str | Path], + *, + cwd: Path, + home: Path, + should_fail: bool = False, + extra_env: Mapping[str, str] | None = None, + ) -> str: + argv = [str(value) for value in command] + shown = " ".join(shell_quote(value) for value in argv) + self.log(f"\n$ (cd {cwd} && {shown})") + environment = self.environment(home) + if extra_env: + environment.update(extra_env) + completed = subprocess.run( + argv, + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + output = completed.stdout or "" + if output: + print(output, end="" if output.endswith("\n") else "\n", flush=True) + with self.log_path.open("a", encoding="utf-8") as handle: + handle.write(output) + if not output.endswith("\n"): + handle.write("\n") + + if should_fail: + if completed.returncode == 0: + raise AssertionError(f"command unexpectedly succeeded: {shown}") + elif completed.returncode != 0: + raise RuntimeError( + f"command failed with exit code {completed.returncode}: {shown}" + ) + return output + + def zed_cmd( + self, + *args: str | Path, + cwd: Path, + home: Path, + should_fail: bool = False, + ) -> str: + return self.run( + [ + self.zed, + "--registry", + self.registry_url, + "--home", + home, + *args, + ], + cwd=cwd, + home=home, + should_fail=should_fail, + ) + + def assert_fixture_clean(self, phase: str) -> None: + output = self.run( + ["git", "status", "--porcelain=v1", "--untracked-files=all"], + cwd=self.fixture, + home=self.root / "git-home", + ) + if output.strip(): + raise AssertionError(f"fixture became dirty during {phase}:\n{output}") + + def copy_fixture(self, destination: Path) -> Path: + def ignore(_directory: str, names: list[str]) -> set[str]: + return {name for name in names if name in IGNORED_SOURCE_NAMES} + + shutil.copytree( + self.fixture, + destination, + symlinks=True, + ignore=ignore, + ) + return destination + + def publish(self, source: Path, home_name: str, *, should_fail: bool = False) -> str: + return self.zed_cmd( + "publish", + "--skip-vcs-checks", + cwd=source, + home=self.root / home_name, + should_fail=should_fail, + ) + + def run_contract(self) -> None: + self.assert_fixture_clean("preflight") + manifest = read_manifest(self.fixture) + packages = expected_packages(manifest) + source = self.copy_fixture(self.root / "source") + + self.log("== Initial immutable publication ==") + self.publish(source, "home-initial") + published = verify_registry(self.registry, packages) + initial_registry = tree_digest(self.registry) + if not initial_registry: + raise AssertionError("initial publication produced no registry objects") + + self.log("== Identical retry must be byte-idempotent ==") + self.publish(source, "home-idempotent") + idempotent_registry = tree_digest(self.registry) + if idempotent_registry != initial_registry: + raise AssertionError( + "identical same-version retry changed registry bytes or metadata" + ) + verify_registry(self.registry, packages) + + self.log("== Same identity with changed bytes must fail closed ==") + tampered = self.copy_fixture(self.root / "tampered") + mutation = choose_packaged_source_file(tampered, published) + mutate_source_file(mutation) + self.log( + f"mutated packaged source path: {mutation.relative_to(tampered).as_posix()}" + ) + + tampered_pack = self.root / "tampered-pack" + self.zed_cmd( + "pack", + "--out", + tampered_pack, + cwd=tampered, + home=self.root / "home-tampered-pack", + ) + packed_files = [ + path + for path in tampered_pack.rglob("*") + if path.is_file() + and (path.name.endswith(".tar.gz") or path.name.endswith(".zip")) + ] + if not packed_files: + packed_files = [ + path for path in tampered_pack.rglob("*") if path.is_file() + ] + tampered_digests = { + hashlib.sha256(path.read_bytes()).hexdigest() for path in packed_files + } + original_digests = {artifact.sha256 for artifact in published} + if not tampered_digests: + raise AssertionError("tampered source produced no packed artifact") + if tampered_digests.issubset(original_digests): + raise AssertionError( + "source mutation did not change the deterministic packed artifact" + ) + + before_conflict = tree_digest(self.registry) + output = self.publish(tampered, "home-conflict", should_fail=True) + after_conflict = tree_digest(self.registry) + if after_conflict != before_conflict: + raise AssertionError( + "rejected same-version mutation changed registry bytes; " + "publication was not atomic" + ) + + lowered = output.lower() + explanatory_terms = ( + "already", + "conflict", + "different", + "exists", + "immutable", + "mismatch", + "published", + ) + if output.strip() and not any(term in lowered for term in explanatory_terms): + raise AssertionError( + "conflicting publication failed without an immutability-oriented " + "diagnostic" + ) + + verify_registry(self.registry, packages) + self.assert_fixture_clean("publication immutability certification") + self.write_summary(packages, mutation, published) + self.log("\nPASS: immutable publication contract") + + def write_summary( + self, + packages: list[PackageRef], + mutation: Path, + published: list[PublishedArtifact], + ) -> None: + payload = { + "schema": "zed.publication-immutability-canary/v1", + "registry": self.registry_url, + "packages": [package.spec for package in packages], + "artifacts": [ + { + "package": artifact.package.spec, + "sha256": artifact.sha256, + "path": artifact.artifact_path.relative_to(self.registry).as_posix(), + } + for artifact in published + ], + "mutated_source": mutation.relative_to(self.root / "tampered").as_posix(), + "identical_retry": "idempotent", + "changed_bytes_retry": "rejected-without-registry-mutation", + } + summary_path = self.root / "summary.json" + summary_path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + github_summary = os.environ.get("GITHUB_STEP_SUMMARY") + if github_summary: + with Path(github_summary).open("a", encoding="utf-8") as handle: + handle.write( + "### Zed publication immutability\n\n" + f"- package outputs: `{len(packages)}`\n" + "- identical same-version retry: **idempotent**\n" + "- changed-byte same-version retry: **rejected**\n" + "- registry bytes after rejection: **unchanged**\n\n" + ) + + +def read_manifest(directory: Path) -> dict: + with (directory / ".zpkg.toml").open("rb") as handle: + return tomllib.load(handle) + + +def expected_packages(manifest: dict) -> list[PackageRef]: + package = manifest.get("package") + if not isinstance(package, dict): + raise AssertionError("manifest has no [package] table") + org = str(package.get("org") or "") + name = str(package.get("name") or "") + version = str(package.get("version") or "") + if not org or not name or not version: + raise AssertionError("manifest package identity is incomplete") + + targets = manifest.get("targets") or {} + if not targets: + return [PackageRef(org, name, version)] + if not isinstance(targets, dict): + raise AssertionError("manifest [targets] must be a table") + + result: list[PackageRef] = [] + for key, value in sorted(targets.items()): + target = value or {} + if not isinstance(target, dict): + raise AssertionError(f"target {key!r} must be a table") + target_name = str(target.get("name") or f"{name}-{key}") + result.append(PackageRef(org, target_name, version)) + return result + + +def verify_registry( + registry: Path, packages: list[PackageRef] +) -> list[PublishedArtifact]: + result: list[PublishedArtifact] = [] + for package in packages: + metadata_path = ( + registry + / "packages" + / package.org + / package.name + / "versions" + / f"{package.version}.json" + ) + if not metadata_path.is_file(): + raise AssertionError( + f"missing registry metadata for {package.spec}: {metadata_path}" + ) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if metadata.get("org") != package.org: + raise AssertionError(f"wrong org in {metadata_path}") + if metadata.get("name") != package.name: + raise AssertionError(f"wrong name in {metadata_path}") + if str(metadata.get("version")) != package.version: + raise AssertionError(f"wrong version in {metadata_path}") + + digest = str(metadata.get("sha256") or "") + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise AssertionError( + f"invalid artifact sha256 for {package.spec}: {digest!r}" + ) + artifact_path = registry / "artifacts" / f"{digest}.tar.gz" + if not artifact_path.is_file(): + raise AssertionError( + f"artifact referenced by {package.spec} is missing: {artifact_path}" + ) + actual = hashlib.sha256(artifact_path.read_bytes()).hexdigest() + if actual != digest: + raise AssertionError( + f"artifact digest mismatch for {package.spec}: {actual} != {digest}" + ) + result.append( + PublishedArtifact(package, metadata_path, artifact_path, digest) + ) + return result + + +def choose_packaged_source_file( + source: Path, artifacts: list[PublishedArtifact] +) -> Path: + archived_names: set[str] = set() + for artifact in artifacts: + with tarfile.open(artifact.artifact_path, mode="r:gz") as archive: + for member in archive.getmembers(): + if member.isfile(): + name = PurePosixPath(member.name).as_posix().lstrip("./") + archived_names.add(name) + + candidates: list[Path] = [] + for path in source.rglob("*"): + if not path.is_file() or path.is_symlink(): + continue + relative = path.relative_to(source).as_posix() + if any(part in IGNORED_SOURCE_NAMES for part in path.relative_to(source).parts): + continue + if any( + archived == relative or archived.endswith(f"/{relative}") + for archived in archived_names + ): + candidates.append(path) + + if not candidates: + raise AssertionError( + "could not map any archived artifact member back to a source file" + ) + + def priority(path: Path) -> tuple[int, str]: + relative = path.relative_to(source).as_posix() + name = path.name.lower() + if name.startswith("readme"): + rank = 0 + elif path.suffix.lower() in {".md", ".txt"}: + rank = 1 + elif path.suffix.lower() in { + ".c", + ".cc", + ".cpp", + ".dart", + ".ex", + ".exs", + ".gleam", + ".go", + ".h", + ".hpp", + ".java", + ".js", + ".kt", + ".mjs", + ".py", + ".rb", + ".rs", + ".ts", + ".zig", + }: + rank = 2 + elif name == ".zpkg.toml": + rank = 4 + else: + rank = 3 + return rank, relative + + return min(candidates, key=priority) + + +def mutate_source_file(path: Path) -> None: + suffix = path.suffix.lower() + if path.name == ".zpkg.toml" or suffix == ".toml": + marker = b"\n# zed publication immutability canary\n" + elif suffix in {".py", ".rb"}: + marker = b"\n# zed publication immutability canary\n" + elif suffix in { + ".c", + ".cc", + ".cpp", + ".dart", + ".ex", + ".exs", + ".gleam", + ".go", + ".h", + ".hpp", + ".java", + ".js", + ".kt", + ".mjs", + ".rs", + ".ts", + ".zig", + }: + marker = b"\n// zed publication immutability canary\n" + else: + marker = b"\nzed publication immutability canary\n" + path.write_bytes(path.read_bytes() + marker) + + +def tree_digest(root: Path) -> tuple[tuple[str, int, str], ...]: + if not root.exists(): + return () + rows: list[tuple[str, int, str]] = [] + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + rows.append((relative, -1, f"symlink:{os.readlink(path)}")) + elif path.is_file(): + data = path.read_bytes() + rows.append((relative, len(data), hashlib.sha256(data).hexdigest())) + return tuple(rows) + + +def shell_quote(value: str) -> str: + if not value: + return "''" + if re.fullmatch(r"[A-Za-z0-9_@%+=:,./-]+", value): + return value + return "'" + value.replace("'", "'\\''") + "'" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--fixture-dir", type=Path, required=True) + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + harness = Harness(parse_args()) + try: + harness.run_contract() + return 0 + except Exception as error: # noqa: BLE001 - CI boundary preserves evidence. + harness.log(f"\nFAIL: publication immutability: {error}") + raise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/publish_ignore_union.py b/scripts/publish_ignore_union.py new file mode 100644 index 00000000..5514b5b5 --- /dev/null +++ b/scripts/publish_ignore_union.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Real-CLI certification for DEN-3017 publish-ignore semantics.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import tarfile +import tempfile +from pathlib import Path +from typing import Iterable + +WARNING_PREFIX = "warning: both [publish].exclude and .zedignore define publish rules" + + +def manifest(name: str, publish_rules: Iterable[str] = (), extra: str = "") -> str: + rules = list(publish_rules) + publish = "" + if rules: + encoded = ", ".join(f'"{rule}"' for rule in rules) + publish = f"\n[publish]\nexclude = [{encoded}]\n" + return f'''[package] +org = "den3017" +name = "{name}" +version = "1.0.0" + +[package.repository] +url = "https://example.invalid/den3017/{name}" +{publish}{extra} +''' + + +def write_package(root: Path, name: str, rules: Iterable[str] = (), extra: str = "") -> None: + root.mkdir(parents=True, exist_ok=True) + (root / ".zpkg.toml").write_text(manifest(name, rules, extra), encoding="utf-8") + (root / "LICENSE").write_text("MIT\n", encoding="utf-8") + + +def pack(zed: Path, root: Path, *, success: bool = True) -> tuple[subprocess.CompletedProcess[str], set[str]]: + out = root / "artifacts" + out.mkdir(exist_ok=True) + env = os.environ.copy() + env["NO_COLOR"] = "1" + proc = subprocess.run( + [str(zed), "pack", "--out", str(out)], + cwd=root, + env=env, + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if success and proc.returncode != 0: + raise AssertionError( + f"zed pack failed in {root.name}:\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}" + ) + if not success: + if proc.returncode == 0: + raise AssertionError(f"zed pack unexpectedly succeeded in {root.name}") + return proc, set() + + archives = sorted(out.glob("*.tar.gz")) + if len(archives) != 1: + raise AssertionError(f"expected one tar.gz in {out}, found {archives}") + with tarfile.open(archives[0], "r:gz") as archive: + entries = {member.name.replace("\\", "/") for member in archive.getmembers()} + return proc, entries + + +def pack_polyglot(zed: Path, root: Path) -> tuple[subprocess.CompletedProcess[str], list[set[str]]]: + out = root / "artifacts" + out.mkdir(exist_ok=True) + proc = subprocess.run( + [str(zed), "pack", "--out", str(out)], + cwd=root, + env={**os.environ, "NO_COLOR": "1"}, + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if proc.returncode != 0: + raise AssertionError(f"polyglot pack failed:\n{proc.stdout}\n{proc.stderr}") + archives = sorted(out.glob("*.tar.gz")) + if len(archives) != 2: + raise AssertionError(f"expected two polyglot archives, found {archives}") + entries: list[set[str]] = [] + for path in archives: + with tarfile.open(path, "r:gz") as archive: + entries.append({member.name.replace("\\", "/") for member in archive.getmembers()}) + return proc, entries + + +def warning_count(proc: subprocess.CompletedProcess[str]) -> int: + return proc.stderr.count(WARNING_PREFIX) + + +def assert_present(entries: set[str], *paths: str) -> None: + missing = [path for path in paths if path not in entries] + if missing: + raise AssertionError(f"archive is missing {missing}; entries={sorted(entries)}") + + +def assert_absent(entries: set[str], *paths: str) -> None: + leaked = [path for path in paths if path in entries] + if leaked: + raise AssertionError(f"archive leaked {leaked}; entries={sorted(entries)}") + + +def scenario_ignore_only(zed: Path, base: Path) -> None: + root = base / "ignore-only" + write_package(root, "ignore-only") + (root / "keep.txt").write_text("keep", encoding="utf-8") + (root / ".env.local").write_text("secret", encoding="utf-8") + (root / ".hidden").mkdir() + (root / ".hidden" / "cache.bin").write_text("cache", encoding="utf-8") + (root / ".zedignore").write_text(".env*\n.hidden/**\n", encoding="utf-8") + proc, entries = pack(zed, root) + assert warning_count(proc) == 0, proc.stderr + assert_present(entries, "pkg/keep.txt", "pkg/.zpkg.toml", "pkg/LICENSE") + assert_absent(entries, "pkg/.env.local", "pkg/.hidden/cache.bin", "pkg/.zedignore") + + +def scenario_manifest_only(zed: Path, base: Path) -> None: + root = base / "manifest-only" + write_package(root, "manifest-only", ["manifest-only.txt"]) + (root / "keep.txt").write_text("keep", encoding="utf-8") + (root / "manifest-only.txt").write_text("drop", encoding="utf-8") + proc, entries = pack(zed, root) + assert warning_count(proc) == 0, proc.stderr + assert_present(entries, "pkg/keep.txt") + assert_absent(entries, "pkg/manifest-only.txt") + + +def scenario_union(zed: Path, base: Path) -> None: + root = base / "union" + write_package(root, "union", ["manifest-only.txt"]) + for name in ["keep.txt", "manifest-only.txt", "ignore-only.txt"]: + (root / name).write_text(name, encoding="utf-8") + (root / ".zedignore").write_text("# local output\n\nignore-only.txt\n", encoding="utf-8") + proc, entries = pack(zed, root) + assert warning_count(proc) == 1, proc.stderr + assert "contradictory paths" not in proc.stderr, proc.stderr + assert_present(entries, "pkg/keep.txt") + assert_absent(entries, "pkg/manifest-only.txt", "pkg/ignore-only.txt", "pkg/.zedignore") + + +def scenario_sorted_conflict(zed: Path, base: Path) -> None: + root = base / "sorted-conflict" + write_package(root, "sorted-conflict", ["dist/**", "cache/**"]) + for directory in ["dist", "cache"]: + (root / directory).mkdir() + (root / directory / "value.txt").write_text(directory, encoding="utf-8") + (root / ".zedignore").write_text("!dist\n!cache\n", encoding="utf-8") + proc, entries = pack(zed, root) + assert warning_count(proc) == 1, proc.stderr + assert "contradictory paths kept excluded: cache, dist" in proc.stderr, proc.stderr + assert_absent(entries, "pkg/dist/value.txt", "pkg/cache/value.txt") + + +def scenario_reverse_conflict(zed: Path, base: Path) -> None: + root = base / "reverse-conflict" + write_package(root, "reverse-conflict", ["!secret"]) + (root / "secret").mkdir() + (root / "secret" / "value.txt").write_text("secret", encoding="utf-8") + (root / ".zedignore").write_text("secret/**\n", encoding="utf-8") + proc, entries = pack(zed, root) + assert warning_count(proc) == 1, proc.stderr + assert "contradictory paths kept excluded: secret" in proc.stderr, proc.stderr + assert_absent(entries, "pkg/secret/value.txt") + + +def scenario_source_local_negation(zed: Path, base: Path) -> None: + manifest_root = base / "manifest-negation" + write_package(manifest_root, "manifest-negation", ["scratch/**", "!scratch"]) + (manifest_root / "scratch").mkdir() + (manifest_root / "scratch" / "value.txt").write_text("kept", encoding="utf-8") + proc, entries = pack(zed, manifest_root) + assert warning_count(proc) == 0, proc.stderr + assert_present(entries, "pkg/scratch/value.txt") + + ignore_root = base / "ignore-negation" + write_package(ignore_root, "ignore-negation") + (ignore_root / "scratch").mkdir() + (ignore_root / "scratch" / "value.txt").write_text("kept", encoding="utf-8") + (ignore_root / ".zedignore").write_text("scratch/**\n!scratch\n", encoding="utf-8") + proc, entries = pack(zed, ignore_root) + assert warning_count(proc) == 0, proc.stderr + assert_present(entries, "pkg/scratch/value.txt") + assert_absent(entries, "pkg/.zedignore") + + +def scenario_invalid_glob(zed: Path, base: Path) -> None: + root = base / "invalid-glob" + write_package(root, "invalid-glob") + (root / "keep.txt").write_text("keep", encoding="utf-8") + (root / ".zedignore").write_text("[broken\n", encoding="utf-8") + proc, _ = pack(zed, root, success=False) + if "invalid glob pattern" not in proc.stderr: + raise AssertionError(f"invalid glob error was not actionable:\n{proc.stderr}") + + +def scenario_polyglot(zed: Path, base: Path) -> None: + root = base / "polyglot" + (root / "clients" / "ts" / "src").mkdir(parents=True) + (root / "clients" / "java" / "src").mkdir(parents=True) + (root / "LICENSE").write_text("MIT\n", encoding="utf-8") + (root / "clients" / "ts" / "package.json").write_text('{"name":"@den3017/client"}', encoding="utf-8") + (root / "clients" / "ts" / "src" / "index.js").write_text("export {};", encoding="utf-8") + (root / "clients" / "ts" / "private.key").write_text("private", encoding="utf-8") + (root / "clients" / "ts" / ".zedignore").write_text("private.key\n", encoding="utf-8") + (root / "clients" / "java" / "pom.xml").write_text("", encoding="utf-8") + (root / "clients" / "java" / "src" / "Client.java").write_text("class Client {}", encoding="utf-8") + (root / ".zpkg.toml").write_text( + manifest( + "polyglot", + extra=''' +[targets.nodejs] +dir = "clients/ts" +adapter = "node" + +[targets.java] +dir = "clients/java" +adapter = "java" +''', + ), + encoding="utf-8", + ) + proc, archives = pack_polyglot(zed, root) + assert warning_count(proc) == 0, proc.stderr + node = next(entries for entries in archives if "pkg/package.json" in entries) + java = next(entries for entries in archives if "pkg/pom.xml" in entries) + assert_present(node, "pkg/package.json", "pkg/src/index.js", "pkg/LICENSE") + assert_absent(node, "pkg/private.key", "pkg/.zedignore", "pkg/pom.xml") + assert_present(java, "pkg/pom.xml", "pkg/src/Client.java", "pkg/LICENSE") + assert_absent(java, "pkg/package.json") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", required=True, type=Path) + args = parser.parse_args() + zed = args.zed.resolve() + if not zed.is_file(): + raise SystemExit(f"zed binary not found: {zed}") + + with tempfile.TemporaryDirectory(prefix="den-3017-") as temp: + base = Path(temp) + scenarios = [ + scenario_ignore_only, + scenario_manifest_only, + scenario_union, + scenario_sorted_conflict, + scenario_reverse_conflict, + scenario_source_local_negation, + scenario_invalid_glob, + scenario_polyglot, + ] + for scenario in scenarios: + scenario(zed, base) + print(f"ok: {scenario.__name__}") + print("DEN-3017 real-CLI certification passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/read-fleet-manifest.mjs b/scripts/read-fleet-manifest.mjs new file mode 100644 index 00000000..49ce3dd9 --- /dev/null +++ b/scripts/read-fleet-manifest.mjs @@ -0,0 +1,18 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import zlib from 'node:zlib'; + +import { applyFleetManifestExtensions } from './fleet-manifest-extensions.mjs'; + +export function readFleetManifest(manifestPath) { + let bytes; + if (fs.existsSync(manifestPath)) { + bytes = fs.readFileSync(manifestPath); + } else { + const partsDir = `${manifestPath}.b64.parts`; + const encoded = fs.readdirSync(partsDir).sort().map((name) => fs.readFileSync(path.join(partsDir, name), 'utf8').trim()).join(''); + bytes = Buffer.from(encoded, 'base64'); + } + const manifest = JSON.parse((manifestPath.endsWith('.gz') ? zlib.gunzipSync(bytes) : bytes).toString('utf8')); + return applyFleetManifestExtensions(manifest, manifestPath); +} diff --git a/scripts/real_k8s_gitops_contract.py b/scripts/real_k8s_gitops_contract.py new file mode 100644 index 00000000..247efb0b --- /dev/null +++ b/scripts/real_k8s_gitops_contract.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Iterable, Sequence + + +SENSITIVE_ENV_FRAGMENTS = ( + "TOKEN", + "SECRET", + "PASSWORD", + "PRIVATE_KEY", + "ACCESS_KEY", + "API_KEY", + "AUTHORIZATION", + "COOKIE", +) + + +class ContractError(RuntimeError): + pass + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ContractError(message) + + +def sanitized_environment() -> dict[str, str]: + environment = dict(os.environ) + for key in list(environment): + upper = key.upper() + if any(fragment in upper for fragment in SENSITIVE_ENV_FRAGMENTS): + environment.pop(key, None) + elif upper.startswith(("ZED_PKG_", "ZED_EXTERNAL_")): + environment.pop(key, None) + environment["CI"] = "true" + return environment + + +def run( + command: Sequence[object], + *, + environment: dict[str, str], + cwd: Path | None = None, + expected: int | Iterable[int] = 0, +) -> subprocess.CompletedProcess[str]: + expected_codes = {expected} if isinstance(expected, int) else set(expected) + result = subprocess.run( + [str(item) for item in command], + cwd=cwd, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=60, + check=False, + ) + if result.returncode not in expected_codes: + rendered = " ".join(repr(str(item)) for item in command) + raise ContractError( + f"command returned {result.returncode}, expected {sorted(expected_codes)}: {rendered}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + +def parse_json_output(result: subprocess.CompletedProcess[str], label: str) -> dict: + try: + value = json.loads(result.stdout) + except json.JSONDecodeError as error: + raise ContractError( + f"{label} did not emit JSON: {error}\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) from error + require(isinstance(value, dict), f"{label} output must be a JSON object") + return value + + +def canonical_digest(value: object) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def git_head(root: Path, environment: dict[str, str]) -> str: + result = run( + ["git", "-C", root, "rev-parse", "HEAD"], environment=environment + ) + return result.stdout.strip() + + +def certify(args: argparse.Namespace) -> dict[str, object]: + environment = sanitized_environment() + zed = args.zed.resolve(strict=True) + zed_gitops = args.zed_gitops.resolve(strict=True) + cluster = args.cluster.resolve(strict=True) + + require( + git_head(cluster, environment) == args.cluster_commit, + "k8s-cluster checkout does not match the pinned commit", + ) + product = args.product.resolve(strict=True) + require( + git_head(product, environment) == args.product_commit, + "zed-cli checkout does not match the pinned commit", + ) + + checks: list[str] = ["immutable-checkouts-match"] + + native_check = parse_json_output( + run( + [ + sys.executable, + cluster / "tools/gitops_composition.py", + "check", + "--root", + cluster, + "--format", + "json", + ], + environment=environment, + ), + "native k8s-cluster validator", + ) + require(native_check.get("valid") is True, f"native report invalid: {native_check}") + require(native_check.get("errors") == 0, f"native errors: {native_check}") + require( + isinstance(native_check.get("records"), int) and native_check["records"] > 0, + f"native validator found no records: {native_check}", + ) + checks.append("native-k8s-validator-passes") + + native_render = parse_json_output( + run( + [ + sys.executable, + cluster / "tools/gitops_composition.py", + "render", + "--root", + cluster, + ], + environment=environment, + ), + "native k8s-cluster renderer", + ) + require( + native_render.get("kind") == "GitOpsApplicationPreviewList", + f"unexpected preview kind: {native_render}", + ) + rendered_items = native_render.get("items") + require(isinstance(rendered_items, list), "preview items must be a list") + require( + len(rendered_items) == native_check["records"], + "preview count does not match validated record count", + ) + checks.append("native-preview-is-deterministic-and-complete") + + validator_args = [ + "validate", + "--root", + cluster, + "--offline", + "--strict", + "--format", + "json", + ] + direct_report = parse_json_output( + run([zed_gitops, *validator_args], environment=environment), + "standalone zed-gitops validator", + ) + root_report = parse_json_output( + run([zed, "gitops", *validator_args], environment=environment), + "root-dispatched zed gitops validator", + ) + require(direct_report == root_report, "root and standalone reports differ") + require(root_report.get("valid") is True, f"zed report invalid: {root_report}") + require(root_report.get("errors") == 0, f"zed report has errors: {root_report}") + require( + root_report.get("records") == native_check["records"], + "native and Zed validators disagree on catalog record count", + ) + checks.append("root-and-standalone-zed-validation-match") + checks.append("native-and-zed-record-counts-match") + + record_path = cluster / "catalog/gitops/apps/dd-fabrication-server.json" + record = json.loads(record_path.read_text(encoding="utf-8")) + spec = record.get("spec", {}) + inventory = spec.get("inventory", {}) + source = spec.get("source", {}) + migration = spec.get("migration", {}) + require(inventory.get("mode") == "git-submodule", "inventory mode drifted") + require(source.get("mode") == "direct-repository", "source mode drifted") + require( + inventory.get("revision") == source.get("targetRevision"), + "inventory gitlink and direct source revisions differ", + ) + require(migration.get("phase") == "pilot-inert", "pilot is no longer inert") + checks.append("catalog-preserves-exact-pin-and-inert-migration") + + application_set_path = ( + cluster + / "remote/argocd/application-sets/gitops-composition-catalog-pilot.applicationset.yaml" + ) + application_set = application_set_path.read_text(encoding="utf-8") + required_fragments = ( + "name: gitops-composition-catalog-pilot", + "oresoftware.dev/activation: inert-not-in-bootstrap", + "missingkey=error", + "path: catalog/gitops/apps/*.json", + "targetRevision: '{{ .spec.source.targetRevision }}'", + "name: 'catalog-pilot-{{ .metadata.name }}'", + ) + for fragment in required_fragments: + require(fragment in application_set, f"ApplicationSet contract omitted: {fragment}") + checks.append("applicationset-template-remains-fail-closed") + + references: list[str] = [] + for path in (cluster / "remote/argocd").rglob("*"): + if not path.is_file() or path == application_set_path: + continue + if path.suffix.lower() not in {".yaml", ".yml", ".json"}: + continue + text = path.read_text(encoding="utf-8", errors="replace") + if ( + "gitops-composition-catalog-pilot" in text + or application_set_path.name in text + ): + references.append(str(path.relative_to(cluster))) + require(not references, f"pilot is referenced by live Argo material: {references}") + checks.append("applicationset-pilot-is-unreferenced") + + return { + "$schema": "zed-pkg-test/real-k8s-gitops-contract/v1", + "productCommit": args.product_commit, + "clusterCommit": args.cluster_commit, + "result": "passed", + "recordCount": root_report["records"], + "renderedApplicationCount": len(rendered_items), + "nativeReportSha256": canonical_digest(native_check), + "zedReportSha256": canonical_digest(root_report), + "previewSha256": canonical_digest(native_render), + "checkCount": len(checks), + "checks": checks, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--zed-gitops", type=Path, required=True) + parser.add_argument("--product", type=Path, required=True) + parser.add_argument("--cluster", type=Path, required=True) + parser.add_argument("--product-commit", required=True) + parser.add_argument("--cluster-commit", required=True) + parser.add_argument("--evidence", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + evidence = certify(args) + except (ContractError, OSError, subprocess.SubprocessError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text( + json.dumps(evidence, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print( + f"certified {evidence['checkCount']} real GitOps checks across " + f"{evidence['recordCount']} catalog record(s)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/recursive_installs.py b/scripts/recursive_installs.py new file mode 100644 index 00000000..53932921 --- /dev/null +++ b/scripts/recursive_installs.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +"""Hermetic end-to-end coverage for recursive zed installs and shared-home locks.""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +import tomllib +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from threading import Barrier +from typing import Mapping, Sequence + +ORG = "zed-recursive-test" +VERSION = "1.0.0" +PACKAGE_GRAPH: dict[str, tuple[str, ...]] = { + "leaf": (), + "left": ("leaf",), + "right": ("leaf",), + "root": ("left", "right"), +} +PREFETCH_RE = re.compile( + r"recursive install prefetch: (?P\d+) package\(s\), " + r"up to (?P\d+) concurrent, (?P\d+) downloaded" +) + + +@dataclass(frozen=True) +class Completed: + command: tuple[str, ...] + returncode: int + output: str + + +def shell_quote(value: str) -> str: + if re.fullmatch(r"[A-Za-z0-9_./:@%+=,-]+", value): + return value + return "'" + value.replace("'", "'\"'\"'") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, +) -> Completed: + argv = tuple(str(value) for value in command) + shown = " ".join(shell_quote(value) for value in argv) + print(f"\n$ (cd {cwd or Path.cwd()} && {shown})", flush=True) + completed = subprocess.run( + argv, + cwd=cwd, + env=dict(env) if env is not None else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + timeout=180, + ) + if completed.stdout: + print(completed.stdout, end="", flush=True) + return Completed(argv, completed.returncode, completed.stdout) + + +def require_success(completed: Completed) -> None: + if completed.returncode != 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise RuntimeError(f"command failed with exit code {completed.returncode}: {shown}") + + +def write_package(root: Path, name: str, dependencies: Sequence[str]) -> None: + root.mkdir(parents=True, exist_ok=True) + lines = [ + "[package]", + f'org = "{ORG}"', + f'name = "{name}"', + f'version = "{VERSION}"', + 'description = "recursive install e2e fixture"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://github.com/zed-pkg-test/zed-pkg-e2e"', + ] + if dependencies: + lines.extend(("", "[dependencies]")) + for dependency in dependencies: + lines.append(f'"{ORG}/{dependency}" = "={VERSION}"') + (root / ".zpkg.toml").write_text("\n".join(lines) + "\n", encoding="utf-8") + (root / "payload.txt").write_text(f"{ORG}/{name}@{VERSION}\n", encoding="utf-8") + + +def write_consumer(root: Path, dependencies: Sequence[str]) -> None: + root.mkdir(parents=True, exist_ok=True) + lines = [ + "[package]", + 'org = "recursive-consumer"', + f'name = "{root.name}"', + 'version = "0.1.0"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://example.invalid/recursive-consumer"', + "", + "[dependencies]", + ] + for dependency in dependencies: + lines.append(f'"{ORG}/{dependency}" = "={VERSION}"') + lines.append("") + (root / ".zpkg.toml").write_text("\n".join(lines), encoding="utf-8") + + +def zed_command(zed: Path, registry: Path, home: Path, *args: str) -> list[str | Path]: + return [ + zed, + "--registry", + f"file://{registry}", + "--home", + home, + *args, + ] + + +def package_keys(names: Sequence[str]) -> set[str]: + return {f"{ORG}/{name}" for name in names} + + +def assert_manifest_dependencies(consumer: Path, expected_names: Sequence[str]) -> None: + with (consumer / ".zpkg.toml").open("rb") as handle: + document = tomllib.load(handle) + actual = set((document.get("dependencies") or {}).keys()) + expected = package_keys(expected_names) + if actual != expected: + raise AssertionError( + f"manifest dependency mismatch: expected {sorted(expected)}, got {sorted(actual)}" + ) + + +def assert_lock_packages(consumer: Path, expected_names: Sequence[str]) -> None: + with (consumer / ".zpkg.lock").open("rb") as handle: + document = tomllib.load(handle) + packages = document.get("package") or [] + actual = {f"{package['org']}/{package['name']}" for package in packages} + expected = package_keys(expected_names) + if actual != expected: + raise AssertionError( + f"lock graph mismatch: expected {sorted(expected)}, got {sorted(actual)}" + ) + + +def assert_symlink_packages( + consumer: Path, + store_root: Path, + expected_names: Sequence[str], +) -> dict[str, Path]: + targets: dict[str, Path] = {} + for name in expected_names: + installed = consumer / "zed_modules" / ORG / name + if not installed.is_symlink(): + raise AssertionError(f"default install did not symlink {installed}") + target = installed.resolve(strict=True) + try: + target.relative_to(store_root) + except ValueError as error: + raise AssertionError( + f"{installed} resolves outside the shared store: {target}" + ) from error + if (target / "payload.txt").read_text(encoding="utf-8") != f"{ORG}/{name}@{VERSION}\n": + raise AssertionError(f"unexpected payload for {installed}") + targets[name] = target + return targets + + +def assert_not_materialized(consumer: Path, names: Sequence[str]) -> None: + for name in names: + installed = consumer / "zed_modules" / ORG / name + if installed.exists() or installed.is_symlink(): + raise AssertionError(f"unexpected materialized package {installed}") + + +def assert_artifact_home(home: Path, expected_count: int) -> None: + cache_files = sorted((home / "cache").glob("*.tar.gz")) + if len(cache_files) != expected_count: + raise AssertionError( + f"expected {expected_count} cached artifacts in {home}, got {cache_files}" + ) + temporary_downloads = list((home / "cache").glob("**/artifact.download")) + if temporary_downloads: + raise AssertionError( + f"temporary downloads leaked into {home}: {temporary_downloads}" + ) + lock_files = sorted((home / "locks").glob("artifact-*.lock")) + if len(lock_files) != expected_count: + raise AssertionError( + f"expected {expected_count} artifact lock files in {home}, got {lock_files}" + ) + + +def parse_prefetch(output: str) -> tuple[int, int, int]: + matches = list(PREFETCH_RE.finditer(output)) + if len(matches) != 1: + raise AssertionError( + f"expected one recursive prefetch summary, got {len(matches)}\n{output}" + ) + match = matches[0] + return tuple( + int(match.group(field)) + for field in ("resolved", "concurrency", "downloaded") + ) + + +def assert_prefetch( + completed: Completed, + *, + resolved: int, + downloaded: int, +) -> tuple[int, int, int]: + summary = parse_prefetch(completed.output) + expected = (resolved, 5, downloaded) + if summary != expected: + raise AssertionError(f"prefetch summary mismatch: expected {expected}, got {summary}") + return summary + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + + zed = args.zed.resolve() + root = args.work_root.resolve() + if not zed.is_file(): + raise RuntimeError(f"zed binary does not exist: {zed}") + if root.exists(): + raise RuntimeError(f"work root must be fresh: {root}") + + registry = root / "registry" + publish_home = root / "publish-home" + shared_home = root / "shared-home" + add_home = root / "add-home" + remove_home = root / "remove-home" + sources = root / "sources" + consumers = [root / "consumer-a", root / "consumer-b"] + add_consumer = root / "consumer-add" + remove_consumer = root / "consumer-remove" + registry.mkdir(parents=True) + + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "RUST_BACKTRACE": "1", + "ZED_PKG_INSTALL_CONCURRENCY": "5", + } + ) + + for name, dependencies in PACKAGE_GRAPH.items(): + package_root = sources / name + write_package(package_root, name, dependencies) + completed = run( + zed_command( + zed, + registry, + publish_home, + "publish", + "--skip-vcs-checks", + ), + cwd=package_root, + env=environment, + ) + require_success(completed) + + for consumer in consumers: + write_consumer(consumer, ("root",)) + + start = Barrier(3) + + def install(consumer: Path) -> Completed: + start.wait() + return run( + zed_command( + zed, + registry, + shared_home, + "install", + "--adapter", + "none", + ), + cwd=consumer, + env=environment, + ) + + with ThreadPoolExecutor(max_workers=2, thread_name_prefix="zed-e2e") as executor: + futures = [executor.submit(install, consumer) for consumer in consumers] + start.wait() + completed_installs = [future.result(timeout=240) for future in futures] + + for completed in completed_installs: + require_success(completed) + + summaries = [parse_prefetch(completed.output) for completed in completed_installs] + if any(resolved != len(PACKAGE_GRAPH) for resolved, _, _ in summaries): + raise AssertionError( + f"each recursive resolver must discover four packages: {summaries}" + ) + if any(concurrency != 5 for _, concurrency, _ in summaries): + raise AssertionError( + f"installer did not honor five-worker concurrency: {summaries}" + ) + if sum(downloaded for _, _, downloaded in summaries) != len(PACKAGE_GRAPH): + raise AssertionError( + "shared-home installs did not deduplicate downloads across processes: " + f"{summaries}" + ) + + first_targets: dict[str, Path] | None = None + store_root = (shared_home / "store").resolve() + all_packages = tuple(PACKAGE_GRAPH) + for consumer in consumers: + assert_manifest_dependencies(consumer, ("root",)) + assert_lock_packages(consumer, all_packages) + targets = assert_symlink_packages(consumer, store_root, all_packages) + if first_targets is None: + first_targets = targets + elif targets != first_targets: + raise AssertionError( + f"consumers did not share identical store entries: {first_targets} != {targets}" + ) + assert_artifact_home(shared_home, len(PACKAGE_GRAPH)) + + # `zed add` installs against an in-memory proposed manifest before writing + # it. This must enter the same recursive prefetch facade as a normal install. + write_consumer(add_consumer, ()) + add_completed = run( + zed_command( + zed, + registry, + add_home, + "add", + f"{ORG}/root@={VERSION}", + ), + cwd=add_consumer, + env=environment, + ) + require_success(add_completed) + add_summary = assert_prefetch( + add_completed, + resolved=len(PACKAGE_GRAPH), + downloaded=len(PACKAGE_GRAPH), + ) + assert_manifest_dependencies(add_consumer, ("root",)) + assert_lock_packages(add_consumer, all_packages) + assert_symlink_packages(add_consumer, (add_home / "store").resolve(), all_packages) + assert_artifact_home(add_home, len(PACKAGE_GRAPH)) + + # `zed remove` also installs an in-memory proposed manifest. Starting with + # root + left and removing root leaves left -> leaf, so exactly two packages + # should be recursively prefetched and materialized. + write_consumer(remove_consumer, ("root", "left")) + remove_completed = run( + zed_command( + zed, + registry, + remove_home, + "remove", + f"{ORG}/root", + ), + cwd=remove_consumer, + env=environment, + ) + require_success(remove_completed) + remove_summary = assert_prefetch(remove_completed, resolved=2, downloaded=2) + assert_manifest_dependencies(remove_consumer, ("left",)) + assert_lock_packages(remove_consumer, ("left", "leaf")) + assert_symlink_packages( + remove_consumer, + (remove_home / "store").resolve(), + ("left", "leaf"), + ) + assert_not_materialized(remove_consumer, ("root", "right")) + assert_artifact_home(remove_home, 2) + + print("\nrecursive install e2e passed", flush=True) + print(f"concurrent install summaries: {summaries}", flush=True) + print(f"add summary: {add_summary}", flush=True) + print(f"remove summary: {remove_summary}", flush=True) + print(f"shared store: {store_root}", flush=True) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 - CI entry point must print context. + print(f"error: {error}", file=sys.stderr, flush=True) + raise diff --git a/scripts/recursive_installs_stress.py b/scripts/recursive_installs_stress.py new file mode 100644 index 00000000..8b109af2 --- /dev/null +++ b/scripts/recursive_installs_stress.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +"""Stress recursive zed installs across processes and a manifestless frozen replay.""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tomllib +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from threading import Barrier +from typing import Mapping, Sequence + +ORG = "zed-recursive-stress" +VERSION = "1.0.0" +LEAVES = tuple(f"leaf-{index:02d}" for index in range(12)) +ROOT = "root" +ALL_PACKAGES = (*LEAVES, ROOT) +PREFETCH_RE = re.compile( + r"recursive install prefetch: (?P\d+) package\(s\), " + r"up to (?P\d+) concurrent, (?P\d+) downloaded" +) + + +@dataclass(frozen=True) +class Completed: + command: tuple[str, ...] + returncode: int + output: str + + +def shell_quote(value: str) -> str: + if re.fullmatch(r"[A-Za-z0-9_./:@%+=,-]+", value): + return value + return "'" + value.replace("'", "'\"'\"'") + "'" + + +def run( + command: Sequence[str | Path], + *, + cwd: Path | None = None, + env: Mapping[str, str] | None = None, +) -> Completed: + argv = tuple(str(value) for value in command) + shown = " ".join(shell_quote(value) for value in argv) + print(f"\n$ (cd {cwd or Path.cwd()} && {shown})", flush=True) + completed = subprocess.run( + argv, + cwd=cwd, + env=dict(env) if env is not None else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + timeout=240, + ) + if completed.stdout: + print(completed.stdout, end="", flush=True) + return Completed(argv, completed.returncode, completed.stdout) + + +def require_success(completed: Completed) -> None: + if completed.returncode != 0: + shown = " ".join(shell_quote(value) for value in completed.command) + raise RuntimeError(f"command failed with exit code {completed.returncode}: {shown}") + + +def write_package(root: Path, name: str, dependencies: Sequence[str]) -> None: + root.mkdir(parents=True, exist_ok=True) + lines = [ + "[package]", + f'org = "{ORG}"', + f'name = "{name}"', + f'version = "{VERSION}"', + 'description = "recursive install stress fixture"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://github.com/zed-pkg-test/zed-pkg-e2e"', + ] + if dependencies: + lines.extend(("", "[dependencies]")) + for dependency in dependencies: + lines.append(f'"{ORG}/{dependency}" = "={VERSION}"') + (root / ".zpkg.toml").write_text("\n".join(lines) + "\n", encoding="utf-8") + (root / "payload.txt").write_text(f"{ORG}/{name}@{VERSION}\n", encoding="utf-8") + + +def write_consumer(root: Path) -> None: + root.mkdir(parents=True, exist_ok=True) + (root / ".zpkg.toml").write_text( + "\n".join( + ( + "[package]", + 'org = "recursive-stress-consumer"', + f'name = "{root.name}"', + 'version = "0.1.0"', + "", + "[package.repository]", + 'vcs = "git"', + 'url = "https://example.invalid/recursive-stress-consumer"', + "", + "[dependencies]", + f'"{ORG}/{ROOT}" = "={VERSION}"', + "", + ) + ), + encoding="utf-8", + ) + + +def zed_command(zed: Path, registry: Path, home: Path, *args: str) -> list[str | Path]: + return [ + zed, + "--registry", + f"file://{registry}", + "--home", + home, + *args, + ] + + +def package_keys(names: Sequence[str]) -> set[str]: + return {f"{ORG}/{name}" for name in names} + + +def assert_manifest_root_only(consumer: Path) -> None: + with (consumer / ".zpkg.toml").open("rb") as handle: + document = tomllib.load(handle) + actual = set((document.get("dependencies") or {}).keys()) + expected = package_keys((ROOT,)) + if actual != expected: + raise AssertionError( + f"manifest dependency mismatch: expected {sorted(expected)}, got {sorted(actual)}" + ) + + +def assert_lock_packages(consumer: Path, expected_names: Sequence[str]) -> None: + with (consumer / ".zpkg.lock").open("rb") as handle: + document = tomllib.load(handle) + packages = document.get("package") or [] + actual = {f"{package['org']}/{package['name']}" for package in packages} + expected = package_keys(expected_names) + if actual != expected: + raise AssertionError( + f"lock graph mismatch: expected {sorted(expected)}, got {sorted(actual)}" + ) + + +def assert_symlink_packages( + consumer: Path, + store_root: Path, + expected_names: Sequence[str], +) -> dict[str, Path]: + targets: dict[str, Path] = {} + for name in expected_names: + installed = consumer / "zed_modules" / ORG / name + if not installed.is_symlink(): + raise AssertionError(f"default install did not symlink {installed}") + target = installed.resolve(strict=True) + try: + target.relative_to(store_root) + except ValueError as error: + raise AssertionError( + f"{installed} resolves outside the shared store: {target}" + ) from error + payload = (target / "payload.txt").read_text(encoding="utf-8") + if payload != f"{ORG}/{name}@{VERSION}\n": + raise AssertionError(f"unexpected payload for {installed}: {payload!r}") + targets[name] = target + return targets + + +def assert_artifact_home(home: Path, expected_count: int) -> None: + cache = home / "cache" + cache_files = sorted(cache.glob("*.tar.gz")) + if len(cache_files) != expected_count: + raise AssertionError( + f"expected {expected_count} cached artifacts in {home}, got {cache_files}" + ) + leaked_directories = sorted(path for path in cache.iterdir() if path.is_dir()) + if leaked_directories: + raise AssertionError(f"temporary cache directories leaked: {leaked_directories}") + temporary_downloads = list(cache.glob("**/artifact.download")) + if temporary_downloads: + raise AssertionError(f"temporary downloads leaked: {temporary_downloads}") + lock_files = sorted((home / "locks").glob("artifact-*.lock")) + if len(lock_files) != expected_count: + raise AssertionError( + f"expected {expected_count} artifact locks in {home}, got {lock_files}" + ) + + +def parse_prefetch(output: str) -> tuple[int, int, int]: + matches = list(PREFETCH_RE.finditer(output)) + if len(matches) != 1: + raise AssertionError( + f"expected one recursive prefetch summary, got {len(matches)}\n{output}" + ) + match = matches[0] + return tuple( + int(match.group(field)) + for field in ("resolved", "concurrency", "downloaded") + ) + + +def assert_prefetch( + completed: Completed, + *, + resolved: int, + downloaded: int, +) -> tuple[int, int, int]: + summary = parse_prefetch(completed.output) + expected = (resolved, 5, downloaded) + if summary != expected: + raise AssertionError(f"prefetch summary mismatch: expected {expected}, got {summary}") + return summary + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed", type=Path, required=True) + parser.add_argument("--work-root", type=Path, required=True) + args = parser.parse_args() + + zed = args.zed.resolve() + root = args.work_root.resolve() + if not zed.is_file(): + raise RuntimeError(f"zed binary does not exist: {zed}") + if root.exists(): + raise RuntimeError(f"work root must be fresh: {root}") + + registry = root / "registry" + publish_home = root / "publish-home" + shared_home = root / "shared-home" + frozen_home = root / "frozen-home" + sources = root / "sources" + consumers = [root / f"consumer-{index}" for index in range(4)] + frozen_consumer = root / "consumer-frozen" + registry.mkdir(parents=True) + + environment = os.environ.copy() + environment.update( + { + "CI": "true", + "GIT_TERMINAL_PROMPT": "0", + "RUST_BACKTRACE": "1", + "ZED_PKG_INSTALL_CONCURRENCY": "5", + } + ) + + for name in LEAVES: + package_root = sources / name + write_package(package_root, name, ()) + completed = run( + zed_command( + zed, + registry, + publish_home, + "publish", + "--skip-vcs-checks", + ), + cwd=package_root, + env=environment, + ) + require_success(completed) + + root_package = sources / ROOT + write_package(root_package, ROOT, LEAVES) + root_publish = run( + zed_command( + zed, + registry, + publish_home, + "publish", + "--skip-vcs-checks", + ), + cwd=root_package, + env=environment, + ) + require_success(root_publish) + + for consumer in consumers: + write_consumer(consumer) + + start = Barrier(len(consumers) + 1) + + def install(consumer: Path) -> Completed: + start.wait() + return run( + zed_command( + zed, + registry, + shared_home, + "install", + "--adapter", + "none", + ), + cwd=consumer, + env=environment, + ) + + with ThreadPoolExecutor( + max_workers=len(consumers), + thread_name_prefix="zed-recursive-stress", + ) as executor: + futures = [executor.submit(install, consumer) for consumer in consumers] + start.wait() + installs = [future.result(timeout=300) for future in futures] + + for completed in installs: + require_success(completed) + + summaries = [parse_prefetch(completed.output) for completed in installs] + expected_count = len(ALL_PACKAGES) + if any(resolved != expected_count for resolved, _, _ in summaries): + raise AssertionError(f"every process must resolve the complete graph: {summaries}") + if any(concurrency != 5 for _, concurrency, _ in summaries): + raise AssertionError(f"five-worker configuration was not honored: {summaries}") + if sum(downloaded for _, _, downloaded in summaries) != expected_count: + raise AssertionError( + "four processes did not deduplicate shared-home downloads: " + f"{summaries}" + ) + + first_targets: dict[str, Path] | None = None + shared_store = (shared_home / "store").resolve() + for consumer in consumers: + assert_manifest_root_only(consumer) + assert_lock_packages(consumer, ALL_PACKAGES) + targets = assert_symlink_packages(consumer, shared_store, ALL_PACKAGES) + if first_targets is None: + first_targets = targets + elif targets != first_targets: + raise AssertionError( + f"consumers did not share identical store entries: {first_targets} != {targets}" + ) + assert_artifact_home(shared_home, expected_count) + + # Replay the complete transitive lockfile in a fresh project with no + # manifest. This exercises the lock-only recursive prefetch facade and its + # normal symlink materialization path. + frozen_consumer.mkdir(parents=True) + source_lock = consumers[0] / ".zpkg.lock" + frozen_lock = frozen_consumer / ".zpkg.lock" + shutil.copyfile(source_lock, frozen_lock) + lock_before = frozen_lock.read_bytes() + frozen_install = run( + zed_command( + zed, + registry, + frozen_home, + "install", + "--frozen", + "--allow-no-manifest", + "--adapter", + "none", + ), + cwd=frozen_consumer, + env=environment, + ) + require_success(frozen_install) + frozen_summary = assert_prefetch( + frozen_install, + resolved=expected_count, + downloaded=expected_count, + ) + if (frozen_consumer / ".zpkg.toml").exists(): + raise AssertionError("manifestless frozen install wrote a new manifest") + if frozen_lock.read_bytes() != lock_before: + raise AssertionError("frozen install modified the lockfile") + assert_lock_packages(frozen_consumer, ALL_PACKAGES) + assert_symlink_packages( + frozen_consumer, + (frozen_home / "store").resolve(), + ALL_PACKAGES, + ) + assert_artifact_home(frozen_home, expected_count) + + frozen_warm = run( + zed_command( + zed, + registry, + frozen_home, + "install", + "--frozen", + "--allow-no-manifest", + "--adapter", + "none", + ), + cwd=frozen_consumer, + env=environment, + ) + require_success(frozen_warm) + frozen_warm_summary = assert_prefetch( + frozen_warm, + resolved=expected_count, + downloaded=0, + ) + if frozen_lock.read_bytes() != lock_before: + raise AssertionError("warm frozen install modified the lockfile") + + print("\nrecursive install stress passed", flush=True) + print(f"concurrent process summaries: {summaries}", flush=True) + print(f"frozen summary: {frozen_summary}", flush=True) + print(f"warm frozen summary: {frozen_warm_summary}", flush=True) + print(f"shared store: {shared_store}", flush=True) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as error: # noqa: BLE001 - CI entry point prints full context. + print(f"error: {error}", file=sys.stderr, flush=True) + raise diff --git a/scripts/repair_shared_auth_candidate.py b/scripts/repair_shared_auth_candidate.py new file mode 100644 index 00000000..43636da2 --- /dev/null +++ b/scripts/repair_shared_auth_candidate.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +from pathlib import Path + + +def replace_once(path: Path, old: str, new: str, label: str) -> None: + text = path.read_text() + count = text.count(old) + if count != 1: + raise SystemExit(f"{label} drifted: expected one match, found {count}") + path.write_text(text.replace(old, new, 1)) + + +handoff = Path("src/handoff/mod.rs") +text = handoff.read_text() +cloud_only = ''' let endpoint = format!( + "https://{}.supabase.co/auth/v1/token?grant_type=password", + project.project_ref + );''' +if text.count(cloud_only) != 1: + raise SystemExit( + f"cloud-only Supabase endpoint drifted: found {text.count(cloud_only)} matches" + ) +text = text.replace( + cloud_only, + " let endpoint = supabase_password_token_endpoint(project);", + 1, +) +helper = '''fn supabase_password_token_endpoint(project: &SupabaseProject) -> String { + format!( + "{}/token?grant_type=password", + project.issuer().trim_end_matches('/') + ) +} + +''' +anchor = "impl HandoffService {\n" +if "fn supabase_password_token_endpoint(" in text: + raise SystemExit("endpoint helper unexpectedly exists at the locked source head") +if text.count(anchor) != 1: + raise SystemExit("HandoffService implementation anchor drifted") +text = text.replace(anchor, helper + anchor, 1) +regression = r''' + +#[cfg(test)] +mod endpoint_tests { + use serde_json::json; + + use super::supabase_password_token_endpoint; + use crate::config::SupabaseProject; + + fn project(project_ref: &str, issuer: Option<&str>) -> SupabaseProject { + let mut value = json!({ + "name": "zpkg-test", + "project_ref": project_ref + }); + if let Some(issuer) = issuer { + value["issuer"] = json!(issuer); + } + serde_json::from_value(value).expect("valid Supabase project fixture") + } + + #[test] + fn derives_hosted_password_endpoint_from_default_issuer() { + let project = project("abcdefghijklmnopqrst", None); + assert_eq!( + supabase_password_token_endpoint(&project), + "https://abcdefghijklmnopqrst.supabase.co/auth/v1/token?grant_type=password" + ); + } + + #[test] + fn honors_explicit_self_hosted_issuer_and_trims_trailing_slash() { + let project = project( + "unused-for-explicit-issuer", + Some("http://127.0.0.1:54321/auth/v1/"), + ); + assert_eq!( + supabase_password_token_endpoint(&project), + "http://127.0.0.1:54321/auth/v1/token?grant_type=password" + ); + } +} +''' +if "mod endpoint_tests" in text: + raise SystemExit("endpoint regression tests unexpectedly exist at the locked source head") +handoff.write_text(text.rstrip() + regression + "\n") + +crypto = Path("src/handoff/crypto.rs") +replace_once( + crypto, + "use hmac::{Hmac, Mac};\nuse rand::RngCore;", + "use hmac::{Hmac, KeyInit as HmacKeyInit, Mac};\nuse rand::Rng;", + "rand/HMAC imports", +) +replace_once( + crypto, + ''' let mut nonce = [0_u8; 12]; + rand::rng().fill_bytes(&mut nonce); + let ciphertext = self + .0 + .encrypt( + Nonce::from_slice(&nonce), + Payload { + msg: &plaintext, + aad, + }, + ) + .map_err(|_| AuthError::Internal)?; + let mut encoded = Vec::with_capacity(nonce.len() + ciphertext.len()); + encoded.extend_from_slice(&nonce);''', + ''' let mut nonce_bytes = [0_u8; 12]; + rand::rng().fill(&mut nonce_bytes); + let nonce = Nonce::from(nonce_bytes); + let ciphertext = self + .0 + .encrypt( + &nonce, + Payload { + msg: &plaintext, + aad, + }, + ) + .map_err(|_| AuthError::Internal)?; + let mut encoded = Vec::with_capacity(nonce_bytes.len() + ciphertext.len()); + encoded.extend_from_slice(&nonce_bytes);''', + "AES-GCM encryption nonce", +) +replace_once( + crypto, + ''' let (nonce, ciphertext) = encoded.split_at(12); + let plaintext = self + .0 + .decrypt( + Nonce::from_slice(nonce), + Payload { + msg: ciphertext, + aad, + }, + )''', + ''' let (nonce_bytes, ciphertext) = encoded.split_at(12); + let nonce_bytes: [u8; 12] = nonce_bytes.try_into().map_err(|_| AuthError::Internal)?; + let nonce = Nonce::from(nonce_bytes); + let plaintext = self + .0 + .decrypt( + &nonce, + Payload { + msg: ciphertext, + aad, + }, + )''', + "AES-GCM decryption nonce", +) +replace_once( + crypto, + "rand::rng().fill_bytes(&mut bytes);", + "rand::rng().fill(&mut bytes);", + "authorization-code randomness", +) +replace_once( + crypto, + "::new_from_slice(SECRET_COMPARISON_KEY)", + "::new_from_slice(SECRET_COMPARISON_KEY)", + "first HMAC constructor", +) +# The source contains the constructor twice. The first replacement above is +# intentionally exact-once per call; repair the remaining constructor now. +replace_once( + crypto, + "::new_from_slice(SECRET_COMPARISON_KEY)", + "::new_from_slice(SECRET_COMPARISON_KEY)", + "second HMAC constructor", +) + +http = Path("src/http/handoff.rs") +replace_once( + http, + "use axum_extra::extract::cookie::{time::Duration as CookieDuration, Cookie, CookieJar, SameSite};", + "use axum_extra::extract::cookie::{Cookie, CookieJar, SameSite};\nuse time::Duration as CookieDuration;", + "cookie duration import", +) diff --git a/scripts/resolve_den2725_replay.py b/scripts/resolve_den2725_replay.py new file mode 100644 index 00000000..49f867ea --- /dev/null +++ b/scripts/resolve_den2725_replay.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from pathlib import Path + + +def replace_once(path: Path, old: str, new: str) -> None: + text = path.read_text(encoding="utf-8") + count = text.count(old) + if count != 1: + raise SystemExit(f"{path}: expected one replay marker, found {count}") + path.write_text(text.replace(old, new, 1), encoding="utf-8") + + +def resolve(root: Path) -> None: + lib = root / "src/lib.rs" + replace_once( + lib, + "pub mod exec;\n", + "pub mod exec;\npub mod external_subcommands;\n", + ) + + main = root / "src/main.rs" + replace_once( + main, + "use zed_cli::{\n cli::{\n", + "use zed_cli::{\n external_subcommands::{ExternalDispatch, external_route, run_external},\n cli::{\n", + ) + replace_once( + main, + ''' if let Some(code) = dev::dispatch(args.clone()) { + std::process::exit(code); + } + + let matches = cli_model::built_in_cli_command() +''', + ''' if let Some(code) = dev::dispatch(args.clone()) { + std::process::exit(code); + } + + if let Some(route) = external_route(&args) { + let code = match run_external(&route) { + Ok(ExternalDispatch::Exited(code)) => code, + Ok(ExternalDispatch::MissingKnown(message)) => { + eprintln!("error: {message}"); + 1 + } + Err(error) => { + eprintln!("error: {error:#}"); + 1 + } + }; + std::process::exit(code); + } + + let matches = cli_model::built_in_cli_command() +''', + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + args = parser.parse_args() + resolve(args.root.resolve(strict=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run-federated-test-org-scenarios.py b/scripts/run-federated-test-org-scenarios.py new file mode 100644 index 00000000..f0cd83fc --- /dev/null +++ b/scripts/run-federated-test-org-scenarios.py @@ -0,0 +1,464 @@ +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import textwrap +import tarfile +from pathlib import Path + +ZED_TOKEN = os.environ.get("ZED_BIN", "zed") +ZED = Path(shutil.which(ZED_TOKEN) or ZED_TOKEN).resolve() +ROOT = Path(os.environ.get("ZED_E2E_ROOT", "/tmp/zed-test-org-e2e")).resolve() +BINARY_ARCHIVE = os.environ.get("ZED_BINARY_ARCHIVE") +SOURCE_COMMIT = os.environ.get("ZED_SOURCE_COMMIT", "unknown") +REPORT_PATH = Path(os.environ.get("ZED_E2E_REPORT", str(ROOT / "report.json"))).resolve() +REPORT = {"binary": {}, "suites": {}} + + +class ScenarioFailure(RuntimeError): + pass + + +def run(args, cwd: Path, *, env=None, ok=True): + merged = os.environ.copy() + merged.update({"ZED_PKG_INTERACTIVE": "0", "RUST_BACKTRACE": "1"}) + if env: + merged.update({str(k): str(v) for k, v in env.items()}) + process = subprocess.run( + [str(arg) for arg in args], + cwd=cwd, + env=merged, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if ok is True and process.returncode != 0: + raise ScenarioFailure( + f"command failed ({process.returncode}): {' '.join(map(str, args))}\n" + f"stdout:\n{process.stdout}\nstderr:\n{process.stderr}" + ) + if ok is False and process.returncode == 0: + raise ScenarioFailure( + f"command unexpectedly succeeded: {' '.join(map(str, args))}\n" + f"stdout:\n{process.stdout}" + ) + return process + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_manifest(path: Path, name: str, version: str, dependencies=None, extra=""): + dependencies = dependencies or {} + dependency_lines = "\n".join( + f'"{coordinate}" = "{requirement}"' + for coordinate, requirement in dependencies.items() + ) + path.write_text( + textwrap.dedent( + f'''\ + [package] + org = "zed-pkg-test" + name = "{name}" + version = "{version}" + description = "test-org executable fixture" + license = "MIT" + + [package.repository] + vcs = "git" + url = "https://github.com/zed-pkg-test/{name}" + + [dependencies] + {dependency_lines} + + [publish] + exclude = [] + {extra} + ''' + ), + encoding="utf-8", + ) + + +def publish(registry: Path, home: Path, name: str, version: str, *, dependencies=None, files=None, extra=""): + package = registry.parent / "sources" / f"{name}-{version}" + package.mkdir(parents=True, exist_ok=True) + write_manifest(package / ".zpkg.toml", name, version, dependencies, extra) + for relative, content in (files or {"payload.txt": f"{name}@{version}\n"}).items(): + target = package / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + run( + [ + ZED, + "--registry", + f"file://{registry}", + "--home", + home, + "publish", + "--skip-vcs-checks", + ], + package, + ) + return package + + +def lock_packages(lock: Path): + import tomllib + + return tomllib.loads(lock.read_text(encoding="utf-8")).get("package", []) + + +def suite(name): + REPORT["suites"][name] = {"checks": [], "status": "running"} + return REPORT["suites"][name] + + +def check(target, name, detail=None): + target["checks"].append( + {"name": name, "status": "pass", **({"detail": detail} if detail else {})} + ) + + +def normalize_error(text: str, roots=()): + for root in roots: + text = text.replace(str(root), "") + text = re.sub(r"[0-9a-f]{8}-[0-9a-f-]{27,}", "", text, flags=re.I) + return "\n".join(line.rstrip() for line in text.splitlines() if line.strip()) + + +def offline_cache_suite(): + target = suite("zed-pkg-test/offline-cache-e2e") + base = ROOT / "offline-cache-e2e" + registry, home, consumer = base / "registry", base / "home", base / "consumer-a" + for directory in (registry, home, consumer): + directory.mkdir(parents=True, exist_ok=True) + publish( + registry, + home, + "offline-cache-lib", + "1.0.0", + files={ + "package.json": '{"name":"@zed-pkg-test/offline-cache-lib","version":"1.0.0","type":"module","exports":"./index.js"}\n', + "index.js": "export const answer = 42;\n", + }, + ) + run( + [ + ZED, + "--registry", + f"file://{registry}", + "--home", + home, + "install", + "zed-pkg-test/offline-cache-lib@^1", + "--install-mode", + "copy", + "--adapter", + "node", + ], + consumer, + ) + run( + [ + "node", + "--input-type=module", + "-e", + "import('./node_modules/@zed-pkg-test/offline-cache-lib/index.js').then(m=>{if(m.answer!==42)process.exit(1)})", + ], + consumer, + ) + packages = lock_packages(consumer / ".zpkg.lock") + assert len(packages) == 1 and packages[0]["version"] == "1.0.0" + digest = packages[0]["sha256"] + cache = home / "cache" / f"{digest}.tar.gz" + store = home / "store" / "v1" / digest[:2] / digest + assert cache.is_file() and store.is_dir() + check(target, "cold publish/install and Node consumer import", digest) + + manifest_bytes = (consumer / ".zpkg.toml").read_bytes() + lock_bytes = (consumer / ".zpkg.lock").read_bytes() + run([ZED, "--registry", f"file://{registry}", "--home", home, "uninstall"], consumer) + offline_registry = base / "registry.offline" + registry.rename(offline_registry) + dead_network = { + "http_proxy": "http://127.0.0.1:9", + "https_proxy": "http://127.0.0.1:9", + "HTTP_PROXY": "http://127.0.0.1:9", + "HTTPS_PROXY": "http://127.0.0.1:9", + "NO_PROXY": "", + "no_proxy": "", + } + frozen = [ + ZED, + "--registry", + f"file://{registry}", + "--home", + home, + "install", + "--frozen", + "--install-mode", + "copy", + "--adapter", + "node", + ] + run(frozen, consumer, env=dead_network) + assert (consumer / ".zpkg.toml").read_bytes() == manifest_bytes + assert (consumer / ".zpkg.lock").read_bytes() == lock_bytes + check(target, "offline frozen reinstall with registry absent and proxies dead") + + consumer_b = base / "consumer-b" + consumer_b.mkdir() + (consumer_b / ".zpkg.toml").write_bytes(manifest_bytes) + (consumer_b / ".zpkg.lock").write_bytes(lock_bytes) + run(frozen, consumer_b, env=dead_network) + assert (consumer_b / "zed_modules/zed-pkg-test/offline-cache-lib/index.js").is_file() + check(target, "shared-home cache reuse in second checkout") + + run([ZED, "--registry", f"file://{registry}", "--home", home, "uninstall"], consumer_b) + shutil.rmtree(store, ignore_errors=True) + cache.write_bytes(b"corrupt-cache-entry") + failed = run(frozen, consumer_b, env=dead_network, ok=False) + assert not (consumer_b / "zed_modules/zed-pkg-test/offline-cache-lib").exists() + assert any(word in failed.stderr.lower() for word in ("sha256", "integrity", "registry")) + check(target, "corrupt cache + offline registry rejected without partial materialization") + + offline_registry.rename(registry) + run(frozen, consumer_b) + assert sha256(cache) == digest + check(target, "corrupt cache recovers from immutable registry artifact") + target["status"] = "pass" + + +def version_solver_suite(): + target = suite("zed-pkg-test/version-solver-e2e") + base, registry, home = ROOT / "version-solver-e2e", ROOT / "version-solver-e2e/registry", ROOT / "version-solver-e2e/home" + registry.mkdir(parents=True) + home.mkdir() + for version in ("1.0.0", "1.1.0", "2.0.0"): + publish(registry, home, "solver-shared", version) + publish(registry, home, "solver-alpha", "1.0.0", dependencies={"zed-pkg-test/solver-shared": "^1"}) + publish(registry, home, "solver-beta", "1.0.0", dependencies={"zed-pkg-test/solver-shared": "^2"}) + + ranged = base / "range" + ranged.mkdir() + run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "zed-pkg-test/solver-shared@^1", "--adapter", "none", "--install-mode", "copy"], ranged) + assert [(item["name"], item["version"]) for item in lock_packages(ranged / ".zpkg.lock")] == [("solver-shared", "1.1.0")] + check(target, "compatible range selects highest matching version", "1.1.0") + + exact = base / "exact" + exact.mkdir() + run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "zed-pkg-test/solver-shared@=1.0.0", "--adapter", "none", "--install-mode", "copy"], exact) + assert lock_packages(exact / ".zpkg.lock")[0]["version"] == "1.0.0" + check(target, "exact version remains exact") + + errors = [] + for index in (1, 2): + consumer = base / f"conflict-{index}" + consumer.mkdir() + write_manifest(consumer / ".zpkg.toml", f"conflict-{index}", "0.0.0", {"zed-pkg-test/solver-alpha": "^1", "zed-pkg-test/solver-beta": "^1"}) + output = run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "--adapter", "none", "--install-mode", "copy"], consumer, ok=False) + assert not (consumer / "zed_modules").exists() + errors.append(normalize_error(output.stderr, (consumer, base))) + cores = [re.sub(r"conflict-[12]", "conflict-N", error) for error in errors] + assert cores[0] == cores[1] + assert "solver-shared" in cores[0] and any(word in cores[0].lower() for word in ("conflict", "satisf")) + check(target, "incompatible transitive constraints fail deterministically", cores[0].splitlines()[-1][:240]) + + for version in ("1.0.0+aaa", "1.0.0+bbb"): + publish(registry, home, "solver-metadata", version) + outcomes = [] + for index in (1, 2): + consumer = base / f"metadata-{index}" + consumer.mkdir() + output = run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "zed-pkg-test/solver-metadata@=1.0.0", "--adapter", "none", "--install-mode", "copy"], consumer, ok=None) + outcomes.append(("resolved", lock_packages(consumer / ".zpkg.lock")[0]["version"]) if output.returncode == 0 else ("rejected", normalize_error(output.stderr, (consumer, base)))) + assert outcomes[0] == outcomes[1] + check(target, "SemVer build-metadata ambiguity has a stable outcome", repr(outcomes[0])[:240]) + target["status"] = "pass" + + +def security_suite(): + target = suite("zed-pkg-test/security-adversarial-e2e") + base, registry, home = ROOT / "security-adversarial-e2e", ROOT / "security-adversarial-e2e/registry", ROOT / "security-adversarial-e2e/home" + registry.mkdir(parents=True) + home.mkdir() + + package = base / "symlink" + package.mkdir() + write_manifest(package / ".zpkg.toml", "escape-symlink", "1.0.0") + (package / "payload.txt").write_text("safe\n") + (package / "escape").symlink_to("/etc/passwd") + output = run([ZED, "--home", home, "pack"], package, ok=None) + if output.returncode == 0: + archives = list((package / ".zed/pack").glob("*.tar.gz")) + assert len(archives) == 1 + with tarfile.open(archives[0], "r:gz") as archive: + members = archive.getmembers() + assert all(member.isfile() for member in members) + assert all(member.name != "pkg/escape" for member in members) + assert all(not member.name.startswith("/") and ".." not in member.name.split("/") for member in members) + check(target, "absolute symlink omitted from deterministic artifact without external bytes") + else: + assert any(word in output.stderr.lower() for word in ("symlink", "absolute", "escape")) + check(target, "absolute symlink rejected during deterministic pack") + + traversal = base / "target-traversal" + traversal.mkdir() + write_manifest(traversal / ".zpkg.toml", "target-traversal", "1.0.0", extra=''' +[targets.nodejs] +dir = "../outside" +adapter = "node" +''') + (traversal / "payload.txt").write_text("safe\n") + output = run([ZED, "--home", home, "pack"], traversal, ok=False) + assert ".." in output.stderr or any(word in output.stderr.lower() for word in ("target", "relative")) + check(target, "target directory traversal rejected") + + publish(registry, home, "hook-fixture", "1.0.0", extra=''' +[build] +command = "sh -c 'echo consented > build-marker.txt'" +''') + no_consent = base / "hook-no-consent" + no_consent.mkdir() + run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "zed-pkg-test/hook-fixture@1.0.0", "--adapter", "none", "--install-mode", "copy"], no_consent) + assert not any(no_consent.rglob("build-marker.txt")) + check(target, "package build hook does not execute without explicit consent") + + consent = base / "hook-consent" + consent.mkdir() + run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "zed-pkg-test/hook-fixture@1.0.0", "--adapter", "none", "--install-mode", "copy", "--allow-build"], consent) + markers = list(consent.rglob("build-marker.txt")) + assert markers + check(target, "explicit --allow-build executes declared hook", str(markers[0].relative_to(consent))) + + publish(registry, home, "tamper-fixture", "1.0.0", files={"package.json": '{"name":"@zed-pkg-test/tamper-fixture","version":"1.0.0"}\n'}) + consumer = base / "tamper-consumer" + consumer.mkdir() + run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "zed-pkg-test/tamper-fixture@1.0.0", "--adapter", "none", "--install-mode", "copy"], consumer) + run([ZED, "--registry", f"file://{registry}", "--home", home, "uninstall"], consumer) + lock = consumer / ".zpkg.lock" + text, count = re.subn(r'^sha256 = "[0-9a-f]{64}"$', 'sha256 = "' + "0" * 64 + '"', lock.read_text(), count=1, flags=re.M) + assert count == 1 + lock.write_text(text) + output = run([ZED, "--registry", f"file://{registry}", "--home", home, "install", "--frozen", "--adapter", "none", "--install-mode", "copy"], consumer, ok=False) + assert not (consumer / "zed_modules/zed-pkg-test/tamper-fixture").exists() + assert any(word in output.stderr.lower() for word in ("sha256", "integrity")) + check(target, "tampered frozen lock rejected without partial install") + target["status"] = "pass" + + +def manager_suite(): + target = suite("zed-pkg-test/manager-interop-e2e") + base, home = ROOT / "manager-interop-e2e", ROOT / "manager-interop-e2e/home" + empty = home / "empty-path" + home.mkdir(parents=True) + empty.mkdir() + isolated = {"HOME": home, "USERPROFILE": home, "XDG_CONFIG_HOME": home / ".config", "ZED_PKG_HOME": home / ".zed-pkg", "PATH": empty} + + mise = base / "mise" + mise.mkdir(parents=True) + (mise / "mise.toml").write_text('[settings]\nlockfile = true\n\n[tools]\nnode = "22"\n') + (mise / "mise.lock").write_text('[[tools.node]]\nversion = "22.4.0"\nbackend = "core:node"\n\n[tools.node.platforms.linux-x64]\nchecksum = "sha256:' + "a" * 64 + '"\nsize = 123\nurl = "https://example.invalid/node.tar.xz"\n') + before = (sha256(mise / "mise.toml"), sha256(mise / "mise.lock")) + verified = json.loads(run([ZED, "env", "verify", "mise", "--config", "mise.toml", "--lock", "mise.lock", "--frozen", "--json"], mise, env=isolated).stdout) + imported = json.loads(run([ZED, "env", "import", "mise", "--config", "mise.toml", "--lock", "mise.lock", "--frozen", "--json"], mise, env=isolated).stdout) + assert verified["verified"] is True and verified["tools"] == 1 + assert imported["tools"]["node"]["resolved"] == "22.4.0" + assert before == (sha256(mise / "mise.toml"), sha256(mise / "mise.lock")) + check(target, "mise frozen import/verify is parser-only, deterministic, and read-only", verified["environment_plan_sha256"]) + + (mise / ".mise.toml").write_text('[tools]\npython = "3.12"\n') + assert "multiple project-local" in run([ZED, "env", "verify", "mise", "--frozen", "--json"], mise, env=isolated, ok=False).stderr + (mise / ".mise.toml").unlink() + unlocked = base / "mise-unlocked" + unlocked.mkdir() + (unlocked / "mise.toml").write_text('[tools]\nnode = "22"\n') + assert "requires a project-local lockfile" in run([ZED, "env", "verify", "mise", "--config", "mise.toml", "--frozen", "--json"], unlocked, env=isolated, ok=False).stderr + check(target, "mise ambiguity and incomplete frozen state fail closed") + + asdf = base / "asdf" + (asdf / ".zed").mkdir(parents=True) + (asdf / ".tool-versions").write_text("nodejs 22.11.0\npython 3.12.4\n") + (asdf / ".zed/asdf.lock.toml").write_text('''schema = 1 + +[plugins.nodejs] +version = "22.11.0" +url = "https://github.com/asdf-vm/asdf-nodejs.git" +revision = "1111111111111111111111111111111111111111" +sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +platforms = ["x86_64-linux"] + +[plugins.python] +version = "3.12.4" +url = "https://github.com/danhper/asdf-python.git" +revision = "2222222222222222222222222222222222222222" +sha256 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +''') + before = (sha256(asdf / ".tool-versions"), sha256(asdf / ".zed/asdf.lock.toml")) + verified = json.loads(run([ZED, "env", "verify", "asdf", "--config", ".tool-versions", "--lock", ".zed/asdf.lock.toml", "--frozen", "--json"], asdf, env=isolated).stdout) + imported = json.loads(run([ZED, "env", "import", "asdf", "--config", ".tool-versions", "--lock", ".zed/asdf.lock.toml", "--frozen", "--json"], asdf, env=isolated).stdout) + assert verified["verified"] is True and verified["tools"] == 2 + assert imported["tools"]["nodejs"]["resolved"] == "22.11.0" + assert before == (sha256(asdf / ".tool-versions"), sha256(asdf / ".zed/asdf.lock.toml")) + check(target, "asdf frozen import/verify validates immutable plugin/artifact provenance read-only", verified["environment_plan_sha256"]) + + moving = base / "asdf-moving" + (moving / ".zed").mkdir(parents=True) + (moving / ".tool-versions").write_text("nodejs ref:main\n") + (moving / ".zed/asdf.lock.toml").write_text('''schema = 1 +[plugins.nodejs] +version = "ref:main" +url = "https://github.com/asdf-vm/asdf-nodejs.git" +revision = "1111111111111111111111111111111111111111" +sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +''') + assert "moving selector" in run([ZED, "env", "verify", "asdf", "--config", ".tool-versions", "--lock", ".zed/asdf.lock.toml", "--frozen"], moving, env=isolated, ok=False).stderr + check(target, "asdf moving selector rejected in frozen mode") + + unsupported = [] + for manager in ("flox", "devbox", "nix"): + output = run([ZED, "env", "import", manager], base, env=isolated, ok=False) + assert "possible values" in output.stderr and "mise" in output.stderr and "asdf" in output.stderr + unsupported.append(manager) + check(target, "unsupported manager labels fail at typed CLI boundary", ",".join(unsupported)) + target["status"] = "pass" + + +def main(): + if ROOT.exists(): + shutil.rmtree(ROOT) + ROOT.mkdir(parents=True) + REPORT["binary"] = { + "version": run([ZED, "--version"], ROOT).stdout.strip(), + "archive_sha256": sha256(Path(BINARY_ARCHIVE)) if BINARY_ARCHIVE else None, + "source_commit": SOURCE_COMMIT, + } + for function in (offline_cache_suite, version_solver_suite, security_suite, manager_suite): + try: + function() + except Exception as error: + for value in REPORT["suites"].values(): + if value.get("status") == "running": + value["status"] = "fail" + value["error"] = str(error) + break + print(json.dumps(REPORT, indent=2), file=sys.stderr) + raise + REPORT_PATH.parent.mkdir(parents=True, exist_ok=True) + REPORT_PATH.write_text(json.dumps(REPORT, indent=2) + "\n") + print(json.dumps(REPORT, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/terminal_context_acceptance.py b/scripts/terminal_context_acceptance.py new file mode 100644 index 00000000..fd09e1e2 --- /dev/null +++ b/scripts/terminal_context_acceptance.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Independent black-box acceptance for Zed terminal-context prompt behavior. + +This script intentionally lives in zed-pkg-test rather than zed-cli. It consumes +an exact built candidate and drives a minimal real mutation (`zed init`) so prompt +safety is certified without depending on registry, network, or package transport. +""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import shutil +import subprocess +import sys + +FORCE_KEYS = { + "ZED_PKG_FORCE_STDIN_TTY", + "ZED_PKG_FORCE_STDOUT_TTY", + "ZED_PKG_FORCE_STDERR_TTY", + "ZED_PKG_FORCE_CI", + "ZED_PKG_FORCE_COLOR", + "ZED_PKG_FORCE_UNICODE", + "ZED_PKG_SHELL", + "F2E_FORCE_STDIN_TTY", + "F2E_FORCE_STDOUT_TTY", + "F2E_FORCE_STDERR_TTY", + "F2E_FORCE_CI", + "F2E_FORCE_COLOR", + "F2E_FORCE_UNICODE", + "F2E_SHELL", +} + + +def clean_env() -> dict[str, str]: + env = os.environ.copy() + for key in FORCE_KEYS: + env.pop(key, None) + return env + + +def run( + command: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, + input_text: str | None = None, + should_succeed: bool = True, +) -> subprocess.CompletedProcess[str]: + print(f"$ {' '.join(command)}", flush=True) + completed = subprocess.run( + command, + cwd=cwd, + env=env, + input=input_text, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.stdout: + print(completed.stdout, end="" if completed.stdout.endswith("\n") else "\n") + if completed.stderr: + print( + completed.stderr, + end="" if completed.stderr.endswith("\n") else "\n", + file=sys.stderr, + ) + + succeeded = completed.returncode == 0 + if succeeded != should_succeed: + expected = "success" if should_succeed else "failure" + raise SystemExit( + f"expected {expected}, got exit {completed.returncode}: {' '.join(command)}" + ) + return completed + + +def assert_contains_prompt_failure(completed: subprocess.CompletedProcess[str]) -> None: + combined = f"{completed.stdout}\n{completed.stderr}".lower() + if "terminal stdin and stderr" not in combined: + raise SystemExit( + "interactive failure did not explain that terminal stdin and stderr are required" + ) + + +def reset_project(project: Path) -> None: + for name in (".zpkg.toml", ".gitignore"): + path = project / name + if path.exists(): + path.unlink() + + +def assert_not_initialized(project: Path) -> None: + if (project / ".zpkg.toml").exists(): + raise SystemExit("rejected confirmation unexpectedly created .zpkg.toml") + + +def assert_initialized(project: Path) -> None: + manifest = project / ".zpkg.toml" + if not manifest.is_file(): + raise SystemExit("accepted confirmation did not create .zpkg.toml") + text = manifest.read_text(encoding="utf-8") + if 'org = "terminal-cert"' not in text or 'name = "prompt-fixture"' not in text: + raise SystemExit("generated manifest did not contain the requested package identity") + + +def interactive_init( + zed: Path, + project: Path, + *, + env: dict[str, str], + should_succeed: bool, +) -> subprocess.CompletedProcess[str]: + return run( + [ + str(zed), + "--interactive", + "init", + "--org", + "terminal-cert", + "--name", + "prompt-fixture", + ], + cwd=project, + env=env, + input_text="yes\n", + should_succeed=should_succeed, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--zed-cli-root", required=True) + args = parser.parse_args() + + zed_cli_root = Path(args.zed_cli_root).resolve() + zed = zed_cli_root / "target" / "debug" / ("zed.exe" if os.name == "nt" else "zed") + if not zed.is_file(): + raise SystemExit(f"built zed candidate not found: {zed}") + + runner_temp = Path(os.environ.get("RUNNER_TEMP", ".")).resolve() + work = runner_temp / "zed-terminal-context-acceptance" + if work.exists(): + shutil.rmtree(work) + project = work / "prompt-fixture" + project.mkdir(parents=True) + + # A pipe must never masquerade as human consent. GitHub Actions is also CI, + # so this independently confirms the ordinary fail-closed path. + ordinary = interactive_init( + zed, + project, + env=clean_env(), + should_succeed=False, + ) + assert_contains_prompt_failure(ordinary) + assert_not_initialized(project) + + # Even when terminal state is forced true, CI remains authoritative. + forced_ci = clean_env() + forced_ci.update( + { + "ZED_PKG_FORCE_STDIN_TTY": "1", + "ZED_PKG_FORCE_STDOUT_TTY": "0", + "ZED_PKG_FORCE_STDERR_TTY": "1", + "ZED_PKG_FORCE_CI": "1", + "TERM": "xterm-256color", + } + ) + ci_failure = interactive_init( + zed, + project, + env=forced_ci, + should_succeed=False, + ) + assert_contains_prompt_failure(ci_failure) + assert_not_initialized(project) + + # TERM=dumb also remains fail-closed even after CI is explicitly cleared. + dumb = clean_env() + dumb.update( + { + "ZED_PKG_FORCE_STDIN_TTY": "1", + "ZED_PKG_FORCE_STDOUT_TTY": "0", + "ZED_PKG_FORCE_STDERR_TTY": "1", + "ZED_PKG_FORCE_CI": "0", + "TERM": "dumb", + } + ) + dumb_failure = interactive_init( + zed, + project, + env=dumb, + should_succeed=False, + ) + assert_contains_prompt_failure(dumb_failure) + assert_not_initialized(project) + + # The documented ZED_PKG_FORCE_* controls may explicitly simulate a safe + # human terminal. stdout remains redirected/captured here, proving stdout + # is not part of the prompt gate. + zed_override = clean_env() + zed_override.update( + { + "ZED_PKG_FORCE_STDIN_TTY": "1", + "ZED_PKG_FORCE_STDOUT_TTY": "0", + "ZED_PKG_FORCE_STDERR_TTY": "1", + "ZED_PKG_FORCE_CI": "0", + "TERM": "xterm-256color", + } + ) + accepted = interactive_init( + zed, + project, + env=zed_override, + should_succeed=True, + ) + if "[y/N]" not in accepted.stderr: + raise SystemExit("accepted interactive mutation did not emit its checkpoint on stderr") + assert_initialized(project) + + # Reset only test-created files, then prove the shared F2E_FORCE_* spellings + # drive the same behavior without any Zed-specific force variables present. + reset_project(project) + f2e_override = clean_env() + f2e_override.update( + { + "F2E_FORCE_STDIN_TTY": "1", + "F2E_FORCE_STDOUT_TTY": "0", + "F2E_FORCE_STDERR_TTY": "1", + "F2E_FORCE_CI": "0", + "TERM": "xterm-256color", + } + ) + accepted_f2e = interactive_init( + zed, + project, + env=f2e_override, + should_succeed=True, + ) + if "[y/N]" not in accepted_f2e.stderr: + raise SystemExit("F2E override path did not emit its checkpoint on stderr") + assert_initialized(project) + + # Unix runners additionally exercise a real PTY through the production + # harness. Windows uses the deterministic override path above because the + # upstream helper relies on forkpty(). + if os.name != "nt": + reset_project(project) + helper = zed_cli_root / "tests" / "interactive_pty.py" + run( + [ + sys.executable, + str(helper), + "yes", + "--", + str(zed), + "--interactive", + "init", + "--org", + "terminal-cert", + "--name", + "prompt-fixture", + ], + cwd=project, + env=clean_env(), + ) + assert_initialized(project) + + print("terminal-context acceptance passed", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-org-fleet-integration-policy.mjs b/scripts/test-org-fleet-integration-policy.mjs new file mode 100644 index 00000000..7856e8a1 --- /dev/null +++ b/scripts/test-org-fleet-integration-policy.mjs @@ -0,0 +1,180 @@ +const UPLOAD_ARTIFACT_SHA = '043fb46d1a93c77aae656e7c1c64a875d1fc6a0a'; // v7 + +function replaceOnce(source, before, after, label) { + const count = source.split(before).length - 1; + if (count !== 1) { + throw new Error(`${label}: expected one generated seam, found ${count}`); + } + return source.replace(before, after); +} + +/** + * Apply source-access and execution semantics to the generated test-org harness. + * + * The generic generated workflow can prove that its protected credential gate + * and generated plan ran. It cannot certify arbitrary product source without a + * product-specific executable overlay, so its status remains explicitly not + * certified even after a successful access/plan run. + */ +export function hardenGeneratedIntegrationPolicy(input) { + let source = input; + + const planSecurityBefore = [ + ' security: {\n', + ' immutableSourcePins: true,\n', + ' pullRequestCredentials: false,\n', + ' leastPrivilege: true,\n', + ' redactSecretsAndRawMedia: true,\n', + ' },', + ].join(''); + const planSecurityAfter = [ + ' security: {\n', + ' immutableSourcePins: true,\n', + ' pullRequestCredentials: false,\n', + ' leastPrivilege: true,\n', + ' redactSecretsAndRawMedia: true,\n', + ' protectedSourceIntegration: true,\n', + " protectedSourceIntegrationStatus: 'not-executed-unless-protected-lane-runs',\n", + ' productOverlayPreserved: true,\n', + ' },', + ].join(''); + source = replaceOnce( + source, + planSecurityBefore, + planSecurityAfter, + 'generated plan security contract', + ); + + const readmeBefore = + 'Full integration is intentionally release-gated until required source repositories and organization read credentials are present.'; + const readmeAfter = + 'Product-specific files outside the generated file set are preserved and must add executable assertions without weakening the base contract. Full integration is intentionally release-gated until required source repositories and organization read credentials are present. The generic protected lane reports source-access status only; source certification requires a product-specific executable overlay. A skipped integration job is not source certification.'; + source = replaceOnce( + source, + readmeBefore, + readmeAfter, + 'generated README integration boundary', + ); + + const concurrencyBefore = [ + 'name: gated integration\\n\\n', + 'on:\\n', + ' workflow_dispatch:\\n', + ' schedule:\\n', + " - cron: '17 8 * * *'\\n\\n", + 'permissions:\\n', + ' contents: read\\n\\n', + 'jobs:\\n', + ].join(''); + const concurrencyAfter = [ + 'name: gated integration\\n\\n', + 'on:\\n', + ' workflow_dispatch:\\n', + ' schedule:\\n', + " - cron: '17 8 * * *'\\n\\n", + 'permissions:\\n', + ' contents: read\\n\\n', + 'concurrency:\\n', + ' group: \\${{ github.workflow }}-\\${{ github.ref }}\\n', + ' cancel-in-progress: true\\n\\n', + 'jobs:\\n', + ].join(''); + source = replaceOnce( + source, + concurrencyBefore, + concurrencyAfter, + 'generated integration concurrency', + ); + + source = replaceOnce( + source, + ' token: \\${{ secrets.TEST_FLEET_READ_TOKEN || github.token }}', + ' token: \\${{ secrets.TEST_FLEET_READ_TOKEN }}', + 'protected checkout token fallback', + ); + + const stepsBefore = + ' steps:\\n - name: Check out source pins and submodules\\n'; + const stepsAfter = [ + ' steps:\\n', + ' - name: Require protected cross-organization read credential\\n', + ' env:\\n', + ' TEST_FLEET_READ_TOKEN: \\${{ secrets.TEST_FLEET_READ_TOKEN }}\\n', + ' run: |\\n', + ' set -euo pipefail\\n', + ' test -n "$TEST_FLEET_READ_TOKEN"\\n', + " printf 'protected source access enabled\\\\n'\\n", + ' - name: Check out source pins and submodules\\n', + ].join(''); + source = replaceOnce( + source, + stepsBefore, + stepsAfter, + 'protected integration credential requirement', + ); + + const tailBefore = [ + ' - name: Explain profile gate\\n', + ' run: |\\n', + ' echo "Profile: ${spec.profile}"\\n', + ' echo "Required checks are defined in test-plan.json."\\n', + ' echo "Add product-specific executable fixtures here without weakening the generated contract."\\n', + '`;', + ].join(''); + const tailAfter = [ + ' - name: Explain profile gate\\n', + ' run: |\\n', + ' echo "Profile: ${spec.profile}"\\n', + ' echo "Required checks are defined in test-plan.json."\\n', + ' echo "Add product-specific executable fixtures here without weakening the generated contract."\\n', + '\\n', + ' certification-status:\\n', + ' name: Record protected source access status\\n', + ' if: always()\\n', + ' needs: integration\\n', + ' runs-on: ubuntu-24.04\\n', + ' timeout-minutes: 5\\n', + ' steps:\\n', + ' - name: Write source-integration status\\n', + ' env:\\n', + ' ENABLED: \\${{ vars.ENABLE_TEST_FLEET_INTEGRATION }}\\n', + ' RESULT: \\${{ needs.integration.result }}\\n', + ' run: |\\n', + " node - <<'NODE'\\n", + " const fs = require('node:fs');\\n", + " const result = process.env.RESULT || 'unknown';\\n", + " const enabled = process.env.ENABLED === 'true';\\n", + " const executed = result !== 'skipped';\\n", + " const sourceAccessPassed = result === 'success';\\n", + ' const report = {\\n', + ' schemaVersion: 2,\\n', + ' enabled,\\n', + ' executed,\\n', + ' sourceAccessPassed,\\n', + ' certified: false,\\n', + ' result,\\n', + " reason: sourceAccessPassed ? 'protected-source-access-passed-product-certification-required'\\n", + " : executed ? 'protected-source-access-did-not-pass'\\n", + " : 'protected-source-integration-not-enabled',\\n", + ' };\\n', + " fs.writeFileSync('source-integration-status.json', JSON.stringify(report, null, 2) + '\\\\n');\\n", + " console.log(JSON.stringify(report));\\n", + ' NODE\\n', + ' - name: Upload source-integration status\\n', + ` uses: actions/upload-artifact@${UPLOAD_ARTIFACT_SHA} # v7\\n`, + ' with:\\n', + ' name: source-integration-status-\\${{ github.sha }}\\n', + ' path: source-integration-status.json\\n', + ' if-no-files-found: error\\n', + ' retention-days: 14\\n', + '`;', + ].join(''); + source = replaceOnce( + source, + tailBefore, + tailAfter, + 'protected integration source-access status', + ); + + return source; +} diff --git a/scripts/test_snapshot_endpoint_policy.py b/scripts/test_snapshot_endpoint_policy.py new file mode 100644 index 00000000..412a7963 --- /dev/null +++ b/scripts/test_snapshot_endpoint_policy.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import unittest + +from check_snapshot_endpoint_policy import ContractError, validate + + +DEPLOYMENT = """\ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app + namespace: daedalus +spec: + template: + spec: + containers: + - name: app + env: + - { name: NATS_URL, value: "nats://nats.messaging.svc:4222" } + - { name: OTEL_EXPORTER_OTLP_ENDPOINT, value: "http://otel.observability.svc:4318" } +""" + +POLICY = """\ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: app + namespace: daedalus +spec: + egress: + - to: [] + ports: + - { protocol: TCP, port: 4222 } + - { protocol: TCP, port: 4318 } +""" + + +class SnapshotEndpointPolicyTests(unittest.TestCase): + def test_declared_endpoints_are_permitted(self) -> None: + result = validate(DEPLOYMENT, POLICY) + self.assertEqual( + {"NATS_URL": 4222, "OTEL_EXPORTER_OTLP_ENDPOINT": 4318}, + result["endpointPorts"], + ) + + def test_otlp_grpc_port_does_not_cover_http_export(self) -> None: + with self.assertRaisesRegex( + ContractError, + r"OTEL_EXPORTER_OTLP_ENDPOINT=4318", + ): + validate(DEPLOYMENT, POLICY.replace("port: 4318", "port: 4317")) + + def test_missing_explicit_endpoint_port_fails_closed(self) -> None: + deployment = DEPLOYMENT.replace( + "http://otel.observability.svc:4318", + "http://otel.observability.svc", + ) + with self.assertRaisesRegex(ContractError, r"explicit port"): + validate(deployment, POLICY) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate-test-org-fleet.mjs b/scripts/validate-test-org-fleet.mjs new file mode 100755 index 00000000..1a23d9b7 --- /dev/null +++ b/scripts/validate-test-org-fleet.mjs @@ -0,0 +1,106 @@ +#!/usr/bin/env node +// generated by zed-pkg-test/test-org-fleet +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { readFleetManifest } from './read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const manifestPath = process.argv.find((arg) => arg.endsWith('.json') || arg.endsWith('.json.gz')) ?? path.join(root, 'bootstrap', 'test-org-fleet.json.gz'); +const manifest = readFleetManifest(manifestPath); +const serialized = JSON.stringify(manifest); +const errors = []; +const warnings = []; +const fail = (message) => errors.push(message); +const warn = (message) => warnings.push(message); +const seen = new Set(); +const excluded = new Set(manifest.policy?.excludedOrganizations ?? []); +const allowedModes = new Set(manifest.policy?.dependencyModes ?? []); + +if (manifest.schemaVersion !== 1) fail(`schemaVersion must be 1, got ${manifest.schemaVersion}`); +if (!Array.isArray(manifest.pairs) || manifest.pairs.length === 0) fail('pairs must be a non-empty array'); +if (!manifest.profiles || typeof manifest.profiles !== 'object') fail('profiles must be an object'); +if (!excluded.has('r2g') || !excluded.has('r2g-test')) fail('r2g and r2g-test must remain explicitly excluded'); + +let createCount = 0; +let existingCount = 0; +let importCount = 0; +const managers = new Map(); + +for (const pair of manifest.pairs ?? []) { + const key = `${pair.sourceOrg}=>${pair.testOrg}`; + if (seen.has(key)) fail(`duplicate pair ${key}`); + seen.add(key); + if (excluded.has(pair.sourceOrg) || excluded.has(pair.testOrg)) fail(`excluded organization leaked into pair ${key}`); + if (!pair.testOrg?.toLowerCase().endsWith('-test')) fail(`test org must end in -test: ${pair.testOrg}`); + if (!['public', 'private'].includes(pair.defaultVisibility)) fail(`invalid defaultVisibility for ${pair.testOrg}`); + if (!Array.isArray(pair.repositories) || pair.repositories.length === 0) fail(`no repositories for ${pair.testOrg}`); + + const repoNames = new Set(); + for (const existing of pair.existingRepositories ?? []) { + existingCount += 1; + if (repoNames.has(existing)) fail(`duplicate existing repository ${pair.testOrg}/${existing}`); + repoNames.add(existing); + } + + for (const repository of pair.repositories ?? []) { + createCount += repository.state === 'create' ? 1 : 0; + if (!/^[a-z0-9._-]+$/.test(repository.name)) fail(`invalid repository name ${pair.testOrg}/${repository.name}`); + if (repoNames.has(repository.name)) fail(`duplicate repository ${pair.testOrg}/${repository.name}`); + repoNames.add(repository.name); + if (!manifest.profiles?.[repository.profile]) fail(`unknown profile ${repository.profile} for ${pair.testOrg}/${repository.name}`); + if (!Array.isArray(repository.sources) || repository.sources.length === 0) fail(`no sources for ${pair.testOrg}/${repository.name}`); + if (!Array.isArray(repository.focus) || repository.focus.length < 2) warn(`shallow focus list for ${pair.testOrg}/${repository.name}`); + for (const source of repository.sources ?? []) { + if (!/^[^/]+\/[^/]+$/.test(source)) fail(`invalid source ${source} for ${pair.testOrg}/${repository.name}`); + if (excluded.has(source.split('/')[0])) fail(`excluded source ${source} in ${pair.testOrg}/${repository.name}`); + } + for (const dependency of repository.imports ?? []) { + importCount += 1; + if (!allowedModes.has(dependency.mode)) fail(`unsupported import mode ${dependency.mode} in ${pair.testOrg}/${repository.name}`); + if (!/^[^/]+\/[^/]+$/.test(dependency.source ?? '')) fail(`invalid import source in ${pair.testOrg}/${repository.name}`); + if (dependency.mode === 'git-submodule' && !dependency.path) fail(`submodule missing path in ${pair.testOrg}/${repository.name}`); + if (dependency.mode === 'native-package') { + if (!dependency.manager) fail(`native package missing manager in ${pair.testOrg}/${repository.name}`); + managers.set(dependency.manager, (managers.get(dependency.manager) ?? 0) + 1); + } + } + } +} + +const fiducia = manifest.pairs?.find((pair) => pair.testOrg === 'fiducia-cloud-test'); +const fiduciaClients = fiducia?.repositories?.filter((repository) => /^fiducia-.+-client-e2e$/.test(repository.name)) ?? []; +if (fiduciaClients.length !== 10) fail(`fiducia-cloud-test must define exactly 10 language client repos; found ${fiduciaClients.length}`); +for (const repository of fiduciaClients) { + const modes = new Set(repository.imports?.map((dependency) => dependency.mode)); + for (const required of ['git-submodule', 'zed', 'native-package']) { + if (!modes.has(required)) fail(`${repository.name} missing ${required} lane`); + } +} + +for (const pattern of [/ghp_[A-Za-z0-9]+/, /github_pat_[A-Za-z0-9_]+/, /Bearer\s+[A-Za-z0-9._-]{20,}/i]) { + if (pattern.test(serialized)) fail(`credential-shaped value found in ${manifestPath}`); +} + +const summary = { + schemaVersion: manifest.schemaVersion, + pairs: manifest.pairs?.length ?? 0, + governanceRepositories: manifest.pairs?.length ?? 0, + createRepositories: createCount, + existingRepositories: existingCount, + imports: importCount, + nativeManagers: Object.fromEntries([...managers.entries()].sort(([a], [b]) => a.localeCompare(b))), + warnings: warnings.length, + errors: errors.length, +}; + +if (process.argv.includes('--json')) { + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); +} else { + console.log(`validated ${summary.pairs} org pairs, ${summary.createRepositories} create repos, ${summary.governanceRepositories} governance repos, ${summary.existingRepositories} existing fixtures, ${summary.imports} dependency lanes`); + for (const message of warnings) console.warn(`warning: ${message}`); +} +if (errors.length) { + for (const message of errors) console.error(`error: ${message}`); + process.exit(1); +} diff --git a/scripts/verify-live-test-org-fleet.mjs b/scripts/verify-live-test-org-fleet.mjs new file mode 100644 index 00000000..95c00609 --- /dev/null +++ b/scripts/verify-live-test-org-fleet.mjs @@ -0,0 +1,176 @@ +#!/usr/bin/env node +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { readFleetManifest } from './read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const manifest = readFleetManifest(path.join(root, 'bootstrap', 'test-org-fleet.json.gz')); +const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; +const apiRoot = process.env.GITHUB_API_URL || 'https://api.github.com'; +const concurrency = Math.max(1, Math.min(8, Number(process.env.FLEET_VERIFY_CONCURRENCY || 4))); + +if (!token) throw new Error('GH_TOKEN or GITHUB_TOKEN is required for live fleet verification'); + +const excluded = new Set(['r2g', 'r2g-test']); +const expected = []; +for (const pair of manifest.pairs) { + const organization = pair.testOrg; + if (excluded.has(organization.toLowerCase())) { + throw new Error(`refusing excluded organization in manifest: ${organization}`); + } + expected.push({ organization, repository: '.github', kind: 'governance' }); + for (const repository of pair.repositories) { + expected.push({ organization, repository: repository.name, kind: 'specialized' }); + } +} + +const expectedSpecialized = expected.filter((entry) => entry.kind === 'specialized').length; +const expectedGovernance = expected.filter((entry) => entry.kind === 'governance').length; +function delay(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function apiRequest(endpoint, { allow404 = false } = {}) { + for (let attempt = 0; attempt < 8; attempt += 1) { + const response = await fetch(`${apiRoot}${endpoint}`, { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'zed-pkg-test-live-fleet-verifier', + }, + }); + const text = await response.text(); + if (response.ok) return text ? JSON.parse(text) : null; + if (allow404 && response.status === 404) return null; + + const secondary = response.status === 403 && /secondary rate limit|abuse detection|temporarily blocked/i.test(text); + const retryable = secondary || response.status === 429 || response.status >= 500; + if (retryable && attempt < 7) { + const retryAfter = Number(response.headers.get('retry-after') || 0) * 1000; + const backoff = Math.min(120000, (secondary ? 10000 : 1000) * (2 ** attempt)); + await delay(Math.max(retryAfter, backoff)); + continue; + } + throw new Error(`GitHub API ${response.status} for ${endpoint}: ${text.slice(0, 500)}`); + } + throw new Error(`exhausted retries for ${endpoint}`); +} + +async function verifyRepository(entry) { + const owner = encodeURIComponent(entry.organization); + const name = encodeURIComponent(entry.repository); + const repository = await apiRequest(`/repos/${owner}/${name}`, { allow404: true }); + if (!repository) { + return { ...entry, present: false, defaultBranch: null, defaultBranchPresent: false, openPullRequests: 0, draftPullRequests: 0 }; + } + + const defaultBranch = repository.default_branch || 'main'; + const ref = await apiRequest( + `/repos/${owner}/${name}/git/ref/heads/${encodeURIComponent(defaultBranch)}`, + { allow404: true }, + ); + const pulls = await apiRequest(`/repos/${owner}/${name}/pulls?state=open&per_page=100`); + return { + ...entry, + present: true, + visibility: repository.visibility, + defaultBranch, + defaultBranchPresent: Boolean(ref), + openPullRequests: pulls.length, + draftPullRequests: pulls.filter((pull) => pull.draft).length, + pullRequests: pulls.map((pull) => ({ + number: pull.number, + title: pull.title, + draft: pull.draft, + head: pull.head?.ref || null, + htmlUrl: pull.html_url, + })), + }; +} + +async function mapConcurrent(items, limit, operation) { + const results = new Array(items.length); + let cursor = 0; + async function worker() { + while (true) { + const index = cursor; + cursor += 1; + if (index >= items.length) return; + if (index % 25 === 0) process.stderr.write(`verifying ${index + 1}/${items.length}\n`); + results[index] = await operation(items[index]); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker())); + return results; +} + +const repositories = await mapConcurrent(expected, concurrency, verifyRepository); +const missingRepositories = repositories.filter((entry) => !entry.present); +const missingDefaultBranches = repositories.filter((entry) => entry.present && !entry.defaultBranchPresent); +const missingOpenPullRequests = repositories.filter((entry) => entry.present && entry.openPullRequests === 0); +const specialized = repositories.filter((entry) => entry.kind === 'specialized'); +const governance = repositories.filter((entry) => entry.kind === 'governance'); + +const summary = { + generatedAt: new Date().toISOString(), + expected: { + organizations: manifest.pairs.length, + specializedRepositories: expectedSpecialized, + governanceRepositories: expectedGovernance, + totalRepositories: expected.length, + }, + verified: { + presentRepositories: repositories.length - missingRepositories.length, + specializedRepositories: specialized.filter((entry) => entry.present).length, + governanceRepositories: governance.filter((entry) => entry.present).length, + repositoriesWithDefaultBranch: repositories.length - missingRepositories.length - missingDefaultBranches.length, + repositoriesWithOpenPullRequest: repositories.filter((entry) => entry.present && entry.openPullRequests > 0).length, + openPullRequests: repositories.reduce((total, entry) => total + entry.openPullRequests, 0), + draftPullRequests: repositories.reduce((total, entry) => total + entry.draftPullRequests, 0), + }, + failures: { + missingRepositories: missingRepositories.map(({ organization, repository, kind }) => ({ organization, repository, kind })), + missingDefaultBranches: missingDefaultBranches.map(({ organization, repository, kind, defaultBranch }) => ({ organization, repository, kind, defaultBranch })), + missingOpenPullRequests: missingOpenPullRequests.map(({ organization, repository, kind }) => ({ organization, repository, kind })), + }, + repositories, +}; + +process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); + +if (process.env.GITHUB_STEP_SUMMARY) { + const failureRows = [ + ['Missing repositories', summary.failures.missingRepositories.length], + ['Missing default branches', summary.failures.missingDefaultBranches.length], + ['Repositories without an open PR', summary.failures.missingOpenPullRequests.length], + ]; + const markdown = [ + '# Live test-organization fleet verification', + '', + `Generated: ${summary.generatedAt}`, + '', + '| Contract | Expected | Verified |', + '|---|---:|---:|', + `| Test organizations | ${summary.expected.organizations} | ${new Set(repositories.filter((entry) => entry.present).map((entry) => entry.organization)).size} |`, + `| Specialized repositories | ${summary.expected.specializedRepositories} | ${summary.verified.specializedRepositories} |`, + `| Governance repositories | ${summary.expected.governanceRepositories} | ${summary.verified.governanceRepositories} |`, + `| Total repositories | ${summary.expected.totalRepositories} | ${summary.verified.presentRepositories} |`, + `| Repositories with a default branch | ${summary.expected.totalRepositories} | ${summary.verified.repositoriesWithDefaultBranch} |`, + `| Repositories with an open PR | ${summary.expected.totalRepositories} | ${summary.verified.repositoriesWithOpenPullRequest} |`, + `| Open PRs observed | — | ${summary.verified.openPullRequests} |`, + `| Draft PRs observed | — | ${summary.verified.draftPullRequests} |`, + '', + '| Failure class | Count |', + '|---|---:|', + ...failureRows.map(([label, count]) => `| ${label} | ${count} |`), + '', + '`r2g` and `r2g-test` are excluded from the manifest-derived verification scope.', + ]; + fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${markdown.join('\n')}\n`); +} + +if (missingRepositories.length || missingDefaultBranches.length || missingOpenPullRequests.length) { + process.exitCode = 1; +} diff --git a/scripts/verify_formal_app_lifecycle_contract.py b/scripts/verify_formal_app_lifecycle_contract.py new file mode 100644 index 00000000..cec38a60 --- /dev/null +++ b/scripts/verify_formal_app_lifecycle_contract.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Cross-repository checks for zed-sync's managed application lifecycle.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +try: + import tomllib +except ImportError as error: # pragma: no cover - hosted runners use Python 3.11+ + raise SystemExit("Python 3.11 or newer is required for tomllib") from error + + +EXPECTED_EVENTS = { + "start_requested", + "start_succeeded", + "start_failed", + "connectivity_changed", + "runtime_failed", + "stop_requested", + "stop_succeeded", + "stop_failed", + "reconcile_requested", +} +EXPECTED_OUTCOMES = {"applied", "stuttered", "stale", "rejected"} +EXPECTED_PHASES = {"stopped", "starting", "online", "offline", "stopping", "failed"} +EXPECTED_WITNESSES = { + "online_reached", + "offline_reached", + "failed_reached", + "stale_completion_reached", + "rejected_transition_reached", + "failure_reconciliation_reached", +} +STATE_KEYS = { + "phase", + "operation", + "generation", + "active_token", + "desired_running", + "online", + "failure", +} + + +class ContractError(ValueError): + """The pinned production tree does not implement the declared contract.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ContractError(message) + + +def load_json(path: Path) -> dict: + require(path.is_file(), f"missing required JSON artifact: {path}") + try: + return json.loads(path.read_text(encoding="utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ContractError(f"invalid JSON artifact {path}: {error}") from error + + +def verify(root: Path) -> None: + root = root.resolve() + manifest_path = root / "formal/app-lifecycle.fm.toml" + model_path = root / "formal/app_lifecycle.qnt" + fixture_path = root / "protocol/formal-app-lifecycle.json" + schema_path = root / "protocol/formal-app-lifecycle.schema.json" + + require(manifest_path.is_file(), f"missing formal manifest: {manifest_path}") + manifest = tomllib.loads(manifest_path.read_text(encoding="utf-8")) + require(manifest.get("schema_version") == 1, "formal manifest schema must be v1") + require(manifest.get("project") == "zed-sync", "formal manifest project drifted") + require(manifest.get("model") == "app-lifecycle-v1", "formal model identity drifted") + require(manifest.get("language") == "quint", "formal model must remain Quint") + require(manifest.get("spec") == "formal/app_lifecycle.qnt", "formal spec path drifted") + require(manifest.get("main") == "app_lifecycle", "Quint main module drifted") + require( + set(manifest.get("invariants", [])) == {"app_lifecycle_safety"}, + "formal manifest must declare exactly app_lifecycle_safety", + ) + require( + set(manifest.get("witnesses", [])) == EXPECTED_WITNESSES, + "formal manifest witness set is incomplete or unexpected", + ) + require( + manifest.get("verification", {}).get("backend") == "tlc" + and manifest.get("verification", {}).get("exhaustive_finite_model") is True, + "formal manifest must retain exhaustive finite TLC verification", + ) + require( + manifest.get("toolchain", {}).get("quint") == "0.32.0", + "Quint verifier version must stay pinned", + ) + + require(model_path.is_file(), f"missing Quint model: {model_path}") + model = model_path.read_text(encoding="utf-8") + for symbol in {"app_lifecycle_safety", *EXPECTED_WITNESSES}: + require(symbol in model, f"Quint model is missing declared symbol {symbol!r}") + + fixture = load_json(fixture_path) + require(fixture.get("schema_version") == 1, "trace fixture schema must be v1") + require(fixture.get("model") == "app-lifecycle-v1", "trace fixture model drifted") + cases = fixture.get("cases") + require(isinstance(cases, list) and len(cases) >= 6, "trace fixture needs six cases") + + events: set[str] = set() + outcomes: set[str] = set() + phases: set[str] = set() + for case in cases: + require(isinstance(case, dict), "every trace case must be an object") + require(isinstance(case.get("name"), str) and case["name"], "trace case needs a name") + steps = case.get("steps") + require(isinstance(steps, list) and len(steps) >= 2, "trace case needs two steps") + for step in steps: + require(isinstance(step, dict), "every trace step must be an object") + event = step.get("event") + state = step.get("state") + require(isinstance(event, dict), "trace step is missing its event object") + require(isinstance(state, dict), "trace step is missing its state object") + require(set(state) == STATE_KEYS, "trace state keys drifted from the contract") + events.add(event.get("type")) + outcomes.add(step.get("outcome")) + phases.add(state.get("phase")) + + require(events == EXPECTED_EVENTS, f"event coverage drifted: {sorted(events)}") + require(outcomes == EXPECTED_OUTCOMES, f"outcome coverage drifted: {sorted(outcomes)}") + require(phases == EXPECTED_PHASES, f"phase coverage drifted: {sorted(phases)}") + + schema = load_json(schema_path) + require( + schema.get("$schema") == "https://json-schema.org/draft/2020-12/schema", + "trace schema must remain Draft 2020-12", + ) + definitions = schema.get("$defs", {}) + require( + set(definitions.get("event", {}).get("properties", {}).get("type", {}).get("enum", [])) + == EXPECTED_EVENTS, + "JSON Schema event enum drifted from the fixture contract", + ) + require( + set(definitions.get("step", {}).get("properties", {}).get("outcome", {}).get("enum", [])) + == EXPECTED_OUTCOMES, + "JSON Schema outcome enum drifted from the fixture contract", + ) + require( + set(definitions.get("state", {}).get("properties", {}).get("phase", {}).get("enum", [])) + == EXPECTED_PHASES, + "JSON Schema phase enum drifted from the fixture contract", + ) + + runtime_contracts = { + "Rust": ( + root / "src/lifecycle.rs", + root / "tests/formal_app_lifecycle.rs", + "formal-app-lifecycle.json", + ), + "JavaScript": ( + root / "sdk/src/lifecycle.mjs", + root / "sdk/test/formal-app-lifecycle.test.mjs", + "formal-app-lifecycle.json", + ), + "Dart": ( + root / "dart/zed_sync/lib/src/lifecycle.dart", + root / "dart/zed_sync/test/formal_app_lifecycle_test.dart", + "formal-app-lifecycle.json", + ), + } + for language, (implementation, refinement, fixture_name) in runtime_contracts.items(): + require(implementation.is_file(), f"missing {language} lifecycle reducer") + require(refinement.is_file(), f"missing {language} lifecycle refinement test") + require( + fixture_name in refinement.read_text(encoding="utf-8"), + f"{language} refinement test no longer consumes the shared trace fixture", + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("zed_sync_root", type=Path) + arguments = parser.parse_args() + verify(arguments.zed_sync_root) + print("formal application lifecycle cross-repository contract: OK") + + +if __name__ == "__main__": + try: + main() + except ContractError as error: + raise SystemExit(f"formal application lifecycle contract failed: {error}") from error diff --git a/scripts/verify_nix_flake_bundle_canary.py b/scripts/verify_nix_flake_bundle_canary.py new file mode 100644 index 00000000..aeca995c --- /dev/null +++ b/scripts/verify_nix_flake_bundle_canary.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Independent persisted-bundle verifier and tamper matrix for Zed → Nix.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import stat +import sys +from pathlib import Path, PurePosixPath +from typing import Any + +SCHEMA = "zed.nix-flake-bundle/v1" +INVENTORY_PATH = "metadata/bundle.json" +EXPECTED_PATHS = { + "README.md", + "artifacts/example-sample-1.2.3.tar.gz", + "flake.lock", + "flake.nix", + "metadata/bundle.json", + "metadata/plan.json", + "package.nix", +} + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def canonical_json(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def safe_relative_path(value: str) -> PurePosixPath: + if not value or value.startswith("/") or "\\" in value: + raise AssertionError("bundle contains an unsafe path") + path = PurePosixPath(value) + if any(part in {"", ".", ".."} for part in path.parts): + raise AssertionError("bundle contains an unsafe path component") + if any(ord(character) < 32 or ord(character) == 127 for character in value): + raise AssertionError("bundle contains a control-bearing path") + return path + + +def regular_files(root: Path) -> dict[str, bytes]: + files: dict[str, bytes] = {} + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise AssertionError("persisted bundle contains a symbolic link") + if path.is_dir(): + continue + if not path.is_file(): + raise AssertionError("persisted bundle contains a special file") + safe_relative_path(relative) + if stat.S_IMODE(path.stat().st_mode) != 0o644: + raise AssertionError("persisted bundle file mode is not canonical") + files[relative] = path.read_bytes() + return files + + +def bundle_digest(entries: list[dict[str, Any]]) -> str: + digest = hashlib.sha256() + digest.update(SCHEMA.encode("utf-8")) + digest.update(b"\0") + for entry in entries: + path = str(entry["path"]) + path_bytes = path.encode("utf-8") + artifact_digest = bytes.fromhex(str(entry["sha256"])) + if len(artifact_digest) != 32: + raise AssertionError("bundle entry digest length is invalid") + digest.update(len(path_bytes).to_bytes(8, "big")) + digest.update(path_bytes) + digest.update(int(entry["size"]).to_bytes(8, "big")) + digest.update(artifact_digest) + return digest.hexdigest() + + +def verify_bundle(root: Path, forbidden: tuple[str, ...]) -> dict[str, Any]: + if not root.is_dir(): + raise AssertionError("bundle root is missing") + files = regular_files(root) + if set(files) != EXPECTED_PATHS: + raise AssertionError("bundle file set differs from the strict v1 contract") + + inventory_bytes = files[INVENTORY_PATH] + inventory = json.loads(inventory_bytes) + if canonical_json(inventory) != inventory_bytes: + raise AssertionError("bundle inventory is not canonical compact JSON") + if inventory.get("schema") != SCHEMA: + raise AssertionError("bundle schema is unsupported") + + entries = inventory.get("entries") + if not isinstance(entries, list): + raise AssertionError("bundle inventory entries are missing") + paths = [entry.get("path") for entry in entries] + if paths != sorted(paths) or len(paths) != len(set(paths)): + raise AssertionError("bundle inventory paths are not sorted and unique") + if paths != sorted(set(files) - {INVENTORY_PATH}): + raise AssertionError("bundle inventory does not cover the exact file set") + + for entry in entries: + path = str(entry["path"]) + safe_relative_path(path) + data = files[path] + if int(entry["size"]) != len(data): + raise AssertionError("bundle entry size evidence does not match") + if str(entry["sha256"]) != sha256(data): + raise AssertionError("bundle entry digest evidence does not match") + + if inventory.get("bundle_sha256") != bundle_digest(entries): + raise AssertionError("domain-separated bundle digest does not match") + if inventory.get("plan_sha256") != sha256(files["metadata/plan.json"]): + raise AssertionError("plan digest does not match") + if inventory.get("flake_lock_sha256") != sha256(files["flake.lock"]): + raise AssertionError("flake lock digest does not match") + + for value in forbidden: + if not value: + continue + needle = value.encode("utf-8") + if any(needle in data for data in files.values()): + raise AssertionError("bundle retained forbidden ambient input") + return inventory + + +def assert_expected_failure(root: Path, mutate) -> None: + candidate = root.parent / f"{root.name}-tamper" + if candidate.exists(): + shutil.rmtree(candidate) + shutil.copytree(root, candidate) + mutate(candidate) + try: + verify_bundle(candidate, ()) + except (AssertionError, json.JSONDecodeError, KeyError, ValueError): + shutil.rmtree(candidate) + return + raise AssertionError("tampered persisted bundle unexpectedly verified") + + +def exercise_tamper_matrix(root: Path) -> None: + assert_expected_failure( + root, + lambda candidate: (candidate / "package.nix").open("ab").write(b"\n# tampered\n"), + ) + assert_expected_failure( + root, + lambda candidate: (candidate / "artifacts/example-sample-1.2.3.tar.gz").unlink(), + ) + assert_expected_failure( + root, + lambda candidate: (candidate / "unexpected.txt").write_text( + "unexpected", encoding="utf-8" + ), + ) + assert_expected_failure( + root, + lambda candidate: (candidate / "flake.lock").open("ab").write(b"\n"), + ) + + def rewrite_inventory(candidate: Path) -> None: + path = candidate / INVENTORY_PATH + value = json.loads(path.read_bytes()) + path.write_text(json.dumps(value, indent=2), encoding="utf-8") + + assert_expected_failure(root, rewrite_inventory) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--bundle-a", required=True, type=Path) + parser.add_argument("--bundle-b", required=True, type=Path) + parser.add_argument("--report", required=True, type=Path) + parser.add_argument("--forbidden", action="append", default=[]) + parser.add_argument("--exercise-tamper", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + bundle_a = args.bundle_a.resolve() + bundle_b = args.bundle_b.resolve() + report_path = args.report.resolve() + forbidden = tuple(args.forbidden) + + inventory_a = verify_bundle(bundle_a, forbidden) + inventory_b = verify_bundle(bundle_b, forbidden) + if regular_files(bundle_a) != regular_files(bundle_b): + raise AssertionError("clean-room bundle replays are not byte-identical") + if inventory_a != inventory_b: + raise AssertionError("clean-room bundle inventories differ") + + report = json.loads(report_path.read_bytes()) + if report.get("schema") != "zed-pkg-test.nix-flake-bundle-canary/v1": + raise AssertionError("external canary report schema is unsupported") + if report.get("bundle_sha256") != inventory_a.get("bundle_sha256"): + raise AssertionError("external report is not bound to the bundle inventory") + if report.get("plan_sha256") != inventory_a.get("plan_sha256"): + raise AssertionError("external report is not bound to the plan") + if report.get("flake_lock_sha256") != inventory_a.get("flake_lock_sha256"): + raise AssertionError("external report is not bound to the flake lock") + if report.get("credential_canaries_retained") is not False: + raise AssertionError("external report did not prove credential redaction") + if report.get("external_registry_required") is not False: + raise AssertionError("external report unexpectedly requires a registry") + + if args.exercise_tamper: + exercise_tamper_matrix(bundle_a) + + print( + json.dumps( + { + "result": "PASS", + "schema": report["schema"], + "candidate": report.get("candidate", ""), + "system": report["system"], + "bundle_sha256": report["bundle_sha256"], + "file_count": report["file_count"], + "tamper_cases": 5 if args.exercise_tamper else 0, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/static-registry-protocol/.gitignore b/static-registry-protocol/.gitignore new file mode 100644 index 00000000..3fa54c6e --- /dev/null +++ b/static-registry-protocol/.gitignore @@ -0,0 +1,2 @@ +tree-a/ +tree-b/ diff --git a/static-registry-protocol/README.md b/static-registry-protocol/README.md new file mode 100644 index 00000000..81d2f6fe --- /dev/null +++ b/static-registry-protocol/README.md @@ -0,0 +1,122 @@ +# static-registry-protocol — fixture + conformance harness (sketch v0) + +Groundwork for the zpkg registry-protocol work in Linear project **zpkg +Self-Hosted Registries and Private Deps**: the protocol RFC (DEN-2854), the +static read-only export (DEN-2857), and the static arm of the e2e certification +(DEN-2861). GitHub anchor: `zed-pkg/zed-interfaces#45`. + +This is **not** the final protocol. It exists to make the load-bearing design +constraint testable *now*: + +> the public read path is plain files — an entire registry read side must be +> servable by dumb object/static hosting with no running daemon. + +What v0 deliberately omits (owned by the DEN-2854 RFC): `registry_id` and trust +roots, signed checkpoints (the `checkpoint.json` here has a `signature: null` +slot), lifecycle states beyond `yanked`, and projection/target semantics. + +## Layout produced + +``` +.well-known/zpkg-registry.json discovery: schema_version, endpoints, + auth_modes=[none], publish_supported=false +index// NDJSON; one line per version, ascending semver: + {version, deps, cksum:"sha256:…", size, yanked} +pkgs///.tar.zst content-addressed deterministic tarballs +checkpoint.json {seq, files:[{path,sha256,size}], tree_sha256, + signature:null} +``` + +Org segments must already be lowercase canonical form. The generator also +requires a new or empty real output directory and rejects fixture/output +symlinks and special files. It validates and renders the complete tree before +publishing through a same-parent staging directory, so a bad later package +cannot expose earlier package blobs or leave a staging tree. + +## Tools + +- `build_static_registry.py --fixtures fixtures --out tree` — deterministic, + failure-atomic build: ustar, sorted entries, `SOURCE_DATE_EPOCH` (default 0), + uid/gid 0, `zstd -19`, same-parent staging, final directory rename. Two + builds of the same fixtures are byte-identical. +- `check_static_registry.py --base ` — conformance: discovery, + checkpoint schema and sorted unique safe paths, checkpoint↔object integrity, + exact local file set, NDJSON/semver/checksum verification, yank semantics, + and absent-package hard miss. Local trees reject path traversal, duplicate + paths, missing or uncheckpointed objects, symlinks, and special files before + unsafe content is fetched. +- `check_static_registry.py --base self-test` — **red tests**: every + declared mutation must turn the checker red. The current nine cases cover + tarball tampering, index tampering, checkpoint-entry removal, yank + resurrection, schema drift, an uncheckpointed object, a removed checkpointed + object, an unsafe `../` checkpoint path, and a duplicate checkpoint path. +- `sync_to_r2.sh ` — verifies the complete local tree, uploads + immutable `pkgs/**` with one-year immutable caching, uploads mutable v0 + discovery/index objects with `no-cache`, and publishes `checkpoint.json` + last. Replacing an existing v0 checkpoint fails closed unless an operator + explicitly sets `ZPKG_ALLOW_V0_REPLACE=true` for a disposable + non-production experiment. + +Version 0 cannot make a rolling update collection-atomic because index paths are +mutable. Checkpoint-last ordering reduces exposure for an initial publication, +but v1 must move indexes under immutable snapshot prefixes selected by a signed +stable checkpoint. + +## Permanent credential-free CI + +`.github/workflows/static-registry-protocol.yml` runs the local contract on +Ubuntu 24.04 and macOS 15 with read-only repository permissions and immutable +Action pins. Each job: + +1. verifies pinned Python, runner-provided `zstd`, Bash syntax, and Python + syntax; +2. runs twelve generator tests covering fresh/empty output, stale-output and + parent-traversal refusal, fixture/output/metadata symlink refusal, + failure-atomic multi-package validation, metadata object/field shape, + semantic-version shape, uppercase-org refusal, and empty-fixture refusal; +3. runs six fake-AWS synchronization tests against the real generated fixture, + covering exact upload order/cache policy, checkpoint-last publication, + replacement refusal/override, and pre-AWS rejection of missing, tampered, or + uncheckpointed state; +4. builds the registry twice in unrelated temporary directories with + `SOURCE_DATE_EPOCH=0`; +5. compares the complete sorted path/size/SHA-256 inventory byte-for-byte; +6. runs the green conformance check; +7. dynamically discovers and requires every adversarial mutation to fail; +8. proves the source checkout remained unchanged; and +9. uploads bounded commit-addressed local evidence for seven days. + +The permanent workflow reads no GitHub, Cloudflare, R2, registry, signing, or +shared-auth secret. Live object-store synchronization remains a separate manual +and environment-gated operation; a pull request can certify the protocol +fixture and uploader contract without network access or infrastructure +authority. + +## Live fixture (2026-08-08) + +- Bucket: `zed-pkg-static-registry-e2e` (dedicated; do **not** reuse + `zed-pkg-artifacts-e2e`, which belongs to the API-path R2 certification, and + never the prod `zed-pkg-artifacts`). +- Public base: `https://pub-9b365e012501426c9de90d2746deac9d.r2.dev` + (r2.dev is rate-limited and not a production surface; fine for e2e). +- `evidence/` holds the recorded local + live runs. + +The recorded live fixture predates the stricter v0 replacement rule. Future +manual live experiments should use a fresh dedicated bucket or explicitly mark +the bucket disposable; the permanent CI path never mutates the live fixture. + +## Operational findings worth carrying into the RFC (DEN-2854) + +1. **r2.dev filters script User-Agents**: `Python-urllib/3.9` gets `403`; + curl's UA and a custom `zpkg-static-registry-check/0` UA pass. Registry + clients must always send an identifying UA; conformance tooling can't rely + on language-default agents. +2. **Managed-domain enablement propagates asynchronously**: the first ~minutes + after enabling the r2.dev domain returned intermittent `403`s. Tooling that + provisions a static registry must retry/verify before declaring it live. +3. Per-object `Cache-Control` survives verbatim through R2. Immutable package + bytes can use long caching, while mutable v0 pointers must revalidate until + v1 snapshot paths make index objects immutable. +4. Deterministic `.tar.zst` needs all of: ustar format, sorted entries, zeroed + mtime/uid/gid/uname, fixed mode, and a pinned zstd level; with those, the + whole tree (including `tree_sha256`) is byte-reproducible. diff --git a/static-registry-protocol/build_static_registry.py b/static-registry-protocol/build_static_registry.py new file mode 100644 index 00000000..21b7c390 --- /dev/null +++ b/static-registry-protocol/build_static_registry.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +"""Build a deterministic static zpkg registry tree (protocol sketch v0). + +Groundwork fixture for the registry protocol RFC (Linear DEN-2854) and the +static-export / e2e phases (DEN-2857, DEN-2861). The signed-checkpoint design +is owned by the RFC; this v0 emits an UNSIGNED checkpoint structured so a +signature slots in without changing the tree layout. + +Layout produced under --out: + .well-known/zpkg-registry.json discovery document + index// NDJSON, one line per version (semver order) + pkgs///.tar.zst content-addressed deterministic tarballs + checkpoint.json {schema_version, seq, generated_at, + files:[{path,sha256,size}], tree_sha256} + +Determinism: tar entries sorted, mtime/uid/gid zeroed via SOURCE_DATE_EPOCH +(default 0), zstd -19 with a pinned-version note recorded in the checkpoint. +Same inputs => byte-identical tree. + +Safety: output must be a new or empty real directory. Fixture symlinks and +special files are rejected rather than followed into the package archive. The +complete tree is written to a same-parent staging directory and renamed into +place only after all fixture validation and file writes succeed. +""" +import argparse +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + +SCHEMA_VERSION = 0 +EPOCH = int(os.environ.get("SOURCE_DATE_EPOCH", "0")) + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def validate_output_target(out: Path) -> tuple[Path, bool]: + if out.name in {"", ".", ".."} or ".." in out.parts: + raise ValueError(f"output path must name a child directory without `..`: {out}") + if out.is_symlink(): + raise ValueError(f"output path is a symbolic link: {out}") + existed_empty = False + if out.exists(): + if not out.is_dir(): + raise ValueError(f"output path is not a directory: {out}") + if any(out.iterdir()): + raise ValueError( + f"output directory must be empty to prevent stale registry objects: {out}" + ) + existed_empty = True + + parent = out.parent if out.parent != Path("") else Path(".") + parent.mkdir(parents=True, exist_ok=True) + if parent.is_symlink() or not parent.is_dir(): + raise ValueError(f"output parent must be a real directory: {parent}") + return parent, existed_empty + + +def publish_tree(out: Path, files: dict[str, bytes]) -> None: + parent, existed_empty = validate_output_target(out) + staging = Path( + tempfile.mkdtemp( + prefix=f".{out.name}.zpkg-static-", + dir=parent, + ) + ) + committed = False + removed_empty_output = False + try: + for relative, data in sorted(files.items()): + destination = staging / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(data) + if existed_empty: + out.rmdir() + removed_empty_output = True + os.replace(staging, out) + committed = True + finally: + if not committed and staging.exists(): + shutil.rmtree(staging, ignore_errors=True) + if not committed and removed_empty_output and not out.exists(): + out.mkdir() + + +def deterministic_tar_zst(pkg_dir: Path) -> bytes: + buffer = io.BytesIO() + paths = sorted( + pkg_dir.rglob("*"), + key=lambda path: path.relative_to(pkg_dir).as_posix(), + ) + with tarfile.open(fileobj=buffer, mode="w", format=tarfile.USTAR_FORMAT) as archive: + for path in paths: + relative = path.relative_to(pkg_dir) + if path.is_symlink(): + raise ValueError( + "fixture contains a symbolic link, which static export will not follow: " + f"{relative}" + ) + if path.is_dir(): + continue + if not path.is_file(): + raise ValueError(f"fixture contains an unsupported special file: {relative}") + data = path.read_bytes() + entry = tarfile.TarInfo(name=relative.as_posix()) + entry.size = len(data) + entry.mtime = EPOCH + entry.uid = entry.gid = 0 + entry.uname = entry.gname = "" + entry.mode = 0o644 + archive.addfile(entry, io.BytesIO(data)) + result = subprocess.run( + ["zstd", "-19", "-q", "--no-progress", "-c"], + input=buffer.getvalue(), + stdout=subprocess.PIPE, + check=True, + ) + return result.stdout + + +def semver_key(version: str): + core = version.split("-")[0].split("+")[0] + components = core.split(".") + if len(components) != 3 or any(not component.isdigit() for component in components): + raise ValueError(f"invalid v0 semantic version: {version}") + return tuple(int(component) for component in components) + + +def read_fixture_metadata(meta_path: Path) -> dict: + if meta_path.is_symlink(): + raise ValueError(f"fixture metadata is a symbolic link: {meta_path}") + metadata = json.loads(meta_path.read_text()) + if not isinstance(metadata, dict): + raise ValueError(f"fixture metadata must be a JSON object: {meta_path}") + dependencies = metadata.get("deps", []) + yanked = metadata.get("yanked", False) + if not isinstance(dependencies, list): + raise ValueError(f"fixture `deps` must be an array: {meta_path}") + if not isinstance(yanked, bool): + raise ValueError(f"fixture `yanked` must be boolean: {meta_path}") + return metadata + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--fixtures", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + parser.add_argument("--seq", type=int, default=1, help="checkpoint sequence number") + args = parser.parse_args() + + if not args.fixtures.is_dir() or args.fixtures.is_symlink(): + print(f"ERROR: fixtures path must be a real directory: {args.fixtures}", file=sys.stderr) + return 2 + + # Validate the caller-owned destination before spending compression work, + # but do not create or mutate it until the complete tree is ready. + try: + validate_output_target(args.out) + except (OSError, ValueError) as error: + print(f"ERROR: {error}", file=sys.stderr) + return 2 + + objects: dict[str, bytes] = {} + + def emit(relative: str, data: bytes) -> None: + if relative in objects: + raise ValueError(f"duplicate static registry object path: {relative}") + objects[relative] = data + + # Packages: fixtures////{zpkg.json, files...} + packages: dict[tuple[str, str], list[dict]] = {} + for meta_path in sorted(args.fixtures.glob("*/*/*/zpkg.json")): + version_dir = meta_path.parent + version = version_dir.name + name = version_dir.parent.name + org = version_dir.parent.parent.name + if org != org.lower(): + print(f"ERROR: org segment must be lowercase canonical form: {org}", file=sys.stderr) + return 2 + try: + semver_key(version) + metadata = read_fixture_metadata(meta_path) + blob = deterministic_tar_zst(version_dir) + except (OSError, ValueError, json.JSONDecodeError, subprocess.CalledProcessError) as error: + print(f"ERROR: cannot export {org}/{name}@{version}: {error}", file=sys.stderr) + return 2 + emit(f"pkgs/{org}/{name}/{version}.tar.zst", blob) + packages.setdefault((org, name), []).append( + { + "version": version, + "deps": metadata.get("deps", []), + "cksum": f"sha256:{sha256_bytes(blob)}", + "size": len(blob), + "yanked": metadata.get("yanked", False), + } + ) + + if not packages: + print("ERROR: fixture set contains no package versions", file=sys.stderr) + return 2 + + try: + for (org, name), lines in sorted(packages.items()): + lines.sort(key=lambda line: semver_key(line["version"])) + ndjson = "".join( + json.dumps(line, sort_keys=True, separators=(",", ":")) + "\n" + for line in lines + ) + emit(f"index/{org}/{name}", ndjson.encode()) + except (ValueError, TypeError) as error: + print(f"ERROR: invalid package version metadata: {error}", file=sys.stderr) + return 2 + + discovery = { + "schema_version": SCHEMA_VERSION, + "registry_kind": "static-export", + "endpoints": { + "index": "/index", + "pkgs": "/pkgs", + "checkpoint": "/checkpoint.json", + }, + "auth_modes": ["none"], + "publish_supported": False, + "notes": "protocol sketch v0 fixture; signed checkpoint + registry_id land with the DEN-2854 RFC", + } + emit( + ".well-known/zpkg-registry.json", + (json.dumps(discovery, indent=2, sort_keys=True) + "\n").encode(), + ) + + files_entry = [ + {"path": relative, "sha256": sha256_bytes(data), "size": len(data)} + for relative, data in sorted(objects.items()) + ] + tree_material = "".join( + f"{entry['path']} {entry['sha256']}\n" for entry in files_entry + ) + checkpoint = { + "schema_version": SCHEMA_VERSION, + "seq": args.seq, + "generated_at_epoch": EPOCH, + "zstd_note": "compressed with zstd -19; version recorded by the sync evidence, not trusted by verifiers", + "signature": None, + "files": files_entry, + "tree_sha256": sha256_bytes(tree_material.encode()), + } + emit( + "checkpoint.json", + (json.dumps(checkpoint, indent=2, sort_keys=True) + "\n").encode(), + ) + + try: + publish_tree(args.out, objects) + except (OSError, ValueError) as error: + print(f"ERROR: cannot publish static registry tree: {error}", file=sys.stderr) + return 2 + + print( + f"built {len(files_entry)} objects + checkpoint " + f"(tree_sha256={checkpoint['tree_sha256'][:12]}…)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/static-registry-protocol/check_static_registry.py b/static-registry-protocol/check_static_registry.py new file mode 100644 index 00000000..5df813e8 --- /dev/null +++ b/static-registry-protocol/check_static_registry.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +"""Conformance checker for the static zpkg registry tree (protocol sketch v0). + +Runs against a local tree (--base /path) or a live static host +(--base https://host). Also provides `self-test`, which proves every +assertion can fail (mutation/red tests per the DEN-2861 acceptance rule: +a green check is meaningless unless red was reachable). +""" +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +import tempfile +import urllib.error +import urllib.request +from pathlib import Path + +SCHEMA_VERSION = 0 +FAILURES: list[str] = [] + + +def fail(msg: str) -> None: + FAILURES.append(msg) + print(f" FAIL: {msg}") + + +def ok(msg: str) -> None: + print(f" ok: {msg}") + + +def safe_relative_path(relative: str) -> bool: + if not isinstance(relative, str) or not relative or relative.startswith("/"): + return False + if "\\" in relative or any( + ord(character) < 32 or ord(character) == 127 for character in relative + ): + return False + return all(part not in {"", ".", ".."} for part in relative.split("/")) + + +def checkpoint_path_allowed(relative: str) -> bool: + if not safe_relative_path(relative) or relative == "checkpoint.json": + return False + parts = relative.split("/") + if relative == ".well-known/zpkg-registry.json": + return True + if len(parts) == 3 and parts[0] == "index": + return parts[1] == parts[1].lower() + if len(parts) == 4 and parts[0] == "pkgs" and parts[3].endswith(".tar.zst"): + return parts[1] == parts[1].lower() + return False + + +class Base: + def __init__(self, base: str): + self.base = base.rstrip("/") + self.http = base.startswith("http://") or base.startswith("https://") + self.root = None if self.http else Path(self.base) + + # r2.dev (and some static hosts) 403 default script User-Agents like + # Python-urllib; real registry clients identify themselves anyway. + UA = {"User-Agent": "zpkg-static-registry-check/0"} + + def local_path(self, rel: str) -> Path: + if self.root is None: + raise ValueError("HTTP base has no local path") + if not safe_relative_path(rel): + raise ValueError(f"unsafe static registry object path: {rel!r}") + if self.root.is_symlink(): + raise ValueError(f"local registry root is a symbolic link: {self.root}") + path = self.root + for part in rel.split("/"): + path = path / part + if path.is_symlink(): + raise ValueError(f"local registry object path crosses a symbolic link: {rel}") + return path + + def fetch(self, rel: str) -> bytes: + if self.http: + request = urllib.request.Request(f"{self.base}/{rel}", headers=self.UA) + with urllib.request.urlopen(request, timeout=30) as response: + return response.read() + return self.local_path(rel).read_bytes() + + def missing(self, rel: str) -> bool: + if self.http: + try: + urllib.request.urlopen( + urllib.request.Request(f"{self.base}/{rel}", headers=self.UA), + timeout=15, + ) + return False + except urllib.error.HTTPError as error: + return error.code == 404 + return not self.local_path(rel).exists() + + +def semver_key(version: str): + core = version.split("-")[0].split("+")[0] + components = core.split(".") + if len(components) != 3 or any(not component.isdigit() for component in components): + raise ValueError(f"invalid v0 semantic version: {version}") + return tuple(int(component) for component in components) + + +def canonical_sha256(value) -> bool: + if not isinstance(value, str) or len(value) != 64 or value != value.lower(): + return False + try: + int(value, 16) + except ValueError: + return False + return True + + +def fetch_json(base: Base, relative: str, label: str): + try: + return json.loads(base.fetch(relative)) + except (OSError, ValueError, json.JSONDecodeError, urllib.error.URLError) as error: + fail(f"cannot read {label} `{relative}`: {error}") + return None + + +def validated_checkpoint_entries(checkpoint: dict) -> list[dict] | None: + if not isinstance(checkpoint, dict): + fail("checkpoint must be a JSON object") + return None + if checkpoint.get("schema_version") != SCHEMA_VERSION: + fail( + f"checkpoint schema_version {checkpoint.get('schema_version')} != {SCHEMA_VERSION}" + ) + if checkpoint.get("signature") is not None: + fail("v0 checkpoint signature slot must be null") + if not canonical_sha256(checkpoint.get("tree_sha256")): + fail("checkpoint tree_sha256 must be canonical lowercase SHA-256") + files = checkpoint.get("files") + if not isinstance(files, list) or not files: + fail("checkpoint files must be a non-empty array") + return None + + validated = [] + paths = [] + for position, entry in enumerate(files): + if not isinstance(entry, dict): + fail(f"checkpoint files[{position}] must be an object") + continue + relative = entry.get("path") + digest = entry.get("sha256") + size = entry.get("size") + if not checkpoint_path_allowed(relative): + fail(f"checkpoint contains unsafe or unsupported object path: {relative!r}") + continue + if not canonical_sha256(digest): + fail(f"checkpoint has invalid canonical SHA-256 for `{relative}`") + continue + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + fail(f"checkpoint has invalid size for `{relative}`") + continue + validated.append(entry) + paths.append(relative) + + if len(paths) != len(set(paths)): + fail("checkpoint contains duplicate object paths") + if paths != sorted(paths): + fail("checkpoint object paths must be in ascending lexical order") + if FAILURES: + return None + return validated + + +def check_local_file_set(base: Base, entries: list[dict]) -> None: + if base.http: + return + root = base.root + assert root is not None + if not root.is_dir() or root.is_symlink(): + fail(f"local registry root must be a real directory: {root}") + return + expected = {entry["path"] for entry in entries} + expected.add("checkpoint.json") + actual: set[str] = set() + for path in root.rglob("*"): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + fail(f"local registry tree contains symbolic link `{relative}`") + elif path.is_file(): + actual.add(relative) + elif not path.is_dir(): + fail(f"local registry tree contains special file `{relative}`") + missing = sorted(expected - actual) + extra = sorted(actual - expected) + if missing: + fail(f"local registry tree is missing checkpointed files: {missing}") + if extra: + fail(f"local registry tree contains uncheckpointed files: {extra}") + if not missing and not extra: + ok("local file set exactly matches checkpoint plus checkpoint.json") + + +def run_checks(base: Base) -> int: + FAILURES.clear() + print(f"== conformance against {base.base}") + + print("[1] discovery document") + discovery = fetch_json(base, ".well-known/zpkg-registry.json", "discovery document") + if discovery is None: + print("== FAIL (discovery unavailable)") + return 1 + if not isinstance(discovery, dict): + fail("discovery document must be a JSON object") + print(f"== FAIL ({len(FAILURES)})") + return 1 + if discovery.get("schema_version") != SCHEMA_VERSION: + fail( + f"discovery schema_version {discovery.get('schema_version')} != {SCHEMA_VERSION}" + ) + else: + ok("schema_version matches") + if ( + discovery.get("publish_supported") is not False + or discovery.get("auth_modes") != ["none"] + ): + fail("static export must advertise publish_supported=false, auth_modes=[none]") + else: + ok("static-export capabilities advertised correctly") + expected_endpoints = { + "index": "/index", + "pkgs": "/pkgs", + "checkpoint": "/checkpoint.json", + } + if discovery.get("endpoints") != expected_endpoints: + fail(f"discovery endpoints differ from v0 contract: {discovery.get('endpoints')}") + else: + ok("v0 endpoint map matches") + + print("[2] checkpoint integrity") + checkpoint = fetch_json(base, "checkpoint.json", "checkpoint") + if checkpoint is None: + print(f"== FAIL ({len(FAILURES)})") + return 1 + entries = validated_checkpoint_entries(checkpoint) + if entries is None: + print(f"== FAIL ({len(FAILURES)})") + return 1 + + tree_material = "".join( + f"{entry['path']} {entry['sha256']}\n" for entry in entries + ) + if hashlib.sha256(tree_material.encode()).hexdigest() != checkpoint.get("tree_sha256"): + fail("tree_sha256 does not match the files list") + else: + ok(f"tree_sha256 consistent over {len(entries)} files") + + object_mismatches = 0 + for entry in entries: + try: + data = base.fetch(entry["path"]) + except (OSError, ValueError, urllib.error.URLError) as error: + fail(f"cannot read checkpointed object `{entry['path']}`: {error}") + object_mismatches += 1 + continue + if ( + hashlib.sha256(data).hexdigest() != entry["sha256"] + or len(data) != entry["size"] + ): + fail(f"checkpoint mismatch for {entry['path']}") + object_mismatches += 1 + if object_mismatches == 0: + ok("every checkpointed object matches its recorded sha256+size") + check_local_file_set(base, entries) + + print("[3] indexes: NDJSON, semver order, tarball checksums") + index_paths = [ + entry["path"] for entry in entries if entry["path"].startswith("index/") + ] + if not index_paths: + fail("no index files listed in checkpoint") + resolvable: dict[str, list[str]] = {} + parsed_indexes: dict[str, list[dict]] = {} + for index_path in index_paths: + _, org, name = index_path.split("/", 2) + try: + raw_index = base.fetch(index_path).decode() + lines = [json.loads(line) for line in raw_index.splitlines() if line.strip()] + except (OSError, ValueError, UnicodeDecodeError, json.JSONDecodeError, urllib.error.URLError) as error: + fail(f"cannot parse index `{index_path}`: {error}") + continue + if not lines or any(not isinstance(line, dict) for line in lines): + fail(f"{index_path}: every NDJSON line must be an object") + continue + versions = [line.get("version") for line in lines] + try: + ordered = sorted(versions, key=semver_key) + except (AttributeError, TypeError, ValueError) as error: + fail(f"{index_path}: invalid semantic version: {error}") + continue + if versions != ordered or len(set(versions)) != len(versions): + fail(f"{index_path}: versions not unique/ascending semver") + for line in lines: + version = line.get("version") + checksum = line.get("cksum") + size = line.get("size") + yanked = line.get("yanked") + if not isinstance(checksum, str) or ":" not in checksum: + fail(f"{index_path}: invalid checksum for version {version!r}") + continue + algo, digest = checksum.split(":", 1) + if algo != "sha256" or not canonical_sha256(digest): + fail(f"{index_path}: unsupported or malformed checksum `{checksum}`") + continue + if not isinstance(size, int) or isinstance(size, bool) or size < 0: + fail(f"{index_path}: invalid size for version {version!r}") + continue + if not isinstance(yanked, bool): + fail(f"{index_path}: yanked must be boolean for version {version!r}") + continue + artifact_path = f"pkgs/{org}/{name}/{version}.tar.zst" + try: + blob = base.fetch(artifact_path) + except (OSError, ValueError, urllib.error.URLError) as error: + fail(f"cannot read artifact `{artifact_path}`: {error}") + continue + if hashlib.sha256(blob).hexdigest() != digest or len(blob) != size: + fail(f"{org}/{name}@{version}: tarball does not match index cksum/size") + parsed_indexes[index_path] = lines + resolvable[f"{org}/{name}"] = [ + line["version"] for line in lines if line.get("yanked") is False + ] + ok(f"{org}/{name}: {len(lines)} versions verified") + + print("[4] yank semantics") + yanked_seen = False + for index_path, lines in parsed_indexes.items(): + _, org, name = index_path.split("/", 2) + for line in lines: + if line.get("yanked") is True: + yanked_seen = True + if line.get("version") in resolvable.get(f"{org}/{name}", []): + fail(f"{org}/{name}@{line.get('version')} yanked but resolvable") + artifact_path = f"pkgs/{org}/{name}/{line.get('version')}.tar.zst" + try: + base.fetch(artifact_path) + except (OSError, ValueError, urllib.error.URLError) as error: + fail(f"yanked package bytes are not retained at `{artifact_path}`: {error}") + else: + ok( + f"{org}/{name}@{line.get('version')}: " + "yanked => unresolvable, bytes retained" + ) + if not yanked_seen: + fail("fixture set contains no yanked version — yank path untested (vacuous)") + + print("[5] absent package => missing/404") + try: + absent = base.missing("index/zpkg-e2e/definitely-not-a-package") + except (OSError, ValueError, urllib.error.URLError) as error: + fail(f"cannot test absent package: {error}") + else: + if absent: + ok("absent index is a hard miss") + else: + fail("absent package index unexpectedly present") + + print(f"== {'PASS' if not FAILURES else f'FAIL ({len(FAILURES)})'}") + return 1 if FAILURES else 0 + + +def rewrite_checkpoint(tree: Path, transform) -> None: + checkpoint_path = tree / "checkpoint.json" + checkpoint = json.loads(checkpoint_path.read_text()) + transform(checkpoint) + files = checkpoint.get("files", []) + tree_material = "".join( + f"{entry['path']} {entry['sha256']}\n" for entry in files + ) + checkpoint["tree_sha256"] = hashlib.sha256(tree_material.encode()).hexdigest() + checkpoint_path.write_text(json.dumps(checkpoint, indent=2, sort_keys=True) + "\n") + + +MUTATIONS = { + "tamper-tarball": lambda tree: _flip_byte(next((tree / "pkgs").rglob("*.tar.zst"))), + "tamper-index": lambda tree: _append( + next(path for path in sorted((tree / "index").rglob("*")) if path.is_file()), + b'{"version":"9.9.9","deps":[],"cksum":"sha256:00","size":2,"yanked":false}\n', + ), + "drop-checkpoint-entry": lambda tree: rewrite_checkpoint( + tree, lambda checkpoint: checkpoint["files"].pop(0) + ), + "resurrect-yanked": lambda tree: _unyank(tree), + "wrong-schema": lambda tree: _bump_schema(tree), + "uncheckpointed-object": lambda tree: _add_uncheckpointed_object(tree), + "remove-checkpointed-object": lambda tree: _remove_checkpointed_object(tree), + "unsafe-checkpoint-path": lambda tree: _unsafe_checkpoint_path(tree), + "duplicate-checkpoint-path": lambda tree: _duplicate_checkpoint_path(tree), +} + + +def _flip_byte(path: Path) -> None: + data = bytearray(path.read_bytes()) + data[len(data) // 2] ^= 0xFF + path.write_bytes(bytes(data)) + + +def _append(path: Path, extra: bytes) -> None: + path.write_bytes(path.read_bytes() + extra) + + +def _unyank(tree: Path) -> None: + for index_path in (tree / "index").rglob("*"): + if index_path.is_file(): + lines = [ + json.loads(line) + for line in index_path.read_text().splitlines() + if line.strip() + ] + changed = False + for line in lines: + if line["yanked"]: + line["yanked"] = False + changed = True + if changed: + index_path.write_text( + "".join( + json.dumps(line, sort_keys=True, separators=(",", ":")) + "\n" + for line in lines + ) + ) + return + raise SystemExit("no yanked line to mutate") + + +def _bump_schema(tree: Path) -> None: + discovery_path = tree / ".well-known/zpkg-registry.json" + document = json.loads(discovery_path.read_text()) + document["schema_version"] = 99 + discovery_path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n") + + +def _add_uncheckpointed_object(tree: Path) -> None: + path = tree / "index/zpkg-e2e/uncheckpointed-package" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + '{"version":"1.0.0","deps":[],"cksum":"sha256:00","size":0,"yanked":false}\n' + ) + + +def _remove_checkpointed_object(tree: Path) -> None: + checkpoint = json.loads((tree / "checkpoint.json").read_text()) + relative = checkpoint["files"][0]["path"] + (tree / relative).unlink() + + +def _unsafe_checkpoint_path(tree: Path) -> None: + rewrite_checkpoint( + tree, + lambda checkpoint: checkpoint["files"][0].update({"path": "../outside-secret"}), + ) + + +def _duplicate_checkpoint_path(tree: Path) -> None: + rewrite_checkpoint( + tree, + lambda checkpoint: checkpoint["files"].append(dict(checkpoint["files"][0])), + ) + + +def self_test(tree: Path) -> int: + print("== red tests: every mutation must make the checker fail") + if not MUTATIONS: + print("== self-test FAIL: no mutations declared") + return 1 + bad = 0 + for name, mutate in MUTATIONS.items(): + with tempfile.TemporaryDirectory() as directory: + copy = Path(directory) / "tree" + shutil.copytree(tree, copy) + mutate(copy) + result = subprocess.run( + [sys.executable, __file__, "--base", str(copy)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + verdict = ( + "red as required" + if result.returncode != 0 + else "STAYED GREEN — checker is vacuous here" + ) + print(f" {name}: {verdict}") + if result.returncode == 0: + bad += 1 + print(f"== self-test {'PASS' if bad == 0 else 'FAIL'} ({len(MUTATIONS)} mutations)") + return 1 if bad else 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base", help="tree path or https base URL") + parser.add_argument( + "cmd", + nargs="?", + choices=["self-test"], + help="self-test runs the mutation red tests against --base (local tree)", + ) + args = parser.parse_args() + if not args.base: + parser.error("--base required") + if args.cmd == "self-test": + return self_test(Path(args.base)) + return run_checks(Base(args.base)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/static-registry-protocol/evidence/2026-08-08-local.txt b/static-registry-protocol/evidence/2026-08-08-local.txt new file mode 100644 index 00000000..9de712c4 --- /dev/null +++ b/static-registry-protocol/evidence/2026-08-08-local.txt @@ -0,0 +1,60 @@ +# final credential-free local replay 2026-08-08T05:25Z + +Environment: +- Python 3.13.5 +- zstd 1.5.7 +- no GitHub, Linear, Cloudflare, R2, registry, signing, or shared-auth credential +- no external network or infrastructure mutation + +== focused test suites == +generator safety and failure atomicity: 12 PASS +fake-AWS R2 synchronization contract: 6 PASS +loopback plain-HTTP transport: 3 PASS +total focused tests: 21 PASS + +The HTTP suite served the generated tree with Python ThreadingHTTPServer, +required User-Agent `zpkg-static-registry-check/0` on every request, exercised +a real 404 for an absent package, and rejected missing/corrupt HTTP objects +without a traceback. + +== determinism == +built 7 checkpointed objects + checkpoint +built 7 checkpointed objects + checkpoint +byte-identical rebuild: yes +complete tree files: 8 +tree_sha256: 473c19d07618d76b3581041aac0f0bba6dd01eb1f51e9e55cad929df488a1f22 + +== conformance against final local tree == +[1] discovery document + ok: schema_version matches + ok: static-export capabilities advertised correctly + ok: v0 endpoint map matches +[2] checkpoint integrity + ok: checkpoint schema/signature/path/hash/size fields validated before fetch + ok: tree_sha256 consistent over 7 files + ok: every checkpointed object matches its recorded sha256+size + ok: local file set exactly matches checkpoint plus checkpoint.json +[3] indexes: NDJSON, semver order, tarball checksums + ok: every NDJSON line is an object + ok: zpkg-e2e/dep-user: 1 version verified + ok: zpkg-e2e/hello-zed: 3 versions verified +[4] yank semantics + ok: zpkg-e2e/hello-zed@1.2.0: yanked => unresolvable, bytes retained +[5] absent package => missing/404 + ok: absent index is a hard miss +== PASS + +== adversarial self-test == + tamper-tarball: red as required + tamper-index: red as required + drop-checkpoint-entry: red as required + resurrect-yanked: red as required + wrong-schema: red as required + uncheckpointed-object: red as required + remove-checkpointed-object: red as required + unsafe-checkpoint-path: red as required + duplicate-checkpoint-path: red as required +== self-test PASS (9 mutations) + +This is supporting reconstruction evidence. The authoritative merge gate is the +exact-head Ubuntu 24.04/macOS 15 GitHub Actions workflow. diff --git a/static-registry-protocol/evidence/2026-08-08-r2.txt b/static-registry-protocol/evidence/2026-08-08-r2.txt new file mode 100644 index 00000000..3b867b39 --- /dev/null +++ b/static-registry-protocol/evidence/2026-08-08-r2.txt @@ -0,0 +1,45 @@ +# historical live r2.dev read evidence 2026-08-08T04:11Z +# +# This transcript predates the stricter failure-atomic generator, exact local +# file-set checker, checkpoint-path validation, v0 replacement refusal, and +# permanent credential-free CI added later on this branch. It records only a +# read against a dedicated non-production test bucket. The current PR does not +# replay, replace, or mutate this live fixture. +# +# No reusable credential, signed URL, account ID, access key, or secret key is +# present in this file. + +base: https://pub-9b365e012501426c9de90d2746deac9d.r2.dev + +== response headers (pkg vs index) == +HTTP/1.1 200 OK +Content-Type: application/zstd +Cache-Control: public, max-age=31536000, immutable +HTTP/1.1 200 OK +Content-Type: application/x-ndjson +Cache-Control: public, max-age=60, stale-while-revalidate=600 + +== UA filtering == +default curl UA -> 200 +Python-urllib/3.9 UA -> 403 + +== conformance against https://pub-9b365e012501426c9de90d2746deac9d.r2.dev +[1] discovery document + ok: schema_version matches + ok: static-export capabilities advertised correctly +[2] checkpoint integrity + ok: tree_sha256 consistent over 7 files + ok: every checkpointed object matches its recorded sha256+size +[3] indexes: NDJSON, semver order, tarball checksums + ok: zpkg-e2e/dep-user: 1 versions verified + ok: zpkg-e2e/hello-zed: 3 versions verified +[4] yank semantics + ok: zpkg-e2e/hello-zed@1.2.0: yanked => unresolvable, bytes retained +[5] absent package => missing/404 + ok: absent index is a hard miss +== PASS + +This historical transcript supports the RFC observations about static hosting, +cache metadata, identifying User-Agents, and managed-domain propagation. It is +not promotion evidence for the final hardened exact head; the permanent local +Ubuntu/macOS workflow is the promotion authority. diff --git a/static-registry-protocol/fixtures/zpkg-e2e/dep-user/0.1.0/README.md b/static-registry-protocol/fixtures/zpkg-e2e/dep-user/0.1.0/README.md new file mode 100644 index 00000000..342fd1c7 --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/dep-user/0.1.0/README.md @@ -0,0 +1 @@ +uses hello-zed diff --git a/static-registry-protocol/fixtures/zpkg-e2e/dep-user/0.1.0/zpkg.json b/static-registry-protocol/fixtures/zpkg-e2e/dep-user/0.1.0/zpkg.json new file mode 100644 index 00000000..4a45f321 --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/dep-user/0.1.0/zpkg.json @@ -0,0 +1 @@ +{"deps": [{"name": "zpkg-e2e/hello-zed", "req": "^1.0"}], "yanked": false} diff --git a/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.0.0/main.rs b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.0.0/main.rs new file mode 100644 index 00000000..d606bf41 --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.0.0/main.rs @@ -0,0 +1 @@ +fn main() { println!("hello zed 1.0.0"); } diff --git a/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.0.0/zpkg.json b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.0.0/zpkg.json new file mode 100644 index 00000000..15ca2cb0 --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.0.0/zpkg.json @@ -0,0 +1 @@ +{"deps": [], "yanked": false} diff --git a/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.1.0/main.rs b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.1.0/main.rs new file mode 100644 index 00000000..2a1ba35b --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.1.0/main.rs @@ -0,0 +1 @@ +fn main() { println!("hello zed 1.1.0"); } diff --git a/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.1.0/zpkg.json b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.1.0/zpkg.json new file mode 100644 index 00000000..15ca2cb0 --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.1.0/zpkg.json @@ -0,0 +1 @@ +{"deps": [], "yanked": false} diff --git a/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.2.0/main.rs b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.2.0/main.rs new file mode 100644 index 00000000..18bd361f --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.2.0/main.rs @@ -0,0 +1 @@ +fn main() { println!("bad release, yanked"); } diff --git a/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.2.0/zpkg.json b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.2.0/zpkg.json new file mode 100644 index 00000000..bbb19975 --- /dev/null +++ b/static-registry-protocol/fixtures/zpkg-e2e/hello-zed/1.2.0/zpkg.json @@ -0,0 +1 @@ +{"deps": [], "yanked": true} diff --git a/static-registry-protocol/sync_to_r2.sh b/static-registry-protocol/sync_to_r2.sh new file mode 100755 index 00000000..219edc35 --- /dev/null +++ b/static-registry-protocol/sync_to_r2.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Upload a built static-registry v0 tree to a dedicated R2 test bucket. +# +# IMPORTANT: v0 uses mutable index paths. It is safe for a first publication to +# an empty/dedicated test bucket, but it cannot provide collection-atomic rolling +# updates. The signed v1 contract uses immutable snapshot prefixes selected by a +# stable checkpoint. Until the fixture is upgraded, replacing an existing v0 +# checkpoint fails closed unless the operator explicitly acknowledges the risk. +# +# Env: +# ZPKG_R2_ACCESS_KEY_ID +# ZPKG_R2_SECRET_ACCESS_KEY +# ZPKG_R2_ENDPOINT +# ZPKG_ALLOW_V0_REPLACE=true # explicit non-production override only +# Usage: sync_to_r2.sh +set -euo pipefail + +if [[ "$#" -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +: "${ZPKG_R2_ACCESS_KEY_ID:?ZPKG_R2_ACCESS_KEY_ID is required}" +: "${ZPKG_R2_SECRET_ACCESS_KEY:?ZPKG_R2_SECRET_ACCESS_KEY is required}" +: "${ZPKG_R2_ENDPOINT:?ZPKG_R2_ENDPOINT is required}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TREE="$(cd "$1" && pwd)" +BUCKET="$2" +CHECKPOINT="checkpoint.json" + +if [[ ! -f "$TREE/$CHECKPOINT" ]]; then + echo "missing $TREE/$CHECKPOINT" >&2 + exit 2 +fi + +# Never publish a tree whose checkpoint, index, or package bytes already fail +# the same local conformance boundary used by CI. +if ! python3 "$SCRIPT_DIR/check_static_registry.py" --base "$TREE" >/dev/null; then + echo "refusing to upload a static registry tree that fails local conformance" >&2 + exit 2 +fi + +export AWS_ACCESS_KEY_ID="$ZPKG_R2_ACCESS_KEY_ID" +export AWS_SECRET_ACCESS_KEY="$ZPKG_R2_SECRET_ACCESS_KEY" +EP=(--endpoint-url "$ZPKG_R2_ENDPOINT" --region auto) + +# A pre-existing v0 checkpoint means readers may already hold the old mutable +# index paths. Updating those paths cannot be made collection-atomic merely by +# writing the new checkpoint last, so require an explicit test-only override. +if aws s3api head-object "${EP[@]}" --bucket "$BUCKET" --key "$CHECKPOINT" \ + >/dev/null 2>&1; then + if [[ "${ZPKG_ALLOW_V0_REPLACE:-false}" != "true" ]]; then + cat >&2 <<'EOF' +refusing to replace an existing v0 checkpoint: mutable v0 indexes cannot provide +atomic rolling publication. Use a fresh dedicated test bucket, upgrade to the +v1 immutable-snapshot layout, or set ZPKG_ALLOW_V0_REPLACE=true only for an +explicitly disposable non-production experiment. +EOF + exit 3 + fi +fi + +put() { # rel content-type cache-control + local rel="$1" + local content_type="$2" + local cache_control="$3" + aws s3api put-object "${EP[@]}" \ + --bucket "$BUCKET" \ + --key "$rel" \ + --body "$TREE/$rel" \ + --content-type "$content_type" \ + --cache-control "$cache_control" \ + --query ETag \ + --output text +} + +upload_object() { + local rel="$1" + case "$rel" in + pkgs/*.tar.zst) + put "$rel" application/zstd "public, max-age=31536000, immutable" >/dev/null + ;; + index/*) + # v0 indexes are mutable. Revalidate rather than serving stale data after + # a checkpoint transition. v1 snapshot indexes become immutable. + put "$rel" application/x-ndjson "no-cache" >/dev/null + ;; + *) + put "$rel" application/json "no-cache" >/dev/null + ;; + esac +} + +UPLOAD_LIST="$(mktemp)" +trap 'rm -f "$UPLOAD_LIST"' EXIT + +# Produce a portable, NUL-delimited lexical inventory. GNU `sort -z` is not +# available on the macOS runner, and newline-delimited filenames are unsafe. +python3 - "$TREE" "$CHECKPOINT" > "$UPLOAD_LIST" <<'PY' +import os +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +checkpoint = sys.argv[2] +entries = [] +for path in root.rglob("*"): + if path.is_symlink(): + raise SystemExit(f"static registry tree contains a symbolic link: {path}") + if path.is_file(): + relative = path.relative_to(root).as_posix() + if relative != checkpoint: + entries.append(relative) +for relative in sorted(entries): + os.write(sys.stdout.fileno(), relative.encode("utf-8") + b"\0") +PY + +count=0 + +# Publish every object except the stable checkpoint first. +while IFS= read -r -d '' rel; do + upload_object "$rel" + count=$((count + 1)) +done < "$UPLOAD_LIST" + +# The stable checkpoint is the only publication pointer and is always last. +put "$CHECKPOINT" application/json "no-cache" >/dev/null +count=$((count + 1)) + +echo "uploaded $count objects to r2://$BUCKET (checkpoint last)" diff --git a/static-registry-protocol/test_http_static_registry.py b/static-registry-protocol/test_http_static_registry.py new file mode 100644 index 00000000..70277dd4 --- /dev/null +++ b/static-registry-protocol/test_http_static_registry.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Credential-free HTTP transport tests for the static registry checker.""" +from __future__ import annotations + +import contextlib +import http.server +import os +import subprocess +import sys +import tempfile +import threading +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +BUILDER = ROOT / "build_static_registry.py" +CHECKER = ROOT / "check_static_registry.py" +EXPECTED_USER_AGENT = "zpkg-static-registry-check/0" + + +class RecordingHandler(http.server.SimpleHTTPRequestHandler): + root: Path + user_agents: list[str] + + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(self.root), **kwargs) + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + self.user_agents.append(self.headers.get("User-Agent", "")) + super().do_GET() + + def log_message(self, format: str, *args) -> None: + del format, args + + +@contextlib.contextmanager +def static_server(root: Path): + user_agents: list[str] = [] + + class Handler(RecordingHandler): + pass + + Handler.root = root + Handler.user_agents = user_agents + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}", user_agents + finally: + server.shutdown() + thread.join(timeout=10) + server.server_close() + + +class StaticRegistryHttpTests(unittest.TestCase): + def build_tree(self, root: Path) -> Path: + tree = root / "tree" + environment = os.environ.copy() + environment["SOURCE_DATE_EPOCH"] = "0" + result = subprocess.run( + [ + sys.executable, + str(BUILDER), + "--fixtures", + str(ROOT / "fixtures"), + "--out", + str(tree), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return tree + + def test_plain_http_server_passes_full_conformance_with_identifying_user_agent(self) -> None: + with tempfile.TemporaryDirectory() as directory: + tree = self.build_tree(Path(directory)) + with static_server(tree) as (base, user_agents): + result = subprocess.run( + [sys.executable, str(CHECKER), "--base", base], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("== PASS", result.stdout) + self.assertGreaterEqual(len(user_agents), 1) + self.assertEqual(set(user_agents), {EXPECTED_USER_AGENT}) + + def test_http_checker_fails_cleanly_when_checkpointed_object_is_missing(self) -> None: + with tempfile.TemporaryDirectory() as directory: + tree = self.build_tree(Path(directory)) + missing = tree / "pkgs/zpkg-e2e/hello-zed/1.0.0.tar.zst" + missing.unlink() + with static_server(tree) as (base, user_agents): + result = subprocess.run( + [sys.executable, str(CHECKER), "--base", base], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("cannot read checkpointed object", result.stdout) + self.assertNotIn("Traceback", result.stdout + result.stderr) + self.assertEqual(set(user_agents), {EXPECTED_USER_AGENT}) + + def test_http_checker_rejects_invalid_discovery_json_without_traceback(self) -> None: + with tempfile.TemporaryDirectory() as directory: + tree = self.build_tree(Path(directory)) + discovery = tree / ".well-known/zpkg-registry.json" + discovery.write_text("not-json\n", encoding="utf-8") + with static_server(tree) as (base, user_agents): + result = subprocess.run( + [sys.executable, str(CHECKER), "--base", base], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual(result.returncode, 1) + self.assertIn("cannot read discovery document", result.stdout) + self.assertNotIn("Traceback", result.stdout + result.stderr) + self.assertEqual(set(user_agents), {EXPECTED_USER_AGENT}) + + +if __name__ == "__main__": + unittest.main() diff --git a/static-registry-protocol/test_static_registry.py b/static-registry-protocol/test_static_registry.py new file mode 100644 index 00000000..49e3d914 --- /dev/null +++ b/static-registry-protocol/test_static_registry.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Focused fail-closed tests for the static registry fixture generator.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +BUILDER = ROOT / "build_static_registry.py" + + +class StaticRegistryGeneratorSafetyTests(unittest.TestCase): + def run_builder(self, fixtures: Path, output: Path) -> subprocess.CompletedProcess[str]: + environment = os.environ.copy() + environment["SOURCE_DATE_EPOCH"] = "0" + return subprocess.run( + [ + sys.executable, + str(BUILDER), + "--fixtures", + str(fixtures), + "--out", + str(output), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + check=False, + ) + + def package( + self, + fixtures: Path, + *, + org: str = "zpkg-e2e", + name: str = "hello-zed", + version: str = "1.0.0", + ) -> Path: + package = fixtures / org / name / version + package.mkdir(parents=True) + (package / "zpkg.json").write_text( + json.dumps({"deps": [], "yanked": False}) + "\n", + encoding="utf-8", + ) + (package / "payload.txt").write_text("payload\n", encoding="utf-8") + return package + + def test_accepts_a_new_or_empty_real_output_directory(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + self.package(fixtures) + + new_output = root / "new-output" + created = self.run_builder(fixtures, new_output) + self.assertEqual(created.returncode, 0, created.stderr) + self.assertTrue((new_output / "checkpoint.json").is_file()) + + empty_output = root / "empty-output" + empty_output.mkdir() + created_in_empty = self.run_builder(fixtures, empty_output) + self.assertEqual(created_in_empty.returncode, 0, created_in_empty.stderr) + self.assertTrue((empty_output / "checkpoint.json").is_file()) + + def test_rejects_nonempty_output_without_touching_caller_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + self.package(fixtures) + output = root / "output" + output.mkdir() + sentinel = output / "caller-owned.txt" + sentinel.write_bytes(b"do not overwrite\n") + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("must be empty", result.stderr) + self.assertEqual(sentinel.read_bytes(), b"do not overwrite\n") + self.assertFalse((output / "checkpoint.json").exists()) + + def test_rejects_parent_traversal_output_path(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + self.package(fixtures) + output = root / "nested" / ".." + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("without `..`", result.stderr) + self.assertFalse((root / "checkpoint.json").exists()) + self.assertFalse((root / "nested").exists()) + + @unittest.skipUnless(hasattr(os, "symlink"), "platform has no symlink support") + def test_rejects_fixture_symlinks_instead_of_following_them(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + package = self.package(fixtures) + outside = root / "outside-secret.txt" + outside.write_text("must not be archived\n", encoding="utf-8") + os.symlink(outside, package / "escape.txt") + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("symbolic link", result.stderr) + self.assertFalse((output / "checkpoint.json").exists()) + + @unittest.skipUnless(hasattr(os, "symlink"), "platform has no symlink support") + def test_rejects_symlinked_metadata_before_reading_it(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + package = self.package(fixtures) + metadata = package / "zpkg.json" + metadata.unlink() + outside = root / "outside-metadata.json" + outside.write_text('{"deps":[],"yanked":false}\n', encoding="utf-8") + os.symlink(outside, metadata) + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("metadata is a symbolic link", result.stderr) + self.assertFalse(output.exists()) + + @unittest.skipUnless(hasattr(os, "symlink"), "platform has no symlink support") + def test_later_invalid_package_leaves_no_partial_output_tree(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + self.package(fixtures, name="a-valid-package") + invalid = self.package(fixtures, name="z-invalid-package") + outside = root / "outside-secret.txt" + outside.write_text("must not be archived\n", encoding="utf-8") + os.symlink(outside, invalid / "escape.txt") + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("symbolic link", result.stderr) + self.assertFalse( + output.exists(), + "validation failure after an earlier valid package must not expose a partial registry", + ) + self.assertEqual( + list(root.glob(".output.zpkg-static-*")), + [], + "validation failure must not leave staging directories", + ) + + @unittest.skipUnless(hasattr(os, "symlink"), "platform has no symlink support") + def test_rejects_a_symlinked_output_directory(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + self.package(fixtures) + real_output = root / "real-output" + real_output.mkdir() + linked_output = root / "linked-output" + os.symlink(real_output, linked_output, target_is_directory=True) + + result = self.run_builder(fixtures, linked_output) + + self.assertEqual(result.returncode, 2) + self.assertIn("output path is a symbolic link", result.stderr) + self.assertEqual(list(real_output.iterdir()), []) + + def test_rejects_non_object_metadata(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + package = self.package(fixtures) + (package / "zpkg.json").write_text("[]\n", encoding="utf-8") + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("metadata must be a JSON object", result.stderr) + self.assertFalse(output.exists()) + + def test_rejects_invalid_metadata_field_types(self) -> None: + for metadata, expected in ( + ({"deps": "not-an-array", "yanked": False}, "`deps` must be an array"), + ({"deps": [], "yanked": "false"}, "`yanked` must be boolean"), + ): + with self.subTest(metadata=metadata): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + package = self.package(fixtures) + (package / "zpkg.json").write_text( + json.dumps(metadata) + "\n", + encoding="utf-8", + ) + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn(expected, result.stderr) + self.assertFalse(output.exists()) + + def test_rejects_invalid_semantic_version_directory(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + self.package(fixtures, version="1.0") + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("invalid v0 semantic version", result.stderr) + self.assertFalse(output.exists()) + + def test_rejects_uppercase_org_segments(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + self.package(fixtures, org="Zpkg-E2E") + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("lowercase canonical form", result.stderr) + self.assertFalse((output / "checkpoint.json").exists()) + + def test_rejects_an_empty_fixture_set(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fixtures = root / "fixtures" + fixtures.mkdir() + output = root / "output" + + result = self.run_builder(fixtures, output) + + self.assertEqual(result.returncode, 2) + self.assertIn("contains no package versions", result.stderr) + self.assertFalse((output / "checkpoint.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/static-registry-protocol/test_sync_to_r2.py b/static-registry-protocol/test_sync_to_r2.py new file mode 100644 index 00000000..9eebc1a7 --- /dev/null +++ b/static-registry-protocol/test_sync_to_r2.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Credential-free regression tests for the v0 R2 synchronization boundary.""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +BUILDER = ROOT / "build_static_registry.py" +SYNC = ROOT / "sync_to_r2.sh" + + +class R2SyncTests(unittest.TestCase): + def fixture_tree(self, root: Path) -> Path: + tree = root / "tree" + environment = os.environ.copy() + environment["SOURCE_DATE_EPOCH"] = "0" + result = subprocess.run( + [ + sys.executable, + str(BUILDER), + "--fixtures", + str(ROOT / "fixtures"), + "--out", + str(tree), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return tree + + def fake_aws(self, root: Path) -> tuple[Path, Path]: + bin_dir = root / "bin" + bin_dir.mkdir(exist_ok=True) + log = root / "aws-calls.jsonl" + program = bin_dir / "aws" + program.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +args = sys.argv[1:] +log = Path(os.environ["FAKE_AWS_LOG"]) +with log.open("a", encoding="utf-8") as stream: + stream.write(json.dumps(args) + "\\n") + +if args[:2] == ["s3api", "head-object"]: + raise SystemExit(0 if os.environ.get("FAKE_CHECKPOINT_EXISTS") == "true" else 1) +if args[:2] == ["s3api", "put-object"]: + body = Path(args[args.index("--body") + 1]) + if not body.is_file(): + raise SystemExit(f"put-object body is not a file: {body}") + print('"fake-etag"') + raise SystemExit(0) +raise SystemExit(f"unexpected fake aws invocation: {args}") +""", + encoding="utf-8", + ) + program.chmod(program.stat().st_mode | stat.S_IXUSR) + return bin_dir, log + + def run_sync( + self, + root: Path, + tree: Path, + *, + checkpoint_exists: bool, + allow_replace: bool = False, + ) -> tuple[subprocess.CompletedProcess[str], list[list[str]]]: + bin_dir, log = self.fake_aws(root) + if log.exists(): + log.unlink() + environment = os.environ.copy() + environment.update( + { + "PATH": f"{bin_dir}{os.pathsep}{environment['PATH']}", + "FAKE_AWS_LOG": str(log), + "FAKE_CHECKPOINT_EXISTS": "true" if checkpoint_exists else "false", + "ZPKG_R2_ACCESS_KEY_ID": "test-access-key", + "ZPKG_R2_SECRET_ACCESS_KEY": "test-secret-key", + "ZPKG_R2_ENDPOINT": "https://example.invalid", + } + ) + if allow_replace: + environment["ZPKG_ALLOW_V0_REPLACE"] = "true" + else: + environment.pop("ZPKG_ALLOW_V0_REPLACE", None) + + result = subprocess.run( + ["bash", str(SYNC), str(tree), "test-bucket"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + check=False, + ) + calls = [] + if log.exists(): + calls = [json.loads(line) for line in log.read_text().splitlines()] + self.assertNotIn("test-secret-key", json.dumps(calls)) + return result, calls + + @staticmethod + def put_calls(calls: list[list[str]]) -> list[list[str]]: + return [call for call in calls if call[:2] == ["s3api", "put-object"]] + + @staticmethod + def head_calls(calls: list[list[str]]) -> list[list[str]]: + return [call for call in calls if call[:2] == ["s3api", "head-object"]] + + @staticmethod + def option(call: list[str], name: str) -> str: + return call[call.index(name) + 1] + + def assert_upload_contract(self, calls: list[list[str]]) -> None: + puts = self.put_calls(calls) + expected_keys = [ + ".well-known/zpkg-registry.json", + "index/zpkg-e2e/dep-user", + "index/zpkg-e2e/hello-zed", + "pkgs/zpkg-e2e/dep-user/0.1.0.tar.zst", + "pkgs/zpkg-e2e/hello-zed/1.0.0.tar.zst", + "pkgs/zpkg-e2e/hello-zed/1.1.0.tar.zst", + "pkgs/zpkg-e2e/hello-zed/1.2.0.tar.zst", + "checkpoint.json", + ] + self.assertEqual( + [self.option(call, "--key") for call in puts], + expected_keys, + ) + self.assertEqual(len(self.head_calls(calls)), 1) + self.assertEqual(self.option(puts[-1], "--key"), "checkpoint.json") + self.assertEqual(self.option(puts[-1], "--cache-control"), "no-cache") + + by_key = {self.option(call, "--key"): call for call in puts} + self.assertEqual( + self.option( + by_key["pkgs/zpkg-e2e/hello-zed/1.0.0.tar.zst"], + "--content-type", + ), + "application/zstd", + ) + self.assertEqual( + self.option( + by_key["pkgs/zpkg-e2e/hello-zed/1.0.0.tar.zst"], + "--cache-control", + ), + "public, max-age=31536000, immutable", + ) + self.assertEqual( + self.option(by_key["index/zpkg-e2e/hello-zed"], "--content-type"), + "application/x-ndjson", + ) + self.assertEqual( + self.option(by_key["index/zpkg-e2e/hello-zed"], "--cache-control"), + "no-cache", + ) + for call in puts: + self.assertEqual(self.option(call, "--bucket"), "test-bucket") + self.assertTrue(Path(self.option(call, "--body")).is_file()) + + def test_fresh_publication_writes_checkpoint_last_with_reviewed_cache_policy(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tree = self.fixture_tree(root) + result, calls = self.run_sync(root, tree, checkpoint_exists=False) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("uploaded 8 objects", result.stdout) + self.assertIn("checkpoint last", result.stdout) + self.assert_upload_contract(calls) + + def test_existing_v0_checkpoint_fails_closed_without_explicit_override(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tree = self.fixture_tree(root) + result, calls = self.run_sync(root, tree, checkpoint_exists=True) + + self.assertEqual(result.returncode, 3) + self.assertIn("refusing to replace an existing v0 checkpoint", result.stderr) + self.assertEqual(self.put_calls(calls), []) + self.assertEqual(len(self.head_calls(calls)), 1) + + def test_disposable_override_still_writes_checkpoint_last(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tree = self.fixture_tree(root) + result, calls = self.run_sync( + root, + tree, + checkpoint_exists=True, + allow_replace=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_upload_contract(calls) + + def test_missing_checkpoint_fails_before_aws(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tree = self.fixture_tree(root) + (tree / "checkpoint.json").unlink() + + result, calls = self.run_sync(root, tree, checkpoint_exists=False) + + self.assertEqual(result.returncode, 2) + self.assertIn("missing", result.stderr) + self.assertEqual(calls, []) + + def test_tampered_checkpointed_object_fails_before_aws(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tree = self.fixture_tree(root) + artifact = tree / "pkgs/zpkg-e2e/hello-zed/1.0.0.tar.zst" + artifact.write_bytes(artifact.read_bytes() + b"tamper") + + result, calls = self.run_sync(root, tree, checkpoint_exists=False) + + self.assertEqual(result.returncode, 2) + self.assertIn("fails local conformance", result.stderr) + self.assertEqual(calls, []) + + def test_uncheckpointed_object_fails_before_aws(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tree = self.fixture_tree(root) + extra = tree / "index/zpkg-e2e/uncheckpointed" + extra.write_text("{}\n", encoding="utf-8") + + result, calls = self.run_sync(root, tree, checkpoint_exists=False) + + self.assertEqual(result.returncode, 2) + self.assertIn("fails local conformance", result.stderr) + self.assertEqual(calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/audit-test-org-fleet.test.mjs b/tests/audit-test-org-fleet.test.mjs new file mode 100644 index 00000000..f6f0da4e --- /dev/null +++ b/tests/audit-test-org-fleet.test.mjs @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const scriptPath = path.join(root, 'scripts', 'audit-test-org-fleet.mjs'); +const launcherPath = path.join(root, 'scripts', 'audit-test-org-fleet-live.mjs'); +const workflowPath = path.join(root, '.github', 'workflows', 'audit-test-org-fleet-once.yml'); +const credentialLiteral = /(?:ghp_|github_pat_)[A-Za-z0-9_]{20,}/; + +test('live audit plan matches the complete physical fleet', () => { + const result = childProcess.spawnSync(process.execPath, [launcherPath, '--plan-json'], { + cwd: root, + encoding: 'utf8', + env: { ...process.env, GH_TOKEN: '', GITHUB_TOKEN: '' }, + }); + assert.equal(result.status, 0, result.stderr); + const plan = JSON.parse(result.stdout); + assert.ok(plan.organizations > 0); + assert.equal(plan.repositories, plan.generatedRepositories + plan.retainedRepositories); + assert.equal(plan.retainedRepositories, 22); + assert.equal(Object.keys(plan.expectedByOrganization).length, plan.organizations); + assert.equal(plan.expectedByOrganization['zed-pkg-test'], 37); + assert.equal(plan.expectedByOrganization['fiducia-cloud-test'], 32); + assert.equal(plan.expectedByOrganization['streempilot-test'], 21); + assert.equal('r2g-test' in plan.expectedByOrganization, false); +}); + +test('live audit requires exact generated branches and draft PRs without embedding credentials', () => { + const script = fs.readFileSync(scriptPath, 'utf8'); + const launcher = fs.readFileSync(launcherPath, 'utf8'); + const workflow = fs.readFileSync(workflowPath, 'utf8'); + + assert.doesNotMatch(script, /summary\.organizations !== \d+/); + assert.doesNotMatch(script, /summary\.expectedRepositories !== \d+/); + assert.doesNotMatch(script, /summary\.expectedGeneratedRepositories !== \d+/); + assert.match(script, /verifiedRepositories !== summary\.expectedRepositories/); + assert.match(script, /missing generated branch/); + assert.match(script, /expected one open generated PR/); + assert.match(script, /is not draft/); + assert.match(script, /differs from branch/); + assert.match(script, /visibility is/); + assert.match(launcher, /branch\.split\('\/'\)\.map\(route\)\.join\('\/'\)/); + assert.match(launcher, /fleet audit branch-path seam changed/); + assert.match(launcher, /API rate limit exceeded/); + assert.match(launcher, /resetDelay = primary/); + assert.match(launcher, /mapConcurrent\(generatedWork, 2,/); + assert.match(launcher, /pathToFileURL\(temporaryAuditorPath\)/); + assert.match(launcher, /\.audit-test-org-fleet-live-/); + assert.doesNotMatch(launcher, /data:text\/javascript/); + assert.match(workflow, /secrets\.FLEET_GH_TOKEN/); + assert.match(workflow, /AUDIT_TEST_FLEET_V1/); + assert.match(workflow, /audit-test-org-fleet-live\.mjs/); + assert.equal(credentialLiteral.test(script), false); + assert.equal(credentialLiteral.test(launcher), false); + assert.equal(credentialLiteral.test(workflow), false); + assert.doesNotMatch(workflow, /workflow_dispatch:/); +}); diff --git a/tests/bootstrap-retry-policy.test.mjs b/tests/bootstrap-retry-policy.test.mjs new file mode 100644 index 00000000..b7d48ff5 --- /dev/null +++ b/tests/bootstrap-retry-policy.test.mjs @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const launcher = fs.readFileSync(path.join(root, 'scripts', 'bootstrap-test-org-fleet.mjs'), 'utf8'); + +test('primary quota exhaustion enters the bounded retry path', () => { + assert.match(launcher, /const primaryRemaining = Number\(response\.headers\.get\('x-ratelimit-remaining'\)/); + assert.match(launcher, /const primaryRateLimit = response\.status === 403 && primaryRemaining === 0;/); + assert.match( + launcher, + /if \(\(primaryRateLimit \|\| secondaryRateLimit \|\| response\.status === 429 \|\| response\.status >= 500\)/, + ); + assert.match(launcher, /Date\.now\(\) \+ 5000/); +}); + +test('secondary rate limits do not inherit a non-exhausted primary reset timestamp', () => { + assert.match(launcher, /const resetDelayMs = primaryRateLimit && resetEpochSeconds > 0/); + assert.match(launcher, /Math\.min\(180000, 15000 \* \(2 \*\* attempt\)\)/); + assert.doesNotMatch(launcher, /const resetDelayMs = resetEpochSeconds > 0\s*\?/); +}); + +test('rate-limit logs expose quota state while git errors redact the token', () => { + assert.match(launcher, /primaryRateLimit,/); + assert.match(launcher, /primaryRemaining,/); + assert.match(launcher, /raw\.replaceAll\(token, '\*\*\*'\)/); + assert.doesNotMatch(launcher, /log\([^\n]*token/); +}); diff --git a/tests/browser/playwright/harness-contract.test.mjs b/tests/browser/playwright/harness-contract.test.mjs new file mode 100644 index 00000000..03203633 --- /dev/null +++ b/tests/browser/playwright/harness-contract.test.mjs @@ -0,0 +1,338 @@ +import assert from 'node:assert/strict'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import path from 'node:path'; +import test from 'node:test'; + +process.env.E2E_FRAMEWORK ||= 'playwright'; + +export const contractEnabled = process.env.E2E_BROWSER_CONTRACT === '1'; + +const CSP = [ + "default-src 'none'", + "script-src 'self'", + "connect-src 'self'", + "img-src 'self' data:", + "style-src 'self'", + "base-uri 'none'", + "form-action 'none'", + "frame-ancestors 'none'", + "object-src 'none'", +].join('; '); + +const html = `E2E Harness Contract

Browser harness contract

booting0
`; +const script = `(async()=>{const state=document.querySelector('#state');const count=document.querySelector('#count');const button=document.querySelector('#increment');const session=await fetch('/api/session',{cache:'no-store'}).then(r=>{if(!r.ok)throw new Error('session request failed: '+r.status);return r.json()});if(!session.cookieSeen)throw new Error('HttpOnly session cookie was not returned to the server');state.textContent='ready';button.addEventListener('click',()=>{count.textContent=String(Number(count.textContent)+1)})})().catch(error=>{document.querySelector('#state').textContent='error';console.error(error)});`; + +function writeResponse(request, response, statusCode, headers, body = '') { + response.statusCode = statusCode; + for (const [name, value] of Object.entries(headers)) response.setHeader(name, value); + if (request.method === 'HEAD') response.end(); + else response.end(body); +} + +async function startHarness() { + const server = createServer((request, response) => { + response.setHeader('Cache-Control', 'no-store'); + response.setHeader('Cross-Origin-Opener-Policy', 'same-origin'); + response.setHeader('Cross-Origin-Resource-Policy', 'same-origin'); + response.setHeader('Permissions-Policy', 'camera=(), microphone=(), geolocation=()'); + response.setHeader('Referrer-Policy', 'no-referrer'); + response.setHeader('X-Content-Type-Options', 'nosniff'); + + if (!['GET', 'HEAD'].includes(request.method ?? '')) { + writeResponse(request, response, 405, { + Allow: 'GET, HEAD', + 'Content-Type': 'text/plain; charset=utf-8', + }, 'method not allowed'); + return; + } + + let pathname; + try { + pathname = new URL(request.url ?? '/', 'http://127.0.0.1').pathname; + } catch { + writeResponse(request, response, 400, { 'Content-Type': 'text/plain; charset=utf-8' }, 'bad request'); + return; + } + + if (pathname === '/') { + writeResponse(request, response, 200, { + 'Content-Security-Policy': CSP, + 'Content-Type': 'text/html; charset=utf-8', + 'Set-Cookie': 'e2e_session=contract; HttpOnly; SameSite=Strict; Path=/', + }, html); + } else if (pathname === '/app.js') { + writeResponse(request, response, 200, { 'Content-Type': 'text/javascript; charset=utf-8' }, script); + } else if (pathname === '/api/session') { + writeResponse(request, response, 200, { 'Content-Type': 'application/json; charset=utf-8' }, JSON.stringify({ + cookieSeen: String(request.headers.cookie ?? '').includes('e2e_session=contract'), + requestId: 'browser-harness-contract', + })); + } else if (pathname === '/favicon.ico') { + writeResponse(request, response, 204, {}); + } else if (pathname === '/healthz') { + writeResponse(request, response, 200, { 'Content-Type': 'text/plain; charset=utf-8' }, 'ok'); + } else { + writeResponse(request, response, 404, { 'Content-Type': 'text/plain; charset=utf-8' }, 'not found'); + } + }); + + server.keepAliveTimeout = 1_000; + server.headersTimeout = 2_000; + server.requestTimeout = 5_000; + + await new Promise((resolve, reject) => { + const onListening = () => { + server.off('error', onError); + resolve(); + }; + const onError = (error) => { + server.off('listening', onListening); + reject(error); + }; + server.once('listening', onListening); + server.once('error', onError); + server.listen(0, '127.0.0.1'); + }); + + const address = server.address(); + assert(address && typeof address === 'object', 'harness did not expose a TCP address'); + + return { + origin: `http://127.0.0.1:${address.port}`, + async close() { + if (!server.listening) return; + const forceClose = setTimeout(() => server.closeAllConnections?.(), 1_000); + forceClose.unref(); + server.closeIdleConnections?.(); + try { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } finally { + clearTimeout(forceClose); + } + }, + }; +} + +async function artifactDirectory(framework) { + const directory = path.join(process.cwd(), 'artifacts', framework); + await mkdir(directory, { recursive: true }); + return directory; +} + +async function writeArtifact(framework, name, value, encoding = 'utf8') { + const directory = await artifactDirectory(framework); + await writeFile(path.join(directory, name), value, encoding); +} + +function assertErrors(errors) { + assert.deepEqual(errors, [], `browser emitted errors:\n${errors.join('\n')}`); +} + +function assertMainResponse(status, headers) { + assert.equal(status, 200); + assert.match(headers['content-security-policy'] ?? '', /default-src 'none'/); + assert.equal(headers['x-content-type-options'], 'nosniff'); + assert.equal(headers['cross-origin-opener-policy'], 'same-origin'); + assert.equal(headers['cross-origin-resource-policy'], 'same-origin'); + assert.match(headers['permissions-policy'] ?? '', /camera=\(\)/); + assert.match(headers['set-cookie'] ?? '', /HttpOnly/i); + assert.match(headers['set-cookie'] ?? '', /SameSite=Strict/i); +} + +async function assertInPageBoundaries(evaluate) { + const result = await evaluate(async () => { + const health = await fetch('/healthz', { cache: 'no-store' }); + return { + cookie: document.cookie, + healthBody: await health.text(), + healthStatus: health.status, + }; + }); + if (result?.contractError) throw new Error(result.contractError); + assert.deepEqual(result, { + cookie: '', + healthBody: 'ok', + healthStatus: 200, + }); +} + +async function capturePlaywright(page, errors) { + await Promise.allSettled([ + page.screenshot({ path: path.join(await artifactDirectory('playwright'), 'harness-contract.png'), fullPage: true }), + page.content().then((content) => writeArtifact('playwright', 'page.html', content)), + writeArtifact('playwright', 'browser-errors.json', JSON.stringify(errors, null, 2)), + ]); +} + +async function runPlaywright(origin) { + const { chromium } = await import('playwright'); + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 1280, height: 720 } }); + const errors = []; + page.on('console', (message) => message.type() === 'error' && errors.push(`console: ${message.text()}`)); + page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); + try { + const response = await page.goto(origin, { waitUntil: 'networkidle', timeout: 15_000 }); + assert(response, 'navigation did not return a response'); + assertMainResponse(response.status(), await response.allHeaders()); + await page.waitForFunction(() => document.querySelector('#state')?.textContent === 'ready'); + assert.equal(await page.title(), 'E2E Harness Contract'); + await page.click('#increment'); + await page.click('#increment'); + assert.equal(await page.textContent('#count'), '2'); + await assertInPageBoundaries((fn) => page.evaluate(fn)); + assertErrors(errors); + } finally { + await capturePlaywright(page, errors); + await page.close().catch(() => {}); + await browser.close().catch(() => {}); + } +} + +async function capturePuppeteer(page, errors) { + await Promise.allSettled([ + page.screenshot({ path: path.join(await artifactDirectory('puppeteer'), 'harness-contract.png'), fullPage: true }), + page.content().then((content) => writeArtifact('puppeteer', 'page.html', content)), + writeArtifact('puppeteer', 'browser-errors.json', JSON.stringify(errors, null, 2)), + ]); +} + +async function runPuppeteer(origin) { + const { default: puppeteer } = await import('puppeteer'); + const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH; + assert(executablePath, 'PUPPETEER_EXECUTABLE_PATH must identify the CI Chrome binary'); + const browser = await puppeteer.launch({ + executablePath, + headless: true, + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], + }); + const page = await browser.newPage(); + await page.setViewport({ width: 1280, height: 720 }); + page.setDefaultNavigationTimeout(15_000); + page.setDefaultTimeout(15_000); + const errors = []; + page.on('console', (message) => message.type() === 'error' && errors.push(`console: ${message.text()}`)); + page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); + try { + const response = await page.goto(origin, { waitUntil: 'networkidle0' }); + assert(response, 'navigation did not return a response'); + assertMainResponse(response.status(), response.headers()); + await page.waitForFunction(() => document.querySelector('#state')?.textContent === 'ready'); + assert.equal(await page.title(), 'E2E Harness Contract'); + await page.click('#increment'); + await page.click('#increment'); + assert.equal(await page.$eval('#count', (element) => element.textContent), '2'); + await assertInPageBoundaries((fn) => page.evaluate(fn)); + assertErrors(errors); + } finally { + await capturePuppeteer(page, errors); + await page.close().catch(() => {}); + await browser.close().catch(() => {}); + } +} + +async function captureSelenium(driver, browserErrors) { + const tasks = [ + driver.takeScreenshot().then((png) => writeArtifact('selenium', 'harness-contract.png', png, 'base64')), + driver.getPageSource().then((source) => writeArtifact('selenium', 'page.html', source)), + writeArtifact('selenium', 'browser-errors.json', JSON.stringify(browserErrors, null, 2)), + ]; + await Promise.allSettled(tasks); +} + +async function runSelenium(origin) { + const [webdriver, chromeModule, loggingModule] = await Promise.all([ + import('selenium-webdriver'), + import('selenium-webdriver/chrome.js'), + import('selenium-webdriver/lib/logging.js'), + ]); + const { Builder, By, until } = webdriver; + const chrome = chromeModule.default ?? chromeModule; + const logging = loggingModule.default ?? loggingModule; + const preferences = new logging.Preferences(); + preferences.setLevel(logging.Type.BROWSER, logging.Level.ALL); + const options = new chrome.Options(); + options.addArguments('--headless=new', '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--window-size=1280,720'); + const driver = await new Builder() + .forBrowser('chrome') + .setLoggingPrefs(preferences) + .setChromeOptions(options) + .build(); + const browserErrors = []; + let testError; + try { + await driver.manage().setTimeouts({ implicit: 0, pageLoad: 15_000, script: 15_000 }); + await driver.get(origin); + const state = await driver.wait(until.elementLocated(By.id('state')), 10_000); + await driver.wait(until.elementTextIs(state, 'ready'), 10_000); + assert.equal(await driver.getTitle(), 'E2E Harness Contract'); + const increment = await driver.findElement(By.id('increment')); + await increment.click(); + await increment.click(); + assert.equal(await driver.findElement(By.id('count')).getText(), '2'); + await assertInPageBoundaries((fn) => driver.executeAsyncScript(` + const done = arguments[arguments.length - 1]; + (${fn.toString()})().then(done, (error) => done({ contractError: error.message })); + `)); + } catch (error) { + testError = error; + } finally { + const entries = await driver.manage().logs().get(logging.Type.BROWSER).catch((error) => { + browserErrors.push(`webdriver-log: ${error.message}`); + return []; + }); + browserErrors.push(...entries + .filter((entry) => entry.level.value >= logging.Level.SEVERE.value) + .map((entry) => `${entry.level.name}: ${entry.message}`)); + await captureSelenium(driver, browserErrors); + await driver.quit().catch(() => {}); + } + if (testError) throw testError; + assertErrors(browserErrors); +} + +export async function runBrowserContract(framework) { + const harness = await startHarness(); + try { + if (framework === 'playwright') await runPlaywright(harness.origin); + else if (framework === 'puppeteer') await runPuppeteer(harness.origin); + else if (framework === 'selenium') await runSelenium(harness.origin); + else throw new TypeError(`unsupported framework: ${framework}`); + } finally { + await harness.close(); + } +} + +test('local harness enforces HTTP and security boundaries', { + skip: process.env.E2E_FRAMEWORK !== 'playwright', + timeout: 10_000, +}, async () => { + const harness = await startHarness(); + try { + const response = await fetch(harness.origin, { redirect: 'error' }); + const headers = Object.fromEntries(response.headers.entries()); + assertMainResponse(response.status, headers); + assert.equal(await response.text(), html); + + const head = await fetch(harness.origin, { method: 'HEAD' }); + assert.equal(head.status, 200); + assert.equal(await head.text(), ''); + + const rejected = await fetch(`${harness.origin}/healthz`, { method: 'POST' }); + assert.equal(rejected.status, 405); + assert.equal(rejected.headers.get('allow'), 'GET, HEAD'); + + const missing = await fetch(`${harness.origin}/missing`); + assert.equal(missing.status, 404); + } finally { + await harness.close(); + } +}); + +test('playwright satisfies the browser harness contract', { + skip: !contractEnabled || process.env.E2E_FRAMEWORK !== 'playwright', + timeout: 45_000, +}, () => runBrowserContract('playwright')); diff --git a/tests/browser/puppeteer/harness-contract.test.mjs b/tests/browser/puppeteer/harness-contract.test.mjs new file mode 100644 index 00000000..d9c6ce6b --- /dev/null +++ b/tests/browser/puppeteer/harness-contract.test.mjs @@ -0,0 +1,6 @@ +import test from 'node:test'; + +process.env.E2E_FRAMEWORK ||= 'puppeteer'; +const { contractEnabled, runBrowserContract } = await import('../playwright/harness-contract.test.mjs'); + +test('puppeteer satisfies the browser harness contract', { skip: !contractEnabled, timeout: 45_000 }, () => runBrowserContract('puppeteer')); diff --git a/tests/browser/selenium/harness-contract.test.mjs b/tests/browser/selenium/harness-contract.test.mjs new file mode 100644 index 00000000..7f04a8e8 --- /dev/null +++ b/tests/browser/selenium/harness-contract.test.mjs @@ -0,0 +1,6 @@ +import test from 'node:test'; + +process.env.E2E_FRAMEWORK ||= 'selenium'; +const { contractEnabled, runBrowserContract } = await import('../playwright/harness-contract.test.mjs'); + +test('selenium satisfies the browser harness contract', { skip: !contractEnabled, timeout: 45_000 }, () => runBrowserContract('selenium')); diff --git a/tests/export-test-org-fleet-index.test.mjs b/tests/export-test-org-fleet-index.test.mjs new file mode 100644 index 00000000..52f691e3 --- /dev/null +++ b/tests/export-test-org-fleet-index.test.mjs @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const exporter = path.join(root, 'scripts', 'export-test-org-fleet-index.mjs'); + +function loadIndex() { + const result = childProcess.spawnSync(process.execPath, [exporter], { + cwd: root, + encoding: 'utf8', + env: { ...process.env, GH_TOKEN: '', GITHUB_TOKEN: '' }, + }); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); +} + +test('exports the exact live 363-repository portfolio', () => { + const index = loadIndex(); + assert.equal(index.schemaVersion, 1); + assert.equal(index.pairCount, 19); + assert.equal(index.retainedRepositoryCount, 22); + assert.equal(index.specializedRepositoryCount, 322); + assert.equal(index.governanceRepositoryCount, 19); + assert.equal(index.managedRepositoryCount, 341); + assert.equal(index.expectedRepositoryCount, 363); + assert.deepEqual(index.excludedOrganizations.sort(), ['r2g', 'r2g-test']); +}); + +test('exports unique expected names for every paired test organization', () => { + const index = loadIndex(); + for (const organization of index.organizations) { + assert.match(organization.testOrganization, /-test$/i); + assert.equal(organization.governanceRepository, '.github'); + assert.equal( + new Set(organization.expectedRepositories).size, + organization.expectedRepositories.length, + organization.testOrganization, + ); + assert.equal( + organization.expectedCount, + organization.retainedCount + organization.specializedCount + 1, + organization.testOrganization, + ); + assert.equal(organization.managedCount, organization.specializedCount + 1); + } +}); + +test('keeps the canonical high-depth product counts stable', () => { + const index = loadIndex(); + const byName = new Map(index.organizations.map((item) => [item.testOrganization, item])); + assert.equal(byName.get('evento-globolo-test').specializedCount, 17); + assert.equal(byName.get('fiducia-cloud-test').specializedCount, 31); + assert.equal(byName.get('shared-auth-test').specializedCount, 19); + assert.equal(byName.get('file-tunnel-test').specializedCount, 17); + assert.equal(byName.get('hacker-house-medellin-test').specializedCount, 18); + assert.equal(byName.get('streempilot-test').specializedCount, 20); + assert.equal(byName.get('quaestor-ledger-test').specializedCount, 19); +}); diff --git a/tests/fleet-manifest-extensions.test.mjs b/tests/fleet-manifest-extensions.test.mjs new file mode 100644 index 00000000..800e3907 --- /dev/null +++ b/tests/fleet-manifest-extensions.test.mjs @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +import { applyFleetManifestExtensions } from '../scripts/fleet-manifest-extensions.mjs'; + +function fixture() { + return { + schemaVersion: 1, + pairs: [ + { + sourceOrg: 'quaestor-ledger', + testOrg: 'quaestor-ledger-test', + existingRepositories: [], + repositories: [{ name: 'existing-e2e' }], + }, + ], + }; +} + +function withExtension(extension, callback) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'fleet-extension-test-')); + const manifestPath = path.join(directory, 'test-org-fleet.json.gz'); + fs.writeFileSync(path.join(directory, 'test-org-fleet.extensions.json'), `${JSON.stringify(extension, null, 2)}\n`); + try { + return callback(manifestPath); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +} + +test('extensions append repositories to one existing organization pair', () => { + const manifest = fixture(); + withExtension({ + schemaVersion: 1, + pairs: [{ + sourceOrg: 'quaestor-ledger', + testOrg: 'quaestor-ledger-test', + repositories: [{ name: 'tenant-isolation-e2e', state: 'create' }], + }], + }, (manifestPath) => applyFleetManifestExtensions(manifest, manifestPath)); + + assert.deepEqual( + manifest.pairs[0].repositories.map((repository) => repository.name), + ['existing-e2e', 'tenant-isolation-e2e'], + ); +}); + +test('extensions reject duplicate repository names', () => { + withExtension({ + schemaVersion: 1, + pairs: [{ + testOrg: 'quaestor-ledger-test', + repositories: [{ name: 'existing-e2e' }], + }], + }, (manifestPath) => { + assert.throws( + () => applyFleetManifestExtensions(fixture(), manifestPath), + /duplicates repository quaestor-ledger-test\/existing-e2e/, + ); + }); +}); + +test('extensions reject unknown organizations and source-pair mismatches', () => { + withExtension({ + schemaVersion: 1, + pairs: [{ testOrg: 'missing-test', repositories: [{ name: 'new-e2e' }] }], + }, (manifestPath) => { + assert.throws(() => applyFleetManifestExtensions(fixture(), manifestPath), /unknown test organization/); + }); + + withExtension({ + schemaVersion: 1, + pairs: [{ + sourceOrg: 'wrong-source', + testOrg: 'quaestor-ledger-test', + repositories: [{ name: 'new-e2e' }], + }], + }, (manifestPath) => { + assert.throws(() => applyFleetManifestExtensions(fixture(), manifestPath), /source organization mismatch/); + }); +}); + +test('no extension file leaves the manifest unchanged', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'fleet-extension-test-')); + try { + const manifest = fixture(); + assert.equal( + applyFleetManifestExtensions(manifest, path.join(directory, 'test-org-fleet.json.gz')), + manifest, + ); + assert.equal(manifest.pairs[0].repositories.length, 1); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/fleet-recovery-control.test.mjs b/tests/fleet-recovery-control.test.mjs new file mode 100644 index 00000000..4517f681 --- /dev/null +++ b/tests/fleet-recovery-control.test.mjs @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const credentialLiteral = /(?:ghp_|github_pat_)[A-Za-z0-9_]{20,}/; + +function read(relativePath) { + return fs.readFileSync(path.join(root, relativePath), 'utf8'); +} + +test('full fleet apply validates git transport and defers nonessential topic writes', () => { + const workflow = read('.github/workflows/apply-test-org-fleet.yml'); + + assert.match(workflow, /node --test tests\/test-org-fleet\.test\.mjs tests\/git-transport-worktree\.test\.mjs tests\/fleet-recovery-control\.test\.mjs/); + assert.match(workflow, /TEST_ORG_FLEET_SKIP_TOPICS: "true"/); + assert.match(workflow, /max-parallel: 1/); + assert.match(workflow, /-f authentication=pat|authentication=pat/); + assert.equal(credentialLiteral.test(workflow), false); +}); + +test('one-time cancellation stops both superseded writers and nothing else', () => { + const workflow = read('.github/workflows/cancel-stale-fleet-recovery.yml'); + const runIds = [...workflow.matchAll(/\b309\d{8}\b/g)].map((match) => match[0]); + + assert.deepEqual([...new Set(runIds)], ['30972717804', '30973196337']); + assert.match(workflow, /actions: write/); + assert.match(workflow, /contents: read/); + assert.match(workflow, /--method POST "\$endpoint\/cancel"/); + assert.equal(credentialLiteral.test(workflow), false); +}); + +test('v3 dispatcher waits for stale writers and launches one PAT-backed all-org apply', () => { + const workflow = read('.github/workflows/dispatch-test-org-fleet-once.yml'); + const marker = read('bootstrap/execute-test-org-fleet-v2.once').trim(); + + assert.equal(marker, 'APPLY_TEST_FLEET_V3'); + assert.match(workflow, /bootstrap\/execute-test-org-fleet-v2\.once/); + assert.match(workflow, /STALE_RUN_IDS: "30972717804 30973196337"/); + assert.match(workflow, /Wait for superseded fleet writers to stop/); + assert.match(workflow, /gh workflow run apply-test-org-fleet\.yml/); + assert.match(workflow, /-f organization=all/); + assert.match(workflow, /-f authentication=pat/); + assert.match(workflow, /-f confirm=APPLY_TEST_FLEET/); + assert.doesNotMatch(workflow, /workflow_dispatch:/); + assert.equal(credentialLiteral.test(workflow), false); +}); diff --git a/tests/generated-plan-validator-gitlinks.test.mjs b/tests/generated-plan-validator-gitlinks.test.mjs new file mode 100644 index 00000000..9713147d --- /dev/null +++ b/tests/generated-plan-validator-gitlinks.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const validatorTemplatePart = path.join( + root, + 'scripts', + 'bootstrap-test-org-fleet.parts', + '002.mjs.part', +); + +function git(cwd, ...args) { + return childProcess.execFileSync('git', args, { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +test('generated validator renders a recursive git tree scan', () => { + const source = fs.readFileSync(validatorTemplatePart, 'utf8'); + assert.match(source, /\['ls-tree', '-r', 'HEAD'\]/); + assert.doesNotMatch(source, /\['ls-tree', 'HEAD'\]/); +}); + +test('recursive tree inspection exposes nested gitlinks and missing paths stay absent', () => { + const repository = fs.mkdtempSync(path.join(os.tmpdir(), 'nested-gitlink-')); + try { + git(repository, 'init', '--quiet'); + git(repository, 'config', 'user.name', 'test-org-fleet'); + git(repository, 'config', 'user.email', 'test-org-fleet@example.invalid'); + fs.writeFileSync(path.join(repository, 'README.md'), '# fixture\n'); + git(repository, 'add', 'README.md'); + git(repository, 'commit', '--quiet', '-m', 'base'); + + const pinnedCommit = '1111111111111111111111111111111111111111'; + const nestedPath = 'vendor/example-client'; + git( + repository, + 'update-index', + '--add', + '--cacheinfo', + `160000,${pinnedCommit},${nestedPath}`, + ); + git(repository, 'commit', '--quiet', '-m', 'add nested gitlink'); + + const topLevel = git(repository, 'ls-tree', 'HEAD'); + const recursive = git(repository, 'ls-tree', '-r', 'HEAD'); + + assert.doesNotMatch(topLevel, /vendor\/example-client/); + assert.match(topLevel, /040000 tree [0-9a-f]{40}\tvendor/); + assert.match( + recursive, + new RegExp(`160000 commit ${pinnedCommit}\\t${nestedPath}`), + ); + assert.doesNotMatch(recursive, /vendor\/missing-client/); + } finally { + fs.rmSync(repository, { recursive: true, force: true }); + } +}); diff --git a/tests/git-transport-worktree.test.mjs b/tests/git-transport-worktree.test.mjs new file mode 100644 index 00000000..93c755f3 --- /dev/null +++ b/tests/git-transport-worktree.test.mjs @@ -0,0 +1,128 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const launcher = fs.readFileSync(path.join(root, 'scripts', 'bootstrap-test-org-fleet.mjs'), 'utf8'); +const compatibilityWrapper = fs.readFileSync( + path.join(root, 'scripts', 'bootstrap-test-org-fleet-worktree-safe.mjs'), + 'utf8', +); + +function git(cwd, ...arguments_) { + const result = childProcess.spawnSync('git', arguments_, { + cwd, + encoding: 'utf8', + }); + assert.equal( + result.status, + 0, + `git ${arguments_.join(' ')} failed:\n${result.stderr || result.stdout}`, + ); + return result.stdout.trim(); +} + +test('git credential helper never dirties a generated repository worktree', () => { + assert.match(launcher, /const worktree = fs\.mkdtempSync\(path\.join\(os\.tmpdir\(\), 'test-org-fleet-worktree-'\)\)/); + assert.match(launcher, /const credentialDirectory = fs\.mkdtempSync\(path\.join\(os\.tmpdir\(\), 'test-org-fleet-credentials-'\)\)/); + assert.match(launcher, /const askpass = path\.join\(credentialDirectory, 'git-askpass\.sh'\)/); + assert.doesNotMatch(launcher, /const askpass = path\.join\(worktree, 'git-askpass\.sh'\)/); + assert.match(launcher, /\['status', '--porcelain=v1', '--untracked-files=no'\]/); + assert.match(launcher, /fs\.rmSync\(credentialDirectory, \{ recursive: true, force: true \}\)/); + assert.match( + launcher, + /fs\.mkdirSync\(gitlinkPath, \{ recursive: true \}\)/, + ); + assert.match(compatibilityWrapper, /if \(!launcher\.includes\(gitlinkWorktreePlaceholder\)\)/); +}); + +test('an existing gitlink remains clean during an idempotent generated-branch rerun', () => { + const repository = fs.mkdtempSync(path.join(os.tmpdir(), 'fleet-gitlink-rerun-')); + const gitlinkPath = path.join(repository, 'vendor', 'dependency'); + + try { + git(repository, 'init', '--quiet'); + git(repository, 'config', 'user.name', 'fleet-regression'); + git(repository, 'config', 'user.email', 'fleet-regression@example.invalid'); + + fs.writeFileSync(path.join(repository, 'README.md'), '# fixture\n'); + git(repository, 'add', 'README.md'); + git(repository, 'commit', '--quiet', '-m', 'initial fixture'); + const dependencySha = git(repository, 'rev-parse', 'HEAD'); + + fs.writeFileSync( + path.join(repository, '.gitmodules'), + '[submodule "vendor/dependency"]\n' + + '\tpath = vendor/dependency\n' + + '\turl = https://github.com/example/dependency.git\n', + ); + git(repository, 'add', '.gitmodules'); + git( + repository, + 'update-index', + '--add', + '--cacheinfo', + `160000,${dependencySha},vendor/dependency`, + ); + fs.mkdirSync(gitlinkPath, { recursive: true }); + git(repository, 'commit', '--quiet', '-m', 'add generated dependency gitlink'); + assert.equal(git(repository, 'status', '--porcelain=v1', '--untracked-files=no'), ''); + + // Simulate fetching an existing generated branch without checking out its + // submodules, then re-applying the same immutable gitlink to the index. + fs.rmSync(gitlinkPath, { recursive: true, force: true }); + git( + repository, + 'update-index', + '--add', + '--cacheinfo', + `160000,${dependencySha},vendor/dependency`, + ); + assert.notEqual( + git(repository, 'status', '--porcelain=v1', '--untracked-files=no'), + '', + 'the missing gitlink worktree should reproduce the deletion seen in fleet reruns', + ); + + fs.mkdirSync(gitlinkPath, { recursive: true }); + assert.equal( + git(repository, 'status', '--porcelain=v1', '--untracked-files=no'), + '', + 'the empty gitlink placeholder must restore a clean idempotent worktree', + ); + } finally { + fs.rmSync(repository, { recursive: true, force: true }); + } +}); + +test('an unchanged gitlink keeps an empty worktree directory during idempotent recovery', () => { + assert.match(launcher, /const gitlinkPath = path\.join\(worktree, gitlink\.path\)/); + assert.match(launcher, /fs\.mkdirSync\(gitlinkPath, \{ recursive: true \}\)/); + + const repository = fs.mkdtempSync(path.join(os.tmpdir(), 'test-org-fleet-gitlink-')); + try { + git(repository, 'init', '--quiet'); + git(repository, 'config', 'user.name', 'test-org-fleet'); + git(repository, 'config', 'user.email', 'test-org-fleet@example.invalid'); + fs.writeFileSync(path.join(repository, 'README.md'), '# fixture\n'); + git(repository, 'add', 'README.md'); + git(repository, 'commit', '--quiet', '-m', 'fixture base'); + + const sourceSha = git(repository, 'rev-parse', 'HEAD'); + git(repository, 'update-index', '--add', '--cacheinfo', `160000,${sourceSha},vendor/sdk`); + fs.mkdirSync(path.join(repository, 'vendor', 'sdk'), { recursive: true }); + git(repository, 'commit', '--quiet', '-m', 'add gitlink'); + assert.equal(git(repository, 'status', '--porcelain=v1', '--untracked-files=no'), ''); + + fs.rmSync(path.join(repository, 'vendor', 'sdk'), { recursive: true, force: true }); + assert.equal(git(repository, 'status', '--porcelain=v1', '--untracked-files=no'), 'D vendor/sdk'); + + fs.mkdirSync(path.join(repository, 'vendor', 'sdk'), { recursive: true }); + assert.equal(git(repository, 'status', '--porcelain=v1', '--untracked-files=no'), ''); + } finally { + fs.rmSync(repository, { recursive: true, force: true }); + } +}); diff --git a/tests/idempotent-fleet-pr-repair.test.mjs b/tests/idempotent-fleet-pr-repair.test.mjs new file mode 100644 index 00000000..40ebeba1 --- /dev/null +++ b/tests/idempotent-fleet-pr-repair.test.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const orchestrationPath = path.join(root, 'scripts', 'bootstrap-test-org-fleet.parts', '005.mjs.part'); +const pullRequestPath = path.join(root, 'scripts', 'bootstrap-test-org-fleet.parts', '004.mjs.part'); +const orchestration = fs.readFileSync(orchestrationPath, 'utf8'); +const pullRequests = fs.readFileSync(pullRequestPath, 'utf8'); +const credentialLiteral = /(?:ghp_|github_pat_)[A-Za-z0-9_]{20,}/; + +test('idempotent apply repairs missing draft PRs even when generated files are unchanged', () => { + const governance = orchestration.match(/async function applyGovernance[\s\S]*?async function applyTestRepository/)?.[0] ?? ''; + const repository = orchestration.match(/async function applyTestRepository[\s\S]*?async function mapConcurrent/)?.[0] ?? ''; + + assert.match(governance, /const result = await writeTree/); + assert.match(governance, /await ensurePullRequest/); + assert.doesNotMatch(governance, /if \(result\.changed\)/); + + assert.match(repository, /const result = await writeTree/); + assert.match(repository, /await ensurePullRequest/); + assert.doesNotMatch(repository, /if \(result\.changed\)/); + assert.equal(credentialLiteral.test(orchestration), false); +}); + +test('generated PR reconciliation retains one draft and closes duplicates or non-drafts', () => { + const ensurePullRequest = pullRequests.match(/async function ensurePullRequest[\s\S]*?async function ens/)?.[0] ?? ''; + + assert.match(ensurePullRequest, /pull\.head\?\.ref === branchName/); + assert.match(ensurePullRequest, /pull\.base\?\.ref === base/); + assert.match(ensurePullRequest, /const drafts = matching\.filter/); + assert.match(ensurePullRequest, /closed duplicate generated pull request/); + assert.match(ensurePullRequest, /closed non-draft generated pull request/); + assert.match(ensurePullRequest, /pulls\/\$\{duplicate\.number\}/); + assert.match(ensurePullRequest, /state: 'closed'/); + assert.match(ensurePullRequest, /draft: true/); + assert.equal(credentialLiteral.test(pullRequests), false); +}); diff --git a/tests/live-fleet-verifier.test.mjs b/tests/live-fleet-verifier.test.mjs new file mode 100644 index 00000000..dbb9a39c --- /dev/null +++ b/tests/live-fleet-verifier.test.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import { readFleetManifest } from '../scripts/read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const verifierPath = path.join(root, 'scripts', 'verify-live-test-org-fleet.mjs'); +const verifier = fs.readFileSync(verifierPath, 'utf8'); +const manifest = readFleetManifest(path.join(root, 'bootstrap', 'test-org-fleet.json.gz')); + +test('live verifier parses and derives the complete non-r2g fleet', () => { + const syntax = childProcess.spawnSync(process.execPath, ['--check', verifierPath], { + cwd: root, + encoding: 'utf8', + }); + assert.equal(syntax.status, 0, syntax.stderr); + + const specialized = manifest.pairs.reduce((total, pair) => total + pair.repositories.length, 0); + assert.ok(manifest.pairs.length > 0); + assert.ok(specialized > manifest.pairs.length); + assert.equal( + manifest.pairs.some((pair) => ['r2g', 'r2g-test'].includes(pair.testOrg.toLowerCase())), + false, + ); + + assert.doesNotMatch(verifier, /expectedSpecialized !== \d+/); + assert.doesNotMatch(verifier, /expectedGovernance !== \d+/); + assert.doesNotMatch(verifier, /expected\.length !== \d+/); + assert.match(verifier, /const expectedSpecialized = expected\.filter/); + assert.match(verifier, /const expectedGovernance = expected\.filter/); + assert.match(verifier, /git\/ref\/heads/); + assert.match(verifier, /pulls\?state=open&per_page=100/); + assert.match(verifier, /missingOpenPullRequests/); + assert.match(verifier, /repositoriesWithOpenPullRequest/); + assert.doesNotMatch(verifier, /(?:ghp_|github_pat_)[A-Za-z0-9_]{20,}/); +}); diff --git a/tests/live-test-org-fleet-audit.test.mjs b/tests/live-test-org-fleet-audit.test.mjs new file mode 100644 index 00000000..ef35b84d --- /dev/null +++ b/tests/live-test-org-fleet-audit.test.mjs @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + auditFleet, + diffRepositoryNames, + expectedRepositoryNames, + parseArguments, + renderMarkdown, +} from '../scripts/audit-live-test-org-fleet.mjs'; + +function repository(name, overrides = {}) { + return { + name, + archived: false, + disabled: false, + default_branch: 'main', + ...overrides, + }; +} + +const pair = { + sourceOrg: 'example', + testOrg: 'example-test', + existingRepositories: ['legacy-fixture'], + repositories: [{ name: 'api-e2e' }, { name: 'web-e2e' }], +}; + +test('expected set includes governance, retained fixtures, and generated repos', () => { + assert.deepEqual( + [...expectedRepositoryNames(pair)].sort(), + ['.github', 'api-e2e', 'legacy-fixture', 'web-e2e'], + ); +}); + +test('diff preserves extras while reporting missing canonical repos', () => { + assert.deepEqual( + diffRepositoryNames(['.github', 'api-e2e'], ['.github', 'extra-e2e']), + { + missing: ['api-e2e'], + extras: ['extra-e2e'], + present: ['.github'], + }, + ); +}); + +test('argument parser supports scoped and strict audits', () => { + assert.deepEqual( + parseArguments(['--json', '--strict-extras', '--org', 'example-test']), + { json: true, strictExtras: true, organizations: ['example-test'] }, + ); + assert.throws(() => parseArguments(['--org']), /requires a value/); + assert.throws(() => parseArguments(['--unknown']), /unknown argument/); +}); + +test('fleet audit checks canonical hygiene but only reports production hygiene', async () => { + const manifest = { + policy: { excludedOrganizations: ['r2g', 'r2g-test'] }, + pairs: [pair], + }; + const client = { + async listOrganizationRepositories(organization) { + if (organization === 'example') return [repository('api', { archived: true })]; + return [ + repository('.github'), + repository('api-e2e'), + repository('web-e2e', { default_branch: '' }), + repository('extra-e2e'), + ]; + }, + }; + const report = await auditFleet(manifest, client); + assert.equal(report.pairCount, 1); + assert.equal(report.expectedRepositories, 4); + assert.equal(report.presentRepositories, 3); + assert.equal(report.missingRepositories, 1); + assert.equal(report.extraRepositories, 1); + assert.equal(report.expectedHygieneFindings, 1); + assert.equal(report.sourceHygieneFindings, 1); + assert.deepEqual(report.reports[0].missing, ['legacy-fixture']); + assert.deepEqual(report.reports[0].extras, ['extra-e2e']); + assert.match(renderMarkdown(report), /legacy-fixture/); +}); + +test('excluded and unknown test organizations cannot enter scope', async () => { + const manifest = { + policy: { excludedOrganizations: ['r2g', 'r2g-test'] }, + pairs: [pair], + }; + const client = { async listOrganizationRepositories() { return []; } }; + await assert.rejects( + () => auditFleet(manifest, client, { organizations: ['r2g-test'] }), + /unknown or excluded/, + ); +}); diff --git a/tests/paced-fleet-executor.test.mjs b/tests/paced-fleet-executor.test.mjs new file mode 100644 index 00000000..ab1d0578 --- /dev/null +++ b/tests/paced-fleet-executor.test.mjs @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const listScript = path.join(root, 'scripts', 'list-test-org-fleet-repositories.mjs'); +const workflow = fs.readFileSync(path.join(root, '.github', 'workflows', 'finish-test-org-fleet-paced.yml'), 'utf8'); +const marker = fs.readFileSync(path.join(root, 'bootstrap', 'execute-test-org-fleet-paced.once'), 'utf8').trim(); +const credentialLiteral = /(?:ghp_|github_pat_)[A-Za-z0-9_]{20,}/; + +const organizations = [ + 'fiducia-cloud-test', + 'evento-globolo-test', + 'opto-sync-test', + 'quaestor-ledger-test', + 'memebank-test', + 'scintilla-run-test', + 'file-tunnel-test', + 'shared-auth-test', + 'hypesiege-test', + 'streempilot-test', + 'sonus-auris-test', + 'messaging-intel-test', + '3fa-app-test', + 'declarative-migrations-test', + 'cliptown-test', + 'claritas-viz-test', + 'embedded-alerts-test', +]; + +test('paced executor lists exactly 290 specialized repositories', () => { + let total = 0; + for (const organization of organizations) { + const result = childProcess.spawnSync(process.execPath, [listScript, '--org', organization, '--json'], { + cwd: root, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr); + const plan = JSON.parse(result.stdout); + assert.equal(plan.organization, organization); + assert.equal(plan.repositoryCount, plan.repositories.length); + assert.equal(new Set(plan.repositories).size, plan.repositories.length); + total += plan.repositoryCount; + } + assert.equal(total, 290); +}); + +test('paced executor rejects excluded organizations and paces write-producing repositories', () => { + const excluded = childProcess.spawnSync(process.execPath, [listScript, '--org', 'r2g-test'], { + cwd: root, + encoding: 'utf8', + }); + assert.notEqual(excluded.status, 0); + assert.match(excluded.stderr, /refusing excluded organization/); + + assert.match(marker, /^APPLY_TEST_FLEET_PACED_(?:V2|ARM64_V[12])$/); + assert.match(workflow, /group: finish-test-org-fleet-paced-v3/); + assert.match(workflow, /cancel-in-progress: false/); + assert.match(workflow, /--repo "\$repository"/); + assert.match(workflow, /for attempt in 1 2 3/); + assert.match(workflow, /sleep 15/); + assert.match(workflow, /TEST_ORG_FLEET_SKIP_TOPICS: "true"/); + assert.match(workflow, /test "\$total" -eq 290/); + assert.match(workflow, /tests\/bootstrap-retry-policy\.test\.mjs/); + assert.match(workflow, /tests\/git-transport-worktree\.test\.mjs/); + assert.match(workflow, /gh_api_with_rate_limit_retry\(\)/); + assert.match(workflow, /gh api rate_limit/); + assert.match(workflow, /reset_epoch - now_epoch \+ 5/); + assert.match(workflow, /gh_api_with_rate_limit_retry "orgs\/\$organization" --silent/); + assert.match(workflow, /gh_api_with_rate_limit_retry --paginate/); + assert.doesNotMatch(workflow, /^\s+gh api "orgs\/\$organization" --silent\s*$/m); + assert.doesNotMatch(workflow, /^\s+r2g(?:-test)?\s*$/m); + assert.equal(credentialLiteral.test(workflow), false); +}); diff --git a/tests/quaestor-fleet-extension.test.mjs b/tests/quaestor-fleet-extension.test.mjs new file mode 100644 index 00000000..01ac70bf --- /dev/null +++ b/tests/quaestor-fleet-extension.test.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; +import test from 'node:test'; + +import { readFleetManifest } from '../scripts/read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const manifest = readFleetManifest(path.join(root, 'bootstrap', 'test-org-fleet.json.gz')); + +test('Quaestor adds dedicated tenant, webhook, and recovery risk repositories', () => { + const pair = manifest.pairs.find((candidate) => candidate.testOrg === 'quaestor-ledger-test'); + assert.ok(pair); + assert.equal(pair.sourceOrg, 'quaestor-ledger'); + assert.equal(pair.repositories.length, 19); + + const repositories = new Map(pair.repositories.map((repository) => [repository.name, repository])); + const expected = new Map([ + ['tenant-isolation-e2e', 'security-e2e'], + ['webhook-auth-replay-e2e', 'security-e2e'], + ['migration-recovery-e2e', 'database-e2e'], + ]); + + for (const [name, profile] of expected) { + const repository = repositories.get(name); + assert.ok(repository, name); + assert.equal(repository.state, 'create'); + assert.equal(repository.profile, profile); + assert.equal(repository.focus.length >= 5, true); + assert.equal(repository.sources.every((source) => /^[^/]+\/[^/]+$/.test(source)), true); + assert.deepEqual(repository.imports, []); + } +}); + +test('Quaestor extension contains no credential-shaped values', () => { + const pair = manifest.pairs.find((candidate) => candidate.testOrg === 'quaestor-ledger-test'); + const serialized = JSON.stringify(pair); + assert.doesNotMatch(serialized, /(?:ghp_|github_pat_|Bearer\s+)[A-Za-z0-9._-]{20,}/i); +}); diff --git a/tests/secure-fleet-handoff-scope.test.mjs b/tests/secure-fleet-handoff-scope.test.mjs new file mode 100644 index 00000000..6233bb10 --- /dev/null +++ b/tests/secure-fleet-handoff-scope.test.mjs @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const workflowPath = path.join( + root, + '.github', + 'workflows', + 'secure-fleet-token-handoff.yml', +); +const workflow = fs.readFileSync(workflowPath, 'utf8'); + +test('encrypted handoff starts only from an explicit reviewed START selection', () => { + assert.match( + workflow, + /push:\n\s+branches: \[main\]\n\s+paths:\n\s+- "\.github\/fleet-handoff\/START"/, + ); + assert.doesNotMatch( + workflow, + /paths:\n[\s\S]*?\.github\/workflows\/secure-fleet-token-handoff\.yml/, + ); + assert.match(workflow, /github\.event\.head_commit\.removed/); + assert.match(workflow, /START must contain exactly one approved test organization or all/); + assert.match(workflow, /HANDOFF_ORGANIZATION=%s/); + assert.match(workflow, /--arg organization "\$handoff_organization"/); +}); + +test('encrypted handoff validates and dispatches only the selected organization', () => { + assert.match(workflow, /organizations=\("\$HANDOFF_ORGANIZATION"\)/); + assert.match(workflow, /gh api "orgs\/\$organization" --silent/); + assert.match(workflow, /-f organization="\$HANDOFF_ORGANIZATION"/); + assert.doesNotMatch(workflow, /-f organization=all/); + assert.match( + workflow, + /The \\`\$HANDOFF_ORGANIZATION\\` apply workflow was dispatched/, + ); +}); + +test('encrypted handoff retains ciphertext-only and actor verification controls', () => { + assert.match(workflow, /RSA-OAEP-SHA256/); + assert.match(workflow, /openssl pkeyutl -decrypt/); + assert.match(workflow, /EXPECTED_LOGIN: ORESoftware/); + assert.match(workflow, /payload_nonce/); + assert.match(workflow, /::add-mask::%s/); + assert.match(workflow, /gh secret set FLEET_GH_TOKEN/); + assert.doesNotMatch(workflow, /(?:ghp_|github_pat_)[A-Za-z0-9_]{20,}/); +}); diff --git a/tests/test-org-fleet-source-certification.test.mjs b/tests/test-org-fleet-source-certification.test.mjs new file mode 100644 index 00000000..5f5a0293 --- /dev/null +++ b/tests/test-org-fleet-source-certification.test.mjs @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +import { hardenGeneratedIntegrationPolicy } from '../scripts/test-org-fleet-integration-policy.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const partsDirectory = path.join(root, 'scripts', 'bootstrap-test-org-fleet.parts'); +const generatedSource = fs + .readdirSync(partsDirectory) + .sort() + .map((name) => fs.readFileSync(path.join(partsDirectory, name), 'utf8')) + .join('') + .replace(/^#![^\n]*\n/, ''); +const sourceCertificationDocumentation = fs.readFileSync( + path.join(root, 'docs', 'test-org-source-certification.md'), + 'utf8', +); + +function hardened() { + return hardenGeneratedIntegrationPolicy(generatedSource); +} + +test('generated plans distinguish snapshots, source access, and product certification', () => { + const source = hardened(); + assert.match(source, /protectedSourceIntegration: true/); + assert.match( + source, + /protectedSourceIntegrationStatus: 'not-executed-unless-protected-lane-runs'/, + ); + assert.match(source, /productOverlayPreserved: true/); + assert.match(source, /generic protected lane reports source-access status only/); + assert.match(source, /source certification requires a product-specific executable overlay/); + assert.match(source, /A skipped integration job is not source certification/); +}); + +test('generated integration never falls back to the repository GITHUB_TOKEN', () => { + const source = hardened(); + assert.doesNotMatch( + source, + /secrets\.TEST_FLEET_READ_TOKEN \|\| github\.token/, + ); + assert.match(source, /test -n \"\$TEST_FLEET_READ_TOKEN\"/); + assert.match(source, /token: \\?\$\{\{ secrets\.TEST_FLEET_READ_TOKEN \}\}/); + assert.match(source, /persist-credentials: false/); +}); + +test('generated integration cancels superseded runs for the same workflow and ref', () => { + const source = hardened(); + const concurrencyContract = [ + 'permissions:\\n', + ' contents: read\\n\\n', + 'concurrency:\\n', + ' group: \\${{ github.workflow }}-\\${{ github.ref }}\\n', + ' cancel-in-progress: true\\n\\n', + 'jobs:\\n', + ' integration:\\n', + ].join(''); + + assert.equal(source.split(concurrencyContract).length - 1, 1); +}); + +test('generic status artifact can report source access but never product certification', () => { + const source = hardened(); + assert.match(source, /name: Record protected source access status/); + assert.match(source, /if: always\(\)/); + assert.match(source, /needs: integration/); + assert.match(source, /const executed = result !== 'skipped'/); + assert.match(source, /const sourceAccessPassed = result === 'success'/); + assert.match(source, /certified: false/); + assert.doesNotMatch(source, /const certified = result === 'success'/); + assert.match(source, /protected-source-access-passed-product-certification-required/); + assert.match(source, /protected-source-integration-not-enabled/); + assert.match(source, /source-integration-status-/); + assert.match( + source, + /actions\/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a/, + ); +}); + +test('generated file set leaves product-specific overlays outside fleet ownership', () => { + const source = hardened(); + const generatedFiles = source.slice( + source.indexOf('const files = new Map(['), + source.indexOf('const zpkg = renderZpkg', source.indexOf('const files = new Map([')), + ); + assert.ok(generatedFiles.length > 0); + for (const productPath of [ + 'CONTRACT.md', + 'contract-lock.json', + 'contract-surface.json', + 'scripts/contract-parser.mjs', + '.github/workflows/api-contract-parity.yml', + ]) { + assert.equal( + generatedFiles.includes(productPath), + false, + `${productPath} must remain product-overlay owned`, + ); + } + assert.match(source, /Product-specific files outside the generated file set are preserved/); +}); + +test('source certification documentation preserves the security and evidence meanings', () => { + assert.match( + sourceCertificationDocumentation, + /A skipped lane is \*\*not\*\* source certification/, + ); + assert.match(sourceCertificationDocumentation, /must not fall back to `github\.token`/); + assert.match(sourceCertificationDocumentation, /Product-specific overlays are preserved/); + assert.match(sourceCertificationDocumentation, /generic generated lane/i); + assert.match(sourceCertificationDocumentation, /source access/i); + assert.match(sourceCertificationDocumentation, /product-specific executable/i); + assert.match(sourceCertificationDocumentation, /`executed`/); + assert.match(sourceCertificationDocumentation, /`certified`/); + assert.match(sourceCertificationDocumentation, /Never choose an entire conflict side/); + assert.doesNotMatch( + sourceCertificationDocumentation, + /ghp_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}/, + ); +}); + +test('policy transform is fail-closed when generated seams change', () => { + assert.throws( + () => hardenGeneratedIntegrationPolicy(hardened()), + /expected one generated seam, found 0/, + ); + assert.throws( + () => + hardenGeneratedIntegrationPolicy( + generatedSource.replace( + ' token: \\${{ secrets.TEST_FLEET_READ_TOKEN || github.token }}', + ' token: changed', + ), + ), + /protected checkout token fallback/, + ); + assert.throws( + () => + hardenGeneratedIntegrationPolicy( + generatedSource.replace( + " - cron: '17 8 * * *'\\n\\npermissions:\\n contents: read\\n\\njobs:\\n", + 'generated integration concurrency seam changed', + ), + ), + /generated integration concurrency/, + ); +}); diff --git a/tests/test-org-fleet.test.mjs b/tests/test-org-fleet.test.mjs new file mode 100644 index 00000000..e926052a --- /dev/null +++ b/tests/test-org-fleet.test.mjs @@ -0,0 +1,215 @@ +// generated by zed-pkg-test/test-org-fleet +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import { readFleetManifest } from '../scripts/read-fleet-manifest.mjs'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const manifestPath = path.join(root, 'bootstrap', 'test-org-fleet.json.gz'); +const manifest = readFleetManifest(manifestPath); +const credentialLiteral = /(?:ghp_|github_pat_)[A-Za-z0-9_]{20,}/; +const interruptedOrganizations = [ + 'evento-globolo-test', + 'fiducia-cloud-test', + 'sonus-auris-test', + 'messaging-intel-test', + 'quaestor-ledger-test', + 'opto-sync-test', + 'memebank-test', + 'file-tunnel-test', + 'scintilla-run-test', + 'streempilot-test', + 'shared-auth-test', + 'hypesiege-test', +]; + +test('fleet contains the intended organization pairs and never r2g', () => { + assert.equal(manifest.pairs.length, 19); + const names = new Set(manifest.pairs.flatMap((pair) => [pair.sourceOrg.toLowerCase(), pair.testOrg.toLowerCase()])); + assert.equal(names.has('r2g'), false); + assert.equal(names.has('r2g-test'), false); + for (const pair of manifest.pairs) assert.match(pair.testOrg, /-test$/i); +}); + +test('repository names are unique inside every test org', () => { + for (const pair of manifest.pairs) { + const names = [...(pair.existingRepositories ?? []), ...pair.repositories.map((repository) => repository.name)]; + assert.equal(new Set(names).size, names.length, pair.testOrg); + } +}); + +test('fiducia has exactly ten three-lane language consumers', () => { + const pair = manifest.pairs.find((candidate) => candidate.testOrg === 'fiducia-cloud-test'); + const clients = pair.repositories.filter((repository) => /^fiducia-.+-client-e2e$/.test(repository.name)); + assert.equal(clients.length, 10); + for (const repository of clients) { + const modes = new Set(repository.imports.map((dependency) => dependency.mode)); + assert.deepEqual([...['git-submodule', 'zed', 'native-package'].filter((mode) => !modes.has(mode))], []); + } +}); + +test('validator succeeds and reports the complete fleet', () => { + const result = childProcess.spawnSync(process.execPath, [path.join(root, 'scripts', 'validate-test-org-fleet.mjs'), '--json'], { cwd: root, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + const summary = JSON.parse(result.stdout); + assert.equal(summary.pairs, 19); + assert.equal(summary.createRepositories, 322); + assert.equal(summary.governanceRepositories, 19); + assert.equal(summary.existingRepositories, 22); + assert.equal(summary.errors, 0); +}); + +test('bootstrap dry run is deterministic and write-free', () => { + const result = childProcess.spawnSync(process.execPath, [path.join(root, 'scripts', 'bootstrap-test-org-fleet.mjs'), '--dry-run', '--summary-json'], { cwd: root, encoding: 'utf8', env: { ...process.env, GH_TOKEN: '', GITHUB_TOKEN: '' } }); + assert.equal(result.status, 0, result.stderr); + const summary = JSON.parse(result.stdout); + assert.equal(summary.mode, 'dry-run'); + assert.equal(summary.selectedOrganizations, 19); + assert.equal(summary.selectedRepositories, 322); + assert.equal(summary.governanceRepositories, 19); + assert.equal(summary.planned, 341); + assert.equal(summary.created, 0); + assert.equal(summary.committed, 0); + assert.equal(summary.pullRequests, 0); +}); + +test('secret-backed apply workflow covers every test organization and never embeds a credential', () => { + const workflowPath = path.join(root, '.github', 'workflows', 'apply-test-org-fleet.yml'); + const documentationPath = path.join(root, 'docs', 'test-org-fleet-app-auth.md'); + const workflow = fs.readFileSync(workflowPath, 'utf8'); + const documentation = fs.readFileSync(documentationPath, 'utf8'); + + for (const pair of manifest.pairs) { + assert.equal(workflow.includes(`- ${pair.testOrg}`), true, `missing ${pair.testOrg}`); + } + + assert.equal(workflow.toLowerCase().includes('r2g-test'), false); + assert.match(workflow, /actions\/create-github-app-token@[0-9a-f]{40}/); + assert.match(workflow, /permission-administration: write/); + assert.match(workflow, /permission-contents: write/); + assert.match(workflow, /permission-metadata: read/); + assert.match(workflow, /permission-pull-requests: write/); + assert.match(workflow, /permission-workflows: write/); + assert.match(workflow, /secrets\.FLEET_GH_TOKEN/); + assert.match(workflow, /authentication:\n[\s\S]*?- auto\n[\s\S]*?- pat\n[\s\S]*?- github-app/); + assert.match(workflow, /APPLY_TEST_FLEET/); + assert.match(workflow, /GH_TOKEN=%s/); + assert.match(documentation, /Actions secret `FLEET_GH_TOKEN`/); + + assert.equal(credentialLiteral.test(workflow), false); + assert.equal(credentialLiteral.test(documentation), false); + assert.doesNotMatch(workflow, /^\s+(?:token|pat):\s*$/m); +}); + +test('fleet apply serializes organizations and retries GitHub secondary content limits', () => { + const workflowPath = path.join(root, '.github', 'workflows', 'apply-test-org-fleet.yml'); + const launcherPath = path.join(root, 'scripts', 'bootstrap-test-org-fleet.mjs'); + const workflow = fs.readFileSync(workflowPath, 'utf8'); + const launcher = fs.readFileSync(launcherPath, 'utf8'); + + assert.match(workflow, /timeout-minutes: 90/); + assert.match(workflow, /max-parallel: 1/); + assert.match(launcher, /attempt < 14/); + assert.match(launcher, /secondary rate limit\|temporarily blocked from content creation\|abuse detection/); + assert.match(launcher, /x-ratelimit-reset/); + assert.match(launcher, /Math\.min\(180000, 15000 \* \(2 \*\* attempt\)\)/); + assert.match(launcher, /retrying GitHub API request after rate limit/); +}); + +test('generated harness branches use authenticated git transport without staging the askpass helper', () => { + const launcherPath = path.join(root, 'scripts', 'bootstrap-test-org-fleet.mjs'); + const launcher = fs.readFileSync(launcherPath, 'utf8'); + + assert.match(launcher, /spawnSync\('git'/); + assert.match(launcher, /GIT_ASKPASS_REQUIRE: 'force'/); + assert.match(launcher, /const credentialDirectory = fs\.mkdtempSync\(path\.join\(os\.tmpdir\(\), 'test-org-fleet-credentials-'\)\)/); + assert.match(launcher, /const askpass = path\.join\(credentialDirectory, 'git-askpass\.sh'\)/); + assert.doesNotMatch(launcher, /const askpass = path\.join\(worktree, 'git-askpass\.sh'\)/); + assert.match(launcher, /fs\.rmSync\(credentialDirectory, \{ recursive: true, force: true \}\)/); + assert.match(launcher, /\['push', '--quiet', 'origin'/); + assert.match(launcher, /update-index/); + assert.match(launcher, /--cacheinfo/); + assert.match(launcher, /TEST_ORG_FLEET_SKIP_TOPICS/); + assert.match(launcher, /pushed generated commit with git transport/); + assert.equal(credentialLiteral.test(launcher), false); +}); + +test('one-time recovery resumes exactly the interrupted organizations in serial', () => { + const workflowPath = path.join(root, '.github', 'workflows', 'resume-test-org-fleet.yml'); + const workflow = fs.readFileSync(workflowPath, 'utf8'); + const listedOrganizations = [...workflow.matchAll(/^\s+- ([a-z0-9-]+-test)$/gm)].map((match) => match[1]); + + assert.deepEqual(listedOrganizations, interruptedOrganizations); + assert.equal(workflow.toLowerCase().includes('r2g-test'), false); + assert.equal(credentialLiteral.test(workflow), false); + assert.match(workflow, /push:\n\s+branches: \[main\]/); + assert.match(workflow, /timeout-minutes: 120/); + assert.match(workflow, /max-parallel: 1/); + assert.match(workflow, /secrets\.FLEET_GH_TOKEN/); + assert.match(workflow, /TEST_ORG_FLEET_SKIP_TOPICS: "true"/); + assert.match(workflow, /--apply/); + assert.match(workflow, /--concurrency 1/); + assert.doesNotMatch(workflow, /workflow_dispatch:/); +}); + +test('one-time handoff persists only ciphertext and dispatches the secret-backed fleet workflow', () => { + const workflowPath = path.join(root, '.github', 'workflows', 'secure-fleet-token-handoff.yml'); + const workflow = fs.readFileSync(workflowPath, 'utf8'); + + for (const pair of manifest.pairs) { + assert.equal(workflow.includes(` ${pair.testOrg}`), true, `handoff missing ${pair.testOrg}`); + } + + assert.equal(workflow.toLowerCase().includes('r2g-test'), false); + assert.equal(credentialLiteral.test(workflow), false); + assert.match(workflow, /permissions:\n contents: write/); + assert.match(workflow, /RSA-OAEP-SHA256/); + assert.match(workflow, /openssl pkeyutl -decrypt/); + assert.match(workflow, /ciphertext\.b64/); + assert.match(workflow, /EXPECTED_LOGIN: ORESoftware/); + assert.match(workflow, /gh secret set FLEET_GH_TOKEN/); + assert.match(workflow, /gh workflow run apply-test-org-fleet\.yml/); + assert.match(workflow, /-f authentication=pat/); + assert.match(workflow, /-f confirm=APPLY_TEST_FLEET/); + assert.match(workflow, /github\.repository == 'zed-pkg-test\/zed-pkg-e2e'/); + assert.doesNotMatch(workflow, /workflow_dispatch:/); +}); + + +test('Hacker House Medellín uses synthetic, privacy-safe product coverage', () => { + const pair = manifest.pairs.find((candidate) => candidate.testOrg === 'hacker-house-medellin-test'); + assert.ok(pair); + assert.equal(pair.sourceOrg, 'hacker-house-medellin'); + assert.equal(pair.defaultVisibility, 'private'); + assert.equal(pair.repositories.length, 18); + const names = new Set(pair.repositories.map((repository) => repository.name)); + for (const required of [ + 'mash-web-e2e', + 'leptos-web-e2e', + 'dioxus-web-e2e', + 'api-contract-e2e', + 'websocket-contract-e2e', + 'cli-contract-e2e', + 'sync-offline-e2e', + 'infra-compose-e2e', + 'postgres-supabase-e2e', + 'booking-race-e2e', + 'tenant-isolation-e2e', + 'payment-idempotency-e2e', + 'access-control-lifecycle-e2e', + 'notification-reconnect-e2e', + 'performance-smoke-e2e', + 'mcp-contract-e2e', + 'clients-rust-consumer', + 'clients-typescript-consumer', + ]) assert.equal(names.has(required), true, required); + const serializedPair = JSON.stringify(pair); + assert.equal(credentialLiteral.test(serializedPair), false); + assert.doesNotMatch(serializedPair, /(?:resident identity document|door credential|payment card|private message|real occupancy record)/i); + for (const repository of pair.repositories) { + assert.equal(repository.sources.every((source) => /^[^/]+\/[^/]+$/.test(source)), true); + assert.equal(repository.focus.length >= 3, true); + } +}); diff --git a/tests/test_candidate_lifecycle.py b/tests/test_candidate_lifecycle.py new file mode 100644 index 00000000..58e8e680 --- /dev/null +++ b/tests/test_candidate_lifecycle.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import argparse +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +import candidate_lifecycle # noqa: E402 + +SHA_A = "a" * 40 +SHA_B = "b" * 40 + + +class FixtureRefParsingTests(unittest.TestCase): + def test_accepts_exact_lowercase_commits(self) -> None: + self.assertEqual( + candidate_lifecycle.parse_fixture_refs( + f'{{"node-app":"{SHA_A}","node-lib":"{SHA_B}"}}' + ), + {"node-app": SHA_A, "node-lib": SHA_B}, + ) + + def test_rejects_non_object_empty_and_mutable_inputs(self) -> None: + invalid = ( + "[]", + "{}", + "not-json", + '{"node-app":"main"}', + '{"node-app":"abc123"}', + f'{{"Node-App":"{SHA_A}"}}', + f'{{"../node-app":"{SHA_A}"}}', + f'{{"node-app":"{SHA_A.upper()}"}}', + ) + for raw in invalid: + with self.subTest(raw=raw): + with self.assertRaises(argparse.ArgumentTypeError): + candidate_lifecycle.parse_fixture_refs(raw) + + def test_rejects_non_string_values(self) -> None: + with self.assertRaises(argparse.ArgumentTypeError): + candidate_lifecycle.parse_fixture_refs('{"node-app":42}') + + +class PinnedHarnessTests(unittest.TestCase): + def _harness(self, refs: dict[str, str]): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + harness = candidate_lifecycle.PinnedHarness.__new__( + candidate_lifecycle.PinnedHarness + ) + harness.repo = "node-app" + harness.fixture = Path(temp.name) / "root" + harness.fixture_refs = refs + harness.clones = {} + harness.dependency_repos = Path(temp.name) / "dependencies" + harness.assert_git_clean = lambda *_args, **_kwargs: None + commands: list[list[str]] = [] + + def run(command, **_kwargs): + argv = [str(value) for value in command] + commands.append(argv) + if "rev-parse" in argv: + return refs.get("node-lib", SHA_B) + "\n" + return "" + + harness.run = run + return harness, commands + + def test_missing_transitive_pin_fails_before_network(self) -> None: + harness, commands = self._harness({"node-app": SHA_A}) + with self.assertRaisesRegex(AssertionError, "has no exact commit"): + harness.source_root("node-lib") + self.assertEqual(commands, []) + + def test_dependency_fetch_uses_only_the_exact_commit(self) -> None: + harness, commands = self._harness({"node-app": SHA_B, "node-lib": SHA_A}) + resolved = harness.source_root("node-lib") + self.assertEqual(resolved, harness.dependency_repos / "node-lib") + + fetch = next(command for command in commands if "fetch" in command) + self.assertEqual(fetch[-1], SHA_A) + self.assertIn("--depth", fetch) + self.assertIn("--no-tags", fetch) + self.assertNotIn("main", fetch) + + checkout = next(command for command in commands if "checkout" in command) + self.assertEqual(checkout[-2:], ["--detach", "FETCH_HEAD"]) + + command_count = len(commands) + self.assertEqual(harness.source_root("node-lib"), resolved) + self.assertEqual(len(commands), command_count, "cached clone should be reused") + + def test_checked_out_dependency_must_match_the_pin(self) -> None: + harness, _commands = self._harness({"node-app": SHA_A, "node-lib": SHA_A}) + + def wrong_head(command, **_kwargs): + argv = [str(value) for value in command] + if "rev-parse" in argv: + return SHA_B + "\n" + return "" + + harness.run = wrong_head + with self.assertRaisesRegex(AssertionError, "ref mismatch"): + harness.source_root("node-lib") + + def test_root_checkout_must_be_present_and_exact(self) -> None: + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + fixture = Path(temp.name) / "fixture" + fixture.mkdir() + + def base_init(instance, args) -> None: + instance.repo = args.repo + instance.fixture = args.fixture_dir + instance.clones = {} + instance.dependency_repos = Path(temp.name) / "dependencies" + + args = SimpleNamespace( + repo="node-app", + fixture_dir=fixture, + fixture_refs_json={"node-app": SHA_A}, + ) + with patch.object(candidate_lifecycle.lifecycle.Harness, "__init__", base_init), patch.object( + candidate_lifecycle.PinnedHarness, "run", return_value=SHA_A + "\n" + ): + candidate_lifecycle.PinnedHarness(args) + + args.fixture_refs_json = {} + with patch.object(candidate_lifecycle.lifecycle.Harness, "__init__", base_init): + with self.assertRaisesRegex(AssertionError, "absent"): + candidate_lifecycle.PinnedHarness(args) + + args.fixture_refs_json = {"node-app": SHA_A} + with patch.object(candidate_lifecycle.lifecycle.Harness, "__init__", base_init), patch.object( + candidate_lifecycle.PinnedHarness, "run", return_value=SHA_B + "\n" + ): + with self.assertRaisesRegex(AssertionError, "root fixture ref mismatch"): + candidate_lifecycle.PinnedHarness(args) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_candidate_procedure.py b/tests/test_candidate_procedure.py new file mode 100644 index 00000000..fee45b62 --- /dev/null +++ b/tests/test_candidate_procedure.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +import procedure_contract # noqa: E402 + +SHA = "a" * 40 + +VALID_WORKFLOW = f'''name: candidate +on: + workflow_call: +permissions: + contents: read +jobs: + contract: + steps: + - run: | + [[ "$ZED_CLI_REF" =~ ^[0-9a-f]{{40}}$ ]] + [[ "$HARNESS_REF" =~ ^[0-9a-f]{{40}}$ ]] + [[ "$ZED_INTERFACES_REF" =~ ^[0-9a-f]{{40}}$ ]] + python3 - <<'PY' + re.fullmatch(r"[0-9a-f]{{40}}", ref) + fixture_refs=refs + PY + build: + steps: + - uses: actions/checkout@{SHA} + with: + ref: ${{{{ env.ZED_CLI_REF }}}} + persist-credentials: false + - uses: actions/checkout@{SHA} + with: + ref: ${{{{ env.ZED_INTERFACES_REF }}}} + persist-credentials: false + - uses: actions/checkout@{SHA} + with: + ref: ${{{{ env.HARNESS_REF }}}} + persist-credentials: false + - uses: actions/checkout@{SHA} + with: + ref: ${{{{ matrix.fixture.ref }}}} + persist-credentials: false + - run: | + python3 scripts/candidate_lifecycle.py --fixture-refs-json "$FIXTURE_REFS" + sha256sum --check SHA256SUMS +''' + +VALID_WRAPPER = ''' +if not re.fullmatch(r"[0-9a-f]{40}", ref): + raise ValueError("root fixture") +raise ValueError("dependency fixture") +command = [ + "fetch", + "--depth", + "1", + "--no-tags", +] +checkout = ["checkout", "--detach", "FETCH_HEAD"] +verify = ["rev-parse", "HEAD"] +''' + +VALID_DOCS = ''' +# Two gates, two purposes +exact 40-character `zed-cli` commit +exact 40-character `zed-pkg-e2e` harness commit +never follows a fixture default branch +## Full candidate certification +.github/workflows/lifecycle.yml +.github/workflows/e2e.yml +.github/workflows/install-boundaries.yml +same immutable candidate and dependency graph +zed-pkg-test certification runbook +product regression, fixture drift, harness defect, or infrastructure failure +''' + + +class WorkflowPolicyTests(unittest.TestCase): + def test_valid_contract_passes(self) -> None: + procedure_contract.audit_workflow(VALID_WORKFLOW) + + def test_mutable_action_ref_is_rejected(self) -> None: + with self.assertRaisesRegex(procedure_contract.ContractViolation, "exact commit"): + procedure_contract.audit_workflow( + VALID_WORKFLOW.replace(f"actions/checkout@{SHA}", "actions/checkout@v7", 1) + ) + + def test_write_permission_is_rejected(self) -> None: + with self.assertRaisesRegex(procedure_contract.ContractViolation, "contents: read"): + procedure_contract.audit_workflow( + VALID_WORKFLOW.replace("contents: read", "contents: write") + ) + + def test_secret_inheritance_is_rejected(self) -> None: + with self.assertRaisesRegex(procedure_contract.ContractViolation, "secrets"): + procedure_contract.audit_workflow(VALID_WORKFLOW + "\nsecrets: inherit\n") + + def test_persisted_checkout_credentials_are_rejected(self) -> None: + with self.assertRaisesRegex(procedure_contract.ContractViolation, "credentials"): + procedure_contract.audit_workflow( + VALID_WORKFLOW.replace("persist-credentials: false", "persist-credentials: true", 1) + ) + + def test_missing_exact_fixture_validation_is_rejected(self) -> None: + with self.assertRaisesRegex(procedure_contract.ContractViolation, "fullmatch"): + procedure_contract.audit_workflow( + VALID_WORKFLOW.replace('re.fullmatch(r"[0-9a-f]{40}", ref)', "pass") + ) + + +class WrapperAndDocumentationTests(unittest.TestCase): + def test_valid_wrapper_and_docs_pass(self) -> None: + procedure_contract.audit_wrapper(VALID_WRAPPER) + procedure_contract.audit_documentation(VALID_DOCS) + + def test_mutable_clone_boundary_is_rejected(self) -> None: + with self.assertRaises(procedure_contract.ContractViolation): + procedure_contract.audit_wrapper(VALID_WRAPPER + '\nref: main\n') + + def test_missing_failure_classification_is_rejected(self) -> None: + with self.assertRaises(procedure_contract.ContractViolation): + procedure_contract.audit_documentation( + VALID_DOCS.replace( + "product regression, fixture drift, harness defect, or infrastructure failure", + "failure", + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_formal_app_lifecycle_contract.py b/tests/test_formal_app_lifecycle_contract.py new file mode 100644 index 00000000..99c9093a --- /dev/null +++ b/tests/test_formal_app_lifecycle_contract.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from verify_formal_app_lifecycle_contract import ( # noqa: E402 + ContractError, + EXPECTED_EVENTS, + EXPECTED_OUTCOMES, + EXPECTED_PHASES, + EXPECTED_WITNESSES, + verify, +) + + +def write(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +class FormalAppLifecycleContractTest(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.root = Path(self._tmp.name) + + witness_lines = "\n".join(f' "{value}",' for value in sorted(EXPECTED_WITNESSES)) + write( + self.root / "formal/app-lifecycle.fm.toml", + 'schema_version = 1\nproject = "zed-sync"\nmodel = "app-lifecycle-v1"\n' + 'language = "quint"\nspec = "formal/app_lifecycle.qnt"\n' + 'main = "app_lifecycle"\ninvariants = ["app_lifecycle_safety"]\n' + f"witnesses = [\n{witness_lines}\n]\n\n" + '[toolchain]\nquint = "0.32.0"\n\n' + '[verification]\nbackend = "tlc"\nexhaustive_finite_model = true\n', + ) + symbols = "\n".join(sorted({"app_lifecycle_safety", *EXPECTED_WITNESSES})) + write(self.root / "formal/app_lifecycle.qnt", symbols) + + events = sorted(EXPECTED_EVENTS) + outcomes = sorted(EXPECTED_OUTCOMES) + phases = sorted(EXPECTED_PHASES) + steps = [] + for index, event in enumerate(events): + steps.append({ + "event": {"type": event}, + "outcome": outcomes[index % len(outcomes)], + "state": { + "phase": phases[index % len(phases)], + "operation": "none", + "generation": index, + "active_token": None, + "desired_running": False, + "online": False, + "failure": None, + }, + }) + cases = [ + {"name": f"case-{index}", "steps": steps[index:index + 2]} + for index in range(5) + ] + cases.append({"name": "case-5", "steps": steps[5:]}) + write( + self.root / "protocol/formal-app-lifecycle.json", + json.dumps({"schema_version": 1, "model": "app-lifecycle-v1", "cases": cases}), + ) + write( + self.root / "protocol/formal-app-lifecycle.schema.json", + json.dumps({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "event": {"properties": {"type": {"enum": sorted(EXPECTED_EVENTS)}}}, + "step": {"properties": {"outcome": {"enum": sorted(EXPECTED_OUTCOMES)}}}, + "state": {"properties": {"phase": {"enum": sorted(EXPECTED_PHASES)}}}, + }, + }), + ) + for implementation, refinement in ( + ("src/lifecycle.rs", "tests/formal_app_lifecycle.rs"), + ("sdk/src/lifecycle.mjs", "sdk/test/formal-app-lifecycle.test.mjs"), + ("dart/zed_sync/lib/src/lifecycle.dart", "dart/zed_sync/test/formal_app_lifecycle_test.dart"), + ): + write(self.root / implementation, "lifecycle reducer\n") + write(self.root / refinement, "formal-app-lifecycle.json\n") + + def test_complete_cross_runtime_contract_passes(self) -> None: + verify(self.root) + + def test_missing_runtime_refinement_fails_closed(self) -> None: + (self.root / "dart/zed_sync/test/formal_app_lifecycle_test.dart").unlink() + with self.assertRaisesRegex(ContractError, "missing Dart lifecycle refinement"): + verify(self.root) + + def test_incomplete_outcome_coverage_fails_closed(self) -> None: + path = self.root / "protocol/formal-app-lifecycle.json" + fixture = json.loads(path.read_text(encoding="utf-8")) + for case in fixture["cases"]: + for step in case["steps"]: + if step["outcome"] == "stale": + step["outcome"] = "applied" + path.write_text(json.dumps(fixture), encoding="utf-8") + with self.assertRaisesRegex(ContractError, "outcome coverage drifted"): + verify(self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lifecycle_metadata.py b/tests/test_lifecycle_metadata.py new file mode 100644 index 00000000..a5abc0a0 --- /dev/null +++ b/tests/test_lifecycle_metadata.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "lifecycle.py" +SPEC = importlib.util.spec_from_file_location("zed_lifecycle", SCRIPT) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"unable to load lifecycle harness from {SCRIPT}") +LIFECYCLE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = LIFECYCLE +SPEC.loader.exec_module(LIFECYCLE) + + +class LifecycleMetadataTests(unittest.TestCase): + def test_shared_schema_is_a_publishable_fixture_and_dependency_source(self) -> None: + self.assertNotIn("shared-schema", LIFECYCLE.NON_PACKAGE_REPOS) + self.assertEqual( + LIFECYCLE.PACKAGE_SOURCES["zedtest/shared-schema"], + ("shared-schema", "."), + ) + + def test_orchestrator_is_the_only_non_package_repository(self) -> None: + self.assertEqual(LIFECYCLE.NON_PACKAGE_REPOS, {"zed-pkg-e2e"}) + + def test_dependency_sections_are_combined_deduplicated_and_sorted(self) -> None: + manifest = { + "dependencies": { + "zedtest/shared-schema": "^1", + "zed-pkg-test/python-lib": "^1", + }, + "build_dependencies": { + "zedtest/shared-schema": "^1", + "zed-pkg-test/go-lib": "^1", + }, + "build-dependencies": { + "zed-pkg-test/rust-lib": "^1", + }, + } + self.assertEqual( + LIFECYCLE.manifest_dependencies(manifest), + [ + "zed-pkg-test/go-lib", + "zed-pkg-test/python-lib", + "zed-pkg-test/rust-lib", + "zedtest/shared-schema", + ], + ) + + def test_every_dependency_source_is_a_repository_path_pair(self) -> None: + for package, source in LIFECYCLE.PACKAGE_SOURCES.items(): + with self.subTest(package=package): + self.assertRegex(package, r"^[^/]+/[^/]+$") + self.assertIsInstance(source, tuple) + self.assertEqual(len(source), 2) + self.assertTrue(all(isinstance(value, str) and value for value in source)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_lifecycle_source_map.py b/tests/test_lifecycle_source_map.py new file mode 100755 index 00000000..e2f3626c --- /dev/null +++ b/tests/test_lifecycle_source_map.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Contract tests for lifecycle package sources and transient outputs.""" + +from __future__ import annotations + +import importlib.util +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import ModuleType + +ROOT = Path(__file__).resolve().parents[1] +LIFECYCLE_PATH = ROOT / "scripts" / "lifecycle.py" +FIXTURE_ROOT = os.environ.get("FIXTURE_ROOT") + + +def load_lifecycle() -> ModuleType: + spec = importlib.util.spec_from_file_location("zed_lifecycle_contract", LIFECYCLE_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load lifecycle module from {LIFECYCLE_PATH}") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +lifecycle = load_lifecycle() + + +class SourceMapShapeTests(unittest.TestCase): + def test_shared_schema_is_a_package_source(self) -> None: + self.assertNotIn("shared-schema", lifecycle.NON_PACKAGE_REPOS) + self.assertEqual( + lifecycle.PACKAGE_SOURCES.get("zedtest/shared-schema"), + ("shared-schema", "."), + ) + + def test_package_repositories_are_not_classified_as_non_packages(self) -> None: + conflicts = { + package: repo + for package, (repo, _relative) in lifecycle.PACKAGE_SOURCES.items() + if repo in lifecycle.NON_PACKAGE_REPOS + } + self.assertEqual(conflicts, {}) + + def test_source_map_keys_and_paths_are_normalized(self) -> None: + for package, (repo, relative) in lifecycle.PACKAGE_SOURCES.items(): + with self.subTest(package=package): + self.assertRegex(package, r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") + self.assertRegex(repo, r"^[A-Za-z0-9_.-]+$") + self.assertFalse(Path(relative).is_absolute()) + self.assertNotIn("..", Path(relative).parts) + + +class TransientPackCleanupTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.git("init", "-q") + self.git("config", "user.name", "Lifecycle Contract") + self.git("config", "user.email", "lifecycle-contract@example.invalid") + (self.root / "tracked.txt").write_text("tracked\n", encoding="utf-8") + self.git("add", "tracked.txt") + self.git("commit", "-qm", "initial fixture") + self.package = lifecycle.PackageRef("zedtest/shared-schema", "1.0.0") + + def tearDown(self) -> None: + self.temporary.cleanup() + + def git(self, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=self.root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + + def pack_path(self, name: str) -> Path: + path = self.root / ".zed" / "pack" / name + path.parent.mkdir(parents=True, exist_ok=True) + return path + + def test_archive_name_matches_the_cli_package_identity_convention(self) -> None: + self.assertEqual( + lifecycle.publish_archive_name(self.package), + "zedtest-shared-schema-1.0.0.tar.gz", + ) + + def test_removes_only_the_expected_untracked_publish_archive(self) -> None: + archive = self.pack_path(lifecycle.publish_archive_name(self.package)) + archive.write_bytes(b"archive") + + lifecycle.remove_transient_pack_outputs(self.root, [self.package]) + + self.assertFalse(archive.exists()) + status = self.git("status", "--porcelain=v1", "--untracked-files=all") + self.assertEqual(status.stdout, "") + + def test_refuses_an_unexpected_untracked_pack_file(self) -> None: + unexpected = self.pack_path("unexpected.tar.gz") + unexpected.write_bytes(b"unexpected") + + with self.assertRaisesRegex(AssertionError, "unexpected pack output"): + lifecycle.remove_transient_pack_outputs(self.root, [self.package]) + + self.assertTrue(unexpected.is_file()) + + def test_refuses_to_delete_a_tracked_pack_archive(self) -> None: + archive = self.pack_path(lifecycle.publish_archive_name(self.package)) + archive.write_bytes(b"tracked archive") + self.git("add", archive.relative_to(self.root).as_posix()) + self.git("commit", "-qm", "track archive") + archive.write_bytes(b"modified archive") + + with self.assertRaisesRegex(AssertionError, "tracked or modified"): + lifecycle.remove_transient_pack_outputs(self.root, [self.package]) + + self.assertEqual(archive.read_bytes(), b"modified archive") + + +@unittest.skipUnless(FIXTURE_ROOT, "FIXTURE_ROOT is required for live fixture checks") +class LiveFixtureContractTests(unittest.TestCase): + fixture_root = Path(FIXTURE_ROOT or ".").resolve() + + def source_for_dependency(self, dependency: str) -> Path: + try: + repo, relative = lifecycle.PACKAGE_SOURCES[dependency] + except KeyError as error: + self.fail(f"dependency {dependency!r} has no PACKAGE_SOURCES entry: {error}") + source = (self.fixture_root / repo / relative).resolve() + self.assertTrue( + source.is_relative_to(self.fixture_root), + f"mapped source escapes fixture root: {source}", + ) + self.assertTrue((source / ".zpkg.toml").is_file(), source) + return source + + def test_python_app_dependencies_resolve_to_matching_package_outputs(self) -> None: + python_app = self.fixture_root / "python-app" + manifest = lifecycle.read_manifest(python_app) + dependencies = lifecycle.manifest_dependencies(manifest) + self.assertIn("zedtest/shared-schema", dependencies) + self.assertIn("zedtest/polyglot-lib-python", dependencies) + + for dependency in dependencies: + with self.subTest(dependency=dependency): + source = self.source_for_dependency(dependency) + outputs = { + package.full_name + for package in lifecycle.expected_packages( + lifecycle.read_manifest(source) + ) + } + self.assertIn(dependency, outputs) + + def test_shared_schema_manifest_identity_matches_source_map_key(self) -> None: + source = self.source_for_dependency("zedtest/shared-schema") + outputs = lifecycle.expected_packages(lifecycle.read_manifest(source)) + self.assertEqual( + [(package.full_name, package.version) for package in outputs], + [("zedtest/shared-schema", "1.0.0")], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_org_fixture_inventory.py b/tests/test_org_fixture_inventory.py new file mode 100644 index 00000000..9f1da9a4 --- /dev/null +++ b/tests/test_org_fixture_inventory.py @@ -0,0 +1,450 @@ +from __future__ import annotations + +import base64 +import importlib.util +import json +import sys +import tempfile +import unittest +import urllib.error +from io import BytesIO +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "org_fixture_inventory.py" +SPEC = importlib.util.spec_from_file_location("org_fixture_inventory", SCRIPT) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"cannot load inventory module from {SCRIPT}") +inventory = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = inventory +SPEC.loader.exec_module(inventory) + + +def make_inventory_payload( + repositories: list[dict[str, object]] | None = None, + *, + orchestrator: str = "zed-pkg-e2e", +) -> dict[str, object]: + return { + "schema": inventory.INVENTORY_SCHEMA, + "organization": "zed-pkg-test", + "orchestrator": orchestrator, + "repositories": repositories + or [ + {"name": "node-lib", "kind": "package", "manifest": ".zpkg.toml"}, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": None, + }, + ], + } + + +def load_payload(payload: dict[str, object]) -> inventory.Inventory: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "inventory.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return inventory.load_inventory(path) + + +class FakeResponse: + def __init__(self, payload: object) -> None: + self.payload = json.dumps(payload).encode("utf-8") + + def __enter__(self) -> "FakeResponse": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return self.payload + + +class InventoryParsingTests(unittest.TestCase): + def test_loads_valid_inventory(self) -> None: + loaded = load_payload(make_inventory_payload()) + self.assertEqual(loaded.organization, "zed-pkg-test") + self.assertEqual(loaded.orchestrator, "zed-pkg-e2e") + self.assertEqual(loaded.names, ("node-lib", "zed-pkg-e2e")) + self.assertEqual(loaded.package_names, ("node-lib",)) + + def test_rejects_unknown_schema(self) -> None: + payload = make_inventory_payload() + payload["schema"] = "unknown" + with self.assertRaisesRegex(inventory.InventoryError, "schema"): + load_payload(payload) + + def test_rejects_duplicate_repository_names(self) -> None: + entries = [ + {"name": "node-lib", "kind": "package", "manifest": ".zpkg.toml"}, + {"name": "node-lib", "kind": "package", "manifest": ".zpkg.toml"}, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": None, + }, + ] + with self.assertRaisesRegex(inventory.InventoryError, "duplicate"): + load_payload(make_inventory_payload(entries)) + + def test_rejects_invalid_repository_names(self) -> None: + entries = [ + {"name": "../node-lib", "kind": "package", "manifest": ".zpkg.toml"}, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": None, + }, + ] + with self.assertRaisesRegex(inventory.InventoryError, "invalid repository"): + load_payload(make_inventory_payload(entries)) + + def test_rejects_unknown_repository_kind(self) -> None: + entries = [ + {"name": "node-lib", "kind": "sample", "manifest": ".zpkg.toml"}, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": None, + }, + ] + with self.assertRaisesRegex(inventory.InventoryError, "invalid kind"): + load_payload(make_inventory_payload(entries)) + + def test_rejects_package_without_manifest(self) -> None: + entries = [ + {"name": "node-lib", "kind": "package", "manifest": None}, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": None, + }, + ] + with self.assertRaisesRegex(inventory.InventoryError, "manifest"): + load_payload(make_inventory_payload(entries)) + + def test_rejects_manifest_path_escape(self) -> None: + entries = [ + {"name": "node-lib", "kind": "package", "manifest": "../.zpkg.toml"}, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": None, + }, + ] + with self.assertRaisesRegex(inventory.InventoryError, "escapes"): + load_payload(make_inventory_payload(entries)) + + def test_rejects_orchestrator_with_manifest(self) -> None: + entries = [ + {"name": "node-lib", "kind": "package", "manifest": ".zpkg.toml"}, + { + "name": "zed-pkg-e2e", + "kind": "orchestrator", + "manifest": ".zpkg.toml", + }, + ] + with self.assertRaisesRegex(inventory.InventoryError, "must be null"): + load_payload(make_inventory_payload(entries)) + + def test_rejects_mismatched_orchestrator_classification(self) -> None: + entries = [ + {"name": "node-lib", "kind": "package", "manifest": ".zpkg.toml"}, + { + "name": "other-orchestrator", + "kind": "orchestrator", + "manifest": None, + }, + ] + with self.assertRaisesRegex(inventory.InventoryError, "exactly"): + load_payload(make_inventory_payload(entries)) + + +class LifecycleMatrixParsingTests(unittest.TestCase): + def test_extracts_repo_matrix_in_declared_order(self) -> None: + workflow = """ +jobs: + lifecycle: + strategy: + matrix: + repo: + - node-lib + - rust-lib + - zed-pkg-e2e + steps: + - run: echo ok +""" + self.assertEqual( + inventory.extract_lifecycle_repositories(workflow), + ("node-lib", "rust-lib", "zed-pkg-e2e"), + ) + + def test_rejects_missing_repo_matrix(self) -> None: + with self.assertRaisesRegex(inventory.InventoryError, "cannot find"): + inventory.extract_lifecycle_repositories("jobs: {}\n") + + def test_rejects_duplicate_repo_matrix_entries(self) -> None: + workflow = """ +jobs: + lifecycle: + strategy: + matrix: + repo: + - node-lib + - node-lib +""" + with self.assertRaisesRegex(inventory.InventoryError, "duplicates"): + inventory.extract_lifecycle_repositories(workflow) + + def test_rejects_dynamic_repo_matrix_entries(self) -> None: + workflow = """ +jobs: + lifecycle: + strategy: + matrix: + repo: + - node-lib + - ${{ fromJSON(needs.plan.outputs.repos) }} +""" + with self.assertRaisesRegex(inventory.InventoryError, "unsupported"): + inventory.extract_lifecycle_repositories(workflow) + + +class StaticInventoryValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.loaded = load_payload(make_inventory_payload()) + self.sources = {"zed-pkg-test/node-lib": ("node-lib", ".")} + + def test_accepts_matching_inventory_matrix_and_sources(self) -> None: + errors = inventory.validate_static_inventory( + self.loaded, + self.loaded.names, + self.sources, + {"zed-pkg-e2e"}, + ) + self.assertEqual(errors, []) + + def test_reports_fixture_missing_from_matrix(self) -> None: + errors = inventory.validate_static_inventory( + self.loaded, + ("node-lib",), + self.sources, + {"zed-pkg-e2e"}, + ) + self.assertTrue(any("missing from lifecycle" in error for error in errors)) + + def test_reports_unknown_matrix_fixture(self) -> None: + errors = inventory.validate_static_inventory( + self.loaded, + (*self.loaded.names, "unknown"), + self.sources, + {"zed-pkg-e2e"}, + ) + self.assertTrue(any("unknown repositories" in error for error in errors)) + + def test_reports_non_package_classification_drift(self) -> None: + errors = inventory.validate_static_inventory( + self.loaded, + self.loaded.names, + self.sources, + {"node-lib"}, + ) + self.assertTrue(any("NON_PACKAGE_REPOS drift" in error for error in errors)) + + def test_reports_unknown_package_source_repository(self) -> None: + errors = inventory.validate_static_inventory( + self.loaded, + self.loaded.names, + {"zedtest/ghost": ("ghost", ".")}, + {"zed-pkg-e2e"}, + ) + self.assertTrue(any("unclassified" in error for error in errors)) + + def test_reports_escaping_package_source_path(self) -> None: + errors = inventory.validate_static_inventory( + self.loaded, + self.loaded.names, + {"zed-pkg-test/node-lib": ("node-lib", "../outside")}, + {"zed-pkg-e2e"}, + ) + self.assertTrue(any("escapes repository" in error for error in errors)) + + def test_reports_orchestrator_as_package_source(self) -> None: + errors = inventory.validate_static_inventory( + self.loaded, + self.loaded.names, + {"zedtest/bad": ("zed-pkg-e2e", ".")}, + {"zed-pkg-e2e"}, + ) + self.assertTrue(any("orchestrator" in error for error in errors)) + + +class LiveRepositoryValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.loaded = load_payload(make_inventory_payload()) + self.good = ( + inventory.LiveRepository( + "node-lib", False, False, False, False, "main" + ), + inventory.LiveRepository( + "zed-pkg-e2e", False, False, False, False, "main" + ), + ) + + def test_accepts_matching_live_repositories(self) -> None: + self.assertEqual( + inventory.validate_live_repositories(self.loaded, self.good), + [], + ) + + def test_reports_missing_live_repository(self) -> None: + errors = inventory.validate_live_repositories(self.loaded, self.good[:1]) + self.assertTrue(any("missing from org" in error for error in errors)) + + def test_reports_unclassified_live_repository(self) -> None: + extra = inventory.LiveRepository( + "new-fixture", False, False, False, False, "main" + ) + errors = inventory.validate_live_repositories( + self.loaded, (*self.good, extra) + ) + self.assertTrue(any("unclassified live" in error for error in errors)) + + def test_reports_private_repository(self) -> None: + bad = ( + inventory.LiveRepository("node-lib", True, False, False, False, "main"), + self.good[1], + ) + errors = inventory.validate_live_repositories(self.loaded, bad) + self.assertTrue(any("must be public" in error for error in errors)) + + def test_reports_archived_repository(self) -> None: + bad = ( + inventory.LiveRepository("node-lib", False, True, False, False, "main"), + self.good[1], + ) + errors = inventory.validate_live_repositories(self.loaded, bad) + self.assertTrue(any("archived" in error for error in errors)) + + def test_reports_disabled_repository(self) -> None: + bad = ( + inventory.LiveRepository("node-lib", False, False, True, False, "main"), + self.good[1], + ) + errors = inventory.validate_live_repositories(self.loaded, bad) + self.assertTrue(any("disabled" in error for error in errors)) + + def test_reports_fork_repository(self) -> None: + bad = ( + inventory.LiveRepository("node-lib", False, False, False, True, "main"), + self.good[1], + ) + errors = inventory.validate_live_repositories(self.loaded, bad) + self.assertTrue(any("must not be a fork" in error for error in errors)) + + def test_reports_non_main_default_branch(self) -> None: + bad = ( + inventory.LiveRepository( + "node-lib", False, False, False, False, "master" + ), + self.good[1], + ) + errors = inventory.validate_live_repositories(self.loaded, bad) + self.assertTrue(any("default branch" in error for error in errors)) + + +class ManifestProbeTests(unittest.TestCase): + def manifest_payload(self, content: str) -> dict[str, object]: + encoded = base64.b64encode(content.encode("utf-8")).decode("ascii") + return {"type": "file", "encoding": "base64", "content": encoded} + + def test_decodes_valid_package_manifest(self) -> None: + manifest = inventory.decode_manifest_payload( + "node-lib", + ".zpkg.toml", + self.manifest_payload( + '[package]\norg="zed-pkg-test"\nname="node-lib"\nversion="1.0.0"\n' + ), + ) + self.assertEqual(manifest["package"]["name"], "node-lib") + + def test_rejects_manifest_without_package_table(self) -> None: + with self.assertRaisesRegex(inventory.InventoryError, r"\[package\]"): + inventory.decode_manifest_payload( + "node-lib", + ".zpkg.toml", + self.manifest_payload("[workspace]\nmembers=[]\n"), + ) + + def test_rejects_invalid_toml(self) -> None: + with self.assertRaisesRegex(inventory.InventoryError, "invalid TOML"): + inventory.decode_manifest_payload( + "node-lib", + ".zpkg.toml", + self.manifest_payload("[package\n"), + ) + + def test_fetch_manifest_probe_reports_identity(self) -> None: + payload = self.manifest_payload( + '[package]\norg="zedtest"\nname="node-lib"\nversion="2.3.4"\n' + ) + opener = lambda _request: FakeResponse(payload) + probe = inventory.fetch_manifest_probe( + "zed-pkg-test", + inventory.RepositorySpec("node-lib", "package", ".zpkg.toml"), + opener=opener, + ) + self.assertEqual(probe.package, "zedtest/node-lib") + self.assertEqual(probe.version, "2.3.4") + self.assertTrue(probe.present) + + def test_fetch_json_wraps_http_errors(self) -> None: + def opener(request: object) -> object: + raise urllib.error.HTTPError( + "https://api.github.test", + 403, + "forbidden", + {}, + BytesIO(b'{"message":"rate limited"}'), + ) + + with self.assertRaisesRegex(inventory.InventoryError, "403"): + inventory.fetch_json( + "https://api.github.test", + opener=opener, + ) + + +class CheckedInContractTests(unittest.TestCase): + def test_checked_in_inventory_matches_lifecycle_contract(self) -> None: + loaded = inventory.load_inventory(ROOT / "fixtures" / "org-repositories.json") + matrix = inventory.extract_lifecycle_repositories( + (ROOT / ".github" / "workflows" / "lifecycle.yml").read_text( + encoding="utf-8" + ) + ) + lifecycle = inventory.load_lifecycle_module(ROOT / "scripts" / "lifecycle.py") + errors = inventory.validate_static_inventory( + loaded, + matrix, + lifecycle.PACKAGE_SOURCES, + lifecycle.NON_PACKAGE_REPOS, + ) + self.assertEqual(errors, []) + + def test_checked_in_inventory_has_twenty_two_unique_repositories(self) -> None: + loaded = inventory.load_inventory(ROOT / "fixtures" / "org-repositories.json") + self.assertEqual(len(loaded.names), 22) + self.assertEqual(len(set(loaded.names)), 22) + self.assertEqual(len(loaded.package_names), 21) + + def test_orchestrator_is_the_only_manifestless_repository(self) -> None: + loaded = inventory.load_inventory(ROOT / "fixtures" / "org-repositories.json") + manifestless = [repo.name for repo in loaded.repositories if repo.manifest is None] + self.assertEqual(manifestless, ["zed-pkg-e2e"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_org_fleet_contract.py b/tests/test_org_fleet_contract.py new file mode 100644 index 00000000..2507bd02 --- /dev/null +++ b/tests/test_org_fleet_contract.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +import org_fleet_contract as contract +import org_fleet_inventory as fleet + + +class PackageOrganizationTests(unittest.TestCase): + def write_inventory(self, package_organizations: object = None) -> Path: + raw: dict[str, object] = { + "organization": "zed-pkg-test", + "package_organizations": ( + ["zed-pkg-test", "zedtest"] + if package_organizations is None + else package_organizations + ), + } + temp = tempfile.NamedTemporaryFile("w", delete=False, encoding="utf-8") + json.dump(raw, temp) + temp.close() + path = Path(temp.name) + self.addCleanup(path.unlink) + return path + + def test_loads_reviewed_package_organizations_in_order(self): + result = contract.load_package_organizations( + self.write_inventory(), "zed-pkg-test" + ) + self.assertEqual(result, ("zed-pkg-test", "zedtest")) + + def test_rejects_missing_package_organizations(self): + path = self.write_inventory([]) + with self.assertRaisesRegex(fleet.FleetError, "non-empty array"): + contract.load_package_organizations(path, "zed-pkg-test") + + def test_rejects_non_string_package_organization(self): + path = self.write_inventory(["zed-pkg-test", 42]) + with self.assertRaisesRegex(fleet.FleetError, "non-empty string"): + contract.load_package_organizations(path, "zed-pkg-test") + + def test_rejects_duplicate_package_organization(self): + path = self.write_inventory(["zed-pkg-test", "zed-pkg-test"]) + with self.assertRaisesRegex(fleet.FleetError, "duplicate"): + contract.load_package_organizations(path, "zed-pkg-test") + + def test_requires_github_organization_in_reviewed_namespaces(self): + path = self.write_inventory(["zedtest"]) + with self.assertRaisesRegex(fleet.FleetError, "must include"): + contract.load_package_organizations(path, "zed-pkg-test") + + +class AliasAwareAuditTests(unittest.TestCase): + def core(self) -> dict[str, fleet.CoreSpec]: + return { + "legacy-lib": fleet.CoreSpec("legacy-lib", "package", ".zpkg.toml"), + "zed-pkg-e2e": fleet.CoreSpec("zed-pkg-e2e", "orchestrator", None), + } + + def repositories(self) -> list[fleet.LiveRepository]: + return [ + fleet.LiveRepository("legacy-lib"), + fleet.LiveRepository("zed-pkg-e2e"), + ] + + def loader_for(self, package_org: str): + def load(repo: fleet.LiveRepository, path: str) -> fleet.ManifestProbe: + if repo.name == "zed-pkg-e2e": + return fleet.ManifestProbe(repo.name, path, False) + return fleet.ManifestProbe( + repo.name, + path, + True, + package_org, + repo.name, + "1.0.0", + ) + + return load + + def test_accepts_legacy_namespace_and_preserves_it_in_evidence(self): + report = contract.audit_with_package_organizations( + "zed-pkg-test", + ("zed-pkg-test", "zedtest"), + self.core(), + self.repositories(), + self.loader_for("zedtest"), + ) + self.assertEqual(report["errors"], []) + self.assertEqual(report["schema"], contract.REPORT_SCHEMA) + self.assertEqual( + report["package_organizations"], ["zed-pkg-test", "zedtest"] + ) + legacy = next(row for row in report["repositories"] if row["name"] == "legacy-lib") + self.assertEqual(legacy["manifest"]["package_org"], "zedtest") + self.assertTrue(legacy["manifest"]["package_org_allowed"]) + + def test_accepts_current_namespace(self): + report = contract.audit_with_package_organizations( + "zed-pkg-test", + ("zed-pkg-test", "zedtest"), + self.core(), + self.repositories(), + self.loader_for("zed-pkg-test"), + ) + self.assertEqual(report["errors"], []) + + def test_rejects_unreviewed_namespace(self): + report = contract.audit_with_package_organizations( + "zed-pkg-test", + ("zed-pkg-test", "zedtest"), + self.core(), + self.repositories(), + self.loader_for("unreviewed"), + ) + self.assertTrue(any("package.org mismatch" in error for error in report["errors"])) + legacy = next(row for row in report["repositories"] if row["name"] == "legacy-lib") + self.assertEqual(legacy["manifest"]["package_org"], "unreviewed") + self.assertFalse(legacy["manifest"]["package_org_allowed"]) + + def test_manifestless_orchestrator_has_no_namespace_decision(self): + report = contract.audit_with_package_organizations( + "zed-pkg-test", + ("zed-pkg-test", "zedtest"), + self.core(), + self.repositories(), + self.loader_for("zedtest"), + ) + orchestrator = next( + row for row in report["repositories"] if row["name"] == "zed-pkg-e2e" + ) + self.assertIsNone(orchestrator["manifest"]["package_org_allowed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_org_fleet_inventory.py b/tests/test_org_fleet_inventory.py new file mode 100644 index 00000000..aa78f46e --- /dev/null +++ b/tests/test_org_fleet_inventory.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import base64 +import io +import json +import tempfile +import unittest +import urllib.error +from pathlib import Path + +import org_fleet_inventory as fleet + + +def core_inventory() -> dict[str, fleet.CoreSpec]: + return { + "node-lib": fleet.CoreSpec("node-lib", "package", ".zpkg.toml"), + "zed-pkg-e2e": fleet.CoreSpec("zed-pkg-e2e", "orchestrator", None), + } + + +def probe( + repository: fleet.LiveRepository, + path: str, +) -> fleet.ManifestProbe: + if repository.name in {"zed-pkg-e2e", ".github", "api-contract-e2e"}: + return fleet.ManifestProbe(repository.name, path, False) + return fleet.ManifestProbe( + repository.name, + path, + True, + "zed-pkg-test", + repository.name, + "0.1.0", + ) + + +class FakeResponse: + def __init__(self, payload: object): + self._payload = json.dumps(payload).encode() + self.headers: dict[str, str] = {} + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self) -> bytes: + return self._payload + + +class CoreInventoryTests(unittest.TestCase): + def write_inventory(self, raw: object) -> Path: + temp = tempfile.NamedTemporaryFile("w", delete=False, encoding="utf-8") + json.dump(raw, temp) + temp.close() + self.addCleanup(Path(temp.name).unlink) + return Path(temp.name) + + def valid_raw(self) -> dict[str, object]: + return { + "schema": fleet.CORE_SCHEMA, + "organization": "zed-pkg-test", + "orchestrator": "zed-pkg-e2e", + "repositories": [ + {"name": "node-lib", "kind": "package", "manifest": ".zpkg.toml"}, + {"name": "zed-pkg-e2e", "kind": "orchestrator", "manifest": None}, + ], + } + + def test_loads_valid_core_inventory(self): + organization, result = fleet.load_core_inventory(self.write_inventory(self.valid_raw())) + self.assertEqual(organization, "zed-pkg-test") + self.assertTrue(result["node-lib"].requires_manifest) + self.assertFalse(result["zed-pkg-e2e"].requires_manifest) + + def test_rejects_unknown_schema(self): + raw = self.valid_raw() + raw["schema"] = "wrong" + with self.assertRaisesRegex(fleet.FleetError, "schema"): + fleet.load_core_inventory(self.write_inventory(raw)) + + def test_rejects_duplicate_repository(self): + raw = self.valid_raw() + raw["repositories"].append(raw["repositories"][0]) + with self.assertRaisesRegex(fleet.FleetError, "duplicate"): + fleet.load_core_inventory(self.write_inventory(raw)) + + def test_rejects_package_without_manifest(self): + raw = self.valid_raw() + raw["repositories"][0]["manifest"] = None + with self.assertRaisesRegex(fleet.FleetError, "manifest"): + fleet.load_core_inventory(self.write_inventory(raw)) + + def test_rejects_orchestrator_with_manifest(self): + raw = self.valid_raw() + raw["repositories"][1]["manifest"] = ".zpkg.toml" + with self.assertRaisesRegex(fleet.FleetError, "must be null"): + fleet.load_core_inventory(self.write_inventory(raw)) + + def test_rejects_manifest_path_escape(self): + raw = self.valid_raw() + raw["repositories"][0]["manifest"] = "../outside.toml" + with self.assertRaisesRegex(fleet.FleetError, "escapes"): + fleet.load_core_inventory(self.write_inventory(raw)) + + def test_rejects_second_orchestrator(self): + raw = self.valid_raw() + raw["repositories"].append( + {"name": "other", "kind": "orchestrator", "manifest": None} + ) + with self.assertRaisesRegex(fleet.FleetError, "exactly"): + fleet.load_core_inventory(self.write_inventory(raw)) + + +class ClassificationTests(unittest.TestCase): + def test_classifies_core_package(self): + self.assertEqual(fleet.classify_repository("node-lib", core_inventory()), "core-package") + + def test_classifies_core_orchestrator(self): + self.assertEqual( + fleet.classify_repository("zed-pkg-e2e", core_inventory()), + "core-orchestrator", + ) + + def test_classifies_governance_repository(self): + self.assertEqual(fleet.classify_repository(".github", core_inventory()), "governance") + + def test_classifies_e2e_harness(self): + self.assertEqual( + fleet.classify_repository("offline-cache-e2e", core_inventory()), + "harness", + ) + + def test_classifies_contract_harness(self): + self.assertEqual( + fleet.classify_repository("registry-api-contract", core_inventory()), + "harness", + ) + + def test_classifies_other_repositories_as_packages(self): + self.assertEqual( + fleet.classify_repository("csharp-app", core_inventory()), + "supplemental-package", + ) + + def test_harness_manifest_is_optional(self): + path, required = fleet.manifest_requirement( + fleet.LiveRepository("browser-e2e"), core_inventory() + ) + self.assertEqual(path, ".zpkg.toml") + self.assertFalse(required) + + def test_supplemental_package_manifest_is_required(self): + _, required = fleet.manifest_requirement( + fleet.LiveRepository("swift-app"), core_inventory() + ) + self.assertTrue(required) + + +class AuditTests(unittest.TestCase): + def healthy_repositories(self): + return [ + fleet.LiveRepository("node-lib"), + fleet.LiveRepository("zed-pkg-e2e"), + fleet.LiveRepository(".github"), + fleet.LiveRepository("api-contract-e2e"), + fleet.LiveRepository("csharp-app"), + ] + + def test_accepts_core_and_supplemental_fleet(self): + report = fleet.audit_fleet( + "zed-pkg-test", core_inventory(), self.healthy_repositories(), probe + ) + self.assertEqual(report["errors"], []) + self.assertEqual(report["live_repository_count"], 5) + self.assertEqual(report["supplemental_repository_count"], 3) + self.assertEqual(report["required_manifest_count"], 2) + self.assertEqual(report["required_manifest_present_count"], 2) + self.assertEqual(report["manifest_present_count"], 2) + + def test_reports_missing_core_repository(self): + report = fleet.audit_fleet( + "zed-pkg-test", + core_inventory(), + [fleet.LiveRepository("node-lib")], + probe, + ) + self.assertIn("zed-pkg-e2e", report["missing_core_repositories"]) + self.assertTrue(any("missing from public org" in error for error in report["errors"])) + + def test_reports_missing_required_supplemental_manifest(self): + def missing(repo, path): + return fleet.ManifestProbe(repo.name, path, False) + + report = fleet.audit_fleet( + "zed-pkg-test", + core_inventory(), + [ + fleet.LiveRepository("node-lib"), + fleet.LiveRepository("zed-pkg-e2e"), + fleet.LiveRepository("java-app"), + ], + missing, + ) + self.assertTrue(any("java-app: required manifest is missing" in error for error in report["errors"])) + + def test_allows_manifestless_harness(self): + report = fleet.audit_fleet( + "zed-pkg-test", + core_inventory(), + [ + fleet.LiveRepository("node-lib"), + fleet.LiveRepository("zed-pkg-e2e"), + fleet.LiveRepository("offline-cache-e2e"), + ], + probe, + ) + self.assertEqual(report["errors"], []) + + def test_reports_manifest_org_mismatch(self): + def wrong_org(repo, path): + return fleet.ManifestProbe(repo.name, path, True, "wrong", repo.name, "1.0.0") + + report = fleet.audit_fleet( + "zed-pkg-test", core_inventory(), self.healthy_repositories()[:2], wrong_org + ) + self.assertTrue(any("package.org mismatch" in error for error in report["errors"])) + + def test_reports_manifest_name_mismatch(self): + def wrong_name(repo, path): + return fleet.ManifestProbe(repo.name, path, True, "zed-pkg-test", "wrong", "1.0.0") + + report = fleet.audit_fleet( + "zed-pkg-test", core_inventory(), self.healthy_repositories()[:2], wrong_name + ) + self.assertTrue(any("package.name mismatch" in error for error in report["errors"])) + + def test_reports_repository_shape_failures(self): + bad = fleet.LiveRepository( + "node-lib", + private=True, + archived=True, + disabled=True, + fork=True, + default_branch="master", + ) + report = fleet.audit_fleet( + "zed-pkg-test", + core_inventory(), + [bad, fleet.LiveRepository("zed-pkg-e2e")], + probe, + ) + joined = "\n".join(report["errors"]) + for fragment in ( + "must be public", + "must not be archived", + "must not be disabled", + "must not be a fork", + "default branch", + ): + self.assertIn(fragment, joined) + + def test_reports_duplicate_live_repository(self): + report = fleet.audit_fleet( + "zed-pkg-test", + core_inventory(), + [ + fleet.LiveRepository("node-lib"), + fleet.LiveRepository("node-lib"), + fleet.LiveRepository("zed-pkg-e2e"), + ], + probe, + ) + self.assertTrue(any("duplicate live repositories" in error for error in report["errors"])) + + def test_converts_manifest_loader_error_to_repository_error(self): + def broken(_repo, _path): + raise fleet.FleetError("bad manifest payload") + + report = fleet.audit_fleet( + "zed-pkg-test", + core_inventory(), + [fleet.LiveRepository("node-lib"), fleet.LiveRepository("zed-pkg-e2e")], + broken, + ) + self.assertTrue(any("bad manifest payload" in error for error in report["errors"])) + + +class ManifestDecodeTests(unittest.TestCase): + def payload(self, text: str) -> dict[str, object]: + return { + "type": "file", + "encoding": "base64", + "content": base64.b64encode(text.encode()).decode(), + } + + def test_decodes_valid_manifest(self): + result = fleet.decode_manifest( + "java-app", + ".zpkg.toml", + self.payload('[package]\norg="zed-pkg-test"\nname="java-app"\nversion="1.2.3"\n'), + ) + self.assertTrue(result.present) + self.assertEqual(result.package_name, "java-app") + self.assertEqual(result.version, "1.2.3") + + def test_rejects_invalid_toml(self): + with self.assertRaisesRegex(fleet.FleetError, "invalid TOML"): + fleet.decode_manifest("x", ".zpkg.toml", self.payload("[package")) + + def test_rejects_missing_package_table(self): + with self.assertRaisesRegex(fleet.FleetError, r"missing \[package\]"): + fleet.decode_manifest("x", ".zpkg.toml", self.payload("value=1\n")) + + def test_rejects_non_file_payload(self): + with self.assertRaisesRegex(fleet.FleetError, "regular file"): + fleet.decode_manifest( + "x", + ".zpkg.toml", + {"type": "dir", "encoding": "base64", "content": ""}, + ) + + +class HttpTests(unittest.TestCase): + def test_fetches_public_repositories(self): + payload = [ + { + "name": "node-lib", + "private": False, + "archived": False, + "disabled": False, + "fork": False, + "default_branch": "main", + } + ] + result = fleet.fetch_public_repositories( + "zed-pkg-test", opener=lambda _request: FakeResponse(payload) + ) + self.assertEqual(result, (fleet.LiveRepository("node-lib"),)) + + def test_manifest_loader_maps_404_to_absent_probe(self): + def not_found(request): + raise urllib.error.HTTPError( + request.full_url, 404, "not found", {}, io.BytesIO(b"{}") + ) + + loader = fleet.make_github_manifest_loader("zed-pkg-test", opener=not_found) + result = loader(fleet.LiveRepository("missing"), ".zpkg.toml") + self.assertFalse(result.present) + + def test_manifest_loader_surfaces_non_404_http_errors(self): + def forbidden(request): + raise urllib.error.HTTPError( + request.full_url, 403, "forbidden", {}, io.BytesIO(b"{}") + ) + + loader = fleet.make_github_manifest_loader("zed-pkg-test", opener=forbidden) + with self.assertRaisesRegex(fleet.FleetError, "403"): + loader(fleet.LiveRepository("private"), ".zpkg.toml") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/workflow-trigger-references.test.mjs b/tests/workflow-trigger-references.test.mjs new file mode 100644 index 00000000..c9952a22 --- /dev/null +++ b/tests/workflow-trigger-references.test.mjs @@ -0,0 +1,110 @@ +// A `workflow_run` trigger is resolved by GitHub against the `name:` declared +// by workflows on the default branch. A reference to a name no workflow +// declares is accepted silently at push time and simply never fires, so the +// dependent workflow looks configured while never running once. That is exactly +// how `verify-live-test-org-fleet.yml` sat dead: it waited on "run authoritative +// test organization fleet on ARM64", a workflow that only ever existed on the +// unmerged `agent/run-fleet-arm64` branch. +// +// Parsed line-by-line on purpose. The repository carries ~70 workflows and +// pulling in a YAML dependency for two keys is not worth the supply-chain +// surface; the two keys involved (`workflows:` under `workflow_run:`, and the +// top-level `name:`) are unambiguous in block style. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '..'); +const workflowDir = path.join(root, '.github', 'workflows'); + +function indentOf(line) { + return line.length - line.trimStart().length; +} + +function declaredName(lines, file) { + for (const line of lines) { + if (line.startsWith('name:')) return line.slice(5).trim().replace(/^['"]|['"]$/g, ''); + } + return path.basename(file); +} + +// Returns the workflow names referenced by every `workflow_run:` trigger. +function workflowRunReferences(lines) { + const references = []; + for (let i = 0; i < lines.length; i += 1) { + if (!lines[i].includes('workflow_run:')) continue; + const triggerIndent = indentOf(lines[i]); + + for (let j = i + 1; j < lines.length; j += 1) { + const line = lines[j]; + if (line.trim() && indentOf(line) <= triggerIndent) break; + if (!line.includes('workflows:')) continue; + + const inline = line.slice(line.indexOf('workflows:') + 'workflows:'.length).trim(); + if (inline.startsWith('[')) { + for (const entry of inline.replace(/^\[|\]$/g, '').split(',')) { + const value = entry.trim().replace(/^['"]|['"]$/g, ''); + if (value) references.push(value); + } + } else { + for (let k = j + 1; k < lines.length; k += 1) { + if (!lines[k].trim()) continue; + if (!lines[k].trimStart().startsWith('-')) break; + const value = lines[k].trimStart().slice(1).trim().replace(/^['"]|['"]$/g, ''); + if (value) references.push(value); + } + } + break; + } + } + return references; +} + +const files = fs + .readdirSync(workflowDir) + .filter((entry) => entry.endsWith('.yml') || entry.endsWith('.yaml')) + .map((entry) => path.join(workflowDir, entry)); + +const parsed = new Map(files.map((file) => [file, fs.readFileSync(file, 'utf8').split('\n')])); +const declaredNames = new Set([...parsed].map(([file, lines]) => declaredName(lines, file))); +const basenames = new Set(files.map((file) => path.basename(file))); + +test('every workflow_run trigger names a workflow this repository declares', () => { + assert.ok(files.length > 0, 'no workflows found to check'); + + const dangling = []; + for (const [file, lines] of parsed) { + for (const reference of workflowRunReferences(lines)) { + if (!declaredNames.has(reference) && !basenames.has(reference)) { + dangling.push(`${path.basename(file)} waits on '${reference}' — no workflow declares it`); + } + } + } + + assert.deepEqual( + dangling, + [], + `dead workflow_run trigger(s); these workflows can never fire:\n ${dangling.join('\n ')}`, + ); +}); + +test('the live fleet verifier chains from fleet writers that exist on main', () => { + const verifier = path.join(workflowDir, 'verify-live-test-org-fleet.yml'); + const references = workflowRunReferences(parsed.get(verifier)); + + assert.ok(references.length > 0, 'the live fleet verifier lost its workflow_run chain'); + for (const reference of references) { + assert.ok( + declaredNames.has(reference), + `live fleet verifier waits on undeclared workflow '${reference}'`, + ); + } + + // The verifier must also be reachable without an upstream run, so a change to + // the verifier itself is exercised by its own pull request. + const text = parsed.get(verifier).join('\n'); + assert.match(text, /^ {2}pull_request:$/m, 'the verifier must run on its own pull requests'); + assert.match(text, /^ {2}workflow_dispatch:$/m, 'the verifier must stay manually dispatchable'); +});