From b2981e1889bbbe9c0f9c773e0f89c1689d69c1fb Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 27 Aug 2026 11:52:10 -0700 Subject: [PATCH 1/5] chore(plugin): lead both manifests with skill evaluation, add discovery metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin and marketplace manifests are what a plugin directory indexes and what the `/plugin` browser shows before install. Both led with "Author, run, and analyze coder-eval suites" — the mechanism, not the job — so the one thing that distinguishes this plugin (does your Claude Code skill actually trigger?) sat behind an em dash where a truncated listing drops it. - Both descriptions now lead with the skill-activation job. - Keywords lead with the skill terms (`claude-code-skills`, `skill-testing`, `skill-activation`) instead of the generic `evaluation`/`testing` pair. - The marketplace entry gains the discovery metadata the schema supports and it was missing: `displayName`, `author`, `homepage`, `repository`, `license`, `tags`. - `plugin.json` gains `displayName` so the listing shows "Coder Eval" rather than the kebab-case id. Deliberately NOT adding `version` to the marketplace entry: it pins updates, and release.yml's seds maintain only the two existing derived pins (action.yml, plugin.json). A third would go stale and strand users on a cached copy — the exact failure test_action_version_pin.py exists to prevent. `version` stays second in plugin.json with its trailing comma, so _PLUGIN_PIN_PATTERN and release.yml's sed still match. Verified: `claude plugin validate --strict` on both manifests, `make lint`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01V5WZ1BbtygK9YVvcgPfNGh --- .claude-plugin/marketplace.json | 20 ++++++++++++++++--- plugins/coder-eval/.claude-plugin/plugin.json | 13 ++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 053eeaec..7a592bf4 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -4,14 +4,28 @@ "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" }, - "description": "Evaluate and benchmark AI coding agents and Claude Code skills.", + "description": "Test whether your Claude Code skills actually trigger, and benchmark AI coding agents against your own tasks.", "plugins": [ { "name": "coder-eval", + "displayName": "Coder Eval", "source": "./plugins/coder-eval", - "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.", + "description": "Test whether your Claude Code skills actually trigger — then author, run, and analyze the eval suite that proves it, locally or as a CI gate.", + "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" }, + "homepage": "https://coder-eval.com", + "repository": "https://github.com/UiPath/coder_eval", + "license": "Apache-2.0", "category": "testing", - "keywords": ["evaluation", "testing", "skills", "benchmark", "ci"] + "keywords": [ + "claude-code-skills", + "skill-testing", + "skill-activation", + "evaluation", + "benchmark", + "agent-testing", + "ci" + ], + "tags": ["skills", "evals", "github-actions", "sandbox", "llm-judge"] } ] } diff --git a/plugins/coder-eval/.claude-plugin/plugin.json b/plugins/coder-eval/.claude-plugin/plugin.json index db19ada8..a811c6c1 100644 --- a/plugins/coder-eval/.claude-plugin/plugin.json +++ b/plugins/coder-eval/.claude-plugin/plugin.json @@ -1,10 +1,19 @@ { "name": "coder-eval", + "displayName": "Coder Eval", "version": "0.11.4", - "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.", + "description": "Test whether your Claude Code skills actually trigger — then author, run, and analyze the eval suite that proves it, locally or as a CI gate.", "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" }, "homepage": "https://coder-eval.com", "repository": "https://github.com/UiPath/coder_eval", "license": "Apache-2.0", - "keywords": ["evaluation", "testing", "claude-code-skills", "benchmark", "ci"] + "keywords": [ + "claude-code-skills", + "skill-testing", + "skill-activation", + "evaluation", + "benchmark", + "agent-testing", + "ci" + ] } From 4ef82e5dcadf6b69901434c883f59c5d15a0a611 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 27 Aug 2026 11:56:03 -0700 Subject: [PATCH 2/5] chore(plugin): pin both manifests to their published JSON schemas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audited both files field-by-field against the two official SchemaStore schemas (`claude-code-plugin-manifest.json`, `claude-code-marketplace.json`). Only one real gap survived: neither manifest declared `$schema`, so an editor gave no autocomplete and no validation on the files whose typos are otherwise invisible until a directory listing renders wrong. Both URLs resolve 200 and are the ones the plugin reference documents. Everything else the schemas allow is deliberately absent, not overlooked: component-path keys (`skills`, `commands`, `agents`, `hooks`, `mcpServers`, …) because the plugin ships only `skills/`, which is auto-discovered; `strict` because its default of true is the wanted behavior; `defaultEnabled` likewise; `dependencies`/`channels`/`userConfig`/`settings`/`monitors`/`themes`/ `outputStyles` because the plugin has none; marketplace `version`/`metadata`/ `forceRemoveDeletedPlugins`/`allowCrossMarketplaceDependenciesOn` because a single-plugin marketplace served from its own repo needs none of them; and the entry's `version` for the pin-drift reason recorded in the previous commit. Note SchemaStore lags the reference on `displayName` — the docs specify it on both a plugin manifest and a marketplace entry, the schemas do not list it. Neither schema sets `additionalProperties: false`, so it validates clean under both, verified below. Verified: both files VALID against the downloaded schemas via jsonschema, `claude plugin validate --strict` on both, `make lint` exit 0. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01V5WZ1BbtygK9YVvcgPfNGh --- .claude-plugin/marketplace.json | 1 + plugins/coder-eval/.claude-plugin/plugin.json | 1 + 2 files changed, 2 insertions(+) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 7a592bf4..60f7b061 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/claude-code-marketplace.json", "name": "coder-eval", "owner": { "name": "UiPath", diff --git a/plugins/coder-eval/.claude-plugin/plugin.json b/plugins/coder-eval/.claude-plugin/plugin.json index a811c6c1..1062db4e 100644 --- a/plugins/coder-eval/.claude-plugin/plugin.json +++ b/plugins/coder-eval/.claude-plugin/plugin.json @@ -1,4 +1,5 @@ { + "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "coder-eval", "displayName": "Coder Eval", "version": "0.11.4", From 1fa0a8f87d92ec08d5ca9f0468108d311cf16d6d Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 27 Aug 2026 13:27:47 -0700 Subject: [PATCH 3/5] chore(plugin): give the marketplace owner a contact address `owner.email` was the one field the marketplace schema documents that this manifest deliberately left blank, for want of a shared address. `coder-eval@uipath.com` is that address, so a plugin directory reviewer or a user with a packaging question has somewhere to write that is not a personal inbox. Verified: `claude plugin validate --strict`, VALID against claude-code-marketplace.json, `make lint` exit 0. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01V5WZ1BbtygK9YVvcgPfNGh --- .claude-plugin/marketplace.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 60f7b061..87b93536 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -3,6 +3,7 @@ "name": "coder-eval", "owner": { "name": "UiPath", + "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, "description": "Test whether your Claude Code skills actually trigger, and benchmark AI coding agents against your own tasks.", From 5db2cd4eeaa5aa0676ec6c5bdf1296bfe0ff17c2 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 27 Aug 2026 13:29:35 -0700 Subject: [PATCH 4/5] chore(plugin): give both author objects the same contact address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `owner.email` reached only the marketplace maintainer. A reader who opens `plugin.json` — the file a plugin directory reads and the one that wins under `strict: true` — still found an `author` with no way to make contact. Both `author` objects now carry `coder-eval@uipath.com`, so the address is present whichever manifest a reviewer or user opens. Surveyed every other author/contact field in the repo. Nothing else needs it: `pyproject.toml` already publishes this same address to PyPI, and `action.yml`'s `author:` is a plain GitHub Actions string with no email form. Verified: `claude plugin validate --strict` on both manifests, both VALID against their published schemas, `make lint` exit 0, version-pin test passing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01V5WZ1BbtygK9YVvcgPfNGh --- .claude-plugin/marketplace.json | 2 +- plugins/coder-eval/.claude-plugin/plugin.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 87b93536..6b7530a8 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -13,7 +13,7 @@ "displayName": "Coder Eval", "source": "./plugins/coder-eval", "description": "Test whether your Claude Code skills actually trigger — then author, run, and analyze the eval suite that proves it, locally or as a CI gate.", - "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" }, + "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, "homepage": "https://coder-eval.com", "repository": "https://github.com/UiPath/coder_eval", "license": "Apache-2.0", diff --git a/plugins/coder-eval/.claude-plugin/plugin.json b/plugins/coder-eval/.claude-plugin/plugin.json index 1062db4e..90f23fe7 100644 --- a/plugins/coder-eval/.claude-plugin/plugin.json +++ b/plugins/coder-eval/.claude-plugin/plugin.json @@ -4,7 +4,7 @@ "displayName": "Coder Eval", "version": "0.11.4", "description": "Test whether your Claude Code skills actually trigger — then author, run, and analyze the eval suite that proves it, locally or as a CI gate.", - "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" }, + "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, "homepage": "https://coder-eval.com", "repository": "https://github.com/UiPath/coder_eval", "license": "Apache-2.0", From d63d51bc9878b504edc7fc1bbb2f525096a6e564 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 27 Aug 2026 14:25:08 -0700 Subject: [PATCH 5/5] chore(plugin): answer "skills only?", fold tags into keywords, add CE044 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review items on #141. `claude code skills only?` (tmatup, on the plugin.json description) — no. The harness runs claude-code, codex and antigravity, and the description read as if skill activation were the whole product. Both descriptions now say "...and benchmark any coding agent" before the mechanism clause, so the skill-activation hook still leads without narrowing the tool to it. `tags` alongside `keywords` (review nit 1) — dropped. The two marketplace fields are described near-identically by the schema, held disjoint values with cross-boundary synonyms, and the plugin-manifest schema has no `tags` property at all, so an installed user's copy silently lost those five strings. The three genuinely distinct terms (`github-actions`, `sandbox`, `llm-judge`) are folded into the single `keywords` list both manifests now share; `skills` and `evals` were already covered by `claude-code-skills`/`skill-testing`/`evaluation`. CE044 (review's harness suggestion) — the commit left eight fields as byte-identical duplicates across two files with nothing comparing them; the only test that read plugin.json read its `version`. `tests/lint/plugin_manifest_parity.py` asserts the shared keys agree and fails on any extra discovery key the manifest cannot mirror unless MARKETPLACE_ONLY records why — so the "which list does a new term go in" rule lives in code, and re-adding `tags` fails `make lint`. `claude plugin validate --strict` passes on both files; make lint 376 passed. Co-Authored-By: Claude Opus 5 (1M context) --- .claude-plugin/marketplace.json | 8 +- plugins/coder-eval/.claude-plugin/plugin.json | 5 +- tests/lint/plugin_manifest_parity.py | 101 ++++++++++++++++++ tests/test_custom_lint.py | 77 +++++++++++++ 4 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 tests/lint/plugin_manifest_parity.py diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6b7530a8..1710351b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "name": "coder-eval", "displayName": "Coder Eval", "source": "./plugins/coder-eval", - "description": "Test whether your Claude Code skills actually trigger — then author, run, and analyze the eval suite that proves it, locally or as a CI gate.", + "description": "Test whether your Claude Code skills actually trigger, and benchmark any coding agent — author, run, and analyze the eval suite that proves it, locally or as a CI gate.", "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, "homepage": "https://coder-eval.com", "repository": "https://github.com/UiPath/coder_eval", @@ -25,9 +25,11 @@ "evaluation", "benchmark", "agent-testing", + "github-actions", + "sandbox", + "llm-judge", "ci" - ], - "tags": ["skills", "evals", "github-actions", "sandbox", "llm-judge"] + ] } ] } diff --git a/plugins/coder-eval/.claude-plugin/plugin.json b/plugins/coder-eval/.claude-plugin/plugin.json index 90f23fe7..afe18c69 100644 --- a/plugins/coder-eval/.claude-plugin/plugin.json +++ b/plugins/coder-eval/.claude-plugin/plugin.json @@ -3,7 +3,7 @@ "name": "coder-eval", "displayName": "Coder Eval", "version": "0.11.4", - "description": "Test whether your Claude Code skills actually trigger — then author, run, and analyze the eval suite that proves it, locally or as a CI gate.", + "description": "Test whether your Claude Code skills actually trigger, and benchmark any coding agent — author, run, and analyze the eval suite that proves it, locally or as a CI gate.", "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, "homepage": "https://coder-eval.com", "repository": "https://github.com/UiPath/coder_eval", @@ -15,6 +15,9 @@ "evaluation", "benchmark", "agent-testing", + "github-actions", + "sandbox", + "llm-judge", "ci" ] } diff --git a/tests/lint/plugin_manifest_parity.py b/tests/lint/plugin_manifest_parity.py new file mode 100644 index 00000000..ba007693 --- /dev/null +++ b/tests/lint/plugin_manifest_parity.py @@ -0,0 +1,101 @@ +"""CE044 — the marketplace entry and the plugin manifest are one metadata surface. + +``.claude-plugin/marketplace.json`` is what the ``/plugin`` browser and the +plugin directories show *before* install; ``plugins/coder-eval/.claude-plugin/ +plugin.json`` is what an installed user's copy carries *after*. Six fields are +byte-identical duplicates across the two files (``description``, ``keywords``, +``author``, ``homepage``, ``repository``, ``license``, plus ``name`` and +``displayName``), and nothing compared them — the only test that reads +``plugin.json`` at all is ``tests/test_action_version_pin.py``, and only its +``version``. A one-sided edit — retitling the plugin in the marketplace but not +the manifest — would ship silently and show two different one-liners in the wild. + +The second half of the rule is the one that has already bitten: the marketplace +entry must not carry a *discovery* field the plugin manifest cannot mirror. The +marketplace schema allows both ``keywords`` ("Tags for plugin discovery and +categorization") and ``tags`` ("Tags for searchability and discovery"); the +plugin-manifest schema has no ``tags`` property at all. Splitting discovery +strings across the two therefore drops half of them from the installed copy, and +leaves a future editor with no rule for which list a new term belongs in. So an +extra key on the entry is a lint failure unless it is listed in +``MARKETPLACE_ONLY`` with a written reason — the allowlist *is* the rule, kept in +code rather than in tribal knowledge. + +Like CE026-CE031 and CE033 this reasons over whole files (JSON, plus resolving a +``source`` path to a directory) rather than one ``.py`` AST, so it is not a +``BaseRule`` in the runner; it is wired as a dedicated ``@pytest.mark.lint`` test +class in ``tests/test_custom_lint.py``. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +# Fields both manifests can express, and which mean the same thing on both sides. +# `version` is deliberately absent: only `plugin.json` carries it (a derived pin of +# pyproject, guarded by tests/test_action_version_pin.py), and an entry-side copy +# would be a third pin no release step maintains. +SHARED_KEYS = ( + "name", + "displayName", + "description", + "author", + "homepage", + "repository", + "license", + "keywords", +) + +# Keys the marketplace entry may carry that the plugin manifest has no counterpart +# for. Each needs a reason: adding one here is the deliberate act of saying "this +# term does not belong in the shared surface". +MARKETPLACE_ONLY: dict[str, str] = { + "source": "the entry's pointer at the plugin directory; meaningless inside the manifest it points at", + "category": "a marketplace-browser facet with a fixed vocabulary, not a free discovery string", +} + + +def _entry_source_dir(repo_root: Path, entry: dict[str, Any]) -> Path: + source = entry.get("source") + if not isinstance(source, str): + raise TypeError(f"marketplace entry {entry.get('name')!r} has no string `source`") + return (repo_root / source).resolve() + + +def check(repo_root: Path) -> list[str]: + """Return one message per parity violation; empty means clean.""" + findings: list[str] = [] + marketplace_path = repo_root / ".claude-plugin" / "marketplace.json" + marketplace = json.loads(marketplace_path.read_text(encoding="utf-8")) + + for entry in marketplace.get("plugins", []): + name = entry.get("name") + manifest_path = _entry_source_dir(repo_root, entry) / ".claude-plugin" / "plugin.json" + if not manifest_path.is_file(): + findings.append( + f"marketplace entry {name!r}: `source` does not resolve to a plugin manifest ({manifest_path})" + ) + continue + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + for key in SHARED_KEYS: + in_entry, in_manifest = entry.get(key), manifest.get(key) + if in_entry != in_manifest: + findings.append( + f"marketplace entry {name!r}: `{key}` differs from its plugin manifest\n" + f" marketplace: {in_entry!r}\n" + f" plugin.json: {in_manifest!r}" + ) + + extras = set(entry) - set(SHARED_KEYS) - set(MARKETPLACE_ONLY) - {"$schema"} + for key in sorted(extras): + findings.append( + f"marketplace entry {name!r}: `{key}` has no counterpart in the plugin manifest, so its " + f"value is dropped from an installed user's copy. Fold it into `keywords`, or add it to " + f"MARKETPLACE_ONLY in tests/lint/plugin_manifest_parity.py with a reason." + ) + + return findings diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index be312a91..1b3d7b7e 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -10,6 +10,7 @@ make lint """ +import json import re from pathlib import Path @@ -3273,3 +3274,79 @@ def sorted_recency_verdict(records): ) violations = permuted_violations(checker, case) assert any("NON-MONOTONIC" in v for v in violations), violations + + +@pytest.mark.lint +class TestCE044PluginManifestParity: + """CE044 — the marketplace entry and the plugin manifest it points at are one surface. + + Eight fields are byte-identical duplicates across the two manifests and nothing + compared them: the only test that read ``plugin.json`` at all was + ``test_action_version_pin.py``, and only its ``version``. A one-sided edit ships + two different one-liners — one in the ``/plugin`` browser, one in the installed copy. + + The second half is the motivating defect: the marketplace schema allows both + ``keywords`` and a near-synonymous ``tags``, while the plugin-manifest schema has no + ``tags`` property at all, so discovery strings parked there are dropped from an + installed user's manifest and a future editor has no rule for where a new term goes. + An extra key on the entry now fails unless ``MARKETPLACE_ONLY`` records why. + + Reasons over JSON files and a ``source`` path, so it is wired here rather than as a + ``BaseRule`` in the AST runner. + """ + + REPO_ROOT = Path(__file__).parent.parent + + def test_manifests_agree_on_every_shared_field(self): + from tests.lint.plugin_manifest_parity import check + + findings = check(self.REPO_ROOT) + assert not findings, "plugin/marketplace manifest parity violations:\n" + "\n".join(f" {f}" for f in findings) + + def test_catches_a_one_sided_description_edit(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path, entry_extra={"description": "drifted"}) + findings = check(tmp_path) + assert any("`description` differs" in f for f in findings), findings + + def test_catches_a_discovery_key_the_manifest_cannot_mirror(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path, entry_extra={"tags": ["skills", "evals"]}) + findings = check(tmp_path) + assert any("`tags` has no counterpart" in f for f in findings), findings + + def test_catches_a_source_that_resolves_nowhere(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path, entry_extra={"source": "./plugins/gone"}) + findings = check(tmp_path) + assert any("does not resolve" in f for f in findings), findings + + def test_a_matching_pair_is_clean(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path) + assert check(tmp_path) == [] + + @staticmethod + def _write_pair(root: Path, entry_extra: dict | None = None) -> None: + """Write a minimal in-parity marketplace/manifest pair, then apply ``entry_extra``.""" + shared = { + "name": "demo", + "displayName": "Demo", + "description": "a demo plugin", + "author": {"name": "UiPath"}, + "homepage": "https://example.invalid", + "repository": "https://example.invalid/repo", + "license": "Apache-2.0", + "keywords": ["demo"], + } + entry = {**shared, "source": "./plugins/demo", "category": "testing", **(entry_extra or {})} + manifest_dir = root / "plugins" / "demo" / ".claude-plugin" + manifest_dir.mkdir(parents=True) + (manifest_dir / "plugin.json").write_text(json.dumps({**shared, "version": "1.0.0"}), encoding="utf-8") + market_dir = root / ".claude-plugin" + market_dir.mkdir(parents=True) + (market_dir / "marketplace.json").write_text(json.dumps({"name": "demo", "plugins": [entry]}), encoding="utf-8")