diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 053eeaec..1710351b 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,17 +1,35 @@ { + "$schema": "https://json.schemastore.org/claude-code-marketplace.json", "name": "coder-eval", "owner": { "name": "UiPath", + "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, - "description": "Evaluate and benchmark AI coding agents and Claude Code skills.", + "description": "Test whether your Claude Code skills actually trigger, and benchmark AI coding agents against your own tasks.", "plugins": [ { "name": "coder-eval", + "displayName": "Coder Eval", "source": "./plugins/coder-eval", - "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.", + "description": "Test whether your Claude Code skills actually trigger, and benchmark any coding agent — author, run, and analyze the eval suite that proves it, locally or as a CI gate.", + "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, + "homepage": "https://coder-eval.com", + "repository": "https://github.com/UiPath/coder_eval", + "license": "Apache-2.0", "category": "testing", - "keywords": ["evaluation", "testing", "skills", "benchmark", "ci"] + "keywords": [ + "claude-code-skills", + "skill-testing", + "skill-activation", + "evaluation", + "benchmark", + "agent-testing", + "github-actions", + "sandbox", + "llm-judge", + "ci" + ] } ] } diff --git a/plugins/coder-eval/.claude-plugin/plugin.json b/plugins/coder-eval/.claude-plugin/plugin.json index db19ada8..afe18c69 100644 --- a/plugins/coder-eval/.claude-plugin/plugin.json +++ b/plugins/coder-eval/.claude-plugin/plugin.json @@ -1,10 +1,23 @@ { + "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "coder-eval", + "displayName": "Coder Eval", "version": "0.11.4", - "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.", - "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" }, + "description": "Test whether your Claude Code skills actually trigger, and benchmark any coding agent — author, run, and analyze the eval suite that proves it, locally or as a CI gate.", + "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" }, "homepage": "https://coder-eval.com", "repository": "https://github.com/UiPath/coder_eval", "license": "Apache-2.0", - "keywords": ["evaluation", "testing", "claude-code-skills", "benchmark", "ci"] + "keywords": [ + "claude-code-skills", + "skill-testing", + "skill-activation", + "evaluation", + "benchmark", + "agent-testing", + "github-actions", + "sandbox", + "llm-judge", + "ci" + ] } diff --git a/tests/lint/plugin_manifest_parity.py b/tests/lint/plugin_manifest_parity.py new file mode 100644 index 00000000..ba007693 --- /dev/null +++ b/tests/lint/plugin_manifest_parity.py @@ -0,0 +1,101 @@ +"""CE044 — the marketplace entry and the plugin manifest are one metadata surface. + +``.claude-plugin/marketplace.json`` is what the ``/plugin`` browser and the +plugin directories show *before* install; ``plugins/coder-eval/.claude-plugin/ +plugin.json`` is what an installed user's copy carries *after*. Six fields are +byte-identical duplicates across the two files (``description``, ``keywords``, +``author``, ``homepage``, ``repository``, ``license``, plus ``name`` and +``displayName``), and nothing compared them — the only test that reads +``plugin.json`` at all is ``tests/test_action_version_pin.py``, and only its +``version``. A one-sided edit — retitling the plugin in the marketplace but not +the manifest — would ship silently and show two different one-liners in the wild. + +The second half of the rule is the one that has already bitten: the marketplace +entry must not carry a *discovery* field the plugin manifest cannot mirror. The +marketplace schema allows both ``keywords`` ("Tags for plugin discovery and +categorization") and ``tags`` ("Tags for searchability and discovery"); the +plugin-manifest schema has no ``tags`` property at all. Splitting discovery +strings across the two therefore drops half of them from the installed copy, and +leaves a future editor with no rule for which list a new term belongs in. So an +extra key on the entry is a lint failure unless it is listed in +``MARKETPLACE_ONLY`` with a written reason — the allowlist *is* the rule, kept in +code rather than in tribal knowledge. + +Like CE026-CE031 and CE033 this reasons over whole files (JSON, plus resolving a +``source`` path to a directory) rather than one ``.py`` AST, so it is not a +``BaseRule`` in the runner; it is wired as a dedicated ``@pytest.mark.lint`` test +class in ``tests/test_custom_lint.py``. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +# Fields both manifests can express, and which mean the same thing on both sides. +# `version` is deliberately absent: only `plugin.json` carries it (a derived pin of +# pyproject, guarded by tests/test_action_version_pin.py), and an entry-side copy +# would be a third pin no release step maintains. +SHARED_KEYS = ( + "name", + "displayName", + "description", + "author", + "homepage", + "repository", + "license", + "keywords", +) + +# Keys the marketplace entry may carry that the plugin manifest has no counterpart +# for. Each needs a reason: adding one here is the deliberate act of saying "this +# term does not belong in the shared surface". +MARKETPLACE_ONLY: dict[str, str] = { + "source": "the entry's pointer at the plugin directory; meaningless inside the manifest it points at", + "category": "a marketplace-browser facet with a fixed vocabulary, not a free discovery string", +} + + +def _entry_source_dir(repo_root: Path, entry: dict[str, Any]) -> Path: + source = entry.get("source") + if not isinstance(source, str): + raise TypeError(f"marketplace entry {entry.get('name')!r} has no string `source`") + return (repo_root / source).resolve() + + +def check(repo_root: Path) -> list[str]: + """Return one message per parity violation; empty means clean.""" + findings: list[str] = [] + marketplace_path = repo_root / ".claude-plugin" / "marketplace.json" + marketplace = json.loads(marketplace_path.read_text(encoding="utf-8")) + + for entry in marketplace.get("plugins", []): + name = entry.get("name") + manifest_path = _entry_source_dir(repo_root, entry) / ".claude-plugin" / "plugin.json" + if not manifest_path.is_file(): + findings.append( + f"marketplace entry {name!r}: `source` does not resolve to a plugin manifest ({manifest_path})" + ) + continue + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + + for key in SHARED_KEYS: + in_entry, in_manifest = entry.get(key), manifest.get(key) + if in_entry != in_manifest: + findings.append( + f"marketplace entry {name!r}: `{key}` differs from its plugin manifest\n" + f" marketplace: {in_entry!r}\n" + f" plugin.json: {in_manifest!r}" + ) + + extras = set(entry) - set(SHARED_KEYS) - set(MARKETPLACE_ONLY) - {"$schema"} + for key in sorted(extras): + findings.append( + f"marketplace entry {name!r}: `{key}` has no counterpart in the plugin manifest, so its " + f"value is dropped from an installed user's copy. Fold it into `keywords`, or add it to " + f"MARKETPLACE_ONLY in tests/lint/plugin_manifest_parity.py with a reason." + ) + + return findings diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index be312a91..1b3d7b7e 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -10,6 +10,7 @@ make lint """ +import json import re from pathlib import Path @@ -3273,3 +3274,79 @@ def sorted_recency_verdict(records): ) violations = permuted_violations(checker, case) assert any("NON-MONOTONIC" in v for v in violations), violations + + +@pytest.mark.lint +class TestCE044PluginManifestParity: + """CE044 — the marketplace entry and the plugin manifest it points at are one surface. + + Eight fields are byte-identical duplicates across the two manifests and nothing + compared them: the only test that read ``plugin.json`` at all was + ``test_action_version_pin.py``, and only its ``version``. A one-sided edit ships + two different one-liners — one in the ``/plugin`` browser, one in the installed copy. + + The second half is the motivating defect: the marketplace schema allows both + ``keywords`` and a near-synonymous ``tags``, while the plugin-manifest schema has no + ``tags`` property at all, so discovery strings parked there are dropped from an + installed user's manifest and a future editor has no rule for where a new term goes. + An extra key on the entry now fails unless ``MARKETPLACE_ONLY`` records why. + + Reasons over JSON files and a ``source`` path, so it is wired here rather than as a + ``BaseRule`` in the AST runner. + """ + + REPO_ROOT = Path(__file__).parent.parent + + def test_manifests_agree_on_every_shared_field(self): + from tests.lint.plugin_manifest_parity import check + + findings = check(self.REPO_ROOT) + assert not findings, "plugin/marketplace manifest parity violations:\n" + "\n".join(f" {f}" for f in findings) + + def test_catches_a_one_sided_description_edit(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path, entry_extra={"description": "drifted"}) + findings = check(tmp_path) + assert any("`description` differs" in f for f in findings), findings + + def test_catches_a_discovery_key_the_manifest_cannot_mirror(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path, entry_extra={"tags": ["skills", "evals"]}) + findings = check(tmp_path) + assert any("`tags` has no counterpart" in f for f in findings), findings + + def test_catches_a_source_that_resolves_nowhere(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path, entry_extra={"source": "./plugins/gone"}) + findings = check(tmp_path) + assert any("does not resolve" in f for f in findings), findings + + def test_a_matching_pair_is_clean(self, tmp_path: Path): + from tests.lint.plugin_manifest_parity import check + + self._write_pair(tmp_path) + assert check(tmp_path) == [] + + @staticmethod + def _write_pair(root: Path, entry_extra: dict | None = None) -> None: + """Write a minimal in-parity marketplace/manifest pair, then apply ``entry_extra``.""" + shared = { + "name": "demo", + "displayName": "Demo", + "description": "a demo plugin", + "author": {"name": "UiPath"}, + "homepage": "https://example.invalid", + "repository": "https://example.invalid/repo", + "license": "Apache-2.0", + "keywords": ["demo"], + } + entry = {**shared, "source": "./plugins/demo", "category": "testing", **(entry_extra or {})} + manifest_dir = root / "plugins" / "demo" / ".claude-plugin" + manifest_dir.mkdir(parents=True) + (manifest_dir / "plugin.json").write_text(json.dumps({**shared, "version": "1.0.0"}), encoding="utf-8") + market_dir = root / ".claude-plugin" + market_dir.mkdir(parents=True) + (market_dir / "marketplace.json").write_text(json.dumps({"name": "demo", "plugins": [entry]}), encoding="utf-8")