chore(plugin): lead both manifests with skill evaluation, add discovery metadata

.claude-plugin/marketplace.json

@@ -1,17 +1,35 @@
 {
+  "$schema": "https://json.schemastore.org/claude-code-marketplace.json",
   "name": "coder-eval",
   "owner": {
     "name": "UiPath",
+    "email": "coder-eval@uipath.com",
     "url": "https://github.com/UiPath/coder_eval"
   },
-  "description": "Evaluate and benchmark AI coding agents and Claude Code skills.",
+  "description": "Test whether your Claude Code skills actually trigger, and benchmark AI coding agents against your own tasks.",
   "plugins": [
     {
       "name": "coder-eval",
+      "displayName": "Coder Eval",
       "source": "./plugins/coder-eval",
-      "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.",
+      "description": "Test whether your Claude Code skills actually trigger, and benchmark any coding agent — author, run, and analyze the eval suite that proves it, locally or as a CI gate.",
+      "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" },
+      "homepage": "https://coder-eval.com",
+      "repository": "https://github.com/UiPath/coder_eval",
+      "license": "Apache-2.0",
       "category": "testing",
-      "keywords": ["evaluation", "testing", "skills", "benchmark", "ci"]
+      "keywords": [
+        "claude-code-skills",
+        "skill-testing",
+        "skill-activation",
+        "evaluation",
+        "benchmark",
+        "agent-testing",
+        "github-actions",
+        "sandbox",
+        "llm-judge",
+        "ci"
+      ]
     }
   ]
 }

plugins/coder-eval/.claude-plugin/plugin.json

@@ -1,10 +1,23 @@
 {
+  "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json",
   "name": "coder-eval",
+  "displayName": "Coder Eval",
   "version": "0.11.4",
-  "description": "Author, run, and analyze coder-eval suites — including whether your Claude Code skills actually trigger.",
-  "author": { "name": "UiPath", "url": "https://github.com/UiPath/coder_eval" },
+  "description": "Test whether your Claude Code skills actually trigger, and benchmark any coding agent — author, run, and analyze the eval suite that proves it, locally or as a CI gate.",
+  "author": { "name": "UiPath", "email": "coder-eval@uipath.com", "url": "https://github.com/UiPath/coder_eval" },
   "homepage": "https://coder-eval.com",
   "repository": "https://github.com/UiPath/coder_eval",
   "license": "Apache-2.0",
-  "keywords": ["evaluation", "testing", "claude-code-skills", "benchmark", "ci"]
+  "keywords": [
+    "claude-code-skills",
+    "skill-testing",
+    "skill-activation",
+    "evaluation",
+    "benchmark",
+    "agent-testing",
+    "github-actions",
+    "sandbox",
+    "llm-judge",
+    "ci"
+  ]
 }

tests/lint/plugin_manifest_parity.py

@@ -0,0 +1,101 @@
+"""CE044 — the marketplace entry and the plugin manifest are one metadata surface.
+
+``.claude-plugin/marketplace.json`` is what the ``/plugin`` browser and the
+plugin directories show *before* install; ``plugins/coder-eval/.claude-plugin/
+plugin.json`` is what an installed user's copy carries *after*. Six fields are
+byte-identical duplicates across the two files (``description``, ``keywords``,
+``author``, ``homepage``, ``repository``, ``license``, plus ``name`` and
+``displayName``), and nothing compared them — the only test that reads
+``plugin.json`` at all is ``tests/test_action_version_pin.py``, and only its
+``version``. A one-sided edit — retitling the plugin in the marketplace but not
+the manifest — would ship silently and show two different one-liners in the wild.
+
+The second half of the rule is the one that has already bitten: the marketplace
+entry must not carry a *discovery* field the plugin manifest cannot mirror. The
+marketplace schema allows both ``keywords`` ("Tags for plugin discovery and
+categorization") and ``tags`` ("Tags for searchability and discovery"); the
+plugin-manifest schema has no ``tags`` property at all. Splitting discovery
+strings across the two therefore drops half of them from the installed copy, and
+leaves a future editor with no rule for which list a new term belongs in. So an
+extra key on the entry is a lint failure unless it is listed in
+``MARKETPLACE_ONLY`` with a written reason — the allowlist *is* the rule, kept in
+code rather than in tribal knowledge.
+
+Like CE026-CE031 and CE033 this reasons over whole files (JSON, plus resolving a
+``source`` path to a directory) rather than one ``.py`` AST, so it is not a
+``BaseRule`` in the runner; it is wired as a dedicated ``@pytest.mark.lint`` test
+class in ``tests/test_custom_lint.py``.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+# Fields both manifests can express, and which mean the same thing on both sides.
+# `version` is deliberately absent: only `plugin.json` carries it (a derived pin of
+# pyproject, guarded by tests/test_action_version_pin.py), and an entry-side copy
+# would be a third pin no release step maintains.
+SHARED_KEYS = (
+    "name",
+    "displayName",
+    "description",
+    "author",
+    "homepage",
+    "repository",
+    "license",
+    "keywords",
+)
+
+# Keys the marketplace entry may carry that the plugin manifest has no counterpart
+# for. Each needs a reason: adding one here is the deliberate act of saying "this
+# term does not belong in the shared surface".
+MARKETPLACE_ONLY: dict[str, str] = {
+    "source": "the entry's pointer at the plugin directory; meaningless inside the manifest it points at",
+    "category": "a marketplace-browser facet with a fixed vocabulary, not a free discovery string",
+}
+
+
+def _entry_source_dir(repo_root: Path, entry: dict[str, Any]) -> Path:
+    source = entry.get("source")
+    if not isinstance(source, str):
+        raise TypeError(f"marketplace entry {entry.get('name')!r} has no string `source`")
+    return (repo_root / source).resolve()
+
+
+def check(repo_root: Path) -> list[str]:
+    """Return one message per parity violation; empty means clean."""
+    findings: list[str] = []
+    marketplace_path = repo_root / ".claude-plugin" / "marketplace.json"
+    marketplace = json.loads(marketplace_path.read_text(encoding="utf-8"))
+
+    for entry in marketplace.get("plugins", []):
+        name = entry.get("name")
+        manifest_path = _entry_source_dir(repo_root, entry) / ".claude-plugin" / "plugin.json"
+        if not manifest_path.is_file():
+            findings.append(
+                f"marketplace entry {name!r}: `source` does not resolve to a plugin manifest ({manifest_path})"
+            )
+            continue
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+
+        for key in SHARED_KEYS:
+            in_entry, in_manifest = entry.get(key), manifest.get(key)
+            if in_entry != in_manifest:
+                findings.append(
+                    f"marketplace entry {name!r}: `{key}` differs from its plugin manifest\n"
+                    f"      marketplace: {in_entry!r}\n"
+                    f"      plugin.json: {in_manifest!r}"
+                )
+
+        extras = set(entry) - set(SHARED_KEYS) - set(MARKETPLACE_ONLY) - {"$schema"}
+        for key in sorted(extras):
+            findings.append(
+                f"marketplace entry {name!r}: `{key}` has no counterpart in the plugin manifest, so its "
+                f"value is dropped from an installed user's copy. Fold it into `keywords`, or add it to "
+                f"MARKETPLACE_ONLY in tests/lint/plugin_manifest_parity.py with a reason."
+            )
+
+    return findings

tests/test_custom_lint.py

@@ -10,6 +10,7 @@
     make lint
 """
 
+import json
 import re
 from pathlib import Path
 
@@ -1357,7 +1358,7 @@
         )
 
     def test_activation_rows_have_both_polarities(self):
         import json
 
         rows = [
             json.loads(line)
@@ -3273,3 +3274,79 @@
         )
         violations = permuted_violations(checker, case)
         assert any("NON-MONOTONIC" in v for v in violations), violations
+
+
+@pytest.mark.lint
+class TestCE044PluginManifestParity:
+    """CE044 — the marketplace entry and the plugin manifest it points at are one surface.
+
+    Eight fields are byte-identical duplicates across the two manifests and nothing
+    compared them: the only test that read ``plugin.json`` at all was
+    ``test_action_version_pin.py``, and only its ``version``. A one-sided edit ships
+    two different one-liners — one in the ``/plugin`` browser, one in the installed copy.
+
+    The second half is the motivating defect: the marketplace schema allows both
+    ``keywords`` and a near-synonymous ``tags``, while the plugin-manifest schema has no
+    ``tags`` property at all, so discovery strings parked there are dropped from an
+    installed user's manifest and a future editor has no rule for where a new term goes.
+    An extra key on the entry now fails unless ``MARKETPLACE_ONLY`` records why.
+
+    Reasons over JSON files and a ``source`` path, so it is wired here rather than as a
+    ``BaseRule`` in the AST runner.
+    """
+
+    REPO_ROOT = Path(__file__).parent.parent
+
+    def test_manifests_agree_on_every_shared_field(self):
+        from tests.lint.plugin_manifest_parity import check
+
+        findings = check(self.REPO_ROOT)
+        assert not findings, "plugin/marketplace manifest parity violations:\n" + "\n".join(f"  {f}" for f in findings)
+
+    def test_catches_a_one_sided_description_edit(self, tmp_path: Path):
+        from tests.lint.plugin_manifest_parity import check
+
+        self._write_pair(tmp_path, entry_extra={"description": "drifted"})
+        findings = check(tmp_path)
+        assert any("`description` differs" in f for f in findings), findings
+
+    def test_catches_a_discovery_key_the_manifest_cannot_mirror(self, tmp_path: Path):
+        from tests.lint.plugin_manifest_parity import check
+
+        self._write_pair(tmp_path, entry_extra={"tags": ["skills", "evals"]})
+        findings = check(tmp_path)
+        assert any("`tags` has no counterpart" in f for f in findings), findings
+
+    def test_catches_a_source_that_resolves_nowhere(self, tmp_path: Path):
+        from tests.lint.plugin_manifest_parity import check
+
+        self._write_pair(tmp_path, entry_extra={"source": "./plugins/gone"})
+        findings = check(tmp_path)
+        assert any("does not resolve" in f for f in findings), findings
+
+    def test_a_matching_pair_is_clean(self, tmp_path: Path):
+        from tests.lint.plugin_manifest_parity import check
+
+        self._write_pair(tmp_path)
+        assert check(tmp_path) == []
+
+    @staticmethod
+    def _write_pair(root: Path, entry_extra: dict | None = None) -> None:
+        """Write a minimal in-parity marketplace/manifest pair, then apply ``entry_extra``."""
+        shared = {
+            "name": "demo",
+            "displayName": "Demo",
+            "description": "a demo plugin",
+            "author": {"name": "UiPath"},
+            "homepage": "https://example.invalid",
+            "repository": "https://example.invalid/repo",
+            "license": "Apache-2.0",
+            "keywords": ["demo"],
+        }
+        entry = {**shared, "source": "./plugins/demo", "category": "testing", **(entry_extra or {})}
+        manifest_dir = root / "plugins" / "demo" / ".claude-plugin"
+        manifest_dir.mkdir(parents=True)
+        (manifest_dir / "plugin.json").write_text(json.dumps({**shared, "version": "1.0.0"}), encoding="utf-8")
+        market_dir = root / ".claude-plugin"
+        market_dir.mkdir(parents=True)
+        (market_dir / "marketplace.json").write_text(json.dumps({"name": "demo", "plugins": [entry]}), encoding="utf-8")