feat(hermes): report setup outcome so catalog installs are counted (v0.2.4)

hermes/README.md

@@ -92,6 +92,10 @@ tinyfish:
 
 When the `tinyfish` MCP server is also configured in Hermes (`mcp_servers.tinyfish.url: https://agent.tinyfish.ai/mcp`), the plugin injects once-per-context guidance steering the model between the generic web tools and the native MCP tools. Disable with `tinyfish.routing_context: false`.
 
+## Telemetry
+
+Each `hermes tinyfish setup` run sends one event to TinyFish, whether it succeeds or fails: the outcome (for example `completed` or `key_rejected`), where the key came from (`flag`, `prompt`, `env` or `none`), the plugin version and OS. It sends a one-way hash of the API key, never the key itself, so the event joins your account. Set `TINYFISH_NO_TELEMETRY=1` in your shell or in `~/.hermes/.env` to turn it off.
+
 ## Development
 
 ```bash

hermes/npm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiny-fish/hermes",
-  "version": "0.2.3",
+  "version": "0.2.4",
   "description": "TinyFish web provider plugin for Hermes Agent, delivered via npm for the TinyFish CLI. Installed by `tinyfish connect hermes`; not a JavaScript library.",
   "license": "MIT",
   "homepage": "https://docs.tinyfish.ai",

hermes/plugin.yaml

@@ -1,5 +1,5 @@
 name: tinyfish
-version: 0.2.3
+version: 0.2.4
 description: "TinyFish: the web, built into Hermes. Free Search and Fetch power Hermes's own web_search and web_extract, and when a site needs clicks, forms or a login, a remote browser and the tinyfish_agent web agent handle it."
 author: TinyFish
 kind: backend

hermes/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "tinyfish-hermes"
-version = "0.2.3"
+version = "0.2.4"
 description = "TinyFish plugin for Hermes Agent - Search, Fetch, Browser, and the tinyfish_agent web agent"
 readme = "README.md"
 license = { text = "MIT" }

hermes/scripts/e2e.sh

@@ -11,6 +11,8 @@ HERMES_HOME=$(mktemp -d)
 if command -v cygpath >/dev/null; then HERMES_HOME=$(cygpath -m "$HERMES_HOME"); fi
 [[ $REPO_ROOT == /* ]] || REPO_ROOT=/$REPO_ROOT
 export HERMES_HOME
+# CI setups would otherwise count as real plugin installs.
+export TINYFISH_NO_TELEMETRY=1
 trap 'rm -rf "$HERMES_HOME"' EXIT
 
 step() { printf '\n== %s\n' "$*"; }

hermes/tests/test_rest_client.py

@@ -12,7 +12,7 @@
     "X-API-Key": "tf_test",
     "Accept": "application/json",
     "X-TF-Client-Name": "hermes",
-    "X-TF-Client-Version": "0.2.3",
+    "X-TF-Client-Version": "0.2.4",
 }
 
 
@@ -450,3 +450,60 @@ def fake_get(url: str, **kwargs: Any) -> httpx.Response:
         match="legacy billing or no Metronome customer yet",
     ):
         rest_client.wallet(api_key="tf_test")
+
+
+def test_report_setup_sends_key_hash_not_key(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("TINYFISH_NO_TELEMETRY", raising=False)
+    captured: dict[str, Any] = {}
+
+    def fake_post(url: str, **kwargs: Any) -> httpx.Response:
+        captured.update(url=url, **kwargs)
+        return _response("POST", url, status=204)
+
+    monkeypatch.setattr(rest_client.httpx, "post", fake_post)
+
+    rest_client.report_setup("completed", key_source="flag", api_key="tf_test")
+
+    assert captured["url"] == "https://agent.tinyfish.ai/api/cli/connect-event"
+    assert captured["headers"]["x-api-key-lookup"] == (
+        "caf63824b113b6c8522e4552c6db58a9a38242c8d0c3460b6a9a140b791e0b42"
+    )
+    assert "tf_test" not in repr(captured)
+    assert captured["json"]["event"] == "harness_plugin_setup"
+    assert captured["json"]["outcome"] == "completed"
+    assert captured["json"]["harness"] == "hermes"
+
+
+def test_report_setup_omits_hash_without_key(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("TINYFISH_NO_TELEMETRY", raising=False)
+    captured: dict[str, Any] = {}
+    monkeypatch.setattr(
+        rest_client.httpx, "post", lambda url, **kwargs: captured.update(kwargs)
+    )
+
+    rest_client.report_setup("no_key", key_source="none")
+
+    assert "x-api-key-lookup" not in captured["headers"]
+
+
+@pytest.mark.parametrize("value", ["1", "true", "yes"])
+def test_report_setup_honours_opt_out(
+    monkeypatch: pytest.MonkeyPatch, value: str
+) -> None:
+    monkeypatch.setenv("TINYFISH_NO_TELEMETRY", value)
+    monkeypatch.setattr(
+        rest_client.httpx, "post", lambda *a, **k: pytest.fail("telemetry sent")
+    )
+
+    rest_client.report_setup("completed", key_source="env", api_key="tf_test")
+
+
+def test_report_setup_swallows_failures(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("TINYFISH_NO_TELEMETRY", raising=False)
+
+    def boom(*args: Any, **kwargs: Any) -> httpx.Response:
+        raise httpx.ConnectError("offline")
+
+    monkeypatch.setattr(rest_client.httpx, "post", boom)
+
+    rest_client.report_setup("completed", key_source="env", api_key="tf_test")

hermes/tests/test_setup_cli.py

@@ -92,6 +92,12 @@ def fake_save_env_secure(name: str, value: str) -> dict[str, Any]:
         "_enable_toolset",
         lambda name: state["enabled_toolsets"].append(name) or True,
     )
+    state["reports"] = []
+    monkeypatch.setattr(
+        rest_client,
+        "report_setup",
+        lambda outcome, **kwargs: state["reports"].append((outcome, kwargs)),
+    )
     monkeypatch.delenv("TINYFISH_API_KEY", raising=False)
     return state
 
@@ -865,3 +871,69 @@ def test_browser_enable_fails_when_agent_browser_install_fails(
     env["config"] = {}
 
     assert cli.dispatch_tinyfish_cli(_parser().parse_args(["browser", "enable"])) == 1
+
+
+def test_setup_reports_completed_with_env_key(env: dict[str, Any]) -> None:
+    assert cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"])) == 0
+
+    assert env["reports"] == [
+        ("completed", {"key_source": "env", "api_key": "tf_test"})
+    ]
+
+
+def test_setup_reports_flag_key(env: dict[str, Any]) -> None:
+    args = _parser().parse_args(["setup", "--yes", "--api-key", "tf_new"])
+
+    cli.dispatch_tinyfish_cli(args)
+
+    assert env["reports"] == [
+        ("completed", {"key_source": "flag", "api_key": "tf_new"})
+    ]
+
+
+def test_setup_reports_rejected_key(
+    env: dict[str, Any], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(cli, "_validate_key", lambda api_key: "rejected")
+    args = _parser().parse_args(["setup", "--yes", "--api-key", "tf_bad"])
+
+    cli.dispatch_tinyfish_cli(args)
+
+    assert [outcome for outcome, _ in env["reports"]] == ["key_rejected"]
+
+
+def test_setup_reports_save_failure(
+    env: dict[str, Any], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(cli, "_read_saved_env", lambda name: "")
+    args = _parser().parse_args(["setup", "--yes", "--api-key", "tf_new"])
+
+    cli.dispatch_tinyfish_cli(args)
+
+    assert [outcome for outcome, _ in env["reports"]] == ["save_failed"]
+
+
+def test_setup_reports_missing_key(env: dict[str, Any]) -> None:
+    env["env"] = {}
+
+    cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"]))
+
+    assert env["reports"] == [("no_key", {"key_source": "none", "api_key": ""})]
+
+
+def test_setup_reports_backend_failure(
+    env: dict[str, Any], monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(cli, "_enable_toolset", lambda name: False)
+
+    cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"]))
+
+    assert [outcome for outcome, _ in env["reports"]] == ["backend_failed"]
+
+
+def test_setup_honours_opt_out_from_hermes_env(env: dict[str, Any]) -> None:
+    env["env"]["TINYFISH_NO_TELEMETRY"] = "1"
+
+    cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"]))
+
+    assert env["reports"] == []

hermes/tests/test_version.py

@@ -93,7 +93,7 @@ def test_version_degrades_to_metadata_when_manifest_is_non_ascii(
 
 def test_manifest_version_matches_the_pinned_header_value() -> None:
     manifest = Path(plugin.__file__).resolve().parents[1] / "plugin.yaml"
-    assert plugin._version_from_plugin_manifest(manifest) == "0.2.3"
+    assert plugin._version_from_plugin_manifest(manifest) == "0.2.4"
 
 
 def _pyproject_version(hermes_root: Path) -> str:

hermes/tinyfish_hermes/rest_client.py

@@ -2,8 +2,12 @@
 
 from __future__ import annotations
 
+import hashlib
 import math
+import os
+import sys
 import time
+import uuid
 from typing import Any, NoReturn, cast
 
 import httpx
@@ -17,6 +21,7 @@
 WALLET_URL = f"{AGENT_URL}/v1/wallet"
 SEARCH_USAGE_URL = f"{SEARCH_URL}/usage"
 FETCH_USAGE_URL = f"{FETCH_URL}/usage"
+TELEMETRY_URL = f"{AGENT_URL}/api/cli/connect-event"
 
 _BROWSER_CLOSE_MAX_ATTEMPTS = 3
 _BROWSER_CLOSE_RETRYABLE_STATUSES = frozenset({409, 429, 500, 503, 504})
@@ -334,6 +339,36 @@ def start_run(
     )
 
 
+def telemetry_disabled(value: str | None = None) -> bool:
+    # Matches the TinyFish CLI: anything but empty, 0 or false opts out.
+    if value is None:
+        value = os.environ.get("TINYFISH_NO_TELEMETRY", "")
+    return value.strip().lower() not in ("", "0", "false")
+
+
+def report_setup(outcome: str, *, key_source: str, api_key: str = "") -> None:
+    """Record a ``hermes tinyfish setup`` outcome; never raises."""
+
+    if telemetry_disabled():
+        return
+    headers = {"Content-Type": "application/json"}
+    if api_key:
+        headers["x-api-key-lookup"] = hashlib.sha256(api_key.encode()).hexdigest()
+    body = {
+        "event": "harness_plugin_setup",
+        "harness": "hermes",
+        "plugin_version": __version__,
+        "outcome": outcome,
+        "key_source": key_source,
+        "runtime_platform": sys.platform,
+        "setup_id": str(uuid.uuid4()),
+    }
+    try:
+        httpx.post(TELEMETRY_URL, headers=headers, json=body, timeout=2.0)
+    except Exception:
+        pass
+
+
 def get_run(run_id: str, *, api_key: str) -> dict[str, Any]:
     return _agent_request("GET", f"/v1/runs/{run_id}", api_key=api_key)
 

hermes/tinyfish_hermes/setup_cli.py

@@ -257,20 +257,36 @@ def cmd_setup(
     provider: TinyFishWebSearchProvider | None = None,
 ) -> int:
     api_key = (getattr(args, "api_key", None) or "").strip()
+    key_source = "flag" if api_key else "none"
     existing = _api_key_env_var()
     if not api_key and existing:
         print(f"Using the TinyFish API key from {existing}")
+        key_source = "env"
     elif not api_key and sys.stdin.isatty():
         api_key = _prompt_for_key()
+        key_source = "prompt" if api_key else "none"
+
+    # Catalog installs skip `tinyfish connect`; this is their install signal.
+    def report(outcome: str) -> None:
+        # Hermes keeps settings in its .env, which never reaches os.environ.
+        if rest_client.telemetry_disabled(_get_env("TINYFISH_NO_TELEMETRY")):
+            return
+        rest_client.report_setup(
+            outcome, key_source=key_source, api_key=api_key or _api_key()
+        )
+
     if api_key:
         rejection = _validate_key(api_key)
         if rejection:
             print(rejection, file=sys.stderr)
+            report("key_rejected")
             return 1
         if not _store_api_key(api_key):
+            report("save_failed")
             return 1
     elif not existing:
         print(MISSING_KEY_ERROR, file=sys.stderr)
+        report("no_key")
         return 1
 
     # Route web tools only once a usable key exists, so a failed setup changes nothing.
@@ -284,8 +300,10 @@ def cmd_setup(
         _save_config(config)
         print("Configured Hermes web backends to use TinyFish")
         if not _enable_toolset("web"):
+            report("backend_failed")
             return 1
 
+    report("completed")
     if getattr(args, "live", False):
         doctor_args = argparse.Namespace(json=False, live=True, live_paid=False)
         return cmd_doctor(doctor_args, provider=provider)