From 5a828b07a6400e5f98992dca9c3d5fb5df4f55b5 Mon Sep 17 00:00:00 2001 From: Zachary Lyon Date: Wed, 21 Oct 2026 14:33:45 -0700 Subject: [PATCH 1/2] feat(hermes): report setup outcome so catalog installs are counted (v0.2.4) Co-Authored-By: Claude Opus 5.5 --- hermes/README.md | 4 ++ hermes/npm/package.json | 2 +- hermes/plugin.yaml | 2 +- hermes/pyproject.toml | 2 +- hermes/scripts/e2e.sh | 2 + hermes/tests/test_rest_client.py | 59 +++++++++++++++++++++++- hermes/tests/test_setup_cli.py | 64 +++++++++++++++++++++++++++ hermes/tests/test_version.py | 2 +- hermes/tinyfish_hermes/rest_client.py | 34 ++++++++++++++ hermes/tinyfish_hermes/setup_cli.py | 15 +++++++ 10 files changed, 181 insertions(+), 5 deletions(-) diff --git a/hermes/README.md b/hermes/README.md index 502af5b..9268f68 100644 --- a/hermes/README.md +++ b/hermes/README.md @@ -92,6 +92,10 @@ tinyfish: When the `tinyfish` MCP server is also configured in Hermes (`mcp_servers.tinyfish.url: https://agent.tinyfish.ai/mcp`), the plugin injects once-per-context guidance steering the model between the generic web tools and the native MCP tools. Disable with `tinyfish.routing_context: false`. +## Telemetry + +`hermes tinyfish setup` sends one event to TinyFish when it finishes: the outcome (for example `completed` or `key_rejected`), where the key came from (`flag`, `prompt`, `env`), the plugin version and OS. It sends a one-way hash of the API key, never the key itself, so the event joins your account. Set `TINYFISH_NO_TELEMETRY=1` to turn it off. + ## Development ```bash diff --git a/hermes/npm/package.json b/hermes/npm/package.json index b14f208..b41de28 100644 --- a/hermes/npm/package.json +++ b/hermes/npm/package.json @@ -1,6 +1,6 @@ { "name": "@tiny-fish/hermes", - "version": "0.2.3", + "version": "0.2.4", "description": "TinyFish web provider plugin for Hermes Agent, delivered via npm for the TinyFish CLI. Installed by `tinyfish connect hermes`; not a JavaScript library.", "license": "MIT", "homepage": "https://docs.tinyfish.ai", diff --git a/hermes/plugin.yaml b/hermes/plugin.yaml index 56f8a3d..d27c237 100644 --- a/hermes/plugin.yaml +++ b/hermes/plugin.yaml @@ -1,5 +1,5 @@ name: tinyfish -version: 0.2.3 +version: 0.2.4 description: "TinyFish: the web, built into Hermes. Free Search and Fetch power Hermes's own web_search and web_extract, and when a site needs clicks, forms or a login, a remote browser and the tinyfish_agent web agent handle it." author: TinyFish kind: backend diff --git a/hermes/pyproject.toml b/hermes/pyproject.toml index 103350c..cdd74a5 100644 --- a/hermes/pyproject.toml +++ b/hermes/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "tinyfish-hermes" -version = "0.2.3" +version = "0.2.4" description = "TinyFish plugin for Hermes Agent - Search, Fetch, Browser, and the tinyfish_agent web agent" readme = "README.md" license = { text = "MIT" } diff --git a/hermes/scripts/e2e.sh b/hermes/scripts/e2e.sh index 01e7d2a..1eeb26c 100755 --- a/hermes/scripts/e2e.sh +++ b/hermes/scripts/e2e.sh @@ -11,6 +11,8 @@ HERMES_HOME=$(mktemp -d) if command -v cygpath >/dev/null; then HERMES_HOME=$(cygpath -m "$HERMES_HOME"); fi [[ $REPO_ROOT == /* ]] || REPO_ROOT=/$REPO_ROOT export HERMES_HOME +# CI setups would otherwise count as real plugin installs. +export TINYFISH_NO_TELEMETRY=1 trap 'rm -rf "$HERMES_HOME"' EXIT step() { printf '\n== %s\n' "$*"; } diff --git a/hermes/tests/test_rest_client.py b/hermes/tests/test_rest_client.py index e22cec6..d82f48a 100644 --- a/hermes/tests/test_rest_client.py +++ b/hermes/tests/test_rest_client.py @@ -12,7 +12,7 @@ "X-API-Key": "tf_test", "Accept": "application/json", "X-TF-Client-Name": "hermes", - "X-TF-Client-Version": "0.2.3", + "X-TF-Client-Version": "0.2.4", } @@ -450,3 +450,60 @@ def fake_get(url: str, **kwargs: Any) -> httpx.Response: match="legacy billing or no Metronome customer yet", ): rest_client.wallet(api_key="tf_test") + + +def test_report_setup_sends_key_hash_not_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TINYFISH_NO_TELEMETRY", raising=False) + captured: dict[str, Any] = {} + + def fake_post(url: str, **kwargs: Any) -> httpx.Response: + captured.update(url=url, **kwargs) + return _response("POST", url, status=204) + + monkeypatch.setattr(rest_client.httpx, "post", fake_post) + + rest_client.report_setup("completed", key_source="flag", api_key="tf_test") + + assert captured["url"] == "https://agent.tinyfish.ai/api/cli/connect-event" + assert captured["headers"]["x-api-key-lookup"] == ( + "caf63824b113b6c8522e4552c6db58a9a38242c8d0c3460b6a9a140b791e0b42" + ) + assert "tf_test" not in repr(captured) + assert captured["json"]["event"] == "harness_plugin_setup" + assert captured["json"]["outcome"] == "completed" + assert captured["json"]["harness"] == "hermes" + + +def test_report_setup_omits_hash_without_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TINYFISH_NO_TELEMETRY", raising=False) + captured: dict[str, Any] = {} + monkeypatch.setattr( + rest_client.httpx, "post", lambda url, **kwargs: captured.update(kwargs) + ) + + rest_client.report_setup("no_key", key_source="none") + + assert "x-api-key-lookup" not in captured["headers"] + + +@pytest.mark.parametrize("value", ["1", "true", "yes"]) +def test_report_setup_honours_opt_out( + monkeypatch: pytest.MonkeyPatch, value: str +) -> None: + monkeypatch.setenv("TINYFISH_NO_TELEMETRY", value) + monkeypatch.setattr( + rest_client.httpx, "post", lambda *a, **k: pytest.fail("telemetry sent") + ) + + rest_client.report_setup("completed", key_source="env", api_key="tf_test") + + +def test_report_setup_swallows_failures(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TINYFISH_NO_TELEMETRY", raising=False) + + def boom(*args: Any, **kwargs: Any) -> httpx.Response: + raise httpx.ConnectError("offline") + + monkeypatch.setattr(rest_client.httpx, "post", boom) + + rest_client.report_setup("completed", key_source="env", api_key="tf_test") diff --git a/hermes/tests/test_setup_cli.py b/hermes/tests/test_setup_cli.py index 4d03d5c..8be25a7 100644 --- a/hermes/tests/test_setup_cli.py +++ b/hermes/tests/test_setup_cli.py @@ -92,6 +92,12 @@ def fake_save_env_secure(name: str, value: str) -> dict[str, Any]: "_enable_toolset", lambda name: state["enabled_toolsets"].append(name) or True, ) + state["reports"] = [] + monkeypatch.setattr( + rest_client, + "report_setup", + lambda outcome, **kwargs: state["reports"].append((outcome, kwargs)), + ) monkeypatch.delenv("TINYFISH_API_KEY", raising=False) return state @@ -865,3 +871,61 @@ def test_browser_enable_fails_when_agent_browser_install_fails( env["config"] = {} assert cli.dispatch_tinyfish_cli(_parser().parse_args(["browser", "enable"])) == 1 + + +def test_setup_reports_completed_with_env_key(env: dict[str, Any]) -> None: + assert cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"])) == 0 + + assert env["reports"] == [ + ("completed", {"key_source": "env", "api_key": "tf_test"}) + ] + + +def test_setup_reports_flag_key(env: dict[str, Any]) -> None: + args = _parser().parse_args(["setup", "--yes", "--api-key", "tf_new"]) + + cli.dispatch_tinyfish_cli(args) + + assert env["reports"] == [ + ("completed", {"key_source": "flag", "api_key": "tf_new"}) + ] + + +def test_setup_reports_rejected_key( + env: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(cli, "_validate_key", lambda api_key: "rejected") + args = _parser().parse_args(["setup", "--yes", "--api-key", "tf_bad"]) + + cli.dispatch_tinyfish_cli(args) + + assert [outcome for outcome, _ in env["reports"]] == ["key_rejected"] + + +def test_setup_reports_save_failure( + env: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(cli, "_read_saved_env", lambda name: "") + args = _parser().parse_args(["setup", "--yes", "--api-key", "tf_new"]) + + cli.dispatch_tinyfish_cli(args) + + assert [outcome for outcome, _ in env["reports"]] == ["save_failed"] + + +def test_setup_reports_missing_key(env: dict[str, Any]) -> None: + env["env"] = {} + + cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"])) + + assert env["reports"] == [("no_key", {"key_source": "none", "api_key": ""})] + + +def test_setup_reports_backend_failure( + env: dict[str, Any], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(cli, "_enable_toolset", lambda name: False) + + cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"])) + + assert [outcome for outcome, _ in env["reports"]] == ["backend_failed"] diff --git a/hermes/tests/test_version.py b/hermes/tests/test_version.py index 9caa27a..37a21c5 100644 --- a/hermes/tests/test_version.py +++ b/hermes/tests/test_version.py @@ -93,7 +93,7 @@ def test_version_degrades_to_metadata_when_manifest_is_non_ascii( def test_manifest_version_matches_the_pinned_header_value() -> None: manifest = Path(plugin.__file__).resolve().parents[1] / "plugin.yaml" - assert plugin._version_from_plugin_manifest(manifest) == "0.2.3" + assert plugin._version_from_plugin_manifest(manifest) == "0.2.4" def _pyproject_version(hermes_root: Path) -> str: diff --git a/hermes/tinyfish_hermes/rest_client.py b/hermes/tinyfish_hermes/rest_client.py index 72f16a4..73e8fe2 100644 --- a/hermes/tinyfish_hermes/rest_client.py +++ b/hermes/tinyfish_hermes/rest_client.py @@ -2,8 +2,12 @@ from __future__ import annotations +import hashlib import math +import os +import sys import time +import uuid from typing import Any, NoReturn, cast import httpx @@ -17,6 +21,7 @@ WALLET_URL = f"{AGENT_URL}/v1/wallet" SEARCH_USAGE_URL = f"{SEARCH_URL}/usage" FETCH_USAGE_URL = f"{FETCH_URL}/usage" +TELEMETRY_URL = f"{AGENT_URL}/api/cli/connect-event" _BROWSER_CLOSE_MAX_ATTEMPTS = 3 _BROWSER_CLOSE_RETRYABLE_STATUSES = frozenset({409, 429, 500, 503, 504}) @@ -334,6 +339,35 @@ def start_run( ) +def telemetry_disabled() -> bool: + # Matches the TinyFish CLI: anything but empty, 0 or false opts out. + value = os.environ.get("TINYFISH_NO_TELEMETRY", "").strip().lower() + return value not in ("", "0", "false") + + +def report_setup(outcome: str, *, key_source: str, api_key: str = "") -> None: + """Record a ``hermes tinyfish setup`` outcome; never raises.""" + + if telemetry_disabled(): + return + headers = {"Content-Type": "application/json"} + if api_key: + headers["x-api-key-lookup"] = hashlib.sha256(api_key.encode()).hexdigest() + body = { + "event": "harness_plugin_setup", + "harness": "hermes", + "plugin_version": __version__, + "outcome": outcome, + "key_source": key_source, + "runtime_platform": sys.platform, + "setup_id": str(uuid.uuid4()), + } + try: + httpx.post(TELEMETRY_URL, headers=headers, json=body, timeout=2.0) + except Exception: + pass + + def get_run(run_id: str, *, api_key: str) -> dict[str, Any]: return _agent_request("GET", f"/v1/runs/{run_id}", api_key=api_key) diff --git a/hermes/tinyfish_hermes/setup_cli.py b/hermes/tinyfish_hermes/setup_cli.py index 86733a3..60f8bc2 100644 --- a/hermes/tinyfish_hermes/setup_cli.py +++ b/hermes/tinyfish_hermes/setup_cli.py @@ -257,20 +257,33 @@ def cmd_setup( provider: TinyFishWebSearchProvider | None = None, ) -> int: api_key = (getattr(args, "api_key", None) or "").strip() + key_source = "flag" if api_key else "none" existing = _api_key_env_var() if not api_key and existing: print(f"Using the TinyFish API key from {existing}") + key_source = "env" elif not api_key and sys.stdin.isatty(): api_key = _prompt_for_key() + key_source = "prompt" if api_key else "none" + + # Catalog installs skip `tinyfish connect`; this is their install signal. + def report(outcome: str) -> None: + rest_client.report_setup( + outcome, key_source=key_source, api_key=api_key or _api_key() + ) + if api_key: rejection = _validate_key(api_key) if rejection: print(rejection, file=sys.stderr) + report("key_rejected") return 1 if not _store_api_key(api_key): + report("save_failed") return 1 elif not existing: print(MISSING_KEY_ERROR, file=sys.stderr) + report("no_key") return 1 # Route web tools only once a usable key exists, so a failed setup changes nothing. @@ -284,8 +297,10 @@ def cmd_setup( _save_config(config) print("Configured Hermes web backends to use TinyFish") if not _enable_toolset("web"): + report("backend_failed") return 1 + report("completed") if getattr(args, "live", False): doctor_args = argparse.Namespace(json=False, live=True, live_paid=False) return cmd_doctor(doctor_args, provider=provider) From 06b1bc4e58b2160ce1c367665d9f12dffa0095f1 Mon Sep 17 00:00:00 2001 From: Zachary Lyon Date: Wed, 21 Oct 2026 15:05:22 -0700 Subject: [PATCH 2/2] fix(hermes): honour TINYFISH_NO_TELEMETRY set in the Hermes .env Co-Authored-By: Claude Opus 5.5 --- hermes/README.md | 2 +- hermes/tests/test_setup_cli.py | 8 ++++++++ hermes/tinyfish_hermes/rest_client.py | 7 ++++--- hermes/tinyfish_hermes/setup_cli.py | 3 +++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/hermes/README.md b/hermes/README.md index 9268f68..c629936 100644 --- a/hermes/README.md +++ b/hermes/README.md @@ -94,7 +94,7 @@ When the `tinyfish` MCP server is also configured in Hermes (`mcp_servers.tinyfi ## Telemetry -`hermes tinyfish setup` sends one event to TinyFish when it finishes: the outcome (for example `completed` or `key_rejected`), where the key came from (`flag`, `prompt`, `env`), the plugin version and OS. It sends a one-way hash of the API key, never the key itself, so the event joins your account. Set `TINYFISH_NO_TELEMETRY=1` to turn it off. +Each `hermes tinyfish setup` run sends one event to TinyFish, whether it succeeds or fails: the outcome (for example `completed` or `key_rejected`), where the key came from (`flag`, `prompt`, `env` or `none`), the plugin version and OS. It sends a one-way hash of the API key, never the key itself, so the event joins your account. Set `TINYFISH_NO_TELEMETRY=1` in your shell or in `~/.hermes/.env` to turn it off. ## Development diff --git a/hermes/tests/test_setup_cli.py b/hermes/tests/test_setup_cli.py index 8be25a7..a86b298 100644 --- a/hermes/tests/test_setup_cli.py +++ b/hermes/tests/test_setup_cli.py @@ -929,3 +929,11 @@ def test_setup_reports_backend_failure( cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"])) assert [outcome for outcome, _ in env["reports"]] == ["backend_failed"] + + +def test_setup_honours_opt_out_from_hermes_env(env: dict[str, Any]) -> None: + env["env"]["TINYFISH_NO_TELEMETRY"] = "1" + + cli.dispatch_tinyfish_cli(_parser().parse_args(["setup", "--yes"])) + + assert env["reports"] == [] diff --git a/hermes/tinyfish_hermes/rest_client.py b/hermes/tinyfish_hermes/rest_client.py index 73e8fe2..7b77e29 100644 --- a/hermes/tinyfish_hermes/rest_client.py +++ b/hermes/tinyfish_hermes/rest_client.py @@ -339,10 +339,11 @@ def start_run( ) -def telemetry_disabled() -> bool: +def telemetry_disabled(value: str | None = None) -> bool: # Matches the TinyFish CLI: anything but empty, 0 or false opts out. - value = os.environ.get("TINYFISH_NO_TELEMETRY", "").strip().lower() - return value not in ("", "0", "false") + if value is None: + value = os.environ.get("TINYFISH_NO_TELEMETRY", "") + return value.strip().lower() not in ("", "0", "false") def report_setup(outcome: str, *, key_source: str, api_key: str = "") -> None: diff --git a/hermes/tinyfish_hermes/setup_cli.py b/hermes/tinyfish_hermes/setup_cli.py index 60f8bc2..1d2497a 100644 --- a/hermes/tinyfish_hermes/setup_cli.py +++ b/hermes/tinyfish_hermes/setup_cli.py @@ -268,6 +268,9 @@ def cmd_setup( # Catalog installs skip `tinyfish connect`; this is their install signal. def report(outcome: str) -> None: + # Hermes keeps settings in its .env, which never reaches os.environ. + if rest_client.telemetry_disabled(_get_env("TINYFISH_NO_TELEMETRY")): + return rest_client.report_setup( outcome, key_source=key_source, api_key=api_key or _api_key() )