From 292c5abef27efdef5e9c27271c8f715114a91b67 Mon Sep 17 00:00:00 2001 From: Mattias Hemmingsson Date: Wed, 18 Nov 2026 18:59:30 +0100 Subject: [PATCH 1/2] Count map callback, publish, list and cache outcomes Callbacks for an unknown map, failed listings and failed NATS publishes all answered 200 (or were swallowed), so the HTTP metrics could not see them. Add bounded counters for each, plus Redis cache hit/miss, and one JSON log line with the mapid when a callback or publish fails. Co-Authored-By: Claude Fable 5.2 --- code/db/redisCache.py | 11 +++++++++- code/map_metrics.py | 50 +++++++++++++++++++++++++++++++++++++++++++ code/maps.py | 33 +++++++++++++++++++++++++--- 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 code/map_metrics.py diff --git a/code/db/redisCache.py b/code/db/redisCache.py index 1997365..e016358 100644 --- a/code/db/redisCache.py +++ b/code/db/redisCache.py @@ -2,6 +2,8 @@ import os import redis +import map_metrics + _VALID_TTL = 3600 _INVALID_TTL = 600 _KEY_PREFIX = "mission_valid:" @@ -70,9 +72,13 @@ def getCachedMission(key): v = client.get(_MISSION_PREFIX + str(key)) except Exception as e: print("Redis GET failed: {}".format(e)) + map_metrics.CACHE_REQUESTS.labels("mission", "error").inc() return None if v is None: + map_metrics.CACHE_REQUESTS.labels("mission", "miss").inc() return None + # A cached "no such mission" is still an answer from the cache. + map_metrics.CACHE_REQUESTS.labels("mission", "hit").inc() if v == "": return "" try: @@ -100,10 +106,13 @@ def getCachedSpaceKey(space_id): if client is None: return None try: - return client.get(_SPACE_KEY_PREFIX + str(space_id)) + v = client.get(_SPACE_KEY_PREFIX + str(space_id)) except Exception as e: print("Redis GET failed: {}".format(e)) + map_metrics.CACHE_REQUESTS.labels("space_key", "error").inc() return None + map_metrics.CACHE_REQUESTS.labels("space_key", "miss" if v is None else "hit").inc() + return v def setCachedSpaceKey(space_id, key): diff --git a/code/map_metrics.py b/code/map_metrics.py new file mode 100644 index 0000000..30bf1ba --- /dev/null +++ b/code/map_metrics.py @@ -0,0 +1,50 @@ +"""Map pipeline outcomes as seen by the API. + +Several map failures are invisible in the HTTP metrics: a callback for an +unknown map and a failed list both answer 200 with an error body, and a failed +NATS publish is swallowed (the map then sits unprocessed forever). These +counters make them visible. + +Labels are bounded: action is a fixed set and result is an enum. A map id is +never a label; a failed callback writes one JSON log line with the mapid. +""" + +import json + +from prometheus_client import Counter + + +MAP_CALLBACKS = Counter( + "api_map_callbacks_total", "map-maker callbacks (POST /maps/, PATCH /maps/)", + ["action", "result"]) +# result: saved | unknown_mapid | unknown_action | invalid | db_error +MAP_CREATES = Counter( + "api_map_creates_total", "Maps registered through PUT /maps/", ["result"]) +# result: ok | invalid | db_error +MAP_NATS_PUBLISH = Counter( + "api_map_nats_publish_total", "Map jobs published to NATS", ["subject", "result"]) +MAP_LISTS = Counter( + "api_map_list_total", "GET /maps/ listings", ["result"]) # ok | error +MAP_SEARCHES = Counter( + "api_map_search_total", "POST /search/ map searches", ["result"]) # ok | error +CACHE_REQUESTS = Counter( + "api_cache_requests_total", "Redis cache lookups", ["cache", "result"]) +# cache: mission | space_key result: hit | miss | error + + +def action_label(action, known): + return action if action in known else "other" + + +def callback(action, known, result, mapid=None): + MAP_CALLBACKS.labels(action_label(action, known), result).inc() + if result != "saved": + print(json.dumps({"event": "map_callback_error", "mapid": str(mapid or ""), + "action": str(action or ""), "result": result}, sort_keys=True), + flush=True) + + +def query_result(result): + """ok/error for the list and search helpers, which return {"error": ...} + instead of raising.""" + return "error" if isinstance(result, dict) and "error" in result else "ok" diff --git a/code/maps.py b/code/maps.py index 01da593..6d10286 100644 --- a/code/maps.py +++ b/code/maps.py @@ -2,8 +2,11 @@ from db.natsQue import addToNats from flask import jsonify import datetime +import json import uuid +import map_metrics + def setJsonValidate(jsonData): JsonStandrad = { @@ -62,7 +65,9 @@ def mapsSearch(payload, groups=None): print(payload) if payload is None: return '{"error":"Missing search"}' - return searchMaps(payload, groups=groups) + result = searchMaps(payload, groups=groups) + map_metrics.MAP_SEARCHES.labels(map_metrics.query_result(result)).inc() + return result @@ -76,6 +81,7 @@ def maps(payload, request, groups=None): print("Update database") if payload is None: print("ERROR {} /maps/ bad payload: missing=['']".format(request.method)) + map_metrics.callback(None, _KNOWN_ACTIONS, "invalid") return jsonify({"error": "missing body", "missing": [""]}), 400 payload["recordtime"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") missing = _validate_post(payload) @@ -83,15 +89,28 @@ def maps(payload, request, groups=None): print("ERROR {} /maps/ bad payload: missing={} mapid={} action={} keys={}".format( request.method, missing, payload.get("mapid"), payload.get("action"), sorted(payload.keys()) )) + map_metrics.callback(payload.get("action"), _KNOWN_ACTIONS, "invalid", payload.get("mapid")) return jsonify({"error": "invalid payload", "missing": missing}), 400 try: - return updateMapDataDb(payload, "maps") + result = updateMapDataDb(payload, "maps") except Exception as e: print("ERROR {} /maps/ db error: mapid={} err={}".format(request.method, payload.get("mapid"), e)) + map_metrics.callback(payload["action"], _KNOWN_ACTIONS, "db_error", payload.get("mapid")) return jsonify({"error": "db error", "detail": str(e)}), 500 + # updateMapDataDb answers {"error": "unknown mapid" | "unknown action"} + # with a 200, so the HTTP metrics cannot see these. + outcome = "saved" + if isinstance(result, dict) and result.get("error"): + outcome = str(result["error"]).replace(" ", "_") + if outcome not in ("unknown_mapid", "unknown_action"): + outcome = "db_error" + map_metrics.callback(payload["action"], _KNOWN_ACTIONS, outcome, payload.get("mapid")) + return result if request.method == "GET": print("List maps (groups={})".format(groups)) - return getDataDb("maps", groups=groups) + result = getDataDb("maps", groups=groups) + map_metrics.MAP_LISTS.labels(map_metrics.query_result(result)).inc() + return result if request.method == "PUT": print("Add to database") if payload is None: @@ -103,14 +122,22 @@ def maps(payload, request, groups=None): uuid.UUID(str(clearJson["space_id"])) except (ValueError, AttributeError, TypeError): print("ERROR PUT /maps/ bad space_id: value={!r}".format(clearJson.get("space_id"))) + map_metrics.MAP_CREATES.labels("invalid").inc() return jsonify({"error": "invalid space_id"}), 400 try: result = addDataDb(clearJson, "maps") except Exception as e: print("ERROR PUT /maps/ db error: mapid={} err={}".format(clearJson.get("mapid"), e)) + map_metrics.MAP_CREATES.labels("db_error").inc() return jsonify({"error": "db error", "detail": str(e)}), 500 + map_metrics.MAP_CREATES.labels("ok").inc() try: addToNats("maps", clearJson) + map_metrics.MAP_NATS_PUBLISH.labels("maps", "ok").inc() except Exception as e: + # The row exists but no worker will ever pick it up. print("NATS publish failed: {}".format(e)) + map_metrics.MAP_NATS_PUBLISH.labels("maps", "error").inc() + print(json.dumps({"event": "map_publish_error", "mapid": str(clearJson.get("mapid") or ""), + "error": str(e)}, sort_keys=True), flush=True) return result From 0d97b3f1a4c33532b92d78e79ffe6074dd9e83ee Mon Sep 17 00:00:00 2001 From: Mattias Hemmingsson Date: Wed, 18 Nov 2026 19:07:21 +0100 Subject: [PATCH 2/2] Document the map usage metrics Co-Authored-By: Claude Fable 5.2 --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index c423fae..a018930 100755 --- a/README.md +++ b/README.md @@ -393,3 +393,20 @@ python3 test/demo_features.py curl -N http://localhost:8888/event//stream # knobs (env): API_BASE, MISSION_KEY, CAT_URL (falls back to a generated image offline) ``` + +# Map metrics + +`/metrics` (in-cluster only) reports map outcomes the HTTP status codes hide +(`code/map_metrics.py`): + +| Metric | What it counts | +|---|---| +| `api_map_callbacks_total{action,result}` | map-maker callbacks: `saved`, `unknown_mapid`, `unknown_action`, `invalid`, `db_error` | +| `api_map_creates_total{result}` | Maps registered with `PUT /maps/` | +| `api_map_nats_publish_total{subject,result}` | Map jobs published to NATS; `error` means the map will never be processed | +| `api_map_list_total{result}`, `api_map_search_total{result}` | Listings and searches; a failed one answers 200 with an error body | +| `api_cache_requests_total{cache,result}` | Redis lookups for missions and space keys | + +A map id is never a label. A failed callback or publish writes one JSON log +line (`"event": "map_callback_error"` / `"map_publish_error"`) with the mapid. +Dashboard: "Ollebo maps" in Grafana.