diff --git a/README.md b/README.md index c423fae..a018930 100755 --- a/README.md +++ b/README.md @@ -393,3 +393,20 @@ python3 test/demo_features.py curl -N http://localhost:8888/event//stream # knobs (env): API_BASE, MISSION_KEY, CAT_URL (falls back to a generated image offline) ``` + +# Map metrics + +`/metrics` (in-cluster only) reports map outcomes the HTTP status codes hide +(`code/map_metrics.py`): + +| Metric | What it counts | +|---|---| +| `api_map_callbacks_total{action,result}` | map-maker callbacks: `saved`, `unknown_mapid`, `unknown_action`, `invalid`, `db_error` | +| `api_map_creates_total{result}` | Maps registered with `PUT /maps/` | +| `api_map_nats_publish_total{subject,result}` | Map jobs published to NATS; `error` means the map will never be processed | +| `api_map_list_total{result}`, `api_map_search_total{result}` | Listings and searches; a failed one answers 200 with an error body | +| `api_cache_requests_total{cache,result}` | Redis lookups for missions and space keys | + +A map id is never a label. A failed callback or publish writes one JSON log +line (`"event": "map_callback_error"` / `"map_publish_error"`) with the mapid. +Dashboard: "Ollebo maps" in Grafana. diff --git a/code/db/redisCache.py b/code/db/redisCache.py index 1997365..e016358 100644 --- a/code/db/redisCache.py +++ b/code/db/redisCache.py @@ -2,6 +2,8 @@ import os import redis +import map_metrics + _VALID_TTL = 3600 _INVALID_TTL = 600 _KEY_PREFIX = "mission_valid:" @@ -70,9 +72,13 @@ def getCachedMission(key): v = client.get(_MISSION_PREFIX + str(key)) except Exception as e: print("Redis GET failed: {}".format(e)) + map_metrics.CACHE_REQUESTS.labels("mission", "error").inc() return None if v is None: + map_metrics.CACHE_REQUESTS.labels("mission", "miss").inc() return None + # A cached "no such mission" is still an answer from the cache. + map_metrics.CACHE_REQUESTS.labels("mission", "hit").inc() if v == "": return "" try: @@ -100,10 +106,13 @@ def getCachedSpaceKey(space_id): if client is None: return None try: - return client.get(_SPACE_KEY_PREFIX + str(space_id)) + v = client.get(_SPACE_KEY_PREFIX + str(space_id)) except Exception as e: print("Redis GET failed: {}".format(e)) + map_metrics.CACHE_REQUESTS.labels("space_key", "error").inc() return None + map_metrics.CACHE_REQUESTS.labels("space_key", "miss" if v is None else "hit").inc() + return v def setCachedSpaceKey(space_id, key): diff --git a/code/map_metrics.py b/code/map_metrics.py new file mode 100644 index 0000000..30bf1ba --- /dev/null +++ b/code/map_metrics.py @@ -0,0 +1,50 @@ +"""Map pipeline outcomes as seen by the API. + +Several map failures are invisible in the HTTP metrics: a callback for an +unknown map and a failed list both answer 200 with an error body, and a failed +NATS publish is swallowed (the map then sits unprocessed forever). These +counters make them visible. + +Labels are bounded: action is a fixed set and result is an enum. A map id is +never a label; a failed callback writes one JSON log line with the mapid. +""" + +import json + +from prometheus_client import Counter + + +MAP_CALLBACKS = Counter( + "api_map_callbacks_total", "map-maker callbacks (POST /maps/, PATCH /maps/)", + ["action", "result"]) +# result: saved | unknown_mapid | unknown_action | invalid | db_error +MAP_CREATES = Counter( + "api_map_creates_total", "Maps registered through PUT /maps/", ["result"]) +# result: ok | invalid | db_error +MAP_NATS_PUBLISH = Counter( + "api_map_nats_publish_total", "Map jobs published to NATS", ["subject", "result"]) +MAP_LISTS = Counter( + "api_map_list_total", "GET /maps/ listings", ["result"]) # ok | error +MAP_SEARCHES = Counter( + "api_map_search_total", "POST /search/ map searches", ["result"]) # ok | error +CACHE_REQUESTS = Counter( + "api_cache_requests_total", "Redis cache lookups", ["cache", "result"]) +# cache: mission | space_key result: hit | miss | error + + +def action_label(action, known): + return action if action in known else "other" + + +def callback(action, known, result, mapid=None): + MAP_CALLBACKS.labels(action_label(action, known), result).inc() + if result != "saved": + print(json.dumps({"event": "map_callback_error", "mapid": str(mapid or ""), + "action": str(action or ""), "result": result}, sort_keys=True), + flush=True) + + +def query_result(result): + """ok/error for the list and search helpers, which return {"error": ...} + instead of raising.""" + return "error" if isinstance(result, dict) and "error" in result else "ok" diff --git a/code/maps.py b/code/maps.py index 01da593..6d10286 100644 --- a/code/maps.py +++ b/code/maps.py @@ -2,8 +2,11 @@ from db.natsQue import addToNats from flask import jsonify import datetime +import json import uuid +import map_metrics + def setJsonValidate(jsonData): JsonStandrad = { @@ -62,7 +65,9 @@ def mapsSearch(payload, groups=None): print(payload) if payload is None: return '{"error":"Missing search"}' - return searchMaps(payload, groups=groups) + result = searchMaps(payload, groups=groups) + map_metrics.MAP_SEARCHES.labels(map_metrics.query_result(result)).inc() + return result @@ -76,6 +81,7 @@ def maps(payload, request, groups=None): print("Update database") if payload is None: print("ERROR {} /maps/ bad payload: missing=['']".format(request.method)) + map_metrics.callback(None, _KNOWN_ACTIONS, "invalid") return jsonify({"error": "missing body", "missing": [""]}), 400 payload["recordtime"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") missing = _validate_post(payload) @@ -83,15 +89,28 @@ def maps(payload, request, groups=None): print("ERROR {} /maps/ bad payload: missing={} mapid={} action={} keys={}".format( request.method, missing, payload.get("mapid"), payload.get("action"), sorted(payload.keys()) )) + map_metrics.callback(payload.get("action"), _KNOWN_ACTIONS, "invalid", payload.get("mapid")) return jsonify({"error": "invalid payload", "missing": missing}), 400 try: - return updateMapDataDb(payload, "maps") + result = updateMapDataDb(payload, "maps") except Exception as e: print("ERROR {} /maps/ db error: mapid={} err={}".format(request.method, payload.get("mapid"), e)) + map_metrics.callback(payload["action"], _KNOWN_ACTIONS, "db_error", payload.get("mapid")) return jsonify({"error": "db error", "detail": str(e)}), 500 + # updateMapDataDb answers {"error": "unknown mapid" | "unknown action"} + # with a 200, so the HTTP metrics cannot see these. + outcome = "saved" + if isinstance(result, dict) and result.get("error"): + outcome = str(result["error"]).replace(" ", "_") + if outcome not in ("unknown_mapid", "unknown_action"): + outcome = "db_error" + map_metrics.callback(payload["action"], _KNOWN_ACTIONS, outcome, payload.get("mapid")) + return result if request.method == "GET": print("List maps (groups={})".format(groups)) - return getDataDb("maps", groups=groups) + result = getDataDb("maps", groups=groups) + map_metrics.MAP_LISTS.labels(map_metrics.query_result(result)).inc() + return result if request.method == "PUT": print("Add to database") if payload is None: @@ -103,14 +122,22 @@ def maps(payload, request, groups=None): uuid.UUID(str(clearJson["space_id"])) except (ValueError, AttributeError, TypeError): print("ERROR PUT /maps/ bad space_id: value={!r}".format(clearJson.get("space_id"))) + map_metrics.MAP_CREATES.labels("invalid").inc() return jsonify({"error": "invalid space_id"}), 400 try: result = addDataDb(clearJson, "maps") except Exception as e: print("ERROR PUT /maps/ db error: mapid={} err={}".format(clearJson.get("mapid"), e)) + map_metrics.MAP_CREATES.labels("db_error").inc() return jsonify({"error": "db error", "detail": str(e)}), 500 + map_metrics.MAP_CREATES.labels("ok").inc() try: addToNats("maps", clearJson) + map_metrics.MAP_NATS_PUBLISH.labels("maps", "ok").inc() except Exception as e: + # The row exists but no worker will ever pick it up. print("NATS publish failed: {}".format(e)) + map_metrics.MAP_NATS_PUBLISH.labels("maps", "error").inc() + print(json.dumps({"event": "map_publish_error", "mapid": str(clearJson.get("mapid") or ""), + "error": str(e)}, sort_keys=True), flush=True) return result