Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -393,3 +393,20 @@ python3 test/demo_features.py
curl -N http://localhost:8888/event/<MISSION_KEY>/stream
# knobs (env): API_BASE, MISSION_KEY, CAT_URL (falls back to a generated image offline)
```

# Map metrics

`/metrics` (in-cluster only) reports map outcomes the HTTP status codes hide
(`code/map_metrics.py`):

| Metric | What it counts |
|---|---|
| `api_map_callbacks_total{action,result}` | map-maker callbacks: `saved`, `unknown_mapid`, `unknown_action`, `invalid`, `db_error` |
| `api_map_creates_total{result}` | Maps registered with `PUT /maps/` |
| `api_map_nats_publish_total{subject,result}` | Map jobs published to NATS; `error` means the map will never be processed |
| `api_map_list_total{result}`, `api_map_search_total{result}` | Listings and searches; a failed one answers 200 with an error body |
| `api_cache_requests_total{cache,result}` | Redis lookups for missions and space keys |

A map id is never a label. A failed callback or publish writes one JSON log
line (`"event": "map_callback_error"` / `"map_publish_error"`) with the mapid.
Dashboard: "Ollebo maps" in Grafana.
11 changes: 10 additions & 1 deletion code/db/redisCache.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import os
import redis

import map_metrics

_VALID_TTL = 3600
_INVALID_TTL = 600
_KEY_PREFIX = "mission_valid:"
Expand Down Expand Up @@ -70,9 +72,13 @@ def getCachedMission(key):
v = client.get(_MISSION_PREFIX + str(key))
except Exception as e:
print("Redis GET failed: {}".format(e))
map_metrics.CACHE_REQUESTS.labels("mission", "error").inc()
return None
if v is None:
map_metrics.CACHE_REQUESTS.labels("mission", "miss").inc()
return None
# A cached "no such mission" is still an answer from the cache.
map_metrics.CACHE_REQUESTS.labels("mission", "hit").inc()
if v == "":
return ""
try:
Expand Down Expand Up @@ -100,10 +106,13 @@ def getCachedSpaceKey(space_id):
if client is None:
return None
try:
return client.get(_SPACE_KEY_PREFIX + str(space_id))
v = client.get(_SPACE_KEY_PREFIX + str(space_id))
except Exception as e:
print("Redis GET failed: {}".format(e))
map_metrics.CACHE_REQUESTS.labels("space_key", "error").inc()
return None
map_metrics.CACHE_REQUESTS.labels("space_key", "miss" if v is None else "hit").inc()
return v


def setCachedSpaceKey(space_id, key):
Expand Down
50 changes: 50 additions & 0 deletions code/map_metrics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
"""Map pipeline outcomes as seen by the API.

Several map failures are invisible in the HTTP metrics: a callback for an
unknown map and a failed list both answer 200 with an error body, and a failed
NATS publish is swallowed (the map then sits unprocessed forever). These
counters make them visible.

Labels are bounded: action is a fixed set and result is an enum. A map id is
never a label; a failed callback writes one JSON log line with the mapid.
"""

import json

from prometheus_client import Counter


MAP_CALLBACKS = Counter(
"api_map_callbacks_total", "map-maker callbacks (POST /maps/, PATCH /maps/<mapid>)",
["action", "result"])
# result: saved | unknown_mapid | unknown_action | invalid | db_error
MAP_CREATES = Counter(
"api_map_creates_total", "Maps registered through PUT /maps/", ["result"])
# result: ok | invalid | db_error
MAP_NATS_PUBLISH = Counter(
"api_map_nats_publish_total", "Map jobs published to NATS", ["subject", "result"])
MAP_LISTS = Counter(
"api_map_list_total", "GET /maps/ listings", ["result"]) # ok | error
MAP_SEARCHES = Counter(
"api_map_search_total", "POST /search/ map searches", ["result"]) # ok | error
CACHE_REQUESTS = Counter(
"api_cache_requests_total", "Redis cache lookups", ["cache", "result"])
# cache: mission | space_key result: hit | miss | error


def action_label(action, known):
return action if action in known else "other"


def callback(action, known, result, mapid=None):
MAP_CALLBACKS.labels(action_label(action, known), result).inc()
if result != "saved":
print(json.dumps({"event": "map_callback_error", "mapid": str(mapid or ""),
"action": str(action or ""), "result": result}, sort_keys=True),
flush=True)


def query_result(result):
"""ok/error for the list and search helpers, which return {"error": ...}
instead of raising."""
return "error" if isinstance(result, dict) and "error" in result else "ok"
33 changes: 30 additions & 3 deletions code/maps.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@
from db.natsQue import addToNats
from flask import jsonify
import datetime
import json
import uuid

import map_metrics


def setJsonValidate(jsonData):
JsonStandrad = {
Expand Down Expand Up @@ -62,7 +65,9 @@ def mapsSearch(payload, groups=None):
print(payload)
if payload is None:
return '{"error":"Missing search"}'
return searchMaps(payload, groups=groups)
result = searchMaps(payload, groups=groups)
map_metrics.MAP_SEARCHES.labels(map_metrics.query_result(result)).inc()
return result



Expand All @@ -76,22 +81,36 @@ def maps(payload, request, groups=None):
print("Update database")
if payload is None:
print("ERROR {} /maps/ bad payload: missing=['<body>']".format(request.method))
map_metrics.callback(None, _KNOWN_ACTIONS, "invalid")
return jsonify({"error": "missing body", "missing": ["<body>"]}), 400
payload["recordtime"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
missing = _validate_post(payload)
if missing:
print("ERROR {} /maps/ bad payload: missing={} mapid={} action={} keys={}".format(
request.method, missing, payload.get("mapid"), payload.get("action"), sorted(payload.keys())
))
map_metrics.callback(payload.get("action"), _KNOWN_ACTIONS, "invalid", payload.get("mapid"))
return jsonify({"error": "invalid payload", "missing": missing}), 400
try:
return updateMapDataDb(payload, "maps")
result = updateMapDataDb(payload, "maps")
except Exception as e:
print("ERROR {} /maps/ db error: mapid={} err={}".format(request.method, payload.get("mapid"), e))
map_metrics.callback(payload["action"], _KNOWN_ACTIONS, "db_error", payload.get("mapid"))
return jsonify({"error": "db error", "detail": str(e)}), 500
# updateMapDataDb answers {"error": "unknown mapid" | "unknown action"}
# with a 200, so the HTTP metrics cannot see these.
outcome = "saved"
if isinstance(result, dict) and result.get("error"):
outcome = str(result["error"]).replace(" ", "_")
if outcome not in ("unknown_mapid", "unknown_action"):
outcome = "db_error"
map_metrics.callback(payload["action"], _KNOWN_ACTIONS, outcome, payload.get("mapid"))
return result
if request.method == "GET":
print("List maps (groups={})".format(groups))
return getDataDb("maps", groups=groups)
result = getDataDb("maps", groups=groups)
map_metrics.MAP_LISTS.labels(map_metrics.query_result(result)).inc()
return result
if request.method == "PUT":
print("Add to database")
if payload is None:
Expand All @@ -103,14 +122,22 @@ def maps(payload, request, groups=None):
uuid.UUID(str(clearJson["space_id"]))
except (ValueError, AttributeError, TypeError):
print("ERROR PUT /maps/ bad space_id: value={!r}".format(clearJson.get("space_id")))
map_metrics.MAP_CREATES.labels("invalid").inc()
return jsonify({"error": "invalid space_id"}), 400
try:
result = addDataDb(clearJson, "maps")
except Exception as e:
print("ERROR PUT /maps/ db error: mapid={} err={}".format(clearJson.get("mapid"), e))
map_metrics.MAP_CREATES.labels("db_error").inc()
return jsonify({"error": "db error", "detail": str(e)}), 500
map_metrics.MAP_CREATES.labels("ok").inc()
try:
addToNats("maps", clearJson)
map_metrics.MAP_NATS_PUBLISH.labels("maps", "ok").inc()
except Exception as e:
# The row exists but no worker will ever pick it up.
print("NATS publish failed: {}".format(e))
map_metrics.MAP_NATS_PUBLISH.labels("maps", "error").inc()
print(json.dumps({"event": "map_publish_error", "mapid": str(clearJson.get("mapid") or ""),
"error": str(e)}, sort_keys=True), flush=True)
return result
Loading