diff --git a/README.md b/README.md index 10baf36..54e6fec 100644 --- a/README.md +++ b/README.md @@ -1 +1,33 @@ # maps-proxy + +## Map metrics + +Besides the default per-route HTTP metrics, `/metrics` reports which maps are +used, what is served from cache and what fails (`code/map_metrics.py`): + +| Metric | What it counts | +|---|---| +| `maps_tile_requests_total{kind,layer,result}` | Vector tiles per layer: `hit`, `miss`, `empty`, `out_of_range`, `upstream_error` | +| `maps_tile_upstream_seconds{layer}` | martin render time on a cache miss | +| `maps_tile_cache_write_failures_total{layer}` | Rendered tiles that could not be stored | +| `maps_object_requests_total{kind,space,result}` | Map files, layers and models: `hit`, `miss`, `not_found`, `s3_error`, `cache_error`, `denied`, `blocked` | +| `maps_object_fetch_seconds{kind}`, `maps_object_bytes_total{kind,result}` | Download time on a miss, bytes served | +| `maps_gate_decisions_total{route,space,result}` | forwardAuth checks for terracotta raster tiles | +| `maps_gate_maxzoom_cache_total{result}` | Public maxzoom cache hit/miss | +| `maps_file_cache_bytes`, `maps_file_cache_files`, `maps_cache_purges_total{scope}` | Local `/file_cache` size and purges | + +A map id is never a label. To see which map is failing, ask the pod (the +endpoint answers 404 through the ingress, like `/metrics`): + + kubectl exec -n ollebo deploy/maps-proxy -- python3 -c \ + "import urllib.request as u; print(u.urlopen('http://localhost:8080/stats/maps?sort=errors').read().decode())" + +`sort` is `requests`, `errors` or `recent`; `top` limits the rows. Counts are +since the pod started, merged across the gunicorn workers. Every failure also +writes one JSON log line with `"event": "map_error"`. + +The catch-all answers 502 when object storage or the cache volume fails, and +404 only when the object does not exist. + +Dashboard: "Ollebo maps" in Grafana (`monitoring/dashboards-src/ollebo.py` in +the manifests repo). diff --git a/code/app.py b/code/app.py index 1ec094a..39edd80 100644 --- a/code/app.py +++ b/code/app.py @@ -31,12 +31,14 @@ def _metrics_path_label(req): def _hide_metrics_from_ingress(): # Prometheus scrapes the pod IP directly; requests via Traefik carry # X-Forwarded-For. Keep /metrics off maps.ollebo.com. - if request.path == "/metrics" and request.headers.get("X-Forwarded-For"): + # /stats/maps lists map ids, so it gets the same treatment. + if request.path in ("/metrics", "/stats/maps") and request.headers.get("X-Forwarded-For"): return jsonify({"error": "not found"}), 404 import ext import gate import k8s +import map_metrics import minioBackend import tiles import utils @@ -93,6 +95,11 @@ def _allowed_origin(): return None +@app.before_request +def _start_map_metrics(): + map_metrics.ensure_started() + + @app.before_request def _cors_preflight(): if request.method == "OPTIONS": @@ -135,6 +142,7 @@ def _cors_headers(resp): def cache(): if not utils.purge_key_ok(request): return jsonify({"error": "forbidden"}), 403 + map_metrics.CACHE_PURGES.labels("all").inc() return utils.cleanCache() @@ -145,11 +153,26 @@ def cache_purge(): body = request.get_json(silent=True) or {} prefix = body.get("prefix") or "" removed = utils.purge_prefix(minioBackend.S3_FILE_BUCKET, prefix) + map_metrics.CACHE_PURGES.labels("prefix").inc() if body.get("space") and body.get("map"): gate.forget_maxzoom(body["space"], body["map"]) return jsonify({"removed": removed}) +# Per-map request and error counts, merged across workers. In-cluster only +# (see _hide_metrics_from_ingress): map ids never go in Prometheus labels, so +# this is where "which map is failing" is answered. +# /stats/maps?sort=requests|errors|recent&top=50 +@app.route('/stats/maps') +@metrics.do_not_track() +def stats_maps(): + try: + top = int(request.args.get("top", "50")) + except ValueError: + top = 50 + return jsonify(map_metrics.snapshot(top=top, sort=request.args.get("sort", "requests"))) + + # Traefik forwardAuth targets for terracotta (spridningstillstand.md 5.2/6). @app.route('/auth/tiles') def auth_private_tiles(): diff --git a/code/gate.py b/code/gate.py index 0396f06..0ab400c 100644 --- a/code/gate.py +++ b/code/gate.py @@ -25,6 +25,7 @@ from flask import jsonify, request import auth +import map_metrics TERRACOTTA_PUBLIC_URL = os.environ.get("TERRACOTTA_PUBLIC_URL", "http://terracotta-public:5000") MAXZOOM_TTL_S = int(os.environ.get("PUBLIC_MAXZOOM_TTL_S", "600")) @@ -44,7 +45,8 @@ def max_public_zoom(max_abs_lat_deg, tile_size=256, pixel_ratio=1): _SEG = r"[A-Za-z0-9_.-]+" _PRIVATE_RE = re.compile( - r"^/private-tiles/(?:rgb|singleband)/(?P{0})/".format(_SEG) + # The map segment is only read for /stats/maps; access is per space. + r"^/private-tiles/(?:rgb|singleband)/(?P{0})/(?:(?P{0})/)?".format(_SEG) ) _PUBLIC_RGB_RE = re.compile( r"^/public/rgb/(?P{0})/(?P{0})/(?P\d+)/(?P\d+)/(?P\d+)\.png$".format(_SEG) @@ -66,15 +68,24 @@ def _forwarded(): return parts.path, parse_qs(parts.query, keep_blank_values=True) +def _decision(route, result, space=None): + # `space` is passed only once the request has proved it exists. + map_metrics.GATE_DECISIONS.labels(route, map_metrics.space_label(space), result).inc() + + def private_tiles(): path, _ = _forwarded() m = _PRIVATE_RE.match(path) if not m: + _decision("private", "not_found") return jsonify({"error": "not found"}), 404 err = auth.check_space_access(request, m.group("space")) if err is not None: status, body = err + _decision("private", "deny") return jsonify(body), status + _decision("private", "allow", m.group("space")) + map_metrics.record_map(m.group("space"), m.group("map"), "private_tile", "allow") resp = jsonify({"ok": True}) return resp, 200 @@ -85,7 +96,9 @@ def _dataset_maxzoom(space, map_id, variant): with _maxzoom_lock: hit = _maxzoom_cache.get(key) if hit and hit[0] > now: + map_metrics.GATE_MAXZOOM_CACHE.labels("hit").inc() return hit[1] + map_metrics.GATE_MAXZOOM_CACHE.labels("miss").inc() url = "{0}/metadata/{1}/{2}/{3}".format(TERRACOTTA_PUBLIC_URL, space, map_id, variant) resp = requests.get(url, timeout=5) if resp.status_code == 404: @@ -109,9 +122,11 @@ def public_tile(): path, query = _forwarded() m = _PUBLIC_RGB_RE.match(path) or _PUBLIC_SB_RE.match(path) if not m: + _decision("public", "not_found") return jsonify({"error": "not found"}), 404 unknown = set(query) - _PUBLIC_PARAMS if unknown: + _decision("public", "bad_request") return jsonify({"error": "parameter not allowed", "params": sorted(unknown)}), 400 groups = m.groupdict() @@ -121,17 +136,29 @@ def public_tile(): # three channels are built from the same coarse source. variant = (query.get("r") or [""])[0] if not variant: + _decision("public", "bad_request") return jsonify({"error": "missing r"}), 400 + space, map_id = groups["space"], groups["map"] try: - maxzoom = _dataset_maxzoom(groups["space"], groups["map"], variant) + maxzoom = _dataset_maxzoom(space, map_id, variant) except Exception as e: print("ERROR public maxzoom lookup {0}: {1}".format(path, e)) + _decision("public", "lookup_error") + map_metrics.record_map(space, map_id, "public_tile", "lookup_error", error=True) return jsonify({"error": "unavailable"}), 503 if maxzoom is None: + # Terracotta has no such dataset. For a map the viewer links to, this + # is the map being broken -- so it counts as an error in /stats/maps. + _decision("public", "not_found") + map_metrics.record_map(space, map_id, "public_tile", "not_found", error=True) return jsonify({"error": "not found"}), 404 if int(groups["z"]) > maxzoom: # Overzoom happens in the client (policy 5.2); never render past maxzoom. + _decision("public", "above_maxzoom", space) + map_metrics.record_map(space, map_id, "public_tile", "above_maxzoom") resp = jsonify({"error": "zoom above public maximum", "maxzoom": maxzoom}) resp.headers["X-Max-Zoom"] = str(maxzoom) return resp, 404 + _decision("public", "allow", space) + map_metrics.record_map(space, map_id, "public_tile", "allow") return jsonify({"ok": True}), 200 diff --git a/code/gunicorn_conf.py b/code/gunicorn_conf.py index cc9e8b3..ab60d3e 100644 --- a/code/gunicorn_conf.py +++ b/code/gunicorn_conf.py @@ -1,8 +1,16 @@ # Gunicorn runs several worker processes. prometheus_client multiprocess mode # aggregates them from files in PROMETHEUS_MULTIPROC_DIR (set by start.sh); # without it every scrape only sees whichever worker answered. +import os + from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics def child_exit(server, worker): GunicornInternalPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid) + # The worker's per-map table (map_metrics.py); /stats/maps merges these. + try: + os.remove(os.path.join( + os.environ.get("MAP_STATS_DIR", "/tmp/map-stats"), "{0}.json".format(worker.pid))) + except OSError: + pass diff --git a/code/map_metrics.py b/code/map_metrics.py new file mode 100644 index 0000000..c162b7e --- /dev/null +++ b/code/map_metrics.py @@ -0,0 +1,261 @@ +"""Which maps are used, what comes from cache, and what fails. + +Two outputs, on purpose: + +Prometheus series carry only bounded labels -- layer, kind, result and space. +A map id is NEVER a label: maps grow without bound and a series per map is how +a Prometheus dies (same rule as map-maker/code/metrics.py). A space is only +used as a label once the request has proved the space exists (membership check +passed, or the object/dataset was really there), so a scanner inventing paths +cannot mint series; `space_label` caps the set as a last resort. + +Per-map detail lives in a small in-memory table that /stats/maps serves as +JSON, plus one JSON log line per failure. gunicorn runs several workers, so +each worker writes its table to MAP_STATS_DIR/.json and /stats/maps merges +the files -- otherwise the answer would depend on which worker took the call. +""" + +import collections +import json +import os +import threading +import time + +from prometheus_client import Counter, Gauge, Histogram + + +# --- prometheus ------------------------------------------------------------ + +TILE_REQUESTS = Counter( + "maps_tile_requests_total", "Self-hosted vector tile requests by layer and cache result", + ["kind", "layer", "result"]) # hit | miss | empty | out_of_range | upstream_error +TILE_UPSTREAM_LATENCY = Histogram( + "maps_tile_upstream_seconds", "martin render latency on a tile cache miss", ["layer"]) +TILE_CACHE_WRITE_FAILURES = Counter( + "maps_tile_cache_write_failures_total", "Rendered tiles that could not be written to the cache", + ["layer"]) + +OBJECT_REQUESTS = Counter( + "maps_object_requests_total", "Catch-all object requests (map files, layers, models)", + ["kind", "space", "result"]) +# kind: public | private | model +# result: hit | miss | not_found | s3_error | cache_error | denied | blocked +OBJECT_FETCH_LATENCY = Histogram( + "maps_object_fetch_seconds", "S3 download latency on a file cache miss", ["kind"], + buckets=(0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120)) +OBJECT_BYTES = Counter( + "maps_object_bytes_total", "Bytes served by the catch-all", ["kind", "result"]) + +GATE_DECISIONS = Counter( + "maps_gate_decisions_total", "forwardAuth decisions for terracotta raster tiles", + ["route", "space", "result"]) +# route: public | private +# result: allow | deny | above_maxzoom | not_found | bad_request | lookup_error +GATE_MAXZOOM_CACHE = Counter( + "maps_gate_maxzoom_cache_total", "Lookups in the public maxzoom cache", ["result"]) + +# Every worker walks the same directory, so the newest live value is the answer. +FILE_CACHE_BYTES = Gauge( + "maps_file_cache_bytes", "Bytes held in the local object cache", + multiprocess_mode="livemax") +FILE_CACHE_FILES = Gauge( + "maps_file_cache_files", "Files held in the local object cache", + multiprocess_mode="livemax") +CACHE_PURGES = Counter( + "maps_cache_purges_total", "Local object cache purges", ["scope"]) # prefix | all + + +MAX_SPACE_LABELS = int(os.environ.get("MAP_METRICS_MAX_SPACES", "500")) +NO_SPACE = "-" +_space_labels = set() + + +def space_label(space): + """Label value for a space the request has already proved to exist.""" + if not space: + return NO_SPACE + if space in _space_labels: + return space + if len(_space_labels) >= MAX_SPACE_LABELS: + return "other" + _space_labels.add(space) + return space + + +# --- per-map table --------------------------------------------------------- + +STATS_DIR = os.environ.get("MAP_STATS_DIR", "/tmp/map-stats") +MAX_MAPS = int(os.environ.get("MAP_STATS_MAX", "2000")) +DUMP_INTERVAL_S = float(os.environ.get("MAP_STATS_DUMP_INTERVAL", "15")) +FILE_CACHE_DIR = os.environ.get("FILE_CACHE_DIR", "/file_cache") +FILE_CACHE_SCAN_INTERVAL_S = float(os.environ.get("FILE_CACHE_SCAN_INTERVAL", "300")) + +_lock = threading.Lock() +_maps = collections.OrderedDict() # (space, map) -> entry, least recently seen first +_dirty = False +_worker_pid = None + + +def _log(event, **fields): + fields["event"] = event + print(json.dumps(fields, sort_keys=True), flush=True) + + +def record_map(space, map_id, kind, result, error=False): + """Count one request against a map. `kind:result` is the breakdown key. + + Unlike the Prometheus labels this takes ids straight from the path, so a + map that fails because it does not exist still shows up. The table is + capped; the least recently seen map is dropped first. + """ + global _dirty + if not map_id: + return + key = (space or NO_SPACE, map_id) + now = time.time() + with _lock: + entry = _maps.get(key) + if entry is None: + entry = {"space": key[0], "map": map_id, "requests": 0, "errors": 0, + "results": {}, "last_seen": now, "last_error": None, + "last_error_at": None} + _maps[key] = entry + while len(_maps) > MAX_MAPS: + _maps.popitem(last=False) + else: + _maps.move_to_end(key) + entry["requests"] += 1 + entry["last_seen"] = now + bucket = "{0}:{1}".format(kind, result) + entry["results"][bucket] = entry["results"].get(bucket, 0) + 1 + if error: + entry["errors"] += 1 + entry["last_error"] = bucket + entry["last_error_at"] = now + _dirty = True + if error: + _log("map_error", space=key[0], map=map_id, kind=kind, result=result) + + +def log_error(kind, result, **fields): + """A failure with no map id to hang it on (a vector layer, a stray key).""" + _log("map_error", kind=kind, result=result, **fields) + + +def _stats_file(pid): + return os.path.join(STATS_DIR, "{0}.json".format(pid)) + + +def _dump(): + global _dirty + with _lock: + if not _dirty: + return + rows = list(_maps.values()) + payload = json.dumps(rows) + _dirty = False + try: + os.makedirs(STATS_DIR, exist_ok=True) + tmp = _stats_file(os.getpid()) + ".tmp" + with open(tmp, "w") as handle: + handle.write(payload) + os.replace(tmp, _stats_file(os.getpid())) + except OSError as error: + print("map stats dump failed: {0}".format(error)) + + +def _scan_file_cache(): + total = files = 0 + for root, _, names in os.walk(FILE_CACHE_DIR): + for name in names: + try: + total += os.path.getsize(os.path.join(root, name)) + files += 1 + except OSError: + pass # purged while we were walking + FILE_CACHE_BYTES.set(total) + FILE_CACHE_FILES.set(files) + + +def _background(): + next_scan = 0 + while True: + try: + if time.time() >= next_scan: + _scan_file_cache() + next_scan = time.time() + FILE_CACHE_SCAN_INTERVAL_S + _dump() + except Exception as error: # never let the reporter thread die + print("map metrics background error: {0}".format(error)) + time.sleep(DUMP_INTERVAL_S) + + +def ensure_started(): + """Start the per-worker reporter thread. Called per request: gunicorn forks + after import, and a thread started at import time would not survive it.""" + global _worker_pid + pid = os.getpid() + if _worker_pid == pid: + return + with _lock: + if _worker_pid == pid: + return + _worker_pid = pid + threading.Thread(target=_background, name="map-metrics", daemon=True).start() + + +def _merge(into, row): + key = (row["space"], row["map"]) + have = into.get(key) + if have is None: + into[key] = dict(row, results=dict(row["results"])) + return + have["requests"] += row["requests"] + have["errors"] += row["errors"] + for bucket, count in row["results"].items(): + have["results"][bucket] = have["results"].get(bucket, 0) + count + have["last_seen"] = max(have["last_seen"], row["last_seen"]) + if (row.get("last_error_at") or 0) > (have.get("last_error_at") or 0): + have["last_error"] = row["last_error"] + have["last_error_at"] = row["last_error_at"] + + +def snapshot(top=50, sort="requests"): + """Per-map counts merged across every worker, since the pod started.""" + _dump() + merged = {} + workers = 0 + try: + names = [n for n in os.listdir(STATS_DIR) if n.endswith(".json")] + except OSError: + names = [] + for name in names: + try: + with open(os.path.join(STATS_DIR, name)) as handle: + rows = json.load(handle) + except (OSError, ValueError): + continue + workers += 1 + for row in rows: + _merge(merged, row) + if not names: + # No stats directory (tests, read-only fs): this worker only. + with _lock: + for row in _maps.values(): + _merge(merged, row) + rows = list(merged.values()) + if sort == "errors": + rows = [r for r in rows if r["errors"]] + rows.sort(key=lambda r: (r["errors"], r["last_error_at"] or 0), reverse=True) + elif sort == "recent": + rows.sort(key=lambda r: r["last_seen"], reverse=True) + else: + sort = "requests" + rows.sort(key=lambda r: r["requests"], reverse=True) + return { + "sort": sort, + "workers": workers, + "tracked": len(merged), + "max_tracked_per_worker": MAX_MAPS, + "maps": rows[:max(0, top)], + } diff --git a/code/minioBackend.py b/code/minioBackend.py index 602ae44..e2ba267 100644 --- a/code/minioBackend.py +++ b/code/minioBackend.py @@ -1,9 +1,15 @@ from minio import Minio +from minio.error import S3Error from flask import make_response, request, jsonify +import errno import os import os.path +import time + +import urllib3 import auth +import map_metrics S3_FILE_BUCKET = os.environ.get('AWS_S3_FILE_BUCKET') @@ -100,6 +106,56 @@ def _space_of(key): return parts[0] if parts else "" +def _kind_of(filename): + if filename.startswith("private/"): + return "private" + if filename.startswith("models/"): + return "model" + return "public" + + +def _map_of(kind, key): + """(space, map/model id) a key belongs to, or (None, None). + + Only for /stats/maps and logs -- these come straight from the path and are + never used as a Prometheus label. + private /cogs/// + public public//// + model // + """ + parts = (key or "").split("/") + if kind == "private" and len(parts) >= 5 and parts[1] == "cogs": + return parts[2], parts[3] + if kind == "public" and len(parts) >= 5 and parts[0] == "public": + return parts[2], parts[3] + if kind == "model" and len(parts) >= 3: + return parts[0], parts[1] + return None, None + + +# OSErrors that mean "this path cannot name an object" (a key that runs through +# an already cached file, a key that is a cached directory, an absurd name). +# Anything else from the filesystem -- a full cache volume above all -- is ours. +_PATH_ERRNOS = frozenset([ + errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG, errno.ENOENT, errno.EINVAL]) + + +def _classify_fetch_error(error): + """not_found | s3_error | cache_error for a failed download. + + Everything used to be a 404, so an object-storage outage looked exactly + like a missing file. Only a failure we can positively pin on storage or on + the cache volume is reported as one; anything unrecognised stays not_found. + """ + if isinstance(error, S3Error): + return "not_found" if error.code in ("NoSuchKey", "NoSuchBucket") else "s3_error" + if isinstance(error, urllib3.exceptions.HTTPError): + return "s3_error" + if isinstance(error, OSError): + return "not_found" if error.errno in _PATH_ERRNOS else "cache_error" + return "not_found" + + def getFile(filename): print(filename) if filename == "" or filename == "index.html": @@ -108,59 +164,72 @@ def getFile(filename): return response bucket, key = resolveBucket(filename) + kind = _kind_of(filename) + map_space, map_id = _map_of(kind, key) + + def count(result, space=None): + # `space` is passed only once the request has proved it exists. + map_metrics.OBJECT_REQUESTS.labels( + kind, map_metrics.space_label(space), result).inc() # spridningstillstand.md 5.3/5.6: the anonymous catch-all never serves # COGs or detections outside the coarse public/ prefix, nor the legacy # /cogs/... layout (pre-compliance public maps were too fine). if bucket == S3_FILE_BUCKET and _blocked_public_key(key): + count("blocked") return jsonify({"error": "not found"}), 404 - if filename.startswith("private/") or filename.startswith("models/"): + # Membership has been checked for private/ and models/, so that space is + # real. A public key's space is only trusted once the object turns up. + known_space = None + if kind != "public": space_id = _space_of(key) if not space_id: + count("denied") return jsonify({"error": "missing space id"}), 400 err = auth.check_space_access(request, space_id) if err is not None: status, body = err + count("denied") return jsonify(body), status + known_space = space_id - WeHaveFile = False - fileIsDownloaded = False FILE_DEST = FILE_PREFIX + bucket + "/" + key - if os.path.isfile(FILE_DEST): - fileIsDownloaded = True - WeHaveFile = True - print("We have a local copy of the file" ) - - - if not fileIsDownloaded: + result = "hit" + if not os.path.isfile(FILE_DEST): + result = "miss" + started = time.time() try: os.makedirs(os.path.dirname(FILE_DEST), exist_ok=True) #Get the file from the bucket client.fget_object(bucket, key, FILE_DEST) - WeHaveFile = True - print("file downloaded") - except: - print(FILE_DEST) - # A bare string is a 200 in Flask, which made every miss look like a - # successful fetch of the body "File not found" -- fine for an , - # fatal for anything that parses the response (MVT, GeoJSON). - return jsonify({"error": "not found"}), 404 - # - ##Read the file - if WeHaveFile: - f = open(FILE_DEST, "rb") + except Exception as error: + failure = _classify_fetch_error(error) + count(failure, known_space) + if map_id: + map_metrics.record_map(map_space, map_id, kind + "_object", failure, error=True) + if failure == "not_found": + # A bare string is a 200 in Flask, which made every miss look like a + # successful fetch of the body "File not found" -- fine for an , + # fatal for anything that parses the response (MVT, GeoJSON). + return jsonify({"error": "not found"}), 404 + print("ERROR fetching {0}/{1}: {2!r}".format(bucket, key, error)) + if not map_id: + map_metrics.log_error(kind + "_object", failure, key=key, error=repr(error)) + return jsonify({"error": "storage unavailable"}), 502 + map_metrics.OBJECT_FETCH_LATENCY.labels(kind).observe(time.time() - started) + + with open(FILE_DEST, "rb") as f: theFile = f.read() - f.close() - #set the content type - response = make_response(theFile) - - #set the content type - filename, file_extension = os.path.splitext(FILE_DEST) - content_type = CONTENT_TYPES.get(file_extension.lower()) - if content_type: - response.headers.set('Content-Type', content_type) - return response - else: - return jsonify({"error": "not found"}), 404 + count(result, known_space or map_space) + map_metrics.OBJECT_BYTES.labels(kind, result).inc(len(theFile)) + if map_id: + map_metrics.record_map(map_space, map_id, kind + "_object", result) + + response = make_response(theFile) + _, file_extension = os.path.splitext(FILE_DEST) + content_type = CONTENT_TYPES.get(file_extension.lower()) + if content_type: + response.headers.set('Content-Type', content_type) + return response diff --git a/code/start.sh b/code/start.sh index 3b247a9..fc537a3 100755 --- a/code/start.sh +++ b/code/start.sh @@ -8,6 +8,9 @@ echo "Atring up the http server" # /metrics aggregates them. Must be empty at start. export PROMETHEUS_MULTIPROC_DIR="${PROMETHEUS_MULTIPROC_DIR:-/tmp/prometheus-multiproc}" rm -rf "$PROMETHEUS_MULTIPROC_DIR" && mkdir -p "$PROMETHEUS_MULTIPROC_DIR" +# Per-worker map tables that /stats/maps merges (map_metrics.py). Same rule. +export MAP_STATS_DIR="${MAP_STATS_DIR:-/tmp/map-stats}" +rm -rf "$MAP_STATS_DIR" && mkdir -p "$MAP_STATS_DIR" gunicorn -c /code/gunicorn_conf.py -w "${GUNICORN_WORKERS:-4}" -k gthread --threads "${GUNICORN_THREADS:-8}" \ --timeout "${GUNICORN_TIMEOUT:-120}" -b 0.0.0.0:8080 'app:app' \ No newline at end of file diff --git a/code/tiles.py b/code/tiles.py index e1ed827..bbcd48f 100644 --- a/code/tiles.py +++ b/code/tiles.py @@ -6,6 +6,7 @@ import requests from flask import Response, jsonify, make_response, request +import map_metrics import minioBackend @@ -71,10 +72,12 @@ def _put_object(bucket, key, data, content_type): try: minioBackend.client.put_object( bucket, key, io.BytesIO(data), len(data), content_type=content_type) + return True except Exception as error: # A cache write failing must not fail the request -- the tile is already # rendered and in hand; the next request just pays for it again. print("tile cache write failed for {0}: {1}".format(key, error)) + return False def _metadata_key(layer, name): @@ -134,7 +137,11 @@ def tile(layer, z, x, y): if metadata is None: return minioBackend.getFile(static_path) + # From here on the layer is one we host, so its name is a safe label. + count = lambda result: map_metrics.TILE_REQUESTS.labels("vector", layer, result).inc() + if z < metadata.get("minzoom", 0) or z > metadata.get("maxzoom", 22): + count("out_of_range") return Response(status=204) version = (metadata.get("ollebo") or {}).get("version", 1) @@ -143,21 +150,30 @@ def tile(layer, z, x, y): cached = _get_object(TILE_CACHE_BUCKET, key) if cached is not None: + count("hit") return _tile_response(cached) + started = time.time() try: data = _fetch_from_martin("tile_" + layer.replace("-", "_"), z, x, y) except requests.RequestException as error: print("martin request failed for {0}/{1}/{2}/{3}: {4}".format( layer, z, x, y, error)) + count("upstream_error") + map_metrics.log_error("vector", "upstream_error", layer=layer, + tile="{0}/{1}/{2}".format(z, x, y), error=str(error)) return jsonify({"error": "tile backend unavailable"}), 502 + map_metrics.TILE_UPSTREAM_LATENCY.labels(layer).observe(time.time() - started) if not data: # An empty tile is a real answer, not an error: the layer has nothing # here. 204 keeps it out of the cache so a later import can fill it. + count("empty") return Response(status=204) - _put_object(TILE_CACHE_BUCKET, key, data, MVT_CONTENT_TYPE) + count("miss") + if not _put_object(TILE_CACHE_BUCKET, key, data, MVT_CONTENT_TYPE): + map_metrics.TILE_CACHE_WRITE_FAILURES.labels(layer).inc() return _tile_response(data)