Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1 +1,33 @@
# maps-proxy

## Map metrics

Besides the default per-route HTTP metrics, `/metrics` reports which maps are
used, what is served from cache and what fails (`code/map_metrics.py`):

| Metric | What it counts |
|---|---|
| `maps_tile_requests_total{kind,layer,result}` | Vector tiles per layer: `hit`, `miss`, `empty`, `out_of_range`, `upstream_error` |
| `maps_tile_upstream_seconds{layer}` | martin render time on a cache miss |
| `maps_tile_cache_write_failures_total{layer}` | Rendered tiles that could not be stored |
| `maps_object_requests_total{kind,space,result}` | Map files, layers and models: `hit`, `miss`, `not_found`, `s3_error`, `cache_error`, `denied`, `blocked` |
| `maps_object_fetch_seconds{kind}`, `maps_object_bytes_total{kind,result}` | Download time on a miss, bytes served |
| `maps_gate_decisions_total{route,space,result}` | forwardAuth checks for terracotta raster tiles |
| `maps_gate_maxzoom_cache_total{result}` | Public maxzoom cache hit/miss |
| `maps_file_cache_bytes`, `maps_file_cache_files`, `maps_cache_purges_total{scope}` | Local `/file_cache` size and purges |

A map id is never a label. To see which map is failing, ask the pod (the
endpoint answers 404 through the ingress, like `/metrics`):

kubectl exec -n ollebo deploy/maps-proxy -- python3 -c \
"import urllib.request as u; print(u.urlopen('http://localhost:8080/stats/maps?sort=errors').read().decode())"

`sort` is `requests`, `errors` or `recent`; `top` limits the rows. Counts are
since the pod started, merged across the gunicorn workers. Every failure also
writes one JSON log line with `"event": "map_error"`.

The catch-all answers 502 when object storage or the cache volume fails, and
404 only when the object does not exist.

Dashboard: "Ollebo maps" in Grafana (`monitoring/dashboards-src/ollebo.py` in
the manifests repo).
25 changes: 24 additions & 1 deletion code/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,14 @@ def _metrics_path_label(req):
def _hide_metrics_from_ingress():
# Prometheus scrapes the pod IP directly; requests via Traefik carry
# X-Forwarded-For. Keep /metrics off maps.ollebo.com.
if request.path == "/metrics" and request.headers.get("X-Forwarded-For"):
# /stats/maps lists map ids, so it gets the same treatment.
if request.path in ("/metrics", "/stats/maps") and request.headers.get("X-Forwarded-For"):
return jsonify({"error": "not found"}), 404

import ext
import gate
import k8s
import map_metrics
import minioBackend
import tiles
import utils
Expand Down Expand Up @@ -93,6 +95,11 @@ def _allowed_origin():
return None


@app.before_request
def _start_map_metrics():
map_metrics.ensure_started()


@app.before_request
def _cors_preflight():
if request.method == "OPTIONS":
Expand Down Expand Up @@ -135,6 +142,7 @@ def _cors_headers(resp):
def cache():
if not utils.purge_key_ok(request):
return jsonify({"error": "forbidden"}), 403
map_metrics.CACHE_PURGES.labels("all").inc()
return utils.cleanCache()


Expand All @@ -145,11 +153,26 @@ def cache_purge():
body = request.get_json(silent=True) or {}
prefix = body.get("prefix") or ""
removed = utils.purge_prefix(minioBackend.S3_FILE_BUCKET, prefix)
map_metrics.CACHE_PURGES.labels("prefix").inc()
if body.get("space") and body.get("map"):
gate.forget_maxzoom(body["space"], body["map"])
return jsonify({"removed": removed})


# Per-map request and error counts, merged across workers. In-cluster only
# (see _hide_metrics_from_ingress): map ids never go in Prometheus labels, so
# this is where "which map is failing" is answered.
# /stats/maps?sort=requests|errors|recent&top=50
@app.route('/stats/maps')
@metrics.do_not_track()
def stats_maps():
try:
top = int(request.args.get("top", "50"))
except ValueError:
top = 50
return jsonify(map_metrics.snapshot(top=top, sort=request.args.get("sort", "requests")))


# Traefik forwardAuth targets for terracotta (spridningstillstand.md 5.2/6).
@app.route('/auth/tiles')
def auth_private_tiles():
Expand Down
31 changes: 29 additions & 2 deletions code/gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from flask import jsonify, request

import auth
import map_metrics

TERRACOTTA_PUBLIC_URL = os.environ.get("TERRACOTTA_PUBLIC_URL", "http://terracotta-public:5000")
MAXZOOM_TTL_S = int(os.environ.get("PUBLIC_MAXZOOM_TTL_S", "600"))
Expand All @@ -44,7 +45,8 @@ def max_public_zoom(max_abs_lat_deg, tile_size=256, pixel_ratio=1):

_SEG = r"[A-Za-z0-9_.-]+"
_PRIVATE_RE = re.compile(
r"^/private-tiles/(?:rgb|singleband)/(?P<space>{0})/".format(_SEG)
# The map segment is only read for /stats/maps; access is per space.
r"^/private-tiles/(?:rgb|singleband)/(?P<space>{0})/(?:(?P<map>{0})/)?".format(_SEG)
)
_PUBLIC_RGB_RE = re.compile(
r"^/public/rgb/(?P<space>{0})/(?P<map>{0})/(?P<z>\d+)/(?P<x>\d+)/(?P<y>\d+)\.png$".format(_SEG)
Expand All @@ -66,15 +68,24 @@ def _forwarded():
return parts.path, parse_qs(parts.query, keep_blank_values=True)


def _decision(route, result, space=None):
# `space` is passed only once the request has proved it exists.
map_metrics.GATE_DECISIONS.labels(route, map_metrics.space_label(space), result).inc()


def private_tiles():
path, _ = _forwarded()
m = _PRIVATE_RE.match(path)
if not m:
_decision("private", "not_found")
return jsonify({"error": "not found"}), 404
err = auth.check_space_access(request, m.group("space"))
if err is not None:
status, body = err
_decision("private", "deny")
return jsonify(body), status
_decision("private", "allow", m.group("space"))
map_metrics.record_map(m.group("space"), m.group("map"), "private_tile", "allow")
resp = jsonify({"ok": True})
return resp, 200

Expand All @@ -85,7 +96,9 @@ def _dataset_maxzoom(space, map_id, variant):
with _maxzoom_lock:
hit = _maxzoom_cache.get(key)
if hit and hit[0] > now:
map_metrics.GATE_MAXZOOM_CACHE.labels("hit").inc()
return hit[1]
map_metrics.GATE_MAXZOOM_CACHE.labels("miss").inc()
url = "{0}/metadata/{1}/{2}/{3}".format(TERRACOTTA_PUBLIC_URL, space, map_id, variant)
resp = requests.get(url, timeout=5)
if resp.status_code == 404:
Expand All @@ -109,9 +122,11 @@ def public_tile():
path, query = _forwarded()
m = _PUBLIC_RGB_RE.match(path) or _PUBLIC_SB_RE.match(path)
if not m:
_decision("public", "not_found")
return jsonify({"error": "not found"}), 404
unknown = set(query) - _PUBLIC_PARAMS
if unknown:
_decision("public", "bad_request")
return jsonify({"error": "parameter not allowed", "params": sorted(unknown)}), 400

groups = m.groupdict()
Expand All @@ -121,17 +136,29 @@ def public_tile():
# three channels are built from the same coarse source.
variant = (query.get("r") or [""])[0]
if not variant:
_decision("public", "bad_request")
return jsonify({"error": "missing r"}), 400
space, map_id = groups["space"], groups["map"]
try:
maxzoom = _dataset_maxzoom(groups["space"], groups["map"], variant)
maxzoom = _dataset_maxzoom(space, map_id, variant)
except Exception as e:
print("ERROR public maxzoom lookup {0}: {1}".format(path, e))
_decision("public", "lookup_error")
map_metrics.record_map(space, map_id, "public_tile", "lookup_error", error=True)
return jsonify({"error": "unavailable"}), 503
if maxzoom is None:
# Terracotta has no such dataset. For a map the viewer links to, this
# is the map being broken -- so it counts as an error in /stats/maps.
_decision("public", "not_found")
map_metrics.record_map(space, map_id, "public_tile", "not_found", error=True)
return jsonify({"error": "not found"}), 404
if int(groups["z"]) > maxzoom:
# Overzoom happens in the client (policy 5.2); never render past maxzoom.
_decision("public", "above_maxzoom", space)
map_metrics.record_map(space, map_id, "public_tile", "above_maxzoom")
resp = jsonify({"error": "zoom above public maximum", "maxzoom": maxzoom})
resp.headers["X-Max-Zoom"] = str(maxzoom)
return resp, 404
_decision("public", "allow", space)
map_metrics.record_map(space, map_id, "public_tile", "allow")
return jsonify({"ok": True}), 200
8 changes: 8 additions & 0 deletions code/gunicorn_conf.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,16 @@
# Gunicorn runs several worker processes. prometheus_client multiprocess mode
# aggregates them from files in PROMETHEUS_MULTIPROC_DIR (set by start.sh);
# without it every scrape only sees whichever worker answered.
import os

from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics


def child_exit(server, worker):
GunicornInternalPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid)
# The worker's per-map table (map_metrics.py); /stats/maps merges these.
try:
os.remove(os.path.join(
os.environ.get("MAP_STATS_DIR", "/tmp/map-stats"), "{0}.json".format(worker.pid)))
except OSError:
pass
Loading
Loading