From 8b82805ffc17c948c5963f6f789e96d712794807 Mon Sep 17 00:00:00 2001 From: Mattias Hemmingsson Date: Wed, 11 Nov 2026 18:53:03 +0100 Subject: [PATCH 1/2] Gate terracotta: forwardAuth for private tiles, maxzoom gate for public - /private-tiles/{rgb,singleband} -> terracotta (space membership check) - /public/{rgb,singleband} -> terracotta-public (z <= max public zoom, query allowlist); /datasets, /metadata etc. no longer routed - catch-all never serves COGs outside public/ or detections - keyed /cache/purge and /cache/clear/; fix space parsing for private//cogs// keys Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 17 +++ charts/templates/deployment.yaml | 11 ++ charts/templates/ingress.yaml | 117 ++++++++++----------- charts/templates/middleware.yaml | 65 +++++++++++- code/app.py | 31 +++++- code/gate.py | 137 +++++++++++++++++++++++++ code/minioBackend.py | 40 +++++++- code/test_gate.py | 171 +++++++++++++++++++++++++++++++ code/utils.py | 28 ++++- 9 files changed, 553 insertions(+), 64 deletions(-) create mode 100644 CLAUDE.md create mode 100644 code/gate.py create mode 100644 code/test_gate.py diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..7a0b53a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,17 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Spridningstillstånd (mandatory for any map/imagery work) + +All map data handling follows Ollebo's spreading policy (LMFS 2016:1). The full rules are in `map-maker/docs/spridningstillstand.md` (local copy: `~/projects/ollebo/docs/spridningstillstand.md`). Read it before changing rendering, tiles, storage, sharing, export, thumbnails or anything that serves map data. + +In short: high-res material (finer than 0.5 m) is visible only to members of the owning space. Anything public or external gets the coarse public version: resampled to 0.6 m in EPSG:3006, with maxzoom from the northernmost latitude. If unsure, keep it private and create a `compliance_review` row. + +Never: +- publish or share externally anything finer than 0.5 m without a registered permit; +- publish DSM/DTM, point clouds, 3D, raw images, oblique images, video or detections; +- make a dataset public by only changing access/ACL (going public = a new coarse render); +- expose originals or high-res COGs/tiles via a public URL, cache or API; +- assume an area exemption (LMFS 2 kap. 3 §) applies without human review; +- remove or bypass audit logging (`publication_audit`), resolution checks or maxzoom gates. diff --git a/charts/templates/deployment.yaml b/charts/templates/deployment.yaml index 9a43202..c3145a8 100644 --- a/charts/templates/deployment.yaml +++ b/charts/templates/deployment.yaml @@ -101,6 +101,17 @@ spec: # re-import invalidates a layer without a purge. - name: TILE_CACHE_BUCKET value: ollebo-maps + # spridningstillstand.md: /auth/public-tile reads dataset bounds from + # the PUBLIC terracotta to compute each map's max public zoom. + - name: TERRACOTTA_PUBLIC_URL + value: http://terracotta-public:5000 + # Shared with map-maker (MAPS_PROXY_PURGE_KEY): guards /cache/purge + # and /cache/clear/. Unset = both refuse. + - name: PURGE_API_KEY + valueFrom: + secretKeyRef: + name: ollebo + key: API_KEY volumeMounts: # /file_cache was the container's writable layer: unbounded, and only # ever emptied by the (unauthenticated) /cache/clear/ route. Bounded diff --git a/charts/templates/ingress.yaml b/charts/templates/ingress.yaml index db30907..a2dd24e 100644 --- a/charts/templates/ingress.yaml +++ b/charts/templates/ingress.yaml @@ -1,10 +1,17 @@ {{/* - Terracotta's public tile routes. Listed once and reused per hostname: every - brand gets the same seven prefixes, and a route added for one site but not the - other is a bug that only shows up as a 404 on whichever site was forgotten. + Terracotta routes (spridningstillstand.md 5.3/5.6/6). Only tile endpoints are + exposed -- /datasets, /metadata, /keys, /colormap and /compute are not routed + at all, so the catalogs cannot be listed from outside. Each gate is a separate + Ingress because Traefik attaches middlewares per Ingress. + /private-tiles/{rgb,singleband}//... -> forwardAuth (space member) + -> terracotta (private catalog) + /public/{rgb,singleband}/... -> forwardAuth (maxzoom gate) + -> terracotta-public + Listed once and reused per hostname: a route added for one brand but not the + other is a bug that only shows up as a 404 on the forgotten site. */}} -{{- define "maps-proxy.terracottaPaths" -}} -{{- range list "/rgb" "/singleband" "/metadata" "/keys" "/colormap" "/datasets" "/compute" }} +{{- define "maps-proxy.privateTilePaths" -}} +{{- range list "/private-tiles/rgb" "/private-tiles/singleband" }} - backend: service: name: terracotta @@ -14,35 +21,69 @@ pathType: Prefix {{- end }} {{- end -}} +{{- define "maps-proxy.publicTilePaths" -}} +{{- range list "/public/rgb" "/public/singleband" }} + - backend: + service: + name: terracotta-public + port: + number: 5000 + path: {{ . }} + pathType: Prefix +{{- end }} +{{- end -}} +{{- range $host := list (dict "name" "ollebo-com" "host" "maps.ollebo.com" "tls" "maps-ollebo-tls") (dict "name" "northamlin-com" "host" "maps.northamlin.com" "tls" "maps-northamlin-tls") }} +--- kind: Ingress apiVersion: networking.k8s.io/v1 metadata: - name: "public-maps-ollebo-com" + name: "private-tiles-{{ $host.name }}" namespace: ollebo annotations: kubernetes.io/ingress.class: "traefik" - cert-manager.io/cluster-issuer: "http" - kubernetes.io/tls-acme: "true" traefik.ingress.kubernetes.io/router.entrypoints: websecure traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: "ollebo-terracotta-cors@kubernetescrd,ollebo-terracotta-cache@kubernetescrd" + traefik.ingress.kubernetes.io/router.middlewares: "ollebo-private-tiles-auth@kubernetescrd,ollebo-private-tiles-strip@kubernetescrd,ollebo-terracotta-cors@kubernetescrd,ollebo-private-tiles-cache@kubernetescrd" spec: tls: - hosts: - - maps.ollebo.com - secretName: maps-ollebo-tls + - {{ $host.host }} + secretName: {{ $host.tls }} rules: - - host: maps.ollebo.com + - host: {{ $host.host }} + http: + paths: +{{- include "maps-proxy.privateTilePaths" $ }} +--- +kind: Ingress +apiVersion: networking.k8s.io/v1 +metadata: + name: "public-tiles-{{ $host.name }}" + namespace: ollebo + annotations: + kubernetes.io/ingress.class: "traefik" + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.middlewares: "ollebo-public-tiles-gate@kubernetescrd,ollebo-public-tiles-strip@kubernetescrd,ollebo-terracotta-cors@kubernetescrd,ollebo-public-tiles-cache@kubernetescrd" +spec: + tls: + - hosts: + - {{ $host.host }} + secretName: {{ $host.tls }} + rules: + - host: {{ $host.host }} http: paths: -{{- include "maps-proxy.terracottaPaths" . }} +{{- include "maps-proxy.publicTilePaths" $ }} +{{- end }} --- # maps-proxy catch-all is a SEPARATE ingress WITHOUT the terracotta-cors # middleware: that middleware sets Access-Control-Allow-Origin: *, which both # duplicates and invalidates the credentialed CORS maps-proxy sends for private # maps/models (a "*" origin is illegal with credentials). maps-proxy does its # own CORS (echoes the ollebo.com origin + allow-credentials), so this route -# must carry no CORS middleware. Reuses the cert issued by the ingress above. +# must carry no CORS middleware. Owns the maps.ollebo.com cert (the open terracotta ingress that used to +# request it is gone). kind: Ingress apiVersion: networking.k8s.io/v1 metadata: @@ -50,6 +91,8 @@ metadata: namespace: ollebo annotations: kubernetes.io/ingress.class: "traefik" + cert-manager.io/cluster-issuer: "http" + kubernetes.io/tls-acme: "true" traefik.ingress.kubernetes.io/router.entrypoints: websecure traefik.ingress.kubernetes.io/router.tls: "true" spec: @@ -97,51 +140,9 @@ spec: path: / pathType: Prefix --- -# northamlin.com (the US site) is the SAME dw image, the same Keycloak realm and -# the same buckets -- so it is the same maps-proxy and the same terracotta, just -# reached under a second name. It needs its own hostname rather than borrowing -# maps.ollebo.com because dw scopes the `access_token` cookie to the apex the -# browser is on (COOKIE_DOMAIN=.northamlin.com): a request to maps.ollebo.com -# from northamlin.com carries no cookie, so every private map, model and layer -# 401s. Public tiles were never affected, which is why this looked fine at -# launch. -# -# These two mirror the ollebo pair above -- terracotta's prefixes with the CORS -# and cache middlewares, then a catch-all to maps-proxy with neither. Both -# namespaced in `ollebo`, because that is where the Services and the Middlewares -# live and traefik runs without --providers.kubernetescrd.allowCrossNamespace. -# -# The cert only issues once maps.northamlin.com resolves to 65.109.40.143. The -# other front-end address, 65.21.254.116, does not hairpin, so cert-manager's -# in-cluster HTTP-01 self-check can never pass against it and the Certificate -# stays pending forever. That is what kept northamlin.com itself dark for eight -# months. -kind: Ingress -apiVersion: networking.k8s.io/v1 -metadata: - name: "public-maps-northamlin-com" - namespace: ollebo - annotations: - kubernetes.io/ingress.class: "traefik" - cert-manager.io/cluster-issuer: "http" - kubernetes.io/tls-acme: "true" - traefik.ingress.kubernetes.io/router.entrypoints: websecure - traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: "ollebo-terracotta-cors@kubernetescrd,ollebo-terracotta-cache@kubernetescrd" -spec: - tls: - - hosts: - - maps.northamlin.com - secretName: maps-northamlin-tls - rules: - - host: maps.northamlin.com - http: - paths: -{{- include "maps-proxy.terracottaPaths" . }} ---- # Catch-all for maps.northamlin.com. No CORS middleware, for the same reason the # ollebo catch-all carries none; maps-proxy answers with the caller's own origin -# and allow-credentials. Reuses the cert requested by the ingress above. +# and allow-credentials. Owns the maps.northamlin.com cert. kind: Ingress apiVersion: networking.k8s.io/v1 metadata: @@ -149,6 +150,8 @@ metadata: namespace: ollebo annotations: kubernetes.io/ingress.class: "traefik" + cert-manager.io/cluster-issuer: "http" + kubernetes.io/tls-acme: "true" traefik.ingress.kubernetes.io/router.entrypoints: websecure traefik.ingress.kubernetes.io/router.tls: "true" spec: diff --git a/charts/templates/middleware.yaml b/charts/templates/middleware.yaml index ce8a96e..5609ee1 100644 --- a/charts/templates/middleware.yaml +++ b/charts/templates/middleware.yaml @@ -37,13 +37,72 @@ spec: accessControlMaxAge: 100 addVaryHeader: true --- -# 1-day browser/CDN cache for terracotta tiles (referenced as ollebo-terracotta-cache@kubernetescrd) +# --- spridningstillstand.md: terracotta is never reachable without a gate --- +# +# Private (full-resolution) tiles: forwardAuth asks maps-proxy whether the +# caller is a member of the space in the path, then /private-tiles is stripped +# and the request goes to the PRIVATE terracotta. Never cached by shared caches. apiVersion: traefik.io/v1alpha1 kind: Middleware metadata: - name: terracotta-cache + name: private-tiles-auth + namespace: ollebo +spec: + forwardAuth: + address: http://{{ .Values.name }}.ollebo.svc.cluster.local:8080/auth/tiles + trustForwardHeader: false +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: private-tiles-strip + namespace: ollebo +spec: + stripPrefix: + prefixes: + - /private-tiles +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: private-tiles-cache + namespace: ollebo +spec: + headers: + customResponseHeaders: + Cache-Control: "private, no-store" +--- +# Public (coarse 0.6 m) tiles: forwardAuth enforces the per-dataset max public +# zoom and a query-parameter allowlist, then /public is stripped and the request +# goes to the PUBLIC terracotta (public catalog only). +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: public-tiles-gate + namespace: ollebo +spec: + forwardAuth: + address: http://{{ .Values.name }}.ollebo.svc.cluster.local:8080/auth/public-tile + trustForwardHeader: false +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: public-tiles-strip + namespace: ollebo +spec: + stripPrefix: + prefixes: + - /public +--- +# Short public cache: an unpublished map must disappear from browsers and any +# intermediate cache quickly (policy 5.5). Was 1 day. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: public-tiles-cache namespace: ollebo spec: headers: customResponseHeaders: - Cache-Control: "public, max-age=86400" + Cache-Control: "public, max-age=3600" diff --git a/code/app.py b/code/app.py index a87bd2e..a30f1bb 100644 --- a/code/app.py +++ b/code/app.py @@ -1,7 +1,7 @@ import os import re -from flask import Flask, make_response, request +from flask import Flask, jsonify, make_response, request from prometheus_flask_exporter import PrometheusMetrics @@ -9,6 +9,7 @@ metrics = PrometheusMetrics(app) import ext +import gate import k8s import minioBackend import tiles @@ -102,12 +103,38 @@ def _cors_headers(resp): app.add_url_rule('/ready', view_func=k8s.ready) app.add_url_rule('/healthz', view_func=k8s.healthz) -#Clear cache +# Cache maintenance. Both need the internal API key: an open cache wipe was a +# free denial of service, and purge is what unpublish relies on (policy 5.5). @app.route('/cache/clear/') def cache(): + if not utils.purge_key_ok(request): + return jsonify({"error": "forbidden"}), 403 return utils.cleanCache() +@app.route('/cache/purge', methods=['POST']) +def cache_purge(): + if not utils.purge_key_ok(request): + return jsonify({"error": "forbidden"}), 403 + body = request.get_json(silent=True) or {} + prefix = body.get("prefix") or "" + removed = utils.purge_prefix(minioBackend.S3_FILE_BUCKET, prefix) + if body.get("space") and body.get("map"): + gate.forget_maxzoom(body["space"], body["map"]) + return jsonify({"removed": removed}) + + +# Traefik forwardAuth targets for terracotta (spridningstillstand.md 5.2/6). +@app.route('/auth/tiles') +def auth_private_tiles(): + return gate.private_tiles() + + +@app.route('/auth/public-tile') +def auth_public_tile(): + return gate.public_tile() + + # Maps we host ourselves: vector tiles rendered by martin from our own PostGIS, # plus the TileJSON and style the viewer reads. These are declared before the # catch-all so //{z}/{x}/{y}.pbf is a tile rather than an object lookup; diff --git a/code/gate.py b/code/gate.py new file mode 100644 index 0000000..0396f06 --- /dev/null +++ b/code/gate.py @@ -0,0 +1,137 @@ +"""Tile gates for terracotta (spridningstillstand.md 5.2, 5.6 and 6). + +Traefik calls these as forwardAuth endpoints before handing a tile request to +terracotta. A 2xx lets the request through; anything else is returned to the +client as is. + +/auth/tiles private, full-resolution tiles under /private-tiles/... + -> the caller must be a member of the space in the path. +/auth/public-tile public, coarse tiles under /public/... + -> only rgb/singleband tiles at or below the dataset's + computed max public zoom, with an allowlisted query. + +Both read the original request from X-Forwarded-Uri, which Traefik sets to the +path + query the client asked for (before any stripPrefix). +""" + +import math +import os +import re +import threading +import time +from urllib.parse import parse_qs, urlsplit + +import requests +from flask import jsonify, request + +import auth + +TERRACOTTA_PUBLIC_URL = os.environ.get("TERRACOTTA_PUBLIC_URL", "http://terracotta-public:5000") +MAXZOOM_TTL_S = int(os.environ.get("PUBLIC_MAXZOOM_TTL_S", "600")) + +# Same formula as map-maker/code/compliance/zoom.py (policy 5.2). Tiles are +# always 256 px here: tile_size is not an allowed query parameter. +EARTH_RES_Z0 = 156543.03392804097 +MIN_RES_M = 0.5 +SAFETY = 1.2 + + +def max_public_zoom(max_abs_lat_deg, tile_size=256, pixel_ratio=1): + base = (EARTH_RES_Z0 * math.cos(math.radians(abs(max_abs_lat_deg))) + * 256 / tile_size / pixel_ratio) + return math.floor(math.log2(base / (MIN_RES_M * SAFETY))) + + +_SEG = r"[A-Za-z0-9_.-]+" +_PRIVATE_RE = re.compile( + r"^/private-tiles/(?:rgb|singleband)/(?P{0})/".format(_SEG) +) +_PUBLIC_RGB_RE = re.compile( + r"^/public/rgb/(?P{0})/(?P{0})/(?P\d+)/(?P\d+)/(?P\d+)\.png$".format(_SEG) +) +_PUBLIC_SB_RE = re.compile( + r"^/public/singleband/(?P{0})/(?P{0})/(?P{0})/(?P\d+)/(?P\d+)/(?P\d+)\.png$".format(_SEG) +) +_PUBLIC_PARAMS = frozenset([ + "r", "g", "b", "r_range", "g_range", "b_range", "colormap", "stretch_range", +]) + +_maxzoom_lock = threading.Lock() +_maxzoom_cache = {} # (space, map, variant) -> (expires_at, maxzoom) + + +def _forwarded(): + uri = request.headers.get("X-Forwarded-Uri", "") + parts = urlsplit(uri) + return parts.path, parse_qs(parts.query, keep_blank_values=True) + + +def private_tiles(): + path, _ = _forwarded() + m = _PRIVATE_RE.match(path) + if not m: + return jsonify({"error": "not found"}), 404 + err = auth.check_space_access(request, m.group("space")) + if err is not None: + status, body = err + return jsonify(body), status + resp = jsonify({"ok": True}) + return resp, 200 + + +def _dataset_maxzoom(space, map_id, variant): + key = (space, map_id, variant) + now = time.time() + with _maxzoom_lock: + hit = _maxzoom_cache.get(key) + if hit and hit[0] > now: + return hit[1] + url = "{0}/metadata/{1}/{2}/{3}".format(TERRACOTTA_PUBLIC_URL, space, map_id, variant) + resp = requests.get(url, timeout=5) + if resp.status_code == 404: + zoom = None + else: + resp.raise_for_status() + west, south, east, north = resp.json()["bounds"] + zoom = max_public_zoom(max(abs(float(south)), abs(float(north)))) + with _maxzoom_lock: + _maxzoom_cache[key] = (now + MAXZOOM_TTL_S, zoom) + return zoom + + +def forget_maxzoom(space, map_id): + with _maxzoom_lock: + for key in [k for k in _maxzoom_cache if k[0] == space and k[1] == map_id]: + del _maxzoom_cache[key] + + +def public_tile(): + path, query = _forwarded() + m = _PUBLIC_RGB_RE.match(path) or _PUBLIC_SB_RE.match(path) + if not m: + return jsonify({"error": "not found"}), 404 + unknown = set(query) - _PUBLIC_PARAMS + if unknown: + return jsonify({"error": "parameter not allowed", "params": sorted(unknown)}), 400 + + groups = m.groupdict() + variant = groups.get("variant") + if variant is None: + # rgb composite: the zoom gate is checked against the red channel; the + # three channels are built from the same coarse source. + variant = (query.get("r") or [""])[0] + if not variant: + return jsonify({"error": "missing r"}), 400 + try: + maxzoom = _dataset_maxzoom(groups["space"], groups["map"], variant) + except Exception as e: + print("ERROR public maxzoom lookup {0}: {1}".format(path, e)) + return jsonify({"error": "unavailable"}), 503 + if maxzoom is None: + return jsonify({"error": "not found"}), 404 + if int(groups["z"]) > maxzoom: + # Overzoom happens in the client (policy 5.2); never render past maxzoom. + resp = jsonify({"error": "zoom above public maximum", "maxzoom": maxzoom}) + resp.headers["X-Max-Zoom"] = str(maxzoom) + return resp, 404 + return jsonify({"ok": True}), 200 diff --git a/code/minioBackend.py b/code/minioBackend.py index 1709e67..602ae44 100644 --- a/code/minioBackend.py +++ b/code/minioBackend.py @@ -68,6 +68,38 @@ def resolveBucket(path): } +_TERRACOTTA_PREFIXES = ("rgb/", "singleband/", "metadata/", "keys", "colormap", + "datasets", "compute/") + + +def _blocked_public_key(key): + lower = key.lower() + if lower.startswith(_TERRACOTTA_PREFIXES): + return True + base = lower.rsplit("/", 1)[-1] + if base == "detections.json" or base.endswith(".geojson") and "detect" in base: + return True + if lower.endswith((".tif", ".tiff")) and not lower.startswith("public/"): + return True + if "/cogs/" in "/" + lower: + return True + return False + + +def _space_of(key): + """Space id that guards a private/ or models/ key. + + Layers and models put the space first (/...); map outputs from + map-maker/geo-detect use /cogs///..., where the first + segment is the Keycloak realm. Reading the realm as the space 403'd every + member, so the cogs layout is recognised explicitly. + """ + parts = (key or "").split("/") + if len(parts) >= 3 and parts[1] == "cogs": + return parts[2] + return parts[0] if parts else "" + + def getFile(filename): print(filename) if filename == "" or filename == "index.html": @@ -77,8 +109,14 @@ def getFile(filename): bucket, key = resolveBucket(filename) + # spridningstillstand.md 5.3/5.6: the anonymous catch-all never serves + # COGs or detections outside the coarse public/ prefix, nor the legacy + # /cogs/... layout (pre-compliance public maps were too fine). + if bucket == S3_FILE_BUCKET and _blocked_public_key(key): + return jsonify({"error": "not found"}), 404 + if filename.startswith("private/") or filename.startswith("models/"): - space_id = key.split("/", 1)[0] if key else "" + space_id = _space_of(key) if not space_id: return jsonify({"error": "missing space id"}), 400 err = auth.check_space_access(request, space_id) diff --git a/code/test_gate.py b/code/test_gate.py new file mode 100644 index 0000000..e6cf096 --- /dev/null +++ b/code/test_gate.py @@ -0,0 +1,171 @@ +"""Gate + catch-all checks for spridningstillstand.md. Run inside the image: + python3 /code/test_gate.py +""" +import os +import sys +import tempfile +import types + +os.environ.setdefault("S3_ENDPOINT", "localhost:9000") +os.environ.setdefault("AWS_ACCESS_KEY_ID", "test") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "test") +os.environ.setdefault("AWS_S3_FILE_BUCKET", "ollebo-maps") +os.environ["PURGE_API_KEY"] = "secret-key" + +MEMBER = "11111111-1111-1111-1111-111111111111" + + +def _check(request, space_id): + token = request.headers.get("Authorization", "") + if not token: + return 401, {"error": "unauthenticated"} + if space_id != MEMBER: + return 403, {"error": "forbidden"} + return None + + +_auth_stub = types.ModuleType("auth") +_auth_stub.check_space_access = _check +sys.modules["auth"] = _auth_stub + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import app # noqa: E402 +import gate # noqa: E402 +import utils # noqa: E402 + + +class _Resp: + def __init__(self, status, body=None): + self.status_code = status + self._body = body or {} + + def json(self): + return self._body + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(self.status_code) + + +# Fake terracotta-public metadata: one dataset near Stockholm (maxzoom 17), +# one near Kiruna (maxzoom 16). +_BOUNDS = { + ("S", "M", "rgb__r"): [18.0, 59.30, 18.01, 59.305], + ("S", "K", "grayscale"): [20.2, 67.85, 20.21, 67.855], +} + + +def _fake_get(url, timeout=None): + parts = url.rstrip("/").split("/") + key = tuple(parts[-3:]) + if key in _BOUNDS: + return _Resp(200, {"bounds": _BOUNDS[key]}) + return _Resp(404) + + +gate.requests.get = _fake_get + +client = app.app.test_client() +failures = [] + + +def expect(label, resp, status): + ok = resp.status_code == status + print("{0} {1:55} -> {2} (want {3})".format("ok " if ok else "FAIL", label, resp.status_code, status)) + if not ok: + failures.append(label) + + +def fwd(uri, **headers): + h = {"X-Forwarded-Uri": uri} + h.update(headers) + return h + + +RGBQ = "?r=rgb__r&g=rgb__g&b=rgb__b&r_range=[0,255]&g_range=[0,255]&b_range=[0,255]" + +# --- private tiles ------------------------------------------------------- +expect("private tile, no token", client.get("/auth/tiles", headers=fwd( + "/private-tiles/rgb/{0}/m/10/1/1.png".format(MEMBER))), 401) +expect("private tile, member", client.get("/auth/tiles", headers=fwd( + "/private-tiles/rgb/{0}/m/10/1/1.png".format(MEMBER), Authorization="Bearer x")), 200) +expect("private tile, non-member", client.get("/auth/tiles", headers=fwd( + "/private-tiles/singleband/22222222-2222-2222-2222-222222222222/m/v/1/1/1.png", + Authorization="Bearer x")), 403) +expect("private datasets listing", client.get("/auth/tiles", headers=fwd( + "/private-tiles/datasets", Authorization="Bearer x")), 404) + +# --- public tiles -------------------------------------------------------- +expect("public rgb at maxzoom 17 (Stockholm)", client.get("/auth/public-tile", headers=fwd( + "/public/rgb/S/M/17/1/1.png" + RGBQ)), 200) +expect("public rgb above maxzoom", client.get("/auth/public-tile", headers=fwd( + "/public/rgb/S/M/18/1/1.png" + RGBQ)), 404) +expect("public singleband at 16 (Kiruna)", client.get("/auth/public-tile", headers=fwd( + "/public/singleband/S/K/grayscale/16/1/1.png?colormap=greys_r&stretch_range=[0,255]")), 200) +expect("public singleband at 17 (Kiruna)", client.get("/auth/public-tile", headers=fwd( + "/public/singleband/S/K/grayscale/17/1/1.png")), 404) +expect("public tile_size param refused", client.get("/auth/public-tile", headers=fwd( + "/public/rgb/S/M/10/1/1.png" + RGBQ + "&tile_size=[512,512]")), 400) +expect("public unknown dataset", client.get("/auth/public-tile", headers=fwd( + "/public/rgb/S/NOPE/10/1/1.png" + RGBQ)), 404) +expect("public metadata endpoint", client.get("/auth/public-tile", headers=fwd( + "/public/metadata/S/M/rgb__r")), 404) + +# --- catch-all ---------------------------------------------------------- +for path in ( + "/master/cogs/S/M/rgb__r.tif", + "/master/cogs/S/M/detections.json", + "/whatever/file.tif", + "/rgb/S/M/1/1/1.png", + "/datasets", + "/metadata/S/M/rgb__r", +): + expect("catch-all blocks " + path, client.get(path), 404) + +# The S3 endpoint is fake, so a miss is also a 404: check the rule itself too. +import minioBackend # noqa: E402 +for key, blocked in ( + ("master/cogs/S/M/rgb__r.tif", True), + ("x/detections.json", True), + ("whatever/file.tif", True), + ("datasets", True), + ("public/master/S/M/rgb__r.tif", False), + ("layers/S/L/layer.geojson", False), + ("tiles/sweden/v1/8/1/1.pbf", False), +): + if minioBackend._blocked_public_key(key) != blocked: + print("FAIL _blocked_public_key({0}) != {1}".format(key, blocked)) + failures.append(key) + +for key, space in ( + ("master/cogs/{0}/m/detections.json".format(MEMBER), MEMBER), + ("{0}/layers/l/layer.geojson".format(MEMBER), MEMBER), + ("{0}/model/file.glb".format(MEMBER), MEMBER), +): + if minioBackend._space_of(key) != space: + print("FAIL _space_of({0})".format(key)) + failures.append(key) + +# --- cache --------------------------------------------------------------- +expect("cache clear without key", client.get("/cache/clear/"), 403) +expect("cache purge without key", client.post("/cache/purge", json={"prefix": "public/x/"}), 403) + +tmp = tempfile.mkdtemp() +utils.FILE_CACHE = tmp + "/" +os.makedirs(os.path.join(tmp, "ollebo-maps", "public", "master", "S", "M")) +open(os.path.join(tmp, "ollebo-maps", "public", "master", "S", "M", "rgb__r.tif"), "w").close() +gate._dataset_maxzoom("S", "M", "rgb__r") +r = client.post("/cache/purge", headers={"X-Api-Key": "secret-key"}, + json={"prefix": "public/master/S/M/", "space": "S", "map": "M"}) +expect("cache purge with key", r, 200) +if r.get_json().get("removed") != 1 or ("S", "M", "rgb__r") in gate._maxzoom_cache: + print("FAIL purge did not remove cached file/maxzoom") + failures.append("purge effect") +r = client.post("/cache/purge", headers={"X-Api-Key": "secret-key"}, json={"prefix": "../../etc/"}) +if r.get_json().get("removed") != 0: + failures.append("purge traversal") + +if failures: + print("\n{0} failure(s)".format(len(failures))) + sys.exit(1) +print("\nall gate checks passed") diff --git a/code/utils.py b/code/utils.py index 27b7aec..2da37aa 100644 --- a/code/utils.py +++ b/code/utils.py @@ -1,7 +1,33 @@ -import os, shutil +import hmac +import os +import shutil CACHE_KEY = os.environ.get('CACHE_KEY') +# Shared with map-maker (MAPS_PROXY_PURGE_KEY). Unset = every purge/clear is +# refused rather than open. +PURGE_API_KEY = os.environ.get('PURGE_API_KEY', '') +FILE_CACHE = '/file_cache/' + + +def purge_key_ok(request): + given = request.headers.get('X-Api-Key', '') + return bool(PURGE_API_KEY) and hmac.compare_digest(given, PURGE_API_KEY) + + +def purge_prefix(bucket, prefix): + """Remove cached copies of bucket/prefix*. Only whole-directory prefixes + (ending in '/') under public/ are accepted.""" + if not prefix.startswith('public/') or not prefix.endswith('/') or '..' in prefix: + return 0 + target = os.path.realpath(os.path.join(FILE_CACHE, bucket, prefix)) + root = os.path.realpath(os.path.join(FILE_CACHE, bucket)) + if not target.startswith(root + os.sep) or not os.path.isdir(target): + return 0 + count = sum(len(files) for _, _, files in os.walk(target)) + shutil.rmtree(target, ignore_errors=True) + print('purged {0} cached files under {1}'.format(count, target)) + return count def cleanCache(): folder = '/file_cache/' From 20ed1f87dc573e2a657c7400024734b73154a0c2 Mon Sep 17 00:00:00 2001 From: Mattias Hemmingsson Date: Thu, 12 Nov 2026 07:55:38 +0100 Subject: [PATCH 2/2] ci: retrigger build (previous run stuck in queue) Co-Authored-By: Claude Opus 5.5