From ef4b91ab087e298d80fbfd3b6a205c9fa142f1c3 Mon Sep 17 00:00:00 2001 From: Mattias Hemmingsson Date: Fri, 6 Nov 2026 12:30:52 +0100 Subject: [PATCH] Restore the map pipeline so the mapserver can start start.py and listMaps.py import mapmaker.maps, but that package was never committed, and terracotta was missing from requirements.txt, so the published image could not start. Restore the package from map-maker (1c17a2b) and use its current conventions: RGB is split into one COG per channel, and tile URLs carry explicit ranges. Also: - Allow the Ollebo dashboards in Terracotta's tile CORS; the default is localhost only. - Pin numpy/rasterio/terracotta and drop the unused ninja2 and geotiff. - Run secure_filename on the /mapmaker name. - Add a .dockerignore. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AwkBcZCmo27zHnnRnAcKiy --- .dockerignore | 11 ++ code/mapmaker/__init__.py | 0 code/mapmaker/makingGeotiff.py | 31 ++-- code/mapmaker/maps/__init__.py | 0 code/mapmaker/maps/cog.py | 140 ++++++++++++++++++ code/mapmaker/maps/detect.py | 54 +++++++ code/mapmaker/maps/ramps/dem.txt | 12 ++ code/mapmaker/maps/ramps/ndvi.txt | 7 + code/mapmaker/maps/terracotta_client.py | 58 ++++++++ code/mapmaker/maps/variants.py | 30 ++++ code/requirements.txt | 10 +- code/start.py | 4 +- docker-compose.yaml | 1 + .../helm/mapserver/templates/configmap.yaml | 1 + install/helm/mapserver/values.yaml | 5 + 15 files changed, 349 insertions(+), 15 deletions(-) create mode 100644 .dockerignore create mode 100644 code/mapmaker/__init__.py create mode 100644 code/mapmaker/maps/__init__.py create mode 100644 code/mapmaker/maps/cog.py create mode 100644 code/mapmaker/maps/detect.py create mode 100644 code/mapmaker/maps/ramps/dem.txt create mode 100644 code/mapmaker/maps/ramps/ndvi.txt create mode 100644 code/mapmaker/maps/terracotta_client.py create mode 100644 code/mapmaker/maps/variants.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..87e0239 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.github +.env +maps +cogs +db +web +test-bild +install +**/__pycache__ +**/*.pyc diff --git a/code/mapmaker/__init__.py b/code/mapmaker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/code/mapmaker/makingGeotiff.py b/code/mapmaker/makingGeotiff.py index 0b6cb9d..f623dad 100644 --- a/code/mapmaker/makingGeotiff.py +++ b/code/mapmaker/makingGeotiff.py @@ -54,13 +54,23 @@ def getGeoTiffData(tiff_file): def _tile_url(space_id, map_id, spec): - base = "singleband" if spec.singleband else "rgb" - url = "{0}/{1}/{2}/{3}/{4}/{{z}}/{{x}}/{{y}}.png".format( - config.terracotta_public_url(), base, space_id, map_id, spec.name - ) - if spec.colormap and spec.singleband: - url += "?colormap=" + spec.colormap - return url + base_url = config.terracotta_public_url() + # Explicit ranges match the 8-bit COGs and the URLs map-maker produces. + if spec.singleband: + url = "{0}/singleband/{1}/{2}/{3}/{{z}}/{{x}}/{{y}}.png".format( + base_url, space_id, map_id, spec.name + ) + params = [] + if spec.colormap: + params.append("colormap=" + spec.colormap) + params.append("stretch_range=[0,255]") + return url + "?" + "&".join(params) + # RGB composite: three single-band datasets named __r/_g/_b. + return ( + "{0}/rgb/{1}/{2}/{{z}}/{{x}}/{{y}}.png" + "?r={3}__r&g={3}__g&b={3}__b" + "&r_range=[0,255]&g_range=[0,255]&b_range=[0,255]" + ).format(base_url, space_id, map_id, spec.name) def makingTiles(map): @@ -79,11 +89,12 @@ def makingTiles(map): variants_meta = [] for spec in VARIANTS[input_type]: - cog_path = os.path.join(cog_out_dir, "{0}.tif".format(spec.name)) status = "ok" try: - spec.build(source_path, cog_path, bands) - terracotta_client.register(space_id, map_id, spec.name, cog_path) + # One variant can produce several datasets (RGB is one per channel). + outputs = spec.build(source_path, cog_out_dir, spec.name, bands) + for variant_key, cog_path in outputs.items(): + terracotta_client.register(space_id, map_id, variant_key, cog_path) except Exception as e: print("variant {0} failed: {1}".format(spec.name, e)) status = "failed" diff --git a/code/mapmaker/maps/__init__.py b/code/mapmaker/maps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/code/mapmaker/maps/cog.py b/code/mapmaker/maps/cog.py new file mode 100644 index 0000000..86d39f7 --- /dev/null +++ b/code/mapmaker/maps/cog.py @@ -0,0 +1,140 @@ +import os +import subprocess +import tempfile + + +RAMPS_DIR = os.path.join(os.path.dirname(__file__), "ramps") + +COG_CO = [ + "-co", "COMPRESS=DEFLATE", + "-co", "PREDICTOR=2", + "-co", "OVERVIEWS=AUTO", + "-co", "BLOCKSIZE=512", +] + + +def _run(argv): + print("[cog] " + " ".join(argv)) + subprocess.run(argv, check=True, capture_output=True, text=True) + + +def _scale_args(dtype): + if dtype == "uint16": + return ["-scale", "0", "65535", "0", "255"] + return ["-scale"] + + +def _channel_key(name, channel): + return "{0}__{1}".format(name, channel) + + +def build_rgb(input_path, out_dir, name, bands): + # Terracotta's /rgb endpoint composites three single-band datasets, so we + # split source R/G/B into three separate single-band COGs. + scale = _scale_args(bands["dtype"]) + outputs = {} + for channel, src_band in (("r", bands["r"]), ("g", bands["g"]), ("b", bands["b"])): + out = os.path.join(out_dir, "{0}.tif".format(_channel_key(name, channel))) + _run([ + "gdal_translate", "-of", "COG", "-ot", "Byte", + "-b", str(src_band), + ] + scale + COG_CO + [input_path, out]) + outputs[_channel_key(name, channel)] = out + return outputs + + +def build_grayscale(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp: + intermediate = tmp.name + try: + _run([ + "gdal_calc.py", + "--calc=0.299*R+0.587*G+0.114*B", + "-R", input_path, f"--R_band={bands['r']}", + "-G", input_path, f"--G_band={bands['g']}", + "-B", input_path, f"--B_band={bands['b']}", + "--type=Byte", + f"--outfile={intermediate}", + "--overwrite", + ]) + _run(["gdal_translate", "-of", "COG"] + COG_CO + [intermediate, out]) + finally: + if os.path.exists(intermediate): + os.unlink(intermediate) + return {name: out} + + +def build_ndvi(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp: + intermediate = tmp.name + try: + _run([ + "gdal_calc.py", + "--calc=(NIR.astype(float)-R)/(NIR+R+1e-6)", + "-R", input_path, f"--R_band={bands['r']}", + "--NIR", input_path, f"--NIR_band={bands['nir']}", + "--type=Float32", + "--NoDataValue=-9999", + f"--outfile={intermediate}", + "--overwrite", + ]) + _run(["gdal_translate", "-of", "COG"] + COG_CO + [intermediate, out]) + finally: + if os.path.exists(intermediate): + os.unlink(intermediate) + return {name: out} + + +def build_hillshade(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + _run([ + "gdaldem", "hillshade", + "-of", "COG", + "-compute_edges", + "-z", "1.0", "-az", "315", "-alt", "45", + "-co", "COMPRESS=DEFLATE", + input_path, out, + ]) + return {name: out} + + +def build_color_relief(input_path, out_dir, name, bands): + # gdaldem color-relief produces a 3-band RGB tif; split into single-band + # COGs so terracotta's /rgb endpoint can composite them. + ramp = os.path.join(RAMPS_DIR, "dem.txt") + with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp: + intermediate = tmp.name + try: + _run([ + "gdaldem", "color-relief", + "-of", "GTiff", + "-co", "COMPRESS=DEFLATE", + input_path, ramp, intermediate, + ]) + outputs = {} + for channel, src_band in (("r", 1), ("g", 2), ("b", 3)): + out = os.path.join(out_dir, "{0}.tif".format(_channel_key(name, channel))) + _run([ + "gdal_translate", "-of", "COG", "-ot", "Byte", + "-b", str(src_band), + ] + COG_CO + [intermediate, out]) + outputs[_channel_key(name, channel)] = out + return outputs + finally: + if os.path.exists(intermediate): + os.unlink(intermediate) + + +def build_slope(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + _run([ + "gdaldem", "slope", + "-of", "COG", + "-compute_edges", + "-s", "1.0", + "-co", "COMPRESS=DEFLATE", + input_path, out, + ]) + return {name: out} diff --git a/code/mapmaker/maps/detect.py b/code/mapmaker/maps/detect.py new file mode 100644 index 0000000..7beca9d --- /dev/null +++ b/code/mapmaker/maps/detect.py @@ -0,0 +1,54 @@ +import re + +import rasterio +from rasterio.enums import ColorInterp + + +NIR_HINT = re.compile(r"nir|842|b5|rededge|red\s*edge", re.IGNORECASE) + + +def _band_looks_like_nir(ds, idx): + desc = (ds.descriptions[idx - 1] or "") + if NIR_HINT.search(desc): + return True + tags = ds.tags(idx) + for value in tags.values(): + if NIR_HINT.search(str(value)): + return True + return False + + +def detect_input_type(path): + with rasterio.open(path) as ds: + count = ds.count + dtype = ds.dtypes[0] + colorinterp = list(ds.colorinterp) + + if count == 1: + has_palette = False + try: + ds.colormap(1) + has_palette = True + except ValueError: + pass + if not has_palette: + return "dem", {"r": None, "g": None, "b": None, "nir": None, "dtype": dtype} + + if count >= 4 and dtype in ("uint8", "uint16"): + nir_idx = None + for i in range(4, count + 1): + if _band_looks_like_nir(ds, i): + nir_idx = i + break + if nir_idx is None and count == 4 and colorinterp[3] in (ColorInterp.undefined, ColorInterp.gray): + nir_idx = 4 + if nir_idx is not None: + return "multispectral", {"r": 1, "g": 2, "b": 3, "nir": nir_idx, "dtype": dtype} + + if count in (3, 4) and ColorInterp.red in colorinterp: + return "rgb", {"r": 1, "g": 2, "b": 3, "nir": None, "dtype": dtype} + + if dtype == "uint8" and count >= 3: + return "rgb", {"r": 1, "g": 2, "b": 3, "nir": None, "dtype": dtype} + + return "dem", {"r": None, "g": None, "b": None, "nir": None, "dtype": dtype} diff --git a/code/mapmaker/maps/ramps/dem.txt b/code/mapmaker/maps/ramps/dem.txt new file mode 100644 index 0000000..bd2cbda --- /dev/null +++ b/code/mapmaker/maps/ramps/dem.txt @@ -0,0 +1,12 @@ +nv 0 0 0 0 +-500 12 44 132 +0 65 105 225 +1 80 200 120 +200 154 205 50 +500 188 187 87 +1000 205 175 100 +1500 165 130 80 +2200 139 105 75 +3000 200 200 200 +5000 255 255 255 +8848 255 255 255 diff --git a/code/mapmaker/maps/ramps/ndvi.txt b/code/mapmaker/maps/ramps/ndvi.txt new file mode 100644 index 0000000..d3ef06c --- /dev/null +++ b/code/mapmaker/maps/ramps/ndvi.txt @@ -0,0 +1,7 @@ +nv 0 0 0 0 +-1.0 0 0 0 +-0.2 200 50 50 +0.0 230 230 130 +0.3 130 200 80 +0.6 30 150 30 +1.0 0 90 0 diff --git a/code/mapmaker/maps/terracotta_client.py b/code/mapmaker/maps/terracotta_client.py new file mode 100644 index 0000000..e3030e1 --- /dev/null +++ b/code/mapmaker/maps/terracotta_client.py @@ -0,0 +1,58 @@ +import os + +from terracotta import get_driver, exceptions + + +KEY_NAMES = ["space", "map", "variant"] +KEY_DESCRIPTIONS = { + "space": "Ollebo space (tenant) ID", + "map": "Map ID within the space", + "variant": "Rendering variant (rgb, grayscale, ndvi, hillshade, ...)", +} + + +def _driver(): + db_path = os.environ.get("TERRACOTTA_DB_PATH", "/data/db/terracotta.sqlite") + provider = os.environ.get("TERRACOTTA_PROVIDER", "sqlite") + return get_driver(db_path, provider=provider) + + +def ensure_schema(): + driver = _driver() + try: + with driver.connect(): + if not driver.key_names: + driver.create(KEY_NAMES, key_descriptions=KEY_DESCRIPTIONS) + except exceptions.InvalidDatabaseError: + driver.create(KEY_NAMES, key_descriptions=KEY_DESCRIPTIONS) + + +def replace_map(space_id, map_id): + driver = _driver() + with driver.connect(): + if not driver.key_names: + return + datasets = driver.get_datasets(where={"space": space_id, "map": map_id}) + for key_tuple in list(datasets.keys()): + driver.delete(dict(zip(driver.key_names, key_tuple))) + + +# Cap the array Terracotta reads when computing band statistics. Without the +# optional `crick` dependency, Terracotta otherwise reads the FULL uncompressed +# raster into memory (rasterio out_shape = full height x width), which OOMKills +# the worker on large orthos. A decimated read at <=1024x1024 is plenty: the +# tile URLs pin an explicit stretch_range=[0,255], so exact percentiles aren't +# used for rendering anyway. See terracotta.raster.compute_image_stats. +_STATS_MAX_SHAPE = (1024, 1024) + + +def register(space_id, map_id, variant_name, s3_uri): + driver = _driver() + with driver.connect(): + metadata = driver.compute_metadata(s3_uri, max_shape=_STATS_MAX_SHAPE) + driver.insert( + {"space": space_id, "map": map_id, "variant": variant_name}, + s3_uri, + override_path=s3_uri, + metadata=metadata, + ) diff --git a/code/mapmaker/maps/variants.py b/code/mapmaker/maps/variants.py new file mode 100644 index 0000000..aeb936b --- /dev/null +++ b/code/mapmaker/maps/variants.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from typing import Callable, Optional + +from mapmaker.maps import cog + + +@dataclass(frozen=True) +class VariantSpec: + name: str + singleband: bool + colormap: Optional[str] + build: Callable + + +VARIANTS = { + "rgb": [ + VariantSpec("rgb", singleband=False, colormap=None, build=cog.build_rgb), + VariantSpec("grayscale", singleband=True, colormap="greys_r", build=cog.build_grayscale), + ], + "multispectral": [ + VariantSpec("rgb", singleband=False, colormap=None, build=cog.build_rgb), + VariantSpec("grayscale", singleband=True, colormap="greys_r", build=cog.build_grayscale), + VariantSpec("ndvi", singleband=True, colormap="rdylgn", build=cog.build_ndvi), + ], + "dem": [ + VariantSpec("hillshade", singleband=True, colormap="greys_r", build=cog.build_hillshade), + VariantSpec("color-relief", singleband=False, colormap=None, build=cog.build_color_relief), + VariantSpec("slope", singleband=True, colormap="magma", build=cog.build_slope), + ], +} diff --git a/code/requirements.txt b/code/requirements.txt index 48d2dcc..ed7f92d 100644 --- a/code/requirements.txt +++ b/code/requirements.txt @@ -1,7 +1,9 @@ +# numpy<2 (and rasterio<1.5) keeps pip's numpy ABI-compatible with the +# apt-installed python3-gdal. Without it gdal_calc.py can't import gdal_array. +numpy<2 +rasterio==1.4.3 +terracotta==0.10.1 flask flask_cors -ninja2 -geotiff -rasterio names -requests \ No newline at end of file +requests diff --git a/code/start.py b/code/start.py index 750b2f7..a08736d 100755 --- a/code/start.py +++ b/code/start.py @@ -58,7 +58,9 @@ def rescan(): @app.route("/mapmaker", methods=["POST"]) def mapmaker(): - name = request.form.get("name") + # Uploads are saved through secure_filename, so this only rejects paths + # that point outside /data/maps. + name = secure_filename(request.form.get("name") or "") tags = request.form.get("tags") or request.form.get("tag") location = request.form.get("locations") or request.form.get("location") mapToMake = { diff --git a/docker-compose.yaml b/docker-compose.yaml index 98628be..1720087 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,5 +33,6 @@ services: environment: - TERRACOTTA_DB_PATH=${TERRACOTTA_DB_PATH:-/data/db/terracotta.sqlite} - TERRACOTTA_PROVIDER=${TERRACOTTA_PROVIDER:-sqlite} + - TC_ALLOWED_ORIGINS_TILES=["https://dash.ollebo.com","https://dash.northamlin.com"] working_dir: /code command: ["sh", "-c", "mkdir -p /data/db && python3 -c 'from mapmaker.maps import terracotta_client; terracotta_client.ensure_schema()' && terracotta serve -d /data/db/terracotta.sqlite --allow-all-ips --port 5000"] diff --git a/install/helm/mapserver/templates/configmap.yaml b/install/helm/mapserver/templates/configmap.yaml index 073b7f0..7dbea35 100644 --- a/install/helm/mapserver/templates/configmap.yaml +++ b/install/helm/mapserver/templates/configmap.yaml @@ -10,3 +10,4 @@ data: TERRACOTTA_PUBLIC_URL: {{ .Values.terracotta.publicUrl | quote }} TERRACOTTA_DB_PATH: {{ .Values.terracotta.dbPath | quote }} TERRACOTTA_PROVIDER: {{ .Values.terracotta.provider | quote }} + TC_ALLOWED_ORIGINS_TILES: {{ .Values.terracotta.allowedOrigins | toJson | quote }} diff --git a/install/helm/mapserver/values.yaml b/install/helm/mapserver/values.yaml index 83c4b8f..17e5ec0 100644 --- a/install/helm/mapserver/values.yaml +++ b/install/helm/mapserver/values.yaml @@ -22,6 +22,11 @@ terracotta: publicUrl: http://localhost:5001 dbPath: /data/db/terracotta.sqlite provider: sqlite + # Web pages allowed to load tiles. Terracotta's default is localhost only, + # which makes the ollebo.com dashboards fail with a CORS error. + allowedOrigins: + - https://dash.ollebo.com + - https://dash.northamlin.com # One Service exposes both container ports. service: