diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..87e0239 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.github +.env +maps +cogs +db +web +test-bild +install +**/__pycache__ +**/*.pyc diff --git a/code/mapmaker/__init__.py b/code/mapmaker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/code/mapmaker/makingGeotiff.py b/code/mapmaker/makingGeotiff.py index 0b6cb9d..f623dad 100644 --- a/code/mapmaker/makingGeotiff.py +++ b/code/mapmaker/makingGeotiff.py @@ -54,13 +54,23 @@ def getGeoTiffData(tiff_file): def _tile_url(space_id, map_id, spec): - base = "singleband" if spec.singleband else "rgb" - url = "{0}/{1}/{2}/{3}/{4}/{{z}}/{{x}}/{{y}}.png".format( - config.terracotta_public_url(), base, space_id, map_id, spec.name - ) - if spec.colormap and spec.singleband: - url += "?colormap=" + spec.colormap - return url + base_url = config.terracotta_public_url() + # Explicit ranges match the 8-bit COGs and the URLs map-maker produces. + if spec.singleband: + url = "{0}/singleband/{1}/{2}/{3}/{{z}}/{{x}}/{{y}}.png".format( + base_url, space_id, map_id, spec.name + ) + params = [] + if spec.colormap: + params.append("colormap=" + spec.colormap) + params.append("stretch_range=[0,255]") + return url + "?" + "&".join(params) + # RGB composite: three single-band datasets named __r/_g/_b. + return ( + "{0}/rgb/{1}/{2}/{{z}}/{{x}}/{{y}}.png" + "?r={3}__r&g={3}__g&b={3}__b" + "&r_range=[0,255]&g_range=[0,255]&b_range=[0,255]" + ).format(base_url, space_id, map_id, spec.name) def makingTiles(map): @@ -79,11 +89,12 @@ def makingTiles(map): variants_meta = [] for spec in VARIANTS[input_type]: - cog_path = os.path.join(cog_out_dir, "{0}.tif".format(spec.name)) status = "ok" try: - spec.build(source_path, cog_path, bands) - terracotta_client.register(space_id, map_id, spec.name, cog_path) + # One variant can produce several datasets (RGB is one per channel). + outputs = spec.build(source_path, cog_out_dir, spec.name, bands) + for variant_key, cog_path in outputs.items(): + terracotta_client.register(space_id, map_id, variant_key, cog_path) except Exception as e: print("variant {0} failed: {1}".format(spec.name, e)) status = "failed" diff --git a/code/mapmaker/maps/__init__.py b/code/mapmaker/maps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/code/mapmaker/maps/cog.py b/code/mapmaker/maps/cog.py new file mode 100644 index 0000000..86d39f7 --- /dev/null +++ b/code/mapmaker/maps/cog.py @@ -0,0 +1,140 @@ +import os +import subprocess +import tempfile + + +RAMPS_DIR = os.path.join(os.path.dirname(__file__), "ramps") + +COG_CO = [ + "-co", "COMPRESS=DEFLATE", + "-co", "PREDICTOR=2", + "-co", "OVERVIEWS=AUTO", + "-co", "BLOCKSIZE=512", +] + + +def _run(argv): + print("[cog] " + " ".join(argv)) + subprocess.run(argv, check=True, capture_output=True, text=True) + + +def _scale_args(dtype): + if dtype == "uint16": + return ["-scale", "0", "65535", "0", "255"] + return ["-scale"] + + +def _channel_key(name, channel): + return "{0}__{1}".format(name, channel) + + +def build_rgb(input_path, out_dir, name, bands): + # Terracotta's /rgb endpoint composites three single-band datasets, so we + # split source R/G/B into three separate single-band COGs. + scale = _scale_args(bands["dtype"]) + outputs = {} + for channel, src_band in (("r", bands["r"]), ("g", bands["g"]), ("b", bands["b"])): + out = os.path.join(out_dir, "{0}.tif".format(_channel_key(name, channel))) + _run([ + "gdal_translate", "-of", "COG", "-ot", "Byte", + "-b", str(src_band), + ] + scale + COG_CO + [input_path, out]) + outputs[_channel_key(name, channel)] = out + return outputs + + +def build_grayscale(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp: + intermediate = tmp.name + try: + _run([ + "gdal_calc.py", + "--calc=0.299*R+0.587*G+0.114*B", + "-R", input_path, f"--R_band={bands['r']}", + "-G", input_path, f"--G_band={bands['g']}", + "-B", input_path, f"--B_band={bands['b']}", + "--type=Byte", + f"--outfile={intermediate}", + "--overwrite", + ]) + _run(["gdal_translate", "-of", "COG"] + COG_CO + [intermediate, out]) + finally: + if os.path.exists(intermediate): + os.unlink(intermediate) + return {name: out} + + +def build_ndvi(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp: + intermediate = tmp.name + try: + _run([ + "gdal_calc.py", + "--calc=(NIR.astype(float)-R)/(NIR+R+1e-6)", + "-R", input_path, f"--R_band={bands['r']}", + "--NIR", input_path, f"--NIR_band={bands['nir']}", + "--type=Float32", + "--NoDataValue=-9999", + f"--outfile={intermediate}", + "--overwrite", + ]) + _run(["gdal_translate", "-of", "COG"] + COG_CO + [intermediate, out]) + finally: + if os.path.exists(intermediate): + os.unlink(intermediate) + return {name: out} + + +def build_hillshade(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + _run([ + "gdaldem", "hillshade", + "-of", "COG", + "-compute_edges", + "-z", "1.0", "-az", "315", "-alt", "45", + "-co", "COMPRESS=DEFLATE", + input_path, out, + ]) + return {name: out} + + +def build_color_relief(input_path, out_dir, name, bands): + # gdaldem color-relief produces a 3-band RGB tif; split into single-band + # COGs so terracotta's /rgb endpoint can composite them. + ramp = os.path.join(RAMPS_DIR, "dem.txt") + with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp: + intermediate = tmp.name + try: + _run([ + "gdaldem", "color-relief", + "-of", "GTiff", + "-co", "COMPRESS=DEFLATE", + input_path, ramp, intermediate, + ]) + outputs = {} + for channel, src_band in (("r", 1), ("g", 2), ("b", 3)): + out = os.path.join(out_dir, "{0}.tif".format(_channel_key(name, channel))) + _run([ + "gdal_translate", "-of", "COG", "-ot", "Byte", + "-b", str(src_band), + ] + COG_CO + [intermediate, out]) + outputs[_channel_key(name, channel)] = out + return outputs + finally: + if os.path.exists(intermediate): + os.unlink(intermediate) + + +def build_slope(input_path, out_dir, name, bands): + out = os.path.join(out_dir, "{0}.tif".format(name)) + _run([ + "gdaldem", "slope", + "-of", "COG", + "-compute_edges", + "-s", "1.0", + "-co", "COMPRESS=DEFLATE", + input_path, out, + ]) + return {name: out} diff --git a/code/mapmaker/maps/detect.py b/code/mapmaker/maps/detect.py new file mode 100644 index 0000000..7beca9d --- /dev/null +++ b/code/mapmaker/maps/detect.py @@ -0,0 +1,54 @@ +import re + +import rasterio +from rasterio.enums import ColorInterp + + +NIR_HINT = re.compile(r"nir|842|b5|rededge|red\s*edge", re.IGNORECASE) + + +def _band_looks_like_nir(ds, idx): + desc = (ds.descriptions[idx - 1] or "") + if NIR_HINT.search(desc): + return True + tags = ds.tags(idx) + for value in tags.values(): + if NIR_HINT.search(str(value)): + return True + return False + + +def detect_input_type(path): + with rasterio.open(path) as ds: + count = ds.count + dtype = ds.dtypes[0] + colorinterp = list(ds.colorinterp) + + if count == 1: + has_palette = False + try: + ds.colormap(1) + has_palette = True + except ValueError: + pass + if not has_palette: + return "dem", {"r": None, "g": None, "b": None, "nir": None, "dtype": dtype} + + if count >= 4 and dtype in ("uint8", "uint16"): + nir_idx = None + for i in range(4, count + 1): + if _band_looks_like_nir(ds, i): + nir_idx = i + break + if nir_idx is None and count == 4 and colorinterp[3] in (ColorInterp.undefined, ColorInterp.gray): + nir_idx = 4 + if nir_idx is not None: + return "multispectral", {"r": 1, "g": 2, "b": 3, "nir": nir_idx, "dtype": dtype} + + if count in (3, 4) and ColorInterp.red in colorinterp: + return "rgb", {"r": 1, "g": 2, "b": 3, "nir": None, "dtype": dtype} + + if dtype == "uint8" and count >= 3: + return "rgb", {"r": 1, "g": 2, "b": 3, "nir": None, "dtype": dtype} + + return "dem", {"r": None, "g": None, "b": None, "nir": None, "dtype": dtype} diff --git a/code/mapmaker/maps/ramps/dem.txt b/code/mapmaker/maps/ramps/dem.txt new file mode 100644 index 0000000..bd2cbda --- /dev/null +++ b/code/mapmaker/maps/ramps/dem.txt @@ -0,0 +1,12 @@ +nv 0 0 0 0 +-500 12 44 132 +0 65 105 225 +1 80 200 120 +200 154 205 50 +500 188 187 87 +1000 205 175 100 +1500 165 130 80 +2200 139 105 75 +3000 200 200 200 +5000 255 255 255 +8848 255 255 255 diff --git a/code/mapmaker/maps/ramps/ndvi.txt b/code/mapmaker/maps/ramps/ndvi.txt new file mode 100644 index 0000000..d3ef06c --- /dev/null +++ b/code/mapmaker/maps/ramps/ndvi.txt @@ -0,0 +1,7 @@ +nv 0 0 0 0 +-1.0 0 0 0 +-0.2 200 50 50 +0.0 230 230 130 +0.3 130 200 80 +0.6 30 150 30 +1.0 0 90 0 diff --git a/code/mapmaker/maps/terracotta_client.py b/code/mapmaker/maps/terracotta_client.py new file mode 100644 index 0000000..e3030e1 --- /dev/null +++ b/code/mapmaker/maps/terracotta_client.py @@ -0,0 +1,58 @@ +import os + +from terracotta import get_driver, exceptions + + +KEY_NAMES = ["space", "map", "variant"] +KEY_DESCRIPTIONS = { + "space": "Ollebo space (tenant) ID", + "map": "Map ID within the space", + "variant": "Rendering variant (rgb, grayscale, ndvi, hillshade, ...)", +} + + +def _driver(): + db_path = os.environ.get("TERRACOTTA_DB_PATH", "/data/db/terracotta.sqlite") + provider = os.environ.get("TERRACOTTA_PROVIDER", "sqlite") + return get_driver(db_path, provider=provider) + + +def ensure_schema(): + driver = _driver() + try: + with driver.connect(): + if not driver.key_names: + driver.create(KEY_NAMES, key_descriptions=KEY_DESCRIPTIONS) + except exceptions.InvalidDatabaseError: + driver.create(KEY_NAMES, key_descriptions=KEY_DESCRIPTIONS) + + +def replace_map(space_id, map_id): + driver = _driver() + with driver.connect(): + if not driver.key_names: + return + datasets = driver.get_datasets(where={"space": space_id, "map": map_id}) + for key_tuple in list(datasets.keys()): + driver.delete(dict(zip(driver.key_names, key_tuple))) + + +# Cap the array Terracotta reads when computing band statistics. Without the +# optional `crick` dependency, Terracotta otherwise reads the FULL uncompressed +# raster into memory (rasterio out_shape = full height x width), which OOMKills +# the worker on large orthos. A decimated read at <=1024x1024 is plenty: the +# tile URLs pin an explicit stretch_range=[0,255], so exact percentiles aren't +# used for rendering anyway. See terracotta.raster.compute_image_stats. +_STATS_MAX_SHAPE = (1024, 1024) + + +def register(space_id, map_id, variant_name, s3_uri): + driver = _driver() + with driver.connect(): + metadata = driver.compute_metadata(s3_uri, max_shape=_STATS_MAX_SHAPE) + driver.insert( + {"space": space_id, "map": map_id, "variant": variant_name}, + s3_uri, + override_path=s3_uri, + metadata=metadata, + ) diff --git a/code/mapmaker/maps/variants.py b/code/mapmaker/maps/variants.py new file mode 100644 index 0000000..aeb936b --- /dev/null +++ b/code/mapmaker/maps/variants.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from typing import Callable, Optional + +from mapmaker.maps import cog + + +@dataclass(frozen=True) +class VariantSpec: + name: str + singleband: bool + colormap: Optional[str] + build: Callable + + +VARIANTS = { + "rgb": [ + VariantSpec("rgb", singleband=False, colormap=None, build=cog.build_rgb), + VariantSpec("grayscale", singleband=True, colormap="greys_r", build=cog.build_grayscale), + ], + "multispectral": [ + VariantSpec("rgb", singleband=False, colormap=None, build=cog.build_rgb), + VariantSpec("grayscale", singleband=True, colormap="greys_r", build=cog.build_grayscale), + VariantSpec("ndvi", singleband=True, colormap="rdylgn", build=cog.build_ndvi), + ], + "dem": [ + VariantSpec("hillshade", singleband=True, colormap="greys_r", build=cog.build_hillshade), + VariantSpec("color-relief", singleband=False, colormap=None, build=cog.build_color_relief), + VariantSpec("slope", singleband=True, colormap="magma", build=cog.build_slope), + ], +} diff --git a/code/requirements.txt b/code/requirements.txt index 48d2dcc..ed7f92d 100644 --- a/code/requirements.txt +++ b/code/requirements.txt @@ -1,7 +1,9 @@ +# numpy<2 (and rasterio<1.5) keeps pip's numpy ABI-compatible with the +# apt-installed python3-gdal. Without it gdal_calc.py can't import gdal_array. +numpy<2 +rasterio==1.4.3 +terracotta==0.10.1 flask flask_cors -ninja2 -geotiff -rasterio names -requests \ No newline at end of file +requests diff --git a/code/start.py b/code/start.py index 750b2f7..a08736d 100755 --- a/code/start.py +++ b/code/start.py @@ -58,7 +58,9 @@ def rescan(): @app.route("/mapmaker", methods=["POST"]) def mapmaker(): - name = request.form.get("name") + # Uploads are saved through secure_filename, so this only rejects paths + # that point outside /data/maps. + name = secure_filename(request.form.get("name") or "") tags = request.form.get("tags") or request.form.get("tag") location = request.form.get("locations") or request.form.get("location") mapToMake = { diff --git a/docker-compose.yaml b/docker-compose.yaml index 98628be..1720087 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,5 +33,6 @@ services: environment: - TERRACOTTA_DB_PATH=${TERRACOTTA_DB_PATH:-/data/db/terracotta.sqlite} - TERRACOTTA_PROVIDER=${TERRACOTTA_PROVIDER:-sqlite} + - TC_ALLOWED_ORIGINS_TILES=["https://dash.ollebo.com","https://dash.northamlin.com"] working_dir: /code command: ["sh", "-c", "mkdir -p /data/db && python3 -c 'from mapmaker.maps import terracotta_client; terracotta_client.ensure_schema()' && terracotta serve -d /data/db/terracotta.sqlite --allow-all-ips --port 5000"] diff --git a/install/helm/mapserver/templates/configmap.yaml b/install/helm/mapserver/templates/configmap.yaml index 073b7f0..7dbea35 100644 --- a/install/helm/mapserver/templates/configmap.yaml +++ b/install/helm/mapserver/templates/configmap.yaml @@ -10,3 +10,4 @@ data: TERRACOTTA_PUBLIC_URL: {{ .Values.terracotta.publicUrl | quote }} TERRACOTTA_DB_PATH: {{ .Values.terracotta.dbPath | quote }} TERRACOTTA_PROVIDER: {{ .Values.terracotta.provider | quote }} + TC_ALLOWED_ORIGINS_TILES: {{ .Values.terracotta.allowedOrigins | toJson | quote }} diff --git a/install/helm/mapserver/values.yaml b/install/helm/mapserver/values.yaml index 83c4b8f..17e5ec0 100644 --- a/install/helm/mapserver/values.yaml +++ b/install/helm/mapserver/values.yaml @@ -22,6 +22,11 @@ terracotta: publicUrl: http://localhost:5001 dbPath: /data/db/terracotta.sqlite provider: sqlite + # Web pages allowed to load tiles. Terracotta's default is localhost only, + # which makes the ollebo.com dashboards fail with a CORS error. + allowedOrigins: + - https://dash.ollebo.com + - https://dash.northamlin.com # One Service exposes both container ports. service: