Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
.git
.github
.env
maps
cogs
db
web
test-bild
install
**/__pycache__
**/*.pyc
Empty file added code/mapmaker/__init__.py
Empty file.
31 changes: 21 additions & 10 deletions code/mapmaker/makingGeotiff.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,23 @@ def getGeoTiffData(tiff_file):


def _tile_url(space_id, map_id, spec):
base = "singleband" if spec.singleband else "rgb"
url = "{0}/{1}/{2}/{3}/{4}/{{z}}/{{x}}/{{y}}.png".format(
config.terracotta_public_url(), base, space_id, map_id, spec.name
)
if spec.colormap and spec.singleband:
url += "?colormap=" + spec.colormap
return url
base_url = config.terracotta_public_url()
# Explicit ranges match the 8-bit COGs and the URLs map-maker produces.
if spec.singleband:
url = "{0}/singleband/{1}/{2}/{3}/{{z}}/{{x}}/{{y}}.png".format(
base_url, space_id, map_id, spec.name
)
params = []
if spec.colormap:
params.append("colormap=" + spec.colormap)
params.append("stretch_range=[0,255]")
return url + "?" + "&".join(params)
# RGB composite: three single-band datasets named <name>__r/_g/_b.
return (
"{0}/rgb/{1}/{2}/{{z}}/{{x}}/{{y}}.png"
"?r={3}__r&g={3}__g&b={3}__b"
"&r_range=[0,255]&g_range=[0,255]&b_range=[0,255]"
).format(base_url, space_id, map_id, spec.name)


def makingTiles(map):
Expand All @@ -79,11 +89,12 @@ def makingTiles(map):

variants_meta = []
for spec in VARIANTS[input_type]:
cog_path = os.path.join(cog_out_dir, "{0}.tif".format(spec.name))
status = "ok"
try:
spec.build(source_path, cog_path, bands)
terracotta_client.register(space_id, map_id, spec.name, cog_path)
# One variant can produce several datasets (RGB is one per channel).
outputs = spec.build(source_path, cog_out_dir, spec.name, bands)
for variant_key, cog_path in outputs.items():
terracotta_client.register(space_id, map_id, variant_key, cog_path)
except Exception as e:
print("variant {0} failed: {1}".format(spec.name, e))
status = "failed"
Expand Down
Empty file added code/mapmaker/maps/__init__.py
Empty file.
140 changes: 140 additions & 0 deletions code/mapmaker/maps/cog.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
import os
import subprocess
import tempfile


RAMPS_DIR = os.path.join(os.path.dirname(__file__), "ramps")

COG_CO = [
"-co", "COMPRESS=DEFLATE",
"-co", "PREDICTOR=2",
"-co", "OVERVIEWS=AUTO",
"-co", "BLOCKSIZE=512",
]


def _run(argv):
print("[cog] " + " ".join(argv))
subprocess.run(argv, check=True, capture_output=True, text=True)


def _scale_args(dtype):
if dtype == "uint16":
return ["-scale", "0", "65535", "0", "255"]
return ["-scale"]


def _channel_key(name, channel):
return "{0}__{1}".format(name, channel)


def build_rgb(input_path, out_dir, name, bands):
# Terracotta's /rgb endpoint composites three single-band datasets, so we
# split source R/G/B into three separate single-band COGs.
scale = _scale_args(bands["dtype"])
outputs = {}
for channel, src_band in (("r", bands["r"]), ("g", bands["g"]), ("b", bands["b"])):
out = os.path.join(out_dir, "{0}.tif".format(_channel_key(name, channel)))
_run([
"gdal_translate", "-of", "COG", "-ot", "Byte",
"-b", str(src_band),
] + scale + COG_CO + [input_path, out])
outputs[_channel_key(name, channel)] = out
return outputs


def build_grayscale(input_path, out_dir, name, bands):
out = os.path.join(out_dir, "{0}.tif".format(name))
with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp:
intermediate = tmp.name
try:
_run([
"gdal_calc.py",
"--calc=0.299*R+0.587*G+0.114*B",
"-R", input_path, f"--R_band={bands['r']}",
"-G", input_path, f"--G_band={bands['g']}",
"-B", input_path, f"--B_band={bands['b']}",
"--type=Byte",
f"--outfile={intermediate}",
"--overwrite",
])
_run(["gdal_translate", "-of", "COG"] + COG_CO + [intermediate, out])
finally:
if os.path.exists(intermediate):
os.unlink(intermediate)
return {name: out}


def build_ndvi(input_path, out_dir, name, bands):
out = os.path.join(out_dir, "{0}.tif".format(name))
with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp:
intermediate = tmp.name
try:
_run([
"gdal_calc.py",
"--calc=(NIR.astype(float)-R)/(NIR+R+1e-6)",
"-R", input_path, f"--R_band={bands['r']}",
"--NIR", input_path, f"--NIR_band={bands['nir']}",
"--type=Float32",
"--NoDataValue=-9999",
f"--outfile={intermediate}",
"--overwrite",
])
_run(["gdal_translate", "-of", "COG"] + COG_CO + [intermediate, out])
finally:
if os.path.exists(intermediate):
os.unlink(intermediate)
return {name: out}


def build_hillshade(input_path, out_dir, name, bands):
out = os.path.join(out_dir, "{0}.tif".format(name))
_run([
"gdaldem", "hillshade",
"-of", "COG",
"-compute_edges",
"-z", "1.0", "-az", "315", "-alt", "45",
"-co", "COMPRESS=DEFLATE",
input_path, out,
])
return {name: out}


def build_color_relief(input_path, out_dir, name, bands):
# gdaldem color-relief produces a 3-band RGB tif; split into single-band
# COGs so terracotta's /rgb endpoint can composite them.
ramp = os.path.join(RAMPS_DIR, "dem.txt")
with tempfile.NamedTemporaryFile(suffix=".tif", delete=False) as tmp:
intermediate = tmp.name
try:
_run([
"gdaldem", "color-relief",
"-of", "GTiff",
"-co", "COMPRESS=DEFLATE",
input_path, ramp, intermediate,
])
outputs = {}
for channel, src_band in (("r", 1), ("g", 2), ("b", 3)):
out = os.path.join(out_dir, "{0}.tif".format(_channel_key(name, channel)))
_run([
"gdal_translate", "-of", "COG", "-ot", "Byte",
"-b", str(src_band),
] + COG_CO + [intermediate, out])
outputs[_channel_key(name, channel)] = out
return outputs
finally:
if os.path.exists(intermediate):
os.unlink(intermediate)


def build_slope(input_path, out_dir, name, bands):
out = os.path.join(out_dir, "{0}.tif".format(name))
_run([
"gdaldem", "slope",
"-of", "COG",
"-compute_edges",
"-s", "1.0",
"-co", "COMPRESS=DEFLATE",
input_path, out,
])
return {name: out}
54 changes: 54 additions & 0 deletions code/mapmaker/maps/detect.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import re

import rasterio
from rasterio.enums import ColorInterp


NIR_HINT = re.compile(r"nir|842|b5|rededge|red\s*edge", re.IGNORECASE)


def _band_looks_like_nir(ds, idx):
desc = (ds.descriptions[idx - 1] or "")
if NIR_HINT.search(desc):
return True
tags = ds.tags(idx)
for value in tags.values():
if NIR_HINT.search(str(value)):
return True
return False


def detect_input_type(path):
with rasterio.open(path) as ds:
count = ds.count
dtype = ds.dtypes[0]
colorinterp = list(ds.colorinterp)

if count == 1:
has_palette = False
try:
ds.colormap(1)
has_palette = True
except ValueError:
pass
if not has_palette:
return "dem", {"r": None, "g": None, "b": None, "nir": None, "dtype": dtype}

if count >= 4 and dtype in ("uint8", "uint16"):
nir_idx = None
for i in range(4, count + 1):
if _band_looks_like_nir(ds, i):
nir_idx = i
break
if nir_idx is None and count == 4 and colorinterp[3] in (ColorInterp.undefined, ColorInterp.gray):
nir_idx = 4
if nir_idx is not None:
return "multispectral", {"r": 1, "g": 2, "b": 3, "nir": nir_idx, "dtype": dtype}

if count in (3, 4) and ColorInterp.red in colorinterp:
return "rgb", {"r": 1, "g": 2, "b": 3, "nir": None, "dtype": dtype}

if dtype == "uint8" and count >= 3:
return "rgb", {"r": 1, "g": 2, "b": 3, "nir": None, "dtype": dtype}

return "dem", {"r": None, "g": None, "b": None, "nir": None, "dtype": dtype}
12 changes: 12 additions & 0 deletions code/mapmaker/maps/ramps/dem.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
nv 0 0 0 0
-500 12 44 132
0 65 105 225
1 80 200 120
200 154 205 50
500 188 187 87
1000 205 175 100
1500 165 130 80
2200 139 105 75
3000 200 200 200
5000 255 255 255
8848 255 255 255
7 changes: 7 additions & 0 deletions code/mapmaker/maps/ramps/ndvi.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
nv 0 0 0 0
-1.0 0 0 0
-0.2 200 50 50
0.0 230 230 130
0.3 130 200 80
0.6 30 150 30
1.0 0 90 0
58 changes: 58 additions & 0 deletions code/mapmaker/maps/terracotta_client.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import os

from terracotta import get_driver, exceptions


KEY_NAMES = ["space", "map", "variant"]
KEY_DESCRIPTIONS = {
"space": "Ollebo space (tenant) ID",
"map": "Map ID within the space",
"variant": "Rendering variant (rgb, grayscale, ndvi, hillshade, ...)",
}


def _driver():
db_path = os.environ.get("TERRACOTTA_DB_PATH", "/data/db/terracotta.sqlite")
provider = os.environ.get("TERRACOTTA_PROVIDER", "sqlite")
return get_driver(db_path, provider=provider)


def ensure_schema():
driver = _driver()
try:
with driver.connect():
if not driver.key_names:
driver.create(KEY_NAMES, key_descriptions=KEY_DESCRIPTIONS)
except exceptions.InvalidDatabaseError:
driver.create(KEY_NAMES, key_descriptions=KEY_DESCRIPTIONS)


def replace_map(space_id, map_id):
driver = _driver()
with driver.connect():
if not driver.key_names:
return
datasets = driver.get_datasets(where={"space": space_id, "map": map_id})
for key_tuple in list(datasets.keys()):
driver.delete(dict(zip(driver.key_names, key_tuple)))


# Cap the array Terracotta reads when computing band statistics. Without the
# optional `crick` dependency, Terracotta otherwise reads the FULL uncompressed
# raster into memory (rasterio out_shape = full height x width), which OOMKills
# the worker on large orthos. A decimated read at <=1024x1024 is plenty: the
# tile URLs pin an explicit stretch_range=[0,255], so exact percentiles aren't
# used for rendering anyway. See terracotta.raster.compute_image_stats.
_STATS_MAX_SHAPE = (1024, 1024)


def register(space_id, map_id, variant_name, s3_uri):
driver = _driver()
with driver.connect():
metadata = driver.compute_metadata(s3_uri, max_shape=_STATS_MAX_SHAPE)
driver.insert(
{"space": space_id, "map": map_id, "variant": variant_name},
s3_uri,
override_path=s3_uri,
metadata=metadata,
)
30 changes: 30 additions & 0 deletions code/mapmaker/maps/variants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
from dataclasses import dataclass
from typing import Callable, Optional

from mapmaker.maps import cog


@dataclass(frozen=True)
class VariantSpec:
name: str
singleband: bool
colormap: Optional[str]
build: Callable


VARIANTS = {
"rgb": [
VariantSpec("rgb", singleband=False, colormap=None, build=cog.build_rgb),
VariantSpec("grayscale", singleband=True, colormap="greys_r", build=cog.build_grayscale),
],
"multispectral": [
VariantSpec("rgb", singleband=False, colormap=None, build=cog.build_rgb),
VariantSpec("grayscale", singleband=True, colormap="greys_r", build=cog.build_grayscale),
VariantSpec("ndvi", singleband=True, colormap="rdylgn", build=cog.build_ndvi),
],
"dem": [
VariantSpec("hillshade", singleband=True, colormap="greys_r", build=cog.build_hillshade),
VariantSpec("color-relief", singleband=False, colormap=None, build=cog.build_color_relief),
VariantSpec("slope", singleband=True, colormap="magma", build=cog.build_slope),
],
}
10 changes: 6 additions & 4 deletions code/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
# numpy<2 (and rasterio<1.5) keeps pip's numpy ABI-compatible with the
# apt-installed python3-gdal. Without it gdal_calc.py can't import gdal_array.
numpy<2
rasterio==1.4.3
terracotta==0.10.1
flask
flask_cors
ninja2
geotiff
rasterio
names
requests
requests
4 changes: 3 additions & 1 deletion code/start.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,9 @@ def rescan():

@app.route("/mapmaker", methods=["POST"])
def mapmaker():
name = request.form.get("name")
# Uploads are saved through secure_filename, so this only rejects paths
# that point outside /data/maps.
name = secure_filename(request.form.get("name") or "")
tags = request.form.get("tags") or request.form.get("tag")
location = request.form.get("locations") or request.form.get("location")
mapToMake = {
Expand Down
1 change: 1 addition & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,5 +33,6 @@ services:
environment:
- TERRACOTTA_DB_PATH=${TERRACOTTA_DB_PATH:-/data/db/terracotta.sqlite}
- TERRACOTTA_PROVIDER=${TERRACOTTA_PROVIDER:-sqlite}
- TC_ALLOWED_ORIGINS_TILES=["https://dash.ollebo.com","https://dash.northamlin.com"]
working_dir: /code
command: ["sh", "-c", "mkdir -p /data/db && python3 -c 'from mapmaker.maps import terracotta_client; terracotta_client.ensure_schema()' && terracotta serve -d /data/db/terracotta.sqlite --allow-all-ips --port 5000"]
Loading
Loading