Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,3 +74,17 @@ Everything is via env vars, read with `os.environ.get(..., <default>)`. Key ones
- **Stale / unused trees.** `code_old/` and `old/` are previous iterations kept on disk; ignore them. `Pipfile` pins Python 3.7 but the actual containers use 3.9 (`Dockerfile`) or 3.12 (`Dockerfile_aws`) — `requirements.txt` is the source of truth for dependencies.
- **CI deploys via a separate manifests repo.** The GitHub Action templates `chart/` into `Ollebo/manifests` and pushes — the cluster picks up the new image from that repo, not directly from this one.
- **README quickstart is out of date** (refers to a "fins" folder workflow and old Meilisearch/Mongo backends); prefer the commands above.

## Spridningstillstånd (mandatory for any map/imagery work)

All map data handling follows Ollebo's spreading policy (LMFS 2016:1). The full rules are in `map-maker/docs/spridningstillstand.md` (local copy: `~/projects/ollebo/docs/spridningstillstand.md`). Read it before changing rendering, tiles, storage, sharing, export, thumbnails or anything that serves map data.

In short: high-res material (finer than 0.5 m) is visible only to members of the owning space. Anything public or external gets the coarse public version: resampled to 0.6 m in EPSG:3006, with maxzoom from the northernmost latitude. If unsure, keep it private and create a `compliance_review` row.

Never:
- publish or share externally anything finer than 0.5 m without a registered permit;
- publish DSM/DTM, point clouds, 3D, raw images, oblique images, video or detections;
- make a dataset public by only changing access/ACL (going public = a new coarse render);
- expose originals or high-res COGs/tiles via a public URL, cache or API;
- assume an area exemption (LMFS 2 kap. 3 §) applies without human review;
- remove or bypass audit logging (`publication_audit`), resolution checks or maxzoom gates.
195 changes: 182 additions & 13 deletions code/db/postgis.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,18 +339,154 @@ def addDataDb(json, db="maps"):


# Worker reports progress via `action`; the maps_status_check constraint only
# permits these four status values, so map every callback to one of them.
# permits four status values, so map every RENDER callback to one of them.
_ACTION_TO_STATUS = {
"makingMap": "processing",
"ready": "ready",
"partial": "ready",
"error": "failed",
}

# Publish-only callbacks (spridningstillstand.md): they never touch the private
# render (status/tilesurl/mapdata), only the publish state.
PUBLISH_ACTIONS = ("publicReady", "publicRefused", "unpublished", "unpublishFailed")

_PUBLISH_STATUSES = ("private", "rendering", "pending_review", "public", "rejected")


def _audit(cur, action, mapid, space_id, actor="map-maker", bbox=None, resolution=None,
max_zoom=None, material=None, legal_basis=None, detail=None):
# publication_audit is append-only (policy 5.7); schema lives in dw (028).
cur.execute(
"INSERT INTO publication_audit (action, dataset_id, space_id, actor, bbox, "
"resolution_m, max_zoom, material_type, legal_basis, detail) "
"VALUES (%s, %s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s)",
[action, mapid, space_id, actor,
json.dumps(bbox) if bbox is not None else None,
resolution, max_zoom, material, legal_basis,
json.dumps(detail) if detail is not None else None],
)


def _open_review(cur, mapid, space_id, reason, detail):
# One open case per map + reason (policy section 10).
cur.execute(
"INSERT INTO compliance_review (id, space_id, object_type, object_id, reason, detail) "
"SELECT gen_random_uuid(), %s::uuid, 'map', %s::uuid, %s, %s "
"WHERE NOT EXISTS (SELECT 1 FROM compliance_review WHERE object_id = %s::uuid "
"AND reason = %s AND status = 'open')",
[space_id, mapid, reason[:80], json.dumps(detail), mapid, reason[:80]],
)


def _apply_publish_outcome(cur, jsonData, render_failed=False):
"""Store the worker's public-render outcome (spridningstillstand.md).

Only applied while the row is still 'rendering': if a member unpublished or
a reviewer rejected meanwhile, the late result is ignored (map-maker removes
that public set when it processes the unpublish queued after it).
Returns the new publish_status, or None when nothing changed."""
mapid = jsonData["mapid"]
status = jsonData.get("publishStatus")
if render_failed:
status, jsonData = "private", {**jsonData, "publishReason": "render_failed"}
if status not in _PUBLISH_STATUSES or status == "rendering":
return None
public_url = jsonData.get("publicTilesURL")
if status == "public" and not public_url:
status = "pending_review"
jsonData = {**jsonData, "publishReason": "public_url_missing"}
is_public = status == "public"
reason = jsonData.get("publishReason") or None
cur.execute(
"UPDATE maps SET publish_status = %s, publish_reason = %s, "
"public_tilesurl = %s, public_maxzoom = %s, public_resolution_m = %s, "
"material_type = COALESCE(%s, material_type), updated_at = now() "
"WHERE mapid = %s::uuid AND publish_status = 'rendering' "
"RETURNING space_id::text, material_type",
[status, (reason or "")[:300] or None,
public_url if is_public else None,
jsonData.get("publicMaxZoom") if is_public else None,
jsonData.get("publicResolutionM") if is_public else None,
jsonData.get("materialType"), mapid],
)
row = cur.fetchone()
if row is None:
print("publish outcome ignored for {} (no longer rendering)".format(mapid))
return None
space_id, material = row[0], row[1]
detail = {
"complianceDetail": jsonData.get("complianceDetail"),
"inputType": jsonData.get("inputType"),
"reason": reason,
}
if is_public:
_audit(cur, "publish", mapid, space_id,
bbox=jsonData.get("publicBbox"),
resolution=jsonData.get("publicResolutionM"),
max_zoom=jsonData.get("publicMaxZoom"),
material=material,
legal_basis=jsonData.get("legalBasis"),
detail={**detail, "crs": jsonData.get("publicCrs"),
"maxZoomRetina": jsonData.get("publicMaxZoomRetina"),
"tilesUrl": public_url})
elif status == "pending_review":
_open_review(cur, mapid, space_id, (reason or "unknown").split(":")[0], detail)
elif status == "rejected":
_audit(cur, "publish_refused", mapid, space_id, material=material, detail=detail)
return status


def _update_publish_only(cur, jsonData):
mapid, action = jsonData["mapid"], jsonData["action"]
if action == "unpublished":
cur.execute(
"UPDATE maps SET action = %s, public_tilesurl = NULL, public_maxzoom = NULL, "
"public_resolution_m = NULL, updated_at = now(), "
"publish_status = CASE WHEN publish_status IN ('public', 'rendering') "
"THEN 'private' ELSE publish_status END "
"WHERE mapid = %s::uuid",
[action, mapid],
)
elif action == "unpublishFailed":
# The public set may still exist: hide it and put it in front of a human.
cur.execute(
"UPDATE maps SET action = %s, public_tilesurl = NULL, "
"publish_status = 'pending_review', publish_reason = %s, updated_at = now() "
"WHERE mapid = %s::uuid RETURNING space_id::text",
[action, (jsonData.get("publishReason") or "unpublish_failed")[:300], mapid],
)
row = cur.fetchone()
if row:
_open_review(cur, mapid, row[0], "unpublish_failed",
{"detail": jsonData.get("publishReason")})
else: # publicReady / publicRefused
cur.execute("UPDATE maps SET action = %s, updated_at = now() WHERE mapid = %s::uuid",
[action, mapid])
_apply_publish_outcome(cur, jsonData)
return cur.rowcount


@pooled
def updateMapDataDb(jsonData, db="maps"):
mapid = jsonData['mapid']
action = jsonData['action']

if action in PUBLISH_ACTIONS:
cur = conn.cursor()
try:
_update_publish_only(cur, jsonData)
cur.execute("SELECT 1 FROM maps WHERE mapid = %s::uuid", [mapid])
found = cur.fetchone() is not None
except Exception as e:
conn.rollback()
print("ERROR updateMapDataDb publish failed: mapid={} action={} err={}".format(mapid, action, e))
raise
conn.commit()
if not found:
return {"error": "unknown mapid", "mapid": mapid}
return {"data": "saved", "mapid": mapid}

status = _ACTION_TO_STATUS.get(action)
if status is None:
print("ERROR updateMapDataDb unknown action: mapid={} action={}".format(mapid, action))
Expand All @@ -366,6 +502,7 @@ def updateMapDataDb(jsonData, db="maps"):
**mapData,
'inputType': jsonData.get('inputType'),
'variants': jsonData.get('variants'),
'publicVariants': jsonData.get('publicVariants') or [],
}))
loc = mapData.get('location')
if loc:
Expand All @@ -377,7 +514,7 @@ def updateMapDataDb(jsonData, db="maps"):
params.append(json.dumps(area))

tiles_url = jsonData.get('tilesURL')
if tiles_url:
if tiles_url and tiles_url != "none":
sets.append("tilesurl = %s")
params.append(tiles_url)

Expand All @@ -387,28 +524,60 @@ def updateMapDataDb(jsonData, db="maps"):
cur = conn.cursor()
try:
cur.execute(query, params)
rowcount = cur.rowcount
if rowcount and action != "makingMap":
_apply_publish_outcome(cur, jsonData, render_failed=(status == "failed"))
except Exception as e:
conn.rollback()
print("ERROR updateMapDataDb failed: mapid={} action={} status={} err={}".format(mapid, action, status, e))
raise
rowcount = cur.rowcount
conn.commit()
if rowcount == 0:
print("ERROR updateMapDataDb no row matched: mapid={} action={}".format(mapid, action))
return {"error": "unknown mapid", "mapid": mapid}
return {"data": "saved", "mapid": mapid}




def _visibility_clause(groups):
# MAPS. `groups` are the caller's authorized space_ids — validated canonical
# UUIDs from jwt_auth.get_auth_context, so the `::uuid[]` cast below is safe.
# A map is visible outside its space only once its coarse public render
# passed (publish_status = 'public'); `access` alone is just a request.
if not groups:
return ("publish_status = %s", ['public'])
return ("(publish_status = %s OR space_id = ANY(%s::uuid[]))", ['public', list(groups)])


def _visibility_clause(groups):
# `groups` are the caller's authorized space_ids — validated canonical UUIDs
# from jwt_auth.get_auth_context, so the `::uuid[]` cast below is safe.
def _model_visibility_clause(groups):
# 3D models are never public (spridningstillstand.md section 3).
if not groups:
return ("access = %s", ['public'])
return ("(access = %s OR space_id = ANY(%s::uuid[]))", ['public', list(groups)])
return ("FALSE", [])
return ("space_id = ANY(%s::uuid[])", [list(groups)])


# Columns that describe the PRIVATE, full-resolution map. Stripped from every
# row the caller is not a member of (spridningstillstand.md 5.6).
_PRIVATE_MAP_FIELDS = ("mapdata", "originfile", "asset_id", "creator_id", "accessid",
"mapserver_id", "mapserver_map_key", "publish_reason")


def public_map_view(rows, groups=None):
"""Rewrite map rows for the caller: members see everything; for anyone else
`tilesurl` becomes the coarse public URL and private fields are dropped."""
member = set(str(g).lower() for g in (groups or ()))
if not isinstance(rows, list):
return rows
out = []
for row in rows:
if str(row.get("space_id", "")).lower() in member:
out.append(row)
continue
if row.get("publish_status") != "public" or not row.get("public_tilesurl"):
continue
clean = {k: v for k, v in row.items() if k not in _PRIVATE_MAP_FIELDS}
clean["tilesurl"] = row["public_tilesurl"]
out.append(clean)
return out


@pooled
Expand All @@ -424,7 +593,7 @@ def getDataDb(db="maps", groups=None):
cur = conn.cursor(cursor_factory=RealDictCursor)
try:
cur.execute(query, vis_params)
return cur.fetchall()
return public_map_view(cur.fetchall(), groups)
except Exception as e:
try:
conn.rollback()
Expand Down Expand Up @@ -637,7 +806,7 @@ def updateModelDataDb(jsonData, db="model"):

@pooled
def getModelsDb(groups=None):
vis_sql, vis_params = _visibility_clause(groups)
vis_sql, vis_params = _model_visibility_clause(groups)
query = (
"SELECT *, "
"TO_CHAR(created_at, 'YYYY-MM-DD') AS created_at, "
Expand Down Expand Up @@ -841,7 +1010,7 @@ def searchMaps(payload, groups=None):
cur = conn.cursor(cursor_factory=RealDictCursor)
try:
cur.execute(query, params)
return cur.fetchall()
return public_map_view(cur.fetchall(), groups)
except Exception as e:
conn.rollback()
print("searchMaps failed: {}".format(e))
Expand Down
7 changes: 5 additions & 2 deletions code/maps.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@ def setJsonValidate(jsonData):
return JsonStandrad


_KNOWN_ACTIONS = ("makingMap", "ready", "error")
_KNOWN_ACTIONS = ("makingMap", "ready", "partial", "error",
# publish-only callbacks (spridningstillstand.md)
"publicReady", "publicRefused", "unpublished", "unpublishFailed")
_PUBLISH_ONLY = ("publicReady", "publicRefused", "unpublished", "unpublishFailed")


def _validate_post(payload):
Expand All @@ -37,7 +40,7 @@ def _validate_post(payload):
action = payload["action"]
if action not in _KNOWN_ACTIONS:
return ["action(unknown:{})".format(action)]
if action == "makingMap":
if action == "makingMap" or action in _PUBLISH_ONLY:
return []
further = []
if action == "ready" and not payload.get("tilesURL"):
Expand Down
15 changes: 15 additions & 0 deletions code/pictures.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import storage
from event import resolve_mission, subject_for
from jwt_auth import get_auth_context, JwtError
from db.postgis import setPictureUploaded
from db.natsQue import addToNats

Expand Down Expand Up @@ -60,6 +61,20 @@ def download_picture(mission_key, picture_id, request):
if mission is None:
return jsonify({"error": "mission not found"}), 404

# Raw drone photos are never spread outside the owning space, even for a
# public mission (spridningstillstand.md section 3: single raw images have
# no exemption). Knowing the mission key is not enough any more.
try:
groups = get_auth_context(request.headers.get)
except JwtError as e:
return jsonify({"error": "unauthorized", "detail": str(e)}), 401
member = {str(g).lower() for g in (groups or ())}
space_id = (mission.get("space_id") or "").lower()
if not groups:
return jsonify({"error": "unauthorized"}), 401
if not space_id or space_id not in member:
return jsonify({"error": "forbidden"}), 403

try:
result = storage.get_picture(mission["id"], picture_id)
except Exception as e:
Expand Down
Loading