From 73983ce082c5afb773028e4b6eac0648a5568860 Mon Sep 17 00:00:00 2001 From: Christian Chwala Date: Wed, 21 Oct 2026 11:56:10 +0200 Subject: [PATCH 1/9] feat(webserver): admin accounts with all-provider map and archive overview Add an optional 'admins:' list to users.yml. Admin accounts are written to users.json with is_admin=true (no DB role, parser or SFTP account) and get /admin/realtime (all CMLs, coloured per provider) and /admin/archive (per-provider summary and daily time series). Cross-tenant queries run as webserver_role in a read-only session behind @admin_required. --- README.md | 23 ++++ scripts/generate_config.py | 40 +++++- scripts/tests/test_generate_config.py | 45 +++++++ users.yml | 12 ++ webserver/main.py | 172 ++++++++++++++++++++++++ webserver/templates/admin_archive.html | 87 ++++++++++++ webserver/templates/admin_realtime.html | 103 ++++++++++++++ webserver/templates/base.html | 17 ++- webserver/tests/test_admin.py | 166 +++++++++++++++++++++++ 9 files changed, 661 insertions(+), 4 deletions(-) create mode 100644 webserver/templates/admin_archive.html create mode 100644 webserver/templates/admin_realtime.html create mode 100644 webserver/tests/test_admin.py diff --git a/README.md b/README.md index 627bae6..f96853c 100644 --- a/README.md +++ b/README.md @@ -215,3 +215,26 @@ users: A user with multiple independent CML subnetworks gets one parser per source; all sources share the same `user_id` so Grafana sees their data unified. +### Admin accounts + +An optional top-level `admins:` list in `users.yml` defines web-login accounts +that see **all providers** at once: + +```yaml +admins: + - id: admin + display_name: Administrator +``` + +After running the generator, set a password with +`python scripts/set_password.py admin`. Admin accounts get two extra pages +(they have no tenant data of their own): + +- `/admin/realtime` — map of every CML, coloured per provider, with per-provider toggles +- `/admin/archive` — per-provider summary and daily record-count / mean-RSL plots + +Admin views query as `webserver_role` (read-only session), which already has +read access across tenants; no database migration is needed. Admin routes are +guarded by `@admin_required` in `webserver/main.py` — use it on any new +cross-tenant route. + diff --git a/scripts/generate_config.py b/scripts/generate_config.py index 8c575e8..4a1f47c 100644 --- a/scripts/generate_config.py +++ b/scripts/generate_config.py @@ -12,7 +12,8 @@ Outputs (all paths relative to the repository root): docker-compose.override.yml sftp_receiver/entrypoint.sh - webserver/configs/users.json (existing password hashes preserved) + webserver/configs/users.json (existing password hashes preserved; + also receives entries from `admins:`) grafana/provisioning/datasources/postgres.yml grafana/init_grafana.py (ORGS/USERS lists replaced) database/migrations/NNN_add_.sql (only emitted for genuinely new users) @@ -42,6 +43,23 @@ def load_users(users_file: Path) -> list[dict]: return users +def load_admins(users_file: Path, users: list[dict]) -> list[dict]: + """Return the ``admins:`` entries of users.yml (web-login-only accounts).""" + with open(users_file) as f: + data = yaml.safe_load(f) or {} + admins = data.get("admins") or [] + user_ids = {u["id"] for u in users} + seen: set[str] = set() + for a in admins: + aid = a.get("id") + if not aid: + raise ValueError("Admin entry missing 'id'") + if aid in user_ids or aid in seen: + raise ValueError(f"Admin id {aid!r} duplicates another user id") + seen.add(aid) + return admins + + def _validate(users: list[dict]) -> None: seen_ids = set() seen_uids = set() @@ -244,15 +262,30 @@ def generate_entrypoint_sh(users: list[dict]) -> str: # --------------------------------------------------------------------------- -def generate_users_json(users: list[dict], existing_json: dict) -> dict: +def generate_users_json( + users: list[dict], existing_json: dict, admins: list[dict] | None = None +) -> dict: """ Merge users.yml into the existing users.json. - Existing password hashes and grafana_org_id are preserved. - New users get a skeleton entry (password must be set via set_password.py). - Users present in users.json but absent from users.yml are left intact (so manual removals require an explicit edit of users.json). + - Admin accounts get ``"is_admin": true`` and no grafana_org_id; they have + no DB role, parser or SFTP account. """ result = dict(existing_json) + for a in admins or []: + aid = a["id"] + entry = result.get(aid, {"password_hash": ""}) + entry["display_name"] = a.get("display_name", aid) + entry["is_admin"] = True + if aid not in result: + print( + f" [users.json] Added skeleton entry for new admin {aid!r}. " + "Run scripts/set_password.py to set their password." + ) + result[aid] = entry for u in users: uid = u["id"] if uid in result: @@ -638,7 +671,8 @@ def main(argv: list[str] | None = None) -> None: existing_json: dict = {} if users_json_path.exists(): existing_json = json.loads(users_json_path.read_text()) - new_json = generate_users_json(users, existing_json) + admins = load_admins(users_file, users) + new_json = generate_users_json(users, existing_json, admins) users_json_path.write_text(json.dumps(new_json, indent=4) + "\n") print(f" [webserver] Written {users_json_path.relative_to(repo_root)}") diff --git a/scripts/tests/test_generate_config.py b/scripts/tests/test_generate_config.py index 83e9802..b20482b 100644 --- a/scripts/tests/test_generate_config.py +++ b/scripts/tests/test_generate_config.py @@ -158,3 +158,48 @@ def test_main_default_ssh_keys_dir_stays_inside_repo(tmp_path): override = (repo_root / "docker-compose.override.yml").read_text() assert "./ssh_keys/alice/authorized_keys" in override assert (repo_root / "ssh_keys" / "alice" / "id_ed25519").exists() + + +# --------------------------------------------------------------------------- +# admins: — web-login-only accounts +# --------------------------------------------------------------------------- + + +def test_admins_written_to_users_json_only(tmp_path): + from generate_config import generate_users_json, load_admins + + users_file = tmp_path / "users.yml" + users_file.write_text( + yaml.dump({"users": USERS, "admins": [{"id": "boss", "display_name": "Boss"}]}) + ) + admins = load_admins(users_file, USERS) + existing = {"boss": {"password_hash": "keepme"}} + + result = generate_users_json(USERS, existing, admins) + + assert result["boss"] == { + "password_hash": "keepme", + "display_name": "Boss", + "is_admin": True, + } + assert "is_admin" not in result["alice"] + + +@pytest.mark.parametrize( + "admins", + [[{"id": "alice"}], [{"id": "x"}, {"id": "x"}], [{"display_name": "no id"}]], +) +def test_load_admins_rejects_invalid_entries(tmp_path, admins): + from generate_config import load_admins + + users_file = tmp_path / "users.yml" + users_file.write_text(yaml.dump({"users": USERS, "admins": admins})) + with pytest.raises(ValueError): + load_admins(users_file, USERS) + + +def test_load_admins_absent_is_empty(tmp_path): + from generate_config import load_admins + + users_file = _make_users_yml(tmp_path) + assert load_admins(users_file, USERS) == [] diff --git a/users.yml b/users.yml index 744dd1a..243df73 100644 --- a/users.yml +++ b/users.yml @@ -59,6 +59,18 @@ # All sources of one user share the same user_id, so Grafana sees their # data unified. Each source gets its own named Docker volume and parser # service (parser__). +# +# Optional top-level `admins:` list — web-login accounts that can see all +# providers (map + archive overview at /admin/realtime and /admin/archive). +# Admins have no DB role, SFTP account, parser or Grafana org; they are only +# written to webserver/configs/users.json. Set a password with +# scripts/set_password.py . +# id — login name (must not equal any user id). +# display_name — label shown in the webserver UI. +# +#admins: +# - id: admin +# display_name: Administrator users: - id: demo_openmrg diff --git a/webserver/main.py b/webserver/main.py index 1dee237..4a715ce 100644 --- a/webserver/main.py +++ b/webserver/main.py @@ -9,6 +9,7 @@ from markupsafe import escape from flask import ( Flask, + abort, render_template, request, jsonify, @@ -34,6 +35,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from contextlib import contextmanager +from functools import wraps import uuid app = Flask(__name__) @@ -75,6 +77,7 @@ def __init__(self, user_id: str): self.id = user_id self.display_name = USERS[user_id].get("display_name", user_id) self.grafana_org_id = USERS[user_id].get("grafana_org_id", 1) + self.is_admin = bool(USERS[user_id].get("is_admin", False)) @login_manager.user_loader @@ -141,6 +144,8 @@ def user_db_scope(user_id: str): """ if user_id not in USERS: raise ValueError(f"Unknown user_id: {user_id!r}") + if USERS[user_id].get("is_admin"): + raise ValueError(f"Admin account {user_id!r} has no tenant data role") conn = psycopg2.connect(os.getenv("DATABASE_URL")) try: @@ -198,6 +203,8 @@ def logout(): @login_required def overview(): """Landing page with overview and processing status""" + if getattr(current_user, "is_admin", False) is True: + return redirect(url_for("admin_realtime")) stats = { "total_cmls": 0, "total_records": 0, @@ -796,6 +803,171 @@ def archive(): ) +# ==================== ADMIN (ALL PROVIDERS) ROUTES ==================== +# Admin accounts (is_admin in users.json) have no tenant DB role. These views +# read across all tenants as webserver_role, which bypasses per-user isolation, +# so every route here must be decorated with @admin_required. + + +def admin_required(view): + """Restrict a view to accounts flagged is_admin in users.json.""" + + @wraps(view) + @login_required + def wrapper(*args, **kwargs): + if getattr(current_user, "is_admin", False) is not True: + abort(403) + return view(*args, **kwargs) + + return wrapper + + +@contextmanager +def admin_db_scope(): + """Read-only cross-tenant connection as webserver_role.""" + conn = psycopg2.connect(os.getenv("DATABASE_URL")) + try: + conn.set_session(readonly=True) + yield conn + finally: + conn.close() + + +def _provider_list(): + return [ + {"id": uid, "display_name": cfg.get("display_name", uid)} + for uid, cfg in USERS.items() + if not cfg.get("is_admin") + ] + + +def get_admin_archive_summary(): + """Per-provider CML count, record count and archived date range.""" + with admin_db_scope() as conn: + cur = conn.cursor() + cur.execute( + "SELECT user_id, COUNT(DISTINCT cml_id) FROM cml_metadata GROUP BY user_id" + ) + cml_counts = dict(cur.fetchall()) + cur.execute( + "SELECT user_id, COALESCE(SUM(total_records), 0) FROM cml_stats GROUP BY user_id" + ) + records = dict(cur.fetchall()) + cur.execute( + "SELECT user_id, MIN(bucket), MAX(bucket) FROM cml_data_1h GROUP BY user_id" + ) + ranges = {row[0]: (row[1], row[2]) for row in cur.fetchall()} + cur.close() + + summary = {} + for uid, (start, end) in ranges.items(): + summary[uid] = {"start": start, "end": end} + for uid in set(cml_counts) | set(records) | set(ranges): + entry = summary.setdefault(uid, {"start": None, "end": None}) + entry["cml_count"] = cml_counts.get(uid, 0) + entry["total_records"] = int(records.get(uid, 0)) + return summary + + +@app.route("/admin/realtime") +@admin_required +def admin_realtime(): + """Map of all CMLs from all providers.""" + return render_template("admin_realtime.html", providers=_provider_list()) + + +@app.route("/admin/archive") +@admin_required +def admin_archive(): + """Per-provider archive overview with time series.""" + try: + summary = get_admin_archive_summary() + except Exception as e: + print(f"Error fetching admin archive summary: {e}") + flash("Could not load archive statistics.") + summary = {} + empty = {"start": None, "end": None, "cml_count": 0, "total_records": 0} + providers = [{**p, **summary.get(p["id"], empty)} for p in _provider_list()] + return render_template( + "admin_archive.html", providers=providers, day_options=_ADMIN_TS_DAYS + ) + + +@app.route("/api/admin/cml-map") +@admin_required +def api_admin_cml_map(): + """CML endpoints of all providers, tagged with user_id.""" + try: + with admin_db_scope() as conn: + cur = conn.cursor() + cur.execute( + """SELECT DISTINCT ON (user_id, cml_id) + user_id, cml_id::text, site_0_lon, site_0_lat, site_1_lon, site_1_lat + FROM cml_metadata + WHERE site_0_lon IS NOT NULL AND site_0_lat IS NOT NULL + AND site_1_lon IS NOT NULL AND site_1_lat IS NOT NULL + ORDER BY user_id, cml_id""" + ) + rows = cur.fetchall() + cur.close() + except Exception as e: + print(f"Error fetching admin CML map data: {e}") + return jsonify({"error": "Database error"}), 500 + + return jsonify( + [ + { + "user_id": row[0], + "cml_id": str(row[1]), + "site_0": {"lon": float(row[2]), "lat": float(row[3])}, + "site_1": {"lon": float(row[4]), "lat": float(row[5])}, + } + for row in rows + ] + ) + + +_ADMIN_TS_DAYS = (7, 30, 90, 365) + + +@app.route("/api/admin/archive-timeseries") +@admin_required +def api_admin_archive_timeseries(): + """Daily record count and mean RSL per provider from the 1 h aggregate.""" + try: + days = int(request.args.get("days", 30)) + except ValueError: + days = 30 + if days not in _ADMIN_TS_DAYS: + days = 30 + + try: + with admin_db_scope() as conn: + cur = conn.cursor() + cur.execute( + """SELECT user_id, time_bucket('1 day', bucket) AS day, + SUM(record_count), AVG(rsl_avg) + FROM cml_data_1h + WHERE bucket >= NOW() - make_interval(days => %s) + GROUP BY user_id, day + ORDER BY user_id, day""", + (days,), + ) + rows = cur.fetchall() + cur.close() + except Exception as e: + print(f"Error fetching admin archive time series: {e}") + return jsonify({"error": "Database error"}), 500 + + series: dict = {} + for user_id, day, records, mean_rsl in rows: + s = series.setdefault(user_id, {"days": [], "records": [], "mean_rsl": []}) + s["days"].append(day.strftime("%Y-%m-%d")) + s["records"].append(int(records or 0)) + s["mean_rsl"].append(safe_float(mean_rsl)) + return jsonify(series) + + # ==================== DATA UPLOADS ROUTES ==================== diff --git a/webserver/templates/admin_archive.html b/webserver/templates/admin_archive.html new file mode 100644 index 0000000..6903858 --- /dev/null +++ b/webserver/templates/admin_archive.html @@ -0,0 +1,87 @@ +{% extends "base.html" %} + +{% block title %}All Providers Archive - GMDI Platform{% endblock %} + +{% block content %} +
+

Archive by Provider

+
+ + +
+
+ +{% with messages = get_flashed_messages() %} +{% for m in messages %}
{{ m }}
{% endfor %} +{% endwith %} + +{% for p in providers %} +
+
+
+ {{ p.display_name }} + {{ p.id }} + {{ p.cml_count }} + CMLs + ~{{ "{:,}".format(p.total_records) + }} records + {% if p.start and p.end %} + + {{ p.start.strftime('%Y-%m-%d') }} → {{ p.end.strftime('%Y-%m-%d') }} + + {% else %} + no data + {% endif %} +
+
+
+
+{% endfor %} +{% endblock %} + +{% block extra_js %} + + +{% endblock %} diff --git a/webserver/templates/admin_realtime.html b/webserver/templates/admin_realtime.html new file mode 100644 index 0000000..21410e1 --- /dev/null +++ b/webserver/templates/admin_realtime.html @@ -0,0 +1,103 @@ +{% extends "base.html" %} + +{% block title %}All Providers Map - GMDI Platform{% endblock %} + +{% block extra_css %} + + +{% endblock %} + +{% block content %} +
+

All Providers

+
+ Loading… +
+
+{% endblock %} + +{% block extra_js %} + + +{% endblock %} diff --git a/webserver/templates/base.html b/webserver/templates/base.html index f12e26c..75b8ebf 100644 --- a/webserver/templates/base.html +++ b/webserver/templates/base.html @@ -23,7 +23,22 @@