diff --git a/.github/workflows/test_integration_e2e.yml b/.github/workflows/test_integration_e2e.yml index 54259ae..f283f99 100644 --- a/.github/workflows/test_integration_e2e.yml +++ b/.github/workflows/test_integration_e2e.yml @@ -26,7 +26,14 @@ on: jobs: e2e-integration-tests: runs-on: ubuntu-latest - + env: + # Use the small 12-hour file (committed to the repo) to avoid a large + # download in CI. The 3-month file is only needed for production. + # Job-level so every `docker compose` call sees the same config and + # does not recreate the running services. + OPENMRG_NETCDF_FILE: /app/example_data/openMRG_cmls_20150827_12hours.nc + OPENMRG_NETCDF_FILE_URL: "" + steps: - uses: actions/checkout@v4 @@ -69,11 +76,6 @@ jobs: cat ssh_keys/known_hosts - name: Start services - env: - # Use the small 12-hour file (committed to the repo) to avoid a large - # download in CI. The 3-month file is only needed for production. - OPENMRG_NETCDF_FILE: /app/example_data/openMRG_cmls_20150827_12hours.nc - OPENMRG_NETCDF_FILE_URL: "" run: | docker compose up -d database sftp_receiver parser_demo_openmrg_main webserver mno_simulator_openmrg sleep 10 @@ -118,7 +120,8 @@ jobs: - name: Run E2E integration tests run: | - docker compose --profile testing run --rm integration_tests + # --no-deps: services are already running; don't let compose recreate them + docker compose --profile testing run --rm --no-deps integration_tests - name: Show logs on failure if: failure() diff --git a/README.md b/README.md index 627bae6..b273679 100644 --- a/README.md +++ b/README.md @@ -215,3 +215,32 @@ users: A user with multiple independent CML subnetworks gets one parser per source; all sources share the same `user_id` so Grafana sees their data unified. +### Admin accounts + +An optional top-level `admins:` list in `users.yml` defines web-login accounts +that see **all providers** at once: + +```yaml +admins: + - id: gmdi_admin + display_name: Administrator +``` + +After running the generator, set a password with +`python scripts/set_password.py gmdi_admin`. Admin accounts get two extra pages +(they have no tenant data of their own): + +- `/admin/realtime` — map of every CML, coloured per provider, with per-provider toggles, + and an embedded Grafana time series (RSL / TSL) below it; clicking a link on the map + selects that provider and CML in the panel +- `/admin/archive` — per-provider summary and daily record-count / mean-RSL plots + +Admin views query as `webserver_role` (read-only session), which already has +read access across tenants; no database migration is needed. All admins share +one extra Grafana org (created by `init_grafana` when `admins:` is set) with a +`webserver_role` datasource and the dashboard in `grafana/admin_dashboards/`. +Auth-proxy logins map to Grafana users by login name, so do not use `admin` as +an admin id: it resolves to Grafana's built-in `admin` account. Admin routes are +guarded by `@admin_required` in `webserver/main.py` — use it on any new +cross-tenant route. + diff --git a/grafana/Dockerfile b/grafana/Dockerfile index fce0fd8..5acc630 100644 --- a/grafana/Dockerfile +++ b/grafana/Dockerfile @@ -1,4 +1,5 @@ FROM python:3.11-slim RUN pip install --no-cache-dir requests COPY init_grafana.py /init_grafana.py +COPY admin_dashboards /admin_dashboards CMD ["python", "/init_grafana.py"] diff --git a/grafana/admin_dashboards/cml-admin-realtime.json b/grafana/admin_dashboards/cml-admin-realtime.json new file mode 100644 index 0000000..4e597f3 --- /dev/null +++ b/grafana/admin_dashboards/cml-admin-realtime.json @@ -0,0 +1,805 @@ +{ + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "refresh": "10s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "cml", + "timeseries", + "realtime", + "admin" + ], + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "CML Real-Time Data (All Providers)", + "uid": "cml-admin-realtime", + "version": 1, + "weekStart": "", + "id": null, + "panels": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "RSL (dBm)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dBm" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sublink_1 min" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.pointSize", + "value": 0 + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2 min" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.pointSize", + "value": 0 + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_1 max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + }, + { + "id": "custom.fillBelowTo", + "value": "sublink_1 min" + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2 max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + }, + { + "id": "custom.fillBelowTo", + "value": "sublink_2 min" + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_1 avg" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2 avg" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_1" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' min' AS metric,\n rsl_min AS value\nFROM cml_data_1h\nWHERE user_id = '${provider}'\n AND cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "A" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' max' AS metric,\n rsl_max AS value\nFROM cml_data_1h\nWHERE user_id = '${provider}'\n AND cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "B" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' avg' AS metric,\n rsl_avg AS value\nFROM cml_data_1h\nWHERE user_id = '${provider}'\n AND cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "C" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH bucketed AS (\n SELECT\n time_bucket('$__interval', time) AS bucket,\n sublink_id,\n MIN(rsl) AS rsl_min,\n MAX(rsl) AS rsl_max,\n AVG(rsl) AS rsl_avg\n FROM cml_data\n WHERE cml_id = '${cml_id}'\n AND user_id = '${provider}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) <= 259200\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\n GROUP BY 1, 2\n)\nSELECT bucket AS \"time\", sublink_id || ' min' AS metric, rsl_min AS value FROM bucketed\nUNION ALL\nSELECT bucket AS \"time\", sublink_id || ' max' AS metric, rsl_max AS value FROM bucketed\nUNION ALL\nSELECT bucket AS \"time\", sublink_id || ' avg' AS metric, rsl_avg AS value FROM bucketed\nORDER BY 1 ASC", + "refId": "D" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n time AS \"time\",\n sublink_id AS metric,\n rsl AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND user_id = '${provider}'\n AND '${interval}' = 'raw'\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "E" + } + ], + "title": "CML Time Series - Received Signal Level", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "TSL (dB)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dB" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "sublink_1 min" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.pointSize", + "value": 0 + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2 min" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.pointSize", + "value": 0 + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_1 max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + }, + { + "id": "custom.fillBelowTo", + "value": "sublink_1 min" + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2 max" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 20 + }, + { + "id": "custom.fillBelowTo", + "value": "sublink_2 min" + }, + { + "id": "custom.hideFrom", + "value": { + "tooltip": true, + "viz": false, + "legend": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_1 avg" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2 avg" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_1" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sublink_2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' min' AS metric,\n tsl_min AS value\nFROM cml_data_1h\nWHERE user_id = '${provider}'\n AND cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "A" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' max' AS metric,\n tsl_max AS value\nFROM cml_data_1h\nWHERE user_id = '${provider}'\n AND cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "B" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n bucket AS \"time\",\n sublink_id || ' avg' AS metric,\n tsl_avg AS value\nFROM cml_data_1h\nWHERE user_id = '${provider}'\n AND cml_id = '${cml_id}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) > 259200\n AND bucket >= $__timeFrom()::timestamptz\n AND bucket <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "C" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n time_bucket('$__interval', time) AS \"time\",\n sublink_id AS metric,\n AVG(tsl) AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND user_id = '${provider}'\n AND '${interval}' = 'auto'\n AND EXTRACT(EPOCH FROM ($__timeTo()::timestamptz - $__timeFrom()::timestamptz)) <= 259200\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nGROUP BY 1, 2\nORDER BY 1 ASC", + "refId": "D" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n time AS \"time\",\n sublink_id AS metric,\n tsl AS value\nFROM cml_data\nWHERE cml_id = '${cml_id}'\n AND user_id = '${provider}'\n AND '${interval}' = 'raw'\n AND time >= $__timeFrom()::timestamptz\n AND time <= $__timeTo()::timestamptz\nORDER BY 1 ASC", + "refId": "E" + } + ], + "title": "CML Time Series - Transmitted Signal Level", + "type": "timeseries" + } + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "type": "datasource", + "label": "Datasource" + }, + { + "name": "provider", + "label": "Provider", + "type": "query", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "definition": "SELECT DISTINCT user_id FROM cml_metadata ORDER BY 1", + "query": "SELECT DISTINCT user_id FROM cml_metadata ORDER BY 1", + "refresh": 1, + "includeAll": false, + "multi": false, + "hide": 0, + "sort": 0, + "current": {}, + "options": [], + "regex": "" + }, + { + "name": "cml_id", + "label": "CML ID", + "type": "query", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${datasource}" + }, + "definition": "SELECT DISTINCT cml_id::text FROM cml_metadata WHERE user_id = '${provider}' ORDER BY 1", + "query": "SELECT DISTINCT cml_id::text FROM cml_metadata WHERE user_id = '${provider}' ORDER BY 1", + "refresh": 1, + "includeAll": false, + "multi": false, + "hide": 0, + "sort": 0, + "current": {}, + "options": [], + "regex": "" + }, + { + "current": { + "selected": true, + "text": "Auto", + "value": "auto" + }, + "description": "Time aggregation interval for downsampling data", + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "Auto", + "value": "auto" + }, + { + "selected": false, + "text": "Raw", + "value": "raw" + } + ], + "query": "Auto : auto,Raw : raw", + "queryValue": "", + "skipUrlSync": true, + "type": "custom" + } + ] + }, + "annotations": { + "list": [] + } +} diff --git a/grafana/init_grafana.py b/grafana/init_grafana.py index 6919783..8e3e265 100644 --- a/grafana/init_grafana.py +++ b/grafana/init_grafana.py @@ -12,8 +12,11 @@ (those orgs do not exist when Grafana starts, so provisioning cannot cover them). """ +import json import sys import time +from pathlib import Path + import requests GRAFANA_URL = "http://grafana:3000" @@ -29,6 +32,14 @@ {"login": "demo_orange_cameroun", "org_id": 2, "role": "Viewer"}, ] +ADMIN_ORG = None + +# Dashboards imported only into the admin org (they query across all tenants). +ADMIN_DASHBOARD_DIR = Path("/admin_dashboards") +# Read-all DB role behind the admin org's datasource (see database/init.sql). +ADMIN_DB_USER = "webserver_role" +ADMIN_DB_PASSWORD = "webserverpassword" + def wait_for_grafana(timeout=120): deadline = time.time() + timeout @@ -50,6 +61,14 @@ def get_or_create_org(org_id, org_name): """Ensure an org with the given numeric id and name exists.""" r = requests.get(f"{GRAFANA_URL}/api/orgs/{org_id}", auth=ADMIN_AUTH) if r.status_code == 200: + # A name mismatch means two configs claim one org id (tenant vs admin org). + if r.json().get("name") != org_name: + print( + f"ERROR: org {org_id} is named {r.json().get('name')!r}, expected " + f"{org_name!r}; refusing to share it.", + file=sys.stderr, + ) + sys.exit(1) print(f"Org {org_id} ({org_name}) already exists.") return # Org id=1 always exists; for id>=2 we create by name. @@ -58,11 +77,17 @@ def get_or_create_org(org_id, org_name): json={"name": org_name}, auth=ADMIN_AUTH, ) - if r.status_code not in (200, 409): + if r.status_code != 200: print( f"ERROR creating org {org_name}: {r.status_code} {r.text}", file=sys.stderr ) sys.exit(1) + if r.json().get("orgId") != org_id: + print( + f"ERROR: org {org_name} got id {r.json().get('orgId')}, expected {org_id}.", + file=sys.stderr, + ) + sys.exit(1) print(f"Created org {org_name}.") @@ -246,6 +271,23 @@ def copy_dashboards_to_org(target_org_id, source_org_id=1): print(f"Copied dashboard {uid} to org {target_org_id}.") +def import_admin_dashboards(org_id): + """Import the cross-tenant dashboards into the admin org only.""" + for path in sorted(ADMIN_DASHBOARD_DIR.glob("*.json")): + dashboard = json.loads(path.read_text()) + dashboard["id"] = None + r = requests.post( + f"{GRAFANA_URL}/api/dashboards/db", + json={"dashboard": dashboard, "overwrite": True, "folderId": 0}, + auth=ADMIN_AUTH, + headers={"X-Grafana-Org-Id": str(org_id)}, + ) + if r.status_code != 200: + print(f"WARN: could not import {path.name}: {r.status_code} {r.text}") + else: + print(f"Imported {path.name} into org {org_id}.") + + if __name__ == "__main__": wait_for_grafana() rename_default_org() @@ -270,6 +312,16 @@ def copy_dashboards_to_org(target_org_id, source_org_id=1): time.sleep(2) for org in ORGS[1:]: copy_dashboards_to_org(target_org_id=org["id"], source_org_id=1) + if ADMIN_ORG: + get_or_create_org(ADMIN_ORG["id"], ADMIN_ORG["name"]) + create_datasource_for_org( + org_id=ADMIN_ORG["id"], + name="PostgreSQL", + uid="ds_admin", + user=ADMIN_DB_USER, + password=ADMIN_DB_PASSWORD, + ) + import_admin_dashboards(ADMIN_ORG["id"]) for user in USERS: get_or_create_user(user["login"], user["org_id"], user["role"]) print("Grafana bootstrap complete.") diff --git a/scripts/generate_config.py b/scripts/generate_config.py index 8c575e8..1495544 100644 --- a/scripts/generate_config.py +++ b/scripts/generate_config.py @@ -12,7 +12,8 @@ Outputs (all paths relative to the repository root): docker-compose.override.yml sftp_receiver/entrypoint.sh - webserver/configs/users.json (existing password hashes preserved) + webserver/configs/users.json (existing password hashes preserved; + also receives entries from `admins:`) grafana/provisioning/datasources/postgres.yml grafana/init_grafana.py (ORGS/USERS lists replaced) database/migrations/NNN_add_.sql (only emitted for genuinely new users) @@ -42,6 +43,23 @@ def load_users(users_file: Path) -> list[dict]: return users +def load_admins(users_file: Path, users: list[dict]) -> list[dict]: + """Return the ``admins:`` entries of users.yml (web-login-only accounts).""" + with open(users_file) as f: + data = yaml.safe_load(f) or {} + admins = data.get("admins") or [] + user_ids = {u["id"] for u in users} + seen: set[str] = set() + for a in admins: + aid = a.get("id") + if not aid: + raise ValueError("Admin entry missing 'id'") + if aid in user_ids or aid in seen: + raise ValueError(f"Admin id {aid!r} duplicates another user id") + seen.add(aid) + return admins + + def _validate(users: list[dict]) -> None: seen_ids = set() seen_uids = set() @@ -244,15 +262,56 @@ def generate_entrypoint_sh(users: list[dict]) -> str: # --------------------------------------------------------------------------- -def generate_users_json(users: list[dict], existing_json: dict) -> dict: +def admin_grafana_org_id(users: list[dict], existing_json: dict | None = None) -> int: + """Grafana org shared by all admins. + + Reuses the org already recorded for an admin in users.json. A tenant must + never share it: that org's datasource reads every provider's data. + """ + tenant_orgs = {u["grafana_org_id"] for u in users} + for uid, entry in (existing_json or {}).items(): + org = entry.get("grafana_org_id") + if entry.get("is_admin") and org is not None: + if org in tenant_orgs: + clash = [u["id"] for u in users if u["grafana_org_id"] == org] + raise ValueError( + f"grafana_org_id {org} is the admin org (admin {uid!r}); " + f"choose another id for {clash}" + ) + return org + return max(tenant_orgs, default=0) + 1 + + +def generate_users_json( + users: list[dict], + existing_json: dict, + admins: list[dict] | None = None, + admin_org_id: int | None = None, +) -> dict: """ Merge users.yml into the existing users.json. - Existing password hashes and grafana_org_id are preserved. - New users get a skeleton entry (password must be set via set_password.py). - Users present in users.json but absent from users.yml are left intact (so manual removals require an explicit edit of users.json). + - Admin accounts get ``"is_admin": true`` and, when ``admin_org_id`` is + given, that shared Grafana org; they have no DB role, parser or SFTP + account. """ result = dict(existing_json) + for a in admins or []: + aid = a["id"] + entry = result.get(aid, {"password_hash": ""}) + entry["display_name"] = a.get("display_name", aid) + entry["is_admin"] = True + if admin_org_id is not None: + entry["grafana_org_id"] = admin_org_id + if aid not in result: + print( + f" [users.json] Added skeleton entry for new admin {aid!r}. " + "Run scripts/set_password.py to set their password." + ) + result[aid] = entry for u in users: uid = u["id"] if uid in result: @@ -473,22 +532,45 @@ def _orgs_list_literal(users: list[dict]) -> str: return "\n".join(lines) -def _users_list_literal(users: list[dict]) -> str: - """Render the USERS list as a Python literal.""" +def _users_list_literal( + users: list[dict], + admins: list[dict] | None = None, + admin_org_id: int | None = None, +) -> str: + """Render the USERS list as a Python literal (admins join the admin org).""" lines = ["USERS = ["] for u in users: lines.append( f' {{"login": "{u["id"]}", "org_id": {u["grafana_org_id"]}, "role": "Viewer"}},', ) + for a in admins or []: + lines.append( + f' {{"login": "{a["id"]}", "org_id": {admin_org_id}, "role": "Viewer"}},', + ) lines.append("]") return "\n".join(lines) -def update_init_grafana(users: list[dict], init_grafana_path: Path) -> None: +def _admin_org_literal(admin_org_id: int | None) -> str: + if admin_org_id is None: + return "ADMIN_ORG = None" + return f'ADMIN_ORG = {{"id": {admin_org_id}, "name": "admin"}}' + + +def update_init_grafana( + users: list[dict], + init_grafana_path: Path, + admins: list[dict] | None = None, + admin_org_id: int | None = None, +) -> None: """ - Replace the ORGS = [...] and USERS = [...] blocks in init_grafana.py with + Replace the ORGS, USERS and ADMIN_ORG definitions in init_grafana.py with versions derived from users.yml. Everything else in the file is preserved. """ + if not admins: + admin_org_id = None + elif admin_org_id is None: + admin_org_id = admin_grafana_org_id(users) src = init_grafana_path.read_text() # Replace ORGS block @@ -507,12 +589,21 @@ def update_init_grafana(users: list[dict], init_grafana_path: Path) -> None: r"^USERS\s*=\s*\[.*?\]", re.MULTILINE | re.DOTALL, ) - new_users = _users_list_literal(users) + new_users = _users_list_literal(users, admins, admin_org_id) if users_pattern.search(src): src = users_pattern.sub(new_users, src, count=1) else: raise RuntimeError("Could not locate USERS = [...] in init_grafana.py") + # Replace ADMIN_ORG line (one-line literal) + admin_pattern = re.compile(r"^ADMIN_ORG\s*=.*$", re.MULTILINE) + if admin_pattern.search(src): + src = admin_pattern.sub( + lambda _: _admin_org_literal(admin_org_id), src, count=1 + ) + elif admins: + raise RuntimeError("Could not locate ADMIN_ORG = ... in init_grafana.py") + init_grafana_path.write_text(src) @@ -638,7 +729,9 @@ def main(argv: list[str] | None = None) -> None: existing_json: dict = {} if users_json_path.exists(): existing_json = json.loads(users_json_path.read_text()) - new_json = generate_users_json(users, existing_json) + admins = load_admins(users_file, users) + admin_org_id = admin_grafana_org_id(users, existing_json) if admins else None + new_json = generate_users_json(users, existing_json, admins, admin_org_id) users_json_path.write_text(json.dumps(new_json, indent=4) + "\n") print(f" [webserver] Written {users_json_path.relative_to(repo_root)}") @@ -653,11 +746,11 @@ def main(argv: list[str] | None = None) -> None: # 6. Update init_grafana.py ORGS/USERS lists init_grafana_path = repo_root / "grafana" / "init_grafana.py" - update_init_grafana(users, init_grafana_path) + update_init_grafana(users, init_grafana_path, admins) print(f" [grafana] Updated {init_grafana_path.relative_to(repo_root)}") # 7. SSH keys - ensure_ssh_keys(users, ssh_keys_dir) + ensure_ssh_keys(users, ssh_keys_dir), admin_org_id print() print("Done. Commit the regenerated files:") diff --git a/scripts/tests/test_generate_config.py b/scripts/tests/test_generate_config.py index 83e9802..c602489 100644 --- a/scripts/tests/test_generate_config.py +++ b/scripts/tests/test_generate_config.py @@ -55,7 +55,6 @@ def test_custom_ssh_keys_dir_written_verbatim(): assert "../ssh_keys/alice/authorized_keys" in output - def test_absolute_ssh_keys_dir_written_verbatim(tmp_path): """An absolute path is also written verbatim.""" abs_path = str(tmp_path / "keys") @@ -106,8 +105,14 @@ def test_main_ssh_keys_dir_override_uses_external_dir(tmp_path, monkeypatch): """ # Build a minimal repo layout inside tmp_path repo_root = tmp_path / "repo" - for subdir in ("sftp_receiver", "webserver/configs", "database/migrations", - "grafana/provisioning/datasources", "grafana", "ssh_keys"): + for subdir in ( + "sftp_receiver", + "webserver/configs", + "database/migrations", + "grafana/provisioning/datasources", + "grafana", + "ssh_keys", + ): (repo_root / subdir).mkdir(parents=True, exist_ok=True) # Minimal users.yml (deployment-level, one real user) @@ -115,19 +120,22 @@ def test_main_ssh_keys_dir_override_uses_external_dir(tmp_path, monkeypatch): # Stub out files that main() reads/updates (repo_root / "webserver" / "configs" / "users.json").write_text("{}") - (repo_root / "grafana" / "init_grafana.py").write_text( - "ORGS = []\nUSERS = []\n" - ) + (repo_root / "grafana" / "init_grafana.py").write_text("ORGS = []\nUSERS = []\n") # External ssh_keys dir lives next to the repo (simulating deployment layout) ext_keys_dir = tmp_path / "ssh_keys" ext_keys_dir.mkdir() - main([ - "--users-file", str(tmp_path / "users.yml"), - "--repo-root", str(repo_root), - "--ssh-keys-dir", str(ext_keys_dir), - ]) + main( + [ + "--users-file", + str(tmp_path / "users.yml"), + "--repo-root", + str(repo_root), + "--ssh-keys-dir", + str(ext_keys_dir), + ] + ) # 1. Compose override uses the external path override = (repo_root / "docker-compose.override.yml").read_text() @@ -143,18 +151,130 @@ def test_main_ssh_keys_dir_override_uses_external_dir(tmp_path, monkeypatch): def test_main_default_ssh_keys_dir_stays_inside_repo(tmp_path): """Without --ssh-keys-dir, keys go into /ssh_keys as before.""" repo_root = tmp_path / "repo" - for subdir in ("sftp_receiver", "webserver/configs", "database/migrations", - "grafana/provisioning/datasources", "grafana", "ssh_keys"): + for subdir in ( + "sftp_receiver", + "webserver/configs", + "database/migrations", + "grafana/provisioning/datasources", + "grafana", + "ssh_keys", + ): (repo_root / subdir).mkdir(parents=True, exist_ok=True) _make_users_yml(repo_root) (repo_root / "webserver" / "configs" / "users.json").write_text("{}") - (repo_root / "grafana" / "init_grafana.py").write_text( - "ORGS = []\nUSERS = []\n" - ) + (repo_root / "grafana" / "init_grafana.py").write_text("ORGS = []\nUSERS = []\n") main(["--repo-root", str(repo_root)]) override = (repo_root / "docker-compose.override.yml").read_text() assert "./ssh_keys/alice/authorized_keys" in override assert (repo_root / "ssh_keys" / "alice" / "id_ed25519").exists() + + +# --------------------------------------------------------------------------- +# admins: — web-login-only accounts +# --------------------------------------------------------------------------- + + +def test_admins_written_to_users_json_only(tmp_path): + from generate_config import generate_users_json, load_admins + + users_file = tmp_path / "users.yml" + users_file.write_text( + yaml.dump({"users": USERS, "admins": [{"id": "boss", "display_name": "Boss"}]}) + ) + admins = load_admins(users_file, USERS) + existing = {"boss": {"password_hash": "keepme"}} + + result = generate_users_json(USERS, existing, admins) + + assert result["boss"] == { + "password_hash": "keepme", + "display_name": "Boss", + "is_admin": True, + } + assert "is_admin" not in result["alice"] + + +@pytest.mark.parametrize( + "admins", + [[{"id": "alice"}], [{"id": "x"}, {"id": "x"}], [{"display_name": "no id"}]], +) +def test_load_admins_rejects_invalid_entries(tmp_path, admins): + from generate_config import load_admins + + users_file = tmp_path / "users.yml" + users_file.write_text(yaml.dump({"users": USERS, "admins": admins})) + with pytest.raises(ValueError): + load_admins(users_file, USERS) + + +def test_load_admins_absent_is_empty(tmp_path): + from generate_config import load_admins + + users_file = _make_users_yml(tmp_path) + assert load_admins(users_file, USERS) == [] + + +# --------------------------------------------------------------------------- +# admins: — shared Grafana org +# --------------------------------------------------------------------------- + + +def test_admin_grafana_org_follows_tenant_orgs(): + from generate_config import admin_grafana_org_id + + users = [ + {**USERS[0], "grafana_org_id": 1}, + {**USERS[0], "id": "b", "grafana_org_id": 4}, + ] + assert admin_grafana_org_id(users) == 5 + + +def test_admin_grafana_org_is_stable_once_assigned(): + from generate_config import admin_grafana_org_id + + users = [{**USERS[0], "grafana_org_id": 1}, {**USERS[0], "id": "b", "grafana_org_id": 4}] + existing = {"boss": {"is_admin": True, "grafana_org_id": 3}} + assert admin_grafana_org_id(users, existing) == 3 + + +def test_tenant_cannot_take_the_admin_grafana_org(): + from generate_config import admin_grafana_org_id + + users = [{**USERS[0], "grafana_org_id": 1}, {**USERS[0], "id": "b", "grafana_org_id": 3}] + existing = {"boss": {"is_admin": True, "grafana_org_id": 3}} + with pytest.raises(ValueError, match="admin org"): + admin_grafana_org_id(users, existing) + + +def test_admin_gets_shared_grafana_org_in_users_json(): + from generate_config import generate_users_json + + result = generate_users_json(USERS, {}, [{"id": "boss"}], admin_org_id=2) + assert result["boss"]["grafana_org_id"] == 2 + assert result["alice"]["grafana_org_id"] == 1 + + +def test_update_init_grafana_adds_admin_org_and_users(tmp_path): + from generate_config import update_init_grafana + + path = tmp_path / "init_grafana.py" + path.write_text("ORGS = []\nUSERS = []\nADMIN_ORG = None\nOTHER = 1\n") + update_init_grafana(USERS, path, [{"id": "boss"}]) + + ns: dict = {} + exec(path.read_text(), ns) + assert ns["ADMIN_ORG"] == {"id": 2, "name": "admin"} + assert {"login": "boss", "org_id": 2, "role": "Viewer"} in ns["USERS"] + assert ns["OTHER"] == 1 + + +def test_update_init_grafana_without_admins_clears_admin_org(tmp_path): + from generate_config import update_init_grafana + + path = tmp_path / "init_grafana.py" + path.write_text('ORGS = []\nUSERS = []\nADMIN_ORG = {"id": 9, "name": "admin"}\n') + update_init_grafana(USERS, path) + assert "ADMIN_ORG = None" in path.read_text() diff --git a/users.yml b/users.yml index 744dd1a..09517b9 100644 --- a/users.yml +++ b/users.yml @@ -59,6 +59,19 @@ # All sources of one user share the same user_id, so Grafana sees their # data unified. Each source gets its own named Docker volume and parser # service (parser__). +# +# Optional top-level `admins:` list — web-login accounts that can see all +# providers (map + archive overview at /admin/realtime and /admin/archive). +# Admins have no DB role, SFTP account or parser. They share one extra Grafana +# org ("admin") whose datasource connects as webserver_role, so the real-time +# page can plot every provider. Set a password with +# scripts/set_password.py . +# id — login name (must not equal any user id). +# display_name — label shown in the webserver UI. +# +#admins: +# - id: admin +# display_name: Administrator users: - id: demo_openmrg diff --git a/webserver/main.py b/webserver/main.py index 1dee237..f03addb 100644 --- a/webserver/main.py +++ b/webserver/main.py @@ -9,6 +9,7 @@ from markupsafe import escape from flask import ( Flask, + abort, render_template, request, jsonify, @@ -34,6 +35,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from contextlib import contextmanager +from functools import wraps import uuid app = Flask(__name__) @@ -74,7 +76,10 @@ class User(UserMixin): def __init__(self, user_id: str): self.id = user_id self.display_name = USERS[user_id].get("display_name", user_id) - self.grafana_org_id = USERS[user_id].get("grafana_org_id", 1) + self.is_admin = bool(USERS[user_id].get("is_admin", False)) + # Admins only have a Grafana org once generate_config.py assigned one. + default_org = None if self.is_admin else 1 + self.grafana_org_id = USERS[user_id].get("grafana_org_id", default_org) @login_manager.user_loader @@ -141,6 +146,8 @@ def user_db_scope(user_id: str): """ if user_id not in USERS: raise ValueError(f"Unknown user_id: {user_id!r}") + if USERS[user_id].get("is_admin"): + raise ValueError(f"Admin account {user_id!r} has no tenant data role") conn = psycopg2.connect(os.getenv("DATABASE_URL")) try: @@ -198,6 +205,8 @@ def logout(): @login_required def overview(): """Landing page with overview and processing status""" + if getattr(current_user, "is_admin", False) is True: + return redirect(url_for("admin_realtime")) stats = { "total_cmls": 0, "total_records": 0, @@ -796,6 +805,214 @@ def archive(): ) +# ==================== ADMIN (ALL PROVIDERS) ROUTES ==================== +# Admin accounts (is_admin in users.json) have no tenant DB role. These views +# read across all tenants as webserver_role, which bypasses per-user isolation, +# so every route here must be decorated with @admin_required. + + +def admin_required(view): + """Restrict a view to accounts flagged is_admin in users.json.""" + + @wraps(view) + @login_required + def wrapper(*args, **kwargs): + if getattr(current_user, "is_admin", False) is not True: + abort(403) + return view(*args, **kwargs) + + return wrapper + + +@contextmanager +def admin_db_scope(): + """Read-only cross-tenant connection as webserver_role.""" + conn = psycopg2.connect(os.getenv("DATABASE_URL")) + try: + conn.set_session(readonly=True) + yield conn + finally: + conn.close() + + +def _provider_list(): + return [ + {"id": uid, "display_name": cfg.get("display_name", uid)} + for uid, cfg in USERS.items() + if not cfg.get("is_admin") + ] + + +def get_admin_archive_summary(): + """Per-provider CML count, record count and archived date range.""" + with admin_db_scope() as conn: + cur = conn.cursor() + cur.execute( + "SELECT user_id, COUNT(DISTINCT cml_id) FROM cml_metadata GROUP BY user_id" + ) + cml_counts = dict(cur.fetchall()) + cur.execute( + "SELECT user_id, COALESCE(SUM(total_records), 0) FROM cml_stats GROUP BY user_id" + ) + records = dict(cur.fetchall()) + cur.execute( + "SELECT user_id, MIN(bucket), MAX(bucket) FROM cml_data_1h GROUP BY user_id" + ) + ranges = {row[0]: (row[1], row[2]) for row in cur.fetchall()} + cur.close() + + summary = {} + for uid, (start, end) in ranges.items(): + summary[uid] = {"start": start, "end": end} + for uid in set(cml_counts) | set(records) | set(ranges): + entry = summary.setdefault(uid, {"start": None, "end": None}) + entry["cml_count"] = cml_counts.get(uid, 0) + entry["total_records"] = int(records.get(uid, 0)) + return summary + + +@app.route("/admin/realtime") +@admin_required +def admin_realtime(): + """Map of all CMLs from all providers plus the all-providers Grafana panel.""" + resp = make_response( + render_template( + "admin_realtime.html", + providers=_provider_list(), + grafana_org_id=current_user.grafana_org_id, + ) + ) + resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate" + return resp + + +@app.route("/admin/archive") +@admin_required +def admin_archive(): + """Per-provider archive overview with time series.""" + try: + summary = get_admin_archive_summary() + except Exception as e: + print(f"Error fetching admin archive summary: {e}") + flash("Could not load archive statistics.") + summary = {} + empty = {"start": None, "end": None, "cml_count": 0, "total_records": 0} + providers = [{**p, **summary.get(p["id"], empty)} for p in _provider_list()] + return render_template( + "admin_archive.html", providers=providers, day_options=_ADMIN_TS_DAYS + ) + + +@app.route("/api/admin/cml-map") +@admin_required +def api_admin_cml_map(): + """CML endpoints of all providers, tagged with user_id.""" + try: + with admin_db_scope() as conn: + cur = conn.cursor() + cur.execute( + """SELECT DISTINCT ON (user_id, cml_id) + user_id, cml_id::text, site_0_lon, site_0_lat, site_1_lon, site_1_lat + FROM cml_metadata + WHERE site_0_lon IS NOT NULL AND site_0_lat IS NOT NULL + AND site_1_lon IS NOT NULL AND site_1_lat IS NOT NULL + ORDER BY user_id, cml_id""" + ) + rows = cur.fetchall() + cur.close() + except Exception as e: + print(f"Error fetching admin CML map data: {e}") + return jsonify({"error": "Database error"}), 500 + + return jsonify( + [ + { + "user_id": row[0], + "cml_id": str(row[1]), + "site_0": {"lon": float(row[2]), "lat": float(row[3])}, + "site_1": {"lon": float(row[4]), "lat": float(row[5])}, + } + for row in rows + ] + ) + + +@app.route("/api/admin/cml-stats") +@admin_required +def api_admin_cml_stats(): + """Live per-CML stats of all providers (same fields as /api/cml-stats), tagged with user_id.""" + try: + with admin_db_scope() as conn: + cur = conn.cursor() + cur.execute( + """SELECT user_id, cml_id::text, completeness_percent_6h, + total_records_6h, valid_records_6h, stddev_rsl_1h, last_rsl + FROM cml_stats + ORDER BY user_id, cml_id""" + ) + rows = cur.fetchall() + cur.close() + except Exception as e: + print(f"Error fetching admin CML stats: {e}") + return jsonify({"error": "Database error"}), 500 + + return jsonify( + [ + { + "user_id": row[0], + "cml_id": str(row[1]), + "completeness_percent": safe_float(row[2]), + "total_records": int(row[3] or 0), + "valid_records": int(row[4] or 0), + "stddev_last_60min": safe_float(row[5]), + "last_rsl": safe_float(row[6]), + } + for row in rows + ] + ) + + +_ADMIN_TS_DAYS = (7, 30, 90, 365) + + +@app.route("/api/admin/archive-timeseries") +@admin_required +def api_admin_archive_timeseries(): + """Daily record count and mean RSL per provider from the 1 h aggregate.""" + try: + days = int(request.args.get("days", 30)) + except ValueError: + days = 30 + if days not in _ADMIN_TS_DAYS: + days = 30 + + try: + with admin_db_scope() as conn: + cur = conn.cursor() + cur.execute( + """SELECT user_id, time_bucket('1 day', bucket) AS day, + SUM(record_count), AVG(rsl_avg) + FROM cml_data_1h + WHERE bucket >= NOW() - make_interval(days => %s) + GROUP BY user_id, day + ORDER BY user_id, day""", + (days,), + ) + rows = cur.fetchall() + cur.close() + except Exception as e: + print(f"Error fetching admin archive time series: {e}") + return jsonify({"error": "Database error"}), 500 + + series: dict = {} + for user_id, day, records, mean_rsl in rows: + s = series.setdefault(user_id, {"days": [], "records": [], "mean_rsl": []}) + s["days"].append(day.strftime("%Y-%m-%d")) + s["records"].append(int(records or 0)) + s["mean_rsl"].append(safe_float(mean_rsl)) + return jsonify(series) + + # ==================== DATA UPLOADS ROUTES ==================== diff --git a/webserver/templates/admin_archive.html b/webserver/templates/admin_archive.html new file mode 100644 index 0000000..6903858 --- /dev/null +++ b/webserver/templates/admin_archive.html @@ -0,0 +1,87 @@ +{% extends "base.html" %} + +{% block title %}All Providers Archive - GMDI Platform{% endblock %} + +{% block content %} +
+

Archive by Provider

+
+ + +
+
+ +{% with messages = get_flashed_messages() %} +{% for m in messages %}
{{ m }}
{% endfor %} +{% endwith %} + +{% for p in providers %} +
+
+
+ {{ p.display_name }} + {{ p.id }} + {{ p.cml_count }} + CMLs + ~{{ "{:,}".format(p.total_records) + }} records + {% if p.start and p.end %} + + {{ p.start.strftime('%Y-%m-%d') }} → {{ p.end.strftime('%Y-%m-%d') }} + + {% else %} + no data + {% endif %} +
+
+
+
+{% endfor %} +{% endblock %} + +{% block extra_js %} + + +{% endblock %} diff --git a/webserver/templates/admin_realtime.html b/webserver/templates/admin_realtime.html new file mode 100644 index 0000000..8b71dc8 --- /dev/null +++ b/webserver/templates/admin_realtime.html @@ -0,0 +1,373 @@ +{% extends "base.html" %} + +{% block title %}All Providers - Real-Time Data - GMDI Platform{% endblock %} + +{% block extra_css %} + + +{% endblock %} + +{% block content %} +
+
+
+
+ +
+
Loading…
+
+
+ {% if grafana_org_id %} +
+
+ +
+ {% endif %} +
+{% endblock %} + +{% block extra_js %} + + +{% endblock %} \ No newline at end of file diff --git a/webserver/templates/base.html b/webserver/templates/base.html index f12e26c..75b8ebf 100644 --- a/webserver/templates/base.html +++ b/webserver/templates/base.html @@ -23,7 +23,22 @@