diff --git a/Dockerfile_uwsgi b/Dockerfile_uwsgi index 176725d..14f39af 100644 --- a/Dockerfile_uwsgi +++ b/Dockerfile_uwsgi @@ -1,4 +1,4 @@ -FROM python:3.11 +FROM python:3.11.13-bookworm # Metadata LABEL Name="SynAnno" Version="1.0.0" @@ -6,8 +6,8 @@ LABEL Name="SynAnno" Version="1.0.0" # Set working directory WORKDIR /app -# Upgrade pip -RUN python -m pip install --no-cache-dir --upgrade pip +# Pin pip to the version in the previous deployed image +RUN python -m pip install --no-cache-dir pip==25.2 # Install system dependencies for PyQt5 RUN apt-get update && apt-get install -y \ @@ -26,16 +26,20 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN cargo --version # Copy application code -COPY setup.py /app/ +COPY setup.py requirements.lock /app/ COPY synanno /app/synanno COPY run_production.py /app/ COPY h01/h01_104_materialization.csv /app/h01/h01_104_materialization.csv -# Install Python dependencies -RUN python -m pip install uv -RUN uv pip install --system --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu -RUN uv pip install --system --no-cache-dir torchsummary tqdm -e . -RUN python -m pip install --no-cache-dir uwsgi +# Install the locked Python dependencies, then the app. torch/torchvision come only +# from the CPU-only PyTorch index; everything else comes only from PyPI. +RUN grep -E '^(torch|torchvision)==' requirements.lock > /tmp/torch.lock \ + && grep -vE '^(torch|torchvision)==' requirements.lock > /tmp/pypi.lock \ + && python -m pip install --no-cache-dir --no-deps -r /tmp/torch.lock \ + --index-url https://download.pytorch.org/whl/cpu \ + && python -m pip install --no-cache-dir --no-deps -r /tmp/pypi.lock \ + && rm /tmp/torch.lock /tmp/pypi.lock +RUN python -m pip install --no-cache-dir --no-deps -e . # Copy uWSGI configuration COPY uwsgi.ini /app diff --git a/README.md b/README.md index 7211fd3..7cc8219 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ SynAnno is a tool designed for interactive guided proofreading and correction of ## Live Demo -A simplified version of the app, starting directly in the Error Categorization view, is available [here](http://54.210.88.222/demo). +A simplified version of the app, starting directly in the Error Categorization view, is available [here](http://54.164.194.68/demo). -For the full functionality and configuration options of SynAnno, try the complete version [here](http://54.210.88.222/reset). +For the full functionality and configuration options of SynAnno, try the complete version [here](http://54.164.194.68/reset). --- diff --git a/docs/deploy.md b/docs/deploy.md new file mode 100644 index 0000000..1927a8c --- /dev/null +++ b/docs/deploy.md @@ -0,0 +1,90 @@ +# Deploying the public demo + +The public demo (linked from the README) runs as a single ECS task on one Graviton +(arm64) EC2 instance in AWS `us-east-1`. The concrete account and resource IDs are kept +out of this public repository; ask a maintainer for them. Placeholders below look like +``. + +## How it is deployed + +- **Image:** `.dkr.ecr.us-east-1.amazonaws.com/synanno/uwsgi:`, + built from `Dockerfile_uwsgi` for `linux/arm64`. Dependencies are pinned in + `requirements.lock`; change a pin there, not in `setup.py`. +- **Task definition:** one container, network mode `host`, port 80 (Flask/uWSGI) and + port 9015 (Neuroglancer), with the environment: + - `PUBLIC_DNS_SYNANNO=`: the embedded Neuroglancer view is loaded from + `http://:9015/v//`, so this must be the public address. + - `SYNANNO_PUBLIC_DEMO=1`: limits the app to the bundled H01 data and refuses + credential uploads. + - `SECRET_KEY` is not set; the app generates one at startup. +- **Address:** an Elastic IP associated with the instance. +- **Firewall:** the instance's security group allows only ports 80 and 9015. +- **Logs:** CloudWatch, in the service's log group. + +The demo keeps one shared state for all visitors, and `/reset` and `/demo` wipe it. +That is intended: the demo is a single-user test deployment. + +## Release a new version + +Pass `--region us-east-1` on every AWS command. + +```bash +REPO=.dkr.ecr.us-east-1.amazonaws.com/synanno/uwsgi +SHA=$(git rev-parse --short HEAD) +aws ecr get-login-password --region us-east-1 \ + | docker login --username AWS --password-stdin "${REPO%%/*}" +docker build --platform linux/arm64 -f Dockerfile_uwsgi -t "$REPO:$SHA" . +docker push "$REPO:$SHA" +``` + +Register a new task definition revision that differs from the current one only in the +image tag, then switch the service. Host networking with fixed ports cannot run two +copies on one instance, so the service stops the old task before starting the new one +(minimum healthy 0%, maximum 100%; Availability Zone rebalancing is disabled because +it requires a maximum above 100%). Expect a short downtime. + +```bash +aws ecs update-service --region us-east-1 --cluster --service \ + --task-definition : --force-new-deployment +aws ecs wait services-stable --region us-east-1 --cluster --services +scripts/smoke_test.sh http:// +``` + +Once the smoke test passes, point the `latest` tag at the new image as well. + +## Roll back + +Switch the service back to an earlier task definition revision with the same +`update-service` command, then run the smoke test. Rollback revisions should reference +their image **by digest** (`…/synanno/uwsgi@sha256:…`), not by a tag such as `latest`, +which moves with every release. + +## If the Auto Scaling group replaces the instance + +A replacement instance gets a new auto-assigned public IP, and the Elastic IP is not +moved to it. Re-associate it, then run the smoke test: + +```bash +NEW_ID=$(aws ec2 describe-instances --region us-east-1 \ + --filters Name=tag:aws:autoscaling:groupName,Values= \ + Name=instance-state-name,Values=running \ + --query 'Reservations[0].Instances[0].InstanceId' --output text) +aws ec2 associate-address --region us-east-1 \ + --allocation-id --instance-id "$NEW_ID" +``` + +The task definition does not need to change: it names the Elastic IP, not the instance. + +## Shell access (SSH) + +Port 22 is closed, and nothing in the deployment needs it. If you need a shell on the +instance, open port 22 for your own IP only, and close it again afterwards: + +```bash +MY_IP=$(curl -s https://checkip.amazonaws.com) +aws ec2 authorize-security-group-ingress --region us-east-1 \ + --group-id --protocol tcp --port 22 --cidr "$MY_IP/32" +ssh -i ec2-user@ +aws ec2 revoke-security-group-ingress --region us-east-1 \ + --group-id --protocol tcp --port 22 --cidr "$MY_IP/32" +``` diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..42767a4 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,144 @@ +# Exact package set of the deployed image (synanno/uwsgi:rollback-2026-10, +# sha256:b8793d31...), captured with pip freeze on Python 3.11.13 (arm64). +# The app itself is installed separately (pip install --no-deps -e .). +# torch and torchvision come from https://download.pytorch.org/whl/cpu (see Dockerfile_uwsgi). +# flask-cors was dropped together with CORS support; tornado was raised from +# 6.5.1 to 6.5.10 for HTTP-parsing DoS fixes (it serves the public NG port). +atomicwrites==1.4.1 +attrs==25.3.0 +blinker==1.9.0 +boto3==1.40.2 +botocore==1.40.2 +Brotli==1.1.0 +cachelib==0.13.0 +cachetools==5.5.2 +certifi==2025.8.3 +chardet==5.2.0 +charset-normalizer==3.4.2 +click==8.2.1 +cloud-files==5.6.0 +cloud_volume==12.4.1 +compressed_segmentation==2.3.2 +contourpy==1.3.3 +crc32c==2.7.1 +cycler==0.12.1 +deflate==0.8.1 +dill==0.4.0 +DracoPy==1.5.0 +fasteners==0.19 +fastremap==1.17.2 +filelock==3.13.1 +Flask==3.1.1 +Flask-Session==0.8.0 +flexcache==0.3 +flexparser==0.4 +fonttools==4.59.0 +fpzip==1.2.4 +fsspec==2024.6.1 +fuzzywuzzy==0.18.0 +gevent==25.5.1 +google-api-core==2.25.1 +google-apitools==0.5.32 +google-auth==2.40.3 +google-cloud-core==2.4.3 +google-cloud-storage==3.2.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +greenlet==3.2.3 +h5py==3.14.0 +httplib2==0.22.0 +idna==3.10 +igraph==0.11.9 +imageio==2.37.0 +inflection==0.5.1 +iniconfig==2.1.0 +intervaltree==3.1.0 +itsdangerous==2.2.0 +Jinja2==3.1.4 +jmespath==1.0.1 +joblib==1.5.1 +json5==0.12.0 +jsonschema==4.25.0 +jsonschema-specifications==2025.4.1 +kiwisolver==1.4.8 +lazy_loader==0.4 +Markdown==3.8.2 +MarkupSafe==2.1.5 +matplotlib==3.10.5 +microviewer==1.17.1 +molesq==0.4.0 +morphops==0.1.13 +mpmath==1.3.0 +msgspec==0.19.0 +multiprocess==0.70.18 +navis==1.10.0 +ncollpyde==0.19.0 +networkx==3.1 +neuroglancer==2.40.1 +numpy==1.26.4 +oauth2client==4.1.3 +opencv-python==4.11.0.86 +orjson==3.11.1 +osteoid==0.3.2 +packaging==25.0 +pandas==2.3.1 +pathos==0.3.4 +pillow==11.0.0 +Pint==0.24.4 +platformdirs==4.3.8 +pluggy==1.6.0 +posix_ipc==1.3.0 +pox==0.3.6 +ppft==1.7.7 +proto-plus==1.26.1 +protobuf==6.31.1 +psutil==7.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +Pygments==2.19.2 +pykdtree==1.4.2 +pynrrd==1.1.3 +pyparsing==3.2.3 +pypng==0.20220715.0 +pysimdjson==7.0.2 +pytest==8.4.1 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +python-jsonschema-objects==0.5.7 +pytz==2025.2 +rdata==0.11.2 +referencing==0.36.2 +requests==2.32.4 +rpds-py==0.26.0 +rsa==4.9.1 +s3transfer==0.13.1 +scikit-image==0.25.2 +scikit-learn==1.7.1 +scipy==1.16.1 +seaborn==0.13.2 +simplejpeg==1.8.2 +six==1.17.0 +skeletor==1.3.0 +sortedcontainers==2.4.0 +sympy==1.13.3 +tenacity==9.1.2 +texttable==1.7.0 +threadpoolctl==3.6.0 +tifffile==2025.6.11 +torch==2.7.1+cpu +torchsummary==1.5.1 +torchvision==0.22.1 +tornado==6.5.10 +tqdm==4.67.1 +trimesh==4.7.1 +typing_extensions==4.12.2 +tzdata==2025.2 +urllib3==2.5.0 +uv==0.8.4 +uWSGI==2.0.30 +Werkzeug==3.1.3 +xarray==2025.7.1 +zope.event==5.1 +zope.interface==7.2 +zstandard==0.23.0 diff --git a/run.py b/run.py index 48c11be..c6e7106 100644 --- a/run.py +++ b/run.py @@ -1,4 +1,5 @@ import logging +import os from dotenv import load_dotenv @@ -14,4 +15,9 @@ if __name__ == "__main__": - app.run(host=app.config["IP"], port=app.config["PORT"], debug=True) + # The dev server binds to localhost and runs without the debugger unless asked + app.run( + host=os.getenv("APP_IP", "127.0.0.1"), + port=app.config["PORT"], + debug=os.getenv("FLASK_DEBUG", "0") == "1", + ) diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh new file mode 100755 index 0000000..efa92a3 --- /dev/null +++ b/scripts/smoke_test.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Smoke test for a deployed SynAnno demo (SYNANNO_PUBLIC_DEMO=1). +# +# Usage: scripts/smoke_test.sh http:// +# +# Calls /reset and /demo, which wipe the shared demo state (by design: the demo +# is single-user). Makes no other state changes. + +set -u + +BASE="${1:?usage: $0 http://}" +BASE="${BASE%/}" +H01_SOURCE="gs://h01-release/data/20210601/4nm_raw" +H01_TARGET="gs://h01-release/data/20210729/c3/synapses/whole_ei_onlyvol" +H01_NEUROPIL="gs://h01-release/data/20210601/proofread_104" + +failures=0 +pass() { echo "PASS $1"; } +fail() { echo "FAIL $1"; failures=$((failures + 1)); } + +status() { curl -s -o /dev/null -m "${2:-30}" -w "%{http_code}" "${@:3}" "$1"; } + +# 1. Pages load +for path in / /reset /demo; do + code=$(status "$BASE$path") + [[ $code == 200 ]] && pass "GET $path -> $code" || fail "GET $path -> $code" +done + +# 2. The demo loads, and its Neuroglancer view is served from :9015 with an +# unguessable token (the same /neuro call the annotation page makes) +code=$(status "$BASE/demo_annotation" 600) +[[ $code == 200 ]] && pass "GET /demo_annotation -> $code" || fail "GET /demo_annotation -> $code" +ng_json=$(curl -s -m 120 -X POST "$BASE/neuro" -d mode=draw -d cz0=0 -d cy0=0 -d cx0=0) +ng_url=$(python3 -c 'import json,sys; print(json.loads(sys.argv[1])["ng_link"])' "$ng_json" 2>/dev/null) +host="${BASE#*://}" +if [[ $ng_url =~ ^http://${host}:9015/v/([A-Za-z0-9_-]{40,})/$ ]]; then + pass "Neuroglancer URL $ng_url" + code=$(status "$ng_url") + [[ $code == 200 ]] && pass "GET Neuroglancer view -> $code" \ + || fail "GET Neuroglancer view -> $code" +else + fail "Neuroglancer URL not on $host:9015 with a long token: '${ng_url:-$ng_json}'" +fi + +# 3. /load_materialization rejects local files and external URLs +for url in /etc/passwd file:///etc/passwd http://example.com/table.csv; do + code=$(status "$BASE/load_materialization" 30 -X POST \ + -H "Content-Type: application/json" \ + -d "{\"materialization_url\": \"$url\"}") + [[ $code =~ ^4 ]] && pass "load_materialization $url -> $code" \ + || fail "load_materialization $url -> $code" +done + +# 4. No wildcard CORS header +if curl -s -o /dev/null -D - -m 30 -H "Origin: http://evil.example" "$BASE/" \ + | grep -qi "^access-control-allow-origin"; then + fail "Access-Control-Allow-Origin header sent" +else + pass "No Access-Control-Allow-Origin header" +fi + +# 5. Credential upload is disabled (rejected before any state change) +creds=$(mktemp) +echo '{"token": "smoke-test-not-a-secret"}' > "$creds" +code=$(status "$BASE/upload" 30 -X POST \ + -F "source_url=$H01_SOURCE" -F "target_url=$H01_TARGET" \ + -F "neuropil_url=$H01_NEUROPIL" -F "view_style=neuron" \ + -F "tiles_per_page=12" -F "secrets_file=@$creds;type=application/json") +rm -f "$creds" +[[ $code == 403 ]] && pass "Credential upload -> $code" || fail "Credential upload -> $code" + +echo +if ((failures)); then + echo "$failures check(s) failed" + exit 1 +fi +echo "All checks passed" diff --git a/setup.py b/setup.py index 18b4689..ec79873 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,6 @@ install_requires=[ "numpy<2", "flask>=2.3.2", - "flask-cors>=4.0.0", "flask-session>=0.5.0", "cloud-volume", "navis==1.10.0", diff --git a/synanno/__init__.py b/synanno/__init__.py index d08b80f..f8790b0 100644 --- a/synanno/__init__.py +++ b/synanno/__init__.py @@ -1,17 +1,20 @@ import logging import os +import secrets import threading from collections import defaultdict from threading import Lock import pandas as pd from flask import Flask -from flask_cors import CORS from flask_session import Session logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +# Repository root (in the Docker image: /app) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + def create_app(): """Factory function to create and configure the Flask app.""" @@ -25,9 +28,6 @@ def create_app(): with app.app_context(): register_routes(app) - # Set up context processor - setup_context_processors(app) - # attach a lock for the data frame access to the app instance app.df_metadata_lock = Lock() @@ -36,12 +36,17 @@ def create_app(): def configure_app(app): """Configure the Flask app with required settings.""" - # Enable CORS - CORS(app) - app.config["DEBUG_APP"] = bool(os.getenv("DEBUG_APP", "True") == "True") - app.config["CORS_HEADERS"] = "Content-Type" + app.config["DEBUG_APP"] = bool(os.getenv("DEBUG_APP", "False") == "True") + + # Public demo mode: restricts data sources to the bundled H01 release + app.config["PUBLIC_DEMO"] = os.getenv("SYNANNO_PUBLIC_DEMO", "0") == "1" # Secret key and session settings + secret_key = os.getenv("SECRET_KEY") + if not secret_key: + logger.warning("SECRET_KEY is not set; generated a random key for this run.") + secret_key = secrets.token_hex(32) + app.config["SECRET_KEY"] = secret_key app.config["SESSION_PERMANENT"] = bool(os.getenv("DEBUG_APP", "False") == "True") app.config["SESSION_TYPE"] = "filesystem" app.config["SESSION_FILE_DIR"] = "/tmp/flask_session" @@ -56,6 +61,17 @@ def configure_app(app): PORT=int(os.getenv("APP_PORT", 80)), NG_IP=os.getenv("PUBLIC_DNS_SYNANNO", "0.0.0.0"), NG_PORT=os.getenv("NG_PORT", "9015"), + # Directories /load_materialization may read from when not in demo mode + DATA_DIRS=[ + os.path.realpath(d) + for d in os.getenv( + "SYNANNO_DATA_DIRS", os.path.join(BASE_DIR, "h01") + ).split(os.pathsep) + if d + ], + BUNDLED_MATERIALIZATION=os.path.realpath( + os.path.join(BASE_DIR, "h01", "h01_104_materialization.csv") + ), ) # Initialize global variables @@ -201,11 +217,3 @@ def register_routes(app): app.register_blueprint(auto_annotate_blueprint) app.register_blueprint(fn_blueprint) app.register_blueprint(demo_blueprint) - - -def setup_context_processors(app): - """Set up context processors for the app.""" - - @app.context_processor - def handle_context(): - return dict(os=os) # noqa: C408 diff --git a/synanno/backend/ng_util.py b/synanno/backend/ng_util.py index 00f6c4a..dc94b8e 100644 --- a/synanno/backend/ng_util.py +++ b/synanno/backend/ng_util.py @@ -1,5 +1,5 @@ import logging -from random import randint +import secrets from typing import Union import neuroglancer @@ -144,8 +144,8 @@ def setup_ng( neuropil: Neuropil segmentation volume when undergoing view-centric analysis """ - # generate a version number - app.ng_version = str(randint(0, 3200)) + # unguessable viewer token; it is the only access control on the NG port + app.ng_version = secrets.token_urlsafe(32) # setup a Tornado web server and create viewer instance neuroglancer.set_server_bind_address( diff --git a/synanno/backend/processing.py b/synanno/backend/processing.py index 64f167f..f2cae76 100644 --- a/synanno/backend/processing.py +++ b/synanno/backend/processing.py @@ -119,7 +119,8 @@ def calculate_crop_pad( c11, c21, c31 = max(c11o, c11m), max(c21o, c21m), max(c31o, c31m) c12, c22, c32 = min(c12o, c12m), min(c22o, c22m), min(c32o, c32m) - assert c11 < c12 and c21 < c22 and c31 < c32, "Invalid bounding box." + if not (c11 < c12 and c21 < c22 and c31 < c32): + raise ValueError("Invalid bounding box.") pad = [ [c11 - c11o, c12o - c12], diff --git a/synanno/routes/annotation.py b/synanno/routes/annotation.py index c44af60..7314d53 100644 --- a/synanno/routes/annotation.py +++ b/synanno/routes/annotation.py @@ -12,9 +12,6 @@ # flask util functions from flask import Blueprint, current_app, jsonify, render_template, request -# flask ajax requests -from flask_cors import cross_origin - # for type hinting from jinja2 import Template @@ -72,13 +69,11 @@ def annotation(page: int = 1) -> Template: @blueprint.route("/loading_bar_image_tiles", methods=["GET"]) -@cross_origin() def loading_bar_image_tiles() -> Template: return render_template("loading_bar_image_tiles.html") @blueprint.route("/is_metadata_locked", methods=["GET"]) -@cross_origin() def is_metadata_locked(): """Check if retrieve_instance_metadata is currently running.""" is_locked = current_app.retrieve_instance_metadata_lock.locked() @@ -87,7 +82,6 @@ def is_metadata_locked(): @blueprint.route("/update_image_tiles/", endpoint="update_image_tiles_page") @blueprint.route("/update_image_tiles", methods=["POST"]) -@cross_origin() def update_images(page: int = 1): """Fetch updated image data and load the data for the current page.""" @@ -129,7 +123,6 @@ def update_images(page: int = 1): @blueprint.route("/set_grid_opacity", methods=["POST"]) -@cross_origin() def set_grid_opacity() -> tuple[str, int, Dict[str, str]]: """Serves and Ajax request from annotation.js updating the grid's opacity value @@ -147,7 +140,6 @@ def set_grid_opacity() -> tuple[str, int, Dict[str, str]]: @blueprint.route("/update-card", methods=["POST"]) -@cross_origin() def update_card() -> Dict[str, object]: """Updates the label of an instance - switch between correct, incorrect to unsure diff --git a/synanno/routes/categorize.py b/synanno/routes/categorize.py index 7be1800..c27a332 100644 --- a/synanno/routes/categorize.py +++ b/synanno/routes/categorize.py @@ -2,7 +2,6 @@ import json from flask import Blueprint, current_app, render_template, request -from flask_cors import cross_origin # Define a Blueprint for categorize routes blueprint = Blueprint("categorize", __name__) @@ -53,7 +52,6 @@ def start_categorization_timer(): @blueprint.route("/pass_flags", methods=["GET", "POST"]) -@cross_origin() def pass_flags(): """Serves an Ajax request from categorize.js, retrieving the new error tags from the frontend and updating metadata data frame. diff --git a/synanno/routes/false_negatives.py b/synanno/routes/false_negatives.py index 4bf06a5..f22aca3 100644 --- a/synanno/routes/false_negatives.py +++ b/synanno/routes/false_negatives.py @@ -3,8 +3,7 @@ import numpy as np import pandas as pd from cloudvolume import Bbox -from flask import Blueprint, current_app, jsonify, request -from flask_cors import cross_origin +from flask import Blueprint, abort, current_app, jsonify, request from PIL import Image from synanno.backend.processing import calculate_crop_pad @@ -14,7 +13,6 @@ @blueprint.route("/ng_bbox_fn", methods=["POST"]) -@cross_origin() def ng_bbox_fn() -> dict: """Serves an Ajax request by draw_module.js, passing the coordinates of the center point of a newly marked FN to the front end, enabling the front end to depict the @@ -46,7 +44,6 @@ def ng_bbox_fn() -> dict: @blueprint.route("/ng_bbox_fn_save", methods=["POST"]) -@cross_origin() def ng_bbox_fn_save() -> dict: """Serves an Ajax request by draw_module.js, that passes the manual updated/ corrected bb coordinates to this backend function. Additionally, the @@ -57,16 +54,22 @@ def ng_bbox_fn_save() -> dict: the upper and the lower z bound of the instance as JSON to draw_module.js """ coordinate_order = list(current_app.coordinate_order.keys()) - cz1, cz2, current_app.cz, current_app.cy, current_app.cx = ( - get_corrected_coordinates(request) - ) + previous_center = (current_app.cz, current_app.cy, current_app.cx) + try: + cz1, cz2, current_app.cz, current_app.cy, current_app.cx = ( + get_corrected_coordinates(request) + ) - item = create_new_item(request) + item = create_new_item(request) - bbox = define_bbox(cz1, cz2, coordinate_order) - item["Original_Bbox"] = scale_bbox(bbox, coordinate_order) + bbox = define_bbox(cz1, cz2, coordinate_order) + item["Original_Bbox"] = scale_bbox(bbox, coordinate_order) - crop_bbox, img_padding = calculate_crop_pad(bbox, current_app.vol_dim_scaled) + crop_bbox, img_padding = calculate_crop_pad(bbox, current_app.vol_dim_scaled) + except (KeyError, ValueError): + # leave the shared state as it was before the invalid request + current_app.cz, current_app.cy, current_app.cx = previous_center + abort(400, "Invalid bounding box.") crop_box_dict = map_bbox_to_dict(crop_bbox, coordinate_order) bound = create_bbox_bound(crop_box_dict, coordinate_order) diff --git a/synanno/routes/file_access.py b/synanno/routes/file_access.py index b89d44d..5412b69 100644 --- a/synanno/routes/file_access.py +++ b/synanno/routes/file_access.py @@ -2,7 +2,6 @@ import logging from flask import Blueprint, Response, current_app, request, send_file -from flask_cors import cross_origin logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -22,7 +21,6 @@ def get_swc(): @blueprint.route("/source_and_target_exist//", methods=["GET"]) -@cross_origin() def source_and_target_exist(image_index, slice_id): """Valide that both source and target images are available.""" image_index = str(image_index) @@ -39,7 +37,6 @@ def source_and_target_exist(image_index, slice_id): @blueprint.route("/source_img_exists//", methods=["GET"]) -@cross_origin() def source_img_exists(image_index, slice_id): """Serves EM images from memory.""" image_index = str(image_index) @@ -53,7 +50,6 @@ def source_img_exists(image_index, slice_id): @blueprint.route("/get_source_image//", methods=["GET", "HEAD"]) -@cross_origin() def get_source_image(image_index, slice_id): """Serves EM images from memory.""" image_index = str(image_index) @@ -74,7 +70,6 @@ def get_source_image(image_index, slice_id): @blueprint.route("/get_target_image//", methods=["GET", "HEAD"]) -@cross_origin() def get_target_image(image_index, slice_id): """Serves synapse segmentation images from memory.""" image_index = str(image_index) @@ -94,7 +89,6 @@ def get_target_image(image_index, slice_id): return "Image not found", 404 -@cross_origin() @blueprint.route("/get_curve_image//", methods=["GET", "HEAD"]) def get_curve_image(image_index, slice_id): """Serves curve images from memory.""" @@ -116,7 +110,6 @@ def get_curve_image(image_index, slice_id): return "Image not found", 404 -@cross_origin() @blueprint.route( "/get_auto_curve_image//", methods=["GET", "HEAD"] ) @@ -142,7 +135,6 @@ def get_auto_curve_image(image_index, slice_id): return "Image not found", 404 -@cross_origin() @blueprint.route( "/get_circle_pre_image//", methods=["GET", "HEAD"] ) @@ -168,7 +160,6 @@ def get_circle_pre_image(image_index, slice_id): return "Image not found", 404 -@cross_origin() @blueprint.route( "/get_circle_post_image//", methods=["GET", "HEAD"] ) diff --git a/synanno/routes/manual_annotate.py b/synanno/routes/manual_annotate.py index ea40b26..1c6fad9 100644 --- a/synanno/routes/manual_annotate.py +++ b/synanno/routes/manual_annotate.py @@ -5,9 +5,8 @@ from io import BytesIO import numpy as np -from flask import Blueprint, current_app, jsonify, render_template, request -from flask_cors import cross_origin -from PIL import Image +from flask import Blueprint, abort, current_app, jsonify, render_template, request +from PIL import Image, UnidentifiedImageError from synanno.backend.processing import process_instance, update_slice_number from synanno.backend.utils import img_to_png_bytes, png_bytes_to_pil_img @@ -37,10 +36,21 @@ def get_instance_data(page: int, index: int) -> dict: ).to_dict("records")[0] +# Upper bound for drawn canvases; the canvas is a few hundred pixels wide +MAX_CANVAS_SIDE = 4096 + + def decode_image(image_base64: str) -> Image: - """Decode base64 image data.""" + """Decode base64 image data. + + The canvas always sends PNG, so other formats (and their decoders) are refused, + and oversized images are rejected before their pixels are decoded. + """ image_data = re.sub("^data:image/.+;base64,", "", image_base64) - return Image.open(BytesIO(base64.b64decode(image_data))) + image = Image.open(BytesIO(base64.b64decode(image_data)), formats=["PNG"]) + if max(image.size) > MAX_CANVAS_SIDE: + raise ValueError(f"Canvas image too large: {image.size}") + return image def resize_image(image: Image, crop_axes: tuple) -> Image: @@ -76,7 +86,10 @@ def save_canvas() -> dict: Passes the instance specific session information as JSON to draw.js """ coordinate_order = list(current_app.coordinate_order.keys()) - image = decode_image(request.form["imageBase64"]) + try: + image = decode_image(request.form["imageBase64"]) + except (KeyError, ValueError, UnidentifiedImageError): + abort(400, "Expected a PNG image.") page = int(request.form["page"]) index = int(request.form["data_id"]) viewed_instance_slice = int(request.form["viewedInstanceSlice"]) @@ -129,7 +142,6 @@ def load_missing_slices() -> dict: @blueprint.route("/save_pre_post_coordinates", methods=["POST"]) -@cross_origin() def save_pre_post_coordinates() -> tuple: """Save the pre or post coordinates.""" coordinate_order = get_coordinate_order() diff --git a/synanno/routes/opendata.py b/synanno/routes/opendata.py index c5c9dd6..9f76671 100644 --- a/synanno/routes/opendata.py +++ b/synanno/routes/opendata.py @@ -1,10 +1,19 @@ import json import logging +import os +from urllib.parse import urlparse import numpy as np import pandas as pd -from flask import Blueprint, current_app, flash, jsonify, render_template, request -from flask_cors import cross_origin +from flask import ( + Blueprint, + abort, + current_app, + flash, + jsonify, + render_template, + request, +) from werkzeug.datastructures import MultiDict import synanno.backend.ng_util as ng_util @@ -39,6 +48,51 @@ # Define a Blueprint for opendata routes blueprint = Blueprint("open_data", __name__) +# The only cloud volumes the public demo may load (the public H01 release) +DEMO_SOURCE_URL = "gs://h01-release/data/20210601/4nm_raw" +DEMO_TARGET_URL = "gs://h01-release/data/20210729/c3/synapses/whole_ei_onlyvol" +DEMO_NEUROPIL_URL = "gs://h01-release/data/20210601/proofread_104" +DEMO_VOLUME_URLS = (DEMO_SOURCE_URL, DEMO_TARGET_URL, DEMO_NEUROPIL_URL) + + +def is_demo_volume_set(source_url, target_url, neuropil_url) -> bool: + """Check that the requested volumes are exactly the public H01 release.""" + return (source_url, target_url, neuropil_url) == DEMO_VOLUME_URLS + + +def resolve_materialization_path(materialization_path: str): + """Validate a user-supplied materialization location. + + In public demo mode only the bundled H01 table is accepted. Otherwise local + paths must lie inside one of the configured data directories, and remote + tables must be http(s) URLs. + + Args: + materialization_path: Path or URL provided by the user. + + Returns: + The path or URL to read, or None if the location is not allowed. + """ + parsed = urlparse(materialization_path) + if parsed.scheme in ("http", "https"): + if current_app.config["PUBLIC_DEMO"] or not parsed.netloc: + return None + return materialization_path + if parsed.scheme not in ("", "file"): + return None + + path = os.path.realpath( + parsed.path if parsed.scheme == "file" else materialization_path + ) + if current_app.config["PUBLIC_DEMO"]: + if path == current_app.config["BUNDLED_MATERIALIZATION"]: + return path + return None + for data_dir in current_app.config["DATA_DIRS"]: + if os.path.commonpath([path, data_dir]) == data_dir: + return path + return None + @blueprint.route("/open_data", defaults={"task": "annotate"}) @blueprint.route("/open_data/", methods=["GET"], endpoint="open_data_task") @@ -372,6 +426,16 @@ def upload_file(): Returns: Renders the open-data view, with additional buttons enabled """ + if current_app.config["PUBLIC_DEMO"] and ( + request.files.get("secrets_file") + or not is_demo_volume_set( + request.form.get("source_url"), + request.form.get("target_url"), + request.form.get("neuropil_url"), + ) + ): + abort(403, "The public demo only serves the H01 release.") + current_app.view_style = request.form.get("view_style") current_app.tiles_per_page = int(request.form.get("tiles_per_page")) @@ -514,7 +578,6 @@ def set_data(task: str = "annotate"): @blueprint.route("/get_instance", methods=["POST"]) -@cross_origin() def get_instance(): """Serves one of two Ajax calls from annotation.js, passing instance specific info @@ -570,7 +633,6 @@ def get_instance(): @blueprint.route("/neuro", methods=["POST"]) -@cross_origin() def neuro(): """Serves an Ajax request from annotation.js or draw_module.js, shifting the view focus with in the running NG instance and passing the link for the instance to the @@ -691,6 +753,11 @@ def launch_neuroglancer(): target_url = request.args.get("target_url") neuropil_url = request.args.get("neuropil_url") + if current_app.config["PUBLIC_DEMO"] and not is_demo_volume_set( + source_url, target_url, neuropil_url + ): + abort(403, "The public demo only serves the H01 release.") + if not hasattr(current_app, "ng_viewer") or current_app.ng_viewer is None: ng_util.setup_ng( app=current_app._get_current_object(), @@ -708,13 +775,21 @@ def launch_neuroglancer(): @blueprint.route("/load_materialization", methods=["POST"]) def load_materialization(): - materialization_path = request.json.get("materialization_url") + payload = request.get_json(silent=True) + materialization_path = ( + payload.get("materialization_url") if isinstance(payload, dict) else None + ) - if materialization_path is None or materialization_path == "": + if not isinstance(materialization_path, str) or materialization_path == "": return jsonify({"error": "Materialization path is missing."}), 400 + + # Validate before the cache check so a bad path is always rejected + path = resolve_materialization_path(materialization_path) + if path is None: + return jsonify({"error": "Materialization location is not allowed."}), 400 + try: logger.info("Loading the materialization table...") - path = materialization_path.replace("file://", "") # check if current_app.synapse_data is empty dictionary if not hasattr(current_app, "synapse_data") or not current_app.synapse_data: current_app.synapse_data = pd.read_csv(path) @@ -726,7 +801,7 @@ def load_materialization(): except Exception as e: logger.info(f"Failed to load materialization table: {e}") - return jsonify({"error": str(e)}), 500 + return jsonify({"error": "Failed to load the materialization table."}), 500 @blueprint.route("/get_neuron_id", methods=["GET"]) diff --git a/synanno/static/draw.js b/synanno/static/draw.js index 3bbcb6e..46547c7 100755 --- a/synanno/static/draw.js +++ b/synanno/static/draw.js @@ -66,7 +66,7 @@ $(document).ready(() => { if (xhr.status === 200) { // Only execute if status is 200 (image exists) $(new Image()) .attr("src", "/get_circle_pre_image/" + data_id + "/" + middle_slice) - .load(function () { + .on("load", function () { $(canvas_target_image_circle_pre).attr("src", this.src); }); $(canvas_target_image_circle_pre).removeClass("d-none"); @@ -84,7 +84,7 @@ $(document).ready(() => { if (xhr.status === 200) { // Only execute if status is 200 (image exists) $(new Image()) .attr("src", "/get_circle_post_image/" + data_id + "/" + middle_slice) - .load(function () { + .on("load", function () { $(canvas_target_image_circle_post).attr("src", this.src); }); $(canvas_target_image_circle_post).removeClass("d-none"); diff --git a/synanno/static/vendor/jquery-3.7.1.min.js b/synanno/static/vendor/jquery-3.7.1.min.js new file mode 100644 index 0000000..7f37b5d --- /dev/null +++ b/synanno/static/vendor/jquery-3.7.1.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.7.1 | (c) OpenJS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(ie,e){"use strict";var oe=[],r=Object.getPrototypeOf,ae=oe.slice,g=oe.flat?function(e){return oe.flat.call(e)}:function(e){return oe.concat.apply([],e)},s=oe.push,se=oe.indexOf,n={},i=n.toString,ue=n.hasOwnProperty,o=ue.toString,a=o.call(Object),le={},v=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},y=function(e){return null!=e&&e===e.window},C=ie.document,u={type:!0,src:!0,nonce:!0,noModule:!0};function m(e,t,n){var r,i,o=(n=n||C).createElement("script");if(o.text=e,t)for(r in u)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[i.call(e)]||"object":typeof e}var t="3.7.1",l=/HTML$/i,ce=function(e,t){return new ce.fn.init(e,t)};function c(e){var t=!!e&&"length"in e&&e.length,n=x(e);return!v(e)&&!y(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+ge+")"+ge+"*"),x=new RegExp(ge+"|>"),j=new RegExp(g),A=new RegExp("^"+t+"$"),D={ID:new RegExp("^#("+t+")"),CLASS:new RegExp("^\\.("+t+")"),TAG:new RegExp("^("+t+"|[*])"),ATTR:new RegExp("^"+p),PSEUDO:new RegExp("^"+g),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+ge+"*(even|odd|(([+-]|)(\\d*)n|)"+ge+"*(?:([+-]|)"+ge+"*(\\d+)|))"+ge+"*\\)|)","i"),bool:new RegExp("^(?:"+f+")$","i"),needsContext:new RegExp("^"+ge+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+ge+"*((?:-\\d)?\\d*)"+ge+"*\\)|)(?=[^-]|$)","i")},N=/^(?:input|select|textarea|button)$/i,q=/^h\d$/i,L=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,H=/[+~]/,O=new RegExp("\\\\[\\da-fA-F]{1,6}"+ge+"?|\\\\([^\\r\\n\\f])","g"),P=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},M=function(){V()},R=J(function(e){return!0===e.disabled&&fe(e,"fieldset")},{dir:"parentNode",next:"legend"});try{k.apply(oe=ae.call(ye.childNodes),ye.childNodes),oe[ye.childNodes.length].nodeType}catch(e){k={apply:function(e,t){me.apply(e,ae.call(t))},call:function(e){me.apply(e,ae.call(arguments,1))}}}function I(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(V(e),e=e||T,C)){if(11!==p&&(u=L.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return k.call(n,a),n}else if(f&&(a=f.getElementById(i))&&I.contains(e,a)&&a.id===i)return k.call(n,a),n}else{if(u[2])return k.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&e.getElementsByClassName)return k.apply(n,e.getElementsByClassName(i)),n}if(!(h[t+" "]||d&&d.test(t))){if(c=t,f=e,1===p&&(x.test(t)||m.test(t))){(f=H.test(t)&&U(e.parentNode)||e)==e&&le.scope||((s=e.getAttribute("id"))?s=ce.escapeSelector(s):e.setAttribute("id",s=S)),o=(l=Y(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+Q(l[o]);c=l.join(",")}try{return k.apply(n,f.querySelectorAll(c)),n}catch(e){h(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return re(t.replace(ve,"$1"),e,n,r)}function W(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function F(e){return e[S]=!0,e}function $(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function B(t){return function(e){return fe(e,"input")&&e.type===t}}function _(t){return function(e){return(fe(e,"input")||fe(e,"button"))&&e.type===t}}function z(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&R(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function X(a){return F(function(o){return o=+o,F(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function U(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}function V(e){var t,n=e?e.ownerDocument||e:ye;return n!=T&&9===n.nodeType&&n.documentElement&&(r=(T=n).documentElement,C=!ce.isXMLDoc(T),i=r.matches||r.webkitMatchesSelector||r.msMatchesSelector,r.msMatchesSelector&&ye!=T&&(t=T.defaultView)&&t.top!==t&&t.addEventListener("unload",M),le.getById=$(function(e){return r.appendChild(e).id=ce.expando,!T.getElementsByName||!T.getElementsByName(ce.expando).length}),le.disconnectedMatch=$(function(e){return i.call(e,"*")}),le.scope=$(function(){return T.querySelectorAll(":scope")}),le.cssHas=$(function(){try{return T.querySelector(":has(*,:jqfake)"),!1}catch(e){return!0}}),le.getById?(b.filter.ID=function(e){var t=e.replace(O,P);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(O,P);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):t.querySelectorAll(e)},b.find.CLASS=function(e,t){if("undefined"!=typeof t.getElementsByClassName&&C)return t.getElementsByClassName(e)},d=[],$(function(e){var t;r.appendChild(e).innerHTML="",e.querySelectorAll("[selected]").length||d.push("\\["+ge+"*(?:value|"+f+")"),e.querySelectorAll("[id~="+S+"-]").length||d.push("~="),e.querySelectorAll("a#"+S+"+*").length||d.push(".#.+[+~]"),e.querySelectorAll(":checked").length||d.push(":checked"),(t=T.createElement("input")).setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),r.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&d.push(":enabled",":disabled"),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||d.push("\\["+ge+"*name"+ge+"*="+ge+"*(?:''|\"\")")}),le.cssHas||d.push(":has"),d=d.length&&new RegExp(d.join("|")),l=function(e,t){if(e===t)return a=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!le.sortDetached&&t.compareDocumentPosition(e)===n?e===T||e.ownerDocument==ye&&I.contains(ye,e)?-1:t===T||t.ownerDocument==ye&&I.contains(ye,t)?1:o?se.call(o,e)-se.call(o,t):0:4&n?-1:1)}),T}for(e in I.matches=function(e,t){return I(e,null,null,t)},I.matchesSelector=function(e,t){if(V(e),C&&!h[t+" "]&&(!d||!d.test(t)))try{var n=i.call(e,t);if(n||le.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){h(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(O,P),e[3]=(e[3]||e[4]||e[5]||"").replace(O,P),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||I.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&I.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return D.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&j.test(n)&&(t=Y(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(O,P).toLowerCase();return"*"===e?function(){return!0}:function(e){return fe(e,t)}},CLASS:function(e){var t=s[e+" "];return t||(t=new RegExp("(^|"+ge+")"+e+"("+ge+"|$)"))&&s(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=I.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function T(e,n,r){return v(n)?ce.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?ce.grep(e,function(e){return e===n!==r}):"string"!=typeof n?ce.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(ce.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||k,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:S.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof ce?t[0]:t,ce.merge(this,ce.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:C,!0)),w.test(r[1])&&ce.isPlainObject(t))for(r in t)v(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=C.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):v(e)?void 0!==n.ready?n.ready(e):e(ce):ce.makeArray(e,this)}).prototype=ce.fn,k=ce(C);var E=/^(?:parents|prev(?:Until|All))/,j={children:!0,contents:!0,next:!0,prev:!0};function A(e,t){while((e=e[t])&&1!==e.nodeType);return e}ce.fn.extend({has:function(e){var t=ce(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,Ce=/^$|^module$|\/(?:java|ecma)script/i;xe=C.createDocumentFragment().appendChild(C.createElement("div")),(be=C.createElement("input")).setAttribute("type","radio"),be.setAttribute("checked","checked"),be.setAttribute("name","t"),xe.appendChild(be),le.checkClone=xe.cloneNode(!0).cloneNode(!0).lastChild.checked,xe.innerHTML="",le.noCloneChecked=!!xe.cloneNode(!0).lastChild.defaultValue,xe.innerHTML="",le.option=!!xe.lastChild;var ke={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function Se(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&fe(e,t)?ce.merge([e],n):n}function Ee(e,t){for(var n=0,r=e.length;n",""]);var je=/<|&#?\w+;/;function Ae(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function Re(e,t){return fe(e,"table")&&fe(11!==t.nodeType?t:t.firstChild,"tr")&&ce(e).children("tbody")[0]||e}function Ie(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function We(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Fe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(_.hasData(e)&&(s=_.get(e).events))for(i in _.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),C.head.appendChild(r[0])},abort:function(){i&&i()}}});var Jt,Kt=[],Zt=/(=)\?(?=&|$)|\?\?/;ce.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Kt.pop()||ce.expando+"_"+jt.guid++;return this[e]=!0,e}}),ce.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Zt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Zt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=v(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Zt,"$1"+r):!1!==e.jsonp&&(e.url+=(At.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||ce.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=ie[r],ie[r]=function(){o=arguments},n.always(function(){void 0===i?ce(ie).removeProp(r):ie[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Kt.push(r)),o&&v(i)&&i(o[0]),o=i=void 0}),"script"}),le.createHTMLDocument=((Jt=C.implementation.createHTMLDocument("").body).innerHTML="
",2===Jt.childNodes.length),ce.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(le.createHTMLDocument?((r=(t=C.implementation.createHTMLDocument("")).createElement("base")).href=C.location.href,t.head.appendChild(r)):t=C),o=!n&&[],(i=w.exec(e))?[t.createElement(i[1])]:(i=Ae([e],t,o),o&&o.length&&ce(o).remove(),ce.merge([],i.childNodes)));var r,i,o},ce.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(ce.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},ce.expr.pseudos.animated=function(t){return ce.grep(ce.timers,function(e){return t===e.elem}).length},ce.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=ce.css(e,"position"),c=ce(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=ce.css(e,"top"),u=ce.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),v(t)&&(t=t.call(e,n,ce.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},ce.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){ce.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===ce.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===ce.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=ce(e).offset()).top+=ce.css(e,"borderTopWidth",!0),i.left+=ce.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-ce.css(r,"marginTop",!0),left:t.left-i.left-ce.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===ce.css(e,"position"))e=e.offsetParent;return e||J})}}),ce.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;ce.fn[t]=function(e){return M(this,function(e,t,n){var r;if(y(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),ce.each(["top","left"],function(e,n){ce.cssHooks[n]=Ye(le.pixelPosition,function(e,t){if(t)return t=Ge(e,n),_e.test(t)?ce(e).position()[n]+"px":t})}),ce.each({Height:"height",Width:"width"},function(a,s){ce.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){ce.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return M(this,function(e,t,n){var r;return y(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?ce.css(e,t,i):ce.style(e,t,n,i)},s,n?e:void 0,n)}})}),ce.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){ce.fn[t]=function(e){return this.on(t,e)}}),ce.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.on("mouseenter",e).on("mouseleave",t||e)}}),ce.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){ce.fn[n]=function(e,t){return 0 - + diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..4e80ee9 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,214 @@ +"""Regression tests for the security fixes (public demo hardening).""" + +import base64 +import io +import os +import runpy + +import pytest + +from synanno import create_app + + +@pytest.fixture +def make_app(monkeypatch): + """Create a fresh app with the given environment variables.""" + + def _make(**env): + for key in ("SYNANNO_PUBLIC_DEMO", "SYNANNO_DATA_DIRS", "SECRET_KEY"): + monkeypatch.delenv(key, raising=False) + for key, value in env.items(): + monkeypatch.setenv(key, value) + app = create_app() + app.config["TESTING"] = True + return app + + return _make + + +def load(client, url): + return client.post("/load_materialization", json={"materialization_url": url}) + + +H01 = { + "source_url": "gs://h01-release/data/20210601/4nm_raw", + "target_url": "gs://h01-release/data/20210729/c3/synapses/whole_ei_onlyvol", + "neuropil_url": "gs://h01-release/data/20210601/proofread_104", +} + + +def test_demo_materialization_allowlist(make_app): + app = make_app(SYNANNO_PUBLIC_DEMO="1") + client = app.test_client() + for url in ("/etc/passwd", "file:///etc/passwd", "http://example.com/t.csv"): + response = load(client, url) + assert response.status_code == 400 + assert "root:" not in response.get_data(as_text=True) + + bundled = "file://" + app.config["BUNDLED_MATERIALIZATION"] + assert load(client, bundled).status_code == 200 + # still rejected once a table is cached + assert load(client, "/etc/passwd").status_code == 400 + + +def test_local_materialization_limited_to_data_dirs(make_app, tmp_path): + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "table.csv").write_text("x,y,z\n1,2,3\n") + (data_dir / "empty.csv").write_text("") + (tmp_path / "outside.csv").write_text("x,y,z\n1,2,3\n") + client = make_app(SYNANNO_DATA_DIRS=str(data_dir)).test_client() + + outside = str(data_dir / ".." / "outside.csv") + for url in (outside, "gs://bucket/t.csv", "/etc/passwd"): + assert load(client, url).status_code == 400 + + # errors do not echo exception details + response = load(client, str(data_dir / "empty.csv")) + assert response.get_json() == {"error": "Failed to load the materialization table."} + + assert load(client, "file://" + str(data_dir / "table.csv")).status_code == 200 + + +def test_demo_refuses_credentials_and_custom_buckets(make_app): + app = make_app(SYNANNO_PUBLIC_DEMO="1") + client = app.test_client() + + form = dict(H01, view_style="neuron", tiles_per_page="12") + form["secrets_file"] = (io.BytesIO(b'{"token": "x"}'), "secrets.json") + assert client.post("/upload", data=form).status_code == 403 + assert not hasattr(app, "view_style") # rejected before any state change + + form = dict(H01, source_url="gs://other/bucket", tiles_per_page="12") + assert client.post("/upload", data=form).status_code == 403 + query = dict(H01, source_url="file:///etc") + response = client.get("/launch_neuroglancer", query_string=query) + assert response.status_code == 403 + + +def test_neuroglancer_token_is_unguessable(make_app, monkeypatch): + import synanno.backend.ng_util as ng_util + + tokens = [] + + class Stop(Exception): + pass + + def fake_viewer(token): + tokens.append(token) + raise Stop + + monkeypatch.setattr(ng_util.neuroglancer, "set_server_bind_address", lambda **_: 0) + monkeypatch.setattr(ng_util.neuroglancer, "Viewer", fake_viewer) + app = make_app() + for _ in range(2): + with pytest.raises(Stop): + ng_util.setup_ng( + app, "precomputed://a", "precomputed://b", "precomputed://c" + ) + + assert len(tokens[0]) >= 43 and tokens[0] != tokens[1] + + +def test_no_cors_headers(make_app): + client = make_app().test_client() + response = client.get("/", headers={"Origin": "http://evil.example"}) + assert "Access-Control-Allow-Origin" not in response.headers + preflight = { + "Origin": "http://evil.example", + "Access-Control-Request-Method": "POST", + } + response = client.options("/get_instance", headers=preflight) + assert "Access-Control-Allow-Origin" not in response.headers + + +def test_no_debug_by_default(make_app, monkeypatch): + import dotenv + from flask import Flask + + calls = [] + monkeypatch.setattr(dotenv, "load_dotenv", lambda *a, **k: None) + monkeypatch.setattr(Flask, "run", lambda self, **kwargs: calls.append(kwargs)) + for key in ("APP_IP", "FLASK_DEBUG", "DEBUG_APP"): + monkeypatch.delenv(key, raising=False) + + run_py = os.path.join(os.path.dirname(os.path.dirname(__file__)), "run.py") + runpy.run_path(run_py, run_name="__main__") + assert calls[0]["host"] == "127.0.0.1" and calls[0]["debug"] is False + + app = make_app() + assert app.debug is False and app.config["DEBUG_APP"] is False + + +def test_secret_key(make_app): + assert make_app(SECRET_KEY="from-env").config["SECRET_KEY"] == "from-env" + first, second = make_app().config["SECRET_KEY"], make_app().config["SECRET_KEY"] + assert len(first) == 64 and first != second + + +def test_os_module_not_exposed_to_templates(make_app): + from flask import render_template_string + + with make_app().test_request_context("/"): + assert render_template_string("{{ os is defined }}") == "False" + + +def test_pages_use_vendored_jquery(make_app): + html = make_app().test_client().get("/").get_data(as_text=True) + assert "/static/vendor/jquery-3.7.1.min.js" in html + assert "jquery-2.1.3" not in html + + +@pytest.mark.parametrize( + "path", + [ + "/get_source_image/..%2F..%2Fetc%2Fpasswd/0", + "/get_curve_image/../../etc/passwd/0", + ], +) +def test_file_routes_serve_only_in_memory_data(make_app, path): + response = make_app().test_client().get(path) + assert response.status_code in (204, 404) + assert b"root:" not in response.data + + +def test_fn_save_rejects_invalid_bbox(make_app): + from synanno.backend.processing import calculate_crop_pad + + with pytest.raises(ValueError): # not an assert, so it survives python -O + calculate_crop_pad([10, 5, 0, 10, 0, 10], (100, 100, 100)) + + app = make_app() + client = app.test_client() + center = (app.cz, app.cy, app.cx) + assert client.post("/ng_bbox_fn_save", data={}).status_code == 400 + form = {"z1": "10", "z2": "2", "my": "5", "mx": "5", "currentPage": "1"} + assert client.post("/ng_bbox_fn_save", data=form).status_code == 400 + assert (app.cz, app.cy, app.cx) == center # shared state left untouched + + +def _encoded(fmt, size=(4, 4)): + from PIL import Image + + buffer = io.BytesIO() + Image.new("1", size).save(buffer, format=fmt) + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +def test_canvas_upload_accepts_only_small_png(make_app): + from PIL import UnidentifiedImageError + + from synanno.routes.manual_annotate import decode_image + + assert decode_image(_encoded("PNG")).format == "PNG" + with pytest.raises(UnidentifiedImageError): + decode_image(_encoded("TIFF")) + with pytest.raises(ValueError): + decode_image(_encoded("PNG", size=(5000, 1))) + + client = make_app().test_client() + form = {"page": "1", "data_id": "0", "viewedInstanceSlice": "0"} + form["canvas_type"] = "curve" + for image in (_encoded("TIFF"), _encoded("PNG", size=(5000, 1))): + response = client.post("/save_canvas", data=dict(form, imageBase64=image)) + assert response.status_code == 400