fix(deps): remediate high severity SQLAlchemy dependency alerts

.github/workflows/dependency-security.yml

@@ -0,0 +1,36 @@
+name: Dependency security regression
+
+on:
+  pull_request:
+    branches: [main]
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  locked-unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - run: python -m pip install pipenv
+      - run: pipenv verify && pipenv sync --dev
+      - run: pipenv run pytest --confcutdir=tests/unit -o addopts='' tests/unit -q
+
+  superset-compatibility:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - uses: astral-sh/setup-uv@v6
+      - run: uv venv
+      - run: uv pip install -e . -r tests/superset_requirements.txt pytest
+      - run: uv pip check
+      - run: uv run --no-project python -m pytest --confcutdir=tests/security -o addopts='' tests/security -q
+      - run: uv run --no-project python -m pytest --confcutdir=tests/unit -o addopts='' tests/unit -q

.github/workflows/test.yml

@@ -15,10 +15,11 @@ jobs:
         env:
           QUERY_DEFAULT_USER: tidbcloudlake
           QUERY_DEFAULT_PASSWORD: tidbcloudlake
-          MINIO_ENABLED: true
+          # No object-storage I/O is exercised by this suite. Avoid the image
+          # bootstrap downloading a MinIO binary from a retired endpoint.
+          QUERY_STORAGE_TYPE: fs
         ports:
           - 8000:8000
-          - 9000:9000
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -31,14 +32,29 @@ jobs:
       - name: Pip Install
         run: |
           pip install pipenv
-          pipenv install --dev --skip-lock
+          pipenv verify
+          pipenv sync --dev
 
       - name: Verify Service Running
         run: |
           cid=$(docker ps -aq --filter "ancestor=datafuselabs/databend:nightly" | head -n1)
           test -n "${cid}"
-          docker logs "${cid}"
-          curl -v http://localhost:8000/v1/health
+          if ! curl --fail --silent --show-error \
+            --retry 30 --retry-all-errors --retry-delay 2 --max-time 5 \
+            --user tidbcloudlake:tidbcloudlake \
+            --header 'Content-Type: application/json' \
+            --data '{"sql":"SELECT 1"}' \
+            http://localhost:8000/v1/query > /tmp/lake-readiness.json; then
+            docker logs "${cid}"
+            exit 1
+          fi
+          python - <<'PYCODE'
+          import json
+          with open("/tmp/lake-readiness.json") as response:
+              result = json.load(response)
+          assert not result.get("error"), result
+          assert result.get("data"), result
+          PYCODE
 
       - name: SQLAlchemy Test Suite
         env:

.gitignore

@@ -107,3 +107,7 @@ ENV/
 
 # vim
 *.swp
+
+# Local dependency validation environments
+.venv/
+.venv-superset/

Pipfile

@@ -4,10 +4,12 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-databend-sqlalchemy = {file = ".", editable = true}
+tidbcloudlake-sqlalchemy = {file = ".", editable = true}
+# The upstream conformance tests import SQLAlchemy 2.0-only test classes.
+sqlalchemy = ">=2.0,<2.1"
 
 [dev-packages]
-black = "*"
+black = ">=26.3.1"
 flake8 = "*"
 pytest = "*"
 pytest-xdist = "*"

tests/security/test_dependency_security.py

@@ -0,0 +1,104 @@
+"""Dependency floors and Flask compatibility without a live database."""
+
+import json
+from importlib.metadata import version
+from pathlib import Path
+
+import pytest
+from packaging.requirements import Requirement
+from packaging.version import Version
+
+# This optional profile runs in the dedicated Superset CI environment.
+flask = pytest.importorskip("flask")
+Flask, request, session = flask.Flask, flask.request, flask.session
+
+ROOT = Path(__file__).parents[2]
+
+
+@pytest.mark.parametrize(
+    ("package", "minimum"), [("Flask", "3.1.3"), ("Werkzeug", "3.1.9")]
+)
+def test_installed_web_dependencies_are_patched(package, minimum):
+    assert Version(version(package)) >= Version(minimum)
+
+
+def test_lockfile_security_floors():
+    lock = json.loads((ROOT / "Pipfile.lock").read_text())
+    assert Version(lock["develop"]["black"]["version"].removeprefix("==")) >= Version(
+        "26.3.1"
+    )
+    for group in ("default", "develop"):
+        for package, minimum in (("flask", "2.2.5"), ("werkzeug", "3.0.3")):
+            if package in lock[group]:
+                assert Version(
+                    lock[group][package]["version"].removeprefix("==")
+                ) >= Version(minimum)
+    assert "tidbcloudlake-sqlalchemy" in lock["default"]
+    assert "databend-sqlalchemy" not in lock["default"]
+
+
+def test_superset_requirements_exclude_vulnerable_web_versions():
+    requirements = {}
+    for line in (ROOT / "tests" / "superset_requirements.txt").read_text().splitlines():
+        if line and not line.startswith("#"):
+            requirement = Requirement(line)
+            requirements[requirement.name.lower()] = requirement
+    assert "2.2.4" not in requirements["flask"].specifier
+    assert "3.0.2" not in requirements["werkzeug"].specifier
+    assert "3.1.3" in requirements["flask"].specifier
+    assert "3.1.9" in requirements["werkzeug"].specifier
+
+
+def test_permanent_session_response_varies_on_cookie():
+    app = Flask(__name__)
+    app.config["SECRET_KEY"] = "test-only-session-key"
+
+    @app.get("/init")
+    def init():
+        session.permanent = True
+        session["visited"] = True
+        return "ok"
+
+    @app.get("/")
+    def index():
+        # Refresh a permanent cookie without accessing or modifying the session.
+        return "ok"
+
+    with app.test_client() as client:
+        assert client.get("/init").status_code == 200
+        response = client.get("/")
+        assert response.status_code == 200
+        assert "Set-Cookie" in response.headers
+        assert "Cookie" in response.vary
+
+
+def test_multipart_field_limit_is_enforced():
+    app = Flask(__name__)
+    app.config["MAX_FORM_PARTS"] = 2
+
+    @app.post("/")
+    def form():
+        return dict(request.form)
+
+    with app.test_client() as client:
+        accepted = client.post(
+            "/", data={"a": "1", "b": "2"}, content_type="multipart/form-data"
+        )
+        assert accepted.status_code == 200
+        rejected = client.post(
+            "/", data={"a": "1", "b": "2", "c": "3"}, content_type="multipart/form-data"
+        )
+        assert rejected.status_code == 413
+
+
+def test_superset_imports_and_loads_lake_dialect():
+    pytest.importorskip("superset")
+    from sqlalchemy.dialects import registry
+    from superset.app import create_app
+    from superset.db_engine_specs.base import BaseEngineSpec
+
+    from tidbcloudlake_sqlalchemy.tidbcloudlake_dialect import TiDBCloudLakeDialect
+
+    assert callable(create_app)
+    assert hasattr(BaseEngineSpec, "engine")
+    assert registry.load("lake") is TiDBCloudLakeDialect

tests/superset_requirements.txt

@@ -1,6 +1,7 @@
-apache_superset>=2.0.0
-flask==2.0.3
-MarkupSafe==2.0.1
-Jinja2==3.0.3
-werkzeug==2.0.3
-WTForms==2.3.3
+# Superset 6 supports Flask 3; old Flask/Werkzeug 2.0 pins were vulnerable.
+apache-superset>=6.1.0,<7
+Flask>=3.1.3,<4
+Werkzeug>=3.1.9,<4
+MarkupSafe>=2.1.5,<4
+Jinja2>=3.1.6,<4
+WTForms>=3.1.0,<4

tests/unit/test_tidbcloudlake_dialect.py

@@ -46,7 +46,7 @@ def test_create_connect_args(self, dialect: TiDBCloudLakeDialect):
         u = url.make_url("lake://user:pass@host:443/db?warehouse=test&secure=True")
         args, kwargs = dialect.create_connect_args(u)
         assert args == []
-        assert kwargs["dsn"] == "lake://user:pass@host:443/db?warehouse=test&secure=True"
+        assert url.make_url(kwargs["dsn"]) == u
 
     def test_do_execute(
             self, dialect: TiDBCloudLakeDialect, cursor: mock.Mock(spec=MockCursor)
@@ -110,6 +110,7 @@ def test_view_names(
             self, dialect: TiDBCloudLakeDialect, connection: mock.Mock(spec=MockDBApi)
     ):
         connection.execute.return_value = []
+        dialect.server_version_info = (1, 2, 881)
         assert dialect.get_view_names(connection) == []
 
     def test_indexes(
@@ -129,14 +130,14 @@ def getitem(self, idx):
             return mock.Mock(__getitem__=getitem)
 
         connection.execute.return_value = [
-            multi_column_row(["name1", "INT", "YES"]),
-            multi_column_row(["name2", "date", "NO"]),
-            multi_column_row(["name3", "boolean", "YES"]),
-            multi_column_row(["name4", "binary", "YES"])
+            multi_column_row(["name1", "INT", "YES", None]),
+            multi_column_row(["name2", "date", "NO", None]),
+            multi_column_row(["name3", "boolean", "YES", None]),
+            multi_column_row(["name4", "binary", "YES", "binary column"])
         ]
 
         expected_query = """
-            select column_name, column_type, is_nullable
+            select column_name, data_type, is_nullable, nullif(column_comment, '')
             from information_schema.columns
             where table_name = :table_name
             and table_schema = :schema_name
@@ -153,30 +154,35 @@ def getitem(self, idx):
                 ),
         ):
             result = call()
-            assert result == [
+            # Reflection returns TypeEngine instances, not their classes.
+            assert [{**column, "type": type(column["type"])} for column in result] == [
                 {
                     "name": "name1",
                     "type": INTEGER,
                     "nullable": True,
                     "default": None,
+                    "comment": None,
                 },
                 {
                     "name": "name2",
                     "type": TiDBCloudLakeDate,
                     "nullable": False,
                     "default": None,
+                    "comment": None,
                 },
                 {
                     "name": "name3",
                     "type": BOOLEAN,
                     "nullable": True,
                     "default": None,
+                    "comment": None,
                 },
                 {
                     "name": "name4",
                     "type": BINARY,
                     "nullable": True,
                     "default": None,
+                    "comment": "binary column",
                 },
             ]
             connection.execute.assert_called_once()