feat: enabled cross-origin storage emscripten build for bundler-friendly build

.github/workflows/build-wasm.yml

@@ -73,16 +73,13 @@ jobs:
       - name: Run build
         if: steps.check-branch.outputs.skip != 'true'
         run: |
-          mkdir -p out src/bin
+          mkdir -p src/bin
           docker run --rm \
             -e SQLITE_REF="${{ steps.resolve-ref.outputs.sqlite_ref }}" \
             -e HOST_UID="$(id -u)" \
             -e HOST_GID="$(id -g)" \
-            -v "$(pwd)/out":/out \
             -v "$(pwd)/src/bin":/src/bin \
             sqlite-wasm-builder:env build
-          # Fallback fix for permissions if chown inside docker failed or was incomplete
-          sudo chown -R $(id -u):$(id -g) out src/bin
 
       - name: Check for changes
         if: steps.check-branch.outputs.skip != 'true'

CONTRIBUTING.md

@@ -38,7 +38,6 @@ To rebuild the underlying SQLite Wasm binaries, use Docker:
    ```bash
    docker run --rm \
      -e SQLITE_REF="master" \
-     -v "$(pwd)/out":/out \
      -v "$(pwd)/src/bin":/src/bin \
      sqlite-wasm-builder:env build
    ```
@@ -48,7 +47,6 @@ To rebuild the underlying SQLite Wasm binaries, use Docker:
    ```powershell
    docker run --rm `
      -e SQLITE_REF="master" `
-     -v "${PWD}/out:/out" `
      -v "${PWD}/src/bin:/src/bin" `
      sqlite-wasm-builder:env build
    ```
@@ -58,7 +56,6 @@ To rebuild the underlying SQLite Wasm binaries, use Docker:
    ```cmd
    docker run --rm ^
      -e SQLITE_REF="master" ^
-     -v "%cd%/out:/out" ^
      -v "%cd%/src/bin:/src/bin" ^
      sqlite-wasm-builder:env build
    ```

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-ARG EMSDK_VERSION=5.0.5
+ARG EMSDK_VERSION=6.0.13
 FROM emscripten/emsdk:${EMSDK_VERSION} AS wasm-tools
 
 # The SDK already supplies the build dependencies. SQLite's release build

scripts/build-sqlite3-wasm.sh

@@ -3,140 +3,132 @@ set -euo pipefail
 
 SQLITE_REPO="${SQLITE_REPO:-https://github.com/sqlite/sqlite.git}"
 SQLITE_REF="${SQLITE_REF:-master}"
-OUT_DIR="${OUT_DIR:-/out}"
 SRC_DIR="${WORKDIR:-/build/sqlite-src}"
 HOST_UID="${HOST_UID:-}"
 HOST_GID="${HOST_GID:-}"
+WASM_OPT_WRAPPER=""
+EXTRACT_DIR=""
 
-# Ensure out exists and is writable
-mkdir -p "$OUT_DIR"
-mkdir -p /src/bin
-
-# Check if volumes are mounted
-check_mounts() {
-  if ! mountpoint -q "$OUT_DIR" 2>/dev/null; then
-    # Fallback check using device IDs if mountpoint command is not available or fails
-    root_dev=$(stat -c %d /)
-    out_dev=$(stat -c %d "$OUT_DIR")
-    if [ "$root_dev" = "$out_dev" ]; then
-      log "WARNING: $OUT_DIR does not appear to be a volume mount. Files will be lost when the container exits."
-    fi
-  fi
-  if ! mountpoint -q /src/bin 2>/dev/null; then
-    root_dev=$(stat -c %d /)
-    bin_dev=$(stat -c %d /src/bin)
-    if [ "$root_dev" = "$bin_dev" ]; then
-      log "WARNING: /src/bin does not appear to be a volume mount. Files will be lost when the container exits."
-    fi
-  fi
-}
-
-# Helper: log
 log() { printf '\033[1;34m[build]\033[0m %s\n' "$*"; }
 
-# Handle ownership fixup on exit
-cleanup() {
-  if [ -n "$HOST_UID" ] && [ -n "$HOST_GID" ]; then
-    log "Fixing ownership for /out and /src/bin to $HOST_UID:$HOST_GID"
-    chown -R "$HOST_UID:$HOST_GID" "$OUT_DIR" /src/bin
-  fi
-}
-trap cleanup EXIT
-
 if [ "${1:-}" = "shell" ]; then
   exec /bin/bash
 fi
-
 if [ "${1:-}" != "build" ]; then
-  echo "Usage: <image> build  (or 'shell' for debugging)"
+  echo "Usage: <image> build (or 'shell' for debugging)"
   exit 2
 fi
 
-log "Starting sqlite build"
-check_mounts
-log "Repo: $SQLITE_REPO"
-log "Ref:  $SQLITE_REF"
-log "Out: $OUT_DIR"
-log "Src dir: $SRC_DIR"
-
-# prepare source dir: clone shallow or fetch+reset if exists
-if [ -d "$SRC_DIR/.git" ]; then
-  log "Repository already exists, fetching updates..."
-  git -C "$SRC_DIR" fetch --depth 1 origin "$SQLITE_REF" || git -C "$SRC_DIR" fetch --unshallow || true
-  git -C "$SRC_DIR" checkout --detach FETCH_HEAD || git -C "$SRC_DIR" checkout "$SQLITE_REF" || git -C "$SRC_DIR" reset --hard origin/"$SQLITE_REF"
-else
-  log "Cloning repository..."
-  rm -rf "$SRC_DIR"
-  mkdir -p "$SRC_DIR"
-  git -C "$SRC_DIR" init
-  git -C "$SRC_DIR" remote add origin "$SQLITE_REPO"
-  log "Fetching ref: $SQLITE_REF"
-  if git -C "$SRC_DIR" fetch --depth 1 origin "$SQLITE_REF"; then
-    git -C "$SRC_DIR" checkout --detach FETCH_HEAD
-  else
-    log "Failed to fetch shallow ref, trying full clone..."
-    rm -rf "$SRC_DIR"
-    git clone "$SQLITE_REPO" "$SRC_DIR"
-    git -C "$SRC_DIR" checkout "$SQLITE_REF"
+mkdir -p /src/bin
+cleanup() {
+  local status=$?
+  trap - EXIT
+  if [ -n "$WASM_OPT_WRAPPER" ]; then rm -f "$WASM_OPT_WRAPPER"; fi
+  if [ -n "$EXTRACT_DIR" ]; then rm -rf "$EXTRACT_DIR"; fi
+  if [ -n "$HOST_UID" ] && [ -n "$HOST_GID" ]; then
+    log "Fixing ownership for /src/bin to $HOST_UID:$HOST_GID"
+    chown -R "$HOST_UID:$HOST_GID" /src/bin || status=1
   fi
+  exit "$status"
+}
+trap cleanup EXIT
+
+if ! mountpoint -q /src/bin; then
+  log "WARNING: /src/bin is not a volume mount. Output will be lost when the container exits."
 fi
+log "Building $SQLITE_REPO at $SQLITE_REF"
 
-# Ensure emsdk environment active in this shell
-if [ -f /emsdk/emsdk_env.sh ]; then
-  # shellcheck disable=SC1091
-  source /emsdk/emsdk_env.sh
+# Fail on fetch errors rather than falling back to a stale checkout.
+mkdir -p "$SRC_DIR"
+if [ ! -d "$SRC_DIR/.git" ]; then
+  git -C "$SRC_DIR" init
+fi
+if git -C "$SRC_DIR" remote get-url origin >/dev/null 2>&1; then
+  git -C "$SRC_DIR" remote set-url origin "$SQLITE_REPO"
 else
-  log "Warning: /emsdk/emsdk_env.sh not found — emscripten environment not available"
+  git -C "$SRC_DIR" remote add origin "$SQLITE_REPO"
 fi
+git -C "$SRC_DIR" fetch --depth 1 origin "$SQLITE_REF"
+git -C "$SRC_DIR" checkout --detach FETCH_HEAD
 
-# build steps
+# shellcheck disable=SC1091
+source /emsdk/emsdk_env.sh
 cd "$SRC_DIR"
-
-# regenerate configure if not present
-if [ ! -f ./configure ]; then
-  log "Running autoreconf -f -i"
-  autoreconf -f -i
-fi
-
 log "Running ./configure"
 ./configure
-
 log "make sqlite3.c"
 make sqlite3.c
-
 cd ext/wasm
 
-log "Running make -j4 npm (ext/wasm)"
-make -j4 npm
-
-# ensure artifact exists
-if [ ! -f npm-bundle.zip ]; then
-  log "ERROR: npm-bundle.zip not found in ext/wasm"
-  exit 3
-fi
-
-# copy zip into OUT_DIR
-log "Copying npm-bundle.zip to $OUT_DIR"
-cp -f npm-bundle.zip "$OUT_DIR/npm-bundle.zip"
-chmod a+r "$OUT_DIR/npm-bundle.zip"
-
-# extract into /src/bin (normalize single top-level dir)
-log "Extracting npm-bundle.zip into /src/bin"
-# Clear existing contents to ensure a clean overwrite
-rm -rf /src/bin/*
-mkdir -p /src/bin
-unzip -q -o "$OUT_DIR/npm-bundle.zip" -d /src/bin
-rm -f "$OUT_DIR/npm-bundle.zip"
-
-# If a single top-level directory exists, move its contents up
-shopt -s nullglob
-entries=(/src/bin/*)
-if [ "${#entries[@]}" -eq 1 ] && [ -d "${entries[0]}" ]; then
-  log "Zip contained top-level dir: moving contents up"
-  mv "${entries[0]}"/* /src/bin/ || true
-  rmdir "${entries[0]}" || true
-fi
-shopt -u nullglob
-
+# SQLite invokes wasm-opt with --all-features. Newer Binaryen versions then
+# emit compact imports, which are not supported by our browser/Node targets.
+# Append the compatibility flag after SQLite's flags, preserving its passes.
+WASM_OPT_WRAPPER="$(mktemp /tmp/sqlite-wasm-opt.XXXXXX)"
+printf '#!/usr/bin/env bash\nexec %q "$@" --disable-compact-imports\n' \
+  "$(command -v wasm-opt)" > "$WASM_OPT_WRAPPER"
+chmod +x "$WASM_OPT_WRAPPER"
+
+# Check the generated defaults before overriding the per-loader flags. Refuse
+# to silently discard additional flags introduced by an upstream SQLite update.
+make .wasmbuilds.make WASM_CUSTOM_INSTANTIATE=0
+python3 - <<'PY'
+from pathlib import Path
+import re
+
+defaults = Path('.wasmbuilds.make').read_text()
+for build in ('bundler', 'esm'):
+    assignments = re.findall(r'^emcc\.flags\.' + build + r'\s*[:+?]?=([^\n]*)', defaults, re.M)
+    if any(value.split('#', 1)[0].strip() for value in assignments):
+        raise SystemExit(f'Upstream emcc.flags.{build} is no longer empty; merge its flags before enabling COS')
+PY
+
+log "Running make -j4 npm with Cross-Origin Storage for browser loaders (excluding Node)"
+# SQLite's custom instantiateWasm bypasses Emscripten's COS loading path.
+make -j4 npm WASM_CUSTOM_INSTANTIATE=0 \
+  'emcc.flags.bundler=-sCROSS_ORIGIN_STORAGE' \
+  'emcc.flags.esm=-sCROSS_ORIGIN_STORAGE' \
+  "bin.wasm-opt=$WASM_OPT_WRAPPER"
+
+# Extract to staging and verify before replacing the mounted output.
+log "Extracting npm-bundle.zip"
+EXTRACT_DIR="$(mktemp -d /tmp/sqlite-wasm-bundle.XXXXXX)"
+unzip -q npm-bundle.zip -d "$EXTRACT_DIR"
+cd "$EXTRACT_DIR/jswasm"
+
+log "Updating and verifying the COS hash after Wasm post-processing"
+python3 - <<'PY'
+import hashlib
+from pathlib import Path
+import re
+
+wasm = Path('sqlite3.wasm')
+loaders = [Path('sqlite3-bundler-friendly.mjs'), Path('sqlite3.mjs')]
+digest = hashlib.sha256(wasm.read_bytes()).hexdigest()
+pattern = re.compile(
+    r"(Module\['wasmHash'\]\s*=\s*\{\s*algorithm:\s*'SHA-256',\s*value:\s*')"
+    r"[0-9a-f]{64}('\s*\})"
+)
+for loader in loaders:
+    source, count = pattern.subn(lambda match: match[1] + digest + match[2], loader.read_text())
+    if count != 1:
+        raise SystemExit(f'Expected exactly one COS hash in {loader}, found {count}')
+    if "Module['instantiateWasm'] =" in source:
+        raise SystemExit(f'Custom instantiator would bypass COS in {loader}')
+    loader.write_text(source)
+node_source = Path('sqlite3-node.mjs').read_text()
+if 'crossOriginStorage' in node_source or 'wasmHash' in node_source:
+    raise SystemExit('Node loader must not include COS')
+
+# Verify the files we will actually publish, after SQLite's Wasm post-processing.
+for loader in loaders:
+    match = pattern.search(loader.read_text())
+    if not match or digest not in match[0]:
+        raise SystemExit(f'Final COS hash verification failed in {loader}')
+print(f'COS SHA-256: {digest}')
+PY
+
+log "Publishing verified artifacts to /src/bin"
+find /src/bin -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
+cp -a "$EXTRACT_DIR/jswasm/." /src/bin/
 log "Build complete. Extracted contents are in /src/bin"
-exec /bin/ls -la /src/bin
+ls -la /src/bin