diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6be939836..91fa63b84 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -1,7 +1,7 @@ { "$schema": "https://anthropic.com/claude-code/marketplace.schema.json", "name": "diffusers-workflow", - "description": "Skills for driving a diffusers-workflow server from Claude Code: a composition skill per model family, and one for cutting a series from them.", + "description": "Skills for driving a diffusers-workflow server from Claude Code: a composition skill per model family, one for cutting a series from them, one camera-and-look vocabulary compiled per family, and storyboards with a timed animatic.", "owner": { "name": "Don Kackman" }, @@ -9,7 +9,7 @@ { "name": "dw", "source": "./plugins/dw", - "description": "Compose MiniMax H3 video, MiniMax Music 3 and LTX-2.5 workflows over a dw MCP server, and cut a multi-episode series from them: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides." + "description": "Compose MiniMax H3 video, MiniMax Music 3, LTX-2.5, Kandinsky 6.0 and FLUX 3 Klein workflows over a dw MCP server, cut a multi-episode series from them, direct camera and look in each family's caption format, and storyboard a scene before it is filmed: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides." } ] } diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..1fe46d85f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +# The image builds from a clean tree: no venv, no build products, no outputs +.git +venv +activate +.env +.env.* +**/__pycache__ +**/*.pyc +*.egg-info +build +dist +outputs +log +tmp* +.superpowers +.remember +.claude +docker/.env +ui/node_modules +ui/dist +ui/test-results +ui/playwright-report +dw/server/ui +dw/docs diff --git a/CLAUDE.md b/CLAUDE.md index e3ad6fd70..950bceaac 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,13 +72,13 @@ The guides in `docs/`, by topic: `.claude-plugin/marketplace.json` publishes the `dw` plugin in `plugins/dw/`: one composition skill per model family (`minimax-h3`, `minimax-music3`, `ltx-2.5`, -`kandinsky-6`, `flux3-klein`) that chooses a template for a request's shape and -states the family's hard rules, plus -cross-cutting composition skills (`script-to-video`, `series-episodes`) - shapes above -the families that orchestrate the decision trees and cast consistency across multiple -generations. Every skill the directory holds is named in `plugins/dw/README.md` and here, -pinned by `tests/test_plugin_skills.py`; that README says how the rest is pinned and -versioned. +`kandinsky-6`, `flux3-klein`) that chooses a template for a request's shape and states the +family's hard rules, plus cross-cutting composition skills (`script-to-video`, +`series-episodes`, `cinematography`, `storyboard`) - shapes above the families that +orchestrate the decision trees, cast consistency across multiple generations, one +camera-and-look vocabulary compiled per family, and boards and a timed animatic before +video. Every skill the directory holds is named in `plugins/dw/README.md` and here, pinned +by `tests/test_plugin_skills.py`; that README says how the rest is pinned and versioned. ## Security Rules diff --git a/README.md b/README.md index 5b7e950e9..63fe8cf84 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,7 @@ frame interpolation and more. - [MCP Server](docs/MCP.md) — The agent tool surface (Claude Code, Claude Desktop) - [Server & Web UI](docs/SERVER.md) — The web UI, jobs API, and introspection service - [Remote GPU server](docs/REMOTE.md) — Using the server, UI and MCP from another machine +- [Docker](docs/DOCKER.md) — The server in a container on a Linux NVIDIA box, sharing data with a native install - [Workspaces](docs/WORKSPACES.md) — Where your content lives, run directories, and several workspaces on one server - [Workflow Guide](docs/WORKFLOW_GUIDE.md) — JSON structure, variables, steps, data flow - [Quantization](docs/QUANTIZATION.md) — BitsAndBytes, TorchAO, GGUF, SDNQ diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..4c16770d5 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,84 @@ +# syntax=docker/dockerfile:1 +# diffusers-workflow server image (Linux, NVIDIA). See docs/DOCKER.md. +# +# docker compose -f docker/compose.yaml build +# +# The build context is the repository root. Torch and every dependency are +# baked in; diffusers is baked too, as a fallback - the copy the server uses +# lives on the /data/env volume (docker/entrypoint.py), so it can track main. + +ARG PYTHON_VERSION=3.13 + +# --- the wheel: scripts/build_dist.sh, with the SPA built by npm ----------- +# Node's version follows ui/.nvmrc (tests/test_docker.py pins the two) +FROM node:24.20.0-trixie-slim AS node + +FROM python:${PYTHON_VERSION}-slim-trixie AS wheel +COPY --from=node /usr/local/bin/node /usr/local/bin/node +COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules +RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm +WORKDIR /src +COPY ui/package.json ui/package-lock.json ui/ +RUN cd ui && npm ci --no-audit --no-fund +COPY . . +RUN bash scripts/build_dist.sh + +# --- the runtime ---------------------------------------------------------- +FROM python:${PYTHON_VERSION}-slim-trixie + +# cu130 needs an NVIDIA driver >= 580 and a Turing or newer card. A CUDA 12 +# image for an older driver: --build-arg TORCH_INDEX=https://pypi.org/simple +ARG TORCH_INDEX=https://download.pytorch.org/whl/cu130 +# The baked (fallback) diffusers: a branch, tag or commit +ARG DIFFUSERS_REF=main + +# git for pip's git+ installs; a compiler for Inductor and Triton, which +# build kernels at run time; tini as PID 1, so signals reach dw.serve and +# its worker processes are reaped +RUN apt-get update \ + && apt-get install -y --no-install-recommends git build-essential tini \ + && rm -rf /var/lib/apt/lists/* + +ENV PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONUNBUFFERED=1 + +# install.sh's order: torch first, so the resolver below sees it satisfied; +# then dw; then diffusers from git, forced without its dependencies (forcing +# those would reinstall torch), and a plain install to add any it needs +RUN pip install torch torchvision torchaudio --index-url "${TORCH_INDEX}" +COPY --from=wheel /src/dist/*.whl /tmp/wheels/ +RUN pip install "$(ls /tmp/wheels/*.whl)[server,mcp]" && rm -rf /tmp/wheels +RUN pip install --force-reinstall --no-deps \ + "git+https://github.com/huggingface/diffusers@${DIFFUSERS_REF}" \ + && pip install diffusers + +# The catalog the server offers beside the workspace's own (--examples-dir): +# the repo's workflows/, with the prompts/ and loras/ they reference +COPY workflows /opt/dw/examples/workflows +COPY prompts /opt/dw/examples/prompts +COPY loras /opt/dw/examples/loras + +COPY docker/entrypoint.py /opt/dw/entrypoint.py +RUN python /opt/dw/entrypoint.py record-image + +# Defaults for a container with one /data volume. docker/compose.yaml +# replaces the shared ones with host paths, mounted at the same path. +ENV DW_ENV_DIR=/data/env \ + HOME=/data/env/home \ + DW_WORKSPACE=/data/workspace \ + HF_HOME=/data/hf \ + TORCHINDUCTOR_CACHE_DIR=/data/cache/torchinductor \ + TRITON_CACHE_DIR=/data/cache/triton \ + DIFFUSERS_HELPER_ROOT=/data/settings + +# Any uid may run it (compose passes the host user's); nothing below /data +# is created at build time, so the volume's owner owns everything in it, and +# without a volume /data is world-writable (sticky) for whichever uid it is +RUN mkdir -m 1777 /data +WORKDIR /data +EXPOSE 8765 +HEALTHCHECK --interval=30s --timeout=15s --start-period=10m --retries=3 \ + CMD ["python", "/opt/dw/entrypoint.py", "healthcheck"] +ENTRYPOINT ["tini", "--", "python", "/opt/dw/entrypoint.py", "serve"] +CMD [] diff --git a/docker/compose.yaml b/docker/compose.yaml new file mode 100644 index 000000000..742571854 --- /dev/null +++ b/docker/compose.yaml @@ -0,0 +1,63 @@ +# diffusers-workflow in a container. See docs/DOCKER.md. +# +# bash docker/init-env.sh # writes docker/.env from this host's layout +# docker compose -f docker/compose.yaml up -d --build +# docker compose -f docker/compose.yaml logs -f # the login URL is here +# +# Every shared directory is mounted at the same absolute path it has on the +# host, and the variable dw reads points at it. Run manifests, settings and +# quantized-cache entries record absolute paths, so a container and a native +# dw.serve (systemd, install.sh) can share the workspace and every cache and +# read each other's records. Only DW_ENV_DIR - the diffusers overlay - is the +# container's own. +services: + dw: + build: + context: .. + dockerfile: docker/Dockerfile + image: diffusers-workflow:local + # The host user, so files written to the shared directories are theirs + user: "${DW_UID:?run docker/init-env.sh}:${DW_GID:?run docker/init-env.sh}" + ports: + - "${DW_BIND:-127.0.0.1}:${DW_HOST_PORT:-8765}:8765" + volumes: + - ${DW_ENV_DIR:?run docker/init-env.sh}:/data/env + - ${DW_WORKSPACE:?run docker/init-env.sh}:${DW_WORKSPACE} + - ${HF_HOME:?run docker/init-env.sh}:${HF_HOME} + - ${TORCHINDUCTOR_CACHE_DIR:?run docker/init-env.sh}:${TORCHINDUCTOR_CACHE_DIR} + - ${TRITON_CACHE_DIR:?run docker/init-env.sh}:${TRITON_CACHE_DIR} + - ${DIFFUSERS_HELPER_ROOT:?run docker/init-env.sh}:${DIFFUSERS_HELPER_ROOT} + env_file: + # The checkout's own .env (DW_QUANTIZED_CACHE=1 and the like), the + # file a native dw.serve loads, so both run with the same switches; + # then docker/.env (DW_API_TOKEN, HF_TOKEN, DW_DIFFUSERS_REF, ...), + # which wins where both set a variable + - path: ../.env + required: false + - path: .env + # Only the mounted paths, which must match the mounts above. Anything + # else set here would override both env files, empty or not. + environment: + # docker/.env holds the overlay's host path; inside, it is /data/env + DW_ENV_DIR: /data/env + DW_WORKSPACE: ${DW_WORKSPACE} + HF_HOME: ${HF_HOME} + TORCHINDUCTOR_CACHE_DIR: ${TORCHINDUCTOR_CACHE_DIR} + TRITON_CACHE_DIR: ${TRITON_CACHE_DIR} + DIFFUSERS_HELPER_ROOT: ${DIFFUSERS_HELPER_ROOT} + # dw.serve's arguments; the entrypoint adds --host, --port and --examples-dir + command: ${DW_SERVE_ARGS:---mcp} + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + # Worker processes hand tensors through shared memory + shm_size: "16gb" + ulimits: + nofile: 65536 + # Model loads are slow to interrupt; give a job time to unwind + stop_grace_period: 60s + restart: unless-stopped diff --git a/docker/entrypoint.py b/docker/entrypoint.py new file mode 100644 index 000000000..f92a8b279 --- /dev/null +++ b/docker/entrypoint.py @@ -0,0 +1,420 @@ +"""The container's entrypoint: the diffusers overlay, the API token, then dw.serve. + +The image is immutable and carries torch, the dw wheel and every pinned +dependency, plus a *baked* diffusers - a git commit chosen when the image was +built. diffusers is the one package that tracks GitHub main, so the one that +changes lives on a volume: `DW_ENV_DIR/venv`, a venv made with +`--system-site-packages` over the image's interpreter. It sees every image +package, and its own site-packages comes first, so a diffusers installed there +shadows the baked one. pip run from it leaves the image's copy alone and adds +only the dependencies a new commit needs that the image does not already +satisfy. The server runs under the overlay's interpreter, so the Models page's +updater (`dw/server/updater.py`, which pips into `sys.executable`) lands its +installs on the volume too. + +`DW_ENV_DIR/overlay.json` records which image the overlay was built against +(the image's build id, a hash of its `pip freeze`) and which diffusers it holds. +A restart never moves diffusers. A new image rebuilds the overlay - an old one +can hold dependency versions that would now shadow newer ones the image ships - +and reinstalls the diffusers the old overlay held, not HEAD. When the overlay +does not import (a broken HEAD, no network on the first start), the server runs +on the image's interpreter and the baked diffusers instead, so it still comes +up and the Models page can pin a working commit. + +`DW_DIFFUSERS_REF` picks what a new overlay installs: `head` (the default), +a commit hash (which the overlay is then held to), or `image` (no overlay). + +Subcommands: `serve` (the default; anything after it is passed to dw.serve), +`update-diffusers [--commit SHA | --revert]`, `healthcheck`, `record-image` +(run once at image build) and `probe` (run under the overlay's interpreter). +See docs/DOCKER.md. +""" + +import argparse +import hashlib +import json +import os +import secrets +import shutil +import subprocess +import sys +import urllib.request +from pathlib import Path + +from dw.security import InvalidInputError, sanitize_command_args, validate_commit_hash +from dw.server.updater import build_pip_commands + +BASE_PYTHON = sys.executable +IMAGE_INFO = Path(os.environ.get("DW_IMAGE_INFO", "/opt/dw/image.json")) +DEFAULT_ENV_DIR = "/data/env" +EXAMPLES_DIR = "/opt/dw/examples/workflows" +CONTAINER_PORT = 8765 +TOKEN_FILE_NAME = "api-token" +# pip builds diffusers from a git clone; a cold one is a few minutes +INSTALL_TIMEOUT_SECONDS = 1800 +PROBE_TIMEOUT_SECONDS = 300 + + +def log(message): + print(f"[dw-container] {message}", file=sys.stderr, flush=True) + + +def env_dir(environ): + return Path(environ.get("DW_ENV_DIR") or DEFAULT_ENV_DIR) + + +def overlay_python(environ): + return env_dir(environ) / "venv" / "bin" / "python" + + +def marker_path(environ): + return env_dir(environ) / "overlay.json" + + +def parse_ref(value): + """`DW_DIFFUSERS_REF` as ("head", None), ("image", None) or ("commit", sha).""" + value = (value or "head").strip().lower() + if value in ("head", "image"): + return value, None + try: + return "commit", validate_commit_hash(value) + except InvalidInputError: + raise SystemExit( + f"DW_DIFFUSERS_REF={value!r}: expected 'head', 'image' or a commit hash" + ) + + +def read_json(path): + try: + return json.loads(Path(path).read_text()) + except (OSError, ValueError): + return None + + +def write_json(path, data): + path = Path(path) + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(json.dumps(data, indent=2) + "\n") + os.replace(tmp, path) + + +def run_command(command, run=subprocess.run, timeout=INSTALL_TIMEOUT_SECONDS): + """Run one fixed argument list; True when it exits 0. Output tails are + logged on failure, not on success - pip's progress is noise in `docker logs`.""" + try: + completed = run( + sanitize_command_args(command), + capture_output=True, + text=True, + timeout=timeout, + ) + except (OSError, subprocess.TimeoutExpired) as e: + log(f"{command[0]} failed to run: {e}") + return False + if completed.returncode != 0: + output = ((completed.stdout or "") + (completed.stderr or "")).strip() + log(f"exit {completed.returncode}: {' '.join(command[:6])}") + for line in output.splitlines()[-20:]: + log(f" {line}") + return False + return True + + +def probe(python, run=subprocess.run): + """What diffusers `python` imports ({"version", "commit"}), or None when + dw's server or diffusers fails to import under it.""" + try: + completed = run( + sanitize_command_args( + [str(python), str(Path(__file__).resolve()), "probe"] + ), + capture_output=True, + text=True, + timeout=PROBE_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired) as e: + log(f"overlay probe failed to run: {e}") + return None + if completed.returncode != 0: + for line in (completed.stderr or "").strip().splitlines()[-10:]: + log(f" {line}") + return None + try: + return json.loads(completed.stdout.strip().splitlines()[-1]) + except (ValueError, IndexError): + return None + + +def install_target(kind, commit, marker, previous): + """What a (re)built overlay installs, as build_pip_commands' keywords. + + A pinned ref wins. Otherwise the diffusers the old overlay held, so an image + upgrade never moves it: its commit, or a revert when it was a release (the + Models page's revert). Nothing recorded means a new overlay, so HEAD.""" + if kind == "commit": + return {"commit": commit} + held = previous or marker or {} + held_commit = held.get("commit") or held.get("diffusers_commit") + if held_commit: + return {"commit": held_commit} + held_version = held.get("version") or held.get("diffusers_version") + if held_version and "dev" not in held_version: + return {"revert": True} + return {} + + +def build_overlay(environ, target, run=subprocess.run): + """A fresh overlay venv with the diffusers `target` names. True on success.""" + venv = env_dir(environ) / "venv" + if venv.exists(): + shutil.rmtree(venv) + if not run_command( + [BASE_PYTHON, "-m", "venv", "--system-site-packages", str(venv)], run + ): + return False + what = "release floor" if target.get("revert") else target.get("commit") or "HEAD" + log(f"installing diffusers ({what}) into the overlay at {venv}") + for command in build_pip_commands(python=str(overlay_python(environ)), **target): + if not run_command(command, run): + return False + return True + + +def prepare_overlay(environ, image, run=subprocess.run): + """The interpreter dw.serve runs under: the overlay's, or the image's when + `DW_DIFFUSERS_REF=image` or the overlay will not import.""" + kind, commit = parse_ref(environ.get("DW_DIFFUSERS_REF")) + if kind == "image": + log( + f"DW_DIFFUSERS_REF=image: using the baked diffusers {image.get('diffusers')}" + ) + return BASE_PYTHON + + env_dir(environ).mkdir(parents=True, exist_ok=True) + marker = read_json(marker_path(environ)) or {} + python = overlay_python(environ) + build_id = image.get("build_id") + + current = None + if python.exists() and marker.get("build_id") == build_id: + current = probe(python, run) + if ( + current + and kind == "commit" + and not (current.get("commit") or "").startswith(commit) + ): + log( + f"overlay holds {current.get('commit')}, DW_DIFFUSERS_REF pins {commit}" + ) + current = None + + if current is None: + # The old overlay's own answer beats the marker: the Models page may have + # moved it since the marker was written. It may not run at all once the + # image's Python changed, so the marker is the fallback. + previous = probe(python, run) if python.exists() else None + target = install_target(kind, commit, marker.get("diffusers"), previous) + reason = ( + "new" + if not marker + else "image changed" + if marker.get("build_id") != build_id + else "rebuild" + ) + log(f"building the diffusers overlay ({reason})") + ok = build_overlay(environ, target, run) + current = probe(python, run) if ok else None + if current is None: + # Keep what it was meant to hold, so the next start retries it + write_json( + marker_path(environ), + {"build_id": None, "diffusers": marker.get("diffusers") or previous}, + ) + log( + "the overlay did not install or import - serving the baked diffusers " + f"{image.get('diffusers')}; an update from the Models page now lands in the " + "container, not the volume, and is lost when it is re-created" + ) + return BASE_PYTHON + + write_json( + marker_path(environ), + {"build_id": build_id, "python": image.get("python"), "diffusers": current}, + ) + log( + f"diffusers {current.get('version')} ({current.get('commit') or 'release'}) from the overlay" + ) + return str(python) + + +def token_path(environ): + """Beside settings.json, in the directory the settings mount shares.""" + settings = environ.get("DIFFUSERS_HELPER_ROOT") or Path.home() / ".diffusers_helper" + return Path(settings) / TOKEN_FILE_NAME + + +def saved_token(environ): + try: + return token_path(environ).read_text().strip() + except OSError: + return "" + + +def resolve_token(environ): + """The API token: DW_API_TOKEN, else the one saved in the settings + directory, else a new one saved there. None only with DW_INSECURE_NO_TOKEN=1. + + The server binds every interface inside the container and cannot see + whether the host published its port on localhost only, so it gets a token + unless told otherwise - Jupyter's pattern, but kept across restarts so an + MCP client configured with it keeps working.""" + token = (environ.get("DW_API_TOKEN") or "").strip() + if token: + return token + if environ.get("DW_INSECURE_NO_TOKEN") == "1": + log("DW_INSECURE_NO_TOKEN=1: no API token - publish the port on localhost only") + return None + path = token_path(environ) + token = saved_token(environ) + if not token: + path.parent.mkdir(parents=True, exist_ok=True) + token = secrets.token_hex(32) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as f: + f.write(token + "\n") + log(f"generated an API token and saved it to {path}") + return token + + +def serve_command(python, args): + """dw.serve's argument list: the container's defaults unless `args` names them.""" + command = [python, "-m", "dw.serve"] + if "--host" not in args: + command += ["--host", "0.0.0.0"] + if "--port" not in args: + command += ["--port", str(CONTAINER_PORT)] + if "--examples-dir" not in args and Path(EXAMPLES_DIR).is_dir(): + command += ["--examples-dir", EXAMPLES_DIR] + return command + list(args) + + +def serve(args, environ=None, run=subprocess.run, execve=os.execve): + environ = dict(os.environ if environ is None else environ) + image = read_json(IMAGE_INFO) or {} + home = Path(environ.get("HOME") or env_dir(environ) / "home") + home.mkdir(parents=True, exist_ok=True) + + token = resolve_token(environ) + python = prepare_overlay(environ, image, run) + if token: + # The environment, not --token: argv shows in `ps` on the host + environ["DW_API_TOKEN"] = token + port = environ.get("DW_HOST_PORT") or CONTAINER_PORT + log(f"open http://localhost:{port}/?token={token}") + command = serve_command(python, args) + log(" ".join(command)) + execve(python, sanitize_command_args(command), environ) + + +def update_diffusers(args, environ=None, run=subprocess.run): + """The Models page's update, from `docker exec`: HEAD, a commit, or a revert.""" + parser = argparse.ArgumentParser(prog="update-diffusers") + group = parser.add_mutually_exclusive_group() + group.add_argument("--commit", help="a commit hash to pin (default: main HEAD)") + group.add_argument( + "--revert", action="store_true", help="the pyproject release floor" + ) + options = parser.parse_args(args) + environ = dict(os.environ if environ is None else environ) + python = overlay_python(environ) + if not python.exists(): + log(f"no overlay at {python} - the server is on the baked diffusers") + return 1 + commit = validate_commit_hash(options.commit) if options.commit else None + for command in build_pip_commands( + commit=commit, revert=options.revert, python=str(python) + ): + if not run_command(command, run): + return 1 + current = probe(python, run) + if current is None: + log("the update installed but does not import - pin a commit or --revert") + return 1 + marker = read_json(marker_path(environ)) or {} + marker["diffusers"] = current + write_json(marker_path(environ), marker) + log( + f"diffusers {current.get('version')} ({current.get('commit') or 'release'}) installed" + ) + log("a running job keeps the old import; restart the container to load it") + return 0 + + +def healthcheck(environ=None): + environ = os.environ if environ is None else environ + request = urllib.request.Request(f"http://127.0.0.1:{CONTAINER_PORT}/api/health") + # Never resolve_token: a healthcheck must not mint one + token = (environ.get("DW_API_TOKEN") or "").strip() or saved_token(environ) + if token: + request.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(request, timeout=10) as response: + return 0 if response.status == 200 else 1 + except OSError: + return 1 + + +def record_image(): + """Write IMAGE_INFO at build time: the build id the overlay compares + against, and the diffusers baked in.""" + from dw.server.updater import diffusers_install_info + + freeze = subprocess.run( + sanitize_command_args([BASE_PYTHON, "-m", "pip", "freeze", "--all"]), + capture_output=True, + text=True, + check=True, + ).stdout + python = ".".join(str(part) for part in sys.version_info[:3]) + build_id = hashlib.sha256((python + "\n" + freeze).encode()).hexdigest()[:16] + IMAGE_INFO.parent.mkdir(parents=True, exist_ok=True) + write_json( + IMAGE_INFO, + {"build_id": build_id, "python": python, "diffusers": diffusers_install_info()}, + ) + print(IMAGE_INFO.read_text()) + return 0 + + +def probe_main(): + """Run under the overlay's interpreter: import what dw.serve needs, print + the diffusers it got.""" + import diffusers # noqa: F401 + import dw.server # noqa: F401 + from dw.server.updater import diffusers_install_info + + print(json.dumps(diffusers_install_info())) + return 0 + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + command, rest = ( + (argv[0], argv[1:]) if argv and not argv[0].startswith("-") else ("serve", argv) + ) + if command == "serve": + return serve(rest) + if command == "update-diffusers": + return update_diffusers(rest) + if command == "healthcheck": + return healthcheck() + if command == "record-image": + return record_image() + if command == "probe": + return probe_main() + log(f"unknown command {command!r}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/init-env.sh b/docker/init-env.sh new file mode 100644 index 000000000..1eadcd827 --- /dev/null +++ b/docker/init-env.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Write docker/.env for docker/compose.yaml from this host's native layout: +# the same workspace, Hugging Face cache, compile caches and settings a +# native dw.serve uses, so the two can be switched between (docker/switch.sh). +# Each value can be overridden by exporting it first: +# +# HF_HOME=/mnt/models bash docker/init-env.sh +# +# Refuses to overwrite an existing docker/.env unless --force. +set -euo pipefail +cd "$(dirname "$0")" + +if [ -e .env ] && [ "${1:-}" != "--force" ]; then + echo "docker/.env exists - edit it, or rerun with --force" >&2 + exit 1 +fi + +# The defaults dw resolves natively (dw/__init__.py, dw/disk_caches.py, +# dw/settings.py, dw/workspace.py), absolute. A symlinked directory stays as +# its link path: docker mounts the target there, so the path dw records is +# the one a native run records too. +cache="${XDG_CACHE_HOME:-$HOME/.cache}" +workspace="${DW_WORKSPACE:-$HOME/diffusers-workspace}" +hf_home="${HF_HOME:-$cache/huggingface}" +inductor="${TORCHINDUCTOR_CACHE_DIR:-$cache/dw/torchinductor}" +triton="${TRITON_CACHE_DIR:-${TRITON_HOME:-$HOME}/.triton/cache}" +settings="${DIFFUSERS_HELPER_ROOT:-$HOME/.diffusers_helper}" +env_dir="${DW_ENV_DIR:-$HOME/.local/share/dw-docker/env}" + +# docker would create a missing source as root; create them as this user +mkdir -p "$workspace" "$hf_home" "$inductor" "$triton" "$settings" "$env_dir" + +cat > .env <&2 + exit 2 + ;; +esac diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md index 27152ad84..ca9ab4ccd 100644 --- a/docs/DEPENDENCIES.md +++ b/docs/DEPENDENCIES.md @@ -18,6 +18,12 @@ source ./activate The install scripts handle everything: Python version detection (3.12-3.14), virtual environment creation, PyTorch, diffusers, and all dependencies. +### Docker (Linux, NVIDIA) + +A container image with torch, dw and its dependencies built in, sharing the +workspace and caches with a native install on the same host: +[DOCKER.md](DOCKER.md). + ### Manual Installation If you prefer manual control: diff --git a/docs/DOCKER.md b/docs/DOCKER.md new file mode 100644 index 000000000..d482fde89 --- /dev/null +++ b/docs/DOCKER.md @@ -0,0 +1,134 @@ +# Running dw in Docker + +A container image for a Linux box with an NVIDIA GPU. It holds Python, torch +(CUDA 13), dw and its web UI. You don't need a venv or `install.sh`. The +image is built on the box from this checkout; nothing is published to a +registry yet. + +Docker is for Linux and NVIDIA only. On a Mac, Docker runs a Linux VM with +no access to the GPU (MPS), so use `install.sh` there. + +## What the host needs + +- An NVIDIA driver **580 or newer**, and a Turing or newer card (RTX 20-series + and up). CUDA 13 dropped Maxwell, Pascal and Volta. +- Docker with the compose plugin, and the + [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) + (`nvidia-ctk runtime configure --runtime=docker`, then restart docker). +- Your user in the `docker` group, or every command below under `sudo`. + +Check the GPU reaches a container: +`docker run --rm --gpus all ubuntu nvidia-smi`. + +## Start it + +```bash +bash docker/init-env.sh # writes docker/.env +docker compose -f docker/compose.yaml up -d --build +docker compose -f docker/compose.yaml logs -f # wait for the login URL +``` + +The first build takes a while, because torch alone is several GB. The first +start then installs diffusers from GitHub main into its volume (see below), +which takes a few more minutes before the server answers. The log prints +`open http://localhost:8765/?token=...`. Open that link and the UI saves the +token. + +## Where the data lives: shared with a native install + +`docker/init-env.sh` fills `docker/.env` with the directories a native +dw.serve on the same host already uses: + +| variable | default | holds | +|---|---|---| +| `DW_WORKSPACE` | `~/diffusers-workspace` | workflows, prompts, assets, outputs | +| `HF_HOME` | `~/.cache/huggingface` | model weights, and the quantized cache (`dw-quantized/`) | +| `TORCHINDUCTOR_CACHE_DIR` | `~/.cache/dw/torchinductor` | `torch.compile` kernels | +| `TRITON_CACHE_DIR` | `~/.triton/cache` | Triton kernels | +| `DIFFUSERS_HELPER_ROOT` | `~/.diffusers_helper` | `settings.json`, the log, the container's API token | +| `DW_ENV_DIR` | `~/.local/share/dw-docker/env` | the diffusers overlay (container only) | + +Export any of them before running the script to change it. Each one is +mounted **at the same absolute path** inside the container, and the +container runs as your uid. Run manifests, settings and cache entries +record absolute paths, so a job the container ran opens in a native server +and the other way round. Models download once, and kernels compiled by one +are reused by the other. A symlinked directory, such as an `HF_HOME` that +links to another disk, works: Docker mounts the target at the link's path. + +The checkout's own `.env`, which a native dw.serve loads (for example +`DW_QUANTIZED_CACHE=1`), is passed to the container too, so both run with +the same switches. A path in it, such as `DW_QUANTIZED_CACHE_DIR`, has to be +one of the mounted directories above. + +### Switching between Docker and systemd + +Both use the same port, GPUs and data, so only one should run at a time: + +```bash +bash docker/switch.sh docker # stop the dw-serve user unit, start the container +bash docker/switch.sh native # stop the container, start the unit +bash docker/switch.sh status +``` + +The script assumes the user unit from `scripts/dw-serve.service`. For the +system unit in `contrib/systemd/`, set `DW_SYSTEMCTL="sudo systemctl"`. +To keep the token your MCP clients already use, set `DW_API_TOKEN` in +`docker/.env` to the one the unit passes. + +The examples catalog is one difference: the container offers the copy of +the checkout's `workflows/` that was built into the image, while a native +server offers the live checkout. + +## diffusers tracks main, on a volume + +Everything in the image is fixed when it's built, except diffusers. New +model pipelines land on diffusers' GitHub main before a release, so the +diffusers the server uses lives in `DW_ENV_DIR`. That directory holds a +small venv layered over the image's Python (`--system-site-packages`), with +only diffusers and any dependency a new commit needs that the image lacks. + +- **The first start** installs main HEAD (or `DW_DIFFUSERS_REF`; see below). +- **A restart changes nothing.** diffusers moves only when you ask it to. +- **Update** from the web UI's Models page, as on a native install, or: + `docker compose -f docker/compose.yaml exec dw python /opt/dw/entrypoint.py update-diffusers [--commit SHA | --revert]`, + then restart the container. +- **A rebuilt image** (a new dw) rebuilds the overlay and reinstalls the + diffusers commit you had, not HEAD. +- **If the overlay doesn't import** (a broken HEAD, or no network on the + first start), the server starts on the copy of diffusers built into the + image, and says so in the log. Pin a working commit from the Models page, + then restart. An update made in this state goes into the container + rather than the volume, so it's lost when the container is re-created. + +`DW_DIFFUSERS_REF` in `docker/.env`: `head` (the default), a commit hash +(the overlay is held to it), or `image` (no overlay; use the copy built +into the image). + +## The API token + +Inside the container the server listens on every interface, and it can't +tell how the port is published. So it always gets a token: `DW_API_TOKEN` +if you set one, otherwise one generated on first start and saved to +`DIFFUSERS_HELPER_ROOT/api-token`. Its login URL is logged on every start. +`DW_INSECURE_NO_TOKEN=1` turns this off; only do that with the port +published on localhost (`DW_BIND=127.0.0.1`, the default). + +To reach the server from your LAN, set `DW_BIND=0.0.0.0`. Clients connect +the way [REMOTE.md](REMOTE.md) describes. + +## Options + +`DW_SERVE_ARGS` holds dw.serve's own arguments (default `--mcp`), for example +`--mcp --devices cuda:0,cuda:1` for one worker per card. The entrypoint adds +`--host 0.0.0.0`, `--port 8765` and the examples catalog unless you give +them. `HF_TOKEN` is passed through for gated models. + +Build arguments (`docker compose build --build-arg ...`): `TORCH_INDEX` +(default the cu130 index; `https://pypi.org/simple` gives the CUDA 12 wheel +for an older driver) and `DIFFUSERS_REF` (the diffusers built into the +image; default `main`). + +flash_attn is not in the image, because it needs the CUDA toolkit to +build. The Hub kernels and comfy-kitchen's prebuilt kernels work as they do +natively. diff --git a/docs/REMOTE.md b/docs/REMOTE.md index 05ff4aee0..a4833f178 100644 --- a/docs/REMOTE.md +++ b/docs/REMOTE.md @@ -5,7 +5,9 @@ your laptop use it over the network. Nothing is installed on the laptop. ## On the GPU box -1. Install as usual: `git clone …`, `bash ./install.sh`. +1. Install as usual: `git clone …`, `bash ./install.sh` - or run the + container instead ([DOCKER.md](DOCKER.md), `DW_BIND=0.0.0.0`), which + makes the token for you; skip to the clients. 2. Make a token: `openssl rand -hex 32`. This one string is the only thing between "on your network" and "can run workflows on your GPU" - keep it long and random. diff --git a/docs/TASKS.md b/docs/TASKS.md index 96b593237..37dd665a0 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -526,6 +526,7 @@ add grain last so it is not itself sharpened: | `get_first_frame` | Extract first video frame | | | `get_last_frame` | Extract last video frame | | | `get_frame` | Extract frame at index | `frame_index` | +| `stills_to_video` | Hold storyboard stills for a duration each, as one silent clip | `images`, `durations`, `fps` - see [stills_to_video](#stills_to_video) | | `window_video` | One overlapping, fixed-length window of a long video, with its audio | `index`, `num_frames`, `overlap`, `fps` - see [window_video](#window_video) | | `join_windows` | Blend processed windows back into one video the source's length, with the source's audio | `videos`, `source`, `num_frames`, `overlap`, `curve`, `fps` - see [join_windows](#join_windows) | | `fit_to_model` | Fit a video into a model's working size and exact frame count; returns `{video, fit}` | `width`, `height`, `num_frames`, `mode` (`letterbox`, `stretch`, `crop`), `downscale` - see [fit_to_model and restore_to_source](#fit_to_model-and-restore_to_source) | @@ -535,6 +536,50 @@ The frame commands accept videos in any shape a result carries them: PIL frame lists, numpy or torch frame arrays, and audio+video pairs (LTX-2, MiniMax H3). The extracted frame is always a PIL image. +### stills_to_video + +Hold each still on screen for a set time and get one silent clip back - an +animatic of a storyboard. The result joins with `concat_videos` and takes a +score from `pair_audio` or the `assemble-and-score` template like any shot. + +```json +{ + "name": "animatic", + "task": { + "command": "stills_to_video", + "arguments": { + "images": [ + "previous_result:frame_1", + "previous_result:frame_2", + "previous_result:frame_3" + ], + "durations": [1, 2, 0.5], + "fps": 24 + } + }, + "result": { "content_type": "video/mp4", "save": true } +} +``` + +| Argument | Required | Description | +| -------- | -------- | ----------- | +| `images` | Yes | The stills in order - images, arrays, or `{"media_type": "image", "location": "asset:x.png"}`. All the same size | +| `durations` | Yes | Seconds per still: one number for all, or one per still | +| `fps` | No | Frame rate, default 24, at most 120 | + +Each still gets `max(1, round(duration * fps))` frames, so a duration shorter +than a frame still shows for one. The clip is at most 600 seconds and 36000 +frames in total; those cap its length, not its memory. The frames are held as +uint8 pixels, so the command also refuses, before allocating, a clip over +4 GiB (frames x width x height x 3), naming `durations`, `fps` and the image +size. +Stills of different sizes are refused, naming both sizes. The video has no +audio track; its `shots` record where each still starts. +`validate_workflow` catches a non-positive duration, an `fps` out of range, a +duration count that does not match a literal image list, and a total over 600 +seconds or 36000 frames (the 4 GiB limit depends on the image size, so it is +checked at run time). + ### concat_videos Concatenate videos - and the audio generated with them - into one video. The @@ -2329,6 +2374,42 @@ whole file: | `video_seconds` / `audio_seconds` | Each stream's own duration | | `length_delta_ms` | `audio_seconds` minus `video_seconds` | +### analyze_face_drift + +Whether a recurring character still looks like their cast portrait. It +answers the shape of the probes above but is not one: it measures faces, not +the soundtrack, and its two rules live in the command rather than in the +table below. Needs the SFace and YuNet weights from `opencv/face_recognition_sface` +and `opencv/face_detection_yunet`, which `validate_workflow` lists under +`downloads_required`. + +```json +{ + "command": "analyze_face_drift", + "arguments": { + "video": "output:/latest/final/cut.mp4", + "reference": "asset:cast/ada.png" + } +} +``` + +| Argument | Meaning | +| -------- | ------- | +| `video` | A stored video, or the video an earlier step returned | +| `reference` | The face to hold. With several faces the largest is used and `notes` says so; with none the step is refused | +| `shots` | Shot records, as the probes take them; else the video's own, else the run manifest, else the whole file is one shot | +| `threshold` | Median cosine similarity below which a shot raises `face_drift`, from -1 to 1. Default 0.363, OpenCV's published SFace threshold | +| `samples_per_shot` | Frames read per shot, evenly spaced, 1 to 64. Default 8 | + +Each shot is a `measurements[]` entry: `shot`, `frames_sampled`, +`faces_found` (sampled frames with a face), `similarity_median` and +`similarity_min` - over those frames, the best cosine match among the faces +in each frame. Both are null when no sampled frame had a face. Findings: +`face_drift` (warn) when the median is below `threshold`, and `no_face` +(info) for a shot where no sampled frame had a face. Like the probes, they +are places to look, not verdicts: a turned head or a mask also lowers the +score, so look at the shot before regenerating it. + ### Rules Each rule names the probe and field it reads, how the value is compared to diff --git a/docs/proposals/audits/2026-11-higgsfield/README.md b/docs/proposals/audits/2026-11-higgsfield/README.md new file mode 100644 index 000000000..58a3a0e56 --- /dev/null +++ b/docs/proposals/audits/2026-11-higgsfield/README.md @@ -0,0 +1,284 @@ +# Compile Higgsfield's presets into dw's skills + +dw should adopt Higgsfield's **control vocabulary and planning shape**, not its model zoo. Most of what makes Higgsfield feel directable is a closed set of named choices: camera move, lens, framing, light, genre, era, tempo and grade. Those choices compile into prompt text before a model ever runs, and an agent plans shots from a script on top of them. dw can build that today as catalog and skill text over H3, LTX-2.5, Kandinsky 6 and FLUX 3 Klein, because every family's caption format already asks for one camera statement. The open-source Higgsfield clones from Comfy's challenge converged on the same architecture dw already has: declarative graph JSON, a server, a UI, an agent surface and a provenance record per take. Their ideas port cleanly even though their ComfyUI graphs do not. The features dw cannot reach on its current families are re-shooting a clip from a new angle, swapping an actor, dubbing existing footage, relighting video, and spectacle effects. Re-shooting and angle stills come within reach of an add-on (the LTX-2.5 Layout-To-Render IC-LoRA, or Qwen-Image-Edit-2511 with a multi-angle LoRA). Actor swap and most effect presets need Wan 2.x as a new base family, which collides with the 24 GB rule. The highest-leverage single move is a camera-and-look vocabulary in the plugin skills plus a persisted shot document, measured before it is promoted. That one move turns `script-to-video` from "write a good paragraph" into "fill a shot card", and it is the shape every serious clone and Higgsfield itself landed on. + +## Higgsfield is a preset layer over other people's models + +Higgsfield aggregates **30+ third-party models** (Seedance, Kling, Veo, Sora 2, Wan, Nano Banana, Hailuo) behind single-purpose "apps", plus its own Soul image model and Soul ID identity training ([homepage](https://higgsfield.ai/); [help center](https://higgsfield.ai/creator-hub/help-center/getting-started/what-is-higgsfield)). It grew virally in 2025 on one-click camera presets such as Crash Zoom, Bullet Time and Super Dolly, applied to a single image to make a roughly 5 s clip ([X, Mar 2025](https://x.com/higgsfield_ai/status/1906748661322711305); [memons cheat sheet](https://memons.ai/higgsfield-prompt-cheat-sheet)). In late 2026 its flagship surfaces are: + +- **Cinema Studio 4.0**, a director's panel. +- **Genjutsu**, video-to-video recast and object swap, launched 1 Sept 2026. +- **Supercomputer**, a chat agent that plans and routes jobs. +- **Popcorn**, an up-to-8-frame storyboard. +- **Canvas**, a node graph. +- **Marketing Studio**, for ads. + +A long tail of tools sits beside these: Angles and Shots (re-angled stills), Relight, Upscale, Lipsync Studio, Effects, and Draw-to-Video. + +Cinema Studio shows the product philosophy most clearly. Its build blog says: **"Instead of describing a look in words, you build the frame the way a cinematographer would."** Shot settings and metadata are saved with each still, so a still used as a video start frame carries its look forward ([How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio)). + +The control set grew version by version: + +- **v1:** camera body, lens and focal length. +- **v1.5:** aperture. +- **v3.0:** genres (General, Action, Horror, Comedy, Noir, Drama, Epic) and eight speed ramps (Linear, Auto, Flash In, Flash Out, Slow-mo, Bullet Time, Impact, Ramp Up). Audio, meaning SFX, speech and music, arrived in the same pass. +- **v3.5:** per-shot camera, lens, focal length and aperture, plus global genre, style, lighting, 8 palettes and 10 "MoveSet" styles. This version added the **AI Director**, a Claude chat panel that breaks a script into shots with camera parameters pre-filled. +- **v4.0:** Era (60s–2020s), Tempo (Single Shot to Chaotic), abstract camera bodies ("35mm Film", "8mm Film", "DV Camcorder"), lens families, 50+ grade templates, an emotion slider, up to 50 references, Forward/Backward Extend to 30 s, and regional Edit Video ([Cinema Studio help](https://higgsfield.ai/creator-hub/help-center/tools/how-do-i-use-cinema-studio)). + +The trajectory matters for dw. Higgsfield moved **away from literal hardware names** (ARRI, RED, Cooke) toward abstract looks once multi-shot generation made per-shot rigs less useful. Its blog says camera control "moved up a level to genres" ([How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio)). Cinema Studio 4.0 runs on Seedance 2.5, so these controls are almost certainly prompt composition plus curated per-model templates rather than model conditioning. That inference comes from the research notes, not from a Higgsfield statement, and it is exactly the layer dw keeps in skills. + +Consistency comes in three layers: + +- **Trained identity:** Soul ID, from at least 20 photos in about 3 minutes. +- **Named reusable "Elements":** characters, locations and props, injected as numbered references ("image one") into third-party models. +- **Project-locked style:** lighting, optics and tempo held across the project, plus within-sequence chaining in Popcorn, where each clip's last frame starts the next ([Popcorn help](https://higgsfield.ai/creator-hub/help-center/ai-models/how-do-i-use-popcorn)). + +Supercomputer, the agent spun out of AI Director, plans a job, routes each step to a model, shows per-step credits live and re-runs only the affected steps on revision. Higgsfield also ships an MCP server for Claude ([Supercomputer help](https://higgsfield.ai/creator-hub/help-center/tools-and-workflows/how-do-i-use-supercomputer)). No usage numbers exist for any app, so "most popular" can only be argued from the viral clips, which were the camera presets and Effects. + +### Feature-by-feature: where dw already stands + +Status is judged against dw 0.17.0 as read from the repo; nothing was GPU-tested for this report. + +| Higgsfield feature | dw state | What dw has, or the gap | +|---|---|---| +| Text/image to shot, first+last frame, extend | **Has** | H3 (22 templates), LTX-2.5 (19), Kandinsky 6 (3) ([minimax-h3 skill](../../../../plugins/dw/skills/minimax-h3/SKILL.md); [ltx-2.5 skill](../../../../plugins/dw/skills/ltx-2.5/SKILL.md)) | +| Multi-shot with cuts in one generation | **Has** | H3 `storyboard`, `dialogue-short`; LTX "a hard cut to" 2-4 shots per paragraph | +| Native dialogue/SFX/music, voice | **Has** | H3 dialogue, Music 3, MMAudio SFX, TTS, voice clone ([TASKS.md](../../../../docs/TASKS.md)) | +| Elements / Soul Cast (named reusable cast, locations, props) | **Has (as procedure)** | `keep_output(..., shared=True)` cast assets + `prompt:/` ([series-episodes skill](../../../../plugins/dw/skills/series-episodes/SKILL.md)) | +| AI Director / Supercomputer (script → shots → route → run) | **Has (as skill)** | `script-to-video` shot list with cast, opening/ending state, beat, `must_not`, join; routes H3 vs LTX ([script-to-video skill](../../../../plugins/dw/skills/script-to-video/SKILL.md)) | +| Colorgrade panel, palettes, film grain | **Has** | `grade`, `apply_lut` with `palette`, `film_grain`, `sharpen` | +| Upscale / restore | **Has (exceeds)** | LTX upscale/refine/restore/colourise/alpha-matte, spandrel, diffusion upscale | +| Montage pacing / beat-cut music video | **Has (exceeds)** | `analyze_beats`, `analyze_phrases`, `plan_cuts`, `music-video-cuts` | +| Edit Video (regional) / face fix | **Has** | H3 `regional-edit`, `face-repair`, `crop_face_track`/`paste_face_track` | +| Named camera-move presets (Crash Zoom, Dolly, Orbit…) | **Missing** | Camera lives only inside each family's caption prose; no preset list (repo grep) | +| Lens / framing / light / era / genre / tempo panel | **Missing** | Same as above; no vocabulary | +| Speed ramps / slow-mo | **Missing** | LTX-2.5 Slow-Motion-Control LoRA exists upstream, not onboarded | +| Popcorn storyboard / animatic deliverable | **Partial** | FLUX 3 Klein `multi-reference`, `layout-boxes`, `consistent-set` exist; no skill composes a board | +| Shots / Angles (multi-angle stills) | **Partial** | Klein multi-reference by prompt only; no trained angle vocabulary | +| Automatic judging between generations | **Partial** | `analyze_shots`/`analyze_seams`/`analyze_sync_drift` probes report places to look, not verdicts; no face-identity check | +| Re-shoot from a new angle (video) | **Missing** | Needs depth + Layout-To-Render or CrossView | +| Genjutsu motion transfer / actor swap | **Missing** | Face round-trip tasks exist; no swap model | +| Lipsync Studio onto existing footage | **Partial** | H3 locked-frame lip-sync to a held track (#985); no dubbing model | +| Relight (photo/video) | **Missing** | No relight model in any family | +| Effects library (Set On Fire, Melt…) | **Missing** | Open versions are Wan community LoRAs | +| Draw-to-Video | **Missing** | Klein `layout-boxes` is the nearest primitive | +| Canvas node graph | **Has (different)** | Web UI FlowView/StepEditor; no real-time collaboration | + +The table makes one thing clear: **dw's gaps are mostly vocabulary and presentation, not capability**. The big capability gaps are geometry (re-shoot, angles), identity transfer (swap, dub) and light. Those are the places where Higgsfield is renting Seedance and Kling, and where the clones reach for add-on models. + +## The challenge clones converged on dw's own architecture + +Comfy's #OpenSourceHiggsfield challenge asked entrants to "Rebuild Higgsfield's popular apps, and more, using ComfyUI" under an open-source licence. It had tracks for Cinematic Studio, Re-shoot and "Openjutsu", plus bonuses for LTX, FLUX 3 and MiniMax H3 as the core model ([comfy.org](https://comfy.org/open-source-higgsfield.md)). That bonus list lines up almost exactly with dw's families. + +The CC0 index `jutsuhq/open-source-higgsfield` catalogs **102 entries and 6 older alternatives**, and is the place to start for anything not named here ([jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield)). The entries are overwhelmingly MIT or Apache and open-weights-first: LTX-2.5, H3, FLUX 3/2 Klein, Qwen-Image-Edit-2511, Wan 2.2, SAM 3.1 and SeedVR2. Nearly all share one architecture: a custom web front end and a small server that patches committed ComfyUI API-format JSON per job. Structurally that is dw's declarative workflow, server, UI and MCP stack with a different engine underneath. The research notes infer from this that the clones' UX ideas port to dw even where their graphs do not. + +Four architectural patterns recur, and each has a direct dw counterpart. + +**Preset → compiled prompt.** Aither Cinema Studio (MIT) states the principle best: "A rig is a decision, not a slider stack. `dolly_in` + `85mm` + `close_up` is a different shot from `static` + `85mm` + `close_up`, and the studio makes the model honour the difference — deterministically, in the prompt text." Its vocabulary has **20 moves, 7 lenses, 3 speeds, 10 framings and 9 looks** ([Aither](https://github.com/Aitherium/aither-cinema-studio)). + +Other entries follow the same pattern: + +- **OpenField (MIT):** its Director compiles genre, camera, framing, light, film stock, look, grade, an era scrubber and speed ramps into the shot, and its Video app has 28 motion presets ([OpenField](https://github.com/wexloops/openfield)). +- **CreatorOS Studio (Apache-2.0):** a "shot card instead of a prompt", 25 one-click camera/effect presets, seven relights, and a re-shoot angle set ([CreatorOS](https://github.com/thePloggy/CreatorOS-Studio)). + +In dw the compiled form belongs in the plugin skills and the stored prompt library, keyed per family. That honours the rule that model knowledge lives in the catalog and skills, never in engine code. + +**Plan without render.** Aither's `plan_shot` compiles a rig into the exact prompt with no render. Comfy-Dial has a free "Run dry". Haggs puts the price on every Generate button, and "Nothing is spent on the drawing page" ([Comfy-Dial](https://github.com/ylchen333/comfy-dial); [Haggs](https://github.com/IBFolding/haggs)). dw already has the stronger half of this: `validate_workflow` is free and returns `plan.estimate`, and `acknowledged_cost` is a hard gate ([MCP.md](../../../../docs/MCP.md)). The missing half is a **free view of the compiled prompt per shot**, which `enhance_prompt` approximates only for the families with enhancers. + +**Judges between generations.** Several entries check output automatically: + +- **CreatorOS** runs a CPU clip check in about 1 s for black frames, frozen video, an in-shot jump, smear, and **how close the face stays to the start image**, using OpenCV YuNet + SFace ([CreatorOS](https://github.com/thePloggy/CreatorOS-Studio)). +- **Body Double (Apache-2.0)** runs seam-ratio and background-drift QC before the next segment starts, and re-rolls a flagged segment ([Body Double](https://github.com/lucataco/open-source-higgsfield)). +- **open-higgsfield-skills (MIT)** has a still-QA judge with a fixed fix ladder (image-edit, relight, re-prompt, seed+1) and hard caps on retries and GPU-minutes ([skills](https://github.com/mojomast/open-higgsfield-skills)). +- **Boson (MIT)** persists each judged round as git-backed state that "the next round must read" ([Boson](https://github.com/0xMisterBailey/Boson)). + +dw's `assess_output` probes, `compare_outputs`, `get_output_frames(seams=)` and `set_note` cover the mechanics. What is missing is the **face-identity measurement**: the series skill says outright that "There is no engine check for 'the same character looks the same' across separate runs" ([series-episodes skill](../../../../plugins/dw/skills/series-episodes/SKILL.md)). + +**Shot documents.** A few entries make the shot plan its own artifact: + +- **seriesmith (MIT)** compiles `series.json` and markdown screenplays into a **model-free `shots.json`**, then compiles that per video target. "Every shot is its own job, so a bad shot is rendered again alone." Its assistant `SKILL.md` is generated from its authoring docs ([seriesmith](https://github.com/bgstratt/seriesmith)). +- **Picky** makes each H3 shot "a structured document with style, overview, timed beats, camera, dialogue, soundscape and music", and adds a **prompt linter that knows H3** for overfull beats, cuts with no new information and missing soundscape ([Picky](https://github.com/chenthedesignmaker/ComfyUI-Picky)). Picky is GPL-3.0, so for dw it is ideas only. + +dw's `script-to-video` shot list already has the right fields for continuity: opening and ending state, beat, `must_not` and join. But it lives in the agent's context rather than as a persisted, re-compilable document. Its fields also cover continuity, not cinematography. + +| Clone pattern | Best source to mine | dw seam it maps to | Gap to close | +|---|---|---|---| +| Closed-enum rig compiled to prompt | Aither (MIT), CreatorOS (Apache), OpenField (MIT) | Plugin skill `references/` + `prompt:camera/…` library | Vocabulary plus one compiled sentence per family | +| Plan-without-render | Aither `plan_shot`, Comfy-Dial dry run | `validate_workflow` + `plan.estimate`; `enhance_prompt` | Show the compiled caption per shot before spend | +| Judge + fix ladder + caps | open-higgsfield-skills, CreatorOS QC, Body Double | `assess_output`, `compare_outputs`, `set_note`, skill step 6 | Face-drift probe; written fix ladder | +| Model-free shot document → per-target compile | seriesmith (MIT), Picky (GPL, ideas) | `script-to-video` step 2 shot list | Persist it; add camera/lens/light fields | +| Provenance per take | OpenField, CreatorOS, Shot Board | Run manifests, `export_job`, `rerun_job` | None; dw already leads | +| Hardware/quality tiers | ReAngle, Picky, open-higgsfield-skills | H3 draft/final tiers, 24 GB recipes | None material | +| Preflight of models with install hints | OPENjutsu (MIT) | `validate_workflow` downloads | None material | +| Resume after crash | OPENjutsu, Body Double | Open `resume.md` proposal | Already queued | + +## Adopt now: vocabulary, storyboards and judges from text alone + +Everything in this tier uses the current families and adds only catalog, skill or prompt-library text. The one exception is a small, model-agnostic probe. It is where most of the "agents compose cinematic output" value sits. + +**1. A camera-and-look vocabulary per family.** The plugin should gain a shared reference with a closed set of axes: + +- **Shot size:** extreme wide to extreme close-up. +- **Camera move:** static, dolly in/out, truck left/right, crane/jib up/down, arc/orbit, tracking, whip pan, push-in, handheld, crash zoom. +- **Lens character:** wide, normal, long, anamorphic, vintage. +- **Light:** high-key, low-key, golden hour, practical, noir. +- **Era/stock:** 8mm, 16mm, 35mm, VHS/DV, digital. +- **Genre and tempo.** + +Each family's skill should then state the **one sentence** that value compiles to in that family's caption format. LTX-2.5's trained caption spec already requires "a shot type, a camera motion (say static when none) and a viewpoint" ([ltx-2.5 skill](../../../../plugins/dw/skills/ltx-2.5/SKILL.md)). Kandinsky 6 demands "exactly one camera sentence" ([kandinsky-6 skill](../../../../plugins/dw/skills/kandinsky-6/SKILL.md)). H3's camera language lives inside its Context-IR caption structure ([minimax-h3 skill](../../../../plugins/dw/skills/minimax-h3/SKILL.md)). So this fills an existing slot rather than inventing one. + +Mine these sources: + +- **Aither:** the enum sizes and the "decision, not a slider" principle. +- **CreatorOS:** the preset list and pull-a-palette-from-a-photo. +- **OpenField:** the Director axes and era scrubber. +- **Higgsfield's preset names:** they are the user-facing words a person will type ([memons](https://memons.ai/higgsfield-prompt-cheat-sheet)). +- **Lightricks' camera LoRA set:** dolly in/out/left/right, jib up/down, static. Its card advises "naming in the prompt what should come into view during the move", which is a good compile rule for LTX ([Dolly-Left card](https://huggingface.co/Galatea777/LTX-2-19b-LoRA-Camera-Control-Dolly-Left)). + +Follow Higgsfield's own retreat and **do not ship camera-body brand names** as presets. Abstract looks generalise across families. Brand names only work if the model learned them, and nobody has measured that. + +The caveat is measurement. dw's story-forge audit found the "i2v invents camera moves" lesson **did not reproduce** on LTX (0 of 6 clear moves). Yet one 241-frame clip pushed or arced after about 5 s despite "camera remains static" ([story-forge audit](../../../../docs/proposals/audits/2026-11-story-forge-lessons.md)). Whether H3 or LTX honour *named* moves is unprobed. The vocabulary should therefore ship as `trial`, with a small per-family A/B per move judged from `get_output_frames`. Comfy-Dial's approach is the model: recover the camera from each take and score it ([Comfy-Dial](https://github.com/ylchen333/comfy-dial)). + +**2. A persisted shot document with cinematography fields.** Extend `script-to-video`'s shot list with `shot_size`, `move`, `lens`, `light` and `duration`. Store it as a workspace artifact, or as a prompt-library entry, so a revision recompiles only the changed shots. That is Supercomputer's "re-runs only affected steps", reached through dw's per-shot jobs. + +Mine these sources: + +- **seriesmith:** the model-free shot list and per-target compile, `continuous:` chaining queued up front, the STORY_STRUCTURES docs (three-act, Story Circle, Kishōtenketsu), and generating the skill from the authoring doc ([seriesmith](https://github.com/bgstratt/seriesmith)). +- **OpenReel (MIT):** the three continuity anchors. Inject the actor description verbatim into every keyframe prompt, use the first head sheet as the image anchor, and keep one unified style block ([OpenReel](https://github.com/smithysw7/openreel)). +- **Vision Orchestrator (Apache-2.0):** "only approved content flows into generation" ([Vision Orchestrator](https://github.com/stellartune-studio/vision-orchestrator)). + +The written shot document is also where an **H3 prompt lint** can live as skill text: flag overfull beats, a cut with no new information, and a missing `overall_soundscape`. Take that idea from Picky without its GPL code. + +**3. A storyboard and animatic skill.** Popcorn's shape works on FLUX 3 Klein today: + +- up to 8 frames; +- references cited as "image one"; +- Manual or Auto mode; +- each frame later becomes a start frame, chained by last frame ([Popcorn help](https://higgsfield.ai/creator-hub/help-center/ai-models/how-do-i-use-popcorn)). + +Klein has `multi-reference` (10-reference training limit, not pipeline-enforced) and `layout-boxes` for blocking by bbox, and `consistent-set` produces a subject that differs in one stated way ([flux3-klein skill](../../../../plugins/dw/skills/flux3-klein/SKILL.md)). + +Mine **Shot Board (MIT)** for its **six-cut coverage set**: establishing, medium, face close-up, over-the-shoulder, insert and low angle. Pick a hero, redraw the rest from it, record a pass/fail verdict with a note, and export a manifest ([Shot Board](https://github.com/ArcSolver/shot-board)). Klein multi-reference stands in for Shot Board's Qwen-Image-Edit-2511, with weaker angle control. + +An animatic is then those boards timed to a TTS scratch track and a Music 3 bed through `assemble-and-score`. This is the cheapest pre-vis tier for sitcom and animated work, and it lets an agent show the cut before spending H3 minutes. One unknown remains: the notes do not say whether an existing task, `loop_frames` for example, can hold a still for a stated duration. If none can, a still-to-clip utility task is the only engine work this tier needs. + +**4. A face-drift probe and a written fix ladder.** Add an assessment probe in the style of CreatorOS: YuNet face detection plus SFace embedding similarity of each shot's face against the cast asset ([CreatorOS](https://github.com/thePloggy/CreatorOS-Studio)). It is model-agnostic and CPU-only, and it fills the gap the series skill names. Like the existing probes it should report places to look, not verdicts. The OpenCV model licences are not in the notes and need a check. + +Pair the probe with a written fix ladder and retry/GPU-minute caps in `script-to-video`, taken from open-higgsfield-skills ([skills](https://github.com/mojomast/open-higgsfield-skills)). The ladder runs: regional edit or face repair, then re-prompt, then reseed, then ask. Body Double's "QC before the next segment starts" belongs in the chain guidance ([Body Double](https://github.com/lucataco/open-source-higgsfield)). Its five-frame conditioning and per-segment colour match overlap the parked #1033/#1067. + +**5. Sitcom and multicam grammar.** One-shot Multicam (MIT) cuts between angles **on word boundaries** from a transcript ([One-shot Multicam](https://github.com/apsolut/one-shot-multicam)). dw already has `transcribe_audio`, `analyze_phrases` and `plan_cuts`. A sitcom section in `series-episodes` should therefore describe: + +- a wide master, held throughout; +- per-speaker singles from `dialogue-short`; +- cuts placed on line boundaries. + +The skill's own guidance to "frame recurring characters large and keep a wide master" already points this way ([script-to-video skill](../../../../plugins/dw/skills/script-to-video/SKILL.md)). + +**6. Named grades and speed language.** Higgsfield offers 50+ grade templates and an era control. dw's `apply_lut` palettes plus `film_grain` can carry a small named set: noir, teal-orange, 70s stock, VHS, bleach-bypass. That set lives as stored palettes, with no engine change. Speed ramps stay prompt-only until the LoRA in tier 2 is measured. + +## Add-ons that unlock re-shoot, angles and dubbing + +These need a new adapter or model entry on a base dw already runs, or a new utility-task model. Each needs a 24 GB proof and a licence check before it becomes a template. + +**LTX-2.5 Layout-To-Render IC-LoRA, for blocking-to-shot and re-shoot.** This is the clones' workhorse: + +- **OpenField's Stage** renders greybox blocking to a depth pass, paints a look frame, then renders "a shot that moves like the blocking" with Layout-To-Render ([OpenField](https://github.com/wexloops/openfield)). +- **ReAngle (Apache-2.0)** re-shoots by depth-estimating the clip, warping along a camera path with a z-buffered splat, pre-filling revealed holes from the far background and guiding LTX-2.5, with chunk re-anchoring and LAB colour match for long clips ([ReAngle](https://github.com/Tekholms-LW/ReAngle)). +- **Comfy-Dial** feeds LTX a depth video rendered along an authored path. "The model never sees your camera numbers." It then recovers the camera per take and repairs only the drifted segment ([Comfy-Dial](https://github.com/ylchen333/comfy-dial)). + +The LoRA is on the same base as dw's catalogued IC-LoRAs ([Lightricks listing](https://huggingface.co/api/models?author=Lightricks&search=LoRA&limit=100&sort=lastModified)). There is a warning in dw's own record, though. Clean-Plate and Day-To-Night were **not onboarded** because they "don't fit the distilled IC template" and need a guided dev-transformer route ([ltx2 IC-LoRAs record](../../../../docs/proposals/complete/ltx2-ic-loras-complete.md)). Layout-To-Render's recipe must be checked against that constraint first. + +The depth model is a licence trap. ReAngle defaults to **Video Depth Anything Small because Base and Large are CC-BY-NC-4.0**. The depth estimator and the reprojection would be new utility tasks. This is the best route to Higgsfield's "Re-shoot" and to camera moves you can actually measure, and it is the most engineering in this tier. + +**LTX-2.5 Slow-Motion-Control and Cinemagraph LoRAs.** These are plain LoRAs on the 2.5 base. They give speed-ramp and living-photo presets ([Lightricks listing](https://huggingface.co/api/models?author=Lightricks&search=LoRA&limit=100&sort=lastModified)), and they are dw's first honest "effect presets". Their sampling recipe is unverified. + +**LTX-2.3 adapters on 2.5 (probe only).** Three 2.3 adapters are worth probing: + +- **Relight** is driven by a light-direction ball composited into the reference's corner, and the docs say it is **not yet validated on LTX-2.5** ([relight card](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Relight); [docs.ltx.io](https://docs.ltx.io/open-source-model/feature-guides/editing-effects/relight)). +- **Motion-Track-Control.** +- **DubIt.** Its card was not read. + +An aggregator says 2.3 LoRAs "partly work" on 2.5 ([gradually.ai](https://www.gradually.ai/en/ai-models/ltx-2-5/)). Treat each as a one-afternoon probe, not a plan. + +**Qwen-Image-Edit-2511 + fal Multiple-Angles LoRA, for Shots and Angles.** This is the open standard for "9 angles from one image". It covers **96 camera positions** trained on Gaussian-splat renders and is driven by prompts like "Rotate the camera 45 degrees to the left" ([fal LoRA](https://huggingface.co/fal/Qwen-Image-Edit-2511-Multiple-Angles-LoRA); [Space prompt grammar](https://huggingface.co/spaces/linoyts/Qwen-Image-Edit-Angles/blob/bbbeff133c3a7714b98a8ab99a61895294dfb4cf/app.py)). CreatorOS and Shot Board both use it. + +Two facts from dw's own audit shape the cost: + +- The base is Apache-2.0 but **57.7 GB** at full precision, so it needs quantization or offload to meet the 24 GB rule. +- dw's onboarded **Qwen-Image 2.1** is under the non-commercial Qwen Research License and has none of this LoRA ecosystem ([Qwen audit](../../../../docs/proposals/audits/2026-09-20-qwen-image-2.1-audit.md)). + +A `for_each` over nine fixed angle prompts makes a "Shots" template. Those stills then feed H3 FL2VA or LTX keyframes. The fal LoRA's licence is unchecked. + +**H3 LoRAs from the Genjutsu clones.** Three LoRAs are relevant: + +- **comfy-reshoot** pairs MoGe geometry with the **CrossView LoRA for H3** ([comfy-reshoot](https://github.com/Comfy-Org/comfy-reshoot)). +- **H3 Recast** swaps up to three people with the **BFS Character Swap LoRA** ([H3 Recast](https://github.com/Bambushu/h3-recast)). +- **OPENjutsu** runs H3 Ref2VA int8 with a **DMAD 4-step LoRA** ([OPENjutsu](https://github.com/D3voz/openjutsu)). + +None has been checked for diffusers loading. OPENjutsu's recipe is the most portable idea: + +- a SAM 3.1 track at 1008 px; +- the mask grown 40 px with 10% headroom; +- a grey-mannequin motion reference; +- Ref2VA with the new cast; +- a VLM labelling tracks to roles. + +It also documents its failure modes: reflections leak the original, and clothing words leak. That maps onto dw's `segment`, `regional-edit`, `crop_face_track`/`paste_face_track` and Ref2VA. It is the one route to Genjutsu-style object and character swap without a new base. Expect H3 Recast's caveat to hold: "Big, obvious changes land … Subtle ones fall back to the clip." + +**LatentSync 1.6 as a dubbing task.** For lip-syncing existing footage, as Lipsync Studio does, LatentSync is the clear commercial path: **Apache-2.0 code and OpenRAIL++ weights at about 18 GB VRAM**, with MuseTalk (MIT, about 4 GB) as the light option ([Creative AI News](https://www.creativeainews.com/articles/open-source-lip-sync-licenses-commercial-use-2026/); [LatentSync](https://github.com/bytedance/LatentSync)). It is post-processing on a clip, so it fits as a utility task rather than a family. One Civitai page labels the weights RAIL-M, which conflicts and is likely a mislabel ([HF discussion](https://huggingface.co/chunyu-li/LatentSync/discussions/2)). For new dialogue, H3's joint audio-video generation remains better than dubbing. + +**Location consistency.** 360 Scout to Film (MIT) turns one equirectangular panorama into overlapping perspective location references plus a JSON manifest shared by every shot ([360 Scout](https://github.com/Alvinvision/360-Scout-to-Film-Consistency)). That is a pure image utility over Ref2VA's reference slots, with no new model, and it gives dw a "Cinematic Locations" equivalent. + +## Wan 2.x is the price of actor swap and spectacle effects + +A large share of what the clones and Higgsfield do sits on Wan 2.1/2.2 14B: + +- **Wan2.2-Animate:** character replacement and motion transfer, the core of Genjutsu. diffusers has a `WanAnimatePipeline`, but "integration of these preprocessing steps into Diffusers is planned for a future release" ([diffusers PR #12442](https://github.com/huggingface/diffusers/pull/12442.diff)). +- **VACE, Phantom, MAGREF and Stand-In:** identity-reference options. +- **Remade-AI effect LoRAs:** 360 Orbit and Crash Zoom, with trigger phrases ([Remade-AI](https://huggingface.co/collections/Remade-AI/wan21-14b-480p-i2v-loras)). +- **Uni3C and ReCamMaster:** camera transfer and re-render. ReCamMaster's released weights are a Wan port that "may not match the paper" ([ReCamMaster](https://github.com/KlingAIResearch/ReCamMaster)). +- **InfiniteTalk:** long talking video. + +Wan 2.2 itself is Apache-2.0 ([MindStudio](https://www.mindstudio.ai/blog/what-is-wan-2-2-video-open-source)). Adopting it is a real family onboarding, though. Animate needs pose and mask preprocessing (DWPose, SAM2) as new tasks. The conditioning adapters mostly need custom loaders. And dw's parked #1066 notes VACE reframe was "meant for a cloud GPU with 48 GB or more". + +The recommendation is **defer**. Wan earns a slot only if a field report asks for actor replacement that the H3 mask-and-Ref2VA route cannot do. If it is onboarded, run it through the `model-family-onboarding` audit, and treat Animate's 24 GB fit as unproven. + +The seven named Lightricks camera LoRAs fall in the same "new base" bucket. Their repos say **LTX-2 19B**, while Lightricks' VFX page lists them under LTX-2.3 ([HF](https://huggingface.co/Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In); [ltx.io](https://ltx.io/model/capabilities/vfx)). They do not target 2.5, so for dw they are vocabulary, not weights. + +### Do not adopt + +Leave these out: + +- **Spectacle effects** (Set On Fire, Melt, Cakeify, Explode). Their open forms are Wan community LoRAs with unchecked licences, or LTX-Video 0.9 relics, and they serve virality rather than cinematic or sitcom composition. +- **A Soul ID-style per-person training step.** dw's fixed-asset cast references are the consistency mechanism and are already proven in the series skill. +- **Supercomputer's connectors, app deployment, Playable, AI-influencer and ad factories.** They are outside dw's purpose. +- **Code from no-licence repos**, including the 5,211-star `wide-trace/open-higgsfield`, HeliosGen, Darius, Conjure, FXjutsu and HiggsOpen. These are concept-only ([jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield)). +- **Code from Picky and Open Cinema (GPL-3.0), or from Parametric Studio and SpEdE (AGPL).** Ideas only. Open Cinema's post-hoc physically based depth of field and rack focus from per-frame depth is a good idea to revisit once a depth task exists ([Open Cinema](https://github.com/LaytonGott/open-cinema)). +- **Wav2Lip.** Commercial use is "strictly prohibited". + +Non-commercial weights to flag if they ever appear in a template: Video Depth Anything Base/Large (CC-BY-NC), FLUX.2 (marked non-commercial by OpenField), dw's own Qwen-Image 2.1, and MMAudio (CC-BY-NC 4.0, already shipped). Hunyuan3D 2.1 excludes the EU, UK and South Korea, and the H3 community licence has geographic exclusions too ([ReAngle](https://github.com/Tekholms-LW/ReAngle); [Comfy-Dial](https://github.com/ylchen333/comfy-dial); [OpenField](https://github.com/wexloops/openfield)). + +### Unresolved and conflicting + +Several sources disagree: + +- **Challenge deadline:** Comfy's live page says submissions close 3 Nov with winners on 10 Nov. The ComfyUI Wiki and the jutsuhq list say 30 Oct and 6 Nov ([comfy.org](https://comfy.org/open-source-higgsfield.md); [ComfyUI Wiki](https://comfyui-wiki.com/en/news/2026-10-15-open-source-higgsfield)). Winners are not yet known. +- **Cinema Studio focal lengths:** 12-135 mm versus 8/14/35/50/75 mm ([seedance.tv](https://www.seedance.tv/blog/higgsfield-cinema-studio-camera-settings); [vidpros](https://vidpros.com/higgsfield-cinema-studio/)). +- **Cinema Studio 4.0 resolution:** "up to 720p" conflicts with the homepage's 1080p Seedance option. +- **Genjutsu references:** sources say up to 40 or up to 30. +- **Higgsfield's cease-and-desist** to Comfy rests on Comfy's account alone ([RuntimeWire](https://runtimewire.com/article/comfy-says-higgsfield-sent-a-48-hour-cease-and-desist-over-open-source-challenge)). +- **"Openjutsu"** is the challenge track's name. The product it imitates is Genjutsu. +- **Licence reporting:** GitHub reports `NOASSERTION` for OpenField, open-higgsfield-skills and Vision Orchestrator, whose LICENSE files are MIT, MIT and Apache-2.0. + +Several things nobody has verified: + +- that CrossView, BFS, DMAD, Layout-To-Render, SCAIL-2 or LTX audio-reference-token freezing load in diffusers; +- the licences of the fal angle LoRA, the Remade LoRAs, ReCamMaster, TrajectoryCrafter, InfiniteTalk and the Lightricks camera LoRAs; +- whether most clones run end to end (open-higgsfield-skills admits "Most have not been run"); +- whether H3, LTX or Kandinsky honour named camera moves at all. + +## Conclusion + +The surprise is how little of Higgsfield is model work. Its moat is a **vocabulary and a planning loop**, and dw already owns the harder parts of that loop: free validation with cost, a hard spend gate, replayable per-shot jobs with manifests, assessment probes, and two orchestration skills that already enforce cast lock and continuity states. Higgsfield's AI Director and Supercomputer are, functionally, dw's `script-to-video` with a shot card bolted on. The challenge entries independently rebuilt dw's architecture on ComfyUI, which is a strong signal the design is right. The work left is to give agents the director's words (closed enums compiled per family), a persisted shot document to fill, and a face-drift measurement to judge with. + +The order follows from that. First, vocabulary, the shot document, the storyboard/animatic skill and the face probe, all measured as `trial` before promotion. Second, Layout-To-Render for blocking and re-shoot, Qwen-Image-Edit-2511 angles, and LatentSync, each gated on a 24 GB proof and a licence read. Wan only if a user needs a true actor swap. One thing should be decided before tier 1 lands: whether a named camera move is a promise or a hint. The evidence so far (0 of 6 invented moves, but one clip that drifted after 5 s) says hint. If so, the skills should present presets as hints, the way Higgsfield's own users echo the preset in the prompt and hope, and let the planned verifier turn that hint into a measured claim. diff --git a/docs/proposals/audits/2026-11-higgsfield/notes/dw_current_capabilities.md b/docs/proposals/audits/2026-11-higgsfield/notes/dw_current_capabilities.md new file mode 100644 index 000000000..9a2046ddb --- /dev/null +++ b/docs/proposals/audits/2026-11-higgsfield/notes/dw_current_capabilities.md @@ -0,0 +1,144 @@ +# dw (diffusers-workflow) current capabilities for cinematic / animatic / sitcom video, vs a Higgsfield-style feature set + +Scope: master working tree at `a2285c64 release 0.17.0`, read 2026-11-01. All sources are repo files (paths relative to the repo root `/Users/don/src/dkackman/diffusers-workflow`) or GitHub issues on `dkackman/diffusers-workflow`. Nothing here was run on a GPU; behaviour claims are what the code/docs/skills state. + +## Which templates/workflows exist per shape and per model family? + +### Takeaway +dw ships ~110 catalog workflows: 4 video families (MiniMax H3 = 22 templates, LTX-2.5 = 19, Kandinsky 6 = 3, plus a generic Mochi/compose route), FLUX 3 Klein (5) and a dozen FLUX.1/Z-Image/Qwen/SD image templates, plus audio (Music 3, MMAudio SFX, TTS/voice clone) and assembly templates. Shape is not hand-labelled; it is derived from the steps (`image`, `image-set`, `image-edit`, `shot`, `sequence`, `audio`, `text`, `utility`) and filterable over MCP. + +### Cited Findings +- Shape and traits are derived, not declared: a video workflow that cuts clips together is `sequence`; any other video is `shot`; audio -> `audio`; an edit pipeline or `image`/`mask_image` arg -> `image-edit`; two+ image steps or a workflow step -> `image-set`; nothing generated -> `utility`. Traits: `has-audio`, `chained` (a `chain` block or chain args), `image-conditioned`, `identity-referenced` (`references` arg), `composes-workflows`, `needs-input-media` — [dw/server/catalog_shape.py](dw/server/catalog_shape.py) `_derive_shape`/`_derive_traits` (lines ~224-295) +- **MiniMax H3 (video+audio, 24 fps, 32 kHz stereo)** — `workflows/templates/minimax/`: `video-with-audio` (960x544 baseline), `video-with-audio-768p`, `enhance-prompt`, `enhance-prompt-with-image`, `image-to-video` (first frame, FL2VA), `first-and-last-frame`, `last-frame-only`, `reference-to-video` (Ref2VA identity/voice), `composable-references` (image subject + video ref for framing/camera, ~3.4x cost), `generated-subject-reference` (Z-Image draws subject then Ref2VA), `voice-timbre-reference` (Bark line as voice ref), `storyboard` (one generation, three shots from stills), `dialogue-short` (cast drawn by Z-Image, one shot per `shots` entry, `concat_videos`), `music-video`, `music-video-cuts` (plans cuts from song+lyrics), `shots-batch` (independent shots on one loaded model), `chained-segments`, `chain-video-continuity`, `chain-matched-to-audio`, `chain-matched-and-aligned`, `regional-edit` (mask a region of a good take, keep motion/audio), `face-repair`, `music` (Music 3) — [workflows/templates/minimax/](workflows/templates/minimax/); [plugins/dw/skills/minimax-h3/SKILL.md](plugins/dw/skills/minimax-h3/SKILL.md) +- **LTX-2.5 (video+soundtrack, 24 fps)** — `workflows/templates/ltx2/`: `text-to-video` (960x544, 121 frames), `image-to-video` (401 frames ≈17 s at 768x768 on 24 GB), `keyframes` (first+last pinned), `enhance-prompt`, `two-stage` (half-size + latent upsample + refine to 1536x896), `diffusion-decode` (A/B only), `generative-upscale`, `upscale-clip` (IC-LoRA 2x), `refine-clip` (2x via own latents), `refine-in-place` (`strength` 0-4), `reference-sheet` (cast/props/set from one composite sheet — "the family's only identity route"), `alpha-matte` (soft per-frame alpha, ≤145 frames), `restore-deblur`, `restore-decompression`, `restore-colorize`, `restore-long` (windowed, ≤32 windows), `face-repair`, `extend-clip`, `chained-segments` — [workflows/templates/ltx2/](workflows/templates/ltx2/); [plugins/dw/skills/ltx-2.5/SKILL.md](plugins/dw/skills/ltx-2.5/SKILL.md) +- **Kandinsky 6.0 (3B Lite-distill, 5 s with soundtrack)**: `text-to-video`, `image-to-video` (still = first frame), `generate-and-upscale` (SR pipeline to 1728x960) — [workflows/templates/kandinsky6/](workflows/templates/kandinsky6/) +- **FLUX 3 Image Klein (5.6B distilled)**: `text-to-image`, `upsample-then-generate` (BFL prompt upsampler first), `image-edit`, `multi-reference` (several references combined; 10-reference training limit not enforced by pipeline), `layout-boxes` (elements placed by bbox JSON) — [workflows/templates/flux3/](workflows/templates/flux3/); [plugins/dw/skills/flux3-klein/SKILL.md](plugins/dw/skills/flux3-klein/SKILL.md) +- **Image model entries** (`workflows/models/`): FLUX.1 dev (+compile, GGUF, TorchAO, Krea), FLUX.2 dev, FLUX 3 Klein, Krea 2 Turbo, Qwen-Image 2.1 (+ instruction edit; native RGBA), Z-Image Turbo (+SDNQ, +inpaint) — [workflows/models/](workflows/models/) +- **Generic image templates** relevant to pre-vis/storyboards: `consistent-set` (shape `image-set`: one subject differing in one stated way — generated once then edited), `image-edit` (FLUX.1 Kontext), `multi-image-reference` (FLUX.2), `ip-adapter` (incl. face model), `image-variation` (Redux, `image-edit`), `inpaint`, `outpaint`, `segment-and-inpaint`, `remove-object`/`segment-and-remove` (LaMa), `controlnet`/`controlnet-component` (Canny/depth), `image-processors` (edges, depth, pose, segmentation), `depth-marigold`, `surface-normals`, `recenter-crop` (register stills before they become shots), `lora`/`lora-styles` (ten FLUX in-context LoRA styles), `restore-faces`, `upscale-spandrel`, `upscale-diffusion`, `describe-and-regenerate`, `expand-prompt` — [workflows/templates/](workflows/templates/) (descriptions in each JSON) +- **Audio templates**: `minimax/music` (Music 3), `text-to-sound-effect` and `video-to-sound-effects` (MMAudio large 44.1 kHz v2; the latter muxes under a clip), `generate-speech` (local TTS, default `facebook/mms-tts-eng`), `voice-clone` (Audio8-TTS from a short reference clip), `transcribe-audio`, `attribute-lines` (who sings which line), `check-script` (does a take speak its script), `audio-trim-fade`, `relevel-clip` — [workflows/templates/](workflows/templates/) +- **Assembly/sequence templates**: `assemble-and-score` (cuts existing shots into one scored film; the re-runnable cutting pass), `dissolve-between-shots` (same with cross-dissolves), `compose-workflows` (any T2I + any I2V as still-then-animate), `best-of-n-to-video` (N stills, deterministic pick, animate), `interpolate-frames` — [workflows/templates/assemble-and-score.json](workflows/templates/assemble-and-score.json), [workflows/templates/dissolve-between-shots.json](workflows/templates/dissolve-between-shots.json) +- **Builtins** (`builtin:` step targets, not the runnable examples): `flux3_klein_upsample`, `flux3_klein_upsample_edit` (BFL upsampler, Qwen3.5-9B based), `h3_context_ir` (rewrites a video idea into MiniMax-H3 Context-IR), `test` — [dw/workflows/](dw/workflows/) + +### Inferences +- The family coverage maps well onto Higgsfield's "shot" primitives (T2V, I2V, first/last frame, extend, reference identity, upscale) but every one is a separate template a user/agent must choose; there is no single "shot" UI with a model switch. +- H3 is the only family with dialogue lip-sync and multi-shot-with-cuts templates; LTX-2.5 owns restoration/matte/upscale; Kandinsky 6 is minimal (3 templates). + +### Gaps +- No open-weights family for character/face swap, relighting, novel-view/re-shoot, or camera-trajectory control appears in any template or model entry (grep for "relight", "novel view", "reshoot", "face swap", "character swap", "animatic" across docs/plugins/workflows/dw/ui returned 0 files). +- Exact per-template derived shape was not computed at runtime (the server derives it); the per-family shapes above come from skills (`ltx-2.5` says every LTX template is `shot`; `series-episodes` says `dialogue-short`, `music-video`, `assemble-and-score` are `sequence`). + +## What utility tasks exist that support editing/assembly? + +### Takeaway +dw has an unusually deep edit/finish/audio task layer (~60 registered commands): concat/dissolve/trim/stabilize/window, face-track crop/paste, beat/phrase analysis and a cut planner, a full audio mix chain (slice, gain, fade, crossfade, loop, mix, normalize with LUFS target/limiter, compress, filter, stem separation), grading (grade, sharpen, film_grain, apply_lut with palette), MMAudio SFX, TTS, transcription, script checking, voice attribution, and read-only assessment probes for seams/shots/sync drift. + +### Cited Findings +- Registered task commands (from `@register_command` across `dw/`): analyze_audio, analyze_seams, analyze_shots, analyze_sync_drift, apply_lut, attribute_voices, batch_decode_post_process, check_script, compose_text, compress_audio, concat_videos, crop_face_track, crossfade_audio, decode_h3_latents, diffusion_upscale, dissolve_videos, extract_sections, fade_audio, film_grain, filter_audio, find_loop_bed, fit_to_model, format_chat_message, frame_grid, gain_audio, gather_images/inputs/videos, generate_sound_effects, generate_speech, get_dict_value, grade, image_to_text, ingredients_grid, interpolate_frames, join_into_song, join_windows, judge, loop_audio, loop_frames, mix_audio, normalize_audio, pair_audio, paste_face_track, qr_code, remove_object, resample_audio, restore_faces, restore_to_source, segment, select, separate_stems, sharpen, slice_audio, stabilize_video, text_generation, transcribe_audio, upscale, upscale_h3_latents, video_frames, window_video; plus `analyze_beats` ([dw/tasks/beats.py](dw/tasks/beats.py)), `plan_cuts` ([dw/tasks/cuts_lines.py](dw/tasks/cuts_lines.py)), `analyze_phrases` ([dw/tasks/phrases.py](dw/tasks/phrases.py)), `measure_sync`, and `get_last_frame`/`get_first_frame` (used by joins) — [dw/tasks/](dw/tasks/); [docs/TASKS.md](docs/TASKS.md) +- TASKS.md sections for video: `concat_videos`, `dissolve_videos`, `join_into_song`, `stabilize_video`, `trim_video`, `window_video`, `join_windows`, `fit_to_model`/`restore_to_source`, `crop_face_track`, `paste_face_track`, `video_frames`, `pair_audio`; audio: `slice_audio`, `gain_audio`, `crossfade_audio`, `fade_audio`, `normalize_audio`, `mix_audio`, `loop_audio`, `find_loop_bed`, `resample_audio`, `compress_audio`, `filter_audio`, `analyze_audio`, `measure_sync`, `analyze_beats`, `analyze_phrases`, `plan_cuts`; image finishing: `grade`, `sharpen`, `film_grain`, `apply_lut`, "A finishing chain" — [docs/TASKS.md](docs/TASKS.md) headings (lines 264-2108) +- Assessment probes `analyze_shots`, `analyze_seams`, `analyze_sync_drift` are registered `assessment=True`, stream a 64x36 grey thumbnail per frame plus the soundtrack, and return `findings` that are "places to look, not verdicts: nothing in the engine acts on one" — [docs/TASKS.md](docs/TASKS.md) "Assessment Probes" (line ~2220) +- Other sections: Stem separation (htdemucs), Voice attribution (`attribute_voices`, "Checking the lip-sync target"), `ingredients_grid`, Image/Diffusion upscaling, Face restoration, Object segmentation/removal, Captioning, Prompt expansion ("Writing a prompt from a picture"), Speech generation ("Cloning a voice (Audio8-TTS)", "Generating a voice to condition on"), Sound Effects Generation, Speech Transcription, Script check, Frame Interpolation — [docs/TASKS.md](docs/TASKS.md) +- The five-beat episode pass: **recut** (`concat_videos`, hard cuts, no rescale — clips must share size/fps), **bed** (`loop_audio` to the cut's frames; `find_loop_bed` for room tone under dialogue gaps), **match_levels** (`rms`/`peak`), **normalize** (`balanced` step, -3 dBFS peak ceiling, optional series-wide `target_lufs`), **pair** (mux onto the cut) — [plugins/dw/skills/series-episodes/SKILL.md](plugins/dw/skills/series-episodes/SKILL.md) +- Dialogue/score separation recipe: `separate_stems` on the world track, `previous_result:.vocals` as dialogue and drums+bass+other as the rest, remixed with one gain each (htdemucs keeps H3 speech intact, #1091) — [plugins/dw/skills/series-episodes/SKILL.md](plugins/dw/skills/series-episodes/SKILL.md) +- A uniform series look: `apply_lut` with one shared `palette` (2-16 `#rrggbb`, dark to light) or an uploaded `.cube`, then optional `film_grain`, before the recut — [plugins/dw/skills/series-episodes/SKILL.md](plugins/dw/skills/series-episodes/SKILL.md) and `references/look.md` +- Music-video timing stack: `music-video-cuts` separates vocal stem (htdemucs), transcribes, plans cuts; `music-video` pads-then-trims each shot to its cut; the `num_frames` rule is `max(124, next 17n+5 ≥ lead + cut)` — [plugins/dw/skills/minimax-h3/SKILL.md](plugins/dw/skills/minimax-h3/SKILL.md); [workflows/templates/minimax/music-video-cuts.json](workflows/templates/minimax/music-video-cuts.json) + +### Inferences +- Edit/assembly is a declarative, re-runnable "recut" job (JSON), not a timeline editor; it is closer to an NLE-as-code than to Higgsfield's in-app editor. Strength: reproducible, measurable (LUFS, seams, sync drift). +- `paste_face_track` + `crop_face_track` + `face-repair` templates give a face-region round-trip that could be the basis of a character-swap/face-replace feature, though none exists. + +### Gaps +- No timeline/transition vocabulary beyond hard cut and cross-dissolve (no wipes, J/L-cuts as first-class, titles/captions burn-in task except via the H3 `music-video-subtitle-generator` style pack). +- No colour-match across seams yet (open, parked #1067/#1083/#1033). + +## What do the plugin skills instruct agents to do? + +### Takeaway +Seven skills: five family pickers (template per request shape + hard rules + cost quote + vendor prompt format) and two cross-cutting orchestrators. `script-to-video` turns prose into a structured shot list (cast, locations, opening/ending state, beat, must_not, join type), casts recurring characters once as shared assets, routes each shot to H3 (dialogue/voice/music) or LTX-2.5 (silent/action/establishing), test-renders, gates on cost, judges with probes, and hands off to `assemble-and-score`. `series-episodes` adds the cross-episode cast lock and the five-beat recut/bed/match_levels/normalize/pair pass. + +### Cited Findings +- Plugin inventory: `minimax-h3`, `minimax-music3`, `ltx-2.5`, `kandinsky-6`, `flux3-klein`, `script-to-video`, `series-episodes`; each SKILL.md capped under 12 KB, detail in `references/`, numeric rules pinned by `tests/test_plugin_skills.py` — [plugins/dw/README.md](plugins/dw/README.md) +- script-to-video shot list shape: Cast (voice description, sex, `asset:cast/`), Locations, per shot `opening_state`/`ending_state` (a shot's opening is its source's ending "so the cut between them is written down rather than hoped for"), `beat`, `must_not`, optional `join` ∈ {`continue`, `cut`, `cutaway`, `lead_in`}; user-supplied fields are locked, blanks stay blank — [plugins/dw/skills/script-to-video/SKILL.md](plugins/dw/skills/script-to-video/SKILL.md) step 2 +- Routing rule: silent action/establishing -> LTX-2.5 (`text-to-video`, `image-to-video`, chained); dialogue, held voice or scored montage -> H3 `dialogue-short` or `music-video` with `h3-prompt-writing` structure (`subject_definitions`, `retention_analysis`, `detailed_description`, `overall_soundscape`, `non_diegetic_music`); "keep the sharpest of N" -> `best-of-n-to-video` — [plugins/dw/skills/script-to-video/SKILL.md](plugins/dw/skills/script-to-video/SKILL.md) step 4 +- Process discipline: test-render 2-4 representative shots; `validate_workflow` and quote `plan.estimate` before `run_workflow`; `acknowledged_cost` "is a real gate ... even when running unattended"; render in source order; 3 failed attempts per shot then ask; after a chain call `assess_output(probe="analyze_seams")`; never splice a re-rolled segment with `concat_videos`; new seed redraws an H3 voice unless pinned by an audio reference (#610) — [plugins/dw/skills/script-to-video/SKILL.md](plugins/dw/skills/script-to-video/SKILL.md) steps 5-6 +- Explicitly out of scope: automatic model speed optimisation, authoring new workflows at run time — [plugins/dw/skills/script-to-video/SKILL.md](plugins/dw/skills/script-to-video/SKILL.md) "Not in scope" +- Joins: `cutaway`/`lead_in` are composed from separate runs joined by `concat_videos`; frames taken off a finished shot via `get_last_frame`/`get_first_frame` tasks; "Nothing in a template reads `join`" — [plugins/dw/skills/script-to-video/references/joins.md](plugins/dw/skills/script-to-video/references/joins.md) +- Cast consistency procedure (step 0): draw each portrait (and voice clip) once, `keep_output(..., asset_name="cast/.png", shared=True)`, reference with `from_file` never `from_previous_result`; save portrait and voice description prompts as `prompt:/` and `-voice`; "There is no engine check for 'the same character looks the same' across separate runs; the fixed-asset reference is the only guard" — [plugins/dw/skills/series-episodes/SKILL.md](plugins/dw/skills/series-episodes/SKILL.md) +- Framing guidance (unmeasured): a face small in a wide shot drifts most, frame recurring characters large and keep a wide master — [plugins/dw/skills/script-to-video/SKILL.md](plugins/dw/skills/script-to-video/SKILL.md); [docs/proposals/audits/2026-11-story-forge-lessons.md](docs/proposals/audits/2026-11-story-forge-lessons.md) +- H3 hard rules: `num_frames` = 17n+5, 124-345 at 24 fps (5.17-14.4 s); canvas 768 short edge, ≤768x1344, multiples of 32; Ref2VA ≤9 images/3 videos/3 audio/12 files, audio never sole reference; 24 GB Ref2VA ceiling 243/209/175/141 frames at 1/2/3/4 refs; dialogue shots drop the turbo LoRA (12 steps); "Nothing carries between generations except a passed reference, a held track or a `guides` clip"; one cropped reference per character, never a group picture — [plugins/dw/skills/minimax-h3/SKILL.md](plugins/dw/skills/minimax-h3/SKILL.md) +- H3 `guides`: up to 4 clips (3 on a guide chain) at frames that are multiples of 17, hold look and motion, `"audio": true` holds the soundtrack — [plugins/dw/skills/minimax-h3/SKILL.md](plugins/dw/skills/minimax-h3/SKILL.md); [docs/WORKFLOW_GUIDE.md](docs/WORKFLOW_GUIDE.md) "H3: holding a clip with `guides`" +- LTX-2.5 caption rules: one 150-220 word paragraph, present progressive, "stating for every shot a shot type, a camera motion (say static when none) and a viewpoint"; multi-shot in one generation = one chronological paragraph naming "a hard cut to", 2-4 shots; voice caption steers pitch/age/gender (measured F0 291/221/136 Hz) but not whisper or tempo — [plugins/dw/skills/ltx-2.5/SKILL.md](plugins/dw/skills/ltx-2.5/SKILL.md) "Prompts" +- H3 style: eight MiniMax style packs (papercraft-stop-motion-explainer, paper-collage-explainer-generator, 3d-animation-short-generator, handdrawn-live-video-generator, minimalist-product-ad-generator, music-video-subtitle-generator, brand-promo-video-generator, co-op-game-intro-generator), installed by name; without a pack "name the medium, then what it looks like" (unprobed on dw) — [plugins/dw/skills/minimax-h3/references/style.md](plugins/dw/skills/minimax-h3/references/style.md) +- Other H3 references: `cuts.md`, `dialogue.md` (how long a line fits), `per-character.md`, `staging.md` (crowds, two-person lip-sync, landmark), `regional-edit.md`, `quality-and-speed.md` (draft/final tiers, 1080p tiled latent refine), `checkpoints-and-loras.md` — [plugins/dw/skills/minimax-h3/references/](plugins/dw/skills/minimax-h3/references/) + +### Inferences +- The skills already encode much of what Higgsfield sells as product UX (shot planning, cast lock, continuity states, beat review) — but as agent instructions, not UI. Camera/lens vocabulary lives only inside the vendor caption formats (LTX "shot type, camera motion, viewpoint"; H3 Context-IR), not in a dw preset list. + +### Gaps +- Kandinsky 6 and Music 3 skill bodies were only skimmed (headings); details not captured here. +- No skill covers storyboard/animatic stills-first pre-vis as a deliverable (FLUX 3 Klein `layout-boxes` and `consistent-set` exist but no skill composes a board). + +## What does the MCP surface allow an agent to do end to end? + +### Takeaway +65 MCP tools cover discovery (catalog/guides/tasks/pipelines), authoring and validation (free, with cost plan), cost-gated runs, job waiting/events/rerun, media inspection (images, audio, video as frames), automated assessment and comparison, asset/prompt/LoRA/workspace management, model downloads, and export. An agent can go from a prose script to a scored, exported film without leaving MCP; it judges video only as frames plus audio. + +### Cited Findings +- Tool list: list_guides, get_guide, list_workflows, get_workflow, get_schema, list_pipelines, get_pipeline_signature, list_classes, get_class, list_tasks, get_task, list_models, get_memory, get_health, get_server_info, list_jobs, list_gallery, get_gallery_metadata, get_output_image, get_output_audio, get_output_frames, get_output_text, assess_output, compare_outputs, download_output, delete_output, validate_workflow, list_workspaces, use_workspace, create_workspace, copy_to_workspace, delete_workspace, list_assets, keep_output, set_note, upload_asset, delete_asset, save_workflow, delete_workflow, list_prompts, get_prompt, get_prompt_schema, save_prompt, delete_prompt, list_enhancers, enhance_prompt, list_loras, save_lora, recommend_loras, run_workflow, get_job, get_job_workflow, export_job, get_job_events, wait_for_job, cancel_job, clear_memory, rerun_job, move_job, download_model, list_downloads, cancel_download, delete_model, get_diffusers_state, update_diffusers — [docs/MCP.md](docs/MCP.md) "Tool reference" ("65 tools in six groups"); [dw_mcp/](dw_mcp/) `tools_catalog.py`, `tools_jobs.py`, `tools_media.py`, `tools_authoring.py` +- `list_workflows` without a shape returns summaries only (catalog in full ≈6.8k tokens, #101); filter by `shape`, `traits`, `configures` — [dw_mcp/catalog.py](dw_mcp/catalog.py) +- Cost gate: tools that spend GPU/disk or delete need `acknowledged_cost` = the validate plan's `{fingerprint, minutes, downloads}` — [docs/MCP.md](docs/MCP.md) "The cost gate"; MCP server instructions +- Known limits: no video content type over MCP — a video is seen as frames (`get_output_frames`, incl. `seams=` and `at`) and heard as its track; `get_output_audio` refuses whole clips over a 4 MB base64 budget; `get_job_events` keeps the last 200 events; uploads read the MCP host's disk unless sent inline (≤4 MB); prompts library is global across workspaces — [docs/MCP.md](docs/MCP.md) "Known limits" +- References usable in arguments: `variable:`, `previous_result:`, `prompt:`, `asset:`, `output:`, `constant:`, `item:`, `gather:` — [CLAUDE.md](CLAUDE.md); [docs/WORKFLOW_GUIDE.md](docs/WORKFLOW_GUIDE.md) "References" +- `for_each` = one step per list entry (how `shots` lists fan out) — [docs/WORKFLOW_GUIDE.md](docs/WORKFLOW_GUIDE.md) "One step per entry: `for_each`" (line ~543) +- Enhancers exposed (`list_enhancers`, `enhance_prompt`) map onto the builtins `h3_context_ir`, `flux3_klein_upsample(_edit)` and LTX's own enhancer — [dw/workflows/](dw/workflows/); [workflows/templates/ltx2/enhance-prompt.json](workflows/templates/ltx2/enhance-prompt.json) + +### Inferences +- The MCP surface is agent-first and cost-safe; Higgsfield's equivalent is a consumer UI. dw's differentiator is that every shot is a replayable JSON job with a manifest, notes, rerun-with-new-seed, and measured warnings. + +### Gaps +- No MCP tool for timeline editing per se; assembly is "run another workflow". No vision-based identity check (face similarity across shots) — consistency is judged by eye from frames. + +## How is fine-grained control exposed today? + +### Takeaway +Control is through workflow JSON (variables with declared constraints, step arguments, pipeline/quantization/acceleration blocks), stored prompts/assets, and a Svelte web UI with a step/flow editor, gallery, assets, compare and job pages. There are no named camera-move or lens presets; camera and shot type are written into the prompt caption. + +### Cited Findings +- Workflow schema `dw/workflow_schema.json`; step types pipeline, pipeline-reference, task, workflow; cross-step data flow by variable/previous_result references — [docs/WORKFLOW_GUIDE.md](docs/WORKFLOW_GUIDE.md) "Structure", "Step Types", "Cross-Step Data Flow" +- Model rules about values are declared in the workflow (`dw/variable_constraints.py`), not engine code — [CLAUDE.md](CLAUDE.md) "Before you edit" +- Saving: each saving step has `subfolder` `final` vs `intermediate` and `role`; `list_gallery(subfolder="final")` lists deliverables — [docs/WORKFLOW_GUIDE.md](docs/WORKFLOW_GUIDE.md) "Saying which output is the deliverable"; [plugins/dw/skills/ltx-2.5/SKILL.md](plugins/dw/skills/ltx-2.5/SKILL.md) +- H3 advanced knobs documented: held soundtrack, `guides`, latent upscale/decode to 768p, `refine_strength`, explicit refine sigmas, regional edit with `source_video`+mask, tiled refine above 768p, chained generation — [docs/WORKFLOW_GUIDE.md](docs/WORKFLOW_GUIDE.md) sections at lines ~1745-2993 +- Web UI pages: Assets, Compare, Editor, Gallery, Job (`ui/src/lib/pages/`); editor components include FlowView (step graph), StepEditor, ArgumentsEditor, VariablesForm, LorasEditor, QuantizationEditor, EnhancePanel (prompt enhancer), MediaArgumentInput (asset/prompt/output pickers), ValidationPanel — [ui/src/lib/pages/](ui/src/lib/pages/), [ui/src/lib/editor/](ui/src/lib/editor/) +- Camera control in practice: LTX captions must state shot type, camera motion and viewpoint; H3 `composable-references` takes a video reference "for framing and camera" — [plugins/dw/skills/ltx-2.5/SKILL.md](plugins/dw/skills/ltx-2.5/SKILL.md); [plugins/dw/skills/minimax-h3/SKILL.md](plugins/dw/skills/minimax-h3/SKILL.md) +- Story-forge lesson L2 ("i2v invents camera moves unless pinned"; add "locked-off on a tripod, no pan, no zoom") was **not reproduced** on dw's LTX pipeline (0 of 6 clear moves, 2 ambiguous); one 241-frame i2v did push/arc after ~5 s despite "camera remains static" — [docs/proposals/audits/2026-11-story-forge-lessons.md](docs/proposals/audits/2026-11-story-forge-lessons.md) "Verdicts", "L2" +- Grep: "dolly|crane shot|orbit" appears in 1 file (docs/RELEASING.md, unrelated); "camera motion|camera move" in 16 files (TASKS.md, audits, plans) — no preset list — repo grep, this session + +### Inferences +- A Higgsfield-style camera-preset library (dolly-in, crane, FPV, orbit, crash zoom...) would in dw be a prompt-library folder (`prompt:camera/`) per family caption format, or a video reference (H3 `composable-references`) — no engine work needed for the first form, but efficacy is unmeasured. + +### Gaps +- Whether H3/LTX actually honour named camera moves reliably has not been probed in dw beyond the static-camera L2 test. + +## What is explicitly missing or documented as a limitation; related open/closed issues? + +### Takeaway +Open backlog is small (11 open issues) and mostly parked with Don: generative reframe/outpaint via Wan VACE, colour-match for seams/reframe, chain-seam improvements, FastH3 checkpoint, MOSS-Image text rendering. The Longshot Studio ideas (#1091-#1095) are all closed/shipped (dialogue/effects stems, H3 face refine, cutaway/lead-in joins, odd-then-bridge, phrase timing without Whisper). Lip-sync is H3-native only (no dedicated lip-sync/dubbing model). Relight, novel view, character swap and camera presets are absent. + +### Cited Findings +- Open issues (all): #1139 audio content-hash identity check; #1083/#1067 colour-match generated region to source (reframe, chain seams), plan-approved, parked; #1066 generative video reframe — outpaint to a new aspect ratio with Wan 2.1 VACE (prior art justae-flow, "meant for a cloud GPU with 48 GB or more"), parked; #1044 H3 chain seams carried as latents + crossfade; #1035 LTX finish starting sigma A/B; #1033 colour-match each chained segment start; #1032 last_frame continuity uses sharpest of last N; #1030 decode a segment chain once; #980 FastH3 Omni 8-step Ref2VA transformer; #968 MOSS-Image-SFT (text rendering + JSON layout) — `gh issue list --state open` on dkackman/diffusers-workflow, 2026-11-01 +- Longshot Studio ideas (concept only; source repo has no licence): #1091 dialogue vs effects stem separation [CLOSED]; #1092 H3 face refine, low-denoise redraw of tracked face crop with audio held [CLOSED]; #1093 cutaway and lead-in joins [CLOSED]; #1094 odd-shots-then-bridge rendering for long H3 takes [CLOSED]; #1095 phrase starts from stereo centre, no-Whisper timing for `plan_cuts` [CLOSED, shipped as `analyze_phrases`] — `gh issue view 1091..1095`; [docs/proposals/todo.md](docs/proposals/todo.md) +- Closed lip-sync lineage: #852/#877/#878/#879 Ref2VA keyframe+audio as locked-frame lip-sync; #488/#617 lip-sync target check; #485 `attribute_voices`; #795/#808/#619 `hold_audio` music-video lip-sync A/B; #802 `measure_sync`; #788 soundtrack start-frame sync; #984 H3 audio refinement pass; #1061/#1074 chain following consecutive windows of a long source video; #865 H3 quality tier preset — `gh issue list --state closed --search ...` +- Locked-frame lip-sync to a song: FL2VA with `hold_audio` keeps first-frame pixels and the mouth follows the track (#985) — [plugins/dw/skills/minimax-h3/SKILL.md](plugins/dw/skills/minimax-h3/SKILL.md) +- Proposal backlog remaining: orphaned run directories (Tier 2), resume of crashed multi-step runs and maintenance screen (Tier 3); recent shipped: MMAudio picture-synced SFX (#1124; weights CC-BY-NC 4.0 / Apple AMLR), story-forge lessons check (#1123), phrase timing (#1095) — [docs/proposals/todo.md](docs/proposals/todo.md) +- Other open proposals: `801-music-video-per-entry-references.md`, `h3-guide-vram.md`, `resume.md` — [docs/proposals/](docs/proposals/) +- MMAudio runs 7-25 dB hot — `match_levels` first — [plugins/dw/skills/series-episodes/SKILL.md](plugins/dw/skills/series-episodes/SKILL.md) +- LTX `diffusion-decode` OOMs on 24 GB without a natten build; `alpha-matte` caps at 145 frames; colourise reliable on 24 fps 16:9 only; restore templates each invert one defect — [plugins/dw/skills/ltx-2.5/SKILL.md](plugins/dw/skills/ltx-2.5/SKILL.md) +- Wan/VACE appears only in docs/RECIPES_24GB.md and a stabilization survey; no Wan template exists — repo grep, this session + +### Inferences +- Against Higgsfield: dw covers keyframes, extend/chain, reference identity, multi-shot cuts, upscale/restore/colourise/matte, music/SFX/TTS/voice clone, prompt rewriting and agent orchestration. It lacks: named camera-motion presets, lens/shot-type preset vocabulary, relighting, re-shoot/novel-view (re-angle a shot), character/face swap in video, dedicated lip-sync/dubbing onto existing footage (LatentSync-class; only a passing mention in output-assessment-complete.md), aspect reframe (#1066 parked), seam colour-match (#1067 parked), an automated identity-consistency check, and a visual storyboard/animatic deliverable. +- The 24 GB single-card constraint shapes everything: every template "fits a 24 GB card", which excludes large models (e.g. Wan VACE 14B) that Higgsfield-class products rely on. + +### Gaps +- I did not read docs/ARCHITECTURE.md (97 KB) in full; its limitation rows were not mined beyond the grep results above. +- Closed-issue search was keyword-limited (camera/lip/relight/storyboard/animatic/swap/shot); other relevant closed issues may exist. +- No GPU runs; no measurement of quality vs Higgsfield. diff --git a/docs/proposals/audits/2026-11-higgsfield/notes/higgsfield_feature_catalog.md b/docs/proposals/audits/2026-11-higgsfield/notes/higgsfield_feature_catalog.md new file mode 100644 index 000000000..785104c6d --- /dev/null +++ b/docs/proposals/audits/2026-11-higgsfield/notes/higgsfield_feature_catalog.md @@ -0,0 +1,156 @@ +# Higgsfield AI feature catalog and control surfaces (as of Nov 2026) + +Source caveat for the whole file: most detailed material comes from Higgsfield's own help center / blog (vendor claims, but usually accurate on *what controls exist*). `geo.higgsfield.ai` pages are Higgsfield SEO marketing and were used only for feature names. Third-party guides (vidpros, memons, seedance.tv) disagree on numbers in places; conflicts are flagged. No independent hands-on review with measured quality was found. The name "Openjutsu" from the brief was not found anywhere; the real product is **Genjutsu** (Sept 2026). + +## What is the full list of Higgsfield apps/features as of late 2026, and which are the most popular? + +### Takeaway +Higgsfield is a multi-model aggregator (30+ third-party models: Seedance, Kling, Veo, Sora 2, Wan, Nano Banana, Hailuo) wrapped in preset-driven "apps". Its flagship surfaces in late 2026 are Cinema Studio 4.0, Genjutsu (video-to-video recast/swap), Supercomputer (agent), Marketing Studio, Canvas (node graph), Soul/Soul ID (own image model + identity), plus a long tail of single-purpose apps (Angles, Shots, Relight, Upscale, Effects, Lipsync Studio, Recast, Popcorn, Draw-to-Video). No usage/popularity statistics were found. + +### Cited Findings +**Current homepage (fetched 2026-11-01)** +- Homepage features: Higgsfield Genjutsu ("One upload in. Endless new visions out." - takes motion from a source and recasts it with your characters/locations/products, or swaps specific elements), Genjutsu Lite (faster/cheaper), community Genjutsu presets, Higgsfield Playable (mobile game from one prompt), Content Factory (trending styles -> ready-to-post series), Cinema Studio 4.0, AI Influencer Studio, Supercomputer Agent, Higgsfield MCP for Claude; models named Kling 4.0, Seedance 2.5 (1080p option), Sora 2, Kling 3.0, Veo 3.1; "30+ models", "40+ creative AI tools", face swap, virtual try-on; Academy and Global Film Festival — [higgsfield.ai](https://higgsfield.ai/) + - Note: the homepage text also says Supercomputer is "powered by GPT-6 Astra"; OpenAI customer-story pages reference "Astra" too — [openai.com Astra page](https://openai.com/es-419/index/higgsfield-from-prompt-to-production-with-astra/). Unverified; treat as vendor branding. + +**Higgsfield's own top-level product list (help center "What is Higgsfield")** +- Soul (native image model family, cinematic/fashion/culture visuals); Soul ID (train a character from photos, face reused across images and videos); Cinema Studio (camera control, multi-shot scenes, physics-aware motion); Marketing Studio (product URL or image -> finished video ad in formats like UGC, unboxing, tutorial, TV spot, with audio and lip sync); Canvas (node-based board chaining models into reusable pipelines, real-time collaboration); Supercomputer (plain-language request -> plan -> assets); Higgsfield Audio (voiceover, voice cloning, translation); external models Seedance, Kling, Veo, Sora 2, Nano Banana — [Higgsfield help center](https://higgsfield.ai/creator-hub/help-center/getting-started/what-is-higgsfield) + +**Per-app catalog (input -> knobs -> output -> model)** + +| App | Input | Knobs | Output | Model / notes | Source | +|---|---|---|---|---|---| +| Camera Controls / DoP (2025 origin product) | single image (+ prompt) | pick one camera-move preset (see Q4), aspect 16:9/9:16/1:1 | ~5 s clip | Higgsfield "DoP" model; early Mar 2025 marketing | [memons cheat sheet](https://memons.ai/higgsfield-prompt-cheat-sheet); [X post Mar 31 2025](https://x.com/higgsfield_ai/status/1906748661322711305) | +| Effects library / Effects Mix | image | pick effect(s); Mix stacks two, e.g. "Set On Fire + Action Run", "Melting + Soul Jump", "Crash Zoom In + Invisible" | short clip | Effects Mix was live for all users incl. free trial; blog ~1 yr old (likely 2025) | [Higgsfield blog Effects Mix](https://higgsfield.ai/blog/Higgsfield-Effects-Mix-Lets-You-Tell-Bigger-Stories) (via search summary; page fetch did not surface text) | +| Viral presets | image | named template | short clip | "13 new viral presets" lists exist | [miracamp](https://www.miracamp.com/learn/video-editing/new-higgsfield-viral-presets) | +| Soul / Soul 2.0 | prompt (+ style presets) | aesthetic presets | photo | Higgsfield's own image model | [Soul intro](https://higgsfield.ai/soul-intro) | +| Soul ID | >=20 photos of a person | training (~3 min) | reusable identity for Soul generations | own model | [search summary of Higgsfield help](https://higgsfield.ai/creator-hub/help-center/ai-models/how-do-i-use-popcorn) | +| Popcorn (storyboard) | prompt and/or up to 4 image refs (cited "image one" etc.) | Manual (per-frame subject/setting/style/atmosphere) vs Auto (one prompt + frame count); up to 8 frames; aspect 3:4, 2:3, 3:2, 4:3, 16:9, 1:1, 9:16, Auto; Instadump mode (1 selfie + style pack of up to 15 refs) | consistent frame sequence; frames used as start frames for Kling/Veo, last frame chained to next | model unnamed | [Popcorn help](https://higgsfield.ai/creator-hub/help-center/ai-models/how-do-i-use-popcorn) | +| Shots | one image | select favorites, upscale to 4K | 9 cinematic angles | — | search summary, [Angles app](https://higgsfield.ai/app/angles) | +| Angles / Angles 2.0 | one JPG/PNG | new viewpoint | re-angled image | also shipped as Photoshop + Figma plugins | [Chase Jarvis review](https://chasejarvis.com/blog/higgsfield-angles-2-0-is-here-my-100-honest-review/); [Photoshop plugin](https://higgsfield.ai/plugins/photoshop) | +| Relight | photo (later video) | light-source angle, color temperature, brightness; "understands 3D geometry" | relit image | Jan 2026 photo launch; Aug 2026 "Video Relight and Color Palette" standalone tools | [PetaPixel Jan 2026](https://petapixel.com/2026/01/09/this-ai-app-lets-you-relight-your-photos/); [Relight blog](https://higgsfield.ai/blog/Relight-Director-Style-Cinematic-Lighting) | +| Upscale (video) | video | one-pass deflicker + stabilize + denoise + upscale | upscaled video | — | [Upscale blog](https://higgsfield.ai/blog/Upgrade-the-Quality-of-Old-Videos-using-Upscale) | +| Photoshop plugin | — | Angles, Mockup Studio, Expand, Remove BG, Relight | — | — | [Photoshop plugin](https://higgsfield.ai/plugins/photoshop) | +| Draw-to-Video | sketch/annotations over an image | draw to insert products, outfits, objects into frame | video | — | [UGC/ads blog](https://higgsfield.ai/blog/Create-Selling-Content-Using-Higgsfield) | +| Lipsync Studio / Speak | photo or character + script/audio | engine choice: Speak v2 / Speak 2.0, lipsync-2, InfiniteTalk, Kling AI Avatar (2.0), Kling Lipsync, Veo 3 | talking avatar / redubbed video | Sept 2025 blog; reviewer saw timing drift in close-ups on Kling-based lipsync | [Lipsync Studio blog Sep 18 2025](https://higgsfield.ai/blog/19fPMcKM5T0c8o8HQGdgyW); [best lip sync 2026](https://higgsfield.ai/blog/best-ai-lip-sync-video-generators-2026) | +| UGC Factory (older) -> Marketing Studio (current) | product idea / product URL or image | templates: Unboxing, Product Review, SaaS, Try-On, Tutorial, Localization; one avatar held through clip | creator-style ad with audio + lip sync | — | [UGC Factory blog](https://higgsfield.ai/blog/Higgsfield-UGC-Factory-Explained); [help center](https://higgsfield.ai/creator-hub/help-center/getting-started/what-is-higgsfield) | +| Recast | 5-15 s video + character image | swap subject keeping camera, lighting, voice (voice clone) | recast video | "Recast Studio" under Character, plus simpler Apps version; third-party claims | [perplexityaimagazine](https://perplexityaimagazine.com/?p=1015); [face swap guide](https://higgsfield.ai/blog/AI-Face-Character-Swap-in-Video-Photo-PRO-Guide) | +| Genjutsu (Sept 1 2026) | reference video 3-30 s + up to 40 reference images (another source: up to 30) | modes: Motion Transfer (keep motion/camera/timing/lip sync, rebuild cast + location + look) and Object Swap (change one element: character, wardrobe, product, location); Lite tier; community presets | new video | model unnamed; mixed reception (praise for motion transfer and lip sync; "scam" complaints over cost/access) | [creativeainews](https://www.creativeainews.com/blog/higgsfield-genjutsu-ai-video-motion-transfer-2026/); [postium](https://postium.ru/higgsfield-zapustila-genjutsu/) | +| Wan Animate / character swap | video + character image | — | video | Wan-based | [lilys.ai note](https://lilys.ai/de/notes/higgsfield-ai-20260128/higgsfield-wan-animate-video-magic) | +| Canvas | nodes | chain models, save reusable pipelines, real-time collab | any | — | [help center](https://higgsfield.ai/creator-hub/help-center/getting-started/what-is-higgsfield) | +| Higgsfield Assist | chat | prompt help | — | named only | [memons](https://memons.ai/higgsfield-prompt-cheat-sheet) | +| Supercomputer (agent) | natural-language goal | see Q6 | multi-asset deliverables | see Q6 | [Supercomputer help](https://higgsfield.ai/creator-hub/help-center/tools-and-workflows/how-do-i-use-supercomputer) | +| Faceless channel (Supercomputer skill) | topic | one of 12 visual styles | YouTube video up to 15 min: AI script, voiceover, 2D animation, music, subtitles | — | [Supercomputer help](https://higgsfield.ai/creator-hub/help-center/tools-and-workflows/how-do-i-use-supercomputer) | + +- Directory listing of key features: Cinema Studio 4.0, multi-model generation, Soul ID and AI Cast, Canvas, Marketing Studio — [yespress](https://yespress.io/products/higgsfield.md) +- Video length extended from ~1 minute to over ten minutes (OpenAI customer page, Oct 2026) — [openai.com](https://openai.com/hy-AM/index/higgsfield-longer-videos-voice/) +- Pricing tracker: "Supercomputer" tier launched May 2026; Starter/Plus/Ultra prices raised Q2 2026 — [pricingsaas](https://pricingsaas.com/companies/higgsfield/diffs/2026W21) (contains an "[INVALID]" line; low confidence) + +### Inferences +- Old/likely-deprecated surfaces: standalone "UGC Factory" appears superseded by Marketing Studio; Recast and Wan-Animate-style swap appear superseded by Genjutsu (the homepage now promotes Genjutsu, not Recast); single-preset "DoP" camera clips are superseded by Cinema Studio's genre/MoveSet controls but the preset library still exists. +- Popularity signals (qualitative only): the 2025 viral growth was driven by the one-click camera presets and Effects (Crash Zoom, Bullet Time, Set On Fire etc. on X/TikTok); 2026 attention centers on Cinema Studio and Genjutsu. + +### Gaps +- No usage numbers per app; "most popular" cannot be stated with evidence. +- Effects library size and full effect-name list not retrieved (the Effects Mix blog text did not load). +- Exact status of Speak, Recast, Shots, UGC Factory in Nov 2026 nav was not verifiable (homepage fetch exposes only promoted items). + +## For Cinema Studio / director mode: what exact controls (camera body, lens, focal length, aperture, genre, movement) are exposed? + +### Takeaway +Cinema Studio is a "build the rig" UI: pick camera body, lens, focal length, aperture per shot; pick genre/style/lighting/palette/MoveSet globally. It has been versioned fast (v1 Dec 2025 -> 4.0 by late 2026) and users can still select older versions (2.0-4.0). The control vocabulary changed between versions, and sources disagree on the exact lists. + +### Cited Findings +- Version history: v1 shipped Dec 18 2025 with camera, lens, focal length (movement + slow motion later); v1.5 added aperture and recommended camera presets; v2/2.5 added Elements, 3D Scene, Character Motions, locations, Soul Cinema; v3.0 April 2026 ("turbulent" rollout); v3.5 ten days later added multishot, genre-based camera, video-first generation, projects, AI Director; v4.0 up to 720p, up to 30 s, Montage Pacing, AI Cast, Cinematic Locations, Soul Cinema models, multi-model, "anti-slop camera pipeline" — [How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio) +- Philosophy quote: "Instead of describing a look in words, you build the frame the way a cinematographer would"; a former DP shaped controls; lenses hand-tuned — "we looked for each lens's character, and we amplified it"; settings + metadata saved with each image so a still used as a video start frame carries the look — [How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio) +- Version selector 2.0-4.0, 4.0 default. 4.0 runs on Seedance 2.5 — [Cinema Studio help](https://higgsfield.ai/creator-hub/help-center/tools/how-do-i-use-cinema-studio) +- **2.5:** up to 3 Soul Cast characters (saved Elements or new); location in prompt; post "Colorgrade": Color Temperature, Contrast, Saturation, Sharpness, Film Grain, Highlights, Exposure; "Save to Elements" — same source +- **3.0:** up to 9 references (characters, locations, scene details); genres General, Action, Horror, Comedy, Noir, Drama, Epic; speed ramps Linear, Auto, Flash In, Flash Out, Slow-mo, Bullet Time, Impact, Ramp Up; native audio (SFX, speech, music) in same pass; video uploads up to 15 s — same source +- **3.5:** per-shot Camera, Lens, Focal length, Aperture; global Genre, Style, Lighting, Color Palette, Camera MoveSet Style (8 palettes, 10 MoveSet styles "inspired by real-world cinematographers", 6 lighting presets); Elements = Characters, Locations, Props; AI Director (Claude Chat panel) breaks a script into shots with camera params pre-filled; shared projects — same source +- **4.0 Director's Panel:** Film Setup genres (General, Action, Epic, Drama, Comedy, Horror, Noir), Era (60s-2020s), Tempo (Single Shot ... Chaotic); camera bodies "35mm Film", "8mm Film", "DV Camcorder"; lens families clean sharp / anamorphic / vintage; aperture control; moves incl. "POV", "Robot Arm", "Helicopter Shot"; 50+ color-grading templates; lighting 6 presets or custom color/brightness/diffusion/angle; up to 50 references (images, videos, Soul Cast characters); character **emotion slider** (performance expressiveness); clips up to 30 s; modes References / Forward Extend / Backward Extend (up to 30 s) / Edit Video (regional object/face edits) — same source +- 2.0/3.5-era camera list (third party, June 2026): bodies RED V-Raptor, Sony Venice, ARRI Alexa, IMAX, Panavision DXL2; lenses Petzval, JDC Xtal Xpress, Cooke S4, Panavision C-Series, Lensbaby, Zeiss Ultra Prime, Helios, ARRI Signature Prime, Canon K-35, Laowa Macro, Hawk V-Lite; focal lengths 8, 14, 35, 50, 75 mm — [vidpros](https://vidpros.com/higgsfield-cinema-studio/) +- Conflict on focal length: help center (per search summary) says 12-135 mm; Seedance guide and vidpros say 8/14/35/50/75 mm — [seedance.tv](https://www.seedance.tv/blog/higgsfield-cinema-studio-camera-settings); [vidpros](https://vidpros.com/higgsfield-cinema-studio/) + +### Inferences +- Trajectory: from literal hardware names (ARRI/RED/Cooke) toward abstract "looks" (35mm Film / 8mm / DV, lens families, era, tempo) as multi-shot made per-shot camera specs less useful — the blog says camera control "moved up a level to genres". +- Under the hood these controls are almost certainly prompt-composition + curated per-model prompt templates (an "enhancer" rewrites sparse input), not model conditioning — consistent with running on third-party Seedance. + +### Gaps +- Numeric aperture values and the 10 MoveSet style names were not found. +- Output resolution for 3.x not stated; 4.0 "up to 720p" (blog FAQ) conflicts with homepage "Seedance 2.5 1080p" option. + +## How does Higgsfield achieve character/subject consistency across shots (Soul ID, reference images)? + +### Takeaway +Three layers: a trained identity (Soul ID, >=20 photos, ~3 min) for its own Soul image model; reusable named "Elements"/"Soul Cast"/"AI Cast" assets (characters, locations, props) that are injected as references into third-party video models (up to 9 refs in 3.0, up to 50 in 4.0); and project-level locked style (lighting, optics, tempo) plus within-sequence consistency in Popcorn. + +### Cited Findings +- Soul ID needs >=20 photos, trains ~3 min; holds one face across separate Soul generations; Popcorn instead keeps consistency within one sequence and can take a Soul character as a reference — [Popcorn help](https://higgsfield.ai/creator-hub/help-center/ai-models/how-do-i-use-popcorn) +- Elements (Characters, Locations, Props) saved and reused; Soul Cast characters (3 in 2.5); refs 9 (3.0) -> 50 (4.0) — [Cinema Studio help](https://higgsfield.ai/creator-hub/help-center/tools/how-do-i-use-cinema-studio) +- Projects keep light, optics and tempo consistent across all generations in the project — [How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio) +- Settings metadata saved with each still so it carries into image-to-video — same source +- Popcorn workflow: last frame of one clip chained as start of next — [Popcorn help](https://higgsfield.ai/creator-hub/help-center/ai-models/how-do-i-use-popcorn) +- Marketing Studio/UGC holds one avatar across a clip — [UGC Factory blog](https://higgsfield.ai/blog/Higgsfield-UGC-Factory-Explained) + +### Inferences +- Video-side consistency depends on the underlying model's reference-to-video support (Seedance 2.x / Kling multi-reference); Higgsfield's contribution is asset management + numbered reference citation ("image one") in prompts. + +### Gaps +- How Soul ID is implemented (LoRA vs. adapter) is not disclosed. +- An Oct 2026 analysis on character consistency exists but was not read — [aiproductivity.ai](https://aiproductivity.ai/blog/higgsfield-consistent-ai-character/) + +## What camera-motion preset vocabulary do they use (names of moves)? + +### Takeaway +A named library (counts cited from 50+ to 100+) grouped roughly as zooms, dollies, vertigo, orbit/arc, aerial, signature FX, and time-lapse. Users pick a preset; the cheat-sheet convention is to also echo the preset in the prompt. + +### Cited Findings +- Preset names (Aug 2026 cheat sheet): Zoom — Crash Zoom In, Crash Zoom Out, Rapid Zoom In, Rapid Zoom Out, YoYo Zoom; Dolly — Dolly In/Out/Left/Right, Super Dolly In/Out, Double Dolly; Vertigo — Dolly Zoom In/Out; Orbit/Arc — 360 Orbit, Arc Left, Arc Right, Lazy Susan; Aerial — FPV Drone, Crane Up/Down, Jib Up/Down, Overhead; Signature FX — Bullet Time, Snorricam, Robo Arm, Whip Pan, Through Object In/Out, Object POV; Time — Hyperlapse, Timelapse Human, Timelapse Landscape, Timelapse Glam. Duration ~5 s for Higgsfield's own moves; prompt convention "(Preset: Crash Zoom In · 9:16 · 5s)" — [memons cheat sheet](https://memons.ai/higgsfield-prompt-cheat-sheet) +- Other sources confirm Dolly In, Crane Up/Down, Tracking, Whip Pan, Snorricam, Bullet Time, Boom; counts "50+" vs "20+" vs "100+" — [BibiGPT](https://bibigpt.co/en/features/higgsfield-ai-video-generator-explained); [kolbo](https://kolbo.ai/blog/higgsfield-suite-100-camera-presets) +- Crash Zoom In, bullet time, super dollies, robo arms promoted Mar 31 2025 — [X](https://x.com/higgsfield_ai/status/1906748661322711305) +- Cinema Studio 3.0 speed ramps: Linear, Auto, Flash In, Flash Out, Slow-mo, Bullet Time, Impact, Ramp Up; 4.0 moves incl. POV, Robot Arm, Helicopter Shot — [Cinema Studio help](https://higgsfield.ai/creator-hub/help-center/tools/how-do-i-use-cinema-studio) +- Combo motions (Effects Mix pairs camera + effect, e.g. Crash Zoom In + Invisible) — [Effects Mix blog](https://higgsfield.ai/blog/Higgsfield-Effects-Mix-Lets-You-Tell-Bigger-Stories) + +### Inferences +- Names mix real grip terms (Snorricam, Jib, Lazy Susan, Robo Arm) with TikTok-native ones (YoYo Zoom, Timelapse Glam) — the vocabulary is a UX product in itself. + +### Gaps +- No official, complete, dated preset list; cheat-sheet list is third party. +- Intensity/strength knob per preset: not confirmed in sources. + +## How do they handle multi-shot sequences, storyboards, and audio (dialogue, music, SFX)? + +### Takeaway +Multi-shot arrived in Cinema Studio 3.5 (April 2026) with genre-driven camera and a "Tempo"/"Montage Pacing" control in 4.0; storyboards via Popcorn (up to 8 frames); continuation via Forward/Backward Extend; audio is native from the underlying model (3.0+) plus a separate Higgsfield Audio suite and Lipsync Studio. + +### Cited Findings +- 3.5 added multishot; single continuous camera moves became insufficient so camera control moved to genres — [How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio) +- 4.0: Tempo (Single Shot -> Chaotic), clips up to 30 s, Forward/Backward Extend, Edit Video — [Cinema Studio help](https://higgsfield.ai/creator-hub/help-center/tools/how-do-i-use-cinema-studio) +- Enhancer example input "generate me a 50-shot battle scene in 15 seconds" — [How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio) +- 3.0 audio: SFX, speech, background music in same pass — [Cinema Studio help](https://higgsfield.ai/creator-hub/help-center/tools/how-do-i-use-cinema-studio) +- Higgsfield Audio: voiceover, voice cloning, translation — [help center](https://higgsfield.ai/creator-hub/help-center/getting-started/what-is-higgsfield) +- Lipsync engines: Speak v2, lipsync-2, InfiniteTalk, Kling AI Avatar, Kling Lipsync, Veo 3 — [Lipsync Studio blog](https://higgsfield.ai/blog/19fPMcKM5T0c8o8HQGdgyW) +- Popcorn storyboards up to 8 frames, then animate per frame and chain last frame — [Popcorn help](https://higgsfield.ai/creator-hub/help-center/ai-models/how-do-i-use-popcorn) +- Long-form: videos extended to >10 min (Oct 2026) — [openai.com](https://openai.com/hy-AM/index/higgsfield-longer-videos-voice/) + +### Inferences +- Multi-shot inside one generation is delegated to the base model's native multi-shot ability (Seedance 2.x); Higgsfield supplies pacing/genre presets around it. Long-form >10 min is presumably agent-stitched. + +### Gaps +- Music generation model and SFX controls not identified. + +## Are there agent/assistant features that plan shots from a script? + +### Takeaway +Yes. Cinema Studio 3.5's AI Director (a Claude chat panel) breaks a script into shots with camera parameters pre-filled; it was spun out into Supercomputer, a chat agent that plans multi-step jobs and routes each sub-task to a model. Also MCP for Claude, a ChatGPT plugin, and a CLI. + +### Cited Findings +- AI Director (Claude Chat) breaks a script into shots with camera params pre-filled — [Cinema Studio help](https://higgsfield.ai/creator-hub/help-center/tools/how-do-i-use-cinema-studio); it later became Supercomputer — [How we built Cinema Studio](https://higgsfield.ai/blog/how-we-built-cinema-studio) +- Supercomputer: "agentic workspace"; plans, generates across image/video/audio, re-runs only affected steps on revision; auto-routes generation models (Soul, Cinema Studio, Seedance, Nano Banana); chat LLM selectable (Anthropic, OpenAI, Google, xAI, DeepSeek) or auto; Memory; connectors (Telegram, Slack, TikTok, Gmail, Google Drive, Notion); can build/deploy apps/sites; skills usable via MCP in Claude or Higgsfield Plugin in ChatGPT; per-step credits shown live — [Supercomputer help](https://higgsfield.ai/creator-hub/help-center/tools-and-workflows/how-do-i-use-supercomputer) +- Built on open-source Hermes Agent; memory mostly manual at launch (reviewer); launch date conflicting (May vs later) — [popularaitools review](https://popularaitools.ai/blog/higgsfield-supercomputer-review/); [Higgsfield build blog](https://higgsfield.ai/blog/how-we-built-supercomputer) +- CLI plus agent features cited as signal of a "full AI production studio" (Oct 2026 newsletter) — [mean.ceo](https://blog.mean.ceo/?p=11502) + +### Inferences +- UX philosophy overall: presets first (named moves, genres, eras, templates) with an LLM "enhancer" hiding prompt writing; fine control (lens/aperture) offered but increasingly abstracted; consistency via saved asset libraries + projects; and now an agent on top that picks models and composes steps — closely analogous to dw's workflow catalog + MCP agent loop. + +### Gaps +- No public detail on how AI Director's shot breakdown is structured (fields per shot) beyond "camera parameters pre-filled". diff --git a/docs/proposals/audits/2026-11-higgsfield/notes/open_models_and_agents.md b/docs/proposals/audits/2026-11-higgsfield/notes/open_models_and_agents.md new file mode 100644 index 000000000..3b2bb70bc --- /dev/null +++ b/docs/proposals/audits/2026-11-higgsfield/notes/open_models_and_agents.md @@ -0,0 +1,193 @@ +# Open-weights models, LoRAs and agents for Higgsfield-style cinematic controls (as of 2026-11-01) + +Scope note: about 17 searches and fetches. Primary sources (Hugging Face model listings and cards, GitHub, arXiv, docs.ltx.io) are preferred. Where only aggregators or vendor blogs were found, that is said. Anything I know only from training data, and could not re-verify this session, is in **Inferences** or **Gaps** and marked "(unverified)". The dw engine's supported families are MiniMax H3, LTX-2.5 (22B, `Lightricks/LTX-2.5-Diffusers`), Kandinsky 6 and FLUX 3 Klein. The repo already catalogues these LTX-2.5 IC-LoRAs: Pixel-Spatial-Upscaler, Deblur, Ingredients, Decompression, Restore, Refine-Details, Colorization and Alpha-Gen (grep of `dw/`, `workflows/` and `plugins/`). + +## Q1. Camera motion control in video (dolly, orbit, crane, crash zoom), and which engine families it fits + +### Takeaway +Lightricks publishes the only first-party, named-preset camera LoRAs. These are seven moves: dolly in, dolly out, dolly left, dolly right, jib up, jib down and static. **They target LTX-2 19B, not LTX-2.5 22B**, and I found no LTX-2.5 camera-preset LoRA. On LTX-2.5, camera moves come from the prompt: the trained caption spec requires a stated camera move. Orbit and crash-zoom presets exist only as Wan 2.1 community effect LoRAs. Trajectory-conditioned methods (ReCamMaster, Uni3C, and older ones like CameraCtrl and MotionCtrl) are Wan- or SVD-based and would need a new base model plus custom loaders. + +### Cited Findings +- The Lightricks Hugging Face listing (sorted by last modified, all re-uploaded 2026-10-22) has these camera LoRAs: `Lightricks/LTX-2-19b-LoRA-Camera-Control-{Dolly-In, Dolly-Out, Dolly-Left, Dolly-Right, Jib-Up, Jib-Down, Static}`. LTX-2 19B control IC-LoRAs: `LTX-2-19b-IC-LoRA-{Union-Control, Pose-Control, Depth-Control, Canny-Control, Detailer}`. LTX-2.3 22B IC-LoRAs: Union-Control, Motion-Track-Control, Relight, In-Outpainting, HDR, DubIt, Clean-Plate, Day-To-Night, Water-Simulation, Instant-Shave, Cross-Eyed and others, plus a `LTX-2.3-22b-LoRA-Foley-V2A`. LTX-2.5 22B: IC-LoRAs Layout-To-Render, Day-To-Night, Clean-Plate, Water-Simulation, SDR-To-HDR and the restoration set, plus plain LoRAs **Slow-Motion-Control** and **Cinemagraph**. **There is no LTX-2.5 or LTX-2.3 camera-preset LoRA in the listing.** — [HF API listing, author=Lightricks](https://huggingface.co/api/models?author=Lightricks&search=LoRA&limit=100&sort=lastModified) +- The Dolly-In card says the LoRA "biases outputs toward a forward push-in". It loads through LTX-2's two-stage HQ pipeline at strength 1.0, alongside the base checkpoint, a distilled LoRA and a Gemma 3 text encoder. — [Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In](https://huggingface.co/Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-In); [Dolly-Out](https://huggingface.co/Lightricks/LTX-2-19b-LoRA-Camera-Control-Dolly-Out) +- The Dolly-Left card advises naming in the prompt what should come into view during the move. — [Dolly-Left mirror](https://huggingface.co/Galatea777/LTX-2-19b-LoRA-Camera-Control-Dolly-Left) +- Lightricks' VFX product page lists "seven pre-trained camera LoRAs (dolly in/out/left/right, jib up/down, static)" under LTX-2.3, plus Union Control and Motion Track Control IC-LoRAs. This conflicts with the HF repo names, which say LTX-2-19b, so it is unclear which base the camera LoRAs are validated on. — [ltx.io VFX capabilities](https://ltx.io/model/capabilities/vfx) +- LTX-2.5 was released on 2026-08-11. Camera moves on 2.5 are mainly prompt-driven: the HF examples include a dolly-out prompt. Official ComfyUI examples cover union control and IC-LoRA workflows, including motion tracking. One aggregator says some LTX-2.3 LoRAs "partly work" with 2.5, with variable results. — [gradually.ai](https://www.gradually.ai/en/ai-models/ltx-2-5/); [Lightricks/LTX-2.5](https://huggingface.co/Lightricks/LTX-2.5); [comfyui-wiki LTX-2.5](https://comfyui-wiki.com/es/models/ltx-video/ltx-2-5); [aireiter (aggregator)](https://aireiter.com/es/blog/ltx-2-5-api-pricing-guide) +- RunComfy's hosted LTX 2.5 Pro image-to-video API exposes an optional "camera-motion preset". That is a hosted-API feature, not a published open LoRA. — [RunComfy](https://www.runcomfy.com/zh-CN/models/lightricks/ltx-2.5/image-to-video/pro/api) +- The dw repo's LTX-2.5 caption spec already demands "Camera motion (always stated; if none, explicitly say the camera remains static)" and a viewpoint (front-facing, over-the-shoulder, top-down, low-angle and so on), written as flowing prose. The Kandinsky 6 skill demands "exactly one camera sentence ('The camera is static.' when none is asked for)". — `plugins/dw/skills/ltx-2.5/references/caption-spec.md` lines 25-27; `plugins/dw/skills/kandinsky-6/SKILL.md` lines 77-78 (local repo) +- Remade-AI publishes Wan 2.1 14B I2V 480p effect LoRAs for camera moves, each with a trigger phrase: + - **360 Orbit**: trigger "0rb4it 360 degree orbit". Strength 1.0, guidance 6.0, flow shift 5.0. Trained on about 50 s of video from 10 clips, for 30 epochs. + - **Crash Zoom In**: trigger "cr34sh crash zoom in effect". + - **Crash Zoom Out** and **Rotate**: listed, but no details found. + + The org title says T2V, but the individual cards say I2V. — [Remade-AI org](https://huggingface.co/Remade-AI); [360-Orbit mirror](https://cnb.cool/ai-models/Remade-AI/360-Orbit); [Crash-zoom-in mirror](https://cnb.cool/ai-models/Remade-AI/Crash-zoom-in); [I2V collection](https://huggingface.co/collections/Remade-AI/wan21-14b-480p-i2v-loras) +- Uni3C (ewrfcas/Uni3C) is a "unified 3D-enhanced framework for precise control of both camera and human motion". It has a pose-extraction step and an inference step. It is used in ComfyUI to transfer camera motion from a reference video. — [cnb mirror](https://cnb.cool/wan2222222/uni3cv2); [RunComfy Uni3C workflow](https://www.runcomfy.com/comfyui-workflows/uni3c-comfyui-workflow-video-referenced-camera-motion-transfer) + +### Inferences +- **Named presets:** two sets of LoRAs are trained on camera vocabulary and can be exposed directly as presets. The Lightricks set has dolly in, dolly out, dolly left, dolly right, jib/crane up, jib/crane down and static. The Remade set has orbit 360, crash zoom in, crash zoom out and rotate, each with a trigger phrase. Neither targets LTX-2.5. For dw, the cheapest path is **prompt presets**: a named move maps to a canonical caption sentence in the LTX-2.5 and Kandinsky 6 skill caption specs, which already require one camera statement. LoRA-backed presets would come later, if Lightricks ships 2.5 camera LoRAs or a 2.3 camera LoRA is shown to load on 2.5. +- The LTX-2.3 and LTX-2.5 IC-LoRA family uses the same mechanism that dw already loads for the LTX-2.5 restoration set. So Motion-Track-Control and Union-Control, if ported to 2.5, would be the lowest-friction route to trajectory and reference-video camera control inside the engine. +- Uni3C, ReCamMaster and the Remade LoRAs all need Wan 2.1 or 2.2 as a new base family. Wan has native diffusers pipelines (WanPipeline, WanImageToVideoPipeline, WanVACEPipeline, and WanAnimatePipeline per the PR cited in Q4), but the camera-conditioning paths of ReCamMaster and Uni3C are not in diffusers. +- CameraCtrl (AnimateDiff and SVD), MotionCtrl (VideoCrafter and SVD) and AC3D are older U-Net or CogVideoX-era methods. As of late 2026 they are superseded by Wan-based methods and by first-party LoRAs (unverified, from prior knowledge). + +### Gaps +- No camera-control LoRA for LTX-2.5 was found, and no statement that the LTX-2-19b camera LoRAs load on 2.5. docs.ltx.io's camera-control page returned 404. +- I could not verify licences for the LTX-2 camera LoRAs this session. They are presumably under the LTX-2 community licence, which in earlier versions allowed commercial use below a revenue threshold (unverified). Remade-AI licences are also unverified. +- No camera-control LoRA or adapter was found for MiniMax H3, Kandinsky 6 or FLUX 3 Klein. H3 camera control appears to be prompt-only. The dw staging notes say "a moving camera weren't tested". +- CameraCtrl, MotionCtrl, TrajectoryAttention and AC3D licences and status were not re-verified. + +## Q2. Re-shooting a video from a new angle (novel-view video) + +### Takeaway +The leading open options are ReCamMaster (KlingTeam, ported to Wan 2.1) and TrajectoryCrafter (Tencent ARC, ICCV 2025). Both are re-render-from-source-video models with preset or explicit trajectories. Neither has a diffusers pipeline, and both need a new base (Wan 2.1, or CogVideoX-based for TrajectoryCrafter, unverified) and a custom loader. + +### Cited Findings +- ReCamMaster (ICCV'25 Oral, Best Paper Finalist) is "camera-controlled generative rendering from a single video". The paper's model is an internal Kuaishou T2V model that "can't be open-sourced". The released checkpoint is a port to Wan2.1, at `KlingTeam/ReCamMaster-Wan2.1`, and its results may not match the paper's. — [GitHub KlingAIResearch/ReCamMaster](https://github.com/KlingAIResearch/ReCamMaster); [README](https://raw.githubusercontent.com/KlingAIResearch/ReCamMaster/fcf98bc86e876bb534518cd99e8a65b282f0f16e/README.md); [HF KlingTeam/ReCamMaster-Wan2.1](https://huggingface.co/KlingTeam/ReCamMaster-Wan2.1) +- A ComfyUI workflow host describes a 1.3B ReCamMaster on the Wan 2.1 architecture. This comes from a marketing page. — [floyo.ai](https://www.floyo.ai/workflows/wan-2-1-recammaster-v2v-camera-contr-f9y6ufftq2i6) +- TrajectoryCrafter (Mark Yu, Wenbo Hu, Jinbo Xing, Ying Shan; Tencent ARC and CUHK; ICCV 2025) generates "high-fidelity novel views from casually captured monocular video" with precise pose control. The repo recommends GPUs with at least 28 GB of VRAM. — [arXiv 2503.05638](https://arxiv.org/html/2503.05638v1); [ICCV paper](https://openaccess.thecvf.com/content/ICCV2025/papers/Yu_TrajectoryCrafter_Redirecting_Camera_Trajectory_for_Monocular_Videos_via_Diffusion_Models_ICCV_2025_paper.pdf) +- ViewCrafter, the same group's earlier static-scene novel-view model, is Apache-2.0 on HF. — [Drexubery/ViewCrafter_25](https://huggingface.co/Drexubery/ViewCrafter_25/raw/35af104d9781e9ed5cdaddfcfbbdca5835e1c36f/README.md) + +### Inferences +- In ReCamMaster, the camera is chosen from a fixed set of preset trajectories (pan, arc, translate, zoom and so on, per the README, unverified in detail). That maps naturally onto named "re-shoot" presets. +- Neither model fits the dw engine's current families. Each would be a new pipeline class wrapping a Wan 2.1 transformer (ReCamMaster), or a point-cloud render plus a video-diffusion stage (TrajectoryCrafter), and that is a custom loader. +- The LTX-2.3 Motion-Track-Control IC-LoRA (Q1) is the nearest LTX-family substitute, though it is not novel-view re-rendering. + +### Gaps +- Licences for ReCamMaster (code and weights) and TrajectoryCrafter were not confirmed. The official TrajectoryCrafter repo page was not returned. +- No 2026 successor to either was found in this session, and no newer novel-view video model was checked (for example Wan 2.2-based or LTX-based ones). + +## Q3. Multi-angle stills from one image ("Shots: 9 angles") + +### Takeaway +The open standard is a Qwen-Image-Edit multi-angle LoRA: fal's 96-position LoRA for the 2511 base, or dx8152's for the 2509 base. Both are prompt-driven ("Rotate the camera 45 degrees to the left"), so they map directly to named angle presets. Qwen-Image-Edit is a native diffusers pipeline (QwenImageEditPlusPipeline). dw already ran Qwen in the 2026-09-20 field report, so this is the lowest-cost add. No equivalent LoRA for FLUX 3 Klein was found. + +### Cited Findings +- `fal/Qwen-Image-Edit-2511-Multiple-Angles-LoRA` is a "multi-angle camera control LoRA" with 96 camera positions, trained on more than 3000 Gaussian-splatting renders. It ships a safetensors file and a ComfyUI workflow, with a suggested strength of 0.9. — [HF fal](https://huggingface.co/fal/Qwen-Image-Edit-2511-Multiple-Angles-LoRA) +- `dx8152/Qwen-Edit-2509-Multiple-angles` (weight file `镜头转换.safetensors`) is wrapped by HF Spaces. Those Spaces build the prompt from rotation, zoom, tilt and wide-angle controls, with positive values rotating left and negative rotating right. — [linoyts Space app.py](https://huggingface.co/spaces/linoyts/Qwen-Image-Edit-Angles/blob/bbbeff133c3a7714b98a8ab99a61895294dfb4cf/app.py); [rideshare Space](https://huggingface.co/spaces/rideshare/Qwen-Image-Edit-Angles/blob/main/app.py) +- Replicate's wrapper uses ±180° rotation and a `multiple_angles_strength` of 0.8-1.3. — [Replicate qwen-edit-multiangle](https://replicate.com/qwen/qwen-edit-multiangle/readme) +- A community Rapid-AIO merge variant is tested with "Rotate the camera 45 degrees to the left." — [Manojb/Qwen-Image-Edit-Rapid-AIO-MultipleAngle](https://huggingface.co/Manojb/Qwen-Image-Edit-Rapid-AIO-MultipleAngle) + +### Inferences +- A "9 angles" preset can be a fan-out over one Qwen-Image-Edit plus multi-angle LoRA step, using dw's list-driven steps, with nine fixed prompt strings. Examples: front, ±45° rotate, ±90°, high angle, low angle, close-up and wide. The Space's rotate, zoom and tilt prompt grammar is the vocabulary to pin. +- FLUX 3 Klein multi-reference editing (in dw already) can approximate angle changes from prompting, but it has no trained camera-pose vocabulary that I found. +- MV-Adapter (SDXL and SD2.1, Apache-2.0, diffusers-based custom pipeline; unverified this session) produces orthographic turnaround views. It is better for character sheets than for cinematic angles. + +### Gaps +- Licences of the fal and dx8152 LoRAs were not checked. The Qwen-Image-Edit base is Apache-2.0 (unverified this session). +- No multi-angle LoRA for FLUX 3 Klein or Kandinsky was found. + +## Q4. Character or actor swap in video, and identity consistency + +### Takeaway +Wan2.2-Animate-14B is the strongest open character-replacement model. Its animation and replacement modes have a diffusers `WanAnimatePipeline`, but the pose, mask and face preprocessing is outside diffusers. Identity-reference options (VACE, Phantom, MAGREF, Stand-In) are all Wan 2.1-based. Within dw's current families, identity holding comes from LTX-2.5's Ingredients IC-LoRA (already catalogued) and H3's Ref2VA reference mode. + +### Cited Findings +- Diffusers PR #12442 adds `WanAnimatePipeline`, loading `Wan-AI/Wan2.2-Animate-14B-Diffusers`. It has Animation Mode (animates a character image with motion from a reference video) and Replacement Mode (replaces a character in a background video). Replacement needs pose video and other extracted inputs. The PR notes "integration of these preprocessing steps into Diffusers is planned for a future release". — [diffusers PR #12442 diff](https://github.com/huggingface/diffusers/pull/12442.diff); [issue #12441](https://github.com/huggingface/diffusers/issues/12441) +- The official card's replacement mode uses `generate.py --replace_flag` plus a relighting LoRA, and the card lists diffusers integration. — [HF Wan-AI/Wan2.2-Animate-14B](https://huggingface.co/wan-ai/wan2.2-animate-14b) +- The ComfyUI version uses a 14B FP8 checkpoint, a SAM2 character mask and DWPose extraction. — [comfy.org workflow](https://comfy.org/hub/workflows/wan2-2-animate-character-animation-and-replacement-c2c4b45b7df5/) +- Wan 2.2 is Apache-2.0, released July 2025. TI2V-5B runs 720p at 24 fps on 8 GB of VRAM. — [MindStudio (aggregator)](https://www.mindstudio.ai/blog/what-is-wan-2-2-video-open-source) +- **Stand-In** is a Wan 2.1 identity LoRA that locks identity from a single image. **Phantom** (ByteDance) is built on Wan 2.1 in 14B and 1.3B sizes and handles cross-modal subject consistency. **MAGREF** generates from multiple reference images, with a Wan2.1 I2V 14B base. **VACE** is Wan's all-in-one editing module (inject subjects, animate, transfer motion). — [RunComfy Stand-In](https://www.runcomfy.com/comfyui-workflows/wan2-1-stand-in-in-comfyui-character-consistent-video-workflow); [comfyui-wiki Phantom](https://comfyui-wiki.com/en/models/wan/phantom); [ThinkDiffusion MAGREF](https://learn.thinkdiffusion.com/magref-generate-ai-videos-with-multiple-people-and-objects-from-images/); [Wan2GP VACE doc](https://huggingface.co/spaces/attong39/Wan2GP/blob/main/docs/VACE.md) +- The repo's own LTX-2.5 catalogue already includes `Lightricks/LTX-2.5-22b-IC-LoRA-Ingredients`, its subject-reference adapter. — local grep + +### Inferences +- An actor swap needs a Wan family plus a preprocessing toolchain (DWPose, SAM2), and that toolchain would be new utility tasks in dw. The diffusers pipeline covers only the diffusion stage. +- Wan VACE has a native diffusers `WanVACEPipeline` (unverified this session). Phantom, MAGREF and Stand-In need custom loaders. + +### Gaps +- Merge status of PR #12442 at Nov 2026 was not confirmed, though it very likely merged in late 2025 (unverified). +- Licences of Phantom, MAGREF and Stand-In were not found in this session's results. Wan2.2-Animate VRAM was not measured. + +## Q5. Lip sync, talking heads and dialogue + +### Takeaway +For dubbing an existing video, LatentSync 1.6 (ByteDance) is the default with a clear commercial path: Apache-2.0 code and OpenRAIL++ weights, about 18 GB of VRAM. MuseTalk (MIT) is the lightweight option. For audio-driven generation of long talking or multi-person video, InfiniteTalk and MultiTalk (MeiGen, Wan-based) lead, and LongCat-Video-Avatar 1.5 (MIT) is a newer option. dw already has native audio-video families (H3 dialogue, LTX-2.5, Kandinsky 6) that generate speech and lips jointly. That is the closest match to Higgsfield's "speak" and should come first. LTX-2.3's DubIt IC-LoRA is the in-family lip-redub option. + +### Cited Findings +- From the 2026-09-16 licence comparison: + - **LatentSync**: Apache-2.0 code, commercial use permitted. 8 GB VRAM for v1.5 and 18 GB for v1.6. v1.6 is trained at 512×512. 25 fps video, 16 kHz audio. + - **MuseTalk**: MIT, commercial use permitted. 4 GB minimum (fp16), 256×256 face region. + - **Wav2Lip**: commercial use "strictly prohibited". + - **SadTalker**: Apache-2.0 with third-party carve-outs. + - **LongCat-Video-Avatar 1.5**: MIT. Whisper-Large audio encoder, 8-step DMD2 distillation, 480p or 720p. + + — [Creative AI News, 2026-09-16](https://www.creativeainews.com/articles/open-source-lip-sync-licenses-commercial-use-2026/) +- LatentSync weights are labelled OpenRAIL++ in HF metadata and the code is Apache. A Civitai page shows CreativeML Open RAIL-M text, which conflicts and is likely a mislabel. — [HF discussion](https://huggingface.co/chunyu-li/LatentSync/discussions/2); [GitHub bytedance/LatentSync](https://github.com/bytedance/LatentSync); [Civitai (conflicting)](https://civitai.com/models/license/2409072) +- InfiniteTalk (MeiGen-AI) is a "sparse-frame video dubbing" framework for unlimited length. It runs on a single GPU, on multiple GPUs, or in low-VRAM setups. It claims better lip sync than MultiTalk and fewer hand and body distortions. These are the project's own claims. — [sourcepulse](https://www.sourcepulse.org/projects/12893512); [knightli (2026-06)](https://knightli.com/es/2026/06/15/infinitetalk-audio-driven-video-generation/) +- Lightricks lists `LTX-2.3-22b-IC-LoRA-DubIt` and `LTX-2.3-22b-LoRA-Foley-V2A`. — [HF listing](https://huggingface.co/api/models?author=Lightricks&search=LoRA&limit=100&sort=lastModified) + +### Inferences +- InfiniteTalk and MultiTalk are built on Wan 2.1 I2V 14B with an audio cross-attention adapter (unverified this session). They need the Wan family plus a custom loader, and InfiniteTalk's licence should be checked (Apache-2.0 per prior knowledge, unverified). +- Hallo3 (CogVideoX-based) and Sonic (Tencent) are, to my knowledge, research or non-commercial licences (unverified). Flag them as non-commercial until checked. +- LatentSync is a standalone U-Net latent-diffusion pipeline, not diffusers-native. It could be a dw utility task, because it is post-processing on an existing clip. + +### Gaps +- No licence was found this session for InfiniteTalk, MultiTalk, Hallo3, Sonic, FantasyTalking, OmniAvatar or Wan2.2-S2V. +- The DubIt IC-LoRA card was not read, so its function (lip re-dub to new audio?) and whether it is validated on LTX-2.5 are unconfirmed. + +## Q6. VFX effect presets (explosion, disintegrate and so on) + +### Takeaway +Effect LoRAs are a community ecosystem centred on Wan 2.1 and 2.2 I2V: Remade-AI's set (orbit, crash zoom, rotate and others) and a few Apache-2.0 one-offs. Older Lightricks LTX-Video effect LoRAs (Cakeify, Squish) target the deprecated LTX-Video 0.9 base. On LTX-2.5, Lightricks' own "effect" adapters are utility-grade (Slow-Motion-Control, Cinemagraph, Water-Simulation, Day-To-Night, Clean-Plate), not Higgsfield-style spectacle presets. + +### Cited Findings +- `Ashmotv/exploded_effect_wan` is an Apache-2.0 Wan2.2-I2V-A14B LoRA with trigger "expl0ded effect". It is an exploded-diagram disassembly effect, not a pyrotechnic explosion. — [HF README](https://huggingface.co/Ashmotv/exploded_effect_wan/blob/main/README.md) +- Remade-AI publishes a collection of Wan 2.1 14B 480p I2V effect LoRAs with trigger phrases. — [Remade-AI collection](https://huggingface.co/collections/Remade-AI/wan21-14b-480p-i2v-loras) +- Lightricks: `LTX-Video-Cakeify-LoRA` and `LTX-Video-Squish-LoRA` (2025-03, LTX-Video 0.9 base). LTX-2.5 22B adapters include `LoRA-Slow-Motion-Control`, `LoRA-Cinemagraph`, `IC-LoRA-Water-Simulation`, `IC-LoRA-Day-To-Night` and `IC-LoRA-Clean-Plate` (2026-10-22). — [HF listing](https://huggingface.co/api/models?author=Lightricks&search=LoRA&limit=100&sort=lastModified) +- A dev.to post claims Higgsfield launched Wan 2.2. That is contradicted by the primary attribution to Alibaba, so treat it as unreliable. — [dev.to](https://dev.to/ytosko/higgsfield-ai-launches-wan-22-its-most-advanced-open-source-model-for-video-amp-image-3373) + +### Inferences +- The LTX-2.5 Slow-Motion-Control, Cinemagraph, Water-Simulation and Day-To-Night LoRAs are immediately usable in dw: same base and same loader path as the catalogued IC-LoRAs. They are good first "effect presets" for dw. +- Higgsfield-style spectacle effects (explode, disintegrate, melt, inflate, levitate) exist in open form mainly as Wan 2.1 I2V community LoRAs (Remade, Civitai). Adopting them requires adding Wan 2.1/2.2 I2V 14B as a family. diffusers' Wan pipelines load LoRAs natively via `load_lora_weights`, so no custom loader would be needed once Wan is in. + +### Gaps +- No open "disintegrate" or pyrotechnic-explosion LoRA was located with a confirmed licence. Civitai was not searched. +- No HunyuanVideo effect-LoRA survey was done. Licences on Remade LoRAs were not checked. + +## Q7. Relighting and lens or cinematography look + +### Takeaway +The most directly usable video relight is Lightricks' LTX-2.3 IC-LoRA Relight (gated). The docs say it is not yet validated on LTX-2.5. Training-free alternatives (Light-A-Video, LightCtrl) build on IC-Light, the image relighter fine-tuned from Stable Diffusion, and have no diffusers pipeline. LTX-2.5 already ships SDR-To-HDR and Day-To-Night IC-LoRAs as look controls. I found nothing on lens or depth-of-field control LoRAs. + +### Cited Findings +- `Lightricks/LTX-2.3-22b-IC-LoRA-Relight` relights exterior video to a chosen sun direction and hardness. It is driven by a "light-direction ball" composited into the reference video's corner and runs in a single pass. Access requires accepting the model's conditions. The docs say it has not been validated on LTX-2.5. — [HF card](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Relight); [docs.ltx.io relight](https://docs.ltx.io/open-source-model/feature-guides/editing-effects/relight) +- Light-A-Video is training-free video relighting. It reports a motion error of 1.833 against 5.969 for per-frame IC-Light, which flickers. IC-Light is fine-tuned from Stable Diffusion. — [arXiv 2502.08590](https://arxiv.org/pdf/2502.08590) +- LightCtrl (2026) adds user-defined light trajectories, training-free, on an IC-Light base. — [arXiv 2603.27083](https://arxiv.org/pdf/2603.27083) +- LTX-2.5 has the IC-LoRAs `SDR-To-HDR` (2026-09-30) and `Day-To-Night`. — [HF listing](https://huggingface.co/api/models?author=Lightricks&search=LoRA&limit=100&sort=lastModified) +- Community `FuzzPuppy/LTX-2.3-Black-Magic-LoRA` appears in relight-adjacent results. Its purpose was not verified. — [HF](https://huggingface.co/FuzzPuppy/LTX-2.3-Black-Magic-LoRA) + +### Inferences +- For stills, IC-Light (SD1.5-based, with an FC and FBC model) can be loaded in diffusers with custom UNet channel patching. The IC-Light v2 Flux variant was not released openly (unverified). Relighting a FLUX 3 Klein still is better done by prompt editing with Klein itself. +- Lens and look presets (anamorphic, shallow DOF, 35mm film, golden hour) are best handled as caption-sentence presets in each family's prompt spec, plus the HDR and Day-To-Night IC-LoRAs on LTX-2.5. + +### Gaps +- No open lens, focal-length or DOF-control LoRA was found (the search did not target it directly). IC-Light licence: Apache-2.0 per prior knowledge (unverified). + +## Q8. Shot-planning and storyboard agents, and prompt upsamplers for video + +### Takeaway +ViMax (HKUDS, MIT, about 5.5k stars, paper arXiv 2606.07649) is the most complete open agentic film pipeline. Its decomposition is screenwriting, then shot planning and storyboard, then character design (reference images), then keyframes, then video generation per shot, then VLM quality control with best-of-k, then assembly. It calls external image and video APIs rather than a local engine. FilmAgent (Xu et al. 2025) is a multi-agent research system in virtual 3D spaces. dw's existing `script-to-video` and `series-episodes` skills already implement a comparable script, shot list, cast-once, per-shot template, validate and cost, run, cut-and-score loop. + +### Cited Findings +- ViMax "orchestrates scriptwriting, storyboarding, character creation, and final video generation, all end-to-end". The paper describes "specialized agents for screenwriting, shot planning, character styling, video generation, and VLM-based quality control with best-of-k selection". — [HF papers 2606.07649](https://huggingface.co/papers/2606.07649.md); [alphaxiv](https://www.alphaxiv.org/abs/2606.07649.md) +- ViMax code is at github.com/HKUDS/ViMax and is "fully open-sourced under the MIT license" (secondary source). It reports more than 5,500 stars and 935 forks (news site, may be stale). It recently added Seedance 2.0 Fast as a video backend. Cost is about $1-5 per video in external API fees (unverified blog estimate). — [themenonlab blog](https://themenonlab.blog/blog/vimax-agentic-video-generation-open-source); [mirror repo](https://github.com/2456868764/ViMax) +- FilmAgent is cited by ViMax as "a multi-agent framework for end-to-end film automation in virtual 3d spaces (Xu et al., 2025)". — [HF papers 2606.07649](https://huggingface.co/papers/2606.07649.md) +- The LTX-2.5 caption spec in dw is itself Lightricks' prompt-upsampler system prompt ("Write a single, highly detailed audio-visual caption… in the EXACT style of the training captions"). Kandinsky 6 has a built-in `expand_prompts` text-encoder upsampler. — local `plugins/dw/skills/ltx-2.5/references/caption-spec.md` line 13; `plugins/dw/skills/kandinsky-6/SKILL.md` line 87-88 + +### Inferences +- The common decomposition across ViMax, MovieAgent and AnimDirector (the latter two unverified this session) is: script, then scene breakdown, then shot list (shot size, camera move, duration, dialogue), then a character and location bible with reference images, then a first-frame keyframe per shot (T2I or edit), then I2V per shot, then VLM judge and retry, then edit and score. dw's MCP tools (`assess_output`, `compare_outputs`) can serve as the VLM judge stage, and list-driven steps cover the per-shot fan-out. +- ViMax's value for dw is as a design reference (prompt templates, best-of-k QC) rather than a dependency, because it targets hosted APIs. + +### Gaps +- MovieAgent, StoryAgent, AnimDirector and DreamFactory were not verified this session (repos, licences, decomposition details). +- Open prompt upsamplers dedicated to video, beyond vendor caption specs, were not surveyed. + +## Cross-cutting: fit to dw's supported families (summary of the above) +- **Works now, same base:** + - LTX-2.5 22B LoRAs and IC-LoRAs: Slow-Motion-Control, Cinemagraph, Water-Simulation, Day-To-Night, Clean-Plate, Layout-To-Render, SDR-To-HDR and Ingredients. + - Prompt-level camera, lens and look presets for LTX-2.5, Kandinsky 6 and H3. + - Multi-reference editing on FLUX 3 Klein. +- **Probably works, needs validation:** LTX-2.3 IC-LoRAs (Relight, Motion-Track-Control, Union-Control, DubIt) on LTX-2.5. The docs say Relight is not validated on 2.5, and an aggregator says 2.3 LoRAs "partly work". +- **Needs a new base:** + - Qwen-Image-Edit 2509 or 2511, with the multi-angle LoRAs. It is diffusers-native and the easiest add. + - Wan 2.1 or 2.2 14B: orbit and crash-zoom effect LoRAs, VACE, Animate, Phantom, MAGREF, Stand-In, ReCamMaster, Uni3C, InfiniteTalk and MultiTalk. The base pipelines are diffusers-native. Most of the conditioning adapters need custom loaders and preprocessing tasks. + - LTX-2 19B, for the seven named camera LoRAs. +- **Non-commercial flags:** Wav2Lip is confirmed non-commercial. Hallo3, Sonic and the ReCamMaster and TrajectoryCrafter weights are unconfirmed and should be treated as unknown until checked. Lightricks LoRAs are gated and under their own licence terms (not re-verified). diff --git a/docs/proposals/audits/2026-11-higgsfield/notes/open_source_clones.md b/docs/proposals/audits/2026-11-higgsfield/notes/open_source_clones.md new file mode 100644 index 000000000..21b10ffb1 --- /dev/null +++ b/docs/proposals/audits/2026-11-higgsfield/notes/open_source_clones.md @@ -0,0 +1,164 @@ +# Open-source Higgsfield clones and alternatives (2025-2026, focus on Comfy's #OpenSourceHiggsfield challenge) + +Research date: 2026-11-01. Method: `gh search repos` / `gh api` (repo metadata, licence, commit count via the commits Link header, README) plus web search/fetch. No code was cloned or run; every "does it work" claim below is the project's own README claim unless stated. Star counts are as of 2026-11-01 and moving fast. "Commits" is the default-branch commit count. + +Licence note: GitHub's API reports `NOASSERTION` for some repos whose LICENSE file is in fact a standard licence. I read the LICENSE file for three such repos: OpenField (MIT), open-higgsfield-skills (MIT), Vision Orchestrator (Apache-2.0). The community list marks OpenField and Vision Orchestrator as "none", which is wrong per their LICENSE files. + +## Which open-source Higgsfield clones exist (GitHub, HF Spaces, ComfyUI registry, Reddit/X)? + +### Takeaway +There is a large, already-curated field: the CC0 list `jutsuhq/open-source-higgsfield` catalogs 102 challenge projects and 6 older alternatives. The challenge entries are overwhelmingly MIT/Apache, ComfyUI-backed (a custom web front end that drives ComfyUI through the Comfy SDK + `comfy-api-proxy`), open-weights-first (LTX-2.5, MiniMax H3, FLUX 3/2 Klein, Qwen-Image-Edit 2511, Wan 2.2, SAM 3.1, SeedVR2). The pre-challenge "Higgsfield clones" are mostly API front ends (Muapi, kie.ai, Gemini) or take-home-assignment UIs with no models and often no licence. + +### Cited Findings + +**The challenge itself** +- Comfy's challenge: "Rebuild Higgsfield's popular apps, and more, using ComfyUI"; entries must "Publish your app code with an open-source license and setup instructions"; suggested tracks are Cinematic Studio, Re-shoot a video and Openjutsu; prizes are Best Overall $10,000, Most Innovative / Most Viral / Open Weights Only (RTX 5090 each), 10 runner-ups, plus bonuses for LTX as the core VFX model ($2,000), FLUX 3 as the core model, and MiniMax H3 as the core model. Open Weights Only means "No partner nodes or closed-source APIs." — [comfy.org/open-source-higgsfield](https://comfy.org/open-source-higgsfield); [markdown version](https://comfy.org/open-source-higgsfield.md) +- **Dates conflict.** The live Comfy page (fetched 2026-11-01) says build period "10/15 – 11/3", "Submission window closes: November 3rd 9am PT", winners "11/10", with no mention of an extension — [comfy.org .md](https://comfy.org/open-source-higgsfield.md). The ComfyUI Wiki news item and the jutsuhq list say the deadline was October 30 9am PT and winners November 6 — [ComfyUI Wiki](https://comfyui-wiki.com/en/news/2026-10-15-open-source-higgsfield); [jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield). Most likely the deadline was extended, but I found no source that says so. +- Comfy says Higgsfield sent a 48-hour cease-and-desist objecting to use of Higgsfield's name and its CEO's likeness; Comfy said it would not comply. This rests on Comfy's own account (an X thread); no Higgsfield statement or letter copy was found — [RuntimeWire](https://runtimewire.com/article/comfy-says-higgsfield-sent-a-48-hour-cease-and-desist-over-open-source-challenge); [explainx commentary](https://explainx.ai/blog/higgsfield-cease-and-desist-comfy-open-source-higgsfield-challenge-48-hours-claimed-vs-verified-2026); list timeline cites [ComfyUI on X](https://x.com/ComfyUI/status/2115928993623109648). +- The Comfy team released its own reference entries: OPENjutsu (character swap, Oct 26) and the clonable `comfy-openjutsu` app template; an earlier `comfy-reshoot` reference app (Sep 29) is the template OPENjutsu says it follows — [jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield); [D3voz/openjutsu README](https://github.com/D3voz/openjutsu); [Comfy-Org/comfy-reshoot](https://github.com/Comfy-Org/comfy-reshoot). + +**The curated index (start here for anything not below)** +- `jutsuhq/open-source-higgsfield` (CC0-1.0, 93 stars): "102 challenge projects and 6 older alternatives, with stars and licenses as of 2026-11-01", categorized as Character swap, All-in-one studios, Effects and VFX, Camera/reshoot/previs, Films/music videos/stories, Ads and UGC, Characters and datasets, Canvases/front ends/tools. Entries found "through the #OpenSourceHiggsfield tag on X and through GitHub search". Note it is maintained by Jutsu Studio, a commercial-adjacent Higgsfield desktop client — [repo](https://github.com/jutsuhq/open-source-higgsfield) + +**Catalog: challenge entries examined in detail** (metadata from `gh api` on 2026-11-01; features from each README) + +| Project | Licence | Higgsfield feature recreated | Architecture | Models | Maturity | +|---|---|---|---|---|---| +| [OPENjutsu](https://github.com/D3voz/openjutsu) (Comfy team) | MIT | Genjutsu character swap; object swap/remove/background; face fix; colour fix | Next.js front end over ComfyUI API-format workflows via `@comfyorg/sdk` + `comfy-api-proxy`; own node pack `comfyui-h3-oneclick-swap` | MiniMax H3 Ref2VA int8 + DMAD 4-step LoRA, SAM 3.1, Qwen3-VL 8B describer (or Gemma 4 E2B), FLUX.3 Klein / SolarRenderer for "From picture" — all open weights | 1,050★, 74 commits, created 10-26, pushed 10-30. Tested on RTX 4060 Ti 16 GB + 64 GB RAM; 6 s 480p clip ≈6-13 min | +| [comfy-openjutsu](https://github.com/Comfy-Org/comfy-openjutsu) | MIT | Character swap template + dev guide | Clonable Comfy Developer Platform app | (as above, cloud deployment) | 91★, 2 commits | +| [comfy-reshoot](https://github.com/Comfy-Org/comfy-reshoot) | MIT | Re-shoot from a new angle | Next.js app; `analyze.api.json` (MoGe depth → `.cvgeo`), browser camera preview with sliders, `generate.api.json` (MoGe + CrossView Warp + MiniMax H3 + CrossView LoRA) | MiniMax H3, MoGe, CrossView LoRA | 8★, reference app, needs Comfy platform key | +| [OpenField](https://github.com/wexloops/openfield) | MIT (LICENSE file; GitHub says NOASSERTION) | Whole studio: Video w/ 28 motion presets, Director (genre/camera/framing/light/film stock/grade/era scrubber), Scene (LLM plans 2-6 shots), Stage (3D blocking), Cast (`@mention`), Mirror (motion transfer), Reshoot, Effects, Relight, New angle, Speak, Upscale, HDR finish, loop seam score | Vite+React → Fastify server with TypeScript graph builders (`server/graphs/*.ts`) exported to `workflows/` → comfy-api-proxy → ComfyUI. 31-tool MCP server. Every take "opens as the exact workflow that rendered it" | LTX-2.5 (+ Layout-to-Render IC-LoRA), Qwen image/edit, Z-Image Turbo + Fun ControlNet Union, TRELLIS.2, SeedVR2, SAM3, FLUX.2 (non-commercial), MiniMax H3 bundles; installable per-bundle (72 GB core, 334 GB all) | 126★, 6 commits (squashed), created 10-28. 24 GB VRAM for video; built on RTX 5090 | +| [CreatorOS Studio](https://github.com/thePloggy/CreatorOS-Studio) | Apache-2.0 | One recurring character: one-click reel, 25 camera/effect presets, Cinematic Studio "shot card", Stage, Re-shoot, talking avatar, motion transfer, Mix (character swap), editor, auto clip QC | Node/Fastify server (job queue, recipes, persona, SQLite) → ComfyUI API-format "recipes, patched per job"; desktop + mobile UIs | LTX-2.5 22B, Wan 2.2 I2V A14B, Wan Animate 2, InfiniteTalk (Wan 2.1), Qwen-Image-Edit-2511 + fal Multiple-Angles LoRA, MoGe-2 + CrossView (re-shoot), SeedVR2 3B, TRELLIS.2, SAM3, Krea 2 / FLUX.2 Klein, ACE-Step 1.5, Ollama LLM | 46★, 34 commits; "public edition of a production tool used daily"; public demo on author's GPU | +| [agentY](https://github.com/szprivate/agentY) | MIT | Agent that picks/builds and runs workflows | ComfyUI side panel agent; MCP links to Blender/Maya/Houdini/Nuke; any LLM incl. Ollama | Model-agnostic | 48★, 1,076 commits, since Feb 2026 (oldest/most mature agent here) | +| [OpenKatana](https://github.com/rk0neki/OpenKatana) | Apache-2.0 | Agent-driven editing (cuts, transitions, typography, music) | Python CLI + MCP server + ComfyUI Apps (85 nodes, 72 tools, on Comfy Registry as `openkatana`); agent "setup prompt" install | Model-agnostic editor; WebGL/Blender for 3D | 21★, 65 commits | +| [ReAngle](https://github.com/Tekholms-LW/ReAngle) | Apache-2.0 | Re-shoot a video from a new camera angle | FastAPI backend + Vite/three.js 3D path editor + node pack `comfyui-reangle` (camera paths, depth reprojection) | LTX-2.5 22B distilled GGUF Q4 + Layout-to-Render IC-LoRA (optional LTX-2.3 Union Control), Video Depth Anything **Small** (Base/Large are CC-BY-NC) | 4★, 16 commits; tested on RTX 5080 16 GB; refuses <12 GB | +| [Comfy-Dial](https://github.com/ylchen333/comfy-dial) | MIT | Camera-move control that verifies itself | pnpm TS studio (Author/Generate/Review) + Python verifier; CLI `pnpm loop`, `pnpm repair`, `pnpm preflight` | LTX-2.x driven by a rendered depth video, Depth Anything 3 for camera recovery, optional FLUX.1 schnell + Hunyuan3D 2.1 actors, SeedVR2 | 0★, 42 commits; "MVP, tentative docs", run end to end on Comfy Cloud only | +| [Motion Previs Studio](https://github.com/navarisun1982/comfyui-motion-previs-studio) | MIT | Higgsfield 3D / "3D-Jutsu" previs: 3D camera, character, lighting blocking | three.js studio with dual viewport (soundstage + viewfinder), "Shot Bible" multi-shot manager, auto-rigging, LLM directing assistant (Ollama/LM Studio/Gemini/OpenAI) | Pixal3D / TRELLIS.2 INT8, MoGe, BiRefNet, Z-Image Turbo, MiniMax H3 Ref2VA, LTX-2.5 22B INT8 | 29★, 103 commits | +| [Shot Board](https://github.com/ArcSolver/shot-board) | MIT | Cinema Studio coverage: one scene → six cuts (establishing, medium, face CU, OTS, insert, low angle); pick hero, redraw the rest from it | FastAPI on CPU → Comfy SDK → comfy-api-proxy; pinned `model-lock.json`; export ZIP with per-frame API workflows + manifest | SDXL Base 1.0, Qwen-Image-2512, Qwen-Image-Edit-2511 (GGUF) | 1★, 102 commits | +| [seriesmith](https://github.com/bgstratt/seriesmith) | MIT | Series/episode production (Higgsfield-style "film" pipeline) | `series.json` + markdown episode scripts → model-free `shots.json` → per-target compile → backends (local ComfyUI, Comfy Cloud, Router); browser editor + CLI; ComfyUI-Manager install | Default target MiniMax H3 Ref2VA; any model per shot | 3★, 503 commits; successor to h3pipe | +| [Mondo Music Video](https://github.com/hodgemann/mondo-music-video) | Apache-2.0 | Song → music video | Python app, agent-installable, MCP tools, "Open Actor/Prop Protocols" (signed actor packages) | Beat/downbeat/section analysis + LLM prompts; local or hosted video | 12★, 110 commits, v1.12.2 | +| [Picky](https://github.com/chenthedesignmaker/ComfyUI-Picky) | GPL-3.0 | Cinema Studio for H3: takes, compare, pick | Node pack + browser editor | MiniMax H3, Krea 2 keyframes, SeedVR2 | 31★, 8 commits (squashed) | +| [ComfyUI-Continuity](https://github.com/roadmaus/ComfyUI-Continuity) | MIT | "One node" studio: prompt + `@` media → render, timelines, chat | Single ComfyUI node + dashboard; optional prompt-refiner LLM (LM Studio/Ollama/hosted) | Six families via ComfyUI core (LTX 2.5, H3, Krea 2, …) | 139★, 694 commits, since Aug 2026 | +| [Body Double](https://github.com/lucataco/open-source-higgsfield) | Apache-2.0 | Genjutsu recast keeping plate pixel-identical | Stock ComfyUI, core nodes only, per-segment jobs | SAM 3.1, FLUX 3 Image Klein (first frame), SCAIL-2 | 56★, 4 commits | +| [H3 Recast](https://github.com/Bambushu/h3-recast) | MIT | Swap up to 3 people in one pass | ComfyUI App-mode workflow | MiniMax H3 + BFS Character Swap LoRA, optional FaceRefine lip pass | 39★, 8 commits; 16 GB VRAM + 64 GB RAM | +| [One-shot Multicam](https://github.com/apsolut/one-shot-multicam) | MIT | Re-shoot: one selfie take → multi-camera edit cut on word boundaries | Web app on Comfy Cloud + ComfyUI workflows | LTX-2.3 22B + CrossView IC-LoRA (v2v), Qwen-Image-Edit 2511 angles + LTX-2.3 ID LoRA (v1), faster-whisper | 3★, 60 commits | +| [360 Scout to Film](https://github.com/Alvinvision/360-Scout-to-Film-Consistency) | MIT | Location consistency across shots | ComfyUI extractor (panorama → perspective views + contact sheet + JSON manifest) | Seedance 2.5 (API) and MiniMax H3 | 4★, 32 commits | +| [Aither Cinema Studio](https://github.com/Aitherium/aither-cinema-studio) | MIT | Cinema Studio camera rigs, re-shoot, grade, cut | FastAPI (rig → prompt → Comfy graph) + React studio with 11 WebMCP tools + `cinema-mcp` (10 tools) + Claude Code skill | SDXL, Wan 2.2 (camera, t2v, i2v, flf2v), LTX | 0★, 15 commits | +| [Haggs](https://github.com/IBFolding/haggs) | MIT | Full studio: 26 apps, canvas, Draw-to-video, Series, "Recreate an edit", multitrack editor; price on every button | HTML/JS front end on Comfy (cloud) | Mixed: Comfy partner models (Gemini for shot analysis, ElevenLabs) + labelled open-weights workflows | 0★, 309 commits | +| [open-higgsfield-skills](https://github.com/mojomast/open-higgsfield-skills) | MIT (LICENSE file) | Agent "film crew": one prompt → short video | 37 Agent Skills (SKILL.md + stdlib Python + API-format workflows); `video-director` router with JSON manifest | Open weights only: FLUX 3 Klein, LTX-2.5, Wan Fun Camera, Qwen-Image-Edit 2511, BiRefNet, Chatterbox, H3 swap | 2★, 6 commits; "Most have not been run" — statically checked only | +| [Boson](https://github.com/0xMisterBailey/Boson) | MIT | "Studios instead of presets" | 28 ComfyUI nodes + app at `/boson/app`; studio = git repo of taste state | One switch per capability (local open weights / cloud / hybrid) | 1★, 9 commits (history condensed; says >500 commits) | +| [Parametric Studio](https://github.com/parametricparadigm/ComfyUI_Parametric_Studio) | AGPL-3.0 | Recast, reshoot, frozen orbit, VFX passes, Shot Board, 740-recipe library | ComfyUI studio with guided forms + "Pro" real-graph view (3D); AI "crew" answering from cited notes | 90 families wired, 55 rendered; open weights only "enforced by a test" | 1★, 25 commits; proven only on a 96 GB RTX PRO 6000 | +| [Open Ads Studio](https://github.com/jhoard/open-ads-studio) | Apache-2.0 | Higgsfield Ads Studio | TS app + ComfyUI workflows; deterministic glyph-paste post step | Qwen-Image-Edit 2509, LTX-2 19B distilled (draft), MiniMax H3 FL2VA (final) | 2★, 32 commits | +| [OpenReel](https://github.com/smithysw7/openreel) | MIT | Soul Cast / Soul ID / Soul Cinema: logline → 8-shot storyboard → consistent short | TS app + ComfyUI + ffmpeg assembly | Open weights (keyframe + I2V with native audio) | 2★, 9 commits | +| [Open Cinema](https://github.com/LaytonGott/open-cinema) | GPL-3.0 | Direct a shot, change lens/focus afterwards | Director web app + "Open Optics" node pack (physically based DoF, distortion, CA, anamorphic) | LTX-2 19B GGUF + official LTX-2 camera LoRAs, Depth Anything 3 | 1★, 1,015 commits | +| [OpenSourceHiggsfield (kevin9327)](https://github.com/kevin9327/OpenSourceHiggsfield) | MIT | Cinematic Studio, Re-shoot, OpenJutsu, SDK workbench | Web studio + 5 custom nodes + comfy-sdk | Open weights (claimed) | 10★, 2 commits; README is heavily stylized/marketing | +| [Vision Orchestrator](https://github.com/stellartune-studio/vision-orchestrator) | Apache-2.0 (LICENSE file) | LLM director chat growing a "production blueprint" | Node 24 + pnpm app, optional ComfyUI via proxy | ComfyUI-backed | 0★, 1 commit | +| [ComfyArt](https://github.com/jtydhr88/ComfyArt) | MIT | Design canvas (Higgsfield Canvas / Flora-style) | ComfyUI custom node serving tldraw+React canvas, MCP design agent, SQLite projects | Local image/video/audio/3D | 413★, 61 commits | + +Other challenge entries from the list worth knowing by name (one-line summaries are the list's; not individually verified): Seam (two shots in, AI transition out, 19★ MIT), Element Forge (LTX-2.5 VFX elements tracked and composited, MIT), SpEdE (edit objects in a Gaussian splat, AGPL), Hero Frame Rig (refocus after the fact, Apache), Motion Guard / ShotCheck (CPU camera-motion checkers, Apache/MIT), previz2shot (Blender playblast → shot with FLUX.2 Klein + LTX-2.5, MIT), Toybox Director (toys on a 3D stage → LTX shot, MIT), comfyui-sameface (measures and undoes face-geometry drift from edits, Apache), Character Lock / Dataset Forge (MIT), Xenoform (song → music video, open weights, MIT), ComfyUI-Spool (video timeline for ComfyUI, GPL) — [jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield). + +**No-licence repos (concept-only; do not copy code)** — per the list and/or `gh api`: +- [wide-trace/open-higgsfield](https://github.com/wide-trace/open-higgsfield) (5,211★, 3 commits, Next.js front end needing a platform API key) — no licence. +- [SegFault42/HeliosGen](https://github.com/SegFault42/HeliosGen) (3,224★, Tauri app via kie.ai) — no licence per list. +- Challenge entries listed without licence: [Darius](https://github.com/Niutonian/Comfyui_Darius_OS), [Conjure](https://github.com/Orazen/conjure), [FXjutsu](https://github.com/audriu/fxjutsu), [HiggsOpen](https://github.com/kingofmit/ComfyUI-HiggsOpen) — [jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield). +- Many `*/higgsfield-clone` repos (Sep-Oct 2026) are "8x take-home assignment" UI rebuilds with 0-1★ and mostly no licence — `gh search repos "higgsfield clone"`, e.g. [mudassar531/higgsfield-clone](https://github.com/mudassar531/higgsfield-clone) ("24h rebuild of higgsfield.ai's core generation flow — 8x take-home assignment"). + +**Earlier / non-challenge alternatives** +- [Anil-matcha/Open-Generative-AI](https://github.com/Anil-matcha/Open-Generative-AI), formerly Open-Higgsfield-AI: 32,020★, MIT, 488 commits; "600+ … models across 14 studios", cloud models through Muapi.ai; README leads with a paid white-label upsell. Code is MIT but it is an API front end, not a model pipeline — [README](https://github.com/Anil-matcha/Open-Generative-AI) +- [Lightricks/LTX-Desktop](https://github.com/Lightricks/LTX-Desktop) (Apache-2.0, 2,199★) — local LTX desktop app; [BeatAPI/BeatDesign](https://github.com/BeatAPI/BeatDesign) (Apache-2.0, API) — [jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield) +- [TechBeme/open-higgsfield](https://github.com/TechBeme/open-higgsfield) (MIT, 19★): Freepik/Gemini/Vertex/Vercel AI Gateway front end; [clawnify/OpenStudio](https://github.com/clawnify/OpenStudio) (MIT, 65★): image studio with node workflows — `gh search repos "higgsfield alternative"`. + +### Inferences +- The challenge converged on one architecture: a purpose-built web UI (Next.js/Vite/React) + a small server that patches committed ComfyUI API-format JSON per job and submits through `@comfyorg/sdk` → `comfy-api-proxy` → ComfyUI. This is structurally close to dw's model (declarative JSON + server + UI), which makes the entries' UX ideas directly portable even where their graphs are not. +- Star counts are dominated by the Comfy team's own entry and by older API front ends; most idea-rich entries have <50 stars and are 1-3 weeks old. "Works" is largely unverified outside each author's own GPU. + +### Gaps +- No Hugging Face Spaces or ComfyUI-registry search was done directly (only OpenKatana was confirmed on the registry). HF Spaces for these entries were not found in the search results I ran. +- Reddit threads were not reachable via search; the list links several r/comfyui launch posts (e.g. H3 Recast, Element Forge, RenderRepo) that I did not open. +- Winners are not yet announced (Nov 6 or Nov 10 depending on source). + +## How do they structure presets vs fine-grained control? + +### Takeaway +The dominant pattern is "named app / one-click preset on top, the real graph one click underneath": a small vocabulary of discrete choices (move, lens, framing, light, stock, grade) that compiles deterministically into the prompt and graph, with an "Advanced" fold and an "open the exact ComfyUI workflow" escape hatch. A few entries reject presets outright in favor of persisted taste state (Boson) or measured verification (Comfy-Dial). + +### Cited Findings +- Aither: "A rig is a decision, not a slider stack. `dolly_in` + `85mm` + `close_up` is a different shot from `static` + `85mm` + `close_up`, and the studio makes the model honour the difference — deterministically, in the prompt text"; vocabulary of 20 moves, 7 lenses, 3 speeds, 10 framings, 9 looks; modes keyframe/t2v/i2v/flf2v/reshoot; `plan_shot` compiles a rig into the exact prompt with no render — [Aither README](https://github.com/Aitherium/aither-cinema-studio) +- OpenField's Director: "genre, camera, framing, light, film stock, look, grade, an era scrubber (1960 to 2020) and speed ramps compile into the shot"; Video has "28 motion presets"; every take opens as the exact ComfyUI workflow; user LoRAs are only offered to the model their tensor keys match — [OpenField README](https://github.com/wexloops/openfield) +- CreatorOS "Cinematic Studio: a shot card instead of a prompt. Pick the camera body, lens, focal length and aperture, the framing, the light, the film stock, the look and the grade, add a colour palette (or pull one from a photo)"; 25 one-click camera/effect presets: dolly in/out, jib up/down, side tracking, orbit, arc, crash zoom in/out, bullet time, dolly zoom, plus squish/inflate/cakeify/explode; seven one-click relights; re-shoot angle presets left/right/three-quarter/high/low/sweep/descend/orbit; Library stores "provenance of every render (recipe, seed, LoRAs, final prompt)" and downloads the exact workflow — [CreatorOS README](https://github.com/thePloggy/CreatorOS-Studio) +- OPENjutsu: simple mode is one click; "Advanced" exposes steps, base model, LoRA chain, text-encoder quant, VAEs, motion-reference size (512/768/1376), seed (re-running resumes from last finished clip); `lib/simple.ts` edits the committed API JSON per request by removing/reconnecting nodes (hat fix, no lip-sync) rather than keeping many workflow variants; a preflight checks the workflow against ComfyUI's node and model lists and lists missing packs with install links — [OPENjutsu README](https://github.com/D3voz/openjutsu) +- Parametric Studio: "a form anyone can use, and one click from the exact ComfyUI graph"; apps "Named after what you want to do, not which node to wire"; "Open weights only … enforced by a test" — [Parametric README](https://github.com/parametricparadigm/ComfyUI_Parametric_Studio) +- Boson: "Studios instead of presets: your taste as plain files in git that every next ComfyUI round obeys"; judge a round with green/red dots and one sentence, and "the judgment becomes state the next round must read"; nodes run `stored` (free replay) or `live`; "a studio never names a model" — [Boson README](https://github.com/0xMisterBailey/Boson) +- Continuity: the "pills under the box are the whole setup: duration, aspect, resolution, the model, and a sampler row for when you want it" — [Continuity README](https://github.com/roadmaus/ComfyUI-Continuity) +- Hardware tiers as presets: ReAngle "Hardware detection picks a tier (resolution, chunk length, upscale), refuses under-spec GPUs"; open-higgsfield-skills picks "draft 16 GB / standard 24 GB / max 40 GB"; Picky has preview / turbo HD / 2-pass final quality tiers — [ReAngle](https://github.com/Tekholms-LW/ReAngle); [skills](https://github.com/mojomast/open-higgsfield-skills); [Picky](https://github.com/chenthedesignmaker/ComfyUI-Picky) +- Cost on every control: Haggs shows the credit price "on the Generate button as always" and "Nothing is spent on the drawing page"; Comfy-Dial has a free "Run dry" with synthetic poses and confirms every paid action's credit estimate — [Haggs](https://github.com/IBFolding/haggs); [Comfy-Dial](https://github.com/ylchen333/comfy-dial) +- Open Cinema: physically based lens params (focal length, f-number, focus distance, sensor width) applied after generation from per-frame depth; re-rendering only DoF takes ~7 s; pick-point focus and rack focus — [Open Cinema](https://github.com/LaytonGott/open-cinema) + +### Inferences +- The shared design: presets are a closed enum vocabulary compiled into prompt text and graph parameters, not free sliders. That maps cleanly onto dw variables with constrained enums plus a template that injects phrases — the "rig compiles to prompt, inspectable before render" pattern (Aither `plan_shot`) is analogous to dw's free `validate_workflow`. +- "Every output carries its exact recipe/workflow" is near-universal and is a table-stakes expectation (dw manifests already do this). + +### Gaps +- Exact preset tables (e.g. Aither's CONTRACT.md, OpenField's 28 motion presets) were not opened; only README summaries were read. + +## Which ones include agent/LLM shot planning, storyboard-to-video, multi-shot continuity? + +### Takeaway +Many: shot planning by local LLM (OpenField Scene, CreatorOS reel, Motion Previs, OpenReel), script/series compilers (seriesmith), beat-driven music video planners (Mondo, Xenoform), agent skill packs and MCP servers (open-higgsfield-skills, Aither, OpenField 31-tool MCP, OpenKatana, agentY, ComfyArt). Continuity is handled by casts with `@mentions`, hero-frame propagation, last-frame chaining, location references, and automatic QC. + +### Cited Findings +- seriesmith: show defined once in `series.json` (cast, places, voices, look), episodes as markdown screenplays → model-free `shotlist/shots.json` → compiled per "video target" → rendered per shot on a backend; "Every shot is its own job, so a bad shot is rendered again alone"; "a shot written `continuous:` reads the take before it when its job starts, so a whole chain still queues at once"; docs include STORY_STRUCTURES (three-act, Story Circle, Kishōtenketsu, Hero's Journey…) and an H3 target doc covering "reference slots, sheets, voices, audio policies, pacing"; its assistant prompt `SKILL.md` is generated from `docs/AUTHORING.md` — [seriesmith README](https://github.com/bgstratt/seriesmith) +- open-higgsfield-skills: `video-director` flow: intake (at most 3 questions) → preflight + tier pick → story-bible → shot-planning (approval gate) → stills → still-qa judge (fail → image-edit / relight / re-prompt / seed+1) → per-shot video → assembly; "records every prompt, seed, model and cost in one resumable JSON manifest"; approval gates "on the bible, the shot plan, spend and stills, and hard caps on retries and GPU-minutes"; chaining via `chain_from: s01` — [skills README](https://github.com/mojomast/open-higgsfield-skills) +- OpenReel's three continuity layers: text anchoring (actor appearance description injected verbatim into every keyframe prompt), image anchoring (first head sheet as reference / first frame for all later generations), plus a unified style block; logline → 8-shot storyboard with framing / camera move / dialogue / duration — [OpenReel README](https://github.com/smithysw7/openreel) +- Shot Board: hero-frame propagation — pick one hero cut, the other five are redrawn from it with Qwen-Image-Edit-2511; save pass/fail verdict with a note; export storyboard + per-cut API workflows + manifest — [Shot Board README](https://github.com/ArcSolver/shot-board) +- Picky: each shot is "a structured document with style, overview, timed beats, camera, dialogue, soundscape and music … composed into MiniMax's official H3 prompt format"; "A prompt linter that knows H3" catches "overfull beats, cuts with no new information, wasted quality tokens, missing soundscape"; cast `@Mara` resolves to the right `` with current references at render time; renaming rewrites every mention; extensions run on the shot's own graph plus Motion Context — [Picky README](https://github.com/chenthedesignmaker/ComfyUI-Picky) +- Mondo Music Video: detects "beats and downbeats, measures energy, and identifies musical sections"; "beat-driven cuts and energy-aware montage grammar"; per-actor beat/drums/bass/vocals assignments for dancing; lyric captions from timed lines — [Mondo README](https://github.com/hodgemann/mondo-music-video) +- Haggs "Recreate an edit": a video model "lists every shot: its length, what the camera sees, the camera move, the words on screen" and rebuilds the edit shot-for-shot around your character; "Draw to video": sketch two kinds of arrows (subject motion vs camera) which are turned into editable plain-language motion text — [Haggs README](https://github.com/IBFolding/haggs) +- CreatorOS: one-click reel where "A local LLM plans 3 to 6 shots of one session (same outfit, place and light)"; driver videos with cuts are split per shot and the montage rebuilt "at the exact original durations"; automatic clip check on CPU in ~1 s: "black frames, frozen video, a jump inside the shot, smear, and how close the face stays to the start image (OpenCV YuNet + SFace)" — [CreatorOS README](https://github.com/thePloggy/CreatorOS-Studio) +- Body Double: long takes as 81-frame segments, each its own job conditioned on the previous segment's last five frames with per-segment seed and colour match; "QC (seam ratio and background drift) runs before the next segment starts, and a flagged segment is re-rolled. Runs resume after a crash" — [Body Double README](https://github.com/lucataco/open-source-higgsfield) +- 360 Scout to Film: one 2:1 equirectangular panorama → overlapping perspective location references (16:9, 2048×1152), a contact sheet, a location/lens prompt and a JSON manifest, shared across every shot; the same refs go to Seedance 2.5 and H3 for comparison — [360 Scout README](https://github.com/Alvinvision/360-Scout-to-Film-Consistency) +- Agent/MCP surfaces: OpenField "31 tools let an agent direct, render, check and finish shots"; Aither exposes `list_camera_moves`, `plan_shot`, `render_shot`, `reshoot_shot`, `grade_shot`, `cut_sequence`, `cut_music_video` (beat sheet from bpm + beats/shot); OpenKatana installs via a copy-paste agent setup prompt that says "Ask before downloading models, spending credits, uploading files or publishing" — [OpenField](https://github.com/wexloops/openfield); [Aither](https://github.com/Aitherium/aither-cinema-studio); [OpenKatana](https://github.com/rk0neki/OpenKatana) +- Vision Orchestrator: chat with an LLM that grows a structured "production blueprint — intent, story, characters, materials, constraints — and only approved content flows into generation" — [Vision Orchestrator README](https://github.com/stellartune-studio/vision-orchestrator) + +### Inferences +- seriesmith and Picky are the closest analogues to dw's own `script-to-video` / `series-episodes` skills and H3 prompt tooling; seriesmith's "model-free story → per-target compile" split and Picky's H3 prompt linter are the two most directly comparable designs. +- Automatic, cheap, CPU-side QC between generations (CreatorOS clip check, Body Double seam/drift QC, Comfy-Dial camera verifier, comfyui-sameface face drift) is a recurring idea that dw's `assess_output`/`compare_outputs` partially covers. + +### Gaps +- I did not verify whether any of the agent pipelines run end to end; open-higgsfield-skills explicitly says most workflows were only statically checked. + +## Which open models do they use for camera control, re-shoot from new angle, character swap, multi-angle stills, lip sync? + +### Takeaway +Camera control and re-shoot centre on LTX-2.5 with the Layout-to-Render IC-LoRA fed depth renders/reprojections, CrossView (IC-LoRA for LTX-2.3, LoRA for H3) with MoGe geometry, and the official LTX-2 camera LoRAs / Wan Fun Camera. Character swap centres on MiniMax H3 Ref2VA + SAM 3.1 (OPENjutsu, H3 Recast with BFS LoRA), with SCAIL-2 and Wan 2.2 Animate as alternatives. Multi-angle stills use Qwen-Image-Edit-2511 (+ fal Multiple-Angles LoRA). Lip sync uses H3's native audio, InfiniteTalk, LTX audio-ref tokens, and FaceRefine passes. + +### Cited Findings +- **Re-shoot via depth reprojection + LTX IC-LoRA:** ReAngle depth-estimates the clip (Video Depth Anything Small), warps along one global camera path (z-buffered splat), pre-fills revealed holes from the far background, and guides LTX-2.5 distilled via the Layout-to-Render IC-LoRA; long clips are overlapping chunks re-anchored to the warped source frame and LAB colour-matched — [ReAngle](https://github.com/Tekholms-LW/ReAngle) +- **Re-shoot via CrossView:** Comfy's reference app runs MoGe + CrossView Warp + MiniMax H3 + CrossView LoRA — [comfy-reshoot](https://github.com/Comfy-Org/comfy-reshoot); CreatorOS re-shoot uses MoGe-2 + CrossView and keeps original sound — [CreatorOS](https://github.com/thePloggy/CreatorOS-Studio); One-shot Multicam uses LTX-2.3 22B + CrossView IC-LoRA (Apache-2.0) with the prompt being only a camera move ("crossview. new camera angle: far to the right, same height, closer."), and freezes the real audio into the audio latent (`LTXVSetAudioRefTokens`) so generated frame t = source frame t, measured lag 0-1 frames — [One-shot Multicam](https://github.com/apsolut/one-shot-multicam) +- **3D blocking → shot:** OpenField Stage renders a clay clip + exact depth pass in the browser (three.js), paints the look frame on the depth pass with Z-Image Turbo + Fun ControlNet Union, then LTX-2.5 Layout-to-Render IC-LoRA renders a shot "that moves like the blocking" — [OpenField](https://github.com/wexloops/openfield); Comfy-Dial feeds LTX-2.5 a depth video rendered along the authored path ("The model never sees your camera numbers"), recovers the camera per take with Depth Anything 3, scores per third, and repairs only the drifted segment — [Comfy-Dial](https://github.com/ylchen333/comfy-dial) +- **Camera LoRAs:** Open Cinema uses Lightricks' official LTX-2 camera-control LoRAs for static, dolly in/out, truck left/right, crane up/down, and describes other moves in the prompt — [Open Cinema](https://github.com/LaytonGott/open-cinema); open-higgsfield-skills uses Wan Fun Camera with `MOVES: "arc-left-45"` — [skills](https://github.com/mojomast/open-higgsfield-skills) +- **Character swap:** OPENjutsu: SAM 3.1 tracks at 1008 px, mask grown 40 px with 10% headroom, grey-mannequin motion reference, H3 Ref2VA int8 + DMAD 4-step LoRA; Advanced mode labels tracks with Qwen3-VL 8B to map 1-4 characters to roles; known limit: mirrors/reflections leak the original look — [OPENjutsu](https://github.com/D3voz/openjutsu). H3 Recast: H3 + BFS Character Swap LoRA, up to 3 people, scene reference = one edited frame of the clip; "Big, obvious changes land … Subtle ones fall back to the clip" — [H3 Recast](https://github.com/Bambushu/h3-recast). Body Double: SAM 3.1 + FLUX 3 Image Klein first frame + SCAIL-2 with a virtual follow-cam crop so the performer stays large, pasted back into the untouched plate — [Body Double](https://github.com/lucataco/open-source-higgsfield). Wan 2.2 Animate in Swapjutsu/Kawarimi and Wan Animate 2 in CreatorOS — [jutsuhq list](https://github.com/jutsuhq/open-source-higgsfield); [CreatorOS](https://github.com/thePloggy/CreatorOS-Studio) +- **Multi-angle stills / coverage:** Qwen-Image-Edit-2511 with the fal Multiple-Angles LoRA (¾ left/right, profile, from behind, high/low, wide, close-up) in CreatorOS; Qwen-Image-Edit-2511 hero-propagation in Shot Board; OpenField/CreatorOS "Coverage" (wide, ¾, close-up), "Push in", "Next beat" — [CreatorOS](https://github.com/thePloggy/CreatorOS-Studio); [Shot Board](https://github.com/ArcSolver/shot-board); [OpenField](https://github.com/wexloops/openfield) +- **Lip sync / talking:** CreatorOS talking avatar uses InfiniteTalk on Wan 2.1 chained block-by-block to audio length, or LTX-2.5 from typed lines — [CreatorOS](https://github.com/thePloggy/CreatorOS-Studio); OPENjutsu notes H3 lip-syncs any vocals unless "No lip-sync" feeds silent audio, and offers Fix faces "Match the photo" vs "Follow the mouth" — [OPENjutsu](https://github.com/D3voz/openjutsu); H3 Recast's optional FaceRefine lip pass reruns only the face from cache — [H3 Recast](https://github.com/Bambushu/h3-recast); SingSync does "measured lip-sync retime" with Seedance — [gh search](https://github.com/SasanSeyedi/singsync) +- **Licence traps flagged by entries:** Video Depth Anything Base/Large are CC-BY-NC-4.0 (ReAngle defaults to Small); Hunyuan3D 2.1 licence excludes EU/UK/South Korea (Comfy-Dial keeps meshes out of the repo); FLUX.2 marked non-commercial in OpenField; MiniMax H3 community licence has geographic exclusions (Parametric Studio, OpenField "geo-restricted") — [ReAngle](https://github.com/Tekholms-LW/ReAngle); [Comfy-Dial](https://github.com/ylchen333/comfy-dial); [OpenField](https://github.com/wexloops/openfield); [Parametric](https://github.com/parametricparadigm/ComfyUI_Parametric_Studio) + +### Inferences +- For dw (diffusers, not ComfyUI), the portable parts are the model choices and the pre/post-processing algorithms (depth reprojection + hole prefill, follow-cam crop + plate paste-back, audio-latent freezing, per-segment chaining with QC), not the graphs. Whether CrossView, Layout-to-Render IC-LoRA, BFS swap LoRA, SCAIL-2 and DMAD LoRA load in diffusers was not checked. + +### Gaps +- Did not verify diffusers support for any of these LoRAs/models. +- Lip-sync coverage beyond these few is thin in what I read; dedicated lip-sync entries (e.g. in Haggs' 26 apps) were not inspected. + +### Most idea-rich projects and what to mine (synthesis for the report writer) +- **OpenField** (MIT) — the broadest feature map; mine: Director axis list (genre/camera/framing/light/stock/look/grade/era/speed ramp), Stage greybox→depth→Layout-to-Render pipeline, loop seam score + fixer, LoRA filtering by tensor keys, per-bundle installs with sizes, 31-tool MCP surface. [repo](https://github.com/wexloops/openfield) +- **CreatorOS Studio** (Apache-2.0) — the most complete preset vocabulary (25 camera/effect presets, 7 relights, re-shoot angle set, Expand ratios) and CPU clip QC (black/frozen/jump/smear/face-drift). [repo](https://github.com/thePloggy/CreatorOS-Studio) +- **Aither Cinema Studio** (MIT) — rig-as-enum vocabulary (20/7/3/10/9) that compiles deterministically to prompt, `plan_shot` dry compile, `cut_music_video` beat sheet. [repo](https://github.com/Aitherium/aither-cinema-studio) +- **seriesmith** (MIT) — series bible + markdown scripts → model-free shot list → per-target compile; `continuous:` chaining queued up front; per-shot re-render; story-structure docs; skill generated from authoring docs. [repo](https://github.com/bgstratt/seriesmith) +- **Picky** (GPL-3.0, ideas only if dw is to stay permissive) — structured H3 shot document, H3 prompt linter, `@cast` → `` mapping, takes/lineage, quality tiers. [repo](https://github.com/chenthedesignmaker/ComfyUI-Picky) +- **OPENjutsu** (MIT) — mask/mannequin swap recipe, role-matching via VLM, "Check mask" cheap preview, preflight of nodes/models with install hints, resume from last finished clip, documented failure modes (reflections, clothing words leaking). [repo](https://github.com/D3voz/openjutsu) +- **ReAngle + Comfy-Dial** (Apache / MIT) — re-shoot by depth reprojection with background-only hole fill and chunk re-anchoring; generate-N-takes → recover camera → score per segment → repair only the drifted segment. [ReAngle](https://github.com/Tekholms-LW/ReAngle); [Comfy-Dial](https://github.com/ylchen333/comfy-dial) +- **open-higgsfield-skills** (MIT) — director-agent flow with approval gates, still-QA judge with fixed fix ladder, retry/GPU-minute caps, resumable JSON manifest. [repo](https://github.com/mojomast/open-higgsfield-skills) +- **Shot Board** (MIT) — six-cut coverage template with hero-frame propagation and pass/fail verdict notes. [repo](https://github.com/ArcSolver/shot-board) +- **Body Double / One-shot Multicam** (Apache / MIT) — follow-cam crop + plate paste-back; audio-latent freeze for frame-locked re-shoots and word-boundary multicam cuts. [Body Double](https://github.com/lucataco/open-source-higgsfield); [Multicam](https://github.com/apsolut/one-shot-multicam) +- **Boson** (MIT) — judged rounds persisted as git-backed "studio" state that later rounds must obey; stored vs live replay. [repo](https://github.com/0xMisterBailey/Boson) +- **Open Ads Studio** (Apache-2.0) — protected-region paste-back with a label-match SSIM score (49.0% → 99.8% in its example). [repo](https://github.com/jhoard/open-ads-studio) diff --git a/docs/proposals/cinematic-addons.md b/docs/proposals/cinematic-addons.md new file mode 100644 index 000000000..080c42702 --- /dev/null +++ b/docs/proposals/cinematic-addons.md @@ -0,0 +1,49 @@ +# Cinematic add-ons (tier 2): re-shoot, angles, speed, relight, dubbing, swap + +Source: [audits/2026-11-higgsfield](audits/2026-11-higgsfield/README.md), the section "Add-ons that unlock re-shoot, angles and dubbing". + +**Follows:** the tier 1 plan in [../superpowers/plans/2026-11-01-cinematic-controls.md](../superpowers/plans/2026-11-01-cinematic-controls.md). + +Every stage below opens with a probe on lem. A stage becomes a template only after a 24 GB proof and a licence read. Each probe is a `spend` decision for Don, and some also need a `trust` decision. + +## Stages + +**A. LTX-2.5 Slow-Motion-Control and Cinemagraph LoRAs.** +- **Probe:** get the sampling recipe from the cards, then run 2 clips each. +- **If they pass:** two templates under `workflows/templates/ltx2/`, plus `speed` and `cinemagraph` rows in the `cinematography` vocabulary. +- **Decisions:** spend (2 LoRA downloads, about 30 min on lem). + +**B. Qwen-Image-Edit-2511 + fal Multiple-Angles LoRA ("Shots").** +- **Checks first:** read the LoRA's licence. Choose a quantization that fits 24 GB, against the 57.7 GB base. +- **If it fits:** a `shots-9-angles` template with `for_each` over 9 fixed angle prompts, whose output feeds H3 FL2VA or LTX keyframes. +- **Decisions:** spend (about 58 GB download), trust (a new family). + +**C. LTX-2.5 Layout-To-Render IC-LoRA.** +- **Probe:** does it need the guided dev-transformer route that kept Clean-Plate and Day-To-Night out (`complete/ltx2-ic-loras-complete.md`)? +- **If not:** a blocking-to-shot template (depth video → render). +- **Re-shoot (ReAngle and Comfy-Dial):** depth estimation, warping along a camera path, then guiding the render. This adds a `reproject_depth` task on Video Depth Anything **Small** only, because Base and Large are CC-BY-NC. +- **Decisions:** spend, scope (new task). + +**D. LTX-2.3 Relight / Motion-Track / DubIt on 2.5 (probe only).** +- **Plan:** one afternoon of probes, and a record of each verdict. Nothing becomes a template unless a probe passes. +- **Decisions:** spend. + +**E. H3 swap recipe (OPENjutsu idea, existing tools).** +- **Recipe:** SAM-style `segment` track, the mask grown 40 px, a grey-mannequin motion reference, then Ref2VA with the new cast. +- **Output:** a skill section plus one template. The BFS, CrossView and DMAD LoRAs only if they load in diffusers. +- **Decisions:** spend. + +**F. LatentSync 1.6 dubbing task.** +- **Licence:** Apache-2.0 code, OpenRAIL++ weights, about 18 GB. +- **Output:** a `lip_sync` utility task on an existing clip plus an audio track. +- **Decisions:** trust (new dependency), spend. + +**G. 360 location references.** +- **Output:** an `equirect_views` image task that cuts a panorama into overlapping perspective references plus a manifest. No model. The idea comes from 360 Scout (MIT). +- **Decisions:** scope (new task). + +## Deferred + +**Wan 2.x** as a family: Animate swap, VACE, effect LoRAs, ReCamMaster, InfiniteTalk. Revisit only on a field report asking for an actor swap that stage E can't do. + +**The Lightricks camera LoRAs** target LTX-2 19B, not 2.5, so they count only as vocabulary. diff --git a/docs/stabilization/baseline.json b/docs/stabilization/baseline.json index 562025aee..22ec4ff23 100644 --- a/docs/stabilization/baseline.json +++ b/docs/stabilization/baseline.json @@ -1,5 +1,5 @@ { - "modules": 236, + "modules": 237, "modules_over_size_ceiling": 0, "functions_over_150_lines": 0, "prefix_literals": 0, diff --git a/docs/superpowers/plans/2026-11-01-cinematic-controls.md b/docs/superpowers/plans/2026-11-01-cinematic-controls.md new file mode 100644 index 000000000..564b9bef6 --- /dev/null +++ b/docs/superpowers/plans/2026-11-01-cinematic-controls.md @@ -0,0 +1,312 @@ +# Cinematic Controls (tier 1) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give users and agents Higgsfield's director vocabulary on dw's existing families. That means named camera, lens, light, look and grade choices compiled to each family's caption, a saved shot document, storyboards and timed animatics, a face-drift check between shots, and a sitcom multicam grammar. Every named choice is a hint, never a promise. + +**Architecture:** +- **Model knowledge stays in the plugin.** It goes into two new cross-cutting skills (`cinematography`, `storyboard`) and into reference files under the existing skills, which are all at their 12 KiB cap. +- **Engine surface:** two new task commands, `stills_to_video` and `analyze_face_drift`, and two templates that run them. No MCP tool, REST route or workflow syntax changes. +- **The shot document is stored in the existing prompt library** (`save_prompt` / `prompt:`). + +**Tech Stack:** Python 3 (`dw/tasks/`), OpenCV `cv2.FaceDetectorYN` + `cv2.FaceRecognizerSF`, workflow JSON templates, plugin skill markdown, pytest. + +**Spec:** [docs/proposals/audits/2026-11-higgsfield/README.md](../../proposals/audits/2026-11-higgsfield/README.md), tier "Adopt now". Don ruled on 2026-11-01 that named moves are **hints**. + +## Global Constraints + +- Model knowledge lives in `plugins/dw/` and the catalog, never in engine code (CLAUDE.md, "Before you edit"). +- Every `SKILL.md` stays at or under `SKILL_SIZE_LIMIT = 12 * 1024` bytes (`tests/test_plugin_skills.py:23`). Detail goes in `references/`. +- A new skill is named in `plugins/dw/README.md` and in `CLAUDE.md`'s plugin paragraph, and pinned by `tests/test_plugin_skills.py`. +- **Presets are hints.** Skill text never promises a move happens. It says to describe the move in the caption and check it with `get_output_frames`. +- **No camera-body brand names** (ARRI, RED, Cooke, Venice) as presets. Use abstract looks only. +- Code from no-licence, GPL or AGPL repos is never copied. Ideas only, with the project named. +- Weights are read at a pinned revision. SFace: `opencv/face_recognition_sface`, file `face_recognition_sface_2021dec.onnx`, revision `3d7082438a6e4551e840c9b2bb60b71e8da4b524`, Apache-2.0. YuNet: as in `dw/tasks/face_track.py:65-67`. +- Paths and assets go through `dw/security.py` validators. No `eval`, `exec` or `shell=True`. +- `docs/stabilization/baseline.json` `modules` goes 236 → 237 for the one new module (`dw/tasks/face_drift.py`). That is Q2 below. +- Compare backends with `get_device_type()` (`dw/__init__.py`). +- Warnings a caller must see go through `dw.events.emit_warning`, not `logger.warning`. + +## Review Focus + +1. **`stills_to_video` with stills of different sizes.** It must refuse and name both sizes, not stretch silently. Pinned in Task 3. +2. **`stills_to_video` with a duration shorter than one frame** (0.01 s at 24 fps). Each still must get at least one frame, and the shot map must still land on the frame grid. Pinned in Task 3. +3. **`analyze_face_drift` with a reference image that has no face, or two faces.** No face: refuse with a message naming the reference. Two faces: use the largest and say so in `notes`. Pinned in Task 4. +4. **`analyze_face_drift` on a shot where nobody is in frame** (an establishing shot). It must return a `no_face` finding for that shot, not a drift finding and not a crash. Pinned in Task 4. +5. **An agent following the `cinematography` skill for a family it doesn't cover** (Music 3, an image-only family). The skill must say the vocabulary is for video families plus Klein stills and stop there. Pinned by a text assertion in Task 1. + +--- + +## Stages (the plan Don approves; each is one GitHub stage issue) + +Stage letters match the harnest stage issues. Each stage's **acceptance intent** is what the source-blind tester writes `pending:` cases from. + +### Stage A: Camera-and-look vocabulary (`cinematography` skill) + +**Builds:** +- **A new skill `plugins/dw/skills/cinematography/`** with a closed vocabulary. It has six axes: shot size, camera move, lens character, light, era/stock, and genre/tempo. +- **One compiled sentence per value per family**, for H3, LTX-2.5, Kandinsky 6, and FLUX 3 Klein (stills). +- **A named-grade table**: noir, teal-orange, 70s stock, VHS and bleach-bypass. Each one is an `apply_lut` `palette` plus a `film_grain` setting. +- **A "presets are hints" rule** and the check that goes with it. +- **One pointer line in each family skill.** + +**Acceptance intent (over MCP and the plugin):** +- A cold agent asked for "a slow dolly-in on a close-up, noir look" for LTX-2.5 gets a caption containing that family's compiled move sentence and shot size. It runs the noir grade through `apply_lut` with the table's palette, and that validates clean. +- The same request for Kandinsky 6 produces exactly one camera sentence. +- `validate_workflow` stays clean for a template run with a compiled caption. +- The skill names no camera brands. +- Asked for a camera move on Music 3, the skill says it doesn't apply. + +### Stage B: Shot document, fix ladder, H3 lint (`script-to-video` references) + +**Builds:** +- **New shot-list fields** in `script-to-video`: `shot_size`, `move`, `lens`, `light`, `duration`. +- **Saving the shot list** as a prompt-library entry, `prompt:/shots`, holding the JSON text. Each shot's compiled caption is saved as `prompt:/shot-NN`, and the workflow references it by name. A revision re-saves and re-runs only the changed shots. +- **A written fix ladder**: regional edit or face repair, then re-prompt, then reseed, then ask the user. Caps: 3 tries per shot (the existing budget) and a GPU-minute cap per project, quoted from `plan.estimate`. +- **An H3 prompt lint checklist**: overfull beat, a cut with no new information, missing `overall_soundscape`. + +**Acceptance intent:** +- An agent following the skill on a 3-shot script ends with `list_prompts` showing `/shots` and `/shot-01..03`. +- Each run's arguments reference `prompt:/shot-NN`, which `get_job` shows. +- After a one-shot revision, only that shot's job is queued again. +- `get_prompt` on `/shots` returns JSON with the five new fields on every shot. + +### Stage C: `stills_to_video` task + +**Builds:** the task command `stills_to_video(images, durations, fps=24)`. It holds each still for its duration and returns an `AudioVideo` that is silent, with one shot record per still. It refuses mismatched sizes, durations that are non-positive or the wrong length, an fps outside (0, 120], and a total over 600 s. + +**Acceptance intent:** +- Three 512×512 stills with durations [1, 2, 0.5] at 24 fps produce a 84-frame clip of 3.5 s. Its manifest has 3 shots at frames 0, 24 and 72. +- A single duration, given as a number, applies to every still. +- A duration of 0.01 s still gives one frame. +- Each of these is refused with a message naming the argument: mixed sizes, `durations` with the wrong count, a duration of 0 or less, fps 0, fps 121, and more than 600 s in total. +- The clip goes on to `concat_videos` / `assemble-and-score` like any other clip. + +### Stage D: Storyboard and animatic (`storyboard` skill + `animatic` template) + +**Builds:** +- **A new skill `plugins/dw/skills/storyboard/`.** + - **Popcorn shape on FLUX 3 Klein:** up to 8 frames, references cited as "image one". + - **Shot Board's six-cut coverage set:** establishing, medium, face close-up, over-the-shoulder, insert and low angle. You pick a hero frame and redraw the others from it. + - **Board to start frame:** a board frame becomes an H3, LTX or K6 start frame. +- **An animatic recipe:** boards, then a TTS scratch track, then a Music 3 bed, then the new template `workflows/templates/animatic.json`. The template uses `stills_to_video` and an audio overlay through existing tasks. + +**Acceptance intent:** +- `list_workflows(shape="shot")`, or whichever shape the catalog derives, lists `animatic`. +- `validate_workflow` on it with 3 board assets, durations and an audio asset is clean and quotes a near-zero GPU estimate. +- Running it gives a video whose length is the sum of the durations, whose audio is the provided track trimmed or padded to that length, and whose `get_output_frames` at each shot's midpoint shows that board. + +### Stage E: `analyze_face_drift` task + `face-drift-check` template + +**Builds:** +- **The task command** `analyze_face_drift(video, reference, shots=None, threshold=0.363, samples_per_shot=8)`. + - It runs YuNet detection and SFace embedding on CPU or CUDA. + - It reports the cosine similarity, per shot, between the reference face and the best-matching face in sampled frames. + - Findings are `face_drift` (median below `threshold`) and `no_face`. They are places to look, in the `assess.py` answer shape. +- **A template** `workflows/templates/face-drift-check.json`. +- **Skill pointers** in `series-episodes` and `script-to-video`. + +**Acceptance intent:** +- Each of these produces the expected findings: + - a clip of the cast portrait animated by H3/LTX: `face_drift` absent; + - the same check against a different person's portrait: `face_drift` on every shot; + - a clip with an establishing shot: `no_face` on that shot only; + - a reference with no face: a refusal naming `reference`. +- The plan or estimate lists the SFace weights. +- A threshold outside [-1, 1] is refused. + +### Stage F: Sitcom multicam grammar (`series-episodes` reference) + +**Builds:** +- **A sitcom section** in `series-episodes/references/`: a wide master held throughout, per-speaker singles from `dialogue-short`, and cuts on line boundaries from `transcribe_audio` and `plan_cuts`. +- **One pointer line** in `SKILL.md`. + +**Acceptance intent:** +- A cold agent asked for "a sitcom scene, two characters, multicam" names the master, the singles and the line-boundary cut procedure, and gets each tool name right. +- `validate_workflow` on the cut it builds is clean. + +### Stage G: Measure named moves (trial → measured), spend-gated + +**Builds:** nothing in code. For each family (H3, LTX-2.5, K6), run 6 moves — static, dolly in, dolly out, truck left, crane up, orbit — at one seed, on a fixed subject. Judge each with `get_output_frames` and record honoured / partial / ignored in the `cinematography` skill's per-family table. Moves that are ignored are marked "hint only, rarely honoured". + +**Acceptance intent:** the skill's table carries a measured verdict for each of the 18 cells, with the job ids that support it in `docs/proposals/audits/2026-11-higgsfield/move-trial.md`. + +### Questions for Don + +- **Q1 [scope]:** Add two task commands (`stills_to_video`, `analyze_face_drift`) and two new skills, with no MCP, REST or syntax changes. **Default:** yes. +- **Q2 [none]:** The module ratchet goes 236 → 237 for `dw/tasks/face_drift.py`. **Default:** yes. The alternative is growing `face_track.py` (893 lines) past its ceiling. +- **Q3 [trust]:** A new weight download: SFace, Apache-2.0, 37 MB, at a pinned revision. **Default:** yes. +- **Q4 [spend]:** Stage G uses about 18 lem clips at draft settings, roughly 45–60 GPU minutes. **Default:** yes, after A–F verify. + +--- + +## Implementation tasks + +### Task 1: `cinematography` skill (Stage A) + +**Files:** +- Create: `plugins/dw/skills/cinematography/SKILL.md`, `plugins/dw/skills/cinematography/references/vocabulary.md`, `references/compile-h3.md`, `references/compile-ltx-2.5.md`, `references/compile-kandinsky-6.md`, `references/compile-flux3-klein.md`, `references/grades.md` +- Modify: `plugins/dw/skills/{minimax-h3,ltx-2.5,kandinsky-6,flux3-klein}/SKILL.md`, one pointer line each, staying under the cap (trim elsewhere in the same file if a skill is over). +- Modify: `plugins/dw/README.md`, `CLAUDE.md` plugin paragraph, `.claude-plugin/marketplace.json` and the plugin version, following the README's versioning rule. +- Test: `tests/test_plugin_skills.py` + +**Interfaces:** +- Produces: the axis names and value ids that Tasks 2, 5 and 7 cite: + - `shot_size` ∈ {extreme-wide, wide, full, medium, medium-close, close-up, extreme-close-up} + - `move` ∈ {static, dolly-in, dolly-out, truck-left, truck-right, crane-up, crane-down, orbit, tracking, whip-pan, push-in, handheld, crash-zoom} + - `lens` ∈ {wide, normal, long, anamorphic, vintage} + - `light` ∈ {high-key, low-key, golden-hour, practical, noir} + - `era` ∈ {8mm, 16mm, 35mm, vhs-dv, digital} + - `tempo` ∈ {still, measured, brisk, frantic} + - grade ids ∈ {noir, teal-orange, 70s-stock, vhs, bleach-bypass} + +- [ ] **Step 1: Write the failing tests** in `tests/test_plugin_skills.py`: + - `test_cinematography_skill_is_registered`: the skill dir is in the README's list and in CLAUDE.md. + - `test_cinematography_names_no_camera_brands`: none of `ARRI`, `RED `, `Cooke`, `Venice`, `Alexa` appear anywhere under the skill dir. + - `test_cinematography_vocabulary_covers_every_family`: each of the four `compile-*.md` contains a row for every `move` and `shot_size` id above. + - `test_cinematography_says_presets_are_hints`: `SKILL.md` contains the phrase `a hint, not a promise`. + - `test_cinematography_scopes_out_audio_families`: `SKILL.md` names `minimax-music3` as out of scope. + - `test_grades_are_valid_palettes`: every palette in `grades.md`'s fenced JSON passes `dw.tasks.lut.check_lut_source(palette=...)`. + - `test_family_skills_point_at_cinematography`: each of the 4 family `SKILL.md` contains `cinematography`. +- [ ] **Step 2:** `venv/bin/python -m pytest tests/test_plugin_skills.py -n0 -k cinematograph -v`. Expected: FAIL. +- [ ] **Step 3: Write the skill.** + - Write each family's compiled sentences from that family's own caption spec. Read `ltx-2.5/SKILL.md` and its references for the shot-type/motion/viewpoint rule, `kandinsky-6` for the one camera sentence, and `minimax-h3/references` for Context-IR camera language. + - For LTX, add the Lightricks card's compile rule: name what comes into view during the move. + - The `SKILL.md` body covers: when to use it; the axes; "a hint, not a promise", and how to check with `get_output_frames`; how to pick a grade; and which skill each family's compile table lives under. Attribute ideas to Aither (MIT), CreatorOS (Apache-2.0) and OpenField (MIT) in a references footer. +- [ ] **Step 4:** Run the same pytest. Expected: PASS. Then run `venv/bin/python -m pytest tests/test_plugin_skills.py -n0`. Expected: all PASS, size caps included. +- [ ] **Step 5: Commit** with `feat(plugin): # - cinematography skill: camera/look vocabulary compiled per family`. + +### Task 2: Shot document, fix ladder, H3 lint (Stage B) + +**Files:** +- Create: `plugins/dw/skills/script-to-video/references/shot-document.md`, `references/fix-ladder.md`, `plugins/dw/skills/minimax-h3/references/prompt-lint.md` +- Modify: `plugins/dw/skills/script-to-video/SKILL.md`. Its step 2 shot list gains the 5 fields, and pointer lines go to the three references, staying under the cap. +- Test: `tests/test_plugin_skills.py` + +**Interfaces:** +- Consumes: Task 1's axis ids. +- Produces: prompt names `/shots` (JSON text, an array of shot objects) and `/shot-NN` (two digits, from 01). The shot-object keys are `id, cast, location, opening_state, ending_state, beat, must_not, join, shot_size, move, lens, light, duration`. + +- [ ] **Step 1: Failing tests:** + - `test_shot_document_names_its_prompt_keys`: `shot-document.md` contains `/shots` and `/shot-NN` and every key above. + - `test_shot_document_uses_only_existing_tools`: every backticked `snake_case(` tool name in the three new files is a tool `dw_mcp` registers. Reuse the existing tool-name helper in that test file, if there is one. + - `test_fix_ladder_order`: `fix-ladder.md` lists `regional-edit`, then `face-repair`, then re-prompt, reseed and ask, in that order. + - `test_h3_lint_checks`: `prompt-lint.md` names `overall_soundscape`. +- [ ] **Step 2:** Run with `-k "shot_document or fix_ladder or h3_lint"`. Expected: FAIL. +- [ ] **Step 3: Write the references.** Recompiling only changed shots means re-saving `shot-NN` and calling `rerun_job` or `run_workflow` for that shot alone. Credit seriesmith (MIT), OpenReel (MIT) and open-higgsfield-skills (MIT) for ideas. Picky (GPL) is credited as an idea source only, with no text copied. +- [ ] **Step 4:** Run the full `tests/test_plugin_skills.py -n0`. Expected: PASS. +- [ ] **Step 5: Commit** with `feat(plugin): # - persisted shot document, fix ladder, H3 prompt lint`. + +### Task 3: `stills_to_video` task (Stage C) + +**Files:** +- Modify: `dw/tasks/video_utils.py`. Add `stills_to_video`. +- Modify: `dw/tasks/video_handlers.py`. Register the command with `domains={"fps": POSITIVE}`, plus the `static_check` for the counts it can see before run time. +- Modify: `docs/TASKS.md`. Add an entry beside `loop_frames`. +- Test: `tests/test_stills_to_video.py` (new) + +**Interfaces:** +- Produces: `stills_to_video(images, durations, fps=24) -> AudioVideo`, where: + - `images` is a list of PIL images, arrays, or image location dicts. Load them the way `ingredients_grid` loads its images. + - `durations` is a float or a list of floats in seconds. + - The return is `AudioVideo(frames=uint8 (n,h,w,3), audio=None, sample_rate=None, fps=fps, shots=[...])`. Shot records follow `dw/shots.py`, one per still with its start/end frame. +- Constants in `video_utils.py`: `MAX_STILLS_FPS = 120`, `MAX_STILLS_SECONDS = 600`. +- Frames per still = `max(1, round(duration * fps))`. + +- [ ] **Step 1: Failing tests** in `tests/test_stills_to_video.py`: + - `test_three_stills_hold_for_their_durations`: three 512×512 stills, `[1, 2, 0.5]`, fps 24 → `frames.shape == (84, 512, 512, 3)`, `fps == 24`, shot starts `[0, 24, 72]`. + - `test_one_duration_applies_to_all`: `durations=2` with 2 stills at 12 fps → 48 frames. + - `test_a_sub_frame_duration_still_gets_one_frame`: `[0.01]` at 24 → 1 frame. + - `test_mismatched_sizes_refused`: 512×512 and 512×768 → ValueError naming both sizes. + - `test_wrong_duration_count_refused`: 3 stills and 2 durations → ValueError naming `durations`. + - `test_non_positive_duration_refused`: `[1, 0]` → ValueError. + - `test_fps_bounds`: 0 and 121 refused, 120 accepted. + - `test_total_over_600_seconds_refused`: `[601]` → ValueError naming 600. + - `test_result_concats_with_concat_videos`: the output passed to `concat_videos` with itself gives 2× the frames. + - `test_command_registered`: the registry has `stills_to_video`. +- [ ] **Step 2:** `venv/bin/python -m pytest tests/test_stills_to_video.py -n0 -v`. Expected: FAIL. +- [ ] **Step 3: Implement** the function and its handler, following `_handle_loop_frames`. The silent `AudioVideo` must save as a video, whatever `result` path an image-less `AudioVideo` uses. Check `dw/media_types.py` save handling. If audio `None` isn't accepted, use `audio=numpy.zeros((1, 0))` with `sample_rate=None`, and pin that in a test. +- [ ] **Step 4:** Run the same pytest, then `venv/bin/python -m pytest tests/ -q` and `ruff check dw tests`. Expected: PASS, clean. +- [ ] **Step 5: Commit** with `feat(tasks): # - stills_to_video: hold each still for a duration`. + +### Task 4: `analyze_face_drift` task (Stage E) + +**Files:** +- Create: `dw/tasks/face_drift.py`. +- Modify: `dw/tasks/face_track.py`. Expose the YuNet loader for reuse: rename `_detector` to `face_detector`, or add a public wrapper. Don't copy it. +- Modify: `dw/tasks/video_handlers.py` (or the handler file that holds `analyze_shots`), `dw/tasks/weights.py` if it needs a registry entry, `docs/TASKS.md`, and `docs/stabilization/baseline.json` (`modules` 237). +- Test: `tests/test_face_drift.py` (new) + +**Interfaces:** +- Consumes: `assess.media_from`, `assess.resolve_shots`, and `assess._answer` (or its public equivalent) for the answer shape. +- Produces: `analyze_face_drift(video, reference, shots=None, threshold=0.363, samples_per_shot=8) -> dict`. The dict has these keys: + - `measurements`, a list with one entry per shot: `{shot, frames_sampled, faces_found, similarity_median, similarity_min}`; + - `findings`; + - `rules_applied`; + - `shots_source`; + - `notes`. +- Constants: `SFACE_REPO = "opencv/face_recognition_sface"`, `SFACE_FILE = "face_recognition_sface_2021dec.onnx"`, `SFACE_REVISION = "3d7082438a6e4551e840c9b2bb60b71e8da4b524"`. +- Similarity is `FaceRecognizerSF.match(..., FR_COSINE)`. The 0.363 default is OpenCV's published cosine threshold for SFace. + +- [ ] **Step 1: Failing tests** in `tests/test_face_drift.py`. Inject a fake detector and recognizer, as `tests/test_face_track.py` does, so no weights download. + - `test_same_face_no_drift_finding`. + - `test_other_face_drift_on_every_shot`. + - `test_empty_shot_reports_no_face_only_there`: shot 1 has no detections, shot 2 matches → `no_face` at shot 1 only. + - `test_reference_without_face_refused`: ValueError naming `reference`. + - `test_reference_with_two_faces_uses_largest_and_notes_it`. + - `test_threshold_bounds`: -1.01 and 1.01 refused. + - `test_samples_per_shot_caps_reads`: a 100-frame shot reads 8 frames. + - `test_required_weights_lists_sface_and_yunet`. + - `test_command_registered`. +- [ ] **Step 2:** `venv/bin/python -m pytest tests/test_face_drift.py -n0 -v`. Expected: FAIL. +- [ ] **Step 3: Implement.** + - Per sampled frame: detect all faces, align and crop each with `recognizer.alignCrop`, embed it, and keep the best cosine match against the reference. + - Per shot: the median and minimum of those best values over the frames that had a face. + - Rules: `face_drift` when the median is below `threshold`; `no_face` when no sampled frame had a face. + - Load the recognizer through `model_cache.cached_model`, like YuNet. +- [ ] **Step 4:** Run the same pytest, the full suite, `ruff`, and `venv/bin/python scripts/arch_metrics.py` (or the ratchet test). Expected: PASS, with modules at 237. +- [ ] **Step 5: Commit** with `feat(tasks): # - analyze_face_drift: SFace identity check per shot`. + +### Task 5: `animatic` and `face-drift-check` templates, `storyboard` skill (Stages D + E) + +**Files:** +- Create: `workflows/templates/animatic.json`, `workflows/templates/face-drift-check.json`, `plugins/dw/skills/storyboard/SKILL.md`, `plugins/dw/skills/storyboard/references/coverage-set.md`, `references/animatic.md`. +- Modify: `plugins/dw/README.md`, `CLAUDE.md`, and the plugin version (as in Task 1); `series-episodes/SKILL.md` and `script-to-video/SKILL.md` get one pointer line each to `face-drift-check`. +- Test: `tests/test_plugin_skills.py`, plus whatever the catalog uses to validate every template (find the test that loads every `workflows/templates/**/*.json`, such as `tests/test_templates*.py`, and confirm it picks the new files up). + +**Interfaces:** +- Consumes: `stills_to_video` (Task 3), `analyze_face_drift` (Task 4), and the existing audio tasks for laying a track under a clip. Use the same pairing step `assemble-and-score.json` uses. +- `animatic.json` variables: `boards` (a list of image refs), `durations` (a number or a list), `fps` (default 24), `soundtrack` (an audio ref). +- `face-drift-check.json` variables: `video`, `reference`, `threshold` (default 0.363). + +- [ ] **Step 1: Failing tests:** + - `test_storyboard_skill_registered`. + - `test_storyboard_names_six_coverage_cuts`: establishing, medium, close-up, over-the-shoulder, insert and low angle all appear in `coverage-set.md`. + - `test_storyboard_caps_boards_at_eight`. + - Template validation passes for both new templates (`python -m dw.validate workflows/templates/animatic.json`). +- [ ] **Step 2:** Run them. Expected: FAIL. +- [ ] **Step 3: Write the templates and the skill.** Board generation in the skill uses the existing Klein `multi-reference` and `consistent-set` templates. Credit Higgsfield's Popcorn (concept) and Shot Board (MIT). +- [ ] **Step 4:** Run the full suite and `python -m dw.validate` on both templates. Expected: PASS, clean. +- [ ] **Step 5: Commit** with `feat(catalog): #/# - animatic + face-drift-check templates, storyboard skill`. + +### Task 6: Sitcom multicam grammar (Stage F) + +**Files:** +- Create: `plugins/dw/skills/series-episodes/references/sitcom-multicam.md`. +- Modify: `plugins/dw/skills/series-episodes/SKILL.md` (one pointer line, under the cap). +- Test: `tests/test_plugin_skills.py`. + +- [ ] **Step 1: Failing tests:** + - `test_sitcom_reference_names_its_tools`: names `dialogue-short`, `transcribe_audio` and `plan_cuts`, and each backticked tool or template exists. + - `test_series_points_at_sitcom`. +- [ ] **Step 2:** Run them. Expected: FAIL. +- [ ] **Step 3: Write the reference.** The wide master is held through the scene, singles come per speaker, cuts land on line boundaries from the transcript's word timings, and the master is the fallback when a single fails the face-drift check. Credit One-shot Multicam (MIT). +- [ ] **Step 4:** Run the full `tests/test_plugin_skills.py -n0`. Expected: PASS. +- [ ] **Step 5: Commit** with `feat(plugin): # - sitcom multicam grammar`. + +### Task 7: Move trial (Stage G, after A–F verify and Q4) + +- [ ] For each of H3 (draft tier), LTX-2.5 (distilled) and K6, run the 6 moves at seed 1234 on one subject. The prompt is the family's compiled sentence from Task 1. Each run gets a `note` naming the move. +- [ ] Judge each with `get_output_frames` (first, middle and last frame) as honoured, partial or ignored. +- [ ] Write `docs/proposals/audits/2026-11-higgsfield/move-trial.md` with the 18 cells and their job ids, and update each `compile-*.md` table with a `measured` column. +- [ ] Commit with `docs(plugin): # - named-move trial verdicts`. diff --git a/docs/superpowers/specs/2026-11-01-docker-image-design.md b/docs/superpowers/specs/2026-11-01-docker-image-design.md new file mode 100644 index 000000000..a4f222c29 --- /dev/null +++ b/docs/superpowers/specs/2026-11-01-docker-image-design.md @@ -0,0 +1,318 @@ +# Docker image - design + +Status: built on feat/docker-image (2026-11-01); lem GPU test pending. The guide is docs/DOCKER.md. Tauri desktop notes at the end; they are a +follow-on, not part of this spec's scope. + +## Intent + +Make dw installable on a Linux/NVIDIA box with one `docker run` (or +`docker compose up`): no Python, no venv, no torch-index guesswork. This is +the first step toward packaging dw for end users. The second is a Tauri +desktop app (notes below). + +Don's constraint: **diffusers tracks its GitHub main HEAD**, because the newest +model pipelines land there before a release. That means diffusers can't be +frozen into the image. It lives on a volume where it can be updated and where +updates survive restarts, container re-creation and image upgrades. + +Docker is the Linux/NVIDIA (and remote-server, `docs/REMOTE.md`) channel only. +Docker Desktop on a Mac runs a Linux VM with no MPS, so Apple Silicon users +would get CPU only. Macs are the Tauri app's job. + +## How ComfyUI does it (for reference) + +- Git clone plus pip is the base install. The Windows portable 7z (embedded + Python, torch preinstalled) and comfy-cli wrap it. +- ComfyUI Desktop (Electron) bundles uv and a standalone Python, **installs + torch on first run** for the detected GPU, pins the ComfyUI core inside the + app, and keeps models and user data in a folder outside the app. +- The frontend ships as its own pip package (`comfyui-frontend-package`). + dw already does the equivalent: `scripts/build_dist.sh` copies the SPA + into `dw.server`. +- There is no official first-party Docker image. Community images + (yanwk/comfyui-boot, ai-dock) fill the gap. + +Lessons taken: torch is chosen per machine; user data lives outside the +thing that gets replaced; the engine version is pinned by the package and +not by whatever HEAD was on the day of install. diffusers is the one +deliberate exception, and this design gives it its own volume. + +## Layering + +``` +image (immutable, rebuilt per dw release) + /usr/local/bin/python python:3.13-slim system interpreter + site-packages torch (cu130 or cu12), dw wheel [server,mcp], + every pinned dependency, and a *baked* diffusers + (a git commit chosen at build time) + /opt/dw/examples the repo's workflows/ + prompts/ (read-only catalog) + +volume /data (persistent, owned by the user) + env/venv the diffusers overlay: a venv made with + --system-site-packages over the image's Python + env/overlay.json marker: image build id, Python version, + installed diffusers commit + workspace/ DW_WORKSPACE - workflows, prompts, assets, outputs + hf/ HF_HOME - model weights, incl. dw-quantized/ + cache/ XDG_CACHE_HOME (torchinductor), TRITON_CACHE_DIR + settings/ DIFFUSERS_HELPER_ROOT - settings.json + home/ HOME for the non-root user +``` + +### Why an overlay venv, not the whole venv on the volume + +With the whole venv on the volume, the image would be nothing more than a +bootstrap script, and an upgrade would mean reinstalling torch (several GB) +into the volume. The overlay holds only what differs from the image: + +- `python -m venv --system-site-packages /data/env/venv` sees every image + package. Its own site-packages comes first on `sys.path`, so a diffusers + installed there shadows the baked one. +- pip run from the overlay leaves the image's copy alone ("Not uninstalling + diffusers ... outside environment") and installs into the overlay. It adds + only the dependencies the new diffusers commit needs that the image doesn't + already satisfy. +- **The existing updater works unchanged.** `build_pip_commands` runs + `sys.executable -m pip`. The server runs under the overlay's Python, so the + Models page's HEAD / pinned-commit / revert actions land on the volume, and + the worker restart after an update behaves as it does today. + +A volume holding a git checkout of diffusers (editable install) was rejected. +pip's `git+` install is what install.sh and the updater already do, and +a checkout adds a second update path to keep consistent. + +### Overlay lifecycle (the entrypoint) + +The entrypoint is `docker/entrypoint.py`, run by the image's interpreter. +It's a script outside `dw/`, so it adds no engine module to the ratchet. It +imports `dw.security` (`sanitize_command_args`, `validate_commit_hash`) and +`dw.server.updater` (`build_pip_commands`, which now takes the interpreter +to install into); no `shell=True`. tests/test_docker.py loads it by path. On every start it: + +1. **Creates** the overlay if `/data/env/venv` is missing, then installs the + diffusers that `DW_DIFFUSERS_REF` names. The two install commands are the + same ones `build_pip_commands` builds. +2. **Rebuilds** it if `overlay.json`'s image build id or Python minor version + doesn't match the running image. An old overlay can hold dependency + versions that would now shadow newer ones the image ships. The rebuild + reinstalls the *recorded* diffusers commit, not HEAD, so an image upgrade + never silently moves diffusers. +3. **Smoke-tests** the overlay with `python -c "import diffusers, dw.server"`. + If that fails (a broken HEAD; transformers regressions have broken HEAD + before), it logs the error and serves from the image's Python with the + baked diffusers instead, so the server still comes up and the UI can be + used to pin a working commit. +4. **Execs** ` -m dw.serve --host 0.0.0.0 --examples-dir + /opt/dw/examples ...`. User flags pass through: `docker run ... image + --devices cuda:0,cuda:1 --mcp`. + +`DW_DIFFUSERS_REF` (validated): + +| value | meaning | +|---|---| +| `head` (default) | install main HEAD **when the overlay is created**; afterwards update from the Models page or `entrypoint.py update-diffusers` | +| a 40-hex commit | install that commit at creation | +| `image` | no overlay diffusers; use the baked one | + +**A restart never updates diffusers.** HEAD moves only when the user asks, +so restarting a container is reproducible. Automatic updates on start +(`DW_DIFFUSERS_UPDATE=on-start`) are a possible later addition. + +`python /opt/dw/entrypoint.py update-diffusers [--commit | --revert]` +(through `docker compose exec`) runs the same `build_pip_commands` the UI +button runs. + +If the first start has no network, the overlay install fails, the +entrypoint logs it, and the server falls back to the baked diffusers. The +image works offline. + +## The image + +Multi-stage `Dockerfile` at the repo root: + +1. **ui stage** (`node:`): `npm ci && npm run build`, then the + `build_dist.sh` steps (copy the SPA into `dw/server/ui`, the docs into + `dw/docs`), then `python -m build` produces the wheel. +2. **runtime stage** (`python:3.13-slim-bookworm`): + - apt: `git` (for pip's git installs), `build-essential` (Inductor and + Triton compile kernels at run time), `tini`. + - torch from `ARG TORCH_INDEX`, in install.sh's order: torch first, then + the wheel with `[server,mcp]`, then + `pip install --force-reinstall --no-deps git+...diffusers@${DIFFUSERS_REF}` + and a plain `pip install diffusers` for its dependencies. + - Install pins come from a constraints file generated per release, so a + rebuild of the same tag resolves the same versions. Today only the + pyproject floors exist; the constraints file is new work. The + `<0.3` cap on comfy-kitchen is already a pin. + - No CUDA base image is needed. torch wheels bundle the CUDA runtime + (`nvidia-*` packages), and the host's driver comes in through the NVIDIA + Container Toolkit. + - A non-root `dw` user, `HOME=/data/home`, `WORKDIR /data`. + - Labels: dw version, the baked diffusers commit, the torch build, and the + image build id the overlay marker compares against. + - `HEALTHCHECK` against `GET /api/health`. + +**Decided: cu130 only, and build it yourself for now.** Nothing is +published to a registry yet; `docker compose build` (or `docker build -t +dw .`) on the GPU box is the install. The requirements are an NVIDIA driver +>= 580 and a Turing or newer card: CUDA 13 dropped Maxwell, Pascal and +Volta. The only reason to add a cu12 variant would be a user stuck on +an older driver branch, or on a V100 or a 10-series card. `ARG TORCH_INDEX` +keeps that a one-line build override (`--build-arg +TORCH_INDEX=https://pypi.org/simple`), with no published tag. +linux/amd64 only at first; arm64 (GH200, DGX Spark) comes later if someone +asks. + +Expected size: about 8-10 GB, mostly torch and the CUDA libraries. Model +weights are never baked in; they go to `/data/hf`. + +## Running it + +**As built: shared with a native install.** `docker/init-env.sh` writes +`docker/.env` from the host's native layout (workspace, HF_HOME, the +Inductor and Triton caches, the settings directory). compose mounts each one +**at the same absolute path** and sets the variable dw reads. Manifests, +settings and quantized-cache entries record absolute paths, so the +container and a systemd dw.serve read each other's records. +`docker/switch.sh docker|native|status` stops one and starts the other. +Only `DW_ENV_DIR` (the overlay) is the container's own. The single-`/data` +layout above is still the image's default when it runs without compose. +The sketch below is the original; `docker/compose.yaml` is what shipped. + +`docker-compose.yml` in the repo: + +```yaml +services: + dw: + build: . + image: diffusers-workflow:local + ports: ["127.0.0.1:8765:8765"] + volumes: + - ./dw-data:/data # or split: ~/.cache/huggingface:/data/hf + environment: + DW_API_TOKEN: ${DW_API_TOKEN:-} + HF_TOKEN: ${HF_TOKEN:-} # gated models + deploy: + resources: + reservations: + devices: [{driver: nvidia, count: all, capabilities: [gpu]}] + restart: unless-stopped +``` + +- Every `/data/*` subpath can be its own mount, and each env var can be + overridden, so an existing host HF cache can be shared instead of + downloading models twice. +- Multi-GPU works the way it does on lem: `--devices cuda:0,cuda:1`. The + worker manager's `CUDA_VISIBLE_DEVICES` pinning works inside the + container. +- `--user $(id -u):$(id -g)` with a bind mount keeps output files owned by + the host user. The entrypoint must not need root. + +## Security + +- The server has to bind `0.0.0.0` inside the container, and it cannot see + whether the port is published on localhost only or everywhere. + `dw.serve` warns on a non-loopback host without a token, and refuses + `--mcp` there. **Decided: Jupyter's pattern.** If `DW_API_TOKEN` is unset + on first start, the entrypoint generates a token, saves it to + `/data/settings/token` (mode 0600), and logs a login URL, + `http://127.0.0.1:8765/?token=`, on every start. Unlike Jupyter, + the token persists across restarts, so an MCP client configured with the + header keeps working. Setting `DW_API_TOKEN` overrides it. + `DW_INSECURE_NO_TOKEN=1` is the explicit opt-out, for a port published on + localhost only. + - **UI work:** the SPA has a token prompt and appends the token to media + URLs (`ui/src/lib/apiUrls.ts`), but it does not read `?token=` from the + page URL. It needs to: save the token the way the prompt does, then + remove it from the address bar with `history.replaceState` so it stays + out of history and screenshots. +- `--trust-workflows` stays off by default; the container changes nothing + about the trust model. The usual `docs/SECURITY.md` rules apply. +- The overlay venv lives on a volume the container writes to, so anything + that can write to the volume can change the code the server runs. That's + no different from a venv on disk, but the docs should say it. +- Entrypoint subprocesses use fixed argument lists through + `sanitize_command_args`. `DW_DIFFUSERS_REF` goes through + `validate_commit_hash` or the two literal keywords. + +## Known wrinkles + +- `release_floor()` reads `pyproject.toml` relative to the source tree. From + a wheel it falls back to `FALLBACK_RELEASE_FLOOR`. That already works, but + in the container "revert" could more usefully mean "drop the overlay's + diffusers and use the baked one". That is a small updater change, and + optional. +- If the smoke test failed and the server is running on the image's Python, + the UI updater would pip into the container's writable layer, and that + install is lost when the container is re-created. In that mode the + entrypoint should make the updater rebuild the overlay instead. Exactly + how that hand-off works still needs designing. +- flash_attn needs nvcc, and the image doesn't have it. Out of scope. The + backends that ship as Hub kernels (`kernels`, sage_hub) and comfy-kitchen's + prebuilt Linux kernels do work. +- lem and mini-ai keep their venv installs. Docker is an additional channel + and replaces nothing. + +## Testing + +- pytest for `docker/entrypoint.py` (tests/test_docker.py): marker mismatch leads to a rebuild; the + recorded commit (not HEAD) is reinstalled on a rebuild; `DW_DIFFUSERS_REF` + validation; a failed smoke test falls back to the image's Python; the + argument lists are what `build_pip_commands` builds. Subprocess calls are + mocked. +- CI (no GPU): build the image; run the container with `DW_DIFFUSERS_REF=image`; + check that `/api/health` answers, and that `dw.validate` passes on + `workflows/templates/text-to-image.json` inside the container. +- GPU smoke on lem: `docker compose up`, then `python -m dw.test` inside the + container; then create an overlay at HEAD, update from the UI, re-create + the container, and confirm the commit survived. + +## Stages + +1. `docker/entrypoint.py` + tests (no Docker needed). +2. Dockerfile, compose file, `.dockerignore`, CI build and health check. +3. A per-release constraints file, generated by `release.sh`, that the + Dockerfile installs against. Publishing to a registry (GHCR) is + deferred; it would be a later `release.sh` step. +4. The UI's `?token=` login link (see Security). +5. `docs/DOCKER.md`, linked from README, DEPENDENCIES and REMOTE. + +## Decisions (Don, 2026-11-01) + +1. Token: Jupyter's pattern (generated, persisted, login URL in the log). +2. cu130 only; cu12 only through a build arg. +3. Build it yourself; no registry publishing yet. + +--- + +## Tauri desktop app - notes for the follow-on spec + +Not designed here. What the Docker work sets up for it: + +- **Same layering, minus Docker.** The app bundles uv as a sidecar + (`externalBin`). On first run it does what `docker/entrypoint.py` does, with uv + in place of the image: it picks a Python (uv-managed), installs torch for + the detected accelerator (install.sh's MPS / cu130 / cu12 / CPU branches, + ported), installs the pinned dw wheel from the release's constraints file, + and then the diffusers overlay. The constraints file from stage 3 is the + shared piece. +- **A thin shell.** `dw.serve` already serves the UI. The app starts it as a + child process on 127.0.0.1 with a random port and a per-launch token, + waits for `/api/health`, and points the webview at it. That keeps one + origin, with no CORS and no second copy of the SPA. It also stops other + local processes from driving the GPU through the server. +- **Data outside the app.** The user chooses the folder, and the same + `/data` layout applies (workspace, hf, cache, settings, env), so a + reinstall or app update never touches models or outputs. An existing HF + cache can be pointed at instead of re-downloaded. +- **Updates.** The app updater moves the dw wheel (the engine is pinned per + app version, as in ComfyUI Desktop). diffusers stays on the overlay and + the Models page, as here. Any engine change needs a server restart; the + persistent worker only loads new code when it restarts. +- **Platforms.** macOS arm64 first, since that's the gap Docker can't fill, + then Windows/NVIDIA (WebView2), then Linux, where WebKitGTK quirks need an + early check against the Svelte UI. macOS also needs `fp4-fp8-for-torch-mps`, + which pyproject can't express, so it goes in the bootstrap's + per-platform branch. +- **Signing.** macOS notarization and Windows code signing are release-pipeline + work and need certificates. Budget for it. diff --git a/dw/arguments.py b/dw/arguments.py index c08f2f2cd..67d31640e 100644 --- a/dw/arguments.py +++ b/dw/arguments.py @@ -925,6 +925,7 @@ def resolve_relative_path(path, base_dir): "analyze_shots", "analyze_seams", "analyze_sync_drift", + "analyze_face_drift", "window_video", "trim_video", "generate_sound_effects", diff --git a/dw/server/updater.py b/dw/server/updater.py index 2d68c56ac..2a5c6fb0d 100644 --- a/dw/server/updater.py +++ b/dw/server/updater.py @@ -72,11 +72,14 @@ def diffusers_install_info(): return info -def build_pip_commands(commit=None, revert=False): +def build_pip_commands(commit=None, revert=False, python=None): """The pip commands, run in order, for one of three installs: HEAD, a pinned commit, or a revert to the pyproject.toml release floor. `commit` is validated by the caller before it reaches here - this only assembles the already-validated pieces, never a raw request value. + `python` is the interpreter whose environment they install into - this + one by default; the container entrypoint (docker/entrypoint.py) passes + its diffusers overlay venv's. A git install is two commands. diffusers main keeps one dev version string for a whole release cycle, so `install --upgrade ` saw a @@ -88,7 +91,7 @@ def build_pip_commands(commit=None, revert=False): commit requires - no URL, so no second clone, and no --upgrade, so the git build stays. """ - pip = [sys.executable, "-m", "pip", "install"] + pip = [python or sys.executable, "-m", "pip", "install"] if revert: return [pip + [f"{PYPI_PACKAGE}=={release_floor()}"]] target = DIFFUSERS_GIT_URL if not commit else f"{DIFFUSERS_GIT_URL}@{commit}" diff --git a/dw/task_problems.py b/dw/task_problems.py index 3b273591b..201d28dbe 100644 --- a/dw/task_problems.py +++ b/dw/task_problems.py @@ -4,7 +4,8 @@ is everything a single range cannot say: crop_face_track's crop size on the video model's frame grid, gate_zero above gate_full; analyze_beats' tempo range and anchors that are one kind and in time order; plan_cuts' transcript shape -and render grid; analyze_phrases' `centre` as a real boolean; apply_lut's one-of-two source and its palette; check_script's +and render grid; analyze_face_drift's threshold and sample count; +analyze_phrases' `centre` as a real boolean; apply_lut's one-of-two source and its palette; check_script's `lines` and `shots`; ingredients_grid's choices and image count. Each family's `*_problems` function is the rule, and both sides call it - its `*_errors` static check (registered as the command's `static_check`, run by @@ -75,6 +76,140 @@ def ingredients_grid_errors(arguments): return errors +# stills_to_video's ceilings, read by the command and by its static check. The +# seconds and frame caps bound the length of the clip; they do not bound its +# memory, which also depends on the stills' size. The clip is held as uint8 +# frames, so the command refuses a clip over MAX_STILLS_BYTES before it +# allocates (the size is only known at run time) +MAX_STILLS_FPS = 120 +MAX_STILLS_SECONDS = 600 +MAX_STILLS_FRAMES = 36000 +MAX_STILLS_BYTES = 4 * 1024**3 + + +def stills_to_video_problems(durations, fps, count=None): + """[(argument, message)] for the stills_to_video rules past a domain: `fps` + in 1..MAX_STILLS_FPS, every duration above zero, one duration per still + when both are known, and a total of at most MAX_STILLS_SECONDS. Values + that are references, or not yet known, are the run's to check.""" + problems = [] + rate = as_number(fps) + if rate is not None and not 0 < rate <= MAX_STILLS_FPS: + problems.append( + ( + "fps", + f"stills_to_video needs 'fps' above 0 and at most " + f"{MAX_STILLS_FPS}, got {fps}", + ) + ) + if durations is None or is_ref(DEFERRED, durations): + return problems + many = isinstance(durations, (list, tuple)) + items = list(durations) if many else [durations] + if any(is_ref(DEFERRED, item) or is_ref(GATHER, item) for item in items): + return problems + numbers = [] + for item in items: + number = as_number(item) + if number is None or number <= 0: + problems.append( + ( + "durations", + f"stills_to_video needs every one of 'durations' as a " + f"number of seconds above 0, got {item!r}", + ) + ) + return problems + numbers.append(number) + if many and count is not None and len(numbers) != count: + problems.append( + ( + "durations", + f"stills_to_video was given {count} images but {len(numbers)} " + "'durations' - give one duration per image, or a single number " + "for all of them", + ) + ) + return problems + total = sum(numbers) * (count if not many and count else 1) + if total > MAX_STILLS_SECONDS: + problems.append( + ( + "durations", + f"stills_to_video makes a clip of {total:g} seconds, over the " + f"{MAX_STILLS_SECONDS} second limit - shorten the durations", + ) + ) + rate = as_number(fps) + if rate is not None and 0 < rate <= MAX_STILLS_FPS: + frames = sum(max(1, round(n * rate)) for n in numbers) + if not many and count: + frames *= count + if frames > MAX_STILLS_FRAMES: + problems.append( + ( + "durations", + f"stills_to_video makes {frames} frames, over the " + f"{MAX_STILLS_FRAMES} frame limit - shorten the " + "'durations' or lower 'fps' (each still takes at least " + "one frame)", + ) + ) + return problems + + +def stills_to_video_errors(arguments): + """The stills_to_video rules a literal workflow can break before it runs""" + images = arguments.get("images") + count = None + if isinstance(images, list) and not any(is_ref(GATHER, item) for item in images): + count = len(images) + return stills_to_video_problems( + arguments.get("durations"), arguments.get("fps", 24), count + ) + + +# analyze_face_drift's two settings: a cosine similarity is -1..1, and the +# frames read from each shot are a whole number up to FACE_DRIFT_MAX_SAMPLES +FACE_DRIFT_MAX_SAMPLES = 64 + + +def face_drift_problems(threshold=None, samples_per_shot=None): + """[(argument, message)] for the analyze_face_drift rules these values + break. A value that is None, a reference or not a number is skipped - the + domain check owns that complaint.""" + problems = [] + cosine = as_number(threshold) + if cosine is not None and not -1.0 <= cosine <= 1.0: + problems.append( + ( + "threshold", + "analyze_face_drift: threshold must be a cosine similarity " + f"from -1 to 1, not {threshold}", + ) + ) + samples = as_number(samples_per_shot) + if samples is not None and ( + samples != int(samples) or not 1 <= samples <= FACE_DRIFT_MAX_SAMPLES + ): + problems.append( + ( + "samples_per_shot", + "analyze_face_drift: samples_per_shot must be a whole number " + f"from 1 to {FACE_DRIFT_MAX_SAMPLES}, not {samples_per_shot}", + ) + ) + return problems + + +def face_drift_errors(arguments): + """The analyze_face_drift rules a literal workflow can break before it + runs""" + return face_drift_problems( + arguments.get("threshold"), arguments.get("samples_per_shot") + ) + + # crop_face_track's rules past a plain domain. The crop feeds a video model # whose frame size and count move on a grid the workflow declares (modulus, # remainder and multiple; the defaults are LTX's 8n+1 and 32 pixels), the diff --git a/dw/tasks/face_drift.py b/dw/tasks/face_drift.py new file mode 100644 index 000000000..6d8f4252c --- /dev/null +++ b/dw/tasks/face_drift.py @@ -0,0 +1,299 @@ +"""analyze_face_drift: does a recurring character still look like their cast +portrait (#1146)? + +Given a clip and a reference face, YuNet finds the faces in a few frames of +each shot, SFace embeds them, and the cosine match against the reference is +reported per shot: the median and the minimum, over the sampled frames that +had a face, of the best match among the faces in that frame. The answer has +the shape of the assessment probes (`dw/tasks/assess.py`) - every +measurement, `findings`, `rules_applied`, `shots_source` - and its findings +are places to look, not verdicts. It is not an assessment probe itself +(`assessment=True` is for probes that read a finished cut's soundtrack and +seams) and its two rules live here, as `measure_sync`'s do. + +The reference is one face. With none the call is refused; with several the +largest is used and `notes` says so. + +Models: YuNet through `face_track.face_detector_model` (the one loader) and +SFace the same way, both at a pinned revision and through the task model +cache. OpenCV works in BGR; the pictures here are RGB until they reach it. +""" + +import logging + +import numpy as np +from PIL import Image + +from ..assessment_rules import finding, sort_findings +from ..task_domains import real_number +from ..task_problems import face_drift_problems +from .assess import media_from, resolve_shots +from .face_track import ( + DEFAULT_DETECTOR_FILE, + DEFAULT_DETECTOR_REPO, + face_detector_model, + required_weights as detector_weights, +) +from .weights import file_entries + +logger = logging.getLogger("dw") + +COMMAND = "analyze_face_drift" + +# OpenCV's own upload of SFace, read at a pinned revision so a later push to +# the repo cannot change what an existing workflow measures +SFACE_REPO = "opencv/face_recognition_sface" +SFACE_FILE = "face_recognition_sface_2021dec.onnx" +SFACE_REVISION = "3d7082438a6e4551e840c9b2bb60b71e8da4b524" + +# cv2.FaceRecognizerSF_FR_COSINE +COSINE = 0 + +# OpenCV's published cosine threshold for SFace: at or above it two faces +# are the same person +DEFAULT_THRESHOLD = 0.363 +DEFAULT_SAMPLES = 8 +# The detector score below which a face is ignored, as crop_face_track's +DETECTOR_CONFIDENCE = 0.6 +PLACES = 3 + +RULES = { + "face_drift": { + "name": "face_drift", + "severity": "warn", + "says": ( + "the faces in this shot sit this close to the reference, below the" + " threshold - the character may not look like their portrait here" + ), + }, + "no_face": { + "name": "no_face", + "severity": "info", + "threshold": 1, + "says": "no face was found in the frames sampled from this shot", + }, +} + + +def required_weights(arguments): + """SFace and YuNet's weight files, for the plan (dw/tasks/weights.py).""" + return file_entries(SFACE_REPO, SFACE_FILE, SFACE_REVISION) + detector_weights({}) + + +def _models(device=None): + """(detect, recognizer): `detect(bgr)` is every face YuNet finds as a full + row (box, landmarks, score), `recognizer` an SFace FaceRecognizerSF.""" + import cv2 + + from .model_cache import cached_model + + model = face_detector_model( + DEFAULT_DETECTOR_REPO, DEFAULT_DETECTOR_FILE, DETECTOR_CONFIDENCE, device + ) + + def load(): + from huggingface_hub import hf_hub_download + + path = hf_hub_download( + repo_id=SFACE_REPO, filename=SFACE_FILE, revision=SFACE_REVISION + ) + return cv2.FaceRecognizerSF.create(path, "") + + recognizer = cached_model( + ("analyze_face_drift", SFACE_REPO, SFACE_FILE, SFACE_REVISION), load + ) + + def detect(bgr): + height, width = bgr.shape[:2] + model.setInputSize((width, height)) + _, faces = model.detect(bgr) + return [] if faces is None else list(faces) + + return detect, recognizer + + +def _embeddings(detect, recognizer, picture): + """[(area, embedding)] for each face in an RGB picture, largest first.""" + bgr = np.ascontiguousarray(np.asarray(picture.convert("RGB"))[..., ::-1]) + faces = [] + for row in detect(bgr): + area = float(row[2]) * float(row[3]) + faces.append((area, recognizer.feature(recognizer.alignCrop(bgr, row)))) + return sorted(faces, key=lambda face: -face[0]) + + +def _reference_picture(reference): + """The reference as a PIL image: an image, an array, or a stored path.""" + if reference is None: + raise ValueError(f"{COMMAND} needs 'reference' - a picture of the face to hold") + if isinstance(reference, Image.Image): + return reference + if isinstance(reference, np.ndarray): + return Image.fromarray(reference) + if isinstance(reference, str): + from ..locations import validate_media_path + + path = validate_media_path(reference, None, "the reference picture") + with Image.open(path) as opened: + return opened.convert("RGB") + raise ValueError( + f"{COMMAND}: 'reference' must be an image or the path of one, " + f"not {type(reference).__name__}" + ) + + +def _reference_embedding(detect, recognizer, reference, notes): + faces = _embeddings(detect, recognizer, _reference_picture(reference)) + if not faces: + raise ValueError( + f"{COMMAND}: no face found in 'reference' - give a picture with the " + "character's face in it" + ) + if len(faces) > 1: + notes.append( + f"reference holds {len(faces)} faces; the largest was used. Crop " + "the portrait to the one face to be sure of the match" + ) + return faces[0][1] + + +def _sample_indexes(start, count, total, samples): + """Up to `samples` evenly spaced frame numbers of a shot, inside the clip.""" + first = max(0, min(int(start), total)) + last = max(first, min(int(start) + int(count), total)) + span = last - first + if span <= 0: + return [] + if span <= samples: + return list(range(first, last)) + return sorted({first + (span * i) // samples for i in range(samples)}) + + +def _stored_path(video): + """The validated file a 'video' names, or None for an in-memory clip.""" + from ..locations import validate_media_path + from .video_utils import VideoFileReference + + if isinstance(video, VideoFileReference): + video = video.path + if isinstance(video, str): + return validate_media_path(video, None, "a video to measure") + return None + + +def _picture_at(video, index): + """One frame as a PIL image, seeking in a stored file rather than decoding it.""" + path = _stored_path(video) + if path is not None: + from ..media_frames import frames_at + + return frames_at(path, [f"frame:{index}"])[0]["image"] + from .video_utils import extract_frame + + return extract_frame(video, index) + + +def _best_match(recognizer, embedding, faces): + return max(float(recognizer.match(embedding, face, COSINE)) for _, face in faces) + + +def _shot_measurement(name, matches, sampled, faces_found): + return { + "shot": name, + "frames_sampled": sampled, + "faces_found": faces_found, + "similarity_median": round(float(np.median(matches)), PLACES) + if matches + else None, + "similarity_min": round(float(min(matches)), PLACES) if matches else None, + } + + +def _measure_shot(video, shot, media, models, embedding, samples): + detect, recognizer = models + count = shot.get("num_frames") + if count is None: + count = media.frame_count - int(shot.get("start_frame", 0)) + indexes = _sample_indexes( + shot.get("start_frame", 0), count, media.frame_count, samples + ) + matches = [] + for index in indexes: + faces = _embeddings(detect, recognizer, _picture_at(video, index)) + if faces: + matches.append(_best_match(recognizer, embedding, faces)) + return _shot_measurement(shot.get("name"), matches, len(indexes), len(matches)) + + +def _findings(measurements, threshold): + found = [] + for record in measurements: + at = {"shot": record["shot"]} + if record["faces_found"] == 0: + found.append(finding(RULES["no_face"], 0, at)) + elif record["similarity_median"] < threshold: + rule = {**RULES["face_drift"], "threshold": threshold} + found.append(finding(rule, record["similarity_median"], at)) + return sort_findings(found) + + +def _checked_settings(threshold, samples_per_shot): + """(threshold, samples) as numbers, or ValueError in the static check's + words: `face_drift_problems` is the one rule for both (#692).""" + threshold = real_number(threshold, "threshold", COMMAND, required=True) + samples = real_number(samples_per_shot, "samples_per_shot", COMMAND, required=True) + problems = face_drift_problems(threshold, samples) + if problems: + raise ValueError(problems[0][1]) + return float(threshold), int(samples) + + +def analyze_face_drift( + video, + reference, + shots=None, + threshold=DEFAULT_THRESHOLD, + samples_per_shot=DEFAULT_SAMPLES, + device=None, +): + """Task command: how close the faces in each shot are to a reference face. + + Args: + video: A stored video (asset:, output: or a path) or the video an + earlier step returned + reference: The face to hold - an image or the path of one. With + several faces the largest is used; with none the call is refused + shots: Shot records to measure, else the video's own, else the run + manifest beside the file, else the whole file is one shot + threshold: Median cosine similarity below which a shot is a + `face_drift` finding (-1 to 1); 0.363 is OpenCV's SFace threshold + samples_per_shot: Frames read from each shot, evenly spaced (1 to 64) + device: Where the models run + + Returns: + {measurements: [{shot, frames_sampled, faces_found, similarity_median, + similarity_min}], findings, rules_applied, shots_source, notes} + """ + threshold, samples = _checked_settings(threshold, samples_per_shot) + models = _models(device) + notes = [] + embedding = _reference_embedding(*models, reference, notes) + + media = media_from(video) + records, source = resolve_shots(video, media, shots) + if not records: + records = [{"name": "whole", "start_frame": 0, "num_frames": media.frame_count}] + notes.append( + "no shot boundaries found; the whole file was measured as one shot" + ) + measurements = [ + _measure_shot(video, shot, media, models, embedding, samples) + for shot in records + ] + return { + "measurements": measurements, + "findings": _findings(measurements, threshold), + "rules_applied": list(RULES), + "shots_source": source, + "notes": notes, + } diff --git a/dw/tasks/face_track.py b/dw/tasks/face_track.py index 3fba78096..51f0fba45 100644 --- a/dw/tasks/face_track.py +++ b/dw/tasks/face_track.py @@ -442,8 +442,10 @@ def required_weights(arguments): return file_entries(*detector_source(arguments)) -def _detector(repo, filename, min_confidence, device): - """A callable running YuNet on an RGB array, loaded once per device.""" +def face_detector_model(repo, filename, min_confidence, device): + """The cached cv2.FaceDetectorYN for a source, loaded once per device - + the one place YuNet is loaded, shared with `analyze_face_drift`, which + needs its full rows (landmarks included) rather than `_detector`'s boxes.""" import cv2 from .. import get_device_type @@ -473,10 +475,17 @@ def load(): path, "", (320, 320), min_confidence, 0.3, 5000, backend, target ) - model = cached_model( + return cached_model( ("crop_face_track", repo, filename, revision, min_confidence, use_cuda), load ) + +def _detector(repo, filename, min_confidence, device): + """A callable running YuNet on an RGB array, loaded once per device.""" + import cv2 + + model = face_detector_model(repo, filename, min_confidence, device) + def detect(rgb): height, width = rgb.shape[:2] model.setInputSize((width, height)) diff --git a/dw/tasks/image_utils.py b/dw/tasks/image_utils.py index 4049dc38e..bacea8bc0 100644 --- a/dw/tasks/image_utils.py +++ b/dw/tasks/image_utils.py @@ -892,7 +892,7 @@ def ingredients_grid( ) color = ingredients_background(background) - sources = _grid_load(images) + sources = load_rgb_images(images) if not sources: raise ValueError("ingredients_grid was given no images") if len(sources) > max_images: @@ -923,8 +923,9 @@ def ingredients_grid( return canvas -def _grid_load(images): - """The images as flat list of RGB PIL images, from whatever references resolved to""" +def load_rgb_images(images): + """The images as a flat list of RGB PIL images, from whatever references + resolved to - read by ingredients_grid and, through its handler, stills_to_video""" if images is None: return [] loaded = fetch_image(images) diff --git a/dw/tasks/video_handlers.py b/dw/tasks/video_handlers.py index dabfe64a6..70aa7313f 100644 --- a/dw/tasks/video_handlers.py +++ b/dw/tasks/video_handlers.py @@ -30,9 +30,11 @@ from ..task_problems import ( INGREDIENTS_FITS, INGREDIENTS_LAYOUTS, + face_drift_errors, face_track_errors, ingredients_grid_errors, paste_face_track_errors, + stills_to_video_errors, ) logger = logging.getLogger("dw") @@ -124,6 +126,45 @@ def _handle_loop_frames(task, arguments, previous_pipelines): return loop_frames(**arguments) +@register_command( + "stills_to_video", + implementation="dw.tasks.video_utils.stills_to_video", + domains={"fps": POSITIVE}, + static_check=stills_to_video_errors, +) +def _handle_stills_to_video(task, arguments, previous_pipelines): + """Hold each still for a duration, as one silent clip""" + logger.debug("Holding stills into a clip") + import numpy + + from .image_utils import load_rgb_images + from .video_utils import stills_to_video + + # Read the stills as ingredients_grid reads its images; arrays are + # already pixels and pass through + images = arguments.get("images") + if isinstance(images, (list, tuple)): + stills = [] + for item in images: + if isinstance(item, numpy.ndarray): + stills.append(item) + continue + loaded = load_rgb_images(item) + if not loaded: + raise ValueError( + "stills_to_video: an entry of 'images' resolved to no " + "image - check the reference" + ) + # An entry that resolves to several images (a step that made + # num_images_per_prompt > 1) contributes them in order, as + # ingredients_grid reads the same input + stills.extend(loaded) + arguments["images"] = stills + elif not isinstance(images, numpy.ndarray): + arguments["images"] = load_rgb_images(images) + return stills_to_video(**arguments) + + @register_command( "window_video", implementation="dw.tasks.windows.window_video", @@ -383,6 +424,42 @@ def _handle_crop_face_track(task, arguments, previous_pipelines): return crop_face_track(device=task.device_for(arguments), **arguments) +@register_command( + "analyze_face_drift", + implementation="dw.tasks.face_drift.analyze_face_drift", + returns="json", + consumes_device=True, + required_weights="dw.tasks.face_drift.required_weights", + summary=( + "Measure, per shot, how close the faces in a clip are to a reference " + "face (SFace cosine similarity), and name the shots to look at." + ), + parameter_descriptions={ + "video": "Video to measure - an asset:/output: reference, a path, or an earlier step's video.", + "reference": ( + "Picture of the face to hold, such as the cast portrait. The " + "largest face is used when it holds several; one with none is refused." + ), + "shots": "Shot records to measure; else the video's own, else the whole file.", + "threshold": ( + "Median cosine similarity below which a shot is a face_drift " + "finding, from -1 to 1; 0.363 is OpenCV's SFace threshold." + ), + "samples_per_shot": "Frames read from each shot, evenly spaced (1 to 64).", + }, + domains={"threshold": FINITE, "samples_per_shot": POSITIVE}, + whole_numbers=("samples_per_shot",), + media_arguments=("reference",), + static_check=face_drift_errors, +) +def _handle_analyze_face_drift(task, arguments, previous_pipelines): + """Measure how close each shot's faces are to a reference face""" + logger.debug("Analyzing face drift") + from .face_drift import analyze_face_drift + + return analyze_face_drift(device=task.device_for(arguments), **arguments) + + @register_command( "paste_face_track", implementation="dw.tasks.face_track.paste_face_track", diff --git a/dw/tasks/video_utils.py b/dw/tasks/video_utils.py index 47a4304b0..95713f4cc 100644 --- a/dw/tasks/video_utils.py +++ b/dw/tasks/video_utils.py @@ -31,7 +31,14 @@ FrameList, fit_codec_padding, ) -from ..task_domains import frame_size_error, whole_number +from ..shots import shot_record +from ..task_domains import frame_size_error, real_number, whole_number +from ..task_problems import ( # noqa: F401 + MAX_STILLS_FPS, + MAX_STILLS_SECONDS, + stills_to_video_problems, + MAX_STILLS_BYTES, +) logger = logging.getLogger("dw") @@ -231,6 +238,96 @@ def loop_frames(video, num_frames): return (looped.astype(numpy.float32) / 255.0).clip(0.0, 1.0) +def _stills_durations(durations): + """`durations` with a numpy array as a list, and an empty list refused""" + if isinstance(durations, numpy.ndarray): + durations = durations.tolist() + if isinstance(durations, (list, tuple)) and not durations: + raise ValueError( + "stills_to_video needs 'durations' as a number or a non-empty list" + ) + return durations + + +def _stills_seconds(durations, count): + if isinstance(durations, (list, tuple)): + return list(durations) + return [durations] * count + + +def stills_to_video(images, durations, fps=24): + """Task command: storyboard stills held on screen for a duration each, as + one silent clip - an animatic. + + The clip has no audio track (`audio=None`); `concat_videos` fills a + silent input with silence and `pair_audio` lays a score under it, so it + joins and scores like any generated shot. + + Args: + images: The stills, in order - PIL images or arrays. As a task + argument they may also be `{"media_type": "image", "location": + "asset:x.png"}` dicts or references, read the way + `ingredients_grid` reads its images. Every still must be the + same size + durations: Seconds each still is held - one number for all of them, or + one per still. Each still gets `max(1, round(seconds * fps))` + frames, so a duration shorter than a frame still shows + fps: Frame rate of the clip, up to 120. The clip is at most 600 + seconds long in total + + Returns: + An AudioVideo of uint8 (frames, height, width, 3) frames at `fps`, + with one shot record per still + """ + if isinstance(images, numpy.ndarray) and images.ndim == 4: + images = list(images) + elif isinstance(images, (numpy.ndarray, Image.Image)): + images = [images] + # Locations and dicts are read by the handler (argument_media imports this + # module, so it cannot import that back); what arrives here is in memory + stills = [ + _to_pil(image).convert("RGB") if not isinstance(image, Image.Image) else image + for image in images or [] + ] + if not stills: + raise ValueError("stills_to_video was given no 'images' to hold") + fps = real_number(fps, "fps", "stills_to_video", required=True) + durations = _stills_durations(durations) + problems = stills_to_video_problems(durations, fps, len(stills)) + if durations is None: + problems.append(("durations", "stills_to_video needs 'durations'")) + if problems: + raise ValueError(problems[0][1]) + seconds = _stills_seconds(durations, len(stills)) + + sizes = {still.size for still in stills} + if len(sizes) > 1: + first = stills[0].size + other = next(still.size for still in stills if still.size != first) + raise ValueError( + f"stills_to_video: 'images' must all be one size: got " + f"{first[0]}x{first[1]} and {other[0]}x{other[1]} - resize them " + "to match first" + ) + counts = [max(1, round(float(s) * fps)) for s in seconds] + width, height = stills[0].size + needed = sum(counts) * height * width * 3 + if needed > MAX_STILLS_BYTES: + raise ValueError( + f"stills_to_video would hold {sum(counts)} frames of " + f"{width}x{height} ({needed / 1024**3:.1f} GiB), over the " + f"{MAX_STILLS_BYTES / 1024**3:g} GiB limit - shorten 'durations', " + "lower 'fps', or use smaller images" + ) + frames = numpy.empty((sum(counts), height, width, 3), dtype=numpy.uint8) + shots, start = [], 0 + for index, (still, count) in enumerate(zip(stills, counts), start=1): + frames[start : start + count] = numpy.asarray(still.convert("RGB")) + shots.append(shot_record(f"still_{index}", start, count)) + start += count + return AudioVideo(frames, None, None, fps=fps, shots=shots) + + def frame_grid(video, count=12, columns=None, tile_width=320, label=True): """Task command: tile evenly sampled frames of a video into one contact sheet - a preview of a clip's shape without authoring a frames-extraction diff --git a/plugins/dw/.claude-plugin/plugin.json b/plugins/dw/.claude-plugin/plugin.json index 84a73f061..7a9c81f74 100644 --- a/plugins/dw/.claude-plugin/plugin.json +++ b/plugins/dw/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "dw", - "description": "Compose MiniMax H3 video, MiniMax Music 3, LTX-2.5 and Kandinsky 6.0 workflows over a dw MCP server, and cut a multi-episode series from them: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides.", - "version": "0.17.0", + "description": "Compose MiniMax H3 video, MiniMax Music 3, LTX-2.5, Kandinsky 6.0 and FLUX 3 Klein workflows over a dw MCP server, cut a multi-episode series from them, direct camera and look in each family's caption format, and storyboard a scene before it is filmed: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides.", + "version": "0.18.0-alpha.1", "author": { "name": "Don Kackman" }, @@ -14,6 +14,9 @@ "minimax-music3", "ltx-2.5", "kandinsky-6", + "flux3-klein", + "cinematography", + "storyboard", "mcp" ] } diff --git a/plugins/dw/README.md b/plugins/dw/README.md index 98b3f8534..f364e7c6c 100644 --- a/plugins/dw/README.md +++ b/plugins/dw/README.md @@ -20,6 +20,8 @@ Install once, from Claude Code: | `ltx-2.5` | LTX-2.5 video with a soundtrack: a single clip, first-frame and keyframe conditioning, the three-move two-stage flow, the IC-LoRA upscale, extend and chain; the distilled schedule and the frame and size rules. Prompts follow the trained-caption spec that ships inside diffusers. | | `kandinsky-6` | Kandinsky 6.0 video with a soundtrack: a five-second clip from text or from a picture on the 3B Lite-distill checkpoint; the distilled schedule, the size and frame rules the pipeline's own defaults get wrong, and why Apple Silicon is parked. Prompts follow Kandinsky Lab's own beautifier system prompts. | | `flux3-klein` | FLUX 3 Image Klein images: a picture from a short request through BFL's prompt upsampler, from a full caption, an edit of one photo, several references combined, or a layout placed by boxes; the 16-step distilled schedule with no guidance, the 16-pixel grid, the 10-reference training limit the pipeline does not enforce, and the gated license. Prompts come from BFL's upsampler builtins or their prompting guide. | +| `cinematography` | A director's vocabulary for camera and look - shot size, camera move, lens, light, era, tempo - as closed values, each compiled to the words it becomes in H3, LTX-2.5, Kandinsky 6 and FLUX 3 Klein captions; five named grades as `apply_lut` palettes plus `film_grain`. The words are hints the models may not follow, so the skill says how to check a move in the frames. | +| `storyboard` | Boards before video: a scene drawn on FLUX 3 Klein from character and location references (up to eight boards, cited as "image one"), the six-cut coverage set of one moment, a board as the first frame of an H3, LTX-2.5 or Kandinsky 6 shot, and a timed animatic with a scratch voice and music bed. Takes the Popcorn shape from Higgsfield (concept only) and the coverage set from Shot Board (MIT). | | `script-to-video` | From a prose script to finished video: decompose into shots, cast recurring characters once, pick each shot's template and family, validate cost before spending, run and diagnose, hand off to cut-and-score. The decision tree that turns a script into the shots the other skills know what to do with. | | `series-episodes` | A series rather than one generation: several episodes over one cast, each cut and scored from its own template runs. The five-beat recut-bed-match_levels-normalize-pair procedure, an optional per-shot look (one shared palette through `apply_lut`, then `film_grain`) before the recut, and the one step that must not be skipped or the cast drifts between episodes. | diff --git a/plugins/dw/skills/cinematography/SKILL.md b/plugins/dw/skills/cinematography/SKILL.md new file mode 100644 index 000000000..0e5442493 --- /dev/null +++ b/plugins/dw/skills/cinematography/SKILL.md @@ -0,0 +1,140 @@ +--- +name: cinematography +description: Use when a dw MCP server is connected and the user directs the camera or the look - a shot size, a camera move (dolly, crane, orbit, whip pan, crash zoom), a lens, lighting, a film era, a pace, or a named colour grade (noir, teal-orange, 70s stock, VHS, bleach bypass) - for an H3, LTX-2.5, Kandinsky 6 or FLUX 3 Klein generation. Gives a closed vocabulary and the one sentence each value becomes in that family's own caption format, and the grades as apply_lut palettes plus film_grain. +--- + +# Cinematography on a dw server + +A director's words for the camera and the look, as a closed set of values, +and what each value becomes in each family's caption. The families already +have a slot for this: LTX-2.5's trained caption must state a shot type, a +camera motion and a viewpoint; Kandinsky 6 wants exactly one camera +sentence; H3 writes camera moves inside its Context-IR shots; FLUX 3 Klein +frames a still. This skill fills that slot the same way every time, so a +shot list can say `move: dolly-in` and mean the same thing on every family. + +It does not pick the template, the size or the cost. The family's own skill +does that, and its rules win. + +## When to use it + +- The user names a camera or look in film words ("slow push-in", "low-key", + "shot on 16mm", "teal and orange"), or a shot list needs them per shot. +- A piece with several shots should read as one film: fix the values once + and compile each shot from them. +- The user wants a named grade on a finished clip or image. + +## The axes + +Six axes, each a closed set; a shot sets at most one value per axis. +Before choosing, read `references/vocabulary.md`, which defines every id. + +| Axis | Values | +| --- | --- | +| `shot_size` | `extreme-wide` `wide` `full` `medium` `medium-close` `close-up` `extreme-close-up` | +| `move` | `static` `dolly-in` `dolly-out` `truck-left` `truck-right` `crane-up` `crane-down` `orbit` `tracking` `whip-pan` `push-in` `handheld` `crash-zoom` | +| `lens` | `wide` `normal` `long` `anamorphic` `vintage` | +| `light` | `high-key` `low-key` `golden-hour` `practical` `noir` | +| `era` | `8mm` `16mm` `35mm` `vhs-dv` `digital` | +| `tempo` | `still` `measured` `brisk` `frantic` | + +Each value is a decision, not a slider: pick the one that fits, or leave the +axis unset and let the model choose. Map the user's words onto the ids +("arc shot" is `orbit`, "jib up" is `crane-up`, "slow zoom toward her" is +`push-in`) and say which you picked. Name the look by its effect, never by +a camera body or lens brand: a brand only works if the model learned it, +and nothing here has measured that. Genre is not an axis; keep the user's +genre words in the prompt and state the values it implies. + +## Compile per family + +Each family's table turns every `shot_size` and `move` into words in that +family's caption format, and says where `lens`, `light`, `era` and `tempo` +go. Read the one for the family the shot runs on, alongside that family's +skill: + +| Family skill | Table | +| --- | --- | +| `minimax-h3` | `references/compile-h3.md` - Context-IR, move type with amplitude and speed | +| `ltx-2.5` | `references/compile-ltx-2.5.md` - shot type, camera motion and viewpoint in one sentence; name what comes into view | +| `kandinsky-6` | `references/compile-kandinsky-6.md` - shot size up front, exactly one camera sentence | +| `flux3-klein` | `references/compile-flux3-klein.md` - a still: a move becomes framing | + +The compiled words go into the prompt the family's own rules build; they do +not replace it. The stored prompts (`list_prompts`, then `get_prompt`) show +each family's format with a camera already in it - `ltx2/`, `kandinsky6/`, +`minimax/fox_dawn_context_ir` - so read one before writing. + +## A hint, not a promise + +Every value here is a hint, not a promise. The models were trained on +captions that describe camera work; none was trained on this list, and dw +has not measured which named moves each family honours (one 241-frame LTX +image-to-video clip told "the camera remains static" pushed or arced from +about five seconds, while six clips in the L2 cells showed no clear move; see +`docs/proposals/audits/2026-11-story-forge-lessons.md`). So: + +1. Describe the move in the caption in the family's own words (the table + does that) rather than naming the id alone. +2. After the run, check it: `get_output_frames(count=12)` and compare the + first and last frames. A dolly-in should end tighter than it began, a + crane-up higher, an orbit at a different angle, a static shot unchanged. +3. Tell the user what the frames show. A move that did not land is + reported as missed, not as done. A reseed, a stronger wording ("with + large amplitude"), or on H3 a video reference that carries the camera + (`composable-references`) are the next tries. + +## Grades + +A grade is colour and grain applied after generation, on the CPU: an +`apply_lut` `palette` (2-16 `#rrggbb` colours, dark to light) and then +`film_grain` (`amount`, `size`, `chroma`). Five are named - `noir`, +`teal-orange`, `70s-stock`, `vhs`, `bleach-bypass` - with their palettes, +strengths, grain settings and a two-step workflow in `references/grades.md`. + +Pick a grade by what the user named, else by the `light` and `era` already +chosen (the table there pairs them). `era` is said in the prompt and a grade +is applied afterwards; they agree but are separate. A palette keeps each +pixel's brightness, so a grade cannot add contrast - that comes from +`light`. For a series, one grade on every shot: the `series-episodes` skill's +look recipe takes the palette from here. + +## Run and judge + +1. Generate with the family's skill and its template (for example + `templates/ltx2/text-to-video`, `templates/minimax/video-with-audio`, + `templates/kandinsky6/text-to-video`, `templates/flux3/text-to-image`), + with the compiled words in its prompt. Its validate, cost quote and + `acknowledged_cost` rules apply unchanged. +2. Check the camera with `get_output_frames` as above; check a still with + `get_output_image` against the framing asked for. +3. For a grade, `validate_workflow` the grade workflow (free; it refuses a + bad palette entry by name), then run it on the `output:` reference of the + generated file. It needs no GPU and no downloads. +4. A saving step carries a `subfolder`: the graded file is `final`, the + ungraded step before it `intermediate`, so `list_gallery(subfolder="final")` + lists only deliverables. Keep both in anything you compose, then give the + user the gallery `url`. + +## Not in scope + +- Any family not in the table above. The vocabulary covers H3, LTX-2.5 and + Kandinsky 6 video plus FLUX 3 Klein stills only; for any other family, + image or audio, stop and do not borrow a table. +- Audio-only families: `minimax-music3` has no camera, and nothing here + applies to it. A soundtrack's mood is the music skill's. +- Choosing a template or a family, sizes, frame counts and cost: the family + skill's. +- Recovering the camera from a finished clip automatically, or tuning a + model to follow moves: not built. Judge by the frames. + +## Sources + +The axes follow Higgsfield's user-facing preset words, without its +camera-body presets. Ideas, not code, from: Aither (MIT) - enum-sized axes +and "a decision, not a slider"; CreatorOS (Apache-2.0) - a preset list and +pulling a palette from a photo; OpenField (MIT) - director axes and an era +control. The LTX compile rule is from the LTX-2 camera-control LoRA cards +on Hugging Face; H3's from MiniMax's base prompt-writing guide; Kandinsky's +from Kandinsky Lab's beautifier prompts. Audit: +`docs/proposals/audits/2026-11-higgsfield/README.md`. diff --git a/plugins/dw/skills/cinematography/references/compile-flux3-klein.md b/plugins/dw/skills/cinematography/references/compile-flux3-klein.md new file mode 100644 index 000000000..1529bae63 --- /dev/null +++ b/plugins/dw/skills/cinematography/references/compile-flux3-klein.md @@ -0,0 +1,67 @@ +# Compiling the vocabulary for FLUX 3 Klein + +Part of the `cinematography` skill, for the `flux3-klein` skill's captions. +Read that skill's *Prompts* section first. + +## The rule this follows + +Klein draws a still. A still has no camera move, so a `move` compiles to +framing and composition instead: the frame the move would land on, plus +the visual cue a photograph of that move would show (motion blur for a +fast one). Say so to the user when they ask for a move on Klein: the +picture implies the move; it does not perform it. + +Two uses make this worth doing. A storyboard frame for a video shot: draw +the frame the shot opens on (or, for `templates/ltx2/keyframes` or +`templates/minimax/first-and-last-frame`, both ends of the move as two +stills). And a still that should read as cinematic. + +Klein was trained on long captions from BFL's upsampler. Through +`templates/flux3/upsample-then-generate`, put the values in the short +request in plain words and the upsampler writes them in; in a caption +written by hand (`templates/flux3/text-to-image`), they go in the prose +like any other detail. + +## `shot_size` + +| id | Framing in the caption | +| --- | --- | +| `extreme-wide` | "an extreme wide view of , small in the frame" | +| `wide` | "a wide view of within " | +| `full` | "a full-length view of , head to toe" | +| `medium` | "a waist-up view of " | +| `medium-close` | "a chest-up portrait of " | +| `close-up` | "a close-up of , filling the frame" | +| `extreme-close-up` | "an extreme close-up of " | + +## `move` + +| id | The still it becomes | +| --- | --- | +| `static` | "a steady, balanced composition, everything sharp" | +| `dolly-in` | the closer, end frame: one shot size tighter than asked, the subject centred | +| `dolly-out` | the wider, end frame: one shot size looser, more of the setting around the subject | +| `truck-left` | the subject off-centre to the right, open space on the left where the camera is heading | +| `truck-right` | the subject off-centre to the left, open space on the right | +| `crane-up` | "a high-angle view looking down on " | +| `crane-down` | "an eye-level view of " | +| `orbit` | "a three-quarter view of ", the angle partway round | +| `tracking` | " in motion, sharp, the background streaked with motion blur" | +| `whip-pan` | "a horizontal motion blur smearing the frame, just resolving" | +| `push-in` | a slightly tighter framing on , the background softly out of focus | +| `handheld` | "a slightly tilted, candid framing" | +| `crash-zoom` | "radial zoom blur rushing toward " | + +## The other axes + +In the caption, as visual detail: + +- **lens**: "wide-angle perspective with stretched edges", "telephoto + compression with creamy background blur", "anamorphic widescreen with + oval bokeh and horizontal flares", "soft vintage-lens glow". Name the + effect, not a lens product. +- **light**: the `vocabulary.md` meaning in observable words. +- **era**: "a grainy 8mm film frame", "a 16mm film still", "a 35mm film + still", "a VHS video frame", "a clean digital photograph". +- **tempo**: only through motion blur - `still` and `measured` none, + `brisk` a little on moving parts, `frantic` heavy. diff --git a/plugins/dw/skills/cinematography/references/compile-h3.md b/plugins/dw/skills/cinematography/references/compile-h3.md new file mode 100644 index 000000000..3668f5cb7 --- /dev/null +++ b/plugins/dw/skills/cinematography/references/compile-h3.md @@ -0,0 +1,77 @@ +# Compiling the vocabulary for MiniMax H3 + +Part of the `cinematography` skill, for the `minimax-h3` skill's Context-IR +prompts. Read that skill's *Prompts* section first. If MiniMax's +`h3-prompt-writing` skill is installed, hand it the chosen values in words +and let it write; this table is for writing Context-IR by hand. + +## The rule this follows + +From MiniMax's base prompt-writing guide (section 4.3, read 2026-11-01): +camera language lives inside `integrated_multimodal_description`, in the +shot it belongs to, written as a natural action ("The camera pushes in with +small amplitude at slow speed toward the folded letter in her hands"), never +stacked as labels at the end of a sentence. A complete move has three +parts: the motion type, an amplitude ("with small amplitude", "with large +amplitude") and a speed ("at slow speed", "at fast speed"); add amplitude +and speed only when they matter. MiniMax's motion types: Push In / Pull +Out, Zoom In / Zoom Out, Pan, Tilt, Truck Left / Right, Pedestal Up / Down, +Arc Shot, Tracking Shot, Static Shot, Shake Slightly / Strongly, POV, Roll. + +The opening of `[Shot 1]` states the overall style and the initial +composition, which is where `shot_size`, `lens`, `light` and `era` go. A cut +is "the camera cuts to", and the guide prefers a camera move when only the +distance changes. The stored `minimax/fox_dawn_context_ir` prompt +(`get_prompt`) shows both: "a medium tracking shot moves alongside a red +fox" and "tilts up with small amplitude at slow speed". + +## `shot_size` + +The guide sets no shot-size words; its own examples use the plain ones. + +| id | Composition at the opening of the shot | +| --- | --- | +| `extreme-wide` | "an extreme wide shot frames , small within it" | +| `wide` | "a wide shot frames in " | +| `full` | "a full shot frames head to toe" | +| `medium` | "a medium shot frames from the waist up" | +| `medium-close` | "a medium close-up frames from the chest up" | +| `close-up` | "a close-up frames " | +| `extreme-close-up` | "an extreme close-up frames " | + +## `move` + +| id | The move, in MiniMax's motion types | +| --- | --- | +| `static` | "The camera holds a static shot as ." | +| `dolly-in` | "The camera pushes in with large amplitude toward ." | +| `dolly-out` | "The camera pulls out with large amplitude, revealing ." | +| `truck-left` | "The camera trucks left alongside ." | +| `truck-right` | "The camera trucks right alongside ." | +| `crane-up` | "The camera pedestals up with large amplitude, rising above ." | +| `crane-down` | "The camera pedestals down to 's eye level." | +| `orbit` | "The camera moves in an arc shot around ." | +| `tracking` | "The camera holds a tracking shot, matching 's pace." | +| `whip-pan` | "The camera pans with large amplitude at fast speed, revealing ." | +| `push-in` | "The camera pushes in with small amplitude at slow speed toward ." | +| `handheld` | "The camera shakes slightly as it follows ." | +| `crash-zoom` | "The camera zooms in with large amplitude at fast speed on ." | + +## The other axes + +In the opening of the shot, with the style: + +- **lens**: the effect - "wide-angle", "long-lens compression with a soft + background", "anamorphic widescreen, horizontal flares", "soft vintage + glass". +- **light**: the `vocabulary.md` meaning, in the visual description. +- **era**: "shot on grainy 8mm home-movie film", "16mm documentary film", + "35mm film", "VHS camcorder footage", "clean digital video". For a medium + (clay, paper) the `minimax-h3` skill's `style.md` reference applies too. +- **tempo**: the speed part of the move - `still` and `measured` "at slow + speed", `brisk` none (normal speed is omitted), `frantic` "at fast speed" + plus "shakes strongly" where it fits. In a piece with cuts, tempo is also + how long each `[Shot n]` holds. + +A video reference (`composable-references`) carries camera from a real clip +and is stronger than words: when the move must land, that is the route. diff --git a/plugins/dw/skills/cinematography/references/compile-kandinsky-6.md b/plugins/dw/skills/cinematography/references/compile-kandinsky-6.md new file mode 100644 index 000000000..dfd80ede7 --- /dev/null +++ b/plugins/dw/skills/cinematography/references/compile-kandinsky-6.md @@ -0,0 +1,70 @@ +# Compiling the vocabulary for Kandinsky 6.0 + +Part of the `cinematography` skill, for the `kandinsky-6` skill's prompts. +Read that skill's *Prompts* section first; this table only fills the slots +its format already has. + +## The rule this follows + +Kandinsky Lab's beautifier format is one paragraph: the shot (what is in +the frame and the shot size), then exactly one camera sentence, then +appearance, the five seconds of action in order, and light and style last. +The camera sentence starts with the camera and is never left out: with no +move asked for it is "The camera is static." So: + +- `shot_size` goes into the opening shot clause. +- `move` is the one camera sentence. Never write a second sentence about the + camera, and never fold the move into the action sentences. +- `lens`, `light` and `era` go at the end, with light and style. + +A clip is five seconds: one move, once. `` is the shot's own noun. + +## `shot_size` + +| id | Opening shot clause | +| --- | --- | +| `extreme-wide` | "An extreme wide shot of , with small in it." | +| `wide` | "A wide shot of in ." | +| `full` | "A full shot of , head to toe." | +| `medium` | "A medium shot of from the waist up." | +| `medium-close` | "A medium close-up of from the chest up." | +| `close-up` | "A close-up of ." | +| `extreme-close-up` | "An extreme close-up of ." | + +## `move` + +| id | The one camera sentence | +| --- | --- | +| `static` | "The camera is static." | +| `dolly-in` | "The camera slowly dollies in toward ." | +| `dolly-out` | "The camera slowly dollies back, widening to show ." | +| `truck-left` | "The camera trucks left alongside ." | +| `truck-right` | "The camera trucks right alongside ." | +| `crane-up` | "The camera cranes up and looks down over ." | +| `crane-down` | "The camera cranes down to 's eye level." | +| `orbit` | "The camera circles slowly around ." | +| `tracking` | "The camera tracks alongside , keeping pace." | +| `whip-pan` | "The camera whip-pans quickly to ." | +| `push-in` | "The camera pushes in slightly toward ." | +| `handheld` | "The camera is handheld and sways gently with the action." | +| `crash-zoom` | "The camera crash-zooms in on ." | + +Example opening: "A medium close-up of an old fisherman on a wet pier, +from the chest up. The camera slowly dollies in toward his face. ..." + +## The other axes + +At the end of the paragraph, with light and style: + +- **lens**: the effect in plain words - "wide-angle perspective", "long-lens + compression with a soft background", "anamorphic widescreen with + horizontal flares", "soft vintage glass". +- **light**: "bright, even high-key light", "low-key light from a single + lamp", "warm golden-hour sun", "lit only by the shop's neon signs", "film + noir lighting, hard shadows". +- **era**: "grainy 8mm home-movie look", "16mm documentary film", "35mm + film look", "VHS camcorder look", "clean digital video". The format says + to state a requested style in prose, and these are styles. +- **tempo**: an adverb on the camera sentence - "slowly", "steadily", + "quickly" - and the pace of the action sentences. `still` pairs with + `static`. diff --git a/plugins/dw/skills/cinematography/references/compile-ltx-2.5.md b/plugins/dw/skills/cinematography/references/compile-ltx-2.5.md new file mode 100644 index 000000000..1953976f3 --- /dev/null +++ b/plugins/dw/skills/cinematography/references/compile-ltx-2.5.md @@ -0,0 +1,82 @@ +# Compiling the vocabulary for LTX-2.5 + +Part of the `cinematography` skill, for the `ltx-2.5` skill's captions. Read +that skill's *Prompts* section and its `caption-spec.md` reference first; +this table only fills the slot the spec already has. + +## The rule this follows + +LTX-2.5's trained caption spec requires, for every shot and woven into the +prose, never as tags: + +- a shot type, exactly one of extreme wide shot / wide shot / medium shot / + medium close-up / close-up / extreme close-up; +- a camera motion, always stated: if none, say the camera remains static; +- a camera viewpoint relative to the subject: front-facing, back-facing, + side view, over-the-shoulder, top-down, low-angle or high-angle. + +So one camera sentence carries all three. The vocabulary has no viewpoint +axis: pick the one the request implies, else front-facing, and always write +it. Add one compile rule from the LTX-2 camera-control LoRA cards, which +advise naming what should come into view during the move: for every move +but `static`, say what comes into view as the camera goes (a reveal gives +the model somewhere to move to). The caption is present progressive, one +paragraph, observable words only. + +`` and `` are the shot's own nouns; `` is one of +the spec's seven (for example "a low angle" or "a front-facing angle"). + +## `shot_size` + +| id | Shot type to write | +| --- | --- | +| `extreme-wide` | "an extreme wide shot takes in , small within it" | +| `wide` | "a wide shot frames and " | +| `full` | "a wide shot frames head to toe" (the spec has no full shot; this is the nearest of its six) | +| `medium` | "a medium shot frames from the waist up" | +| `medium-close` | "a medium close-up frames from the chest up" | +| `close-up` | "a close-up fills the frame with " | +| `extreme-close-up` | "an extreme close-up isolates " | + +## `move` + +| id | Camera motion to write (every row carries the viewpoint, as the spec requires) | +| --- | --- | +| `static` | "captured from , the camera remains static" | +| `dolly-in` | "seen from , as the camera dollies in toward , comes into view" | +| `dolly-out` | "seen from , as the camera dollies back from , comes into view around them" | +| `truck-left` | "seen from , as the camera trucks left alongside , slides into view from the left" | +| `truck-right` | "seen from , as the camera trucks right alongside , slides into view from the right" | +| `crane-up` | "seen from , as the camera cranes upward above , comes into view below" | +| `crane-down` | "seen from , as the camera cranes down to 's level, comes into view behind them" | +| `orbit` | "seen from , as the camera arcs around , comes into view behind them" | +| `tracking` | "seen from , the camera tracks alongside at their pace, passing through the frame" | +| `whip-pan` | "seen from , the camera whip-pans in a blur, landing on " | +| `push-in` | "seen from , the camera pushes in slowly toward , until fills the frame" | +| `handheld` | "seen from , the handheld camera sways slightly as it follows , drifting into view" | +| `crash-zoom` | "seen from , the camera crash-zooms in on , until fills the frame" | + +Example: "A medium shot frames a woman in a grey wool coat from the waist up, +captured from a low angle as the camera dollies in toward her, the lit +window behind her coming into view over her shoulder." + +## The other axes + +These go into the visual detail, where the spec asks for lighting, colour +and texture, and in the spec's aesthetic-quality descriptors: + +- **lens**: say what it does to the picture - "a wide-angle view that + stretches the edges", "compressed depth and a shallow focus", "a + widescreen frame with oval bokeh and horizontal flares", "soft, glowing + vintage glass". Say the effect, not a lens name. +- **light**: the `vocabulary.md` meaning in observable words ("a single hard + lamp leaves most of the room in deep shadow"). +- **era**: "grainy 8mm home-movie footage", "16mm documentary film grain", + "35mm film with fine grain", "soft VHS camcorder footage", "clean digital + video". +- **tempo**: speed words on the move - `still` "barely moving", `measured` + "slowly", `brisk` "steadily", `frantic` "quickly and jerkily". The clip + length paces the shot; tempo does not shorten it. + +Multi-shot captions repeat the triple at each cut, as the `ltx-2.5` skill's +*Several shots* paragraph says. diff --git a/plugins/dw/skills/cinematography/references/grades.md b/plugins/dw/skills/cinematography/references/grades.md new file mode 100644 index 000000000..46ce16e7d --- /dev/null +++ b/plugins/dw/skills/cinematography/references/grades.md @@ -0,0 +1,103 @@ +# Named grades + +Part of the `cinematography` skill; its *Grades* section says when to read +this. A grade is applied after generation, on the CPU, with no model: an +`apply_lut` `palette` followed by `film_grain`. It changes colour and +texture, never framing or motion. + +How a palette works (the `apply_lut` task): 2 to 16 `#rrggbb` colours, +ordered dark to light. Each pixel keeps its own brightness and takes its hue +from the palette at that brightness, so shadows lean toward the first +colour and highlights toward the last. `strength` (0..1) blends the result +with the original. `film_grain` takes `amount` (0..1; keep video at or +under about 0.12), `size` (pixels, 1 or above) and `chroma` (0..1, where 0 +is brightness-only grain and 1 is colour noise). Its seed is the run's, so +a rerun reproduces the grain. + +Because a palette keeps brightness, no grade here adds contrast. Contrast +comes from the `light` axis in the prompt (`low-key`, `noir`). + +## The grades + +```json +{ + "noir": { + "palette": ["#000000", "#1c1c1c", "#4a4a4a", "#8c8c8c", "#d4d4d4", "#ffffff"], + "strength": 1.0, + "film_grain": {"amount": 0.08, "size": 1.5, "chroma": 0.0} + }, + "teal-orange": { + "palette": ["#06222b", "#16505a", "#6f8a86", "#d49a5e", "#ffdcae"], + "strength": 0.5, + "film_grain": {"amount": 0.03, "size": 1.0, "chroma": 0.0} + }, + "70s-stock": { + "palette": ["#22261a", "#55502f", "#9a7b4b", "#d6ae74", "#f4e3c0"], + "strength": 0.6, + "film_grain": {"amount": 0.08, "size": 2.0, "chroma": 0.2} + }, + "vhs": { + "palette": ["#140f2a", "#3b3263", "#7d789c", "#c9bfc6", "#f5efdf"], + "strength": 0.5, + "film_grain": {"amount": 0.12, "size": 2.5, "chroma": 0.6} + }, + "bleach-bypass": { + "palette": ["#0a0c0d", "#33383a", "#7a7f80", "#c3c6c4", "#f4f4ef"], + "strength": 0.7, + "film_grain": {"amount": 0.06, "size": 1.2, "chroma": 0.0} + } +} +``` + +| Grade | Looks like | Pairs with | +| --- | --- | --- | +| `noir` | Black and white, fine grain | `light` `noir` or `low-key` | +| `teal-orange` | Teal shadows, warm skin and highlights, the blockbuster look | `light` `golden-hour`, `era` `35mm` or `digital` | +| `70s-stock` | Warm, faded, olive shadows, coarse grain | `era` `16mm` or `35mm`, `lens` `vintage` | +| `vhs` | Washed-out, violet shadows, coarse colour noise | `era` `vhs-dv` | +| `bleach-bypass` | Nearly colourless, cool, gritty | `light` `low-key` | + +The palettes are starting points chosen by eye, not matched to any film +stock. Lower `strength` for a lighter touch. To pull a palette from a +reference picture instead (CreatorOS's idea): pick 3-6 colours from its +shadows to its highlights, dark to light. + +## Applying one + +One clip, one grade. `clip` is an `asset:` or `output:` reference; `look` +and the two grain values come from the grade's entry above. For a whole +episode with one look across shots, use the `series-episodes` skill's +`series-episodes/references/look.md` recipe instead, with the palette from +here. + +```json +{ + "id": "graded-clip", + "variables": { + "clip": "asset:shot.mp4", + "look": ["#06222b", "#16505a", "#6f8a86", "#d49a5e", "#ffdcae"] + }, + "steps": [ + { + "name": "graded", + "task": { + "command": "apply_lut", + "arguments": {"media": "variable:clip", "palette": "variable:look", "strength": 0.5} + }, + "result": {"content_type": "video/mp4", "subfolder": "intermediate"} + }, + { + "name": "grain", + "task": { + "command": "film_grain", + "arguments": {"media": "previous_result:graded", "amount": 0.03, "size": 1.0, "chroma": 0.0} + }, + "result": {"content_type": "video/mp4", "subfolder": "final"} + } + ] +} +``` + +`validate_workflow` it first: free, and it refuses a bad palette entry by +name. Both tasks keep the clip's size, frame rate, frame count and audio. +An image works the same way, with `image/png` as the `content_type`. diff --git a/plugins/dw/skills/cinematography/references/vocabulary.md b/plugins/dw/skills/cinematography/references/vocabulary.md new file mode 100644 index 000000000..aab38bd1e --- /dev/null +++ b/plugins/dw/skills/cinematography/references/vocabulary.md @@ -0,0 +1,100 @@ +# The cinematography vocabulary + +Part of the `cinematography` skill; its *The axes* section says when to read +this. Six axes, each a closed set of ids. A shot picks at most one value per +axis and leaves the rest unset; an unset axis is the model's own choice, not +a default. The ids are what a shot list records and what later skills cite, +so they are spelled exactly as here. The meaning column is the plain-language +idea; the per-family `compile-*.md` tables turn it into the family's own +caption words. + +## `shot_size` + +How much of the subject the frame holds. + +| id | Means | +| --- | --- | +| `extreme-wide` | The place dominates; figures are small in it. An establishing shot. | +| `wide` | The whole setting around the subject, the subject clearly readable. | +| `full` | One figure head to toe, a little room above and below. | +| `medium` | Waist up. | +| `medium-close` | Chest up. The usual dialogue framing. | +| `close-up` | The face, or one object, fills most of the frame. | +| `extreme-close-up` | A detail: the eyes, a hand, a keyhole. | + +## `move` + +What the camera does during the shot. One move per shot. + +| id | Means | +| --- | --- | +| `static` | The camera does not move. | +| `dolly-in` | The camera travels toward the subject, a full move. | +| `dolly-out` | The camera travels away, revealing more of the setting. | +| `truck-left` | The camera slides sideways to the left, parallel to the subject. | +| `truck-right` | The camera slides sideways to the right. | +| `crane-up` | The camera rises, often to look down over the scene. | +| `crane-down` | The camera descends, often settling to the subject's level. | +| `orbit` | The camera arcs around the subject, keeping it centred. | +| `tracking` | The camera moves with a moving subject, keeping pace. | +| `whip-pan` | A very fast turn sideways that blurs to a new view. | +| `push-in` | A slow, small move toward the subject, for emphasis. | +| `handheld` | Unsteady, operator-held movement. | +| `crash-zoom` | A very fast zoom in on one thing. | + +`dolly-in` and `push-in` differ by size and speed: a dolly travels, a push-in +creeps. `crash-zoom` is a zoom, not a travel: the view narrows while the +camera stays put. + +## `lens` + +The look of the glass, not a product. + +| id | Means | +| --- | --- | +| `wide` | Wide-angle: deep space, stretched edges, close objects loom. | +| `normal` | Natural perspective, roughly what the eye sees. | +| `long` | Telephoto: compressed depth, a shallow focus that isolates the subject. | +| `anamorphic` | Widescreen, oval bokeh, horizontal streak flares. | +| `vintage` | Soft edges, lower contrast, warm glow and gentle flare. | + +## `light` + +| id | Means | +| --- | --- | +| `high-key` | Bright, even, few shadows. | +| `low-key` | Mostly dark, one hard source, deep shadows. | +| `golden-hour` | Low, warm sun, long shadows, glowing rim light. | +| `practical` | Light from sources inside the scene: lamps, screens, candles, neon. | +| `noir` | Hard contrast, slatted or venetian-blind shadows, smoke, night. | + +## `era` + +The capture medium the picture should feel shot on. Era is said in the +prompt; a grade (`references/grades.md` in this skill) adds colour and grain +afterwards, and the two pair well but are separate choices. + +| id | Means | +| --- | --- | +| `8mm` | Home-movie film: heavy grain, soft focus, light flicker, faded colour. | +| `16mm` | Documentary film: visible grain, natural colour, slightly soft. | +| `35mm` | Feature film: fine grain, rich colour, shallow depth of field. | +| `vhs-dv` | Consumer tape or early digital video: smeared colour, soft detail, video sharpening. | +| `digital` | Clean modern digital video: sharp, no grain. | + +## `tempo` + +How fast the shot feels. A tempo sets how fast the move goes and, in a piece +with cuts, how long each shot holds. + +| id | Means | +| --- | --- | +| `still` | Almost nothing moves; long holds. | +| `measured` | Slow, deliberate movement. | +| `brisk` | Normal-to-quick movement; shorter holds. | +| `frantic` | Fast, jittery movement; quick cuts. | + +Genre is not an axis: it is the subject's business and stays in the user's +own words. A genre usually implies a few values (a thriller leans +`low-key`, `handheld`, `brisk`); state those as values rather than +naming the genre and hoping. diff --git a/plugins/dw/skills/flux3-klein/SKILL.md b/plugins/dw/skills/flux3-klein/SKILL.md index 73bd71def..736988b6e 100644 --- a/plugins/dw/skills/flux3-klein/SKILL.md +++ b/plugins/dw/skills/flux3-klein/SKILL.md @@ -101,6 +101,9 @@ the caption Klein expects, not as advice for a short prompt. The stored captions, `man_dog` and `balloons_layout`: read one before writing your own. The upsampler writes English only. +A shot size, camera move or look in film words: the `cinematography` +skill's Klein table. A still has no move; it becomes framing. + ## Run and judge 1. `validate_workflow` first - free, and catches an off-grid size. diff --git a/plugins/dw/skills/kandinsky-6/SKILL.md b/plugins/dw/skills/kandinsky-6/SKILL.md index 14ba26896..4959055a6 100644 --- a/plugins/dw/skills/kandinsky-6/SKILL.md +++ b/plugins/dw/skills/kandinsky-6/SKILL.md @@ -88,6 +88,10 @@ user's language stand; the model takes it. The pipeline can also expand a short prompt with its own text encoder (`expand_prompts` true, a shorter version of the same instructions). No template uses it yet. +A shot size, camera move or look in film words (a dolly-in, low-key light, +16mm): the `cinematography` skill's Kandinsky table writes the one camera +sentence. + ## Run and judge 1. `validate_workflow` first - free, and catches bad arguments. diff --git a/plugins/dw/skills/ltx-2.5/SKILL.md b/plugins/dw/skills/ltx-2.5/SKILL.md index b0d7d8e3f..2ec730197 100644 --- a/plugins/dw/skills/ltx-2.5/SKILL.md +++ b/plugins/dw/skills/ltx-2.5/SKILL.md @@ -130,7 +130,8 @@ the caption the motion: open by matching the image faithfully, never contradict it, one continuous take, no cuts. The `ltx2/` stored prompts (`list_prompts`) follow it. The duration predictor times the clip as written: put beats in the prompt ("she pauses") or set a duration. On-screen text and -chaotic physics are unreliable. +chaotic physics are unreliable. Camera or look in film words: the +`cinematography` skill. **Voice**: the caption steers a speaker's pitch and apparent age and gender, not their delivery. Measured 2026-11-01 on mps (one image, seed and line, four voice diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index eae9f050b..440602360 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -10,7 +10,7 @@ H3 generates video and audio together. Every template fits a 24 GB card. ## Before anything 1. `get_server_info`: the device (H3 is CUDA; quantized configs don't - run on mps) and the session's workspace. + run on mps) and the workspace. 2. `list_workflows(shape="shot")`, `list_workflows(shape="sequence")` and `list_workflows(shape="audio")`: the family's templates by name, `summary`, `traits`, `cost`; it outranks names below. @@ -82,7 +82,7 @@ H3 generates video and audio together. Every template fits a 24 GB card. `pair_audio`, in that reference's last section. - **Unrelated shots, no cut**: `templates/minimax/shots-batch` - one H3 step per `shots` entry. `keep_output` each clip, then `templates/assemble-and-score` cuts and scores. -- **Music alone**: `templates/minimax/music` (Music3); the `minimax-music3` +- **Music alone**: `templates/minimax/music`; the `minimax-music3` skill - its `audio_duration` is a ceiling, not a length. - **Replace a shot's own sound**: `templates/video-to-sound-effects` (MMAudio, 7-25 dB hot: `match_levels` first), then `pair_audio`. @@ -152,9 +152,9 @@ the rules are MiniMax's: framed as `Task: T2VA. Duration: 5.17 seconds. Idea: ...`. Either way: write the whole script before the first shot. Repeat a speaker's voice description verbatim across shots. -Several characters: one cropped reference each, never a group picture, which imposes its framing on every shot (`references/per-character.md`). -Before writing lines, read `references/dialogue.md`: how long a line fits, and what not to name. -A wanted look (clay, paper): `references/style.md`. +Several characters: one cropped reference each, never a group picture (`references/per-character.md`). +Lines: `references/dialogue.md` (length, what not to name). Check a draft: `references/prompt-lint.md`. +A wanted look (clay, paper): `references/style.md`. Film-term camera or look: the `cinematography` skill. ## Run and judge diff --git a/plugins/dw/skills/minimax-h3/references/prompt-lint.md b/plugins/dw/skills/minimax-h3/references/prompt-lint.md new file mode 100644 index 000000000..ef8b092ef --- /dev/null +++ b/plugins/dw/skills/minimax-h3/references/prompt-lint.md @@ -0,0 +1,31 @@ +# H3 prompt lint + +Part of the `minimax-h3` skill; `script-to-video` sends a compiled shot +here. A checklist an agent runs by eye over a drafted prompt before +spending a render; nothing in the engine enforces it. The idea of linting a +shot document comes from Picky (GPL-3.0): idea credit only, no text or code +copied. + +Flag, then fix in the shot document and recompile; do not patch only the +prompt: + +1. **A missing `overall_soundscape`.** Every H3 prompt carries one, even a + shot with no dialogue: silence is a stated choice, not an absence. +2. **An overfull beat.** More than one visible state in a shot's span (a + person enters, sits, speaks and leaves). Split it into shots or cut to + the one state the `beat` names. +3. **A cut with no new information.** A `cut` whose next shot repeats the + framing, place and state of the one before. Merge the two or change the + `shot_size`, `move` or `location`. +4. **A speaker without a voice description,** or one described differently + from the last shot. Repeat it verbatim. +5. **A `must_not` the prompt contradicts,** such as text in frame asked for + while `must_not` rules it out. +6. **A `duration` off the grid.** H3 runs on its `17*n+5` frame grid; the + prompt's pacing should fit the snapped length, not the script's. +7. **A camera word outside the closed set.** Take `shot_size`, `move`, + `lens` and `light` from `cinematography/references/vocabulary.md`, in the + caption words its H3 table gives. + +A shot that passes is not thereby good: look at the frames +(`get_output_frames`) as usual. diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index c06caa367..80545899b 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -45,6 +45,13 @@ Fill in this shape before writing any prompt: next shot's first). Read `references/joins.md` before rendering a `cutaway` or `lead_in`. +- **Per shot, every time** `shot_size`, `move`, `lens` and `light` (ids from + `cinematography`, never invented) and `duration` (seconds); `null` leaves + an axis to the model. + +Save the list as `/shots` in the prompt library, one compiled +`/shot-NN` per shot: `references/shot-document.md`. + Whatever the user supplied is locked; copy it, don't improve it. A field they left blank stays blank - ask, or mark it unspecified - and you invent no character, place or line to fill it. @@ -137,8 +144,10 @@ in with `concat_videos`: that join is recorded `hard_cut`, so shot's budget below. Before the keep, run the beat review in `references/beat-review.md` on the shot's `get_output_frames` frames. -Give each shot a budget of three failed attempts, then stop and ask the user -rather than spending more. Each shot's files are organized by `subfolder` - typically `final` +Fix a shot in the order and within the caps of `references/fix-ladder.md`: three failed attempts, then ask. +Check a recurring face with `templates/face-drift-check`. +For H3, lint the prompt first: `minimax-h3/references/prompt-lint.md`. +Each shot's files are organized by `subfolder` - typically `final` for the deliverable video or `intermediate` for test frames; read the manifest to determine what was written where. diff --git a/plugins/dw/skills/script-to-video/references/fix-ladder.md b/plugins/dw/skills/script-to-video/references/fix-ladder.md new file mode 100644 index 000000000..ac439d9c2 --- /dev/null +++ b/plugins/dw/skills/script-to-video/references/fix-ladder.md @@ -0,0 +1,40 @@ +# The fix ladder and its caps + +Part of the `script-to-video` skill; step 5's *Run and judge* says when to +read this. A shot that reads wrong is fixed in a fixed order, cheapest +change to the least of the take first, and each rung has a cap. The ladder +and its caps are adapted from open-higgsfield-skills (MIT). + +## The rungs + +1. **Edit the region, or repair the face.** When the take is good and one + part is wrong, change only that part: `templates/minimax/regional-edit` + for a region of an H3 take (the rest and the audio stay), and + `templates/minimax/face-repair` for a soft small face. Read + `dw:minimax-h3` first. Other families have no such template: go to rung 2. +2. **Re-prompt.** Change the words, keep the seed, so a pinned voice and a + composition that worked survive. Edit the shot object in + `/shots`, recompile, `save_prompt` the shot (see + `references/shot-document.md`). For H3, run + `minimax-h3/references/prompt-lint.md` over the new prompt first. +3. **Reseed.** `rerun_job(new_seed=True)`. A new seed redraws an unpinned + voice as well; use it only when the voice is pinned by a reference or does + not matter. +4. **Ask.** Stop and put the shot, the three attempts and what each changed + in front of the user. Do not widen a cap to keep going. + +## Caps + +- **Three failed attempts per shot**, counted across all rungs. A seam + re-roll counts against the shot's budget too. +- **A GPU-minute cap for the project**, stated with the cost quote before + the batch runs (`plan.estimate`), and spent down by every re-run. When the + next fix would pass it, ask first; `acknowledged_cost` is the user's, not + the loop's. +- **QC before the next segment starts.** In a chain, judge a segment's + frames before the next one is queued: a bad one is cheaper to fix alone + than after the rest were drawn from its last frame. (The idea is Body + Double's, lucataco/open-source-higgsfield.) + +A fix is judged by looking: `get_output_frames` on the new take, then the +beat review in `references/beat-review.md`. diff --git a/plugins/dw/skills/script-to-video/references/shot-document.md b/plugins/dw/skills/script-to-video/references/shot-document.md new file mode 100644 index 000000000..ed4870824 --- /dev/null +++ b/plugins/dw/skills/script-to-video/references/shot-document.md @@ -0,0 +1,117 @@ +# The persisted shot document + +Part of the `script-to-video` skill; step 2 says when to read this. The shot +list is not kept in the chat. It is saved in the prompt library, so a +revision recompiles only the shots it changed and a later session picks the +project up where it stopped. + +## Where it lives + +Two kinds of prompt, both under a project prefix (the library is shared +across workspaces, so the prefix is what keeps projects apart): + +- `/shots`: the whole document, JSON text, an array of shot objects. + `save_prompt(name, ...)` it; `get_prompt(name)` reads it back; + `list_prompts()` finds what a project already holds. +- `/shot-NN`: one compiled prompt per shot, `NN` two digits from + `01`, in source order. A workflow argument names it as + `prompt:/shot-NN`. + +`` is one name segment matching `[\w][\w.-]*`: letters, digits +and `_`, then `.` or `-` as well - no spaces and no further `/`. A prompt +name holds at most one `/`, and the project prefix uses it, so +`gilded_echoes/shots` is a name and `gilded echoes/shots` or +`series/ep1/shots` is refused. + +`/shots` is the source of truth; each `/shot-NN` is what +it compiles to for that shot's family. Never edit a compiled prompt by +hand without writing the change back into the shot object. + +## The shot object + +| Key | Holds | +| --- | --- | +| `id` | `01`, `02`, ...: the `NN` of its compiled prompt | +| `cast` | the characters in frame, by their `asset:cast/` names | +| `location` | a place named once in the document and reused | +| `opening_state` | what is on screen at frame one | +| `ending_state` | the same at the last frame | +| `beat` | one visible state a single frame can show | +| `must_not` | the few things to rule out | +| `join` | `continue`, `cut`, `cutaway` or `lead_in` | +| `shot_size` | the cinematography axis of that name, or `null` | +| `move` | likewise | +| `lens` | likewise | +| `light` | likewise | +| `duration` | seconds the shot should run (a number), or `null` | + +Every shot carries all five cinematography keys - `shot_size`, `move`, +`lens`, `light` and `duration` - so a reader never has to guess whether a +key was forgotten. `shot_size`, `move`, `lens` and `light` take an id from +`cinematography/references/vocabulary.md` and nothing else, or `null`: an +axis set to `null` is left to the model's own choice, not a default. Their +meaning, and the caption words each family gets for them, are that skill's. +`duration` is a number of seconds, not a vocabulary id, or `null` for the +family's default length. It is snapped to the family's own grid when the +prompt is compiled (H3's is in `dw:minimax-h3`), and the document keeps what +the script asked for. + +One shot object: + +```json +{ + "id": "01", + "cast": ["asset:cast/priya.png"], + "location": "the lighthouse kitchen", + "opening_state": "Priya at the window, back to the room, kettle steaming", + "ending_state": "Priya turned to face the door, mug in both hands", + "beat": "she hears the knock and turns", + "must_not": ["text in frame", "a second figure"], + "join": "cut", + "shot_size": "medium", + "move": "push-in", + "lens": null, + "light": "practical", + "duration": 4.0 +} +``` + +`/shots` is an array of them, in source order. A two-shot +document, saved with `save_prompt("lighthouse/shots", ...)`: + +```json +[ + {"id": "01", "cast": ["asset:cast/priya.png"], + "location": "the lighthouse kitchen", + "opening_state": "Priya at the window, kettle steaming", + "ending_state": "Priya facing the door, mug in hand", + "beat": "she hears the knock and turns", "must_not": ["text in frame"], + "join": "cut", "shot_size": "medium", "move": "push-in", "lens": null, + "light": "practical", "duration": 4.0}, + {"id": "02", "cast": ["asset:cast/priya.png"], + "location": "the lighthouse kitchen", + "opening_state": "Priya facing the door, mug in hand", + "ending_state": "Priya's hand on the latch", + "beat": "she crosses to the door", "must_not": ["text in frame"], + "join": "continue", "shot_size": null, "move": "tracking", + "lens": "normal", "light": "practical", "duration": null} +] +``` + +## Recompiling only what changed + +1. Edit the shot object, then `save_prompt` the whole `/shots`. +2. Compile only the changed shot, per `cinematography`, and `save_prompt` its + `/shot-NN` alone. +3. `rerun_job` that shot's job, or `run_workflow` for that shot alone, and + quote its cost first. Shots that did not change are not rendered again. + +A changed `ending_state` also changes the next shot's `opening_state`: the +shot that has it as its source is a changed shot too. A changed `join` +changes the shots on both sides of the cut (`references/joins.md`). + +## Credit + +The shape borrows ideas from seriesmith (MIT: a model-free shot list +compiled per target) and OpenReel (MIT: one unified style block and the +actor description repeated verbatim in every prompt). No text was copied. diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index d94dbf82e..cf1e84f6a 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -142,6 +142,8 @@ composes into, not something to re-derive: - **pair**: the normalized track is muxed onto the cut - the episode's deliverable. +For a dialogue scene cut master-and-singles, read `references/sitcom-multicam.md`. + For smeary two-shot faces, read `references/cinema-path.md` (1344x768 base model); choose the path once, at episode 1. @@ -176,7 +178,8 @@ Before keeping each episode shot, run `dw:script-to-video`'s beat review Beyond `minimax-h3`'s own judging checklist (a character that changes between shots, a portrait imposing its framing, a voice without affect): watch two episodes back to back and check the cast reads as the same -people - the failure mode step 0 exists to prevent. Compare episodes' +people - the failure mode step 0 exists to prevent. Measure it per shot with +`templates/face-drift-check` (a flag to look at, not a verdict). Compare episodes' `integrated_lufs` (`get_gallery_metadata`), not `peak_dbfs`; the second half of `references/loudness.md` reads the gap. To confirm every line rendered rather than judging it by ear, run `templates/check-script` diff --git a/plugins/dw/skills/series-episodes/references/sitcom-multicam.md b/plugins/dw/skills/series-episodes/references/sitcom-multicam.md new file mode 100644 index 000000000..a722d28ac --- /dev/null +++ b/plugins/dw/skills/series-episodes/references/sitcom-multicam.md @@ -0,0 +1,87 @@ +# Sitcom multicam: a master, per-speaker singles, cuts on the words + +Dialogue scenes cut the way a multicam sitcom does: one wide camera that never +leaves, and a close camera per speaker you cut to when that person talks. The +cut-on-word-boundaries idea is One-shot Multicam's (MIT licence); the pieces +below are this repo's. + +## The grammar + +- **Master**: one wide take of the whole scene, held start to finish. Its shot + fields are `shot_size: wide`. It carries the room, the blocking and the full + dialogue audio, and is the fallback for anything below. +- **Singles**: one `templates/minimax/dialogue-short` clip per speaker line or + run of lines, `shot_size: medium-close` or `close-up`, the speaker facing + camera. Cast the speaker from the same reference as the master so the face + matches. +- **Cuts**: on line boundaries, never mid-word. Cut to a speaker's single as + they start, back to the master for a reaction or when the line ends. +- **Fallback**: a single whose face drifts from the cast reference is dropped + and the master covers that span. + +Field names and values for each shot come from +`cinematography/references/vocabulary.md`; persist the scene as a shot document +per `script-to-video/references/shot-document.md` (master is shot 1, singles +follow) rather than keeping the list in your head. + +## Procedure + +1. Render the master, then the dialogue audio it carries (the scene's speech). +2. Run `transcribe_audio` on that audio with `timestamps` set to `"segment"`. + Without `lyrics`, `plan_cuts` takes each transcript chunk as one line, so + `"word"` would make one shot per word; a segment is a spoken line. Cut + times come from the chunk starts, which is what keeps a cut off the middle + of a word. Do not guess them from the script. +3. Run `plan_cuts` with that `transcript`, `segment_by: "line"`, `fps` of the + family (H3: `24`), `include_instrumental_gaps: false` (a pause stays with + the line before it; no music shot in a dialogue scene), `duration_s` the + audio's length, and the family's frame grid so each single lands on it + (H3: `modulus: 17`, `remainder: 5`, `min_frames: 124`, `max_frames: 345`; + past 345 frames split the line). Read each planned shot's `start_frame` + and `cut_frames` (where the cut sits on the scene timeline and how long it + holds), `num_frames` (what the single renders; more than `cut_frames` + when rounded up to the grid, so `trim_video` it to `cut_frames`) and + `lyric` (the line's text). The shots tile the audio, so every line + boundary is a cut. +4. Map each planned line to its speaker. Run `templates/attribute-lines` (its + `attribute_voices` step scores each transcript line against each voice by + timbre) on the dialogue audio with `voices`: two or more names, each mapped + to reference spans into that audio, taken from lines you know only that + person speaks (read off the script and the transcript's times), or to a + separate clip of the voice. Match its lines to the shots by time. A line + it marks uncertain, or that disagrees with the script, is resolved from + the script; the script is the check, not the source. Consecutive shots by + one speaker join into one single: the first's `start_frame`, the summed + `cut_frames`. +5. Generate each single with `templates/minimax/dialogue-short`, sized to its + planned length. +6. Run `templates/face-drift-check` (task `analyze_face_drift`) on each single + against the cast reference. A `face_drift` or `no_face` finding replaces + that single with the master's span for the same times. The check is a flag + to look at, not a verdict. +7. Cut each master span out of the master with `trim_video`: `video` the + master (`keep_output(name, asset_name="scenes/-master")` keeps it + as `asset:scenes/-master.mp4`), and + `start_frame` and `num_frames` that span's planned `start_frame` and + `cut_frames`, so the span sits at the same times as the line it covers: + + ```json + {"name": "span_03", "task": {"command": "trim_video", "arguments": { + "video": "asset:scenes/s1-master.mp4", + "start_frame": 248, "num_frames": 131}}} + ``` + + Each single is trimmed the same way from `start_frame` 0 to its + `cut_frames` (step 3). +8. Assemble the cut list in order, master spans and singles interleaved, with + `templates/assemble-and-score` (its `concat_videos` joins them). Pass the + master as `score` - `slice_audio` takes a video's soundtrack, so the + scene's dialogue runs under the whole cut unbroken, from + `score_start_frame` 0 for `total_frames` the summed `cut_frames` - and + `world_gain: 0`. The template otherwise mixes each shot's own sound under + the score (`world_gain` defaults to 1.8), and every line would be heard + twice: once from the master track, once from the single or span that + speaks it, a few frames apart. + +Review the assembled scene with `assess_output` before keeping it, the same as +any episode shot. diff --git a/plugins/dw/skills/storyboard/SKILL.md b/plugins/dw/skills/storyboard/SKILL.md new file mode 100644 index 000000000..e1ef36bb5 --- /dev/null +++ b/plugins/dw/skills/storyboard/SKILL.md @@ -0,0 +1,103 @@ +--- +name: storyboard +description: Use when a dw MCP server is connected and the user wants a storyboard - a set of boards (stills) for a scene drawn on FLUX 3 Klein from a character reference, a coverage set of one moment from six camera positions, boards used as the first frames of H3, LTX-2.5 or Kandinsky 6 shots, or a timed animatic of the boards with a scratch voice and music bed before any video is paid for. +--- + +# Storyboards on a dw server + +A board is a still that says what a shot will look like before a video model +is paid for. This skill draws the boards on FLUX 3 Klein, builds a coverage +set of one moment, hands boards to a video family as start frames, and cuts +them into a timed animatic. It does not pick a video family's template or +size: that family's skill does, and its rules win. Camera and look words come +from `cinematography`; the shot list's fields from `script-to-video`. + +## Before anything + +1. `get_server_info`: Klein needs a CUDA card (`flux3-klein`'s rules apply: + 16 steps, no guidance, sides in multiples of 16, gated license). +2. Boards must be on the server: `keep_output(name, asset_name="boards/-NN")` + after each draw. Leave the extension off - the engine adds the file's own + (Klein writes `.jpg`) - and cite `asset:boards/-NN.jpg` later. +3. All boards of one animatic share one size: keep the same first reference, + whose aspect ratio sets the board's, for every board of the scene. + +## 1. Draw the boards (Popcorn shape) + +Higgsfield's Popcorn (the concept only; no code is borrowed) draws a whole +scene's boards from reference images in one pass. Here that is +`templates/flux3/multi-reference`: `images` is the character and location +references in order, and the `instruction` names each one by position, "image +one" the character, "image two" the location. It draws up to eight boards +per scene. Call the template once per board and keep the same references and +`seed` so the cast and place hold; the instruction changes the camera and the +action only, in `cinematography`'s `shot_size`, `move` and `light` words. + +- One idea per board: the single state a frame can show, as in the shot list. +- Past eight boards, split the scene. A scene that needs more has several + beats and wants a second reference set. +- A board is a draw, not a promise. Look at it (`get_output_image`) against + the reference before it goes anywhere else. +- To change one board and hold the rest, `templates/flux3/image-edit` with + an instruction naming only the change. + +## 2. Coverage of one moment (Shot Board shape) + +Shot Board (MIT) names six cuts that cover any moment: establishing, medium, +face close-up, over-the-shoulder, insert and low angle. Pick the hero board +first - the one that best shows the moment - then redraw the other five from +it with `templates/flux3/multi-reference` or `image-edit`, the hero as +"image one". `references/coverage-set.md` has the six, what each frames, and +the instruction pattern. The eight-board cap holds across the set. + +## 3. A board as a start frame + +A board that passes review becomes a shot's first frame. The video family's +skill picks the template and size; the board must match that size, so resize +it first rather than letting the pipeline crop a face off. + +| Family | Template | Board goes in | +| --- | --- | --- | +| H3 | `templates/minimax/image-to-video` | `image` | +| LTX-2.5 | `templates/ltx2/image-to-video` | `image` | +| Kandinsky 6 | `templates/kandinsky6/image-to-video` | `image` | + +Write that shot's prompt to say what happens next, not what the board +already shows. The board fixes the first frame only; later frames may drift +from it, so check the last frame (`get_output_frames`) against the board. +After the shots exist, `templates/face-drift-check` measures whether the +character's face held. + +## 4. Animatic + +Boards, a scratch voice and a music bed cut into one timed clip, cheaply, to +judge pacing and order before video. `references/animatic.md` has the recipe: +`templates/generate-speech` for the voice, `templates/minimax/music` for the +bed, then `templates/animatic` with the boards, one duration per board and +the soundtrack. Change a duration and re-run: it costs seconds, not GPU +hours. + +## Run and judge + +`validate_workflow`, quote `plan.estimate`, `run_workflow`, `wait_for_job`, +`get_job`. A template's `subfolder` sorts what it writes: `final` for the +deliverable (the animatic), `intermediate` for scratch. Boards you keep as +assets are the deliverable of the drawing pass; set `"role": "intermediate"` +on a board that is only a step toward a hero. Read a project's shot list with +`get_prompt("/shots")` (`list_prompts` finds the project). Judge boards +with `get_output_image` and the animatic with `get_output_frames`. + +## Not in scope + +- Video generation, the cut and the score of the final film: the family + skills, `script-to-video` and `series-episodes`. +- Layout by box from a short request (`flux3-klein`'s `layout-boxes` is by + hand). +- Sound timed to the picture: the animatic holds boards, it does not align + them to audio. + +## Sources + +`dw:flux3-klein`, `dw:cinematography`, `dw:script-to-video`; Higgsfield +Popcorn (concept only) and Shot Board (MIT) for the two shapes; +`workflows/templates/animatic.json`; `docs/TASKS.md` (`stills_to_video`). diff --git a/plugins/dw/skills/storyboard/references/animatic.md b/plugins/dw/skills/storyboard/references/animatic.md new file mode 100644 index 000000000..0a9a3f3b9 --- /dev/null +++ b/plugins/dw/skills/storyboard/references/animatic.md @@ -0,0 +1,53 @@ +# The animatic recipe + +Part of the `storyboard` skill; step 4 says when to read this. An animatic +is boards held for a time with a voice and a music bed under them. It tests +the cut's pace and order for the price of a few seconds of CPU. + +## 1. The boards + +Boards at one size, kept with `keep_output(name, asset_name="boards/-NN")` +with no extension: the engine appends the kept file's own, and Klein writes +JPEG, so each is an `asset:boards/-NN.jpg`. A name ending in another +extension is refused. A mixed-size set is refused by the task, naming both +sizes. + +## 2. A scratch voice + +`templates/generate-speech` speaks one line as a wav. Run it per line, or +once for a whole read; it is a scratch voice for timing, not the final +performance. Keep the wav as an asset. + +## 3. A music bed + +`templates/minimax/music` makes a track from a prompt and lyrics (instrumental +if the lyrics are empty), by `dw:minimax-music3`'s rules. Make it at least +as long as the animatic: `animatic` trims a longer track and pads a shorter +one with silence, either way with a warning. To lay voice and bed together, +mix them first with `mix_audio` into one asset. + +## 4. The animatic + +Run `templates/animatic`: + +``` +validate_workflow(name="templates/animatic", arguments={ + "boards": ["asset:boards/s1-01.jpg", "asset:boards/s1-02.jpg", ...], + "durations": [2.0, 1.5, 3.0], + "soundtrack": "asset:audio/s1-scratch.wav", + "fps": 24}) +``` + +`durations` is a list with one duration per board, in seconds, as long as +`boards`; repeat the number to hold every board alike. Each board gets +`max(1, round(duration * fps))` frames, at most 600 seconds and 36000 frames +in total. The output's audio is the soundtrack trimmed or padded to the +clip's length. A voice line only lines up with its board if the soundtrack +was cut to the same durations: the template does not time boards to audio. +Read the finished clip with `get_output_frames`. + +## 5. Iterate + +Change a duration or swap a board and re-run. When the timing reads right, +the durations become each shot's `duration` in the shot document +(`script-to-video`'s `references/shot-document.md`). diff --git a/plugins/dw/skills/storyboard/references/coverage-set.md b/plugins/dw/skills/storyboard/references/coverage-set.md new file mode 100644 index 000000000..3e146c81b --- /dev/null +++ b/plugins/dw/skills/storyboard/references/coverage-set.md @@ -0,0 +1,41 @@ +# The six-cut coverage set + +Part of the `storyboard` skill; step 2 says when to read this. Shot Board +(MIT) covers a moment with six cuts. They are a checklist of what an editor +would want, not a rule that every moment needs all six. + +## The six + +| Cut | Frames | Use it to | +| --- | --- | --- | +| establishing | the whole place, people small | place the scene before it begins | +| medium | the character from the waist up, in the place | carry action and dialogue | +| face close-up | the face fills the frame | show what the character feels | +| over-the-shoulder | one character's shoulder in the foreground, the other facing us | tie two people into one exchange | +| insert | one object, hand or detail | give the plot a thing to look at | +| low angle | the camera below, looking up | give a character or a place weight | + +Each cut has a `shot_size` or camera value in `cinematography`'s +vocabulary; use that word in the instruction rather than a new one. + +## Procedure + +1. Draw the hero: the cut that best shows the moment, usually the medium, with + `templates/flux3/multi-reference` (the character and location references). +2. Keep the hero. Redraw each other cut with the hero as "image one" and the + character reference as "image two", so the face and the place are + the hero's. +3. Per redraw the instruction names the cut and changes nothing else: + "image one is the scene. Redraw it as a low angle shot, the camera below + the character looking up. Keep the character, clothing and light." +4. Review each against the hero: same face, same clothes, same light. A cut + whose face changed is redrawn once with the close-up reference first; a + second miss is asked about rather than rerolled again. + +An over-the-shoulder needs both characters in the references. Two people in +one still is the pipeline's weakest case: faces can swap or merge, so check +both. + +For `templates/consistent-set` (FLUX.1 Kontext, not Klein), the same idea +applies as edits of one base picture; it is the older route and its look +differs from Klein's. diff --git a/pyproject.toml b/pyproject.toml index 6ccaf833f..aef3bd0d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ name = "diffusers-workflow" # runtime (static in TOML because importing dw at build time would drag # torch into the build environment). Release tags must match it - see # docs/RELEASING.md -version = "0.17.0" +version = "0.18.0-alpha.1" description = "Declarative workflow engine and web UI for Hugging Face Diffusers" readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index 3db07e78e..61ea03128 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -128,9 +128,11 @@ def load(path): # Templates that genuinely are utilities - processing with no generation. # Anything else that derives 'utility' is a rule that missed. UTILITIES = { + "workflows/templates/animatic.json", "workflows/templates/attribute-lines.json", "workflows/templates/audio-trim-fade.json", "workflows/templates/check-script.json", + "workflows/templates/face-drift-check.json", "workflows/templates/image-processors.json", "workflows/templates/minimax/music-video-cuts.json", "workflows/templates/recenter-crop.json", @@ -543,7 +545,14 @@ def test_no_stale_entry_in_the_allowlist(): # Then to 12_650 for `video-to-sound-effects` and `text-to-sound-effect` # (#1124, stage #1133, 2026-11-01), measured at 12_609: two new entries, the # catalog's only sound-effect generation (MMAudio), with no cost block yet. -COMPACT_BUDGET = 12_650 +# Then to 12_800 for `animatic` and `face-drift-check` (#1145/#1146, +# 2026-11-01), measured at 12_791: two new entries, the catalog's only timed +# stills-to-clip and its only per-shot identity check. `animatic` first +# shipped an unmeasured CUDA cost block; the final review (#1141) removed it, +# since the template loads no model and only encodes on the CPU, so neither +# entry carries a cost block now. Re-measured at 12_780 after that and +# face-drift-check's `samples_per_shot`, so the budget stands. +COMPACT_BUDGET = 12_800 FILTERED_BUDGET = 1_500 diff --git a/tests/test_docker.py b/tests/test_docker.py new file mode 100644 index 000000000..170bcecd5 --- /dev/null +++ b/tests/test_docker.py @@ -0,0 +1,283 @@ +"""docker/entrypoint.py - the diffusers overlay's lifecycle, the API token and +dw.serve's command line - and the pins between docker/ and the rest of the +repo. Subprocesses are faked: nothing here runs pip, a venv or docker.""" + +import importlib.util +import json +import re +import stat +from pathlib import Path +from types import SimpleNamespace + +import pytest + +REPO = Path(__file__).resolve().parents[1] +DOCKER = REPO / "docker" + + +@pytest.fixture(scope="module") +def entry(): + spec = importlib.util.spec_from_file_location( + "dw_docker_entrypoint", DOCKER / "entrypoint.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeRun: + """Stands in for subprocess.run: records each command, creates the venv's + interpreter when `-m venv` runs, and answers `probe` with the diffusers + the overlay imports - `before` until pip has installed, `installed` after - + or fails it when that is None.""" + + def __init__(self, installed=None, fail_pip=False, before=None): + self.installed = installed + self.before = installed if before is None else before + self.fail_pip = fail_pip + self.pip_ran = False + self.commands = [] + + def __call__(self, command, **kwargs): + self.commands.append(command) + if command[1:3] == ["-m", "venv"]: + python = Path(command[-1]) / "bin" / "python" + python.parent.mkdir(parents=True) + python.write_text("") + return SimpleNamespace(returncode=0, stdout="", stderr="") + if command[-1] == "probe": + installed = self.installed if self.pip_ran else self.before + if installed is None: + return SimpleNamespace(returncode=1, stdout="", stderr="ImportError") + return SimpleNamespace( + returncode=0, stdout=json.dumps(installed) + "\n", stderr="" + ) + if "pip" in command: + if self.fail_pip: + return SimpleNamespace(returncode=1, stdout="", stderr="build failed") + self.pip_ran = True + return SimpleNamespace(returncode=0, stdout="", stderr="") + + def pip_targets(self): + return [c[-1] for c in self.commands if c[1:4] == ["-m", "pip", "install"]] + + +IMAGE = { + "build_id": "b1", + "python": "3.13.9", + "diffusers": {"version": "0.43.0.dev0", "commit": "c" * 40}, +} +HEAD = {"version": "0.43.0.dev0", "commit": "a" * 40} + + +def environ_for(tmp_path, **extra): + return {"DW_ENV_DIR": str(tmp_path / "env"), **extra} + + +def marker(entry, environ): + return json.loads(entry.marker_path(environ).read_text()) + + +class TestParseRef: + def test_keywords(self, entry): + assert entry.parse_ref(None) == ("head", None) + assert entry.parse_ref(" HEAD ") == ("head", None) + assert entry.parse_ref("image") == ("image", None) + + def test_commit(self, entry): + assert entry.parse_ref("abc1234") == ("commit", "abc1234") + + def test_anything_else_refuses_to_start(self, entry): + with pytest.raises(SystemExit): + entry.parse_ref("main; rm -rf /") + + +class TestInstallTarget: + def test_pinned_ref_wins(self, entry): + assert entry.install_target( + "commit", "abc1234", {"commit": "f" * 40}, None + ) == {"commit": "abc1234"} + + def test_old_overlay_beats_marker(self, entry): + """The Models page may have moved diffusers since the marker was written.""" + assert entry.install_target( + "head", None, {"commit": "f" * 40}, {"commit": "e" * 40} + ) == {"commit": "e" * 40} + + def test_marker_when_old_overlay_will_not_run(self, entry): + assert entry.install_target("head", None, {"commit": "f" * 40}, None) == { + "commit": "f" * 40 + } + + def test_held_release_reverts(self, entry): + assert entry.install_target( + "head", None, {"version": "0.42.0", "commit": None}, None + ) == {"revert": True} + + def test_nothing_held_is_head(self, entry): + assert entry.install_target("head", None, None, None) == {} + + +class TestPrepareOverlay: + def test_image_ref_skips_the_overlay(self, entry, tmp_path): + run = FakeRun(installed=HEAD) + environ = environ_for(tmp_path, DW_DIFFUSERS_REF="image") + assert entry.prepare_overlay(environ, IMAGE, run) == entry.BASE_PYTHON + assert run.commands == [] + + def test_first_start_installs_head(self, entry, tmp_path): + run = FakeRun(installed=HEAD) + environ = environ_for(tmp_path) + python = entry.prepare_overlay(environ, IMAGE, run) + assert python == str(entry.overlay_python(environ)) + assert "--system-site-packages" in run.commands[0] + targets = run.pip_targets() + assert targets[0] == "git+https://github.com/huggingface/diffusers" + assert all(c[0] == python for c in run.commands if "pip" in c) + assert marker(entry, environ) == { + "build_id": "b1", + "python": "3.13.9", + "diffusers": HEAD, + } + + def test_restart_installs_nothing(self, entry, tmp_path): + environ = environ_for(tmp_path) + entry.prepare_overlay(environ, IMAGE, FakeRun(installed=HEAD)) + run = FakeRun(installed=HEAD) + assert entry.prepare_overlay(environ, IMAGE, run) == str( + entry.overlay_python(environ) + ) + assert run.pip_targets() == [] + + def test_restart_records_a_models_page_update(self, entry, tmp_path): + environ = environ_for(tmp_path) + entry.prepare_overlay(environ, IMAGE, FakeRun(installed=HEAD)) + moved = {"version": "0.43.0.dev0", "commit": "e" * 40} + entry.prepare_overlay(environ, IMAGE, FakeRun(installed=moved)) + assert marker(entry, environ)["diffusers"] == moved + + def test_new_image_reinstalls_the_held_commit_not_head(self, entry, tmp_path): + environ = environ_for(tmp_path) + entry.prepare_overlay(environ, IMAGE, FakeRun(installed=HEAD)) + run = FakeRun(installed=HEAD) + entry.prepare_overlay(environ, {**IMAGE, "build_id": "b2"}, run) + assert ( + run.pip_targets()[0] + == "git+https://github.com/huggingface/diffusers@" + "a" * 40 + ) + assert marker(entry, environ)["build_id"] == "b2" + + def test_pinned_ref_moves_an_overlay_that_differs(self, entry, tmp_path): + environ = environ_for(tmp_path) + entry.prepare_overlay(environ, IMAGE, FakeRun(installed=HEAD)) + environ["DW_DIFFUSERS_REF"] = "b" * 40 + run = FakeRun( + installed={"version": "0.43.0.dev0", "commit": "b" * 40}, before=HEAD + ) + entry.prepare_overlay(environ, IMAGE, run) + assert run.pip_targets()[0].endswith("@" + "b" * 40) + + def test_failed_install_serves_the_baked_diffusers(self, entry, tmp_path): + environ = environ_for(tmp_path) + run = FakeRun(installed=None, fail_pip=True) + assert entry.prepare_overlay(environ, IMAGE, run) == entry.BASE_PYTHON + # build_id None: the next start retries + assert marker(entry, environ)["build_id"] is None + + def test_overlay_that_will_not_import_serves_the_baked_diffusers( + self, entry, tmp_path + ): + environ = environ_for(tmp_path) + assert ( + entry.prepare_overlay(environ, IMAGE, FakeRun(installed=None)) + == entry.BASE_PYTHON + ) + + +class TestToken: + def test_environment_wins(self, entry, tmp_path): + environ = {"DW_API_TOKEN": " xyz ", "DIFFUSERS_HELPER_ROOT": str(tmp_path)} + assert entry.resolve_token(environ) == "xyz" + assert not (tmp_path / entry.TOKEN_FILE_NAME).exists() + + def test_insecure_opt_out(self, entry, tmp_path): + environ = {"DW_INSECURE_NO_TOKEN": "1", "DIFFUSERS_HELPER_ROOT": str(tmp_path)} + assert entry.resolve_token(environ) is None + + def test_generated_once_private_and_kept(self, entry, tmp_path): + environ = {"DIFFUSERS_HELPER_ROOT": str(tmp_path / "settings")} + token = entry.resolve_token(environ) + path = tmp_path / "settings" / entry.TOKEN_FILE_NAME + assert re.fullmatch(r"[0-9a-f]{64}", token) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert entry.resolve_token(environ) == token + + def test_healthcheck_never_mints_one(self, entry, tmp_path, monkeypatch): + environ = {"DIFFUSERS_HELPER_ROOT": str(tmp_path)} + monkeypatch.setattr( + entry.urllib.request, + "urlopen", + lambda *a, **k: (_ for _ in ()).throw(OSError()), + ) + assert entry.healthcheck(environ) == 1 + assert not (tmp_path / entry.TOKEN_FILE_NAME).exists() + + +class TestServe: + def test_defaults_unless_named(self, entry): + command = entry.serve_command("/py", ["--mcp"]) + assert command[:3] == ["/py", "-m", "dw.serve"] + assert command[command.index("--host") + 1] == "0.0.0.0" + assert command[-1] == "--mcp" + command = entry.serve_command("/py", ["--host", "127.0.0.1", "--port", "9000"]) + assert command.count("--host") == 1 and command.count("--port") == 1 + + def test_token_goes_in_the_environment_not_argv(self, entry, tmp_path): + calls = [] + environ = environ_for( + tmp_path, + DW_DIFFUSERS_REF="image", + DW_API_TOKEN="xyz", + HOME=str(tmp_path / "home"), + ) + entry.serve( + ["--mcp"], + environ, + FakeRun(), + lambda path, argv, env: calls.append((path, argv, env)), + ) + path, argv, env = calls[0] + assert path == entry.BASE_PYTHON + assert env["DW_API_TOKEN"] == "xyz" + assert "xyz" not in argv and "--token" not in argv + + +class TestPins: + def test_node_follows_ui_nvmrc(self): + nvmrc = (REPO / "ui" / ".nvmrc").read_text().strip().lstrip("v") + assert f"FROM node:{nvmrc}-" in (DOCKER / "Dockerfile").read_text() + + def test_shared_volumes_mount_at_their_host_path(self): + """Manifests and cache entries record absolute paths; a container and a + native server share them only when the paths agree.""" + compose = (DOCKER / "compose.yaml").read_text() + mounts = re.findall(r"^\s+- \$\{(\w+)[^}]*\}:(\S+)$", compose, re.MULTILINE) + assert mounts + for variable, target in mounts: + if variable == "DW_ENV_DIR": + # docker/.env reaches the container too: the host path must not + assert target == "/data/env" + assert re.search(r"^\s+DW_ENV_DIR: /data/env$", compose, re.MULTILINE) + else: + assert target == "${%s}" % variable + assert re.search( + rf"^\s+{variable}: \${{{variable}}}$", compose, re.MULTILINE + ) + + def test_init_env_writes_every_variable_compose_requires(self): + compose = (DOCKER / "compose.yaml").read_text() + required = set(re.findall(r"\$\{(\w+):\?", compose)) + init_env = (DOCKER / "init-env.sh").read_text() + assert required + for variable in required: + assert re.search(rf"^{variable}=", init_env, re.MULTILINE), variable diff --git a/tests/test_face_drift.py b/tests/test_face_drift.py new file mode 100644 index 000000000..bc656e0fa --- /dev/null +++ b/tests/test_face_drift.py @@ -0,0 +1,280 @@ +"""Tests for analyze_face_drift (dw/tasks/face_drift.py, #1146). + +YuNet and SFace are replaced by fakes, so no weights are downloaded: a face +is a flat coloured square on a black frame, the detector finds each distinct +colour's bounding box, and the recognizer embeds a crop as its mean colour - +two squares of the same colour are the same person, different colours are +orthogonal (cosine 0). +""" + +import numpy as np +import pytest +from PIL import Image + +import dw.tasks.face_drift as fd +from dw.media_types import AudioVideo +from dw.shots import shot_record +from dw.tasks.registry import required_weights_hook +from dw.tasks.task import task_command_info + +PERSON_A = (255, 0, 0) +PERSON_B = (0, 255, 0) +SIZE = 64 + + +def frame(*faces): + """A black frame; each face is (colour, x, y, side).""" + pixels = np.zeros((SIZE, SIZE, 3), dtype=np.uint8) + for colour, x, y, side in faces: + pixels[y : y + side, x : x + side] = colour + return Image.fromarray(pixels) + + +def a_face(colour=PERSON_A, side=24): + return frame((colour, 8, 8, side)) + + +class FakeRecognizer: + def alignCrop(self, bgr, row): + x, y, w, h = (int(v) for v in row[:4]) + return bgr[y : y + h, x : x + w] + + def feature(self, crop): + return crop.reshape(-1, 3).mean(axis=0).astype(np.float32) + + def match(self, first, second, flag): + assert flag == fd.COSINE + return float( + first @ second / (np.linalg.norm(first) * np.linalg.norm(second) + 1e-9) + ) + + +class Probe: + """Counts the frames the fake detector was shown.""" + + def __init__(self): + self.calls = 0 + + def detect(self, bgr): + self.calls += 1 + rows = [] + for colour in {tuple(p) for p in bgr.reshape(-1, 3)} - {(0, 0, 0)}: + ys, xs = np.nonzero((bgr == colour).all(axis=2)) + row = [xs.min(), ys.min(), xs.max() - xs.min() + 1, ys.max() - ys.min() + 1] + rows.append(row + [0.0] * 10 + [0.9]) + return rows + + +@pytest.fixture +def probe(monkeypatch): + seen = Probe() + monkeypatch.setattr( + fd, "_models", lambda device=None: (seen.detect, FakeRecognizer()) + ) + return seen + + +def clip(frames, shots=None): + return AudioVideo(frames, None, None, fps=24, shots=shots) + + +def shots_of(*counts): + start = 0 + records = [] + for index, count in enumerate(counts): + records.append(shot_record(f"shot{index}", start, count)) + start += count + return records + + +def test_same_face_no_drift_finding(probe): + answer = fd.analyze_face_drift( + clip([a_face()] * 6, shots_of(3, 3)), a_face(), samples_per_shot=3 + ) + assert answer["findings"] == [] + assert [m["faces_found"] for m in answer["measurements"]] == [3, 3] + assert all(m["similarity_median"] > 0.99 for m in answer["measurements"]) + assert answer["shots_source"] == "artifact" + assert answer["rules_applied"] == ["face_drift", "no_face"] + + +def test_other_face_drift_on_every_shot(probe): + answer = fd.analyze_face_drift( + clip([a_face(PERSON_B)] * 6, shots_of(3, 3)), a_face(), samples_per_shot=3 + ) + drifted = [f for f in answer["findings"] if f["rule"] == "face_drift"] + assert [f["at"] for f in drifted] == [{"shot": "shot0"}, {"shot": "shot1"}] + assert drifted[0]["threshold"] == 0.363 + assert drifted[0]["value"] < 0.363 + + +def test_empty_shot_reports_no_face_only_there(probe): + frames = [frame()] * 3 + [a_face()] * 3 + answer = fd.analyze_face_drift( + clip(frames, shots_of(3, 3)), a_face(), samples_per_shot=3 + ) + assert [(f["rule"], f["at"]) for f in answer["findings"]] == [ + ("no_face", {"shot": "shot0"}) + ] + first, second = answer["measurements"] + assert first["faces_found"] == 0 and first["similarity_median"] is None + assert first["similarity_min"] is None + assert second["faces_found"] == 3 + + +def test_reference_without_face_refused(probe): + with pytest.raises(ValueError, match="reference"): + fd.analyze_face_drift(clip([a_face()]), frame()) + + +def test_reference_with_two_faces_uses_largest_and_notes_it(probe): + # the larger face is person A; a second, smaller person B stands beside it + reference = frame((PERSON_A, 4, 4, 30), (PERSON_B, 40, 4, 12)) + answer = fd.analyze_face_drift(clip([a_face()] * 2), reference) + assert answer["findings"] == [] + assert any("reference" in note and "largest" in note for note in answer["notes"]) + swapped = fd.analyze_face_drift(clip([a_face(PERSON_B)] * 2), reference) + assert [f["rule"] for f in swapped["findings"]] == ["face_drift"] + + +def test_whole_file_is_one_shot_without_boundaries(probe): + answer = fd.analyze_face_drift(clip([a_face()] * 4), a_face()) + assert answer["shots_source"] == "none" + assert len(answer["measurements"]) == 1 + + +@pytest.mark.parametrize("threshold", [-1.01, 1.01]) +def test_threshold_bounds(probe, threshold): + with pytest.raises(ValueError, match="threshold"): + fd.analyze_face_drift(clip([a_face()]), a_face(), threshold=threshold) + + +@pytest.mark.parametrize("samples", [0, -2, 1.5]) +def test_samples_per_shot_must_be_a_positive_whole_number(probe, samples): + with pytest.raises(ValueError, match="samples_per_shot"): + fd.analyze_face_drift(clip([a_face()]), a_face(), samples_per_shot=samples) + + +def test_samples_per_shot_caps_reads(probe): + answer = fd.analyze_face_drift(clip([a_face()] * 100), a_face()) + # one detection for the reference, eight for the shot + assert probe.calls == 1 + 8 + assert answer["measurements"][0]["frames_sampled"] == 8 + + +def test_a_shot_shorter_than_the_cap_reads_each_frame_once(probe): + answer = fd.analyze_face_drift(clip([a_face()] * 3), a_face()) + assert answer["measurements"][0]["frames_sampled"] == 3 + + +def test_required_weights_lists_sface_and_yunet(): + entries = fd.required_weights({}) + assert { + "repo": fd.SFACE_REPO, + "filename": fd.SFACE_FILE, + "revision": fd.SFACE_REVISION, + } in entries + assert any(entry["repo"] == "opencv/face_detection_yunet" for entry in entries) + + +def test_command_registered(): + import dw.tasks # noqa: F401 registers the handlers + + info = task_command_info("analyze_face_drift") + assert info["returns"] == "json" + assert required_weights_hook("analyze_face_drift") is fd.required_weights + + +class TestStaticCheck: + """The settings refused at validate, in the run's words (one rule).""" + + @staticmethod + def _errors(**arguments): + from dw.task_problems import face_drift_errors + + return face_drift_errors(arguments) + + @pytest.mark.parametrize( + "arguments, argument", + [ + ({"threshold": 1.5}, "threshold"), + ({"threshold": -1.01}, "threshold"), + ({"samples_per_shot": 0}, "samples_per_shot"), + ({"samples_per_shot": 65}, "samples_per_shot"), + ({"samples_per_shot": 1.5}, "samples_per_shot"), + ], + ) + def test_out_of_range_found(self, arguments, argument): + assert [a for a, _ in self._errors(**arguments)] == [argument] + + def test_clean_and_referenced_values_pass(self): + assert not self._errors(threshold=0.363, samples_per_shot=64) + assert not self._errors(threshold=-1, samples_per_shot=1) + assert not self._errors( + threshold="variable:t", samples_per_shot="previous_result:x" + ) + + def test_the_run_refuses_in_the_same_words(self, probe): + (_, message), *_ = self._errors(threshold=1.5) + with pytest.raises(ValueError) as refused: + fd.analyze_face_drift(clip([a_face()]), a_face(), threshold=1.5) + assert str(refused.value) == message + + +TEMPLATE = "workflows/templates/face-drift-check.json" + + +class TestTemplateValidation: + @staticmethod + def _errors(tmp_path, arguments): + import os + + from dw.validation import workflow_errors + from dw.workflow import workflow_from_file + from tests.test_examples import REPO_ROOT + + workflow = workflow_from_file(os.path.join(REPO_ROOT, TEMPLATE), str(tmp_path)) + return workflow_errors(workflow, arguments) + + @pytest.mark.parametrize( + "arguments, argument", + [ + ({"threshold": 1.5}, "threshold"), + ({"samples_per_shot": 0}, "samples_per_shot"), + ({"samples_per_shot": 65}, "samples_per_shot"), + ], + ) + def test_validate_refuses_out_of_range_settings( + self, tmp_path, arguments, argument + ): + errors = self._errors(tmp_path, arguments) + assert errors + assert all(error["path"].endswith(f".{argument}") for error in errors) + + def test_defaults_are_clean(self, tmp_path): + assert self._errors(tmp_path, {}) == [] + + def test_template_declares_and_wires_samples_per_shot(self): + import json + import os + + from tests.test_examples import REPO_ROOT + + with open(os.path.join(REPO_ROOT, TEMPLATE), encoding="utf-8") as handle: + definition = json.load(handle) + assert definition["variables"]["samples_per_shot"] == fd.DEFAULT_SAMPLES + arguments = definition["steps"][0]["task"]["arguments"] + assert arguments["samples_per_shot"] == "variable:samples_per_shot" + + +def test_sface_cache_key_carries_the_revision(monkeypatch): + """A revision bump must not be answered by the recognizer cached before it.""" + import dw.tasks.model_cache as model_cache + + keys = [] + monkeypatch.setattr(fd, "face_detector_model", lambda *args: object()) + monkeypatch.setattr( + model_cache, "cached_model", lambda key, load: keys.append(key) or object() + ) + fd._models() + assert keys and fd.SFACE_REVISION in keys[0] diff --git a/tests/test_plugin_skills.py b/tests/test_plugin_skills.py index f9b27023e..71022925c 100644 --- a/tests/test_plugin_skills.py +++ b/tests/test_plugin_skills.py @@ -997,6 +997,186 @@ def test_the_validation_is_real(self, tmp_path): self._validate(workflow, tmp_path) +CINE_DIR = os.path.join(PLUGIN_DIR, "skills", "cinematography") +CINE_SKILL = os.path.join(CINE_DIR, "SKILL.md") +CINE_REFS = os.path.join(CINE_DIR, "references") + +# The closed vocabulary. Later stages (the shot document, the storyboard +# skill, the A/B probes) cite these ids verbatim, so a rename fails here. +CINE_AXES = { + "shot_size": [ + "extreme-wide", + "wide", + "full", + "medium", + "medium-close", + "close-up", + "extreme-close-up", + ], + "move": [ + "static", + "dolly-in", + "dolly-out", + "truck-left", + "truck-right", + "crane-up", + "crane-down", + "orbit", + "tracking", + "whip-pan", + "push-in", + "handheld", + "crash-zoom", + ], + "lens": ["wide", "normal", "long", "anamorphic", "vintage"], + "light": ["high-key", "low-key", "golden-hour", "practical", "noir"], + "era": ["8mm", "16mm", "35mm", "vhs-dv", "digital"], + "tempo": ["still", "measured", "brisk", "frantic"], +} +CINE_GRADES = ["noir", "teal-orange", "70s-stock", "vhs", "bleach-bypass"] +CINE_COMPILES = { + "minimax-h3": "compile-h3.md", + "ltx-2.5": "compile-ltx-2.5.md", + "kandinsky-6": "compile-kandinsky-6.md", + "flux3-klein": "compile-flux3-klein.md", +} + + +def _table_ids(text, section): + """The backticked ids in the first column of a markdown table, read + from `section` (a heading) to the next heading of any level.""" + start = text.index(section) + end = text.find("\n#", start + len(section)) + block = text[start : end if end != -1 else len(text)] + return re.findall(r"^\| `([^`]+)` \|", block, re.M) + + +class TestCinematographySkill: + """A closed director vocabulary compiled to one sentence per family's + caption format (#1142). The words are hints the models may ignore, so + the skill says so; the grades are real apply_lut palettes and real + film_grain arguments.""" + + def _ref(self, name): + return open(os.path.join(CINE_REFS, name), encoding="utf-8").read() + + def _grades(self): + return json.loads(_fenced_block_after(self._ref("grades.md"), "## The grades")) + + def test_cinematography_skill_is_registered(self): + for document in ( + os.path.join(PLUGIN_DIR, "README.md"), + os.path.join(REPO_ROOT, "CLAUDE.md"), + ): + assert "`cinematography`" in open(document, encoding="utf-8").read() + assert CINE_SKILL in SKILLS + + def test_cinematography_names_no_camera_brands(self): + """Higgsfield retreated from camera-body presets; a brand only works + if the model learned it, and nobody has measured that.""" + for path in glob.glob(os.path.join(CINE_DIR, "**", "*"), recursive=True): + if not os.path.isfile(path): + continue + text = open(path, encoding="utf-8").read() + for brand in ("ARRI", "RED ", "Cooke", "Venice", "Alexa"): + assert brand not in text, f"{brand!r} in {path}" + + def test_cinematography_vocabulary_defines_every_axis(self): + text = self._ref("vocabulary.md") + for axis, ids in CINE_AXES.items(): + assert _table_ids(text, f"## `{axis}`") == ids, axis + + @pytest.mark.parametrize("family", sorted(CINE_COMPILES)) + def test_cinematography_vocabulary_covers_every_family(self, family): + text = self._ref(CINE_COMPILES[family]) + assert _table_ids(text, "## `shot_size`") == CINE_AXES["shot_size"] + assert _table_ids(text, "## `move`") == CINE_AXES["move"] + assert f"`{family}`" in text + + def test_cinematography_says_presets_are_hints(self): + body = " ".join(skill_body(CINE_SKILL).split()) + assert "a hint, not a promise" in body + assert "get_output_frames" in body + + def test_cinematography_scopes_out_audio_families(self): + body = skill_body(CINE_SKILL) + section = body[body.index("## Not in scope") :] + assert "`minimax-music3`" in section + flat = " ".join(section.split()) + assert ( + "covers H3, LTX-2.5 and Kandinsky 6 video plus FLUX 3 Klein stills only" + in flat + ) + assert "do not borrow a table" in flat + + def test_cinematography_ltx_every_move_row_carries_a_viewpoint(self): + text = self._ref("compile-ltx-2.5.md") + section = text[text.index("## `move`") :] + section = section[: section.index("Example:")] + rows = [r for r in section.splitlines() if r.startswith("| `")] + assert len(rows) == len(CINE_AXES["move"]) + for row in rows: + assert "" in row, row + + def test_cinematography_whip_pan_direction_is_the_shots(self): + for name in CINE_COMPILES.values(): + for row in self._ref(name).splitlines(): + if row.startswith("| `whip-pan`"): + assert "to the right" not in row and "pans right" not in row + + def test_cinematography_compiles_to_each_family_s_own_spec(self): + """Each table follows its family's caption rules, not a shared one.""" + ltx = " ".join(self._ref("compile-ltx-2.5.md").split()) + assert "remains static" in ltx and "viewpoint" in ltx + assert "comes into view" in ltx + k6 = " ".join(self._ref("compile-kandinsky-6.md").split()) + assert "exactly one camera sentence" in k6 + assert "The camera is static." in k6 + h3 = " ".join(self._ref("compile-h3.md").split()) + assert "integrated_multimodal_description" in h3 + assert "amplitude" in h3 and "speed" in h3 + klein = " ".join(self._ref("compile-flux3-klein.md").split()) + assert "still" in klein and "no camera move" in klein + + def test_grades_are_valid_palettes(self): + from dw.task_problems import check_lut_source + + grades = self._grades() + assert sorted(grades) == sorted(CINE_GRADES) + for name, grade in grades.items(): + check_lut_source(palette=grade["palette"]) + assert 0 < grade["strength"] <= 1, name + + def test_grades_name_real_film_grain_arguments(self): + from dw.introspection import list_tasks + from dw.task_domains import check_arguments + from dw.tasks.finish import film_grain + + list_tasks() # registers the command domains + accepted = set(inspect.signature(film_grain).parameters) - {"media", "seed"} + for name, grade in self._grades().items(): + grain = grade["film_grain"] + assert set(grain) <= accepted, name + check_arguments("film_grain", **grain) + + def test_the_grade_recipe_validates(self, tmp_path): + from dw.workflow import workflow_from_file + + workflow = json.loads( + _fenced_block_after(self._ref("grades.md"), "## Applying one") + ) + steps = workflow["steps"] + assert [s["task"]["command"] for s in steps] == ["apply_lut", "film_grain"] + path = tmp_path / "grade.json" + path.write_text(json.dumps(workflow), encoding="utf-8") + workflow_from_file(str(path), str(tmp_path)).validate() + + @pytest.mark.parametrize("family", sorted(CINE_COMPILES)) + def test_family_skills_point_at_cinematography(self, family): + path = os.path.join(PLUGIN_DIR, "skills", family, "SKILL.md") + assert "`cinematography`" in skill_body(path) + + MUSIC_SKILL = os.path.join(PLUGIN_DIR, "skills", "minimax-music3", "SKILL.md") @@ -1205,3 +1385,336 @@ def test_no_rule_lands_for_a_lesson_not_reproduced(self): assert "locked-off on a tripod" not in text # L2's remedy wording assert "all-caps" not in text and "all caps" not in text # L4 assert "just before contact" not in text # L3 + + +STB_REFS = os.path.join(os.path.dirname(SCRIPT_SKILL), "references") +SHOT_DOCUMENT = os.path.join(STB_REFS, "shot-document.md") +FIX_LADDER = os.path.join(STB_REFS, "fix-ladder.md") +H3_LINT = os.path.join( + PLUGIN_DIR, "skills", "minimax-h3", "references", "prompt-lint.md" +) +SHOT_KEYS = ( + "id cast location opening_state ending_state beat must_not join " + "shot_size move lens light duration" +).split() + + +def _read(path): + return open(path, encoding="utf-8").read() + + +def _registered_mcp_tools(): + import asyncio + + import httpx + + pytest.importorskip("mcp", reason="the mcp extra is not installed") + from dw_mcp.client import DwClient + from dw_mcp.server import build_server + + client = DwClient(transport=httpx.MockTransport(lambda r: httpx.Response(200))) + server = build_server(client) + return {tool.name for tool in asyncio.run(server.list_tools())} + + +class TestShotDocumentAndFixLadder: + """Stage B (#1143): the shot list persisted in the prompt library, the + fix ladder with its caps, and the H3 prompt lint.""" + + def test_shot_document_names_its_prompt_keys(self): + text = _read(SHOT_DOCUMENT) + assert "/shots" in text and "/shot-NN" in text + for key in SHOT_KEYS: + assert f"`{key}`" in text, key + + def test_shot_document_points_at_the_vocabulary_not_a_copy(self): + text = _read(SHOT_DOCUMENT) + assert "cinematography/references/vocabulary.md" in text + assert "extreme-close-up" not in text # ids live in one place + + def test_shot_document_uses_only_existing_tools(self): + tools = _registered_mcp_tools() + for path in (SHOT_DOCUMENT, FIX_LADDER, H3_LINT): + for name in re.findall(r"`([a-z][a-z_]+)\(", _read(path)): + assert name in tools, f"{os.path.basename(path)}: {name}" + + def test_shot_document_templates_resolve(self): + for path in (SHOT_DOCUMENT, FIX_LADDER, H3_LINT): + for name in CATALOG_NAME.findall(_read(path)): + assert os.path.exists( + os.path.join(REPO_ROOT, "workflows", name + ".json") + ) or os.path.exists(os.path.join(REPO_ROOT, "workflows", name)), name + + def test_fix_ladder_order(self): + text = _read(FIX_LADDER).lower() + at = [ + text.index(marker) + for marker in ("regional-edit", "face-repair", "re-prompt", "reseed", "ask") + ] + assert at == sorted(at) + + def test_fix_ladder_has_caps(self): + text = " ".join(_read(FIX_LADDER).lower().split()) + assert "three failed attempts" in text and "gpu-minute" in text + + def test_h3_lint_checks(self): + text = _read(H3_LINT) + for check in ("overall_soundscape", "overfull", "no new information"): + assert check in text + + def test_credits(self): + text = _read(SHOT_DOCUMENT) + _read(FIX_LADDER) + _read(H3_LINT) + for name in ("seriesmith", "OpenReel", "open-higgsfield-skills", "Picky"): + assert name in text + assert "GPL" in text + + def test_script_to_video_points_at_the_three_references(self): + body = skill_body(SCRIPT_SKILL) + for ref in ("references/shot-document.md", "references/fix-ladder.md"): + assert f"`{ref}`" in body + assert "minimax-h3/references/prompt-lint.md" in body + for key in ("shot_size", "move", "lens", "light", "duration"): + assert f"`{key}`" in body + + +STORY_DIR = os.path.join(PLUGIN_DIR, "skills", "storyboard") +STORY_SKILL = os.path.join(STORY_DIR, "SKILL.md") + + +class TestStoryboardSkill: + """Boards on FLUX 3 Klein, a six-cut coverage set, boards as start + frames, and an animatic of them (#1145, #1146).""" + + def _ref(self, name): + return open( + os.path.join(STORY_DIR, "references", name), encoding="utf-8" + ).read() + + def test_storyboard_skill_registered(self): + for document in ( + os.path.join(PLUGIN_DIR, "README.md"), + os.path.join(REPO_ROOT, "CLAUDE.md"), + ): + assert "`storyboard`" in open(document, encoding="utf-8").read() + assert STORY_SKILL in SKILLS + + def test_storyboard_names_six_coverage_cuts(self): + text = self._ref("coverage-set.md").lower() + for cut in ( + "establishing", + "medium", + "close-up", + "over-the-shoulder", + "insert", + "low angle", + ): + assert cut in text, cut + + def test_storyboard_caps_boards_at_eight(self): + flat = " ".join(skill_body(STORY_SKILL).split()) + assert "up to eight boards" in flat + assert "image one" in flat + + def test_storyboard_credits_its_sources(self): + text = skill_text(STORY_SKILL) + assert "Higgsfield" in text and "Popcorn" in text + assert "Shot Board" in text and "MIT" in text + + def test_storyboard_animatic_recipe_names_the_template(self): + text = self._ref("animatic.md") + assert "`templates/animatic`" in text + assert "`templates/generate-speech`" in text + assert "`templates/minimax/music`" in text + + def test_storyboard_maps_boards_to_each_family_s_start_frame(self): + text = skill_text(STORY_SKILL) + for template in ( + "templates/minimax/image-to-video", + "templates/ltx2/image-to-video", + "templates/kandinsky6/image-to-video", + ): + assert f"`{template}`" in text + + @pytest.mark.parametrize("template", ["animatic", "face-drift-check"]) + def test_new_templates_validate(self, template, tmp_path): + from dw.workflow import workflow_from_file + + path = os.path.join(REPO_ROOT, "workflows", "templates", f"{template}.json") + workflow_from_file(path, str(tmp_path)).validate() + + def test_face_drift_check_is_pointed_at_from_the_series_skills(self): + for skill in ("script-to-video", "series-episodes"): + body = skill_body(os.path.join(PLUGIN_DIR, "skills", skill, "SKILL.md")) + assert "`templates/face-drift-check`" in body + + +SITCOM = os.path.join(os.path.dirname(SERIES_SKILL), "references", "sitcom-multicam.md") + + +class TestSitcomMulticam: + """Stage F (#1147): the multicam grammar as a series-episodes reference.""" + + def test_sitcom_reference_names_its_tools(self): + from dw.introspection import list_tasks + + text = _read(SITCOM) + tasks = list_tasks() + for name in ("transcribe_audio", "plan_cuts", "analyze_face_drift"): + assert f"`{name}`" in text, name + assert name in tasks["commands"], name + assert "`templates/minimax/dialogue-short`" in text + assert "`templates/face-drift-check`" in text + assert "`templates/assemble-and-score`" in text + for name in CATALOG_NAME.findall(text): + assert os.path.isfile( + os.path.join(REPO_ROOT, "workflows", name + ".json") + ), name + assert "One-shot Multicam" in text and "MIT" in text + assert "cinematography/references/vocabulary.md" in text + assert "`templates/attribute-lines`" in text and "`attribute_voices`" in text + assert '`timestamps` set to `"segment"`' in text + assert '`timestamps` set to `"word"`' not in text + for field in ("start_frame", "cut_frames", "num_frames", "modulus: 17"): + assert field in text, field + assert "script-to-video/references/shot-document.md" in text + + def test_series_points_at_sitcom(self): + assert "`references/sitcom-multicam.md`" in skill_body(SERIES_SKILL) + + +def _fenced_json(text): + """Every ```json block in `text`, parsed.""" + return [json.loads(block) for block in re.findall(r"```json\n(.*?)```", text, re.S)] + + +CINEMATOGRAPHY_KEYS = ("shot_size", "move", "lens", "light", "duration") + + +class TestShotDocumentShape: + """Final review of #1141: every shot carries the five cinematography keys, + with null for the model's choice, a worked example, and the project name + pattern the prompt library accepts.""" + + def test_example_shot_and_two_shot_array_carry_every_key(self): + blocks = _fenced_json(_read(SHOT_DOCUMENT)) + objects = [block for block in blocks if isinstance(block, dict)] + arrays = [block for block in blocks if isinstance(block, list)] + assert objects and any(len(array) == 2 for array in arrays) + for shot in objects + [shot for array in arrays for shot in array]: + for key in SHOT_KEYS: + assert key in shot, (shot.get("id"), key) + nulls = [ + key + for shot in objects + [s for a in arrays for s in a] + for key in CINEMATOGRAPHY_KEYS + if shot[key] is None + ] + assert nulls, "the example must show an axis left to the model as null" + + def test_null_and_seconds_are_stated(self): + flat = " ".join(_read(SHOT_DOCUMENT).split()) + assert "all five cinematography keys" in flat + assert "`null`" in flat and "model's own choice" in flat + assert "`duration` is a number of seconds, not a vocabulary id" in flat + body = " ".join(skill_body(SCRIPT_SKILL).split()) + assert "`null`" in body and "`duration` (seconds)" in body + assert "ids from `cinematography`, never invented) and `duration`" in body + + def test_project_name_pattern_matches_the_prompt_library(self): + from dw.security import PROMPT_REFERENCE_PATTERN + + segment = PROMPT_REFERENCE_PATTERN[1:].split("(")[0] + assert segment == r"[\w][\w.-]*" + text = _read(SHOT_DOCUMENT) + assert f"`{segment}`" in text + assert "no spaces and no further `/`" in " ".join(text.split()) + pattern = re.compile(PROMPT_REFERENCE_PATTERN) + assert pattern.match("gilded_echoes/shots") + assert not pattern.match("gilded echoes/shots") + assert not pattern.match("series/ep1/shots") + + +class TestStoryboardFinalReview: + """Final review of #1141: names keep_output accepts, one duration per + board in the animatic, and one plugin description.""" + + def test_no_storyboard_keep_output_name_ends_in_png(self): + names = [ + (path, name) + for path in glob.glob(os.path.join(STORY_DIR, "**", "*.md"), recursive=True) + for name in re.findall(r'asset_name="([^"]*)"', _read(path)) + ] + assert len(names) >= 2 + for path, name in names: + assert not name.endswith(".png"), (path, name) + assert "asset:boards/-NN.jpg" in skill_body(STORY_SKILL) + + def test_animatic_takes_one_float_duration_per_board(self, tmp_path): + from dw.variables import argument_errors + + path = os.path.join(REPO_ROOT, "workflows", "templates", "animatic.json") + with open(path, encoding="utf-8") as handle: + definition = json.load(handle) + variables = definition["variables"] + assert variables["durations"] == [2.0] * len(variables["boards"]) + assert isinstance(variables["fps"], float) + assert argument_errors(definition, {"durations": [1, 2, 0.5]}) == [] + assert argument_errors(definition, {"fps": 30}) == [] + assert "one duration per board" in definition["description"] + assert "one number holds every board" not in definition["description"] + + def test_animatic_docs_say_one_duration_per_board(self): + recipe = " ".join( + open(os.path.join(STORY_DIR, "references", "animatic.md"), encoding="utf-8") + .read() + .split() + ) + assert "one duration per board" in recipe + assert "one number for every board" not in recipe + assert "one duration per board" in " ".join(skill_body(STORY_SKILL).split()) + assert "`width`" not in skill_body(STORY_SKILL) + + def test_plugin_and_marketplace_describe_the_plugin_alike(self): + with open( + os.path.join(PLUGIN_DIR, ".claude-plugin", "plugin.json"), encoding="utf-8" + ) as handle: + plugin = json.load(handle) + with open( + os.path.join(REPO_ROOT, ".claude-plugin", "marketplace.json"), + encoding="utf-8", + ) as handle: + marketplace = json.load(handle) + entry = next(p for p in marketplace["plugins"] if p["name"] == "dw") + assert plugin["description"] == entry["description"] + assert "storyboard" in plugin["description"] + + +class TestSitcomAssembly: + """Final review of #1141: the master's dialogue is the score with the + shots' own sound muted, and master spans are cut with trim_video.""" + + def test_master_dialogue_is_the_score_with_world_gain_zero(self): + text = " ".join(_read(SITCOM).split()) + assert "Pass the master as `score`" in text + assert "`world_gain: 0`" in text + path = os.path.join( + REPO_ROOT, "workflows", "templates", "assemble-and-score.json" + ) + with open(path, encoding="utf-8") as handle: + variables = json.load(handle)["variables"] + for name in ("score", "world_gain", "score_start_frame", "total_frames"): + assert name in variables, name + assert f"`{name}" in text, name + + def test_master_spans_are_cut_with_trim_video(self): + text = _read(SITCOM) + steps = [b for b in _fenced_json(text) if isinstance(b, dict)] + trims = [s for s in steps if s.get("task", {}).get("command") == "trim_video"] + assert trims + tasks_doc = _read(os.path.join(REPO_ROOT, "docs", "TASKS.md")) + section = tasks_doc[tasks_doc.index("### trim_video") :] + section = section[: section.index("\n### ", 4)] + for name in trims[0]["task"]["arguments"]: + assert f"| `{name}` |" in section, name + flat = " ".join(text.split()) + assert "planned `start_frame` and `cut_frames`" in flat diff --git a/tests/test_shots.py b/tests/test_shots.py index 242892cc8..5a31a836b 100644 --- a/tests/test_shots.py +++ b/tests/test_shots.py @@ -82,6 +82,7 @@ def audio_video(num_frames, level, fps=4, sample_rate=100): ("dw/tasks/interpolate_frames.py", "interpolate_frames"): ("rescales", 1), ("dw/tasks/pair_audio.py", "pair_audio"): ("remeasures", 1), ("dw/tasks/video_utils.py", "_decode_audio_video"): ("none", 1), + ("dw/tasks/video_utils.py", "stills_to_video"): ("populates", 1), ("dw/output_extraction.py", "pair_audio_with_frames"): ("none", 1), ("dw/tasks/h3_latent_upscale.py", "decode_h3_latents"): ("none", 1), ("dw/tasks/face_track.py", "crop_face_track"): ("none", 1), diff --git a/tests/test_step_cache.py b/tests/test_step_cache.py index cd3c26063..a62d34aaf 100644 --- a/tests/test_step_cache.py +++ b/tests/test_step_cache.py @@ -623,6 +623,7 @@ def test_normalized_downstream_false_when_nothing_references_it(): # moved when it defaulted to sparse instead of null (#931): the backend is part of # the pipeline's identity, sparse_tau is not. TEMPLATE_PIPELINE_KEYS = { + "animatic.json": {}, "assemble-and-score.json": {}, "attention-processor.json": { "generate_with_sdpa": "8dc3e92052c445d5dbf20bf6e8b6fe91794cfa3772bb7366e2390889b4e99459" @@ -665,6 +666,7 @@ def test_normalized_downstream_false_when_nothing_references_it(): "generate": "bfe2e09d70b6bf6e7799266cf735abd779e5a9e3c283b8287d962eab17d95425" }, "expand-prompt.json": {}, + "face-drift-check.json": {}, "flux3/image-edit.json": { "edit": "58be5755e37a15b2780cc4fa3eead66e91e2eeec21e15f15624f60a9a072f235" }, diff --git a/tests/test_stills_to_video.py b/tests/test_stills_to_video.py new file mode 100644 index 000000000..e6f23a72c --- /dev/null +++ b/tests/test_stills_to_video.py @@ -0,0 +1,232 @@ +"""stills_to_video: storyboard stills held for a duration into a silent clip (#1144).""" + +import numpy +import pytest +from PIL import Image + +from dw.media_types import AudioVideo +from dw.tasks.video_utils import MAX_STILLS_FPS, MAX_STILLS_SECONDS, stills_to_video + + +def still(width=512, height=512, color=(255, 0, 0)): + return Image.new("RGB", (width, height), color) + + +class TestHold: + def test_three_stills_hold_for_their_durations(self): + video = stills_to_video([still(), still(), still()], [1, 2, 0.5], fps=24) + assert isinstance(video, AudioVideo) + assert video.frames.shape == (84, 512, 512, 3) + assert video.frames.dtype == numpy.uint8 + assert video.fps == 24 + assert [shot["start_frame"] for shot in video.shots] == [0, 24, 72] + assert [shot["num_frames"] for shot in video.shots] == [24, 48, 12] + + def test_each_still_fills_its_own_frames(self): + video = stills_to_video( + [still(color=(255, 0, 0)), still(color=(0, 0, 255))], [1, 1], fps=2 + ) + assert tuple(video.frames[0, 0, 0]) == (255, 0, 0) + assert tuple(video.frames[3, 0, 0]) == (0, 0, 255) + + def test_one_duration_applies_to_all(self): + video = stills_to_video([still(), still()], 2, fps=12) + assert len(video.frames) == 48 + + def test_a_sub_frame_duration_still_gets_one_frame(self): + video = stills_to_video([still()], [0.01], fps=24) + assert len(video.frames) == 1 + + def test_arrays_are_accepted(self): + array = numpy.zeros((64, 64, 3), dtype=numpy.uint8) + video = stills_to_video([array], 1, fps=4) + assert video.frames.shape == (4, 64, 64, 3) + + def test_silent_clip_carries_no_audio(self): + video = stills_to_video([still()], 1, fps=4) + assert video.audio is None + assert video.sample_rate is None + + +class TestRefusals: + def test_mismatched_sizes_refused(self): + with pytest.raises(ValueError, match=r"'images' must all.*512x512.*512x768"): + stills_to_video([still(512, 512), still(512, 768)], 1) + + def test_wrong_duration_count_refused(self): + with pytest.raises(ValueError, match="durations"): + stills_to_video([still(), still(), still()], [1, 2]) + + @pytest.mark.parametrize("durations", [[1, 0], [-1], 0]) + def test_non_positive_duration_refused(self, durations): + with pytest.raises(ValueError, match="durations"): + stills_to_video( + [still(), still()][ + : len(durations) if isinstance(durations, list) else 1 + ], + durations, + ) + + def test_fps_bounds(self): + for bad in (0, 121): + with pytest.raises(ValueError, match="fps"): + stills_to_video([still()], 1, fps=bad) + assert MAX_STILLS_FPS == 120 + assert len(stills_to_video([still()], 1, fps=120).frames) == 120 + + def test_total_over_600_seconds_refused(self): + assert MAX_STILLS_SECONDS == 600 + with pytest.raises(ValueError, match="600"): + stills_to_video([still()], [601], fps=1) + + def test_no_images_refused(self): + with pytest.raises(ValueError, match="images"): + stills_to_video([], 1) + + +class TestWithConcat: + def test_result_concats_with_concat_videos(self): + from dw.tasks.concat_videos import concat_videos + + video = stills_to_video([still(64, 64)], 1, fps=4) + joined = concat_videos([video, video], fps=4) + assert len(joined.frames) == 2 * len(video.frames) + + def test_silent_clip_saves_as_a_video(self, tmp_path): + from dw.result import Result + + video = stills_to_video([still(64, 64)], 1, fps=4) + result = Result({"content_type": "video/mp4"}) + result.add_result([video]) + result.save(str(tmp_path), "silent") + files = list(tmp_path.glob("silent*.mp4")) + assert files and files[0].stat().st_size > 0 + + +class TestRegistered: + def test_command_registered(self): + from dw.tasks.task import _COMMAND_REGISTRY + + assert "stills_to_video" in _COMMAND_REGISTRY + + +class TestStaticValidation: + def _errors(self, **arguments): + from dw.task_problems import stills_to_video_errors + + return stills_to_video_errors(arguments) + + def test_literal_bad_duration_found(self): + assert [a for a, _ in self._errors(images=["a"], durations=[1, 0])] == [ + "durations" + ] + + def test_fps_out_of_range_found(self): + assert [a for a, _ in self._errors(durations=1, fps=121)] == ["fps"] + + def test_count_mismatch_found(self): + errors = self._errors(images=["a", "b", "c"], durations=[1, 2]) + assert [a for a, _ in errors] == ["durations"] + + def test_total_found(self): + assert self._errors(durations=[601]) + + def test_references_are_left_alone(self): + assert not self._errors(images="previous_result:x", durations="variable:d") + + def test_clean_arguments_pass(self): + assert not self._errors(images=["a", "b"], durations=[1, 2], fps=24) + + +class TestHandlerLoadsLocations: + def test_handler_reads_image_dicts(self, tmp_path): + from dw.tasks.task import _COMMAND_REGISTRY + + path = tmp_path / "a.png" + still(32, 32).save(path) + handler = _COMMAND_REGISTRY["stills_to_video"] + video = handler( + None, + { + "images": [{"media_type": "image", "location": str(path)}], + "durations": 1, + "fps": 4, + }, + {}, + ) + assert video.frames.shape == (4, 32, 32, 3) + + +class TestByteBudget: + def test_over_budget_refused_before_allocating(self, monkeypatch): + monkeypatch.setattr("dw.tasks.video_utils.MAX_STILLS_BYTES", 1000) + with pytest.raises(ValueError) as error: + stills_to_video([still(16, 16)], [1], fps=24) + message = str(error.value) + assert "durations" in message and "fps" in message and "16x16" in message + + def test_within_budget_passes(self, monkeypatch): + monkeypatch.setattr("dw.tasks.video_utils.MAX_STILLS_BYTES", 4 * 16 * 16 * 3) + assert len(stills_to_video([still(16, 16)], [1], fps=4).frames) == 4 + + def test_many_tiny_durations_cap_the_frame_count(self): + from dw.task_problems import MAX_STILLS_FRAMES, stills_to_video_problems + + durations = [0.001] * (MAX_STILLS_FRAMES + 1) + problems = stills_to_video_problems(durations, 24, len(durations)) + assert [a for a, _ in problems] == ["durations"] + assert "frame" in problems[0][1] + with pytest.raises(ValueError, match="frame"): + stills_to_video([still(2, 2)] * len(durations), durations, fps=24) + + def test_frame_cap_applies_to_a_scalar_duration(self): + from dw.task_problems import MAX_STILLS_FRAMES, stills_to_video_problems + + count = MAX_STILLS_FRAMES + 1 + assert stills_to_video_problems(0.001, 24, count) + + +class TestDurationsShapes: + def test_numpy_durations(self): + video = stills_to_video( + [still(16, 16), still(16, 16)], numpy.array([1, 2]), fps=4 + ) + assert [s["num_frames"] for s in video.shots] == [4, 8] + + def test_empty_durations_refused(self): + with pytest.raises(ValueError, match="durations"): + stills_to_video([still(16, 16)], [], fps=4) + + +class TestHandlerEntries: + def _run(self, images): + from dw.tasks.task import _COMMAND_REGISTRY + + return _COMMAND_REGISTRY["stills_to_video"]( + None, {"images": images, "durations": 1, "fps": 2}, {} + ) + + def test_an_entry_of_several_images_is_flattened_in_order(self): + red, blue = still(16, 16, (255, 0, 0)), still(16, 16, (0, 0, 255)) + video = self._run([[red, blue], still(16, 16, (0, 255, 0))]) + assert len(video.shots) == 3 + assert tuple(video.frames[0, 0, 0]) == (255, 0, 0) + assert tuple(video.frames[2, 0, 0]) == (0, 0, 255) + + def test_an_entry_of_no_images_is_refused(self): + with pytest.raises(ValueError, match="images"): + self._run([[]]) + + +class TestSavedManifest: + def test_saved_shots_start_frames(self, tmp_path): + from dw.result import Result + from dw.shots import step_shots + + video = stills_to_video([still(16, 16)] * 3, [1, 2, 0.5], fps=24) + result = Result({"content_type": "video/mp4", "save": True}) + result.add_result(video) + saved = result.save(str(tmp_path), "animatic.0") + assert list(tmp_path.glob("animatic.0*.mp4")) + shots = step_shots(result.saved_shots, saved) + assert [shot["start_frame"] for shot in shots] == [0, 24, 72] diff --git a/tests/test_updater.py b/tests/test_updater.py index 519696642..4d0636fec 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -71,6 +71,15 @@ def test_revert_ignores_commit(self): assert "git+" not in commands[0][-1] assert commands[0][-1] == f"{PYPI_PACKAGE}=={release_floor()}" + def test_python_names_the_interpreter(self): + """The container entrypoint installs into its overlay venv, not the + interpreter it runs under.""" + for commands in ( + build_pip_commands(python="/data/env/venv/bin/python"), + build_pip_commands(revert=True, python="/data/env/venv/bin/python"), + ): + assert all(c[0] == "/data/env/venv/bin/python" for c in commands) + class TestReleaseFloor: def test_matches_the_pin_in_pyproject_toml(self): diff --git a/ui/src/lib/editor/FlowView.svelte b/ui/src/lib/editor/FlowView.svelte index 3352aee39..8f553581c 100644 --- a/ui/src/lib/editor/FlowView.svelte +++ b/ui/src/lib/editor/FlowView.svelte @@ -20,23 +20,36 @@ let { workflow, onselect = undefined, + selectable = undefined, + selectHint = 'click a step to open it in the form', activeStep = undefined, + failedStep = undefined, doneSteps = [], activeMember = undefined, + failedMember = undefined, doneMembers = [], nodeTimes = {}, memberTimes = {}, }: { workflow: Record onselect?: (stepName: string) => void + /** Which steps onselect has somewhere to take the reader - every step + * when not given. A step it rules out is drawn as no control at all. */ + selectable?: (stepName: string) => boolean + /** What a click on a step does, for the legend. */ + selectHint?: string /** The step a run is executing right now, when the graph is showing a * live job rather than a workflow being edited. */ activeStep?: string + /** The step a failed run stopped in. */ + failedStep?: string /** Steps that run has already finished. */ doneSteps?: string[] /** The for_each member (`group@entry`) the run is on right now - a * finer grain than activeStep, which names the group. */ activeMember?: string + /** The for_each member a failed run stopped in. */ + failedMember?: string /** The for_each members that run has already finished, engine names. */ doneMembers?: string[] /** Seconds each finished node took, from the run's own event clocks. */ @@ -45,15 +58,46 @@ memberTimes?: Record } = $props() - const showsRun = $derived(activeStep !== undefined || doneSteps.length > 0) + const showsRun = $derived( + activeStep !== undefined || + failedStep !== undefined || + doneSteps.length > 0, + ) + // On a run, a step that is none of running, failed or finished has not + // run yet - drawn faded, so what the run got through reads at a glance const stateOf = $derived((name: string) => - name === activeStep ? 'active' : doneSteps.includes(name) ? 'done' : '', + name === activeStep + ? 'active' + : name === failedStep + ? 'failed' + : doneSteps.includes(name) + ? 'done' + : showsRun + ? 'pending' + : '', ) const memberStateOf = $derived((full: string) => - full === activeMember ? 'active' : doneMembers.includes(full) ? 'done' : '', + full === activeMember + ? 'active' + : full === failedMember + ? 'failed' + : doneMembers.includes(full) + ? 'done' + : '', ) const graph = $derived(dataFlowGraph(workflow)) + const hasForEach = $derived(graph.nodes.some((node) => node.forEach)) + const hasPending = $derived( + graph.nodes.some((node) => stateOf(node.name) === 'pending'), + ) + // One plain step has no flow to draw: it is shown as a line, not a canvas + const lone = $derived( + graph.nodes.length === 1 && !graph.nodes[0].forEach + ? graph.nodes[0] + : undefined, + ) + const canSelect = (name: string) => !!onselect && (selectable?.(name) ?? true) /** The parts of a node's labels that did not fit, in full, for its tooltip. */ function overflowTitle(node: FlowNode): string { @@ -97,7 +141,7 @@ // the run - a node is not a control: no focus stop, no pointer, nothing // announced as a button. const nodeAttributes = (name: string) => - onselect + canSelect(name) ? { role: 'button', tabindex: 0, @@ -109,20 +153,78 @@ {#if (graph.nodes ?? []).length === 0}

No steps yet.

+{:else if lone} + {@const state = stateOf(lone.name)} +
+
+ + {lone.name} + {kindLabel(lone.kind)} + {#if lone.detail}{lone.detail}{/if} + + {#if state === 'done' && nodeTimes[lone.name] !== undefined} + {formatDuration(nodeTimes[lone.name])} + {:else if state === 'active' || state === 'failed'} + {state === 'active' ? 'running' : 'failed'} + {/if} +
+
{:else} -
-

- Read-only data-flow view: boxes are steps, arrows are - previous_result references labeled with the argument they - feed. A step with more than one incoming arrow multiplies its inputs - together (docs/WORKFLOW_GUIDE.md, "Several previous_result references - multiply") - its border is highlighted and the multiplier is noted. A step - carrying - for_each shows the entries it runs inset.{#if showsRun} - A finished step is outlined in green, the one running now in amber - colour.{/if}{#if onselect} - Click a step to jump to it in the form view.{/if} -

+
+ +
    +
  • + entry +
  • + {#if graph.edges.length} +
  • + previous_result +
  • + {/if} + {#if graph.fanIn.size} +
  • + ×inputs multiply +
  • + {/if} + {#if hasForEach} +
  • + for_each entries +
  • + {/if} + {#if doneSteps.length} +
  • + finished +
  • + {/if} + {#if activeStep !== undefined} +
  • + running +
  • + {/if} + {#if failedStep !== undefined} +
  • + failed +
  • + {/if} + {#if hasPending} +
  • + not run +
  • + {/if} + {#if onselect} +
  • {selectHint}
  • + {/if} +
BOX_H @@ -228,6 +332,7 @@ class="member" class:done={state === 'done'} class:active={state === 'active'} + class:failed={state === 'failed'} aria-label={`member ${full}${state ? ', ' + state : ''}`} > {full} @@ -277,7 +382,7 @@ {/each}
-
+ {/if}