From 81593c4be25e28deb5ad2239476b9b4060722bba Mon Sep 17 00:00:00 2001 From: cjkindel Date: Mon, 3 Aug 2026 09:33:37 -0700 Subject: [PATCH 1/3] initial pre-merge minimax h3 node --- docs/nodes/create-noise-latents.md | 6 + docs/nodes/decode_media_latent.md | 23 +- docs/nodes/encode_media_latent.md | 6 + docs/nodes/generate_media_latents.md | 1 + docs/nodes/pipeline_builder.md | 8 +- griptape-nodes-library.json | 2 +- .../latent_pipeline_drivers/driver_factory.py | 2 + .../latent_pipeline_drivers/minimax_h3.py | 570 ++++++++++++++++++ .../nodes/decode_hdr_node.py | 12 +- .../nodes/vae_decoder.py | 26 +- .../parameters/pipelinetype_parameters.py | 28 + .../parameters/providers.py | 1 + .../minimax_h3_runtime_parameters.py | 175 ++++++ .../runtime_params_registry.py | 4 + .../minimax_h3_parameters.py | 92 +++ pyproject.toml | 4 +- uv.lock | 63 +- 17 files changed, 981 insertions(+), 42 deletions(-) create mode 100644 modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py create mode 100644 modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py create mode 100644 modular_diffusion_nodes_library/standard_parameters/minimax_h3_parameters.py diff --git a/docs/nodes/create-noise-latents.md b/docs/nodes/create-noise-latents.md index 09509e2..41de342 100644 --- a/docs/nodes/create-noise-latents.md +++ b/docs/nodes/create-noise-latents.md @@ -41,6 +41,12 @@ Pipeline Builder → [Create Noise Latents] → Generate Media Latents → Decod | `seed` | int | random | Reproducibility. | | `num_inference_steps` | int | `20` | **Only shown for SDXL** — used by SDXL to scale the initial noise. Other pipelines ignore this. | +## Provider / model behavior + +| Provider | Behavior | +| --- | --- | +| MiniMax-H3 | This node's `width`, `height` and `num_frames` are authoritative — [Generate Media Latents](generate_media_latents.md) takes its dimensions from the latent and warns if its own `num_frames` disagrees. `width` and `height` must be multiples of **32**; MiniMax-H3's trained canvas is a 768-pixel short edge (e.g. 1344x768, or 960x544 for roughly 2.3x faster steps). `num_frames` is snapped up to the next `17 * n + 5` (124, 141, 158, … 345) and the result must land between 5 and 15 seconds at the fixed 24 fps. The noise latent also carries the audio noise for the jointly generated soundtrack. | + ## Tips & pitfalls - **`width` / `height` must respect VAE divisibility.** Most VAEs require multiples of 8 or 16. Pick standard dimensions (512, 768, 1024, …) to keep shapes valid. diff --git a/docs/nodes/decode_media_latent.md b/docs/nodes/decode_media_latent.md index a7e3045..00940bf 100644 --- a/docs/nodes/decode_media_latent.md +++ b/docs/nodes/decode_media_latent.md @@ -5,8 +5,9 @@ Category: `ModularDiffusion/Encode\Decode` ## TL;DR -- Output is **dynamic**: `output_image` for image pipelines, `output_video` (+ `fps`) for video pipelines (LTX, LTX2, WAN). It swaps automatically when you connect a `pipeline`. +- Output is **dynamic**: `output_image` for image pipelines, `output_video` (+ `fps`) for video pipelines (LTX, LTX2, WAN, HunyuanVideo 1.5, MiniMax-H3). It swaps automatically when you connect a `pipeline`. - Almost always the last node in the flow. Connect to a Save Image / Save Video node downstream. +- **MiniMax-H3 videos come out with sound.** The soundtrack is generated jointly with the picture and muxed into the same MP4 here. Connect Generate Media Latents **directly** to this node. ## Typical workflow position ```text @@ -37,6 +38,26 @@ Generate Media Latents → [Decode Media Latent] → Save Image / Save Video | --- | --- | --- | --- | | `fps` | int (1–120) | `25` | Output frame rate. **Only shown for video pipelines.** | +## Provider / model behavior + +| Provider | Behavior | +| --- | --- | +| Image pipelines | `output_image` as an `ImageArtifact`. | +| LTX, LTX2, WAN, HunyuanVideo 1.5 | `output_video` as a silent MP4 at `fps`. | +| MiniMax-H3 | `output_video` as an MP4 **with an audio track**. Video and audio are generated jointly by one denoising loop, and are muxed together here. Leave `fps` at `24` — MiniMax-H3 generates at a fixed 24 fps, and any other value desynchronises the soundtrack. | + +### MiniMax-H3: keep the edge direct + +MiniMax-H3's audio latent travels in the latent's *metadata*, not in its tensor. Only a direct +`Generate Media Latents → Decode Media Latent` edge preserves it: + +- **Empty Latents, Save/Load Latent Tensor** drop the audio metadata. Decoding still works and + produces a silent video, with a warning in the logs. +- **Add / Subtract / Multiply Latents, Latents Composite Mask, Latent Upsampler** are worse: they + change the video latent but carry the *old* audio latent through unchanged. This node detects that + mismatch and **fails with an error** rather than muxing a soundtrack that no longer matches the + picture. + ## Tips & pitfalls - **Use the same pipeline that produced the latent.** Each pipeline carries the VAE it was trained with — decoding a latent with a mismatched VAE produces corrupt output. diff --git a/docs/nodes/encode_media_latent.md b/docs/nodes/encode_media_latent.md index e108973..552d9b2 100644 --- a/docs/nodes/encode_media_latent.md +++ b/docs/nodes/encode_media_latent.md @@ -35,6 +35,12 @@ Load Image → [Encode Media Latent] → Generate Media Latents → Decode Media | --- | --- | --- | | `latent_tensor` | `LatentArtifact` | Encoded latent in the pipeline's canonical latent space. | +## Provider / model behavior + +| Provider | Behavior | +| --- | --- | +| MiniMax-H3 | **Not supported.** MiniMax-H3 has no general VAE-encode path — its only encoder is keyframe-specific and produces conditioning rows rather than a reusable latent. This node raises an error. For keyframe conditioning, connect a [Media Gen Conditioning](media_gen_conditioning.md) node to `conditioning_images` on [Generate Media Latents](generate_media_latents.md) instead. | + ## Tips & pitfalls - **The input slot adapts to the connected pipeline.** Image pipelines (Flux, SD3, etc.) show an `image` input; video pipelines (LTX, LTX2, WAN, etc.) show `input_video` instead. Switching pipeline types replaces the slot — rewire the input after switching. diff --git a/docs/nodes/generate_media_latents.md b/docs/nodes/generate_media_latents.md index 880f4e2..bc20123 100644 --- a/docs/nodes/generate_media_latents.md +++ b/docs/nodes/generate_media_latents.md @@ -63,6 +63,7 @@ The exact list depends on the connected pipeline. Common parameters: - **ControlNet:** when `pipeline` is a `ControlNetDiffusionPipelineArtifact`, the `controlnet_parameters` input is added automatically. - **Inpainting:** when `input_latent` is an `InpaintMaskArtifact` (from [Encode Masked Media Latent](encode_masked_media_latent.md)), the node automatically routes through the inpaint pipeline class and uses the artifact's `strength`. +- **MiniMax-H3:** generates the video **and its soundtrack** in one denoising loop. There is no `guidance_scale` and no `negative_prompt` — the checkpoint is guidance-distilled, so guidance is baked into the weights and every step runs a single forward pass. Keyframes are optional: leave `conditioning_images` unconnected for text-only generation, or connect a [Media Gen Conditioning](media_gen_conditioning.md) node for a first and/or last frame. The output latent carries the audio in its metadata, so wire this node **directly** to [Decode Media Latent](decode_media_latent.md). `return_fully_denoised` is not supported. Inpainting, ControlNet, image-to-video and video-to-video are unavailable. ## Tips & pitfalls diff --git a/docs/nodes/pipeline_builder.md b/docs/nodes/pipeline_builder.md index 95e08d3..1a5db61 100644 --- a/docs/nodes/pipeline_builder.md +++ b/docs/nodes/pipeline_builder.md @@ -38,7 +38,7 @@ Category: `ModularDiffusion/Pipeline` | Name | Type | Notes | | --- | --- | --- | -| `provider` | choice | `Flux`, `Flux2`, `Stable Diffusion`, `Stable Diffusion 3`, `Qwen`, `Z-Image`, `HunyuanVideo 1.5`, `LTX`, `LTX2`, `WAN`. Changing this swaps every parameter below. | +| `provider` | choice | `Flux`, `Flux2`, `Stable Diffusion`, `Stable Diffusion 3`, `Qwen`, `Z-Image`, `HunyuanVideo 1.5`, `LTX`, `LTX2`, `MiniMax-H3`, `WAN`. Changing this swaps every parameter below. | | `pipeline_type` | choice | Per-provider pipeline class (e.g. `FluxPipeline`, `WanImageToVideoPipeline`). Determines what the pipeline can do. | | `` | HF repo picker | Hugging Face repo ID. Diffusers-format only — single-file `.safetensors` checkpoints are not loaded directly. | @@ -55,6 +55,12 @@ Category: `ModularDiffusion/Pipeline` Enable only what you need — each option trades speed for memory. +### Provider / model behavior + +| Provider | Behavior | +| --- | --- | +| MiniMax-H3 | The **Memory optimization** knobs above are ignored. MiniMax-H3 is a Modular Diffusers pipeline whose transformer (61.7 GB in bfloat16) and Qwen3-VL conditioner (62.1 GB) cannot be placed by the post-load optimizer, so the builder loads it in bfloat16 and registers the components for automatic CPU offload instead. Expect a single 80 GB accelerator plus ample host RAM. LoRAs are not supported. | + ## Tips & pitfalls - **Pipeline cache after restart.** The cache lives in process memory only; the node re-resolves automatically on the next run. diff --git a/griptape-nodes-library.json b/griptape-nodes-library.json index d2cf8bb..7ea4dc8 100644 --- a/griptape-nodes-library.json +++ b/griptape-nodes-library.json @@ -41,7 +41,7 @@ "beautifulsoup4>=4.13.4", "controlnet-aux>=0.0.9", "static-ffmpeg>=2.8", - "diffusers==0.39.0", + "diffusers @ git+https://github.com/huggingface/diffusers.git@abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc", "imageio[ffmpeg]>=2.37.2", "ninja>=1.13.0", "numpy>=2.2.4", diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py index ada9cce..f7c091f 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/driver_factory.py @@ -16,6 +16,7 @@ ) from modular_diffusion_nodes_library.latent_pipeline_drivers.ltx import LTXLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.ltx2 import LTX2PipelineDriver +from modular_diffusion_nodes_library.latent_pipeline_drivers.minimax_h3 import MiniMaxH3LatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.qwen import QwenLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.qwen_edit import QwenEditLatentPipelineDriver from modular_diffusion_nodes_library.latent_pipeline_drivers.stable_diffusion_3 import ( @@ -40,6 +41,7 @@ "Flux2Pipeline": Flux2LatentPipelineDriver, "Flux2KleinPipeline": Flux2KleinLatentPipelineDriver, "LTX2Pipeline": LTX2PipelineDriver, + "MiniMaxH3ModularPipeline": MiniMaxH3LatentPipelineDriver, "QwenImagePipeline": QwenLatentPipelineDriver, "QwenImageEditPipeline": QwenEditLatentPipelineDriver, "StableDiffusion3Pipeline": StableDiffusion3LatentPipelineDriver, diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py new file mode 100644 index 0000000..016ac05 --- /dev/null +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py @@ -0,0 +1,570 @@ +"""MiniMax-H3 driver: joint video + audio generation from a Modular-Diffusers-only integration. + +MiniMax-H3 is the first model in this library with no ``DiffusionPipeline`` half, so this driver +owns the denoise loop instead of delegating to ``super().denoise_latent()``. That is what +``base_driver.denoise_latent`` asks for when it sees a ``ModularPipeline``. + +Owning the loop is what makes partial denoise, live preview and mid-run cancellation available here +even though HunyuanVideo 1.5 (whose loop is sealed inside diffusers) cannot offer the latter two. + +Two latent streams share one public artifact: the video latent is the artifact's tensor, and the +audio latent rides in this driver's namespaced ``meta`` sub-bag. That channel is only safe on a +direct Generate -> Decode edge. Nodes that rebuild meta from scratch (Empty Latent, save/load) drop +the audio, and latent math is worse: it sums the video latent while keeping the left operand's +*unsummed* audio, so the audio is present but no longer matches the picture. ``decode_latent`` +therefore checks a fingerprint of the video latent the audio was paired with, and refuses rather +than muxing a desynchronised soundtrack. +""" + +import logging +import math +from typing import Any, ClassVar, cast, override + +import torch # type: ignore[reportMissingImports] +from diffusers.modular_pipelines.minimax_h3.denoise import ( # type: ignore[reportMissingImports] + MiniMaxH3DenoiseLoopWrapper, + MiniMaxH3LoopDenoiser, + MiniMaxH3LoopSchedulerStep, +) +from diffusers.modular_pipelines.minimax_h3.modular_pipeline import ( # type: ignore[reportMissingImports] + MiniMaxH3ModularPipeline, +) +from diffusers.modular_pipelines.minimax_h3.packing import ( # type: ignore[reportMissingImports] + MINIMAX_H3_AUDIO_CHANNELS, + MINIMAX_H3_FPS, + align_num_frames, + audio_latent_num_frames, + patchify_video_latents, + unpack_audio_tokens, + unpatchify_video_tokens, + video_latent_num_frames, +) +from diffusers.modular_pipelines.modular_pipeline import ( # type: ignore[reportMissingImports] + ModularPipeline, + ModularPipelineBlocks, + PipelineState, + SequentialPipelineBlocks, +) +from diffusers.modular_pipelines.modular_pipeline_utils import ( # type: ignore[reportMissingImports] + InputParam, + OutputParam, +) +from diffusers.pipelines.pipeline_utils import DiffusionPipeline # type: ignore[reportMissingImports] +from diffusers.utils.torch_utils import randn_tensor # type: ignore[reportMissingImports] + +from modular_diffusion_nodes_library.artifact_utils.inpaint_mask_artifact import InpaintMaskArtifact +from modular_diffusion_nodes_library.artifact_utils.latent_artifact import LatentArtifact +from modular_diffusion_nodes_library.latent_pipeline_drivers.base_driver import LatentPipelineDriver +from modular_diffusion_nodes_library.latent_pipeline_drivers.driver_types import ( + DecodeResult, + GeneratorState, + ImageMedia, + VideoMedia, + read_driver_meta, +) +from modular_diffusion_nodes_library.parameters.media_gen_conditioning.conditioning_payload import ( + normalize_to_payloads, +) +from modular_diffusion_nodes_library.utils.conditioning_utils import ( + ConditioningMode, + MediaGenConditioningKey, + resolve_conditioning_image, + resolve_frame_index, +) + +logger = logging.getLogger("modular_diffusers_nodes_library") + +#: Key under which the audio latent rides in this driver's namespaced ``meta`` sub-bag. +AUDIO_LATENTS_META_KEY = "audio_latents" + +#: Fingerprint of the video latent the audio latent was denoised with. Latent math merges meta +#: left-operand-wins over a *shallow* copy, so a summed video latent keeps the left operand's +#: unsummed audio: the audio is still present but no longer corresponds to the video. Comparing +#: fingerprints at decode time turns that from a silently desynchronised soundtrack into an error. +AUDIO_PAIRED_WITH_META_KEY = "audio_paired_with" + + +def _video_fingerprint(tensor: torch.Tensor) -> tuple[tuple[int, ...], float, float]: + """Cheap value-sensitive fingerprint of a video latent. + + Shape alone would not do: latent math preserves shape and changes only values. + """ + flat = tensor.detach().to(device="cpu", dtype=torch.float64) + return (tuple(tensor.shape), float(flat.sum()), float(flat.square().sum())) + + +def _fingerprints_match( + left: tuple[tuple[int, ...], float, float] | None, + right: tuple[tuple[int, ...], float, float], +) -> bool: + if left is None: + return False + if tuple(left[0]) != tuple(right[0]): + return False + return math.isclose(left[1], right[1], rel_tol=1e-9, abs_tol=1e-6) and math.isclose( + left[2], right[2], rel_tol=1e-9, abs_tol=1e-6 + ) + + +class _MiniMaxH3PrepareNoiseStep(ModularPipelineBlocks): + """Draw the video and audio noise, leaving the video latent unpacked. + + Upstream's ``MiniMaxH3PrepareLatentsStep`` returns patchified rows, which the public latent + surface forbids. This draws the same two tensors in the same order — video then audio, which is + what the request generator reproduces — and returns the video one as a 5-D latent. + """ + + model_name = "minimax-h3" + + @property + def inputs(self) -> list[InputParam]: + return [ + InputParam("num_latent_frames", required=True), + InputParam("latent_height", required=True), + InputParam("latent_width", required=True), + InputParam("num_audio_latents", required=True), + InputParam("generator"), + ] + + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam( + "latents", + type_hint=torch.Tensor, + description="Unpacked video noise of shape (1, C, num_latent_frames, latent_height, latent_width).", + ), + OutputParam( + "audio_latents", + type_hint=torch.Tensor, + description="Audio noise of shape (2, audio_latent_channels, num_audio_latents).", + ), + ] + + @torch.no_grad() + def __call__( + self, components: MiniMaxH3ModularPipeline, state: PipelineState + ) -> tuple[MiniMaxH3ModularPipeline, PipelineState]: + block_state = cast(Any, self.get_block_state(state)) + device = components._execution_device + + block_state.latents = randn_tensor( + ( + 1, + components.vae_latent_channels, + block_state.num_latent_frames, + block_state.latent_height, + block_state.latent_width, + ), + generator=block_state.generator, + device=device, + dtype=torch.float32, + ) + # Drawn in row layout upstream, then reshaped to the (2, C, N) shape `prepare_latents` + # accepts back, so the generator sees the same draw either way. + audio_rows = randn_tensor( + (block_state.num_audio_latents * MINIMAX_H3_AUDIO_CHANNELS, components.audio_latent_channels), + generator=block_state.generator, + device=device, + dtype=torch.float32, + ) + block_state.audio_latents = audio_rows.reshape( + MINIMAX_H3_AUDIO_CHANNELS, block_state.num_audio_latents, components.audio_latent_channels + ).permute(0, 2, 1) + + self.set_block_state(state, block_state) + return components, state + + +class _MiniMaxH3CallbackDenoiseStep(MiniMaxH3DenoiseLoopWrapper): + """``MiniMaxH3DenoiseStep`` plus partial denoise, a step-end callback and an interrupt break. + + ``block_classes`` / ``block_names`` are re-declared rather than inherited because + ``LoopSequentialPipelineBlocks.__init__`` builds ``sub_blocks`` from them. + """ + + block_classes: ClassVar[list[type]] = [MiniMaxH3LoopDenoiser, MiniMaxH3LoopSchedulerStep] + block_names: ClassVar[list[str]] = ["denoiser", "update"] + + #: Assigned per-instance after construction. A block's ``__init__`` takes no arguments and its + #: signature is introspected for config, so these must not become constructor kwargs. + callback: Any = None + start_step: int = 0 + end_step: int = -1 + + @property + def description(self) -> str: + return ( + "Runs the MiniMax-H3 denoising loop over an optional step window, with a step-end " + "callback and cancellation support." + ) + + def _resolve_window(self, num_steps: int) -> tuple[int, int]: + """Clamp the requested step window against the schedule's real length. + + ``num_inference_steps`` counts sigma grid points including the terminal zero, so the + schedule drives one fewer model evaluation than the UI number. ``end_step`` equal to the UI + value is therefore normal and must clamp rather than raise. + """ + if self.start_step < 0: + raise ValueError(f"start_step must be non-negative, got {self.start_step}.") + if self.end_step != -1 and self.start_step >= self.end_step: + raise ValueError(f"start_step must be less than end_step, got {self.start_step} and {self.end_step}.") + + begin = min(self.start_step, num_steps - 1) + if self.end_step == -1: + end = num_steps + else: + end = min(max(self.end_step, begin + 1), num_steps) + return begin, end + + @torch.no_grad() + def __call__( + self, components: MiniMaxH3ModularPipeline, state: PipelineState + ) -> tuple[MiniMaxH3ModularPipeline, PipelineState]: + block_state = self.get_block_state(state) + + if len(block_state.audio_timesteps) != len(block_state.timesteps): + raise ValueError( + f"MiniMax-H3's video and audio schedules must have equal length, got " + f"{len(block_state.timesteps)} and {len(block_state.audio_timesteps)}." + ) + + begin, end = self._resolve_window(len(block_state.timesteps)) + # Slice all three per-step arrays by the identical window. The scheduler step block indexes + # audio_timesteps[i] and row_timestep_plan[i] with the loop counter over the sliced + # timesteps, so a shared window is what keeps video and audio in lockstep. + block_state.timesteps = block_state.timesteps[begin:end] + block_state.audio_timesteps = block_state.audio_timesteps[begin:end] + block_state.row_timestep_plan = block_state.row_timestep_plan[begin:end] + + # The schedulers' own sigma grids stay whole: step() needs sigmas[i + 1], and + # index_for_timestep searches the unsliced timesteps, so slicing either would desynchronise + # them. set_begin_index pins where in the whole grid this window starts, and MUST run after + # the set_timesteps block, which resets it to None. + components.scheduler.set_begin_index(begin) + components.audio_scheduler.set_begin_index(begin) + + with self.progress_bar(total=len(block_state.timesteps)) as progress_bar: + for i, t in enumerate(block_state.timesteps): + components, block_state = self.loop_step(components, block_state, i=i, t=t) + progress_bar.update() + if self.callback is not None: + # The return value is deliberately discarded: the framework's callback returns + # {} on its normal path, and merging that would clobber the loop's latents. + self.callback(components, i, t, {"latents": block_state.latents}) + if getattr(components, "_interrupt", False): + break + self.set_block_state(state, block_state) + return components, state + + +class MiniMaxH3LatentPipelineDriver(LatentPipelineDriver): + produces_video: ClassVar[bool] = True + # MiniMax-H3 generates at a fixed 24 fps; any other rate desynchronises the soundtrack. + video_fps: ClassVar[int] = MINIMAX_H3_FPS + + def __init__(self, pipe: DiffusionPipeline): + super().__init__(pipe) + # Set by decode_latent, read by the VAE Decode node in the same _decode call so the + # soundtrack can be muxed into the video that decode_latent returns. + self.last_audio: torch.Tensor | None = None + self.last_sampling_rate: int | None = None + + @property + @override + def modular_pipe(self) -> ModularPipeline: + # self._pipe already IS the modular pipeline. The base implementation would round-trip + # `update_components(**pipe.components)`, which drops the load spec of any component lacking + # a _diffusers_load_id and re-registers everything with the ComponentsManager that owns this + # model's CPU-offload hooks. + return cast(ModularPipeline, self._pipe) + + @override + def _create_modular_pipe(self) -> ModularPipeline: + return cast(ModularPipeline, self._pipe) + + @classmethod + @override + def can_make_control_pipe_from_standard(cls, control_net_model_lists: list[str] | str | None) -> bool: + return False + + # ------------------------------------------------------------------ + # Geometry + # ------------------------------------------------------------------ + + def _latent_geometry(self, source_shape: tuple[int, ...]) -> dict[str, int]: + """Latent-space dimensions for a pixel-space ``source_shape`` of (..., T, H, W).""" + pipe = cast(MiniMaxH3ModularPipeline, self.modular_pipe) + num_frames = align_num_frames(source_shape[-3]) + compression = pipe.vae_spatial_compression_ratio + return { + "num_frames": num_frames, + "num_latent_frames": video_latent_num_frames(num_frames), + "latent_height": source_shape[-2] // compression, + "latent_width": source_shape[-1] // compression, + "num_audio_latents": audio_latent_num_frames(num_frames), + } + + def _run_blocks(self, blocks: Any, **kwargs: Any) -> PipelineState: + """Run ``blocks`` over one shared ``PipelineState`` and return it. + + Deliberately not ``_call_block``: that builds a fresh state per call (so a multi-block + prefix would lose every intermediate) and runs under ``inference_mode``, whose tensors + cannot be mutated in place — which the scheduler step does. + """ + state = PipelineState() + for param in blocks.inputs: + if param.name in kwargs: + state.set(param.name, kwargs[param.name], param.kwargs_type) + with torch.no_grad(): + _, state = blocks(self.modular_pipe, state) + return state + + # ------------------------------------------------------------------ + # Public latent surface + # ------------------------------------------------------------------ + + @override + def create_noise_latent(self, source_shape: tuple[int, ...], generator_state: GeneratorState) -> LatentArtifact: + generator = generator_state.to_generator() + geometry = self._latent_geometry(source_shape) + state = self._run_blocks( + _MiniMaxH3PrepareNoiseStep(), + num_latent_frames=geometry["num_latent_frames"], + latent_height=geometry["latent_height"], + latent_width=geometry["latent_width"], + num_audio_latents=geometry["num_audio_latents"], + generator=generator, + ) + latents = self._get_required(state.values, "latents", torch.Tensor) + audio_latents = self._get_required(state.values, "audio_latents", torch.Tensor) + return self._make_latent_artifact( + latents, + source_shape=source_shape, + meta={ + AUDIO_LATENTS_META_KEY: audio_latents, + AUDIO_PAIRED_WITH_META_KEY: _video_fingerprint(latents), + **GeneratorState.from_generator(generator).as_meta(), + }, + ) + + @override + def decode_latent(self, latent: LatentArtifact) -> DecodeResult: + """Decode the video, and its soundtrack when the artifact still carries the audio latent.""" + pipe = cast(MiniMaxH3ModularPipeline, self.modular_pipe) + device, _ = self._get_device_and_type() + geometry = self._latent_geometry(latent.source_shape) + + # The decode blocks take packed rows and unpatchify internally. + latents = latent.to_torch(device=device, dtype=torch.float32) + video_rows = patchify_video_latents(latents, pipe.patch_size) + + audio_latents = read_driver_meta(latent, AUDIO_LATENTS_META_KEY, self.driver_namespace) + self.last_audio = None + self.last_sampling_rate = None + + # A present-but-stale audio latent is the dangerous case: latent math sums the video latent + # while carrying the left operand's unsummed audio, so the soundtrack would no longer match + # the picture. Refuse instead of muxing a desynchronised track. + if audio_latents is not None: + paired_with = read_driver_meta(latent, AUDIO_PAIRED_WITH_META_KEY, self.driver_namespace) + if not _fingerprints_match(paired_with, _video_fingerprint(latents)): + raise ValueError( + f"{self.driver_namespace}: Attempted to decode a MiniMax-H3 latent. Failed " + f"because its audio latent belongs to a different video latent, so the " + f"soundtrack would not match the picture. MiniMax-H3 generates video and audio " + f"jointly and the audio travels in the latent's metadata, which latent math, " + f"composite and upsampler nodes do not recompute. Connect Generate Media " + f"Latents directly to Decode Media Latent." + ) + + video_state = self._run_blocks( + pipe.blocks.sub_blocks["decode"].sub_blocks["video"], + latents=video_rows, + num_latent_frames=geometry["num_latent_frames"], + latent_height=geometry["latent_height"], + latent_width=geometry["latent_width"], + output_type="pil", + ) + video_frames = self._get_required(video_state.values, "videos", list)[0] + + # The live-preview path decodes a latent it rebuilt without meta, so a missing soundtrack is + # normal there and must not raise. Callers that need the audio check `last_audio`. + if audio_latents is None: + logger.warning( + "%s: decoding video only because the latent carries no audio latents in driver meta.", + self.driver_namespace, + ) + return video_frames + + audio_rows = ( + audio_latents.to(device=device, dtype=torch.float32) + .permute(0, 2, 1) + .reshape(-1, pipe.audio_latent_channels) + ) + audio_state = self._run_blocks( + pipe.blocks.sub_blocks["decode"].sub_blocks["audio"], + audio_latents=audio_rows, + num_audio_latents=geometry["num_audio_latents"], + output_type="pil", + ) + self.last_audio = self._get_required(audio_state.values, "audio", torch.Tensor) + self.last_sampling_rate = self._get_required(audio_state.values, "sampling_rate", int) + return video_frames + + @override + def encode_media(self, media: ImageMedia | VideoMedia, generator_state: GeneratorState) -> LatentArtifact: + # MiniMax-H3's only VAE-encode block is keyframe-specific: it seeds the posterior with a + # fixed seed, rounds to fp16 and emits noise-augmented conditioning rows rather than a + # public latent. Keyframes reach the model through `conditioning_images` on the Generate + # node instead. There is no general image or video encode in the blockset. + raise NotImplementedError( + f"{self.driver_namespace} does not support encoding media into a latent. MiniMax-H3 has " + f"no general VAE-encode path; supply keyframes via the Media Gen Conditioning input on " + f"Generate Media Latents instead." + ) + + @override + def add_noise_to_latent( + self, + latent: LatentArtifact, + generator_state: GeneratorState, + num_inference_steps: int, + strength: float, + ) -> LatentArtifact: + # Video-to-video would need to resume a partially-noised latent mid-schedule, but + # MiniMaxH3Scheduler.set_timesteps always rebuilds the full linspace(1, 0, n) grid and the + # blockset exposes no encode path to noise from in the first place. + raise NotImplementedError( + f"{self.driver_namespace} does not support adding noise to an existing latent " + f"(no video-to-video path). Use Create Noise Latents for a fresh latent." + ) + + # ------------------------------------------------------------------ + # Denoise + # ------------------------------------------------------------------ + + def _resolve_keyframes(self, kwargs: dict[str, Any], num_frames: int) -> None: + """Translate a Media Gen Conditioning payload into ``image`` / ``last_image`` kwargs.""" + payloads = normalize_to_payloads(kwargs.pop(MediaGenConditioningKey.OUTPUT, None)) + if payloads is None: + return + + for payload in payloads: + if payload.mode is not ConditioningMode.IMAGE: + raise ValueError( + f"Attempted to build MiniMax-H3 keyframe conditioning. Failed with mode " + f"'{payload.mode.value}' because only image payloads are supported." + ) + for entry in payload.entries: + image = resolve_conditioning_image(entry.artifact) + frame_index = resolve_frame_index(entry.frame_index, num_frames) + if frame_index == 0: + kwargs["image"] = image + elif frame_index in (-1, num_frames - 1): + kwargs["last_image"] = image + else: + raise ValueError( + f"Attempted to build MiniMax-H3 keyframe conditioning. Failed with " + f"frame_index={frame_index} because only the first frame (0) and the last " + f"({num_frames - 1} or -1) are supported." + ) + + @override + def denoise_latent( + self, + latent: LatentArtifact | InpaintMaskArtifact, + num_inference_steps: int, + generator_state: GeneratorState, + callback: Any = None, + start_step: int = 0, + end_step: int = -1, + return_fully_denoised: bool = False, + **kwargs: Any, + ) -> LatentArtifact: + if isinstance(latent, InpaintMaskArtifact): + raise NotImplementedError(f"{self.driver_namespace} does not support inpainting.") + if return_fully_denoised: + # Reaching the terminal sigma from a truncated window would step the schedulers past + # the window while their sigma grids stay whole, pairing each step with the wrong sigma. + raise NotImplementedError( + f"{self.driver_namespace} does not support 'return_fully_denoised' because " + f"MiniMax-H3's two schedulers cannot express a non-contiguous schedule." + ) + + pipe = cast(MiniMaxH3ModularPipeline, self.modular_pipe) + device, _ = self._get_device_and_type() + source_shape = latent.source_shape + geometry = self._latent_geometry(source_shape) + + update_kwargs = kwargs.copy() + requested_num_frames = update_kwargs.pop("num_frames", None) + if requested_num_frames is not None and align_num_frames(int(requested_num_frames)) != geometry["num_frames"]: + logger.warning( + "%s: ignoring num_frames=%s because the input latent holds %d frames.", + self.driver_namespace, + requested_num_frames, + geometry["num_frames"], + ) + # The latent's own shape is what gets denoised, so source_shape is the single source of truth. + update_kwargs["num_frames"] = geometry["num_frames"] + update_kwargs.setdefault("height", source_shape[-2]) + update_kwargs.setdefault("width", source_shape[-1]) + self._resolve_keyframes(update_kwargs, geometry["num_frames"]) + + generator = update_kwargs.pop("generator", generator_state.to_generator()) + audio_latents = read_driver_meta(latent, AUDIO_LATENTS_META_KEY, self.driver_namespace) + if audio_latents is not None: + audio_latents = audio_latents.to(device=device, dtype=torch.float32) + + prefix_blocks = dict(zip(pipe.blocks.block_names, pipe.blocks.sub_blocks.values(), strict=True)) + prefix_blocks.pop("decode") + denoise_step = _MiniMaxH3CallbackDenoiseStep() + denoise_step.callback = callback + denoise_step.start_step = start_step + denoise_step.end_step = end_step + prefix_blocks["denoise"] = denoise_step + blocks = SequentialPipelineBlocks.from_blocks_dict(prefix_blocks) + + # The framework signals cancellation by setting `_interrupt` on the pipe, but only when the + # attribute already exists — and ModularPipeline has none. The pipe is cached across runs, + # so a leaked True would break the next run at step 0. + pipe._interrupt = False + try: + state = self._run_blocks( + blocks, + latents=latent.to_torch(device=device, dtype=torch.float32), + audio_latents=audio_latents, + num_inference_steps=num_inference_steps, + generator=generator, + **update_kwargs, + ) + finally: + pipe._interrupt = False + + denoised_video_rows = self._get_required(state.values, "latents", torch.Tensor) + denoised_audio_rows = self._get_required(state.values, "audio_latents", torch.Tensor) + num_condition_video_rows = state.values.get("num_condition_video_rows", 0) + num_condition_audio_rows = state.values.get("num_condition_audio_rows", 0) + + video_latents = unpatchify_video_tokens( + denoised_video_rows[num_condition_video_rows:], + geometry["num_latent_frames"], + geometry["latent_height"], + geometry["latent_width"], + pipe.vae_latent_channels, + pipe.patch_size, + ) + audio_out = unpack_audio_tokens(denoised_audio_rows[num_condition_audio_rows:], geometry["num_audio_latents"]) + + return self._make_latent_artifact( + video_latents, + source_shape=source_shape, + upstream=latent, + meta={ + AUDIO_LATENTS_META_KEY: audio_out, + AUDIO_PAIRED_WITH_META_KEY: _video_fingerprint(video_latents), + **GeneratorState.from_generator(generator).as_meta(), + }, + ) diff --git a/modular_diffusion_nodes_library/nodes/decode_hdr_node.py b/modular_diffusion_nodes_library/nodes/decode_hdr_node.py index d144eb7..a7c543a 100644 --- a/modular_diffusion_nodes_library/nodes/decode_hdr_node.py +++ b/modular_diffusion_nodes_library/nodes/decode_hdr_node.py @@ -106,9 +106,17 @@ def _decode(self) -> None: self.progress_bar_component.reset() super()._decode() - def _encode_video_output(self, output: Any, dest_path: Path, fps: int) -> None: + def _encode_video_output( + self, + output: Any, + dest_path: Path, + fps: int, + *, + audio: Any = None, + audio_sample_rate: int | None = None, + ) -> None: if not isinstance(output, np.ndarray): - super()._encode_video_output(output, dest_path, fps) + super()._encode_video_output(output, dest_path, fps, audio=audio, audio_sample_rate=audio_sample_rate) return frames = output[0] diff --git a/modular_diffusion_nodes_library/nodes/vae_decoder.py b/modular_diffusion_nodes_library/nodes/vae_decoder.py index 1cd6ad8..59fdb0a 100644 --- a/modular_diffusion_nodes_library/nodes/vae_decoder.py +++ b/modular_diffusion_nodes_library/nodes/vae_decoder.py @@ -7,6 +7,7 @@ import diffusers # type: ignore[reportMissingImports] import numpy as np from diffusers.pipelines.ltx2.export_utils import encode_hdr_tensor_to_mp4 # type: ignore[reportMissingImports] +from diffusers.utils.export_utils import encode_video # type: ignore[reportMissingImports] from griptape.artifacts.video_url_artifact import VideoUrlArtifact from griptape_nodes.exe_types.core_types import Parameter, ParameterMode from griptape_nodes.exe_types.node_types import AsyncResult, SuccessFailureNode @@ -243,7 +244,11 @@ def _decode(self) -> None: temp_path = Path(temp_file_obj.name) try: fps = int(self.get_parameter_value("fps") or latents_pipeline_driver.video_fps) - self._encode_video_output(output, temp_path, fps) + # Drivers whose model generates a soundtrack alongside the video expose it here so + # it can be muxed into the same file (MiniMax-H3). + audio = getattr(latents_pipeline_driver, "last_audio", None) + audio_sample_rate = getattr(latents_pipeline_driver, "last_sampling_rate", None) + self._encode_video_output(output, temp_path, fps, audio=audio, audio_sample_rate=audio_sample_rate) self._publish_output_video(temp_path) finally: if temp_path.exists(): @@ -251,10 +256,27 @@ def _decode(self) -> None: else: self._handle_image_output(output) - def _encode_video_output(self, output: Any, dest_path: Path, fps: int) -> None: + def _encode_video_output( + self, + output: Any, + dest_path: Path, + fps: int, + *, + audio: Any = None, + audio_sample_rate: int | None = None, + ) -> None: """Encode a video output to ``dest_path``. Override to customize HDR/tone-mapping.""" if isinstance(output, np.ndarray): encode_hdr_tensor_to_mp4(output[0], str(dest_path), frame_rate=fps) + elif audio is not None and audio_sample_rate is not None: + # The soundtrack arrives batched as (1, 2, num_samples); encode_video wants (2, N). + encode_video( + output, + fps, + str(dest_path), + audio=audio[0], + audio_sample_rate=audio_sample_rate, + ) else: diffusers.utils.export_to_video(output, str(dest_path), fps=fps) # type: ignore[attr-defined] diff --git a/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py b/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py index 671ac88..4ce70a6 100644 --- a/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py +++ b/modular_diffusion_nodes_library/parameters/pipelinetype_parameters.py @@ -37,6 +37,9 @@ from modular_diffusion_nodes_library.standard_parameters.ltx_parameters import ( LTXPipelineParameters, ) +from modular_diffusion_nodes_library.standard_parameters.minimax_h3_parameters import ( + MiniMaxH3PipelineParameters, +) from modular_diffusion_nodes_library.standard_parameters.qwen_edit_parameters import ( QwenEditPipelineParameters, ) @@ -249,6 +252,30 @@ def get_pipeline_type_dict(cls) -> dict[str, type[ModularDiffusionPipelineTypePi } +class LatentMiniMaxH3PipelineTypeParameters(LatentPipelineTypeParameters): + @property + def pipeline_type_badge_message(self) -> str: + return ( + "- `MiniMaxH3ModularPipeline` — Text-to-video and keyframe-to-video generation with a " + "**jointly generated soundtrack** (MiniMax-H3).\n\n" + "Video and audio come out of one denoising loop, and the Decode Media Latent node muxes " + "them into a single MP4. The audio latent travels in the latent's metadata, so connect " + "Generate Media Latents **directly** to Decode Media Latent — latent math, composite, " + "upsampler and save/load nodes drop it.\n\n" + "Fixed 24 fps, 5 to 15 seconds. Frame count is snapped up to the next `17 * n + 5` " + "(124, 141, 158, … 345). Height and width must be multiples of 32 and default to " + "MiniMax-H3's own canvas. Guidance is baked into the weights, so there is no " + "`guidance_scale` and no `negative_prompt`. Image-to-video, video-to-video, ControlNet " + "and inpainting are not supported." + ) + + @classmethod + def get_pipeline_type_dict(cls) -> dict[str, type[ModularDiffusionPipelineTypePipelineParameters]]: + return { + "MiniMaxH3ModularPipeline": MiniMaxH3PipelineParameters, + } + + class LatentQwenPipelineTypeParameters(LatentPipelineTypeParameters): @property def pipeline_type_badge_message(self) -> str: @@ -360,6 +387,7 @@ def get_pipeline_type_dict(cls) -> dict[str, type[ModularDiffusionPipelineTypePi Provider.HUNYUAN_VIDEO_1_5: LatentHunyuanVideo15PipelineTypeParameters, Provider.LTX: LatentLTXPipelineTypeParameters, Provider.LTX2: LatentLTX2PipelineTypeParameters, + Provider.MINIMAX_H3: LatentMiniMaxH3PipelineTypeParameters, Provider.QWEN: LatentQwenPipelineTypeParameters, Provider.STABLE_DIFFUSION: LatentStableDiffusionPipelineTypeParameters, Provider.STABLE_DIFFUSION_3: LatentStableDiffusion3PipelineTypeParameters, diff --git a/modular_diffusion_nodes_library/parameters/providers.py b/modular_diffusion_nodes_library/parameters/providers.py index 586a4ce..250295b 100644 --- a/modular_diffusion_nodes_library/parameters/providers.py +++ b/modular_diffusion_nodes_library/parameters/providers.py @@ -7,6 +7,7 @@ class Provider(StrEnum): HUNYUAN_VIDEO_1_5 = "HunyuanVideo 1.5" LTX = "LTX" LTX2 = "LTX2" + MINIMAX_H3 = "MiniMax-H3" QWEN = "Qwen" STABLE_DIFFUSION = "Stable Diffusion" STABLE_DIFFUSION_3 = "Stable Diffusion 3" diff --git a/modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py b/modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py new file mode 100644 index 0000000..c1af315 --- /dev/null +++ b/modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py @@ -0,0 +1,175 @@ +import logging +from typing import ClassVar + +from diffusers.modular_pipelines.minimax_h3.packing import ( # type: ignore[reportMissingImports] + MINIMAX_H3_CANVAS_MULTIPLE, + MINIMAX_H3_FPS, + MINIMAX_H3_MAX_DURATION, + MINIMAX_H3_MIN_DURATION, + align_num_frames, +) +from griptape_nodes.exe_types.core_types import Parameter +from griptape_nodes.exe_types.node_types import BaseNode + +from modular_diffusion_nodes_library.parameters.media_gen_conditioning.conditioning_layout import ( + PRESET_FIRST, + PRESET_FIRST_LAST, + MediaGenConditioningConfig, + PresetCatalogImageConfig, +) +from modular_diffusion_nodes_library.runtime_parameters.conditioning_runtime_parameter import ( + MediaGenConditioningRuntimeParameter, +) +from modular_diffusion_nodes_library.runtime_parameters.runtime_parameters import ( + DiffusionPipelineRuntimeParameters, +) +from modular_diffusion_nodes_library.utils.conditioning_utils import ConditioningMode + +logger = logging.getLogger("diffusers_nodes_library") + +# `num_frames` is snapped up to the next `17 * n + 5` the video VAE can decode, and the resulting +# duration must land in the 5-15 s window MiniMax-H3 generates. That makes 108 the smallest +# requestable count (-> 124 frames, 5.167 s) and 345 the largest (14.375 s): 346 would snap to 362, +# i.e. 15.083 s, which upstream rejects rather than silently stretching. +MIN_NUM_FRAMES = 108 +MAX_NUM_FRAMES = 345 +DEFAULT_NUM_FRAMES = 124 + + +class MiniMaxH3PipelineRuntimeParameters(DiffusionPipelineRuntimeParameters): + CONDITIONING_CONFIG: ClassVar[MediaGenConditioningConfig | None] = MediaGenConditioningConfig( + image=PresetCatalogImageConfig(presets=(PRESET_FIRST_LAST, PRESET_FIRST), expose_strength=False), + ) + + def __init__(self, node: BaseNode): + super().__init__(node) + self._media_gen_conditioning_param = MediaGenConditioningRuntimeParameter( + node, + param_name="conditioning_images", + accepted_modes=(ConditioningMode.IMAGE,), + tooltip="First/last keyframes for MiniMax-H3 `fl2va`, from a Media Gen Conditioning node.", + badge_title="First/last keyframes", + badge_message=( + "Connect a **Media Gen Conditioning** node here to supply the frame the video starts " + "from (`image`) and/or the frame it ends on (`last_image`). Leave it unconnected for " + "text-only generation (`t2va`). Only **image**-mode payloads are accepted.\n\n" + "**Note:** the canvas follows the **first** keyframe's aspect ratio, so `height` and " + "`width` are derived from it unless you set them explicitly." + ), + ) + + def _add_input_parameters(self) -> None: + self._node.add_parameter( + Parameter( + name="prompt", + default_value="", + type="str", + tooltip="The prompt to guide generation of the video and its soundtrack.", + ) + ) + self._media_gen_conditioning_param.add_input_parameters() + num_frames_param = Parameter( + name="num_frames", + default_value=DEFAULT_NUM_FRAMES, + type="int", + tooltip=( + f"Number of frames to generate, at the fixed {MINIMAX_H3_FPS} fps. Snapped up to the " + f"next 17 * n + 5 the video VAE can decode; the resulting duration must stay between " + f"{MINIMAX_H3_MIN_DURATION:g} and {MINIMAX_H3_MAX_DURATION:g} seconds." + ), + ui_options={"min": MIN_NUM_FRAMES, "max": MAX_NUM_FRAMES}, + ) + num_frames_param.set_badge( + variant="help", + title="Frame count is snapped", + message=( + "MiniMax-H3 only decodes frame counts of the form **17 x n + 5**, so your value is " + "rounded **up** to the next one: 124, 141, 158, 175, 192, 209, 226, 243, 260, 277, " + "294, 311, 328, 345.\n\n" + "The Create Noise Latents node's frame count wins if the two disagree, because the " + "latent's own shape is what gets denoised." + ), + ) + self._node.add_parameter(num_frames_param) + self._node.add_parameter( + Parameter( + name="height", + default_value=0, + type="int", + tooltip=( + f"Height of the generated video in pixels, a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}. " + "Leave at 0 to use MiniMax-H3's own canvas for the aspect ratio of the first " + "keyframe, or 16:9 without one." + ), + ) + ) + self._node.add_parameter( + Parameter( + name="width", + default_value=0, + type="int", + tooltip=( + f"Width of the generated video in pixels, a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}. " + "Leave at 0 to use MiniMax-H3's own canvas for the aspect ratio of the first " + "keyframe, or 16:9 without one." + ), + ) + ) + + def _remove_input_parameters(self) -> None: + self._media_gen_conditioning_param.remove_input_parameters() + self._node.remove_parameter_element_by_name("prompt") + self._node.remove_parameter_element_by_name("num_frames") + self._node.remove_parameter_element_by_name("height") + self._node.remove_parameter_element_by_name("width") + + def _get_pipe_kwargs(self) -> dict: + # 0 means "let MiniMax-H3 resolve its own canvas", which the blocks express as None. + height = self._node.get_parameter_value("height") + width = self._node.get_parameter_value("width") + kwargs = { + "prompt": self._node.get_parameter_value("prompt"), + "num_frames": int(self._node.get_parameter_value("num_frames")), + **self._media_gen_conditioning_param.get_pipe_kwargs(), + } + if height: + kwargs["height"] = int(height) + if width: + kwargs["width"] = int(width) + return kwargs + + def validate_before_node_run(self) -> list[Exception] | None: + errors = super().validate_before_node_run() or [] + conditioning_errors = self._media_gen_conditioning_param.validate_before_node_run() + if conditioning_errors: + errors.extend(conditioning_errors) + + # Validate here rather than at denoise time so a bad canvas or duration surfaces before the + # ~124 GB load, not after it. + num_frames = int(self._node.get_parameter_value("num_frames")) + if num_frames < 1: + errors.append(ValueError(f"'num_frames' must be positive, got {num_frames}.")) + else: + duration = align_num_frames(num_frames) / MINIMAX_H3_FPS + if not MINIMAX_H3_MIN_DURATION <= duration <= MINIMAX_H3_MAX_DURATION: + errors.append( + ValueError( + f"Attempted to configure MiniMax-H3 generation. Failed with " + f"num_frames={num_frames} because it snaps up to {align_num_frames(num_frames)} " + f"frames ({duration:.3f} s), outside the {MINIMAX_H3_MIN_DURATION:g}-" + f"{MINIMAX_H3_MAX_DURATION:g} s window MiniMax-H3 generates. Use " + f"{MIN_NUM_FRAMES}-{MAX_NUM_FRAMES}." + ) + ) + + for name in ("height", "width"): + value = int(self._node.get_parameter_value(name)) + if value and value % MINIMAX_H3_CANVAS_MULTIPLE: + errors.append( + ValueError( + f"Attempted to configure MiniMax-H3 generation. Failed with {name}={value} " + f"because it must be a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}." + ) + ) + + return errors or None diff --git a/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py b/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py index 9237402..58d0d9a 100644 --- a/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py +++ b/modular_diffusion_nodes_library/runtime_parameters/runtime_params_registry.py @@ -37,6 +37,9 @@ from modular_diffusion_nodes_library.runtime_parameters.ltx_runtime_parameters import ( LTXPipelineRuntimeParameters, ) +from modular_diffusion_nodes_library.runtime_parameters.minimax_h3_runtime_parameters import ( + MiniMaxH3PipelineRuntimeParameters, +) from modular_diffusion_nodes_library.runtime_parameters.qwen_edit_runtime_parameters import ( QwenEditPipelineRuntimeParameters, ) @@ -78,6 +81,7 @@ "HunyuanVideo15ImageToVideoPipeline": HunyuanVideo15ImageToVideoPipelineRuntimeParameters, "LTXPipeline": LTXPipelineRuntimeParameters, "LTX2Pipeline": LTX2PipelineRuntimeParameters, + "MiniMaxH3ModularPipeline": MiniMaxH3PipelineRuntimeParameters, "QwenImagePipeline": QwenPipelineRuntimeParameters, "QwenImageEditPipeline": QwenEditPipelineRuntimeParameters, "StableDiffusion3Pipeline": StableDiffusion3PipelineRuntimeParameters, diff --git a/modular_diffusion_nodes_library/standard_parameters/minimax_h3_parameters.py b/modular_diffusion_nodes_library/standard_parameters/minimax_h3_parameters.py new file mode 100644 index 0000000..17e8728 --- /dev/null +++ b/modular_diffusion_nodes_library/standard_parameters/minimax_h3_parameters.py @@ -0,0 +1,92 @@ +import logging +from typing import Any + +import torch # type: ignore[reportMissingImports] +from diffusers import ComponentsManager, ModularPipeline # type: ignore[reportMissingImports] +from diffusers.modular_pipelines.minimax_h3.modular_pipeline import ( # type: ignore[reportMissingImports] + MiniMaxH3ModularPipeline, +) +from griptape_nodes.exe_types.node_types import BaseNode +from griptape_nodes.exe_types.param_components.huggingface.huggingface_repo_parameter import HuggingFaceRepoParameter + +from modular_diffusion_nodes_library.parameters.modular_pipeline_type_parameters import ( + ModularDiffusionPipelineTypePipelineParameters, +) +from modular_diffusion_nodes_library.utils.torch_utils import get_best_device + +logger = logging.getLogger("modular_diffusers_nodes_library") + +# The transformer is 61.7 GB in bfloat16 and the Qwen3-VL conditioner another 62.1 GB, so nothing +# fits alongside the accelerator's own working set. The margin is the model card's recommendation +# for a single 80 GB card. +AUTO_CPU_OFFLOAD_MEMORY_RESERVE_MARGIN = "12GB" + + +class MiniMaxH3PipelineParameters(ModularDiffusionPipelineTypePipelineParameters): + def __init__(self, node: BaseNode, *, list_all_models: bool = False): + super().__init__(node) + self._model_repo_parameter = HuggingFaceRepoParameter( + node, + repo_ids=["MiniMaxAI/MiniMax-H3"], + parameter_name="model", + list_all_models=list_all_models, + ) + + def add_input_parameters(self) -> None: + self._model_repo_parameter.add_input_parameters() + + def remove_input_parameters(self) -> None: + self._model_repo_parameter.remove_input_parameters() + + def get_config_kwargs(self) -> dict: + return { + "model": self._node.get_parameter_value("model"), + } + + @property + def pipeline_class(self) -> type: + return MiniMaxH3ModularPipeline + + def validate_before_node_run(self) -> list[Exception] | None: + errors = [] + model_errors = self._model_repo_parameter.validate_before_node_run() + if model_errors: + errors.extend(model_errors) + return errors or None + + def get_build_data(self) -> dict[str, Any]: + repo_id, revision = self._model_repo_parameter.get_repo_revision() + return { + "repo_id": repo_id, + "revision": revision, + } + + def requires_device_map(self) -> bool: + # Used here as an opt-out from post-hoc pipeline optimization, not literally to request an + # accelerate device_map. MiniMax-H3 is a ModularPipeline, on which + # `optimize_diffusion_pipeline` would call `.to(device)` (a guaranteed OOM at ~124 GB) while + # both `enable_*_cpu_offload` calls are hasattr-gated and silently skipped. Placement is + # instead owned by `build_pipeline_from_build_data` via the ComponentsManager below. + return True + + def is_prequantized(self) -> bool: + # Suppresses quantization and layerwise casting, which fire before the requires_device_map + # short-circuit. Neither is safe to apply on top of the ComponentsManager offload hooks. + return True + + @classmethod + def build_pipeline_from_build_data(cls, build_data: dict[str, Any]) -> ModularPipeline: + # `from_pretrained` resolves the component specs but loads no weights; `load_components` + # fetches them. Only the `t2va` / `fl2va` half is touched, never `transformer_ref/`. + manager = ComponentsManager() + pipe = ModularPipeline.from_pretrained( + build_data["repo_id"], + revision=build_data["revision"], + components_manager=manager, + ) + pipe.load_components(dtype=torch.bfloat16) + manager.enable_auto_cpu_offload( + device=get_best_device(), + memory_reserve_margin=AUTO_CPU_OFFLOAD_MEMORY_RESERVE_MARGIN, + ) + return pipe diff --git a/pyproject.toml b/pyproject.toml index 22af5ce..fb2f06a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,9 @@ dependencies = [ "beautifulsoup4>=4.13.4", "controlnet-aux>=0.0.9", "static-ffmpeg>=2.8", - "diffusers==0.39.0", + # TODO(diffusers-pin): temporary dev pin to the MiniMax-H3 branch (PR #14355, still an open + # draft). Revert to a tagged release once that PR merges; our own change cannot land before it. + "diffusers @ git+https://github.com/huggingface/diffusers.git@abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc", "imageio[ffmpeg]>=2.37.2", "ninja>=1.13.0", "numpy>=2.2.4", diff --git a/uv.lock b/uv.lock index b52c8f9..3ff9afe 100644 --- a/uv.lock +++ b/uv.lock @@ -153,9 +153,9 @@ name = "bitsandbytes" version = "0.49.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "packaging", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "numpy", marker = "sys_platform == 'win32'" }, + { name = "packaging", marker = "sys_platform == 'win32'" }, + { name = "torch", version = "2.7.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/b6/d4/501655842ad6771fb077f576d78cbedb5445d15b1c3c91343ed58ca46f0e/bitsandbytes-0.49.2-py3-none-win_amd64.whl", hash = "sha256:2e0ddd09cd778155388023cbe81f00afbb7c000c214caef3ce83386e7144df7d", size = 55372289, upload-time = "2026-02-16T21:26:16.267Z" }, @@ -261,14 +261,14 @@ wheels = [ [[package]] name = "click" -version = "8.4.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9b/98/518d8e5081007684232226f475082b30087d0f585e8457db087298259f49/click-8.4.1.tar.gz", hash = "sha256:918b5633eddf6b41c32d4f454bf0de810065c74e3f7dbf8ee5452f8be88d3e96", size = 353007, upload-time = "2026-05-22T04:08:37.769Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -461,8 +461,8 @@ wheels = [ [[package]] name = "diffusers" -version = "0.39.0" -source = { registry = "https://pypi.org/simple" } +version = "0.40.0.dev0" +source = { git = "https://github.com/huggingface/diffusers.git?rev=abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc#abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc" } dependencies = [ { name = "filelock" }, { name = "httpx" }, @@ -474,10 +474,6 @@ dependencies = [ { name = "requests" }, { name = "safetensors" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/81/6095237b86a3116c4789f28c4435d5296c00c0fc74ffde99008fd6b3a36c/diffusers-0.39.0.tar.gz", hash = "sha256:14bb1d98c85a0e463d734c99aaa73b480a7bc9bad22af30fbf730ef8f09c1d67", size = 4651240, upload-time = "2026-07-03T08:48:47.904Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/3f/7469c46e9d22307ea686bab687d70e6bf328722952f9d10339f5e913e608/diffusers-0.39.0-py3-none-any.whl", hash = "sha256:912aca51b5787365110806e984d5555735bf8a461073bb8459029d0bca7870ef", size = 5631176, upload-time = "2026-07-03T08:48:45.337Z" }, -] [[package]] name = "distro" @@ -812,7 +808,7 @@ requires-dist = [ { name = "bitsandbytes", marker = "sys_platform == 'win32'", specifier = ">=0.46.0" }, { name = "cmake", specifier = "==3.31.6" }, { name = "controlnet-aux", specifier = ">=0.0.9" }, - { name = "diffusers", specifier = "==0.39.0" }, + { name = "diffusers", git = "https://github.com/huggingface/diffusers.git?rev=abc5e9bf71fd38f53cd471bc3acaa84bc5ecbfdc" }, { name = "ftfy", specifier = ">=6.3.1" }, { name = "griptape", extras = ["drivers-prompt-amazon-bedrock", "drivers-prompt-anthropic", "drivers-prompt-cohere", "drivers-prompt-ollama", "drivers-web-scraper-trafilatura", "drivers-web-search-duckduckgo", "drivers-web-search-exa", "loaders-image"], specifier = ">=1.9.4" }, { name = "griptape-nodes", specifier = ">=0.77.5" }, @@ -864,18 +860,18 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.5.0" +version = "1.5.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/39/67be8d71f900d9a55761b6022821d6679fb56c64f1b6063d5af2c2606727/hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47", size = 903674, upload-time = "2026-07-16T17:29:56.833Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, - { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, - { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, - { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, - { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, + { url = "https://files.pythonhosted.org/packages/de/ba/2b70603c7552db82baeb2623e2336898304a17328845151be4fe1f48d420/hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799", size = 4033760, upload-time = "2026-07-16T17:29:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/60/ac/b097a86a1e4a6098f3a79382643ab09d5733d87ccc864877ad1e12b49b70/hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed", size = 3841438, upload-time = "2026-07-16T17:29:45.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/35/db860aa3a0780660324a506ad4b3d322ddc6ecbba4b9340aed0942cbf21c/hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c", size = 4428006, upload-time = "2026-07-16T17:29:46.996Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/832dd980af4b0c3ae0660e309285f2ffcdff2faa38129390dbb47aa4a3f9/hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025", size = 4221099, upload-time = "2026-07-16T17:29:48.525Z" }, + { url = "https://files.pythonhosted.org/packages/9e/05/ae50f0d34e3254e6c3e208beb2519f6b8673016fc4b3643badaf6450d186/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f", size = 4420766, upload-time = "2026-07-16T17:29:50.092Z" }, + { url = "https://files.pythonhosted.org/packages/07/a9/c050bc2743a2bcd68928bfee157b08681667a164a24ec95fbfcfcd717e08/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097", size = 4636716, upload-time = "2026-07-16T17:29:51.62Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f8/68b01c5c2edb56ac9a67b3d076ffddcb90867abaee923923eb34e7a14e76/hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65", size = 3988373, upload-time = "2026-07-16T17:29:53.395Z" }, + { url = "https://files.pythonhosted.org/packages/39/c6/988383e9dc17294d536fcbcd6fd16eed882e411ad16c954984a53e47b09c/hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d", size = 3816957, upload-time = "2026-07-16T17:29:54.991Z" }, ] [[package]] @@ -933,7 +929,7 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "1.17.0" +version = "1.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -944,12 +940,11 @@ dependencies = [ { name = "packaging" }, { name = "pyyaml" }, { name = "tqdm" }, - { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bd/65/9826515abb600b5722bcf53f8b4a2fb58340b1f8bfcaee19f83561c13a44/huggingface_hub-1.17.0.tar.gz", hash = "sha256:fad842b6763ef70ebc3919665b1b9273645203185400a7d6c5eddc2323cc3435", size = 797082, upload-time = "2026-05-28T15:12:13.347Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/db/3582597f8be0d34bd6881365a26d390854f12893eabdd62dd36de9df5a47/huggingface_hub-1.26.0.tar.gz", hash = "sha256:c8cd4e2df1ba9402f77fce9b509ec1d52debb502551789473f34016acc14e361", size = 936665, upload-time = "2026-07-30T14:12:04.156Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/28/d7cef5e477b855c25d415b8f57e5bc7347c7a90cad3acf1725d0c92ca294/huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c", size = 671546, upload-time = "2026-05-28T15:12:11.441Z" }, + { url = "https://files.pythonhosted.org/packages/97/bb/63a644c75b545f3ff394b822e9bd1c4a9586489c618b77a4d8a44a33a23b/huggingface_hub-1.26.0-py3-none-any.whl", hash = "sha256:e8cca670caa5d8dfa7e45bf45e86b466698198cd8150c021bcdb4a86b9252364", size = 780357, upload-time = "2026-07-30T14:12:01.998Z" }, ] [[package]] @@ -1562,7 +1557,7 @@ name = "nvidia-cudnn-cu12" version = "9.7.1.26" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/25/dc/dc825c4b1c83b538e207e34f48f86063c88deaa35d46c651c7c181364ba2/nvidia_cudnn_cu12-9.7.1.26-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:6d011159a158f3cfc47bf851aea79e31bcff60d530b70ef70474c84cac484d07", size = 726851421, upload-time = "2025-02-06T22:18:29.812Z" }, @@ -1573,7 +1568,7 @@ name = "nvidia-cufft-cu12" version = "11.3.3.41" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ac/26/b53c493c38dccb1f1a42e1a21dc12cba2a77fbe36c652f7726d9ec4aba28/nvidia_cufft_cu12-11.3.3.41-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:da650080ab79fcdf7a4b06aa1b460e99860646b176a43f6208099bdc17836b6a", size = 193118795, upload-time = "2025-01-23T17:56:30.536Z" }, @@ -1600,9 +1595,9 @@ name = "nvidia-cusolver-cu12" version = "11.7.2.55" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/08/953675873a136d96bb12f93b49ba045d1107bc94d2551c52b12fa6c7dec3/nvidia_cusolver_cu12-11.7.2.55-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4d1354102f1e922cee9db51920dba9e2559877cf6ff5ad03a00d853adafb191b", size = 260373342, upload-time = "2025-01-23T17:58:56.406Z" }, @@ -1613,7 +1608,7 @@ name = "nvidia-cusparse-cu12" version = "12.5.7.53" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c2/ab/31e8149c66213b846c082a3b41b1365b831f41191f9f40c6ddbc8a7d550e/nvidia_cusparse_cu12-12.5.7.53-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3c1b61eb8c85257ea07e9354606b26397612627fdcd327bfd91ccf6155e7c86d", size = 292064180, upload-time = "2025-01-23T18:00:23.233Z" }, @@ -2947,7 +2942,7 @@ name = "triton" version = "3.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "setuptools", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "setuptools", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/11/53/ce18470914ab6cfbec9384ee565d23c4d1c55f0548160b1c7b33000b11fd/triton-3.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b68c778f6c4218403a6bd01be7484f6dc9e20fe2083d22dd8aef33e3b87a10a3", size = 156504509, upload-time = "2025-04-09T20:27:40.413Z" }, From 6b60d65ab6016360d7b207275641867c374f06a8 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Mon, 3 Aug 2026 12:08:34 -0700 Subject: [PATCH 2/3] moar minimax h3 changes --- docs/nodes/create-noise-latents.md | 2 +- docs/nodes/decode_media_latent.md | 4 +- docs/nodes/generate_media_latents.md | 2 +- .../latent_pipeline_drivers/minimax_h3.py | 171 +++++++++++++++--- .../nodes/vae_decoder.py | 5 +- .../minimax_h3_runtime_parameters.py | 123 ++----------- 6 files changed, 166 insertions(+), 141 deletions(-) diff --git a/docs/nodes/create-noise-latents.md b/docs/nodes/create-noise-latents.md index 41de342..e5ba059 100644 --- a/docs/nodes/create-noise-latents.md +++ b/docs/nodes/create-noise-latents.md @@ -45,7 +45,7 @@ Pipeline Builder → [Create Noise Latents] → Generate Media Latents → Decod | Provider | Behavior | | --- | --- | -| MiniMax-H3 | This node's `width`, `height` and `num_frames` are authoritative — [Generate Media Latents](generate_media_latents.md) takes its dimensions from the latent and warns if its own `num_frames` disagrees. `width` and `height` must be multiples of **32**; MiniMax-H3's trained canvas is a 768-pixel short edge (e.g. 1344x768, or 960x544 for roughly 2.3x faster steps). `num_frames` is snapped up to the next `17 * n + 5` (124, 141, 158, … 345) and the result must land between 5 and 15 seconds at the fixed 24 fps. The noise latent also carries the audio noise for the jointly generated soundtrack. | +| MiniMax-H3 | This node sets the generated geometry — Generate Media Latents exposes no dimensions of its own. **The defaults do not work:** set `width`/`height` to a MiniMax-H3 canvas (a 768-pixel short edge, e.g. **1344x768** for 16:9, or **960x544** for roughly 2.3x faster steps) and `num_frames` to **108–345**. Both axes must be multiples of **32**, the area must not exceed 1032192 pixels, the aspect ratio must sit between 1:4 and 4:1, and `num_frames` is snapped up to the next `17 * n + 5` (124, 141, … 345), which must land between 5 and 15 seconds at the fixed 24 fps. Anything outside that is rejected with a specific error before the model loads. The noise latent also carries the audio noise for the jointly generated soundtrack. | ## Tips & pitfalls diff --git a/docs/nodes/decode_media_latent.md b/docs/nodes/decode_media_latent.md index 00940bf..f38a0be 100644 --- a/docs/nodes/decode_media_latent.md +++ b/docs/nodes/decode_media_latent.md @@ -36,7 +36,7 @@ Generate Media Latents → [Decode Media Latent] → Save Image / Save Video | Name | Type | Default | Notes | | --- | --- | --- | --- | -| `fps` | int (1–120) | `25` | Output frame rate. **Only shown for video pipelines.** | +| `fps` | int (1–120) | model's native rate | Output frame rate. **Only shown for video pipelines.** Defaults to the rate the selected model generates at (LTX 25, MiniMax-H3 24, WAN 16, HunyuanVideo 1.5 15), so leaving it alone plays back at the correct speed. | ## Provider / model behavior @@ -44,7 +44,7 @@ Generate Media Latents → [Decode Media Latent] → Save Image / Save Video | --- | --- | | Image pipelines | `output_image` as an `ImageArtifact`. | | LTX, LTX2, WAN, HunyuanVideo 1.5 | `output_video` as a silent MP4 at `fps`. | -| MiniMax-H3 | `output_video` as an MP4 **with an audio track**. Video and audio are generated jointly by one denoising loop, and are muxed together here. Leave `fps` at `24` — MiniMax-H3 generates at a fixed 24 fps, and any other value desynchronises the soundtrack. | +| MiniMax-H3 | `output_video` as an MP4 **with an audio track**. Video and audio are generated jointly by one denoising loop, and are muxed together here. `fps` defaults to the model's fixed **24** — changing it desynchronises the soundtrack, since the audio is muxed at its own true sample rate. | ### MiniMax-H3: keep the edge direct diff --git a/docs/nodes/generate_media_latents.md b/docs/nodes/generate_media_latents.md index bc20123..662ac60 100644 --- a/docs/nodes/generate_media_latents.md +++ b/docs/nodes/generate_media_latents.md @@ -63,7 +63,7 @@ The exact list depends on the connected pipeline. Common parameters: - **ControlNet:** when `pipeline` is a `ControlNetDiffusionPipelineArtifact`, the `controlnet_parameters` input is added automatically. - **Inpainting:** when `input_latent` is an `InpaintMaskArtifact` (from [Encode Masked Media Latent](encode_masked_media_latent.md)), the node automatically routes through the inpaint pipeline class and uses the artifact's `strength`. -- **MiniMax-H3:** generates the video **and its soundtrack** in one denoising loop. There is no `guidance_scale` and no `negative_prompt` — the checkpoint is guidance-distilled, so guidance is baked into the weights and every step runs a single forward pass. Keyframes are optional: leave `conditioning_images` unconnected for text-only generation, or connect a [Media Gen Conditioning](media_gen_conditioning.md) node for a first and/or last frame. The output latent carries the audio in its metadata, so wire this node **directly** to [Decode Media Latent](decode_media_latent.md). `return_fully_denoised` is not supported. Inpainting, ControlNet, image-to-video and video-to-video are unavailable. +- **MiniMax-H3:** generates the video **and its soundtrack** in one denoising loop. Only `prompt` and `conditioning_images` are exposed — there is no `guidance_scale` and no `negative_prompt` (the checkpoint is guidance-distilled, so guidance is baked into the weights and every step runs a single forward pass), and no dimensions (they come from the input latent, so set them on [Create Noise Latents](create-noise-latents.md)). Keyframes are optional: leave `conditioning_images` unconnected for text-only generation, or connect a [Media Gen Conditioning](media_gen_conditioning.md) node for a first and/or last frame. The output latent carries the audio in its metadata, so wire this node **directly** to [Decode Media Latent](decode_media_latent.md). Partial denoise must chain from another MiniMax-H3 generate — a `start_step` above 0 on a latent with no audio is rejected, since the soundtrack would restart from noise mid-schedule. `return_fully_denoised` is not supported. Inpainting, ControlNet, image-to-video and video-to-video are unavailable. ## Tips & pitfalls diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py index 016ac05..b158b83 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py @@ -31,7 +31,13 @@ ) from diffusers.modular_pipelines.minimax_h3.packing import ( # type: ignore[reportMissingImports] MINIMAX_H3_AUDIO_CHANNELS, + MINIMAX_H3_CANVAS_MULTIPLE, MINIMAX_H3_FPS, + MINIMAX_H3_MAX_ASPECT_RATIO, + MINIMAX_H3_MAX_DURATION, + MINIMAX_H3_MAX_PIXELS, + MINIMAX_H3_MIN_ASPECT_RATIO, + MINIMAX_H3_MIN_DURATION, align_num_frames, audio_latent_num_frames, patchify_video_latents, @@ -77,6 +83,14 @@ #: Key under which the audio latent rides in this driver's namespaced ``meta`` sub-bag. AUDIO_LATENTS_META_KEY = "audio_latents" +#: Frame counts a user can request. ``num_frames`` is snapped up to the next ``17 * n + 5`` the +#: video VAE can decode, and the resulting duration must land in MiniMax-H3's 5-15 s window. That +#: makes 108 the smallest usable count (snaps to 124 frames, 5.167 s) and 345 the largest +#: (14.375 s); 346 would snap to 362, i.e. 15.083 s, which upstream rejects. +MIN_REQUESTABLE_NUM_FRAMES = 108 +MAX_REQUESTABLE_NUM_FRAMES = 345 +DEFAULT_NUM_FRAMES = 124 + #: Fingerprint of the video latent the audio latent was denoised with. Latent math merges meta #: left-operand-wins over a *shallow* copy, so a summed video latent keeps the left operand's #: unsummed audio: the audio is still present but no longer corresponds to the video. Comparing @@ -106,6 +120,30 @@ def _fingerprints_match( ) +def _unpack_video_rows( + rows: torch.Tensor, + components: MiniMaxH3ModularPipeline, + *, + num_condition_video_rows: int, + num_latent_frames: int, + latent_height: int, + latent_width: int, +) -> torch.Tensor: + """Turn denoised video rows back into an unpacked 5-D latent, dropping conditioning rows. + + Used both for the final output and for the step-end preview, which must hand the framework a + latent in the public shape rather than the loop's internal row layout. + """ + return unpatchify_video_tokens( + rows[num_condition_video_rows:], + num_latent_frames, + latent_height, + latent_width, + components.vae_latent_channels, + components.patch_size, + ) + + class _MiniMaxH3PrepareNoiseStep(ModularPipelineBlocks): """Draw the video and audio noise, leaving the video latent unpacked. @@ -199,6 +237,17 @@ def description(self) -> str: "callback and cancellation support." ) + @property + def loop_inputs(self) -> list[InputParam]: + # The video geometry is not part of upstream's loop contract, but the step-end preview needs + # it to unpack the in-flight rows into the public latent shape. + return [ + *super().loop_inputs, + InputParam("num_latent_frames", required=True), + InputParam("latent_height", required=True), + InputParam("latent_width", required=True), + ] + def _resolve_window(self, num_steps: int) -> tuple[int, int]: """Clamp the requested step window against the schedule's real length. @@ -250,9 +299,19 @@ def __call__( components, block_state = self.loop_step(components, block_state, i=i, t=t) progress_bar.update() if self.callback is not None: - # The return value is deliberately discarded: the framework's callback returns - # {} on its normal path, and merging that would clobber the loop's latents. - self.callback(components, i, t, {"latents": block_state.latents}) + # The loop works in packed rows, but the framework feeds this straight back into + # decode_latent for the live preview, which expects the public 5-D shape. + # The return value is deliberately discarded: the callback returns {} on its + # normal path, and merging that would clobber the loop's latents. + preview_latents = _unpack_video_rows( + block_state.latents, + components, + num_condition_video_rows=block_state.num_condition_video_rows, + num_latent_frames=block_state.num_latent_frames, + latent_height=block_state.latent_height, + latent_width=block_state.latent_width, + ) + self.callback(components, i, t, {"latents": preview_latents}) if getattr(components, "_interrupt", False): break self.set_block_state(state, block_state) @@ -294,24 +353,77 @@ def can_make_control_pipe_from_standard(cls, control_net_model_lists: list[str] # ------------------------------------------------------------------ def _latent_geometry(self, source_shape: tuple[int, ...]) -> dict[str, int]: - """Latent-space dimensions for a pixel-space ``source_shape`` of (..., T, H, W).""" + """Latent-space dimensions for a pixel-space ``source_shape`` of (..., T, H, W). + + ``source_shape`` comes from the latent, which is the single source of truth for what gets + denoised, so it is validated here rather than in the runtime parameters: the Create Noise + Latents node owns these dimensions and has no MiniMax-H3-specific validation hook. + """ pipe = cast(MiniMaxH3ModularPipeline, self.modular_pipe) + height, width = source_shape[-2], source_shape[-1] + self._validate_canvas(height, width) + num_frames = align_num_frames(source_shape[-3]) + duration = num_frames / MINIMAX_H3_FPS + if not MINIMAX_H3_MIN_DURATION <= duration <= MINIMAX_H3_MAX_DURATION: + raise ValueError( + f"{self.driver_namespace}: Attempted to size a MiniMax-H3 request. Failed with " + f"num_frames={source_shape[-3]} because it snaps up to {num_frames} frames " + f"({duration:.3f} s at {MINIMAX_H3_FPS} fps), outside the " + f"{MINIMAX_H3_MIN_DURATION:g}-{MINIMAX_H3_MAX_DURATION:g} s window MiniMax-H3 " + f"generates. Set num_frames between {MIN_REQUESTABLE_NUM_FRAMES} and " + f"{MAX_REQUESTABLE_NUM_FRAMES} on the node that created this latent." + ) + compression = pipe.vae_spatial_compression_ratio return { "num_frames": num_frames, "num_latent_frames": video_latent_num_frames(num_frames), - "latent_height": source_shape[-2] // compression, - "latent_width": source_shape[-1] // compression, + "latent_height": height // compression, + "latent_width": width // compression, "num_audio_latents": audio_latent_num_frames(num_frames), } + def _validate_canvas(self, height: int, width: int) -> None: + """Reject a canvas MiniMax-H3 cannot express. + + Upstream treats an explicit ``height``/``width`` as a canvas override and skips its own + ``resolve_canvas_size`` checks, so these constraints have to be enforced here or they are + never enforced at all. + """ + for name, value in (("height", height), ("width", width)): + if value % MINIMAX_H3_CANVAS_MULTIPLE: + raise ValueError( + f"{self.driver_namespace}: Attempted to size a MiniMax-H3 request. Failed with " + f"{name}={value} because it must be a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}." + ) + + # Aspect ratio first: an extreme ratio usually also blows the pixel budget, and naming the + # ratio is the more actionable of the two messages. + ratio = width / height + if not MINIMAX_H3_MIN_ASPECT_RATIO <= ratio <= MINIMAX_H3_MAX_ASPECT_RATIO: + raise ValueError( + f"{self.driver_namespace}: Attempted to size a MiniMax-H3 request. Failed with " + f"{width}x{height} (ratio {ratio:g}) because MiniMax-H3 supports aspect ratios from " + f"1:4 to 4:1." + ) + + if height * width > MINIMAX_H3_MAX_PIXELS: + raise ValueError( + f"{self.driver_namespace}: Attempted to size a MiniMax-H3 request. Failed with " + f"{width}x{height} ({height * width} pixels) because MiniMax-H3 generates at most " + f"{MINIMAX_H3_MAX_PIXELS} pixels. Try 1344x768 (16:9) or 960x544 for faster steps." + ) + def _run_blocks(self, blocks: Any, **kwargs: Any) -> PipelineState: """Run ``blocks`` over one shared ``PipelineState`` and return it. - Deliberately not ``_call_block``: that builds a fresh state per call (so a multi-block - prefix would lose every intermediate) and runs under ``inference_mode``, whose tensors - cannot be mutated in place — which the scheduler step does. + Deliberately not ``_call_block``, for two reasons. It builds a fresh state per call, so a + multi-block prefix would lose every intermediate. And it runs under ``inference_mode``, + which mints inference tensors: legal to mutate in place *within* that scope, but the + scheduler step mutates ``latents`` in a later call than the one that created them, and + in-place mutation of an inference tensor outside ``inference_mode`` raises. ``no_grad`` + produces ordinary tensors and avoids the whole class of problem. """ state = PipelineState() for param in blocks.inputs: @@ -499,22 +611,27 @@ def denoise_latent( geometry = self._latent_geometry(source_shape) update_kwargs = kwargs.copy() - requested_num_frames = update_kwargs.pop("num_frames", None) - if requested_num_frames is not None and align_num_frames(int(requested_num_frames)) != geometry["num_frames"]: - logger.warning( - "%s: ignoring num_frames=%s because the input latent holds %d frames.", - self.driver_namespace, - requested_num_frames, - geometry["num_frames"], - ) - # The latent's own shape is what gets denoised, so source_shape is the single source of truth. + # The latent's own shape is what gets denoised, so it dictates the geometry outright. These + # are assigned rather than merged: a layout built for different dimensions than the latent + # tensor either crashes deep in attention or, when the row counts happen to collide, + # silently produces garbage. Nothing upstream of here is allowed to disagree. update_kwargs["num_frames"] = geometry["num_frames"] - update_kwargs.setdefault("height", source_shape[-2]) - update_kwargs.setdefault("width", source_shape[-1]) + update_kwargs["height"] = source_shape[-2] + update_kwargs["width"] = source_shape[-1] self._resolve_keyframes(update_kwargs, geometry["num_frames"]) generator = update_kwargs.pop("generator", generator_state.to_generator()) audio_latents = read_driver_meta(latent, AUDIO_LATENTS_META_KEY, self.driver_namespace) + if audio_latents is None and start_step > 0: + # Upstream draws fresh audio noise when none is supplied, but with a begin index set + # both schedulers would step that pure noise as if it were already partly denoised, + # yielding a garbled soundtrack that the output fingerprint would then certify as valid. + raise ValueError( + f"{self.driver_namespace}: Attempted to resume a MiniMax-H3 denoise at step " + f"{start_step}. Failed because the input latent carries no audio latent, so the " + f"soundtrack would restart from pure noise mid-schedule. Chain partial denoise " + f"directly from a previous Generate Media Latents run." + ) if audio_latents is not None: audio_latents = audio_latents.to(device=device, dtype=torch.float32) @@ -548,13 +665,13 @@ def denoise_latent( num_condition_video_rows = state.values.get("num_condition_video_rows", 0) num_condition_audio_rows = state.values.get("num_condition_audio_rows", 0) - video_latents = unpatchify_video_tokens( - denoised_video_rows[num_condition_video_rows:], - geometry["num_latent_frames"], - geometry["latent_height"], - geometry["latent_width"], - pipe.vae_latent_channels, - pipe.patch_size, + video_latents = _unpack_video_rows( + denoised_video_rows, + pipe, + num_condition_video_rows=num_condition_video_rows, + num_latent_frames=geometry["num_latent_frames"], + latent_height=geometry["latent_height"], + latent_width=geometry["latent_width"], ) audio_out = unpack_audio_tokens(denoised_audio_rows[num_condition_audio_rows:], geometry["num_audio_latents"]) diff --git a/modular_diffusion_nodes_library/nodes/vae_decoder.py b/modular_diffusion_nodes_library/nodes/vae_decoder.py index 59fdb0a..8e115a8 100644 --- a/modular_diffusion_nodes_library/nodes/vae_decoder.py +++ b/modular_diffusion_nodes_library/nodes/vae_decoder.py @@ -136,10 +136,13 @@ def _update_output_parameter(self) -> None: self.remove_parameter_element_by_name("output_image") # Add FPS parameter for video output (before output to appear above it in GUI) if not self.get_parameter_by_name("fps"): + # Default to the driver's own rate. Models that generate audio alongside the video + # (MiniMax-H3) only stay in sync at their trained rate, so a generic default would + # silently drift the soundtrack. self.add_parameter( Parameter( name="fps", - default_value=25, + default_value=driver_cls.video_fps, type="int", tooltip="Frames per second for video output.", allowed_modes={ParameterMode.PROPERTY}, diff --git a/modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py b/modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py index c1af315..561a2d5 100644 --- a/modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py +++ b/modular_diffusion_nodes_library/runtime_parameters/minimax_h3_runtime_parameters.py @@ -1,13 +1,6 @@ import logging from typing import ClassVar -from diffusers.modular_pipelines.minimax_h3.packing import ( # type: ignore[reportMissingImports] - MINIMAX_H3_CANVAS_MULTIPLE, - MINIMAX_H3_FPS, - MINIMAX_H3_MAX_DURATION, - MINIMAX_H3_MIN_DURATION, - align_num_frames, -) from griptape_nodes.exe_types.core_types import Parameter from griptape_nodes.exe_types.node_types import BaseNode @@ -27,16 +20,15 @@ logger = logging.getLogger("diffusers_nodes_library") -# `num_frames` is snapped up to the next `17 * n + 5` the video VAE can decode, and the resulting -# duration must land in the 5-15 s window MiniMax-H3 generates. That makes 108 the smallest -# requestable count (-> 124 frames, 5.167 s) and 345 the largest (14.375 s): 346 would snap to 362, -# i.e. 15.083 s, which upstream rejects rather than silently stretching. -MIN_NUM_FRAMES = 108 -MAX_NUM_FRAMES = 345 -DEFAULT_NUM_FRAMES = 124 - class MiniMaxH3PipelineRuntimeParameters(DiffusionPipelineRuntimeParameters): + """Runtime surface for MiniMax-H3. + + Deliberately narrow. The checkpoint is guidance-distilled, so there is no ``guidance_scale`` and + no ``negative_prompt``. Frame count and canvas are not exposed either: they come from the input + latent, like every other video pipeline in this library. + """ + CONDITIONING_CONFIG: ClassVar[MediaGenConditioningConfig | None] = MediaGenConditioningConfig( image=PresetCatalogImageConfig(presets=(PRESET_FIRST_LAST, PRESET_FIRST), expose_strength=False), ) @@ -47,14 +39,15 @@ def __init__(self, node: BaseNode): node, param_name="conditioning_images", accepted_modes=(ConditioningMode.IMAGE,), - tooltip="First/last keyframes for MiniMax-H3 `fl2va`, from a Media Gen Conditioning node.", + tooltip="First/last keyframes for keyframe-to-video generation, from a Media Gen Conditioning node.", badge_title="First/last keyframes", badge_message=( "Connect a **Media Gen Conditioning** node here to supply the frame the video starts " - "from (`image`) and/or the frame it ends on (`last_image`). Leave it unconnected for " - "text-only generation (`t2va`). Only **image**-mode payloads are accepted.\n\n" - "**Note:** the canvas follows the **first** keyframe's aspect ratio, so `height` and " - "`width` are derived from it unless you set them explicitly." + "from and/or the frame it ends on. Leave it unconnected for text-only " + "video-and-audio generation. Only **image**-mode payloads are accepted.\n\n" + "**Note:** the generated canvas comes from the input latent, not from the keyframe — " + "set the dimensions on the **Create Noise Latents** node, matching your keyframe's " + "aspect ratio if you want the framing preserved." ), ) @@ -68,108 +61,20 @@ def _add_input_parameters(self) -> None: ) ) self._media_gen_conditioning_param.add_input_parameters() - num_frames_param = Parameter( - name="num_frames", - default_value=DEFAULT_NUM_FRAMES, - type="int", - tooltip=( - f"Number of frames to generate, at the fixed {MINIMAX_H3_FPS} fps. Snapped up to the " - f"next 17 * n + 5 the video VAE can decode; the resulting duration must stay between " - f"{MINIMAX_H3_MIN_DURATION:g} and {MINIMAX_H3_MAX_DURATION:g} seconds." - ), - ui_options={"min": MIN_NUM_FRAMES, "max": MAX_NUM_FRAMES}, - ) - num_frames_param.set_badge( - variant="help", - title="Frame count is snapped", - message=( - "MiniMax-H3 only decodes frame counts of the form **17 x n + 5**, so your value is " - "rounded **up** to the next one: 124, 141, 158, 175, 192, 209, 226, 243, 260, 277, " - "294, 311, 328, 345.\n\n" - "The Create Noise Latents node's frame count wins if the two disagree, because the " - "latent's own shape is what gets denoised." - ), - ) - self._node.add_parameter(num_frames_param) - self._node.add_parameter( - Parameter( - name="height", - default_value=0, - type="int", - tooltip=( - f"Height of the generated video in pixels, a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}. " - "Leave at 0 to use MiniMax-H3's own canvas for the aspect ratio of the first " - "keyframe, or 16:9 without one." - ), - ) - ) - self._node.add_parameter( - Parameter( - name="width", - default_value=0, - type="int", - tooltip=( - f"Width of the generated video in pixels, a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}. " - "Leave at 0 to use MiniMax-H3's own canvas for the aspect ratio of the first " - "keyframe, or 16:9 without one." - ), - ) - ) def _remove_input_parameters(self) -> None: self._media_gen_conditioning_param.remove_input_parameters() self._node.remove_parameter_element_by_name("prompt") - self._node.remove_parameter_element_by_name("num_frames") - self._node.remove_parameter_element_by_name("height") - self._node.remove_parameter_element_by_name("width") def _get_pipe_kwargs(self) -> dict: - # 0 means "let MiniMax-H3 resolve its own canvas", which the blocks express as None. - height = self._node.get_parameter_value("height") - width = self._node.get_parameter_value("width") - kwargs = { + return { "prompt": self._node.get_parameter_value("prompt"), - "num_frames": int(self._node.get_parameter_value("num_frames")), **self._media_gen_conditioning_param.get_pipe_kwargs(), } - if height: - kwargs["height"] = int(height) - if width: - kwargs["width"] = int(width) - return kwargs def validate_before_node_run(self) -> list[Exception] | None: errors = super().validate_before_node_run() or [] conditioning_errors = self._media_gen_conditioning_param.validate_before_node_run() if conditioning_errors: errors.extend(conditioning_errors) - - # Validate here rather than at denoise time so a bad canvas or duration surfaces before the - # ~124 GB load, not after it. - num_frames = int(self._node.get_parameter_value("num_frames")) - if num_frames < 1: - errors.append(ValueError(f"'num_frames' must be positive, got {num_frames}.")) - else: - duration = align_num_frames(num_frames) / MINIMAX_H3_FPS - if not MINIMAX_H3_MIN_DURATION <= duration <= MINIMAX_H3_MAX_DURATION: - errors.append( - ValueError( - f"Attempted to configure MiniMax-H3 generation. Failed with " - f"num_frames={num_frames} because it snaps up to {align_num_frames(num_frames)} " - f"frames ({duration:.3f} s), outside the {MINIMAX_H3_MIN_DURATION:g}-" - f"{MINIMAX_H3_MAX_DURATION:g} s window MiniMax-H3 generates. Use " - f"{MIN_NUM_FRAMES}-{MAX_NUM_FRAMES}." - ) - ) - - for name in ("height", "width"): - value = int(self._node.get_parameter_value(name)) - if value and value % MINIMAX_H3_CANVAS_MULTIPLE: - errors.append( - ValueError( - f"Attempted to configure MiniMax-H3 generation. Failed with {name}={value} " - f"because it must be a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}." - ) - ) - return errors or None From 47467b0e3caca22bbeba63742fc205de2e4be133 Mon Sep 17 00:00:00 2001 From: cjkindel Date: Mon, 3 Aug 2026 13:46:40 -0700 Subject: [PATCH 3/3] even more --- docs/nodes/create-noise-latents.md | 2 +- .../latent_pipeline_drivers/base_driver.py | 7 ++ .../latent_pipeline_drivers/ltx2.py | 2 + .../latent_pipeline_drivers/minimax_h3.py | 68 ++++++++++--------- .../nodes/vae_decoder.py | 24 +++++-- 5 files changed, 64 insertions(+), 39 deletions(-) diff --git a/docs/nodes/create-noise-latents.md b/docs/nodes/create-noise-latents.md index e5ba059..ebad3be 100644 --- a/docs/nodes/create-noise-latents.md +++ b/docs/nodes/create-noise-latents.md @@ -45,7 +45,7 @@ Pipeline Builder → [Create Noise Latents] → Generate Media Latents → Decod | Provider | Behavior | | --- | --- | -| MiniMax-H3 | This node sets the generated geometry — Generate Media Latents exposes no dimensions of its own. **The defaults do not work:** set `width`/`height` to a MiniMax-H3 canvas (a 768-pixel short edge, e.g. **1344x768** for 16:9, or **960x544** for roughly 2.3x faster steps) and `num_frames` to **108–345**. Both axes must be multiples of **32**, the area must not exceed 1032192 pixels, the aspect ratio must sit between 1:4 and 4:1, and `num_frames` is snapped up to the next `17 * n + 5` (124, 141, … 345), which must land between 5 and 15 seconds at the fixed 24 fps. Anything outside that is rejected with a specific error before the model loads. The noise latent also carries the audio noise for the jointly generated soundtrack. | +| MiniMax-H3 | This node sets the generated geometry — Generate Media Latents exposes no dimensions of its own. **The defaults do not work:** set `width`/`height` to a MiniMax-H3 canvas (a 768-pixel short edge, e.g. **1344x768** for 16:9, or **960x544** for roughly 2.3x faster steps) and `num_frames` to **108–345**. Both axes must be positive multiples of **32**, the area must not exceed 1032192 pixels, the aspect ratio must sit between 1:4 and 4:1, and `num_frames` is snapped up to the next `17 * n + 5` (124, 141, … 345), which must land between 5 and 15 seconds at the fixed 24 fps. Anything outside that is rejected with an error naming the value and the constraint. The noise latent also carries the audio noise for the jointly generated soundtrack. | ## Tips & pitfalls diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py index 5c611b9..dc4ea3f 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/base_driver.py @@ -118,6 +118,13 @@ def __init__(self, pipe: DiffusionPipeline): self._pipe = pipe self._modular_pipe: ModularPipeline | None = None + # Soundtrack published by ``decode_latent`` for models that generate audio jointly with the + # video, read by the VAE Decode node in the same call so it can be muxed into the output + # file. ``decode_latent`` owns these: it must set them on every call, clearing them when the + # decode produced no audio. Drivers for silent models leave them ``None``. + self.last_audio: torch.Tensor | None = None + self.last_sampling_rate: int | None = None + @property def pipe(self) -> DiffusionPipeline: return self._pipe diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/ltx2.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/ltx2.py index 983c8b1..1208308 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/ltx2.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/ltx2.py @@ -57,6 +57,8 @@ class LTX2PipelineDriver(LatentPipelineDriver): produces_video: ClassVar[bool] = True + # frame_rate defaults to 24.0 in LTX2Pipeline.__call__ and its EXAMPLE_DOC_STRING. + video_fps: ClassVar[int] = 24 _HDR_LORA_ADAPTER_TOKEN: ClassVar[str] = "ic-lora-hdr" _IC_LORA_REFERENCE_KEY: ClassVar[str] = "ltx2_ic_lora_reference" diff --git a/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py b/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py index b158b83..799aa6e 100644 --- a/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py +++ b/modular_diffusion_nodes_library/latent_pipeline_drivers/minimax_h3.py @@ -55,7 +55,6 @@ InputParam, OutputParam, ) -from diffusers.pipelines.pipeline_utils import DiffusionPipeline # type: ignore[reportMissingImports] from diffusers.utils.torch_utils import randn_tensor # type: ignore[reportMissingImports] from modular_diffusion_nodes_library.artifact_utils.inpaint_mask_artifact import InpaintMaskArtifact @@ -323,13 +322,6 @@ class MiniMaxH3LatentPipelineDriver(LatentPipelineDriver): # MiniMax-H3 generates at a fixed 24 fps; any other rate desynchronises the soundtrack. video_fps: ClassVar[int] = MINIMAX_H3_FPS - def __init__(self, pipe: DiffusionPipeline): - super().__init__(pipe) - # Set by decode_latent, read by the VAE Decode node in the same _decode call so the - # soundtrack can be muxed into the video that decode_latent returns. - self.last_audio: torch.Tensor | None = None - self.last_sampling_rate: int | None = None - @property @override def modular_pipe(self) -> ModularPipeline: @@ -392,10 +384,11 @@ def _validate_canvas(self, height: int, width: int) -> None: never enforced at all. """ for name, value in (("height", height), ("width", width)): - if value % MINIMAX_H3_CANVAS_MULTIPLE: + if value <= 0 or value % MINIMAX_H3_CANVAS_MULTIPLE: raise ValueError( f"{self.driver_namespace}: Attempted to size a MiniMax-H3 request. Failed with " - f"{name}={value} because it must be a multiple of {MINIMAX_H3_CANVAS_MULTIPLE}." + f"{name}={value} because it must be a positive multiple of " + f"{MINIMAX_H3_CANVAS_MULTIPLE}." ) # Aspect ratio first: an extreme ratio usually also blows the pixel budget, and naming the @@ -415,6 +408,32 @@ def _validate_canvas(self, height: int, width: int) -> None: f"{MINIMAX_H3_MAX_PIXELS} pixels. Try 1344x768 (16:9) or 960x544 for faster steps." ) + def _read_paired_audio_latents( + self, latent: LatentArtifact, video_latents: torch.Tensor, *, action: str + ) -> torch.Tensor | None: + """Return the artifact's audio latent, or ``None`` when it carries none. + + Raises when an audio latent is present but was paired with a *different* video latent. That + is the dangerous case: latent math shallow-merges meta left-operand-wins, so a summed video + latent keeps the left operand's unsummed audio. Both the denoise and the decode entry points + check this, so a stale pairing cannot be laundered by passing through a second denoise. + """ + audio_latents = read_driver_meta(latent, AUDIO_LATENTS_META_KEY, self.driver_namespace) + if audio_latents is None: + return None + + paired_with = read_driver_meta(latent, AUDIO_PAIRED_WITH_META_KEY, self.driver_namespace) + if not _fingerprints_match(paired_with, _video_fingerprint(video_latents)): + raise ValueError( + f"{self.driver_namespace}: Attempted to {action} a MiniMax-H3 latent. Failed " + f"because its audio latent belongs to a different video latent, so the soundtrack " + f"would not match the picture. MiniMax-H3 generates video and audio jointly and the " + f"audio travels in the latent's metadata, which latent math, composite and upsampler " + f"nodes do not recompute. Connect Generate Media Latents directly to Decode Media " + f"Latent." + ) + return audio_latents + def _run_blocks(self, blocks: Any, **kwargs: Any) -> PipelineState: """Run ``blocks`` over one shared ``PipelineState`` and return it. @@ -472,24 +491,9 @@ def decode_latent(self, latent: LatentArtifact) -> DecodeResult: latents = latent.to_torch(device=device, dtype=torch.float32) video_rows = patchify_video_latents(latents, pipe.patch_size) - audio_latents = read_driver_meta(latent, AUDIO_LATENTS_META_KEY, self.driver_namespace) self.last_audio = None self.last_sampling_rate = None - - # A present-but-stale audio latent is the dangerous case: latent math sums the video latent - # while carrying the left operand's unsummed audio, so the soundtrack would no longer match - # the picture. Refuse instead of muxing a desynchronised track. - if audio_latents is not None: - paired_with = read_driver_meta(latent, AUDIO_PAIRED_WITH_META_KEY, self.driver_namespace) - if not _fingerprints_match(paired_with, _video_fingerprint(latents)): - raise ValueError( - f"{self.driver_namespace}: Attempted to decode a MiniMax-H3 latent. Failed " - f"because its audio latent belongs to a different video latent, so the " - f"soundtrack would not match the picture. MiniMax-H3 generates video and audio " - f"jointly and the audio travels in the latent's metadata, which latent math, " - f"composite and upsampler nodes do not recompute. Connect Generate Media " - f"Latents directly to Decode Media Latent." - ) + audio_latents = self._read_paired_audio_latents(latent, latents, action="decode") video_state = self._run_blocks( pipe.blocks.sub_blocks["decode"].sub_blocks["video"], @@ -502,9 +506,10 @@ def decode_latent(self, latent: LatentArtifact) -> DecodeResult: video_frames = self._get_required(video_state.values, "videos", list)[0] # The live-preview path decodes a latent it rebuilt without meta, so a missing soundtrack is - # normal there and must not raise. Callers that need the audio check `last_audio`. + # normal there and must not raise. Debug rather than warning because that path decodes once + # per denoise step. Callers that need the audio check `last_audio`. if audio_latents is None: - logger.warning( + logger.debug( "%s: decoding video only because the latent carries no audio latents in driver meta.", self.driver_namespace, ) @@ -621,11 +626,12 @@ def denoise_latent( self._resolve_keyframes(update_kwargs, geometry["num_frames"]) generator = update_kwargs.pop("generator", generator_state.to_generator()) - audio_latents = read_driver_meta(latent, AUDIO_LATENTS_META_KEY, self.driver_namespace) + video_latents_in = latent.to_torch(device=device, dtype=torch.float32) + audio_latents = self._read_paired_audio_latents(latent, video_latents_in, action="denoise") if audio_latents is None and start_step > 0: # Upstream draws fresh audio noise when none is supplied, but with a begin index set # both schedulers would step that pure noise as if it were already partly denoised, - # yielding a garbled soundtrack that the output fingerprint would then certify as valid. + # yielding a garbled soundtrack that this run would then stamp as validly paired. raise ValueError( f"{self.driver_namespace}: Attempted to resume a MiniMax-H3 denoise at step " f"{start_step}. Failed because the input latent carries no audio latent, so the " @@ -651,7 +657,7 @@ def denoise_latent( try: state = self._run_blocks( blocks, - latents=latent.to_torch(device=device, dtype=torch.float32), + latents=video_latents_in, audio_latents=audio_latents, num_inference_steps=num_inference_steps, generator=generator, diff --git a/modular_diffusion_nodes_library/nodes/vae_decoder.py b/modular_diffusion_nodes_library/nodes/vae_decoder.py index 8e115a8..c9a6b8b 100644 --- a/modular_diffusion_nodes_library/nodes/vae_decoder.py +++ b/modular_diffusion_nodes_library/nodes/vae_decoder.py @@ -136,9 +136,8 @@ def _update_output_parameter(self) -> None: self.remove_parameter_element_by_name("output_image") # Add FPS parameter for video output (before output to appear above it in GUI) if not self.get_parameter_by_name("fps"): - # Default to the driver's own rate. Models that generate audio alongside the video - # (MiniMax-H3) only stay in sync at their trained rate, so a generic default would - # silently drift the soundtrack. + # Default to the driver's own rate rather than a generic one, so playback speed is + # right out of the box. self.add_parameter( Parameter( name="fps", @@ -247,10 +246,21 @@ def _decode(self) -> None: temp_path = Path(temp_file_obj.name) try: fps = int(self.get_parameter_value("fps") or latents_pipeline_driver.video_fps) - # Drivers whose model generates a soundtrack alongside the video expose it here so - # it can be muxed into the same file (MiniMax-H3). - audio = getattr(latents_pipeline_driver, "last_audio", None) - audio_sample_rate = getattr(latents_pipeline_driver, "last_sampling_rate", None) + # Drivers whose model generates a soundtrack alongside the video publish it here so + # it can be muxed into the same file. + audio = latents_pipeline_driver.last_audio + audio_sample_rate = latents_pipeline_driver.last_sampling_rate + if audio is not None and fps != latents_pipeline_driver.video_fps: + # A jointly generated soundtrack is muxed at its own true sample rate, so any + # frame rate other than the one the model generated at drifts the two apart. + logger.warning( + "Encoding at the model's native %d fps instead of %d: %s generates audio " + "and video together, and another rate would desynchronise them.", + latents_pipeline_driver.video_fps, + fps, + type(latents_pipeline_driver).__name__, + ) + fps = latents_pipeline_driver.video_fps self._encode_video_output(output, temp_path, fps, audio=audio, audio_sample_rate=audio_sample_rate) self._publish_output_video(temp_path) finally: