From 0145ce0d5ee837937107f2d6039b139c90e8bd5e Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Thu, 16 Jul 2026 10:24:00 +0800 Subject: [PATCH 01/17] init dit --- diffsynth/configs/model_configs.py | 7 + diffsynth/models/wan_animate_2_dit.py | 1132 +++++++++++++++++++++++++ 2 files changed, 1139 insertions(+) create mode 100644 diffsynth/models/wan_animate_2_dit.py diff --git a/diffsynth/configs/model_configs.py b/diffsynth/configs/model_configs.py index 7f5b95ce..9906f0ca 100644 --- a/diffsynth/configs/model_configs.py +++ b/diffsynth/configs/model_configs.py @@ -81,6 +81,13 @@ ] wan_series = [ + { + # Example: ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors") + "model_hash": "4536c21ad8740ba78367af4216ae85bf", + "model_name": "wan_animate_2_dit", + "model_class": "diffsynth.models.wan_animate_2_dit.WanAnimate2Transformer", + "extra_kwargs": {}, + }, { # Example: ModelConfig(model_id="krea/krea-realtime-video", origin_file_pattern="krea-realtime-video-14b.safetensors") "model_hash": "5ec04e02b42d2580483ad69f4e76346a", diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py new file mode 100644 index 00000000..a95e7b2c --- /dev/null +++ b/diffsynth/models/wan_animate_2_dit.py @@ -0,0 +1,1132 @@ +# Author: Guangyuan Wang +# Faithfully ported from Wan-Animate-2 (wanxiang/models/wan_animate_2_model.py + attention.py). +# Modifications vs. source are limited to: (1) inlining the attention helpers, +# (2) dropping the `wanxiang.ops` context-parallel branches (single-GPU integration). +import numpy as np +import torch +import torch.nn as nn +import math +from torch.nn.attention.flex_attention import create_block_mask +from torch.nn.attention.flex_attention import flex_attention as flex_attention_func + + +try: + from flash_attn_interface import flash_attn_varlen_func + + FLASH_VER = 3 + +except ModuleNotFoundError: + try: + from flash_attn import flash_attn_varlen_func + + FLASH_VER = 2 + except ModuleNotFoundError: + flash_attn_varlen_func = None # in compatible with CPU machines + FLASH_VER = None + +print(f"[PreInfo] Use flash attention={FLASH_VER}") + +flex_attention_func = torch.compile( + flex_attention_func, + dynamic=False, + mode="max-autotune", + fullgraph=True, + backend="inductor" +) + + +def flash_attention( + q, + k, + v, + q_lens=None, + k_lens=None, + dropout_p=0.0, + softmax_scale=None, + q_scale=None, + causal=False, + window_size=(-1, -1), + deterministic=False, + dtype=torch.bfloat16, +): + """ + q: [B, Lq, Nq, C1]. + k: [B, Lk, Nk, C1]. + v: [B, Lk, Nk, C2]. Nq must be divisible by Nk. + q_lens: [B]. + k_lens: [B]. + dropout_p: float. Dropout probability. + softmax_scale: float. The scaling of QK^T before applying softmax. + causal: bool. Whether to apply causal attention mask. + window_size: (left right). If not (-1, -1), apply sliding window local attention. + deterministic: bool. If True, slightly slower and uses more memory. + dtype: torch.dtype. Apply when dtype of q/k/v is not float16/bfloat16. + """ + half_dtypes = (torch.float16, torch.bfloat16) + assert dtype in half_dtypes + assert q.device.type == "cuda" and q.size(-1) <= 256 + + # params + b, lq, lk, out_dtype = q.size(0), q.size(1), k.size(1), q.dtype + + def half(x): + return x if x.dtype in half_dtypes else x.to(dtype) + + # preprocess query + if q_lens is None: + q = half(q.flatten(0, 1)) + q_lens = torch.tensor([lq] * b, dtype=torch.int32).to( + device=q.device, non_blocking=True + ) + else: + q = half(torch.cat([u[:v] for u, v in zip(q, q_lens)])) + + # preprocess key, value + if k_lens is None: + k = half(k.flatten(0, 1)) + v = half(v.flatten(0, 1)) + k_lens = torch.tensor([lk] * b, dtype=torch.int32).to( + device=k.device, non_blocking=True + ) + else: + k = half(torch.cat([u[:v] for u, v in zip(k, k_lens)])) + v = half(torch.cat([u[:v] for u, v in zip(v, k_lens)])) + + q = q.to(v.dtype) + k = k.to(v.dtype) + + if q_scale is not None: + q = q * q_scale + # apply attention + if FLASH_VER == 3: + # Note: dropout_p, window_size are not supported in FA3 now. + x = flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]) + .cumsum(0, dtype=torch.int32) + .to(q.device, non_blocking=True), + cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]) + .cumsum(0, dtype=torch.int32) + .to(q.device, non_blocking=True), + max_seqlen_q=lq, + max_seqlen_k=lk, + softmax_scale=softmax_scale, + causal=causal, + deterministic=deterministic, + )[0].unflatten(0, (b, lq)) + else: + assert FLASH_VER == 2 + x = flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]) + .cumsum(0, dtype=torch.int32) + .to(q.device, non_blocking=True), + cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]) + .cumsum(0, dtype=torch.int32) + .to(q.device, non_blocking=True), + max_seqlen_q=lq, + max_seqlen_k=lk, + dropout_p=dropout_p, + softmax_scale=softmax_scale, + causal=causal, + window_size=window_size, + deterministic=deterministic, + ).unflatten(0, (b, lq)) + + # output + return x.type(out_dtype) + + +def flex_attention( + q, + k, + v, + q_lens=None, + k_lens=None, + block_mask=None, + kernel_options=None, + dtype=torch.bfloat16, + score_mod=None +): + """ + q: [B, Lq, Nq, C1]. + k: [B, Lk, Nk, C1]. + v: [B, Lk, Nk, C2]. Nq must be divisible by Nk. + q_lens: [B]. + k_lens: [B]. + dtype: torch.dtype. Apply when dtype of q/k/v is not float16/bfloat16. + """ + half_dtypes = (torch.float16, torch.bfloat16) + assert dtype in half_dtypes + assert q.device.type == 'cuda' + # params + b, lq, lk, out_dtype = q.size(0), q.size(1), k.size(1), q.dtype + def half(x): return x if x.dtype in half_dtypes else x.to(dtype) + + assert lq % 128 == 0, "q_len must be divisible by 128." + assert lk % 128 == 0, "k_len must be divisible by 128." + + # preprocess query + if q_lens is None: + q = half(q) + else: + q = half(q) + assert q_lens.max() == q_lens.min(), 'varlen of query is not supported' + + # preprocess key, value + if k_lens is None: + k, v = half(k), half(v) + else: + k, v = half(k), half(v) + assert k_lens.max() == k_lens.min(), 'varlen of key is not supported' + + q = q.to(v.dtype) + k = k.to(v.dtype) + + + x = flex_attention_func( + query=q.transpose(2,1), + key=k.transpose(2,1), + value=v.transpose(2,1), + block_mask=block_mask, + kernel_options=kernel_options, + score_mod=score_mod + ).transpose(2, 1) + + return x.type(out_dtype) + + +def sinusoidal_embedding_1d(dim, position): + # preprocess + assert dim % 2 == 0 + half = dim // 2 + position = position.type(torch.float64) + + # calculation + sinusoid = torch.outer( + position, torch.pow(10000, -torch.arange(half).to(position).div(half)) + ) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + return x + + +@torch.amp.autocast(device_type='cuda', enabled=False) +def rope_params(max_seq_len, dim, theta=10000, offset=0): + assert dim % 2 == 0 + freqs = torch.outer( + torch.arange(max_seq_len)+offset, + 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float64).div(dim)) + ) + freqs = torch.polar(torch.ones_like(freqs), freqs) + return freqs + + +@torch.amp.autocast(device_type='cuda', enabled=False) +def rope_apply(x, grid_sizes, freqs, time_stride=1): + n, c = x.size(2), x.size(3) // 2 + + # split freqs + freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1) + + # loop over samples + output = [] + for i, (f, h, w) in enumerate(grid_sizes.tolist()): + seq_len = f * h * w + + # precompute multipliers + x_i = torch.view_as_complex( + x[i, :seq_len].to(torch.float64).reshape(seq_len, n, -1, 2) + ) + freqs_i = torch.cat([ + freqs[0][:f*time_stride:time_stride].view(f, 1, 1, -1).expand(f, h, w, -1), + freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1), + freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1) + ], dim=-1).reshape(seq_len, 1, -1) + + # apply rotary embedding + x_i = torch.view_as_real(x_i * freqs_i).flatten(2) + x_i = torch.cat([x_i, x[i, seq_len:]]) + + # append to collection + output.append(x_i) + return torch.stack(output).float() + +def pad_freqs(original_tensor, target_len): + seq_len, s1, s2 = original_tensor.shape + pad_size = target_len - seq_len + padding_tensor = torch.ones( + pad_size, + s1, + s2, + dtype=original_tensor.dtype, + device=original_tensor.device) + padded_tensor = torch.cat([original_tensor, padding_tensor], dim=0) + return padded_tensor + + +class RMSNorm(nn.Module): + + def __init__(self, dim, eps=1e-5): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + return self._norm(x.float()).type_as(x) * self.weight + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) + + +class LayerNorm(nn.LayerNorm): + """ + LayerNorm without learnable affine parameters. + """ + + def __init__(self, dim, eps=1e-6, elementwise_affine=False): + super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) + + def forward(self, x): + return super().forward(x.float()).type_as(x) + + +class SelfAttention(nn.Module): + + def __init__( + self, + dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + eps=1e-6 + ): + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.eps = eps + + # layers + self.q = nn.Linear(dim, dim) + self.k = nn.Linear(dim, dim) + self.v = nn.Linear(dim, dim) + self.o = nn.Linear(dim, dim) + self.norm_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + self.norm_k = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + def forward(self, *args, method, **kwargs): + return getattr(self, method)(*args, **kwargs) + + def pre_attention(self, x): + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + + # query, key, value function + def qkv_fn(x): + q = self.norm_q(self.q(x)).view(b, s, n, d) + k = self.norm_k(self.k(x)).view(b, s, n, d) + v = self.v(x).view(b, s, n, d) + return q, k, v + + q, k, v = qkv_fn(x) + + return q, k, v + + def post_attention(self, x): + # output + x = x.flatten(2) + x = self.o(x) + return x + + +class CrossAttention(SelfAttention): + def __init__(self, dim, num_heads, window_size=(-1, -1), qk_norm=True, eps=1e-6, use_img_emb=True): + super().__init__(dim, num_heads, window_size, qk_norm, eps) + self.use_img_emb = use_img_emb + if use_img_emb: + self.k_img = nn.Linear(dim, dim) + self.v_img = nn.Linear(dim, dim) + self.norm_k_img = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + def forward(self, x, context, context_lens, counter=0): + """ + x: [B, L1, C]. + context: [B, L2, C]. + context_lens: [B]. + """ + if self.use_img_emb: + context_img = context[:, :257] + context = context[:, 257:] + else: + context = context + + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.q(x)).view(b, -1, n, d) + k = self.norm_k(self.k(context)).view(b, -1, n, d) + v = self.v(context).view(b, -1, n, d) + + if self.use_img_emb: + k_img = self.norm_k_img(self.k_img(context_img)).view(b, -1, n, d) + v_img = self.v_img(context_img).view(b, -1, n, d) + img_x = flash_attention(q, k_img, v_img, k_lens=None) + # compute attention + x = flash_attention(q, k, v, k_lens=context_lens) + + # output + x = x.flatten(2) + if self.use_img_emb: + img_x = img_x.flatten(2) + x = x + img_x + x = self.o(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__( + self, + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + use_img_emb=True + ): + super().__init__() + self.dim = dim + self.ffn_dim = ffn_dim + self.num_heads = num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + + # layers + self.norm1 = LayerNorm(dim, eps) + + self.self_attn = SelfAttention(dim, num_heads, window_size, qk_norm, eps) + + self.norm3 = LayerNorm( + dim, eps, elementwise_affine=True + ) if cross_attn_norm else nn.Identity() + + self.cross_attn = CrossAttention(dim, num_heads, (-1, -1), qk_norm, eps, use_img_emb=use_img_emb) + + self.norm2 = LayerNorm(dim, eps) + self.ffn = nn.Sequential( + nn.Linear(dim, ffn_dim), + nn.GELU(approximate='tanh'), + nn.Linear(ffn_dim, dim) + ) + # modulation + self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim ** 0.5) + + def forward(self, *args, method, **kwargs): + return getattr(self, method)(*args, **kwargs) + + def pre_self_attention(self, x, e): + assert e.dtype == torch.float32 + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + e = (self.modulation + e).chunk(6, dim=1) + assert e[0].dtype == torch.float32 + + q, k, v = self.self_attn(self.norm1(x).float() * (1 + e[1]) + e[0], method="pre_attention") + return q, k, v, e + + def post_self_attention(self, x): + x = self.self_attn(x, method="post_attention") + return x + + def cross_attention(self, x, context, context_lens, e): + x = x + self.cross_attn(self.norm3(x), context, context_lens) + y = self.ffn(self.norm2(x).float() * (1 + e[4]) + e[3]) + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + x = x + y * e[5] + return x + + +class Incontext_AttentionBlock(nn.Module): + def __init__( + self, + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + refer_stride=1, + use_img_emb=True, + use_context_parallel=False, + sparse_type=0 + ): + super().__init__() + self.dim = dim + self.ffn_dim = ffn_dim + self.num_heads = num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + self.refer_stride = refer_stride + self.use_context_parallel = use_context_parallel + self.sparse_type = sparse_type + + self.block = AttentionBlock( + dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps, use_img_emb=use_img_emb + ) + + def forward(self, *args, method, **kwargs): + return getattr(self, method)(*args, **kwargs) + + def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, grid_sizes_ref, e_ref, context_lens): + q_ref, k_ref, v_ref, e_ref = self.block(x_ref, e_ref, method='pre_self_attention') + + k_cache[index] = k_ref + v_cache[index] = v_ref + q_ref_add_rope = rope_apply(q_ref, grid_sizes_ref, freqs_ref, self.refer_stride) + k_ref_add_rope = rope_apply(k_ref, grid_sizes_ref, freqs_ref, self.refer_stride) + + ref_f, ref_h, ref_w = grid_sizes_ref[0].tolist() + ref_vail_len = ref_f * ref_h * ref_w + + xout_ref = flash_attention( + q=q_ref_add_rope, + k=k_ref_add_rope, + v=v_ref, + k_lens=torch.tensor([ref_vail_len], dtype=torch.long), + window_size=self.window_size + ) + + y_ref = self.block(xout_ref, method='post_self_attention') + + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + x_ref = x_ref + y_ref * e_ref[2] + + x_ref = self.block(x_ref, context_ref, context_lens, e_ref, method='cross_attention') + + return x_ref + + def forward_gen(self, x, index, k_cache, v_cache, block_mask, context, freqs, freqs_ref, grid_sizes, grid_sizes_ref, + origin_len, origin_area, e, context_lens): + + origin_latent_f = origin_len // 4 + 1 + origin_latent_hw = origin_area[0] * origin_area[1] // 256 + origin_max_len = (origin_latent_f + 1) * origin_latent_hw + origin_ref_max_len = origin_latent_f * origin_latent_hw + + f, h, w = grid_sizes[0].tolist() + vail_len = f * h * w + hw = h * w + + ref_f, ref_h, ref_w = grid_sizes_ref[0].tolist() + ref_vail_len = ref_f * ref_h * ref_w + ref_hw = ref_h * ref_w + + q, k, v, e = self.block(x, e, method='pre_self_attention') + + q = rope_apply(q, grid_sizes, freqs) + k = rope_apply(k, grid_sizes, freqs) + k_ref, v_ref = k_cache[index], v_cache[index] + k_ref = rope_apply(k_ref, grid_sizes_ref, freqs_ref, self.refer_stride) + + B, _, N, C = q.shape + device, dtype = q.device, q.dtype + + + target_q_len = math.ceil(origin_max_len / 128) * 128 + target_ref_len = math.ceil(origin_ref_max_len / 128) * 128 + target_kv_len = target_q_len + target_ref_len + + q_padding = q[:, vail_len:].clone() + + q_incontext = torch.zeros(B, target_q_len, N, C, device=device, dtype=dtype) + k_incontext = torch.zeros(B, target_kv_len, N, C, device=device, dtype=dtype) + v_incontext = torch.zeros(B, target_kv_len, N, C, device=device, dtype=dtype) + + q_src = q[:, :vail_len].view(B, f, hw, N, C) + k_src = k[:, :vail_len].view(B, f, hw, N, C) + v_src = v[:, :vail_len].view(B, f, hw, N, C) + + q_incontext[:, :f * origin_latent_hw].view(B, f, origin_latent_hw, N, C)[:, :, :hw] = q_src + k_incontext[:, :f * origin_latent_hw].view(B, f, origin_latent_hw, N, C)[:, :, :hw] = k_src + v_incontext[:, :f * origin_latent_hw].view(B, f, origin_latent_hw, N, C)[:, :, :hw] = v_src + + k_ref_src = k_ref[:, :ref_vail_len].view(B, ref_f, ref_hw, N, C) + v_ref_src = v_ref[:, :ref_vail_len].view(B, ref_f, ref_hw, N, C) + + k_incontext[:, target_q_len : target_q_len + ref_f * origin_latent_hw]\ + .view(B, ref_f, origin_latent_hw, N, C)[:, :, :ref_hw] = k_ref_src + v_incontext[:, target_q_len : target_q_len + ref_f * origin_latent_hw]\ + .view(B, ref_f, origin_latent_hw, N, C)[:, :, :ref_hw] = v_ref_src + + xout_full = flex_attention( + q=q_incontext, + k=k_incontext, + v=v_incontext, + block_mask=block_mask, + kernel_options=None + ) + + xout_valid = xout_full[:, :f * origin_latent_hw] + xout_valid = xout_valid.view(B, f, origin_latent_hw, N, C) + xout_vail = xout_valid[:, :, :hw] + xout_vail = xout_vail.reshape(B, f * hw, N, C) # [B, f*hw, N, C] + xout = torch.cat([xout_vail, q_padding], dim=1) + + y = self.block(xout, method='post_self_attention') + + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + x = x + y * e[2] + + x = self.block(x, context, context_lens, e, method='cross_attention') + return x + + +class Head(nn.Module): + + def __init__(self, dim, out_dim, patch_size, eps=1e-6): + super().__init__() + self.dim = dim + self.out_dim = out_dim + self.patch_size = patch_size + self.eps = eps + + # layers + out_dim = math.prod(patch_size) * out_dim + self.norm = LayerNorm(dim, eps) + self.head = nn.Linear(dim, out_dim) + + # modulation + self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5) + + def forward(self, x, e): + assert e.dtype == torch.float32 + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1) + x = self.head(self.norm(x) * (1 + e[1]) + e[0]) + return x + + +class MLPProj(torch.nn.Module): + def __init__(self, in_dim, out_dim): + super().__init__() + + self.proj = torch.nn.Sequential( + torch.nn.LayerNorm(in_dim), + torch.nn.Linear(in_dim, in_dim), + torch.nn.GELU(), + torch.nn.Linear(in_dim, out_dim), + torch.nn.LayerNorm(out_dim), + ) + + def forward(self, image_embeds): + clip_extra_context_tokens = self.proj(image_embeds) + return clip_extra_context_tokens + + +class WanAnimate2Transformer(nn.Module): + + def __init__( + self, + patch_size=(1, 2, 2), + text_len=512, + in_dim=36, + dim=5120, + ffn_dim=13824, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=40, + num_layers=40, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6, + use_img_emb=True, + refer_offset_t=1, + refer_offset_h=0, + refer_offset_w=-1, + refer_stride=1, + sparse_type=0, + use_context_parallel=False + ): + super().__init__() + self.patch_size = patch_size + self.text_len = text_len + self.in_dim = in_dim + self.dim = dim + self.ffn_dim = ffn_dim + self.freq_dim = freq_dim + self.text_dim = text_dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + self.use_img_emb = use_img_emb + self.refer_offset_t = refer_offset_t + self.refer_offset_h = refer_offset_h + self.refer_offset_w = refer_offset_w + self.refer_stride = refer_stride + self.sparse_type=sparse_type + self.use_context_parallel = use_context_parallel + + # [Denoising Transformer] + # embeddings + self.patch_embedding = nn.Conv3d( + in_dim, dim, kernel_size=patch_size, stride=patch_size + ) + self.text_embedding = nn.Sequential( + nn.Linear(text_dim, dim), + nn.GELU(approximate='tanh'), + nn.Linear(dim, dim) + ) + + self.time_embedding = nn.Sequential( + nn.Linear(freq_dim, dim), + nn.SiLU(), + nn.Linear(dim, dim) + ) + self.time_projection = nn.Sequential( + nn.SiLU(), + nn.Linear(dim, dim * 6) + ) + + # blocks + self.blocks = nn.ModuleList([Incontext_AttentionBlock( + dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps, refer_stride, use_img_emb=use_img_emb, use_context_parallel=self.use_context_parallel, + sparse_type=self.sparse_type + ) for _ in range(num_layers)]) + + # head + self.head = Head(dim, out_dim, patch_size, eps) + + if use_img_emb: + self.img_emb = MLPProj(1280, dim) + + # initialize weights + self.init_weights() + self.gradient_checkpointing = True + self.block_masks = dict() + self.block_mask_grid_sizes = dict() + + # Author: Guangyuan Wang + def create_mask(self, origin_len, origin_area, device): + origin_latent_f = origin_len // 4 + 1 + hw = int(np.prod(origin_area).item() // 256) + + q_len = (origin_latent_f + 1) * hw + k_len = origin_latent_f * hw + + q_len_total = math.ceil(q_len / 128) * 128 + k_extra_len_total = math.ceil(k_len / 128) * 128 + k_len_total = q_len_total + k_extra_len_total + + q_limit = q_len + k_limit = k_len + q_total = q_len_total + + def attention_mask_logic(b, h, q_idx, kv_idx): + q_valid = q_idx < q_limit + is_base_attention = kv_idx < q_limit + + q_frame = q_idx // hw + is_first_part = kv_idx < q_total + + kv_frame_1 = kv_idx // hw + kv_is_valid_1 = kv_idx < q_limit + + rel_kv_idx = kv_idx - q_total + kv_frame_2 = (rel_kv_idx // hw) + 1 + kv_is_valid_2 = rel_kv_idx < k_limit + + kv_frame = torch.where(is_first_part, kv_frame_1, kv_frame_2) + kv_is_valid = torch.where(is_first_part, kv_is_valid_1, kv_is_valid_2) + + is_cond_attention = (q_frame == kv_frame) & kv_is_valid + + return q_valid & (is_base_attention | is_cond_attention) + + block_mask = create_block_mask( + attention_mask_logic, + B=None, + H=None, + Q_LEN=q_len_total, + KV_LEN=k_len_total, + device=device, + _compile=True + ) + return block_mask + + + def forward(self, *args, method, **kwargs): + return getattr(self, method)(*args, **kwargs) + + def forward_ref( + self, + x_ref, + grid_sizes, + k_cache, + v_cache, + clip_fea_ref, + y_ref, + context_ref, + seq_len_ref, + t + ): + device = self.patch_embedding.weight.device + # [reference] + x_ref = [torch.cat([u, v], dim=0) for u, v in zip(x_ref, y_ref)] + # embeddings + x_ref = [self.patch_embedding(u.unsqueeze(0)) for u in x_ref] + grid_sizes_ref = torch.stack([ + torch.tensor(u.shape[2:], dtype=torch.long) for u in x_ref + ]) + x_ref = [u.flatten(2).transpose(1, 2) for u in x_ref] + seq_lens_ref = torch.tensor([u.size(1) for u in x_ref], dtype=torch.long) + assert seq_lens_ref.max() <= seq_len_ref + x_ref = torch.cat([torch.cat([ + u, u.new_zeros(1, seq_len_ref - u.size(1), u.size(2)) + ], dim=1) for u in x_ref]) + + assert (self.dim % self.num_heads) == 0 and (self.dim // self.num_heads) % 2 == 0 + d = self.dim // self.num_heads + + if self.refer_offset_t < 0: + self.refer_offset_t = grid_sizes[0][0].item() + if self.refer_offset_h < 0: + self.refer_offset_h = grid_sizes[0][1].item() + if self.refer_offset_w < 0: + self.refer_offset_w = grid_sizes[0][2].item() + + self.freqs_ref = torch.cat([ + rope_params(512, d - 4 * (d // 6), offset=self.refer_offset_t), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_h), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_w) + ], dim=1) + if self.freqs_ref.device != device: + self.freqs_ref = self.freqs_ref.to(device) + + # time embeddings ref + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + e_ref = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t*0+1).float() + ) + e0_ref = self.time_projection(e_ref).unflatten(1, (6, self.dim)) + assert e_ref.dtype == torch.float32 and e0_ref.dtype == torch.float32 + + # [context_ref] + context_ref = self.text_embedding(torch.stack([torch.cat([ + u, u.new_zeros(self.text_len - u.size(0), u.size(1)) + ]) for u in context_ref])) + + if self.use_img_emb: + context_clip_ref = self.img_emb(clip_fea_ref) # bs x 257 x dim + context_ref = torch.concat([context_clip_ref, context_ref], dim=1) + + context_lens = None + # arguments + kwargs = dict( + e_ref=e0_ref, + grid_sizes_ref=grid_sizes_ref, + freqs_ref=self.freqs_ref, + context_ref=context_ref, + context_lens=context_lens + ) + + for idx, block in enumerate(self.blocks): + x_ref = block(x_ref, idx, k_cache, v_cache, method='forward_ref', **kwargs) + + + def forward_gen( + self, + x, + k_cache, + v_cache, + clip_fea, + y, + context, + seq_len, + t, + grid_sizes_ref, + origin_len, + origin_area, + is_uncondtion=False, + ): + # [denoising] + # params + device = self.patch_embedding.weight.device + x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + grid_sizes = torch.stack([ + torch.tensor(u.shape[2:], dtype=torch.long) for u in x + ]) + x = [u.flatten(2).transpose(1, 2) for u in x] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + assert seq_lens.max() <= seq_len + x = torch.cat([torch.cat([ + u, u.new_zeros(1, seq_len - u.size(1), u.size(2)) + ], dim=1) for u in x]) + + assert (self.dim % self.num_heads) == 0 and (self.dim // self.num_heads) % 2 == 0 + d = self.dim // self.num_heads + self.freqs = torch.cat([ + rope_params(512, d - 4 * (d // 6)), + rope_params(512, 2 * (d // 6)), + rope_params(512, 2 * (d // 6)) + ], dim=1) + if self.freqs.device != device: + self.freqs = self.freqs.to(device) + + if self.refer_offset_t < 0: + self.refer_offset_t = grid_sizes[0][0].item() + if self.refer_offset_h < 0: + self.refer_offset_h = grid_sizes[0][1].item() + if self.refer_offset_w < 0: + self.refer_offset_w = grid_sizes[0][2].item() + + self.freqs_ref = torch.cat([ + rope_params(512, d - 4 * (d // 6), offset=self.refer_offset_t), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_h), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_w) + ], dim=1) + if self.freqs_ref.device != device: + self.freqs_ref = self.freqs_ref.to(device) + + # time embeddings + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t).float() + ) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + assert e.dtype == torch.float32 and e0.dtype == torch.float32 + + # [context] + context_lens = None + context = self.text_embedding(torch.stack([torch.cat([ + u, u.new_zeros(self.text_len - u.size(0), u.size(1)) + ]) for u in context])) + + if self.use_img_emb: + context_clip = self.img_emb(clip_fea) # bs x 257 x dim + context = torch.concat([context_clip, context], dim=1) + + block_mask_id = (origin_len, origin_area[0], origin_area[1]) + if block_mask_id not in self.block_masks: + self.block_masks[block_mask_id] = self.create_mask(origin_len, origin_area, x.device) + block_mask = self.block_masks[block_mask_id] + + # arguments + kwargs = dict( + e=e0, + block_mask=block_mask, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + grid_sizes_ref=grid_sizes_ref, + freqs_ref=self.freqs_ref, + context_lens=context_lens, + origin_area=origin_area, + origin_len=origin_len + ) + + for idx, block in enumerate(self.blocks): + if is_uncondtion and idx==9: + continue + x = block(x, idx, k_cache, v_cache, method='forward_gen', **kwargs) + + # head + x = self.head(x, e) + + # unpatchify + x = self.unpatchify(x, grid_sizes) + return [u.float() for u in x] + + def forward_origin( + self, + x, + clip_fea, + y, + context, + seq_len, + x_ref, + clip_fea_ref, + y_ref, + context_ref, + seq_len_ref, + t, + is_uncondition=False, + ): + """ + x: A list of videos each with shape [C, T, H, W]. + context: A list of text embeddings each with shape [L, C]. + x_ref: A list of reference videos each with shape [C, T, H, W]. + context_ref: A list of reference text embeddings each with shape [L, C]. + t: [B]. + """ + + # [denoising] + # params + device = self.patch_embedding.weight.device + x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + grid_sizes = torch.stack([ + torch.tensor(u.shape[2:], dtype=torch.long) for u in x + ]) + x = [u.flatten(2).transpose(1, 2) for u in x] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + assert seq_lens.max() <= seq_len + x = torch.cat([torch.cat([ + u, u.new_zeros(1, seq_len - u.size(1), u.size(2)) + ], dim=1) for u in x]) + + # [reference] + # params + x_ref = [torch.cat([u, v], dim=0) for u, v in zip(x_ref, y_ref)] + # embeddings + x_ref = [self.patch_embedding(u.unsqueeze(0)) for u in x_ref] + grid_sizes_ref = torch.stack([ + torch.tensor(u.shape[2:], dtype=torch.long) for u in x_ref + ]) + x_ref = [u.flatten(2).transpose(1, 2) for u in x_ref] + seq_lens_ref = torch.tensor([u.size(1) for u in x_ref], dtype=torch.long) + assert seq_lens_ref.max() <= seq_len_ref + x_ref = torch.cat([torch.cat([ + u, u.new_zeros(1, seq_len_ref - u.size(1), u.size(2)) + ], dim=1) for u in x_ref]) + + assert (self.dim % self.num_heads) == 0 and (self.dim // self.num_heads) % 2 == 0 + d = self.dim // self.num_heads + self.freqs = torch.cat([ + rope_params(512, d - 4 * (d // 6)), + rope_params(512, 2 * (d // 6)), + rope_params(512, 2 * (d // 6)) + ], dim=1) + if self.freqs.device != device: + self.freqs = self.freqs.to(device) + + if self.refer_offset_t < 0: + self.refer_offset_t = grid_sizes[0][0].item() + if self.refer_offset_h < 0: + self.refer_offset_h = grid_sizes[0][1].item() + if self.refer_offset_w < 0: + self.refer_offset_w = grid_sizes[0][2].item() + + self.freqs_ref = torch.cat([ + rope_params(512, d - 4 * (d // 6), offset=self.refer_offset_t), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_h), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_w) + ], dim=1) + if self.freqs_ref.device != device: + self.freqs_ref = self.freqs_ref.to(device) + + # time embeddings + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t).float() + ) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + assert e.dtype == torch.float32 and e0.dtype == torch.float32 + + # time embeddings ref + with torch.amp.autocast(device_type='cuda', dtype=torch.float32): + e_ref = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t*0+1).float() + ) + e0_ref = self.time_projection(e_ref).unflatten(1, (6, self.dim)) + assert e_ref.dtype == torch.float32 and e0_ref.dtype == torch.float32 + + # [context] + context_lens = None + context = self.text_embedding(torch.stack([torch.cat([ + u, u.new_zeros(self.text_len - u.size(0), u.size(1)) + ]) for u in context])) + + if self.use_img_emb: + context_clip = self.img_emb(clip_fea) # bs x 257 x dim + context = torch.concat([context_clip, context], dim=1) + + # [context_ref] + context_ref = self.text_embedding(torch.stack([torch.cat([ + u, u.new_zeros(self.text_len - u.size(0), u.size(1)) + ]) for u in context_ref])) + + if self.use_img_emb: + context_clip_ref = self.img_emb(clip_fea_ref) # bs x 257 x dim + context_ref = torch.concat([context_clip_ref, context_ref], dim=1) + + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + e_ref=e0_ref, + seq_lens_ref=seq_lens_ref, + grid_sizes_ref=grid_sizes_ref, + freqs_ref=self.freqs_ref, + context_ref=context_ref, + context_lens=context_lens + ) + + for idx, block in enumerate(self.blocks): + if is_uncondition and idx==9: + continue + x, x_ref = block(x, x_ref, method='forward_origin', **kwargs) + + # head + x = self.head(x, e) + + # unpatchify + x = self.unpatchify(x, grid_sizes) + return [u.float() for u in x] + + def unpatchify(self, x, grid_sizes): + c = self.out_dim + out = [] + for u, v in zip(x, grid_sizes.tolist()): + u = u[:math.prod(v)].view(*v, *self.patch_size, c) + u = torch.einsum('fhwpqrc->cfphqwr', u) + u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) + out.append(u) + return out + + def init_weights(self): + # basic init + for m in self.modules(): + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.zeros_(m.bias) + + # init embeddings + nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1)) + for m in self.text_embedding.modules(): + if isinstance(m, nn.Linear): + nn.init.normal_(m.weight, std=.02) + for m in self.time_embedding.modules(): + if isinstance(m, nn.Linear): + nn.init.normal_(m.weight, std=.02) + + # init output layer + nn.init.zeros_(self.head.head.weight) + + +Transformer = WanAnimate2Transformer From 522cd0cfd9f876848fe49a6c85ee3be48e7fb14a Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Thu, 16 Jul 2026 12:46:34 +0800 Subject: [PATCH 02/17] attention reimplement --- diffsynth/models/wan_animate_2_dit.py | 368 ++------------------------ 1 file changed, 18 insertions(+), 350 deletions(-) diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index a95e7b2c..61b601f8 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -1,31 +1,13 @@ -# Author: Guangyuan Wang -# Faithfully ported from Wan-Animate-2 (wanxiang/models/wan_animate_2_model.py + attention.py). -# Modifications vs. source are limited to: (1) inlining the attention helpers, -# (2) dropping the `wanxiang.ops` context-parallel branches (single-GPU integration). import numpy as np import torch import torch.nn as nn import math from torch.nn.attention.flex_attention import create_block_mask from torch.nn.attention.flex_attention import flex_attention as flex_attention_func +from .wan_video_dit import sinusoidal_embedding_1d, RMSNorm, MLP +from ..core.attention.attention import attention_forward -try: - from flash_attn_interface import flash_attn_varlen_func - - FLASH_VER = 3 - -except ModuleNotFoundError: - try: - from flash_attn import flash_attn_varlen_func - - FLASH_VER = 2 - except ModuleNotFoundError: - flash_attn_varlen_func = None # in compatible with CPU machines - FLASH_VER = None - -print(f"[PreInfo] Use flash attention={FLASH_VER}") - flex_attention_func = torch.compile( flex_attention_func, dynamic=False, @@ -35,112 +17,6 @@ ) -def flash_attention( - q, - k, - v, - q_lens=None, - k_lens=None, - dropout_p=0.0, - softmax_scale=None, - q_scale=None, - causal=False, - window_size=(-1, -1), - deterministic=False, - dtype=torch.bfloat16, -): - """ - q: [B, Lq, Nq, C1]. - k: [B, Lk, Nk, C1]. - v: [B, Lk, Nk, C2]. Nq must be divisible by Nk. - q_lens: [B]. - k_lens: [B]. - dropout_p: float. Dropout probability. - softmax_scale: float. The scaling of QK^T before applying softmax. - causal: bool. Whether to apply causal attention mask. - window_size: (left right). If not (-1, -1), apply sliding window local attention. - deterministic: bool. If True, slightly slower and uses more memory. - dtype: torch.dtype. Apply when dtype of q/k/v is not float16/bfloat16. - """ - half_dtypes = (torch.float16, torch.bfloat16) - assert dtype in half_dtypes - assert q.device.type == "cuda" and q.size(-1) <= 256 - - # params - b, lq, lk, out_dtype = q.size(0), q.size(1), k.size(1), q.dtype - - def half(x): - return x if x.dtype in half_dtypes else x.to(dtype) - - # preprocess query - if q_lens is None: - q = half(q.flatten(0, 1)) - q_lens = torch.tensor([lq] * b, dtype=torch.int32).to( - device=q.device, non_blocking=True - ) - else: - q = half(torch.cat([u[:v] for u, v in zip(q, q_lens)])) - - # preprocess key, value - if k_lens is None: - k = half(k.flatten(0, 1)) - v = half(v.flatten(0, 1)) - k_lens = torch.tensor([lk] * b, dtype=torch.int32).to( - device=k.device, non_blocking=True - ) - else: - k = half(torch.cat([u[:v] for u, v in zip(k, k_lens)])) - v = half(torch.cat([u[:v] for u, v in zip(v, k_lens)])) - - q = q.to(v.dtype) - k = k.to(v.dtype) - - if q_scale is not None: - q = q * q_scale - # apply attention - if FLASH_VER == 3: - # Note: dropout_p, window_size are not supported in FA3 now. - x = flash_attn_varlen_func( - q=q, - k=k, - v=v, - cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]) - .cumsum(0, dtype=torch.int32) - .to(q.device, non_blocking=True), - cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]) - .cumsum(0, dtype=torch.int32) - .to(q.device, non_blocking=True), - max_seqlen_q=lq, - max_seqlen_k=lk, - softmax_scale=softmax_scale, - causal=causal, - deterministic=deterministic, - )[0].unflatten(0, (b, lq)) - else: - assert FLASH_VER == 2 - x = flash_attn_varlen_func( - q=q, - k=k, - v=v, - cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]) - .cumsum(0, dtype=torch.int32) - .to(q.device, non_blocking=True), - cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]) - .cumsum(0, dtype=torch.int32) - .to(q.device, non_blocking=True), - max_seqlen_q=lq, - max_seqlen_k=lk, - dropout_p=dropout_p, - softmax_scale=softmax_scale, - causal=causal, - window_size=window_size, - deterministic=deterministic, - ).unflatten(0, (b, lq)) - - # output - return x.type(out_dtype) - - def flex_attention( q, k, @@ -200,20 +76,6 @@ def half(x): return x if x.dtype in half_dtypes else x.to(dtype) return x.type(out_dtype) -def sinusoidal_embedding_1d(dim, position): - # preprocess - assert dim % 2 == 0 - half = dim // 2 - position = position.type(torch.float64) - - # calculation - sinusoid = torch.outer( - position, torch.pow(10000, -torch.arange(half).to(position).div(half)) - ) - x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) - return x - - @torch.amp.autocast(device_type='cuda', enabled=False) def rope_params(max_seq_len, dim, theta=10000, offset=0): assert dim % 2 == 0 @@ -268,21 +130,6 @@ def pad_freqs(original_tensor, target_len): return padded_tensor -class RMSNorm(nn.Module): - - def __init__(self, dim, eps=1e-5): - super().__init__() - self.dim = dim - self.eps = eps - self.weight = nn.Parameter(torch.ones(dim)) - - def forward(self, x): - return self._norm(x.float()).type_as(x) * self.weight - - def _norm(self, x): - return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) - - class LayerNorm(nn.LayerNorm): """ LayerNorm without learnable affine parameters. @@ -377,9 +224,15 @@ def forward(self, x, context, context_lens, counter=0): if self.use_img_emb: k_img = self.norm_k_img(self.k_img(context_img)).view(b, -1, n, d) v_img = self.v_img(context_img).view(b, -1, n, d) - img_x = flash_attention(q, k_img, v_img, k_lens=None) + img_x = attention_forward( + q, k_img, v_img, + q_pattern="b s n d", k_pattern="b s n d", v_pattern="b s n d", out_pattern="b s n d", + ) # compute attention - x = flash_attention(q, k, v, k_lens=context_lens) + x = attention_forward( + q, k, v, + q_pattern="b s n d", k_pattern="b s n d", v_pattern="b s n d", out_pattern="b s n d", + ) # output x = x.flatten(2) @@ -499,15 +352,14 @@ def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, gr k_ref_add_rope = rope_apply(k_ref, grid_sizes_ref, freqs_ref, self.refer_stride) ref_f, ref_h, ref_w = grid_sizes_ref[0].tolist() - ref_vail_len = ref_f * ref_h * ref_w + ref_vail_len = ref_f * ref_h * ref_w - xout_ref = flash_attention( - q=q_ref_add_rope, - k=k_ref_add_rope, - v=v_ref, - k_lens=torch.tensor([ref_vail_len], dtype=torch.long), - window_size=self.window_size - ) + xout_ref = attention_forward( + q_ref_add_rope.to(v_ref.dtype), + k_ref_add_rope[:, :ref_vail_len].to(v_ref.dtype), + v_ref[:, :ref_vail_len], + q_pattern="b s n d", k_pattern="b s n d", v_pattern="b s n d", out_pattern="b s n d", + ).to(q_ref_add_rope.dtype) y_ref = self.block(xout_ref, method='post_self_attention') @@ -619,23 +471,6 @@ def forward(self, x, e): return x -class MLPProj(torch.nn.Module): - def __init__(self, in_dim, out_dim): - super().__init__() - - self.proj = torch.nn.Sequential( - torch.nn.LayerNorm(in_dim), - torch.nn.Linear(in_dim, in_dim), - torch.nn.GELU(), - torch.nn.Linear(in_dim, out_dim), - torch.nn.LayerNorm(out_dim), - ) - - def forward(self, image_embeds): - clip_extra_context_tokens = self.proj(image_embeds) - return clip_extra_context_tokens - - class WanAnimate2Transformer(nn.Module): def __init__( @@ -716,10 +551,8 @@ def __init__( self.head = Head(dim, out_dim, patch_size, eps) if use_img_emb: - self.img_emb = MLPProj(1280, dim) + self.img_emb = MLP(1280, dim, has_pos_emb=False) - # initialize weights - self.init_weights() self.gradient_checkpointing = True self.block_masks = dict() self.block_mask_grid_sizes = dict() @@ -867,7 +700,6 @@ def forward_gen( origin_area, is_uncondtion=False, ): - # [denoising] # params device = self.patch_embedding.weight.device x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] @@ -957,147 +789,6 @@ def forward_gen( x = self.unpatchify(x, grid_sizes) return [u.float() for u in x] - def forward_origin( - self, - x, - clip_fea, - y, - context, - seq_len, - x_ref, - clip_fea_ref, - y_ref, - context_ref, - seq_len_ref, - t, - is_uncondition=False, - ): - """ - x: A list of videos each with shape [C, T, H, W]. - context: A list of text embeddings each with shape [L, C]. - x_ref: A list of reference videos each with shape [C, T, H, W]. - context_ref: A list of reference text embeddings each with shape [L, C]. - t: [B]. - """ - - # [denoising] - # params - device = self.patch_embedding.weight.device - x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] - # embeddings - x = [self.patch_embedding(u.unsqueeze(0)) for u in x] - grid_sizes = torch.stack([ - torch.tensor(u.shape[2:], dtype=torch.long) for u in x - ]) - x = [u.flatten(2).transpose(1, 2) for u in x] - seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) - assert seq_lens.max() <= seq_len - x = torch.cat([torch.cat([ - u, u.new_zeros(1, seq_len - u.size(1), u.size(2)) - ], dim=1) for u in x]) - - # [reference] - # params - x_ref = [torch.cat([u, v], dim=0) for u, v in zip(x_ref, y_ref)] - # embeddings - x_ref = [self.patch_embedding(u.unsqueeze(0)) for u in x_ref] - grid_sizes_ref = torch.stack([ - torch.tensor(u.shape[2:], dtype=torch.long) for u in x_ref - ]) - x_ref = [u.flatten(2).transpose(1, 2) for u in x_ref] - seq_lens_ref = torch.tensor([u.size(1) for u in x_ref], dtype=torch.long) - assert seq_lens_ref.max() <= seq_len_ref - x_ref = torch.cat([torch.cat([ - u, u.new_zeros(1, seq_len_ref - u.size(1), u.size(2)) - ], dim=1) for u in x_ref]) - - assert (self.dim % self.num_heads) == 0 and (self.dim // self.num_heads) % 2 == 0 - d = self.dim // self.num_heads - self.freqs = torch.cat([ - rope_params(512, d - 4 * (d // 6)), - rope_params(512, 2 * (d // 6)), - rope_params(512, 2 * (d // 6)) - ], dim=1) - if self.freqs.device != device: - self.freqs = self.freqs.to(device) - - if self.refer_offset_t < 0: - self.refer_offset_t = grid_sizes[0][0].item() - if self.refer_offset_h < 0: - self.refer_offset_h = grid_sizes[0][1].item() - if self.refer_offset_w < 0: - self.refer_offset_w = grid_sizes[0][2].item() - - self.freqs_ref = torch.cat([ - rope_params(512, d - 4 * (d // 6), offset=self.refer_offset_t), - rope_params(512, 2 * (d // 6), offset=self.refer_offset_h), - rope_params(512, 2 * (d // 6), offset=self.refer_offset_w) - ], dim=1) - if self.freqs_ref.device != device: - self.freqs_ref = self.freqs_ref.to(device) - - # time embeddings - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - e = self.time_embedding( - sinusoidal_embedding_1d(self.freq_dim, t).float() - ) - e0 = self.time_projection(e).unflatten(1, (6, self.dim)) - assert e.dtype == torch.float32 and e0.dtype == torch.float32 - - # time embeddings ref - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - e_ref = self.time_embedding( - sinusoidal_embedding_1d(self.freq_dim, t*0+1).float() - ) - e0_ref = self.time_projection(e_ref).unflatten(1, (6, self.dim)) - assert e_ref.dtype == torch.float32 and e0_ref.dtype == torch.float32 - - # [context] - context_lens = None - context = self.text_embedding(torch.stack([torch.cat([ - u, u.new_zeros(self.text_len - u.size(0), u.size(1)) - ]) for u in context])) - - if self.use_img_emb: - context_clip = self.img_emb(clip_fea) # bs x 257 x dim - context = torch.concat([context_clip, context], dim=1) - - # [context_ref] - context_ref = self.text_embedding(torch.stack([torch.cat([ - u, u.new_zeros(self.text_len - u.size(0), u.size(1)) - ]) for u in context_ref])) - - if self.use_img_emb: - context_clip_ref = self.img_emb(clip_fea_ref) # bs x 257 x dim - context_ref = torch.concat([context_clip_ref, context_ref], dim=1) - - # arguments - kwargs = dict( - e=e0, - seq_lens=seq_lens, - grid_sizes=grid_sizes, - freqs=self.freqs, - context=context, - e_ref=e0_ref, - seq_lens_ref=seq_lens_ref, - grid_sizes_ref=grid_sizes_ref, - freqs_ref=self.freqs_ref, - context_ref=context_ref, - context_lens=context_lens - ) - - for idx, block in enumerate(self.blocks): - if is_uncondition and idx==9: - continue - x, x_ref = block(x, x_ref, method='forward_origin', **kwargs) - - # head - x = self.head(x, e) - - # unpatchify - x = self.unpatchify(x, grid_sizes) - return [u.float() for u in x] - def unpatchify(self, x, grid_sizes): c = self.out_dim out = [] @@ -1107,26 +798,3 @@ def unpatchify(self, x, grid_sizes): u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) out.append(u) return out - - def init_weights(self): - # basic init - for m in self.modules(): - if isinstance(m, nn.Linear): - nn.init.xavier_uniform_(m.weight) - if m.bias is not None: - nn.init.zeros_(m.bias) - - # init embeddings - nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1)) - for m in self.text_embedding.modules(): - if isinstance(m, nn.Linear): - nn.init.normal_(m.weight, std=.02) - for m in self.time_embedding.modules(): - if isinstance(m, nn.Linear): - nn.init.normal_(m.weight, std=.02) - - # init output layer - nn.init.zeros_(self.head.head.weight) - - -Transformer = WanAnimate2Transformer From 8ec26f08b8aff2c18edec9d1f581d59cc625fc2b Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Thu, 16 Jul 2026 13:13:56 +0800 Subject: [PATCH 03/17] gradientcheckpointing --- diffsynth/models/wan_animate_2_dit.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 61b601f8..801e381b 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -6,6 +6,7 @@ from torch.nn.attention.flex_attention import flex_attention as flex_attention_func from .wan_video_dit import sinusoidal_embedding_1d, RMSNorm, MLP from ..core.attention.attention import attention_forward +from ..core.gradient import gradient_checkpoint_forward flex_attention_func = torch.compile( @@ -553,7 +554,6 @@ def __init__( if use_img_emb: self.img_emb = MLP(1280, dim, has_pos_emb=False) - self.gradient_checkpointing = True self.block_masks = dict() self.block_mask_grid_sizes = dict() @@ -619,7 +619,9 @@ def forward_ref( y_ref, context_ref, seq_len_ref, - t + t, + use_gradient_checkpointing: bool = False, + use_gradient_checkpointing_offload: bool = False, ): device = self.patch_embedding.weight.device # [reference] @@ -682,7 +684,12 @@ def forward_ref( ) for idx, block in enumerate(self.blocks): - x_ref = block(x_ref, idx, k_cache, v_cache, method='forward_ref', **kwargs) + x_ref = gradient_checkpoint_forward( + block, + use_gradient_checkpointing, + use_gradient_checkpointing_offload, + x_ref, idx, k_cache, v_cache, method='forward_ref', **kwargs + ) def forward_gen( @@ -699,6 +706,8 @@ def forward_gen( origin_len, origin_area, is_uncondtion=False, + use_gradient_checkpointing: bool = False, + use_gradient_checkpointing_offload: bool = False, ): # params device = self.patch_embedding.weight.device @@ -780,7 +789,12 @@ def forward_gen( for idx, block in enumerate(self.blocks): if is_uncondtion and idx==9: continue - x = block(x, idx, k_cache, v_cache, method='forward_gen', **kwargs) + x = gradient_checkpoint_forward( + block, + use_gradient_checkpointing, + use_gradient_checkpointing_offload, + x, idx, k_cache, v_cache, method='forward_gen', **kwargs + ) # head x = self.head(x, e) From f1cf6bf5c52015926776fb866c6faf697d6b2eb4 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Thu, 16 Jul 2026 13:42:29 +0800 Subject: [PATCH 04/17] resolve flex --- diffsynth/models/wan_animate_2_dit.py | 102 ++++---------------------- 1 file changed, 13 insertions(+), 89 deletions(-) diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 801e381b..7e863529 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -2,81 +2,11 @@ import torch import torch.nn as nn import math -from torch.nn.attention.flex_attention import create_block_mask -from torch.nn.attention.flex_attention import flex_attention as flex_attention_func from .wan_video_dit import sinusoidal_embedding_1d, RMSNorm, MLP from ..core.attention.attention import attention_forward from ..core.gradient import gradient_checkpoint_forward -flex_attention_func = torch.compile( - flex_attention_func, - dynamic=False, - mode="max-autotune", - fullgraph=True, - backend="inductor" -) - - -def flex_attention( - q, - k, - v, - q_lens=None, - k_lens=None, - block_mask=None, - kernel_options=None, - dtype=torch.bfloat16, - score_mod=None -): - """ - q: [B, Lq, Nq, C1]. - k: [B, Lk, Nk, C1]. - v: [B, Lk, Nk, C2]. Nq must be divisible by Nk. - q_lens: [B]. - k_lens: [B]. - dtype: torch.dtype. Apply when dtype of q/k/v is not float16/bfloat16. - """ - half_dtypes = (torch.float16, torch.bfloat16) - assert dtype in half_dtypes - assert q.device.type == 'cuda' - # params - b, lq, lk, out_dtype = q.size(0), q.size(1), k.size(1), q.dtype - def half(x): return x if x.dtype in half_dtypes else x.to(dtype) - - assert lq % 128 == 0, "q_len must be divisible by 128." - assert lk % 128 == 0, "k_len must be divisible by 128." - - # preprocess query - if q_lens is None: - q = half(q) - else: - q = half(q) - assert q_lens.max() == q_lens.min(), 'varlen of query is not supported' - - # preprocess key, value - if k_lens is None: - k, v = half(k), half(v) - else: - k, v = half(k), half(v) - assert k_lens.max() == k_lens.min(), 'varlen of key is not supported' - - q = q.to(v.dtype) - k = k.to(v.dtype) - - - x = flex_attention_func( - query=q.transpose(2,1), - key=k.transpose(2,1), - value=v.transpose(2,1), - block_mask=block_mask, - kernel_options=kernel_options, - score_mod=score_mod - ).transpose(2, 1) - - return x.type(out_dtype) - - @torch.amp.autocast(device_type='cuda', enabled=False) def rope_params(max_seq_len, dim, theta=10000, offset=0): assert dim % 2 == 0 @@ -371,7 +301,7 @@ def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, gr return x_ref - def forward_gen(self, x, index, k_cache, v_cache, block_mask, context, freqs, freqs_ref, grid_sizes, grid_sizes_ref, + def forward_gen(self, x, index, k_cache, v_cache, attn_mask, context, freqs, freqs_ref, grid_sizes, grid_sizes_ref, origin_len, origin_area, e, context_lens): origin_latent_f = origin_len // 4 + 1 @@ -424,12 +354,10 @@ def forward_gen(self, x, index, k_cache, v_cache, block_mask, context, freqs, fr v_incontext[:, target_q_len : target_q_len + ref_f * origin_latent_hw]\ .view(B, ref_f, origin_latent_hw, N, C)[:, :, :ref_hw] = v_ref_src - xout_full = flex_attention( - q=q_incontext, - k=k_incontext, - v=v_incontext, - block_mask=block_mask, - kernel_options=None + xout_full = attention_forward( + q_incontext, k_incontext, v_incontext, + q_pattern="b s n d", k_pattern="b s n d", v_pattern="b s n d", out_pattern="b s n d", + attn_mask=attn_mask, ) xout_valid = xout_full[:, :f * origin_latent_hw] @@ -594,16 +522,12 @@ def attention_mask_logic(b, h, q_idx, kv_idx): return q_valid & (is_base_attention | is_cond_attention) - block_mask = create_block_mask( - attention_mask_logic, - B=None, - H=None, - Q_LEN=q_len_total, - KV_LEN=k_len_total, - device=device, - _compile=True - ) - return block_mask + # Materialize the flex mask_fn as a dense boolean mask for torch SDPA. + # True => attention allowed (same convention as flex block_mask / SDPA bool attn_mask). + q_idx = torch.arange(q_len_total, device=device)[:, None] + kv_idx = torch.arange(k_len_total, device=device)[None, :] + attn_mask = attention_mask_logic(None, None, q_idx, kv_idx) + return attn_mask[None, None] # [1, 1, q_len_total, k_len_total] def forward(self, *args, method, **kwargs): @@ -770,12 +694,12 @@ def forward_gen( block_mask_id = (origin_len, origin_area[0], origin_area[1]) if block_mask_id not in self.block_masks: self.block_masks[block_mask_id] = self.create_mask(origin_len, origin_area, x.device) - block_mask = self.block_masks[block_mask_id] + attn_mask = self.block_masks[block_mask_id] # arguments kwargs = dict( e=e0, - block_mask=block_mask, + attn_mask=attn_mask, grid_sizes=grid_sizes, freqs=self.freqs, context=context, From 8477c0d2291c4342ce28ef07687c517059d34fc0 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Thu, 16 Jul 2026 17:10:30 +0800 Subject: [PATCH 05/17] wan animate pipeline code --- diffsynth/configs/model_configs.py | 2 +- diffsynth/models/wan_animate_2_dit.py | 14 +- diffsynth/pipelines/wan_video.py | 214 +++++++++++++++++- .../wanvideo/model_inference/Wan-Animate-2.py | 46 ++++ 4 files changed, 268 insertions(+), 8 deletions(-) create mode 100644 examples/wanvideo/model_inference/Wan-Animate-2.py diff --git a/diffsynth/configs/model_configs.py b/diffsynth/configs/model_configs.py index 9906f0ca..a048c73b 100644 --- a/diffsynth/configs/model_configs.py +++ b/diffsynth/configs/model_configs.py @@ -84,7 +84,7 @@ { # Example: ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors") "model_hash": "4536c21ad8740ba78367af4216ae85bf", - "model_name": "wan_animate_2_dit", + "model_name": "wan_video_dit", "model_class": "diffsynth.models.wan_animate_2_dit.WanAnimate2Transformer", "extra_kwargs": {}, }, diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 7e863529..61864976 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -62,9 +62,6 @@ def pad_freqs(original_tensor, target_len): class LayerNorm(nn.LayerNorm): - """ - LayerNorm without learnable affine parameters. - """ def __init__(self, dim, eps=1e-6, elementwise_affine=False): super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) @@ -529,7 +526,6 @@ def attention_mask_logic(b, h, q_idx, kv_idx): attn_mask = attention_mask_logic(None, None, q_idx, kv_idx) return attn_mask[None, None] # [1, 1, q_len_total, k_len_total] - def forward(self, *args, method, **kwargs): return getattr(self, method)(*args, **kwargs) @@ -736,3 +732,13 @@ def unpatchify(self, x, grid_sizes): u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) out.append(u) return out + + +def _animate2_get_i2v_mask(lat_t, lat_h, lat_w, mask_len=1, device="cuda"): + # Mirror of wanxiang.eval_i2v.get_i2v_mask (Wan-Animate-2 target library). + msk = torch.zeros(1, (lat_t - 1) * 4 + 1, lat_h, lat_w, device=device) + msk[:, :mask_len] = 1 + msk = torch.concat([torch.repeat_interleave(msk[:, 0:1], repeats=4, dim=1), msk[:, 1:]], dim=1) + msk = msk.view(1, msk.shape[1] // 4, 4, lat_h, lat_w) + msk = msk.transpose(1, 2)[0] + return msk diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index c1e4dfb3..0ce3272d 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -1,4 +1,4 @@ -import torch, types +import torch, types, math import numpy as np from PIL import Image from einops import repeat @@ -27,6 +27,7 @@ from ..models.wan_video_mot import MotWanModel from ..models.wav2vec import WanS2VAudioEncoder from ..models.longcat_video_dit import LongCatVideoTransformer3DModel +from ..models.wan_animate_2_dit import WanAnimate2Transformer, _animate2_get_i2v_mask class WanVideoPipeline(BasePipeline): @@ -78,6 +79,10 @@ def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16): WanVideoUnit_WanToDance_ProcessInputs(), WanVideoUnit_WanToDance_RefImageEmbedder(), WanVideoUnit_WanToDance_ImageKeyframesEmbedder(), + WanVideoUnit_Animate2RefPrompt(), + WanVideoUnit_Animate2Clip(), + WanVideoUnit_Animate2VAEEmbedder(), + WanVideoUnit_Animate2RefKVCache(), ] self.post_units = [ WanVideoPostUnit_S2V(), @@ -223,6 +228,10 @@ def __call__( animate_face_video: list[Image.Image] = None, animate_inpaint_video: list[Image.Image] = None, animate_mask_video: list[Image.Image] = None, + # Wan-Animate-2 (in-context reference KV-cache) + animate2_prompt_ref: str = " ", + animate2_reference_image: Image.Image = None, + animate2_reference_video: list[Image.Image] = None, # VAP vap_video: list[Image.Image] = None, vap_prompt: str = " ", @@ -273,11 +282,13 @@ def __call__( # Inputs inputs_posi = { "prompt": prompt, + "positive": True, "vap_prompt": vap_prompt, "tea_cache_l1_thresh": tea_cache_l1_thresh, "tea_cache_model_id": tea_cache_model_id, "num_inference_steps": num_inference_steps, } inputs_nega = { "negative_prompt": negative_prompt, + "positive": False, "negative_vap_prompt": negative_vap_prompt, "tea_cache_l1_thresh": tea_cache_l1_thresh, "tea_cache_model_id": tea_cache_model_id, "num_inference_steps": num_inference_steps, } @@ -298,6 +309,7 @@ def __call__( "sliding_window_size": sliding_window_size, "sliding_window_stride": sliding_window_stride, "input_audio": input_audio, "audio_sample_rate": audio_sample_rate, "s2v_pose_video": s2v_pose_video, "audio_embeds": audio_embeds, "s2v_pose_latents": s2v_pose_latents, "motion_video": motion_video, "animate_pose_video": animate_pose_video, "animate_face_video": animate_face_video, "animate_inpaint_video": animate_inpaint_video, "animate_mask_video": animate_mask_video, + "animate2_prompt_ref": animate2_prompt_ref, "animate2_reference_image": animate2_reference_image, "animate2_reference_video": animate2_reference_video, "vap_video": vap_video, "wantodance_music_path": wantodance_music_path, "wantodance_reference_image": wantodance_reference_image, "wantodance_fps": wantodance_fps, "wantodance_keyframes": wantodance_keyframes, "wantodance_keyframes_mask": wantodance_keyframes_mask, @@ -342,6 +354,9 @@ def __call__( else: f = 1 inputs_shared["latents"] = inputs_shared["latents"][:, :, f:] + # Wan-Animate-2: drop the leading reference latent slot before decoding #TODO: remove it + if animate2_reference_video is not None: + inputs_shared["latents"] = inputs_shared["latents"][:, :, 1:].to(dtype=self.torch_dtype) # post-denoising, pre-decoding processing logic for unit in self.post_units: inputs_shared, _, _ = self.unit_runner(unit, self, inputs_shared, inputs_posi, inputs_nega) @@ -376,12 +391,15 @@ def process(self, pipe: WanVideoPipeline, height, width, num_frames): class WanVideoUnit_NoiseInitializer(PipelineUnit): def __init__(self): super().__init__( - input_params=("height", "width", "num_frames", "seed", "rand_device", "vace_reference_image"), + input_params=("height", "width", "num_frames", "seed", "rand_device", "vace_reference_image", "animate2_reference_video"), output_params=("noise",) ) - def process(self, pipe: WanVideoPipeline, height, width, num_frames, seed, rand_device, vace_reference_image): + def process(self, pipe: WanVideoPipeline, height, width, num_frames, seed, rand_device, vace_reference_image, animate2_reference_video): length = (num_frames - 1) // 4 + 1 + if animate2_reference_video is not None: + # Wan-Animate-2 prepends one in-context reference latent frame. + length += 1 if vace_reference_image is not None: f = len(vace_reference_image) if isinstance(vace_reference_image, list) else 1 length += f @@ -1151,6 +1169,148 @@ def process(self, pipe: WanVideoPipeline, wantodance_keyframes, wantodance_keyfr return {"clip_feature": clip_context, "y": y} +class WanVideoUnit_Animate2RefPrompt(PipelineUnit): + # Wan-Animate-2: T5 embedding of the reference-branch prompt (pipeline_single.py:274). + def __init__(self): + super().__init__( + input_params=("animate2_prompt_ref", "animate2_reference_video"), + output_params=("context_ref",), + onload_model_names=("text_encoder",) + ) + + def process(self, pipe: WanVideoPipeline, animate2_prompt_ref, animate2_reference_video): + if animate2_reference_video is None or animate2_prompt_ref is None: + return {} + pipe.load_models_to_device(self.onload_model_names) + context_ref = WanVideoUnit_PromptEmbedder().encode_prompt(pipe, animate2_prompt_ref) + return {"context_ref": context_ref} + + +class WanVideoUnit_Animate2Clip(PipelineUnit): + # Wan-Animate-2: CLIP features for the reference image and the reference video first frame + # (pipeline_single.py:211 / :252). + def __init__(self): + super().__init__( + input_params=("animate2_reference_image", "animate2_reference_video", "height", "width"), + output_params=("clip_feature", "clip_fea_ref"), + onload_model_names=("image_encoder",) + ) + + def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_reference_video, height, width): + if animate2_reference_image is None or animate2_reference_video is None: + return {} + pipe.load_models_to_device(self.onload_model_names) + ref = pipe.preprocess_image(animate2_reference_image.resize((width, height))).to(pipe.device) + clip_feature = pipe.image_encoder.encode_image([ref]).to(dtype=pipe.torch_dtype, device=pipe.device) + ref_video_0 = pipe.preprocess_image(animate2_reference_video[0].resize((width, height))).to(pipe.device) + clip_fea_ref = pipe.image_encoder.encode_image([ref_video_0]).to(dtype=pipe.torch_dtype, device=pipe.device) + return {"clip_feature": clip_feature, "clip_fea_ref": clip_fea_ref} + + +class WanVideoUnit_Animate2VAEEmbedder(PipelineUnit): + # Wan-Animate-2: all VAE encodings + conditioning packing for the single-clip path + # (pipeline_single.py:204-269, :276-278). Only the VAE model is used here. + def __init__(self): + super().__init__( + input_params=("animate2_reference_image", "animate2_reference_video", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride"), + output_params=("y", "condition_latents", "condition_y", "grid_sizes", "grid_sizes_ref", "seq_len", "seq_len_ref", "origin_len", "origin_area"), + onload_model_names=("vae",) + ) + + def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_reference_video, num_frames, height, width, tiled, tile_size, tile_stride): + if animate2_reference_image is None or animate2_reference_video is None: + return {} + pipe.load_models_to_device(self.onload_model_names) + device, dtype = pipe.device, pipe.torch_dtype + H, W = height, width + clip_len = num_frames + T = clip_len + 1 + lat_h, lat_w = H // 8, W // 8 + lat_t = T // 4 + 1 + 1 + + grid_sizes = torch.stack([torch.tensor([lat_t, lat_h // 2, lat_w // 2], dtype=torch.long)]) + + # Reference image -> y_ref (mask + latent), single latent frame (:204-208) + ref_img = pipe.preprocess_image(animate2_reference_image.resize((W, H))).to(device) # (1, C, H, W) + ref_pixel = ref_img.transpose(0, 1) # (C, 1, H, W) + ref_latents = pipe.vae.encode([ref_pixel.to(dtype)], device=device).to(dtype=dtype, device=device) # (1, 16, 1, lat_h, lat_w) + mask_ref = _animate2_get_i2v_mask(1, lat_h, lat_w, mask_len=1, device=device) + y_ref = torch.concat([mask_ref, ref_latents[0]]).to(dtype=dtype, device=device) # (20, 1, lat_h, lat_w) + + # Reference-temporal slot: zeros for the single-clip case, mask_reft_len == 0 (:228-241) + zeros_vid = torch.zeros(3, T - 1, H, W, device=device, dtype=dtype) + y_reft = pipe.vae.encode([zeros_vid], device=device)[0].to(dtype=dtype, device=device) # (16, lat_t-1, lat_h, lat_w) + msk_reft = _animate2_get_i2v_mask(lat_t - 1, lat_h, lat_w, mask_len=0, device=device) + y_reft = torch.concat([msk_reft, y_reft]).to(dtype=dtype, device=device) + y = torch.concat([y_ref, y_reft], dim=1) # (20, lat_t, lat_h, lat_w) + + # Reference video -> condition_latents (:243-249) + ref_video = pipe.preprocess_video([f.resize((W, H)) for f in animate2_reference_video[:clip_len]]).to(device) # (1, C, clip_len, H, W) + condition_latents = pipe.vae.encode([ref_video[0].to(dtype)], device=device).to(dtype=dtype, device=device) # (1, 16, lat_t_c, lat_h, lat_w) + + B, C, T, H, W = ref_video.shape + B, C, lat_t_c, lat_h, lat_w = condition_latents.shape + grid_sizes_ref = torch.stack([torch.tensor([lat_t_c, lat_h // 2, lat_w // 2], dtype=torch.long)]) + + # Reference video -> condition_y (mask + latent), mask_len = T (:254-269) + mask_len = T + cond_y_input = torch.nn.functional.interpolate(ref_video[0][:, :mask_len].to("cpu"), size=(H, W), mode="bicubic") + if T - mask_len > 0: + cond_y_input = torch.concat([cond_y_input, torch.zeros(3, T - mask_len, H, W)], dim=1) + condition_y = pipe.vae.encode([cond_y_input.to(device=device, dtype=dtype)], device=device)[0].to(dtype=dtype, device=device) + condition_msk_y = _animate2_get_i2v_mask(lat_t_c, lat_h, lat_w, mask_len=mask_len, device=device) + condition_y = torch.concat([condition_msk_y, condition_y]).to(dtype=dtype, device=device) # (20, lat_t_c, lat_h, lat_w) + + # Gen grid + sequence lengths (:173, :276-278) + seq_len = int(math.ceil(lat_t * lat_h * lat_w / 4)) + seq_len_ref = int(math.ceil(lat_t_c * lat_h * lat_w / 4)) + + return { + "y": y.unsqueeze(0), + "condition_latents": condition_latents, + "condition_y": condition_y, + "grid_sizes": grid_sizes, + "grid_sizes_ref": grid_sizes_ref, + "seq_len": seq_len, + "seq_len_ref": seq_len_ref, + "origin_len": clip_len, + "origin_area": [W, H], + } + + +class WanVideoUnit_Animate2RefKVCache(PipelineUnit): + # Wan-Animate-2: prefill the reference KV-cache once per clip via DiT forward_ref + # (pipeline_single.py:315-321). Only the DiT model is used here. + def __init__(self): + super().__init__( + input_params=("animate2_reference_video", "condition_latents", "condition_y", "context_ref", "clip_fea_ref", "grid_sizes", "seq_len_ref"), + output_params=("animate2_k_cache", "animate2_v_cache"), + onload_model_names=("dit",) + ) + + def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref): + if animate2_reference_video is None: + return {} + pipe.load_models_to_device(self.onload_model_names) + animate2_k_cache, animate2_v_cache = {}, {} + t = pipe.scheduler.timesteps[0] + timestep = t.unsqueeze(0).to(dtype=pipe.torch_dtype, device=pipe.device) + with torch.autocast(device_type=pipe.device.split(":")[0], dtype=pipe.torch_dtype, enabled=True): + pipe.dit( + condition_latents, + grid_sizes=grid_sizes.to(pipe.device), + animate2_k_cache=animate2_k_cache, + animate2_v_cache=animate2_v_cache, + clip_fea_ref=clip_fea_ref, + y_ref=[condition_y], + context_ref=[context_ref[0]], + seq_len_ref=seq_len_ref, + t=timestep, + method="forward_ref", + ) + return {"animate2_k_cache": animate2_k_cache, "animate2_v_cache": animate2_v_cache} + + class TeaCache: def __init__(self, num_inference_steps, rel_l1_thresh, model_id): self.num_inference_steps = num_inference_steps @@ -1313,6 +1473,7 @@ def model_fn_wan_video( skip_9th_layer: bool = False, **kwargs, ): + # Wan-Animate-2 (in-context KV-cache, two-pass forward_ref/forward_gen) if sliding_window_size is not None and sliding_window_stride is not None: model_kwargs = dict( dit=dit, @@ -1349,6 +1510,15 @@ def model_fn_wan_video( use_gradient_checkpointing=use_gradient_checkpointing, use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) + # Wan-Animate-2 + if isinstance(dit, WanAnimate2Transformer): + return model_fn_wananimate( + dit=dit, latents=latents, timestep=timestep, context=context, + clip_feature=clip_feature, y=y, + use_gradient_checkpointing=use_gradient_checkpointing, + use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, + **kwargs, + ) # wan2.2 s2v if audio_embeds is not None: @@ -1715,3 +1885,41 @@ def custom_forward(*inputs): # make compatible with wan video x = torch.cat([origin_ref_latents, x], dim=2) return x + + +def model_fn_wananimate( + dit: WanAnimate2Transformer, + latents: torch.Tensor = None, + timestep: torch.Tensor = None, + context: torch.Tensor = None, + clip_feature: torch.Tensor = None, + y: torch.Tensor = None, + animate2_k_cache: dict = None, + animate2_v_cache: dict = None, + grid_sizes_ref: torch.Tensor = None, + origin_len: int = None, + origin_area: list = None, + seq_len: int = None, + positive: bool = True, + use_gradient_checkpointing_offload=False, + use_gradient_checkpointing=False, + **kwargs, +): + is_uncondtion = not positive + with torch.autocast(device_type=latents.device.type, dtype=next(dit.parameters()).dtype, enabled=True): + out = dit( + [latents[0]], + k_cache=animate2_k_cache, + v_cache=animate2_v_cache, + clip_fea=clip_feature, + y=[y[0]], + context=[context[0]], + seq_len=seq_len, + t=timestep, + grid_sizes_ref=grid_sizes_ref.to(latents.device), + origin_len=origin_len, + origin_area=origin_area, + is_uncondtion=is_uncondtion, + method="forward_gen", + ) + return out[0].unsqueeze(0) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2.py b/examples/wanvideo/model_inference/Wan-Animate-2.py new file mode 100644 index 00000000..ae76784a --- /dev/null +++ b/examples/wanvideo/model_inference/Wan-Animate-2.py @@ -0,0 +1,46 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + redirect_common_files=False, + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors"), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), +) + +# Character animation: reference image (identity) + reference video (motion) -> animated video. +reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage_640x352.jpg").convert("RGB") +reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video_640x352.mp4").raw_data() + + +num_frames = 81 +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + num_frames=num_frames, height=640, width=352, + num_inference_steps=40, cfg_scale=3.0, sigma_shift=5.0, + seed=0, tiled=False, +) +save_video(video, "video_Wan-Animate-2.mp4", fps=24, quality=5) From ba2a2f731dc5be7a32a36b1bca87724c35fbf621 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Thu, 16 Jul 2026 18:29:41 +0800 Subject: [PATCH 06/17] clean code --- diffsynth/models/wan_animate_2_dit.py | 15 +-- diffsynth/pipelines/wan_video.py | 106 +++++++++--------- .../wanvideo/model_inference/Wan-Animate-2.py | 2 +- 3 files changed, 57 insertions(+), 66 deletions(-) diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 61864976..181c40de 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -526,9 +526,6 @@ def attention_mask_logic(b, h, q_idx, kv_idx): attn_mask = attention_mask_logic(None, None, q_idx, kv_idx) return attn_mask[None, None] # [1, 1, q_len_total, k_len_total] - def forward(self, *args, method, **kwargs): - return getattr(self, method)(*args, **kwargs) - def forward_ref( self, x_ref, @@ -612,7 +609,7 @@ def forward_ref( ) - def forward_gen( + def forward( self, x, k_cache, @@ -732,13 +729,3 @@ def unpatchify(self, x, grid_sizes): u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) out.append(u) return out - - -def _animate2_get_i2v_mask(lat_t, lat_h, lat_w, mask_len=1, device="cuda"): - # Mirror of wanxiang.eval_i2v.get_i2v_mask (Wan-Animate-2 target library). - msk = torch.zeros(1, (lat_t - 1) * 4 + 1, lat_h, lat_w, device=device) - msk[:, :mask_len] = 1 - msk = torch.concat([torch.repeat_interleave(msk[:, 0:1], repeats=4, dim=1), msk[:, 1:]], dim=1) - msk = msk.view(1, msk.shape[1] // 4, 4, lat_h, lat_w) - msk = msk.transpose(1, 2)[0] - return msk diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index 0ce3272d..a3cdccf6 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -27,7 +27,7 @@ from ..models.wan_video_mot import MotWanModel from ..models.wav2vec import WanS2VAudioEncoder from ..models.longcat_video_dit import LongCatVideoTransformer3DModel -from ..models.wan_animate_2_dit import WanAnimate2Transformer, _animate2_get_i2v_mask +from ..models.wan_animate_2_dit import WanAnimate2Transformer class WanVideoPipeline(BasePipeline): @@ -79,13 +79,14 @@ def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16): WanVideoUnit_WanToDance_ProcessInputs(), WanVideoUnit_WanToDance_RefImageEmbedder(), WanVideoUnit_WanToDance_ImageKeyframesEmbedder(), - WanVideoUnit_Animate2RefPrompt(), - WanVideoUnit_Animate2Clip(), + WanVideoUnit_Animate2RefPromptEmbedder(), + WanVideoUnit_Animate2CLIPEmbedder(), WanVideoUnit_Animate2VAEEmbedder(), - WanVideoUnit_Animate2RefKVCache(), + WanVideoUnit_Animate2RefKVCacheEmbedder(), ] self.post_units = [ WanVideoPostUnit_S2V(), + WanVideoPostUnit_Animate2(), ] self.model_fn = model_fn_wan_video self.compilable_models = ["dit", "dit2"] @@ -228,7 +229,7 @@ def __call__( animate_face_video: list[Image.Image] = None, animate_inpaint_video: list[Image.Image] = None, animate_mask_video: list[Image.Image] = None, - # Wan-Animate-2 (in-context reference KV-cache) + # Wan-Animate-2 animate2_prompt_ref: str = " ", animate2_reference_image: Image.Image = None, animate2_reference_video: list[Image.Image] = None, @@ -354,9 +355,6 @@ def __call__( else: f = 1 inputs_shared["latents"] = inputs_shared["latents"][:, :, f:] - # Wan-Animate-2: drop the leading reference latent slot before decoding #TODO: remove it - if animate2_reference_video is not None: - inputs_shared["latents"] = inputs_shared["latents"][:, :, 1:].to(dtype=self.torch_dtype) # post-denoising, pre-decoding processing logic for unit in self.post_units: inputs_shared, _, _ = self.unit_runner(unit, self, inputs_shared, inputs_posi, inputs_nega) @@ -398,7 +396,6 @@ def __init__(self): def process(self, pipe: WanVideoPipeline, height, width, num_frames, seed, rand_device, vace_reference_image, animate2_reference_video): length = (num_frames - 1) // 4 + 1 if animate2_reference_video is not None: - # Wan-Animate-2 prepends one in-context reference latent frame. length += 1 if vace_reference_image is not None: f = len(vace_reference_image) if isinstance(vace_reference_image, list) else 1 @@ -1169,8 +1166,7 @@ def process(self, pipe: WanVideoPipeline, wantodance_keyframes, wantodance_keyfr return {"clip_feature": clip_context, "y": y} -class WanVideoUnit_Animate2RefPrompt(PipelineUnit): - # Wan-Animate-2: T5 embedding of the reference-branch prompt (pipeline_single.py:274). +class WanVideoUnit_Animate2RefPromptEmbedder(PipelineUnit): def __init__(self): super().__init__( input_params=("animate2_prompt_ref", "animate2_reference_video"), @@ -1186,9 +1182,7 @@ def process(self, pipe: WanVideoPipeline, animate2_prompt_ref, animate2_referenc return {"context_ref": context_ref} -class WanVideoUnit_Animate2Clip(PipelineUnit): - # Wan-Animate-2: CLIP features for the reference image and the reference video first frame - # (pipeline_single.py:211 / :252). +class WanVideoUnit_Animate2CLIPEmbedder(PipelineUnit): def __init__(self): super().__init__( input_params=("animate2_reference_image", "animate2_reference_video", "height", "width"), @@ -1208,8 +1202,6 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_ref class WanVideoUnit_Animate2VAEEmbedder(PipelineUnit): - # Wan-Animate-2: all VAE encodings + conditioning packing for the single-clip path - # (pipeline_single.py:204-269, :276-278). Only the VAE model is used here. def __init__(self): super().__init__( input_params=("animate2_reference_image", "animate2_reference_video", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride"), @@ -1217,51 +1209,53 @@ def __init__(self): onload_model_names=("vae",) ) + @staticmethod + def animate2_get_i2v_mask(lat_t, lat_h, lat_w, mask_len=1, device="cuda"): + msk = torch.zeros(1, (lat_t - 1) * 4 + 1, lat_h, lat_w, device=device) + msk[:, :mask_len] = 1 + msk = torch.concat([torch.repeat_interleave(msk[:, 0:1], repeats=4, dim=1), msk[:, 1:]], dim=1) + msk = msk.view(1, msk.shape[1] // 4, 4, lat_h, lat_w) + msk = msk.transpose(1, 2)[0] + return msk + + def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_reference_video, num_frames, height, width, tiled, tile_size, tile_stride): if animate2_reference_image is None or animate2_reference_video is None: return {} pipe.load_models_to_device(self.onload_model_names) device, dtype = pipe.device, pipe.torch_dtype H, W = height, width - clip_len = num_frames - T = clip_len + 1 + T = num_frames + 1 lat_h, lat_w = H // 8, W // 8 lat_t = T // 4 + 1 + 1 - grid_sizes = torch.stack([torch.tensor([lat_t, lat_h // 2, lat_w // 2], dtype=torch.long)]) - - # Reference image -> y_ref (mask + latent), single latent frame (:204-208) + # Reference image -> y_ref (mask + latent), single latent frame ref_img = pipe.preprocess_image(animate2_reference_image.resize((W, H))).to(device) # (1, C, H, W) ref_pixel = ref_img.transpose(0, 1) # (C, 1, H, W) ref_latents = pipe.vae.encode([ref_pixel.to(dtype)], device=device).to(dtype=dtype, device=device) # (1, 16, 1, lat_h, lat_w) - mask_ref = _animate2_get_i2v_mask(1, lat_h, lat_w, mask_len=1, device=device) + mask_ref = self.animate2_get_i2v_mask(1, lat_h, lat_w, mask_len=1, device=device) y_ref = torch.concat([mask_ref, ref_latents[0]]).to(dtype=dtype, device=device) # (20, 1, lat_h, lat_w) - # Reference-temporal slot: zeros for the single-clip case, mask_reft_len == 0 (:228-241) + # Reference-temporal slot: zeros for the single-clip case, mask_reft_len == 0 + # TODO: check reft zeros_vid = torch.zeros(3, T - 1, H, W, device=device, dtype=dtype) y_reft = pipe.vae.encode([zeros_vid], device=device)[0].to(dtype=dtype, device=device) # (16, lat_t-1, lat_h, lat_w) - msk_reft = _animate2_get_i2v_mask(lat_t - 1, lat_h, lat_w, mask_len=0, device=device) + msk_reft = self.animate2_get_i2v_mask(lat_t - 1, lat_h, lat_w, mask_len=0, device=device) y_reft = torch.concat([msk_reft, y_reft]).to(dtype=dtype, device=device) y = torch.concat([y_ref, y_reft], dim=1) # (20, lat_t, lat_h, lat_w) - # Reference video -> condition_latents (:243-249) - ref_video = pipe.preprocess_video([f.resize((W, H)) for f in animate2_reference_video[:clip_len]]).to(device) # (1, C, clip_len, H, W) + # Reference video -> condition_latents + ref_video = pipe.preprocess_video([f.resize((W, H)) for f in animate2_reference_video[:num_frames]]).to(device) # (1, C, num_frames, H, W) condition_latents = pipe.vae.encode([ref_video[0].to(dtype)], device=device).to(dtype=dtype, device=device) # (1, 16, lat_t_c, lat_h, lat_w) + # Reference video -> condition_y (mask + latent), mask_len = T + _, _, lat_t_c, lat_h_c, lat_w_c = condition_latents.shape + condition_y = condition_latents.clone()[0] + condition_msk_y = self.animate2_get_i2v_mask(lat_t_c, lat_h_c, lat_w_c, mask_len=ref_video.shape[2], device=device) + condition_y = torch.concat([condition_msk_y, condition_y]).to(dtype=dtype, device=device) # (20, lat_t_c, lat_h_c, lat_w_c) - B, C, T, H, W = ref_video.shape - B, C, lat_t_c, lat_h, lat_w = condition_latents.shape - grid_sizes_ref = torch.stack([torch.tensor([lat_t_c, lat_h // 2, lat_w // 2], dtype=torch.long)]) - - # Reference video -> condition_y (mask + latent), mask_len = T (:254-269) - mask_len = T - cond_y_input = torch.nn.functional.interpolate(ref_video[0][:, :mask_len].to("cpu"), size=(H, W), mode="bicubic") - if T - mask_len > 0: - cond_y_input = torch.concat([cond_y_input, torch.zeros(3, T - mask_len, H, W)], dim=1) - condition_y = pipe.vae.encode([cond_y_input.to(device=device, dtype=dtype)], device=device)[0].to(dtype=dtype, device=device) - condition_msk_y = _animate2_get_i2v_mask(lat_t_c, lat_h, lat_w, mask_len=mask_len, device=device) - condition_y = torch.concat([condition_msk_y, condition_y]).to(dtype=dtype, device=device) # (20, lat_t_c, lat_h, lat_w) - - # Gen grid + sequence lengths (:173, :276-278) + # sequence lengths + grid_sizes = torch.stack([torch.tensor([lat_t, lat_h // 2, lat_w // 2], dtype=torch.long)]) + grid_sizes_ref = torch.stack([torch.tensor([lat_t_c, lat_h_c // 2, lat_w_c // 2], dtype=torch.long)]) seq_len = int(math.ceil(lat_t * lat_h * lat_w / 4)) seq_len_ref = int(math.ceil(lat_t_c * lat_h * lat_w / 4)) @@ -1273,22 +1267,20 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_ref "grid_sizes_ref": grid_sizes_ref, "seq_len": seq_len, "seq_len_ref": seq_len_ref, - "origin_len": clip_len, + "origin_len": num_frames, "origin_area": [W, H], } -class WanVideoUnit_Animate2RefKVCache(PipelineUnit): - # Wan-Animate-2: prefill the reference KV-cache once per clip via DiT forward_ref - # (pipeline_single.py:315-321). Only the DiT model is used here. +class WanVideoUnit_Animate2RefKVCacheEmbedder(PipelineUnit): def __init__(self): super().__init__( - input_params=("animate2_reference_video", "condition_latents", "condition_y", "context_ref", "clip_fea_ref", "grid_sizes", "seq_len_ref"), + input_params=("animate2_reference_video", "condition_latents", "condition_y", "context_ref", "clip_fea_ref", "grid_sizes", "seq_len_ref", "use_gradient_checkpointing", "use_gradient_checkpointing_offload"), output_params=("animate2_k_cache", "animate2_v_cache"), onload_model_names=("dit",) ) - def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref): + def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref, use_gradient_checkpointing, use_gradient_checkpointing_offload): if animate2_reference_video is None: return {} pipe.load_models_to_device(self.onload_model_names) @@ -1296,21 +1288,32 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_la t = pipe.scheduler.timesteps[0] timestep = t.unsqueeze(0).to(dtype=pipe.torch_dtype, device=pipe.device) with torch.autocast(device_type=pipe.device.split(":")[0], dtype=pipe.torch_dtype, enabled=True): - pipe.dit( + pipe.dit.forward_ref( condition_latents, grid_sizes=grid_sizes.to(pipe.device), - animate2_k_cache=animate2_k_cache, - animate2_v_cache=animate2_v_cache, + k_cache=animate2_k_cache, + v_cache=animate2_v_cache, clip_fea_ref=clip_fea_ref, y_ref=[condition_y], context_ref=[context_ref[0]], seq_len_ref=seq_len_ref, t=timestep, - method="forward_ref", + use_gradient_checkpointing=use_gradient_checkpointing, + use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) return {"animate2_k_cache": animate2_k_cache, "animate2_v_cache": animate2_v_cache} +class WanVideoPostUnit_Animate2(PipelineUnit): + def __init__(self): + super().__init__(input_params=("latents", "animate2_reference_video")) + + def process(self, pipe: WanVideoPipeline, latents, animate2_reference_video): + if animate2_reference_video is None: + return {} + return {"latents": latents[:, :, 1:].to(pipe.torch_dtype)} + + class TeaCache: def __init__(self, num_inference_steps, rel_l1_thresh, model_id): self.num_inference_steps = num_inference_steps @@ -1920,6 +1923,7 @@ def model_fn_wananimate( origin_len=origin_len, origin_area=origin_area, is_uncondtion=is_uncondtion, - method="forward_gen", + use_gradient_checkpointing=use_gradient_checkpointing, + use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) return out[0].unsqueeze(0) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2.py b/examples/wanvideo/model_inference/Wan-Animate-2.py index ae76784a..d304243b 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2.py +++ b/examples/wanvideo/model_inference/Wan-Animate-2.py @@ -32,7 +32,7 @@ reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video_640x352.mp4").raw_data() -num_frames = 81 +num_frames = 41 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", From c9ed29b94051e83934578d95fba591348d1df548 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Fri, 17 Jul 2026 11:27:25 +0800 Subject: [PATCH 07/17] remove autocast&offload kvcache --- .../configs/vram_management_module_maps.py | 9 ++ diffsynth/models/wan_animate_2_dit.py | 97 +++++++------------ diffsynth/pipelines/wan_video.py | 68 ++++++------- .../wanvideo/model_inference/Wan-Animate-2.py | 12 ++- .../model_inference_low_vram/Wan-Animate-2.py | 49 ++++++++++ 5 files changed, 136 insertions(+), 99 deletions(-) create mode 100644 examples/wanvideo/model_inference_low_vram/Wan-Animate-2.py diff --git a/diffsynth/configs/vram_management_module_maps.py b/diffsynth/configs/vram_management_module_maps.py index 92f84e26..074b636f 100644 --- a/diffsynth/configs/vram_management_module_maps.py +++ b/diffsynth/configs/vram_management_module_maps.py @@ -75,6 +75,15 @@ "diffsynth.models.wan_video_dit.RMSNorm": "diffsynth.core.vram.layers.AutoWrappedModule", "torch.nn.Conv2d": "diffsynth.core.vram.layers.AutoWrappedModule", }, + "diffsynth.models.wan_animate_2_dit.WanAnimate2Transformer": { + "diffsynth.models.wan_video_dit.MLP": "diffsynth.core.vram.layers.AutoWrappedModule", + "diffsynth.models.wan_animate_2_dit.Incontext_AttentionBlock": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule", + "diffsynth.models.wan_animate_2_dit.Head": "diffsynth.core.vram.layers.AutoWrappedModule", + "torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear", + "torch.nn.Conv3d": "diffsynth.core.vram.layers.AutoWrappedModule", + "torch.nn.LayerNorm": "diffsynth.core.vram.layers.AutoWrappedModule", + "diffsynth.models.wan_video_dit.RMSNorm": "diffsynth.core.vram.layers.AutoWrappedModule", + }, "diffsynth.models.wan_video_dit.WanModel": { "diffsynth.models.wan_video_dit.MLP": "diffsynth.core.vram.layers.AutoWrappedModule", "diffsynth.models.wan_video_dit.DiTBlock": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule", diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 181c40de..4f323838 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -7,7 +7,6 @@ from ..core.gradient import gradient_checkpoint_forward -@torch.amp.autocast(device_type='cuda', enabled=False) def rope_params(max_seq_len, dim, theta=10000, offset=0): assert dim % 2 == 0 freqs = torch.outer( @@ -18,7 +17,6 @@ def rope_params(max_seq_len, dim, theta=10000, offset=0): return freqs -@torch.amp.autocast(device_type='cuda', enabled=False) def rope_apply(x, grid_sizes, freqs, time_stride=1): n, c = x.size(2), x.size(3) // 2 @@ -46,7 +44,8 @@ def rope_apply(x, grid_sizes, freqs, time_stride=1): # append to collection output.append(x_i) - return torch.stack(output).float() + return torch.stack(output).to(x.dtype) + def pad_freqs(original_tensor, target_len): seq_len, s1, s2 = original_tensor.shape @@ -67,7 +66,7 @@ def __init__(self, dim, eps=1e-6, elementwise_affine=False): super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) def forward(self, x): - return super().forward(x.float()).type_as(x) + return super().forward(x).type_as(x) class SelfAttention(nn.Module): @@ -217,12 +216,8 @@ def forward(self, *args, method, **kwargs): return getattr(self, method)(*args, **kwargs) def pre_self_attention(self, x, e): - assert e.dtype == torch.float32 - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - e = (self.modulation + e).chunk(6, dim=1) - assert e[0].dtype == torch.float32 - - q, k, v = self.self_attn(self.norm1(x).float() * (1 + e[1]) + e[0], method="pre_attention") + e = (self.modulation.to(x.device) + e).chunk(6, dim=1) + q, k, v = self.self_attn(self.norm1(x) * (1 + e[1]) + e[0], method="pre_attention") return q, k, v, e def post_self_attention(self, x): @@ -231,9 +226,8 @@ def post_self_attention(self, x): def cross_attention(self, x, context, context_lens, e): x = x + self.cross_attn(self.norm3(x), context, context_lens) - y = self.ffn(self.norm2(x).float() * (1 + e[4]) + e[3]) - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - x = x + y * e[5] + y = self.ffn(self.norm2(x) * (1 + e[4]) + e[3]) + x = x + y * e[5] return x @@ -271,11 +265,11 @@ def __init__( def forward(self, *args, method, **kwargs): return getattr(self, method)(*args, **kwargs) - def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, grid_sizes_ref, e_ref, context_lens): + def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, grid_sizes_ref, e_ref, context_lens, animate2_offload_kv): q_ref, k_ref, v_ref, e_ref = self.block(x_ref, e_ref, method='pre_self_attention') - k_cache[index] = k_ref - v_cache[index] = v_ref + k_cache[index] = k_ref if not animate2_offload_kv else k_ref.to('cpu') + v_cache[index] = v_ref if not animate2_offload_kv else v_ref.to('cpu') q_ref_add_rope = rope_apply(q_ref, grid_sizes_ref, freqs_ref, self.refer_stride) k_ref_add_rope = rope_apply(k_ref, grid_sizes_ref, freqs_ref, self.refer_stride) @@ -291,8 +285,7 @@ def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, gr y_ref = self.block(xout_ref, method='post_self_attention') - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - x_ref = x_ref + y_ref * e_ref[2] + x_ref = x_ref + y_ref * e_ref[2] x_ref = self.block(x_ref, context_ref, context_lens, e_ref, method='cross_attention') @@ -306,19 +299,19 @@ def forward_gen(self, x, index, k_cache, v_cache, attn_mask, context, freqs, fre origin_max_len = (origin_latent_f + 1) * origin_latent_hw origin_ref_max_len = origin_latent_f * origin_latent_hw - f, h, w = grid_sizes[0].tolist() - vail_len = f * h * w - hw = h * w + f, h, w = grid_sizes[0].tolist() + vail_len = f * h * w + hw = h * w ref_f, ref_h, ref_w = grid_sizes_ref[0].tolist() - ref_vail_len = ref_f * ref_h * ref_w - ref_hw = ref_h * ref_w + ref_vail_len = ref_f * ref_h * ref_w + ref_hw = ref_h * ref_w q, k, v, e = self.block(x, e, method='pre_self_attention') - q = rope_apply(q, grid_sizes, freqs) - k = rope_apply(k, grid_sizes, freqs) - k_ref, v_ref = k_cache[index], v_cache[index] + q = rope_apply(q, grid_sizes, freqs) + k = rope_apply(k, grid_sizes, freqs) + k_ref, v_ref = k_cache[index].to(x.device), v_cache[index].to(x.device) k_ref = rope_apply(k_ref, grid_sizes_ref, freqs_ref, self.refer_stride) B, _, N, C = q.shape @@ -358,15 +351,14 @@ def forward_gen(self, x, index, k_cache, v_cache, attn_mask, context, freqs, fre ) xout_valid = xout_full[:, :f * origin_latent_hw] - xout_valid = xout_valid.view(B, f, origin_latent_hw, N, C) - xout_vail = xout_valid[:, :, :hw] - xout_vail = xout_vail.reshape(B, f * hw, N, C) # [B, f*hw, N, C] - xout = torch.cat([xout_vail, q_padding], dim=1) + xout_valid = xout_valid.view(B, f, origin_latent_hw, N, C) + xout_vail = xout_valid[:, :, :hw] + xout_vail = xout_vail.reshape(B, f * hw, N, C) # [B, f*hw, N, C] + xout = torch.cat([xout_vail, q_padding], dim=1) y = self.block(xout, method='post_self_attention') - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - x = x + y * e[2] + x = x + y * e[2] x = self.block(x, context, context_lens, e, method='cross_attention') return x @@ -390,10 +382,8 @@ def __init__(self, dim, out_dim, patch_size, eps=1e-6): self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5) def forward(self, x, e): - assert e.dtype == torch.float32 - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1) - x = self.head(self.norm(x) * (1 + e[1]) + e[0]) + e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1) + x = self.head(self.norm(x) * (1 + e[1]) + e[0]) return x @@ -537,10 +527,10 @@ def forward_ref( context_ref, seq_len_ref, t, + animate2_offload_kv, use_gradient_checkpointing: bool = False, use_gradient_checkpointing_offload: bool = False, ): - device = self.patch_embedding.weight.device # [reference] x_ref = [torch.cat([u, v], dim=0) for u, v in zip(x_ref, y_ref)] # embeddings @@ -569,17 +559,11 @@ def forward_ref( rope_params(512, d - 4 * (d // 6), offset=self.refer_offset_t), rope_params(512, 2 * (d // 6), offset=self.refer_offset_h), rope_params(512, 2 * (d // 6), offset=self.refer_offset_w) - ], dim=1) - if self.freqs_ref.device != device: - self.freqs_ref = self.freqs_ref.to(device) + ], dim=1).to(x_ref.device) # time embeddings ref - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - e_ref = self.time_embedding( - sinusoidal_embedding_1d(self.freq_dim, t*0+1).float() - ) - e0_ref = self.time_projection(e_ref).unflatten(1, (6, self.dim)) - assert e_ref.dtype == torch.float32 and e0_ref.dtype == torch.float32 + e_ref = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t*0+1).to(x_ref.dtype)) + e0_ref = self.time_projection(e_ref).unflatten(1, (6, self.dim)) # [context_ref] context_ref = self.text_embedding(torch.stack([torch.cat([ @@ -597,7 +581,8 @@ def forward_ref( grid_sizes_ref=grid_sizes_ref, freqs_ref=self.freqs_ref, context_ref=context_ref, - context_lens=context_lens + context_lens=context_lens, + animate2_offload_kv=animate2_offload_kv ) for idx, block in enumerate(self.blocks): @@ -647,9 +632,7 @@ def forward( rope_params(512, d - 4 * (d // 6)), rope_params(512, 2 * (d // 6)), rope_params(512, 2 * (d // 6)) - ], dim=1) - if self.freqs.device != device: - self.freqs = self.freqs.to(device) + ], dim=1).to(x.device) if self.refer_offset_t < 0: self.refer_offset_t = grid_sizes[0][0].item() @@ -662,17 +645,11 @@ def forward( rope_params(512, d - 4 * (d // 6), offset=self.refer_offset_t), rope_params(512, 2 * (d // 6), offset=self.refer_offset_h), rope_params(512, 2 * (d // 6), offset=self.refer_offset_w) - ], dim=1) - if self.freqs_ref.device != device: - self.freqs_ref = self.freqs_ref.to(device) + ], dim=1).to(x.device) # time embeddings - with torch.amp.autocast(device_type='cuda', dtype=torch.float32): - e = self.time_embedding( - sinusoidal_embedding_1d(self.freq_dim, t).float() - ) - e0 = self.time_projection(e).unflatten(1, (6, self.dim)) - assert e.dtype == torch.float32 and e0.dtype == torch.float32 + e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t).to(x.dtype)) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) # [context] context_lens = None @@ -718,7 +695,7 @@ def forward( # unpatchify x = self.unpatchify(x, grid_sizes) - return [u.float() for u in x] + return [u for u in x] def unpatchify(self, x, grid_sizes): c = self.out_dim diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index a3cdccf6..1d6fffe6 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -233,6 +233,7 @@ def __call__( animate2_prompt_ref: str = " ", animate2_reference_image: Image.Image = None, animate2_reference_video: list[Image.Image] = None, + animate2_offload_kv: bool = False, # VAP vap_video: list[Image.Image] = None, vap_prompt: str = " ", @@ -310,7 +311,7 @@ def __call__( "sliding_window_size": sliding_window_size, "sliding_window_stride": sliding_window_stride, "input_audio": input_audio, "audio_sample_rate": audio_sample_rate, "s2v_pose_video": s2v_pose_video, "audio_embeds": audio_embeds, "s2v_pose_latents": s2v_pose_latents, "motion_video": motion_video, "animate_pose_video": animate_pose_video, "animate_face_video": animate_face_video, "animate_inpaint_video": animate_inpaint_video, "animate_mask_video": animate_mask_video, - "animate2_prompt_ref": animate2_prompt_ref, "animate2_reference_image": animate2_reference_image, "animate2_reference_video": animate2_reference_video, + "animate2_prompt_ref": animate2_prompt_ref, "animate2_reference_image": animate2_reference_image, "animate2_reference_video": animate2_reference_video, "animate2_offload_kv": animate2_offload_kv, "vap_video": vap_video, "wantodance_music_path": wantodance_music_path, "wantodance_reference_image": wantodance_reference_image, "wantodance_fps": wantodance_fps, "wantodance_keyframes": wantodance_keyframes, "wantodance_keyframes_mask": wantodance_keyframes_mask, @@ -1275,32 +1276,32 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_ref class WanVideoUnit_Animate2RefKVCacheEmbedder(PipelineUnit): def __init__(self): super().__init__( - input_params=("animate2_reference_video", "condition_latents", "condition_y", "context_ref", "clip_fea_ref", "grid_sizes", "seq_len_ref", "use_gradient_checkpointing", "use_gradient_checkpointing_offload"), + input_params=("animate2_reference_video", "condition_latents", "condition_y", "context_ref", "clip_fea_ref", "grid_sizes", "seq_len_ref", "animate2_offload_kv", "use_gradient_checkpointing", "use_gradient_checkpointing_offload"), output_params=("animate2_k_cache", "animate2_v_cache"), onload_model_names=("dit",) ) - def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref, use_gradient_checkpointing, use_gradient_checkpointing_offload): + def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref, animate2_offload_kv, use_gradient_checkpointing, use_gradient_checkpointing_offload): if animate2_reference_video is None: return {} pipe.load_models_to_device(self.onload_model_names) animate2_k_cache, animate2_v_cache = {}, {} t = pipe.scheduler.timesteps[0] timestep = t.unsqueeze(0).to(dtype=pipe.torch_dtype, device=pipe.device) - with torch.autocast(device_type=pipe.device.split(":")[0], dtype=pipe.torch_dtype, enabled=True): - pipe.dit.forward_ref( - condition_latents, - grid_sizes=grid_sizes.to(pipe.device), - k_cache=animate2_k_cache, - v_cache=animate2_v_cache, - clip_fea_ref=clip_fea_ref, - y_ref=[condition_y], - context_ref=[context_ref[0]], - seq_len_ref=seq_len_ref, - t=timestep, - use_gradient_checkpointing=use_gradient_checkpointing, - use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, - ) + pipe.dit.forward_ref( + condition_latents, + grid_sizes=grid_sizes.to(pipe.device), + k_cache=animate2_k_cache, + v_cache=animate2_v_cache, + clip_fea_ref=clip_fea_ref, + y_ref=[condition_y], + context_ref=[context_ref[0]], + seq_len_ref=seq_len_ref, + t=timestep, + animate2_offload_kv=animate2_offload_kv, + use_gradient_checkpointing=use_gradient_checkpointing, + use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, + ) return {"animate2_k_cache": animate2_k_cache, "animate2_v_cache": animate2_v_cache} @@ -1909,21 +1910,20 @@ def model_fn_wananimate( **kwargs, ): is_uncondtion = not positive - with torch.autocast(device_type=latents.device.type, dtype=next(dit.parameters()).dtype, enabled=True): - out = dit( - [latents[0]], - k_cache=animate2_k_cache, - v_cache=animate2_v_cache, - clip_fea=clip_feature, - y=[y[0]], - context=[context[0]], - seq_len=seq_len, - t=timestep, - grid_sizes_ref=grid_sizes_ref.to(latents.device), - origin_len=origin_len, - origin_area=origin_area, - is_uncondtion=is_uncondtion, - use_gradient_checkpointing=use_gradient_checkpointing, - use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, - ) + out = dit( + [latents[0]], + k_cache=animate2_k_cache, + v_cache=animate2_v_cache, + clip_fea=clip_feature, + y=[y[0]], + context=[context[0]], + seq_len=seq_len, + t=timestep, + grid_sizes_ref=grid_sizes_ref.to(latents.device), + origin_len=origin_len, + origin_area=origin_area, + is_uncondtion=is_uncondtion, + use_gradient_checkpointing=use_gradient_checkpointing, + use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, + ) return out[0].unsqueeze(0) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2.py b/examples/wanvideo/model_inference/Wan-Animate-2.py index d304243b..89e548fa 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2.py +++ b/examples/wanvideo/model_inference/Wan-Animate-2.py @@ -19,7 +19,7 @@ device="cuda", redirect_common_files=False, model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors"), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), @@ -28,17 +28,19 @@ ) # Character animation: reference image (identity) + reference video (motion) -> animated video. -reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage_640x352.jpg").convert("RGB") -reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video_640x352.mp4").raw_data() +# reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage_640x352.jpg").convert("RGB") +# reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video_640x352.mp4").raw_data() +reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage.jpg").convert("RGB") +reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video.mp4").raw_data() - -num_frames = 41 +num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", animate2_prompt_ref="视频中的人在做动作，背景静止", animate2_reference_image=reference_image, animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, num_frames=num_frames, height=640, width=352, num_inference_steps=40, cfg_scale=3.0, sigma_shift=5.0, seed=0, tiled=False, diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2.py b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2.py new file mode 100644 index 00000000..989997d5 --- /dev/null +++ b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2.py @@ -0,0 +1,49 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + redirect_common_files=False, + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + # vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, + vram_limit=2, +) + +# Character animation: reference image (identity) + reference video (motion) -> animated video. + +reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage.jpg").convert("RGB") +reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video.mp4").raw_data() + +num_frames = 81 +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + num_frames=num_frames, height=1280, width=720, + num_inference_steps=40, cfg_scale=3.0, sigma_shift=5.0, + seed=0, tiled=False, +) +save_video(video, "video_Wan-Animate-2.mp4", fps=24, quality=5) From 16ab67bf9173a3a252fdc9ae2294e375b499fb64 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Fri, 17 Jul 2026 14:28:22 +0800 Subject: [PATCH 08/17] merger flex to attention_forward --- diffsynth/core/attention/attention.py | 22 ++++++++++++++++++++-- diffsynth/models/wan_animate_2_dit.py | 20 ++++++++++++-------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/diffsynth/core/attention/attention.py b/diffsynth/core/attention/attention.py index 642424cd..41e7652f 100644 --- a/diffsynth/core/attention/attention.py +++ b/diffsynth/core/attention/attention.py @@ -26,6 +26,13 @@ except ModuleNotFoundError: XFORMERS_AVAILABLE = False +try: + from torch.nn.attention.flex_attention import flex_attention as flex_attention_func + flex_attention_func = torch.compile(flex_attention_func, dynamic=False, mode="max-autotune", fullgraph=True, backend="inductor") + FLEX_ATTN_AVAILABLE = True +except (ModuleNotFoundError, ImportError): + FLEX_ATTN_AVAILABLE = False + try: if "enable_gqa" in inspect.signature(torch.nn.functional.scaled_dot_product_attention).parameters: TORCH_SUPPORT_GQA = True @@ -169,9 +176,20 @@ def xformers_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_patt return out -def attention_forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None, is_causal=False, compatibility_mode=False, window_size=None): +def flex_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None): + assert FLEX_ATTN_AVAILABLE, "Flex Attention is not available. Please upgrade torch to 2.5.0 or later." + required_in_pattern, required_out_pattern = "b n s d", "b n s d" + q, k, v = rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, required_in_pattern, dims) + out = flex_attention_func(query=q, key=k, value=v, block_mask=attn_mask, scale=scale) + out = rearrange_out(out, out_pattern, required_out_pattern, dims) + return out + + +def attention_forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None, is_causal=False, compatibility_mode=False, window_size=None, use_flex=False): if compatibility_mode or (attn_mask is not None) or ATTENTION_IMPLEMENTATION == "torch": - if window_size is None: + if use_flex: + return flex_attention(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, attn_mask=attn_mask, scale=scale) + elif window_size is None: return torch_sdpa(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, attn_mask=attn_mask, scale=scale, is_causal=is_causal) else: # Sliding Window Attention is not compatible with `is_causal` and `attn_mask`. diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 4f323838..69075c91 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -2,6 +2,7 @@ import torch import torch.nn as nn import math +from torch.nn.attention.flex_attention import create_block_mask from .wan_video_dit import sinusoidal_embedding_1d, RMSNorm, MLP from ..core.attention.attention import attention_forward from ..core.gradient import gradient_checkpoint_forward @@ -347,7 +348,7 @@ def forward_gen(self, x, index, k_cache, v_cache, attn_mask, context, freqs, fre xout_full = attention_forward( q_incontext, k_incontext, v_incontext, q_pattern="b s n d", k_pattern="b s n d", v_pattern="b s n d", out_pattern="b s n d", - attn_mask=attn_mask, + use_flex=True, attn_mask=attn_mask, ) xout_valid = xout_full[:, :f * origin_latent_hw] @@ -472,7 +473,6 @@ def __init__( self.block_masks = dict() self.block_mask_grid_sizes = dict() - # Author: Guangyuan Wang def create_mask(self, origin_len, origin_area, device): origin_latent_f = origin_len // 4 + 1 hw = int(np.prod(origin_area).item() // 256) @@ -509,12 +509,16 @@ def attention_mask_logic(b, h, q_idx, kv_idx): return q_valid & (is_base_attention | is_cond_attention) - # Materialize the flex mask_fn as a dense boolean mask for torch SDPA. - # True => attention allowed (same convention as flex block_mask / SDPA bool attn_mask). - q_idx = torch.arange(q_len_total, device=device)[:, None] - kv_idx = torch.arange(k_len_total, device=device)[None, :] - attn_mask = attention_mask_logic(None, None, q_idx, kv_idx) - return attn_mask[None, None] # [1, 1, q_len_total, k_len_total] + block_mask = create_block_mask( + attention_mask_logic, + B=None, + H=None, + Q_LEN=q_len_total, + KV_LEN=k_len_total, + device=device, + _compile=True + ) + return block_mask def forward_ref( self, From 20173a0eac6afef2a48326b104213b3a4f67b26e Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Sun, 19 Jul 2026 20:41:51 +0800 Subject: [PATCH 09/17] context parallel --- diffsynth/models/wan_animate_2_dit.py | 81 +++++++++++++++++-- diffsynth/pipelines/wan_video.py | 16 ++-- diffsynth/utils/xfuser/__init__.py | 2 +- .../utils/xfuser/xdit_context_parallel.py | 15 ++++ 4 files changed, 101 insertions(+), 13 deletions(-) diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 69075c91..1aa1b784 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -266,9 +266,31 @@ def __init__( def forward(self, *args, method, **kwargs): return getattr(self, method)(*args, **kwargs) - def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, grid_sizes_ref, e_ref, context_lens, animate2_offload_kv): + def forward_ref( + self, + x_ref, + index, + k_cache, + v_cache, + context_ref, + freqs_ref, + grid_sizes_ref, + e_ref, + context_lens, + animate2_offload_kv, + use_context_parallel=False, + ): q_ref, k_ref, v_ref, e_ref = self.block(x_ref, e_ref, method='pre_self_attention') + if use_context_parallel: + from ..utils.xfuser import all_to_all_4d, is_evenly_divisible, get_sequence_parallel_world_size + assert is_evenly_divisible(q_ref.shape[2]), ( + f"num_heads ({q_ref.shape[2]}) must be divisible by sequence parallel world size ({get_sequence_parallel_world_size()}) " + ) + qkv_ref = torch.cat([q_ref, k_ref, v_ref], dim=0) + qkv_ref = all_to_all_4d(qkv_ref, scatter_dim=2, gather_dim=1) + q_ref, k_ref, v_ref = qkv_ref.chunk(3, dim=0) + k_cache[index] = k_ref if not animate2_offload_kv else k_ref.to('cpu') v_cache[index] = v_ref if not animate2_offload_kv else v_ref.to('cpu') q_ref_add_rope = rope_apply(q_ref, grid_sizes_ref, freqs_ref, self.refer_stride) @@ -284,6 +306,9 @@ def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, gr q_pattern="b s n d", k_pattern="b s n d", v_pattern="b s n d", out_pattern="b s n d", ).to(q_ref_add_rope.dtype) + if use_context_parallel: + xout_ref = all_to_all_4d(xout_ref, scatter_dim=1, gather_dim=2) + y_ref = self.block(xout_ref, method='post_self_attention') x_ref = x_ref + y_ref * e_ref[2] @@ -292,8 +317,24 @@ def forward_ref(self, x_ref, index, k_cache, v_cache, context_ref, freqs_ref, gr return x_ref - def forward_gen(self, x, index, k_cache, v_cache, attn_mask, context, freqs, freqs_ref, grid_sizes, grid_sizes_ref, - origin_len, origin_area, e, context_lens): + def forward_gen( + self, + x, + index, + k_cache, + v_cache, + attn_mask, + context, + freqs, + freqs_ref, + grid_sizes, + grid_sizes_ref, + origin_len, + origin_area, + e, + context_lens, + use_context_parallel, + ): origin_latent_f = origin_len // 4 + 1 origin_latent_hw = origin_area[0] * origin_area[1] // 256 @@ -309,6 +350,14 @@ def forward_gen(self, x, index, k_cache, v_cache, attn_mask, context, freqs, fre ref_hw = ref_h * ref_w q, k, v, e = self.block(x, e, method='pre_self_attention') + if use_context_parallel: + from ..utils.xfuser import all_to_all_4d, is_evenly_divisible, get_sequence_parallel_world_size + assert is_evenly_divisible(q.shape[2]), ( + f"num_heads ({q.shape[2]}) must be divisible by sequence parallel world size ({get_sequence_parallel_world_size()}) " + ) + qkv = torch.cat([q, k, v], dim=0) + qkv = all_to_all_4d(qkv, scatter_dim=2, gather_dim=1) + q, k, v = qkv.chunk(3, dim=0) q = rope_apply(q, grid_sizes, freqs) k = rope_apply(k, grid_sizes, freqs) @@ -356,6 +405,8 @@ def forward_gen(self, x, index, k_cache, v_cache, attn_mask, context, freqs, fre xout_vail = xout_valid[:, :, :hw] xout_vail = xout_vail.reshape(B, f * hw, N, C) # [B, f*hw, N, C] xout = torch.cat([xout_vail, q_padding], dim=1) + if use_context_parallel: + xout = all_to_all_4d(xout, scatter_dim=1, gather_dim=2) y = self.block(xout, method='post_self_attention') @@ -532,6 +583,7 @@ def forward_ref( seq_len_ref, t, animate2_offload_kv, + use_unified_sequence_parallel: bool = False, use_gradient_checkpointing: bool = False, use_gradient_checkpointing_offload: bool = False, ): @@ -586,9 +638,15 @@ def forward_ref( freqs_ref=self.freqs_ref, context_ref=context_ref, context_lens=context_lens, - animate2_offload_kv=animate2_offload_kv + animate2_offload_kv=animate2_offload_kv, + use_context_parallel=use_unified_sequence_parallel, ) - + if use_unified_sequence_parallel: + from ..utils.xfuser import get_current_chunk, is_evenly_divisible, get_sequence_parallel_world_size + assert is_evenly_divisible(x_ref.shape[1]), ( + f"x_ref sequence length ({x_ref.shape[1]}) must be divisible by sequence parallel world size ({get_sequence_parallel_world_size()}) " + ) + x_ref = get_current_chunk(x_ref, dim=1) for idx, block in enumerate(self.blocks): x_ref = gradient_checkpoint_forward( block, @@ -612,6 +670,7 @@ def forward( origin_len, origin_area, is_uncondtion=False, + use_unified_sequence_parallel: bool = False, use_gradient_checkpointing: bool = False, use_gradient_checkpointing_offload: bool = False, ): @@ -681,8 +740,15 @@ def forward( freqs_ref=self.freqs_ref, context_lens=context_lens, origin_area=origin_area, - origin_len=origin_len + origin_len=origin_len, + use_context_parallel=use_unified_sequence_parallel, ) + if use_unified_sequence_parallel: + from ..utils.xfuser import get_current_chunk, is_evenly_divisible, get_sequence_parallel_world_size + assert is_evenly_divisible(x.shape[1]), ( + f"sequence length ({x.shape[1]}) must be divisible by sequence parallel world size ({get_sequence_parallel_world_size()}) " + ) + x = get_current_chunk(x, dim=1) for idx, block in enumerate(self.blocks): if is_uncondtion and idx==9: @@ -696,6 +762,9 @@ def forward( # head x = self.head(x, e) + if use_unified_sequence_parallel: + from ..utils.xfuser import gather_all_chunks + x = gather_all_chunks(x, dim=1) # unpatchify x = self.unpatchify(x, grid_sizes) diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index 1d6fffe6..0097a56a 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -94,6 +94,10 @@ def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16): def enable_usp(self): from ..utils.xfuser import get_sequence_parallel_world_size, usp_attn_forward, usp_dit_forward, usp_vace_forward + self.sp_size = get_sequence_parallel_world_size() + self.use_unified_sequence_parallel = True + if isinstance(self.dit, WanAnimate2Transformer): + return for block in self.dit.blocks: block.self_attn.forward = types.MethodType(usp_attn_forward, block.self_attn) @@ -110,8 +114,6 @@ def enable_usp(self): for block in self.vace2.vace_blocks: block.self_attn.forward = types.MethodType(usp_attn_forward, block.self_attn) self.vace2.forward = types.MethodType(usp_vace_forward, self.vace2) - self.sp_size = get_sequence_parallel_world_size() - self.use_unified_sequence_parallel = True @staticmethod @@ -1299,6 +1301,7 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_la seq_len_ref=seq_len_ref, t=timestep, animate2_offload_kv=animate2_offload_kv, + use_unified_sequence_parallel=pipe.use_unified_sequence_parallel, use_gradient_checkpointing=use_gradient_checkpointing, use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) @@ -1477,7 +1480,6 @@ def model_fn_wan_video( skip_9th_layer: bool = False, **kwargs, ): - # Wan-Animate-2 (in-context KV-cache, two-pass forward_ref/forward_gen) if sliding_window_size is not None and sliding_window_stride is not None: model_kwargs = dict( dit=dit, @@ -1518,7 +1520,7 @@ def model_fn_wan_video( if isinstance(dit, WanAnimate2Transformer): return model_fn_wananimate( dit=dit, latents=latents, timestep=timestep, context=context, - clip_feature=clip_feature, y=y, + clip_feature=clip_feature, y=y, use_unified_sequence_parallel=use_unified_sequence_parallel, use_gradient_checkpointing=use_gradient_checkpointing, use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, **kwargs, @@ -1905,8 +1907,9 @@ def model_fn_wananimate( origin_area: list = None, seq_len: int = None, positive: bool = True, - use_gradient_checkpointing_offload=False, - use_gradient_checkpointing=False, + use_unified_sequence_parallel: bool = False, + use_gradient_checkpointing_offload: bool = False, + use_gradient_checkpointing: bool = False, **kwargs, ): is_uncondtion = not positive @@ -1923,6 +1926,7 @@ def model_fn_wananimate( origin_len=origin_len, origin_area=origin_area, is_uncondtion=is_uncondtion, + use_unified_sequence_parallel=use_unified_sequence_parallel, use_gradient_checkpointing=use_gradient_checkpointing, use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) diff --git a/diffsynth/utils/xfuser/__init__.py b/diffsynth/utils/xfuser/__init__.py index 8e5faca2..5fbe8466 100644 --- a/diffsynth/utils/xfuser/__init__.py +++ b/diffsynth/utils/xfuser/__init__.py @@ -1 +1 @@ -from .xdit_context_parallel import usp_attn_forward, usp_dit_forward, usp_vace_forward, get_sequence_parallel_world_size, initialize_usp, get_current_chunk, gather_all_chunks +from .xdit_context_parallel import usp_attn_forward, usp_dit_forward, usp_vace_forward, get_sequence_parallel_world_size, initialize_usp, get_current_chunk, gather_all_chunks, all_to_all_4d, is_evenly_divisible diff --git a/diffsynth/utils/xfuser/xdit_context_parallel.py b/diffsynth/utils/xfuser/xdit_context_parallel.py index abf0f3fe..db630e5f 100644 --- a/diffsynth/utils/xfuser/xdit_context_parallel.py +++ b/diffsynth/utils/xfuser/xdit_context_parallel.py @@ -2,6 +2,7 @@ from typing import Optional from einops import rearrange from yunchang.kernels import AttnType +from yunchang.comm.all_to_all import SeqAllToAll4D from xfuser.core.distributed import (get_sequence_parallel_rank, get_sequence_parallel_world_size, get_sp_group) @@ -204,3 +205,17 @@ def gather_all_chunks(x, seq_len=None, dim=1): slices[dim] = slice(0, seq_len) x = x[tuple(slices)] return x + + +def all_to_all_4d(x, scatter_dim, gather_dim): + world_size = get_sequence_parallel_world_size() + if world_size == 1: + return x + return SeqAllToAll4D.apply(get_sp_group().ulysses_group, x, scatter_dim, gather_dim) + + +def is_evenly_divisible(seq_len): + world_size = get_sequence_parallel_world_size() + return seq_len % world_size == 0 + + From e7420b6a76a945b861b64928022142ed98d37722 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Mon, 20 Jul 2026 10:45:21 +0800 Subject: [PATCH 10/17] distilled model --- diffsynth/core/attention/attention.py | 10 ++-- diffsynth/models/wan_animate_2_dit.py | 24 +++++++-- diffsynth/pipelines/wan_video.py | 11 ++-- .../Wan-Animate-2-14B-usp.py} | 25 +++++---- .../Wan-Animate-2-14B-Distilled.py | 52 ++++++++++++++++++ .../Wan-Animate-2-14B.py} | 23 ++++---- .../Wan-Animate-2-14B-Distilled.py | 53 +++++++++++++++++++ .../Wan-Animate-2-14B.py | 51 ++++++++++++++++++ 8 files changed, 215 insertions(+), 34 deletions(-) rename examples/wanvideo/{model_inference/Wan-Animate-2.py => acceleration/Wan-Animate-2-14B-usp.py} (74%) create mode 100644 examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py rename examples/wanvideo/{model_inference_low_vram/Wan-Animate-2.py => model_inference/Wan-Animate-2-14B.py} (79%) create mode 100644 examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py create mode 100644 examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py diff --git a/diffsynth/core/attention/attention.py b/diffsynth/core/attention/attention.py index 41e7652f..bddd75b2 100644 --- a/diffsynth/core/attention/attention.py +++ b/diffsynth/core/attention/attention.py @@ -176,19 +176,19 @@ def xformers_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_patt return out -def flex_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None): +def flex_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None, score_mod=None): assert FLEX_ATTN_AVAILABLE, "Flex Attention is not available. Please upgrade torch to 2.5.0 or later." required_in_pattern, required_out_pattern = "b n s d", "b n s d" q, k, v = rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, required_in_pattern, dims) - out = flex_attention_func(query=q, key=k, value=v, block_mask=attn_mask, scale=scale) + out = flex_attention_func(query=q, key=k, value=v, block_mask=attn_mask, scale=scale, score_mod=score_mod) out = rearrange_out(out, out_pattern, required_out_pattern, dims) return out -def attention_forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None, is_causal=False, compatibility_mode=False, window_size=None, use_flex=False): +def attention_forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None, is_causal=False, compatibility_mode=False, window_size=None, use_flex=False, score_mod=None): if compatibility_mode or (attn_mask is not None) or ATTENTION_IMPLEMENTATION == "torch": - if use_flex: - return flex_attention(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, attn_mask=attn_mask, scale=scale) + if use_flex or score_mod is not None: + return flex_attention(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, attn_mask=attn_mask, scale=scale, score_mod=score_mod) elif window_size is None: return torch_sdpa(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, attn_mask=attn_mask, scale=scale, is_causal=is_causal) else: diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index 1aa1b784..e603c366 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -6,6 +6,17 @@ from .wan_video_dit import sinusoidal_embedding_1d, RMSNorm, MLP from ..core.attention.attention import attention_forward from ..core.gradient import gradient_checkpoint_forward +from functools import lru_cache, partial + + +def _score_mod_impl(score, b_idx, h_idx, q_idx, kv_idx, hw: int, log_scale: float): + condition = (kv_idx >= hw) & (kv_idx < 2 * hw) + return torch.where(condition, score + log_scale, score) + + +@lru_cache(maxsize=32) +def _get_score_mod(hw: int, log_scale: float = -1.0): + return partial(_score_mod_impl, hw=hw, log_scale=log_scale) def rope_params(max_seq_len, dim, theta=10000, offset=0): @@ -245,7 +256,7 @@ def __init__( refer_stride=1, use_img_emb=True, use_context_parallel=False, - sparse_type=0 + sparse_type=0, ): super().__init__() self.dim = dim @@ -334,6 +345,7 @@ def forward_gen( e, context_lens, use_context_parallel, + log_scale=0.0, ): origin_latent_f = origin_len // 4 + 1 @@ -394,10 +406,12 @@ def forward_gen( v_incontext[:, target_q_len : target_q_len + ref_f * origin_latent_hw]\ .view(B, ref_f, origin_latent_hw, N, C)[:, :, :ref_hw] = v_ref_src + score_mod = _get_score_mod(hw=int(origin_latent_hw), log_scale=log_scale) + xout_full = attention_forward( q_incontext, k_incontext, v_incontext, q_pattern="b s n d", k_pattern="b s n d", v_pattern="b s n d", out_pattern="b s n d", - use_flex=True, attn_mask=attn_mask, + use_flex=True, attn_mask=attn_mask, score_mod=score_mod ) xout_valid = xout_full[:, :f * origin_latent_hw] @@ -463,7 +477,7 @@ def __init__( refer_offset_w=-1, refer_stride=1, sparse_type=0, - use_context_parallel=False + use_context_parallel=False, ): super().__init__() self.patch_size = patch_size @@ -512,7 +526,7 @@ def __init__( # blocks self.blocks = nn.ModuleList([Incontext_AttentionBlock( dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps, refer_stride, use_img_emb=use_img_emb, use_context_parallel=self.use_context_parallel, - sparse_type=self.sparse_type + sparse_type=self.sparse_type ) for _ in range(num_layers)]) # head @@ -671,6 +685,7 @@ def forward( origin_area, is_uncondtion=False, use_unified_sequence_parallel: bool = False, + log_scale=0.0, use_gradient_checkpointing: bool = False, use_gradient_checkpointing_offload: bool = False, ): @@ -742,6 +757,7 @@ def forward( origin_area=origin_area, origin_len=origin_len, use_context_parallel=use_unified_sequence_parallel, + log_scale=log_scale, ) if use_unified_sequence_parallel: from ..utils.xfuser import get_current_chunk, is_evenly_divisible, get_sequence_parallel_world_size diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index 0097a56a..2caedaec 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -236,6 +236,7 @@ def __call__( animate2_reference_image: Image.Image = None, animate2_reference_video: list[Image.Image] = None, animate2_offload_kv: bool = False, + animate2_log_scale: float = 0.0, # VAP vap_video: list[Image.Image] = None, vap_prompt: str = " ", @@ -313,7 +314,7 @@ def __call__( "sliding_window_size": sliding_window_size, "sliding_window_stride": sliding_window_stride, "input_audio": input_audio, "audio_sample_rate": audio_sample_rate, "s2v_pose_video": s2v_pose_video, "audio_embeds": audio_embeds, "s2v_pose_latents": s2v_pose_latents, "motion_video": motion_video, "animate_pose_video": animate_pose_video, "animate_face_video": animate_face_video, "animate_inpaint_video": animate_inpaint_video, "animate_mask_video": animate_mask_video, - "animate2_prompt_ref": animate2_prompt_ref, "animate2_reference_image": animate2_reference_image, "animate2_reference_video": animate2_reference_video, "animate2_offload_kv": animate2_offload_kv, + "animate2_prompt_ref": animate2_prompt_ref, "animate2_reference_image": animate2_reference_image, "animate2_reference_video": animate2_reference_video, "animate2_offload_kv": animate2_offload_kv, "animate2_log_scale": animate2_log_scale, "vap_video": vap_video, "wantodance_music_path": wantodance_music_path, "wantodance_reference_image": wantodance_reference_image, "wantodance_fps": wantodance_fps, "wantodance_keyframes": wantodance_keyframes, "wantodance_keyframes_mask": wantodance_keyframes_mask, @@ -1278,12 +1279,12 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_ref class WanVideoUnit_Animate2RefKVCacheEmbedder(PipelineUnit): def __init__(self): super().__init__( - input_params=("animate2_reference_video", "condition_latents", "condition_y", "context_ref", "clip_fea_ref", "grid_sizes", "seq_len_ref", "animate2_offload_kv", "use_gradient_checkpointing", "use_gradient_checkpointing_offload"), + input_params=("animate2_reference_video", "condition_latents", "condition_y", "context_ref", "clip_fea_ref", "grid_sizes", "seq_len_ref", "animate2_offload_kv", "use_gradient_checkpointing", "use_gradient_checkpointing_offload", "use_unified_sequence_parallel"), output_params=("animate2_k_cache", "animate2_v_cache"), onload_model_names=("dit",) ) - def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref, animate2_offload_kv, use_gradient_checkpointing, use_gradient_checkpointing_offload): + def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref, animate2_offload_kv, use_gradient_checkpointing, use_gradient_checkpointing_offload, use_unified_sequence_parallel): if animate2_reference_video is None: return {} pipe.load_models_to_device(self.onload_model_names) @@ -1301,7 +1302,7 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_la seq_len_ref=seq_len_ref, t=timestep, animate2_offload_kv=animate2_offload_kv, - use_unified_sequence_parallel=pipe.use_unified_sequence_parallel, + use_unified_sequence_parallel=use_unified_sequence_parallel, use_gradient_checkpointing=use_gradient_checkpointing, use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) @@ -1908,6 +1909,7 @@ def model_fn_wananimate( seq_len: int = None, positive: bool = True, use_unified_sequence_parallel: bool = False, + animate2_log_scale: float = 0.0, use_gradient_checkpointing_offload: bool = False, use_gradient_checkpointing: bool = False, **kwargs, @@ -1927,6 +1929,7 @@ def model_fn_wananimate( origin_area=origin_area, is_uncondtion=is_uncondtion, use_unified_sequence_parallel=use_unified_sequence_parallel, + log_scale=animate2_log_scale, use_gradient_checkpointing=use_gradient_checkpointing, use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2.py b/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py similarity index 74% rename from examples/wanvideo/model_inference/Wan-Animate-2.py rename to examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py index 89e548fa..6b14a976 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2.py +++ b/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py @@ -2,6 +2,8 @@ from PIL import Image from diffsynth.utils.data import save_video, VideoData from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +import torch.distributed as dist +from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, @@ -17,7 +19,7 @@ pipe = WanVideoPipeline.from_pretrained( torch_dtype=torch.bfloat16, device="cuda", - redirect_common_files=False, + use_usp=True, model_configs=[ ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), @@ -28,11 +30,13 @@ ) # Character animation: reference image (identity) + reference video (motion) -> animated video. -# reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage_640x352.jpg").convert("RGB") -# reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video_640x352.mp4").raw_data() -reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage.jpg").convert("RGB") -reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video.mp4").raw_data() - +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -41,8 +45,9 @@ animate2_reference_image=reference_image, animate2_reference_video=reference_video[:num_frames], animate2_offload_kv=True, - num_frames=num_frames, height=640, width=352, - num_inference_steps=40, cfg_scale=3.0, sigma_shift=5.0, - seed=0, tiled=False, + num_frames=num_frames, height=1280, width=720, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2.mp4", fps=24, quality=5) +if dist.get_rank() == 0: + save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py new file mode 100644 index 00000000..8131527d --- /dev/null +++ b/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py @@ -0,0 +1,52 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), +) + +# Character animation: reference image (identity) + reference video (motion) -> animated video. +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() + +num_frames = 81 +# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + animate2_log_scale=-1.3, + num_frames=num_frames, height=1280, width=720, + num_inference_steps=10, cfg_scale=1.0, + seed=0, tiled=True, +) +save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2.py b/examples/wanvideo/model_inference/Wan-Animate-2-14B.py similarity index 79% rename from examples/wanvideo/model_inference_low_vram/Wan-Animate-2.py rename to examples/wanvideo/model_inference/Wan-Animate-2-14B.py index 989997d5..623d82bf 100644 --- a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2.py +++ b/examples/wanvideo/model_inference/Wan-Animate-2-14B.py @@ -2,12 +2,13 @@ from PIL import Image from diffsynth.utils.data import save_video, VideoData from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, "offload_device": "cpu", "onload_dtype": torch.bfloat16, - "onload_device": "cpu", + "onload_device": "cuda", "preparing_dtype": torch.bfloat16, "preparing_device": "cuda", "computation_dtype": torch.bfloat16, @@ -17,7 +18,6 @@ pipe = WanVideoPipeline.from_pretrained( torch_dtype=torch.bfloat16, device="cuda", - redirect_common_files=False, model_configs=[ ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), @@ -25,15 +25,16 @@ ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), - # vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, - vram_limit=2, ) # Character animation: reference image (identity) + reference video (motion) -> animated video. - -reference_image = Image.open("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/refimage.jpg").convert("RGB") -reference_video = VideoData("/mnt/nas1/zhanghong/project26/main_project/opencode/packages/wan2.2/Wan-Animate-2/examples/video.mp4").raw_data() - +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -43,7 +44,7 @@ animate2_reference_video=reference_video[:num_frames], animate2_offload_kv=True, num_frames=num_frames, height=1280, width=720, - num_inference_steps=40, cfg_scale=3.0, sigma_shift=5.0, - seed=0, tiled=False, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2.mp4", fps=24, quality=5) +save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py new file mode 100644 index 00000000..44fbdeee --- /dev/null +++ b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py @@ -0,0 +1,53 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# Character animation: reference image (identity) + reference video (motion) -> animated video. +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() + +num_frames = 81 +# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + animate2_log_scale=-1.3, + num_frames=num_frames, height=1280, width=720, + num_inference_steps=10, cfg_scale=1.0, + seed=0, tiled=True, +) +save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py new file mode 100644 index 00000000..11cebfbc --- /dev/null +++ b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py @@ -0,0 +1,51 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.bfloat16, + "onload_device": "cpu", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# Character animation: reference image (identity) + reference video (motion) -> animated video. +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +num_frames = 81 +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + num_frames=num_frames, height=1280, width=720, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, +) +save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) From 7aed872a760be7f0b866f7428c532ff4460deb6c Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Mon, 20 Jul 2026 11:22:22 +0800 Subject: [PATCH 11/17] fix bug for low vram --- diffsynth/configs/vram_management_module_maps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diffsynth/configs/vram_management_module_maps.py b/diffsynth/configs/vram_management_module_maps.py index 074b636f..c5d81c96 100644 --- a/diffsynth/configs/vram_management_module_maps.py +++ b/diffsynth/configs/vram_management_module_maps.py @@ -77,7 +77,7 @@ }, "diffsynth.models.wan_animate_2_dit.WanAnimate2Transformer": { "diffsynth.models.wan_video_dit.MLP": "diffsynth.core.vram.layers.AutoWrappedModule", - "diffsynth.models.wan_animate_2_dit.Incontext_AttentionBlock": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule", + "diffsynth.models.wan_animate_2_dit.AttentionBlock": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule", "diffsynth.models.wan_animate_2_dit.Head": "diffsynth.core.vram.layers.AutoWrappedModule", "torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear", "torch.nn.Conv3d": "diffsynth.core.vram.layers.AutoWrappedModule", From 33ce83d0566f0882ece81388bb21d12e5566cf3e Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Mon, 20 Jul 2026 18:08:01 +0800 Subject: [PATCH 12/17] train --- diffsynth/core/attention/attention.py | 2 +- diffsynth/models/wan_animate_2_dit.py | 176 +++++++++++++++--- diffsynth/pipelines/wan_video.py | 76 +++++--- .../full/Wan-Animate-2-14B-Distilled.sh | 36 ++++ .../model_training/full/Wan-Animate-2-14B.sh | 36 ++++ .../lora/Wan-Animate-2-14B-Distilled.sh | 40 ++++ .../model_training/lora/Wan-Animate-2-14B.sh | 40 ++++ examples/wanvideo/model_training/train.py | 2 +- .../Wan-Animate-2-14B-Distilled.py | 53 ++++++ .../validate_full/Wan-Animate-2-14B.py | 52 ++++++ .../Wan-Animate-2-14B-Distilled.py | 51 +++++ .../validate_lora/Wan-Animate-2-14B.py | 50 +++++ 12 files changed, 567 insertions(+), 47 deletions(-) create mode 100644 examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh create mode 100644 examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh create mode 100644 examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh create mode 100644 examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh create mode 100644 examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py create mode 100644 examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py create mode 100644 examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py create mode 100644 examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py diff --git a/diffsynth/core/attention/attention.py b/diffsynth/core/attention/attention.py index bddd75b2..7cff7c14 100644 --- a/diffsynth/core/attention/attention.py +++ b/diffsynth/core/attention/attention.py @@ -28,7 +28,7 @@ try: from torch.nn.attention.flex_attention import flex_attention as flex_attention_func - flex_attention_func = torch.compile(flex_attention_func, dynamic=False, mode="max-autotune", fullgraph=True, backend="inductor") + flex_attention_func = torch.compile(flex_attention_func, dynamic=False, mode="max-autotune-no-cudagraphs", fullgraph=True, backend="inductor") FLEX_ATTN_AVAILABLE = True except (ModuleNotFoundError, ImportError): FLEX_ATTN_AVAILABLE = False diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index e603c366..f744a9fc 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -288,7 +288,7 @@ def forward_ref( grid_sizes_ref, e_ref, context_lens, - animate2_offload_kv, + animate2_offload_kv=False, use_context_parallel=False, ): q_ref, k_ref, v_ref, e_ref = self.block(x_ref, e_ref, method='pre_self_attention') @@ -344,7 +344,7 @@ def forward_gen( origin_area, e, context_lens, - use_context_parallel, + use_context_parallel=False, log_scale=0.0, ): @@ -429,6 +429,12 @@ def forward_gen( x = self.block(x, context, context_lens, e, method='cross_attention') return x + def forward_origin(self, x, x_ref, ref_args, gen_args): + k_cache, v_cache = {}, {} + x_ref = self.forward_ref(x_ref, 0, k_cache, v_cache, **ref_args) + x = self.forward_gen(x, 0, k_cache, v_cache, **gen_args) + return x, x_ref + class Head(nn.Module): @@ -596,10 +602,8 @@ def forward_ref( context_ref, seq_len_ref, t, - animate2_offload_kv, + animate2_offload_kv=False, use_unified_sequence_parallel: bool = False, - use_gradient_checkpointing: bool = False, - use_gradient_checkpointing_offload: bool = False, ): # [reference] x_ref = [torch.cat([u, v], dim=0) for u, v in zip(x_ref, y_ref)] @@ -662,15 +666,10 @@ def forward_ref( ) x_ref = get_current_chunk(x_ref, dim=1) for idx, block in enumerate(self.blocks): - x_ref = gradient_checkpoint_forward( - block, - use_gradient_checkpointing, - use_gradient_checkpointing_offload, - x_ref, idx, k_cache, v_cache, method='forward_ref', **kwargs - ) + x_ref = block(x_ref, idx, k_cache, v_cache, method='forward_ref', **kwargs) - def forward( + def forward_gen( self, x, k_cache, @@ -686,11 +685,8 @@ def forward( is_uncondtion=False, use_unified_sequence_parallel: bool = False, log_scale=0.0, - use_gradient_checkpointing: bool = False, - use_gradient_checkpointing_offload: bool = False, ): # params - device = self.patch_embedding.weight.device x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] # embeddings x = [self.patch_embedding(u.unsqueeze(0)) for u in x] @@ -769,12 +765,7 @@ def forward( for idx, block in enumerate(self.blocks): if is_uncondtion and idx==9: continue - x = gradient_checkpoint_forward( - block, - use_gradient_checkpointing, - use_gradient_checkpointing_offload, - x, idx, k_cache, v_cache, method='forward_gen', **kwargs - ) + x = block(x, idx, k_cache, v_cache, method='forward_gen', **kwargs) # head x = self.head(x, e) @@ -786,6 +777,149 @@ def forward( x = self.unpatchify(x, grid_sizes) return [u for u in x] + def forward_origin( + self, + x, + clip_fea, + y, + context, + seq_len, + x_ref, + clip_fea_ref, + y_ref, + context_ref, + seq_len_ref, + t, + origin_len, + origin_area, + log_scale=0.0, + use_gradient_checkpointing: bool = False, + use_gradient_checkpointing_offload: bool = False, + ): + # params + x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + grid_sizes = torch.stack([ + torch.tensor(u.shape[2:], dtype=torch.long) for u in x + ]) + x = [u.flatten(2).transpose(1, 2) for u in x] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + assert seq_lens.max() <= seq_len + x = torch.cat([torch.cat([ + u, u.new_zeros(1, seq_len - u.size(1), u.size(2)) + ], dim=1) for u in x]) + + # [reference] + # params + x_ref = [torch.cat([u, v], dim=0) for u, v in zip(x_ref, y_ref)] + # embeddings + x_ref = [self.patch_embedding(u.unsqueeze(0)) for u in x_ref] + grid_sizes_ref = torch.stack([ + torch.tensor(u.shape[2:], dtype=torch.long) for u in x_ref + ]) + x_ref = [u.flatten(2).transpose(1, 2) for u in x_ref] + seq_lens_ref = torch.tensor([u.size(1) for u in x_ref], dtype=torch.long) + assert seq_lens_ref.max() <= seq_len_ref + x_ref = torch.cat([torch.cat([ + u, u.new_zeros(1, seq_len_ref - u.size(1), u.size(2)) + ], dim=1) for u in x_ref]) + + assert (self.dim % self.num_heads) == 0 and (self.dim // self.num_heads) % 2 == 0 + d = self.dim // self.num_heads + self.freqs = torch.cat([ + rope_params(512, d - 4 * (d // 6)), + rope_params(512, 2 * (d // 6)), + rope_params(512, 2 * (d // 6)) + ], dim=1).to(x.device) + + if self.refer_offset_t < 0: + self.refer_offset_t = grid_sizes[0][0].item() + if self.refer_offset_h < 0: + self.refer_offset_h = grid_sizes[0][1].item() + if self.refer_offset_w < 0: + self.refer_offset_w = grid_sizes[0][2].item() + + self.freqs_ref = torch.cat([ + rope_params(512, d - 4 * (d // 6), offset=self.refer_offset_t), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_h), + rope_params(512, 2 * (d // 6), offset=self.refer_offset_w) + ], dim=1).to(x.device) + + # time embeddings + e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t).to(x.dtype)) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + + # time embeddings ref + e_ref = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t*0+1).to(x.dtype)) + e0_ref = self.time_projection(e_ref).unflatten(1, (6, self.dim)) + + # [context] + context_lens = None + context = self.text_embedding(torch.stack([torch.cat([ + u, u.new_zeros(self.text_len - u.size(0), u.size(1)) + ]) for u in context])) + + if self.use_img_emb: + context_clip = self.img_emb(clip_fea) # bs x 257 x dim + context = torch.concat([context_clip, context], dim=1) + + # [context_ref] + context_ref = self.text_embedding(torch.stack([torch.cat([ + u, u.new_zeros(self.text_len - u.size(0), u.size(1)) + ]) for u in context_ref])) + + if self.use_img_emb: + context_clip_ref = self.img_emb(clip_fea_ref) # bs x 257 x dim + context_ref = torch.concat([context_clip_ref, context_ref], dim=1) + + block_mask_id = (origin_len, origin_area[0], origin_area[1]) + if block_mask_id not in self.block_masks: + self.block_masks[block_mask_id] = self.create_mask(origin_len, origin_area, x.device) + attn_mask = self.block_masks[block_mask_id] + + # arguments + ref_args = dict( + e_ref=e0_ref, + grid_sizes_ref=grid_sizes_ref, + freqs_ref=self.freqs_ref, + context_ref=context_ref, + context_lens=context_lens, + ) + gen_args = dict( + e=e0, + attn_mask=attn_mask, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + grid_sizes_ref=grid_sizes_ref, + freqs_ref=self.freqs_ref, + context_lens=context_lens, + origin_area=origin_area, + origin_len=origin_len, + log_scale=log_scale, + ) + for idx, block in enumerate(self.blocks): + x, x_ref = gradient_checkpoint_forward( + block, + use_gradient_checkpointing, + use_gradient_checkpointing_offload, + x, x_ref, ref_args, gen_args, method='forward_origin' + ) + + # head + x = self.head(x, e) + + # Context Parallel + if self.use_context_parallel: + x = ops.gather_forward_split_backward( + x, dim=1, group=sp_group, grad_scale="up" + ) + + # unpatchify + x = self.unpatchify(x, grid_sizes) + return [u.float() for u in x] + def unpatchify(self, x, grid_sizes): c = self.out_dim out = [] diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index 2caedaec..4996732e 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -409,18 +409,17 @@ def process(self, pipe: WanVideoPipeline, height, width, num_frames, seed, rand_ if vace_reference_image is not None: noise = torch.concat((noise[:, :, -f:], noise[:, :, :-f]), dim=2) return {"noise": noise} - class WanVideoUnit_InputVideoEmbedder(PipelineUnit): def __init__(self): super().__init__( - input_params=("input_video", "noise", "tiled", "tile_size", "tile_stride", "vace_reference_image", "framewise_decoding"), + input_params=("input_video", "noise", "tiled", "tile_size", "tile_stride", "vace_reference_image", "framewise_decoding", "animate2_reference_image", "animate2_reference_video"), output_params=("latents", "input_latents"), onload_model_names=("vae",) ) - def process(self, pipe: WanVideoPipeline, input_video, noise, tiled, tile_size, tile_stride, vace_reference_image, framewise_decoding): + def process(self, pipe: WanVideoPipeline, input_video, noise, tiled, tile_size, tile_stride, vace_reference_image, framewise_decoding, animate2_reference_image, animate2_reference_video): if input_video is None: return {"latents": noise} pipe.load_models_to_device(self.onload_model_names) @@ -436,6 +435,12 @@ def process(self, pipe: WanVideoPipeline, input_video, noise, tiled, tile_size, vace_reference_latents = pipe.vae.encode(vace_reference_image, device=pipe.device).to(dtype=pipe.torch_dtype, device=pipe.device) input_latents = torch.concat([vace_reference_latents, input_latents], dim=2) if pipe.scheduler.training: + if animate2_reference_image is not None and animate2_reference_video is not None: + vh, vw = input_video.shape[-2], input_video.shape[-1] + ref_img = pipe.preprocess_image(animate2_reference_image.resize((vw, vh))).to(pipe.device) # (1, C, H, W) + ref_pixel = ref_img.transpose(0, 1) # (C, 1, H, W) + ref_latent = pipe.vae.encode([ref_pixel.to(pipe.torch_dtype)], device=pipe.device).to(dtype=pipe.torch_dtype, device=pipe.device) + input_latents = torch.concat([ref_latent, input_latents], dim=2) return {"latents": noise, "input_latents": input_latents} else: latents = pipe.scheduler.add_noise(input_latents, noise, timestep=pipe.scheduler.timesteps[0]) @@ -1285,7 +1290,7 @@ def __init__(self): ) def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_latents, condition_y, context_ref, clip_fea_ref, grid_sizes, seq_len_ref, animate2_offload_kv, use_gradient_checkpointing, use_gradient_checkpointing_offload, use_unified_sequence_parallel): - if animate2_reference_video is None: + if animate2_reference_video is None or pipe.scheduler.training: return {} pipe.load_models_to_device(self.onload_model_names) animate2_k_cache, animate2_v_cache = {}, {} @@ -1303,8 +1308,6 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_video, condition_la t=timestep, animate2_offload_kv=animate2_offload_kv, use_unified_sequence_parallel=use_unified_sequence_parallel, - use_gradient_checkpointing=use_gradient_checkpointing, - use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, ) return {"animate2_k_cache": animate2_k_cache, "animate2_v_cache": animate2_v_cache} @@ -1908,6 +1911,11 @@ def model_fn_wananimate( origin_area: list = None, seq_len: int = None, positive: bool = True, + condition_latents: torch.Tensor = None, + clip_fea_ref: torch.Tensor = None, + condition_y: torch.Tensor = None, + context_ref: torch.Tensor = None, + seq_len_ref: int = None, use_unified_sequence_parallel: bool = False, animate2_log_scale: float = 0.0, use_gradient_checkpointing_offload: bool = False, @@ -1915,22 +1923,42 @@ def model_fn_wananimate( **kwargs, ): is_uncondtion = not positive - out = dit( - [latents[0]], - k_cache=animate2_k_cache, - v_cache=animate2_v_cache, - clip_fea=clip_feature, - y=[y[0]], - context=[context[0]], - seq_len=seq_len, - t=timestep, - grid_sizes_ref=grid_sizes_ref.to(latents.device), - origin_len=origin_len, - origin_area=origin_area, - is_uncondtion=is_uncondtion, - use_unified_sequence_parallel=use_unified_sequence_parallel, - log_scale=animate2_log_scale, - use_gradient_checkpointing=use_gradient_checkpointing, - use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, - ) + if animate2_k_cache is None or animate2_v_cache is None: + # training + out = dit.forward_origin( + x=[latents[0]], + clip_fea=clip_feature, + y=[y[0]], + context=[context[0]], + seq_len=seq_len, + x_ref=condition_latents, + clip_fea_ref=clip_fea_ref, + y_ref=[condition_y], + context_ref=[context_ref[0]], + seq_len_ref=seq_len_ref, + t=timestep, + origin_len=origin_len, + origin_area=origin_area, + log_scale=animate2_log_scale, + use_gradient_checkpointing=use_gradient_checkpointing, + use_gradient_checkpointing_offload=use_gradient_checkpointing_offload, + ) + else: + # inference + out = dit.forward_gen( + x=[latents[0]], + k_cache=animate2_k_cache, + v_cache=animate2_v_cache, + clip_fea=clip_feature, + y=[y[0]], + context=[context[0]], + seq_len=seq_len, + t=timestep, + grid_sizes_ref=grid_sizes_ref.to(latents.device), + origin_len=origin_len, + origin_area=origin_area, + is_uncondtion=is_uncondtion, + use_unified_sequence_parallel=use_unified_sequence_parallel, + log_scale=animate2_log_scale, + ) return out[0].unsqueeze(0) diff --git a/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh b/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh new file mode 100644 index 00000000..12c0e7d8 --- /dev/null +++ b/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh @@ -0,0 +1,36 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B-Distilled/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/wanvideo/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/metadata.json \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 41 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B-Distilled_full_splited_cache" \ + --trainable_models "dit" \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ + --dataset_base_path models/train/Wan-Animate-2-14B-Distilled_full_splited_cache \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 41 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16_distillation.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B-Distilled_full" \ + --trainable_models "dit" \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh b/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh new file mode 100644 index 00000000..3ad383ba --- /dev/null +++ b/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh @@ -0,0 +1,36 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/wanvideo/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/metadata.json \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 41 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B_full_splited_cache" \ + --trainable_models "dit" \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ + --dataset_base_path models/train/Wan-Animate-2-14B_full_splited_cache \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 41 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16.safetensors" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B_full" \ + --trainable_models "dit" \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh new file mode 100644 index 00000000..4b90ccf9 --- /dev/null +++ b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh @@ -0,0 +1,40 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B-Distilled/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/wanvideo/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/metadata.json \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 81 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B-Distilled_lora_splited_cache" \ + --lora_base_model "dit" \ + --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_rank 32 \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ + --dataset_base_path models/train/Wan-Animate-2-14B-Distilled_lora_splited_cache \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 81 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16_distillation.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B-Distilled_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_rank 32 \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh new file mode 100644 index 00000000..378df440 --- /dev/null +++ b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh @@ -0,0 +1,40 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch examples/wanvideo/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/metadata.json \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 81 \ + --dataset_repeat 1 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B_lora_splited_cache" \ + --lora_base_model "dit" \ + --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_rank 32 \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:data_process" + +accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ + --dataset_base_path models/train/Wan-Animate-2-14B_lora_splited_cache \ + --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ + --height 640 \ + --width 352 \ + --num_frames 81 \ + --dataset_repeat 100 \ + --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16.safetensors" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/Wan-Animate-2-14B_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_rank 32 \ + --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ + --use_gradient_checkpointing \ + --task "sft:train" diff --git a/examples/wanvideo/model_training/train.py b/examples/wanvideo/model_training/train.py index ce8fedcd..45c94135 100644 --- a/examples/wanvideo/model_training/train.py +++ b/examples/wanvideo/model_training/train.py @@ -70,7 +70,7 @@ def parse_extra_inputs(self, data, extra_inputs, inputs_shared): inputs_shared["input_image"] = data["video"][0] elif extra_input == "end_image": inputs_shared["end_image"] = data["video"][-1] - elif extra_input == "reference_image" or extra_input == "vace_reference_image": + elif extra_input in ("reference_image", "vace_reference_image", "animate2_reference_image"): inputs_shared[extra_input] = data[extra_input][0] else: inputs_shared[extra_input] = data[extra_input] diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py new file mode 100644 index 00000000..7362bfb3 --- /dev/null +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py @@ -0,0 +1,53 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.core import load_state_dict +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), +) +state_dict = load_state_dict("models/train/Wan-Animate-2-14B-Distilled_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict, strict=False) + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +num_frames = 41 +# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + animate2_log_scale=-1.3, + num_frames=num_frames, height=640, width=352, + num_inference_steps=10, cfg_scale=1.0, + seed=0, tiled=True, +) +save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py new file mode 100644 index 00000000..b18ffbfa --- /dev/null +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py @@ -0,0 +1,52 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.core import load_state_dict +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), +) +state_dict = load_state_dict("models/train/Wan-Animate-2-14B_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict, strict=False) + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +num_frames = 41 +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + num_frames=num_frames, height=640, width=352, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, +) +save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py new file mode 100644 index 00000000..dece5404 --- /dev/null +++ b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py @@ -0,0 +1,51 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), +) +pipe.load_lora(pipe.dit, "models/train/Wan-Animate-2-14B-Distilled_lora/epoch-4.safetensors", alpha=1) + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +num_frames = 81 +# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + animate2_log_scale=-1.3, + num_frames=num_frames, height=640, width=352, + num_inference_steps=10, cfg_scale=1.0, + seed=0, tiled=True, +) +save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py new file mode 100644 index 00000000..7b996e83 --- /dev/null +++ b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py @@ -0,0 +1,50 @@ +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": torch.bfloat16, + "offload_device": "cpu", + "onload_dtype": torch.bfloat16, + "onload_device": "cuda", + "preparing_dtype": torch.bfloat16, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = WanVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), + ], + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), +) +pipe.load_lora(pipe.dit, "models/train/Wan-Animate-2-14B_lora/epoch-4.safetensors", alpha=1) + +dataset_snapshot_download( + "DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" +) +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +num_frames = 81 +video = pipe( + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_reference_image=reference_image, + animate2_reference_video=reference_video[:num_frames], + animate2_offload_kv=True, + num_frames=num_frames, height=640, width=352, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, +) +save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) From 413b0d6b0a8f5c9737438bf737a7b95da5f00f06 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Mon, 20 Jul 2026 19:59:22 +0800 Subject: [PATCH 13/17] multi-clp generation --- diffsynth/pipelines/wan_video.py | 29 ++++++--- .../acceleration/Wan-Animate-2-14B-usp.py | 64 +++++++++++++++++++ .../Wan-Animate-2-14B-Distilled.py | 63 ++++++++++++++++++ .../model_inference/Wan-Animate-2-14B.py | 63 ++++++++++++++++++ .../Wan-Animate-2-14B-Distilled.py | 64 +++++++++++++++++++ .../Wan-Animate-2-14B.py | 63 ++++++++++++++++++ .../Wan-Animate-2-14B-Distilled.py | 5 -- .../validate_full/Wan-Animate-2-14B.py | 5 -- .../Wan-Animate-2-14B-Distilled.py | 5 -- .../validate_lora/Wan-Animate-2-14B.py | 5 -- 10 files changed, 336 insertions(+), 30 deletions(-) diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index 4996732e..7f600fde 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -235,6 +235,7 @@ def __call__( animate2_prompt_ref: str = " ", animate2_reference_image: Image.Image = None, animate2_reference_video: list[Image.Image] = None, + animate2_refert_images: list[Image.Image] = None, animate2_offload_kv: bool = False, animate2_log_scale: float = 0.0, # VAP @@ -314,7 +315,7 @@ def __call__( "sliding_window_size": sliding_window_size, "sliding_window_stride": sliding_window_stride, "input_audio": input_audio, "audio_sample_rate": audio_sample_rate, "s2v_pose_video": s2v_pose_video, "audio_embeds": audio_embeds, "s2v_pose_latents": s2v_pose_latents, "motion_video": motion_video, "animate_pose_video": animate_pose_video, "animate_face_video": animate_face_video, "animate_inpaint_video": animate_inpaint_video, "animate_mask_video": animate_mask_video, - "animate2_prompt_ref": animate2_prompt_ref, "animate2_reference_image": animate2_reference_image, "animate2_reference_video": animate2_reference_video, "animate2_offload_kv": animate2_offload_kv, "animate2_log_scale": animate2_log_scale, + "animate2_prompt_ref": animate2_prompt_ref, "animate2_reference_image": animate2_reference_image, "animate2_reference_video": animate2_reference_video, "animate2_refert_images": animate2_refert_images, "animate2_offload_kv": animate2_offload_kv, "animate2_log_scale": animate2_log_scale, "vap_video": vap_video, "wantodance_music_path": wantodance_music_path, "wantodance_reference_image": wantodance_reference_image, "wantodance_fps": wantodance_fps, "wantodance_keyframes": wantodance_keyframes, "wantodance_keyframes_mask": wantodance_keyframes_mask, @@ -1213,7 +1214,7 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_ref class WanVideoUnit_Animate2VAEEmbedder(PipelineUnit): def __init__(self): super().__init__( - input_params=("animate2_reference_image", "animate2_reference_video", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride"), + input_params=("animate2_reference_image", "animate2_reference_video", "animate2_refert_images", "num_frames", "height", "width", "tiled", "tile_size", "tile_stride"), output_params=("y", "condition_latents", "condition_y", "grid_sizes", "grid_sizes_ref", "seq_len", "seq_len_ref", "origin_len", "origin_area"), onload_model_names=("vae",) ) @@ -1228,7 +1229,7 @@ def animate2_get_i2v_mask(lat_t, lat_h, lat_w, mask_len=1, device="cuda"): return msk - def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_reference_video, num_frames, height, width, tiled, tile_size, tile_stride): + def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_reference_video, animate2_refert_images, num_frames, height, width, tiled, tile_size, tile_stride): if animate2_reference_image is None or animate2_reference_video is None: return {} pipe.load_models_to_device(self.onload_model_names) @@ -1241,21 +1242,29 @@ def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_ref # Reference image -> y_ref (mask + latent), single latent frame ref_img = pipe.preprocess_image(animate2_reference_image.resize((W, H))).to(device) # (1, C, H, W) ref_pixel = ref_img.transpose(0, 1) # (C, 1, H, W) - ref_latents = pipe.vae.encode([ref_pixel.to(dtype)], device=device).to(dtype=dtype, device=device) # (1, 16, 1, lat_h, lat_w) + ref_latents = pipe.vae.encode([ref_pixel.to(dtype)], device, tiled, tile_size, tile_stride).to(dtype=dtype, device=device) # (1, 16, 1, lat_h, lat_w) mask_ref = self.animate2_get_i2v_mask(1, lat_h, lat_w, mask_len=1, device=device) y_ref = torch.concat([mask_ref, ref_latents[0]]).to(dtype=dtype, device=device) # (20, 1, lat_h, lat_w) - # Reference-temporal slot: zeros for the single-clip case, mask_reft_len == 0 - # TODO: check reft - zeros_vid = torch.zeros(3, T - 1, H, W, device=device, dtype=dtype) - y_reft = pipe.vae.encode([zeros_vid], device=device)[0].to(dtype=dtype, device=device) # (16, lat_t-1, lat_h, lat_w) - msk_reft = self.animate2_get_i2v_mask(lat_t - 1, lat_h, lat_w, mask_len=0, device=device) + # Reference-temporal slot: for multi-clip long video, the first `mask_reft_len` temporal + # positions carry the previous clip's tail frames (continuation); the rest are zeros. + # Single/first clip: animate2_refert_images is None -> mask_reft_len == 0 (all zeros). + if animate2_refert_images is not None and len(animate2_refert_images) > 0: + mask_reft_len = len(animate2_refert_images) + refert = pipe.preprocess_video([f.resize((W, H)) for f in animate2_refert_images]).to(device) # (1, 3, mask_reft_len, H, W) + zeros_tail = torch.zeros(3, T - 1 - mask_reft_len, H, W, device=device, dtype=dtype) + reft_vid = torch.concat([refert[0].to(dtype), zeros_tail], dim=1) # (3, T-1, H, W) + else: + mask_reft_len = 0 + reft_vid = torch.zeros(3, T - 1, H, W, device=device, dtype=dtype) + y_reft = pipe.vae.encode([reft_vid], device, tiled, tile_size, tile_stride)[0].to(dtype=dtype, device=device) # (16, lat_t-1, lat_h, lat_w) + msk_reft = self.animate2_get_i2v_mask(lat_t - 1, lat_h, lat_w, mask_len=mask_reft_len, device=device) y_reft = torch.concat([msk_reft, y_reft]).to(dtype=dtype, device=device) y = torch.concat([y_ref, y_reft], dim=1) # (20, lat_t, lat_h, lat_w) # Reference video -> condition_latents ref_video = pipe.preprocess_video([f.resize((W, H)) for f in animate2_reference_video[:num_frames]]).to(device) # (1, C, num_frames, H, W) - condition_latents = pipe.vae.encode([ref_video[0].to(dtype)], device=device).to(dtype=dtype, device=device) # (1, 16, lat_t_c, lat_h, lat_w) + condition_latents = pipe.vae.encode([ref_video[0].to(dtype)], device, tiled, tile_size, tile_stride).to(dtype=dtype, device=device) # (1, 16, lat_t_c, lat_h, lat_w) # Reference video -> condition_y (mask + latent), mask_len = T _, _, lat_t_c, lat_h_c, lat_w_c = condition_latents.shape condition_y = condition_latents.clone()[0] diff --git a/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py b/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py index 6b14a976..02536b72 100644 --- a/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py +++ b/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py @@ -37,6 +37,7 @@ ) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +# ===== Example 1: single-clip generation (direct pipeline call) ===== num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -51,3 +52,66 @@ ) if dist.get_rank() == 0: save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) + + +# ===== Example 2: multi-clip long-video generation ===== +def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): + assert clip_len > first_num, "clip_len must be greater than first_num" + + def zigzag_padding(array, target_len): + if len(array) == 1: + return [array[0]] * target_len + idx, flip, out = 0, False, [] + while len(out) < target_len: + out.append(array[idx]) + idx += -1 if flip else 1 + if idx == 0 or idx == len(array) - 1: + flip = not flip + return out[:target_len] + + real_len = len(cond_images) + if real_len == 0: + return [] + step = clip_len - first_num + # Precompute clip count so clips of `clip_len` stepping by `step` tile the (padded) driving video. + num_clips = 1 if real_len <= clip_len else (real_len - clip_len + step - 1) // step + 1 + target_len = clip_len + (num_clips - 1) * step + if real_len < target_len: + cond_images = zigzag_padding(cond_images, target_len) + + all_frames = [] + prev_tail = None + for i in range(num_clips): + start = i * step + seg_driving = cond_images[start:start + clip_len] + seg_out = pipe( + animate2_reference_image=reference_image, + animate2_reference_video=seg_driving, + animate2_refert_images=None if i == 0 else prev_tail, + num_frames=clip_len, + **kwargs, + ) + prev_tail = seg_out[-first_num:] + if i != 0: + seg_out = seg_out[first_num:] + all_frames.extend(seg_out) + return all_frames[:real_len] + + +clip_len = 81 +long_video = generate_long_video( + pipe, + reference_image=reference_image, + cond_images=reference_video, + clip_len=clip_len, + first_num=1, + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_offload_kv=True, + height=1280, width=720, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, +) +if dist.get_rank() == 0: + save_video(long_video, "video_Wan-Animate-2-14B-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py index 8131527d..0e30b7b8 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py @@ -36,6 +36,7 @@ reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +# ===== Example 1: single-clip generation (direct pipeline call) ===== num_frames = 81 # For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. video = pipe( @@ -50,3 +51,65 @@ seed=0, tiled=True, ) save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) + + +# ===== Example 2: multi-clip long-video generation ===== +def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): + assert clip_len > first_num, "clip_len must be greater than first_num" + + def zigzag_padding(array, target_len): + if len(array) == 1: + return [array[0]] * target_len + idx, flip, out = 0, False, [] + while len(out) < target_len: + out.append(array[idx]) + idx += -1 if flip else 1 + if idx == 0 or idx == len(array) - 1: + flip = not flip + return out[:target_len] + + real_len = len(cond_images) + if real_len == 0: + return [] + step = clip_len - first_num + # Precompute clip count so clips of `clip_len` stepping by `step` tile the (padded) driving video. + num_clips = 1 if real_len <= clip_len else (real_len - clip_len + step - 1) // step + 1 + target_len = clip_len + (num_clips - 1) * step + if real_len < target_len: + cond_images = zigzag_padding(cond_images, target_len) + + all_frames = [] + prev_tail = None + for i in range(num_clips): + start = i * step + seg_driving = cond_images[start:start + clip_len] + seg_out = pipe( + animate2_reference_image=reference_image, + animate2_reference_video=seg_driving, + animate2_refert_images=None if i == 0 else prev_tail, + num_frames=clip_len, + **kwargs, + ) + prev_tail = seg_out[-first_num:] + if i != 0: + seg_out = seg_out[first_num:] + all_frames.extend(seg_out) + return all_frames[:real_len] + + +# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. +long_video = generate_long_video( + pipe, + reference_image=reference_image, + cond_images=reference_video, + clip_len=81, + first_num=1, + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_offload_kv=True, + animate2_log_scale=-1.3, + height=1280, width=720, + num_inference_steps=10, cfg_scale=1.0, + seed=0, tiled=True, +) +save_video(long_video, "video_Wan-Animate-2-14B-Distilled-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2-14B.py b/examples/wanvideo/model_inference/Wan-Animate-2-14B.py index 623d82bf..fab06304 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_inference/Wan-Animate-2-14B.py @@ -35,6 +35,7 @@ ) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +# ===== Example 1: single-clip generation (direct pipeline call) ===== num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -48,3 +49,65 @@ seed=0, tiled=True, ) save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) + + +# ===== Example 2: multi-clip long-video generation ===== +def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): + assert clip_len > first_num, "clip_len must be greater than first_num" + + def zigzag_padding(array, target_len): + if len(array) == 1: + return [array[0]] * target_len + idx, flip, out = 0, False, [] + while len(out) < target_len: + out.append(array[idx]) + idx += -1 if flip else 1 + if idx == 0 or idx == len(array) - 1: + flip = not flip + return out[:target_len] + + real_len = len(cond_images) + if real_len == 0: + return [] + step = clip_len - first_num + # Precompute clip count so clips of `clip_len` stepping by `step` tile the (padded) driving video. + num_clips = 1 if real_len <= clip_len else (real_len - clip_len + step - 1) // step + 1 + target_len = clip_len + (num_clips - 1) * step + if real_len < target_len: + cond_images = zigzag_padding(cond_images, target_len) + + all_frames = [] + prev_tail = None + for i in range(num_clips): + start = i * step + seg_driving = cond_images[start:start + clip_len] + seg_out = pipe( + animate2_reference_image=reference_image, + animate2_reference_video=seg_driving, + animate2_refert_images=None if i == 0 else prev_tail, + num_frames=clip_len, + **kwargs, + ) + prev_tail = seg_out[-first_num:] + if i != 0: + seg_out = seg_out[first_num:] + all_frames.extend(seg_out) + return all_frames[:real_len] + + +clip_len = 81 +long_video = generate_long_video( + pipe, + reference_image=reference_image, + cond_images=reference_video, + clip_len=clip_len, + first_num=1, + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_offload_kv=True, + height=1280, width=720, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, +) +save_video(long_video, "video_Wan-Animate-2-14B-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py index 44fbdeee..0a949d4f 100644 --- a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py @@ -37,6 +37,7 @@ reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +# ===== Example 1: single-clip generation (direct pipeline call) ===== num_frames = 81 # For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. video = pipe( @@ -51,3 +52,66 @@ seed=0, tiled=True, ) save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) + + +# ===== Example 2: multi-clip long-video generation ===== +def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): + assert clip_len > first_num, "clip_len must be greater than first_num" + + def zigzag_padding(array, target_len): + if len(array) == 1: + return [array[0]] * target_len + idx, flip, out = 0, False, [] + while len(out) < target_len: + out.append(array[idx]) + idx += -1 if flip else 1 + if idx == 0 or idx == len(array) - 1: + flip = not flip + return out[:target_len] + + real_len = len(cond_images) + if real_len == 0: + return [] + step = clip_len - first_num + # Precompute clip count so clips of `clip_len` stepping by `step` tile the (padded) driving video. + num_clips = 1 if real_len <= clip_len else (real_len - clip_len + step - 1) // step + 1 + target_len = clip_len + (num_clips - 1) * step + if real_len < target_len: + cond_images = zigzag_padding(cond_images, target_len) + + all_frames = [] + prev_tail = None + for i in range(num_clips): + start = i * step + seg_driving = cond_images[start:start + clip_len] + seg_out = pipe( + animate2_reference_image=reference_image, + animate2_reference_video=seg_driving, + animate2_refert_images=None if i == 0 else prev_tail, + num_frames=clip_len, + **kwargs, + ) + prev_tail = seg_out[-first_num:] + if i != 0: + seg_out = seg_out[first_num:] + all_frames.extend(seg_out) + return all_frames[:real_len] + + +clip_len = 81 +# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. +long_video = generate_long_video( + pipe, + reference_image=reference_image, + cond_images=reference_video, + clip_len=clip_len, + first_num=1, + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_offload_kv=True, + animate2_log_scale=-1.3, + height=1280, width=720, + num_inference_steps=10, cfg_scale=1.0, + seed=0, tiled=True, +) +save_video(long_video, "video_Wan-Animate-2-14B-Distilled-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py index 11cebfbc..504eac65 100644 --- a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py @@ -36,6 +36,7 @@ ) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +# ===== Example 1: single-clip generation (direct pipeline call) ===== num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -49,3 +50,65 @@ seed=0, tiled=True, ) save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) + + +# ===== Example 2: multi-clip long-video generation ===== +def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): + assert clip_len > first_num, "clip_len must be greater than first_num" + + def zigzag_padding(array, target_len): + if len(array) == 1: + return [array[0]] * target_len + idx, flip, out = 0, False, [] + while len(out) < target_len: + out.append(array[idx]) + idx += -1 if flip else 1 + if idx == 0 or idx == len(array) - 1: + flip = not flip + return out[:target_len] + + real_len = len(cond_images) + if real_len == 0: + return [] + step = clip_len - first_num + # Precompute clip count so clips of `clip_len` stepping by `step` tile the (padded) driving video. + num_clips = 1 if real_len <= clip_len else (real_len - clip_len + step - 1) // step + 1 + target_len = clip_len + (num_clips - 1) * step + if real_len < target_len: + cond_images = zigzag_padding(cond_images, target_len) + + all_frames = [] + prev_tail = None + for i in range(num_clips): + start = i * step + seg_driving = cond_images[start:start + clip_len] + seg_out = pipe( + animate2_reference_image=reference_image, + animate2_reference_video=seg_driving, + animate2_refert_images=None if i == 0 else prev_tail, + num_frames=clip_len, + **kwargs, + ) + prev_tail = seg_out[-first_num:] + if i != 0: + seg_out = seg_out[first_num:] + all_frames.extend(seg_out) + return all_frames[:real_len] + + +clip_len = 81 +long_video = generate_long_video( + pipe, + reference_image=reference_image, + cond_images=reference_video, + clip_len=clip_len, + first_num=1, + prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + animate2_prompt_ref="视频中的人在做动作，背景静止", + animate2_offload_kv=True, + height=1280, width=720, + num_inference_steps=40, cfg_scale=3.0, + seed=0, tiled=True, +) +save_video(long_video, "video_Wan-Animate-2-14B-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py index 7362bfb3..178d5ab0 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py @@ -30,11 +30,6 @@ state_dict = load_state_dict("models/train/Wan-Animate-2-14B-Distilled_full/epoch-1.safetensors") pipe.dit.load_state_dict(state_dict, strict=False) -dataset_snapshot_download( - "DiffSynth-Studio/diffsynth_example_dataset", - local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" -) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() num_frames = 41 diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py index b18ffbfa..ec896a7a 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py @@ -30,11 +30,6 @@ state_dict = load_state_dict("models/train/Wan-Animate-2-14B_full/epoch-1.safetensors") pipe.dit.load_state_dict(state_dict, strict=False) -dataset_snapshot_download( - "DiffSynth-Studio/diffsynth_example_dataset", - local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" -) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() num_frames = 41 diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py index dece5404..6d90d39d 100644 --- a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py @@ -28,11 +28,6 @@ ) pipe.load_lora(pipe.dit, "models/train/Wan-Animate-2-14B-Distilled_lora/epoch-4.safetensors", alpha=1) -dataset_snapshot_download( - "DiffSynth-Studio/diffsynth_example_dataset", - local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" -) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() num_frames = 81 diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py index 7b996e83..c0837b98 100644 --- a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py @@ -28,11 +28,6 @@ ) pipe.load_lora(pipe.dit, "models/train/Wan-Animate-2-14B_lora/epoch-4.safetensors", alpha=1) -dataset_snapshot_download( - "DiffSynth-Studio/diffsynth_example_dataset", - local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" -) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() num_frames = 81 From 405aa2d73fa2558f93672d4800eafad60bd265c2 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Mon, 20 Jul 2026 20:36:57 +0800 Subject: [PATCH 14/17] style check --- diffsynth/models/wan_animate_2_dit.py | 30 ++++--------------- diffsynth/pipelines/wan_video.py | 1 - .../utils/xfuser/xdit_context_parallel.py | 2 -- .../acceleration/Wan-Animate-2-14B-usp.py | 6 ++-- .../Wan-Animate-2-14B-Distilled.py | 4 +-- .../model_inference/Wan-Animate-2-14B.py | 4 +-- .../Wan-Animate-2-14B-Distilled.py | 4 +-- .../Wan-Animate-2-14B.py | 4 +-- .../Wan-Animate-2-14B-Distilled.py | 1 - .../validate_full/Wan-Animate-2-14B.py | 1 - .../Wan-Animate-2-14B-Distilled.py | 1 - .../validate_lora/Wan-Animate-2-14B.py | 1 - 12 files changed, 17 insertions(+), 42 deletions(-) diff --git a/diffsynth/models/wan_animate_2_dit.py b/diffsynth/models/wan_animate_2_dit.py index f744a9fc..54750db9 100644 --- a/diffsynth/models/wan_animate_2_dit.py +++ b/diffsynth/models/wan_animate_2_dit.py @@ -142,16 +142,9 @@ def __init__(self, dim, num_heads, window_size=(-1, -1), qk_norm=True, eps=1e-6, self.norm_k_img = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() def forward(self, x, context, context_lens, counter=0): - """ - x: [B, L1, C]. - context: [B, L2, C]. - context_lens: [B]. - """ if self.use_img_emb: context_img = context[:, :257] context = context[:, 257:] - else: - context = context b, n, d = x.size(0), self.num_heads, self.head_dim @@ -379,7 +372,6 @@ def forward_gen( B, _, N, C = q.shape device, dtype = q.device, q.dtype - target_q_len = math.ceil(origin_max_len / 128) * 128 target_ref_len = math.ceil(origin_ref_max_len / 128) * 128 target_kv_len = target_q_len + target_ref_len @@ -483,7 +475,6 @@ def __init__( refer_offset_w=-1, refer_stride=1, sparse_type=0, - use_context_parallel=False, ): super().__init__() self.patch_size = patch_size @@ -505,8 +496,7 @@ def __init__( self.refer_offset_h = refer_offset_h self.refer_offset_w = refer_offset_w self.refer_stride = refer_stride - self.sparse_type=sparse_type - self.use_context_parallel = use_context_parallel + self.sparse_type = sparse_type # [Denoising Transformer] # embeddings @@ -531,8 +521,7 @@ def __init__( # blocks self.blocks = nn.ModuleList([Incontext_AttentionBlock( - dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps, refer_stride, use_img_emb=use_img_emb, use_context_parallel=self.use_context_parallel, - sparse_type=self.sparse_type + dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps, refer_stride, use_img_emb=use_img_emb, sparse_type=self.sparse_type ) for _ in range(num_layers)]) # head @@ -668,7 +657,6 @@ def forward_ref( for idx, block in enumerate(self.blocks): x_ref = block(x_ref, idx, k_cache, v_cache, method='forward_ref', **kwargs) - def forward_gen( self, x, @@ -809,7 +797,7 @@ def forward_origin( x = torch.cat([torch.cat([ u, u.new_zeros(1, seq_len - u.size(1), u.size(2)) ], dim=1) for u in x]) - + # [reference] # params x_ref = [torch.cat([u, v], dim=0) for u, v in zip(x_ref, y_ref)] @@ -849,11 +837,11 @@ def forward_origin( # time embeddings e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t).to(x.dtype)) e0 = self.time_projection(e).unflatten(1, (6, self.dim)) - + # time embeddings ref e_ref = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, t*0+1).to(x.dtype)) e0_ref = self.time_projection(e_ref).unflatten(1, (6, self.dim)) - + # [context] context_lens = None context = self.text_embedding(torch.stack([torch.cat([ @@ -868,7 +856,7 @@ def forward_origin( context_ref = self.text_embedding(torch.stack([torch.cat([ u, u.new_zeros(self.text_len - u.size(0), u.size(1)) ]) for u in context_ref])) - + if self.use_img_emb: context_clip_ref = self.img_emb(clip_fea_ref) # bs x 257 x dim context_ref = torch.concat([context_clip_ref, context_ref], dim=1) @@ -910,12 +898,6 @@ def forward_origin( # head x = self.head(x, e) - # Context Parallel - if self.use_context_parallel: - x = ops.gather_forward_split_backward( - x, dim=1, group=sp_group, grad_scale="up" - ) - # unpatchify x = self.unpatchify(x, grid_sizes) return [u.float() for u in x] diff --git a/diffsynth/pipelines/wan_video.py b/diffsynth/pipelines/wan_video.py index 7f600fde..e2ab76f9 100644 --- a/diffsynth/pipelines/wan_video.py +++ b/diffsynth/pipelines/wan_video.py @@ -1228,7 +1228,6 @@ def animate2_get_i2v_mask(lat_t, lat_h, lat_w, mask_len=1, device="cuda"): msk = msk.transpose(1, 2)[0] return msk - def process(self, pipe: WanVideoPipeline, animate2_reference_image, animate2_reference_video, animate2_refert_images, num_frames, height, width, tiled, tile_size, tile_stride): if animate2_reference_image is None or animate2_reference_video is None: return {} diff --git a/diffsynth/utils/xfuser/xdit_context_parallel.py b/diffsynth/utils/xfuser/xdit_context_parallel.py index db630e5f..23b8508b 100644 --- a/diffsynth/utils/xfuser/xdit_context_parallel.py +++ b/diffsynth/utils/xfuser/xdit_context_parallel.py @@ -217,5 +217,3 @@ def all_to_all_4d(x, scatter_dim, gather_dim): def is_evenly_divisible(seq_len): world_size = get_sequence_parallel_world_size() return seq_len % world_size == 0 - - diff --git a/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py b/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py index 02536b72..bb1690b5 100644 --- a/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py +++ b/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py @@ -1,8 +1,8 @@ import torch +import torch.distributed as dist from PIL import Image from diffsynth.utils.data import save_video, VideoData from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig -import torch.distributed as dist from modelscope import dataset_snapshot_download vram_config = { @@ -37,7 +37,7 @@ ) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() -# ===== Example 1: single-clip generation (direct pipeline call) ===== +# Example 1: single-clip generation num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -54,7 +54,7 @@ save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) -# ===== Example 2: multi-clip long-video generation ===== +# Example 2: multi-clip long-video generation def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): assert clip_len > first_num, "clip_len must be greater than first_num" diff --git a/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py index 0e30b7b8..2aeb656a 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py @@ -36,7 +36,7 @@ reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() -# ===== Example 1: single-clip generation (direct pipeline call) ===== +# Example 1: single-clip generation num_frames = 81 # For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. video = pipe( @@ -53,7 +53,7 @@ save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) -# ===== Example 2: multi-clip long-video generation ===== +# Example 2: multi-clip long-video generation def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): assert clip_len > first_num, "clip_len must be greater than first_num" diff --git a/examples/wanvideo/model_inference/Wan-Animate-2-14B.py b/examples/wanvideo/model_inference/Wan-Animate-2-14B.py index fab06304..927c2326 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_inference/Wan-Animate-2-14B.py @@ -35,7 +35,7 @@ ) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() -# ===== Example 1: single-clip generation (direct pipeline call) ===== +# Example 1: single-clip generation num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -51,7 +51,7 @@ save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) -# ===== Example 2: multi-clip long-video generation ===== +# Example 2: multi-clip long-video generation def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): assert clip_len > first_num, "clip_len must be greater than first_num" diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py index 0a949d4f..f198e0f4 100644 --- a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py @@ -37,7 +37,7 @@ reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() -# ===== Example 1: single-clip generation (direct pipeline call) ===== +# Example 1: single-clip generation num_frames = 81 # For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. video = pipe( @@ -54,7 +54,7 @@ save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) -# ===== Example 2: multi-clip long-video generation ===== +# Example 2: multi-clip long-video generation def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): assert clip_len > first_num, "clip_len must be greater than first_num" diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py index 504eac65..f366e1e3 100644 --- a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py @@ -36,7 +36,7 @@ ) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() -# ===== Example 1: single-clip generation (direct pipeline call) ===== +# Example 1: single-clip generation num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -52,7 +52,7 @@ save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) -# ===== Example 2: multi-clip long-video generation ===== +# Example 2: multi-clip long-video generation def generate_long_video(pipe, reference_image, cond_images, clip_len, first_num=1, **kwargs): assert clip_len > first_num, "clip_len must be greater than first_num" diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py index 178d5ab0..383c3d71 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py @@ -3,7 +3,6 @@ from diffsynth.utils.data import save_video, VideoData from diffsynth.core import load_state_dict from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig -from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py index ec896a7a..38c0d062 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py @@ -3,7 +3,6 @@ from diffsynth.utils.data import save_video, VideoData from diffsynth.core import load_state_dict from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig -from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py index 6d90d39d..c4f9a156 100644 --- a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py @@ -2,7 +2,6 @@ from PIL import Image from diffsynth.utils.data import save_video, VideoData from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig -from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py index c0837b98..2eae767b 100644 --- a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py @@ -2,7 +2,6 @@ from PIL import Image from diffsynth.utils.data import save_video, VideoData from diffsynth.pipelines.wan_video import WanVideoPipeline, ModelConfig -from modelscope import dataset_snapshot_download vram_config = { "offload_dtype": torch.bfloat16, From 711f9d97765130df94ccd82d868b4cb85f193d53 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Mon, 20 Jul 2026 20:46:17 +0800 Subject: [PATCH 15/17] Wan-Animate-2-14B docs --- README.md | 74 +++++++++++++++++++----------------- README_zh.md | 74 +++++++++++++++++++----------------- docs/en/Model_Details/Wan.md | 9 +++++ docs/zh/Model_Details/Wan.md | 9 +++++ 4 files changed, 96 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 78e04472..18673a3e 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ We believe that a well-developed open-source code framework can lower the thresh > Currently, the development personnel of this project are limited, with most of the work handled by [Artiprocher](https://github.com/Artiprocher) and [mi804](https://github.com/mi804). Therefore, the progress of new feature development will be relatively slow, and the speed of responding to and resolving issues is limited. We apologize for this and ask developers to understand. +- **July 22, 2026** We add support for Wan-Animate-2 in the Wan series. Given a reference image and a driving video, it makes the reference character perform the motions in the driving video, generating high-quality character animation, with both standard and distilled variants. For details, please refer to the [documentation](/docs/en/Model_Details/Wan.md) and [example code](/examples/wanvideo/). + - **June 29, 2026** Boogu-Image open-sourced. Support includes text-to-image generation, image editing, low VRAM inference, and training capabilities. For details, please refer to the [documentation](/docs/en/Model_Details/Boogu-Image.md) and [example code](/examples/boogu_image/). - **June 24, 2026** Krea-2 is now open-source, and we have provided full support. For more details, please refer to the [documentation](/docs/en/Model_Details/Krea-2.md) and [example code](/examples/krea2/). @@ -1405,41 +1407,43 @@ Example code for Wan is available at: [/examples/wanvideo/](/examples/wanvideo/) | Model ID | Extra Inputs | Inference | Low VRAM Inference | Full Training | Validation After Full Training | LoRA Training | Validation After LoRA Training | |-|-|-|-|-|-|-|-| -|[Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-T2V-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-T2V-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-T2V-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-1.3B.py)| -|[Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-T2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-T2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-T2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-14B.py)| -|[Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-I2V-14B-480P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-480P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-480P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-480P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-480P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-480P.py)| -|[Wan-AI/Wan2.1-I2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-I2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-720P.py)| -|[Wan-AI/Wan2.1-FLF2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-FLF2V-14B-720P)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-FLF2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-FLF2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-FLF2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-FLF2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-FLF2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-FLF2V-14B-720P.py)| -|[iic/VACE-Wan2.1-1.3B-Preview](https://modelscope.cn/models/iic/VACE-Wan2.1-1.3B-Preview)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B-Preview.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B-Preview.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B-Preview.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B-Preview.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B-Preview.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B-Preview.py)| -|[Wan-AI/Wan2.1-VACE-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-1.3B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B.py)| -|[Wan-AI/Wan2.1-VACE-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-VACE-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-VACE-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-VACE-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-14B.py)| -|[PAI/Wan2.1-Fun-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-InP.py)| -|[PAI/Wan2.1-Fun-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control)|`control_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-Control.py)| -|[PAI/Wan2.1-Fun-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-InP.py)| -|[PAI/Wan2.1-Fun-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control)|`control_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-Control.py)| -|[PAI/Wan2.1-Fun-V1.1-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)|`control_video`, `reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control.py)| -|[PAI/Wan2.1-Fun-V1.1-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)|`control_video`, `reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control.py)| -|[PAI/Wan2.1-Fun-V1.1-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-InP.py)| -|[PAI/Wan2.1-Fun-V1.1-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-InP.py)| -|[PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)|`control_camera_video`, `input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)| -|[PAI/Wan2.1-Fun-V1.1-14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)|`control_camera_video`, `input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control-Camera.py)| -|[DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1](https://modelscope.cn/models/DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1)|`motion_bucket_id`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-1.3b-speedcontrol-v1.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-1.3b-speedcontrol-v1.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-1.3b-speedcontrol-v1.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-1.3b-speedcontrol-v1.py)| -|[krea/krea-realtime-video](https://www.modelscope.cn/models/krea/krea-realtime-video)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/krea-realtime-video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/krea-realtime-video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/krea-realtime-video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/krea-realtime-video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/krea-realtime-video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/krea-realtime-video.py)| -|[meituan-longcat/LongCat-Video](https://www.modelscope.cn/models/meituan-longcat/LongCat-Video)|`longcat_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/LongCat-Video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/LongCat-Video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/LongCat-Video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/LongCat-Video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/LongCat-Video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/LongCat-Video.py)| -|[ByteDance/Video-As-Prompt-Wan2.1-14B](https://modelscope.cn/models/ByteDance/Video-As-Prompt-Wan2.1-14B)|`vap_video`, `vap_prompt`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Video-As-Prompt-Wan2.1-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Video-As-Prompt-Wan2.1-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Video-As-Prompt-Wan2.1-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Video-As-Prompt-Wan2.1-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Video-As-Prompt-Wan2.1-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Video-As-Prompt-Wan2.1-14B.py)| -|[Wan-AI/Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-T2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-T2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-T2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-T2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-T2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-T2V-A14B.py)| -|[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| -|[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| -|[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| -|[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| -|[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| -|[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| -|[PAI/Wan2.2-Fun-A14B-Control](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)|`control_video`, `reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control.py)| -|[PAI/Wan2.2-Fun-A14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)|`control_camera_video`, `input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control-Camera.py)| -|[openmoss/MOVA-360p](https://modelscope.cn/models/openmoss/MOVA-360p)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference/MOVA-360p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference_low_vram/MOVA-360p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/full/MOVA-360P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_full/MOVA-360p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/lora/MOVA-360P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_lora/MOVA-360p-I2AV.py)| -|[openmoss/MOVA-720p](https://modelscope.cn/models/openmoss/MOVA-720p)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference/MOVA-720p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference_low_vram/MOVA-720p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/full/MOVA-720P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_full/MOVA-720p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/lora/MOVA-720P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_lora/MOVA-720p-I2AV.py)| -|[Wan-AI/Wan-Dancer-14B (global model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Dancer-14B-global.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-global.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Dancer-14B-global.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-global.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Dancer-14B-global.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-global.py)| -|[Wan-AI/Wan-Dancer-14B (local model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Dancer-14B-local.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-local.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Dancer-14B-local.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-local.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Dancer-14B-local.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-local.py)| +|[Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B)||[code](/examples/wanvideo/model_inference/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-T2V-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-1.3B.py)| +|[Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B)||[code](/examples/wanvideo/model_inference/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-T2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-14B.py)| +|[Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-480P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-480P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-480P.py)| +|[Wan-AI/Wan2.1-I2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-720P.py)| +|[Wan-AI/Wan2.1-FLF2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-FLF2V-14B-720P)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-FLF2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-FLF2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-FLF2V-14B-720P.py)| +|[iic/VACE-Wan2.1-1.3B-Preview](https://modelscope.cn/models/iic/VACE-Wan2.1-1.3B-Preview)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B-Preview.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B-Preview.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B-Preview.py)| +|[Wan-AI/Wan2.1-VACE-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-1.3B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B.py)| +|[Wan-AI/Wan2.1-VACE-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-14B.py)| +|[PAI/Wan2.1-Fun-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-InP.py)| +|[PAI/Wan2.1-Fun-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control)|`control_video`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-Control.py)| +|[PAI/Wan2.1-Fun-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-InP.py)| +|[PAI/Wan2.1-Fun-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control)|`control_video`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-Control.py)| +|[PAI/Wan2.1-Fun-V1.1-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control.py)| +|[PAI/Wan2.1-Fun-V1.1-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control.py)| +|[PAI/Wan2.1-Fun-V1.1-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-InP.py)| +|[PAI/Wan2.1-Fun-V1.1-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-InP.py)| +|[PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)| +|[PAI/Wan2.1-Fun-V1.1-14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control-Camera.py)| +|[DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1](https://modelscope.cn/models/DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1)|`motion_bucket_id`|[code](/examples/wanvideo/model_inference/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-1.3b-speedcontrol-v1.py)| +|[krea/krea-realtime-video](https://www.modelscope.cn/models/krea/krea-realtime-video)||[code](/examples/wanvideo/model_inference/krea-realtime-video.py)|[code](/examples/wanvideo/model_inference_low_vram/krea-realtime-video.py)|[code](/examples/wanvideo/model_training/full/krea-realtime-video.sh)|[code](/examples/wanvideo/model_training/validate_full/krea-realtime-video.py)|[code](/examples/wanvideo/model_training/lora/krea-realtime-video.sh)|[code](/examples/wanvideo/model_training/validate_lora/krea-realtime-video.py)| +|[meituan-longcat/LongCat-Video](https://www.modelscope.cn/models/meituan-longcat/LongCat-Video)|`longcat_video`|[code](/examples/wanvideo/model_inference/LongCat-Video.py)|[code](/examples/wanvideo/model_inference_low_vram/LongCat-Video.py)|[code](/examples/wanvideo/model_training/full/LongCat-Video.sh)|[code](/examples/wanvideo/model_training/validate_full/LongCat-Video.py)|[code](/examples/wanvideo/model_training/lora/LongCat-Video.sh)|[code](/examples/wanvideo/model_training/validate_lora/LongCat-Video.py)| +|[ByteDance/Video-As-Prompt-Wan2.1-14B](https://modelscope.cn/models/ByteDance/Video-As-Prompt-Wan2.1-14B)|`vap_video`, `vap_prompt`|[code](/examples/wanvideo/model_inference/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_training/full/Video-As-Prompt-Wan2.1-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_training/lora/Video-As-Prompt-Wan2.1-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Video-As-Prompt-Wan2.1-14B.py)| +|[Wan-AI/Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B)||[code](/examples/wanvideo/model_inference/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-T2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-T2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-T2V-A14B.py)| +|[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| +|[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| +|[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| +|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| +|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| +|[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| +|[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| +|[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| +|[PAI/Wan2.2-Fun-A14B-Control](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control.py)| +|[PAI/Wan2.2-Fun-A14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control-Camera.py)| +|[openmoss/MOVA-360p](https://modelscope.cn/models/openmoss/MOVA-360p)|`input_image`|[code](/examples/mova/model_inference/MOVA-360p-I2AV.py)|[code](/examples/mova/model_inference_low_vram/MOVA-360p-I2AV.py)|[code](/examples/mova/model_training/full/MOVA-360P-I2AV.sh)|[code](/examples/mova/model_training/validate_full/MOVA-360p-I2AV.py)|[code](/examples/mova/model_training/lora/MOVA-360P-I2AV.sh)|[code](/examples/mova/model_training/validate_lora/MOVA-360p-I2AV.py)| +|[openmoss/MOVA-720p](https://modelscope.cn/models/openmoss/MOVA-720p)|`input_image`|[code](/examples/mova/model_inference/MOVA-720p-I2AV.py)|[code](/examples/mova/model_inference_low_vram/MOVA-720p-I2AV.py)|[code](/examples/mova/model_training/full/MOVA-720P-I2AV.sh)|[code](/examples/mova/model_training/validate_full/MOVA-720p-I2AV.py)|[code](/examples/mova/model_training/lora/MOVA-720P-I2AV.sh)|[code](/examples/mova/model_training/validate_lora/MOVA-720p-I2AV.py)| +|[Wan-AI/Wan-Dancer-14B (global model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](/examples/wanvideo/model_inference/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_training/full/Wan-Dancer-14B-global.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_training/lora/Wan-Dancer-14B-global.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-global.py)| +|[Wan-AI/Wan-Dancer-14B (local model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](/examples/wanvideo/model_inference/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_training/full/Wan-Dancer-14B-local.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_training/lora/Wan-Dancer-14B-local.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-local.py)| diff --git a/README_zh.md b/README_zh.md index dfa7f77b..7c029117 100644 --- a/README_zh.md +++ b/README_zh.md @@ -34,6 +34,8 @@ DiffSynth 目前包括两个开源项目： > 目前本项目的开发人员有限，大部分工作由 [Artiprocher](https://github.com/Artiprocher) 和 [mi804](https://github.com/mi804) 负责，因此新功能的开发进展会比较缓慢，issue 的回复和解决速度有限，我们对此感到非常抱歉，请各位开发者理解。 +- **2026年7月22日** 我们为 Wan 系列新增了 Wan-Animate-2，输入一张参考图和一段驱动视频，即可让参考角色演绎驱动视频中的动作，生成高质量角色动画，包含标准与蒸馏两个变体。详情请参考[文档](/docs/zh/Model_Details/Wan.md)和[示例代码](/examples/wanvideo/)。 + - **2026年6月29日** Boogu-Image 开源，已支持文生图推理、图像编辑、低显存推理和训练能力。详情请参考[文档](/docs/zh/Model_Details/Boogu-Image.md)和[示例代码](/examples/boogu_image/)。 - **2026年6月24日** Krea-2 开源，我们已提供全面支持。详情请参考[文档](/docs/zh/Model_Details/Krea-2.md)和[示例代码](/examples/krea2/)。 @@ -1405,41 +1407,43 @@ Wan 的示例代码位于：[/examples/wanvideo/](/examples/wanvideo/) |模型 ID|额外参数|推理|低显存推理|全量训练|全量训练后验证|LoRA 训练|LoRA 训练后验证| |-|-|-|-|-|-|-|-| -|[Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-T2V-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-T2V-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-T2V-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-1.3B.py)| -|[Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-T2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-T2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-T2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-14B.py)| -|[Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-I2V-14B-480P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-480P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-480P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-480P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-480P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-480P.py)| -|[Wan-AI/Wan2.1-I2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-I2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-720P.py)| -|[Wan-AI/Wan2.1-FLF2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-FLF2V-14B-720P)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-FLF2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-FLF2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-FLF2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-FLF2V-14B-720P.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-FLF2V-14B-720P.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-FLF2V-14B-720P.py)| -|[iic/VACE-Wan2.1-1.3B-Preview](https://modelscope.cn/models/iic/VACE-Wan2.1-1.3B-Preview)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B-Preview.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B-Preview.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B-Preview.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B-Preview.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B-Preview.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B-Preview.py)| -|[Wan-AI/Wan2.1-VACE-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-1.3B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B.py)| -|[Wan-AI/Wan2.1-VACE-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-VACE-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-VACE-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-VACE-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-14B.py)| -|[PAI/Wan2.1-Fun-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-InP.py)| -|[PAI/Wan2.1-Fun-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control)|`control_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-Control.py)| -|[PAI/Wan2.1-Fun-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-InP.py)| -|[PAI/Wan2.1-Fun-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control)|`control_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-Control.py)| -|[PAI/Wan2.1-Fun-V1.1-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)|`control_video`, `reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control.py)| -|[PAI/Wan2.1-Fun-V1.1-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)|`control_video`, `reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control.py)| -|[PAI/Wan2.1-Fun-V1.1-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-InP.py)| -|[PAI/Wan2.1-Fun-V1.1-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-InP.py)| -|[PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)|`control_camera_video`, `input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)| -|[PAI/Wan2.1-Fun-V1.1-14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)|`control_camera_video`, `input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control-Camera.py)| -|[DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1](https://modelscope.cn/models/DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1)|`motion_bucket_id`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.1-1.3b-speedcontrol-v1.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-1.3b-speedcontrol-v1.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.1-1.3b-speedcontrol-v1.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.1-1.3b-speedcontrol-v1.py)| -|[krea/krea-realtime-video](https://www.modelscope.cn/models/krea/krea-realtime-video)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/krea-realtime-video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/krea-realtime-video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/krea-realtime-video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/krea-realtime-video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/krea-realtime-video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/krea-realtime-video.py)| -|[meituan-longcat/LongCat-Video](https://www.modelscope.cn/models/meituan-longcat/LongCat-Video)|`longcat_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/LongCat-Video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/LongCat-Video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/LongCat-Video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/LongCat-Video.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/LongCat-Video.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/LongCat-Video.py)| -|[ByteDance/Video-As-Prompt-Wan2.1-14B](https://modelscope.cn/models/ByteDance/Video-As-Prompt-Wan2.1-14B)|`vap_video`, `vap_prompt`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Video-As-Prompt-Wan2.1-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Video-As-Prompt-Wan2.1-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Video-As-Prompt-Wan2.1-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Video-As-Prompt-Wan2.1-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Video-As-Prompt-Wan2.1-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Video-As-Prompt-Wan2.1-14B.py)| -|[Wan-AI/Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B)||[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-T2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-T2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-T2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-T2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-T2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-T2V-A14B.py)| -|[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| -|[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| -|[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| -|[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| -|[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| -|[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| -|[PAI/Wan2.2-Fun-A14B-Control](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)|`control_video`, `reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control.py)| -|[PAI/Wan2.2-Fun-A14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)|`control_camera_video`, `input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control-Camera.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control-Camera.py)| -|[openmoss/MOVA-360p](https://modelscope.cn/models/openmoss/MOVA-360p)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference/MOVA-360p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference_low_vram/MOVA-360p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/full/MOVA-360P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_full/MOVA-360p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/lora/MOVA-360P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_lora/MOVA-360p-I2AV.py)| -|[openmoss/MOVA-720p](https://modelscope.cn/models/openmoss/MOVA-720p)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference/MOVA-720p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_inference_low_vram/MOVA-720p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/full/MOVA-720P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_full/MOVA-720p-I2AV.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/lora/MOVA-720P-I2AV.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/mova/model_training/validate_lora/MOVA-720p-I2AV.py)| -|[Wan-AI/Wan-Dancer-14B (global model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Dancer-14B-global.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-global.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Dancer-14B-global.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-global.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Dancer-14B-global.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-global.py)| -|[Wan-AI/Wan-Dancer-14B (local model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Dancer-14B-local.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-local.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Dancer-14B-local.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-local.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Dancer-14B-local.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-local.py)| +|[Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B)||[code](/examples/wanvideo/model_inference/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-T2V-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-1.3B.py)| +|[Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B)||[code](/examples/wanvideo/model_inference/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-T2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-14B.py)| +|[Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-480P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-480P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-480P.py)| +|[Wan-AI/Wan2.1-I2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-720P.py)| +|[Wan-AI/Wan2.1-FLF2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-FLF2V-14B-720P)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-FLF2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-FLF2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-FLF2V-14B-720P.py)| +|[iic/VACE-Wan2.1-1.3B-Preview](https://modelscope.cn/models/iic/VACE-Wan2.1-1.3B-Preview)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B-Preview.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B-Preview.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B-Preview.py)| +|[Wan-AI/Wan2.1-VACE-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-1.3B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B.py)| +|[Wan-AI/Wan2.1-VACE-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-14B.py)| +|[PAI/Wan2.1-Fun-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-InP.py)| +|[PAI/Wan2.1-Fun-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control)|`control_video`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-Control.py)| +|[PAI/Wan2.1-Fun-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-InP.py)| +|[PAI/Wan2.1-Fun-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control)|`control_video`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-Control.py)| +|[PAI/Wan2.1-Fun-V1.1-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control.py)| +|[PAI/Wan2.1-Fun-V1.1-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control.py)| +|[PAI/Wan2.1-Fun-V1.1-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-InP.py)| +|[PAI/Wan2.1-Fun-V1.1-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-InP.py)| +|[PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)| +|[PAI/Wan2.1-Fun-V1.1-14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control-Camera.py)| +|[DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1](https://modelscope.cn/models/DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1)|`motion_bucket_id`|[code](/examples/wanvideo/model_inference/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-1.3b-speedcontrol-v1.py)| +|[krea/krea-realtime-video](https://www.modelscope.cn/models/krea/krea-realtime-video)||[code](/examples/wanvideo/model_inference/krea-realtime-video.py)|[code](/examples/wanvideo/model_inference_low_vram/krea-realtime-video.py)|[code](/examples/wanvideo/model_training/full/krea-realtime-video.sh)|[code](/examples/wanvideo/model_training/validate_full/krea-realtime-video.py)|[code](/examples/wanvideo/model_training/lora/krea-realtime-video.sh)|[code](/examples/wanvideo/model_training/validate_lora/krea-realtime-video.py)| +|[meituan-longcat/LongCat-Video](https://www.modelscope.cn/models/meituan-longcat/LongCat-Video)|`longcat_video`|[code](/examples/wanvideo/model_inference/LongCat-Video.py)|[code](/examples/wanvideo/model_inference_low_vram/LongCat-Video.py)|[code](/examples/wanvideo/model_training/full/LongCat-Video.sh)|[code](/examples/wanvideo/model_training/validate_full/LongCat-Video.py)|[code](/examples/wanvideo/model_training/lora/LongCat-Video.sh)|[code](/examples/wanvideo/model_training/validate_lora/LongCat-Video.py)| +|[ByteDance/Video-As-Prompt-Wan2.1-14B](https://modelscope.cn/models/ByteDance/Video-As-Prompt-Wan2.1-14B)|`vap_video`, `vap_prompt`|[code](/examples/wanvideo/model_inference/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_training/full/Video-As-Prompt-Wan2.1-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_training/lora/Video-As-Prompt-Wan2.1-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Video-As-Prompt-Wan2.1-14B.py)| +|[Wan-AI/Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B)||[code](/examples/wanvideo/model_inference/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-T2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-T2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-T2V-A14B.py)| +|[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| +|[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| +|[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| +|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| +|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| +|[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| +|[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| +|[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| +|[PAI/Wan2.2-Fun-A14B-Control](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control.py)| +|[PAI/Wan2.2-Fun-A14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control-Camera.py)| +|[openmoss/MOVA-360p](https://modelscope.cn/models/openmoss/MOVA-360p)|`input_image`|[code](/examples/mova/model_inference/MOVA-360p-I2AV.py)|[code](/examples/mova/model_inference_low_vram/MOVA-360p-I2AV.py)|[code](/examples/mova/model_training/full/MOVA-360P-I2AV.sh)|[code](/examples/mova/model_training/validate_full/MOVA-360p-I2AV.py)|[code](/examples/mova/model_training/lora/MOVA-360P-I2AV.sh)|[code](/examples/mova/model_training/validate_lora/MOVA-360p-I2AV.py)| +|[openmoss/MOVA-720p](https://modelscope.cn/models/openmoss/MOVA-720p)|`input_image`|[code](/examples/mova/model_inference/MOVA-720p-I2AV.py)|[code](/examples/mova/model_inference_low_vram/MOVA-720p-I2AV.py)|[code](/examples/mova/model_training/full/MOVA-720P-I2AV.sh)|[code](/examples/mova/model_training/validate_full/MOVA-720p-I2AV.py)|[code](/examples/mova/model_training/lora/MOVA-720P-I2AV.sh)|[code](/examples/mova/model_training/validate_lora/MOVA-720p-I2AV.py)| +|[Wan-AI/Wan-Dancer-14B (global model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](/examples/wanvideo/model_inference/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_training/full/Wan-Dancer-14B-global.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_training/lora/Wan-Dancer-14B-global.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-global.py)| +|[Wan-AI/Wan-Dancer-14B (local model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](/examples/wanvideo/model_inference/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_training/full/Wan-Dancer-14B-local.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_training/lora/Wan-Dancer-14B-local.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-local.py)| diff --git a/docs/en/Model_Details/Wan.md b/docs/en/Model_Details/Wan.md index ce5ca3a3..a6e0f1dd 100644 --- a/docs/en/Model_Details/Wan.md +++ b/docs/en/Model_Details/Wan.md @@ -91,6 +91,7 @@ graph LR; Wan-AI/Wan2.1-T2V-14B-->krea/krea-realtime-video; Wan-AI/Wan2.1-I2V-14B-720P-->ByteDance/Video-As-Prompt-Wan2.1-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-Animate-14B; + Wan2.2-Series-->Wan-AI/Wan-Animate-2-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-S2V-14B; Wan2.2-Series-->Wan-AI/Wan2.2-T2V-A14B; Wan2.2-Series-->Wan-AI/Wan2.2-I2V-A14B; @@ -132,6 +133,8 @@ graph LR; |[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| |[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| |[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| +|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| +|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| |[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| |[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| |[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| @@ -184,6 +187,12 @@ Input parameters for `WanVideoPipeline` inference include: * `audio_sample_rate`: Audio sampling rate, default value is 16000. * `s2v_pose_video`: S2V model pose video. * `motion_video`: S2V model motion video. +* `animate2_reference_image`: Wan-Animate-2 reference image, providing the character identity. +* `animate2_reference_video`: Wan-Animate-2 driving video, providing the motion. +* `animate2_prompt_ref`: Wan-Animate-2 reference prompt for the driving video, describing its content. +* `animate2_refert_images`: Wan-Animate-2 reference frame images for temporal continuation, used in long video chunked generation. +* `animate2_offload_kv`: Whether Wan-Animate-2 offloads the reference video KV cache to memory, default value is `False`. +* `animate2_log_scale`: Wan-Animate-2 guidance log scale, default value is 0.0, recommended to set to -1.3 for the distilled model. * `height`: Video height, must be a multiple of 16. * `width`: Video width, must be a multiple of 16. * `num_frames`: Number of video frames, default value is 81, must be a multiple of 4 + 1. diff --git a/docs/zh/Model_Details/Wan.md b/docs/zh/Model_Details/Wan.md index afda8616..e96a5255 100644 --- a/docs/zh/Model_Details/Wan.md +++ b/docs/zh/Model_Details/Wan.md @@ -92,6 +92,7 @@ graph LR; Wan-AI/Wan2.1-T2V-14B-->meituan-longcat/LongCat-Video; Wan-AI/Wan2.1-I2V-14B-720P-->ByteDance/Video-As-Prompt-Wan2.1-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-Animate-14B; + Wan2.2-Series-->Wan-AI/Wan-Animate-2-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-S2V-14B; Wan2.2-Series-->Wan-AI/Wan2.2-T2V-A14B; Wan2.2-Series-->Wan-AI/Wan2.2-I2V-A14B; @@ -133,6 +134,8 @@ graph LR; |[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| |[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| |[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| +|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| +|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| |[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| |[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| |[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| @@ -185,6 +188,12 @@ DeepSpeed ZeRO 3 训练：Wan 系列模型支持 DeepSpeed ZeRO 3 训练，将 * `audio_sample_rate`: 音频采样率，默认值为 16000。 * `s2v_pose_video`: S2V 模型的姿态视频。 * `motion_video`: S2V 模型的运动视频。 +* `animate2_reference_image`: Wan-Animate-2 模型的参考图像，提供角色身份。 +* `animate2_reference_video`: Wan-Animate-2 模型的驱动视频，提供动作。 +* `animate2_prompt_ref`: Wan-Animate-2 模型驱动视频的参考提示词，描述驱动视频中的内容。 +* `animate2_refert_images`: Wan-Animate-2 模型用于时序续接的参考帧图像，用于长视频分块生成。 +* `animate2_offload_kv`: Wan-Animate-2 模型是否将参考视频的 KV 缓存卸载到内存，默认值为 `False`。 +* `animate2_log_scale`: Wan-Animate-2 模型的引导对数缩放系数，默认值为 0.0，蒸馏模型建议设为 -1.3。 * `height`: 视频高度，需保证高度为 16 的倍数。 * `width`: 视频宽度，需保证宽度为 16 的倍数。 * `num_frames`: 视频帧数，默认值为 81，需保证为 4 的倍数 + 1。 From 013174c8df509a7a398e7d71957641e544523076 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Tue, 21 Jul 2026 17:59:40 +0800 Subject: [PATCH 16/17] update training --- .../model_training/full/Wan-Animate-2-14B-Distilled.sh | 7 +++---- .../wanvideo/model_training/full/Wan-Animate-2-14B.sh | 7 +++---- .../model_training/lora/Wan-Animate-2-14B-Distilled.sh | 4 ++-- .../wanvideo/model_training/lora/Wan-Animate-2-14B.sh | 4 ++-- .../validate_full/Wan-Animate-2-14B-Distilled.py | 8 ++++---- .../model_training/validate_full/Wan-Animate-2-14B.py | 3 ++- .../validate_lora/Wan-Animate-2-14B-Distilled.py | 5 ++--- 7 files changed, 18 insertions(+), 20 deletions(-) diff --git a/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh b/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh index 12c0e7d8..a77ba9f9 100644 --- a/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh +++ b/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh @@ -6,14 +6,13 @@ accelerate launch examples/wanvideo/model_training/train.py \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ - --num_frames 41 \ + --num_frames 81 \ --dataset_repeat 1 \ --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B-Distilled_full_splited_cache" \ - --trainable_models "dit" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ --task "sft:data_process" @@ -23,14 +22,14 @@ accelerate launch --config_file examples/wanvideo/model_training/full/accelerate --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ - --num_frames 41 \ + --num_frames 81 \ --dataset_repeat 100 \ --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16_distillation.safetensors" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B-Distilled_full" \ - --trainable_models "dit" \ + --trainable_models "dit.blocks.0.block.self_attn.q,dit.blocks.0.block.self_attn.k,dit.blocks.0.block.self_attn.v,dit.blocks.1.block.self_attn.q,dit.blocks.1.block.self_attn.k,dit.blocks.1.block.self_attn.v,dit.blocks.2.block.self_attn.q,dit.blocks.2.block.self_attn.k,dit.blocks.2.block.self_attn.v,dit.blocks.3.block.self_attn.q,dit.blocks.3.block.self_attn.k,dit.blocks.3.block.self_attn.v,dit.blocks.4.block.self_attn.q,dit.blocks.4.block.self_attn.k,dit.blocks.4.block.self_attn.v,dit.blocks.5.block.self_attn.q,dit.blocks.5.block.self_attn.k,dit.blocks.5.block.self_attn.v,dit.blocks.6.block.self_attn.q,dit.blocks.6.block.self_attn.k,dit.blocks.6.block.self_attn.v,dit.blocks.7.block.self_attn.q,dit.blocks.7.block.self_attn.k,dit.blocks.7.block.self_attn.v,dit.blocks.8.block.self_attn.q,dit.blocks.8.block.self_attn.k,dit.blocks.8.block.self_attn.v,dit.blocks.9.block.self_attn.q,dit.blocks.9.block.self_attn.k,dit.blocks.9.block.self_attn.v,dit.blocks.10.block.self_attn.q,dit.blocks.10.block.self_attn.k,dit.blocks.10.block.self_attn.v,dit.blocks.11.block.self_attn.q,dit.blocks.11.block.self_attn.k,dit.blocks.11.block.self_attn.v,dit.blocks.12.block.self_attn.q,dit.blocks.12.block.self_attn.k,dit.blocks.12.block.self_attn.v,dit.blocks.13.block.self_attn.q,dit.blocks.13.block.self_attn.k,dit.blocks.13.block.self_attn.v,dit.blocks.14.block.self_attn.q,dit.blocks.14.block.self_attn.k,dit.blocks.14.block.self_attn.v,dit.blocks.15.block.self_attn.q,dit.blocks.15.block.self_attn.k,dit.blocks.15.block.self_attn.v,dit.blocks.16.block.self_attn.q,dit.blocks.16.block.self_attn.k,dit.blocks.16.block.self_attn.v,dit.blocks.17.block.self_attn.q,dit.blocks.17.block.self_attn.k,dit.blocks.17.block.self_attn.v,dit.blocks.18.block.self_attn.q,dit.blocks.18.block.self_attn.k,dit.blocks.18.block.self_attn.v,dit.blocks.19.block.self_attn.q,dit.blocks.19.block.self_attn.k,dit.blocks.19.block.self_attn.v,dit.blocks.20.block.self_attn.q,dit.blocks.20.block.self_attn.k,dit.blocks.20.block.self_attn.v,dit.blocks.21.block.self_attn.q,dit.blocks.21.block.self_attn.k,dit.blocks.21.block.self_attn.v,dit.blocks.22.block.self_attn.q,dit.blocks.22.block.self_attn.k,dit.blocks.22.block.self_attn.v,dit.blocks.23.block.self_attn.q,dit.blocks.23.block.self_attn.k,dit.blocks.23.block.self_attn.v,dit.blocks.24.block.self_attn.q,dit.blocks.24.block.self_attn.k,dit.blocks.24.block.self_attn.v,dit.blocks.25.block.self_attn.q,dit.blocks.25.block.self_attn.k,dit.blocks.25.block.self_attn.v,dit.blocks.26.block.self_attn.q,dit.blocks.26.block.self_attn.k,dit.blocks.26.block.self_attn.v,dit.blocks.27.block.self_attn.q,dit.blocks.27.block.self_attn.k,dit.blocks.27.block.self_attn.v,dit.blocks.28.block.self_attn.q,dit.blocks.28.block.self_attn.k,dit.blocks.28.block.self_attn.v,dit.blocks.29.block.self_attn.q,dit.blocks.29.block.self_attn.k,dit.blocks.29.block.self_attn.v,dit.blocks.30.block.self_attn.q,dit.blocks.30.block.self_attn.k,dit.blocks.30.block.self_attn.v,dit.blocks.31.block.self_attn.q,dit.blocks.31.block.self_attn.k,dit.blocks.31.block.self_attn.v,dit.blocks.32.block.self_attn.q,dit.blocks.32.block.self_attn.k,dit.blocks.32.block.self_attn.v,dit.blocks.33.block.self_attn.q,dit.blocks.33.block.self_attn.k,dit.blocks.33.block.self_attn.v,dit.blocks.34.block.self_attn.q,dit.blocks.34.block.self_attn.k,dit.blocks.34.block.self_attn.v,dit.blocks.35.block.self_attn.q,dit.blocks.35.block.self_attn.k,dit.blocks.35.block.self_attn.v,dit.blocks.36.block.self_attn.q,dit.blocks.36.block.self_attn.k,dit.blocks.36.block.self_attn.v,dit.blocks.37.block.self_attn.q,dit.blocks.37.block.self_attn.k,dit.blocks.37.block.self_attn.v,dit.blocks.38.block.self_attn.q,dit.blocks.38.block.self_attn.k,dit.blocks.38.block.self_attn.v,dit.blocks.39.block.self_attn.q,dit.blocks.39.block.self_attn.k,dit.blocks.39.block.self_attn.v" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ --task "sft:train" diff --git a/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh b/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh index 3ad383ba..20236a35 100644 --- a/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh +++ b/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh @@ -6,14 +6,13 @@ accelerate launch examples/wanvideo/model_training/train.py \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ - --num_frames 41 \ + --num_frames 81 \ --dataset_repeat 1 \ --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B_full_splited_cache" \ - --trainable_models "dit" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ --task "sft:data_process" @@ -23,14 +22,14 @@ accelerate launch --config_file examples/wanvideo/model_training/full/accelerate --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ - --num_frames 41 \ + --num_frames 81 \ --dataset_repeat 100 \ --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16.safetensors" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B_full" \ - --trainable_models "dit" \ + --trainable_models "dit.blocks.0.block.self_attn.q,dit.blocks.0.block.self_attn.k,dit.blocks.0.block.self_attn.v,dit.blocks.1.block.self_attn.q,dit.blocks.1.block.self_attn.k,dit.blocks.1.block.self_attn.v,dit.blocks.2.block.self_attn.q,dit.blocks.2.block.self_attn.k,dit.blocks.2.block.self_attn.v,dit.blocks.3.block.self_attn.q,dit.blocks.3.block.self_attn.k,dit.blocks.3.block.self_attn.v,dit.blocks.4.block.self_attn.q,dit.blocks.4.block.self_attn.k,dit.blocks.4.block.self_attn.v,dit.blocks.5.block.self_attn.q,dit.blocks.5.block.self_attn.k,dit.blocks.5.block.self_attn.v,dit.blocks.6.block.self_attn.q,dit.blocks.6.block.self_attn.k,dit.blocks.6.block.self_attn.v,dit.blocks.7.block.self_attn.q,dit.blocks.7.block.self_attn.k,dit.blocks.7.block.self_attn.v,dit.blocks.8.block.self_attn.q,dit.blocks.8.block.self_attn.k,dit.blocks.8.block.self_attn.v,dit.blocks.9.block.self_attn.q,dit.blocks.9.block.self_attn.k,dit.blocks.9.block.self_attn.v,dit.blocks.10.block.self_attn.q,dit.blocks.10.block.self_attn.k,dit.blocks.10.block.self_attn.v,dit.blocks.11.block.self_attn.q,dit.blocks.11.block.self_attn.k,dit.blocks.11.block.self_attn.v,dit.blocks.12.block.self_attn.q,dit.blocks.12.block.self_attn.k,dit.blocks.12.block.self_attn.v,dit.blocks.13.block.self_attn.q,dit.blocks.13.block.self_attn.k,dit.blocks.13.block.self_attn.v,dit.blocks.14.block.self_attn.q,dit.blocks.14.block.self_attn.k,dit.blocks.14.block.self_attn.v,dit.blocks.15.block.self_attn.q,dit.blocks.15.block.self_attn.k,dit.blocks.15.block.self_attn.v,dit.blocks.16.block.self_attn.q,dit.blocks.16.block.self_attn.k,dit.blocks.16.block.self_attn.v,dit.blocks.17.block.self_attn.q,dit.blocks.17.block.self_attn.k,dit.blocks.17.block.self_attn.v,dit.blocks.18.block.self_attn.q,dit.blocks.18.block.self_attn.k,dit.blocks.18.block.self_attn.v,dit.blocks.19.block.self_attn.q,dit.blocks.19.block.self_attn.k,dit.blocks.19.block.self_attn.v,dit.blocks.20.block.self_attn.q,dit.blocks.20.block.self_attn.k,dit.blocks.20.block.self_attn.v,dit.blocks.21.block.self_attn.q,dit.blocks.21.block.self_attn.k,dit.blocks.21.block.self_attn.v,dit.blocks.22.block.self_attn.q,dit.blocks.22.block.self_attn.k,dit.blocks.22.block.self_attn.v,dit.blocks.23.block.self_attn.q,dit.blocks.23.block.self_attn.k,dit.blocks.23.block.self_attn.v,dit.blocks.24.block.self_attn.q,dit.blocks.24.block.self_attn.k,dit.blocks.24.block.self_attn.v,dit.blocks.25.block.self_attn.q,dit.blocks.25.block.self_attn.k,dit.blocks.25.block.self_attn.v,dit.blocks.26.block.self_attn.q,dit.blocks.26.block.self_attn.k,dit.blocks.26.block.self_attn.v,dit.blocks.27.block.self_attn.q,dit.blocks.27.block.self_attn.k,dit.blocks.27.block.self_attn.v,dit.blocks.28.block.self_attn.q,dit.blocks.28.block.self_attn.k,dit.blocks.28.block.self_attn.v,dit.blocks.29.block.self_attn.q,dit.blocks.29.block.self_attn.k,dit.blocks.29.block.self_attn.v,dit.blocks.30.block.self_attn.q,dit.blocks.30.block.self_attn.k,dit.blocks.30.block.self_attn.v,dit.blocks.31.block.self_attn.q,dit.blocks.31.block.self_attn.k,dit.blocks.31.block.self_attn.v,dit.blocks.32.block.self_attn.q,dit.blocks.32.block.self_attn.k,dit.blocks.32.block.self_attn.v,dit.blocks.33.block.self_attn.q,dit.blocks.33.block.self_attn.k,dit.blocks.33.block.self_attn.v,dit.blocks.34.block.self_attn.q,dit.blocks.34.block.self_attn.k,dit.blocks.34.block.self_attn.v,dit.blocks.35.block.self_attn.q,dit.blocks.35.block.self_attn.k,dit.blocks.35.block.self_attn.v,dit.blocks.36.block.self_attn.q,dit.blocks.36.block.self_attn.k,dit.blocks.36.block.self_attn.v,dit.blocks.37.block.self_attn.q,dit.blocks.37.block.self_attn.k,dit.blocks.37.block.self_attn.v,dit.blocks.38.block.self_attn.q,dit.blocks.38.block.self_attn.k,dit.blocks.38.block.self_attn.v,dit.blocks.39.block.self_attn.q,dit.blocks.39.block.self_attn.k,dit.blocks.39.block.self_attn.v" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ --task "sft:train" diff --git a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh index 4b90ccf9..103c41d7 100644 --- a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh +++ b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh @@ -14,7 +14,7 @@ accelerate launch examples/wanvideo/model_training/train.py \ --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B-Distilled_lora_splited_cache" \ --lora_base_model "dit" \ - --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ @@ -33,7 +33,7 @@ accelerate launch --config_file examples/wanvideo/model_training/full/accelerate --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B-Distilled_lora" \ --lora_base_model "dit" \ - --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ diff --git a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh index 378df440..cd9caddd 100644 --- a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh +++ b/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh @@ -14,7 +14,7 @@ accelerate launch examples/wanvideo/model_training/train.py \ --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B_lora_splited_cache" \ --lora_base_model "dit" \ - --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ @@ -33,7 +33,7 @@ accelerate launch --config_file examples/wanvideo/model_training/full/accelerate --remove_prefix_in_ckpt "pipe.dit." \ --output_path "./models/train/Wan-Animate-2-14B_lora" \ --lora_base_model "dit" \ - --lora_target_modules "q,k,v,o,ffn.0,ffn.2" \ + --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py index 383c3d71..fbf3db6e 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py @@ -27,21 +27,21 @@ tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) state_dict = load_state_dict("models/train/Wan-Animate-2-14B-Distilled_full/epoch-1.safetensors") +state_dict = {k.replace(".block.self_attn.", ".block.module.self_attn."): v for k, v in state_dict.items()} pipe.dit.load_state_dict(state_dict, strict=False) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() -num_frames = 41 -# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. +num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", animate2_prompt_ref="视频中的人在做动作，背景静止", animate2_reference_image=reference_image, animate2_reference_video=reference_video[:num_frames], animate2_offload_kv=True, - animate2_log_scale=-1.3, num_frames=num_frames, height=640, width=352, - num_inference_steps=10, cfg_scale=1.0, + num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py index 38c0d062..92906d40 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py @@ -27,11 +27,12 @@ tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) state_dict = load_state_dict("models/train/Wan-Animate-2-14B_full/epoch-1.safetensors") +state_dict = {k.replace(".block.self_attn.", ".block.module.self_attn."): v for k, v in state_dict.items()} pipe.dit.load_state_dict(state_dict, strict=False) reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() -num_frames = 41 +num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py index c4f9a156..094d3fc5 100644 --- a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py @@ -30,16 +30,15 @@ reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() num_frames = 81 -# For distilled model, set animate2_log_scale to -1.3, num_inference_steps to 10, and cfg_scale to 1.0. video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", + negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", animate2_prompt_ref="视频中的人在做动作，背景静止", animate2_reference_image=reference_image, animate2_reference_video=reference_video[:num_frames], animate2_offload_kv=True, - animate2_log_scale=-1.3, num_frames=num_frames, height=640, width=352, - num_inference_steps=10, cfg_scale=1.0, + num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) From c446a8329198ae513e438714dadba506584a1954 Mon Sep 17 00:00:00 2001 From: mi804 <1576993271@qq.com> Date: Thu, 6 Aug 2026 17:28:45 +0800 Subject: [PATCH 17/17] update modelid --- README.md | 6 +++--- README_zh.md | 6 +++--- diffsynth/configs/model_configs.py | 2 +- docs/en/Model_Details/Wan.md | 6 +++--- docs/zh/Model_Details/Wan.md | 6 +++--- ...-14B-usp.py => Wan2.2-Animate-2-14B-usp.py} | 18 +++++++++--------- ...ed.py => Wan2.2-Animate-2-14B-Distilled.py} | 18 +++++++++--------- ...nimate-2-14B.py => Wan2.2-Animate-2-14B.py} | 18 +++++++++--------- ...ed.py => Wan2.2-Animate-2-14B-Distilled.py} | 18 +++++++++--------- ...nimate-2-14B.py => Wan2.2-Animate-2-14B.py} | 18 +++++++++--------- ...ed.sh => Wan2.2-Animate-2-14B-Distilled.sh} | 16 ++++++++-------- ...nimate-2-14B.sh => Wan2.2-Animate-2-14B.sh} | 16 ++++++++-------- ...ed.sh => Wan2.2-Animate-2-14B-Distilled.sh} | 16 ++++++++-------- ...nimate-2-14B.sh => Wan2.2-Animate-2-14B.sh} | 16 ++++++++-------- ...ed.py => Wan2.2-Animate-2-14B-Distilled.py} | 16 ++++++++-------- ...nimate-2-14B.py => Wan2.2-Animate-2-14B.py} | 16 ++++++++-------- ...ed.py => Wan2.2-Animate-2-14B-Distilled.py} | 16 ++++++++-------- ...nimate-2-14B.py => Wan2.2-Animate-2-14B.py} | 16 ++++++++-------- 18 files changed, 122 insertions(+), 122 deletions(-) rename examples/wanvideo/acceleration/{Wan-Animate-2-14B-usp.py => Wan2.2-Animate-2-14B-usp.py} (85%) rename examples/wanvideo/model_inference/{Wan-Animate-2-14B-Distilled.py => Wan2.2-Animate-2-14B-Distilled.py} (82%) rename examples/wanvideo/model_inference/{Wan-Animate-2-14B.py => Wan2.2-Animate-2-14B.py} (84%) rename examples/wanvideo/model_inference_low_vram/{Wan-Animate-2-14B-Distilled.py => Wan2.2-Animate-2-14B-Distilled.py} (82%) rename examples/wanvideo/model_inference_low_vram/{Wan-Animate-2-14B.py => Wan2.2-Animate-2-14B.py} (85%) rename examples/wanvideo/model_training/full/{Wan-Animate-2-14B-Distilled.sh => Wan2.2-Animate-2-14B-Distilled.sh} (86%) rename examples/wanvideo/model_training/full/{Wan-Animate-2-14B.sh => Wan2.2-Animate-2-14B.sh} (87%) rename examples/wanvideo/model_training/lora/{Wan-Animate-2-14B-Distilled.sh => Wan2.2-Animate-2-14B-Distilled.sh} (63%) rename examples/wanvideo/model_training/lora/{Wan-Animate-2-14B.sh => Wan2.2-Animate-2-14B.sh} (65%) rename examples/wanvideo/model_training/validate_full/{Wan-Animate-2-14B-Distilled.py => Wan2.2-Animate-2-14B-Distilled.py} (70%) rename examples/wanvideo/model_training/validate_full/{Wan-Animate-2-14B.py => Wan2.2-Animate-2-14B.py} (71%) rename examples/wanvideo/model_training/validate_lora/{Wan-Animate-2-14B-Distilled.py => Wan2.2-Animate-2-14B-Distilled.py} (68%) rename examples/wanvideo/model_training/validate_lora/{Wan-Animate-2-14B.py => Wan2.2-Animate-2-14B.py} (69%) diff --git a/README.md b/README.md index 18673a3e..0c637bbe 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ We believe that a well-developed open-source code framework can lower the thresh > Currently, the development personnel of this project are limited, with most of the work handled by [Artiprocher](https://github.com/Artiprocher) and [mi804](https://github.com/mi804). Therefore, the progress of new feature development will be relatively slow, and the speed of responding to and resolving issues is limited. We apologize for this and ask developers to understand. -- **July 22, 2026** We add support for Wan-Animate-2 in the Wan series. Given a reference image and a driving video, it makes the reference character perform the motions in the driving video, generating high-quality character animation, with both standard and distilled variants. For details, please refer to the [documentation](/docs/en/Model_Details/Wan.md) and [example code](/examples/wanvideo/). +- **August 7, 2026** We add support for Wan-Animate-2 in the Wan series. Given a reference image and a driving video, it makes the reference character perform the motions in the driving video, generating high-quality character animation, with both standard and distilled variants. For details, please refer to the [documentation](/docs/en/Model_Details/Wan.md) and [example code](/examples/wanvideo/). - **June 29, 2026** Boogu-Image open-sourced. Support includes text-to-image generation, image editing, low VRAM inference, and training capabilities. For details, please refer to the [documentation](/docs/en/Model_Details/Boogu-Image.md) and [example code](/examples/boogu_image/). @@ -1433,8 +1433,8 @@ Example code for Wan is available at: [/examples/wanvideo/](/examples/wanvideo/) |[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| |[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| |[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| -|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| -|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| +|[Wan-AI/Wan2.2-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py)| +|[Wan-AI/Wan2.2-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py)| |[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| |[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| |[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| diff --git a/README_zh.md b/README_zh.md index 7c029117..0ce33245 100644 --- a/README_zh.md +++ b/README_zh.md @@ -34,7 +34,7 @@ DiffSynth 目前包括两个开源项目： > 目前本项目的开发人员有限，大部分工作由 [Artiprocher](https://github.com/Artiprocher) 和 [mi804](https://github.com/mi804) 负责，因此新功能的开发进展会比较缓慢，issue 的回复和解决速度有限，我们对此感到非常抱歉，请各位开发者理解。 -- **2026年7月22日** 我们为 Wan 系列新增了 Wan-Animate-2，输入一张参考图和一段驱动视频，即可让参考角色演绎驱动视频中的动作，生成高质量角色动画，包含标准与蒸馏两个变体。详情请参考[文档](/docs/zh/Model_Details/Wan.md)和[示例代码](/examples/wanvideo/)。 +- **2026年8月7日** 我们为 Wan 系列新增了 Wan-Animate-2，输入一张参考图和一段驱动视频，即可让参考角色演绎驱动视频中的动作，生成高质量角色动画，包含标准与蒸馏两个变体。详情请参考[文档](/docs/zh/Model_Details/Wan.md)和[示例代码](/examples/wanvideo/)。 - **2026年6月29日** Boogu-Image 开源，已支持文生图推理、图像编辑、低显存推理和训练能力。详情请参考[文档](/docs/zh/Model_Details/Boogu-Image.md)和[示例代码](/examples/boogu_image/)。 @@ -1433,8 +1433,8 @@ Wan 的示例代码位于：[/examples/wanvideo/](/examples/wanvideo/) |[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| |[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| |[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| -|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| -|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| +|[Wan-AI/Wan2.2-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py)| +|[Wan-AI/Wan2.2-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py)| |[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| |[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| |[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| diff --git a/diffsynth/configs/model_configs.py b/diffsynth/configs/model_configs.py index a048c73b..90fd8fa6 100644 --- a/diffsynth/configs/model_configs.py +++ b/diffsynth/configs/model_configs.py @@ -82,7 +82,7 @@ wan_series = [ { - # Example: ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors") + # Example: ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors") "model_hash": "4536c21ad8740ba78367af4216ae85bf", "model_name": "wan_video_dit", "model_class": "diffsynth.models.wan_animate_2_dit.WanAnimate2Transformer", diff --git a/docs/en/Model_Details/Wan.md b/docs/en/Model_Details/Wan.md index a6e0f1dd..632f007b 100644 --- a/docs/en/Model_Details/Wan.md +++ b/docs/en/Model_Details/Wan.md @@ -91,7 +91,7 @@ graph LR; Wan-AI/Wan2.1-T2V-14B-->krea/krea-realtime-video; Wan-AI/Wan2.1-I2V-14B-720P-->ByteDance/Video-As-Prompt-Wan2.1-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-Animate-14B; - Wan2.2-Series-->Wan-AI/Wan-Animate-2-14B; + Wan2.2-Series-->Wan-AI/Wan2.2-Animate-2-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-S2V-14B; Wan2.2-Series-->Wan-AI/Wan2.2-T2V-A14B; Wan2.2-Series-->Wan-AI/Wan2.2-I2V-A14B; @@ -133,8 +133,8 @@ graph LR; |[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| |[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| |[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| -|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| -|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| +|[Wan-AI/Wan2.2-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py)| +|[Wan-AI/Wan2.2-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py)| |[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| |[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| |[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| diff --git a/docs/zh/Model_Details/Wan.md b/docs/zh/Model_Details/Wan.md index e96a5255..55fd0885 100644 --- a/docs/zh/Model_Details/Wan.md +++ b/docs/zh/Model_Details/Wan.md @@ -92,7 +92,7 @@ graph LR; Wan-AI/Wan2.1-T2V-14B-->meituan-longcat/LongCat-Video; Wan-AI/Wan2.1-I2V-14B-720P-->ByteDance/Video-As-Prompt-Wan2.1-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-Animate-14B; - Wan2.2-Series-->Wan-AI/Wan-Animate-2-14B; + Wan2.2-Series-->Wan-AI/Wan2.2-Animate-2-14B; Wan-AI/Wan2.1-T2V-14B-->Wan-AI/Wan2.2-S2V-14B; Wan2.2-Series-->Wan-AI/Wan2.2-T2V-A14B; Wan2.2-Series-->Wan-AI/Wan2.2-I2V-A14B; @@ -134,8 +134,8 @@ graph LR; |[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)| |[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)| |[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)| -|[Wan-AI/Wan-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py)| -|[Wan-AI/Wan-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py)| +|[Wan-AI/Wan2.2-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py)| +|[Wan-AI/Wan2.2-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py)| |[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)| |[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)| |[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)| diff --git a/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py b/examples/wanvideo/acceleration/Wan2.2-Animate-2-14B-usp.py similarity index 85% rename from examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py rename to examples/wanvideo/acceleration/Wan2.2-Animate-2-14B-usp.py index bb1690b5..8553a8a1 100644 --- a/examples/wanvideo/acceleration/Wan-Animate-2-14B-usp.py +++ b/examples/wanvideo/acceleration/Wan2.2-Animate-2-14B-usp.py @@ -21,22 +21,22 @@ device="cuda", use_usp=True, model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) # Character animation: reference image (identity) + reference video (motion) -> animated video. dataset_snapshot_download( "DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" + allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B/*" ) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refvideo.mp4").raw_data() # Example 1: single-clip generation num_frames = 81 video = pipe( @@ -51,7 +51,7 @@ seed=0, tiled=True, ) if dist.get_rank() == 0: - save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) + save_video(video, "video_Wan2.2-Animate-2-14B.mp4", fps=24, quality=5) # Example 2: multi-clip long-video generation @@ -114,4 +114,4 @@ def zigzag_padding(array, target_len): seed=0, tiled=True, ) if dist.get_rank() == 0: - save_video(long_video, "video_Wan-Animate-2-14B-long.mp4", fps=24, quality=5) + save_video(long_video, "video_Wan2.2-Animate-2-14B-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py similarity index 82% rename from examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py rename to examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py index 2aeb656a..2ed11991 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py @@ -19,22 +19,22 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) # Character animation: reference image (identity) + reference video (motion) -> animated video. dataset_snapshot_download( "DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" + allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B-Distilled/*" ) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refvideo.mp4").raw_data() # Example 1: single-clip generation num_frames = 81 @@ -50,7 +50,7 @@ num_inference_steps=10, cfg_scale=1.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B-Distilled.mp4", fps=24, quality=5) # Example 2: multi-clip long-video generation @@ -112,4 +112,4 @@ def zigzag_padding(array, target_len): num_inference_steps=10, cfg_scale=1.0, seed=0, tiled=True, ) -save_video(long_video, "video_Wan-Animate-2-14B-Distilled-long.mp4", fps=24, quality=5) +save_video(long_video, "video_Wan2.2-Animate-2-14B-Distilled-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference/Wan-Animate-2-14B.py b/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py similarity index 84% rename from examples/wanvideo/model_inference/Wan-Animate-2-14B.py rename to examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py index 927c2326..b74e56da 100644 --- a/examples/wanvideo/model_inference/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py @@ -19,22 +19,22 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) # Character animation: reference image (identity) + reference video (motion) -> animated video. dataset_snapshot_download( "DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" + allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B/*" ) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refvideo.mp4").raw_data() # Example 1: single-clip generation num_frames = 81 video = pipe( @@ -48,7 +48,7 @@ num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B.mp4", fps=24, quality=5) # Example 2: multi-clip long-video generation @@ -110,4 +110,4 @@ def zigzag_padding(array, target_len): num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(long_video, "video_Wan-Animate-2-14B-long.mp4", fps=24, quality=5) +save_video(long_video, "video_Wan2.2-Animate-2-14B-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py similarity index 82% rename from examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py rename to examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py index f198e0f4..0d4e5aa5 100644 --- a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py @@ -19,12 +19,12 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, ) @@ -32,10 +32,10 @@ dataset_snapshot_download( "DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B-Distilled/*" + allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B-Distilled/*" ) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refvideo.mp4").raw_data() # Example 1: single-clip generation num_frames = 81 @@ -51,7 +51,7 @@ num_inference_steps=10, cfg_scale=1.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B-Distilled.mp4", fps=24, quality=5) # Example 2: multi-clip long-video generation @@ -114,4 +114,4 @@ def zigzag_padding(array, target_len): num_inference_steps=10, cfg_scale=1.0, seed=0, tiled=True, ) -save_video(long_video, "video_Wan-Animate-2-14B-Distilled-long.mp4", fps=24, quality=5) +save_video(long_video, "video_Wan2.2-Animate-2-14B-Distilled-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py b/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py similarity index 85% rename from examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py rename to examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py index f366e1e3..d89a5129 100644 --- a/examples/wanvideo/model_inference_low_vram/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py @@ -19,12 +19,12 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, ) @@ -32,10 +32,10 @@ dataset_snapshot_download( "DiffSynth-Studio/diffsynth_example_dataset", local_dir="data/diffsynth_example_dataset", - allow_file_pattern="wanvideo/Wan-Animate-2-14B/*" + allow_file_pattern="wanvideo/Wan2.2-Animate-2-14B/*" ) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refvideo.mp4").raw_data() # Example 1: single-clip generation num_frames = 81 video = pipe( @@ -49,7 +49,7 @@ num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B.mp4", fps=24, quality=5) # Example 2: multi-clip long-video generation @@ -111,4 +111,4 @@ def zigzag_padding(array, target_len): num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(long_video, "video_Wan-Animate-2-14B-long.mp4", fps=24, quality=5) +save_video(long_video, "video_Wan2.2-Animate-2-14B-long.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh b/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh similarity index 86% rename from examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh rename to examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh index a77ba9f9..fdc45026 100644 --- a/examples/wanvideo/model_training/full/Wan-Animate-2-14B-Distilled.sh +++ b/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh @@ -1,34 +1,34 @@ -modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B-Distilled/*" --local_dir ./data/diffsynth_example_dataset +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan2.2-Animate-2-14B-Distilled/*" --local_dir ./data/diffsynth_example_dataset accelerate launch examples/wanvideo/model_training/train.py \ - --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled \ - --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/metadata.json \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/metadata.json \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 1 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B-Distilled_full_splited_cache" \ + --output_path "./models/train/Wan2.2-Animate-2-14B-Distilled_full_splited_cache" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ --task "sft:data_process" accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ - --dataset_base_path models/train/Wan-Animate-2-14B-Distilled_full_splited_cache \ + --dataset_base_path models/train/Wan2.2-Animate-2-14B-Distilled_full_splited_cache \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 100 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16_distillation.safetensors" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:wan_animate_2/wan_animate_2_bf16_distillation.safetensors" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B-Distilled_full" \ + --output_path "./models/train/Wan2.2-Animate-2-14B-Distilled_full" \ --trainable_models "dit.blocks.0.block.self_attn.q,dit.blocks.0.block.self_attn.k,dit.blocks.0.block.self_attn.v,dit.blocks.1.block.self_attn.q,dit.blocks.1.block.self_attn.k,dit.blocks.1.block.self_attn.v,dit.blocks.2.block.self_attn.q,dit.blocks.2.block.self_attn.k,dit.blocks.2.block.self_attn.v,dit.blocks.3.block.self_attn.q,dit.blocks.3.block.self_attn.k,dit.blocks.3.block.self_attn.v,dit.blocks.4.block.self_attn.q,dit.blocks.4.block.self_attn.k,dit.blocks.4.block.self_attn.v,dit.blocks.5.block.self_attn.q,dit.blocks.5.block.self_attn.k,dit.blocks.5.block.self_attn.v,dit.blocks.6.block.self_attn.q,dit.blocks.6.block.self_attn.k,dit.blocks.6.block.self_attn.v,dit.blocks.7.block.self_attn.q,dit.blocks.7.block.self_attn.k,dit.blocks.7.block.self_attn.v,dit.blocks.8.block.self_attn.q,dit.blocks.8.block.self_attn.k,dit.blocks.8.block.self_attn.v,dit.blocks.9.block.self_attn.q,dit.blocks.9.block.self_attn.k,dit.blocks.9.block.self_attn.v,dit.blocks.10.block.self_attn.q,dit.blocks.10.block.self_attn.k,dit.blocks.10.block.self_attn.v,dit.blocks.11.block.self_attn.q,dit.blocks.11.block.self_attn.k,dit.blocks.11.block.self_attn.v,dit.blocks.12.block.self_attn.q,dit.blocks.12.block.self_attn.k,dit.blocks.12.block.self_attn.v,dit.blocks.13.block.self_attn.q,dit.blocks.13.block.self_attn.k,dit.blocks.13.block.self_attn.v,dit.blocks.14.block.self_attn.q,dit.blocks.14.block.self_attn.k,dit.blocks.14.block.self_attn.v,dit.blocks.15.block.self_attn.q,dit.blocks.15.block.self_attn.k,dit.blocks.15.block.self_attn.v,dit.blocks.16.block.self_attn.q,dit.blocks.16.block.self_attn.k,dit.blocks.16.block.self_attn.v,dit.blocks.17.block.self_attn.q,dit.blocks.17.block.self_attn.k,dit.blocks.17.block.self_attn.v,dit.blocks.18.block.self_attn.q,dit.blocks.18.block.self_attn.k,dit.blocks.18.block.self_attn.v,dit.blocks.19.block.self_attn.q,dit.blocks.19.block.self_attn.k,dit.blocks.19.block.self_attn.v,dit.blocks.20.block.self_attn.q,dit.blocks.20.block.self_attn.k,dit.blocks.20.block.self_attn.v,dit.blocks.21.block.self_attn.q,dit.blocks.21.block.self_attn.k,dit.blocks.21.block.self_attn.v,dit.blocks.22.block.self_attn.q,dit.blocks.22.block.self_attn.k,dit.blocks.22.block.self_attn.v,dit.blocks.23.block.self_attn.q,dit.blocks.23.block.self_attn.k,dit.blocks.23.block.self_attn.v,dit.blocks.24.block.self_attn.q,dit.blocks.24.block.self_attn.k,dit.blocks.24.block.self_attn.v,dit.blocks.25.block.self_attn.q,dit.blocks.25.block.self_attn.k,dit.blocks.25.block.self_attn.v,dit.blocks.26.block.self_attn.q,dit.blocks.26.block.self_attn.k,dit.blocks.26.block.self_attn.v,dit.blocks.27.block.self_attn.q,dit.blocks.27.block.self_attn.k,dit.blocks.27.block.self_attn.v,dit.blocks.28.block.self_attn.q,dit.blocks.28.block.self_attn.k,dit.blocks.28.block.self_attn.v,dit.blocks.29.block.self_attn.q,dit.blocks.29.block.self_attn.k,dit.blocks.29.block.self_attn.v,dit.blocks.30.block.self_attn.q,dit.blocks.30.block.self_attn.k,dit.blocks.30.block.self_attn.v,dit.blocks.31.block.self_attn.q,dit.blocks.31.block.self_attn.k,dit.blocks.31.block.self_attn.v,dit.blocks.32.block.self_attn.q,dit.blocks.32.block.self_attn.k,dit.blocks.32.block.self_attn.v,dit.blocks.33.block.self_attn.q,dit.blocks.33.block.self_attn.k,dit.blocks.33.block.self_attn.v,dit.blocks.34.block.self_attn.q,dit.blocks.34.block.self_attn.k,dit.blocks.34.block.self_attn.v,dit.blocks.35.block.self_attn.q,dit.blocks.35.block.self_attn.k,dit.blocks.35.block.self_attn.v,dit.blocks.36.block.self_attn.q,dit.blocks.36.block.self_attn.k,dit.blocks.36.block.self_attn.v,dit.blocks.37.block.self_attn.q,dit.blocks.37.block.self_attn.k,dit.blocks.37.block.self_attn.v,dit.blocks.38.block.self_attn.q,dit.blocks.38.block.self_attn.k,dit.blocks.38.block.self_attn.v,dit.blocks.39.block.self_attn.q,dit.blocks.39.block.self_attn.k,dit.blocks.39.block.self_attn.v" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ diff --git a/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh b/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh similarity index 87% rename from examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh rename to examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh index 20236a35..c4fdf46b 100644 --- a/examples/wanvideo/model_training/full/Wan-Animate-2-14B.sh +++ b/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh @@ -1,34 +1,34 @@ -modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B/*" --local_dir ./data/diffsynth_example_dataset +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan2.2-Animate-2-14B/*" --local_dir ./data/diffsynth_example_dataset accelerate launch examples/wanvideo/model_training/train.py \ - --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B \ - --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/metadata.json \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/metadata.json \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 1 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B_full_splited_cache" \ + --output_path "./models/train/Wan2.2-Animate-2-14B_full_splited_cache" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ --task "sft:data_process" accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ - --dataset_base_path models/train/Wan-Animate-2-14B_full_splited_cache \ + --dataset_base_path models/train/Wan2.2-Animate-2-14B_full_splited_cache \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 100 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16.safetensors" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:wan_animate_2/wan_animate_2_bf16.safetensors" \ --learning_rate 1e-5 \ --num_epochs 2 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B_full" \ + --output_path "./models/train/Wan2.2-Animate-2-14B_full" \ --trainable_models "dit.blocks.0.block.self_attn.q,dit.blocks.0.block.self_attn.k,dit.blocks.0.block.self_attn.v,dit.blocks.1.block.self_attn.q,dit.blocks.1.block.self_attn.k,dit.blocks.1.block.self_attn.v,dit.blocks.2.block.self_attn.q,dit.blocks.2.block.self_attn.k,dit.blocks.2.block.self_attn.v,dit.blocks.3.block.self_attn.q,dit.blocks.3.block.self_attn.k,dit.blocks.3.block.self_attn.v,dit.blocks.4.block.self_attn.q,dit.blocks.4.block.self_attn.k,dit.blocks.4.block.self_attn.v,dit.blocks.5.block.self_attn.q,dit.blocks.5.block.self_attn.k,dit.blocks.5.block.self_attn.v,dit.blocks.6.block.self_attn.q,dit.blocks.6.block.self_attn.k,dit.blocks.6.block.self_attn.v,dit.blocks.7.block.self_attn.q,dit.blocks.7.block.self_attn.k,dit.blocks.7.block.self_attn.v,dit.blocks.8.block.self_attn.q,dit.blocks.8.block.self_attn.k,dit.blocks.8.block.self_attn.v,dit.blocks.9.block.self_attn.q,dit.blocks.9.block.self_attn.k,dit.blocks.9.block.self_attn.v,dit.blocks.10.block.self_attn.q,dit.blocks.10.block.self_attn.k,dit.blocks.10.block.self_attn.v,dit.blocks.11.block.self_attn.q,dit.blocks.11.block.self_attn.k,dit.blocks.11.block.self_attn.v,dit.blocks.12.block.self_attn.q,dit.blocks.12.block.self_attn.k,dit.blocks.12.block.self_attn.v,dit.blocks.13.block.self_attn.q,dit.blocks.13.block.self_attn.k,dit.blocks.13.block.self_attn.v,dit.blocks.14.block.self_attn.q,dit.blocks.14.block.self_attn.k,dit.blocks.14.block.self_attn.v,dit.blocks.15.block.self_attn.q,dit.blocks.15.block.self_attn.k,dit.blocks.15.block.self_attn.v,dit.blocks.16.block.self_attn.q,dit.blocks.16.block.self_attn.k,dit.blocks.16.block.self_attn.v,dit.blocks.17.block.self_attn.q,dit.blocks.17.block.self_attn.k,dit.blocks.17.block.self_attn.v,dit.blocks.18.block.self_attn.q,dit.blocks.18.block.self_attn.k,dit.blocks.18.block.self_attn.v,dit.blocks.19.block.self_attn.q,dit.blocks.19.block.self_attn.k,dit.blocks.19.block.self_attn.v,dit.blocks.20.block.self_attn.q,dit.blocks.20.block.self_attn.k,dit.blocks.20.block.self_attn.v,dit.blocks.21.block.self_attn.q,dit.blocks.21.block.self_attn.k,dit.blocks.21.block.self_attn.v,dit.blocks.22.block.self_attn.q,dit.blocks.22.block.self_attn.k,dit.blocks.22.block.self_attn.v,dit.blocks.23.block.self_attn.q,dit.blocks.23.block.self_attn.k,dit.blocks.23.block.self_attn.v,dit.blocks.24.block.self_attn.q,dit.blocks.24.block.self_attn.k,dit.blocks.24.block.self_attn.v,dit.blocks.25.block.self_attn.q,dit.blocks.25.block.self_attn.k,dit.blocks.25.block.self_attn.v,dit.blocks.26.block.self_attn.q,dit.blocks.26.block.self_attn.k,dit.blocks.26.block.self_attn.v,dit.blocks.27.block.self_attn.q,dit.blocks.27.block.self_attn.k,dit.blocks.27.block.self_attn.v,dit.blocks.28.block.self_attn.q,dit.blocks.28.block.self_attn.k,dit.blocks.28.block.self_attn.v,dit.blocks.29.block.self_attn.q,dit.blocks.29.block.self_attn.k,dit.blocks.29.block.self_attn.v,dit.blocks.30.block.self_attn.q,dit.blocks.30.block.self_attn.k,dit.blocks.30.block.self_attn.v,dit.blocks.31.block.self_attn.q,dit.blocks.31.block.self_attn.k,dit.blocks.31.block.self_attn.v,dit.blocks.32.block.self_attn.q,dit.blocks.32.block.self_attn.k,dit.blocks.32.block.self_attn.v,dit.blocks.33.block.self_attn.q,dit.blocks.33.block.self_attn.k,dit.blocks.33.block.self_attn.v,dit.blocks.34.block.self_attn.q,dit.blocks.34.block.self_attn.k,dit.blocks.34.block.self_attn.v,dit.blocks.35.block.self_attn.q,dit.blocks.35.block.self_attn.k,dit.blocks.35.block.self_attn.v,dit.blocks.36.block.self_attn.q,dit.blocks.36.block.self_attn.k,dit.blocks.36.block.self_attn.v,dit.blocks.37.block.self_attn.q,dit.blocks.37.block.self_attn.k,dit.blocks.37.block.self_attn.v,dit.blocks.38.block.self_attn.q,dit.blocks.38.block.self_attn.k,dit.blocks.38.block.self_attn.v,dit.blocks.39.block.self_attn.q,dit.blocks.39.block.self_attn.k,dit.blocks.39.block.self_attn.v" \ --extra_inputs "animate2_prompt_ref,animate2_reference_image,animate2_reference_video" \ --use_gradient_checkpointing \ diff --git a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh b/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh similarity index 63% rename from examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh rename to examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh index 103c41d7..481086f0 100644 --- a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B-Distilled.sh +++ b/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh @@ -1,18 +1,18 @@ -modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B-Distilled/*" --local_dir ./data/diffsynth_example_dataset +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan2.2-Animate-2-14B-Distilled/*" --local_dir ./data/diffsynth_example_dataset accelerate launch examples/wanvideo/model_training/train.py \ - --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled \ - --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/metadata.json \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/metadata.json \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 1 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ --learning_rate 1e-4 \ --num_epochs 5 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B-Distilled_lora_splited_cache" \ + --output_path "./models/train/Wan2.2-Animate-2-14B-Distilled_lora_splited_cache" \ --lora_base_model "dit" \ --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ @@ -21,17 +21,17 @@ accelerate launch examples/wanvideo/model_training/train.py \ --task "sft:data_process" accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ - --dataset_base_path models/train/Wan-Animate-2-14B-Distilled_lora_splited_cache \ + --dataset_base_path models/train/Wan2.2-Animate-2-14B-Distilled_lora_splited_cache \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 100 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16_distillation.safetensors" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:wan_animate_2/wan_animate_2_bf16_distillation.safetensors" \ --learning_rate 1e-4 \ --num_epochs 5 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B-Distilled_lora" \ + --output_path "./models/train/Wan2.2-Animate-2-14B-Distilled_lora" \ --lora_base_model "dit" \ --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ diff --git a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh b/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh similarity index 65% rename from examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh rename to examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh index cd9caddd..2a812fcf 100644 --- a/examples/wanvideo/model_training/lora/Wan-Animate-2-14B.sh +++ b/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh @@ -1,18 +1,18 @@ -modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan-Animate-2-14B/*" --local_dir ./data/diffsynth_example_dataset +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "wanvideo/Wan2.2-Animate-2-14B/*" --local_dir ./data/diffsynth_example_dataset accelerate launch examples/wanvideo/model_training/train.py \ - --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B \ - --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/metadata.json \ + --dataset_base_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B \ + --dataset_metadata_path data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/metadata.json \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 1 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth,Wan-AI/Wan2.2-Animate-2-14B:videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth,Wan-AI/Wan2.1-T2V-14B:Wan2.1_VAE.pth" \ --learning_rate 1e-4 \ --num_epochs 5 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B_lora_splited_cache" \ + --output_path "./models/train/Wan2.2-Animate-2-14B_lora_splited_cache" \ --lora_base_model "dit" \ --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ @@ -21,17 +21,17 @@ accelerate launch examples/wanvideo/model_training/train.py \ --task "sft:data_process" accelerate launch --config_file examples/wanvideo/model_training/full/accelerate_config_14B.yaml examples/wanvideo/model_training/train.py \ - --dataset_base_path models/train/Wan-Animate-2-14B_lora_splited_cache \ + --dataset_base_path models/train/Wan2.2-Animate-2-14B_lora_splited_cache \ --data_file_keys "video,animate2_reference_image,animate2_reference_video" \ --height 640 \ --width 352 \ --num_frames 81 \ --dataset_repeat 100 \ - --model_id_with_origin_paths "Wan-AI/Wan-Animate-2-14B:wan_animate_2/wan_animate_2_bf16.safetensors" \ + --model_id_with_origin_paths "Wan-AI/Wan2.2-Animate-2-14B:wan_animate_2/wan_animate_2_bf16.safetensors" \ --learning_rate 1e-4 \ --num_epochs 5 \ --remove_prefix_in_ckpt "pipe.dit." \ - --output_path "./models/train/Wan-Animate-2-14B_lora" \ + --output_path "./models/train/Wan2.2-Animate-2-14B_lora" \ --lora_base_model "dit" \ --lora_target_modules "self_attn.q,self_attn.k,self_attn.v" \ --lora_rank 32 \ diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py similarity index 70% rename from examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py rename to examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py index fbf3db6e..acb601ca 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py @@ -19,19 +19,19 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) -state_dict = load_state_dict("models/train/Wan-Animate-2-14B-Distilled_full/epoch-1.safetensors") +state_dict = load_state_dict("models/train/Wan2.2-Animate-2-14B-Distilled_full/epoch-1.safetensors") state_dict = {k.replace(".block.self_attn.", ".block.module.self_attn."): v for k, v in state_dict.items()} pipe.dit.load_state_dict(state_dict, strict=False) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refvideo.mp4").raw_data() num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -44,4 +44,4 @@ num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B-Distilled.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py similarity index 71% rename from examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py rename to examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py index 92906d40..2643b37c 100644 --- a/examples/wanvideo/model_training/validate_full/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py @@ -19,19 +19,19 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) -state_dict = load_state_dict("models/train/Wan-Animate-2-14B_full/epoch-1.safetensors") +state_dict = load_state_dict("models/train/Wan2.2-Animate-2-14B_full/epoch-1.safetensors") state_dict = {k.replace(".block.self_attn.", ".block.module.self_attn."): v for k, v in state_dict.items()} pipe.dit.load_state_dict(state_dict, strict=False) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refvideo.mp4").raw_data() num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -44,4 +44,4 @@ num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py b/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py similarity index 68% rename from examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py rename to examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py index 094d3fc5..44c2b6a7 100644 --- a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B-Distilled.py +++ b/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py @@ -18,17 +18,17 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16_distillation.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) -pipe.load_lora(pipe.dit, "models/train/Wan-Animate-2-14B-Distilled_lora/epoch-4.safetensors", alpha=1) +pipe.load_lora(pipe.dit, "models/train/Wan2.2-Animate-2-14B-Distilled_lora/epoch-4.safetensors", alpha=1) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B-Distilled/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B-Distilled/refvideo.mp4").raw_data() num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -41,4 +41,4 @@ num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B-Distilled.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B-Distilled.mp4", fps=24, quality=5) diff --git a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py b/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py similarity index 69% rename from examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py rename to examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py index 2eae767b..6e3e2a4a 100644 --- a/examples/wanvideo/model_training/validate_lora/Wan-Animate-2-14B.py +++ b/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py @@ -18,17 +18,17 @@ torch_dtype=torch.bfloat16, device="cuda", model_configs=[ - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), - ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_t5_umt5-xxl-enc-bf16.pth", **vram_config), + ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", **vram_config), ModelConfig(model_id="Wan-AI/Wan2.1-T2V-14B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config), ], - tokenizer_config=ModelConfig(model_id="Wan-AI/Wan-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), + tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="videomodel/Wan-AI/umt5-xxl/"), ) -pipe.load_lora(pipe.dit, "models/train/Wan-Animate-2-14B_lora/epoch-4.safetensors", alpha=1) +pipe.load_lora(pipe.dit, "models/train/Wan2.2-Animate-2-14B_lora/epoch-4.safetensors", alpha=1) -reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refimage.jpg").convert("RGB") -reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan-Animate-2-14B/refvideo.mp4").raw_data() +reference_image = Image.open("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refimage.jpg").convert("RGB") +reference_video = VideoData("data/diffsynth_example_dataset/wanvideo/Wan2.2-Animate-2-14B/refvideo.mp4").raw_data() num_frames = 81 video = pipe( prompt="人物外观描述：一名长黑发女性，穿着白色半透明蕾丝长袖上衣，衣身带有花卉刺绣，下身搭配白色百褶短裙和黑色腰带，脚穿米白色厚底运动鞋。 背景描述：背景为现代室内空间，墙面和柜体以浅灰色为主，后方设有两扇深色落地窗或玻璃门，顶部安装长条形灯具，中央有一块浅色长方形台面。", @@ -41,4 +41,4 @@ num_inference_steps=40, cfg_scale=3.0, seed=0, tiled=True, ) -save_video(video, "video_Wan-Animate-2-14B.mp4", fps=24, quality=5) +save_video(video, "video_Wan2.2-Animate-2-14B.mp4", fps=24, quality=5)