let5see/AI-Video
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持视频生成时插图没必填,大幅提升视频生成速度

Browse Source
This commit is contained in:
puke
2025-11-07 14:09:32 +08:00
parent 514dbfaa1b
commit 8d5c578958
11 changed files with 674 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
pixelle_video/pipelines/custom.py
View File

@@ -32,10 +32,24 @@ class CustomPipeline(BasePipeline):
You can customize:
- Content processing logic
- Narration generation strategy
- Image prompt generation
- Image prompt generation (conditional based on template)
- Frame composition
- Video assembly
KEY OPTIMIZATION: Conditional Image Generation
-----------------------------------------------
This pipeline supports automatic detection of template image requirements.
If your template doesn't use {{image}}, the entire image generation pipeline
can be skipped, providing:
⚡ Faster generation (no image API calls)
💰 Lower cost (no LLM calls for image prompts)
🚀 Reduced dependencies (no ComfyUI needed for text-only videos)
Usage patterns:
1. Text-only videos: Use templates/1080x1920/simple.html
2. AI-generated images: Use templates with {{image}} placeholder
3. Custom logic: Modify template or override the detection logic in your subclass
Example usage:
# 1. Create your own pipeline by copying this file
# 2. Modify the __call__ method with your custom logic
@@ -90,6 +104,11 @@ class CustomPipeline(BasePipeline):
Returns:
VideoGenerationResult
Image Generation Logic:
- If template has {{image}} → automatically generates images
- If template has no {{image}} → skips image generation (faster, cheaper)
- To customize: Override the template_requires_image logic in your subclass
"""
logger.info("Starting CustomPipeline")
logger.info(f"Input text length: {len(text)} chars")
@@ -114,6 +133,22 @@ class CustomPipeline(BasePipeline):
user_specified_output = output_path
output_path = get_task_final_video_path(task_id)
# ========== Step 0.5: Check template requirements ==========
# Detect if template requires {{image}} parameter
# This allows skipping the entire image generation pipeline for text-only templates
from pixelle_video.services.frame_html import HTMLFrameGenerator
from pixelle_video.utils.template_util import resolve_template_path
template_path = resolve_template_path(frame_template)
generator = HTMLFrameGenerator(template_path)
template_requires_image = generator.requires_image()
if template_requires_image:
logger.info(f"📸 Template requires image generation")
else:
logger.info(f"⚡ Template does not require images - skipping image generation pipeline")
logger.info(f" 💡 Benefits: Faster generation + Lower cost + No ComfyUI dependency")
# ========== Step 1: Process content (CUSTOMIZE THIS) ==========
self._report_progress(progress_callback, "processing_content", 0.10)
@@ -138,29 +173,37 @@ class CustomPipeline(BasePipeline):
logger.info(f"Generated {len(narrations)} narrations")
# ========== Step 2: Generate image prompts (CUSTOMIZE THIS) ==========
# ========== Step 2: Generate image prompts (CONDITIONAL - CUSTOMIZE THIS) ==========
self._report_progress(progress_callback, "generating_image_prompts", 0.25)
# Example: Generate image prompts using LLM
from pixelle_video.utils.content_generators import generate_image_prompts
image_prompts = await generate_image_prompts(
self.llm,
narrations=narrations,
min_words=30,
max_words=60
)
# Example: Apply custom prompt prefix
from pixelle_video.utils.prompt_helper import build_image_prompt
custom_prefix = "cinematic style, professional lighting" # Customize this
final_image_prompts = []
for base_prompt in image_prompts:
final_prompt = build_image_prompt(base_prompt, custom_prefix)
final_image_prompts.append(final_prompt)
logger.info(f"Generated {len(final_image_prompts)} image prompts")
# IMPORTANT: Check if template actually needs images
# If your template doesn't use {{image}}, you can skip this entire step!
if template_requires_image:
# Template requires images - generate image prompts using LLM
from pixelle_video.utils.content_generators import generate_image_prompts
image_prompts = await generate_image_prompts(
self.llm,
narrations=narrations,
min_words=30,
max_words=60
)
# Example: Apply custom prompt prefix
from pixelle_video.utils.prompt_helper import build_image_prompt
custom_prefix = "cinematic style, professional lighting" # Customize this
final_image_prompts = []
for base_prompt in image_prompts:
final_prompt = build_image_prompt(base_prompt, custom_prefix)
final_image_prompts.append(final_prompt)
logger.info(f"✅ Generated {len(final_image_prompts)} image prompts")
else:
# Template doesn't need images - skip image generation entirely
final_image_prompts = [None] * len(narrations)
logger.info(f"⚡ Skipped image prompt generation (template doesn't need images)")
logger.info(f" 💡 Savings: {len(narrations)} LLM calls + {len(narrations)} image generations")
# ========== Step 3: Create storyboard ==========
config = StoryboardConfig(
@@ -317,8 +360,8 @@ class CustomPipeline(BasePipeline):
# ==================== Usage Examples ====================
"""
Example 1: Register and use custom pipeline
----------------------------------------
Example 1: Text-only video (no AI image generation)
---------------------------------------------------
from pixelle_video import pixelle_video
from pixelle_video.pipelines.custom import CustomPipeline
@@ -328,15 +371,27 @@ await pixelle_video.initialize()
# Register custom pipeline
pixelle_video.pipelines["my_custom"] = CustomPipeline(pixelle_video)
# Use it
# Use text-only template - no image generation!
result = await pixelle_video.generate_video(
text="Your input content here",
text="Your content here",
pipeline="my_custom",
custom_param_example="custom_value"
frame_template="1080x1920/simple.html" # Template without {{image}}
)
# Benefits: ⚡ Fast, 💰 Cheap, 🚀 No ComfyUI needed
Example 2: Create your own pipeline class
Example 2: AI-generated image video
---------------------------------------------------
# Use template with {{image}} - automatic image generation
result = await pixelle_video.generate_video(
text="Your content here",
pipeline="my_custom",
frame_template="1080x1920/default.html" # Template with {{image}}
)
# Will automatically generate images via LLM + ComfyUI
Example 3: Create your own pipeline class
----------------------------------------
from pixelle_video.pipelines.custom import CustomPipeline
@@ -351,7 +406,7 @@ class MySpecialPipeline(CustomPipeline):
return result
Example 3: Inline custom pipeline
Example 4: Inline custom pipeline
----------------------------------------
from pixelle_video.pipelines.base import BasePipeline

133
pixelle_video/pipelines/standard.py
View File

@@ -250,6 +250,14 @@ class StandardPipeline(BasePipeline):
created_at=datetime.now()
)
# ========== Step 0.8: Check template requirements ==========
template_requires_image = self._check_template_requires_image(config.frame_template)
if template_requires_image:
logger.info(f"📸 Template requires image generation")
else:
logger.info(f"⚡ Template does not require images - skipping image generation pipeline")
logger.info(f" 💡 Benefits: Faster generation + Lower cost + No ComfyUI dependency")
try:
# ========== Step 1: Generate/Split narrations ==========
if mode == "generate":
@@ -268,54 +276,61 @@ class StandardPipeline(BasePipeline):
logger.info(f"✅ Split script into {len(narrations)} segments (by lines)")
logger.info(f" Note: n_scenes={n_scenes} is ignored in fixed mode")
# ========== Step 2: Generate image prompts ==========
self._report_progress(progress_callback, "generating_image_prompts", 0.15)
# Override prompt_prefix if provided
original_prefix = None
if prompt_prefix is not None:
image_config = self.core.config.get("comfyui", {}).get("image", {})
original_prefix = image_config.get("prompt_prefix")
image_config["prompt_prefix"] = prompt_prefix
logger.info(f"Using custom prompt_prefix: '{prompt_prefix}'")
try:
# Create progress callback wrapper for image prompt generation
def image_prompt_progress(completed: int, total: int, message: str):
batch_progress = completed / total if total > 0 else 0
overall_progress = 0.15 + (batch_progress * 0.15)
self._report_progress(
progress_callback,
"generating_image_prompts",
overall_progress,
extra_info=message
# ========== Step 2: Generate image prompts (conditional) ==========
if template_requires_image:
self._report_progress(progress_callback, "generating_image_prompts", 0.15)
# Override prompt_prefix if provided
original_prefix = None
if prompt_prefix is not None:
image_config = self.core.config.get("comfyui", {}).get("image", {})
original_prefix = image_config.get("prompt_prefix")
image_config["prompt_prefix"] = prompt_prefix
logger.info(f"Using custom prompt_prefix: '{prompt_prefix}'")
try:
# Create progress callback wrapper for image prompt generation
def image_prompt_progress(completed: int, total: int, message: str):
batch_progress = completed / total if total > 0 else 0
overall_progress = 0.15 + (batch_progress * 0.15)
self._report_progress(
progress_callback,
"generating_image_prompts",
overall_progress,
extra_info=message
)
# Generate base image prompts
base_image_prompts = await generate_image_prompts(
self.llm,
narrations=narrations,
min_words=min_image_prompt_words,
max_words=max_image_prompt_words,
progress_callback=image_prompt_progress
)
# Apply prompt prefix
from pixelle_video.utils.prompt_helper import build_image_prompt
image_config = self.core.config.get("comfyui", {}).get("image", {})
prompt_prefix_to_use = prompt_prefix if prompt_prefix is not None else image_config.get("prompt_prefix", "")
image_prompts = []
for base_prompt in base_image_prompts:
final_prompt = build_image_prompt(base_prompt, prompt_prefix_to_use)
image_prompts.append(final_prompt)
finally:
# Restore original prompt_prefix
if original_prefix is not None:
image_config["prompt_prefix"] = original_prefix
# Generate base image prompts
base_image_prompts = await generate_image_prompts(
self.llm,
narrations=narrations,
min_words=min_image_prompt_words,
max_words=max_image_prompt_words,
progress_callback=image_prompt_progress
)
# Apply prompt prefix
from pixelle_video.utils.prompt_helper import build_image_prompt
image_config = self.core.config.get("comfyui", {}).get("image", {})
prompt_prefix_to_use = prompt_prefix if prompt_prefix is not None else image_config.get("prompt_prefix", "")
image_prompts = []
for base_prompt in base_image_prompts:
final_prompt = build_image_prompt(base_prompt, prompt_prefix_to_use)
image_prompts.append(final_prompt)
finally:
# Restore original prompt_prefix
if original_prefix is not None:
image_config["prompt_prefix"] = original_prefix
logger.info(f"✅ Generated {len(image_prompts)} image prompts")
logger.info(f"✅ Generated {len(image_prompts)} image prompts")
else:
# Skip image prompt generation
image_prompts = [None] * len(narrations)
self._report_progress(progress_callback, "preparing_frames", 0.15)
logger.info(f"⚡ Skipped image prompt generation (template doesn't need images)")
logger.info(f" 💡 Savings: {len(narrations)} LLM calls + {len(narrations)} image generations")
# ========== Step 3: Create frames ==========
for i, (narration, image_prompt) in enumerate(zip(narrations, image_prompts)):
@@ -418,4 +433,30 @@ class StandardPipeline(BasePipeline):
except Exception as e:
logger.error(f"❌ Video generation failed: {e}")
raise
def _check_template_requires_image(self, frame_template: str) -> bool:
"""
Check if template requires image generation
This is checked at pipeline level to avoid unnecessary:
- LLM calls (generating image_prompts)
- Image generation API calls
- ComfyUI dependency
Args:
frame_template: Template path (e.g., "1080x1920/default.html")
Returns:
True if template contains {{image}}, False otherwise
"""
from pixelle_video.services.frame_html import HTMLFrameGenerator
from pixelle_video.utils.template_util import resolve_template_path
template_path = resolve_template_path(frame_template)
generator = HTMLFrameGenerator(template_path)
requires = generator.requires_image()
logger.debug(f"Template '{frame_template}' requires_image={requires}")
return requires