1026 lines
36 KiB
Python
1026 lines
36 KiB
Python
# Copyright (C) 2025 AIDC-AI
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""
|
|
Editor API router for timeline editor operations
|
|
|
|
Provides endpoints for:
|
|
- Fetching storyboard data
|
|
- Reordering frames
|
|
- Updating frame duration
|
|
- Generating preview
|
|
"""
|
|
|
|
from fastapi import APIRouter, HTTPException, Path
|
|
from loguru import logger
|
|
|
|
from api.schemas.editor import (
|
|
StoryboardSchema,
|
|
StoryboardFrameSchema,
|
|
ReorderFramesRequest,
|
|
UpdateDurationRequest,
|
|
PreviewRequest,
|
|
PreviewResponse,
|
|
UpdateFrameRequest,
|
|
UpdateFrameResponse,
|
|
RegenerateImageRequest,
|
|
RegenerateImageResponse,
|
|
RegenerateAudioRequest,
|
|
RegenerateAudioResponse,
|
|
InpaintRequest,
|
|
InpaintResponse,
|
|
ExportRequest,
|
|
ExportResponse,
|
|
ExportStatusResponse,
|
|
AlignPromptRequest,
|
|
AlignPromptResponse,
|
|
)
|
|
from fastapi import BackgroundTasks
|
|
import asyncio
|
|
import uuid as uuid_module
|
|
|
|
router = APIRouter(prefix="/editor", tags=["Editor"])
|
|
|
|
# Export task storage
|
|
_export_tasks: dict = {}
|
|
|
|
|
|
def _path_to_url(file_path: str, base_url: str = "http://localhost:8000") -> str:
|
|
"""Convert local file path to URL accessible through API"""
|
|
if not file_path:
|
|
return None
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# Normalize path separators
|
|
file_path = file_path.replace("\\", "/")
|
|
|
|
# Extract relative path from output directory
|
|
parts = file_path.split("/")
|
|
try:
|
|
output_idx = parts.index("output")
|
|
relative_parts = parts[output_idx + 1:]
|
|
relative_path = "/".join(relative_parts)
|
|
except ValueError:
|
|
relative_path = Path(file_path).name
|
|
|
|
return f"{base_url}/api/files/{relative_path}"
|
|
|
|
|
|
# In-memory cache for demo (in production, use database)
|
|
_storyboard_cache: dict = {}
|
|
|
|
|
|
# Demo data for testing
|
|
_demo_storyboard = {
|
|
"id": "demo-1",
|
|
"title": "演示视频",
|
|
"total_duration": 15.5,
|
|
"final_video_path": None,
|
|
"created_at": None,
|
|
"frames": [
|
|
{"id": "frame-0", "index": 0, "order": 0, "narration": "在一个宁静的早晨，阳光洒满了整个城市", "image_prompt": "A peaceful morning", "duration": 3.2},
|
|
{"id": "frame-1", "index": 1, "order": 1, "narration": "小明决定出门去探索这个美丽的世界", "image_prompt": "A young man stepping out", "duration": 2.8},
|
|
{"id": "frame-2", "index": 2, "order": 2, "narration": "他走过熟悉的街道，感受着微风的吹拂", "image_prompt": "Walking through streets", "duration": 3.5},
|
|
{"id": "frame-3", "index": 3, "order": 3, "narration": "公园里的花朵正在盛开，散发着迷人的芬芳", "image_prompt": "Blooming flowers", "duration": 3.0},
|
|
{"id": "frame-4", "index": 4, "order": 4, "narration": "这是新的一天的开始，充满了无限可能", "image_prompt": "New day begins", "duration": 3.0},
|
|
],
|
|
}
|
|
# Import task manager
|
|
from api.tasks.manager import task_manager
|
|
|
|
|
|
@router.get("/storyboard/{storyboard_id}", response_model=StoryboardSchema)
|
|
async def get_storyboard(storyboard_id: str = Path(..., description="Storyboard/task ID")):
|
|
"""
|
|
Get storyboard by ID
|
|
|
|
Supports:
|
|
- 'demo-1': Returns demo data for testing
|
|
- Any task_id: Loads real storyboard from completed video generation tasks
|
|
- History tasks: Loads from persistence service
|
|
"""
|
|
# Return demo data for demo-1
|
|
if storyboard_id == "demo-1":
|
|
if "demo-1" not in _storyboard_cache:
|
|
_storyboard_cache["demo-1"] = _demo_storyboard.copy()
|
|
return _storyboard_cache["demo-1"]
|
|
|
|
# Try to get from cache first
|
|
if storyboard_id in _storyboard_cache:
|
|
return _storyboard_cache[storyboard_id]
|
|
|
|
# Try to load from task manager (in-memory task)
|
|
task = task_manager.get_task(storyboard_id)
|
|
if task and task.result:
|
|
# Extract storyboard from task result
|
|
result = task.result
|
|
|
|
# Handle different result formats
|
|
storyboard_data = None
|
|
|
|
if hasattr(result, 'storyboard'):
|
|
storyboard_data = result.storyboard
|
|
elif isinstance(result, dict) and 'storyboard' in result:
|
|
storyboard_data = result['storyboard']
|
|
|
|
if storyboard_data:
|
|
# Convert to editor schema format
|
|
schema = _convert_storyboard_to_schema(storyboard_id, storyboard_data)
|
|
_storyboard_cache[storyboard_id] = schema
|
|
logger.info(f"Loaded storyboard from task {storyboard_id}")
|
|
return schema
|
|
|
|
# Try to load from persistence service (history)
|
|
try:
|
|
from pixelle_video.services.persistence import PersistenceService
|
|
persistence = PersistenceService(output_dir="output")
|
|
|
|
# Load storyboard from disk (await since we're in an async function)
|
|
storyboard = await persistence.load_storyboard(storyboard_id)
|
|
|
|
if storyboard:
|
|
schema = _convert_storyboard_to_schema(storyboard_id, storyboard)
|
|
_storyboard_cache[storyboard_id] = schema
|
|
logger.info(f"Loaded storyboard from persistence {storyboard_id}")
|
|
return schema
|
|
except Exception as e:
|
|
logger.warning(f"Failed to load from persistence: {e}")
|
|
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found")
|
|
|
|
|
|
def _convert_storyboard_to_schema(storyboard_id: str, storyboard) -> dict:
|
|
"""Convert internal Storyboard model to API schema format."""
|
|
frames = []
|
|
|
|
# Handle both object and dict formats
|
|
if hasattr(storyboard, 'frames'):
|
|
frame_list = storyboard.frames
|
|
title = getattr(storyboard, 'title', storyboard_id)
|
|
total_duration = getattr(storyboard, 'total_duration', 0)
|
|
final_video_path = getattr(storyboard, 'final_video_path', None)
|
|
created_at = getattr(storyboard, 'created_at', None)
|
|
elif isinstance(storyboard, dict):
|
|
frame_list = storyboard.get('frames', [])
|
|
title = storyboard.get('title', storyboard_id)
|
|
total_duration = storyboard.get('total_duration', 0)
|
|
final_video_path = storyboard.get('final_video_path')
|
|
created_at = storyboard.get('created_at')
|
|
else:
|
|
frame_list = []
|
|
title = storyboard_id
|
|
total_duration = 0
|
|
final_video_path = None
|
|
created_at = None
|
|
|
|
for i, frame in enumerate(frame_list):
|
|
if hasattr(frame, 'narration'):
|
|
# Object format
|
|
frames.append({
|
|
"id": f"frame-{i}",
|
|
"index": getattr(frame, 'index', i),
|
|
"order": i,
|
|
"narration": frame.narration or "",
|
|
"image_prompt": getattr(frame, 'image_prompt', ""),
|
|
"image_path": _path_to_url(getattr(frame, 'image_path', None)),
|
|
"audio_path": _path_to_url(getattr(frame, 'audio_path', None)),
|
|
"video_segment_path": _path_to_url(getattr(frame, 'video_segment_path', None)),
|
|
"duration": getattr(frame, 'duration', 3.0),
|
|
})
|
|
elif isinstance(frame, dict):
|
|
# Dict format
|
|
frames.append({
|
|
"id": f"frame-{i}",
|
|
"index": frame.get('index', i),
|
|
"order": i,
|
|
"narration": frame.get('narration', ""),
|
|
"image_prompt": frame.get('image_prompt', ""),
|
|
"image_path": _path_to_url(frame.get('image_path')),
|
|
"audio_path": _path_to_url(frame.get('audio_path')),
|
|
"video_segment_path": _path_to_url(frame.get('video_segment_path')),
|
|
"duration": frame.get('duration', 3.0),
|
|
})
|
|
|
|
return {
|
|
"id": storyboard_id,
|
|
"title": title,
|
|
"frames": frames,
|
|
"total_duration": total_duration or sum(f.get('duration', 3.0) for f in frames),
|
|
"final_video_path": final_video_path,
|
|
"created_at": created_at.isoformat() if created_at else None,
|
|
}
|
|
|
|
|
|
@router.patch("/storyboard/{storyboard_id}/reorder", response_model=StoryboardSchema)
|
|
async def reorder_frames(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
request: ReorderFramesRequest = None
|
|
):
|
|
"""
|
|
Reorder frames in storyboard
|
|
|
|
Updates the order of frames based on the provided frame ID list.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found in cache")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Create ID to frame mapping
|
|
frame_map = {f["id"]: f for f in frames}
|
|
|
|
# Validate all IDs exist
|
|
for frame_id in request.order:
|
|
if frame_id not in frame_map:
|
|
raise HTTPException(status_code=400, detail=f"Frame {frame_id} not found")
|
|
|
|
# Reorder frames
|
|
reordered = []
|
|
for idx, frame_id in enumerate(request.order):
|
|
frame = frame_map[frame_id].copy()
|
|
frame["order"] = idx
|
|
reordered.append(frame)
|
|
|
|
storyboard["frames"] = reordered
|
|
_storyboard_cache[storyboard_id] = storyboard
|
|
|
|
logger.info(f"Reordered {len(reordered)} frames in storyboard {storyboard_id}")
|
|
|
|
return storyboard
|
|
|
|
|
|
@router.patch(
|
|
"/storyboard/{storyboard_id}/frames/{frame_id}/duration",
|
|
response_model=StoryboardFrameSchema
|
|
)
|
|
async def update_frame_duration(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
frame_id: str = Path(..., description="Frame ID"),
|
|
request: UpdateDurationRequest = None
|
|
):
|
|
"""
|
|
Update frame duration
|
|
|
|
Changes the duration of a specific frame and recalculates total duration.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found in cache")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Find and update frame
|
|
updated_frame = None
|
|
for frame in frames:
|
|
if frame["id"] == frame_id:
|
|
frame["duration"] = request.duration
|
|
updated_frame = frame
|
|
break
|
|
|
|
if not updated_frame:
|
|
raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found")
|
|
|
|
# Recalculate total duration
|
|
storyboard["total_duration"] = sum(f["duration"] for f in frames)
|
|
_storyboard_cache[storyboard_id] = storyboard
|
|
|
|
logger.info(f"Updated frame {frame_id} duration to {request.duration}s")
|
|
|
|
return updated_frame
|
|
|
|
|
|
@router.post("/storyboard/{storyboard_id}/preview", response_model=PreviewResponse)
|
|
async def generate_preview(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
request: PreviewRequest = None
|
|
):
|
|
"""
|
|
Generate preview video for selected frames
|
|
|
|
Creates a preview video from the specified frame range.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found in cache")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Determine frame range
|
|
start = request.start_frame if request else 0
|
|
end = request.end_frame if request and request.end_frame else len(frames)
|
|
|
|
if start >= len(frames):
|
|
raise HTTPException(status_code=400, detail="Start frame out of range")
|
|
|
|
preview_frames = frames[start:end]
|
|
total_duration = sum(f["duration"] for f in preview_frames)
|
|
|
|
# TODO: Implement actual preview generation logic
|
|
# For now, return mock response
|
|
preview_path = f"/output/{storyboard_id}/preview_{start}_{end}.mp4"
|
|
|
|
logger.info(f"Generated preview for frames {start}-{end} ({len(preview_frames)} frames)")
|
|
|
|
return PreviewResponse(
|
|
preview_path=preview_path,
|
|
duration=total_duration,
|
|
frames_count=len(preview_frames)
|
|
)
|
|
|
|
|
|
def _storyboard_to_schema(storyboard_id: str, storyboard) -> dict:
|
|
"""Convert internal Storyboard to API schema format"""
|
|
frames = []
|
|
for i, frame in enumerate(storyboard.frames):
|
|
frames.append({
|
|
"id": f"frame-{i}",
|
|
"index": frame.index,
|
|
"order": i,
|
|
"narration": frame.narration,
|
|
"image_prompt": frame.image_prompt,
|
|
"image_path": frame.image_path,
|
|
"audio_path": frame.audio_path,
|
|
"video_segment_path": frame.video_segment_path,
|
|
"duration": frame.duration,
|
|
})
|
|
|
|
return {
|
|
"id": storyboard_id,
|
|
"title": storyboard.title,
|
|
"frames": frames,
|
|
"total_duration": storyboard.total_duration,
|
|
"final_video_path": storyboard.final_video_path,
|
|
"created_at": storyboard.created_at,
|
|
}
|
|
|
|
|
|
@router.put(
|
|
"/storyboard/{storyboard_id}/frames/{frame_id}",
|
|
response_model=UpdateFrameResponse
|
|
)
|
|
async def update_frame(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
frame_id: str = Path(..., description="Frame ID"),
|
|
request: UpdateFrameRequest = None
|
|
):
|
|
"""
|
|
Update frame content (narration and/or image prompt)
|
|
|
|
Updates the text content of a frame without regenerating media.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found in cache")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Find and update frame
|
|
updated_frame = None
|
|
for frame in frames:
|
|
if frame["id"] == frame_id:
|
|
if request.narration is not None:
|
|
frame["narration"] = request.narration
|
|
if request.image_prompt is not None:
|
|
frame["image_prompt"] = request.image_prompt
|
|
updated_frame = frame
|
|
break
|
|
|
|
if not updated_frame:
|
|
raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found")
|
|
|
|
_storyboard_cache[storyboard_id] = storyboard
|
|
|
|
logger.info(f"Updated frame {frame_id} content")
|
|
|
|
return UpdateFrameResponse(
|
|
id=frame_id,
|
|
narration=updated_frame["narration"],
|
|
image_prompt=updated_frame.get("image_prompt"),
|
|
updated=True
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/storyboard/{storyboard_id}/frames/{frame_id}/regenerate-image",
|
|
response_model=RegenerateImageResponse
|
|
)
|
|
async def regenerate_frame_image(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
frame_id: str = Path(..., description="Frame ID"),
|
|
request: RegenerateImageRequest = None
|
|
):
|
|
"""
|
|
Regenerate image for a frame
|
|
|
|
Uses the frame's image_prompt (or override) to generate a new image.
|
|
Requires ComfyUI service to be running.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Find frame
|
|
target_frame = None
|
|
frame_index = 0
|
|
for i, frame in enumerate(frames):
|
|
if frame["id"] == frame_id:
|
|
target_frame = frame
|
|
frame_index = i
|
|
break
|
|
|
|
if not target_frame:
|
|
raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found")
|
|
|
|
# Get prompt to use
|
|
prompt = request.image_prompt if request and request.image_prompt else target_frame.get("image_prompt", "")
|
|
|
|
if not prompt:
|
|
raise HTTPException(status_code=400, detail="No image prompt available")
|
|
|
|
try:
|
|
# Import and use PixelleVideo core services
|
|
from api.dependencies import get_pixelle_video
|
|
from api.routers.quality import _style_anchors
|
|
|
|
logger.debug(f"[REGEN-IMG] Starting image regeneration for frame {frame_id}")
|
|
logger.debug(f"[REGEN-IMG] Original prompt: {prompt[:100]}...")
|
|
|
|
pixelle_video = await get_pixelle_video()
|
|
|
|
# Get style anchor prefix if available
|
|
style_prefix = ""
|
|
logger.debug(f"[REGEN-IMG] Checking style anchors for storyboard {storyboard_id}")
|
|
logger.debug(f"[REGEN-IMG] Available style anchors: {list(_style_anchors.keys())}")
|
|
|
|
if storyboard_id in _style_anchors:
|
|
style_data = _style_anchors[storyboard_id]
|
|
style_prefix = style_data.get("style_prefix", "")
|
|
logger.info(f"[REGEN-IMG] Found style anchor: {style_prefix[:80] if style_prefix else 'EMPTY'}...")
|
|
else:
|
|
logger.warning(f"[REGEN-IMG] No style anchor found for {storyboard_id}")
|
|
|
|
# Get character descriptions for prompt injection
|
|
character_prefix = ""
|
|
from api.routers.quality import _character_stores
|
|
if storyboard_id in _character_stores:
|
|
char_descriptions = []
|
|
for char_data in _character_stores[storyboard_id].values():
|
|
appearance = char_data.get("appearance_description", "")
|
|
clothing = char_data.get("clothing_description", "")
|
|
name = char_data.get("name", "character")
|
|
|
|
if appearance or clothing:
|
|
parts = [f"{name}:"]
|
|
if appearance:
|
|
parts.append(appearance)
|
|
if clothing:
|
|
parts.append(f"wearing {clothing}")
|
|
char_descriptions.append(" ".join(parts))
|
|
|
|
if char_descriptions:
|
|
character_prefix = "Characters: " + "; ".join(char_descriptions) + ". "
|
|
logger.info(f"[REGEN-IMG] Injecting character descriptions: {character_prefix[:80]}...")
|
|
|
|
# Apply style prefix and character descriptions to prompt
|
|
final_prompt = ""
|
|
if style_prefix:
|
|
final_prompt += f"{style_prefix}, "
|
|
if character_prefix:
|
|
final_prompt += character_prefix
|
|
final_prompt += prompt
|
|
logger.info(f"[REGEN-IMG] Final prompt: {final_prompt[:120]}...")
|
|
|
|
# Use MediaService to generate image via RunningHub workflow
|
|
# Use image_flux2 workflow (FLUX.1 Kontext model for better consistency)
|
|
logger.debug(f"[REGEN-IMG] Calling pixelle_video.image with workflow=runninghub/image_flux2.json")
|
|
result = await pixelle_video.image(
|
|
prompt=final_prompt,
|
|
media_type="image",
|
|
workflow="runninghub/image_flux2.json",
|
|
)
|
|
|
|
if result and result.url:
|
|
# Download and save image
|
|
import aiohttp
|
|
import os
|
|
|
|
output_dir = f"output/{storyboard_id}"
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
image_path = f"{output_dir}/frame_{frame_index}_regenerated.png"
|
|
|
|
# Check if URL is remote or local
|
|
if result.url.startswith("http"):
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(result.url) as resp:
|
|
if resp.status == 200:
|
|
with open(image_path, 'wb') as f:
|
|
f.write(await resp.read())
|
|
else:
|
|
# Local file, copy it
|
|
import shutil
|
|
if os.path.exists(result.url):
|
|
shutil.copy2(result.url, image_path)
|
|
else:
|
|
image_path = result.url
|
|
|
|
# Update frame
|
|
target_frame["image_path"] = _path_to_url(image_path)
|
|
_storyboard_cache[storyboard_id] = storyboard
|
|
|
|
# Persist changes to storyboard.json
|
|
try:
|
|
from pixelle_video.services.persistence import PersistenceService
|
|
persistence = PersistenceService()
|
|
|
|
# Load existing storyboard model
|
|
storyboard_model = await persistence.load_storyboard(storyboard_id)
|
|
if storyboard_model:
|
|
# Update the specific frame's image_path
|
|
for frame in storyboard_model.frames:
|
|
if f"frame-{frame.index}" == frame_id:
|
|
frame.image_path = image_path
|
|
logger.debug(f"[PERSIST] Updated frame {frame_id} image_path in model")
|
|
break
|
|
|
|
# Save back to JSON
|
|
await persistence.save_storyboard(storyboard_id, storyboard_model)
|
|
logger.info(f"[PERSIST] Saved storyboard to JSON for {storyboard_id}")
|
|
except Exception as pe:
|
|
logger.warning(f"[PERSIST] Failed to persist storyboard: {pe}")
|
|
|
|
logger.info(f"Regenerated image for frame {frame_id} via RunningHub")
|
|
|
|
return RegenerateImageResponse(
|
|
image_path=target_frame["image_path"],
|
|
success=True
|
|
)
|
|
else:
|
|
raise HTTPException(status_code=500, detail="Image generation failed")
|
|
|
|
except ImportError as e:
|
|
logger.error(f"Failed to import dependencies: {e}")
|
|
raise HTTPException(status_code=500, detail="Image generation service not available")
|
|
except Exception as e:
|
|
logger.error(f"Image regeneration failed: {e}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.post(
|
|
"/storyboard/{storyboard_id}/frames/{frame_id}/regenerate-audio",
|
|
response_model=RegenerateAudioResponse
|
|
)
|
|
async def regenerate_frame_audio(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
frame_id: str = Path(..., description="Frame ID"),
|
|
request: RegenerateAudioRequest = None
|
|
):
|
|
"""
|
|
Regenerate audio for a frame
|
|
|
|
Uses the frame's narration (or override) to generate new audio via TTS.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Find frame
|
|
target_frame = None
|
|
frame_index = 0
|
|
for i, frame in enumerate(frames):
|
|
if frame["id"] == frame_id:
|
|
target_frame = frame
|
|
frame_index = i
|
|
break
|
|
|
|
if not target_frame:
|
|
raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found")
|
|
|
|
# Get narration to use
|
|
narration = request.narration if request and request.narration else target_frame.get("narration", "")
|
|
|
|
if not narration:
|
|
raise HTTPException(status_code=400, detail="No narration text available")
|
|
|
|
try:
|
|
from api.dependencies import get_pixelle_video
|
|
import os
|
|
|
|
pixelle_video = await get_pixelle_video()
|
|
|
|
# Create output path
|
|
output_dir = f"output/{storyboard_id}"
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
audio_path = f"{output_dir}/frame_{frame_index}_audio_regenerated.mp3"
|
|
|
|
# Generate audio using TTS service
|
|
voice = request.voice if request and request.voice else None
|
|
|
|
result_path = await pixelle_video.tts(
|
|
text=narration,
|
|
voice=voice,
|
|
output_path=audio_path
|
|
)
|
|
|
|
# Get audio duration
|
|
from mutagen.mp3 import MP3
|
|
try:
|
|
audio = MP3(result_path)
|
|
duration = audio.info.length
|
|
except:
|
|
duration = 3.0 # Default duration
|
|
|
|
# Update frame
|
|
target_frame["audio_path"] = _path_to_url(result_path)
|
|
target_frame["duration"] = duration
|
|
|
|
# Recalculate total duration
|
|
storyboard["total_duration"] = sum(f.get("duration", 3.0) for f in frames)
|
|
_storyboard_cache[storyboard_id] = storyboard
|
|
|
|
# Persist changes to storyboard.json
|
|
try:
|
|
from pixelle_video.services.persistence import PersistenceService
|
|
persistence = PersistenceService()
|
|
|
|
# Load existing storyboard model
|
|
storyboard_model = await persistence.load_storyboard(storyboard_id)
|
|
if storyboard_model:
|
|
# Update the specific frame's audio_path and duration
|
|
for frame in storyboard_model.frames:
|
|
if f"frame-{frame.index}" == frame_id:
|
|
frame.audio_path = result_path
|
|
frame.duration = duration
|
|
logger.debug(f"[PERSIST] Updated frame {frame_id} audio_path in model")
|
|
break
|
|
|
|
# Update total duration
|
|
storyboard_model.total_duration = sum(f.duration or 3.0 for f in storyboard_model.frames)
|
|
|
|
# Save back to JSON
|
|
await persistence.save_storyboard(storyboard_id, storyboard_model)
|
|
logger.info(f"[PERSIST] Saved storyboard to JSON for {storyboard_id}")
|
|
except Exception as pe:
|
|
logger.warning(f"[PERSIST] Failed to persist storyboard: {pe}")
|
|
|
|
logger.info(f"Regenerated audio for frame {frame_id}, duration: {duration}s")
|
|
|
|
return RegenerateAudioResponse(
|
|
audio_path=target_frame["audio_path"],
|
|
duration=duration,
|
|
success=True
|
|
)
|
|
|
|
except ImportError as e:
|
|
logger.error(f"Failed to import dependencies: {e}")
|
|
raise HTTPException(status_code=500, detail="TTS service not available")
|
|
except Exception as e:
|
|
logger.error(f"Audio regeneration failed: {e}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
@router.post(
|
|
"/storyboard/{storyboard_id}/frames/{frame_id}/align-prompt",
|
|
response_model=AlignPromptResponse
|
|
)
|
|
async def align_frame_prompt(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
frame_id: str = Path(..., description="Frame ID"),
|
|
request: AlignPromptRequest = None
|
|
):
|
|
"""
|
|
Align image prompt with narration
|
|
|
|
Regenerates the image prompt based on the frame's narration using
|
|
enhanced core imagery extraction for better semantic relevance.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Find frame
|
|
target_frame = None
|
|
for frame in frames:
|
|
if frame["id"] == frame_id:
|
|
target_frame = frame
|
|
break
|
|
|
|
if not target_frame:
|
|
raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found")
|
|
|
|
# Get narration to use
|
|
narration = request.narration if request and request.narration else target_frame.get("narration", "")
|
|
|
|
if not narration:
|
|
raise HTTPException(status_code=400, detail="No narration text available")
|
|
|
|
try:
|
|
from api.dependencies import get_pixelle_video
|
|
|
|
pixelle_video = await get_pixelle_video()
|
|
|
|
# Use LLM to generate aligned image prompt
|
|
from pixelle_video.prompts import build_image_prompt_prompt
|
|
|
|
prompt = build_image_prompt_prompt(
|
|
narrations=[narration],
|
|
min_words=30,
|
|
max_words=60
|
|
)
|
|
|
|
response = await pixelle_video.llm(
|
|
prompt=prompt,
|
|
temperature=0.7,
|
|
max_tokens=500
|
|
)
|
|
|
|
# Parse response
|
|
import json
|
|
import re
|
|
|
|
# Try to extract JSON
|
|
try:
|
|
result = json.loads(response)
|
|
except json.JSONDecodeError:
|
|
# Try markdown code block
|
|
match = re.search(r'```(?:json)?\s*([\s\S]+?)\s*```', response)
|
|
if match:
|
|
result = json.loads(match.group(1))
|
|
else:
|
|
raise ValueError("Failed to parse LLM response")
|
|
|
|
if "image_prompts" not in result or len(result["image_prompts"]) == 0:
|
|
raise ValueError("No image prompts in response")
|
|
|
|
new_prompt = result["image_prompts"][0]
|
|
|
|
# Update frame
|
|
target_frame["image_prompt"] = new_prompt
|
|
_storyboard_cache[storyboard_id] = storyboard
|
|
|
|
logger.info(f"Aligned image prompt for frame {frame_id}")
|
|
|
|
return AlignPromptResponse(
|
|
image_prompt=new_prompt,
|
|
success=True
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Prompt alignment failed: {e}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
@router.post(
|
|
"/storyboard/{storyboard_id}/frames/{frame_id}/inpaint",
|
|
response_model=InpaintResponse
|
|
)
|
|
async def inpaint_frame_image(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
frame_id: str = Path(..., description="Frame ID"),
|
|
request: InpaintRequest = None
|
|
):
|
|
"""
|
|
Inpaint (局部重绘) frame image
|
|
|
|
Uses mask to selectively regenerate parts of the image.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Find frame
|
|
target_frame = None
|
|
frame_index = 0
|
|
for i, frame in enumerate(frames):
|
|
if frame["id"] == frame_id:
|
|
target_frame = frame
|
|
frame_index = i
|
|
break
|
|
|
|
if not target_frame:
|
|
raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found")
|
|
|
|
# Get original image path
|
|
original_image = target_frame.get("image_path")
|
|
if not original_image:
|
|
raise HTTPException(status_code=400, detail="No image to inpaint")
|
|
|
|
if not request or not request.mask:
|
|
raise HTTPException(status_code=400, detail="Mask is required")
|
|
|
|
try:
|
|
from api.dependencies import get_pixelle_video
|
|
import base64
|
|
import tempfile
|
|
import os
|
|
|
|
pixelle_video = await get_pixelle_video()
|
|
|
|
# Save mask to temp file
|
|
mask_data = base64.b64decode(request.mask)
|
|
output_dir = f"output/{storyboard_id}"
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
mask_path = f"{output_dir}/mask_{frame_index}.png"
|
|
with open(mask_path, 'wb') as f:
|
|
f.write(mask_data)
|
|
|
|
# Get prompt
|
|
prompt = request.prompt or target_frame.get("image_prompt", "")
|
|
|
|
# Call inpaint service
|
|
# Convert URL back to file path (URL format: http://localhost:8000/api/files/{relative_path})
|
|
image_file_path = original_image
|
|
if "/api/files/" in original_image:
|
|
image_file_path = "output/" + original_image.split("/api/files/")[-1]
|
|
|
|
result = await pixelle_video.media.inpaint(
|
|
image_path=image_file_path,
|
|
mask_path=mask_path,
|
|
prompt=prompt,
|
|
denoise_strength=request.denoise_strength,
|
|
)
|
|
|
|
if result and result.url:
|
|
# Save inpainted image
|
|
import aiohttp
|
|
image_path = f"{output_dir}/frame_{frame_index}_inpainted.png"
|
|
|
|
async with aiohttp.ClientSession() as session:
|
|
async with session.get(result.url) as resp:
|
|
if resp.status == 200:
|
|
with open(image_path, 'wb') as f:
|
|
f.write(await resp.read())
|
|
|
|
# Update frame
|
|
target_frame["image_path"] = _path_to_url(image_path)
|
|
_storyboard_cache[storyboard_id] = storyboard
|
|
|
|
logger.info(f"Inpainted image for frame {frame_id}")
|
|
|
|
return InpaintResponse(
|
|
image_path=target_frame["image_path"],
|
|
success=True
|
|
)
|
|
else:
|
|
raise HTTPException(status_code=500, detail="Inpainting failed")
|
|
|
|
except ImportError as e:
|
|
logger.error(f"Failed to import dependencies: {e}")
|
|
raise HTTPException(status_code=500, detail="Inpainting service not available")
|
|
except Exception as e:
|
|
logger.error(f"Inpainting failed: {e}")
|
|
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
|
|
# ============================================================
|
|
# Video Export Endpoints
|
|
# ============================================================
|
|
|
|
@router.post(
|
|
"/storyboard/{storyboard_id}/export",
|
|
response_model=ExportResponse
|
|
)
|
|
async def export_video(
|
|
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
|
request: ExportRequest = None,
|
|
background_tasks: BackgroundTasks = None
|
|
):
|
|
"""
|
|
Export edited video
|
|
|
|
Concatenates all video segments in order and optionally adds BGM.
|
|
Returns immediately with task_id for tracking progress.
|
|
"""
|
|
if storyboard_id not in _storyboard_cache:
|
|
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found")
|
|
|
|
storyboard = _storyboard_cache[storyboard_id]
|
|
frames = storyboard["frames"]
|
|
|
|
# Check if all frames have video segments
|
|
missing_segments = [f["id"] for f in frames if not f.get("video_segment_path")]
|
|
if missing_segments:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail=f"Missing video segments for frames: {missing_segments}"
|
|
)
|
|
|
|
# Create export task
|
|
task_id = str(uuid_module.uuid4())
|
|
_export_tasks[task_id] = {
|
|
"status": "pending",
|
|
"progress": 0.0,
|
|
"video_path": None,
|
|
"error": None
|
|
}
|
|
|
|
# Run export in background
|
|
async def run_export():
|
|
try:
|
|
_export_tasks[task_id]["status"] = "processing"
|
|
_export_tasks[task_id]["progress"] = 0.1
|
|
|
|
from pixelle_video.services.video import VideoService
|
|
import os
|
|
|
|
video_service = VideoService()
|
|
|
|
# Get video segment paths (sorted by order)
|
|
sorted_frames = sorted(frames, key=lambda f: f.get("order", 0))
|
|
|
|
# Convert URLs back to file paths
|
|
video_segments = []
|
|
for frame in sorted_frames:
|
|
path = frame.get("video_segment_path", "")
|
|
if path.startswith("http"):
|
|
# Extract path from URL
|
|
path = path.replace("http://localhost:8000/api/files/", "output/")
|
|
video_segments.append(path)
|
|
|
|
_export_tasks[task_id]["progress"] = 0.3
|
|
|
|
# Create output directory
|
|
output_dir = f"output/{storyboard_id}"
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
|
|
# Output path
|
|
output_path = f"{output_dir}/exported_{task_id[:8]}.mp4"
|
|
|
|
# BGM settings
|
|
bgm_path = request.bgm_path if request else None
|
|
bgm_volume = request.bgm_volume if request else 0.2
|
|
|
|
_export_tasks[task_id]["progress"] = 0.5
|
|
|
|
# Concatenate videos
|
|
video_service.concat_videos(
|
|
videos=video_segments,
|
|
output=output_path,
|
|
method="demuxer",
|
|
bgm_path=bgm_path,
|
|
bgm_volume=bgm_volume
|
|
)
|
|
|
|
_export_tasks[task_id]["progress"] = 1.0
|
|
_export_tasks[task_id]["status"] = "completed"
|
|
_export_tasks[task_id]["video_path"] = output_path
|
|
|
|
logger.info(f"Exported video for storyboard {storyboard_id}: {output_path}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Export failed: {e}")
|
|
_export_tasks[task_id]["status"] = "failed"
|
|
_export_tasks[task_id]["error"] = str(e)
|
|
|
|
background_tasks.add_task(run_export)
|
|
|
|
return ExportResponse(
|
|
task_id=task_id,
|
|
status="pending"
|
|
)
|
|
|
|
|
|
@router.get(
|
|
"/export/{task_id}/status",
|
|
response_model=ExportStatusResponse
|
|
)
|
|
async def get_export_status(task_id: str = Path(..., description="Export task ID")):
|
|
"""
|
|
Get export task status and progress
|
|
"""
|
|
if task_id not in _export_tasks:
|
|
raise HTTPException(status_code=404, detail=f"Export task {task_id} not found")
|
|
|
|
task = _export_tasks[task_id]
|
|
|
|
download_url = None
|
|
if task["video_path"]:
|
|
download_url = _path_to_url(task["video_path"])
|
|
|
|
return ExportStatusResponse(
|
|
task_id=task_id,
|
|
status=task["status"],
|
|
progress=task["progress"],
|
|
video_path=task["video_path"],
|
|
download_url=download_url,
|
|
error=task["error"]
|
|
)
|