feat: Add inpainting (局部重绘) feature for timeline editor
- Add canvas-based mask drawing tools (brush, eraser, rect, lasso) - Add undo/redo history support for mask editing - Integrate inpainting UI into preview player - Add backend API endpoint for inpainting requests - Add MediaService.inpaint method with ComfyUI workflow support - Add Flux inpainting workflows for selfhost and RunningHub 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -36,6 +36,8 @@ from api.schemas.editor import (
|
||||
RegenerateImageResponse,
|
||||
RegenerateAudioRequest,
|
||||
RegenerateAudioResponse,
|
||||
InpaintRequest,
|
||||
InpaintResponse,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/editor", tags=["Editor"])
|
||||
@@ -577,3 +579,107 @@ async def regenerate_frame_audio(
|
||||
except Exception as e:
|
||||
logger.error(f"Audio regeneration failed: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post(
|
||||
"/storyboard/{storyboard_id}/frames/{frame_id}/inpaint",
|
||||
response_model=InpaintResponse
|
||||
)
|
||||
async def inpaint_frame_image(
|
||||
storyboard_id: str = Path(..., description="Storyboard/task ID"),
|
||||
frame_id: str = Path(..., description="Frame ID"),
|
||||
request: InpaintRequest = None
|
||||
):
|
||||
"""
|
||||
Inpaint (局部重绘) frame image
|
||||
|
||||
Uses mask to selectively regenerate parts of the image.
|
||||
"""
|
||||
if storyboard_id not in _storyboard_cache:
|
||||
raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found")
|
||||
|
||||
storyboard = _storyboard_cache[storyboard_id]
|
||||
frames = storyboard["frames"]
|
||||
|
||||
# Find frame
|
||||
target_frame = None
|
||||
frame_index = 0
|
||||
for i, frame in enumerate(frames):
|
||||
if frame["id"] == frame_id:
|
||||
target_frame = frame
|
||||
frame_index = i
|
||||
break
|
||||
|
||||
if not target_frame:
|
||||
raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found")
|
||||
|
||||
# Get original image path
|
||||
original_image = target_frame.get("image_path")
|
||||
if not original_image:
|
||||
raise HTTPException(status_code=400, detail="No image to inpaint")
|
||||
|
||||
if not request or not request.mask:
|
||||
raise HTTPException(status_code=400, detail="Mask is required")
|
||||
|
||||
try:
|
||||
from api.dependencies import get_pixelle_video
|
||||
import base64
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
pixelle_video = await get_pixelle_video()
|
||||
|
||||
# Save mask to temp file
|
||||
mask_data = base64.b64decode(request.mask)
|
||||
output_dir = f"output/{storyboard_id}"
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
mask_path = f"{output_dir}/mask_{frame_index}.png"
|
||||
with open(mask_path, 'wb') as f:
|
||||
f.write(mask_data)
|
||||
|
||||
# Get prompt
|
||||
prompt = request.prompt or target_frame.get("image_prompt", "")
|
||||
|
||||
# Call inpaint service
|
||||
# Convert URL back to file path (URL format: http://localhost:8000/api/files/{relative_path})
|
||||
image_file_path = original_image
|
||||
if "/api/files/" in original_image:
|
||||
image_file_path = "output/" + original_image.split("/api/files/")[-1]
|
||||
|
||||
result = await pixelle_video.media.inpaint(
|
||||
image_path=image_file_path,
|
||||
mask_path=mask_path,
|
||||
prompt=prompt,
|
||||
denoise_strength=request.denoise_strength,
|
||||
)
|
||||
|
||||
if result and result.url:
|
||||
# Save inpainted image
|
||||
import aiohttp
|
||||
image_path = f"{output_dir}/frame_{frame_index}_inpainted.png"
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(result.url) as resp:
|
||||
if resp.status == 200:
|
||||
with open(image_path, 'wb') as f:
|
||||
f.write(await resp.read())
|
||||
|
||||
# Update frame
|
||||
target_frame["image_path"] = _path_to_url(image_path)
|
||||
_storyboard_cache[storyboard_id] = storyboard
|
||||
|
||||
logger.info(f"Inpainted image for frame {frame_id}")
|
||||
|
||||
return InpaintResponse(
|
||||
image_path=target_frame["image_path"],
|
||||
success=True
|
||||
)
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail="Inpainting failed")
|
||||
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import dependencies: {e}")
|
||||
raise HTTPException(status_code=500, detail="Inpainting service not available")
|
||||
except Exception as e:
|
||||
logger.error(f"Inpainting failed: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
@@ -108,3 +108,16 @@ class RegenerateAudioResponse(BaseModel):
|
||||
duration: float
|
||||
success: bool = True
|
||||
|
||||
|
||||
class InpaintRequest(BaseModel):
|
||||
"""Request to inpaint (局部重绘) frame image"""
|
||||
mask: str = Field(..., description="Base64 encoded mask image (white=inpaint, black=keep)")
|
||||
prompt: Optional[str] = Field(None, description="Optional prompt for inpainted region")
|
||||
denoise_strength: float = Field(0.8, ge=0.0, le=1.0, description="Denoise strength (0.0-1.0)")
|
||||
|
||||
|
||||
class InpaintResponse(BaseModel):
|
||||
"""Response after inpainting"""
|
||||
image_path: str
|
||||
success: bool = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user