From 79a6c2ef3e1ad8dbe6c0804beedcbba521c56cca Mon Sep 17 00:00:00 2001 From: empty Date: Mon, 5 Jan 2026 23:44:51 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20Add=20inpainting=20(=E5=B1=80=E9=83=A8?= =?UTF-8?q?=E9=87=8D=E7=BB=98)=20feature=20for=20timeline=20editor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add canvas-based mask drawing tools (brush, eraser, rect, lasso) - Add undo/redo history support for mask editing - Integrate inpainting UI into preview player - Add backend API endpoint for inpainting requests - Add MediaService.inpaint method with ComfyUI workflow support - Add Flux inpainting workflows for selfhost and RunningHub 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- api/routers/editor.py | 106 ++++++ api/schemas/editor.py | 13 + .../inpainting/hooks/use-canvas-history.ts | 65 ++++ frontend/src/components/inpainting/index.ts | 8 + .../inpainting/inpainting-canvas.tsx | 340 ++++++++++++++++++ .../inpainting/inpainting-toolbar.tsx | 115 ++++++ .../components/inpainting/tools/brush-tool.ts | 69 ++++ .../inpainting/tools/eraser-tool.ts | 69 ++++ .../src/components/inpainting/tools/index.ts | 10 + .../components/inpainting/tools/lasso-tool.ts | 107 ++++++ .../components/inpainting/tools/rect-tool.ts | 93 +++++ .../src/components/preview/preview-player.tsx | 152 +++++++- frontend/src/services/editor-api.ts | 36 ++ frontend/src/stores/editor-store.ts | 39 ++ pixelle_video/services/media.py | 73 ++++ workflows/runninghub/image_inpaint.json | 5 + workflows/selfhost/image_inpaint_flux.json | 149 ++++++++ 17 files changed, 1444 insertions(+), 5 deletions(-) create mode 100644 frontend/src/components/inpainting/hooks/use-canvas-history.ts create mode 100644 frontend/src/components/inpainting/index.ts create mode 100644 frontend/src/components/inpainting/inpainting-canvas.tsx create mode 100644 frontend/src/components/inpainting/inpainting-toolbar.tsx create mode 100644 frontend/src/components/inpainting/tools/brush-tool.ts create mode 100644 frontend/src/components/inpainting/tools/eraser-tool.ts create mode 100644 frontend/src/components/inpainting/tools/index.ts create mode 100644 frontend/src/components/inpainting/tools/lasso-tool.ts create mode 100644 frontend/src/components/inpainting/tools/rect-tool.ts create mode 100644 workflows/runninghub/image_inpaint.json create mode 100644 workflows/selfhost/image_inpaint_flux.json diff --git a/api/routers/editor.py b/api/routers/editor.py index 7147b49..8de29c8 100644 --- a/api/routers/editor.py +++ b/api/routers/editor.py @@ -36,6 +36,8 @@ from api.schemas.editor import ( RegenerateImageResponse, RegenerateAudioRequest, RegenerateAudioResponse, + InpaintRequest, + InpaintResponse, ) router = APIRouter(prefix="/editor", tags=["Editor"]) @@ -577,3 +579,107 @@ async def regenerate_frame_audio( except Exception as e: logger.error(f"Audio regeneration failed: {e}") raise HTTPException(status_code=500, detail=str(e)) + + +@router.post( + "/storyboard/{storyboard_id}/frames/{frame_id}/inpaint", + response_model=InpaintResponse +) +async def inpaint_frame_image( + storyboard_id: str = Path(..., description="Storyboard/task ID"), + frame_id: str = Path(..., description="Frame ID"), + request: InpaintRequest = None +): + """ + Inpaint (局部重绘) frame image + + Uses mask to selectively regenerate parts of the image. + """ + if storyboard_id not in _storyboard_cache: + raise HTTPException(status_code=404, detail=f"Storyboard {storyboard_id} not found") + + storyboard = _storyboard_cache[storyboard_id] + frames = storyboard["frames"] + + # Find frame + target_frame = None + frame_index = 0 + for i, frame in enumerate(frames): + if frame["id"] == frame_id: + target_frame = frame + frame_index = i + break + + if not target_frame: + raise HTTPException(status_code=404, detail=f"Frame {frame_id} not found") + + # Get original image path + original_image = target_frame.get("image_path") + if not original_image: + raise HTTPException(status_code=400, detail="No image to inpaint") + + if not request or not request.mask: + raise HTTPException(status_code=400, detail="Mask is required") + + try: + from api.dependencies import get_pixelle_video + import base64 + import tempfile + import os + + pixelle_video = await get_pixelle_video() + + # Save mask to temp file + mask_data = base64.b64decode(request.mask) + output_dir = f"output/{storyboard_id}" + os.makedirs(output_dir, exist_ok=True) + mask_path = f"{output_dir}/mask_{frame_index}.png" + with open(mask_path, 'wb') as f: + f.write(mask_data) + + # Get prompt + prompt = request.prompt or target_frame.get("image_prompt", "") + + # Call inpaint service + # Convert URL back to file path (URL format: http://localhost:8000/api/files/{relative_path}) + image_file_path = original_image + if "/api/files/" in original_image: + image_file_path = "output/" + original_image.split("/api/files/")[-1] + + result = await pixelle_video.media.inpaint( + image_path=image_file_path, + mask_path=mask_path, + prompt=prompt, + denoise_strength=request.denoise_strength, + ) + + if result and result.url: + # Save inpainted image + import aiohttp + image_path = f"{output_dir}/frame_{frame_index}_inpainted.png" + + async with aiohttp.ClientSession() as session: + async with session.get(result.url) as resp: + if resp.status == 200: + with open(image_path, 'wb') as f: + f.write(await resp.read()) + + # Update frame + target_frame["image_path"] = _path_to_url(image_path) + _storyboard_cache[storyboard_id] = storyboard + + logger.info(f"Inpainted image for frame {frame_id}") + + return InpaintResponse( + image_path=target_frame["image_path"], + success=True + ) + else: + raise HTTPException(status_code=500, detail="Inpainting failed") + + except ImportError as e: + logger.error(f"Failed to import dependencies: {e}") + raise HTTPException(status_code=500, detail="Inpainting service not available") + except Exception as e: + logger.error(f"Inpainting failed: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/api/schemas/editor.py b/api/schemas/editor.py index f1cf35c..c5bfae8 100644 --- a/api/schemas/editor.py +++ b/api/schemas/editor.py @@ -108,3 +108,16 @@ class RegenerateAudioResponse(BaseModel): duration: float success: bool = True + +class InpaintRequest(BaseModel): + """Request to inpaint (局部重绘) frame image""" + mask: str = Field(..., description="Base64 encoded mask image (white=inpaint, black=keep)") + prompt: Optional[str] = Field(None, description="Optional prompt for inpainted region") + denoise_strength: float = Field(0.8, ge=0.0, le=1.0, description="Denoise strength (0.0-1.0)") + + +class InpaintResponse(BaseModel): + """Response after inpainting""" + image_path: str + success: bool = True + diff --git a/frontend/src/components/inpainting/hooks/use-canvas-history.ts b/frontend/src/components/inpainting/hooks/use-canvas-history.ts new file mode 100644 index 0000000..635cbec --- /dev/null +++ b/frontend/src/components/inpainting/hooks/use-canvas-history.ts @@ -0,0 +1,65 @@ +/** + * useCanvasHistory - Undo/Redo hook for canvas operations + */ + +import { useState, useCallback } from 'react' + +interface UseCanvasHistoryOptions { + maxHistory?: number +} + +export function useCanvasHistory(options: UseCanvasHistoryOptions = {}) { + const { maxHistory = 50 } = options + + const [history, setHistory] = useState([]) + const [historyIndex, setHistoryIndex] = useState(-1) + + // Save current state to history + const saveState = useCallback((dataUrl: string) => { + setHistory(prev => { + // Remove any future states if we're not at the end + const newHistory = prev.slice(0, historyIndex + 1) + newHistory.push(dataUrl) + + // Limit history size + if (newHistory.length > maxHistory) { + newHistory.shift() + return newHistory + } + return newHistory + }) + setHistoryIndex(prev => Math.min(prev + 1, maxHistory - 1)) + }, [historyIndex, maxHistory]) + + // Undo - return previous state + const undo = useCallback((): string | null => { + if (historyIndex <= 0) return null + const newIndex = historyIndex - 1 + setHistoryIndex(newIndex) + return history[newIndex] + }, [history, historyIndex]) + + // Redo - return next state + const redo = useCallback((): string | null => { + if (historyIndex >= history.length - 1) return null + const newIndex = historyIndex + 1 + setHistoryIndex(newIndex) + return history[newIndex] + }, [history, historyIndex]) + + // Clear history + const clearHistory = useCallback(() => { + setHistory([]) + setHistoryIndex(-1) + }, []) + + return { + saveState, + undo, + redo, + clearHistory, + canUndo: historyIndex > 0, + canRedo: historyIndex < history.length - 1, + historyLength: history.length, + } +} diff --git a/frontend/src/components/inpainting/index.ts b/frontend/src/components/inpainting/index.ts new file mode 100644 index 0000000..5589e2e --- /dev/null +++ b/frontend/src/components/inpainting/index.ts @@ -0,0 +1,8 @@ +/** + * Inpainting Components - Export all components + */ + +export { InpaintingCanvas, type InpaintingCanvasProps, type InpaintingCanvasRef } from './inpainting-canvas' +export { InpaintingToolbar, type InpaintingToolbarProps } from './inpainting-toolbar' +export { useCanvasHistory } from './hooks/use-canvas-history' +export * from './tools' diff --git a/frontend/src/components/inpainting/inpainting-canvas.tsx b/frontend/src/components/inpainting/inpainting-canvas.tsx new file mode 100644 index 0000000..42cd74b --- /dev/null +++ b/frontend/src/components/inpainting/inpainting-canvas.tsx @@ -0,0 +1,340 @@ +'use client' + +/** + * InpaintingCanvas - Core canvas component for mask drawing + */ + +import { useRef, useEffect, useCallback, useState, forwardRef, useImperativeHandle } from 'react' +import { BrushTool, EraserTool, RectTool, LassoTool, type ToolType, type Point } from './tools' +import { useCanvasHistory } from './hooks/use-canvas-history' + +export interface InpaintingCanvasProps { + imageSrc: string + width: number + height: number + activeTool: ToolType + brushSize: number + onMaskChange?: (maskDataUrl: string | null) => void +} + +export interface InpaintingCanvasRef { + undo: () => void + redo: () => void + clear: () => void + exportMask: () => string | null + canUndo: boolean + canRedo: boolean +} + +const MASK_COLOR = 'rgba(255, 0, 0, 0.5)' + +export const InpaintingCanvas = forwardRef(function InpaintingCanvas({ + imageSrc, + width, + height, + activeTool, + brushSize, + onMaskChange, +}, ref) { + // Canvas refs + const containerRef = useRef(null) + const imageCanvasRef = useRef(null) + const maskCanvasRef = useRef(null) + const previewCanvasRef = useRef(null) + + // Tool refs + const brushToolRef = useRef(null) + const eraserToolRef = useRef(null) + const rectToolRef = useRef(null) + const lassoToolRef = useRef(null) + + // History + const { saveState, undo, redo, clearHistory, canUndo, canRedo } = useCanvasHistory() + + // State + const [isLoaded, setIsLoaded] = useState(false) + + // Initialize tools when mask canvas is ready + useEffect(() => { + const maskCanvas = maskCanvasRef.current + const previewCanvas = previewCanvasRef.current + if (!maskCanvas || !previewCanvas) return + + const ctx = maskCanvas.getContext('2d') + if (!ctx) return + + brushToolRef.current = new BrushTool(ctx, { size: brushSize, color: MASK_COLOR }) + eraserToolRef.current = new EraserTool(ctx, { size: brushSize }) + rectToolRef.current = new RectTool(ctx, { color: MASK_COLOR }) + lassoToolRef.current = new LassoTool(ctx, { color: MASK_COLOR }) + + rectToolRef.current.setPreviewCanvas(previewCanvas) + lassoToolRef.current.setPreviewCanvas(previewCanvas) + }, [brushSize]) + + // Update brush size + useEffect(() => { + brushToolRef.current?.setSize(brushSize) + eraserToolRef.current?.setSize(brushSize) + }, [brushSize]) + + // Load image + useEffect(() => { + const imageCanvas = imageCanvasRef.current + if (!imageCanvas || !imageSrc) return + + const ctx = imageCanvas.getContext('2d') + if (!ctx) return + + setIsLoaded(false) + const img = new Image() + // Only set crossOrigin for external URLs + if (!imageSrc.startsWith('data:') && !imageSrc.includes('localhost')) { + img.crossOrigin = 'anonymous' + } + img.onload = () => { + ctx.clearRect(0, 0, width, height) + ctx.drawImage(img, 0, 0, width, height) + setIsLoaded(true) + } + img.onerror = (e) => { + console.error('Failed to load image:', e) + // Still mark as loaded to show canvas (user can see error) + setIsLoaded(true) + } + img.src = imageSrc + }, [imageSrc, width, height]) + + // Get mouse position relative to canvas + const getCanvasPoint = useCallback((e: React.MouseEvent | React.TouchEvent): Point => { + const canvas = maskCanvasRef.current + if (!canvas) return { x: 0, y: 0 } + + const rect = canvas.getBoundingClientRect() + const scaleX = canvas.width / rect.width + const scaleY = canvas.height / rect.height + + if ('touches' in e) { + const touch = e.touches[0] + return { + x: (touch.clientX - rect.left) * scaleX, + y: (touch.clientY - rect.top) * scaleY, + } + } + + return { + x: (e.clientX - rect.left) * scaleX, + y: (e.clientY - rect.top) * scaleY, + } + }, []) + + // Save mask state after drawing + const saveMaskState = useCallback(() => { + const maskCanvas = maskCanvasRef.current + if (!maskCanvas) return + + const dataUrl = maskCanvas.toDataURL('image/png') + saveState(dataUrl) + onMaskChange?.(dataUrl) + }, [saveState, onMaskChange]) + + // Mouse/Touch handlers + const handlePointerDown = useCallback((e: React.MouseEvent | React.TouchEvent) => { + const point = getCanvasPoint(e) + + switch (activeTool) { + case 'brush': + brushToolRef.current?.startStroke(point) + break + case 'eraser': + eraserToolRef.current?.startErase(point) + break + case 'rect': + rectToolRef.current?.startRect(point) + break + case 'lasso': + lassoToolRef.current?.addPoint(point) + break + } + }, [activeTool, getCanvasPoint]) + + const handlePointerMove = useCallback((e: React.MouseEvent | React.TouchEvent) => { + const point = getCanvasPoint(e) + + switch (activeTool) { + case 'brush': + brushToolRef.current?.continueStroke(point) + break + case 'eraser': + eraserToolRef.current?.continueErase(point) + break + case 'rect': + rectToolRef.current?.drawPreview(point) + break + } + }, [activeTool, getCanvasPoint]) + + const handlePointerUp = useCallback((e: React.MouseEvent | React.TouchEvent) => { + const point = getCanvasPoint(e) + + switch (activeTool) { + case 'brush': + brushToolRef.current?.endStroke() + saveMaskState() + break + case 'eraser': + eraserToolRef.current?.endErase() + saveMaskState() + break + case 'rect': + rectToolRef.current?.endRect(point) + saveMaskState() + break + } + }, [activeTool, getCanvasPoint, saveMaskState]) + + // Double click to close lasso + const handleDoubleClick = useCallback(() => { + if (activeTool === 'lasso') { + lassoToolRef.current?.closePath() + saveMaskState() + } + }, [activeTool, saveMaskState]) + + // Undo/Redo handlers + const handleUndo = useCallback(() => { + const prevState = undo() + if (prevState && maskCanvasRef.current) { + const ctx = maskCanvasRef.current.getContext('2d') + if (!ctx) return + + const img = new Image() + img.onload = () => { + ctx.clearRect(0, 0, width, height) + ctx.drawImage(img, 0, 0) + onMaskChange?.(prevState) + } + img.src = prevState + } + }, [undo, width, height, onMaskChange]) + + const handleRedo = useCallback(() => { + const nextState = redo() + if (nextState && maskCanvasRef.current) { + const ctx = maskCanvasRef.current.getContext('2d') + if (!ctx) return + + const img = new Image() + img.onload = () => { + ctx.clearRect(0, 0, width, height) + ctx.drawImage(img, 0, 0) + onMaskChange?.(nextState) + } + img.src = nextState + } + }, [redo, width, height, onMaskChange]) + + // Clear mask + const handleClear = useCallback(() => { + const maskCanvas = maskCanvasRef.current + if (!maskCanvas) return + + const ctx = maskCanvas.getContext('2d') + if (!ctx) return + + ctx.clearRect(0, 0, width, height) + clearHistory() + onMaskChange?.(null) + }, [width, height, clearHistory, onMaskChange]) + + // Export mask as black/white base64 + const exportMask = useCallback((): string | null => { + const maskCanvas = maskCanvasRef.current + if (!maskCanvas) return null + + const ctx = maskCanvas.getContext('2d') + if (!ctx) return null + + // Create temp canvas for black/white conversion + const tempCanvas = document.createElement('canvas') + tempCanvas.width = width + tempCanvas.height = height + const tempCtx = tempCanvas.getContext('2d') + if (!tempCtx) return null + + // Get mask data + const imageData = ctx.getImageData(0, 0, width, height) + const data = imageData.data + + // Convert to black/white (alpha > 0 = white, else black) + for (let i = 0; i < data.length; i += 4) { + const alpha = data[i + 3] + const value = alpha > 0 ? 255 : 0 + data[i] = value // R + data[i + 1] = value // G + data[i + 2] = value // B + data[i + 3] = 255 // A + } + + tempCtx.putImageData(imageData, 0, 0) + return tempCanvas.toDataURL('image/png').split(',')[1] + }, [width, height]) + + // Expose methods via ref + useImperativeHandle(ref, () => ({ + undo: handleUndo, + redo: handleRedo, + clear: handleClear, + exportMask, + canUndo, + canRedo, + }), [handleUndo, handleRedo, handleClear, exportMask, canUndo, canRedo]) + + return ( +
+ {/* Image layer */} + + + {/* Mask layer */} + + + {/* Preview layer (for rect/lasso) */} + + + {/* Loading overlay */} + {!isLoaded && ( +
+ Loading... +
+ )} +
+ ) +}) diff --git a/frontend/src/components/inpainting/inpainting-toolbar.tsx b/frontend/src/components/inpainting/inpainting-toolbar.tsx new file mode 100644 index 0000000..36e606a --- /dev/null +++ b/frontend/src/components/inpainting/inpainting-toolbar.tsx @@ -0,0 +1,115 @@ +'use client' + +/** + * InpaintingToolbar - Toolbar for inpainting tools + */ + +import { Paintbrush, Eraser, Square, Lasso, Undo2, Redo2, Trash2, X, Check } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { Slider } from '@/components/ui/slider' +import type { ToolType } from './tools' + +export interface InpaintingToolbarProps { + activeTool: ToolType + brushSize: number + canUndo: boolean + canRedo: boolean + onToolChange: (tool: ToolType) => void + onBrushSizeChange: (size: number) => void + onUndo: () => void + onRedo: () => void + onClear: () => void + onCancel: () => void + onApply: () => void + isApplying?: boolean +} + +const tools: { type: ToolType; icon: typeof Paintbrush; label: string }[] = [ + { type: 'brush', icon: Paintbrush, label: '画笔' }, + { type: 'eraser', icon: Eraser, label: '橡皮擦' }, + { type: 'rect', icon: Square, label: '矩形' }, + { type: 'lasso', icon: Lasso, label: '套索' }, +] + +export function InpaintingToolbar({ + activeTool, + brushSize, + canUndo, + canRedo, + onToolChange, + onBrushSizeChange, + onUndo, + onRedo, + onClear, + onCancel, + onApply, + isApplying = false, +}: InpaintingToolbarProps) { + return ( +
+ {/* Tool buttons */} +
+ {tools.map(({ type, icon: Icon, label }) => ( + + ))} +
+ +
+ + {/* Brush size slider */} +
+ 大小 + onBrushSizeChange(v)} + min={1} + max={100} + step={1} + className="w-20" + /> + {brushSize} +
+ +
+ + {/* Undo/Redo/Clear */} +
+ + + +
+ +
+ + {/* Cancel/Apply */} +
+ + +
+
+ ) +} diff --git a/frontend/src/components/inpainting/tools/brush-tool.ts b/frontend/src/components/inpainting/tools/brush-tool.ts new file mode 100644 index 0000000..54a5a21 --- /dev/null +++ b/frontend/src/components/inpainting/tools/brush-tool.ts @@ -0,0 +1,69 @@ +/** + * Brush Tool - Free drawing tool for mask creation + */ + +export interface Point { + x: number + y: number +} + +export interface BrushToolOptions { + size: number + color: string +} + +export class BrushTool { + private ctx: CanvasRenderingContext2D + private isDrawing: boolean = false + private lastPoint: Point | null = null + private options: BrushToolOptions + + constructor(ctx: CanvasRenderingContext2D, options: BrushToolOptions) { + this.ctx = ctx + this.options = options + } + + setSize(size: number) { + this.options.size = size + } + + setColor(color: string) { + this.options.color = color + } + + startStroke(point: Point) { + this.isDrawing = true + this.lastPoint = point + + // Draw initial dot + this.ctx.fillStyle = this.options.color + this.ctx.beginPath() + this.ctx.arc(point.x, point.y, this.options.size / 2, 0, Math.PI * 2) + this.ctx.fill() + } + + continueStroke(point: Point) { + if (!this.isDrawing || !this.lastPoint) return + + this.ctx.strokeStyle = this.options.color + this.ctx.lineWidth = this.options.size + this.ctx.lineCap = 'round' + this.ctx.lineJoin = 'round' + + this.ctx.beginPath() + this.ctx.moveTo(this.lastPoint.x, this.lastPoint.y) + this.ctx.lineTo(point.x, point.y) + this.ctx.stroke() + + this.lastPoint = point + } + + endStroke() { + this.isDrawing = false + this.lastPoint = null + } + + isActive(): boolean { + return this.isDrawing + } +} diff --git a/frontend/src/components/inpainting/tools/eraser-tool.ts b/frontend/src/components/inpainting/tools/eraser-tool.ts new file mode 100644 index 0000000..b02b527 --- /dev/null +++ b/frontend/src/components/inpainting/tools/eraser-tool.ts @@ -0,0 +1,69 @@ +/** + * Eraser Tool - Erase mask areas + */ + +import { Point } from './brush-tool' + +export interface EraserToolOptions { + size: number +} + +export class EraserTool { + private ctx: CanvasRenderingContext2D + private isErasing: boolean = false + private lastPoint: Point | null = null + private options: EraserToolOptions + + constructor(ctx: CanvasRenderingContext2D, options: EraserToolOptions) { + this.ctx = ctx + this.options = options + } + + setSize(size: number) { + this.options.size = size + } + + startErase(point: Point) { + this.isErasing = true + this.lastPoint = point + + // Save composite operation + const prevOp = this.ctx.globalCompositeOperation + this.ctx.globalCompositeOperation = 'destination-out' + + // Erase initial dot + this.ctx.beginPath() + this.ctx.arc(point.x, point.y, this.options.size / 2, 0, Math.PI * 2) + this.ctx.fill() + + this.ctx.globalCompositeOperation = prevOp + } + + continueErase(point: Point) { + if (!this.isErasing || !this.lastPoint) return + + const prevOp = this.ctx.globalCompositeOperation + this.ctx.globalCompositeOperation = 'destination-out' + + this.ctx.lineWidth = this.options.size + this.ctx.lineCap = 'round' + this.ctx.lineJoin = 'round' + + this.ctx.beginPath() + this.ctx.moveTo(this.lastPoint.x, this.lastPoint.y) + this.ctx.lineTo(point.x, point.y) + this.ctx.stroke() + + this.ctx.globalCompositeOperation = prevOp + this.lastPoint = point + } + + endErase() { + this.isErasing = false + this.lastPoint = null + } + + isActive(): boolean { + return this.isErasing + } +} diff --git a/frontend/src/components/inpainting/tools/index.ts b/frontend/src/components/inpainting/tools/index.ts new file mode 100644 index 0000000..271764c --- /dev/null +++ b/frontend/src/components/inpainting/tools/index.ts @@ -0,0 +1,10 @@ +/** + * Inpainting Tools - Export all tools + */ + +export { BrushTool, type Point, type BrushToolOptions } from './brush-tool' +export { EraserTool, type EraserToolOptions } from './eraser-tool' +export { RectTool, type RectToolOptions } from './rect-tool' +export { LassoTool, type LassoToolOptions } from './lasso-tool' + +export type ToolType = 'brush' | 'eraser' | 'rect' | 'lasso' diff --git a/frontend/src/components/inpainting/tools/lasso-tool.ts b/frontend/src/components/inpainting/tools/lasso-tool.ts new file mode 100644 index 0000000..118a447 --- /dev/null +++ b/frontend/src/components/inpainting/tools/lasso-tool.ts @@ -0,0 +1,107 @@ +/** + * Lasso Tool - Free polygon selection for mask + */ + +import { Point } from './brush-tool' + +export interface LassoToolOptions { + color: string +} + +export class LassoTool { + private ctx: CanvasRenderingContext2D + private isDrawing: boolean = false + private points: Point[] = [] + private options: LassoToolOptions + private previewCanvas: HTMLCanvasElement | null = null + + constructor(ctx: CanvasRenderingContext2D, options: LassoToolOptions) { + this.ctx = ctx + this.options = options + } + + setColor(color: string) { + this.options.color = color + } + + setPreviewCanvas(canvas: HTMLCanvasElement) { + this.previewCanvas = canvas + } + + // Add point on click + addPoint(point: Point) { + if (!this.isDrawing) { + this.isDrawing = true + this.points = [] + } + this.points.push(point) + this.drawPreview() + } + + // Draw preview polygon + private drawPreview() { + if (!this.previewCanvas || this.points.length === 0) return + + const previewCtx = this.previewCanvas.getContext('2d') + if (!previewCtx) return + + previewCtx.clearRect(0, 0, this.previewCanvas.width, this.previewCanvas.height) + + // Draw polygon outline + previewCtx.strokeStyle = this.options.color + previewCtx.lineWidth = 2 + previewCtx.setLineDash([5, 5]) + + previewCtx.beginPath() + previewCtx.moveTo(this.points[0].x, this.points[0].y) + for (let i = 1; i < this.points.length; i++) { + previewCtx.lineTo(this.points[i].x, this.points[i].y) + } + previewCtx.stroke() + + // Draw points + this.points.forEach((p, i) => { + previewCtx.fillStyle = i === 0 ? '#00ff00' : this.options.color + previewCtx.beginPath() + previewCtx.arc(p.x, p.y, 4, 0, Math.PI * 2) + previewCtx.fill() + }) + } + + // Close and fill polygon on double-click + closePath() { + if (!this.isDrawing || this.points.length < 3) { + this.cancel() + return + } + + // Fill polygon on mask + this.ctx.fillStyle = this.options.color + this.ctx.beginPath() + this.ctx.moveTo(this.points[0].x, this.points[0].y) + for (let i = 1; i < this.points.length; i++) { + this.ctx.lineTo(this.points[i].x, this.points[i].y) + } + this.ctx.closePath() + this.ctx.fill() + + this.cancel() + } + + cancel() { + this.isDrawing = false + this.points = [] + if (this.previewCanvas) { + const ctx = this.previewCanvas.getContext('2d') + ctx?.clearRect(0, 0, this.previewCanvas.width, this.previewCanvas.height) + } + } + + isActive(): boolean { + return this.isDrawing + } + + getPointCount(): number { + return this.points.length + } +} diff --git a/frontend/src/components/inpainting/tools/rect-tool.ts b/frontend/src/components/inpainting/tools/rect-tool.ts new file mode 100644 index 0000000..da56aac --- /dev/null +++ b/frontend/src/components/inpainting/tools/rect-tool.ts @@ -0,0 +1,93 @@ +/** + * Rectangle Tool - Draw rectangular mask areas + */ + +import { Point } from './brush-tool' + +export interface RectToolOptions { + color: string +} + +export class RectTool { + private ctx: CanvasRenderingContext2D + private isDrawing: boolean = false + private startPoint: Point | null = null + private options: RectToolOptions + private previewCanvas: HTMLCanvasElement | null = null + + constructor(ctx: CanvasRenderingContext2D, options: RectToolOptions) { + this.ctx = ctx + this.options = options + } + + setColor(color: string) { + this.options.color = color + } + + setPreviewCanvas(canvas: HTMLCanvasElement) { + this.previewCanvas = canvas + } + + startRect(point: Point) { + this.isDrawing = true + this.startPoint = point + } + + // Draw preview rectangle (on preview layer) + drawPreview(currentPoint: Point) { + if (!this.isDrawing || !this.startPoint || !this.previewCanvas) return + + const previewCtx = this.previewCanvas.getContext('2d') + if (!previewCtx) return + + // Clear preview + previewCtx.clearRect(0, 0, this.previewCanvas.width, this.previewCanvas.height) + + // Draw preview rectangle + const x = Math.min(this.startPoint.x, currentPoint.x) + const y = Math.min(this.startPoint.y, currentPoint.y) + const width = Math.abs(currentPoint.x - this.startPoint.x) + const height = Math.abs(currentPoint.y - this.startPoint.y) + + previewCtx.strokeStyle = this.options.color + previewCtx.lineWidth = 2 + previewCtx.setLineDash([5, 5]) + previewCtx.strokeRect(x, y, width, height) + } + + // Finalize rectangle (on main mask layer) + endRect(endPoint: Point) { + if (!this.isDrawing || !this.startPoint) return + + const x = Math.min(this.startPoint.x, endPoint.x) + const y = Math.min(this.startPoint.y, endPoint.y) + const width = Math.abs(endPoint.x - this.startPoint.x) + const height = Math.abs(endPoint.y - this.startPoint.y) + + // Draw filled rectangle on mask + this.ctx.fillStyle = this.options.color + this.ctx.fillRect(x, y, width, height) + + // Clear preview + if (this.previewCanvas) { + const previewCtx = this.previewCanvas.getContext('2d') + previewCtx?.clearRect(0, 0, this.previewCanvas.width, this.previewCanvas.height) + } + + this.isDrawing = false + this.startPoint = null + } + + cancel() { + this.isDrawing = false + this.startPoint = null + if (this.previewCanvas) { + const previewCtx = this.previewCanvas.getContext('2d') + previewCtx?.clearRect(0, 0, this.previewCanvas.width, this.previewCanvas.height) + } + } + + isActive(): boolean { + return this.isDrawing + } +} diff --git a/frontend/src/components/preview/preview-player.tsx b/frontend/src/components/preview/preview-player.tsx index 9279baf..e1bb322 100644 --- a/frontend/src/components/preview/preview-player.tsx +++ b/frontend/src/components/preview/preview-player.tsx @@ -1,10 +1,12 @@ 'use client' -import { useEffect, useRef, useCallback } from 'react' -import { Play, Pause, SkipBack, SkipForward, Maximize } from 'lucide-react' +import { useEffect, useRef, useCallback, useState } from 'react' +import { Play, Pause, SkipBack, SkipForward, Maximize, Paintbrush } from 'lucide-react' import { Button } from '@/components/ui/button' import { Slider } from '@/components/ui/slider' import { useEditorStore } from '@/stores/editor-store' +import { InpaintingCanvas, InpaintingToolbar, type InpaintingCanvasRef } from '@/components/inpainting' +import { editorApi } from '@/services/editor-api' export function PreviewPlayer() { const { @@ -14,10 +16,25 @@ export function PreviewPlayer() { setPlaying, setCurrentTime, selectedFrameId, - setSelectedFrameId + setSelectedFrameId, + // Inpainting state + isInpaintingMode, + inpaintingTool, + brushSize, + setInpaintingMode, + setInpaintingTool, + setBrushSize, + setCurrentMask, + resetInpainting, } = useEditorStore() const timerRef = useRef(null) + const canvasContainerRef = useRef(null) + const inpaintingCanvasRef = useRef(null) + const [canvasSize, setCanvasSize] = useState({ width: 0, height: 0 }) + const [isApplying, setIsApplying] = useState(false) + const [canUndo, setCanUndo] = useState(false) + const [canRedo, setCanRedo] = useState(false) // Get current frame based on currentTime const getCurrentFrameIndex = useCallback(() => { @@ -69,12 +86,96 @@ export function PreviewPlayer() { const selectedFrame = storyboard?.frames.find((f) => f.id === selectedFrameId) + // Calculate canvas size when entering inpainting mode + useEffect(() => { + if (isInpaintingMode && selectedFrame?.imagePath) { + const img = new Image() + img.onload = () => { + const container = canvasContainerRef.current + if (!container) return + const containerRect = container.getBoundingClientRect() + const scale = Math.min( + containerRect.width / img.width, + containerRect.height / img.height + ) + setCanvasSize({ + width: Math.floor(img.width * scale), + height: Math.floor(img.height * scale), + }) + } + img.src = selectedFrame.imagePath + } + }, [isInpaintingMode, selectedFrame?.imagePath]) + const formatTime = (seconds: number) => { const mins = Math.floor(seconds / 60) const secs = Math.floor(seconds % 60) return `${mins}:${secs.toString().padStart(2, '0')}` } + // Inpainting handlers + const handleEnterInpainting = () => { + if (selectedFrame?.imagePath) { + setPlaying(false) + setInpaintingMode(true) + } + } + + const handleCancelInpainting = () => { + resetInpainting() + } + + const handleApplyInpainting = async () => { + if (!storyboard || !selectedFrameId) return + + // Get mask from canvas ref + const maskBase64 = inpaintingCanvasRef.current?.exportMask() + if (!maskBase64) { + console.warn('No mask to apply') + return + } + + setIsApplying(true) + try { + const result = await editorApi.inpaintImage( + storyboard.id, + selectedFrameId, + maskBase64 + ) + if (result.success) { + // Update frame with new image + const { updateFrame } = useEditorStore.getState() + updateFrame(selectedFrameId, { imagePath: result.image_path }) + } + } catch (err) { + console.error('Inpainting failed:', err) + } finally { + setIsApplying(false) + resetInpainting() + } + } + + const handleMaskChange = (maskDataUrl: string | null) => { + setCurrentMask(maskDataUrl) + // Update undo/redo state from canvas ref + if (inpaintingCanvasRef.current) { + setCanUndo(inpaintingCanvasRef.current.canUndo) + setCanRedo(inpaintingCanvasRef.current.canRedo) + } + } + + const handleUndo = () => { + inpaintingCanvasRef.current?.undo() + } + + const handleRedo = () => { + inpaintingCanvasRef.current?.redo() + } + + const handleClear = () => { + inpaintingCanvasRef.current?.clear() + } + // Navigate to previous/next frame const goToPrevFrame = () => { if (!storyboard?.frames.length) return @@ -109,8 +210,40 @@ export function PreviewPlayer() { return (
{/* Video/Image Preview Area - Fixed height */} -
- {selectedFrame?.imagePath ? ( +
+ {isInpaintingMode && selectedFrame?.imagePath && canvasSize.width > 0 ? ( + <> + + {/* Inpainting Toolbar */} +
+ +
+ + ) : selectedFrame?.imagePath ? ( Preview
+ diff --git a/frontend/src/services/editor-api.ts b/frontend/src/services/editor-api.ts index 7885357..ed21a25 100644 --- a/frontend/src/services/editor-api.ts +++ b/frontend/src/services/editor-api.ts @@ -31,6 +31,11 @@ export interface PreviewResponse { frames_count: number } +export interface InpaintResponse { + image_path: string + success: boolean +} + class EditorApiClient { private baseUrl: string @@ -191,6 +196,37 @@ class EditorApiClient { return response.json() } + + /** + * Inpaint (局部重绘) image for a frame + */ + async inpaintImage( + storyboardId: string, + frameId: string, + mask: string, + prompt?: string, + denoiseStrength?: number + ): Promise { + const response = await fetch( + `${this.baseUrl}/editor/storyboard/${storyboardId}/frames/${frameId}/inpaint`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + mask, + prompt, + denoise_strength: denoiseStrength ?? 0.8, + }), + } + ) + + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: response.statusText })) + throw new Error(error.detail || `Failed to inpaint image: ${response.statusText}`) + } + + return response.json() + } } // Export singleton instance diff --git a/frontend/src/stores/editor-store.ts b/frontend/src/stores/editor-store.ts index 81c8e71..4f43335 100644 --- a/frontend/src/stores/editor-store.ts +++ b/frontend/src/stores/editor-store.ts @@ -1,5 +1,8 @@ import { create } from 'zustand' +// Inpainting types +export type InpaintingTool = 'brush' | 'eraser' | 'rect' | 'lasso' + export interface StoryboardFrame { id: string index: number @@ -24,6 +27,13 @@ interface EditorState { isPlaying: boolean currentTime: number + // Inpainting state + isInpaintingMode: boolean + inpaintingTool: InpaintingTool + brushSize: number + currentMask: string | null + inpaintPrompt: string + // Actions setStoryboard: (storyboard: Storyboard) => void selectFrame: (frameId: string | null) => void @@ -33,6 +43,14 @@ interface EditorState { updateFrame: (frameId: string, updates: Partial) => void setPlaying: (playing: boolean) => void setCurrentTime: (time: number | ((prev: number) => number)) => void + + // Inpainting actions + setInpaintingMode: (mode: boolean) => void + setInpaintingTool: (tool: InpaintingTool) => void + setBrushSize: (size: number) => void + setCurrentMask: (mask: string | null) => void + setInpaintPrompt: (prompt: string) => void + resetInpainting: () => void } export const useEditorStore = create((set, get) => ({ @@ -41,6 +59,13 @@ export const useEditorStore = create((set, get) => ({ isPlaying: false, currentTime: 0, + // Inpainting initial state + isInpaintingMode: false, + inpaintingTool: 'brush' as InpaintingTool, + brushSize: 20, + currentMask: null, + inpaintPrompt: '', + setStoryboard: (storyboard) => set({ storyboard }), selectFrame: (frameId) => set({ selectedFrameId: frameId }), @@ -117,6 +142,20 @@ export const useEditorStore = create((set, get) => ({ set({ currentTime: time }) } }, + + // Inpainting actions + setInpaintingMode: (mode) => set({ isInpaintingMode: mode }), + setInpaintingTool: (tool) => set({ inpaintingTool: tool }), + setBrushSize: (size) => set({ brushSize: size }), + setCurrentMask: (mask) => set({ currentMask: mask }), + setInpaintPrompt: (prompt) => set({ inpaintPrompt: prompt }), + resetInpainting: () => set({ + isInpaintingMode: false, + inpaintingTool: 'brush', + brushSize: 20, + currentMask: null, + inpaintPrompt: '', + }), })) diff --git a/pixelle_video/services/media.py b/pixelle_video/services/media.py index d894339..17cc2fa 100644 --- a/pixelle_video/services/media.py +++ b/pixelle_video/services/media.py @@ -285,3 +285,76 @@ class MediaService(ComfyBaseService): except Exception as e: logger.error(f"Media generation error: {e}") raise + + async def inpaint( + self, + image_path: str, + mask_path: str, + prompt: Optional[str] = None, + denoise_strength: float = 0.8, + workflow: Optional[str] = None, + **params + ) -> MediaResult: + """ + Inpaint image using mask + + Args: + image_path: Path to original image + mask_path: Path to mask image (white=inpaint, black=keep) + prompt: Optional prompt for inpainted region + denoise_strength: How much to change masked area (0.0-1.0) + workflow: Inpainting workflow to use + + Returns: + MediaResult with inpainted image URL + """ + logger.info(f"🎨 Inpainting image: {image_path}") + logger.info(f" Mask: {mask_path}") + logger.info(f" Denoise: {denoise_strength}") + + try: + # Resolve workflow + if workflow is None: + workflow = "selfhost/image_inpaint_flux.json" + + workflow_info = self._resolve_workflow(workflow=workflow) + + # Build workflow parameters + workflow_params = { + "image": image_path, + "mask": mask_path, + "denoise": denoise_strength, + } + + if prompt: + workflow_params["prompt"] = prompt + + workflow_params.update(params) + + logger.debug(f"Inpaint workflow parameters: {workflow_params}") + + # Execute workflow + kit = await self.core._get_or_create_comfykit() + + if workflow_info["source"] == "runninghub" and "workflow_id" in workflow_info: + workflow_input = workflow_info["workflow_id"] + else: + workflow_input = workflow_info["path"] + + result = await kit.execute(workflow_input, workflow_params) + + if not result.images: + logger.error("No image generated from inpainting") + raise Exception("Inpainting failed - no image generated") + + image_url = result.images[0] + logger.info(f"✅ Inpainted image: {image_url}") + + return MediaResult( + media_type="image", + url=image_url + ) + + except Exception as e: + logger.error(f"Inpainting error: {e}") + raise diff --git a/workflows/runninghub/image_inpaint.json b/workflows/runninghub/image_inpaint.json new file mode 100644 index 0000000..1fcaddb --- /dev/null +++ b/workflows/runninghub/image_inpaint.json @@ -0,0 +1,5 @@ +{ + "source": "runninghub", + "workflow_id": "YOUR_INPAINT_WORKFLOW_ID", + "description": "Flux Inpainting workflow on RunningHub - Replace workflow_id with your actual RunningHub workflow ID" +} diff --git a/workflows/selfhost/image_inpaint_flux.json b/workflows/selfhost/image_inpaint_flux.json new file mode 100644 index 0000000..e87b6ef --- /dev/null +++ b/workflows/selfhost/image_inpaint_flux.json @@ -0,0 +1,149 @@ +{ + "10": { + "inputs": { + "image": "" + }, + "class_type": "LoadImage", + "_meta": { + "title": "$image.value!" + } + }, + "11": { + "inputs": { + "image": "", + "channel": "alpha" + }, + "class_type": "LoadImageMask", + "_meta": { + "title": "$mask.value!" + } + }, + "12": { + "inputs": { + "grow_mask_by": 6, + "pixels": ["10", 0], + "vae": ["49", 0], + "mask": ["11", 0] + }, + "class_type": "VAEEncodeForInpaint", + "_meta": { + "title": "VAE Encode (for Inpainting)" + } + }, + "29": { + "inputs": { + "seed": 1067822190154760, + "steps": 20, + "cfg": 1, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 0.8, + "model": ["48", 0], + "positive": ["35", 0], + "negative": ["33", 0], + "latent_image": ["12", 0] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "30": { + "inputs": { + "value": 0.8 + }, + "class_type": "easy float", + "_meta": { + "title": "$denoise.value" + } + }, + "31": { + "inputs": { + "text": ["46", 0], + "clip": ["47", 0] + }, + "class_type": "CLIPTextEncode", + "_meta": { + "title": "CLIP Text Encode (Prompt)" + } + }, + "33": { + "inputs": { + "conditioning": ["31", 0] + }, + "class_type": "ConditioningZeroOut", + "_meta": { + "title": "ConditioningZeroOut" + } + }, + "35": { + "inputs": { + "guidance": 3.5, + "conditioning": ["31", 0] + }, + "class_type": "FluxGuidance", + "_meta": { + "title": "FluxGuidance" + } + }, + "36": { + "inputs": { + "filename_prefix": "inpaint", + "images": ["37", 0] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "37": { + "inputs": { + "samples": ["29", 0], + "vae": ["49", 0] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "46": { + "inputs": { + "value": "inpaint region" + }, + "class_type": "PrimitiveStringMultiline", + "_meta": { + "title": "$prompt.value!" + } + }, + "47": { + "inputs": { + "clip_name1": "clip_l.safetensors", + "clip_name2": "t5xxl_fp8_e4m3fn.safetensors", + "type": "flux", + "device": "default" + }, + "class_type": "DualCLIPLoader", + "_meta": { + "title": "DualCLIPLoader" + } + }, + "48": { + "inputs": { + "unet_name": "flux1-dev.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "49": { + "inputs": { + "vae_name": "ae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + } +}