import { ArrowsExpandIcon, DownloadIcon, EyeIcon, } from '@heroicons/react/outline' import React, { SyntheticEvent, useCallback, useEffect, useRef, useState, } from 'react' import { ReactZoomPanPinchRef, TransformComponent, TransformWrapper, } from 'react-zoom-pan-pinch' import { useRecoilState, useRecoilValue } from 'recoil' import { useWindowSize, useKey, useKeyPressEvent } from 'react-use' import inpaint from '../../adapters/inpainting' import Button from '../shared/Button' import Slider from './Slider' import SizeSelector from './SizeSelector' import { downloadImage, isMidClick, isRightClick, loadImage, srcToFile, useImage, } from '../../utils' import { croperState, isInpaintingState, isSDState, propmtState, runManuallyState, seedState, settingState, toastState, } from '../../store/Atoms' import useHotKey from '../../hooks/useHotkey' import Croper from '../Croper/Croper' import emitter, { EVENT_PROMPT } from '../../event' const TOOLBAR_SIZE = 200 const BRUSH_COLOR = '#ffcc00bb' interface EditorProps { file: File } interface Line { size?: number pts: { x: number; y: number }[] } type LineGroup = Array function drawLines( ctx: CanvasRenderingContext2D, lines: LineGroup, color = BRUSH_COLOR ) { ctx.strokeStyle = color ctx.lineCap = 'round' ctx.lineJoin = 'round' lines.forEach(line => { if (!line?.pts.length || !line.size) { return } ctx.lineWidth = line.size ctx.beginPath() ctx.moveTo(line.pts[0].x, line.pts[0].y) line.pts.forEach(pt => ctx.lineTo(pt.x, pt.y)) ctx.stroke() }) } function mouseXY(ev: SyntheticEvent) { const mouseEvent = ev.nativeEvent as MouseEvent return { x: mouseEvent.offsetX, y: mouseEvent.offsetY } } export default function Editor(props: EditorProps) { const { file } = props const promptVal = useRecoilValue(propmtState) const settings = useRecoilValue(settingState) const [seedVal, setSeed] = useRecoilState(seedState) const croperRect = useRecoilValue(croperState) const [toastVal, setToastState] = useRecoilState(toastState) const [isInpainting, setIsInpainting] = useRecoilState(isInpaintingState) const runMannually = useRecoilValue(runManuallyState) const isSD = useRecoilValue(isSDState) const [brushSize, setBrushSize] = useState(40) const [original, isOriginalLoaded] = useImage(file) const [renders, setRenders] = useState([]) const [context, setContext] = useState() const [maskCanvas] = useState(() => { return document.createElement('canvas') }) const [lineGroups, setLineGroups] = useState([]) const [lastLineGroup, setLastLineGroup] = useState([]) const [curLineGroup, setCurLineGroup] = useState([]) const [{ x, y }, setCoords] = useState({ x: -1, y: -1 }) const [showBrush, setShowBrush] = useState(false) const [showRefBrush, setShowRefBrush] = useState(false) const [isPanning, setIsPanning] = useState(false) const [showOriginal, setShowOriginal] = useState(false) const [scale, setScale] = useState(1) const [panned, setPanned] = useState(false) const [minScale, setMinScale] = useState(1.0) const [sizeLimit, setSizeLimit] = useState(1080) const windowSize = useWindowSize() const windowCenterX = windowSize.width / 2 const windowCenterY = windowSize.height / 2 const viewportRef = useRef() // Indicates that the image has been loaded and is centered on first load const [initialCentered, setInitialCentered] = useState(false) const [isDraging, setIsDraging] = useState(false) const [isMultiStrokeKeyPressed, setIsMultiStrokeKeyPressed] = useState(false) const [sliderPos, setSliderPos] = useState(0) // redo 相关 const [redoRenders, setRedoRenders] = useState([]) const [redoCurLines, setRedoCurLines] = useState([]) const [redoLineGroups, setRedoLineGroups] = useState([]) const draw = useCallback( (render: HTMLImageElement, lineGroup: LineGroup) => { if (!context) { return } context.clearRect(0, 0, context.canvas.width, context.canvas.height) context.drawImage( render, 0, 0, original.naturalWidth, original.naturalHeight ) drawLines(context, lineGroup) }, [context, original] ) const drawLinesOnMask = useCallback( (_lineGroups: LineGroup[]) => { if (!context?.canvas.width || !context?.canvas.height) { throw new Error('canvas has invalid size') } maskCanvas.width = context?.canvas.width maskCanvas.height = context?.canvas.height const ctx = maskCanvas.getContext('2d') if (!ctx) { throw new Error('could not retrieve mask canvas') } _lineGroups.forEach(lineGroup => { drawLines(ctx, lineGroup, 'white') }) }, [context, maskCanvas] ) const hadDrawSomething = useCallback(() => { return curLineGroup.length !== 0 }, [curLineGroup]) const drawOnCurrentRender = useCallback( (lineGroup: LineGroup) => { if (renders.length === 0) { draw(original, lineGroup) } else { draw(renders[renders.length - 1], lineGroup) } }, [original, renders, draw] ) const runInpainting = useCallback( async (prompt?: string, useLastLineGroup?: boolean) => { // useLastLineGroup 的影响 // 1. 使用上一次的 mask // 2. 结果替换当前 render console.log('runInpainting') let maskLineGroup = [] if (useLastLineGroup === true) { if (lastLineGroup.length === 0) { return } maskLineGroup = lastLineGroup } else { if (!hadDrawSomething()) { return } setLastLineGroup(curLineGroup) maskLineGroup = curLineGroup } const newLineGroups = [...lineGroups, maskLineGroup] setCurLineGroup([]) setIsDraging(false) setIsInpainting(true) if (settings.graduallyInpainting) { drawLinesOnMask([maskLineGroup]) } else { drawLinesOnMask(newLineGroups) } let targetFile = file if (settings.graduallyInpainting === true) { if (useLastLineGroup === true) { // renders.length == 1 还是用原来的 if (renders.length > 1) { const lastRender = renders[renders.length - 2] targetFile = await srcToFile( lastRender.currentSrc, file.name, file.type ) } } else if (renders.length > 0) { console.info('gradually inpainting on last result') const lastRender = renders[renders.length - 1] targetFile = await srcToFile( lastRender.currentSrc, file.name, file.type ) } } let sdSeed = settings.sdSeedFixed ? settings.sdSeed : -1 if (useLastLineGroup === true) { sdSeed = -1 } try { const res = await inpaint( targetFile, maskCanvas.toDataURL(), settings, croperRect, prompt, sizeLimit.toString(), sdSeed ) if (!res) { throw new Error('empty response') } const { blob, seed } = res if (seed) { setSeed(parseInt(seed, 10)) } const newRender = new Image() await loadImage(newRender, blob) if (useLastLineGroup === true) { const prevRenders = renders.slice(0, -1) const newRenders = [...prevRenders, newRender] setRenders(newRenders) } else { const newRenders = [...renders, newRender] setRenders(newRenders) } draw(newRender, []) // Only append new LineGroup after inpainting success setLineGroups(newLineGroups) // clear redo stack resetRedoState() } catch (e: any) { setToastState({ open: true, desc: e.message ? e.message : e.toString(), state: 'error', duration: 4000, }) drawOnCurrentRender([]) } setIsInpainting(false) }, [ lineGroups, curLineGroup, maskCanvas, settings.graduallyInpainting, settings, croperRect, sizeLimit, promptVal, drawOnCurrentRender, hadDrawSomething, drawLinesOnMask, ] ) useEffect(() => { emitter.on(EVENT_PROMPT, () => { if (hadDrawSomething()) { runInpainting(promptVal) } else if (lastLineGroup.length !== 0) { runInpainting(promptVal, true) } else { setToastState({ open: true, desc: 'Please draw mask on picture', state: 'error', duration: 1500, }) } }) return () => { emitter.off(EVENT_PROMPT) } }, [hadDrawSomething, runInpainting, prompt]) const hadRunInpainting = () => { return renders.length !== 0 } const handleMultiStrokeKeyDown = () => { if (isInpainting) { return } setIsMultiStrokeKeyPressed(true) } const handleMultiStrokeKeyup = () => { if (!isMultiStrokeKeyPressed) { return } if (isInpainting) { return } setIsMultiStrokeKeyPressed(false) if (!runMannually) { runInpainting() } } const predicate = (event: KeyboardEvent) => { return event.key === 'Control' || event.key === 'Meta' } useKey(predicate, handleMultiStrokeKeyup, { event: 'keyup' }, [ isInpainting, isMultiStrokeKeyPressed, hadDrawSomething, ]) useKey( predicate, handleMultiStrokeKeyDown, { event: 'keydown', }, [isInpainting] ) // Draw once the original image is loaded useEffect(() => { if (!isOriginalLoaded) { return } const rW = windowSize.width / original.naturalWidth const rH = (windowSize.height - TOOLBAR_SIZE) / original.naturalHeight let s = 1.0 if (rW < 1 || rH < 1) { s = Math.min(rW, rH) } setMinScale(s) setScale(s) if (context?.canvas) { context.canvas.width = original.naturalWidth context.canvas.height = original.naturalHeight drawOnCurrentRender([]) } if (!initialCentered) { viewportRef.current?.centerView(s, 1) setInitialCentered(true) const imageSizeLimit = Math.max(original.width, original.height) setSizeLimit(imageSizeLimit) } }, [ context?.canvas, viewportRef, original, isOriginalLoaded, windowSize, initialCentered, drawOnCurrentRender, ]) // Zoom reset const resetZoom = useCallback(() => { if (!minScale || !original || !windowSize) { return } const viewport = viewportRef.current if (!viewport) { return } const offsetX = (windowSize.width - original.width * minScale) / 2 const offsetY = (windowSize.height - original.height * minScale) / 2 viewport.setTransform(offsetX, offsetY, minScale, 200, 'easeOutQuad') viewport.state.scale = minScale setScale(minScale) setPanned(false) }, [ viewportRef, windowSize, original, original.width, windowSize.height, minScale, ]) const resetRedoState = () => { setRedoCurLines([]) setRedoLineGroups([]) setRedoRenders([]) } useEffect(() => { window.addEventListener('resize', () => { resetZoom() }) return () => { window.removeEventListener('resize', () => { resetZoom() }) } }, [windowSize, resetZoom]) const handleEscPressed = () => { if (isInpainting) { return } if (isDraging || isMultiStrokeKeyPressed) { setIsDraging(false) setCurLineGroup([]) drawOnCurrentRender([]) } else { resetZoom() } } useKey( 'Escape', handleEscPressed, { event: 'keydown', }, [ isDraging, isInpainting, isMultiStrokeKeyPressed, resetZoom, drawOnCurrentRender, ] ) const onMouseMove = (ev: SyntheticEvent) => { const mouseEvent = ev.nativeEvent as MouseEvent setCoords({ x: mouseEvent.pageX, y: mouseEvent.pageY }) } const onMouseDrag = (ev: SyntheticEvent) => { if (isPanning) { return } if (!isDraging) { return } if (curLineGroup.length === 0) { return } const lineGroup = [...curLineGroup] lineGroup[lineGroup.length - 1].pts.push(mouseXY(ev)) setCurLineGroup(lineGroup) drawOnCurrentRender(lineGroup) } const onPointerUp = (ev: SyntheticEvent) => { if (isMidClick(ev)) { setIsPanning(false) } if (isPanning) { return } if (!original.src) { return } const canvas = context?.canvas if (!canvas) { return } if (isInpainting) { return } if (!isDraging) { return } if (isMultiStrokeKeyPressed) { setIsDraging(false) return } if (runMannually) { setIsDraging(false) } else { runInpainting() } } const isOutsideCroper = (clickPnt: { x: number; y: number }) => { if (clickPnt.x < croperRect.x) { return true } if (clickPnt.y < croperRect.y) { return true } if (clickPnt.x > croperRect.x + croperRect.width) { return true } if (clickPnt.y > croperRect.y + croperRect.height) { return true } return false } const onMouseDown = (ev: SyntheticEvent) => { if (isPanning) { return } if (!original.src) { return } const canvas = context?.canvas if (!canvas) { return } if (isInpainting) { return } if (isRightClick(ev)) { return } if (isMidClick(ev)) { setIsPanning(true) return } if (isSD && settings.showCroper && isOutsideCroper(mouseXY(ev))) { return } setIsDraging(true) let lineGroup: LineGroup = [] if (isMultiStrokeKeyPressed || runMannually) { lineGroup = [...curLineGroup] } lineGroup.push({ size: brushSize, pts: [mouseXY(ev)] }) setCurLineGroup(lineGroup) drawOnCurrentRender(lineGroup) } const undoStroke = useCallback(() => { if (curLineGroup.length === 0) { return } setLastLineGroup([]) const lastLine = curLineGroup.pop()! const newRedoCurLines = [...redoCurLines, lastLine] setRedoCurLines(newRedoCurLines) const newLineGroup = [...curLineGroup] setCurLineGroup(newLineGroup) drawOnCurrentRender(newLineGroup) }, [curLineGroup, redoCurLines, drawOnCurrentRender]) const undoRender = useCallback(() => { if (!renders.length) { return } // save line Group const latestLineGroup = lineGroups.pop()! setRedoLineGroups([...redoLineGroups, latestLineGroup]) // If render is undo, clear strokes setRedoCurLines([]) setLineGroups([...lineGroups]) setCurLineGroup([]) setIsDraging(false) // save render const lastRender = renders.pop()! setRedoRenders([...redoRenders, lastRender]) const newRenders = [...renders] setRenders(newRenders) if (newRenders.length === 0) { draw(original, []) } else { draw(newRenders[newRenders.length - 1], []) } }, [draw, renders, redoRenders, redoLineGroups, lineGroups, original]) const undo = () => { if (runMannually && curLineGroup.length !== 0) { undoStroke() } else { undoRender() } } // Handle Cmd+Z const undoPredicate = (event: KeyboardEvent) => { const isCmdZ = (event.metaKey || event.ctrlKey) && !event.shiftKey && event.key === 'z' // Handle tab switch if (event.key === 'Tab') { event.preventDefault() } if (isCmdZ) { event.preventDefault() console.log('undo') return true } return false } useKey(undoPredicate, undo, undefined, [undoStroke, undoRender]) const disableUndo = () => { if (isInpainting) { return true } if (renders.length > 0) { return false } if (runMannually) { if (curLineGroup.length === 0) { return true } } else if (renders.length === 0) { return true } return false } const redoStroke = useCallback(() => { if (redoCurLines.length === 0) { return } const line = redoCurLines.pop()! setRedoCurLines([...redoCurLines]) const newLineGroup = [...curLineGroup, line] setCurLineGroup(newLineGroup) drawOnCurrentRender(newLineGroup) }, [curLineGroup, redoCurLines, drawOnCurrentRender]) const redoRender = useCallback(() => { if (redoRenders.length === 0) { return } const lineGroup = redoLineGroups.pop()! setRedoLineGroups([...redoLineGroups]) setLineGroups([...lineGroups, lineGroup]) setCurLineGroup([]) setIsDraging(false) const render = redoRenders.pop()! const newRenders = [...renders, render] setRenders(newRenders) draw(newRenders[newRenders.length - 1], []) }, [draw, renders, redoRenders, redoLineGroups, lineGroups, original]) const redo = () => { if (runMannually && redoCurLines.length !== 0) { redoStroke() } else { redoRender() } } // Handle Cmd+shift+Z const redoPredicate = (event: KeyboardEvent) => { const isCmdZ = (event.metaKey || event.ctrlKey) && event.shiftKey && event.key.toLowerCase() === 'z' // Handle tab switch if (event.key === 'Tab') { event.preventDefault() } if (isCmdZ) { event.preventDefault() console.log('redo') return true } return false } useKey(redoPredicate, redo, undefined, [redoStroke, redoRender]) const disableRedo = () => { if (isInpainting) { return true } if (redoRenders.length > 0) { return false } if (runMannually) { if (redoCurLines.length === 0) { return true } } else if (redoRenders.length === 0) { return true } return false } useKeyPressEvent( 'Tab', ev => { ev?.preventDefault() ev?.stopPropagation() if (hadRunInpainting()) { setShowOriginal(() => { window.setTimeout(() => { setSliderPos(100) }, 10) return true }) } }, ev => { ev?.preventDefault() ev?.stopPropagation() if (hadRunInpainting()) { setSliderPos(0) window.setTimeout(() => { setShowOriginal(false) }, 350) } } ) function download() { const name = file.name.replace(/(\.[\w\d_-]+)$/i, '_cleanup$1') const curRender = renders[renders.length - 1] downloadImage(curRender.currentSrc, name) if (settings.downloadMask) { let maskFileName = file.name.replace(/(\.[\w\d_-]+)$/i, '_mask$1') maskFileName = maskFileName.replace(/\.[^/.]+$/, '.jpg') drawLinesOnMask(lineGroups) // Create a link const aDownloadLink = document.createElement('a') // Add the name of the file to the link aDownloadLink.download = maskFileName // Attach the data to the link aDownloadLink.href = maskCanvas.toDataURL('image/jpeg') // Get the code to click the download link aDownloadLink.click() } } const onSizeLimitChange = (_sizeLimit: number) => { setSizeLimit(_sizeLimit) } const toggleShowBrush = (newState: boolean) => { if (newState !== showBrush && !isPanning) { setShowBrush(newState) } } const getCursor = useCallback(() => { if (isPanning) { return 'grab' } if (showBrush) { return 'none' } return undefined }, [showBrush, isPanning]) // Standard Hotkeys for Brush Size useHotKey('[', () => { setBrushSize(currentBrushSize => { if (currentBrushSize > 10) { return currentBrushSize - 10 } if (currentBrushSize <= 10 && currentBrushSize > 0) { return currentBrushSize - 5 } return currentBrushSize }) }) useHotKey(']', () => { setBrushSize(currentBrushSize => { return currentBrushSize + 10 }) }) // Manual Inpainting Hotkey useHotKey( 'shift+r', () => { if (runMannually && hadDrawSomething()) { runInpainting() } }, {}, [runMannually] ) // Toggle clean/zoom tool on spacebar. useKeyPressEvent( ' ', ev => { ev?.preventDefault() ev?.stopPropagation() setShowBrush(false) setIsPanning(true) }, ev => { ev?.preventDefault() ev?.stopPropagation() setShowBrush(true) setIsPanning(false) } ) const getCurScale = (): number => { let s = minScale if (viewportRef.current?.state.scale !== undefined) { s = viewportRef.current?.state.scale } return s! } const getBrushStyle = (_x: number, _y: number) => { const curScale = getCurScale() return { width: `${brushSize * curScale}px`, height: `${brushSize * curScale}px`, left: `${_x}px`, top: `${_y}px`, transform: 'translate(-50%, -50%)', } } const handleSliderChange = (value: number) => { setBrushSize(value) if (!showRefBrush) { setShowRefBrush(true) window.setTimeout(() => { setShowRefBrush(false) }, 10000) } } return (