import { ArrowsExpandIcon, DownloadIcon, EyeIcon, } from '@heroicons/react/outline' import React, { SyntheticEvent, useCallback, useEffect, useRef, useState, } from 'react' import { ReactZoomPanPinchRef, TransformComponent, TransformWrapper, } from 'react-zoom-pan-pinch' import { useRecoilState, useRecoilValue } from 'recoil' import { useWindowSize, useKey, useKeyPressEvent } from 'react-use' import inpaint from '../../adapters/inpainting' import Button from '../shared/Button' import Slider from './Slider' import SizeSelector from './SizeSelector' import { downloadImage, isMidClick, isRightClick, loadImage, useImage, } from '../../utils' import { settingState, toastState } from '../../store/Atoms' const TOOLBAR_SIZE = 200 const BRUSH_COLOR = '#ffcc00bb' interface EditorProps { file: File } interface Line { size?: number pts: { x: number; y: number }[] } type LineGroup = Array function drawLines( ctx: CanvasRenderingContext2D, lines: LineGroup, color = BRUSH_COLOR ) { ctx.strokeStyle = color ctx.lineCap = 'round' ctx.lineJoin = 'round' lines.forEach(line => { if (!line?.pts.length || !line.size) { return } ctx.lineWidth = line.size ctx.beginPath() ctx.moveTo(line.pts[0].x, line.pts[0].y) line.pts.forEach(pt => ctx.lineTo(pt.x, pt.y)) ctx.stroke() }) } function mouseXY(ev: SyntheticEvent) { const mouseEvent = ev.nativeEvent as MouseEvent return { x: mouseEvent.offsetX, y: mouseEvent.offsetY } } export default function Editor(props: EditorProps) { const { file } = props const settings = useRecoilValue(settingState) const [toastVal, setToastState] = useRecoilState(toastState) const [brushSize, setBrushSize] = useState(40) const [original, isOriginalLoaded] = useImage(file) const [renders, setRenders] = useState([]) const [context, setContext] = useState() const [maskCanvas] = useState(() => { return document.createElement('canvas') }) const [lineGroups, setLineGroups] = useState([]) const [curLineGroup, setCurLineGroup] = useState([]) const [{ x, y }, setCoords] = useState({ x: -1, y: -1 }) const [showBrush, setShowBrush] = useState(false) const [showRefBrush, setShowRefBrush] = useState(false) const [isPanning, setIsPanning] = useState(false) const [showOriginal, setShowOriginal] = useState(false) const [isInpaintingLoading, setIsInpaintingLoading] = useState(false) const [scale, setScale] = useState(1) const [panned, setPanned] = useState(false) const [minScale, setMinScale] = useState(1.0) const [sizeLimit, setSizeLimit] = useState(1080) const windowSize = useWindowSize() const windowCenterX = windowSize.width / 2 const windowCenterY = windowSize.height / 2 const viewportRef = useRef() // Indicates that the image has been loaded and is centered on first load const [initialCentered, setInitialCentered] = useState(false) const [isDraging, setIsDraging] = useState(false) const [isMultiStrokeKeyPressed, setIsMultiStrokeKeyPressed] = useState(false) const [sliderPos, setSliderPos] = useState(0) const draw = useCallback( (render: HTMLImageElement, lineGroup: LineGroup) => { if (!context) { return } context.clearRect(0, 0, context.canvas.width, context.canvas.height) context.drawImage( render, 0, 0, original.naturalWidth, original.naturalHeight ) drawLines(context, lineGroup) }, [context, original] ) const drawAllLinesOnMask = (_lineGroups: LineGroup[]) => { if (!context?.canvas.width || !context?.canvas.height) { throw new Error('canvas has invalid size') } maskCanvas.width = context?.canvas.width maskCanvas.height = context?.canvas.height const ctx = maskCanvas.getContext('2d') if (!ctx) { throw new Error('could not retrieve mask canvas') } _lineGroups.forEach(lineGroup => { drawLines(ctx, lineGroup, 'white') }) } const runInpainting = async () => { if (!hadDrawSomething()) { return } const newLineGroups = [...lineGroups, curLineGroup] setCurLineGroup([]) setIsDraging(false) setIsInpaintingLoading(true) drawAllLinesOnMask(newLineGroups) try { const res = await inpaint( file, maskCanvas.toDataURL(), settings, sizeLimit.toString() ) if (!res) { throw new Error('empty response') } const newRender = new Image() await loadImage(newRender, res) const newRenders = [...renders, newRender] setRenders(newRenders) draw(newRender, []) // Only append new LineGroup after inpainting success setLineGroups(newLineGroups) } catch (e: any) { setToastState({ open: true, desc: e.message ? e.message : e.toString(), state: 'error', duration: 2000, }) drawOnCurrentRender([]) } setIsInpaintingLoading(false) } const hadDrawSomething = () => { return curLineGroup.length !== 0 } const hadRunInpainting = () => { return renders.length !== 0 } const drawOnCurrentRender = useCallback( (lineGroup: LineGroup) => { if (renders.length === 0) { draw(original, lineGroup) } else { draw(renders[renders.length - 1], lineGroup) } }, [original, renders, draw] ) const handleMultiStrokeKeyDown = () => { if (isInpaintingLoading) { return } setIsMultiStrokeKeyPressed(true) } const handleMultiStrokeKeyup = () => { if (!isMultiStrokeKeyPressed) { return } if (isInpaintingLoading) { return } setIsMultiStrokeKeyPressed(false) if (!settings.runInpaintingManually) { runInpainting() } } const predicate = (event: KeyboardEvent) => { return event.key === 'Control' || event.key === 'Meta' } useKey(predicate, handleMultiStrokeKeyup, { event: 'keyup' }, [ isInpaintingLoading, isMultiStrokeKeyPressed, hadDrawSomething, ]) useKey( predicate, handleMultiStrokeKeyDown, { event: 'keydown', }, [isInpaintingLoading] ) // Draw once the original image is loaded useEffect(() => { if (!isOriginalLoaded) { return } const rW = windowSize.width / original.naturalWidth const rH = (windowSize.height - TOOLBAR_SIZE) / original.naturalHeight let s = 1.0 if (rW < 1 || rH < 1) { s = Math.min(rW, rH) } setMinScale(s) setScale(s) if (context?.canvas) { context.canvas.width = original.naturalWidth context.canvas.height = original.naturalHeight drawOnCurrentRender([]) } if (!initialCentered) { viewportRef.current?.centerView(s, 1) setInitialCentered(true) const imageSizeLimit = Math.max(original.width, original.height) setSizeLimit(imageSizeLimit) } }, [ context?.canvas, viewportRef, original, isOriginalLoaded, windowSize, initialCentered, drawOnCurrentRender, ]) // Zoom reset const resetZoom = useCallback(() => { if (!minScale || !original || !windowSize) { return } const viewport = viewportRef.current if (!viewport) { throw new Error('no viewport') } const offsetX = (windowSize.width - original.width * minScale) / 2 const offsetY = (windowSize.height - original.height * minScale) / 2 viewport.setTransform(offsetX, offsetY, minScale, 200, 'easeOutQuad') viewport.state.scale = minScale setScale(minScale) setPanned(false) }, [viewportRef, minScale, original, windowSize]) useEffect(() => { window.addEventListener('resize', () => { resetZoom() }) return () => { window.removeEventListener('resize', () => { resetZoom() }) } }, [windowSize, resetZoom]) const handleEscPressed = () => { if (isInpaintingLoading) { return } if (isDraging || isMultiStrokeKeyPressed) { setIsDraging(false) setCurLineGroup([]) drawOnCurrentRender([]) } else { resetZoom() } } useKey( 'Escape', handleEscPressed, { event: 'keydown', }, [ isDraging, isInpaintingLoading, isMultiStrokeKeyPressed, resetZoom, drawOnCurrentRender, ] ) const onMouseMove = (ev: SyntheticEvent) => { const mouseEvent = ev.nativeEvent as MouseEvent setCoords({ x: mouseEvent.pageX, y: mouseEvent.pageY }) } const onMouseDrag = (ev: SyntheticEvent) => { if (isPanning) { return } if (!isDraging) { return } if (curLineGroup.length === 0) { return } const lineGroup = [...curLineGroup] lineGroup[lineGroup.length - 1].pts.push(mouseXY(ev)) setCurLineGroup(lineGroup) drawOnCurrentRender(lineGroup) } const onPointerUp = (ev: SyntheticEvent) => { if (isMidClick(ev)) { setIsPanning(false) } if (isPanning) { return } if (!original.src) { return } const canvas = context?.canvas if (!canvas) { return } if (isInpaintingLoading) { return } if (!isDraging) { return } if (isMultiStrokeKeyPressed) { setIsDraging(false) return } if (settings.runInpaintingManually) { setIsDraging(false) } else { runInpainting() } } const onMouseDown = (ev: SyntheticEvent) => { if (isPanning) { return } if (!original.src) { return } const canvas = context?.canvas if (!canvas) { return } if (isInpaintingLoading) { return } if (isRightClick(ev)) { return } if (isMidClick(ev)) { setIsPanning(true) return } setIsDraging(true) let lineGroup: LineGroup = [] if (isMultiStrokeKeyPressed || settings.runInpaintingManually) { lineGroup = [...curLineGroup] } lineGroup.push({ size: brushSize, pts: [mouseXY(ev)] }) setCurLineGroup(lineGroup) drawOnCurrentRender(lineGroup) } const undoStroke = useCallback(() => { if (curLineGroup.length === 0) { return } const newLineGroup = curLineGroup.slice(0, curLineGroup.length - 1) setCurLineGroup(newLineGroup) drawOnCurrentRender(newLineGroup) }, [curLineGroup, drawOnCurrentRender]) const undoRender = useCallback(() => { if (!renders.length) { return } const groups = lineGroups.slice(0, lineGroups.length - 1) setLineGroups(groups) setCurLineGroup([]) setIsDraging(false) const newRenders = renders.slice(0, renders.length - 1) setRenders(newRenders) if (newRenders.length === 0) { draw(original, []) } else { draw(newRenders[newRenders.length - 1], []) } }, [draw, renders, lineGroups, original]) const undo = () => { if (settings.runInpaintingManually && curLineGroup.length !== 0) { undoStroke() } else { undoRender() } } // Handle Cmd+Z const undoPredicate = (event: KeyboardEvent) => { const isCmdZ = (event.metaKey || event.ctrlKey) && event.key === 'z' // Handle tab switch if (event.key === 'Tab') { event.preventDefault() } if (isCmdZ) { event.preventDefault() return true } return false } useKey(undoPredicate, undo, undefined, [undoStroke, undoRender]) const disableUndo = () => { if (renders.length > 0) { return false } if (settings.runInpaintingManually) { if (curLineGroup.length === 0) { return true } } else if (renders.length === 0) { return true } return false } useKeyPressEvent( 'Tab', ev => { ev?.preventDefault() ev?.stopPropagation() if (hadRunInpainting()) { setShowOriginal(() => { window.setTimeout(() => { setSliderPos(100) }, 10) return true }) } }, ev => { ev?.preventDefault() ev?.stopPropagation() if (hadRunInpainting()) { setSliderPos(0) window.setTimeout(() => { setShowOriginal(false) }, 350) } } ) function download() { const name = file.name.replace(/(\.[\w\d_-]+)$/i, '_cleanup$1') const curRender = renders[renders.length - 1] downloadImage(curRender.currentSrc, name) } const onSizeLimitChange = (_sizeLimit: number) => { setSizeLimit(_sizeLimit) } const toggleShowBrush = (newState: boolean) => { if (newState !== showBrush && !isPanning) { setShowBrush(newState) } } const getCursor = useCallback(() => { if (isPanning) { return 'grab' } if (showBrush) { return 'none' } return undefined }, [showBrush, isPanning]) // Standard Hotkeys for Brush Size useKeyPressEvent('[', () => { setBrushSize(currentBrushSize => { if (currentBrushSize > 10) { return currentBrushSize - 10 } if (currentBrushSize <= 10 && currentBrushSize > 0) { return currentBrushSize - 5 } return currentBrushSize }) }) useKeyPressEvent(']', () => { setBrushSize(currentBrushSize => { return currentBrushSize + 10 }) }) // Manual Inpainting Hotkey useKeyPressEvent('R', () => { if (settings.runInpaintingManually && hadDrawSomething()) { runInpainting() } }) // Toggle clean/zoom tool on spacebar. useKeyPressEvent( ' ', ev => { ev?.preventDefault() ev?.stopPropagation() setShowBrush(false) setIsPanning(true) }, ev => { ev?.preventDefault() ev?.stopPropagation() setShowBrush(true) setIsPanning(false) } ) const getCurScale = (): number => { let s = minScale if (viewportRef.current?.state.scale !== undefined) { s = viewportRef.current?.state.scale } return s! } const getBrushStyle = (_x: number, _y: number) => { const curScale = getCurScale() return { width: `${brushSize * curScale}px`, height: `${brushSize * curScale}px`, left: `${_x}px`, top: `${_y}px`, transform: 'translate(-50%, -50%)', } } const handleSliderChange = (value: number) => { setBrushSize(value) if (!showRefBrush) { setShowRefBrush(true) window.setTimeout(() => { setShowRefBrush(false) }, 10000) } } return (