lots update

2023-12-27 22:00:07 +08:00
parent 0ba6c121e0
commit f0b852725f
33 changed files with 4085 additions and 1000 deletions
--- a/web_app/src/components/Extender.tsx
+++ b/web_app/src/components/Extender.tsx
@@ -1,5 +1,5 @@
-import { EXTENDER_ALL, EXTENDER_X, EXTENDER_Y } from "@/lib/const"
 import { useStore } from "@/lib/states"
+import { ExtenderDirection } from "@/lib/types"
 import { cn } from "@/lib/utils"
 import React, { useEffect, useState } from "react"
 import { twMerge } from "tailwind-merge"
@@ -107,7 +107,7 @@ const Extender = (props: Props) => {
      const newY = evData.initY + offsetY
      let clampedY = newY
      let clampedHeight = newHeight
-      if (extenderDirection === EXTENDER_ALL) {
+      if (extenderDirection === ExtenderDirection.xy) {
        if (clampedY > 0) {
          clampedY = 0
          clampedHeight = evData.initHeight - Math.abs(evData.initY)
@@ -124,7 +124,7 @@ const Extender = (props: Props) => {
    const moveBottom = () => {
      const newHeight = evData.initHeight + offsetY
      let [clampedY, clampedHeight] = clampTopBottom(evData.initY, newHeight)
-      if (extenderDirection === EXTENDER_ALL) {
+      if (extenderDirection === ExtenderDirection.xy) {
        if (clampedHeight < Math.abs(clampedY) + imageHeight) {
          clampedHeight = Math.abs(clampedY) + imageHeight
        }
@@ -138,7 +138,7 @@ const Extender = (props: Props) => {
      const newX = evData.initX + offsetX
      let clampedX = newX
      let clampedWidth = newWidth
-      if (extenderDirection === EXTENDER_ALL) {
+      if (extenderDirection === ExtenderDirection.xy) {
        if (clampedX > 0) {
          clampedX = 0
          clampedWidth = evData.initWidth - Math.abs(evData.initX)
@@ -155,7 +155,7 @@ const Extender = (props: Props) => {
    const moveRight = () => {
      const newWidth = evData.initWidth + offsetX
      let [clampedX, clampedWidth] = clampLeftRight(evData.initX, newWidth)
-      if (extenderDirection === EXTENDER_ALL) {
+      if (extenderDirection === ExtenderDirection.xy) {
        if (clampedWidth < Math.abs(clampedX) + imageWdith) {
          clampedWidth = Math.abs(clampedX) + imageWdith
        }
@@ -296,7 +296,9 @@ const Extender = (props: Props) => {
        onPointerDown={onCropPointerDown}
        className="absolute top-0 h-full w-full"
      >
-        {[EXTENDER_Y, EXTENDER_ALL].includes(extenderDirection) ? (
+        {[ExtenderDirection.y, ExtenderDirection.xy].includes(
+          extenderDirection
+        ) ? (
          <>
            <div
              className="absolute pointer-events-auto top-0 left-0 w-full cursor-ns-resize h-[12px] mt-[-6px]"
@@ -313,7 +315,9 @@ const Extender = (props: Props) => {
          <></>
        )}

-        {[EXTENDER_X, EXTENDER_ALL].includes(extenderDirection) ? (
+        {[ExtenderDirection.x, ExtenderDirection.xy].includes(
+          extenderDirection
+        ) ? (
          <>
            <div
              className="absolute pointer-events-auto top-0 right-0 h-full cursor-ew-resize w-[12px] mr-[-6px]"
@@ -330,7 +334,7 @@ const Extender = (props: Props) => {
          <></>
        )}

-        {extenderDirection === EXTENDER_ALL ? (
+        {extenderDirection === ExtenderDirection.xy ? (
          <>
            {createDragHandle("cursor-nw-resize", "top", "left")}
            {createDragHandle("cursor-ne-resize", "top", "right")}
--- a/web_app/src/components/PromptInput.tsx
+++ b/web_app/src/components/PromptInput.tsx
@@ -36,9 +36,9 @@ const PromptInput = () => {
    updateSettings({ prompt: target.value })
  }

-  const handleRepaintClick = async () => {
-    if (prompt.length !== 0 && !isProcessing) {
-      await runInpainting()
+  const handleRepaintClick = () => {
+    if (!isProcessing) {
+      runInpainting()
    }
  }

@@ -69,7 +69,7 @@ const PromptInput = () => {
      <Button
        size="sm"
        onClick={handleRepaintClick}
-        disabled={prompt.length === 0 || isProcessing}
+        disabled={isProcessing}
        onMouseEnter={onMouseEnter}
        onMouseLeave={onMouseLeave}
      >
--- a/web_app/src/components/Settings.tsx
+++ b/web_app/src/components/Settings.tsx
@@ -1,7 +1,7 @@
 import { IconButton } from "@/components/ui/button"
 import { useToggle } from "@uidotdev/usehooks"
 import { Dialog, DialogContent, DialogTitle, DialogTrigger } from "./ui/dialog"
-import { HelpCircle, Settings } from "lucide-react"
+import { Settings } from "lucide-react"
 import { zodResolver } from "@hookform/resolvers/zod"
 import { useForm } from "react-hook-form"
 import * as z from "zod"
@@ -179,12 +179,12 @@ export function SettingsDialog() {
          <div key={info.name} onClick={() => onModelSelect(info)}>
            <div
              className={cn([
-                info.name === model.name ? "bg-muted " : "hover:bg-muted",
+                info.name === model.name ? "bg-muted" : "hover:bg-muted",
                "rounded-md px-2 py-1 my-1",
                "cursor-default",
              ])}
            >
-              <div className="text-base max-w-sm">{info.name}</div>
+              <div className="text-base">{info.name}</div>
            </div>
            <Separator />
          </div>
@@ -223,13 +223,13 @@ export function SettingsDialog() {
        <div className="space-y-4  rounded-md">
          <div className="flex gap-1 items-center justify-start">
            <div className="font-medium">Available models</div>
-            {/* <IconButton tooltip="How to download new model" asChild>
-              <HelpCircle size={16} strokeWidth={1.5} className="opacity-50" />
+            {/* <IconButton tooltip="How to download new model">
+              <Info size={20} strokeWidth={2} className="opacity-50" />
            </IconButton> */}
          </div>
          <Tabs defaultValue={defaultTab}>
            <TabsList>
-              <TabsTrigger value={MODEL_TYPE_INPAINT}>Erase</TabsTrigger>
+              <TabsTrigger value={MODEL_TYPE_INPAINT}>Inpaint</TabsTrigger>
              <TabsTrigger value={MODEL_TYPE_DIFFUSERS_SD}>
                Stable Diffusion
              </TabsTrigger>
--- a/web_app/src/components/SidePanel/DiffusionOptions.tsx
+++ b/web_app/src/components/SidePanel/DiffusionOptions.tsx
@@ -11,21 +11,14 @@ import {
  SelectValue,
 } from "../ui/select"
 import { Textarea } from "../ui/textarea"
-import { SDSampler } from "@/lib/types"
+import { ExtenderDirection, PowerPaintTask, SDSampler } from "@/lib/types"
 import { Separator } from "../ui/separator"
-import { Move, MoveHorizontal, MoveVertical, Upload } from "lucide-react"
 import { Button, ImageUploadButton } from "../ui/button"
 import { Slider } from "../ui/slider"
 import { useImage } from "@/hooks/useImage"
-import {
-  EXTENDER_ALL,
-  EXTENDER_X,
-  EXTENDER_Y,
-  INSTRUCT_PIX2PIX,
-  PAINT_BY_EXAMPLE,
-} from "@/lib/const"
-import { Tabs, TabsContent, TabsList, TabsTrigger } from "../ui/tabs"
+import { INSTRUCT_PIX2PIX, PAINT_BY_EXAMPLE, POWERPAINT } from "@/lib/const"
 import { RowContainer, LabelTitle } from "./LabelTitle"
+import { Upload } from "lucide-react"

 const ExtenderButton = ({
  text,
@@ -38,8 +31,7 @@ const ExtenderButton = ({
  return (
    <Button
      variant="outline"
-      size="sm"
-      className="p-1"
+      className="p-1 h-7"
      disabled={!showExtender}
      onClick={onClick}
    >
@@ -129,6 +121,7 @@ const DiffusionOptions = () => {

          <div className="pr-2">
            <Select
+              defaultValue={settings.controlnetMethod}
              value={settings.controlnetMethod}
              onValueChange={(value) => {
                updateSettings({ controlnetMethod: value })
@@ -467,96 +460,104 @@ const DiffusionOptions = () => {
            />
          </RowContainer>

-          <Tabs
-            defaultValue={settings.extenderDirection}
-            onValueChange={(value) => updateExtenderDirection(value)}
-            className="flex flex-col justify-center items-center"
-          >
-            <TabsList className="w-[140px] mb-2">
-              <TabsTrigger value={EXTENDER_X} disabled={!settings.showExtender}>
-                <MoveHorizontal size={20} strokeWidth={1} />
-              </TabsTrigger>
-              <TabsTrigger value={EXTENDER_Y} disabled={!settings.showExtender}>
-                <MoveVertical size={20} strokeWidth={1} />
-              </TabsTrigger>
-              <TabsTrigger
-                value={EXTENDER_ALL}
+          <RowContainer>
+            <Select
+              defaultValue={settings.extenderDirection}
+              value={settings.extenderDirection}
+              onValueChange={(value) => {
+                updateExtenderDirection(value as ExtenderDirection)
+              }}
+            >
+              <SelectTrigger
+                className="w-[65px] h-7"
                disabled={!settings.showExtender}
              >
-                <Move size={20} strokeWidth={1} />
-              </TabsTrigger>
-            </TabsList>
+                <SelectValue placeholder="Select axis" />
+              </SelectTrigger>
+              <SelectContent align="end">
+                <SelectGroup>
+                  {Object.values(ExtenderDirection).map((v) => (
+                    <SelectItem key={v} value={v}>
+                      {v}
+                    </SelectItem>
+                  ))}
+                </SelectGroup>
+              </SelectContent>
+            </Select>

-            <TabsContent
-              value={EXTENDER_X}
-              className="flex gap-2 justify-center mt-0"
-            >
+            <div className="flex gap-1 justify-center mt-0">
              <ExtenderButton
                text="1.25x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_X, 1.25)}
+                onClick={() =>
+                  updateExtenderByBuiltIn(settings.extenderDirection, 1.25)
+                }
              />
              <ExtenderButton
                text="1.5x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_X, 1.5)}
+                onClick={() =>
+                  updateExtenderByBuiltIn(settings.extenderDirection, 1.5)
+                }
              />
              <ExtenderButton
                text="1.75x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_X, 1.75)}
+                onClick={() =>
+                  updateExtenderByBuiltIn(settings.extenderDirection, 1.75)
+                }
              />
              <ExtenderButton
                text="2.0x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_X, 2.0)}
+                onClick={() =>
+                  updateExtenderByBuiltIn(settings.extenderDirection, 2.0)
+                }
              />
-            </TabsContent>
-            <TabsContent
-              value={EXTENDER_Y}
-              className="flex gap-2 justify-center mt-0"
-            >
-              <ExtenderButton
-                text="1.25x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_Y, 1.25)}
-              />
-              <ExtenderButton
-                text="1.5x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_Y, 1.5)}
-              />
-              <ExtenderButton
-                text="1.75x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_Y, 1.75)}
-              />
-              <ExtenderButton
-                text="2.0x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_Y, 2.0)}
-              />
-            </TabsContent>
-            <TabsContent
-              value={EXTENDER_ALL}
-              className="flex gap-2 justify-center mt-0"
-            >
-              <ExtenderButton
-                text="1.25x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_ALL, 1.25)}
-              />
-              <ExtenderButton
-                text="1.5x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_ALL, 1.5)}
-              />
-              <ExtenderButton
-                text="1.75x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_ALL, 1.75)}
-              />
-              <ExtenderButton
-                text="2.0x"
-                onClick={() => updateExtenderByBuiltIn(EXTENDER_ALL, 2.0)}
-              />
-            </TabsContent>
-          </Tabs>
+            </div>
+          </RowContainer>
        </div>
        <Separator />
      </>
    )
  }

+  const renderPowerPaintTaskType = () => {
+    if (settings.model.name !== POWERPAINT) {
+      return null
+    }
+
+    return (
+      <RowContainer>
+        <LabelTitle
+          text="Task"
+          toolTip="When using extender, image-outpainting task will be auto used. For object-removal and image-outpainting, it is recommended to set the guidance_scale at 10 or above."
+        />
+        <Select
+          defaultValue={settings.powerpaintTask}
+          value={settings.powerpaintTask}
+          onValueChange={(value: PowerPaintTask) => {
+            updateSettings({ powerpaintTask: value })
+          }}
+          disabled={settings.showExtender}
+        >
+          <SelectTrigger className="w-[140px]">
+            <SelectValue placeholder="Select task" />
+          </SelectTrigger>
+          <SelectContent align="end">
+            <SelectGroup>
+              {[
+                PowerPaintTask.text_guided,
+                PowerPaintTask.object_remove,
+                PowerPaintTask.shape_guided,
+              ].map((task) => (
+                <SelectItem key={task} value={task}>
+                  {task}
+                </SelectItem>
+              ))}
+            </SelectGroup>
+          </SelectContent>
+        </Select>
+      </RowContainer>
+    )
+  }
+
  return (
    <div className="flex flex-col gap-4 mt-4">
      <RowContainer>
@@ -577,6 +578,7 @@ const DiffusionOptions = () => {
      </RowContainer>

      {renderExtender()}
+      {renderPowerPaintTaskType()}

      <div className="flex flex-col gap-1">
        <LabelTitle
@@ -642,20 +644,20 @@ const DiffusionOptions = () => {
      <RowContainer>
        <LabelTitle text="Sampler" />
        <Select
-          value={settings.sdSampler as string}
-          onValueChange={(value) => {
-            const sampler = value as SDSampler
-            updateSettings({ sdSampler: sampler })
+          defaultValue={settings.sdSampler}
+          value={settings.sdSampler}
+          onValueChange={(value: SDSampler) => {
+            updateSettings({ sdSampler: value })
          }}
        >
-          <SelectTrigger className="w-[100px]">
+          <SelectTrigger className="w-[120px]">
            <SelectValue placeholder="Select sampler" />
          </SelectTrigger>
          <SelectContent align="end">
            <SelectGroup>
              {Object.values(SDSampler).map((sampler) => (
-                <SelectItem key={sampler as string} value={sampler as string}>
-                  {sampler as string}
+                <SelectItem key={sampler} value={sampler}>
+                  {sampler}
                </SelectItem>
              ))}
            </SelectGroup>
@@ -707,9 +709,9 @@ const DiffusionOptions = () => {
        <RowContainer>
          <Slider
            className="w-[180px]"
-            defaultValue={[5]}
+            defaultValue={[settings.sdMaskBlur]}
            min={0}
-            max={35}
+            max={96}
            step={1}
            value={[Math.floor(settings.sdMaskBlur)]}
            onValueChange={(vals) => updateSettings({ sdMaskBlur: vals[0] })}
--- a/web_app/src/lib/api.ts
+++ b/web_app/src/lib/api.ts
@@ -1,4 +1,4 @@
-import { ModelInfo, Rect } from "@/lib/types"
+import { ModelInfo, PowerPaintTask, Rect } from "@/lib/types"
 import { Settings } from "@/lib/states"
 import { srcToFile } from "@/lib/utils"
 import axios from "axios"
@@ -22,7 +22,6 @@ export default async function inpaint(
  const fd = new FormData()
  fd.append("image", imageFile)
  fd.append("mask", mask)
-
  fd.append("ldmSteps", settings.ldmSteps.toString())
  fd.append("ldmSampler", settings.ldmSampler.toString())
  fd.append("zitsWireframe", settings.zitsWireframe.toString())
@@ -51,6 +50,7 @@ export default async function inpaint(
  fd.append("sdSteps", settings.sdSteps.toString())
  fd.append("sdGuidanceScale", settings.sdGuidanceScale.toString())
  fd.append("sdSampler", settings.sdSampler.toString())
+
  if (settings.seedFixed) {
    fd.append("sdSeed", settings.seed.toString())
  } else {
@@ -76,13 +76,20 @@ export default async function inpaint(
  fd.append("p2pImageGuidanceScale", settings.p2pImageGuidanceScale.toString())

  // ControlNet
-  fd.append("controlnet_enabled", settings.enableControlnet.toString())
+  fd.append("enable_controlnet", settings.enableControlnet.toString())
  fd.append(
    "controlnet_conditioning_scale",
    settings.controlnetConditioningScale.toString()
  )
  fd.append("controlnet_method", settings.controlnetMethod.toString())

+  // PowerPaint
+  if (settings.showExtender) {
+    fd.append("powerpaintTask", PowerPaintTask.outpainting)
+  } else {
+    fd.append("powerpaintTask", settings.powerpaintTask)
+  }
+
  try {
    const res = await fetch(`${API_ENDPOINT}/inpaint`, {
      method: "POST",
--- a/web_app/src/lib/const.ts
+++ b/web_app/src/lib/const.ts
@@ -8,14 +8,13 @@ export const MODEL_TYPE_DIFFUSERS_SDXL_INPAINT = "diffusers_sdxl_inpaint"
 export const MODEL_TYPE_OTHER = "diffusers_other"
 export const BRUSH_COLOR = "#ffcc00bb"

-export const EXTENDER_X = "extender_x"
-export const EXTENDER_Y = "extender_y"
-export const EXTENDER_ALL = "extender_all"
-
 export const LDM = "ldm"
 export const CV2 = "cv2"
+
 export const PAINT_BY_EXAMPLE = "Fantasy-Studio/Paint-by-Example"
 export const INSTRUCT_PIX2PIX = "timbrooks/instruct-pix2pix"
 export const KANDINSKY_2_2 = "kandinsky-community/kandinsky-2-2-decoder-inpaint"
+export const POWERPAINT = "Sanster/PowerPaint-V1-stable-diffusion-inpainting"
+
 export const DEFAULT_NEGATIVE_PROMPT =
  "out of frame, lowres, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, disfigured, gross proportions, malformed limbs, watermark, signature"
--- a/web_app/src/lib/states.ts
+++ b/web_app/src/lib/states.ts
@@ -5,6 +5,7 @@ import { castDraft } from "immer"
 import { createWithEqualityFn } from "zustand/traditional"
 import {
  CV2Flag,
+  ExtenderDirection,
  FreeuConfig,
  LDMSampler,
  Line,
@@ -12,6 +13,7 @@ import {
  ModelInfo,
  PluginParams,
  Point,
+  PowerPaintTask,
  SDSampler,
  Size,
  SortBy,
@@ -21,9 +23,6 @@ import {
  BRUSH_COLOR,
  DEFAULT_BRUSH_SIZE,
  DEFAULT_NEGATIVE_PROMPT,
-  EXTENDER_ALL,
-  EXTENDER_X,
-  EXTENDER_Y,
  MODEL_TYPE_INPAINT,
  PAINT_BY_EXAMPLE,
 } from "./const"
@@ -60,7 +59,7 @@ export type Settings = {
  enableUploadMask: boolean
  showCropper: boolean
  showExtender: boolean
-  extenderDirection: string
+  extenderDirection: ExtenderDirection

  // For LDM
  ldmSteps: number
@@ -99,6 +98,9 @@ export type Settings = {
  enableLCMLora: boolean
  enableFreeu: boolean
  freeuConfig: FreeuConfig
+
+  // PowerPaint
+  powerpaintTask: PowerPaintTask
 }

 type ServerConfig = {
@@ -178,9 +180,9 @@ type AppAction = {
  setExtenderWidth: (newValue: number) => void
  setExtenderHeight: (newValue: number) => void
  setIsCropperExtenderResizing: (newValue: boolean) => void
-  updateExtenderDirection: (newValue: string) => void
+  updateExtenderDirection: (newValue: ExtenderDirection) => void
  resetExtender: (width: number, height: number) => void
-  updateExtenderByBuiltIn: (direction: string, scale: number) => void
+  updateExtenderByBuiltIn: (direction: ExtenderDirection, scale: number) => void

  setServerConfig: (newValue: ServerConfig) => void
  setSeed: (newValue: number) => void
@@ -296,7 +298,7 @@ const defaultValues: AppState = {
    enableControlnet: false,
    showCropper: false,
    showExtender: false,
-    extenderDirection: EXTENDER_ALL,
+    extenderDirection: ExtenderDirection.xy,
    enableDownloadMask: false,
    enableManualInpainting: false,
    enableUploadMask: false,
@@ -309,7 +311,7 @@ const defaultValues: AppState = {
    negativePrompt: DEFAULT_NEGATIVE_PROMPT,
    seed: 42,
    seedFixed: false,
-    sdMaskBlur: 5,
+    sdMaskBlur: 35,
    sdStrength: 1.0,
    sdSteps: 50,
    sdGuidanceScale: 7.5,
@@ -322,6 +324,7 @@ const defaultValues: AppState = {
    enableLCMLora: false,
    enableFreeu: false,
    freeuConfig: { s1: 0.9, s2: 0.2, b1: 1.2, b2: 1.4 },
+    powerpaintTask: PowerPaintTask.text_guided,
  },
 }

@@ -894,7 +897,7 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
          state.isCropperExtenderResizing = newValue
        }),

-      updateExtenderDirection: (newValue: string) => {
+      updateExtenderDirection: (newValue: ExtenderDirection) => {
        console.log(
          `updateExtenderDirection: ${JSON.stringify(get().extenderState)}`
        )
@@ -908,7 +911,10 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
        get().updateExtenderByBuiltIn(newValue, 1.5)
      },

-      updateExtenderByBuiltIn: (direction: string, scale: number) => {
+      updateExtenderByBuiltIn: (
+        direction: ExtenderDirection,
+        scale: number
+      ) => {
        const newExtenderState = { ...defaultValues.extenderState }
        let { x, y, width, height } = newExtenderState
        const { imageWidth, imageHeight } = get()
@@ -916,15 +922,15 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
        height = imageHeight

        switch (direction) {
-          case EXTENDER_X:
+          case ExtenderDirection.x:
            x = -Math.ceil((imageWidth * (scale - 1)) / 2)
            width = Math.ceil(imageWidth * scale)
            break
-          case EXTENDER_Y:
+          case ExtenderDirection.y:
            y = -Math.ceil((imageHeight * (scale - 1)) / 2)
            height = Math.ceil(imageHeight * scale)
            break
-          case EXTENDER_ALL:
+          case ExtenderDirection.xy:
            x = -Math.ceil((imageWidth * (scale - 1)) / 2)
            y = -Math.ceil((imageHeight * (scale - 1)) / 2)
            width = Math.ceil(imageWidth * scale)
--- a/web_app/src/lib/types.ts
+++ b/web_app/src/lib/types.ts
@@ -93,3 +93,16 @@ export interface Size {
  width: number
  height: number
 }
+
+export enum ExtenderDirection {
+  x = "x",
+  y = "y",
+  xy = "xy",
+}
+
+export enum PowerPaintTask {
+  text_guided = "text-guided",
+  shape_guided = "shape-guided",
+  object_remove = "object-remove",
+  outpainting = "outpainting",
+}