update plugins

2024-01-02 11:07:35 +08:00
parent b0e009f879
commit a2fd5bb3ea
19 changed files with 337 additions and 227 deletions
--- a/web_app/src/App.tsx
+++ b/web_app/src/App.tsx
@@ -42,7 +42,7 @@ function Home() {

  useEffect(() => {
    const fetchServerConfig = async () => {
-      const serverConfig = await getServerConfig().then((res) => res.json())
+      const serverConfig = await getServerConfig()
      setServerConfig(serverConfig)
      if (serverConfig.isDesktop) {
        // Keeping GUI Window Open
--- a/web_app/src/components/Editor.tsx
+++ b/web_app/src/components/Editor.tsx
@@ -363,9 +363,6 @@ export default function Editor(props: EditorProps) {
        undefined,
        newClicks
      )
-      if (!res) {
-        throw new Error("Something went wrong on server side.")
-      }
      const { blob } = res
      const img = new Image()
      img.onload = () => {
--- a/web_app/src/components/FileManager.tsx
+++ b/web_app/src/components/FileManager.tsx
@@ -78,6 +78,7 @@ export default function FileManager(props: Props) {
  const ref = useRef(null)
  const debouncedSearchText = useDebounce(fileManagerState.searchText, 300)
  const [tab, setTab] = useState(IMAGE_TAB)
+  const [filenames, setFilenames] = useState<Filename[]>([])
  const [photos, setPhotos] = useState<Photo[]>([])
  const [photoIndex, setPhotoIndex] = useState(0)

@@ -131,13 +132,28 @@ export default function FileManager(props: Props) {
    [open, closeScrollTop]
  )

+  useEffect(() => {
+    const fetchData = async () => {
+      try {
+        const filenames = await getMedias(tab)
+        setFilenames(filenames)
+      } catch (e: any) {
+        toast({
+          variant: "destructive",
+          title: "Uh oh! Something went wrong.",
+          description: e.message ? e.message : e.toString(),
+        })
+      }
+    }
+    fetchData()
+  }, [tab])
+
  useEffect(() => {
    if (!open) {
      return
    }
    const fetchData = async () => {
      try {
-        const filenames = await getMedias(tab)
        let filteredFilenames = filenames
        if (debouncedSearchText) {
          const fuse = new Fuse(filteredFilenames, {
@@ -173,7 +189,7 @@ export default function FileManager(props: Props) {
      }
    }
    fetchData()
-  }, [tab, debouncedSearchText, fileManagerState, photoWidth, open])
+  }, [filenames, debouncedSearchText, fileManagerState, photoWidth, open])

  const onScroll = (event: SyntheticEvent) => {
    setScrollTop(event.currentTarget.scrollTop)
--- a/web_app/src/components/Settings.tsx
+++ b/web_app/src/components/Settings.tsx
@@ -99,7 +99,7 @@ export function SettingsDialog() {
    },
  })

-  function onSubmit(values: z.infer<typeof formSchema>) {
+  async function onSubmit(values: z.infer<typeof formSchema>) {
    // Do something with the form values. ✅ This will be type-safe and validated.
    updateSettings({
      enableDownloadMask: values.enableDownloadMask,
@@ -116,24 +116,22 @@ export function SettingsDialog() {
    if (model.name !== settings.model.name) {
      toggleOpenModelSwitching()
      updateAppState({ disableShortCuts: true })
-      switchModel(model.name)
-        .then((res) => {
-          toast({
-            title: `Switch to ${model.name} success`,
-          })
-          setAppModel(model)
+      try {
+        const newModel = await switchModel(model.name)
+        toast({
+          title: `Switch to ${newModel.name} success`,
        })
-        .catch((error: any) => {
-          toast({
-            variant: "destructive",
-            title: `Switch to ${model.name} failed: ${error}`,
-          })
-          setModel(settings.model)
-        })
-        .finally(() => {
-          toggleOpenModelSwitching()
-          updateAppState({ disableShortCuts: false })
+        setAppModel(model)
+      } catch (error: any) {
+        toast({
+          variant: "destructive",
+          title: `Switch to ${model.name} failed: ${error}`,
        })
+        setModel(settings.model)
+      } finally {
+        toggleOpenModelSwitching()
+        updateAppState({ disableShortCuts: false })
+      }
    }
  }

--- a/web_app/src/components/SidePanel/DiffusionOptions.tsx
+++ b/web_app/src/components/SidePanel/DiffusionOptions.tsx
@@ -69,6 +69,27 @@ const DiffusionOptions = () => {
    }
  }

+  const renderCropper = () => {
+    return (
+      <RowContainer>
+        <LabelTitle
+          text="Cropper"
+          toolTip="Inpainting on part of image, improve inference speed and reduce memory usage."
+        />
+        <Switch
+          id="cropper"
+          checked={settings.showCropper}
+          onCheckedChange={(value) => {
+            updateSettings({ showCropper: value })
+            if (value) {
+              updateSettings({ showExtender: false })
+            }
+          }}
+        />
+      </RowContainer>
+    )
+  }
+
  const renderConterNetSetting = () => {
    if (!settings.model.support_controlnet) {
      return null
@@ -558,28 +579,8 @@ const DiffusionOptions = () => {
    )
  }

-  return (
-    <div className="flex flex-col gap-4 mt-4">
-      <RowContainer>
-        <LabelTitle
-          text="Cropper"
-          toolTip="Inpainting on part of image, improve inference speed and reduce memory usage."
-        />
-        <Switch
-          id="cropper"
-          checked={settings.showCropper}
-          onCheckedChange={(value) => {
-            updateSettings({ showCropper: value })
-            if (value) {
-              updateSettings({ showExtender: false })
-            }
-          }}
-        />
-      </RowContainer>
-
-      {renderExtender()}
-      {renderPowerPaintTaskType()}
-
+  const renderSteps = () => {
+    return (
      <div className="flex flex-col gap-1">
        <LabelTitle
          htmlFor="steps"
@@ -607,7 +608,11 @@ const DiffusionOptions = () => {
          />
        </RowContainer>
      </div>
+    )
+  }

+  const renderGuidanceScale = () => {
+    return (
      <div className="flex flex-col gap-1">
        <LabelTitle
          text="Guidance scale"
@@ -637,10 +642,11 @@ const DiffusionOptions = () => {
          />
        </RowContainer>
      </div>
+    )
+  }

-      {renderP2PImageGuidanceScale()}
-      {renderStrength()}
-
+  const renderSampler = () => {
+    return (
      <RowContainer>
        <LabelTitle text="Sampler" />
        <Select
@@ -664,7 +670,11 @@ const DiffusionOptions = () => {
          </SelectContent>
        </Select>
      </RowContainer>
+    )
+  }

+  const renderSeed = () => {
+    return (
      <RowContainer>
        {/* 每次会从服务器返回更新该值 */}
        <LabelTitle
@@ -692,15 +702,11 @@ const DiffusionOptions = () => {
          />
        </div>
      </RowContainer>
+    )
+  }

-      {renderNegativePrompt()}
-
-      <Separator />
-
-      {renderConterNetSetting()}
-      {renderFreeu()}
-      {renderLCMLora()}
-
+  const renderMaskBlur = () => {
+    return (
      <div className="flex flex-col gap-1">
        <LabelTitle
          text="Mask blur"
@@ -727,24 +733,49 @@ const DiffusionOptions = () => {
          />
        </RowContainer>
      </div>
+    )
+  }

-      <RowContainer>
-        <LabelTitle
-          text="Match histograms"
-          toolTip="Match the inpainting result histogram to the source image histogram"
-          url="https://github.com/Sanster/lama-cleaner/pull/143#issuecomment-1325859307"
-        />
-        <Switch
-          id="match-histograms"
-          checked={settings.sdMatchHistograms}
-          onCheckedChange={(value) => {
-            updateSettings({ sdMatchHistograms: value })
-          }}
-        />
-      </RowContainer>
+  const renderMatchHistograms = () => {
+    return (
+      <>
+        <RowContainer>
+          <LabelTitle
+            text="Match histograms"
+            toolTip="Match the inpainting result histogram to the source image histogram"
+            url="https://github.com/Sanster/lama-cleaner/pull/143#issuecomment-1325859307"
+          />
+          <Switch
+            id="match-histograms"
+            checked={settings.sdMatchHistograms}
+            onCheckedChange={(value) => {
+              updateSettings({ sdMatchHistograms: value })
+            }}
+          />
+        </RowContainer>
+        <Separator />
+      </>
+    )
+  }

+  return (
+    <div className="flex flex-col gap-4 mt-4">
+      {renderCropper()}
+      {renderExtender()}
+      {renderPowerPaintTaskType()}
+      {renderSteps()}
+      {renderGuidanceScale()}
+      {renderP2PImageGuidanceScale()}
+      {renderStrength()}
+      {renderSampler()}
+      {renderSeed()}
+      {renderNegativePrompt()}
      <Separator />
-
+      {renderConterNetSetting()}
+      {renderLCMLora()}
+      {renderMaskBlur()}
+      {renderMatchHistograms()}
+      {renderFreeu()}
      {renderPaintByExample()}
    </div>
  )
--- a/web_app/src/components/Workspace.tsx
+++ b/web_app/src/components/Workspace.tsx
@@ -14,11 +14,11 @@ const Workspace = () => {
  ])

  useEffect(() => {
-    currentModel()
-      .then((res) => res.json())
-      .then((model) => {
-        updateSettings({ model })
-      })
+    const fetchCurrentModel = async () => {
+      const model = await currentModel()
+      updateSettings({ model })
+    }
+    fetchCurrentModel()
  }, [])

  return (
--- a/web_app/src/lib/api.ts
+++ b/web_app/src/lib/api.ts
@@ -1,4 +1,11 @@
-import { Filename, ModelInfo, PowerPaintTask, Rect } from "@/lib/types"
+import {
+  Filename,
+  GenInfo,
+  ModelInfo,
+  PowerPaintTask,
+  Rect,
+  ServerConfig,
+} from "@/lib/types"
 import { Settings } from "@/lib/states"
 import { convertToBase64, srcToFile } from "@/lib/utils"
 import axios from "axios"
@@ -24,124 +31,113 @@ export default async function inpaint(
  const exampleImageBase64 = paintByExampleImage
    ? await convertToBase64(paintByExampleImage)
    : null
-  try {
-    const res = await fetch(`${API_ENDPOINT}/inpaint`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        image: imageBase64,
-        mask: maskBase64,
-        ldm_steps: settings.ldmSteps,
-        ldm_sampler: settings.ldmSampler,
-        zits_wireframe: settings.zitsWireframe,
-        cv2_flag: settings.cv2Flag,
-        cv2_radius: settings.cv2Radius,
-        hd_strategy: "Crop",
-        hd_strategy_crop_triger_size: 640,
-        hd_strategy_crop_margin: 128,
-        hd_trategy_resize_imit: 2048,
-        prompt: settings.prompt,
-        negative_prompt: settings.negativePrompt,
-        use_croper: settings.showCropper,
-        croper_x: croperRect.x,
-        croper_y: croperRect.y,
-        croper_height: croperRect.height,
-        croper_width: croperRect.width,
-        use_extender: settings.showExtender,
-        extender_x: extenderState.x,
-        extender_y: extenderState.y,
-        extender_height: extenderState.height,
-        extender_width: extenderState.width,
-        sd_mask_blur: settings.sdMaskBlur,
-        sd_strength: settings.sdStrength,
-        sd_steps: settings.sdSteps,
-        sd_guidance_scale: settings.sdGuidanceScale,
-        sd_sampler: settings.sdSampler,
-        sd_seed: settings.seedFixed ? settings.seed : -1,
-        sd_match_histograms: settings.sdMatchHistograms,
-        sd_freeu: settings.enableFreeu,
-        sd_freeu_config: settings.freeuConfig,
-        sd_lcm_lora: settings.enableLCMLora,
-        paint_by_example_example_image: exampleImageBase64,
-        p2p_image_guidance_scale: settings.p2pImageGuidanceScale,
-        enable_controlnet: settings.enableControlnet,
-        controlnet_conditioning_scale: settings.controlnetConditioningScale,
-        controlnet_method: settings.controlnetMethod
-          ? settings.controlnetMethod
-          : "",
-        powerpaint_task: settings.showExtender
-          ? PowerPaintTask.outpainting
-          : settings.powerpaintTask,
-      }),
-    })
+
+  const res = await fetch(`${API_ENDPOINT}/inpaint`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      image: imageBase64,
+      mask: maskBase64,
+      ldm_steps: settings.ldmSteps,
+      ldm_sampler: settings.ldmSampler,
+      zits_wireframe: settings.zitsWireframe,
+      cv2_flag: settings.cv2Flag,
+      cv2_radius: settings.cv2Radius,
+      hd_strategy: "Crop",
+      hd_strategy_crop_triger_size: 640,
+      hd_strategy_crop_margin: 128,
+      hd_trategy_resize_imit: 2048,
+      prompt: settings.prompt,
+      negative_prompt: settings.negativePrompt,
+      use_croper: settings.showCropper,
+      croper_x: croperRect.x,
+      croper_y: croperRect.y,
+      croper_height: croperRect.height,
+      croper_width: croperRect.width,
+      use_extender: settings.showExtender,
+      extender_x: extenderState.x,
+      extender_y: extenderState.y,
+      extender_height: extenderState.height,
+      extender_width: extenderState.width,
+      sd_mask_blur: settings.sdMaskBlur,
+      sd_strength: settings.sdStrength,
+      sd_steps: settings.sdSteps,
+      sd_guidance_scale: settings.sdGuidanceScale,
+      sd_sampler: settings.sdSampler,
+      sd_seed: settings.seedFixed ? settings.seed : -1,
+      sd_match_histograms: settings.sdMatchHistograms,
+      sd_freeu: settings.enableFreeu,
+      sd_freeu_config: settings.freeuConfig,
+      sd_lcm_lora: settings.enableLCMLora,
+      paint_by_example_example_image: exampleImageBase64,
+      p2p_image_guidance_scale: settings.p2pImageGuidanceScale,
+      enable_controlnet: settings.enableControlnet,
+      controlnet_conditioning_scale: settings.controlnetConditioningScale,
+      controlnet_method: settings.controlnetMethod
+        ? settings.controlnetMethod
+        : "",
+      powerpaint_task: settings.showExtender
+        ? PowerPaintTask.outpainting
+        : settings.powerpaintTask,
+    }),
+  })
+  if (res.ok) {
    const blob = await res.blob()
    return {
      blob: URL.createObjectURL(blob),
      seed: res.headers.get("X-Seed"),
    }
-  } catch (error: any) {
-    throw new Error(`Something went wrong: ${JSON.stringify(error.message)}`)
  }
+  const errors = await res.json()
+  throw new Error(`Something went wrong: ${errors.errors}`)
 }

-export function getServerConfig() {
-  return fetch(`${API_ENDPOINT}/server-config`, {
-    method: "GET",
-  })
+export async function getServerConfig(): Promise<ServerConfig> {
+  const res = await api.get(`/server-config`)
+  return res.data
 }

-export function switchModel(name: string) {
-  return axios.post(`${API_ENDPOINT}/model`, { name })
+export async function switchModel(name: string): Promise<ModelInfo> {
+  const res = await api.post(`/model`, { name })
+  return res.data
 }

-export function currentModel() {
-  return fetch(`${API_ENDPOINT}/model`, {
-    method: "GET",
-  })
+export async function currentModel(): Promise<ModelInfo> {
+  const res = await api.get("/model")
+  return res.data
 }

 export function fetchModelInfos(): Promise<ModelInfo[]> {
  return api.get("/models").then((response) => response.data)
 }

-export function modelDownloaded(name: string) {
-  return fetch(`${API_ENDPOINT}/model_downloaded/${name}`, {
-    method: "GET",
-  })
-}
-
 export async function runPlugin(
  name: string,
  imageFile: File,
  upscale?: number,
  clicks?: number[][]
 ) {
-  const fd = new FormData()
-  fd.append("name", name)
-  fd.append("image", imageFile)
-  if (upscale) {
-    fd.append("upscale", upscale.toString())
-  }
-  if (clicks) {
-    fd.append("clicks", JSON.stringify(clicks))
-  }
-
-  try {
-    const res = await fetch(`${API_ENDPOINT}/run_plugin`, {
-      method: "POST",
-      body: fd,
-    })
-    if (res.ok) {
-      const blob = await res.blob()
-      return { blob: URL.createObjectURL(blob) }
-    }
-    const errMsg = await res.text()
-    throw new Error(errMsg)
-  } catch (error) {
-    throw new Error(`Something went wrong: ${error}`)
+  const imageBase64 = await convertToBase64(imageFile)
+  const res = await fetch(`${API_ENDPOINT}/run_plugin`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      name,
+      image: imageBase64,
+      upscale,
+      clicks,
+    }),
+  })
+  if (res.ok) {
+    const blob = await res.blob()
+    return { blob: URL.createObjectURL(blob) }
  }
+  const errMsg = await res.json()
+  throw new Error(errMsg)
 }

 export async function getMediaFile(tab: string, filename: string) {
@@ -160,12 +156,12 @@ export async function getMediaFile(tab: string, filename: string) {
    })
    return file
  }
-  const errMsg = await res.text()
-  throw new Error(errMsg)
+  const errMsg = await res.json()
+  throw new Error(errMsg.errors)
 }

 export async function getMedias(tab: string): Promise<Filename[]> {
-  const res = await axios.get(`${API_ENDPOINT}/medias`, { params: { tab } })
+  const res = await api.get(`medias`, { params: { tab } })
  return res.data
 }

@@ -191,3 +187,10 @@ export async function downloadToOutput(
    throw new Error(`Something went wrong: ${error}`)
  }
 }
+
+export async function getGenInfo(file: File): Promise<GenInfo> {
+  const fd = new FormData()
+  fd.append("file", file)
+  const res = await api.post(`/gen-info`, fd)
+  return res.data
+}
--- a/web_app/src/lib/states.ts
+++ b/web_app/src/lib/states.ts
@@ -15,6 +15,7 @@ import {
  Point,
  PowerPaintTask,
  SDSampler,
+  ServerConfig,
  Size,
  SortBy,
  SortOrder,
@@ -33,7 +34,7 @@ import {
  loadImage,
  srcToFile,
 } from "./utils"
-import inpaint, { runPlugin } from "./api"
+import inpaint, { getGenInfo, runPlugin } from "./api"
 import { toast } from "@/components/ui/use-toast"

 type FileManagerState = {
@@ -57,6 +58,7 @@ export type Settings = {
  enableDownloadMask: boolean
  enableManualInpainting: boolean
  enableUploadMask: boolean
+  enableAutoExtractPrompt: boolean
  showCropper: boolean
  showExtender: boolean
  extenderDirection: ExtenderDirection
@@ -103,16 +105,6 @@ export type Settings = {
  powerpaintTask: PowerPaintTask
 }

-type ServerConfig = {
-  plugins: string[]
-  enableFileManager: boolean
-  enableAutoSaving: boolean
-  enableControlnet: boolean
-  controlnetMethod: string
-  disableModelSwitch: boolean
-  isDesktop: boolean
-}
-
 type InteractiveSegState = {
  isInteractiveSeg: boolean
  interactiveSegMask: HTMLImageElement | null
@@ -162,7 +154,7 @@ type AppState = {

 type AppAction = {
  updateAppState: (newState: Partial<AppState>) => void
-  setFile: (file: File) => void
+  setFile: (file: File) => Promise<void>
  setCustomFile: (file: File) => void
  setIsInpainting: (newValue: boolean) => void
  setIsPluginRunning: (newValue: boolean) => void
@@ -304,6 +296,7 @@ const defaultValues: AppState = {
    enableDownloadMask: false,
    enableManualInpainting: false,
    enableUploadMask: false,
+    enableAutoExtractPrompt: true,
    ldmSteps: 30,
    ldmSampler: LDMSampler.ddim,
    zitsWireframe: true,
@@ -540,9 +533,6 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
          const start = new Date()
          const targetFile = await get().getCurrentTargetFile()
          const res = await runPlugin(pluginName, targetFile, params.upscale)
-          if (!res) {
-            throw new Error("Something went wrong on server side.")
-          }
          const { blob } = res
          const newRender = new Image()
          await loadImage(newRender, blob)
@@ -818,7 +808,27 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
          state.isPluginRunning = newValue
        }),

-      setFile: (file: File) => {
+      setFile: async (file: File) => {
+        if (get().settings.enableAutoExtractPrompt) {
+          try {
+            const res = await getGenInfo(file)
+            if (res.prompt) {
+              set((state) => {
+                state.settings.prompt = res.prompt
+              })
+            }
+            if (res.negative_prompt) {
+              set((state) => {
+                state.settings.negativePrompt = res.negative_prompt
+              })
+            }
+          } catch (e: any) {
+            toast({
+              variant: "destructive",
+              description: e.message ? e.message : e.toString(),
+            })
+          }
+        }
        set((state) => {
          state.file = file
          state.interactiveSegState = castDraft(
--- a/web_app/src/lib/types.ts
+++ b/web_app/src/lib/types.ts
@@ -6,6 +6,21 @@ export interface Filename {
  mtime: number
 }

+export interface ServerConfig {
+  plugins: string[]
+  enableFileManager: boolean
+  enableAutoSaving: boolean
+  enableControlnet: boolean
+  controlnetMethod: string
+  disableModelSwitch: boolean
+  isDesktop: boolean
+}
+
+export interface GenInfo {
+  prompt: string
+  negative_prompt: string
+}
+
 export interface ModelInfo {
  name: string
  path: string