update plugins

2024-01-02 11:07:35 +08:00
parent b0e009f879
commit a2fd5bb3ea
19 changed files with 337 additions and 227 deletions
--- a/web_app/src/lib/api.ts
+++ b/web_app/src/lib/api.ts
@@ -1,4 +1,11 @@
-import { Filename, ModelInfo, PowerPaintTask, Rect } from "@/lib/types"
+import {
+  Filename,
+  GenInfo,
+  ModelInfo,
+  PowerPaintTask,
+  Rect,
+  ServerConfig,
+} from "@/lib/types"
 import { Settings } from "@/lib/states"
 import { convertToBase64, srcToFile } from "@/lib/utils"
 import axios from "axios"
@@ -24,124 +31,113 @@ export default async function inpaint(
  const exampleImageBase64 = paintByExampleImage
    ? await convertToBase64(paintByExampleImage)
    : null
-  try {
-    const res = await fetch(`${API_ENDPOINT}/inpaint`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        image: imageBase64,
-        mask: maskBase64,
-        ldm_steps: settings.ldmSteps,
-        ldm_sampler: settings.ldmSampler,
-        zits_wireframe: settings.zitsWireframe,
-        cv2_flag: settings.cv2Flag,
-        cv2_radius: settings.cv2Radius,
-        hd_strategy: "Crop",
-        hd_strategy_crop_triger_size: 640,
-        hd_strategy_crop_margin: 128,
-        hd_trategy_resize_imit: 2048,
-        prompt: settings.prompt,
-        negative_prompt: settings.negativePrompt,
-        use_croper: settings.showCropper,
-        croper_x: croperRect.x,
-        croper_y: croperRect.y,
-        croper_height: croperRect.height,
-        croper_width: croperRect.width,
-        use_extender: settings.showExtender,
-        extender_x: extenderState.x,
-        extender_y: extenderState.y,
-        extender_height: extenderState.height,
-        extender_width: extenderState.width,
-        sd_mask_blur: settings.sdMaskBlur,
-        sd_strength: settings.sdStrength,
-        sd_steps: settings.sdSteps,
-        sd_guidance_scale: settings.sdGuidanceScale,
-        sd_sampler: settings.sdSampler,
-        sd_seed: settings.seedFixed ? settings.seed : -1,
-        sd_match_histograms: settings.sdMatchHistograms,
-        sd_freeu: settings.enableFreeu,
-        sd_freeu_config: settings.freeuConfig,
-        sd_lcm_lora: settings.enableLCMLora,
-        paint_by_example_example_image: exampleImageBase64,
-        p2p_image_guidance_scale: settings.p2pImageGuidanceScale,
-        enable_controlnet: settings.enableControlnet,
-        controlnet_conditioning_scale: settings.controlnetConditioningScale,
-        controlnet_method: settings.controlnetMethod
-          ? settings.controlnetMethod
-          : "",
-        powerpaint_task: settings.showExtender
-          ? PowerPaintTask.outpainting
-          : settings.powerpaintTask,
-      }),
-    })
+
+  const res = await fetch(`${API_ENDPOINT}/inpaint`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      image: imageBase64,
+      mask: maskBase64,
+      ldm_steps: settings.ldmSteps,
+      ldm_sampler: settings.ldmSampler,
+      zits_wireframe: settings.zitsWireframe,
+      cv2_flag: settings.cv2Flag,
+      cv2_radius: settings.cv2Radius,
+      hd_strategy: "Crop",
+      hd_strategy_crop_triger_size: 640,
+      hd_strategy_crop_margin: 128,
+      hd_trategy_resize_imit: 2048,
+      prompt: settings.prompt,
+      negative_prompt: settings.negativePrompt,
+      use_croper: settings.showCropper,
+      croper_x: croperRect.x,
+      croper_y: croperRect.y,
+      croper_height: croperRect.height,
+      croper_width: croperRect.width,
+      use_extender: settings.showExtender,
+      extender_x: extenderState.x,
+      extender_y: extenderState.y,
+      extender_height: extenderState.height,
+      extender_width: extenderState.width,
+      sd_mask_blur: settings.sdMaskBlur,
+      sd_strength: settings.sdStrength,
+      sd_steps: settings.sdSteps,
+      sd_guidance_scale: settings.sdGuidanceScale,
+      sd_sampler: settings.sdSampler,
+      sd_seed: settings.seedFixed ? settings.seed : -1,
+      sd_match_histograms: settings.sdMatchHistograms,
+      sd_freeu: settings.enableFreeu,
+      sd_freeu_config: settings.freeuConfig,
+      sd_lcm_lora: settings.enableLCMLora,
+      paint_by_example_example_image: exampleImageBase64,
+      p2p_image_guidance_scale: settings.p2pImageGuidanceScale,
+      enable_controlnet: settings.enableControlnet,
+      controlnet_conditioning_scale: settings.controlnetConditioningScale,
+      controlnet_method: settings.controlnetMethod
+        ? settings.controlnetMethod
+        : "",
+      powerpaint_task: settings.showExtender
+        ? PowerPaintTask.outpainting
+        : settings.powerpaintTask,
+    }),
+  })
+  if (res.ok) {
    const blob = await res.blob()
    return {
      blob: URL.createObjectURL(blob),
      seed: res.headers.get("X-Seed"),
    }
-  } catch (error: any) {
-    throw new Error(`Something went wrong: ${JSON.stringify(error.message)}`)
  }
+  const errors = await res.json()
+  throw new Error(`Something went wrong: ${errors.errors}`)
 }

-export function getServerConfig() {
-  return fetch(`${API_ENDPOINT}/server-config`, {
-    method: "GET",
-  })
+export async function getServerConfig(): Promise<ServerConfig> {
+  const res = await api.get(`/server-config`)
+  return res.data
 }

-export function switchModel(name: string) {
-  return axios.post(`${API_ENDPOINT}/model`, { name })
+export async function switchModel(name: string): Promise<ModelInfo> {
+  const res = await api.post(`/model`, { name })
+  return res.data
 }

-export function currentModel() {
-  return fetch(`${API_ENDPOINT}/model`, {
-    method: "GET",
-  })
+export async function currentModel(): Promise<ModelInfo> {
+  const res = await api.get("/model")
+  return res.data
 }

 export function fetchModelInfos(): Promise<ModelInfo[]> {
  return api.get("/models").then((response) => response.data)
 }

-export function modelDownloaded(name: string) {
-  return fetch(`${API_ENDPOINT}/model_downloaded/${name}`, {
-    method: "GET",
-  })
-}
-
 export async function runPlugin(
  name: string,
  imageFile: File,
  upscale?: number,
  clicks?: number[][]
 ) {
-  const fd = new FormData()
-  fd.append("name", name)
-  fd.append("image", imageFile)
-  if (upscale) {
-    fd.append("upscale", upscale.toString())
-  }
-  if (clicks) {
-    fd.append("clicks", JSON.stringify(clicks))
-  }
-
-  try {
-    const res = await fetch(`${API_ENDPOINT}/run_plugin`, {
-      method: "POST",
-      body: fd,
-    })
-    if (res.ok) {
-      const blob = await res.blob()
-      return { blob: URL.createObjectURL(blob) }
-    }
-    const errMsg = await res.text()
-    throw new Error(errMsg)
-  } catch (error) {
-    throw new Error(`Something went wrong: ${error}`)
+  const imageBase64 = await convertToBase64(imageFile)
+  const res = await fetch(`${API_ENDPOINT}/run_plugin`, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      name,
+      image: imageBase64,
+      upscale,
+      clicks,
+    }),
+  })
+  if (res.ok) {
+    const blob = await res.blob()
+    return { blob: URL.createObjectURL(blob) }
  }
+  const errMsg = await res.json()
+  throw new Error(errMsg)
 }

 export async function getMediaFile(tab: string, filename: string) {
@@ -160,12 +156,12 @@ export async function getMediaFile(tab: string, filename: string) {
    })
    return file
  }
-  const errMsg = await res.text()
-  throw new Error(errMsg)
+  const errMsg = await res.json()
+  throw new Error(errMsg.errors)
 }

 export async function getMedias(tab: string): Promise<Filename[]> {
-  const res = await axios.get(`${API_ENDPOINT}/medias`, { params: { tab } })
+  const res = await api.get(`medias`, { params: { tab } })
  return res.data
 }

@@ -191,3 +187,10 @@ export async function downloadToOutput(
    throw new Error(`Something went wrong: ${error}`)
  }
 }
+
+export async function getGenInfo(file: File): Promise<GenInfo> {
+  const fd = new FormData()
+  fd.append("file", file)
+  const res = await api.post(`/gen-info`, fd)
+  return res.data
+}
--- a/web_app/src/lib/states.ts
+++ b/web_app/src/lib/states.ts
@@ -15,6 +15,7 @@ import {
  Point,
  PowerPaintTask,
  SDSampler,
+  ServerConfig,
  Size,
  SortBy,
  SortOrder,
@@ -33,7 +34,7 @@ import {
  loadImage,
  srcToFile,
 } from "./utils"
-import inpaint, { runPlugin } from "./api"
+import inpaint, { getGenInfo, runPlugin } from "./api"
 import { toast } from "@/components/ui/use-toast"

 type FileManagerState = {
@@ -57,6 +58,7 @@ export type Settings = {
  enableDownloadMask: boolean
  enableManualInpainting: boolean
  enableUploadMask: boolean
+  enableAutoExtractPrompt: boolean
  showCropper: boolean
  showExtender: boolean
  extenderDirection: ExtenderDirection
@@ -103,16 +105,6 @@ export type Settings = {
  powerpaintTask: PowerPaintTask
 }

-type ServerConfig = {
-  plugins: string[]
-  enableFileManager: boolean
-  enableAutoSaving: boolean
-  enableControlnet: boolean
-  controlnetMethod: string
-  disableModelSwitch: boolean
-  isDesktop: boolean
-}
-
 type InteractiveSegState = {
  isInteractiveSeg: boolean
  interactiveSegMask: HTMLImageElement | null
@@ -162,7 +154,7 @@ type AppState = {

 type AppAction = {
  updateAppState: (newState: Partial<AppState>) => void
-  setFile: (file: File) => void
+  setFile: (file: File) => Promise<void>
  setCustomFile: (file: File) => void
  setIsInpainting: (newValue: boolean) => void
  setIsPluginRunning: (newValue: boolean) => void
@@ -304,6 +296,7 @@ const defaultValues: AppState = {
    enableDownloadMask: false,
    enableManualInpainting: false,
    enableUploadMask: false,
+    enableAutoExtractPrompt: true,
    ldmSteps: 30,
    ldmSampler: LDMSampler.ddim,
    zitsWireframe: true,
@@ -540,9 +533,6 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
          const start = new Date()
          const targetFile = await get().getCurrentTargetFile()
          const res = await runPlugin(pluginName, targetFile, params.upscale)
-          if (!res) {
-            throw new Error("Something went wrong on server side.")
-          }
          const { blob } = res
          const newRender = new Image()
          await loadImage(newRender, blob)
@@ -818,7 +808,27 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
          state.isPluginRunning = newValue
        }),

-      setFile: (file: File) => {
+      setFile: async (file: File) => {
+        if (get().settings.enableAutoExtractPrompt) {
+          try {
+            const res = await getGenInfo(file)
+            if (res.prompt) {
+              set((state) => {
+                state.settings.prompt = res.prompt
+              })
+            }
+            if (res.negative_prompt) {
+              set((state) => {
+                state.settings.negativePrompt = res.negative_prompt
+              })
+            }
+          } catch (e: any) {
+            toast({
+              variant: "destructive",
+              description: e.message ? e.message : e.toString(),
+            })
+          }
+        }
        set((state) => {
          state.file = file
          state.interactiveSegState = castDraft(
--- a/web_app/src/lib/types.ts
+++ b/web_app/src/lib/types.ts
@@ -6,6 +6,21 @@ export interface Filename {
  mtime: number
 }

+export interface ServerConfig {
+  plugins: string[]
+  enableFileManager: boolean
+  enableAutoSaving: boolean
+  enableControlnet: boolean
+  controlnetMethod: string
+  disableModelSwitch: boolean
+  isDesktop: boolean
+}
+
+export interface GenInfo {
+  prompt: string
+  negative_prompt: string
+}
+
 export interface ModelInfo {
  name: string
  path: string