add controlnet inpainting

lama_cleaner/app/src/App.tsx

@@ -7,6 +7,7 @@ import Workspace from './components/Workspace'
 import {
   enableFileManagerState,
   fileState,
+  isControlNetState,
   isDisableModelSwitchState,
   isEnableAutoSavingState,
   toastState,
@@ -17,6 +18,7 @@ import useHotKey from './hooks/useHotkey'
 import {
   getEnableAutoSaving,
   getEnableFileManager,
+  getIsControlNet,
   getIsDisableModelSwitch,
   isDesktop,
 } from './adapters/inpainting'
@@ -37,6 +39,7 @@ function App() {
   const setIsDisableModelSwitch = useSetRecoilState(isDisableModelSwitchState)
   const setEnableFileManager = useSetRecoilState(enableFileManagerState)
   const setIsEnableAutoSavingState = useSetRecoilState(isEnableAutoSavingState)
+  const setIsControlNet = useSetRecoilState(isControlNetState)
 
   // Set Input Image
   useEffect(() => {
@@ -75,10 +78,17 @@ function App() {
       setIsEnableAutoSavingState(isEnabled === 'true')
     }
     fetchData3()
+
+    const fetchData4 = async () => {
+      const isEnabled = await getIsControlNet().then(res => res.text())
+      setIsControlNet(isEnabled === 'true')
+    }
+    fetchData4()
   }, [
     setEnableFileManager,
     setIsDisableModelSwitch,
     setIsEnableAutoSavingState,
+    setIsControlNet,
   ])
 
   // Dark Mode Hotkey

lama_cleaner/app/src/adapters/inpainting.ts

@@ -87,6 +87,12 @@ export default async function inpaint(
   fd.append('p2pImageGuidanceScale', settings.p2pImageGuidanceScale.toString())
   fd.append('p2pGuidanceScale', settings.p2pGuidanceScale.toString())
 
+  // ControlNet
+  fd.append(
+    'controlnet_conditioning_scale',
+    settings.controlnetConditioningScale.toString()
+  )
+
   if (sizeLimit === undefined) {
     fd.append('sizeLimit', '1080')
   } else {
@@ -116,6 +122,12 @@ export function getIsDisableModelSwitch() {
   })
 }
 
+export function getIsControlNet() {
+  return fetch(`${API_ENDPOINT}/is_controlnet`, {
+    method: 'GET',
+  })
+}
+
 export function getEnableFileManager() {
   return fetch(`${API_ENDPOINT}/is_enable_file_manager`, {
     method: 'GET',

lama_cleaner/app/src/components/SidePanel/SidePanel.tsx

@@ -3,6 +3,7 @@ import { useRecoilState, useRecoilValue } from 'recoil'
 import * as PopoverPrimitive from '@radix-ui/react-popover'
 import { useToggle } from 'react-use'
 import {
+  isControlNetState,
   isInpaintingState,
   negativePropmtState,
   propmtState,
@@ -26,6 +27,7 @@ const SidePanel = () => {
     useRecoilState(negativePropmtState)
   const isInpainting = useRecoilValue(isInpaintingState)
   const prompt = useRecoilValue(propmtState)
+  const isControlNet = useRecoilValue(isControlNetState)
 
   const handleOnInput = (evt: FormEvent<HTMLTextAreaElement>) => {
     evt.preventDefault()
@@ -115,6 +117,22 @@ const SidePanel = () => {
               }}
             />
 
+            {isControlNet && (
+              <NumberInputSetting
+                title="ControlNet Weight"
+                width={INPUT_WIDTH}
+                allowFloat
+                value={`${setting.controlnetConditioningScale}`}
+                desc="Lowered this value if there is a big misalignment between the text prompt and the control image"
+                onValue={value => {
+                  const val = value.length === 0 ? 0 : parseFloat(value)
+                  setSettingState(old => {
+                    return { ...old, controlnetConditioningScale: val }
+                  })
+                }}
+              />
+            )}
+
             <NumberInputSetting
               title="Mask Blur"
               width={INPUT_WIDTH}

lama_cleaner/app/src/store/Atoms.tsx

@@ -51,6 +51,7 @@ interface AppState {
   enableFileManager: boolean
   gifImage: HTMLImageElement | undefined
   brushSize: number
+  isControlNet: boolean
 }
 
 export const appState = atom<AppState>({
@@ -70,6 +71,7 @@ export const appState = atom<AppState>({
     enableFileManager: false,
     gifImage: undefined,
     brushSize: 40,
+    isControlNet: false,
   },
 })
 
@@ -239,6 +241,18 @@ export const isDisableModelSwitchState = selector({
   },
 })
 
+export const isControlNetState = selector({
+  key: 'isControlNetState',
+  get: ({ get }) => {
+    const app = get(appState)
+    return app.isControlNet
+  },
+  set: ({ get, set }, newValue: any) => {
+    const app = get(appState)
+    set(appState, { ...app, isControlNet: newValue })
+  },
+})
+
 export const isEnableAutoSavingState = selector({
   key: 'isEnableAutoSavingState',
   get: ({ get }) => {
@@ -379,6 +393,9 @@ export interface Settings {
   p2pSteps: number
   p2pImageGuidanceScale: number
   p2pGuidanceScale: number
+
+  // ControlNet
+  controlnetConditioningScale: number
 }
 
 const defaultHDSettings: ModelsHDSettings = {
@@ -482,6 +499,7 @@ export enum SDSampler {
   kEuler = 'k_euler',
   kEulerA = 'k_euler_a',
   dpmPlusPlus = 'dpm++',
+  uni_pc = 'uni_pc',
 }
 
 export enum SDMode {
@@ -510,7 +528,7 @@ export const settingStateDefault: Settings = {
   sdStrength: 0.75,
   sdSteps: 50,
   sdGuidanceScale: 7.5,
-  sdSampler: SDSampler.pndm,
+  sdSampler: SDSampler.uni_pc,
   sdSeed: 42,
   sdSeedFixed: false,
   sdNumSamples: 1,
@@ -533,6 +551,9 @@ export const settingStateDefault: Settings = {
   p2pSteps: 50,
   p2pImageGuidanceScale: 1.5,
   p2pGuidanceScale: 7.5,
+
+  // ControlNet
+  controlnetConditioningScale: 0.4,
 }
 
 const localStorageEffect =