add DiffusionInpaintModel

lama_cleaner/model/sd.py

@@ -8,9 +8,8 @@ from diffusers import PNDMScheduler, DDIMScheduler, LMSDiscreteScheduler, EulerD
     EulerAncestralDiscreteScheduler, DPMSolverMultistepScheduler
 from loguru import logger
 
-from lama_cleaner.helper import resize_max_size
-from lama_cleaner.model.base import InpaintModel
-from lama_cleaner.model.utils import torch_gc
+from lama_cleaner.model.base import DiffusionInpaintModel
+from lama_cleaner.model.utils import torch_gc, set_seed
 from lama_cleaner.schema import Config, SDSampler
 
 
@@ -28,7 +27,7 @@ class CPUTextEncoderWrapper:
         return [self.text_encoder(x.to(self.text_encoder.device), **kwargs)[0].to(input_device).to(self.torch_dtype)]
 
 
-class SD(InpaintModel):
+class SD(DiffusionInpaintModel):
     pad_mod = 8
     min_size = 512
 
@@ -73,25 +72,6 @@ class SD(InpaintModel):
 
         self.callback = kwargs.pop("callback", None)
 
-    def _scaled_pad_forward(self, image, mask, config: Config):
-        longer_side_length = int(config.sd_scale * max(image.shape[:2]))
-        origin_size = image.shape[:2]
-        downsize_image = resize_max_size(image, size_limit=longer_side_length)
-        downsize_mask = resize_max_size(mask, size_limit=longer_side_length)
-        logger.info(
-            f"Resize image to do sd inpainting: {image.shape} -> {downsize_image.shape}"
-        )
-        inpaint_result = self._pad_forward(downsize_image, downsize_mask, config)
-        # only paste masked area result
-        inpaint_result = cv2.resize(
-            inpaint_result,
-            (origin_size[1], origin_size[0]),
-            interpolation=cv2.INTER_CUBIC,
-        )
-        original_pixel_indices = mask < 127
-        inpaint_result[original_pixel_indices] = image[:, :, ::-1][original_pixel_indices]
-        return inpaint_result
-
     def forward(self, image, mask, config: Config):
         """Input image and output image have same size
         image: [H, W, C] RGB
@@ -118,11 +98,7 @@ class SD(InpaintModel):
 
         self.model.scheduler = scheduler
 
-        seed = config.sd_seed
-        random.seed(seed)
-        np.random.seed(seed)
-        torch.manual_seed(seed)
-        torch.cuda.manual_seed_all(seed)
+        set_seed(config.sd_seed)
 
         if config.sd_mask_blur != 0:
             k = 2 * config.sd_mask_blur + 1
@@ -147,24 +123,6 @@ class SD(InpaintModel):
         output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR)
         return output
 
-    @torch.no_grad()
-    def __call__(self, image, mask, config: Config):
-        """
-        images: [H, W, C] RGB, not normalized
-        masks: [H, W]
-        return: BGR IMAGE
-        """
-        # boxes = boxes_from_mask(mask)
-        if config.use_croper:
-            crop_img, crop_mask, (l, t, r, b) = self._apply_cropper(image, mask, config)
-            crop_image = self._scaled_pad_forward(crop_img, crop_mask, config)
-            inpaint_result = image[:, :, ::-1]
-            inpaint_result[t:b, l:r, :] = crop_image
-        else:
-            inpaint_result = self._scaled_pad_forward(image, mask, config)
-
-        return inpaint_result
-
     def forward_post_process(self, result, image, mask, config):
         if config.sd_match_histograms:
             result = self._match_histograms(result, image[:, :, ::-1], mask)