Code cleanup (final commit unless I fix controlnet)

arrmansa · arrmansa · commit 84c4aaec0df7 · 2025-01-02T01:56:05.000+05:30
optional vector for sdxl
Better functions, better cache
Tested everything
diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py
@@ -15,7 +15,8 @@
 # For sdxl - OpenAIWrapper will be called, which will call the underlying diffusion_model
 # When controlnet is enabled, the underlying model is not available to use, therefore we skip
 
-def find_noise_for_image(p, cond, uncond, cfg_scale, steps):
+@torch.no_grad()
+def find_noise_for_image(p, cond, uncond, cfg_scale, steps, skip_sdxl_vector):
     x = p.init_latent.clone()
 
     s_in = x.new_ones([x.shape[0]])
@@ -36,53 +37,27 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps):
         sigma_in = torch.cat([sigmas[i] * s_in] * 2)
 
         if shared.sd_model.is_sdxl:
-            cond_tensor = cond['crossattn']
-            uncond_tensor = uncond['crossattn']
-            cond_in = torch.cat([uncond_tensor, cond_tensor])
+            cond_in = {"crossattn": [torch.cat([uncond['crossattn'], cond['crossattn']])], "vector": [torch.cat([uncond['vector'], cond['vector']])]}
         else:
-            cond_in = torch.cat([uncond, cond])
+            cond_in = {"c_concat": [torch.cat([p.image_conditioning] * 2)], "c_crossattn": [torch.cat([uncond, cond])]}
 
-        image_conditioning = torch.cat([p.image_conditioning] * 2)
-        cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]}
-
-        c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]]
         t = dnw.sigma_to_t(sigma_in)
-
-        if shared.sd_model.is_sdxl:
-            num_classes_hack = shared.sd_model.model.diffusion_model.num_classes
-            shared.sd_model.model.diffusion_model.num_classes = None
-            try:
-                eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} )
-            finally:
-                shared.sd_model.model.diffusion_model.num_classes = num_classes_hack
-        else:
-            eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in)
-
-        denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2)
-
-        denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale
-
-        d = (x - denoised) / sigmas[i]
         dt = sigmas[i] - sigmas[i - 1]
-
-        x = x + d * dt
+        x += noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector)
 
         sd_samplers_common.store_latent(x)
 
         # This shouldn't be necessary, but solved some VRAM issues
-        del x_in, sigma_in, cond_in, c_out, c_in, t,
-        del eps, denoised_uncond, denoised_cond, denoised, d, dt
+        del x_in, sigma_in, cond_in, t, dt
 
     shared.state.nextjob()
 
-    return x / x.std()
-
-
-Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment", "second_order_correction", "noise_sigma_intensity"])
+    return x, sigmas[-1]
 
 
 # Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/736
-def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, correction_factor, sigma_intensity):
+@torch.no_grad()
+def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, correction_factor, skip_sdxl_vector):
     x = p.init_latent.clone()
 
     s_in = x.new_ones([x.shape[0]])
@@ -100,14 +75,9 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, cor
         shared.state.sampling_step += 1
         sigma_in = torch.cat([sigmas[i - 1] * s_in] * 2)
         if shared.sd_model.is_sdxl:
-            cond_tensor = cond['crossattn']
-            uncond_tensor = uncond['crossattn']
-            cond_in = torch.cat([uncond_tensor, cond_tensor])
+            cond_in = {"crossattn": [torch.cat([uncond['crossattn'], cond['crossattn']])], "vector": [torch.cat([uncond['vector'], cond['vector']])]}
         else:
-            cond_in = torch.cat([uncond, cond])
-
-        image_conditioning = torch.cat([p.image_conditioning] * 2)
-        cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]}
+            cond_in = {"c_concat": [torch.cat([p.image_conditioning] * 2)], "c_crossattn": [torch.cat([uncond, cond])]}
 
         if i == 1:
             t = dnw.sigma_to_t(torch.cat([sigmas[i] * s_in] * 2))
@@ -116,37 +86,35 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, cor
             t = dnw.sigma_to_t(sigma_in)
             dt = (sigmas[i] - sigmas[i - 1]) / sigmas[i - 1]
 
-        noise = noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip)
+        noise = noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector)
 
         if correction_factor > 0: # runs model with previously calculated noise
-            recalculated_noise = noise_from_model(x + noise, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip)
+            recalculated_noise = noise_from_model(x + noise, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector)
             noise = recalculated_noise * correction_factor + noise * (1 - correction_factor)
 
         x += noise
 
         sd_samplers_common.store_latent(x)
 
-        # This shouldn't be necessary, but solved some VRAM issues
-        #del x_in, sigma_in, cond_in, c_out, c_in, t
-        #del eps, denoised_uncond, denoised_cond, denoised, dt
-
     shared.state.nextjob()
 
-    # Chooses between std and sigmas[-1]
-    return x / (x.std()*(1 - sigma_intensity) + sigmas[-1]*sigma_intensity)
+    return x, sigmas[-1]
 
-def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip):
+@torch.no_grad()
+def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector):
 
     if cfg_scale == 1:  # Case where denoised_uncond should not be calculated - 50% speedup, also good for sdxl in experiments
         x_in = x
         sigma_in = sigma_in[1:2]
         c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]]
-        cond_in = {"c_concat":[cond_in["c_concat"][0][1:2]], "c_crossattn": [cond_in["c_crossattn"][0][1:2]]}
+        cond_in = {k:[v[0][1:2]] for k, v in cond_in.items()}
         if shared.sd_model.is_sdxl:
             num_classes_hack = shared.sd_model.model.diffusion_model.num_classes
-            shared.sd_model.model.diffusion_model.num_classes = None
+            if skip_sdxl_vector:
+                shared.sd_model.model.diffusion_model.num_classes = None
+                cond_in["vector"][0] = None
             try:
-                eps = shared.sd_model.model(x_in * c_in, t[1:2], {"crossattn": cond_in["c_crossattn"][0]})
+                eps = shared.sd_model.model(x_in * c_in, t[1:2], {"crossattn": cond_in["crossattn"][0], "vector": cond_in["vector"][0]})
             finally:
                 shared.sd_model.model.diffusion_model.num_classes = num_classes_hack
         else:
@@ -160,9 +128,11 @@ def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip):
 
         if shared.sd_model.is_sdxl:
             num_classes_hack = shared.sd_model.model.diffusion_model.num_classes
-            shared.sd_model.model.diffusion_model.num_classes = None
+            if skip_sdxl_vector:
+                shared.sd_model.model.diffusion_model.num_classes = None
+                cond_in["vector"][0] = None
             try:
-                eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} )
+                eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["crossattn"][0], "vector": cond_in["vector"][0]} )
             finally:
                 shared.sd_model.model.diffusion_model.num_classes = num_classes_hack
         else:
@@ -174,6 +144,9 @@ def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip):
 
         return -denoised * dt
 
+Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment", "second_order_correction", "skip_sdxl_vector"])
+
+
 class Script(scripts.Script):
     def __init__(self):
         self.cache = None
@@ -189,35 +162,38 @@ def ui(self, is_img2img):
         * `CFG Scale` should be 2 or lower.
         ''')
 
-        override_sampler = gr.Checkbox(label="Override `Sampling method` to Euler?(this method is built for it)", value=True, elem_id=self.elem_id("override_sampler"))
+        override_sampler = gr.Checkbox(label="Override `Sampling method` to Euler?(this method is built for it)", value=False, elem_id=self.elem_id("override_sampler"))
 
-        override_prompt = gr.Checkbox(label="Override `prompt` to the same value as `original prompt`?(and `negative prompt`)", value=True, elem_id=self.elem_id("override_prompt"))
+        override_prompt = gr.Checkbox(label="Override `prompt` to the same value as `original prompt`?(and `negative prompt`)", value=False, elem_id=self.elem_id("override_prompt"))
         original_prompt = gr.Textbox(label="Original prompt", lines=1, elem_id=self.elem_id("original_prompt"))
         original_negative_prompt = gr.Textbox(label="Original negative prompt", lines=1, elem_id=self.elem_id("original_negative_prompt"))
 
         override_steps = gr.Checkbox(label="Override `Sampling Steps` to the same value as `Decode steps`?", value=True, elem_id=self.elem_id("override_steps"))
-        st = gr.Slider(label="Decode steps", minimum=1, maximum=150, step=1, value=50, elem_id=self.elem_id("st"))
+        st = gr.Slider(label="Decode steps", minimum=1, maximum=150, step=1, value=20, elem_id=self.elem_id("st"))
 
         override_strength = gr.Checkbox(label="Override `Denoising strength` to 1?", value=True, elem_id=self.elem_id("override_strength"))
 
         cfg = gr.Slider(label="Decode CFG scale", minimum=0.0, maximum=15.0, step=0.1, value=1.0, elem_id=self.elem_id("cfg"))
         randomness = gr.Slider(label="Randomness", minimum=0.0, maximum=1.0, step=0.01, value=0.0, elem_id=self.elem_id("randomness"))
-        sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=False, elem_id=self.elem_id("sigma_adjustment"))
+        sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=True, elem_id=self.elem_id("sigma_adjustment"))
         second_order_correction = gr.Slider(label="Correct noise by running model again", minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id=self.elem_id("second_order_correction"),
                                             info="use 0 (disabled) for original script behaviour, 0.5 reccomended value. Runs the model again to recalculate noise and correct it by given factor. Higher adheres to original image more.")
         noise_sigma_intensity = gr.Slider(label="Weight scaling std vs sigma based", minimum=-1.0, maximum=2.0, step=0.01, value=0.5, elem_id=self.elem_id("noise_sigma_intensity"),
                                           info="use 1 for original script behaviour, 0.5 reccomended value. Decides whether to use fixed sigma value or dynamic standard deviation to scale noise. Lower gives softer images.")
+        skip_sdxl_vector = gr.Checkbox(label="Skip sdxl vectors", info="may cause distortion if false", value=True, elem_id=self.elem_id("skip_sdxl_vector"))
+
         return [
             info,
             override_sampler,
             override_prompt, original_prompt, original_negative_prompt,
             override_steps, st,
             override_strength,
             cfg, randomness, sigma_adjustment, second_order_correction,
-            noise_sigma_intensity
+            noise_sigma_intensity, skip_sdxl_vector
         ]
 
-    def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment, second_order_correction, noise_sigma_intensity):
+    @torch.no_grad()
+    def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment, second_order_correction, noise_sigma_intensity, skip_sdxl_vector):
         # Override
         if override_sampler:
             p.sampler_name = "Euler"
@@ -237,15 +213,16 @@ def sample_extra(conditioning, unconditional_conditioning, seeds, subseeds, subs
                                 and self.cache.original_negative_prompt == original_negative_prompt \
                                 and self.cache.sigma_adjustment == sigma_adjustment \
                                 and self.cache.second_order_correction == second_order_correction \
-                                and self.cache.noise_sigma_intensity == noise_sigma_intensity
+                                and self.cache.skip_sdxl_vector == skip_sdxl_vector
+
             same_everything = same_params and self.cache.latent.shape == lat.shape and np.abs(self.cache.latent-lat).sum() < 100
 
             rand_noise = processing.create_random_tensors(p.init_latent.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, p=p)
 
             if same_everything:
-                rec_noise = self.cache.noise
+                rec_noise, sigma_val = self.cache.noise
             else:
-                # This prevents a crash, because I don't know how to access the underlying .diffusion_model yet when controlnet is enabled. WIP
+                # This prevents a crash, because I don't know how to access the underlying .diffusion_model yet when controlnet is enabled.
                 # modules.sd_unet -> we're good
                 # scripts.hook -> we're cooked
                 if "scripts.hook" in str(shared.sd_model.model.diffusion_model.forward.__module__):
@@ -257,21 +234,23 @@ def sample_extra(conditioning, unconditional_conditioning, seeds, subseeds, subs
                 cond = p.sd_model.get_learned_conditioning(p.batch_size * [original_prompt])
                 uncond = p.sd_model.get_learned_conditioning(p.batch_size * [original_negative_prompt])
                 if sigma_adjustment:
-                    rec_noise = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st, second_order_correction, noise_sigma_intensity)
+                    rec_noise, sigma_val = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st, second_order_correction, skip_sdxl_vector)
                 else:
-                    rec_noise = find_noise_for_image(p, cond, uncond, cfg, st)
-                self.cache = Cached(rec_noise, cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment, second_order_correction, noise_sigma_intensity)
+                    rec_noise, sigma_val = find_noise_for_image(p, cond, uncond, cfg, st, skip_sdxl_vector)
+                self.cache = Cached((rec_noise, sigma_val), cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment, second_order_correction, skip_sdxl_vector)
+
+            rec_noise = rec_noise / (rec_noise.std()*(1 - noise_sigma_intensity) + sigma_val*noise_sigma_intensity)
 
             combined_noise = ((1 - randomness) * rec_noise + randomness * rand_noise) / ((randomness**2 + (1-randomness)**2) ** 0.5)
 
             sampler = sd_samplers.create_sampler(p.sampler_name, p.sd_model)
 
+            p.seed = p.seed + 1
+
             sigmas = sampler.model_wrap.get_sigmas(p.steps)
 
             noise_dt = combined_noise - (p.init_latent / sigmas[0])
 
-            p.seed = p.seed + 1
-
             return sampler.sample_img2img(p, p.init_latent, noise_dt, conditioning, unconditional_conditioning, image_conditioning=p.image_conditioning)
 
         p.sample = sample_extra
