Merge branch 'main' into tests/unbloat-lora-utils

.github/workflows/pr_tests.yml

@@ -265,11 +265,11 @@ jobs:
     - name: Run fast PyTorch LoRA tests with PEFT
       run: |
         python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
-        python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \
+        python -m pytest -n 6 --max-worker-restart=0 --dist=loadfile \
           -s -v \
           --make-reports=tests_peft_main \
           tests/lora/
-        python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \
+        python -m pytest -n 6 --max-worker-restart=0 --dist=loadfile \
           -s -v \
           --make-reports=tests_models_lora_peft_main \
           tests/models/ -k "lora"

tests/lora/test_lora_layers_auraflow.py

@@ -103,34 +103,6 @@ class AuraFlowLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    @unittest.skip("Not supported in AuraFlow.")
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in AuraFlow.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
-
     @unittest.skip("Not supported in AuraFlow.")
     def test_modify_padding_mode(self):
         pass
-
-    @unittest.skip("Text encoder LoRA is not supported in AuraFlow.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in AuraFlow.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in AuraFlow.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in AuraFlow.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in AuraFlow.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass

tests/lora/test_lora_layers_cogvideox.py

@@ -120,11 +120,25 @@ class CogVideoXLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",)])
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3)
+    def test_lora_set_adapters_scenarios(self, scenario):
+        super()._test_lora_set_adapters_scenarios(scenario, expected_atol=9e-3)
 
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+    @parameterized.expand(
-        super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3)
+        [
+            # Test actions on text_encoder LoRA only
+            ("fused", "text_encoder_only"),
+            ("unloaded", "text_encoder_only"),
+            ("save_load", "text_encoder_only"),
+            # Test actions on both text_encoder and denoiser LoRA
+            ("fused", "text_and_denoiser"),
+            ("unloaded", "text_and_denoiser"),
+            ("unfused", "text_and_denoiser"),
+            ("save_load", "text_and_denoiser"),
+        ]
+    )
+    def test_lora_actions(self, action, components_to_add):
+        super()._test_lora_actions(action, components_to_add, expected_atol=9e-3)
 
     def test_lora_scale_kwargs_match_fusion(self):
         super().test_lora_scale_kwargs_match_fusion(expected_atol=9e-3, expected_rtol=9e-3)
@@ -136,38 +150,8 @@ class CogVideoXLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
         # The reason for this can be found here: https://github.com/huggingface/diffusers/pull/11804#issuecomment-3013325338
         super()._test_group_offloading_inference_denoiser(offload_type, use_stream)
 
-    @unittest.skip("Not supported in CogVideoX.")
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in CogVideoX.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
-
     @unittest.skip("Not supported in CogVideoX.")
     def test_modify_padding_mode(self):
         pass
 
-    @unittest.skip("Text encoder LoRA is not supported in CogVideoX.")
+    # TODO: skip them properly
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogVideoX.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogVideoX.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogVideoX.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogVideoX.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass
-
-    @unittest.skip("Not supported in CogVideoX.")
-    def test_simple_inference_with_text_denoiser_multi_adapter_block_lora(self):
-        pass

tests/lora/test_lora_layers_cogview4.py

@@ -13,10 +13,8 @@
 # limitations under the License.
 
 import sys
-import tempfile
 import unittest
 
-import numpy as np
 import torch
 from parameterized import parameterized
 from transformers import AutoTokenizer, GlmModel
@@ -27,7 +25,6 @@ from diffusers.utils.testing_utils import (
     require_peft_backend,
     require_torch_accelerator,
     skip_mps,
-    torch_device,
 )
 
 
@@ -113,40 +110,21 @@ class CogView4LoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand(
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3)
+        [
-
+            # Test actions on text_encoder LoRA only
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+            ("fused", "text_encoder_only"),
-        super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3)
+            ("unloaded", "text_encoder_only"),
-
+            ("save_load", "text_encoder_only"),
-    def test_simple_inference_save_pretrained(self):
+            # Test actions on both text_encoder and denoiser LoRA
-        """
+            ("fused", "text_and_denoiser"),
-        Tests a simple usecase where users could use saving utilities for LoRA through save_pretrained
+            ("unloaded", "text_and_denoiser"),
-        """
+            ("unfused", "text_and_denoiser"),
-        for scheduler_cls in self.scheduler_classes:
+            ("save_load", "text_and_denoiser"),
-            components, _, _ = self.get_dummy_components(scheduler_cls)
+        ]
-            pipe = self.pipeline_class(**components)
+    )
-            pipe = pipe.to(torch_device)
+    def test_lora_actions(self, action, components_to_add):
-            pipe.set_progress_bar_config(disable=None)
+        super()._test_lora_actions(action, components_to_add, expected_atol=9e-3)
-            _, _, inputs = self.get_dummy_inputs(with_generator=False)
-
-            output_no_lora = pipe(**inputs, generator=torch.manual_seed(0))[0]
-            self.assertTrue(output_no_lora.shape == self.output_shape)
-
-            images_lora = pipe(**inputs, generator=torch.manual_seed(0))[0]
-
-            with tempfile.TemporaryDirectory() as tmpdirname:
-                pipe.save_pretrained(tmpdirname)
-
-                pipe_from_pretrained = self.pipeline_class.from_pretrained(tmpdirname)
-                pipe_from_pretrained.to(torch_device)
-
-            images_lora_save_pretrained = pipe_from_pretrained(**inputs, generator=torch.manual_seed(0))[0]
-
-            self.assertTrue(
-                np.allclose(images_lora, images_lora_save_pretrained, atol=1e-3, rtol=1e-3),
-                "Loading from saved checkpoints should give same results.",
-            )
 
     @parameterized.expand([("block_level", True), ("leaf_level", False)])
     @require_torch_accelerator
@@ -155,34 +133,6 @@ class CogView4LoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
         # The reason for this can be found here: https://github.com/huggingface/diffusers/pull/11804#issuecomment-3013325338
         super()._test_group_offloading_inference_denoiser(offload_type, use_stream)
 
-    @unittest.skip("Not supported in CogView4.")
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in CogView4.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
-
     @unittest.skip("Not supported in CogView4.")
     def test_modify_padding_mode(self):
         pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogView4.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogView4.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogView4.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogView4.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in CogView4.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass

tests/lora/test_lora_layers_flux.py

@@ -263,21 +263,11 @@ class FluxLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
             "LoRA should lead to different results.",
         )
 
-    @unittest.skip("Not supported in Flux.")
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in Flux.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
-
     @unittest.skip("Not supported in Flux.")
     def test_modify_padding_mode(self):
         pass
 
-    @unittest.skip("Not supported in Flux.")
+    # TODO: skip them properly
-    def test_simple_inference_with_text_denoiser_multi_adapter_block_lora(self):
-        pass
 
 
 class FluxControlLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
@@ -791,21 +781,11 @@ class FluxControlLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
         self.assertTrue(pipe.transformer.x_embedder.weight.data.shape[1] == in_features * 2)
         self.assertTrue(pipe.transformer.config.in_channels == in_features * 2)
 
-    @unittest.skip("Not supported in Flux.")
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in Flux.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
-
     @unittest.skip("Not supported in Flux.")
     def test_modify_padding_mode(self):
         pass
 
-    @unittest.skip("Not supported in Flux.")
+    # TODO: skip them properly
-    def test_simple_inference_with_text_denoiser_multi_adapter_block_lora(self):
-        pass
 
 
 @slow

tests/lora/test_lora_layers_hunyuanvideo.py

@@ -19,6 +19,7 @@ import unittest
 import numpy as np
 import pytest
 import torch
+from parameterized import parameterized
 from transformers import CLIPTextModel, CLIPTokenizer, LlamaModel, LlamaTokenizerFast
 
 from diffusers import (
@@ -150,49 +151,33 @@ class HunyuanVideoLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",)])
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3)
+    def test_lora_set_adapters_scenarios(self, scenario):
+        expected_atol = 9e-3
+        if scenario == "weighted":
+            expected_atol = 1e-3
+        super()._test_lora_set_adapters_scenarios(scenario, expected_atol=expected_atol)
 
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+    @parameterized.expand(
-        super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3)
+        [
-
+            # Test actions on text_encoder LoRA only
-    # TODO(aryan): Fix the following test
+            ("fused", "text_encoder_only"),
-    @unittest.skip("This test fails with an error I haven't been able to debug yet.")
+            ("unloaded", "text_encoder_only"),
-    def test_simple_inference_save_pretrained(self):
+            ("save_load", "text_encoder_only"),
-        pass
+            # Test actions on both text_encoder and denoiser LoRA
-
+            ("fused", "text_and_denoiser"),
-    @unittest.skip("Not supported in HunyuanVideo.")
+            ("unloaded", "text_and_denoiser"),
-    def test_simple_inference_with_text_denoiser_block_scale(self):
+            ("unfused", "text_and_denoiser"),
-        pass
+            ("save_load", "text_and_denoiser"),
-
+        ]
-    @unittest.skip("Not supported in HunyuanVideo.")
+    )
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
+    def test_lora_actions(self, action, components_to_add):
-        pass
+        super()._test_lora_actions(action, components_to_add, expected_atol=9e-3)
 
     @unittest.skip("Not supported in HunyuanVideo.")
     def test_modify_padding_mode(self):
         pass
 
-    @unittest.skip("Text encoder LoRA is not supported in HunyuanVideo.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in HunyuanVideo.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in HunyuanVideo.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in HunyuanVideo.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in HunyuanVideo.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass
-
 
 @nightly
 @require_torch_accelerator

tests/lora/test_lora_layers_ltx_video.py

@@ -16,6 +16,7 @@ import sys
 import unittest
 
 import torch
+from parameterized import parameterized
 from transformers import AutoTokenizer, T5EncoderModel
 
 from diffusers import (
@@ -108,40 +109,26 @@ class LTXVideoLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",)])
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3)
+    def test_lora_set_adapters_scenarios(self, scenario):
+        super()._test_lora_set_adapters_scenarios(scenario, expected_atol=9e-3)
 
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+    @parameterized.expand(
-        super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3)
+        [
-
+            # Test actions on text_encoder LoRA only
-    @unittest.skip("Not supported in LTXVideo.")
+            ("fused", "text_encoder_only"),
-    def test_simple_inference_with_text_denoiser_block_scale(self):
+            ("unloaded", "text_encoder_only"),
-        pass
+            ("save_load", "text_encoder_only"),
-
+            # Test actions on both text_encoder and denoiser LoRA
-    @unittest.skip("Not supported in LTXVideo.")
+            ("fused", "text_and_denoiser"),
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
+            ("unloaded", "text_and_denoiser"),
-        pass
+            ("unfused", "text_and_denoiser"),
+            ("save_load", "text_and_denoiser"),
+        ]
+    )
+    def test_lora_actions(self, action, components_to_add):
+        super()._test_lora_actions(action, components_to_add, expected_atol=9e-3)
 
     @unittest.skip("Not supported in LTXVideo.")
     def test_modify_padding_mode(self):
         pass
-
-    @unittest.skip("Text encoder LoRA is not supported in LTXVideo.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in LTXVideo.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in LTXVideo.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in LTXVideo.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in LTXVideo.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass

tests/lora/test_lora_layers_lumina2.py

@@ -101,38 +101,11 @@ class Lumina2LoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    @unittest.skip("Not supported in Lumina2.")
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in Lumina2.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
-
     @unittest.skip("Not supported in Lumina2.")
     def test_modify_padding_mode(self):
         pass
 
     @unittest.skip("Text encoder LoRA is not supported in Lumina2.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Lumina2.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Lumina2.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Lumina2.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Lumina2.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass
-
     @skip_mps
     @pytest.mark.xfail(
         condition=torch.device(torch_device).type == "cpu" and is_torch_version(">=", "2.5"),
@@ -141,11 +114,9 @@ class Lumina2LoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
     )
     def test_lora_fuse_nan(self):
         for scheduler_cls in self.scheduler_classes:
-            components, text_lora_config, denoiser_lora_config = self.get_dummy_components(scheduler_cls)
+            pipe, inputs, output_no_lora, text_lora_config, denoiser_lora_config = (
-            pipe = self.pipeline_class(**components)
+                self._setup_pipeline_and_get_base_output(scheduler_cls)
-            pipe = pipe.to(torch_device)
+            )
-            pipe.set_progress_bar_config(disable=None)
-            _, _, inputs = self.get_dummy_inputs(with_generator=False)
 
             if "text_encoder" in self.pipeline_class._lora_loadable_modules:
                 pipe.text_encoder.add_adapter(text_lora_config, "adapter-1")

tests/lora/test_lora_layers_mochi.py

@@ -16,14 +16,11 @@ import sys
 import unittest
 
 import torch
+from parameterized import parameterized
 from transformers import AutoTokenizer, T5EncoderModel
 
 from diffusers import AutoencoderKLMochi, FlowMatchEulerDiscreteScheduler, MochiPipeline, MochiTransformer3DModel
-from diffusers.utils.testing_utils import (
+from diffusers.utils.testing_utils import floats_tensor, require_peft_backend, skip_mps
-    floats_tensor,
-    require_peft_backend,
-    skip_mps,
-)
 
 
 sys.path.append(".")
@@ -99,44 +96,28 @@ class MochiLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",)])
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3)
+    def test_lora_set_adapters_scenarios(self, scenario):
+        super()._test_lora_set_adapters_scenarios(scenario, expected_atol=9e-3)
 
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+    @parameterized.expand(
-        super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3)
+        [
-
+            # Test actions on text_encoder LoRA only
-    @unittest.skip("Not supported in Mochi.")
+            ("fused", "text_encoder_only"),
-    def test_simple_inference_with_text_denoiser_block_scale(self):
+            ("unloaded", "text_encoder_only"),
-        pass
+            ("save_load", "text_encoder_only"),
-
+            # Test actions on both text_encoder and denoiser LoRA
-    @unittest.skip("Not supported in Mochi.")
+            ("fused", "text_and_denoiser"),
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
+            ("unloaded", "text_and_denoiser"),
-        pass
+            ("unfused", "text_and_denoiser"),
+            ("save_load", "text_and_denoiser"),
+        ]
+    )
+    def test_lora_actions(self, action, components_to_add):
+        super()._test_lora_actions(action, components_to_add, expected_atol=9e-3)
 
     @unittest.skip("Not supported in Mochi.")
     def test_modify_padding_mode(self):
         pass
 
-    @unittest.skip("Text encoder LoRA is not supported in Mochi.")
+    # TODO: skip them properly
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Mochi.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Mochi.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Mochi.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Mochi.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass
-
-    @unittest.skip("Not supported in CogVideoX.")
-    def test_simple_inference_with_text_denoiser_multi_adapter_block_lora(self):
-        pass

tests/lora/test_lora_layers_sana.py

@@ -108,31 +108,3 @@ class SanaLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
     @unittest.skip("Not supported in SANA.")
     def test_modify_padding_mode(self):
         pass
-
-    @unittest.skip("Not supported in SANA.")
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in SANA.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in SANA.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in SANA.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in SANA.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in SANA.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in SANA.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass

tests/lora/test_lora_layers_sd.py

@@ -20,6 +20,7 @@ import numpy as np
 import torch
 import torch.nn as nn
 from huggingface_hub import hf_hub_download
+from parameterized import parameterized
 from safetensors.torch import load_file
 from transformers import CLIPTextModel, CLIPTokenizer
 
@@ -208,6 +209,19 @@ class StableDiffusionLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase):
             if "lora_" in name:
                 self.assertNotEqual(param.device, torch.device("cpu"))
 
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",)])
+    def test_lora_set_adapters_scenarios(self, scenario):
+        if torch.cuda.is_available():
+            expected_atol = 9e-2
+            expected_rtol = 9e-2
+        else:
+            expected_atol = 1e-3
+            expected_rtol = 1e-3
+
+        super()._test_lora_set_adapters_scenarios(
+            scenario=scenario, expected_atol=expected_atol, expected_rtol=expected_rtol
+        )
+
     @slow
     @require_torch_accelerator
     def test_integration_set_lora_device_different_target_layers(self):

tests/lora/test_lora_layers_sd3.py

@@ -114,17 +114,7 @@ class SD3LoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
         lora_filename = "lora_peft_format.safetensors"
         pipe.load_lora_weights(lora_model_id, weight_name=lora_filename)
 
-    @unittest.skip("Not supported in SD3.")
+    # TODO: skip them properly
-    def test_simple_inference_with_text_denoiser_block_scale(self):
-        pass
-
-    @unittest.skip("Not supported in SD3.")
-    def test_simple_inference_with_text_denoiser_multi_adapter_block_lora(self):
-        pass
-
-    @unittest.skip("Not supported in SD3.")
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
-        pass
 
     @unittest.skip("Not supported in SD3.")
     def test_modify_padding_mode(self):

tests/lora/test_lora_layers_sdxl.py

@@ -22,6 +22,7 @@ import unittest
 import numpy as np
 import torch
 from packaging import version
+from parameterized import parameterized
 from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer
 
 from diffusers import (
@@ -117,7 +118,20 @@ class StableDiffusionXLLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase):
     def test_multiple_wrong_adapter_name_raises_error(self):
         super().test_multiple_wrong_adapter_name_raises_error()
 
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+    @parameterized.expand(
+        [
+            # Test actions on text_encoder LoRA only
+            ("fused", "text_encoder_only"),
+            ("unloaded", "text_encoder_only"),
+            ("save_load", "text_encoder_only"),
+            # Test actions on both text_encoder and denoiser LoRA
+            ("fused", "text_and_denoiser"),
+            ("unloaded", "text_and_denoiser"),
+            ("unfused", "text_and_denoiser"),
+            ("save_load", "text_and_denoiser"),
+        ]
+    )
+    def test_lora_actions(self, action, components_to_add):
         if torch.cuda.is_available():
             expected_atol = 9e-2
             expected_rtol = 9e-2
@@ -125,11 +139,10 @@ class StableDiffusionXLLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase):
             expected_atol = 1e-3
             expected_rtol = 1e-3
 
-        super().test_simple_inference_with_text_denoiser_lora_unfused(
+        super()._test_lora_actions(action, components_to_add, expected_atol=expected_atol, expected_rtol=expected_rtol)
-            expected_atol=expected_atol, expected_rtol=expected_rtol
-        )
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",), ("fused_multi",)])
+    def test_lora_set_adapters_scenarios(self, scenario):
         if torch.cuda.is_available():
             expected_atol = 9e-2
             expected_rtol = 9e-2
@@ -137,8 +150,8 @@ class StableDiffusionXLLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase):
             expected_atol = 1e-3
             expected_rtol = 1e-3
 
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(
+        super()._test_lora_set_adapters_scenarios(
-            expected_atol=expected_atol, expected_rtol=expected_rtol
+            scenario=scenario, expected_atol=expected_atol, expected_rtol=expected_rtol
         )
 
     def test_lora_scale_kwargs_match_fusion(self):

tests/lora/test_lora_layers_wan.py

@@ -16,19 +16,11 @@ import sys
 import unittest
 
 import torch
+from parameterized import parameterized
 from transformers import AutoTokenizer, T5EncoderModel
 
-from diffusers import (
+from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler, WanPipeline, WanTransformer3DModel
-    AutoencoderKLWan,
+from diffusers.utils.testing_utils import floats_tensor, require_peft_backend, skip_mps
-    FlowMatchEulerDiscreteScheduler,
-    WanPipeline,
-    WanTransformer3DModel,
-)
-from diffusers.utils.testing_utils import (
-    floats_tensor,
-    require_peft_backend,
-    skip_mps,
-)
 
 
 sys.path.append(".")
@@ -104,40 +96,26 @@ class WanLoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",)])
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3)
+    def test_lora_set_adapters_scenarios(self, scenario):
+        super()._test_lora_set_adapters_scenarios(scenario, expected_atol=9e-3)
 
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+    @parameterized.expand(
-        super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3)
+        [
-
+            # Test actions on text_encoder LoRA only
-    @unittest.skip("Not supported in Wan.")
+            ("fused", "text_encoder_only"),
-    def test_simple_inference_with_text_denoiser_block_scale(self):
+            ("unloaded", "text_encoder_only"),
-        pass
+            ("save_load", "text_encoder_only"),
-
+            # Test actions on both text_encoder and denoiser LoRA
-    @unittest.skip("Not supported in Wan.")
+            ("fused", "text_and_denoiser"),
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
+            ("unloaded", "text_and_denoiser"),
-        pass
+            ("unfused", "text_and_denoiser"),
+            ("save_load", "text_and_denoiser"),
+        ]
+    )
+    def test_lora_actions(self, action, components_to_add):
+        super()._test_lora_actions(action, components_to_add, expected_atol=9e-3)
 
     @unittest.skip("Not supported in Wan.")
     def test_modify_padding_mode(self):
         pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass

tests/lora/test_lora_layers_wanvace.py

@@ -21,19 +21,13 @@ import numpy as np
 import pytest
 import safetensors.torch
 import torch
+from parameterized import parameterized
 from PIL import Image
 from transformers import AutoTokenizer, T5EncoderModel
 
 from diffusers import AutoencoderKLWan, FlowMatchEulerDiscreteScheduler, WanVACEPipeline, WanVACETransformer3DModel
 from diffusers.utils.import_utils import is_peft_available
-from diffusers.utils.testing_utils import (
+from diffusers.utils.testing_utils import floats_tensor, require_peft_backend, skip_mps, torch_device
-    floats_tensor,
-    is_flaky,
-    require_peft_backend,
-    require_peft_version_greater,
-    skip_mps,
-    torch_device,
-)
 
 
 if is_peft_available():
@@ -121,44 +115,30 @@ class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
 
         return noise, input_ids, pipeline_inputs
 
-    def test_simple_inference_with_text_lora_denoiser_fused_multi(self):
+    @parameterized.expand([("simple",), ("weighted",), ("block_lora",), ("delete_adapter",)])
-        super().test_simple_inference_with_text_lora_denoiser_fused_multi(expected_atol=9e-3)
+    def test_lora_set_adapters_scenarios(self, scenario):
+        super()._test_lora_set_adapters_scenarios(scenario, expected_atol=9e-3)
 
-    def test_simple_inference_with_text_denoiser_lora_unfused(self):
+    @parameterized.expand(
-        super().test_simple_inference_with_text_denoiser_lora_unfused(expected_atol=9e-3)
+        [
-
+            # Test actions on text_encoder LoRA only
-    @unittest.skip("Not supported in Wan VACE.")
+            ("fused", "text_encoder_only"),
-    def test_simple_inference_with_text_denoiser_block_scale(self):
+            ("unloaded", "text_encoder_only"),
-        pass
+            ("save_load", "text_encoder_only"),
-
+            # Test actions on both text_encoder and denoiser LoRA
-    @unittest.skip("Not supported in Wan VACE.")
+            ("fused", "text_and_denoiser"),
-    def test_simple_inference_with_text_denoiser_block_scale_for_all_dict_options(self):
+            ("unloaded", "text_and_denoiser"),
-        pass
+            ("unfused", "text_and_denoiser"),
+            ("save_load", "text_and_denoiser"),
+        ]
+    )
+    def test_lora_actions(self, action, components_to_add):
+        super()._test_lora_actions(action, components_to_add, expected_atol=9e-3)
 
     @unittest.skip("Not supported in Wan VACE.")
     def test_modify_padding_mode(self):
         pass
 
-    @unittest.skip("Text encoder LoRA is not supported in Wan VACE.")
-    def test_simple_inference_with_partial_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan VACE.")
-    def test_simple_inference_with_text_lora(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan VACE.")
-    def test_simple_inference_with_text_lora_and_scale(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan VACE.")
-    def test_simple_inference_with_text_lora_fused(self):
-        pass
-
-    @unittest.skip("Text encoder LoRA is not supported in Wan VACE.")
-    def test_simple_inference_with_text_lora_save_load(self):
-        pass
-
     @pytest.mark.xfail(
         condition=True,
         reason="RuntimeError: Input type (float) and bias type (c10::BFloat16) should be the same",
@@ -167,7 +147,6 @@ class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
     def test_layerwise_casting_inference_denoiser(self):
         super().test_layerwise_casting_inference_denoiser()
 
-    @require_peft_version_greater("0.13.2")
     def test_lora_exclude_modules_wanvace(self):
         scheduler_cls = self.scheduler_classes[0]
         exclude_module_name = "vace_blocks.0.proj_out"
@@ -216,7 +195,3 @@ class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests):
                 np.allclose(output_lora_exclude_modules, output_lora_pretrained, atol=1e-3, rtol=1e-3),
                 "Lora outputs should match.",
             )
-
-    @is_flaky
-    def test_simple_inference_with_text_denoiser_lora_and_scale(self):
-        super().test_simple_inference_with_text_denoiser_lora_and_scale()