Merge branch 'convert-cascade-lite' of https://github.com/huggingface/diffusers into convert-cascade-lite

.github/workflows/nightly_tests.yml

@@ -12,7 +12,6 @@ env:
   PYTEST_TIMEOUT: 600
   RUN_SLOW: yes
   RUN_NIGHTLY: yes
-  SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
 
 jobs:
   run_nightly_tests:
@@ -65,7 +64,6 @@ jobs:
           python -m uv pip install -e [quality,test]
           python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers
           python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate
-          python -m uv pip install pytest-reportlog
 
       - name: Environment
         run: |
@@ -80,8 +78,7 @@ jobs:
           python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
             -s -v -k "not Flax and not Onnx" \
             --make-reports=tests_${{ matrix.config.report }} \
-            --report-log=${{ matrix.config.report }}.log \
-            tests/ 
+            tests/
 
       - name: Run nightly Flax TPU tests
         if: ${{ matrix.config.framework == 'flax' }}
@@ -92,7 +89,6 @@ jobs:
           python -m pytest -n 0 \
             -s -v -k "Flax" \
             --make-reports=tests_${{ matrix.config.report }} \
-            --report-log=${{ matrix.config.report }}.log \
             tests/
 
       - name: Run nightly ONNXRuntime CUDA tests
@@ -104,7 +100,6 @@ jobs:
           python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
             -s -v -k "Onnx" \
             --make-reports=tests_${{ matrix.config.report }} \
-            --report-log=${{ matrix.config.report }}.log \ 
             tests/
 
       - name: Failure short reports
@@ -117,12 +112,6 @@ jobs:
         with:
           name: ${{ matrix.config.report }}_test_reports
           path: reports
-      
-      - name: Generate Report and Notify Channel
-        if: always()
-        run: |
-          pip install slack_sdk tabulate
-          python scripts/log_reports.py >> $GITHUB_STEP_SUMMARY
 
   run_nightly_tests_apple_m1:
     name: Nightly PyTorch MPS tests on MacOS
@@ -151,7 +140,6 @@ jobs:
           ${CONDA_RUN} python -m uv pip install -e [quality,test]
           ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
           ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate
-          ${CONDA_RUN} python -m uv pip install pytest-reportlog
 
       - name: Environment
         shell: arch -arch arm64 bash {0}
@@ -164,9 +152,7 @@ jobs:
           HF_HOME: /System/Volumes/Data/mnt/cache
           HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
         run: |
-          ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
-            --report-log=tests_torch_mps.log \
-            tests/
+          ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps tests/
 
       - name: Failure short reports
         if: ${{ failure() }}
@@ -178,9 +164,3 @@ jobs:
         with:
           name: torch_mps_test_reports
           path: reports
-
-      - name: Generate Report and Notify Channel
-        if: always()
-        run: |
-          pip install slack_sdk tabulate
-          python scripts/log_reports.py >> $GITHUB_STEP_SUMMARY

.github/workflows/notify_slack_about_release.yml

@@ -1,23 +0,0 @@
-name: Notify Slack about a release
-
-on:
-  workflow_dispatch:
-  release:
-    types: [published]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    
-    - name: Setup Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.8'
-    
-    - name: Notify Slack about the release
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-      run: pip install requests && python utils/notify_slack_about_release.py

.github/workflows/pypi_publish.yaml

@@ -1,79 +0,0 @@
-# Adapted from https://blog.deepjyoti30.dev/pypi-release-github-action
-
-name: PyPI release
-
-on:
-  workflow_dispatch:
-  push:
-    tags:
-      - "*"
-
-jobs:
-  find-and-checkout-latest-branch:
-    runs-on: ubuntu-latest
-    outputs:
-      latest_branch: ${{ steps.set_latest_branch.outputs.latest_branch }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-
-      - name: Fetch latest branch
-        id: fetch_latest_branch
-        run: |
-          pip install -U requests packaging
-          LATEST_BRANCH=$(python utils/fetch_latest_release_branch.py)
-          echo "Latest branch: $LATEST_BRANCH"
-          echo "latest_branch=$LATEST_BRANCH" >> $GITHUB_ENV
-          
-      - name: Set latest branch output
-        id: set_latest_branch
-        run: echo "::set-output name=latest_branch::${{ env.latest_branch }}"
-
-  release:
-    needs: find-and-checkout-latest-branch
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ needs.find-and-checkout-latest-branch.outputs.latest_branch }}
-          
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.8"
-      
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -U setuptools wheel twine
-      
-      - name: Build the dist files
-        run: python setup.py bdist_wheel && python setup.py sdist
-      
-      - name: Publish to the test PyPI
-        env:
-          TWINE_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }}
-        run: twine upload dist/* -r pypitest --repository-url=https://test.pypi.org/legacy/    
-
-      - name: Test installing diffusers and importing
-        run: |
-          pip install diffusers && pip uninstall diffusers -y
-          pip install -i https://testpypi.python.org/pypi diffusers
-          python -c "from diffusers import __version__; print(__version__)"
-          python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('fusing/unet-ldm-dummy-update'); pipe()"
-          python -c "from diffusers import DiffusionPipeline; pipe = DiffusionPipeline.from_pretrained('hf-internal-testing/tiny-stable-diffusion-pipe', safety_checker=None); pipe('ah suh du')"
-          python -c "from diffusers import *"
-
-      - name: Publish to PyPI
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: twine upload dist/* -r pypi

docs/source/en/_toctree.yml

@@ -104,8 +104,6 @@
       title: Latent Consistency Model-LoRA
     - local: using-diffusers/inference_with_lcm
       title: Latent Consistency Model
-    - local: using-diffusers/inference_with_tcd_lora
-      title: Trajectory Consistency Distillation-LoRA
     - local: using-diffusers/svd
       title: Stable Video Diffusion
     title: Specific pipeline examples
@@ -306,8 +304,6 @@
       title: Latent Consistency Models
     - local: api/pipelines/latent_diffusion
       title: Latent Diffusion
-    - local: api/pipelines/ledits_pp
-      title: LEDITS++
     - local: api/pipelines/panorama
       title: MultiDiffusion
     - local: api/pipelines/musicldm
@@ -404,10 +400,6 @@
       title: EulerAncestralDiscreteScheduler
     - local: api/schedulers/euler
       title: EulerDiscreteScheduler
-    - local: api/schedulers/edm_euler
-      title: EDMEulerScheduler
-    - local: api/schedulers/edm_multistep_dpm_solver
-      title: EDMDPMSolverMultistepScheduler
     - local: api/schedulers/heun
       title: HeunDiscreteScheduler
     - local: api/schedulers/ipndm

docs/source/en/api/pipelines/ledits_pp.md

@@ -1,54 +0,0 @@
-<!--Copyright 2023 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
-an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-specific language governing permissions and limitations under the License.
--->
-
-# LEDITS++
-
-LEDITS++ was proposed in [LEDITS++: Limitless Image Editing using Text-to-Image Models](https://huggingface.co/papers/2311.16711) by Manuel Brack, Felix Friedrich, Katharina Kornmeier, Linoy Tsaban, Patrick Schramowski, Kristian Kersting, Apolinário Passos.
-
-The abstract from the paper is:
-
-*Text-to-image diffusion models have recently received increasing interest for their astonishing ability to produce high-fidelity images from solely text inputs. Subsequent research efforts aim to exploit and apply their capabilities to real image editing. However, existing image-to-image methods are often inefficient, imprecise, and of limited versatility. They either require time-consuming fine-tuning, deviate unnecessarily strongly from the input image, and/or lack support for multiple, simultaneous edits. To address these issues, we introduce LEDITS++, an efficient yet versatile and precise textual image manipulation technique. LEDITS++'s novel inversion approach requires no tuning nor optimization and produces high-fidelity results with a few diffusion steps. Second, our methodology supports multiple simultaneous edits and is architecture-agnostic. Third, we use a novel implicit masking technique that limits changes to relevant image regions. We propose the novel TEdBench++ benchmark as part of our exhaustive evaluation. Our results demonstrate the capabilities of LEDITS++ and its improvements over previous methods. The project page is available at https://leditsplusplus-project.static.hf.space .*
-
-<Tip>
-
-You can find additional information about LEDITS++ on the [project page](https://leditsplusplus-project.static.hf.space/index.html) and try it out in a [demo](https://huggingface.co/spaces/editing-images/leditsplusplus).
-
-</Tip>
-
-<Tip warning={true}>
-Due to some backward compatability issues with the current diffusers implementation of [`~schedulers.DPMSolverMultistepScheduler`] this implementation of LEdits++ can no longer guarantee perfect inversion. 
-This issue is unlikely to have any noticeable effects on applied use-cases. However, we provide an alternative implementation that guarantees perfect inversion in a dedicated [GitHub repo](https://github.com/ml-research/ledits_pp). 
-</Tip>
-
-We provide two distinct pipelines based on different pre-trained models. 
-
-## LEditsPPPipelineStableDiffusion
-[[autodoc]] pipelines.ledits_pp.LEditsPPPipelineStableDiffusion
-	- all
-	- __call__
-	- invert
-
-## LEditsPPPipelineStableDiffusionXL
-[[autodoc]] pipelines.ledits_pp.LEditsPPPipelineStableDiffusionXL
-	- all
-	- __call__
-	- invert
-
-
-
-## LEditsPPDiffusionPipelineOutput
-[[autodoc]] pipelines.ledits_pp.pipeline_output.LEditsPPDiffusionPipelineOutput
-	- all
-
-## LEditsPPInversionPipelineOutput
-[[autodoc]] pipelines.ledits_pp.pipeline_output.LEditsPPInversionPipelineOutput
-	- all

docs/source/en/api/pipelines/overview.md

@@ -57,7 +57,6 @@ The table below lists all the pipelines currently available in 🤗 Diffusers an
 | [Latent Consistency Models](latent_consistency_models) | text2image |
 | [Latent Diffusion](latent_diffusion) | text2image, super-resolution |
 | [LDM3D](stable_diffusion/ldm3d_diffusion) | text2image, text-to-3D, text-to-pano, upscaling |
-| [LEDITS++](ledits_pp) | image editing |
 | [MultiDiffusion](panorama) | text2image |
 | [MusicLDM](musicldm) | text2audio |
 | [Paint by Example](paint_by_example) | inpainting |

docs/source/en/api/pipelines/semantic_stable_diffusion.md

@@ -30,6 +30,6 @@ Make sure to check out the Schedulers [guide](../../using-diffusers/schedulers)
 	- all
 	- __call__
 
-## SemanticStableDiffusionPipelineOutput
+## StableDiffusionSafePipelineOutput
 [[autodoc]] pipelines.semantic_stable_diffusion.pipeline_output.SemanticStableDiffusionPipelineOutput
 	- all

docs/source/en/api/schedulers/edm_euler.md

@@ -1,22 +0,0 @@
-<!--Copyright 2024 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
-an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-specific language governing permissions and limitations under the License.
--->
-
-# EDMEulerScheduler
-
-The Karras formulation of the Euler scheduler (Algorithm 2) from the [Elucidating the Design Space of Diffusion-Based Generative Models](https://huggingface.co/papers/2206.00364) paper by Karras et al. This is a fast scheduler which can often generate good outputs in 20-30 steps. The scheduler is based on the original [k-diffusion](https://github.com/crowsonkb/k-diffusion/blob/481677d114f6ea445aa009cf5bd7a9cdee909e47/k_diffusion/sampling.py#L51) implementation by [Katherine Crowson](https://github.com/crowsonkb/).
-
-
-## EDMEulerScheduler
-[[autodoc]] EDMEulerScheduler
-
-## EDMEulerSchedulerOutput
-[[autodoc]] schedulers.scheduling_edm_euler.EDMEulerSchedulerOutput

docs/source/en/api/schedulers/edm_multistep_dpm_solver.md

@@ -1,24 +0,0 @@
-<!--Copyright 2024 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
-an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-specific language governing permissions and limitations under the License.
--->
-
-# EDMDPMSolverMultistepScheduler
-
-`EDMDPMSolverMultistepScheduler` is a [Karras formulation](https://huggingface.co/papers/2206.00364) of `DPMSolverMultistep`, a multistep scheduler from [DPM-Solver: A Fast ODE Solver for Diffusion Probabilistic Model Sampling in Around 10 Steps](https://huggingface.co/papers/2206.00927) and [DPM-Solver++: Fast Solver for Guided Sampling of Diffusion Probabilistic Models](https://huggingface.co/papers/2211.01095) by Cheng Lu, Yuhao Zhou, Fan Bao, Jianfei Chen, Chongxuan Li, and Jun Zhu.
-
-DPMSolver (and the improved version DPMSolver++) is a fast dedicated high-order solver for diffusion ODEs with convergence order guarantee. Empirically, DPMSolver sampling with only 20 steps can generate high-quality
-samples, and it can generate quite good samples even in 10 steps.
-
-## EDMDPMSolverMultistepScheduler
-[[autodoc]] EDMDPMSolverMultistepScheduler
-
-## SchedulerOutput
-[[autodoc]] schedulers.scheduling_utils.SchedulerOutput

docs/source/en/using-diffusers/inference_with_tcd_lora.md

@@ -1,438 +0,0 @@
-<!--Copyright 2024 The HuggingFace Team. All rights reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
-an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
-specific language governing permissions and limitations under the License.
--->
-
-[[open-in-colab]]
-
-# Trajectory Consistency Distillation-LoRA
-
-Trajectory Consistency Distillation (TCD) enables a model to generate higher quality and more detailed images with fewer steps. Moreover, owing to the effective error mitigation during the distillation process, TCD demonstrates superior performance even under conditions of large inference steps.
-
-The major advantages of TCD are:
-
-- Better than Teacher: TCD demonstrates superior generative quality at both small and large inference steps and exceeds the performance of [DPM-Solver++(2S)](../../api/schedulers/multistep_dpm_solver) with Stable Diffusion XL (SDXL). There is no additional discriminator or LPIPS supervision included during TCD training.
-
-- Flexible Inference Steps: The inference steps for TCD sampling can be freely adjusted without adversely affecting the image quality.
-
-- Freely change detail level: During inference, the level of detail in the image can be adjusted with a single hyperparameter, *gamma*.
-
-> [!TIP]
-> For more technical details of TCD, please refer to the [paper](https://arxiv.org/abs/2402.19159) or official [project page](https://mhh0318.github.io/tcd/)).
-
-For large models like SDXL, TCD is trained with [LoRA](https://huggingface.co/docs/peft/conceptual_guides/adapter#low-rank-adaptation-lora) to reduce memory usage. This is also useful because you can reuse LoRAs between different finetuned models, as long as they share the same base model, without further training.
-
-
-
-This guide will show you how to perform inference with TCD-LoRAs for a variety of tasks like text-to-image and inpainting, as well as how you can easily combine TCD-LoRAs with other adapters. Choose one of the supported base model and it's corresponding TCD-LoRA checkpoint from the table below to get started.
-
-| Base model                                                                                      | TCD-LoRA checkpoint                                            |
-|-------------------------------------------------------------------------------------------------|----------------------------------------------------------------|
-| [stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5)                  | [TCD-SD15](https://huggingface.co/h1t/TCD-SD15-LoRA)           |
-| [stable-diffusion-2-1-base](https://huggingface.co/stabilityai/stable-diffusion-2-1-base)       | [TCD-SD21-base](https://huggingface.co/h1t/TCD-SD21-base-LoRA) |
-| [stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) | [TCD-SDXL](https://huggingface.co/h1t/TCD-SDXL-LoRA)           |
-
-
-Make sure you have [PEFT](https://github.com/huggingface/peft) installed for better LoRA support.
-
-```bash
-pip install -U peft
-```
-
-## General tasks
-
-In this guide, let's use the [`StableDiffusionXLPipeline`] and the [`TCDScheduler`]. Use the [`~StableDiffusionPipeline.load_lora_weights`] method to load the SDXL-compatible TCD-LoRA weights.
-
-A few tips to keep in mind for TCD-LoRA inference are to:
-
-- Keep the `num_inference_steps` between 4 and 50
-- Set `eta` (used to control stochasticity at each step) between 0 and 1. You should use a higher `eta` when increasing the number of inference steps, but the downside is that a larger `eta` in [`TCDScheduler`] leads to blurrier images. A value of 0.3 is recommended to produce good results.
-
-<hfoptions id="tasks">
-<hfoption id="text-to-image">
-
-```python
-import torch
-from diffusers import StableDiffusionXLPipeline, TCDScheduler
-
-device = "cuda"
-base_model_id = "stabilityai/stable-diffusion-xl-base-1.0"
-tcd_lora_id = "h1t/TCD-SDXL-LoRA"
-
-pipe = StableDiffusionXLPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16, variant="fp16").to(device)
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-pipe.load_lora_weights(tcd_lora_id)
-pipe.fuse_lora()
-
-prompt = "Painting of the orange cat Otto von Garfield, Count of Bismarck-Schönhausen, Duke of Lauenburg, Minister-President of Prussia. Depicted wearing a Prussian Pickelhaube and eating his favorite meal - lasagna."
-
-image = pipe(
-    prompt=prompt,
-    num_inference_steps=4,
-    guidance_scale=0,
-    eta=0.3, 
-    generator=torch.Generator(device=device).manual_seed(0),
-).images[0]
-```
-
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/demo_image.png)
-
-</hfoption>
-
-<hfoption id="inpainting">
-
-```python
-import torch
-from diffusers import AutoPipelineForInpainting, TCDScheduler
-from diffusers.utils import load_image, make_image_grid
-
-device = "cuda"
-base_model_id = "diffusers/stable-diffusion-xl-1.0-inpainting-0.1"
-tcd_lora_id = "h1t/TCD-SDXL-LoRA"
-
-pipe = AutoPipelineForInpainting.from_pretrained(base_model_id, torch_dtype=torch.float16, variant="fp16").to(device)
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-pipe.load_lora_weights(tcd_lora_id)
-pipe.fuse_lora()
-
-img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
-mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
-
-init_image = load_image(img_url).resize((1024, 1024))
-mask_image = load_image(mask_url).resize((1024, 1024))
-
-prompt = "a tiger sitting on a park bench"
-
-image = pipe(
-  prompt=prompt,
-  image=init_image,
-  mask_image=mask_image,
-  num_inference_steps=8,
-  guidance_scale=0,
-  eta=0.3,
-  strength=0.99,  # make sure to use `strength` below 1.0
-  generator=torch.Generator(device=device).manual_seed(0),
-).images[0]
-
-grid_image = make_image_grid([init_image, mask_image, image], rows=1, cols=3)
-```
-
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/inpainting_tcd.png)
-
-
-</hfoption>
-</hfoptions>
-
-## Community models
-
-TCD-LoRA also works with many community finetuned models and plugins. For example, load the [animagine-xl-3.0](https://huggingface.co/cagliostrolab/animagine-xl-3.0) checkpoint which is a community finetuned version of SDXL for generating anime images.
-
-```python
-import torch
-from diffusers import StableDiffusionXLPipeline, TCDScheduler
-
-device = "cuda"
-base_model_id = "cagliostrolab/animagine-xl-3.0"
-tcd_lora_id = "h1t/TCD-SDXL-LoRA"
-
-pipe = StableDiffusionXLPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16, variant="fp16").to(device)
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-pipe.load_lora_weights(tcd_lora_id)
-pipe.fuse_lora()
-
-prompt = "A man, clad in a meticulously tailored military uniform, stands with unwavering resolve. The uniform boasts intricate details, and his eyes gleam with determination. Strands of vibrant, windswept hair peek out from beneath the brim of his cap."
-
-image = pipe(
-    prompt=prompt,
-    num_inference_steps=8,
-    guidance_scale=0,
-    eta=0.3, 
-    generator=torch.Generator(device=device).manual_seed(0),
-).images[0]
-```
-
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/animagine_xl.png)
-
-TCD-LoRA also supports other LoRAs trained on different styles. For example, let's load the [TheLastBen/Papercut_SDXL](https://huggingface.co/TheLastBen/Papercut_SDXL) LoRA and fuse it with the TCD-LoRA with the [`~loaders.UNet2DConditionLoadersMixin.set_adapters`] method. 
-
-> [!TIP]
-> Check out the [Merge LoRAs](merge_loras) guide to learn more about efficient merging methods.
-
-```python
-import torch
-from diffusers import StableDiffusionXLPipeline
-from scheduling_tcd import TCDScheduler 
-
-device = "cuda"
-base_model_id = "stabilityai/stable-diffusion-xl-base-1.0"
-tcd_lora_id = "h1t/TCD-SDXL-LoRA"
-styled_lora_id = "TheLastBen/Papercut_SDXL"
-
-pipe = StableDiffusionXLPipeline.from_pretrained(base_model_id, torch_dtype=torch.float16, variant="fp16").to(device)
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-pipe.load_lora_weights(tcd_lora_id, adapter_name="tcd")
-pipe.load_lora_weights(styled_lora_id, adapter_name="style")
-pipe.set_adapters(["tcd", "style"], adapter_weights=[1.0, 1.0])
-
-prompt = "papercut of a winter mountain, snow"
-
-image = pipe(
-    prompt=prompt,
-    num_inference_steps=4,
-    guidance_scale=0,
-    eta=0.3, 
-    generator=torch.Generator(device=device).manual_seed(0),
-).images[0]
-```
-
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/styled_lora.png)
-
-
-## Adapters
-
-TCD-LoRA is very versatile, and it can be combined with other adapter types like ControlNets, IP-Adapter, and AnimateDiff.
-
-<hfoptions id="adapters">
-<hfoption id="ControlNet">
-
-### Depth ControlNet
-
-```python
-import torch
-import numpy as np
-from PIL import Image
-from transformers import DPTFeatureExtractor, DPTForDepthEstimation
-from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline
-from diffusers.utils import load_image, make_image_grid
-from scheduling_tcd import TCDScheduler 
-
-device = "cuda"
-depth_estimator = DPTForDepthEstimation.from_pretrained("Intel/dpt-hybrid-midas").to(device)
-feature_extractor = DPTFeatureExtractor.from_pretrained("Intel/dpt-hybrid-midas")
-
-def get_depth_map(image):
-    image = feature_extractor(images=image, return_tensors="pt").pixel_values.to(device)
-    with torch.no_grad(), torch.autocast(device):
-        depth_map = depth_estimator(image).predicted_depth
-
-    depth_map = torch.nn.functional.interpolate(
-        depth_map.unsqueeze(1),
-        size=(1024, 1024),
-        mode="bicubic",
-        align_corners=False,
-    )
-    depth_min = torch.amin(depth_map, dim=[1, 2, 3], keepdim=True)
-    depth_max = torch.amax(depth_map, dim=[1, 2, 3], keepdim=True)
-    depth_map = (depth_map - depth_min) / (depth_max - depth_min)
-    image = torch.cat([depth_map] * 3, dim=1)
-
-    image = image.permute(0, 2, 3, 1).cpu().numpy()[0]
-    image = Image.fromarray((image * 255.0).clip(0, 255).astype(np.uint8))
-    return image
-
-base_model_id = "stabilityai/stable-diffusion-xl-base-1.0"
-controlnet_id = "diffusers/controlnet-depth-sdxl-1.0"
-tcd_lora_id = "h1t/TCD-SDXL-LoRA"
-
-controlnet = ControlNetModel.from_pretrained(
-    controlnet_id,
-    torch_dtype=torch.float16,
-    variant="fp16",
-).to(device)
-pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
-    base_model_id,
-    controlnet=controlnet,
-    torch_dtype=torch.float16,
-    variant="fp16",
-).to(device)
-pipe.enable_model_cpu_offload()
-
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-pipe.load_lora_weights(tcd_lora_id)
-pipe.fuse_lora()
-
-prompt = "stormtrooper lecture, photorealistic"
-
-image = load_image("https://huggingface.co/lllyasviel/sd-controlnet-depth/resolve/main/images/stormtrooper.png")
-depth_image = get_depth_map(image)
-
-controlnet_conditioning_scale = 0.5  # recommended for good generalization
-
-image = pipe(
-    prompt, 
-    image=depth_image, 
-    num_inference_steps=4, 
-    guidance_scale=0,
-    eta=0.3,
-    controlnet_conditioning_scale=controlnet_conditioning_scale,
-    generator=torch.Generator(device=device).manual_seed(0),
-).images[0]
-
-grid_image = make_image_grid([depth_image, image], rows=1, cols=2)
-```
-
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/controlnet_depth_tcd.png)
-
-### Canny ControlNet
-```python
-import torch
-from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline
-from diffusers.utils import load_image, make_image_grid
-from scheduling_tcd import TCDScheduler 
-
-device = "cuda"
-base_model_id = "stabilityai/stable-diffusion-xl-base-1.0"
-controlnet_id = "diffusers/controlnet-canny-sdxl-1.0"
-tcd_lora_id = "h1t/TCD-SDXL-LoRA"
-
-controlnet = ControlNetModel.from_pretrained(
-    controlnet_id,
-    torch_dtype=torch.float16,
-    variant="fp16",
-).to(device)
-pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
-    base_model_id,
-    controlnet=controlnet,
-    torch_dtype=torch.float16,
-    variant="fp16",
-).to(device)
-pipe.enable_model_cpu_offload()
-
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-pipe.load_lora_weights(tcd_lora_id)
-pipe.fuse_lora()
-
-prompt = "ultrarealistic shot of a furry blue bird"
-
-canny_image = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png")
-
-controlnet_conditioning_scale = 0.5  # recommended for good generalization
-
-image = pipe(
-    prompt, 
-    image=canny_image, 
-    num_inference_steps=4, 
-    guidance_scale=0,
-    eta=0.3,
-    controlnet_conditioning_scale=controlnet_conditioning_scale,
-    generator=torch.Generator(device=device).manual_seed(0),
-).images[0]
-
-grid_image = make_image_grid([canny_image, image], rows=1, cols=2)
-```
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/controlnet_canny_tcd.png)
-
-<Tip>
-The inference parameters in this example might not work for all examples, so we recommend you to try different values for `num_inference_steps`, `guidance_scale`, `controlnet_conditioning_scale` and `cross_attention_kwargs` parameters and choose the best one. 
-</Tip>
-
-</hfoption>
-<hfoption id="IP-Adapter">
-
-This example shows how to use the TCD-LoRA with the [IP-Adapter](https://github.com/tencent-ailab/IP-Adapter/tree/main) and SDXL.
-
-```python
-import torch
-from diffusers import StableDiffusionXLPipeline
-from diffusers.utils import load_image, make_image_grid
-
-from ip_adapter import IPAdapterXL
-from scheduling_tcd import TCDScheduler 
-
-device = "cuda"
-base_model_path = "stabilityai/stable-diffusion-xl-base-1.0"
-image_encoder_path = "sdxl_models/image_encoder"
-ip_ckpt = "sdxl_models/ip-adapter_sdxl.bin"
-tcd_lora_id = "h1t/TCD-SDXL-LoRA"
-
-pipe = StableDiffusionXLPipeline.from_pretrained(
-    base_model_path, 
-    torch_dtype=torch.float16, 
-    variant="fp16"
-)
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-pipe.load_lora_weights(tcd_lora_id)
-pipe.fuse_lora()
-
-ip_model = IPAdapterXL(pipe, image_encoder_path, ip_ckpt, device)
-
-ref_image = load_image("https://raw.githubusercontent.com/tencent-ailab/IP-Adapter/main/assets/images/woman.png").resize((512, 512))
-
-prompt = "best quality, high quality, wearing sunglasses"
-
-image = ip_model.generate(
-    pil_image=ref_image, 
-    prompt=prompt,
-    scale=0.5,
-    num_samples=1, 
-    num_inference_steps=4, 
-    guidance_scale=0,
-    eta=0.3, 
-    seed=0,
-)[0]
-
-grid_image = make_image_grid([ref_image, image], rows=1, cols=2)
-```
-
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/ip_adapter.png)
-
-
-
-</hfoption>
-<hfoption id="AnimateDiff">
-
-[`AnimateDiff`] allows animating images using Stable Diffusion models. TCD-LoRA can substantially accelerate the process without degrading image quality. The quality of animation with TCD-LoRA and AnimateDiff has a more lucid outcome.
-
-```python
-import torch
-from diffusers import MotionAdapter, AnimateDiffPipeline, DDIMScheduler
-from scheduling_tcd import TCDScheduler
-from diffusers.utils import export_to_gif
-
-adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5")
-pipe = AnimateDiffPipeline.from_pretrained(
-    "frankjoshua/toonyou_beta6",
-    motion_adapter=adapter,
-).to("cuda")
-
-# set TCDScheduler
-pipe.scheduler = TCDScheduler.from_config(pipe.scheduler.config)
-
-# load TCD LoRA
-pipe.load_lora_weights("h1t/TCD-SD15-LoRA", adapter_name="tcd")
-pipe.load_lora_weights("guoyww/animatediff-motion-lora-zoom-in", weight_name="diffusion_pytorch_model.safetensors", adapter_name="motion-lora")
-
-pipe.set_adapters(["tcd", "motion-lora"], adapter_weights=[1.0, 1.2])
-
-prompt = "best quality, masterpiece, 1girl, looking at viewer, blurry background, upper body, contemporary, dress"
-generator = torch.manual_seed(0)
-frames = pipe(
-    prompt=prompt,
-    num_inference_steps=5,
-    guidance_scale=0,
-    cross_attention_kwargs={"scale": 1},
-    num_frames=24,
-    eta=0.3,
-    generator=generator
-).frames[0]
-export_to_gif(frames, "animation.gif")
-```
-
-![](https://github.com/jabir-zheng/TCD/raw/main/assets/animation_example.gif)
-
-</hfoption>
-</hfoptions>

examples/advanced_diffusion_training/README.md

@@ -259,50 +259,6 @@ pip install git+https://github.com/huggingface/peft.git
 **Inference** 
 The inference is the same as if you train a regular LoRA 🤗
 
-## Conducting EDM-style training
-
-It's now possible to perform EDM-style training as proposed in [Elucidating the Design Space of Diffusion-Based Generative Models](https://arxiv.org/abs/2206.00364). 
-
-simply set:
-
-```diff
-+  --do_edm_style_training \
-```
-
-Other SDXL-like models that use the EDM formulation, such as [playgroundai/playground-v2.5-1024px-aesthetic](https://huggingface.co/playgroundai/playground-v2.5-1024px-aesthetic), can also be DreamBooth'd with the script. Below is an example command:
-
-```bash
-accelerate launch train_dreambooth_lora_sdxl_advanced.py \
-  --pretrained_model_name_or_path="playgroundai/playground-v2.5-1024px-aesthetic"  \
-  --dataset_name="linoyts/3d_icon" \
-  --instance_prompt="3d icon in the style of TOK" \
-  --validation_prompt="a TOK icon of an astronaut riding a horse, in the style of TOK" \
-  --output_dir="3d-icon-SDXL-LoRA" \
-  --do_edm_style_training \
-  --caption_column="prompt" \
-  --mixed_precision="bf16" \
-  --resolution=1024 \
-  --train_batch_size=3 \
-  --repeats=1 \
-  --report_to="wandb"\
-  --gradient_accumulation_steps=1 \
-  --gradient_checkpointing \
-  --learning_rate=1.0 \
-  --text_encoder_lr=1.0 \
-  --optimizer="prodigy"\
-  --train_text_encoder_ti\
-  --train_text_encoder_ti_frac=0.5\
-  --lr_scheduler="constant" \
-  --lr_warmup_steps=0 \
-  --rank=8 \
-  --max_train_steps=1000 \
-  --checkpointing_steps=2000 \
-  --seed="0" \
-  --push_to_hub
-```
-
-> [!CAUTION]
-> Min-SNR gamma is not supported with the EDM-style training yet. When training with the PlaygroundAI model, it's recommended to not pass any "variant".
 
 ### Tips and Tricks
 Check out [these recommended practices](https://huggingface.co/blog/sdxl_lora_advanced_script#additional-good-practices)

examples/advanced_diffusion_training/train_dreambooth_lora_sd15_advanced.py

@@ -70,7 +70,7 @@ from diffusers.utils.import_utils import is_xformers_available
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/advanced_diffusion_training/train_dreambooth_lora_sdxl_advanced.py

@@ -14,11 +14,9 @@
 # See the License for the specific language governing permissions and
 
 import argparse
-import contextlib
 import gc
 import hashlib
 import itertools
-import json
 import logging
 import math
 import os
@@ -39,7 +37,7 @@ import transformers
 from accelerate import Accelerator
 from accelerate.logging import get_logger
 from accelerate.utils import DistributedDataParallelKwargs, ProjectConfiguration, set_seed
-from huggingface_hub import create_repo, hf_hub_download, upload_folder
+from huggingface_hub import create_repo, upload_folder
 from packaging import version
 from peft import LoraConfig, set_peft_model_state_dict
 from peft.utils import get_peft_model_state_dict
@@ -57,8 +55,6 @@ from diffusers import (
     AutoencoderKL,
     DDPMScheduler,
     DPMSolverMultistepScheduler,
-    EDMEulerScheduler,
-    EulerDiscreteScheduler,
     StableDiffusionXLPipeline,
     UNet2DConditionModel,
 )
@@ -78,25 +74,11 @@ from diffusers.utils.torch_utils import is_compiled_module
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 
 
-def determine_scheduler_type(pretrained_model_name_or_path, revision):
-    model_index_filename = "model_index.json"
-    if os.path.isdir(pretrained_model_name_or_path):
-        model_index = os.path.join(pretrained_model_name_or_path, model_index_filename)
-    else:
-        model_index = hf_hub_download(
-            repo_id=pretrained_model_name_or_path, filename=model_index_filename, revision=revision
-        )
-
-    with open(model_index, "r") as f:
-        scheduler_type = json.load(f)["scheduler"][1]
-    return scheduler_type
-
-
 def save_model_card(
     repo_id: str,
     use_dora: bool,
@@ -388,11 +370,6 @@ def parse_args(input_args=None):
             " `args.validation_prompt` multiple times: `args.num_validation_images`."
         ),
     )
-    parser.add_argument(
-        "--do_edm_style_training",
-        action="store_true",
-        help="Flag to conduct training using the EDM formulation as introduced in https://arxiv.org/abs/2206.00364.",
-    )
     parser.add_argument(
         "--with_prior_preservation",
         default=False,
@@ -1140,8 +1117,6 @@ def main(args):
             "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
             " Please use `huggingface-cli login` to authenticate with the Hub."
         )
-    if args.do_edm_style_training and args.snr_gamma is not None:
-        raise ValueError("Min-SNR formulation is not supported when conducting EDM-style training.")
 
     logging_dir = Path(args.output_dir, args.logging_dir)
 
@@ -1259,19 +1234,7 @@ def main(args):
     )
 
     # Load scheduler and models
-    scheduler_type = determine_scheduler_type(args.pretrained_model_name_or_path, args.revision)
-    if "EDM" in scheduler_type:
-        args.do_edm_style_training = True
-        noise_scheduler = EDMEulerScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler")
-        logger.info("Performing EDM-style training!")
-    elif args.do_edm_style_training:
-        noise_scheduler = EulerDiscreteScheduler.from_pretrained(
-            args.pretrained_model_name_or_path, subfolder="scheduler"
-        )
-        logger.info("Performing EDM-style training!")
-    else:
-        noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler")
-
+    noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler")
     text_encoder_one = text_encoder_cls_one.from_pretrained(
         args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision, variant=args.variant
     )
@@ -1289,12 +1252,7 @@ def main(args):
         revision=args.revision,
         variant=args.variant,
     )
-    latents_mean = latents_std = None
-    if hasattr(vae.config, "latents_mean") and vae.config.latents_mean is not None:
-        latents_mean = torch.tensor(vae.config.latents_mean).view(1, 4, 1, 1)
-    if hasattr(vae.config, "latents_std") and vae.config.latents_std is not None:
-        latents_std = torch.tensor(vae.config.latents_std).view(1, 4, 1, 1)
-
+    vae_scaling_factor = vae.config.scaling_factor
     unet = UNet2DConditionModel.from_pretrained(
         args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision, variant=args.variant
     )
@@ -1832,19 +1790,6 @@ def main(args):
         disable=not accelerator.is_local_main_process,
     )
 
-    def get_sigmas(timesteps, n_dim=4, dtype=torch.float32):
-        # TODO: revisit other sampling algorithms
-        sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype)
-        schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device)
-        timesteps = timesteps.to(accelerator.device)
-
-        step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps]
-
-        sigma = sigmas[step_indices].flatten()
-        while len(sigma.shape) < n_dim:
-            sigma = sigma.unsqueeze(-1)
-        return sigma
-
     if args.train_text_encoder:
         num_train_epochs_text_encoder = int(args.train_text_encoder_frac * args.num_train_epochs)
     elif args.train_text_encoder_ti:  # args.train_text_encoder_ti
@@ -1896,15 +1841,9 @@ def main(args):
                     pixel_values = batch["pixel_values"].to(dtype=vae.dtype)
                     model_input = vae.encode(pixel_values).latent_dist.sample()
 
-                if latents_mean is None and latents_std is None:
-                    model_input = model_input * vae.config.scaling_factor
-                    if args.pretrained_vae_model_name_or_path is None:
-                        model_input = model_input.to(weight_dtype)
-                else:
-                    latents_mean = latents_mean.to(device=model_input.device, dtype=model_input.dtype)
-                    latents_std = latents_std.to(device=model_input.device, dtype=model_input.dtype)
-                    model_input = (model_input - latents_mean) * vae.config.scaling_factor / latents_std
-                    model_input = model_input.to(dtype=weight_dtype)
+                model_input = model_input * vae_scaling_factor
+                if args.pretrained_vae_model_name_or_path is None:
+                    model_input = model_input.to(weight_dtype)
 
                 # Sample noise that we'll add to the latents
                 noise = torch.randn_like(model_input)
@@ -1915,32 +1854,15 @@ def main(args):
                     )
 
                 bsz = model_input.shape[0]
-
                 # Sample a random timestep for each image
-                if not args.do_edm_style_training:
-                    timesteps = torch.randint(
-                        0, noise_scheduler.config.num_train_timesteps, (bsz,), device=model_input.device
-                    )
-                    timesteps = timesteps.long()
-                else:
-                    # in EDM formulation, the model is conditioned on the pre-conditioned noise levels
-                    # instead of discrete timesteps, so here we sample indices to get the noise levels
-                    # from `scheduler.timesteps`
-                    indices = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,))
-                    timesteps = noise_scheduler.timesteps[indices].to(device=model_input.device)
+                timesteps = torch.randint(
+                    0, noise_scheduler.config.num_train_timesteps, (bsz,), device=model_input.device
+                )
+                timesteps = timesteps.long()
 
                 # Add noise to the model input according to the noise magnitude at each timestep
                 # (this is the forward diffusion process)
                 noisy_model_input = noise_scheduler.add_noise(model_input, noise, timesteps)
-                # For EDM-style training, we first obtain the sigmas based on the continuous timesteps.
-                # We then precondition the final model inputs based on these sigmas instead of the timesteps.
-                # Follow: Section 5 of https://arxiv.org/abs/2206.00364.
-                if args.do_edm_style_training:
-                    sigmas = get_sigmas(timesteps, len(noisy_model_input.shape), noisy_model_input.dtype)
-                    if "EDM" in scheduler_type:
-                        inp_noisy_latents = noise_scheduler.precondition_inputs(noisy_model_input, sigmas)
-                    else:
-                        inp_noisy_latents = noisy_model_input / ((sigmas**2 + 1) ** 0.5)
 
                 # time ids
                 add_time_ids = torch.cat(
@@ -1966,7 +1888,7 @@ def main(args):
                     }
                     prompt_embeds_input = prompt_embeds.repeat(elems_to_repeat_text_embeds, 1, 1)
                     model_pred = unet(
-                        inp_noisy_latents if args.do_edm_style_training else noisy_model_input,
+                        noisy_model_input,
                         timesteps,
                         prompt_embeds_input,
                         added_cond_kwargs=unet_added_conditions,
@@ -1984,42 +1906,14 @@ def main(args):
                     )
                     prompt_embeds_input = prompt_embeds.repeat(elems_to_repeat_text_embeds, 1, 1)
                     model_pred = unet(
-                        inp_noisy_latents if args.do_edm_style_training else noisy_model_input,
-                        timesteps,
-                        prompt_embeds_input,
-                        added_cond_kwargs=unet_added_conditions,
+                        noisy_model_input, timesteps, prompt_embeds_input, added_cond_kwargs=unet_added_conditions
                     ).sample
 
-                weighting = None
-                if args.do_edm_style_training:
-                    # Similar to the input preconditioning, the model predictions are also preconditioned
-                    # on noised model inputs (before preconditioning) and the sigmas.
-                    # Follow: Section 5 of https://arxiv.org/abs/2206.00364.
-                    if "EDM" in scheduler_type:
-                        model_pred = noise_scheduler.precondition_outputs(noisy_model_input, model_pred, sigmas)
-                    else:
-                        if noise_scheduler.config.prediction_type == "epsilon":
-                            model_pred = model_pred * (-sigmas) + noisy_model_input
-                        elif noise_scheduler.config.prediction_type == "v_prediction":
-                            model_pred = model_pred * (-sigmas / (sigmas**2 + 1) ** 0.5) + (
-                                noisy_model_input / (sigmas**2 + 1)
-                            )
-                    # We are not doing weighting here because it tends result in numerical problems.
-                    # See: https://github.com/huggingface/diffusers/pull/7126#issuecomment-1968523051
-                    # There might be other alternatives for weighting as well:
-                    # https://github.com/huggingface/diffusers/pull/7126#discussion_r1505404686
-                    if "EDM" not in scheduler_type:
-                        weighting = (sigmas**-2.0).float()
-
                 # Get the target for loss depending on the prediction type
                 if noise_scheduler.config.prediction_type == "epsilon":
-                    target = model_input if args.do_edm_style_training else noise
+                    target = noise
                 elif noise_scheduler.config.prediction_type == "v_prediction":
-                    target = (
-                        model_input
-                        if args.do_edm_style_training
-                        else noise_scheduler.get_velocity(model_input, noise, timesteps)
-                    )
+                    target = noise_scheduler.get_velocity(model_input, noise, timesteps)
                 else:
                     raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}")
 
@@ -2029,28 +1923,10 @@ def main(args):
                     target, target_prior = torch.chunk(target, 2, dim=0)
 
                     # Compute prior loss
-                    if weighting is not None:
-                        prior_loss = torch.mean(
-                            (weighting.float() * (model_pred_prior.float() - target_prior.float()) ** 2).reshape(
-                                target_prior.shape[0], -1
-                            ),
-                            1,
-                        )
-                        prior_loss = prior_loss.mean()
-                    else:
-                        prior_loss = F.mse_loss(model_pred_prior.float(), target_prior.float(), reduction="mean")
+                    prior_loss = F.mse_loss(model_pred_prior.float(), target_prior.float(), reduction="mean")
 
                 if args.snr_gamma is None:
-                    if weighting is not None:
-                        loss = torch.mean(
-                            (weighting.float() * (model_pred.float() - target.float()) ** 2).reshape(
-                                target.shape[0], -1
-                            ),
-                            1,
-                        )
-                        loss = loss.mean()
-                    else:
-                        loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean")
+                    loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean")
                 else:
                     # Compute loss-weights as per Section 3.4 of https://arxiv.org/abs/2303.09556.
                     # Since we predict the noise instead of x_0, the original formulation is slightly changed.
@@ -2173,18 +2049,17 @@ def main(args):
                 # We train on the simplified learning objective. If we were previously predicting a variance, we need the scheduler to ignore it
                 scheduler_args = {}
 
-                if not args.do_edm_style_training:
-                    if "variance_type" in pipeline.scheduler.config:
-                        variance_type = pipeline.scheduler.config.variance_type
+                if "variance_type" in pipeline.scheduler.config:
+                    variance_type = pipeline.scheduler.config.variance_type
 
-                        if variance_type in ["learned", "learned_range"]:
-                            variance_type = "fixed_small"
+                    if variance_type in ["learned", "learned_range"]:
+                        variance_type = "fixed_small"
 
-                        scheduler_args["variance_type"] = variance_type
+                    scheduler_args["variance_type"] = variance_type
 
-                    pipeline.scheduler = DPMSolverMultistepScheduler.from_config(
-                        pipeline.scheduler.config, **scheduler_args
-                    )
+                pipeline.scheduler = DPMSolverMultistepScheduler.from_config(
+                    pipeline.scheduler.config, **scheduler_args
+                )
 
                 pipeline = pipeline.to(accelerator.device)
                 pipeline.set_progress_bar_config(disable=True)
@@ -2192,13 +2067,8 @@ def main(args):
                 # run inference
                 generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) if args.seed else None
                 pipeline_args = {"prompt": args.validation_prompt}
-                inference_ctx = (
-                    contextlib.nullcontext()
-                    if "playground" in args.pretrained_model_name_or_path
-                    else torch.cuda.amp.autocast()
-                )
 
-                with inference_ctx:
+                with torch.cuda.amp.autocast():
                     images = [
                         pipeline(**pipeline_args, generator=generator).images[0]
                         for _ in range(args.num_validation_images)
@@ -2274,18 +2144,15 @@ def main(args):
             # We train on the simplified learning objective. If we were previously predicting a variance, we need the scheduler to ignore it
             scheduler_args = {}
 
-            if not args.do_edm_style_training:
-                if "variance_type" in pipeline.scheduler.config:
-                    variance_type = pipeline.scheduler.config.variance_type
+            if "variance_type" in pipeline.scheduler.config:
+                variance_type = pipeline.scheduler.config.variance_type
 
-                    if variance_type in ["learned", "learned_range"]:
-                        variance_type = "fixed_small"
+                if variance_type in ["learned", "learned_range"]:
+                    variance_type = "fixed_small"
 
-                    scheduler_args["variance_type"] = variance_type
+                scheduler_args["variance_type"] = variance_type
 
-                pipeline.scheduler = DPMSolverMultistepScheduler.from_config(
-                    pipeline.scheduler.config, **scheduler_args
-                )
+            pipeline.scheduler = DPMSolverMultistepScheduler.from_config(pipeline.scheduler.config, **scheduler_args)
 
             # load attention processors
             pipeline.load_lora_weights(args.output_dir)

examples/community/latent_consistency_img2img.py

@@ -513,7 +513,9 @@ class LCMSchedulerWithTimestamp(SchedulerMixin, ConfigMixin):
             there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the alpha value at step 0.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         prediction_type (`str`, defaults to `epsilon`, *optional*):
             Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
             `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen

examples/community/latent_consistency_txt2img.py

@@ -418,7 +418,9 @@ class LCMScheduler(SchedulerMixin, ConfigMixin):
             there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the alpha value at step 0.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         prediction_type (`str`, defaults to `epsilon`, *optional*):
             Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
             `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen

examples/community/marigold_depth_estimation.py

@@ -40,7 +40,8 @@ from diffusers.utils import BaseOutput, check_min_version
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
+
 
 class MarigoldDepthOutput(BaseOutput):
     """

examples/community/scheduling_ufogen.py

@@ -171,7 +171,9 @@ class UFOGenScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         rescale_betas_zero_snr (`bool`, defaults to `False`):
             Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
             dark samples instead of limiting it to samples with medium brightness. Loosely related to

examples/consistency_distillation/train_lcm_distill_lora_sd_wds.py

@@ -72,7 +72,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/consistency_distillation/train_lcm_distill_lora_sdxl.py

@@ -65,7 +65,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/consistency_distillation/train_lcm_distill_lora_sdxl_wds.py

@@ -78,7 +78,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/consistency_distillation/train_lcm_distill_sd_wds.py

@@ -71,7 +71,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/consistency_distillation/train_lcm_distill_sdxl_wds.py

@@ -77,7 +77,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/controlnet/train_controlnet.py

@@ -60,7 +60,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/controlnet/train_controlnet_flax.py

@@ -60,7 +60,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = logging.getLogger(__name__)
 

examples/controlnet/train_controlnet_sdxl.py

@@ -61,7 +61,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/custom_diffusion/train_custom_diffusion.py

@@ -63,7 +63,7 @@ from diffusers.utils.import_utils import is_xformers_available
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 
@@ -1178,7 +1178,7 @@ def main(args):
                         grads_text_encoder = text_encoder.get_input_embeddings().weight.grad
                     # Get the index for tokens that we want to zero the grads for
                     index_grads_to_zero = torch.arange(len(tokenizer)) != modifier_token_id[0]
-                    for i in range(1, len(modifier_token_id)):
+                    for i in range(len(modifier_token_id[1:])):
                         index_grads_to_zero = index_grads_to_zero & (
                             torch.arange(len(tokenizer)) != modifier_token_id[i]
                         )

examples/dreambooth/train_dreambooth.py

@@ -63,7 +63,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/dreambooth/train_dreambooth_flax.py

@@ -35,7 +35,7 @@ from diffusers.utils import check_min_version
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 # Cache compiled models across invocations of this script.
 cc.initialize_cache(os.path.expanduser("~/.cache/jax/compilation_cache"))

examples/dreambooth/train_dreambooth_lora.py

@@ -70,7 +70,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/dreambooth/train_dreambooth_lora_sdxl.py

@@ -75,7 +75,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/instruct_pix2pix/train_instruct_pix2pix.py

@@ -53,7 +53,7 @@ from diffusers.utils.torch_utils import is_compiled_module
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/instruct_pix2pix/train_instruct_pix2pix_sdxl.py

@@ -59,7 +59,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/kandinsky2_2/text_to_image/train_text_to_image_decoder.py

@@ -52,7 +52,7 @@ if is_wandb_available():
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/kandinsky2_2/text_to_image/train_text_to_image_lora_decoder.py

@@ -46,7 +46,7 @@ from diffusers.utils import check_min_version, is_wandb_available
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/kandinsky2_2/text_to_image/train_text_to_image_lora_prior.py

@@ -46,7 +46,7 @@ from diffusers.utils import check_min_version, is_wandb_available
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/kandinsky2_2/text_to_image/train_text_to_image_prior.py

@@ -51,7 +51,7 @@ if is_wandb_available():
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/t2i_adapter/train_t2i_adapter_sdxl.py

@@ -60,7 +60,7 @@ if is_wandb_available():
     import wandb
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/text_to_image/train_text_to_image.py

@@ -56,7 +56,7 @@ if is_wandb_available():
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/text_to_image/train_text_to_image_flax.py

@@ -49,7 +49,7 @@ from diffusers.utils import check_min_version
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = logging.getLogger(__name__)
 

examples/text_to_image/train_text_to_image_lora.py

@@ -52,7 +52,7 @@ from diffusers.utils.torch_utils import is_compiled_module
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/text_to_image/train_text_to_image_lora_sdxl.py

@@ -64,7 +64,7 @@ from diffusers.utils.torch_utils import is_compiled_module
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 
@@ -425,11 +425,6 @@ def parse_args(input_args=None):
         default=4,
         help=("The dimension of the LoRA update matrices."),
     )
-    parser.add_argument(
-        "--debug_loss",
-        action="store_true",
-        help="debug loss for each image, if filenames are awailable in the dataset",
-    )
 
     if input_args is not None:
         args = parser.parse_args(input_args)
@@ -608,7 +603,6 @@ def main(args):
     # Move unet, vae and text_encoder to device and cast to weight_dtype
     # The VAE is in float32 to avoid NaN losses.
     unet.to(accelerator.device, dtype=weight_dtype)
-
     if args.pretrained_vae_model_name_or_path is None:
         vae.to(accelerator.device, dtype=torch.float32)
     else:
@@ -896,17 +890,13 @@ def main(args):
         tokens_one, tokens_two = tokenize_captions(examples)
         examples["input_ids_one"] = tokens_one
         examples["input_ids_two"] = tokens_two
-        if args.debug_loss:
-            fnames = [os.path.basename(image.filename) for image in examples[image_column] if image.filename]
-            if fnames:
-                examples["filenames"] = fnames
         return examples
 
     with accelerator.main_process_first():
         if args.max_train_samples is not None:
             dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples))
         # Set the training transforms
-        train_dataset = dataset["train"].with_transform(preprocess_train, output_all_columns=True)
+        train_dataset = dataset["train"].with_transform(preprocess_train)
 
     def collate_fn(examples):
         pixel_values = torch.stack([example["pixel_values"] for example in examples])
@@ -915,7 +905,7 @@ def main(args):
         crop_top_lefts = [example["crop_top_lefts"] for example in examples]
         input_ids_one = torch.stack([example["input_ids_one"] for example in examples])
         input_ids_two = torch.stack([example["input_ids_two"] for example in examples])
-        result = {
+        return {
             "pixel_values": pixel_values,
             "input_ids_one": input_ids_one,
             "input_ids_two": input_ids_two,
@@ -923,11 +913,6 @@ def main(args):
             "crop_top_lefts": crop_top_lefts,
         }
 
-        filenames = [example["filenames"] for example in examples if "filenames" in example]
-        if filenames:
-            result["filenames"] = filenames
-        return result
-
     # DataLoaders creation:
     train_dataloader = torch.utils.data.DataLoader(
         train_dataset,
@@ -1120,9 +1105,7 @@ def main(args):
                     loss = F.mse_loss(model_pred.float(), target.float(), reduction="none")
                     loss = loss.mean(dim=list(range(1, len(loss.shape)))) * mse_loss_weights
                     loss = loss.mean()
-                if args.debug_loss and "filenames" in batch:
-                    for fname in batch["filenames"]:
-                        accelerator.log({"loss_for_" + fname: loss}, step=global_step)
+
                 # Gather the losses across all processes for logging (if we use distributed training).
                 avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean()
                 train_loss += avg_loss.item() / args.gradient_accumulation_steps

examples/text_to_image/train_text_to_image_sdxl.py

@@ -54,7 +54,7 @@ from diffusers.utils.torch_utils import is_compiled_module
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/textual_inversion/textual_inversion.py

@@ -80,7 +80,7 @@ else:
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/textual_inversion/textual_inversion_flax.py

@@ -56,7 +56,7 @@ else:
 # ------------------------------------------------------------------------------
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = logging.getLogger(__name__)
 

examples/textual_inversion/textual_inversion_sdxl.py

@@ -76,7 +76,7 @@ else:
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__)
 

examples/unconditional_image_generation/train_unconditional.py

@@ -29,7 +29,7 @@ from diffusers.utils.import_utils import is_xformers_available
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/wuerstchen/text_to_image/train_text_to_image_lora_prior.py

@@ -50,7 +50,7 @@ if is_wandb_available():
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

examples/wuerstchen/text_to_image/train_text_to_image_prior.py

@@ -51,7 +51,7 @@ if is_wandb_available():
 
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
-check_min_version("0.27.0")
+check_min_version("0.27.0.dev0")
 
 logger = get_logger(__name__, log_level="INFO")
 

scripts/log_reports.py

@@ -1,139 +0,0 @@
-import argparse
-import json
-import os
-from datetime import date
-from pathlib import Path
-
-from slack_sdk import WebClient
-from tabulate import tabulate
-
-
-MAX_LEN_MESSAGE = 2900  # slack endpoint has a limit of 3001 characters
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--slack_channel_name", default="diffusers-ci-nightly")
-
-
-def main(slack_channel_name=None):
-    failed = []
-    passed = []
-
-    group_info = []
-
-    total_num_failed = 0
-    empty_file = False or len(list(Path().glob("*.log"))) == 0
-
-    total_empty_files = []
-
-    for log in Path().glob("*.log"):
-        section_num_failed = 0
-        i = 0
-        with open(log) as f:
-            for line in f:
-                line = json.loads(line)
-                i += 1
-                if line.get("nodeid", "") != "":
-                    test = line["nodeid"]
-                    if line.get("duration", None) is not None:
-                        duration = f'{line["duration"]:.4f}'
-                        if line.get("outcome", "") == "failed":
-                            section_num_failed += 1
-                            failed.append([test, duration, log.name.split("_")[0]])
-                            total_num_failed += 1
-                        else:
-                            passed.append([test, duration, log.name.split("_")[0]])
-            empty_file = i == 0
-        group_info.append([str(log), section_num_failed, failed])
-        total_empty_files.append(empty_file)
-        os.remove(log)
-        failed = []
-    text = (
-        "🌞 There were no failures!"
-        if not any(total_empty_files)
-        else "Something went wrong there is at least one empty file - please check GH action results."
-    )
-    no_error_payload = {
-        "type": "section",
-        "text": {
-            "type": "plain_text",
-            "text": text,
-            "emoji": True,
-        },
-    }
-
-    message = ""
-    payload = [
-        {
-            "type": "header",
-            "text": {
-                "type": "plain_text",
-                "text": "🤗 Results of the Diffusers scheduled nightly tests.",
-            },
-        },
-    ]
-    if total_num_failed > 0:
-        for i, (name, num_failed, failed_tests) in enumerate(group_info):
-            if num_failed > 0:
-                if num_failed == 1:
-                    message += f"*{name}: {num_failed} failed test*\n"
-                else:
-                    message += f"*{name}: {num_failed} failed tests*\n"
-                failed_table = []
-                for test in failed_tests:
-                    failed_table.append(test[0].split("::"))
-                failed_table = tabulate(
-                    failed_table,
-                    headers=["Test Location", "Test Case", "Test Name"],
-                    showindex="always",
-                    tablefmt="grid",
-                    maxcolwidths=[12, 12, 12],
-                )
-                message += "\n```\n" + failed_table + "\n```"
-
-            if total_empty_files[i]:
-                message += f"\n*{name}: Warning! Empty file - please check the GitHub action job *\n"
-        print(f"### {message}")
-    else:
-        payload.append(no_error_payload)
-
-    if len(message) > MAX_LEN_MESSAGE:
-        print(f"Truncating long message from {len(message)} to {MAX_LEN_MESSAGE}")
-        message = message[:MAX_LEN_MESSAGE] + "..."
-
-    if len(message) != 0:
-        md_report = {
-            "type": "section",
-            "text": {"type": "mrkdwn", "text": message},
-        }
-        payload.append(md_report)
-        action_button = {
-            "type": "section",
-            "text": {"type": "mrkdwn", "text": "*For more details:*"},
-            "accessory": {
-                "type": "button",
-                "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
-                "url": f"https://github.com/huggingface/diffusers/actions/runs/{os.environ['GITHUB_RUN_ID']}",
-            },
-        }
-        payload.append(action_button)
-
-    date_report = {
-        "type": "context",
-        "elements": [
-            {
-                "type": "plain_text",
-                "text": f"Nightly test results for {date.today()}",
-            },
-        ],
-    }
-    payload.append(date_report)
-
-    print(payload)
-
-    client = WebClient(token=os.environ.get("SLACK_API_TOKEN"))
-    client.chat_postMessage(channel=f"#{slack_channel_name}", text=message, blocks=payload)
-
-
-if __name__ == "__main__":
-    args = parser.parse_args()
-    main(args.slack_channel_name)

setup.py

@@ -249,7 +249,7 @@ version_range_max = max(sys.version_info[1], 10) + 1
 
 setup(
     name="diffusers",
-    version="0.27.2",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
+    version="0.27.0.dev0",  # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots)
     description="State-of-the-art diffusion in PyTorch and JAX.",
     long_description=open("README.md", "r", encoding="utf-8").read(),
     long_description_content_type="text/markdown",

src/diffusers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.27.2"
+__version__ = "0.27.0.dev0"
 
 from typing import TYPE_CHECKING
 
@@ -253,8 +253,6 @@ else:
             "LatentConsistencyModelImg2ImgPipeline",
             "LatentConsistencyModelPipeline",
             "LDMTextToImagePipeline",
-            "LEditsPPPipelineStableDiffusion",
-            "LEditsPPPipelineStableDiffusionXL",
             "MusicLDMPipeline",
             "PaintByExamplePipeline",
             "PIAPipeline",
@@ -625,8 +623,6 @@ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
             LatentConsistencyModelImg2ImgPipeline,
             LatentConsistencyModelPipeline,
             LDMTextToImagePipeline,
-            LEditsPPPipelineStableDiffusion,
-            LEditsPPPipelineStableDiffusionXL,
             MusicLDMPipeline,
             PaintByExamplePipeline,
             PIAPipeline,

src/diffusers/models/unets/unet_2d_condition.py

@@ -80,7 +80,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin,
         in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample.
         out_channels (`int`, *optional*, defaults to 4): Number of channels in the output.
         center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample.
-        flip_sin_to_cos (`bool`, *optional*, defaults to `True`):
+        flip_sin_to_cos (`bool`, *optional*, defaults to `False`):
             Whether to flip the sin to cos in the time embedding.
         freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding.
         down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`):
@@ -109,7 +109,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin,
             The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for
             [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
             [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`].
-        reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None):
+       reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None):
             The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling
             blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for
             [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`],
@@ -147,9 +147,9 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin,
             The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`.
         time_cond_proj_dim (`int`, *optional*, defaults to `None`):
             The dimension of `cond_proj` layer in the timestep embedding.
-        conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer.
-        conv_out_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_out` layer.
-        projection_class_embeddings_input_dim (`int`, *optional*): The dimension of the `class_labels` input when
+        conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. conv_out_kernel (`int`,
+        *optional*, default to `3`): The kernel size of `conv_out` layer. projection_class_embeddings_input_dim (`int`,
+        *optional*): The dimension of the `class_labels` input when
             `class_embed_type="projection"`. Required when `class_embed_type="projection"`.
         class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time
             embeddings with the class embeddings.
@@ -1081,8 +1081,6 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin,
                 A tuple of tensors that if specified are added to the residuals of down unet blocks.
             mid_block_additional_residual: (`torch.Tensor`, *optional*):
                 A tensor that if specified is added to the residual of the middle unet block.
-            down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*):
-                additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s)
             encoder_attention_mask (`torch.Tensor`):
                 A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If
                 `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias,
@@ -1090,6 +1088,18 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin,
             return_dict (`bool`, *optional*, defaults to `True`):
                 Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain
                 tuple.
+            cross_attention_kwargs (`dict`, *optional*):
+                A kwargs dictionary that if specified is passed along to the [`AttnProcessor`].
+            added_cond_kwargs: (`dict`, *optional*):
+                A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that
+                are passed along to the UNet blocks.
+            down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*):
+                additional residuals to be added to UNet long skip connections from down blocks to up blocks for
+                example from ControlNet side model(s)
+            mid_block_additional_residual (`torch.Tensor`, *optional*):
+                additional residual to be added to UNet mid block output, for example from ControlNet side model
+            down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*):
+                additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s)
 
         Returns:
             [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] or `tuple`:
@@ -1175,14 +1185,7 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin,
             cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)}
 
         # 3. down
-        # we're popping the `scale` instead of getting it because otherwise `scale` will be propagated
-        # to the internal blocks and will raise deprecation warnings. this will be confusing for our users.
-        if cross_attention_kwargs is not None:
-            cross_attention_kwargs = cross_attention_kwargs.copy()
-            lora_scale = cross_attention_kwargs.pop("scale", 1.0)
-        else:
-            lora_scale = 1.0
-
+        lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0
         if USE_PEFT_BACKEND:
             # weight the lora layers by setting `lora_scale` for each PEFT layer
             scale_lora_layers(self, lora_scale)

src/diffusers/pipelines/__init__.py

@@ -23,7 +23,6 @@ _import_structure = {
     "controlnet_xs": [],
     "deprecated": [],
     "latent_diffusion": [],
-    "ledits_pp": [],
     "stable_diffusion": [],
     "stable_diffusion_xl": [],
 }
@@ -172,12 +171,6 @@ else:
         "LatentConsistencyModelPipeline",
     ]
     _import_structure["latent_diffusion"].extend(["LDMTextToImagePipeline"])
-    _import_structure["ledits_pp"].extend(
-        [
-            "LEditsPPPipelineStableDiffusion",
-            "LEditsPPPipelineStableDiffusionXL",
-        ]
-    )
     _import_structure["musicldm"] = ["MusicLDMPipeline"]
     _import_structure["paint_by_example"] = ["PaintByExamplePipeline"]
     _import_structure["pia"] = ["PIAPipeline"]
@@ -431,12 +424,6 @@ if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
             LatentConsistencyModelPipeline,
         )
         from .latent_diffusion import LDMTextToImagePipeline
-        from .ledits_pp import (
-            LEditsPPDiffusionPipelineOutput,
-            LEditsPPInversionPipelineOutput,
-            LEditsPPPipelineStableDiffusion,
-            LEditsPPPipelineStableDiffusionXL,
-        )
         from .musicldm import MusicLDMPipeline
         from .paint_by_example import PaintByExamplePipeline
         from .pia import PIAPipeline

src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py

@@ -528,12 +528,15 @@ class StableDiffusionInpaintPipelineLegacy(
                     f" {negative_prompt_embeds.shape}."
                 )
 
+    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps
     def get_timesteps(self, num_inference_steps, strength, device):
         # get the original timestep using init_timestep
         init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
 
         t_start = max(num_inference_steps - init_timestep, 0)
         timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
+        if hasattr(self.scheduler, "set_begin_index"):
+            self.scheduler.set_begin_index(t_start * self.scheduler.order)
 
         return timesteps, num_inference_steps - t_start
 

src/diffusers/pipelines/ledits_pp/__init__.py

@@ -1,55 +0,0 @@
-from typing import TYPE_CHECKING
-
-from ...utils import (
-    DIFFUSERS_SLOW_IMPORT,
-    OptionalDependencyNotAvailable,
-    _LazyModule,
-    get_objects_from_module,
-    is_torch_available,
-    is_transformers_available,
-)
-
-
-_dummy_objects = {}
-_import_structure = {}
-
-try:
-    if not (is_transformers_available() and is_torch_available()):
-        raise OptionalDependencyNotAvailable()
-except OptionalDependencyNotAvailable:
-    from ...utils import dummy_torch_and_transformers_objects  # noqa F403
-
-    _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects))
-else:
-    _import_structure["pipeline_leditspp_stable_diffusion"] = ["LEditsPPPipelineStableDiffusion"]
-    _import_structure["pipeline_leditspp_stable_diffusion_xl"] = ["LEditsPPPipelineStableDiffusionXL"]
-
-    _import_structure["pipeline_output"] = ["LEditsPPDiffusionPipelineOutput", "LEditsPPDiffusionPipelineOutput"]
-
-if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:
-    try:
-        if not (is_transformers_available() and is_torch_available()):
-            raise OptionalDependencyNotAvailable()
-
-    except OptionalDependencyNotAvailable:
-        from ...utils.dummy_torch_and_transformers_objects import *
-    else:
-        from .pipeline_leditspp_stable_diffusion import (
-            LEditsPPDiffusionPipelineOutput,
-            LEditsPPInversionPipelineOutput,
-            LEditsPPPipelineStableDiffusion,
-        )
-        from .pipeline_leditspp_stable_diffusion_xl import LEditsPPPipelineStableDiffusionXL
-
-else:
-    import sys
-
-    sys.modules[__name__] = _LazyModule(
-        __name__,
-        globals()["__file__"],
-        _import_structure,
-        module_spec=__spec__,
-    )
-
-    for name, value in _dummy_objects.items():
-        setattr(sys.modules[__name__], name, value)

src/diffusers/pipelines/ledits_pp/pipeline_output.py

@@ -1,43 +0,0 @@
-from dataclasses import dataclass
-from typing import List, Optional, Union
-
-import numpy as np
-import PIL.Image
-
-from ...utils import BaseOutput
-
-
-@dataclass
-class LEditsPPDiffusionPipelineOutput(BaseOutput):
-    """
-    Output class for LEdits++ Diffusion pipelines.
-
-    Args:
-        images (`List[PIL.Image.Image]` or `np.ndarray`)
-            List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width,
-            num_channels)`.
-        nsfw_content_detected (`List[bool]`)
-            List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or
-            `None` if safety checking could not be performed.
-    """
-
-    images: Union[List[PIL.Image.Image], np.ndarray]
-    nsfw_content_detected: Optional[List[bool]]
-
-
-@dataclass
-class LEditsPPInversionPipelineOutput(BaseOutput):
-    """
-    Output class for LEdits++ Diffusion pipelines.
-
-    Args:
-        input_images (`List[PIL.Image.Image]` or `np.ndarray`)
-            List of the cropped and resized input images as PIL images of length `batch_size` or NumPy array of shape `
-            (batch_size, height, width, num_channels)`.
-        vae_reconstruction_images (`List[PIL.Image.Image]` or `np.ndarray`)
-            List of VAE reconstruction of all input images as PIL images of length `batch_size` or NumPy array of shape `
-            (batch_size, height, width, num_channels)`.
-    """
-
-    images: Union[List[PIL.Image.Image], np.ndarray]
-    vae_reconstruction_images: Union[List[PIL.Image.Image], np.ndarray]

src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py

@@ -100,10 +100,8 @@ class StableCascadeDecoderPipeline(DiffusionPipeline):
         )
         self.register_to_config(latent_dim_scale=latent_dim_scale)
 
-    def prepare_latents(
-        self, batch_size, image_embeddings, num_images_per_prompt, dtype, device, generator, latents, scheduler
-    ):
-        _, channels, height, width = image_embeddings.shape
+    def prepare_latents(self, image_embeddings, num_images_per_prompt, dtype, device, generator, latents, scheduler):
+        batch_size, channels, height, width = image_embeddings.shape
         latents_shape = (
             batch_size * num_images_per_prompt,
             4,
@@ -385,19 +383,7 @@ class StableCascadeDecoderPipeline(DiffusionPipeline):
         )
         if isinstance(image_embeddings, list):
             image_embeddings = torch.cat(image_embeddings, dim=0)
-
-        if prompt is not None and isinstance(prompt, str):
-            batch_size = 1
-        elif prompt is not None and isinstance(prompt, list):
-            batch_size = len(prompt)
-        else:
-            batch_size = prompt_embeds.shape[0]
-
-        # Compute the effective number of images per prompt
-        # We must account for the fact that the image embeddings from the prior can be generated with num_images_per_prompt > 1
-        # This results in a case where a single prompt is associated with multiple image embeddings
-        # Divide the number of image embeddings by the batch size to determine if this is the case.
-        num_images_per_prompt = num_images_per_prompt * (image_embeddings.shape[0] // batch_size)
+        batch_size = image_embeddings.shape[0]
 
         # 2. Encode caption
         if prompt_embeds is None and negative_prompt_embeds is None:
@@ -431,7 +417,7 @@ class StableCascadeDecoderPipeline(DiffusionPipeline):
 
         # 5. Prepare latents
         latents = self.prepare_latents(
-            batch_size, image_embeddings, num_images_per_prompt, dtype, device, generator, latents, self.scheduler
+            image_embeddings, num_images_per_prompt, dtype, device, generator, latents, self.scheduler
         )
 
         # 6. Run denoising loop

src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py

@@ -716,12 +716,15 @@ class StableDiffusionDiffEditPipeline(
                     f" `source_negative_prompt_embeds` {source_negative_prompt_embeds.shape}."
                 )
 
+    # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps
     def get_timesteps(self, num_inference_steps, strength, device):
         # get the original timestep using init_timestep
         init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
 
         t_start = max(num_inference_steps - init_timestep, 0)
         timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :]
+        if hasattr(self.scheduler, "set_begin_index"):
+            self.scheduler.set_begin_index(t_start * self.scheduler.order)
 
         return timesteps, num_inference_steps - t_start
 

src/diffusers/schedulers/scheduling_consistency_models.py

@@ -434,11 +434,7 @@ class CMStochasticIterativeScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_ddim.py

@@ -157,7 +157,9 @@ class DDIMScheduler(SchedulerMixin, ConfigMixin):
             there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the alpha value at step 0.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         prediction_type (`str`, defaults to `epsilon`, *optional*):
             Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
             `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen

src/diffusers/schedulers/scheduling_ddim_flax.py

@@ -93,7 +93,9 @@ class FlaxDDIMScheduler(FlaxSchedulerMixin, ConfigMixin):
             step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the value of alpha at step 0.
         steps_offset (`int`, default `0`):
-            An offset added to the inference steps, as required by some model families.
+            an offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False`, to make the last step use step 0 for the previous alpha product, as done in
+            stable diffusion.
         prediction_type (`str`, default `epsilon`):
             indicates whether the model predicts the noise (epsilon), or the samples. One of `epsilon`, `sample`.
             `v-prediction` is not supported for this scheduler.

src/diffusers/schedulers/scheduling_ddim_inverse.py

@@ -155,7 +155,9 @@ class DDIMInverseScheduler(SchedulerMixin, ConfigMixin):
             there is no previous alpha. When this option is `True` the previous alpha product is fixed to 0, otherwise
             it uses the alpha value at step `num_train_timesteps - 1`.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use `num_train_timesteps - 1` for the previous alpha
+            product.
         prediction_type (`str`, defaults to `epsilon`, *optional*):
             Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
             `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen

src/diffusers/schedulers/scheduling_ddim_parallel.py

@@ -159,7 +159,9 @@ class DDIMParallelScheduler(SchedulerMixin, ConfigMixin):
             step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the value of alpha at step 0.
         steps_offset (`int`, default `0`):
-            An offset added to the inference steps, as required by some model families.
+            an offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False`, to make the last step use step 0 for the previous alpha product, as done in
+            stable diffusion.
         prediction_type (`str`, default `epsilon`, optional):
             prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion
             process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4

src/diffusers/schedulers/scheduling_ddpm.py

@@ -167,7 +167,9 @@ class DDPMScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         rescale_betas_zero_snr (`bool`, defaults to `False`):
             Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
             dark samples instead of limiting it to samples with medium brightness. Loosely related to

src/diffusers/schedulers/scheduling_ddpm_parallel.py

@@ -173,7 +173,9 @@ class DDPMParallelScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2. of [Common Diffusion Noise Schedules and Sample
             Steps are Flawed](https://arxiv.org/abs/2305.08891) for more information.
         steps_offset (`int`, default `0`):
-            An offset added to the inference steps, as required by some model families.
+            an offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False`, to make the last step use step 0 for the previous alpha product, as done in
+            stable diffusion.
         rescale_betas_zero_snr (`bool`, defaults to `False`):
             Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
             dark samples instead of limiting it to samples with medium brightness. Loosely related to

src/diffusers/schedulers/scheduling_deis_multistep.py

@@ -115,7 +115,9 @@ class DEISMultistepScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -768,14 +770,10 @@ class DEISMultistepScheduler(SchedulerMixin, ConfigMixin):
             schedule_timesteps = self.timesteps.to(original_samples.device)
             timesteps = timesteps.to(original_samples.device)
 
-        # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
+        # begin_index is None when the scheduler is used for training
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_dpmsolver_multistep.py

@@ -178,7 +178,9 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         rescale_betas_zero_snr (`bool`, defaults to `False`):
             Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
             dark samples instead of limiting it to samples with medium brightness. Loosely related to
@@ -897,7 +899,6 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
         timestep: int,
         sample: torch.FloatTensor,
         generator=None,
-        variance_noise: Optional[torch.FloatTensor] = None,
         return_dict: bool = True,
     ) -> Union[SchedulerOutput, Tuple]:
         """
@@ -913,9 +914,6 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
                 A current instance of a sample created by the diffusion process.
             generator (`torch.Generator`, *optional*):
                 A random number generator.
-            variance_noise (`torch.FloatTensor`):
-                Alternative to generating noise with `generator` by directly providing the noise for the variance
-                itself. Useful for methods such as [`LEdits++`].
             return_dict (`bool`):
                 Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`.
 
@@ -950,12 +948,11 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
 
         # Upcast to avoid precision issues when computing prev_sample
         sample = sample.to(torch.float32)
-        if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"] and variance_noise is None:
+
+        if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]:
             noise = randn_tensor(
                 model_output.shape, generator=generator, device=model_output.device, dtype=torch.float32
             )
-        elif self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]:
-            noise = variance_noise.to(device=model_output.device, dtype=torch.float32)
         else:
             noise = None
 
@@ -1011,14 +1008,10 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
             schedule_timesteps = self.timesteps.to(original_samples.device)
             timesteps = timesteps.to(original_samples.device)
 
-        # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
+        # begin_index is None when the scheduler is used for training
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py

@@ -134,7 +134,9 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -790,7 +792,6 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
         timestep: int,
         sample: torch.FloatTensor,
         generator=None,
-        variance_noise: Optional[torch.FloatTensor] = None,
         return_dict: bool = True,
     ) -> Union[SchedulerOutput, Tuple]:
         """
@@ -806,9 +807,6 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
                 A current instance of a sample created by the diffusion process.
             generator (`torch.Generator`, *optional*):
                 A random number generator.
-            variance_noise (`torch.FloatTensor`):
-                Alternative to generating noise with `generator` by directly providing the noise for the variance
-                itself. Useful for methods such as [`CycleDiffusion`].
             return_dict (`bool`):
                 Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`.
 
@@ -839,12 +837,10 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
             self.model_outputs[i] = self.model_outputs[i + 1]
         self.model_outputs[-1] = model_output
 
-        if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"] and variance_noise is None:
+        if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]:
             noise = randn_tensor(
                 model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype
             )
-        elif self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]:
-            noise = variance_noise
         else:
             noise = None
 

src/diffusers/schedulers/scheduling_dpmsolver_sde.py

@@ -153,7 +153,9 @@ class DPMSolverSDEScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -543,11 +545,7 @@ class DPMSolverSDEScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py

@@ -223,8 +223,6 @@ class DPMSolverSinglestepScheduler(SchedulerMixin, ConfigMixin):
         """
         steps = num_inference_steps
         order = self.config.solver_order
-        if order > 3:
-            raise ValueError("Order > 3 is not supported by this scheduler")
         if self.config.lower_order_final:
             if order == 3:
                 if steps % 3 == 0:
@@ -961,14 +959,10 @@ class DPMSolverSinglestepScheduler(SchedulerMixin, ConfigMixin):
             schedule_timesteps = self.timesteps.to(original_samples.device)
             timesteps = timesteps.to(original_samples.device)
 
-        # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
+        # begin_index is None when the scheduler is used for training
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py

@@ -669,11 +669,7 @@ class EDMDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_edm_euler.py

@@ -367,11 +367,7 @@ class EDMEulerScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py

@@ -156,7 +156,9 @@ class EulerAncestralDiscreteScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         rescale_betas_zero_snr (`bool`, defaults to `False`):
             Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
             dark samples instead of limiting it to samples with medium brightness. Loosely related to
@@ -467,11 +469,7 @@ class EulerAncestralDiscreteScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_euler_discrete.py

@@ -162,7 +162,9 @@ class EulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         rescale_betas_zero_snr (`bool`, defaults to `False`):
             Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and
             dark samples instead of limiting it to samples with medium brightness. Loosely related to
@@ -562,11 +564,7 @@ class EulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_heun_discrete.py

@@ -101,7 +101,9 @@ class HeunDiscreteScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -468,11 +470,7 @@ class HeunDiscreteScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py

@@ -99,7 +99,9 @@ class KDPM2AncestralDiscreteScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -494,11 +496,7 @@ class KDPM2AncestralDiscreteScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py

@@ -98,7 +98,9 @@ class KDPM2DiscreteScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -469,11 +471,7 @@ class KDPM2DiscreteScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_lcm.py

@@ -165,7 +165,9 @@ class LCMScheduler(SchedulerMixin, ConfigMixin):
             there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the alpha value at step 0.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         prediction_type (`str`, defaults to `epsilon`, *optional*):
             Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
             `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen

src/diffusers/schedulers/scheduling_lms_discrete.py

@@ -119,7 +119,9 @@ class LMSDiscreteScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -461,11 +463,7 @@ class LMSDiscreteScheduler(SchedulerMixin, ConfigMixin):
         # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/schedulers/scheduling_pndm.py

@@ -104,7 +104,9 @@ class PNDMScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]

src/diffusers/schedulers/scheduling_pndm_flax.py

@@ -99,7 +99,9 @@ class FlaxPNDMScheduler(FlaxSchedulerMixin, ConfigMixin):
             step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the value of alpha at step 0.
         steps_offset (`int`, default `0`):
-            An offset added to the inference steps, as required by some model families.
+            an offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False`, to make the last step use step 0 for the previous alpha product, as done in
+            stable diffusion.
         prediction_type (`str`, default `epsilon`, optional):
             prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion
             process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4

src/diffusers/schedulers/scheduling_sasolver.py

@@ -131,7 +131,9 @@ class SASolverScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]

src/diffusers/schedulers/scheduling_tcd.py

@@ -166,7 +166,9 @@ class TCDScheduler(SchedulerMixin, ConfigMixin):
             there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`,
             otherwise it uses the alpha value at step 0.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
         prediction_type (`str`, defaults to `epsilon`, *optional*):
             Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process),
             `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen

src/diffusers/schedulers/scheduling_unipc_multistep.py

@@ -126,7 +126,9 @@ class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin):
             The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
             Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
         steps_offset (`int`, defaults to 0):
-            An offset added to the inference steps, as required by some model families.
+            An offset added to the inference steps. You can use a combination of `offset=1` and
+            `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable
+            Diffusion.
     """
 
     _compatibles = [e.name for e in KarrasDiffusionSchedulers]
@@ -862,14 +864,10 @@ class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin):
             schedule_timesteps = self.timesteps.to(original_samples.device)
             timesteps = timesteps.to(original_samples.device)
 
-        # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index
+        # begin_index is None when the scheduler is used for training
         if self.begin_index is None:
             step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps]
-        elif self.step_index is not None:
-            # add_noise is called after first denoising step (for inpainting)
-            step_indices = [self.step_index] * timesteps.shape[0]
         else:
-            # add noise is called bevore first denoising step to create inital latent(img2img)
             step_indices = [self.begin_index] * timesteps.shape[0]
 
         sigma = sigmas[step_indices].flatten()

src/diffusers/utils/dummy_torch_and_transformers_objects.py

@@ -647,36 +647,6 @@ class LDMTextToImagePipeline(metaclass=DummyObject):
         requires_backends(cls, ["torch", "transformers"])
 
 
-class LEditsPPPipelineStableDiffusion(metaclass=DummyObject):
-    _backends = ["torch", "transformers"]
-
-    def __init__(self, *args, **kwargs):
-        requires_backends(self, ["torch", "transformers"])
-
-    @classmethod
-    def from_config(cls, *args, **kwargs):
-        requires_backends(cls, ["torch", "transformers"])
-
-    @classmethod
-    def from_pretrained(cls, *args, **kwargs):
-        requires_backends(cls, ["torch", "transformers"])
-
-
-class LEditsPPPipelineStableDiffusionXL(metaclass=DummyObject):
-    _backends = ["torch", "transformers"]
-
-    def __init__(self, *args, **kwargs):
-        requires_backends(self, ["torch", "transformers"])
-
-    @classmethod
-    def from_config(cls, *args, **kwargs):
-        requires_backends(cls, ["torch", "transformers"])
-
-    @classmethod
-    def from_pretrained(cls, *args, **kwargs):
-        requires_backends(cls, ["torch", "transformers"])
-
-
 class MusicLDMPipeline(metaclass=DummyObject):
     _backends = ["torch", "transformers"]
 

tests/lora/test_lora_layers_peft.py

@@ -158,7 +158,7 @@ class PeftLoraLoaderMixinTests:
 
         pipeline_inputs = {
             "prompt": "A painting of a squirrel eating a burger",
-            "num_inference_steps": 5,
+            "num_inference_steps": 2,
             "guidance_scale": 6.0,
             "output_type": "np",
         }
@@ -589,7 +589,7 @@ class PeftLoraLoaderMixinTests:
                 **inputs, generator=torch.manual_seed(0), cross_attention_kwargs={"scale": 0.5}
             ).images
             self.assertTrue(
-                not np.allclose(output_lora, output_lora_scale, atol=1e-4, rtol=1e-4),
+                not np.allclose(output_lora, output_lora_scale, atol=1e-3, rtol=1e-3),
                 "Lora + scale should change the output",
             )
 
@@ -1300,11 +1300,6 @@ class StableDiffusionLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase):
         pipe.load_lora_weights(lora_id)
         pipe = pipe.to("cuda")
 
-        self.assertTrue(
-            self.check_if_lora_correctly_set(pipe.unet),
-            "Lora not correctly set in UNet",
-        )
-
         self.assertTrue(
             self.check_if_lora_correctly_set(pipe.text_encoder),
             "Lora not correctly set in text encoder 2",

tests/models/autoencoders/test_models_vae.py

@@ -829,7 +829,7 @@ class AutoencoderKLIntegrationTests(unittest.TestCase):
             "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/blob/main/vae-ft-mse-840000-ema-pruned.safetensors",
         )
 
-        assert vae_default.config.scaling_factor == 0.18215
+        assert vae_default.config.scaling_factor == 0.18125
         assert vae_default.config.sample_size == 512
         assert vae_default.dtype == torch.float32
 

tests/models/unets/test_models_unet_stable_cascade.py

@@ -50,7 +50,9 @@ class StableCascadeUNetModelSlowTests(unittest.TestCase):
         gc.collect()
         torch.cuda.empty_cache()
 
-        unet = StableCascadeUNet.from_pretrained("stabilityai/stable-cascade-prior", subfolder="prior", variant="bf16")
+        unet = StableCascadeUNet.from_pretrained(
+            "stabilityai/stable-cascade-prior", subfolder="prior", revision="refs/pr/2", variant="bf16"
+        )
         unet_config = unet.config
         del unet
         gc.collect()
@@ -72,7 +74,9 @@ class StableCascadeUNetModelSlowTests(unittest.TestCase):
         gc.collect()
         torch.cuda.empty_cache()
 
-        unet = StableCascadeUNet.from_pretrained("stabilityai/stable-cascade", subfolder="decoder", variant="bf16")
+        unet = StableCascadeUNet.from_pretrained(
+            "stabilityai/stable-cascade", subfolder="decoder", revision="refs/pr/44", variant="bf16"
+        )
         unet_config = unet.config
         del unet
         gc.collect()

tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion.py

@@ -1,244 +0,0 @@
-# coding=utf-8
-# Copyright 2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import gc
-import random
-import unittest
-
-import numpy as np
-import torch
-from PIL import Image
-from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer
-
-from diffusers import (
-    AutoencoderKL,
-    DPMSolverMultistepScheduler,
-    LEditsPPPipelineStableDiffusion,
-    UNet2DConditionModel,
-)
-from diffusers.utils.testing_utils import (
-    enable_full_determinism,
-    floats_tensor,
-    load_image,
-    require_torch_gpu,
-    skip_mps,
-    slow,
-    torch_device,
-)
-
-
-enable_full_determinism()
-
-
-@skip_mps
-class LEditsPPPipelineStableDiffusionFastTests(unittest.TestCase):
-    pipeline_class = LEditsPPPipelineStableDiffusion
-
-    def get_dummy_components(self):
-        torch.manual_seed(0)
-        unet = UNet2DConditionModel(
-            block_out_channels=(32, 64, 64),
-            layers_per_block=2,
-            sample_size=32,
-            in_channels=4,
-            out_channels=4,
-            down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"),
-            up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"),
-            cross_attention_dim=32,
-        )
-        scheduler = DPMSolverMultistepScheduler(algorithm_type="sde-dpmsolver++", solver_order=2)
-        torch.manual_seed(0)
-        vae = AutoencoderKL(
-            block_out_channels=[32, 64],
-            in_channels=3,
-            out_channels=3,
-            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],
-            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],
-            latent_channels=4,
-        )
-        torch.manual_seed(0)
-        text_encoder_config = CLIPTextConfig(
-            bos_token_id=0,
-            eos_token_id=2,
-            hidden_size=32,
-            intermediate_size=37,
-            layer_norm_eps=1e-05,
-            num_attention_heads=4,
-            num_hidden_layers=5,
-            pad_token_id=1,
-            vocab_size=1000,
-        )
-        text_encoder = CLIPTextModel(text_encoder_config)
-        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")
-
-        components = {
-            "unet": unet,
-            "scheduler": scheduler,
-            "vae": vae,
-            "text_encoder": text_encoder,
-            "tokenizer": tokenizer,
-            "safety_checker": None,
-            "feature_extractor": None,
-        }
-        return components
-
-    def get_dummy_inputs(self, device, seed=0):
-        if str(device).startswith("mps"):
-            generator = torch.manual_seed(seed)
-        else:
-            generator = torch.Generator(device=device).manual_seed(seed)
-        inputs = {
-            "generator": generator,
-            "editing_prompt": ["wearing glasses", "sunshine"],
-            "reverse_editing_direction": [False, True],
-            "edit_guidance_scale": [10.0, 5.0],
-        }
-        return inputs
-
-    def get_dummy_inversion_inputs(self, device, seed=0):
-        images = floats_tensor((2, 3, 32, 32), rng=random.Random(0)).cpu().permute(0, 2, 3, 1)
-        images = 255 * images
-        image_1 = Image.fromarray(np.uint8(images[0])).convert("RGB")
-        image_2 = Image.fromarray(np.uint8(images[1])).convert("RGB")
-
-        if str(device).startswith("mps"):
-            generator = torch.manual_seed(seed)
-        else:
-            generator = torch.Generator(device=device).manual_seed(seed)
-
-        inputs = {
-            "image": [image_1, image_2],
-            "source_prompt": "",
-            "source_guidance_scale": 3.5,
-            "num_inversion_steps": 20,
-            "skip": 0.15,
-            "generator": generator,
-        }
-        return inputs
-
-    def test_ledits_pp_inversion(self):
-        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
-        components = self.get_dummy_components()
-        sd_pipe = LEditsPPPipelineStableDiffusion(**components)
-        sd_pipe = sd_pipe.to(torch_device)
-        sd_pipe.set_progress_bar_config(disable=None)
-
-        inputs = self.get_dummy_inversion_inputs(device)
-        inputs["image"] = inputs["image"][0]
-        sd_pipe.invert(**inputs)
-        assert sd_pipe.init_latents.shape == (
-            1,
-            4,
-            int(32 / sd_pipe.vae_scale_factor),
-            int(32 / sd_pipe.vae_scale_factor),
-        )
-
-        latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device)
-        print(latent_slice.flatten())
-        expected_slice = np.array([-0.9084, -0.0367, 0.2940, 0.0839, 0.6890, 0.2651, -0.7104, 2.1090, -0.7822])
-        assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3
-
-    def test_ledits_pp_inversion_batch(self):
-        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
-        components = self.get_dummy_components()
-        sd_pipe = LEditsPPPipelineStableDiffusion(**components)
-        sd_pipe = sd_pipe.to(torch_device)
-        sd_pipe.set_progress_bar_config(disable=None)
-
-        inputs = self.get_dummy_inversion_inputs(device)
-        sd_pipe.invert(**inputs)
-        assert sd_pipe.init_latents.shape == (
-            2,
-            4,
-            int(32 / sd_pipe.vae_scale_factor),
-            int(32 / sd_pipe.vae_scale_factor),
-        )
-
-        latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device)
-        print(latent_slice.flatten())
-        expected_slice = np.array([0.2528, 0.1458, -0.2166, 0.4565, -0.5657, -1.0286, -0.9961, 0.5933, 1.1173])
-        assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3
-
-        latent_slice = sd_pipe.init_latents[1, -1, -3:, -3:].to(device)
-        print(latent_slice.flatten())
-        expected_slice = np.array([-0.0796, 2.0583, 0.5501, 0.5358, 0.0282, -0.2803, -1.0470, 0.7023, -0.0072])
-        assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3
-
-    def test_ledits_pp_warmup_steps(self):
-        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
-        components = self.get_dummy_components()
-        pipe = LEditsPPPipelineStableDiffusion(**components)
-        pipe = pipe.to(torch_device)
-        pipe.set_progress_bar_config(disable=None)
-
-        inversion_inputs = self.get_dummy_inversion_inputs(device)
-        pipe.invert(**inversion_inputs)
-
-        inputs = self.get_dummy_inputs(device)
-
-        inputs["edit_warmup_steps"] = [0, 5]
-        pipe(**inputs).images
-
-        inputs["edit_warmup_steps"] = [5, 0]
-        pipe(**inputs).images
-
-        inputs["edit_warmup_steps"] = [5, 10]
-        pipe(**inputs).images
-
-        inputs["edit_warmup_steps"] = [10, 5]
-        pipe(**inputs).images
-
-
-@slow
-@require_torch_gpu
-class LEditsPPPipelineStableDiffusionSlowTests(unittest.TestCase):
-    def tearDown(self):
-        super().tearDown()
-        gc.collect()
-        torch.cuda.empty_cache()
-
-    @classmethod
-    def setUpClass(cls):
-        raw_image = load_image(
-            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png"
-        )
-        raw_image = raw_image.convert("RGB").resize((512, 512))
-        cls.raw_image = raw_image
-
-    def test_ledits_pp_editing(self):
-        pipe = LEditsPPPipelineStableDiffusion.from_pretrained(
-            "runwayml/stable-diffusion-v1-5", safety_checker=None, torch_dtype=torch.float16
-        )
-        pipe = pipe.to(torch_device)
-        pipe.set_progress_bar_config(disable=None)
-
-        generator = torch.manual_seed(0)
-        _ = pipe.invert(image=self.raw_image, generator=generator)
-        generator = torch.manual_seed(0)
-        inputs = {
-            "generator": generator,
-            "editing_prompt": ["cat", "dog"],
-            "reverse_editing_direction": [True, False],
-            "edit_guidance_scale": [5.0, 5.0],
-            "edit_threshold": [0.8, 0.8],
-        }
-        reconstruction = pipe(**inputs, output_type="np").images[0]
-
-        output_slice = reconstruction[150:153, 140:143, -1]
-        output_slice = output_slice.flatten()
-        expected_slice = np.array(
-            [0.9453125, 0.93310547, 0.84521484, 0.94628906, 0.9111328, 0.80859375, 0.93847656, 0.9042969, 0.8144531]
-        )
-        assert np.abs(output_slice - expected_slice).max() < 1e-2

tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion_xl.py

@@ -1,289 +0,0 @@
-# coding=utf-8
-# Copyright 2023 HuggingFace Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import random
-import unittest
-
-import numpy as np
-import torch
-from PIL import Image
-from transformers import (
-    CLIPImageProcessor,
-    CLIPTextConfig,
-    CLIPTextModel,
-    CLIPTextModelWithProjection,
-    CLIPTokenizer,
-    CLIPVisionConfig,
-    CLIPVisionModelWithProjection,
-)
-
-from diffusers import (
-    AutoencoderKL,
-    DPMSolverMultistepScheduler,
-    LEditsPPPipelineStableDiffusionXL,
-    UNet2DConditionModel,
-)
-
-# from diffusers.image_processor import VaeImageProcessor
-from diffusers.utils.testing_utils import (
-    enable_full_determinism,
-    floats_tensor,
-    load_image,
-    require_torch_gpu,
-    skip_mps,
-    slow,
-    torch_device,
-)
-
-
-enable_full_determinism()
-
-
-@skip_mps
-class LEditsPPPipelineStableDiffusionXLFastTests(unittest.TestCase):
-    pipeline_class = LEditsPPPipelineStableDiffusionXL
-
-    def get_dummy_components(self, skip_first_text_encoder=False, time_cond_proj_dim=None):
-        torch.manual_seed(0)
-        unet = UNet2DConditionModel(
-            block_out_channels=(32, 64),
-            layers_per_block=2,
-            sample_size=32,
-            in_channels=4,
-            out_channels=4,
-            time_cond_proj_dim=time_cond_proj_dim,
-            down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"),
-            up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"),
-            # SD2-specific config below
-            attention_head_dim=(2, 4),
-            use_linear_projection=True,
-            addition_embed_type="text_time",
-            addition_time_embed_dim=8,
-            transformer_layers_per_block=(1, 2),
-            projection_class_embeddings_input_dim=80,  # 6 * 8 + 32
-            cross_attention_dim=64 if not skip_first_text_encoder else 32,
-        )
-        scheduler = DPMSolverMultistepScheduler(algorithm_type="sde-dpmsolver++", solver_order=2)
-        torch.manual_seed(0)
-        vae = AutoencoderKL(
-            block_out_channels=[32, 64],
-            in_channels=3,
-            out_channels=3,
-            down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"],
-            up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"],
-            latent_channels=4,
-            sample_size=128,
-        )
-        torch.manual_seed(0)
-        image_encoder_config = CLIPVisionConfig(
-            hidden_size=32,
-            image_size=224,
-            projection_dim=32,
-            intermediate_size=37,
-            num_attention_heads=4,
-            num_channels=3,
-            num_hidden_layers=5,
-            patch_size=14,
-        )
-        image_encoder = CLIPVisionModelWithProjection(image_encoder_config)
-
-        feature_extractor = CLIPImageProcessor(
-            crop_size=224,
-            do_center_crop=True,
-            do_normalize=True,
-            do_resize=True,
-            image_mean=[0.48145466, 0.4578275, 0.40821073],
-            image_std=[0.26862954, 0.26130258, 0.27577711],
-            resample=3,
-            size=224,
-        )
-
-        torch.manual_seed(0)
-        text_encoder_config = CLIPTextConfig(
-            bos_token_id=0,
-            eos_token_id=2,
-            hidden_size=32,
-            intermediate_size=37,
-            layer_norm_eps=1e-05,
-            num_attention_heads=4,
-            num_hidden_layers=5,
-            pad_token_id=1,
-            vocab_size=1000,
-            # SD2-specific config below
-            hidden_act="gelu",
-            projection_dim=32,
-        )
-        text_encoder = CLIPTextModel(text_encoder_config)
-        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")
-
-        text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config)
-        tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")
-
-        components = {
-            "unet": unet,
-            "scheduler": scheduler,
-            "vae": vae,
-            "text_encoder": text_encoder if not skip_first_text_encoder else None,
-            "tokenizer": tokenizer if not skip_first_text_encoder else None,
-            "text_encoder_2": text_encoder_2,
-            "tokenizer_2": tokenizer_2,
-            "image_encoder": image_encoder,
-            "feature_extractor": feature_extractor,
-        }
-        return components
-
-    def get_dummy_inputs(self, device, seed=0):
-        if str(device).startswith("mps"):
-            generator = torch.manual_seed(seed)
-        else:
-            generator = torch.Generator(device=device).manual_seed(seed)
-        inputs = {
-            "generator": generator,
-            "editing_prompt": ["wearing glasses", "sunshine"],
-            "reverse_editing_direction": [False, True],
-            "edit_guidance_scale": [10.0, 5.0],
-        }
-        return inputs
-
-    def get_dummy_inversion_inputs(self, device, seed=0):
-        images = floats_tensor((2, 3, 32, 32), rng=random.Random(0)).cpu().permute(0, 2, 3, 1)
-        images = 255 * images
-        image_1 = Image.fromarray(np.uint8(images[0])).convert("RGB")
-        image_2 = Image.fromarray(np.uint8(images[1])).convert("RGB")
-
-        if str(device).startswith("mps"):
-            generator = torch.manual_seed(seed)
-        else:
-            generator = torch.Generator(device=device).manual_seed(seed)
-
-        inputs = {
-            "image": [image_1, image_2],
-            "source_prompt": "",
-            "source_guidance_scale": 3.5,
-            "num_inversion_steps": 20,
-            "skip": 0.15,
-            "generator": generator,
-        }
-        return inputs
-
-    def test_ledits_pp_inversion(self):
-        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
-        components = self.get_dummy_components()
-        sd_pipe = LEditsPPPipelineStableDiffusionXL(**components)
-        sd_pipe = sd_pipe.to(torch_device)
-        sd_pipe.set_progress_bar_config(disable=None)
-
-        inputs = self.get_dummy_inversion_inputs(device)
-        inputs["image"] = inputs["image"][0]
-        sd_pipe.invert(**inputs)
-        assert sd_pipe.init_latents.shape == (
-            1,
-            4,
-            int(32 / sd_pipe.vae_scale_factor),
-            int(32 / sd_pipe.vae_scale_factor),
-        )
-
-        latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device)
-        expected_slice = np.array([-0.9084, -0.0367, 0.2940, 0.0839, 0.6890, 0.2651, -0.7103, 2.1090, -0.7821])
-        assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3
-
-    def test_ledits_pp_inversion_batch(self):
-        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
-        components = self.get_dummy_components()
-        sd_pipe = LEditsPPPipelineStableDiffusionXL(**components)
-        sd_pipe = sd_pipe.to(torch_device)
-        sd_pipe.set_progress_bar_config(disable=None)
-
-        inputs = self.get_dummy_inversion_inputs(device)
-        sd_pipe.invert(**inputs)
-        assert sd_pipe.init_latents.shape == (
-            2,
-            4,
-            int(32 / sd_pipe.vae_scale_factor),
-            int(32 / sd_pipe.vae_scale_factor),
-        )
-
-        latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device)
-        print(latent_slice.flatten())
-        expected_slice = np.array([0.2528, 0.1458, -0.2166, 0.4565, -0.5656, -1.0286, -0.9961, 0.5933, 1.1172])
-        assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3
-
-        latent_slice = sd_pipe.init_latents[1, -1, -3:, -3:].to(device)
-        print(latent_slice.flatten())
-        expected_slice = np.array([-0.0796, 2.0583, 0.5500, 0.5358, 0.0282, -0.2803, -1.0470, 0.7024, -0.0072])
-        print(latent_slice.flatten())
-        assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3
-
-    def test_ledits_pp_warmup_steps(self):
-        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
-        components = self.get_dummy_components()
-        pipe = LEditsPPPipelineStableDiffusionXL(**components)
-        pipe = pipe.to(torch_device)
-        pipe.set_progress_bar_config(disable=None)
-
-        inversion_inputs = self.get_dummy_inversion_inputs(device)
-        inversion_inputs["image"] = inversion_inputs["image"][0]
-        pipe.invert(**inversion_inputs)
-
-        inputs = self.get_dummy_inputs(device)
-
-        inputs["edit_warmup_steps"] = [0, 5]
-        pipe(**inputs).images
-
-        inputs["edit_warmup_steps"] = [5, 0]
-        pipe(**inputs).images
-
-        inputs["edit_warmup_steps"] = [5, 10]
-        pipe(**inputs).images
-
-        inputs["edit_warmup_steps"] = [10, 5]
-        pipe(**inputs).images
-
-
-@slow
-@require_torch_gpu
-class LEditsPPPipelineStableDiffusionXLSlowTests(unittest.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        raw_image = load_image(
-            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png"
-        )
-        raw_image = raw_image.convert("RGB").resize((512, 512))
-        cls.raw_image = raw_image
-
-    def test_ledits_pp_edit(self):
-        pipe = LEditsPPPipelineStableDiffusionXL.from_pretrained(
-            "stabilityai/stable-diffusion-xl-base-1.0", safety_checker=None, add_watermarker=None
-        )
-        pipe = pipe.to(torch_device)
-        pipe.set_progress_bar_config(disable=None)
-
-        generator = torch.manual_seed(0)
-        _ = pipe.invert(image=self.raw_image, generator=generator, num_zero_noise_steps=0)
-        inputs = {
-            "generator": generator,
-            "editing_prompt": ["cat", "dog"],
-            "reverse_editing_direction": [True, False],
-            "edit_guidance_scale": [2.0, 4.0],
-            "edit_threshold": [0.8, 0.8],
-        }
-        reconstruction = pipe(**inputs, output_type="np").images[0]
-
-        output_slice = reconstruction[150:153, 140:143, -1]
-        output_slice = output_slice.flatten()
-        expected_slice = np.array(
-            [0.56419, 0.44121838, 0.2765603, 0.5708484, 0.42763475, 0.30945742, 0.5387106, 0.4735807, 0.3547244]
-        )
-        assert np.abs(output_slice - expected_slice).max() < 1e-3

tests/pipelines/stable_cascade/test_stable_cascade_decoder.py

@@ -21,19 +21,18 @@ import torch
 from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer
 
 from diffusers import DDPMWuerstchenScheduler, StableCascadeDecoderPipeline
+from diffusers.image_processor import VaeImageProcessor
 from diffusers.models import StableCascadeUNet
 from diffusers.pipelines.wuerstchen import PaellaVQModel
 from diffusers.utils.testing_utils import (
     enable_full_determinism,
-    load_numpy,
+    load_image,
     load_pt,
-    numpy_cosine_similarity_distance,
     require_torch_gpu,
     skip_mps,
     slow,
     torch_device,
 )
-from diffusers.utils.torch_utils import randn_tensor
 
 from ..test_pipelines_common import PipelineTesterMixin
 
@@ -247,66 +246,6 @@ class StableCascadeDecoderPipelineFastTests(PipelineTesterMixin, unittest.TestCa
 
         assert np.abs(decoder_output_prompt.images - decoder_output_prompt_embeds.images).max() < 1e-5
 
-    def test_stable_cascade_decoder_single_prompt_multiple_image_embeddings(self):
-        device = "cpu"
-        components = self.get_dummy_components()
-
-        pipe = StableCascadeDecoderPipeline(**components)
-        pipe.set_progress_bar_config(disable=None)
-
-        prior_num_images_per_prompt = 2
-        decoder_num_images_per_prompt = 2
-        prompt = ["a cat"]
-        batch_size = len(prompt)
-
-        generator = torch.Generator(device)
-        image_embeddings = randn_tensor(
-            (batch_size * prior_num_images_per_prompt, 4, 4, 4), generator=generator.manual_seed(0)
-        )
-        decoder_output = pipe(
-            image_embeddings=image_embeddings,
-            prompt=prompt,
-            num_inference_steps=1,
-            output_type="np",
-            guidance_scale=0.0,
-            generator=generator.manual_seed(0),
-            num_images_per_prompt=decoder_num_images_per_prompt,
-        )
-
-        assert decoder_output.images.shape[0] == (
-            batch_size * prior_num_images_per_prompt * decoder_num_images_per_prompt
-        )
-
-    def test_stable_cascade_decoder_single_prompt_multiple_image_embeddings_with_guidance(self):
-        device = "cpu"
-        components = self.get_dummy_components()
-
-        pipe = StableCascadeDecoderPipeline(**components)
-        pipe.set_progress_bar_config(disable=None)
-
-        prior_num_images_per_prompt = 2
-        decoder_num_images_per_prompt = 2
-        prompt = ["a cat"]
-        batch_size = len(prompt)
-
-        generator = torch.Generator(device)
-        image_embeddings = randn_tensor(
-            (batch_size * prior_num_images_per_prompt, 4, 4, 4), generator=generator.manual_seed(0)
-        )
-        decoder_output = pipe(
-            image_embeddings=image_embeddings,
-            prompt=prompt,
-            num_inference_steps=1,
-            output_type="np",
-            guidance_scale=2.0,
-            generator=generator.manual_seed(0),
-            num_images_per_prompt=decoder_num_images_per_prompt,
-        )
-
-        assert decoder_output.images.shape[0] == (
-            batch_size * prior_num_images_per_prompt * decoder_num_images_per_prompt
-        )
-
 
 @slow
 @require_torch_gpu
@@ -319,7 +258,7 @@ class StableCascadeDecoderPipelineIntegrationTests(unittest.TestCase):
 
     def test_stable_cascade_decoder(self):
         pipe = StableCascadeDecoderPipeline.from_pretrained(
-            "stabilityai/stable-cascade", variant="bf16", torch_dtype=torch.bfloat16
+            "diffusers/StableCascade-decoder", torch_dtype=torch.bfloat16
         )
         pipe.enable_model_cpu_offload()
         pipe.set_progress_bar_config(disable=None)
@@ -332,16 +271,18 @@ class StableCascadeDecoderPipelineIntegrationTests(unittest.TestCase):
         )
 
         image = pipe(
-            prompt=prompt,
-            image_embeddings=image_embedding,
-            output_type="np",
-            num_inference_steps=2,
-            generator=generator,
+            prompt=prompt, image_embeddings=image_embedding, num_inference_steps=10, generator=generator
         ).images[0]
 
-        assert image.shape == (1024, 1024, 3)
-        expected_image = load_numpy(
-            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_cascade/stable_cascade_decoder_image.npy"
+        assert image.size == (1024, 1024)
+
+        expected_image = load_image(
+            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_cascade/t2i.png"
         )
-        max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten())
-        assert max_diff < 1e-4
+
+        image_processor = VaeImageProcessor()
+
+        image_np = image_processor.pil_to_numpy(image)
+        expected_image_np = image_processor.pil_to_numpy(expected_image)
+
+        self.assertTrue(np.allclose(image_np, expected_image_np, atol=53e-2))

tests/pipelines/stable_cascade/test_stable_cascade_prior.py

@@ -29,8 +29,7 @@ from diffusers.models.attention_processor import LoRAAttnProcessor, LoRAAttnProc
 from diffusers.utils.import_utils import is_peft_available
 from diffusers.utils.testing_utils import (
     enable_full_determinism,
-    load_numpy,
-    numpy_cosine_similarity_distance,
+    load_pt,
     require_peft_backend,
     require_torch_gpu,
     skip_mps,
@@ -320,9 +319,7 @@ class StableCascadePriorPipelineIntegrationTests(unittest.TestCase):
         torch.cuda.empty_cache()
 
     def test_stable_cascade_prior(self):
-        pipe = StableCascadePriorPipeline.from_pretrained(
-            "stabilityai/stable-cascade-prior", variant="bf16", torch_dtype=torch.bfloat16
-        )
+        pipe = StableCascadePriorPipeline.from_pretrained("diffusers/StableCascade-prior", torch_dtype=torch.bfloat16)
         pipe.enable_model_cpu_offload()
         pipe.set_progress_bar_config(disable=None)
 
@@ -330,12 +327,17 @@ class StableCascadePriorPipelineIntegrationTests(unittest.TestCase):
 
         generator = torch.Generator(device="cpu").manual_seed(0)
 
-        output = pipe(prompt, num_inference_steps=2, output_type="np", generator=generator)
+        output = pipe(prompt, num_inference_steps=10, generator=generator)
         image_embedding = output.image_embeddings
-        expected_image_embedding = load_numpy(
-            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_cascade/stable_cascade_prior_image_embeddings.npy"
+
+        expected_image_embedding = load_pt(
+            "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_cascade/image_embedding.pt"
         )
+
         assert image_embedding.shape == (1, 16, 24, 24)
 
-        max_diff = numpy_cosine_similarity_distance(image_embedding.flatten(), expected_image_embedding.flatten())
-        assert max_diff < 1e-4
+        self.assertTrue(
+            np.allclose(
+                image_embedding.cpu().float().numpy(), expected_image_embedding.cpu().float().numpy(), atol=5e-2
+            )
+        )

tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py

@@ -29,7 +29,6 @@ from diffusers import (
     AutoencoderKL,
     DDIMScheduler,
     DPMSolverMultistepScheduler,
-    EulerAncestralDiscreteScheduler,
     LCMScheduler,
     LMSDiscreteScheduler,
     PNDMScheduler,
@@ -558,29 +557,6 @@ class StableDiffusionSimpleInpaintPipelineFastTests(StableDiffusionInpaintPipeli
         image_slice2 = images[1, -3:, -3:, -1]
         assert np.abs(image_slice1.flatten() - image_slice2.flatten()).max() > 1e-2
 
-    def test_stable_diffusion_inpaint_euler(self):
-        device = "cpu"  # ensure determinism for the device-dependent torch.Generator
-        components = self.get_dummy_components(time_cond_proj_dim=256)
-        sd_pipe = StableDiffusionInpaintPipeline(**components)
-        sd_pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(sd_pipe.scheduler.config)
-        sd_pipe = sd_pipe.to(device)
-        sd_pipe.set_progress_bar_config(disable=None)
-
-        inputs = self.get_dummy_inputs(device, output_pil=False)
-        half_dim = inputs["image"].shape[2] // 2
-        inputs["mask_image"][0, 0, :half_dim, :half_dim] = 0
-
-        inputs["num_inference_steps"] = 4
-        image = sd_pipe(**inputs).images
-        image_slice = image[0, -3:, -3:, -1]
-
-        assert image.shape == (1, 64, 64, 3)
-
-        expected_slice = np.array(
-            [[0.6387283, 0.5564158, 0.58631873, 0.5539942, 0.5494673, 0.6461868, 0.5251618, 0.5497595, 0.5508756]]
-        )
-        assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-4
-
 
 @slow
 @require_torch_gpu

utils/fetch_latest_release_branch.py

@@ -1,68 +0,0 @@
-# coding=utf-8
-# Copyright 2024 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import requests
-from packaging.version import parse
-
-
-# GitHub repository details
-USER = "huggingface"
-REPO = "diffusers"
-
-
-def fetch_all_branches(user, repo):
-    branches = []  # List to store all branches
-    page = 1  # Start from first page
-    while True:
-        # Make a request to the GitHub API for the branches
-        response = requests.get(f"https://api.github.com/repos/{user}/{repo}/branches", params={"page": page})
-
-        # Check if the request was successful
-        if response.status_code == 200:
-            # Add the branches from the current page to the list
-            branches.extend([branch["name"] for branch in response.json()])
-
-            # Check if there is a 'next' link for pagination
-            if "next" in response.links:
-                page += 1  # Move to the next page
-            else:
-                break  # Exit loop if there is no next page
-        else:
-            print("Failed to retrieve branches:", response.status_code)
-            break
-
-    return branches
-
-
-def main():
-    # Fetch all branches
-    branches = fetch_all_branches(USER, REPO)
-
-    # Filter branches.
-    # print(f"Total branches: {len(branches)}")
-    filtered_branches = []
-    for branch in branches:
-        if branch.startswith("v") and ("-release" in branch or "-patch" in branch):
-            filtered_branches.append(branch)
-            # print(f"Filtered: {branch}")
-
-    sorted_branches = sorted(filtered_branches, key=lambda x: parse(x.split("-")[0][1:]), reverse=True)
-    latest_branch = sorted_branches[0]
-    # print(f"Latest branch: {latest_branch}")
-    return latest_branch
-
-
-if __name__ == "__main__":
-    print(main())

utils/notify_slack_about_release.py

@@ -1,80 +0,0 @@
-# coding=utf-8
-# Copyright 2024 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-import requests
-
-
-# Configuration
-LIBRARY_NAME = "diffusers"
-GITHUB_REPO = "huggingface/diffusers"
-SLACK_WEBHOOK_URL = os.getenv("SLACK_WEBHOOK_URL")
-
-
-def check_pypi_for_latest_release(library_name):
-    """Check PyPI for the latest release of the library."""
-    response = requests.get(f"https://pypi.org/pypi/{library_name}/json")
-    if response.status_code == 200:
-        data = response.json()
-        return data["info"]["version"]
-    else:
-        print("Failed to fetch library details from PyPI.")
-        return None
-
-
-def get_github_release_info(github_repo):
-    """Fetch the latest release info from GitHub."""
-    url = f"https://api.github.com/repos/{github_repo}/releases/latest"
-    response = requests.get(url)
-
-    if response.status_code == 200:
-        data = response.json()
-        return {"tag_name": data["tag_name"], "url": data["html_url"], "release_time": data["published_at"]}
-
-    else:
-        print("Failed to fetch release info from GitHub.")
-        return None
-
-
-def notify_slack(webhook_url, library_name, version, release_info):
-    """Send a notification to a Slack channel."""
-    message = (
-        f"🚀 New release for {library_name} available: version **{version}** 🎉\n"
-        f"📜 Release Notes: {release_info['url']}\n"
-        f"⏱️ Release time: {release_info['release_time']}"
-    )
-    payload = {"text": message}
-    response = requests.post(webhook_url, json=payload)
-
-    if response.status_code == 200:
-        print("Notification sent to Slack successfully.")
-    else:
-        print("Failed to send notification to Slack.")
-
-
-def main():
-    latest_version = check_pypi_for_latest_release(LIBRARY_NAME)
-    release_info = get_github_release_info(GITHUB_REPO)
-    parsed_version = release_info["tag_name"].replace("v", "")
-
-    if latest_version and release_info and latest_version == parsed_version:
-        notify_slack(SLACK_WEBHOOK_URL, LIBRARY_NAME, latest_version, release_info)
-    else:
-        raise ValueError("There were some problems.")
-
-
-if __name__ == "__main__":
-    main()