Compare commits

..

15 Commits

Author SHA1 Message Date
DN6 3aabef5de4 update 2025-07-24 22:18:15 +05:30
DN6 39be374591 update 2025-07-21 09:03:32 +05:30
DN6 54e17f3084 update 2025-07-21 08:56:47 +05:30
DN6 80702d222d update 2025-07-17 13:05:43 +05:30
DN6 625cc8ede8 update 2025-07-17 07:14:35 +05:30
yiyixuxu a2a9e4eadb Merge branch 'modular-test' of github.com:huggingface/diffusers into modular-test 2025-07-16 12:03:09 +02:00
yiyixuxu 0998bd75ad up 2025-07-16 12:02:58 +02:00
yiyixuxu 5f560d05a2 up 2025-07-16 11:58:23 +02:00
yiyixuxu 4b7a9e9fa9 prepare_latents_inpaint always return noise and image_latents 2025-07-16 11:57:29 +02:00
yiyixuxu d8fa2de36f remove more unused func 2025-07-16 04:29:27 +02:00
YiYi Xu 4df2739a5e Merge branch 'main' into modular-test 2025-07-15 16:27:33 -10:00
yiyixuxu d92855ddf0 style 2025-07-16 04:26:27 +02:00
yiyixuxu 0a5c90ed47 add names property to pipeline blocks 2025-07-16 04:25:26 +02:00
yiyixuxu 0fa58127f8 make style 2025-07-15 03:05:36 +02:00
yiyixuxu b165cf3742 rearrage the params to groups: default params /image params /batch params / callback params 2025-07-15 03:03:29 +02:00
271 changed files with 2909 additions and 15274 deletions
@@ -79,14 +79,14 @@ jobs:
# Check secret is set # Check secret is set
- name: whoami - name: whoami
run: hf auth whoami run: huggingface-cli whoami
env: env:
HF_TOKEN: ${{ secrets.HF_TOKEN_MIRROR_COMMUNITY_PIPELINES }} HF_TOKEN: ${{ secrets.HF_TOKEN_MIRROR_COMMUNITY_PIPELINES }}
# Push to HF! (under subfolder based on checkout ref) # Push to HF! (under subfolder based on checkout ref)
# https://huggingface.co/datasets/diffusers/community-pipelines-mirror # https://huggingface.co/datasets/diffusers/community-pipelines-mirror
- name: Mirror community pipeline to HF - name: Mirror community pipeline to HF
run: hf upload diffusers/community-pipelines-mirror ./examples/community ${PATH_IN_REPO} --repo-type dataset run: huggingface-cli upload diffusers/community-pipelines-mirror ./examples/community ${PATH_IN_REPO} --repo-type dataset
env: env:
PATH_IN_REPO: ${{ env.PATH_IN_REPO }} PATH_IN_REPO: ${{ env.PATH_IN_REPO }}
HF_TOKEN: ${{ secrets.HF_TOKEN_MIRROR_COMMUNITY_PIPELINES }} HF_TOKEN: ${{ secrets.HF_TOKEN_MIRROR_COMMUNITY_PIPELINES }}
+141
View File
@@ -0,0 +1,141 @@
name: Fast PR tests for Modular
on:
pull_request:
branches: [main]
paths:
- "src/diffusers/modular_pipelines/**.py"
- "src/diffusers/models/modeling_utils.py"
- "src/diffusers/models/model_loading_utils.py"
- "src/diffusers/pipelines/pipeline_utils.py"
- "src/diffusers/pipeline_loading_utils.py"
- "src/diffusers/loaders/lora_base.py"
- "src/diffusers/loaders/lora_pipeline.py"
- "src/diffusers/loaders/peft.py"
- "tests/modular_pipelines/**.py"
- ".github/**.yml"
- "utils/**.py"
- "setup.py"
push:
branches:
- ci-*
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
DIFFUSERS_IS_CI: yes
HF_HUB_ENABLE_HF_TRANSFER: 1
OMP_NUM_THREADS: 4
MKL_NUM_THREADS: 4
PYTEST_TIMEOUT: 60
jobs:
check_code_quality:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install .[quality]
- name: Check quality
run: make quality
- name: Check if failure
if: ${{ failure() }}
run: |
echo "Quality check failed. Please ensure the right dependency versions are installed with 'pip install -e .[quality]' and run 'make style && make quality'" >> $GITHUB_STEP_SUMMARY
check_repository_consistency:
needs: check_code_quality
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install .[quality]
- name: Check repo consistency
run: |
python utils/check_copies.py
python utils/check_dummies.py
python utils/check_support_list.py
make deps_table_check_updated
- name: Check if failure
if: ${{ failure() }}
run: |
echo "Repo consistency check failed. Please ensure the right dependency versions are installed with 'pip install -e .[quality]' and run 'make fix-copies'" >> $GITHUB_STEP_SUMMARY
run_fast_tests:
needs: [check_code_quality, check_repository_consistency]
strategy:
fail-fast: false
matrix:
config:
- name: Fast PyTorch Modular Pipeline CPU tests
framework: pytorch_pipelines
runner: aws-highmemory-32-plus
image: diffusers/diffusers-pytorch-cpu
report: torch_cpu_modular_pipelines
name: ${{ matrix.config.name }}
runs-on:
group: ${{ matrix.config.runner }}
container:
image: ${{ matrix.config.image }}
options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
defaults:
run:
shell: bash
steps:
- name: Checkout diffusers
uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Install dependencies
run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python -m uv pip install -e [quality,test]
pip uninstall transformers -y && python -m uv pip install -U transformers@git+https://github.com/huggingface/transformers.git --no-deps
pip uninstall accelerate -y && python -m uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps
- name: Environment
run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python utils/print_env.py
- name: Run fast PyTorch Pipeline CPU tests
if: ${{ matrix.config.framework == 'pytorch_pipelines' }}
run: |
python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
python -m pytest -n 8 --max-worker-restart=0 --dist=loadfile \
-s -v -k "not Flax and not Onnx" \
--make-reports=tests_${{ matrix.config.report }} \
tests/modular_pipelines
- name: Failure short reports
if: ${{ failure() }}
run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt
- name: Test suite reports artifacts
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: pr_${{ matrix.config.framework }}_${{ matrix.config.report }}_test_reports
path: reports
-1
View File
@@ -13,7 +13,6 @@ on:
- "src/diffusers/loaders/peft.py" - "src/diffusers/loaders/peft.py"
- "tests/pipelines/test_pipelines_common.py" - "tests/pipelines/test_pipelines_common.py"
- "tests/models/test_modeling_common.py" - "tests/models/test_modeling_common.py"
- "examples/**/*.py"
workflow_dispatch: workflow_dispatch:
concurrency: concurrency:
+1 -1
View File
@@ -31,7 +31,7 @@ pip install -r requirements.txt
We need to be authenticated to access some of the checkpoints used during benchmarking: We need to be authenticated to access some of the checkpoints used during benchmarking:
```sh ```sh
hf auth login huggingface-cli login
``` ```
We use an L40 GPU with 128GB RAM to run the benchmark CI. As such, the benchmarks are configured to run on NVIDIA GPUs. So, make sure you have access to a similar machine (or modify the benchmarking scripts accordingly). We use an L40 GPU with 128GB RAM to run the benchmark CI. As such, the benchmarks are configured to run on NVIDIA GPUs. So, make sure you have access to a similar machine (or modify the benchmarking scripts accordingly).
+174 -189
View File
@@ -1,39 +1,36 @@
- title: Get started - sections:
sections:
- local: index - local: index
title: Diffusers title: 🧨 Diffusers
- local: installation
title: Installation
- local: quicktour - local: quicktour
title: Quicktour title: Quicktour
- local: stable_diffusion - local: stable_diffusion
title: Effective and efficient diffusion title: Effective and efficient diffusion
- local: installation
- title: DiffusionPipeline title: Installation
isExpanded: false title: Get started
sections: - sections:
- local: using-diffusers/loading - local: tutorials/tutorial_overview
title: Load pipelines title: Overview
- local: using-diffusers/write_own_pipeline
title: Understanding pipelines, models and schedulers
- local: tutorials/autopipeline - local: tutorials/autopipeline
title: AutoPipeline title: AutoPipeline
- local: tutorials/basic_training
title: Train a diffusion model
title: Tutorials
- sections:
- local: using-diffusers/loading
title: Load pipelines
- local: using-diffusers/custom_pipeline_overview - local: using-diffusers/custom_pipeline_overview
title: Load community pipelines and components title: Load community pipelines and components
- local: using-diffusers/callback
title: Pipeline callbacks
- local: using-diffusers/reusing_seeds
title: Reproducible pipelines
- local: using-diffusers/schedulers - local: using-diffusers/schedulers
title: Load schedulers and models title: Load schedulers and models
- local: using-diffusers/scheduler_features
title: Scheduler features
- local: using-diffusers/other-formats - local: using-diffusers/other-formats
title: Model files and layouts title: Model files and layouts
- local: using-diffusers/push_to_hub - local: using-diffusers/push_to_hub
title: Push files to the Hub title: Push files to the Hub
title: Load pipelines and adapters
- title: Adapters - sections:
isExpanded: false
sections:
- local: tutorials/using_peft_for_inference - local: tutorials/using_peft_for_inference
title: LoRA title: LoRA
- local: using-diffusers/ip_adapter - local: using-diffusers/ip_adapter
@@ -46,12 +43,25 @@
title: DreamBooth title: DreamBooth
- local: using-diffusers/textual_inversion_inference - local: using-diffusers/textual_inversion_inference
title: Textual inversion title: Textual inversion
title: Adapters
- title: Inference
isExpanded: false isExpanded: false
sections: - sections:
- local: using-diffusers/weighted_prompts - local: using-diffusers/unconditional_image_generation
title: Prompt techniques title: Unconditional image generation
- local: using-diffusers/conditional_image_generation
title: Text-to-image
- local: using-diffusers/img2img
title: Image-to-image
- local: using-diffusers/inpaint
title: Inpainting
- local: using-diffusers/text-img2vid
title: Video generation
- local: using-diffusers/depth2img
title: Depth-to-image
title: Generative tasks
- sections:
- local: using-diffusers/overview_techniques
title: Overview
- local: using-diffusers/create_a_server - local: using-diffusers/create_a_server
title: Create a server title: Create a server
- local: using-diffusers/batched_inference - local: using-diffusers/batched_inference
@@ -66,38 +76,14 @@
title: Reproducible pipelines title: Reproducible pipelines
- local: using-diffusers/image_quality - local: using-diffusers/image_quality
title: Controlling image quality title: Controlling image quality
- local: using-diffusers/weighted_prompts
- title: Inference optimization title: Prompt techniques
isExpanded: false title: Inference techniques
sections: - sections:
- local: optimization/fp16 - local: advanced_inference/outpaint
title: Accelerate inference title: Outpainting
- local: optimization/cache title: Advanced inference
title: Caching - sections:
- local: optimization/memory
title: Reduce memory usage
- local: optimization/speed-memory-optims
title: Compile and offloading quantized models
- title: Community optimizations
sections:
- local: optimization/pruna
title: Pruna
- local: optimization/xformers
title: xFormers
- local: optimization/tome
title: Token merging
- local: optimization/deepcache
title: DeepCache
- local: optimization/tgate
title: TGATE
- local: optimization/xdit
title: xDiT
- local: optimization/para_attn
title: ParaAttention
- title: Hybrid Inference
isExpanded: false
sections:
- local: hybrid_inference/overview - local: hybrid_inference/overview
title: Overview title: Overview
- local: hybrid_inference/vae_decode - local: hybrid_inference/vae_decode
@@ -106,10 +92,8 @@
title: VAE Encode title: VAE Encode
- local: hybrid_inference/api_reference - local: hybrid_inference/api_reference
title: API Reference title: API Reference
title: Hybrid Inference
- title: Modular Diffusers - sections:
isExpanded: false
sections:
- local: modular_diffusers/overview - local: modular_diffusers/overview
title: Overview title: Overview
- local: modular_diffusers/modular_pipeline - local: modular_diffusers/modular_pipeline
@@ -128,88 +112,8 @@
title: Auto Pipeline Blocks title: Auto Pipeline Blocks
- local: modular_diffusers/end_to_end_guide - local: modular_diffusers/end_to_end_guide
title: End-to-End Example title: End-to-End Example
title: Modular Diffusers
- title: Training - sections:
isExpanded: false
sections:
- local: training/overview
title: Overview
- local: training/create_dataset
title: Create a dataset for training
- local: training/adapt_a_model
title: Adapt a model to a new task
- local: tutorials/basic_training
title: Train a diffusion model
- title: Models
sections:
- local: training/unconditional_training
title: Unconditional image generation
- local: training/text2image
title: Text-to-image
- local: training/sdxl
title: Stable Diffusion XL
- local: training/kandinsky
title: Kandinsky 2.2
- local: training/wuerstchen
title: Wuerstchen
- local: training/controlnet
title: ControlNet
- local: training/t2i_adapters
title: T2I-Adapters
- local: training/instructpix2pix
title: InstructPix2Pix
- local: training/cogvideox
title: CogVideoX
- title: Methods
sections:
- local: training/text_inversion
title: Textual Inversion
- local: training/dreambooth
title: DreamBooth
- local: training/lora
title: LoRA
- local: training/custom_diffusion
title: Custom Diffusion
- local: training/lcm_distill
title: Latent Consistency Distillation
- local: training/ddpo
title: Reinforcement learning training with DDPO
- title: Quantization
isExpanded: false
sections:
- local: quantization/overview
title: Getting started
- local: quantization/bitsandbytes
title: bitsandbytes
- local: quantization/gguf
title: gguf
- local: quantization/torchao
title: torchao
- local: quantization/quanto
title: quanto
- title: Model accelerators and hardware
isExpanded: false
sections:
- local: using-diffusers/stable_diffusion_jax_how_to
title: JAX/Flax
- local: optimization/onnx
title: ONNX
- local: optimization/open_vino
title: OpenVINO
- local: optimization/coreml
title: Core ML
- local: optimization/mps
title: Metal Performance Shaders (MPS)
- local: optimization/habana
title: Intel Gaudi
- local: optimization/neuron
title: AWS Neuron
- title: Specific pipeline examples
isExpanded: false
sections:
- local: using-diffusers/consisid - local: using-diffusers/consisid
title: ConsisID title: ConsisID
- local: using-diffusers/sdxl - local: using-diffusers/sdxl
@@ -234,30 +138,106 @@
title: Stable Video Diffusion title: Stable Video Diffusion
- local: using-diffusers/marigold_usage - local: using-diffusers/marigold_usage
title: Marigold Computer Vision title: Marigold Computer Vision
title: Specific pipeline examples
- title: Resources - sections:
isExpanded: false - local: training/overview
sections: title: Overview
- title: Task recipes - local: training/create_dataset
title: Create a dataset for training
- local: training/adapt_a_model
title: Adapt a model to a new task
- isExpanded: false
sections: sections:
- local: using-diffusers/unconditional_image_generation - local: training/unconditional_training
title: Unconditional image generation title: Unconditional image generation
- local: using-diffusers/conditional_image_generation - local: training/text2image
title: Text-to-image title: Text-to-image
- local: using-diffusers/img2img - local: training/sdxl
title: Image-to-image title: Stable Diffusion XL
- local: using-diffusers/inpaint - local: training/kandinsky
title: Inpainting title: Kandinsky 2.2
- local: advanced_inference/outpaint - local: training/wuerstchen
title: Outpainting title: Wuerstchen
- local: using-diffusers/text-img2vid - local: training/controlnet
title: Video generation title: ControlNet
- local: using-diffusers/depth2img - local: training/t2i_adapters
title: Depth-to-image title: T2I-Adapters
- local: using-diffusers/write_own_pipeline - local: training/instructpix2pix
title: Understanding pipelines, models and schedulers title: InstructPix2Pix
- local: community_projects - local: training/cogvideox
title: Projects built with Diffusers title: CogVideoX
title: Models
- isExpanded: false
sections:
- local: training/text_inversion
title: Textual Inversion
- local: training/dreambooth
title: DreamBooth
- local: training/lora
title: LoRA
- local: training/custom_diffusion
title: Custom Diffusion
- local: training/lcm_distill
title: Latent Consistency Distillation
- local: training/ddpo
title: Reinforcement learning training with DDPO
title: Methods
title: Training
- sections:
- local: quantization/overview
title: Getting Started
- local: quantization/bitsandbytes
title: bitsandbytes
- local: quantization/gguf
title: gguf
- local: quantization/torchao
title: torchao
- local: quantization/quanto
title: quanto
title: Quantization Methods
- sections:
- local: optimization/fp16
title: Accelerate inference
- local: optimization/cache
title: Caching
- local: optimization/memory
title: Reduce memory usage
- local: optimization/speed-memory-optims
title: Compile and offloading quantized models
- local: optimization/pruna
title: Pruna
- local: optimization/xformers
title: xFormers
- local: optimization/tome
title: Token merging
- local: optimization/deepcache
title: DeepCache
- local: optimization/tgate
title: TGATE
- local: optimization/xdit
title: xDiT
- local: optimization/para_attn
title: ParaAttention
- sections:
- local: using-diffusers/stable_diffusion_jax_how_to
title: JAX/Flax
- local: optimization/onnx
title: ONNX
- local: optimization/open_vino
title: OpenVINO
- local: optimization/coreml
title: Core ML
title: Optimized model formats
- sections:
- local: optimization/mps
title: Metal Performance Shaders (MPS)
- local: optimization/habana
title: Intel Gaudi
- local: optimization/neuron
title: AWS Neuron
title: Optimized hardware
title: Accelerate inference and reduce memory
- sections:
- local: conceptual/philosophy - local: conceptual/philosophy
title: Philosophy title: Philosophy
- local: using-diffusers/controlling_generation - local: using-diffusers/controlling_generation
@@ -268,11 +248,13 @@
title: Diffusers' Ethical Guidelines title: Diffusers' Ethical Guidelines
- local: conceptual/evaluation - local: conceptual/evaluation
title: Evaluating Diffusion Models title: Evaluating Diffusion Models
title: Conceptual Guides
- title: API - sections:
isExpanded: false - local: community_projects
sections: title: Projects built with Diffusers
- title: Main Classes title: Community Projects
- sections:
- isExpanded: false
sections: sections:
- local: api/configuration - local: api/configuration
title: Configuration title: Configuration
@@ -282,7 +264,8 @@
title: Outputs title: Outputs
- local: api/quantization - local: api/quantization
title: Quantization title: Quantization
- title: Loaders title: Main Classes
- isExpanded: false
sections: sections:
- local: api/loaders/ip_adapter - local: api/loaders/ip_adapter
title: IP-Adapter title: IP-Adapter
@@ -298,14 +281,14 @@
title: SD3Transformer2D title: SD3Transformer2D
- local: api/loaders/peft - local: api/loaders/peft
title: PEFT title: PEFT
- title: Models title: Loaders
- isExpanded: false
sections: sections:
- local: api/models/overview - local: api/models/overview
title: Overview title: Overview
- local: api/models/auto_model - local: api/models/auto_model
title: AutoModel title: AutoModel
- title: ControlNets - sections:
sections:
- local: api/models/controlnet - local: api/models/controlnet
title: ControlNetModel title: ControlNetModel
- local: api/models/controlnet_union - local: api/models/controlnet_union
@@ -320,8 +303,8 @@
title: SD3ControlNetModel title: SD3ControlNetModel
- local: api/models/controlnet_sparsectrl - local: api/models/controlnet_sparsectrl
title: SparseControlNetModel title: SparseControlNetModel
- title: Transformers title: ControlNets
sections: - sections:
- local: api/models/allegro_transformer3d - local: api/models/allegro_transformer3d
title: AllegroTransformer3DModel title: AllegroTransformer3DModel
- local: api/models/aura_flow_transformer2d - local: api/models/aura_flow_transformer2d
@@ -370,8 +353,6 @@
title: SanaTransformer2DModel title: SanaTransformer2DModel
- local: api/models/sd3_transformer2d - local: api/models/sd3_transformer2d
title: SD3Transformer2DModel title: SD3Transformer2DModel
- local: api/models/skyreels_v2_transformer_3d
title: SkyReelsV2Transformer3DModel
- local: api/models/stable_audio_transformer - local: api/models/stable_audio_transformer
title: StableAudioDiTModel title: StableAudioDiTModel
- local: api/models/transformer2d - local: api/models/transformer2d
@@ -380,8 +361,8 @@
title: TransformerTemporalModel title: TransformerTemporalModel
- local: api/models/wan_transformer_3d - local: api/models/wan_transformer_3d
title: WanTransformer3DModel title: WanTransformer3DModel
- title: UNets title: Transformers
sections: - sections:
- local: api/models/stable_cascade_unet - local: api/models/stable_cascade_unet
title: StableCascadeUNet title: StableCascadeUNet
- local: api/models/unet - local: api/models/unet
@@ -396,8 +377,8 @@
title: UNetMotionModel title: UNetMotionModel
- local: api/models/uvit2d - local: api/models/uvit2d
title: UViT2DModel title: UViT2DModel
- title: VAEs title: UNets
sections: - sections:
- local: api/models/asymmetricautoencoderkl - local: api/models/asymmetricautoencoderkl
title: AsymmetricAutoencoderKL title: AsymmetricAutoencoderKL
- local: api/models/autoencoder_dc - local: api/models/autoencoder_dc
@@ -428,7 +409,9 @@
title: Tiny AutoEncoder title: Tiny AutoEncoder
- local: api/models/vq - local: api/models/vq
title: VQModel title: VQModel
- title: Pipelines title: VAEs
title: Models
- isExpanded: false
sections: sections:
- local: api/pipelines/overview - local: api/pipelines/overview
title: Overview title: Overview
@@ -564,14 +547,11 @@
title: Semantic Guidance title: Semantic Guidance
- local: api/pipelines/shap_e - local: api/pipelines/shap_e
title: Shap-E title: Shap-E
- local: api/pipelines/skyreels_v2
title: SkyReels-V2
- local: api/pipelines/stable_audio - local: api/pipelines/stable_audio
title: Stable Audio title: Stable Audio
- local: api/pipelines/stable_cascade - local: api/pipelines/stable_cascade
title: Stable Cascade title: Stable Cascade
- title: Stable Diffusion - sections:
sections:
- local: api/pipelines/stable_diffusion/overview - local: api/pipelines/stable_diffusion/overview
title: Overview title: Overview
- local: api/pipelines/stable_diffusion/depth2img - local: api/pipelines/stable_diffusion/depth2img
@@ -608,6 +588,7 @@
title: T2I-Adapter title: T2I-Adapter
- local: api/pipelines/stable_diffusion/text2img - local: api/pipelines/stable_diffusion/text2img
title: Text-to-image title: Text-to-image
title: Stable Diffusion
- local: api/pipelines/stable_unclip - local: api/pipelines/stable_unclip
title: Stable unCLIP title: Stable unCLIP
- local: api/pipelines/text_to_video - local: api/pipelines/text_to_video
@@ -626,7 +607,8 @@
title: Wan title: Wan
- local: api/pipelines/wuerstchen - local: api/pipelines/wuerstchen
title: Wuerstchen title: Wuerstchen
- title: Schedulers title: Pipelines
- isExpanded: false
sections: sections:
- local: api/schedulers/overview - local: api/schedulers/overview
title: Overview title: Overview
@@ -696,7 +678,8 @@
title: UniPCMultistepScheduler title: UniPCMultistepScheduler
- local: api/schedulers/vq_diffusion - local: api/schedulers/vq_diffusion
title: VQDiffusionScheduler title: VQDiffusionScheduler
- title: Internal classes title: Schedulers
- isExpanded: false
sections: sections:
- local: api/internal_classes_overview - local: api/internal_classes_overview
title: Overview title: Overview
@@ -714,3 +697,5 @@
title: VAE Image Processor title: VAE Image Processor
- local: api/video_processor - local: api/video_processor
title: Video Processor title: Video Processor
title: Internal classes
title: API
+1 -1
View File
@@ -16,7 +16,7 @@ Schedulers from [`~schedulers.scheduling_utils.SchedulerMixin`] and models from
<Tip> <Tip>
To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with `hf auth login`. To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with `huggingface-cli login`.
</Tip> </Tip>
+2 -7
View File
@@ -26,7 +26,6 @@ LoRA is a fast and lightweight training method that inserts and trains a signifi
- [`HunyuanVideoLoraLoaderMixin`] provides similar functions for [HunyuanVideo](https://huggingface.co/docs/diffusers/main/en/api/pipelines/hunyuan_video). - [`HunyuanVideoLoraLoaderMixin`] provides similar functions for [HunyuanVideo](https://huggingface.co/docs/diffusers/main/en/api/pipelines/hunyuan_video).
- [`Lumina2LoraLoaderMixin`] provides similar functions for [Lumina2](https://huggingface.co/docs/diffusers/main/en/api/pipelines/lumina2). - [`Lumina2LoraLoaderMixin`] provides similar functions for [Lumina2](https://huggingface.co/docs/diffusers/main/en/api/pipelines/lumina2).
- [`WanLoraLoaderMixin`] provides similar functions for [Wan](https://huggingface.co/docs/diffusers/main/en/api/pipelines/wan). - [`WanLoraLoaderMixin`] provides similar functions for [Wan](https://huggingface.co/docs/diffusers/main/en/api/pipelines/wan).
- [`SkyReelsV2LoraLoaderMixin`] provides similar functions for [SkyReels-V2](https://huggingface.co/docs/diffusers/main/en/api/pipelines/skyreels_v2).
- [`CogView4LoraLoaderMixin`] provides similar functions for [CogView4](https://huggingface.co/docs/diffusers/main/en/api/pipelines/cogview4). - [`CogView4LoraLoaderMixin`] provides similar functions for [CogView4](https://huggingface.co/docs/diffusers/main/en/api/pipelines/cogview4).
- [`AmusedLoraLoaderMixin`] is for the [`AmusedPipeline`]. - [`AmusedLoraLoaderMixin`] is for the [`AmusedPipeline`].
- [`HiDreamImageLoraLoaderMixin`] provides similar functions for [HiDream Image](https://huggingface.co/docs/diffusers/main/en/api/pipelines/hidream) - [`HiDreamImageLoraLoaderMixin`] provides similar functions for [HiDream Image](https://huggingface.co/docs/diffusers/main/en/api/pipelines/hidream)
@@ -93,10 +92,6 @@ To learn more about how to load LoRA weights, see the [LoRA](../../using-diffuse
[[autodoc]] loaders.lora_pipeline.WanLoraLoaderMixin [[autodoc]] loaders.lora_pipeline.WanLoraLoaderMixin
## SkyReelsV2LoraLoaderMixin
[[autodoc]] loaders.lora_pipeline.SkyReelsV2LoraLoaderMixin
## AmusedLoraLoaderMixin ## AmusedLoraLoaderMixin
[[autodoc]] loaders.lora_pipeline.AmusedLoraLoaderMixin [[autodoc]] loaders.lora_pipeline.AmusedLoraLoaderMixin
@@ -105,6 +100,6 @@ To learn more about how to load LoRA weights, see the [LoRA](../../using-diffuse
[[autodoc]] loaders.lora_pipeline.HiDreamImageLoraLoaderMixin [[autodoc]] loaders.lora_pipeline.HiDreamImageLoraLoaderMixin
## LoraBaseMixin ## WanLoraLoaderMixin
[[autodoc]] loaders.lora_base.LoraBaseMixin [[autodoc]] loaders.lora_pipeline.WanLoraLoaderMixin
@@ -1,30 +0,0 @@
<!-- Copyright 2024 The HuggingFace Team. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License. -->
# SkyReelsV2Transformer3DModel
A Diffusion Transformer model for 3D video-like data was introduced in [SkyReels-V2](https://github.com/SkyworkAI/SkyReels-V2) by the Skywork AI.
The model can be loaded with the following code snippet.
```python
from diffusers import SkyReelsV2Transformer3DModel
transformer = SkyReelsV2Transformer3DModel.from_pretrained("Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers", subfolder="transformer", torch_dtype=torch.bfloat16)
```
## SkyReelsV2Transformer3DModel
[[autodoc]] SkyReelsV2Transformer3DModel
## Transformer2DModelOutput
[[autodoc]] models.modeling_outputs.Transformer2DModelOutput
-367
View File
@@ -1,367 +0,0 @@
<!-- Copyright 2024 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License. -->
<div style="float: right;">
<div class="flex flex-wrap space-x-1">
<a href="https://huggingface.co/docs/diffusers/main/en/tutorials/using_peft_for_inference" target="_blank" rel="noopener">
<img alt="LoRA" src="https://img.shields.io/badge/LoRA-d8b4fe?style=flat"/>
</a>
</div>
</div>
# SkyReels-V2: Infinite-length Film Generative model
[SkyReels-V2](https://huggingface.co/papers/2504.13074) by the SkyReels Team.
*Recent advances in video generation have been driven by diffusion models and autoregressive frameworks, yet critical challenges persist in harmonizing prompt adherence, visual quality, motion dynamics, and duration: compromises in motion dynamics to enhance temporal visual quality, constrained video duration (5-10 seconds) to prioritize resolution, and inadequate shot-aware generation stemming from general-purpose MLLMs' inability to interpret cinematic grammar, such as shot composition, actor expressions, and camera motions. These intertwined limitations hinder realistic long-form synthesis and professional film-style generation. To address these limitations, we propose SkyReels-V2, an Infinite-length Film Generative Model, that synergizes Multi-modal Large Language Model (MLLM), Multi-stage Pretraining, Reinforcement Learning, and Diffusion Forcing Framework. Firstly, we design a comprehensive structural representation of video that combines the general descriptions by the Multi-modal LLM and the detailed shot language by sub-expert models. Aided with human annotation, we then train a unified Video Captioner, named SkyCaptioner-V1, to efficiently label the video data. Secondly, we establish progressive-resolution pretraining for the fundamental video generation, followed by a four-stage post-training enhancement: Initial concept-balanced Supervised Fine-Tuning (SFT) improves baseline quality; Motion-specific Reinforcement Learning (RL) training with human-annotated and synthetic distortion data addresses dynamic artifacts; Our diffusion forcing framework with non-decreasing noise schedules enables long-video synthesis in an efficient search space; Final high-quality SFT refines visual fidelity. All the code and models are available at [this https URL](https://github.com/SkyworkAI/SkyReels-V2).*
You can find all the original SkyReels-V2 checkpoints under the [Skywork](https://huggingface.co/collections/Skywork/skyreels-v2-6801b1b93df627d441d0d0d9) organization.
The following SkyReels-V2 models are supported in Diffusers:
- [SkyReels-V2 DF 1.3B - 540P](https://huggingface.co/Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers)
- [SkyReels-V2 DF 14B - 540P](https://huggingface.co/Skywork/SkyReels-V2-DF-14B-540P-Diffusers)
- [SkyReels-V2 DF 14B - 720P](https://huggingface.co/Skywork/SkyReels-V2-DF-14B-720P-Diffusers)
- [SkyReels-V2 T2V 14B - 540P](https://huggingface.co/Skywork/SkyReels-V2-T2V-14B-540P-Diffusers)
- [SkyReels-V2 T2V 14B - 720P](https://huggingface.co/Skywork/SkyReels-V2-T2V-14B-720P-Diffusers)
- [SkyReels-V2 I2V 1.3B - 540P](https://huggingface.co/Skywork/SkyReels-V2-I2V-1.3B-540P-Diffusers)
- [SkyReels-V2 I2V 14B - 540P](https://huggingface.co/Skywork/SkyReels-V2-I2V-14B-540P-Diffusers)
- [SkyReels-V2 I2V 14B - 720P](https://huggingface.co/Skywork/SkyReels-V2-I2V-14B-720P-Diffusers)
- [SkyReels-V2 FLF2V 1.3B - 540P](https://huggingface.co/Skywork/SkyReels-V2-FLF2V-1.3B-540P-Diffusers)
> [!TIP]
> Click on the SkyReels-V2 models in the right sidebar for more examples of video generation.
### A _Visual_ Demonstration
An example with these parameters:
base_num_frames=97, num_frames=97, num_inference_steps=30, ar_step=5, causal_block_size=5
vae_scale_factor_temporal -> 4
num_latent_frames: (97-1)//vae_scale_factor_temporal+1 = 25 frames -> 5 blocks of 5 frames each
base_num_latent_frames = (97-1)//vae_scale_factor_temporal+1 = 25 → blocks = 25//5 = 5 blocks
This 5 blocks means the maximum context length of the model is 25 frames in the latent space.
Asynchronous Processing Timeline:
┌─────────────────────────────────────────────────────────────────┐
│ Steps: 1 6 11 16 21 26 31 36 41 46 50 │
│ Block 1: [■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■] │
│ Block 2: [■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■] │
│ Block 3: [■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■] │
│ Block 4: [■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■] │
│ Block 5: [■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■] │
└─────────────────────────────────────────────────────────────────┘
For Long Videos (num_frames > base_num_frames):
base_num_frames acts as the "sliding window size" for processing long videos.
Example: 257-frame video with base_num_frames=97, overlap_history=17
┌──── Iteration 1 (frames 1-97) ────┐
│ Processing window: 97 frames │ → 5 blocks, async processing
│ Generates: frames 1-97 │
└───────────────────────────────────┘
┌────── Iteration 2 (frames 81-177) ──────┐
│ Processing window: 97 frames │
│ Overlap: 17 frames (81-97) from prev │ → 5 blocks, async processing
│ Generates: frames 98-177 │
└─────────────────────────────────────────┘
┌────── Iteration 3 (frames 161-257) ──────┐
│ Processing window: 97 frames │
│ Overlap: 17 frames (161-177) from prev │ → 5 blocks, async processing
│ Generates: frames 178-257 │
└──────────────────────────────────────────┘
Each iteration independently runs the asynchronous processing with its own 5 blocks.
base_num_frames controls:
1. Memory usage (larger window = more VRAM)
2. Model context length (must match training constraints)
3. Number of blocks per iteration (base_num_latent_frames // causal_block_size)
Each block takes 30 steps to complete denoising.
Block N starts at step: 1 + (N-1) x ar_step
Total steps: 30 + (5-1) x 5 = 50 steps
Synchronous mode (ar_step=0) would process all blocks/frames simultaneously:
┌──────────────────────────────────────────────┐
│ Steps: 1 ... 30 │
│ All blocks: [■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■] │
└──────────────────────────────────────────────┘
Total steps: 30 steps
An example on how the step matrix is constructed for asynchronous processing:
Given the parameters: (num_inference_steps=30, flow_shift=8, num_frames=97, ar_step=5, causal_block_size=5)
- num_latent_frames = (97 frames - 1) // (4 temporal downsampling) + 1 = 25
- step_template = [999, 995, 991, 986, 980, 975, 969, 963, 956, 948,
941, 932, 922, 912, 901, 888, 874, 859, 841, 822,
799, 773, 743, 708, 666, 615, 551, 470, 363, 216]
The algorithm creates a 50x25 step_matrix where:
- Row 1: [999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999]
- Row 2: [995, 995, 995, 995, 995, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999]
- Row 3: [991, 991, 991, 991, 991, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999]
- ...
- Row 7: [969, 969, 969, 969, 969, 995, 995, 995, 995, 995, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999, 999]
- ...
- Row 21: [799, 799, 799, 799, 799, 888, 888, 888, 888, 888, 941, 941, 941, 941, 941, 975, 975, 975, 975, 975, 999, 999, 999, 999, 999]
- ...
- Row 35: [ 0, 0, 0, 0, 0, 216, 216, 216, 216, 216, 666, 666, 666, 666, 666, 822, 822, 822, 822, 822, 901, 901, 901, 901, 901]
- ...
- Row 42: [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 551, 551, 551, 551, 773, 773, 773, 773, 773]
- ...
- Row 50: [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 216, 216, 216, 216]
Detailed Row 6 Analysis:
- step_matrix[5]: [ 975, 975, 975, 975, 975, 999, 999, 999, 999, 999, 999, ..., 999]
- step_index[5]: [ 6, 6, 6, 6, 6, 1, 1, 1, 1, 1, 0, ..., 0]
- step_update_mask[5]: [True,True,True,True,True,True,True,True,True,True,False, ...,False]
- valid_interval[5]: (0, 25)
Key Pattern: Block i lags behind Block i-1 by exactly ar_step=5 timesteps, creating the
staggered "diffusion forcing" effect where later blocks condition on cleaner earlier blocks.
### Text-to-Video Generation
The example below demonstrates how to generate a video from text.
<hfoptions id="T2V usage">
<hfoption id="T2V memory">
Refer to the [Reduce memory usage](../../optimization/memory) guide for more details about the various memory saving techniques.
From the original repo:
>You can use --ar_step 5 to enable asynchronous inference. When asynchronous inference, --causal_block_size 5 is recommended while it is not supposed to be set for synchronous generation... Asynchronous inference will take more steps to diffuse the whole sequence which means it will be SLOWER than synchronous mode. In our experiments, asynchronous inference may improve the instruction following and visual consistent performance.
```py
# pip install ftfy
import torch
from diffusers import AutoModel, SkyReelsV2DiffusionForcingPipeline, UniPCMultistepScheduler
from diffusers.utils import export_to_video
vae = AutoModel.from_pretrained("Skywork/SkyReels-V2-DF-14B-540P-Diffusers", subfolder="vae", torch_dtype=torch.float32)
transformer = AutoModel.from_pretrained("Skywork/SkyReels-V2-DF-14B-540P-Diffusers", subfolder="transformer", torch_dtype=torch.bfloat16)
pipeline = SkyReelsV2DiffusionForcingPipeline.from_pretrained(
"Skywork/SkyReels-V2-DF-14B-540P-Diffusers",
vae=vae,
transformer=transformer,
torch_dtype=torch.bfloat16
)
flow_shift = 8.0 # 8.0 for T2V, 5.0 for I2V
pipeline.scheduler = UniPCMultistepScheduler.from_config(pipeline.scheduler.config, flow_shift=flow_shift)
pipeline = pipeline.to("cuda")
prompt = "A cat and a dog baking a cake together in a kitchen. The cat is carefully measuring flour, while the dog is stirring the batter with a wooden spoon. The kitchen is cozy, with sunlight streaming through the window."
output = pipeline(
prompt=prompt,
num_inference_steps=30,
height=544, # 720 for 720P
width=960, # 1280 for 720P
num_frames=97,
base_num_frames=97, # 121 for 720P
ar_step=5, # Controls asynchronous inference (0 for synchronous mode)
causal_block_size=5, # Number of frames in each block for asynchronous processing
overlap_history=None, # Number of frames to overlap for smooth transitions in long videos; 17 for long video generations
addnoise_condition=20, # Improves consistency in long video generation
).frames[0]
export_to_video(output, "T2V.mp4", fps=24, quality=8)
```
</hfoption>
</hfoptions>
### First-Last-Frame-to-Video Generation
The example below demonstrates how to use the image-to-video pipeline to generate a video using a text description, a starting frame, and an ending frame.
<hfoptions id="FLF2V usage">
<hfoption id="usage">
```python
import numpy as np
import torch
import torchvision.transforms.functional as TF
from diffusers import AutoencoderKLWan, SkyReelsV2DiffusionForcingImageToVideoPipeline, UniPCMultistepScheduler
from diffusers.utils import export_to_video, load_image
model_id = "Skywork/SkyReels-V2-DF-14B-720P-Diffusers"
vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
pipeline = SkyReelsV2DiffusionForcingImageToVideoPipeline.from_pretrained(
model_id, vae=vae, torch_dtype=torch.bfloat16
)
flow_shift = 5.0 # 8.0 for T2V, 5.0 for I2V
pipeline.scheduler = UniPCMultistepScheduler.from_config(pipeline.scheduler.config, flow_shift=flow_shift)
pipeline.to("cuda")
first_frame = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_first_frame.png")
last_frame = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_last_frame.png")
def aspect_ratio_resize(image, pipeline, max_area=720 * 1280):
aspect_ratio = image.height / image.width
mod_value = pipeline.vae_scale_factor_spatial * pipeline.transformer.config.patch_size[1]
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
image = image.resize((width, height))
return image, height, width
def center_crop_resize(image, height, width):
# Calculate resize ratio to match first frame dimensions
resize_ratio = max(width / image.width, height / image.height)
# Resize the image
width = round(image.width * resize_ratio)
height = round(image.height * resize_ratio)
size = [width, height]
image = TF.center_crop(image, size)
return image, height, width
first_frame, height, width = aspect_ratio_resize(first_frame, pipeline)
if last_frame.size != first_frame.size:
last_frame, _, _ = center_crop_resize(last_frame, height, width)
prompt = "CG animation style, a small blue bird takes off from the ground, flapping its wings. The bird's feathers are delicate, with a unique pattern on its chest. The background shows a blue sky with white clouds under bright sunshine. The camera follows the bird upward, capturing its flight and the vastness of the sky from a close-up, low-angle perspective."
output = pipeline(
image=first_frame, last_image=last_frame, prompt=prompt, height=height, width=width, guidance_scale=5.0
).frames[0]
export_to_video(output, "output.mp4", fps=24, quality=8)
```
</hfoption>
</hfoptions>
### Video-to-Video Generation
<hfoptions id="V2V usage">
<hfoption id="usage">
`SkyReelsV2DiffusionForcingVideoToVideoPipeline` extends a given video.
```python
import numpy as np
import torch
import torchvision.transforms.functional as TF
from diffusers import AutoencoderKLWan, SkyReelsV2DiffusionForcingVideoToVideoPipeline, UniPCMultistepScheduler
from diffusers.utils import export_to_video, load_video
model_id = "Skywork/SkyReels-V2-DF-14B-540P-Diffusers"
vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32)
pipeline = SkyReelsV2DiffusionForcingVideoToVideoPipeline.from_pretrained(
model_id, vae=vae, torch_dtype=torch.bfloat16
)
flow_shift = 5.0 # 8.0 for T2V, 5.0 for I2V
pipeline.scheduler = UniPCMultistepScheduler.from_config(pipeline.scheduler.config, flow_shift=flow_shift)
pipeline.to("cuda")
video = load_video("input_video.mp4")
prompt = "CG animation style, a small blue bird takes off from the ground, flapping its wings. The bird's feathers are delicate, with a unique pattern on its chest. The background shows a blue sky with white clouds under bright sunshine. The camera follows the bird upward, capturing its flight and the vastness of the sky from a close-up, low-angle perspective."
output = pipeline(
video=video, prompt=prompt, height=544, width=960, guidance_scale=5.0,
num_inference_steps=30, num_frames=257, base_num_frames=97#, ar_step=5, causal_block_size=5,
).frames[0]
export_to_video(output, "output.mp4", fps=24, quality=8)
# Total frames will be the number of frames of given video + 257
```
</hfoption>
</hfoptions>
## Notes
- SkyReels-V2 supports LoRAs with [`~loaders.SkyReelsV2LoraLoaderMixin.load_lora_weights`].
<details>
<summary>Show example code</summary>
```py
# pip install ftfy
import torch
from diffusers import AutoModel, SkyReelsV2DiffusionForcingPipeline
from diffusers.utils import export_to_video
vae = AutoModel.from_pretrained(
"Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers", subfolder="vae", torch_dtype=torch.float32
)
pipeline = SkyReelsV2DiffusionForcingPipeline.from_pretrained(
"Skywork/SkyReels-V2-DF-1.3B-540P-Diffusers", vae=vae, torch_dtype=torch.bfloat16
)
pipeline.to("cuda")
pipeline.load_lora_weights("benjamin-paine/steamboat-willie-1.3b", adapter_name="steamboat-willie")
pipeline.set_adapters("steamboat-willie")
pipeline.enable_model_cpu_offload()
# use "steamboat willie style" to trigger the LoRA
prompt = """
steamboat willie style, golden era animation, The camera rushes from far to near in a low-angle shot,
revealing a white ferret on a log. It plays, leaps into the water, and emerges, as the camera zooms in
for a close-up. Water splashes berry bushes nearby, while moss, snow, and leaves blanket the ground.
Birch trees and a light blue sky frame the scene, with ferns in the foreground. Side lighting casts dynamic
shadows and warm highlights. Medium composition, front view, low angle, with depth of field.
"""
output = pipeline(
prompt=prompt,
num_frames=97,
guidance_scale=6.0,
).frames[0]
export_to_video(output, "output.mp4", fps=24)
```
</details>
## SkyReelsV2DiffusionForcingPipeline
[[autodoc]] SkyReelsV2DiffusionForcingPipeline
- all
- __call__
## SkyReelsV2DiffusionForcingImageToVideoPipeline
[[autodoc]] SkyReelsV2DiffusionForcingImageToVideoPipeline
- all
- __call__
## SkyReelsV2DiffusionForcingVideoToVideoPipeline
[[autodoc]] SkyReelsV2DiffusionForcingVideoToVideoPipeline
- all
- __call__
## SkyReelsV2Pipeline
[[autodoc]] SkyReelsV2Pipeline
- all
- __call__
## SkyReelsV2ImageToVideoPipeline
[[autodoc]] SkyReelsV2ImageToVideoPipeline
- all
- __call__
## SkyReelsV2PipelineOutput
[[autodoc]] pipelines.skyreels_v2.pipeline_output.SkyReelsV2PipelineOutput
@@ -31,7 +31,7 @@ _As the model is gated, before using it with diffusers you first need to go to t
Use the command below to log in: Use the command below to log in:
```bash ```bash
hf auth login huggingface-cli login
``` ```
<Tip> <Tip>
+4 -4
View File
@@ -27,19 +27,19 @@ Learn how to quantize models in the [Quantization](../quantization/overview) gui
## BitsAndBytesConfig ## BitsAndBytesConfig
[[autodoc]] quantizers.quantization_config.BitsAndBytesConfig [[autodoc]] BitsAndBytesConfig
## GGUFQuantizationConfig ## GGUFQuantizationConfig
[[autodoc]] quantizers.quantization_config.GGUFQuantizationConfig [[autodoc]] GGUFQuantizationConfig
## QuantoConfig ## QuantoConfig
[[autodoc]] quantizers.quantization_config.QuantoConfig [[autodoc]] QuantoConfig
## TorchAoConfig ## TorchAoConfig
[[autodoc]] quantizers.quantization_config.TorchAoConfig [[autodoc]] TorchAoConfig
## DiffusersQuantizer ## DiffusersQuantizer
+26 -13
View File
@@ -12,24 +12,37 @@ specific language governing permissions and limitations under the License.
<p align="center"> <p align="center">
<br> <br>
<img src="https://raw.githubusercontent.com/huggingface/diffusers/77aadfee6a891ab9fcfb780f87c693f7a5beeb8e/docs/source/imgs/diffusers_library.jpg" width="400" style="border: none;"/> <img src="https://raw.githubusercontent.com/huggingface/diffusers/77aadfee6a891ab9fcfb780f87c693f7a5beeb8e/docs/source/imgs/diffusers_library.jpg" width="400"/>
<br> <br>
</p> </p>
# Diffusers # Diffusers
Diffusers is a library of state-of-the-art pretrained diffusion models for generating videos, images, and audio. 🤗 Diffusers is the go-to library for state-of-the-art pretrained diffusion models for generating images, audio, and even 3D structures of molecules. Whether you're looking for a simple inference solution or want to train your own diffusion model, 🤗 Diffusers is a modular toolbox that supports both. Our library is designed with a focus on [usability over performance](conceptual/philosophy#usability-over-performance), [simple over easy](conceptual/philosophy#simple-over-easy), and [customizability over abstractions](conceptual/philosophy#tweakable-contributorfriendly-over-abstraction).
The library revolves around the [`DiffusionPipeline`], an API designed for: The library has three main components:
- easy inference with only a few lines of code - State-of-the-art diffusion pipelines for inference with just a few lines of code. There are many pipelines in 🤗 Diffusers, check out the table in the pipeline [overview](api/pipelines/overview) for a complete list of available pipelines and the task they solve.
- flexibility to mix-and-match pipeline components (models, schedulers) - Interchangeable [noise schedulers](api/schedulers/overview) for balancing trade-offs between generation speed and quality.
- loading and using adapters like LoRA - Pretrained [models](api/models) that can be used as building blocks, and combined with schedulers, for creating your own end-to-end diffusion systems.
Diffusers also comes with optimizations - such as offloading and quantization - to ensure even the largest models are accessible on memory-constrained devices. If memory is not an issue, Diffusers supports torch.compile to boost inference speed. <div class="mt-10">
<div class="w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-2 md:gap-y-4 md:gap-x-5">
Get started right away with a Diffusers model on the [Hub](https://huggingface.co/models?library=diffusers&sort=trending) today! <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./tutorials/tutorial_overview"
><div class="w-full text-center bg-gradient-to-br from-blue-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Tutorials</div>
## Learn <p class="text-gray-700">Learn the fundamental skills you need to start generating outputs, build your own diffusion system, and train a diffusion model. We recommend starting here if you're using 🤗 Diffusers for the first time!</p>
</a>
If you're a beginner, we recommend starting with the [Hugging Face Diffusion Models Course](https://huggingface.co/learn/diffusion-course/unit0/1). You'll learn the theory behind diffusion models, and learn how to use the Diffusers library to generate images, fine-tune your own models, and more. <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./using-diffusers/loading_overview"
><div class="w-full text-center bg-gradient-to-br from-indigo-400 to-indigo-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">How-to guides</div>
<p class="text-gray-700">Practical guides for helping you load pipelines, models, and schedulers. You'll also learn how to use pipelines for specific tasks, control how outputs are generated, optimize for inference speed, and different training techniques.</p>
</a>
<a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./conceptual/philosophy"
><div class="w-full text-center bg-gradient-to-br from-pink-400 to-pink-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Conceptual guides</div>
<p class="text-gray-700">Understand why the library was designed the way it was, and learn more about the ethical guidelines and safety implementations for using the library.</p>
</a>
<a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./api/models/overview"
><div class="w-full text-center bg-gradient-to-br from-purple-400 to-purple-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Reference</div>
<p class="text-gray-700">Technical descriptions of how 🤗 Diffusers classes and methods work.</p>
</a>
</div>
</div>
-8
View File
@@ -239,12 +239,6 @@ The `step()` function is [called](https://github.com/huggingface/diffusers/blob/
In general, the `sigmas` should [stay on the CPU](https://github.com/huggingface/diffusers/blob/35a969d297cba69110d175ee79c59312b9f49e1e/src/diffusers/schedulers/scheduling_euler_discrete.py#L240) to avoid the communication sync and latency. In general, the `sigmas` should [stay on the CPU](https://github.com/huggingface/diffusers/blob/35a969d297cba69110d175ee79c59312b9f49e1e/src/diffusers/schedulers/scheduling_euler_discrete.py#L240) to avoid the communication sync and latency.
<Tip>
Refer to the [torch.compile and Diffusers: A Hands-On Guide to Peak Performance](https://pytorch.org/blog/torch-compile-and-diffusers-a-hands-on-guide-to-peak-performance/) blog post for maximizing performance with `torch.compile` for diffusion models.
</Tip>
### Benchmarks ### Benchmarks
Refer to the [diffusers/benchmarks](https://huggingface.co/datasets/diffusers/benchmarks) dataset to see inference latency and memory usage data for compiled pipelines. Refer to the [diffusers/benchmarks](https://huggingface.co/datasets/diffusers/benchmarks) dataset to see inference latency and memory usage data for compiled pipelines.
@@ -305,5 +299,3 @@ pipeline.fuse_qkv_projections()
- Read the [Presenting Flux Fast: Making Flux go brrr on H100s](https://pytorch.org/blog/presenting-flux-fast-making-flux-go-brrr-on-h100s/) blog post to learn more about how you can combine all of these optimizations with [TorchInductor](https://docs.pytorch.org/docs/stable/torch.compiler.html) and [AOTInductor](https://docs.pytorch.org/docs/stable/torch.compiler_aot_inductor.html) for a ~2.5x speedup using recipes from [flux-fast](https://github.com/huggingface/flux-fast). - Read the [Presenting Flux Fast: Making Flux go brrr on H100s](https://pytorch.org/blog/presenting-flux-fast-making-flux-go-brrr-on-h100s/) blog post to learn more about how you can combine all of these optimizations with [TorchInductor](https://docs.pytorch.org/docs/stable/torch.compiler.html) and [AOTInductor](https://docs.pytorch.org/docs/stable/torch.compiler_aot_inductor.html) for a ~2.5x speedup using recipes from [flux-fast](https://github.com/huggingface/flux-fast).
These recipes support AMD hardware and [Flux.1 Kontext Dev](https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev). These recipes support AMD hardware and [Flux.1 Kontext Dev](https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev).
- Read the [torch.compile and Diffusers: A Hands-On Guide to Peak Performance](https://pytorch.org/blog/torch-compile-and-diffusers-a-hands-on-guide-to-peak-performance/) blog post
to maximize performance when using `torch.compile`.
+10 -16
View File
@@ -11,7 +11,7 @@ specific language governing permissions and limitations under the License.
--> -->
# Getting started # Quantization
Quantization focuses on representing data with fewer bits while also trying to preserve the precision of the original data. This often means converting a data type to represent the same information with fewer bits. For example, if your model weights are stored as 32-bit floating points and they're quantized to 16-bit floating points, this halves the model size which makes it easier to store and reduces memory usage. Lower precision can also speedup inference because it takes less time to perform calculations with fewer bits. Quantization focuses on representing data with fewer bits while also trying to preserve the precision of the original data. This often means converting a data type to represent the same information with fewer bits. For example, if your model weights are stored as 32-bit floating points and they're quantized to 16-bit floating points, this halves the model size which makes it easier to store and reduces memory usage. Lower precision can also speedup inference because it takes less time to perform calculations with fewer bits.
@@ -19,25 +19,19 @@ Diffusers supports multiple quantization backends to make large diffusion models
## Pipeline-level quantization ## Pipeline-level quantization
There are two ways to use [`~quantizers.PipelineQuantizationConfig`] depending on how much customization you want to apply to the quantization configuration. There are two ways you can use [`~quantizers.PipelineQuantizationConfig`] depending on the level of control you want over the quantization specifications of each model in the pipeline.
- for basic use cases, define the `quant_backend`, `quant_kwargs`, and `components_to_quantize` arguments - for more basic and simple use cases, you only need to define the `quant_backend`, `quant_kwargs`, and `components_to_quantize`
- for granular quantization control, define a `quant_mapping` that provides the quantization configuration for individual model components - for more granular quantization control, provide a `quant_mapping` that provides the quantization specifications for the individual model components
### Basic quantization ### Simple quantization
Initialize [`~quantizers.PipelineQuantizationConfig`] with the following parameters. Initialize [`~quantizers.PipelineQuantizationConfig`] with the following parameters.
- `quant_backend` specifies which quantization backend to use. Currently supported backends include: `bitsandbytes_4bit`, `bitsandbytes_8bit`, `gguf`, `quanto`, and `torchao`. - `quant_backend` specifies which quantization backend to use. Currently supported backends include: `bitsandbytes_4bit`, `bitsandbytes_8bit`, `gguf`, `quanto`, and `torchao`.
- `quant_kwargs` specifies the quantization arguments to use. - `quant_kwargs` contains the specific quantization arguments to use.
> [!TIP]
> These `quant_kwargs` arguments are different for each backend. Refer to the [Quantization API](../api/quantization) docs to view the arguments for each backend.
- `components_to_quantize` specifies which components of the pipeline to quantize. Typically, you should quantize the most compute intensive components like the transformer. The text encoder is another component to consider quantizing if a pipeline has more than one such as [`FluxPipeline`]. The example below quantizes the T5 text encoder in [`FluxPipeline`] while keeping the CLIP model intact. - `components_to_quantize` specifies which components of the pipeline to quantize. Typically, you should quantize the most compute intensive components like the transformer. The text encoder is another component to consider quantizing if a pipeline has more than one such as [`FluxPipeline`]. The example below quantizes the T5 text encoder in [`FluxPipeline`] while keeping the CLIP model intact.
The example below loads the bitsandbytes backend with the following arguments from [`~quantizers.quantization_config.BitsAndBytesConfig`], `load_in_4bit`, `bnb_4bit_quant_type`, and `bnb_4bit_compute_dtype`.
```py ```py
import torch import torch
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
@@ -62,13 +56,13 @@ pipe = DiffusionPipeline.from_pretrained(
image = pipe("photo of a cute dog").images[0] image = pipe("photo of a cute dog").images[0]
``` ```
### Advanced quantization ### quant_mapping
The `quant_mapping` argument provides more options for how to quantize each individual component in a pipeline, like combining different quantization backends. The `quant_mapping` argument provides more flexible options for how to quantize each individual component in a pipeline, like combining different quantization backends.
Initialize [`~quantizers.PipelineQuantizationConfig`] and pass a `quant_mapping` to it. The `quant_mapping` allows you to specify the quantization options for each component in the pipeline such as the transformer and text encoder. Initialize [`~quantizers.PipelineQuantizationConfig`] and pass a `quant_mapping` to it. The `quant_mapping` allows you to specify the quantization options for each component in the pipeline such as the transformer and text encoder.
The example below uses two quantization backends, [`~quantizers.quantization_config.QuantoConfig`] and [`transformers.BitsAndBytesConfig`], for the transformer and text encoder. The example below uses two quantization backends, [`~quantizers.QuantoConfig`] and [`transformers.BitsAndBytesConfig`], for the transformer and text encoder.
```py ```py
import torch import torch
@@ -91,7 +85,7 @@ pipeline_quant_config = PipelineQuantizationConfig(
There is a separate bitsandbytes backend in [Transformers](https://huggingface.co/docs/transformers/main_classes/quantization#transformers.BitsAndBytesConfig). You need to import and use [`transformers.BitsAndBytesConfig`] for components that come from Transformers. For example, `text_encoder_2` in [`FluxPipeline`] is a [`~transformers.T5EncoderModel`] from Transformers so you need to use [`transformers.BitsAndBytesConfig`] instead of [`diffusers.BitsAndBytesConfig`]. There is a separate bitsandbytes backend in [Transformers](https://huggingface.co/docs/transformers/main_classes/quantization#transformers.BitsAndBytesConfig). You need to import and use [`transformers.BitsAndBytesConfig`] for components that come from Transformers. For example, `text_encoder_2` in [`FluxPipeline`] is a [`~transformers.T5EncoderModel`] from Transformers so you need to use [`transformers.BitsAndBytesConfig`] instead of [`diffusers.BitsAndBytesConfig`].
> [!TIP] > [!TIP]
> Use the [basic quantization](#basic-quantization) method above if you don't want to manage these distinct imports or aren't sure where each pipeline component comes from. > Use the [simple quantization](#simple-quantization) method above if you don't want to manage these distinct imports or aren't sure where each pipeline component comes from.
```py ```py
import torch import torch
+2 -2
View File
@@ -145,10 +145,10 @@ When running `accelerate config`, if you use torch.compile, there can be dramati
If you would like to push your model to the Hub after training is completed with a neat model card, make sure you're logged in: If you would like to push your model to the Hub after training is completed with a neat model card, make sure you're logged in:
```bash ```bash
hf auth login huggingface-cli login
# Alternatively, you could upload your model manually using: # Alternatively, you could upload your model manually using:
# hf upload my-cool-account-name/my-cool-lora-name /path/to/awesome/lora # huggingface-cli upload my-cool-account-name/my-cool-lora-name /path/to/awesome/lora
``` ```
Make sure your data is prepared as described in [Data Preparation](#data-preparation). When ready, you can begin training! Make sure your data is prepared as described in [Data Preparation](#data-preparation). When ready, you can begin training!
+1 -1
View File
@@ -67,7 +67,7 @@ dataset = load_dataset(
Then use the [`~datasets.Dataset.push_to_hub`] method to upload the dataset to the Hub: Then use the [`~datasets.Dataset.push_to_hub`] method to upload the dataset to the Hub:
```python ```python
# assuming you have ran the hf auth login command in a terminal # assuming you have ran the huggingface-cli login command in a terminal
dataset.push_to_hub("name_of_your_dataset") dataset.push_to_hub("name_of_your_dataset")
# if you want to push to a private repo, simply pass private=True: # if you want to push to a private repo, simply pass private=True:
+1 -1
View File
@@ -42,7 +42,7 @@ We encourage you to share your model with the community, and in order to do that
Or login in from the terminal: Or login in from the terminal:
```bash ```bash
hf auth login huggingface-cli login
``` ```
Since the model checkpoints are quite large, install [Git-LFS](https://git-lfs.com/) to version these large files: Since the model checkpoints are quite large, install [Git-LFS](https://git-lfs.com/) to version these large files:
@@ -0,0 +1,23 @@
<!--Copyright 2025 The HuggingFace Team. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
-->
# Overview
Welcome to 🧨 Diffusers! If you're new to diffusion models and generative AI, and want to learn more, then you've come to the right place. These beginner-friendly tutorials are designed to provide a gentle introduction to diffusion models and help you understand the library fundamentals - the core components and how 🧨 Diffusers is meant to be used.
You'll learn how to use a pipeline for inference to rapidly generate things, and then deconstruct that pipeline to really understand how to use the library as a modular toolbox for building your own diffusion systems. In the next lesson, you'll learn how to train your own diffusion model to generate what you want.
After completing the tutorials, you'll have gained the necessary skills to start exploring the library on your own and see how to use it for your own projects and applications.
Feel free to join our community on [Discord](https://discord.com/invite/JfAtkvEtRb) or the [forums](https://discuss.huggingface.co/c/discussion-related-to-httpsgithubcomhuggingfacediffusers/63) to connect and collaborate with other users and developers!
Let's start diffusing! 🧨
@@ -319,19 +319,6 @@ If you expect to varied resolutions during inference with this feature, then mak
There are still scenarios where recompulation is unavoidable, such as when the hotswapped LoRA targets more layers than the initial adapter. Try to load the LoRA that targets the most layers *first*. For more details about this limitation, refer to the PEFT [hotswapping](https://huggingface.co/docs/peft/main/en/package_reference/hotswap#peft.utils.hotswap.hotswap_adapter) docs. There are still scenarios where recompulation is unavoidable, such as when the hotswapped LoRA targets more layers than the initial adapter. Try to load the LoRA that targets the most layers *first*. For more details about this limitation, refer to the PEFT [hotswapping](https://huggingface.co/docs/peft/main/en/package_reference/hotswap#peft.utils.hotswap.hotswap_adapter) docs.
<details>
<summary>Technical details of hotswapping</summary>
The [`~loaders.lora_base.LoraBaseMixin.enable_lora_hotswap`] method converts the LoRA scaling factor from floats to torch.tensors and pads the shape of the weights to the largest required shape to avoid reassigning the whole attribute when the data in the weights are replaced.
This is why the `max_rank` argument is important. The results are unchanged even when the values are padded with zeros. Computation may be slower though depending on the padding size.
Since no new LoRA attributes are added, each subsequent LoRA is only allowed to target the same layers, or subset of layers, the first LoRA targets. Choosing the LoRA loading order is important because if the LoRAs target disjoint layers, you may end up creating a dummy LoRA that targets the union of all target layers.
For more implementation details, take a look at the [`hotswap.py`](https://github.com/huggingface/peft/blob/92d65cafa51c829484ad3d95cf71d09de57ff066/src/peft/utils/hotswap.py) file.
</details>
## Merge ## Merge
The weights from each LoRA can be merged together to produce a blend of multiple existing styles. There are several methods for merging LoRAs, each of which differ in *how* the weights are merged (may affect generation quality). The weights from each LoRA can be merged together to produce a blend of multiple existing styles. There are several methods for merging LoRAs, each of which differ in *how* the weights are merged (may affect generation quality).
@@ -687,5 +674,3 @@ Browse the [LoRA Studio](https://lorastudio.co/models) for different LoRAs to us
></iframe> ></iframe>
You can find additional LoRAs in the [FLUX LoRA the Explorer](https://huggingface.co/spaces/multimodalart/flux-lora-the-explorer) and [LoRA the Explorer](https://huggingface.co/spaces/multimodalart/LoraTheExplorer) Spaces. You can find additional LoRAs in the [FLUX LoRA the Explorer](https://huggingface.co/spaces/multimodalart/flux-lora-the-explorer) and [LoRA the Explorer](https://huggingface.co/spaces/multimodalart/LoraTheExplorer) Spaces.
Check out the [Fast LoRA inference for Flux with Diffusers and PEFT](https://huggingface.co/blog/lora-fast) blog post to learn how to optimize LoRA inference with methods like FlashAttention-3 and fp8 quantization.
@@ -0,0 +1,18 @@
<!--Copyright 2025 The HuggingFace Team. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
-->
# Overview
The inference pipeline supports and enables a wide range of techniques that are divided into two categories:
* Pipeline functionality: these techniques modify the pipeline or extend it for other applications. For example, pipeline callbacks add new features to a pipeline and a pipeline can also be extended for distributed inference.
* Improve inference quality: these techniques increase the visual quality of the generated images. For example, you can enhance your prompts with GPT2 to create better images with lower effort.
+1 -1
View File
@@ -37,7 +37,7 @@ Diffusers는 Stable Diffusion 추론을 위해 PyTorch `mps`를 사용해 Apple
```python ```python
# `hf auth login`에 로그인되어 있음을 확인 # `huggingface-cli login`에 로그인되어 있음을 확인
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
pipe = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5") pipe = DiffusionPipeline.from_pretrained("stable-diffusion-v1-5/stable-diffusion-v1-5")
+1 -1
View File
@@ -75,7 +75,7 @@ dataset = load_dataset(
[push_to_hub(https://huggingface.co/docs/datasets/v2.13.1/en/package_reference/main_classes#datasets.Dataset.push_to_hub) 을 사용해서 Hub에 데이터셋을 업로드 합니다: [push_to_hub(https://huggingface.co/docs/datasets/v2.13.1/en/package_reference/main_classes#datasets.Dataset.push_to_hub) 을 사용해서 Hub에 데이터셋을 업로드 합니다:
```python ```python
# 터미널에서 hf auth login 커맨드를 이미 실행했다고 가정합니다 # 터미널에서 huggingface-cli login 커맨드를 이미 실행했다고 가정합니다
dataset.push_to_hub("name_of_your_dataset") dataset.push_to_hub("name_of_your_dataset")
# 개인 repo로 push 하고 싶다면, `private=True` 을 추가하세요: # 개인 repo로 push 하고 싶다면, `private=True` 을 추가하세요:
+1 -1
View File
@@ -39,7 +39,7 @@ specific language governing permissions and limitations under the License.
모델을 저장하거나 커뮤니티와 공유하려면 Hugging Face 계정에 로그인하세요(아직 계정이 없는 경우 [생성](https://huggingface.co/join)하세요): 모델을 저장하거나 커뮤니티와 공유하려면 Hugging Face 계정에 로그인하세요(아직 계정이 없는 경우 [생성](https://huggingface.co/join)하세요):
```bash ```bash
hf auth login huggingface-cli login
``` ```
## Text-to-image ## Text-to-image
+1 -1
View File
@@ -42,7 +42,7 @@ Unconditional 이미지 생성은 학습에 사용된 데이터셋과 유사한
또는 터미널로 로그인할 수 있습니다: 또는 터미널로 로그인할 수 있습니다:
```bash ```bash
hf auth login huggingface-cli login
``` ```
모델 체크포인트가 상당히 크기 때문에 [Git-LFS](https://git-lfs.com/)에서 대용량 파일의 버전 관리를 할 수 있습니다. 모델 체크포인트가 상당히 크기 때문에 [Git-LFS](https://git-lfs.com/)에서 대용량 파일의 버전 관리를 할 수 있습니다.
@@ -42,7 +42,7 @@ Stable Diffusion 모델들은 학습 및 저장된 프레임워크와 다운로
시작하기 전에 스크립트를 실행할 🤗 Diffusers의 로컬 클론(clone)이 있는지 확인하고 Hugging Face 계정에 로그인하여 pull request를 열고 변환된 모델을 허브에 푸시할 수 있도록 하세요. 시작하기 전에 스크립트를 실행할 🤗 Diffusers의 로컬 클론(clone)이 있는지 확인하고 Hugging Face 계정에 로그인하여 pull request를 열고 변환된 모델을 허브에 푸시할 수 있도록 하세요.
```bash ```bash
hf auth login huggingface-cli login
``` ```
스크립트를 사용하려면: 스크립트를 사용하려면:
@@ -69,7 +69,7 @@ Note also that we use PEFT library as backend for LoRA training, make sure to ha
Lastly, we recommend logging into your HF account so that your trained LoRA is automatically uploaded to the hub: Lastly, we recommend logging into your HF account so that your trained LoRA is automatically uploaded to the hub:
```bash ```bash
hf auth login huggingface-cli login
``` ```
This command will prompt you for a token. Copy-paste yours from your [settings/tokens](https://huggingface.co/settings/tokens),and press Enter. This command will prompt you for a token. Copy-paste yours from your [settings/tokens](https://huggingface.co/settings/tokens),and press Enter.
@@ -67,7 +67,7 @@ Note also that we use PEFT library as backend for LoRA training, make sure to ha
Lastly, we recommend logging into your HF account so that your trained LoRA is automatically uploaded to the hub: Lastly, we recommend logging into your HF account so that your trained LoRA is automatically uploaded to the hub:
```bash ```bash
hf auth login huggingface-cli login
``` ```
This command will prompt you for a token. Copy-paste yours from your [settings/tokens](https://huggingface.co/settings/tokens),and press Enter. This command will prompt you for a token. Copy-paste yours from your [settings/tokens](https://huggingface.co/settings/tokens),and press Enter.
@@ -1,24 +1,3 @@
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "torch",
# "torchvision",
# "diffusers @ git+https://github.com/huggingface/diffusers.git@main",
# "transformers",
# "accelerate",
# "peft",
# "safetensors",
# "huggingface_hub",
# "datasets",
# "Pillow",
# "tqdm",
# "bitsandbytes",
# "sentencepiece",
# "protobuf",
# "prodigyopt",
# ]
# ///
#!/usr/bin/env python #!/usr/bin/env python
# coding=utf-8 # coding=utf-8
# Copyright 2025 The HuggingFace Inc. team. All rights reserved. # Copyright 2025 The HuggingFace Inc. team. All rights reserved.
@@ -34,20 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# /// script
# dependencies = [
# "diffusers @ git+https://github.com/huggingface/diffusers.git",
# "torch>=2.0.0",
# "accelerate>=0.31.0",
# "transformers>=4.41.2",
# "ftfy",
# "tensorboard",
# "Jinja2",
# "peft>=0.11.1",
# "sentencepiece",
# ]
# ///
import argparse import argparse
import copy import copy
import itertools import itertools
@@ -1006,7 +971,6 @@ class DreamBoothDataset(Dataset):
def __init__( def __init__(
self, self,
args,
instance_data_root, instance_data_root,
instance_prompt, instance_prompt,
class_prompt, class_prompt,
@@ -1016,8 +980,10 @@ class DreamBoothDataset(Dataset):
class_num=None, class_num=None,
size=1024, size=1024,
repeats=1, repeats=1,
center_crop=False,
): ):
self.size = size self.size = size
self.center_crop = center_crop
self.instance_prompt = instance_prompt self.instance_prompt = instance_prompt
self.custom_instance_prompts = None self.custom_instance_prompts = None
@@ -1092,7 +1058,7 @@ class DreamBoothDataset(Dataset):
if interpolation is None: if interpolation is None:
raise ValueError(f"Unsupported interpolation mode {interpolation=}.") raise ValueError(f"Unsupported interpolation mode {interpolation=}.")
train_resize = transforms.Resize(size, interpolation=interpolation) train_resize = transforms.Resize(size, interpolation=interpolation)
train_crop = transforms.CenterCrop(size) if args.center_crop else transforms.RandomCrop(size) train_crop = transforms.CenterCrop(size) if center_crop else transforms.RandomCrop(size)
train_flip = transforms.RandomHorizontalFlip(p=1.0) train_flip = transforms.RandomHorizontalFlip(p=1.0)
train_transforms = transforms.Compose( train_transforms = transforms.Compose(
[ [
@@ -1109,11 +1075,11 @@ class DreamBoothDataset(Dataset):
# flip # flip
image = train_flip(image) image = train_flip(image)
if args.center_crop: if args.center_crop:
y1 = max(0, int(round((image.height - self.size) / 2.0))) y1 = max(0, int(round((image.height - args.resolution) / 2.0)))
x1 = max(0, int(round((image.width - self.size) / 2.0))) x1 = max(0, int(round((image.width - args.resolution) / 2.0)))
image = train_crop(image) image = train_crop(image)
else: else:
y1, x1, h, w = train_crop.get_params(image, (self.size, self.size)) y1, x1, h, w = train_crop.get_params(image, (args.resolution, args.resolution))
image = crop(image, y1, x1, h, w) image = crop(image, y1, x1, h, w)
image = train_transforms(image) image = train_transforms(image)
self.pixel_values.append(image) self.pixel_values.append(image)
@@ -1136,7 +1102,7 @@ class DreamBoothDataset(Dataset):
self.image_transforms = transforms.Compose( self.image_transforms = transforms.Compose(
[ [
transforms.Resize(size, interpolation=interpolation), transforms.Resize(size, interpolation=interpolation),
transforms.CenterCrop(size) if args.center_crop else transforms.RandomCrop(size), transforms.CenterCrop(size) if center_crop else transforms.RandomCrop(size),
transforms.ToTensor(), transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]), transforms.Normalize([0.5], [0.5]),
] ]
@@ -1356,7 +1322,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
@@ -1861,7 +1827,6 @@ def main(args):
# Dataset and DataLoaders creation: # Dataset and DataLoaders creation:
train_dataset = DreamBoothDataset( train_dataset = DreamBoothDataset(
args=args,
instance_data_root=args.instance_data_dir, instance_data_root=args.instance_data_dir,
instance_prompt=args.instance_prompt, instance_prompt=args.instance_prompt,
train_text_encoder_ti=args.train_text_encoder_ti, train_text_encoder_ti=args.train_text_encoder_ti,
@@ -1871,6 +1836,7 @@ def main(args):
class_num=args.num_class_images, class_num=args.num_class_images,
size=args.resolution, size=args.resolution,
repeats=args.repeats, repeats=args.repeats,
center_crop=args.center_crop,
) )
train_dataloader = torch.utils.data.DataLoader( train_dataloader = torch.utils.data.DataLoader(
@@ -13,20 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# /// script
# dependencies = [
# "diffusers @ git+https://github.com/huggingface/diffusers.git",
# "torch>=2.0.0",
# "accelerate>=0.31.0",
# "transformers>=4.41.2",
# "ftfy",
# "tensorboard",
# "Jinja2",
# "peft>=0.11.1",
# "sentencepiece",
# ]
# ///
import argparse import argparse
import gc import gc
import hashlib import hashlib
@@ -1064,7 +1050,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -13,20 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# /// script
# dependencies = [
# "diffusers @ git+https://github.com/huggingface/diffusers.git",
# "torch>=2.0.0",
# "accelerate>=0.31.0",
# "transformers>=4.41.2",
# "ftfy",
# "tensorboard",
# "Jinja2",
# "peft>=0.11.1",
# "sentencepiece",
# ]
# ///
import argparse import argparse
import gc import gc
import itertools import itertools
@@ -1306,7 +1292,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if args.do_edm_style_training and args.snr_gamma is not None: if args.do_edm_style_training and args.snr_gamma is not None:
+2 -2
View File
@@ -125,10 +125,10 @@ When running `accelerate config`, if we specify torch compile mode to True there
If you would like to push your model to the HF Hub after training is completed with a neat model card, make sure you're logged in: If you would like to push your model to the HF Hub after training is completed with a neat model card, make sure you're logged in:
``` ```
hf auth login huggingface-cli login
# Alternatively, you could upload your model manually using: # Alternatively, you could upload your model manually using:
# hf upload my-cool-account-name/my-cool-lora-name /path/to/awesome/lora # huggingface-cli upload my-cool-account-name/my-cool-lora-name /path/to/awesome/lora
``` ```
Make sure your data is prepared as described in [Data Preparation](#data-preparation). When ready, you can begin training! Make sure your data is prepared as described in [Data Preparation](#data-preparation). When ready, you can begin training!
@@ -962,7 +962,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
+1 -1
View File
@@ -984,7 +984,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
+1 -1
View File
@@ -10,7 +10,7 @@ To incorporate additional condition latents, we expand the input features of Cog
> As the model is gated, before using it with diffusers you first need to go to the [CogView4 Hugging Face page](https://huggingface.co/THUDM/CogView4-6B), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in: > As the model is gated, before using it with diffusers you first need to go to the [CogView4 Hugging Face page](https://huggingface.co/THUDM/CogView4-6B), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in:
```bash ```bash
hf auth login huggingface-cli login
``` ```
The example command below shows how to launch fine-tuning for pose conditions. The dataset ([`raulc0399/open_pose_controlnet`](https://huggingface.co/datasets/raulc0399/open_pose_controlnet)) being used here already has the pose conditions of the original images, so we don't have to compute them. The example command below shows how to launch fine-tuning for pose conditions. The dataset ([`raulc0399/open_pose_controlnet`](https://huggingface.co/datasets/raulc0399/open_pose_controlnet)) being used here already has the pose conditions of the original images, so we don't have to compute them.
@@ -705,7 +705,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_out_dir = Path(args.output_dir, args.logging_dir) logging_out_dir = Path(args.output_dir, args.logging_dir)
+1 -1
View File
@@ -3129,7 +3129,7 @@ from io import BytesIO
from diffusers import DiffusionPipeline from diffusers import DiffusionPipeline
# load the pipeline # load the pipeline
# make sure you're logged in with `hf auth login` # make sure you're logged in with `huggingface-cli login`
model_id_or_path = "stable-diffusion-v1-5/stable-diffusion-v1-5" model_id_or_path = "stable-diffusion-v1-5/stable-diffusion-v1-5"
# can also be used with dreamlike-art/dreamlike-photoreal-2.0 # can also be used with dreamlike-art/dreamlike-photoreal-2.0
pipe = DiffusionPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16, custom_pipeline="pipeline_fabric").to("cuda") pipe = DiffusionPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16, custom_pipeline="pipeline_fabric").to("cuda")
@@ -877,7 +877,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -709,7 +709,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -872,7 +872,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -842,7 +842,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -882,7 +882,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
+1 -1
View File
@@ -359,7 +359,7 @@ wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/ma
We encourage you to store or share your model with the community. To use huggingface hub, please login to your Hugging Face account, or ([create one](https://huggingface.co/docs/diffusers/main/en/training/hf.co/join) if you dont have one already): We encourage you to store or share your model with the community. To use huggingface hub, please login to your Hugging Face account, or ([create one](https://huggingface.co/docs/diffusers/main/en/training/hf.co/join) if you dont have one already):
```sh ```sh
hf auth login huggingface-cli login
``` ```
Make sure you have the `MODEL_DIR`,`OUTPUT_DIR` and `HUB_MODEL_ID` environment variables set. The `OUTPUT_DIR` and `HUB_MODEL_ID` variables specify where to save the model to on the Hub: Make sure you have the `MODEL_DIR`,`OUTPUT_DIR` and `HUB_MODEL_ID` environment variables set. The `OUTPUT_DIR` and `HUB_MODEL_ID` variables specify where to save the model to on the Hub:
+2 -2
View File
@@ -22,7 +22,7 @@ Here is a gpu memory consumption for reference, tested on a single A100 with 80G
> **Gated access** > **Gated access**
> >
> As the model is gated, before using it with diffusers you first need to go to the [FLUX.1 [dev] Hugging Face page](https://huggingface.co/black-forest-labs/FLUX.1-dev), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in: `hf auth login` > As the model is gated, before using it with diffusers you first need to go to the [FLUX.1 [dev] Hugging Face page](https://huggingface.co/black-forest-labs/FLUX.1-dev), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in: `huggingface-cli login`
## Running locally with PyTorch ## Running locally with PyTorch
@@ -88,7 +88,7 @@ wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/ma
wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_2.png wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_2.png
``` ```
Then run `hf auth login` to log into your Hugging Face account. This is needed to be able to push the trained ControlNet parameters to Hugging Face Hub. Then run `huggingface-cli login` to log into your Hugging Face account. This is needed to be able to push the trained ControlNet parameters to Hugging Face Hub.
we can define the num_layers, num_single_layers, which determines the size of the control(default values are num_layers=4, num_single_layers=10) we can define the num_layers, num_single_layers, which determines the size of the control(default values are num_layers=4, num_single_layers=10)
+1 -1
View File
@@ -56,7 +56,7 @@ First download the SD3 model from [Hugging Face Hub](https://huggingface.co/stab
> As the model is gated, before using it with diffusers you first need to go to the [Stable Diffusion 3 Medium Hugging Face page](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers) or [Stable Diffusion 3.5 Large Hugging Face page](https://huggingface.co/stabilityai/stable-diffusion-3.5-medium), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in: > As the model is gated, before using it with diffusers you first need to go to the [Stable Diffusion 3 Medium Hugging Face page](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers) or [Stable Diffusion 3.5 Large Hugging Face page](https://huggingface.co/stabilityai/stable-diffusion-3.5-medium), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in:
```bash ```bash
hf auth login huggingface-cli login
``` ```
This will also allow us to push the trained model parameters to the Hugging Face Hub platform. This will also allow us to push the trained model parameters to the Hugging Face Hub platform.
+1 -1
View File
@@ -58,7 +58,7 @@ wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/ma
wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_2.png wget https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/controlnet_training/conditioning_image_2.png
``` ```
Then run `hf auth login` to log into your Hugging Face account. This is needed to be able to push the trained ControlNet parameters to Hugging Face Hub. Then run `huggingface-cli login` to log into your Hugging Face account. This is needed to be able to push the trained ControlNet parameters to Hugging Face Hub.
```bash ```bash
export MODEL_DIR="stabilityai/stable-diffusion-xl-base-1.0" export MODEL_DIR="stabilityai/stable-diffusion-xl-base-1.0"
+1 -1
View File
@@ -734,7 +734,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
+1 -1
View File
@@ -665,7 +665,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging.basicConfig( logging.basicConfig(
+1 -1
View File
@@ -814,7 +814,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_out_dir = Path(args.output_dir, args.logging_dir) logging_out_dir = Path(args.output_dir, args.logging_dir)
+1 -1
View File
@@ -928,7 +928,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
+1 -1
View File
@@ -829,7 +829,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -663,7 +663,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
+1 -1
View File
@@ -330,7 +330,7 @@ For this example we want to directly store the trained LoRA embeddings on the Hu
we need to be logged in and add the `--push_to_hub` flag. we need to be logged in and add the `--push_to_hub` flag.
```bash ```bash
hf auth login huggingface-cli login
``` ```
Now we can start training! Now we can start training!
+1 -1
View File
@@ -19,7 +19,7 @@ The `train_dreambooth_flux.py` script shows how to implement the training proced
> As the model is gated, before using it with diffusers you first need to go to the [FLUX.1 [dev] Hugging Face page](https://huggingface.co/black-forest-labs/FLUX.1-dev), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in: > As the model is gated, before using it with diffusers you first need to go to the [FLUX.1 [dev] Hugging Face page](https://huggingface.co/black-forest-labs/FLUX.1-dev), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in:
```bash ```bash
hf auth login huggingface-cli login
``` ```
This will also allow us to push the trained model parameters to the Hugging Face Hub platform. This will also allow us to push the trained model parameters to the Hugging Face Hub platform.
+1 -1
View File
@@ -95,7 +95,7 @@ accelerate launch train_dreambooth_lora_hidream.py \
For using `push_to_hub`, make you're logged into your Hugging Face account: For using `push_to_hub`, make you're logged into your Hugging Face account:
```bash ```bash
hf auth login huggingface-cli login
``` ```
To better track our training experiments, we're using the following flags in the command above: To better track our training experiments, we're using the following flags in the command above:
+1 -1
View File
@@ -101,7 +101,7 @@ accelerate launch train_dreambooth_lora_lumina2.py \
For using `push_to_hub`, make you're logged into your Hugging Face account: For using `push_to_hub`, make you're logged into your Hugging Face account:
```bash ```bash
hf auth login huggingface-cli login
``` ```
To better track our training experiments, we're using the following flags in the command above: To better track our training experiments, we're using the following flags in the command above:
+1 -1
View File
@@ -101,7 +101,7 @@ accelerate launch train_dreambooth_lora_sana.py \
For using `push_to_hub`, make you're logged into your Hugging Face account: For using `push_to_hub`, make you're logged into your Hugging Face account:
```bash ```bash
hf auth login huggingface-cli login
``` ```
To better track our training experiments, we're using the following flags in the command above: To better track our training experiments, we're using the following flags in the command above:
+1 -1
View File
@@ -8,7 +8,7 @@ The `train_dreambooth_sd3.py` script shows how to implement the training procedu
> As the model is gated, before using it with diffusers you first need to go to the [Stable Diffusion 3 Medium Hugging Face page](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in: > As the model is gated, before using it with diffusers you first need to go to the [Stable Diffusion 3 Medium Hugging Face page](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in:
```bash ```bash
hf auth login huggingface-cli login
``` ```
This will also allow us to push the trained model parameters to the Hugging Face Hub platform. This will also allow us to push the trained model parameters to the Hugging Face Hub platform.
+1 -1
View File
@@ -807,7 +807,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
+1 -15
View File
@@ -13,20 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# /// script
# dependencies = [
# "diffusers @ git+https://github.com/huggingface/diffusers.git",
# "torch>=2.0.0",
# "accelerate>=0.31.0",
# "transformers>=4.41.2",
# "ftfy",
# "tensorboard",
# "Jinja2",
# "peft>=0.11.1",
# "sentencepiece",
# ]
# ///
import argparse import argparse
import copy import copy
import gc import gc
@@ -1027,7 +1013,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
+1 -1
View File
@@ -756,7 +756,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -13,20 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# /// script
# dependencies = [
# "diffusers @ git+https://github.com/huggingface/diffusers.git",
# "torch>=2.0.0",
# "accelerate>=0.31.0",
# "transformers>=4.41.2",
# "ftfy",
# "tensorboard",
# "Jinja2",
# "peft>=0.11.1",
# "sentencepiece",
# ]
# ///
import argparse import argparse
import copy import copy
import itertools import itertools
@@ -1065,7 +1051,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
@@ -1199,7 +1199,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
@@ -1614,7 +1614,7 @@ def main(args):
) )
if args.cond_image_column is not None: if args.cond_image_column is not None:
logger.info("I2I fine-tuning enabled.") logger.info("I2I fine-tuning enabled.")
batch_sampler = BucketBatchSampler(train_dataset, batch_size=args.train_batch_size, drop_last=True) batch_sampler = BucketBatchSampler(train_dataset, batch_size=args.train_batch_size, drop_last=False)
train_dataloader = torch.utils.data.DataLoader( train_dataloader = torch.utils.data.DataLoader(
train_dataset, train_dataset,
batch_sampler=batch_sampler, batch_sampler=batch_sampler,
@@ -58,7 +58,6 @@ from diffusers.training_utils import (
compute_density_for_timestep_sampling, compute_density_for_timestep_sampling,
compute_loss_weighting_for_sd3, compute_loss_weighting_for_sd3,
free_memory, free_memory,
offload_models,
) )
from diffusers.utils import ( from diffusers.utils import (
check_min_version, check_min_version,
@@ -936,7 +935,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
@@ -1365,34 +1364,43 @@ def main(args):
# provided (i.e. the --instance_prompt is used for all images), we encode the instance prompt once to avoid # provided (i.e. the --instance_prompt is used for all images), we encode the instance prompt once to avoid
# the redundant encoding. # the redundant encoding.
if not train_dataset.custom_instance_prompts: if not train_dataset.custom_instance_prompts:
with offload_models(text_encoding_pipeline, device=accelerator.device, offload=args.offload): if args.offload:
( text_encoding_pipeline = text_encoding_pipeline.to(accelerator.device)
instance_prompt_hidden_states_t5, (
instance_prompt_hidden_states_llama3, instance_prompt_hidden_states_t5,
instance_pooled_prompt_embeds, instance_prompt_hidden_states_llama3,
_, instance_pooled_prompt_embeds,
_, _,
_, _,
) = compute_text_embeddings(args.instance_prompt, text_encoding_pipeline) _,
) = compute_text_embeddings(args.instance_prompt, text_encoding_pipeline)
if args.offload:
text_encoding_pipeline = text_encoding_pipeline.to("cpu")
# Handle class prompt for prior-preservation. # Handle class prompt for prior-preservation.
if args.with_prior_preservation: if args.with_prior_preservation:
with offload_models(text_encoding_pipeline, device=accelerator.device, offload=args.offload): if args.offload:
(class_prompt_hidden_states_t5, class_prompt_hidden_states_llama3, class_pooled_prompt_embeds, _, _, _) = ( text_encoding_pipeline = text_encoding_pipeline.to(accelerator.device)
compute_text_embeddings(args.class_prompt, text_encoding_pipeline) (class_prompt_hidden_states_t5, class_prompt_hidden_states_llama3, class_pooled_prompt_embeds, _, _, _) = (
) compute_text_embeddings(args.class_prompt, text_encoding_pipeline)
)
if args.offload:
text_encoding_pipeline = text_encoding_pipeline.to("cpu")
validation_embeddings = {} validation_embeddings = {}
if args.validation_prompt is not None: if args.validation_prompt is not None:
with offload_models(text_encoding_pipeline, device=accelerator.device, offload=args.offload): if args.offload:
( text_encoding_pipeline = text_encoding_pipeline.to(accelerator.device)
validation_embeddings["prompt_embeds_t5"], (
validation_embeddings["prompt_embeds_llama3"], validation_embeddings["prompt_embeds_t5"],
validation_embeddings["pooled_prompt_embeds"], validation_embeddings["prompt_embeds_llama3"],
validation_embeddings["negative_prompt_embeds_t5"], validation_embeddings["pooled_prompt_embeds"],
validation_embeddings["negative_prompt_embeds_llama3"], validation_embeddings["negative_prompt_embeds_t5"],
validation_embeddings["negative_pooled_prompt_embeds"], validation_embeddings["negative_prompt_embeds_llama3"],
) = compute_text_embeddings(args.validation_prompt, text_encoding_pipeline) validation_embeddings["negative_pooled_prompt_embeds"],
) = compute_text_embeddings(args.validation_prompt, text_encoding_pipeline)
if args.offload:
text_encoding_pipeline = text_encoding_pipeline.to("cpu")
# If custom instance prompts are NOT provided (i.e. the instance prompt is used for all images), # If custom instance prompts are NOT provided (i.e. the instance prompt is used for all images),
# pack the statically computed variables appropriately here. This is so that we don't # pack the statically computed variables appropriately here. This is so that we don't
@@ -1573,10 +1581,12 @@ def main(args):
if args.cache_latents: if args.cache_latents:
model_input = latents_cache[step].sample() model_input = latents_cache[step].sample()
else: else:
with offload_models(vae, device=accelerator.device, offload=args.offload): if args.offload:
pixel_values = batch["pixel_values"].to(dtype=vae.dtype) vae = vae.to(accelerator.device)
pixel_values = batch["pixel_values"].to(dtype=vae.dtype)
model_input = vae.encode(pixel_values).latent_dist.sample() model_input = vae.encode(pixel_values).latent_dist.sample()
if args.offload:
vae = vae.to("cpu")
model_input = (model_input - vae_config_shift_factor) * vae_config_scaling_factor model_input = (model_input - vae_config_shift_factor) * vae_config_scaling_factor
model_input = model_input.to(dtype=weight_dtype) model_input = model_input.to(dtype=weight_dtype)
@@ -859,7 +859,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
@@ -13,20 +13,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# /// script
# dependencies = [
# "diffusers @ git+https://github.com/huggingface/diffusers.git",
# "torch>=2.0.0",
# "accelerate>=1.0.0",
# "transformers>=4.47.0",
# "ftfy",
# "tensorboard",
# "Jinja2",
# "peft>=0.14.0",
# "sentencepiece",
# ]
# ///
import argparse import argparse
import copy import copy
import itertools import itertools
@@ -866,7 +852,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
@@ -1063,7 +1063,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
@@ -983,7 +983,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if args.do_edm_style_training and args.snr_gamma is not None: if args.do_edm_style_training and args.snr_gamma is not None:
+1 -1
View File
@@ -988,7 +988,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
+1 -1
View File
@@ -13,7 +13,7 @@ To incorporate additional condition latents, we expand the input features of Flu
> As the model is gated, before using it with diffusers you first need to go to the [FLUX.1 [dev] Hugging Face page](https://huggingface.co/black-forest-labs/FLUX.1-dev), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in: > As the model is gated, before using it with diffusers you first need to go to the [FLUX.1 [dev] Hugging Face page](https://huggingface.co/black-forest-labs/FLUX.1-dev), fill in the form and accept the gate. Once you are in, you need to log in so that your system knows youve accepted the gate. Use the command below to log in:
```bash ```bash
hf auth login huggingface-cli login
``` ```
The example command below shows how to launch fine-tuning for pose conditions. The dataset ([`raulc0399/open_pose_controlnet`](https://huggingface.co/datasets/raulc0399/open_pose_controlnet)) being used here already has the pose conditions of the original images, so we don't have to compute them. The example command below shows how to launch fine-tuning for pose conditions. The dataset ([`raulc0399/open_pose_controlnet`](https://huggingface.co/datasets/raulc0399/open_pose_controlnet)) being used here already has the pose conditions of the original images, so we don't have to compute them.
+1 -1
View File
@@ -697,7 +697,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_out_dir = Path(args.output_dir, args.logging_dir) logging_out_dir = Path(args.output_dir, args.logging_dir)
@@ -725,7 +725,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if args.use_lora_bias and args.gaussian_init_lora: if args.use_lora_bias and args.gaussian_init_lora:
raise ValueError("`gaussian` LoRA init scheme isn't supported when `use_lora_bias` is True.") raise ValueError("`gaussian` LoRA init scheme isn't supported when `use_lora_bias` is True.")
@@ -430,7 +430,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if args.non_ema_revision is not None: if args.non_ema_revision is not None:
@@ -483,7 +483,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if args.non_ema_revision is not None: if args.non_ema_revision is not None:
@@ -41,7 +41,7 @@ For all our examples, we will directly store the trained weights on the Hub, so
Run the following command to authenticate your token Run the following command to authenticate your token
```bash ```bash
hf auth login huggingface-cli login
``` ```
We also use [Weights and Biases](https://docs.wandb.ai/quickstart) logging by default, because it is really useful to monitor the training progress by regularly generating sample images during training. To install wandb, run We also use [Weights and Biases](https://docs.wandb.ai/quickstart) logging by default, because it is really useful to monitor the training progress by regularly generating sample images during training. To install wandb, run
@@ -444,7 +444,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = os.path.join(args.output_dir, args.logging_dir) logging_dir = os.path.join(args.output_dir, args.logging_dir)
@@ -330,7 +330,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -342,7 +342,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -445,7 +445,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = os.path.join(args.output_dir, args.logging_dir) logging_dir = os.path.join(args.output_dir, args.logging_dir)
+6 -6
View File
@@ -1249,7 +1249,7 @@ class EasyPipelineForText2Image(AutoPipelineForText2Image):
<Tip> <Tip>
To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
`hf auth login`. `huggingface-cli login`.
</Tip> </Tip>
@@ -1358,7 +1358,7 @@ class EasyPipelineForText2Image(AutoPipelineForText2Image):
<Tip> <Tip>
To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
`hf auth login`. `huggingface-cli login`.
</Tip> </Tip>
@@ -1507,7 +1507,7 @@ class EasyPipelineForImage2Image(AutoPipelineForImage2Image):
<Tip> <Tip>
To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
`hf auth login`. `huggingface-cli login`.
</Tip> </Tip>
@@ -1617,7 +1617,7 @@ class EasyPipelineForImage2Image(AutoPipelineForImage2Image):
<Tip> <Tip>
To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
`hf auth login`. `huggingface-cli login`.
</Tip> </Tip>
@@ -1766,7 +1766,7 @@ class EasyPipelineForInpainting(AutoPipelineForInpainting):
<Tip> <Tip>
To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
`hf auth login `huggingface-cli login`.
</Tip> </Tip>
@@ -1875,7 +1875,7 @@ class EasyPipelineForInpainting(AutoPipelineForInpainting):
<Tip> <Tip>
To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with
`hf auth login `huggingface-cli login`.
</Tip> </Tip>
@@ -568,7 +568,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -789,7 +789,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir)
@@ -899,7 +899,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -470,7 +470,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -512,7 +512,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -502,7 +502,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -609,7 +609,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -39,7 +39,7 @@ python compute_embeddings.py
It should create a file named `embeddings.parquet`. We're then ready to launch training. First, authenticate so that you can access the Flux.1 Dev model: It should create a file named `embeddings.parquet`. We're then ready to launch training. First, authenticate so that you can access the Flux.1 Dev model:
```bash ```bash
hf auth login huggingface-cli
``` ```
Then launch: Then launch:
@@ -587,7 +587,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if torch.backends.mps.is_available() and args.mixed_precision == "bf16": if torch.backends.mps.is_available() and args.mixed_precision == "bf16":
+7 -7
View File
@@ -47,11 +47,11 @@ pip install git+https://github.com/xinyu1205/recognize-anything.git --no-deps
Download the pre-trained model: Download the pre-trained model:
```bash ```bash
hf download --resume-download xinyu1205/recognize_anything_model ram_swin_large_14m.pth huggingface-cli download --resume-download xinyu1205/recognize_anything_model ram_swin_large_14m.pth
hf download --resume-download IDEA-Research/grounding-dino-base huggingface-cli download --resume-download IDEA-Research/grounding-dino-base
hf download --resume-download Salesforce/blip2-flan-t5-xxl huggingface-cli download --resume-download Salesforce/blip2-flan-t5-xxl
hf download --resume-download clip-vit-large-patch14 huggingface-cli download --resume-download clip-vit-large-patch14
hf download --resume-download masterful/gligen-1-4-generation-text-box huggingface-cli download --resume-download masterful/gligen-1-4-generation-text-box
``` ```
Make the training data on 8 GPUs: Make the training data on 8 GPUs:
@@ -66,7 +66,7 @@ torchrun --master_port 17673 --nproc_per_node=8 make_datasets.py \
You can download the COCO training data from You can download the COCO training data from
```bash ```bash
hf download --resume-download Hzzone/GLIGEN_COCO coco_train2017.pth huggingface-cli download --resume-download Hzzone/GLIGEN_COCO coco_train2017.pth
``` ```
It's in the format of It's in the format of
@@ -125,7 +125,7 @@ Note that although the pre-trained GLIGEN model has been loaded, the parameters
The trained model can be downloaded from The trained model can be downloaded from
```bash ```bash
hf download --resume-download Hzzone/GLIGEN_COCO config.json diffusion_pytorch_model.safetensors huggingface-cli download --resume-download Hzzone/GLIGEN_COCO config.json diffusion_pytorch_model.safetensors
``` ```
You can run `demo.ipynb` to visualize the generated images. You can run `demo.ipynb` to visualize the generated images.
@@ -488,7 +488,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if args.non_ema_revision is not None: if args.non_ema_revision is not None:
@@ -366,7 +366,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = os.path.join(args.output_dir, args.logging_dir) logging_dir = os.path.join(args.output_dir, args.logging_dir)
+1 -1
View File
@@ -34,7 +34,7 @@ For this example we want to directly store the trained LoRA embeddings on the Hu
we need to be logged in and add the `--push_to_hub` flag. we need to be logged in and add the `--push_to_hub` flag.
```bash ```bash
hf auth login huggingface-cli login
``` ```
Now we can start training! Now we can start training!
@@ -396,7 +396,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = os.path.join(args.output_dir, args.logging_dir) logging_dir = os.path.join(args.output_dir, args.logging_dir)
@@ -684,7 +684,7 @@ def main(args):
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = Path(args.output_dir, args.logging_dir) logging_dir = Path(args.output_dir, args.logging_dir)
@@ -60,7 +60,7 @@ You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need
Run the following command to authenticate your token Run the following command to authenticate your token
```bash ```bash
hf auth login huggingface-cli login
``` ```
If you have already cloned the repo, then you won't need to go through these steps. If you have already cloned the repo, then you won't need to go through these steps.
@@ -551,7 +551,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
logging_dir = os.path.join(args.output_dir, args.logging_dir) logging_dir = os.path.join(args.output_dir, args.logging_dir)
@@ -153,7 +153,7 @@ def parse_args():
"--use_auth_token", "--use_auth_token",
action="store_true", action="store_true",
help=( help=(
"Will use the token generated when running `hf auth login` (necessary to use this script with" "Will use the token generated when running `huggingface-cli login` (necessary to use this script with"
" private models)." " private models)."
), ),
) )
@@ -41,7 +41,7 @@ You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need
Run the following command to authenticate your token Run the following command to authenticate your token
```bash ```bash
hf auth login huggingface-cli login
``` ```
If you have already cloned the repo, then you won't need to go through these steps. If you have already cloned the repo, then you won't need to go through these steps.
@@ -415,7 +415,7 @@ def main():
if args.report_to == "wandb" and args.hub_token is not None: if args.report_to == "wandb" and args.hub_token is not None:
raise ValueError( raise ValueError(
"You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token."
" Please use `hf auth login` to authenticate with the Hub." " Please use `huggingface-cli login` to authenticate with the Hub."
) )
if args.non_ema_revision is not None: if args.non_ema_revision is not None:
@@ -46,7 +46,7 @@ You have to be a registered user in 🤗 Hugging Face Hub, and you'll also need
Run the following command to authenticate your token Run the following command to authenticate your token
```bash ```bash
hf auth login huggingface-cli login
``` ```
If you have already cloned the repo, then you won't need to go through these steps. If you have already cloned the repo, then you won't need to go through these steps.

Some files were not shown because too many files have changed in this diff Show More