obscura/vllm
2
0
Fork 0
mirror of https://github.com/vllm-project/vllm.git synced 2026-08-10 07:48:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deprecations for v0.23 and v0.24 (#44992)

Browse Source 
Signed-off-by: Harry Mellor <[email protected]>
This commit is contained in:
Harry Mellor
2026-06-11 14:35:38 +00:00
committed by GitHub
parent 55911db580
commit 03878d1c22
35 changed files with 100 additions and 674 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml
+1 -3
View File
@@ -6,9 +6,7 @@ tasks:
value: 0.7142
- name: "exact_match,flexible-extract"
value: 0.4579
env_vars:
VLLM_USE_FLASHINFER_MOE_FP8: "1"
VLLM_FLASHINFER_MOE_BACKEND: "throughput"
moe_backend: "flashinfer_cutlass"
limit: 1319
num_fewshot: 5
max_model_len: 262144
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+4
View File
@@ -68,6 +68,10 @@ def launch_lm_eval(eval_config, tp_size):
if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
model_args += "attention_backend=TRITON_ATTN"
moe_backend = eval_config.get("moe_backend", None)
if moe_backend is not None:
model_args += f"moe_backend={moe_backend},"
env_vars = eval_config.get("env_vars", None)
with scoped_env_vars(env_vars):
results = lm_eval.simple_evaluate(
docs/design/moe_kernel_features.md
+1 -1
View File
@@ -42,7 +42,7 @@ th {
1. All types: mxfp4, nvfp4, int4, int8, fp8
2. A,T quantization occurs after dispatch.
3. All quantization happens after dispatch.
4. Controlled by different env vars (`VLLM_FLASHINFER_MOE_BACKEND` "throughput" or "latency")
4. Controlled by `--moe-backend` (`flashinfer_cutlass` or `flashinfer_trtllm`)
5. This is a no-op dispatcher that can be used to pair with any modular experts to produce a modular kernel that runs without dispatch or combine. These cannot be selected via environment variable. These are generally use for testing or adapting an expert subclass to the `fused_experts` API.
6. This depends on the experts implementation.
docs/models/pooling_models/reward.md
+1 -1
View File
@@ -143,4 +143,4 @@ More examples can be found here: [examples/pooling/reward](../../../examples/poo
### `LLM.reward`
`llm.reward` api is deprecated and will be removed in v0.23. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead.
`llm.reward` API is deprecated and was removed in v0.24. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead.
tests/compile/correctness_e2e/test_async_tp.py
+3 -3
View File
@@ -102,7 +102,7 @@ def test_async_tp_pass_correctness(
@create_new_process_for_each_test()
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
if (
not current_platform.is_cuda()
or not current_platform.is_device_capability_family(100)
@@ -111,8 +111,6 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
if not has_flashinfer():
pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", "flashinfer-cutlass")
tp_size = 2
if num_gpus_available < tp_size:
pytest.skip(f"Need at least {tp_size} GPUs")
@@ -126,6 +124,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
"8",
"--load-format",
"dummy",
"--linear-backend",
"flashinfer_cutlass",
"--hf-overrides",
json.dumps(NVFP4_HF_OVERRIDES),
]
tests/conftest.py
-4
View File
@@ -1226,10 +1226,6 @@ class VllmRunner:
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
return [req_output.outputs.data for req_output in req_outputs]
def reward(self, prompts: list[str]) -> list[list[float]]:
req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
return [req_output.outputs.data for req_output in req_outputs]
def score(
self,
text_1: list[str] | str,
tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py
+5 -6
View File
@@ -37,6 +37,7 @@ class TestConfig:
hidden_size: int
intermediate_size: int
num_tokens: int
moe_backend: str
def make_fused_moe_layer(
@@ -114,6 +115,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
vllm_config = VllmConfig()
vllm_config.parallel_config.data_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.kernel_config.moe_backend = test_config.moe_backend
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
@@ -250,7 +252,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("intermediate_size", [256])
@pytest.mark.parametrize("num_tokens", [256])
@pytest.mark.parametrize("backend", ["latency", "throughput"])
@pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"])
def test_eplb_fml(
world_size: int,
num_layers: int,
@@ -258,12 +260,8 @@ def test_eplb_fml(
hidden_size: int,
intermediate_size: int,
num_tokens: int,
backend: str,
monkeypatch,
moe_backend: str,
):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", backend)
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs to run the test")
@@ -278,6 +276,7 @@ def test_eplb_fml(
hidden_size=hidden_size,
intermediate_size=intermediate_size,
num_tokens=num_tokens,
moe_backend=moe_backend,
)
distributed_run(
tests/entrypoints/pooling/reward/test_token_reward_offline.py
+2 -1
View File
@@ -45,9 +45,10 @@ def test_config(llm: LLM):
def test_pooling_params(llm: LLM):
def get_outputs(use_activation):
outputs = llm.reward(
outputs = llm.encode(
prompts,
pooling_params=PoolingParams(use_activation=use_activation),
pooling_task="token_classify",
use_tqdm=False,
)
return torch.cat([x.outputs.data for x in outputs])
tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16.yaml → tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml
+1 -3
View File
@@ -3,6 +3,4 @@
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: "1"
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass"
tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_trtllm"
tests/evals/gpt_oss/configs/gpt-oss-20b-marlin.yaml
+1 -3
View File
@@ -3,6 +3,4 @@
model_name: "openai/gpt-oss-20b"
metric_threshold: 0.568
reasoning_effort: "low"
server_args: "--tensor-parallel-size 2"
env:
VLLM_MXFP4_USE_MARLIN: "1"
server_args: "--tensor-parallel-size 2 --moe-backend marlin --linear-backend marlin"
tests/evals/gpt_oss/configs/models-b200.txt
+1 -1
View File
@@ -1,5 +1,5 @@
# B200 model configurations for GPQA evaluation
# Tests different environment variable combinations
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml
gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml
gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml
tests/evals/gpt_oss/configs/models-h100.txt
+1 -1
View File
@@ -1,5 +1,5 @@
# H100 model configurations for GPQA evaluation
# Tests different environment variable combinations
gpt-oss-20b-baseline.yaml
gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml
gpt-oss-20b-marlin.yaml
tests/evals/gsm8k/README.md
+2 -2
View File
@@ -30,9 +30,9 @@ model_name: "Qwen/Qwen2.5-1.5B-Instruct"
accuracy_threshold: 0.54 # Minimum expected accuracy
num_questions: 1319 # Number of questions (default: full test set)
num_fewshot: 5 # Few-shot examples from train set
server_args: "--max-model-len 4096 --tensor-parallel-size 2" # Server arguments
server_args: "--max-model-len 4096 --tensor-parallel-size 2 --moe-backend flashinfer_cutlass" # Server arguments
env: # Environment variables (optional)
VLLM_USE_FLASHINFER_MOE_FP4: "1"
VLLM_LOGGING_LEVEL: "DEBUG"
```
The `server_args` field accepts any arguments that can be passed to `vllm serve`.
tests/kernels/moe/test_moe.py
+1 -3
View File
@@ -1585,7 +1585,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
e: int,
topk: int,
dtype: torch.dtype,
monkeypatch,
workspace_init,
):
"""
@@ -1593,8 +1592,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
"""
set_random_seed(7)
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
@@ -1626,6 +1623,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
in_dtype=dtype,
routing_method=RoutingMethodType.Renormalize,
max_num_tokens=next_power_of_2(m),
moe_backend="flashinfer_trtllm",
)
with set_current_vllm_config(vllm_config):
tests/kernels/moe/test_moe_layer.py
+3 -6
View File
@@ -1804,12 +1804,9 @@ def test_moe_layer(
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
# TODO
# VLLM_FLASHINFER_MOE_BACKEND=latency
# VLLM_USE_FLASHINFER_MOE_FP16=1
# VLLM_USE_FLASHINFER_MOE_FP8
# VLLM_USE_FLASHINFER_MOE_FP4
# VLLM_USE_FLASHINFER_MOE_INT4
# TODO: cover FlashInfer MoE backends via moe_backend, e.g.
# moe_backend=flashinfer_trtllm / flashinfer_cutlass / flashinfer_cutedsl
# (BF16, FP8 and NVFP4 paths), and VLLM_USE_FLASHINFER_MOE_INT4=1.
parallel_config = ParallelConfig(
pipeline_parallel_size=1,
tests/kernels/moe/test_unquantized_backend_selection.py
+4 -41
View File
@@ -123,7 +123,7 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
)
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeypatch):
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
@@ -134,9 +134,8 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeyp
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True),
):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
moe_config = make_dummy_moe_config()
moe_config.moe_backend = "flashinfer_trtllm"
# TRTLLM requires EP and does not support DP
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False
@@ -168,7 +167,6 @@ def test_select_cuda_flashinfer_cutlass_backend(
mock_has_flashinfer,
mock_is_supported_trtllm,
mock_is_supported_cutlass,
monkeypatch,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
and FlashInfer CUTLASS is available."""
@@ -181,10 +179,9 @@ def test_select_cuda_flashinfer_cutlass_backend(
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True),
):
# Enable FlashInfer via env var
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
moe_config = make_dummy_moe_config()
# Select FlashInfer CUTLASS explicitly
moe_config.moe_backend = "flashinfer_cutlass"
# CUTLASS requires EP and does not support DP
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False
@@ -241,37 +238,3 @@ def test_select_explicit_triton_backend(is_lora_enabled):
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
@skipif_not_cuda_rocm
def test_select_explicit_triton_ignores_flashinfer_env(monkeypatch):
"""Explicit triton backend should override FlashInfer env selection."""
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
moe_config = make_dummy_moe_config()
moe_config.is_lora_enabled = False
moe_config.moe_backend = "triton"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
@skipif_not_cuda_rocm
def test_select_lora_ignores_flashinfer_env(monkeypatch):
"""LoRA path should still choose Triton even if FlashInfer env is on."""
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
moe_config = make_dummy_moe_config()
moe_config.is_lora_enabled = True
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
tests/lora/test_gptoss_tp.py
+36 -38
View File
@@ -83,57 +83,55 @@ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
@pytest.mark.parametrize("specialize_active_lora", [True, False])
def test_gpt_oss_lora(
monkeypatch: pytest.MonkeyPatch,
gptoss20b_lora_files,
mxfp4_use_marlin,
specialize_active_lora,
):
with monkeypatch.context() as m:
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
enable_lora=True,
max_loras=4,
max_lora_rank=8,
max_num_seqs=2,
max_num_batched_tokens=2048,
specialize_active_lora=specialize_active_lora,
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
cudagraph_specialize_lora=False,
),
)
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
enable_lora=True,
max_loras=4,
max_lora_rank=8,
max_num_seqs=2,
max_num_batched_tokens=2048,
specialize_active_lora=specialize_active_lora,
moe_backend="marlin" if mxfp4_use_marlin else "auto",
linear_backend="marlin" if mxfp4_use_marlin else "auto",
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
cudagraph_specialize_lora=False,
),
)
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("fully_sharded_loras", [False, True])
@pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
def test_gpt_oss_lora_tp2(
monkeypatch: pytest.MonkeyPatch,
gptoss20b_lora_files,
fully_sharded_loras,
mxfp4_use_marlin,
):
with monkeypatch.context() as m:
m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
enable_lora=True,
max_loras=2,
max_num_seqs=2,
max_num_batched_tokens=2048,
tensor_parallel_size=2,
gpu_memory_utilization=0.8,
fully_sharded_loras=fully_sharded_loras,
enable_expert_parallel=not fully_sharded_loras,
compilation_config=vllm.config.CompilationConfig(
cudagraph_specialize_lora=False,
),
)
llm = vllm.LLM(
MODEL_PATH,
max_model_len=1024,
enable_lora=True,
max_loras=2,
max_num_seqs=2,
max_num_batched_tokens=2048,
tensor_parallel_size=2,
gpu_memory_utilization=0.8,
fully_sharded_loras=fully_sharded_loras,
enable_expert_parallel=not fully_sharded_loras,
moe_backend="marlin" if mxfp4_use_marlin else "auto",
linear_backend="marlin" if mxfp4_use_marlin else "auto",
compilation_config=vllm.config.CompilationConfig(
cudagraph_specialize_lora=False,
),
)
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
tests/models/language/pooling/test_pooler_config_init_behaviour.py
+2 -2
View File
@@ -106,7 +106,7 @@ def test_reward_models_using_activation(
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False),
) as vllm_model:
wo_activation = vllm_model.reward(example_prompts)
wo_activation = vllm_model.token_classify(example_prompts)
with vllm_runner(
model,
@@ -114,7 +114,7 @@ def test_reward_models_using_activation(
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True),
) as vllm_model:
w_activation = vllm_model.reward(example_prompts)
w_activation = vllm_model.token_classify(example_prompts)
for wo, w in zip(wo_activation, w_activation):
wo = torch.tensor(wo)
tests/models/language/pooling/test_reward.py
+2 -2
View File
@@ -107,7 +107,7 @@ def test_prm_models(
pytest.skip("CPU only supports V1")
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.reward(math_step_prompts)
vllm_outputs = vllm_model.token_classify(math_step_prompts)
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
hf_model = step_reward_patch_hf_model(hf_model)
@@ -146,7 +146,7 @@ def test_prm_models_with_golden_outputs(
pytest.skip(f"No available golden outputs for {model}.")
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.reward(math_step_prompts)
vllm_outputs = vllm_model.token_classify(math_step_prompts)
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
tests/models/quantization/test_nvfp4.py
+2 -2
View File
@@ -133,11 +133,11 @@ def test_nvfp4(vllm_runner, model, eager, backend):
not current_platform.is_rocm(),
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
)
def test_nvfp4_moe(vllm_runner, model, backend, monkeypatch):
monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend)
def test_nvfp4_moe(vllm_runner, model, backend):
with vllm_runner(
model,
moe_backend=backend,
linear_backend=backend,
load_format="dummy",
hf_overrides={"num_hidden_layers": 2},
) as llm:
tests/quantization/test_blackwell_moe.py
+5 -2
View File
@@ -185,8 +185,11 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1")
can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
can_initialize(
"openai/gpt-oss-20b",
hf_overrides=HF_OVERRIDE_TEXT,
extra_args=["--moe-backend=flashinfer_trtllm"],
)
def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
vllm/benchmarks/datasets/datasets.py
+1 -7
View File
@@ -1617,7 +1617,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"custom",
"custom_audio",
"custom_image",
"custom_mm",
"prefix_repetition",
"spec_bench",
"speed_bench",
@@ -2106,12 +2105,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
no_oversample=args.no_oversample,
)
elif args.dataset_name in ("custom_image", "custom_mm"):
if args.dataset_name == "custom_mm":
logger.warning(
"Dataset name 'custom_mm' is deprecated and will be removed in v0.24. "
"Use '--dataset-name custom_image' instead."
)
elif args.dataset_name == "custom_image":
dataset = CustomImageDataset(
dataset_path=args.dataset_path,
disable_shuffle=args.disable_shuffle,
vllm/config/compilation.py
-11
View File
@@ -134,10 +134,6 @@ class PassConfig:
"""Enable async TP."""
fuse_allreduce_rms: bool = None # type: ignore[assignment]
"""Enable flashinfer allreduce fusion."""
fuse_minimax_qk_norm: bool = None # type: ignore[assignment]
"""Deprecated. The MiniMax QK norm fusion is now applied automatically at
runtime (see `MiniMaxText01RMSNormTP.forward_qkv`). This flag is kept for
backward compatibility and has no effect; it will be removed in v0.23."""
enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
"""Enable fused Q/K RMSNorm + RoPE pass."""
fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment]
@@ -296,13 +292,6 @@ class PassConfig:
"current platform is not CUDA or ROCm. The fusion will be disabled."
)
self.fuse_rope_kvcache_cat_mla = False
if self.fuse_minimax_qk_norm is not None:
logger.warning_once(
"`fuse_minimax_qk_norm` is deprecated and has no effect; "
"the MiniMax QK norm fusion is now applied automatically at "
"runtime when its conditions are met. This flag will be "
"removed in v0.23."
)
def log_enabled_passes(self) -> None:
"""
vllm/config/kernel.py
+2
View File
@@ -142,6 +142,7 @@ LinearBackend = Literal[
"flashinfer_cutlass",
"flashinfer_trtllm",
"flashinfer_cudnn",
"flashinfer_b12x",
"marlin",
"triton",
"deep_gemm",
@@ -197,6 +198,7 @@ class KernelConfig:
- "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
- "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
- "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
- "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+)
- "marlin": Use Marlin kernels
- "triton": Use Triton-based kernels
- "deep_gemm": Use DeepGEMM kernels
vllm/entrypoints/pooling/offline.py
-44
View File
@@ -286,50 +286,6 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
return [ClassificationRequestOutput.from_base(item) for item in items]
def reward(
self,
prompts: PromptType | Sequence[PromptType],
/,
*,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[PoolingRequestOutput]:
"""
Generate rewards for each prompt.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `PoolingRequestOutput` objects containing the
pooled hidden states in the same order as the input prompts.
"""
logger.warning_once(
"`llm.reward` api is deprecated and will be removed in v0.23. "
'Please use `LLM.encode` with `pooling_task="classify"` or '
'`pooling_task="token_classify"` instead.'
)
return self.encode(
prompts,
use_tqdm=use_tqdm,
lora_request=lora_request,
pooling_params=pooling_params,
pooling_task="token_classify",
tokenization_kwargs=tokenization_kwargs,
)
def score(
self,
data_1: ScoreInput | list[ScoreInput],
vllm/envs.py
-168
View File
@@ -8,7 +8,6 @@ import os
import sys
import tempfile
import uuid
import warnings
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Literal
@@ -167,7 +166,6 @@ if TYPE_CHECKING:
VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None
VLLM_HUMMING_USE_F16_ACCUM: bool = False
VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None
VLLM_MXFP4_USE_MARLIN: bool | None = None
VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
VLLM_V1_USE_OUTLINES_CACHE: bool = False
VLLM_TPU_BUCKET_PADDING_GAP: int = 0
@@ -184,13 +182,7 @@ if TYPE_CHECKING:
] = "relax"
VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
VLLM_USE_FLASHINFER_MOE_FP16: bool = False
VLLM_USE_FLASHINFER_MOE_FP8: bool = False
VLLM_USE_FLASHINFER_MOE_FP4: bool = False
VLLM_USE_FLASHINFER_MOE_INT4: bool = False
VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency", "masked_gemm"] = (
"latency"
)
VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
@@ -212,7 +204,6 @@ if TYPE_CHECKING:
VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
VLLM_USE_NVFP4_CT_EMULATIONS: bool = False
VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
"FP", "INT8", "INT6", "INT4", "NONE"
] = "NONE"
@@ -225,12 +216,8 @@ if TYPE_CHECKING:
VLLM_LOOPBACK_IP: str = ""
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
VLLM_NVFP4_GEMM_BACKEND: str | None = None
VLLM_HAS_FLASHINFER_CUBIN: bool = False
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
VLLM_TUNED_CONFIG_FOLDER: str | None = None
@@ -257,7 +244,6 @@ if TYPE_CHECKING:
VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
VLLM_USE_NCCL_SYMM_MEM: bool = False
VLLM_NCCL_INCLUDE_PATH: str | None = None
VLLM_USE_FBGEMM: bool = False
VLLM_GC_DEBUG: str = ""
VLLM_DEBUG_WORKSPACE: bool = False
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
@@ -350,27 +336,6 @@ def use_mega_aot_artifact():
return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1"
def deprecated_env(
env_name: str,
removal_version: str,
replacement: str,
getter: Callable[[], Any],
) -> Callable[[], Any]:
"""Wrap an env-var getter to emit a FutureWarning when the var is set."""
def _read() -> Any:
if env_name in os.environ:
warnings.warn(
f"{env_name} is deprecated and will be removed in "
f"{removal_version}. {replacement}",
FutureWarning,
stacklevel=2,
)
return getter()
return _read
def env_with_choices(
env_name: str,
default: str | None,
@@ -1371,15 +1336,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MARLIN_USE_ATOMIC_ADD": lambda: (
os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1"
),
# Whether to use marlin kernel in mxfp4 quantization method
# Deprecated: use --moe-backend marlin (MoE) or --linear-backend marlin
# (linear) instead.
"VLLM_MXFP4_USE_MARLIN": deprecated_env(
"VLLM_MXFP4_USE_MARLIN",
"v0.23",
"Use --moe-backend marlin or --linear-backend marlin.",
lambda: maybe_convert_bool(os.environ.get("VLLM_MXFP4_USE_MARLIN", None)),
),
# The activation dtype for marlin kernel
"VLLM_MARLIN_INPUT_DTYPE": env_with_choices(
"VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"]
@@ -1472,68 +1428,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1"))
),
# Allow use of FlashInfer BF16 MoE kernels for fused moe ops.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_FP16": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_FP16",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP16", "0"))),
),
# Allow use of FlashInfer FP8 MoE kernels for fused moe ops.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_FP8": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_FP8",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP8", "0"))),
),
# Allow use of FlashInfer NVFP4 MoE kernels for fused moe ops.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_FP4": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_FP4",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass, "
"flashinfer_cutedsl).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP4", "0"))),
),
# Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops.
"VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool(
int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0"))
),
# If set to 1, use the FlashInfer
# MXFP8 (activation) x MXFP4 (weight) MoE backend.
# Deprecated: use --moe-backend flashinfer_trtllm combined with
# --quantization_config.moe.activation mxfp8.
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8",
"v0.23",
"Use --moe-backend flashinfer_trtllm with "
"--quantization_config.moe.activation mxfp8.",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))),
),
# If set to 1, use the FlashInfer CUTLASS backend for
# MXFP8 (activation) x MXFP4 (weight) MoE.
# Deprecated: use --moe-backend flashinfer_cutlass combined with
# --quantization_config.moe.activation mxfp8.
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS",
"v0.23",
"Use --moe-backend flashinfer_cutlass with "
"--quantization_config.moe.activation mxfp8.",
lambda: bool(
int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0"))
),
),
# If set to 1, use the FlashInfer
# BF16 (activation) x MXFP4 (weight) MoE backend.
# Deprecated: use --moe-backend to select a kernel explicitly.
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": deprecated_env(
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16",
"v0.23",
"Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "0"))),
),
# Control the cache sized used by the xgrammar compiler. The default
# of 512 MB should be enough for roughly 1000 JSON schemas.
# It can be changed with this variable if needed for some reason.
@@ -1585,25 +1483,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
),
# Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support.
# Both require compute capability 10.0 or above.
# Available options:
# - "throughput": [default]
# Uses CUTLASS kernels optimized for high-throughput batch inference.
# - "latency":
# Uses TensorRT-LLM kernels optimized for low-latency inference.
# Deprecated: pass --moe-backend flashinfer_{trtllm,cutlass,cutedsl} directly.
"VLLM_FLASHINFER_MOE_BACKEND": deprecated_env(
"VLLM_FLASHINFER_MOE_BACKEND",
"v0.23",
"Use --moe-backend flashinfer_trtllm, flashinfer_cutlass, or "
"flashinfer_cutedsl.",
env_with_choices(
"VLLM_FLASHINFER_MOE_BACKEND",
"latency",
["throughput", "latency", "masked_gemm"],
),
),
# Override the directory for the FlashInfer autotune config cache.
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
"VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
@@ -1681,16 +1560,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
),
# Controls whether or not emulations are used for NVFP4
# generations on machines < 100 for compressed-tensors
# models
# Deprecated: use --linear-backend emulation instead.
"VLLM_USE_NVFP4_CT_EMULATIONS": deprecated_env(
"VLLM_USE_NVFP4_CT_EMULATIONS",
"v0.23",
"Use --linear-backend emulation.",
lambda: bool(int(os.getenv("VLLM_USE_NVFP4_CT_EMULATIONS", "0"))),
),
# Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
"VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
@@ -1700,35 +1569,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0"))
),
# Supported options:
# - "flashinfer-cudnn": use flashinfer cudnn GEMM backend
# - "flashinfer-trtllm": use flashinfer trtllm GEMM backend
# - "flashinfer-cutlass": use flashinfer cutlass GEMM backend
# - "marlin": use marlin GEMM backend (for GPUs without native FP4 support)
# - "emulation":
# use BF16/FP16 GEMM, dequantizing weights and running QDQ on activations.
# This is only meant for research purposes to run on devices where NVFP4
# GEMM kernels are not available.
# - <none>: automatically pick an available backend
# Deprecated: use --linear-backend instead.
"VLLM_NVFP4_GEMM_BACKEND": deprecated_env(
"VLLM_NVFP4_GEMM_BACKEND",
"v0.23",
"Use --linear-backend.",
env_with_choices(
"VLLM_NVFP4_GEMM_BACKEND",
None,
[
"flashinfer-b12x",
"flashinfer-cudnn",
"flashinfer-trtllm",
"flashinfer-cutlass",
"cutlass",
"marlin",
"emulation",
],
),
),
# Controls garbage collection during CUDA graph capture.
# If set to 0 (default), enables GC freezing to speed up capture time.
# If set to 1, allows GC to run during capture.
@@ -1892,14 +1732,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
),
# NCCL header path
"VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None),
# Flag to enable FBGemm kernels on model execution
# Deprecated: use --linear-backend fbgemm instead.
"VLLM_USE_FBGEMM": deprecated_env(
"VLLM_USE_FBGEMM",
"v0.23",
"Use --linear-backend fbgemm.",
lambda: bool(int(os.getenv("VLLM_USE_FBGEMM", "0"))),
),
# GC debug config
# - VLLM_GC_DEBUG=0: disable GC debugger
# - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times
vllm/model_executor/kernels/linear/__init__.py
+8 -47
View File
@@ -212,6 +212,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"flashinfer_cudnn": {
FlashInferCudnnNvFp4LinearKernel,
},
"flashinfer_b12x": {
FlashInferB12xNvFp4LinearKernel,
},
"marlin": {
MarlinFP8ScaledMMLinearKernel,
MarlinLinearKernel,
@@ -392,7 +395,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
PlatformEnum.CUDA: [
# FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
# upstream CUTLASS SM121 MMA op guard is resolved; use
# VLLM_NVFP4_GEMM_BACKEND=flashinfer-b12x to opt in explicitly.
# --linear-backend flashinfer_b12x to opt in explicitly.
FlashInferCutlassNvFp4LinearKernel,
CutlassNvFp4LinearKernel,
MarlinNvFp4LinearKernel,
@@ -752,20 +755,6 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
current platform."""
linear_backend = _get_linear_backend()
force_kernel: type[MxFp4LinearKernel] | None = None
if linear_backend == "auto" and envs.VLLM_MXFP4_USE_MARLIN:
force_kernel = MarlinMxFp4LinearKernel
if force_kernel is not None:
is_supported, reason = force_kernel.is_supported()
if not is_supported:
raise ValueError(
f"Forced MXFP4 kernel {force_kernel.__name__} is not "
f"supported: {reason}"
)
logger.info_once("Using %s for MXFP4 GEMM", force_kernel.__name__)
return force_kernel(MxFp4LinearLayerConfig())
platform = current_platform._enum
possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
@@ -836,18 +825,6 @@ def init_wfp8_a16_linear_kernel(
)
# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes.
_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = {
"flashinfer-b12x": FlashInferB12xNvFp4LinearKernel,
"flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel,
"cutlass": CutlassNvFp4LinearKernel,
"marlin": MarlinNvFp4LinearKernel,
"flashinfer-trtllm": FlashInferTrtllmNvFp4LinearKernel,
"flashinfer-cudnn": FlashInferCudnnNvFp4LinearKernel,
"emulation": EmulationNvFp4LinearKernel,
}
def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
"""Select and instantiate the best NVFP4 linear kernel for the
current platform."""
@@ -855,8 +832,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
# batch-invariant CUTLASS implementation when available, otherwise fall
# back to emulation. It overrides both --linear-backend and the deprecated
# env vars below.
# back to emulation. It overrides --linear-backend.
force_kernel: type[NvFp4LinearKernel] | None = None
linear_backend = _get_linear_backend()
if envs.VLLM_BATCH_INVARIANT:
@@ -888,24 +864,9 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
reason,
)
force_kernel = EmulationNvFp4LinearKernel
elif linear_backend == "auto":
# Deprecated env-var overrides — only honoured when --linear-backend
# is "auto". Deprecation warnings are emitted from vllm/envs.py.
if use_a16: # force a16 if running weight-only quantization
force_kernel = MarlinNvFp4LinearKernel
elif envs.VLLM_USE_FBGEMM:
force_kernel = FbgemmNvFp4LinearKernel
elif envs.VLLM_USE_NVFP4_CT_EMULATIONS:
force_kernel = EmulationNvFp4LinearKernel
elif envs.VLLM_NVFP4_GEMM_BACKEND is not None:
backend_name = envs.VLLM_NVFP4_GEMM_BACKEND
force_kernel = _NVFP4_BACKEND_TO_KERNEL.get(backend_name)
if force_kernel is None:
raise ValueError(
f"Unknown VLLM_NVFP4_GEMM_BACKEND={backend_name!r}. "
f"Valid choices: "
f"{list(_NVFP4_BACKEND_TO_KERNEL.keys())}"
)
elif linear_backend == "auto" and use_a16:
# Force a16 (Marlin) when running weight-only quantization.
force_kernel = MarlinNvFp4LinearKernel
if force_kernel is not None:
is_supported, reason = force_kernel.is_supported()
vllm/model_executor/layers/fused_moe/oracle/fp8.py
-50
View File
@@ -20,8 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
get_flashinfer_moe_backend,
prepare_fp8_moe_layer_for_fi,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
@@ -321,54 +319,6 @@ def select_fp8_moe_backend(
requested_backend, config, weight_key, activation_key, activation_format
)
# Handle explicit FlashInfer FP8 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP8"):
if not envs.VLLM_USE_FLASHINFER_MOE_FP8:
# If the user rejects FlashInfer remove those backends.
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_TRTLLM)
AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_CUTLASS)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
fi_backend = get_flashinfer_moe_backend()
if fi_backend == FlashinferMoeBackend.CUTLASS:
backend = Fp8MoeBackend.FLASHINFER_CUTLASS
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
else:
raise ValueError(
f"FlashInfer MOE backend {fi_backend} does not support FP8 MoE."
)
k_cls = backend_to_kernel_cls(backend)[0]
return _return_or_raise(
backend, config, weight_key, activation_key, activation_format
)
else:
# If the user is not explicit about the backend, try both.
for backend in [
Fp8MoeBackend.FLASHINFER_TRTLLM,
Fp8MoeBackend.FLASHINFER_CUTLASS,
]:
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls,
config,
weight_key,
activation_key,
activation_format,
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"Found VLLM_USE_FLASHINFER_MOE_FP8=1, but no "
"FlashInfer FP8 MoE backend supports the configuration."
)
# Handle explicit DeepGEMM FP8 configuration.
if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"):
if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM:
vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-69
View File
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Literal, Union
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend
from vllm.config.quantization import QuantizationConfigArgs
@@ -465,74 +464,6 @@ def select_mxfp4_moe_backend(
_get_priority_backends_for_gpt_oss(), requested_activation_key
)
# Handle explicit FlashInfer MXFP4 BF16 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
for _b in (
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
):
if _b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(_b)
else:
if current_platform.is_device_capability(90):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
config,
kMxfp4Static,
None,
activation_format,
)
if current_platform.is_device_capability_family(100):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
config,
kMxfp4Static,
None,
activation_format,
)
raise ValueError(
"VLLM_USE_FLASHINFER_MOE_MXFP4_BF16=1 is set but the "
"current device capability is not supported. "
"Only SM90 (CUTLASS) and SM100+ (TRTLLM) are supported."
)
# Handle explicit FlashInfer MXFP4 MXFP8 TRTLLM configuration.
if (
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8")
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8
):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
config,
kMxfp4Static,
kMxfp8Dynamic,
activation_format,
)
# Handle explicit FlashInfer MXFP4 MXFP8 CUTLASS configuration.
if (
envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS")
and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS
):
return _return_or_raise(
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
config,
kMxfp4Static,
kMxfp8Dynamic,
activation_format,
)
# Handle explicit Marlin MXFP4 configuration.
if envs.is_set("VLLM_MXFP4_USE_MARLIN") and envs.VLLM_MXFP4_USE_MARLIN:
return _return_or_raise(
Mxfp4MoeBackend.MARLIN,
config,
kMxfp4Static,
None,
activation_format,
)
for backend in AVAILABLE_BACKENDS:
# Use requested_activation_key if provided, otherwise use backend default
act_key = (
vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
-59
View File
@@ -22,10 +22,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
get_flashinfer_moe_backend,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_nvfp4_moe_layer_for_marlin,
)
@@ -58,12 +54,6 @@ FLASHINFER_NVFP4_MOE_BACKENDS = [
NvFp4MoeBackend.FLASHINFER_B12X,
]
fi_2_vllm_backend_map: dict[FlashinferMoeBackend, NvFp4MoeBackend] = {
FlashinferMoeBackend.CUTLASS: NvFp4MoeBackend.FLASHINFER_CUTLASS,
FlashinferMoeBackend.TENSORRT_LLM: NvFp4MoeBackend.FLASHINFER_TRTLLM,
FlashinferMoeBackend.CUTEDSL: NvFp4MoeBackend.FLASHINFER_CUTEDSL,
}
def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool:
# Checks whether `backend` supports quantizing with scaling factors
@@ -258,55 +248,6 @@ def select_nvfp4_moe_backend(
requested_backend, config, weight_key, activation_key, activation_format
)
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP4"):
if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
# If the user rejects FlashInfer remove those backends.
for b in FLASHINFER_NVFP4_MOE_BACKENDS:
if b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(b)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
if (
config.swiglu_limit is not None
and backend not in NVFP4_BACKENDS_WITH_CLAMP
):
raise ValueError(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
f"({backend.value}) does not apply the SwiGLU clamp."
)
return _return_or_raise(
backend, config, weight_key, activation_key, activation_format
)
else:
# If the user is not explicit about the backend, try each.
fi_backends = [
b
for b in FLASHINFER_NVFP4_MOE_BACKENDS
if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
]
for backend in fi_backends:
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls,
config,
weight_key,
activation_key,
activation_format,
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"Found VLLM_USE_FLASHINFER_MOE_FP4=1, but no "
"FlashInfer NVFP4 MoE backend supports the configuration."
)
if envs.VLLM_TEST_FORCE_FP8_MARLIN:
backend = NvFp4MoeBackend.MARLIN
return _return_or_raise(
vllm/model_executor/layers/fused_moe/oracle/unquantized.py
-45
View File
@@ -19,9 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
FlashinferMoeBackend,
convert_moe_weights_to_flashinfer_trtllm_block_layout,
get_flashinfer_moe_backend,
swap_w13_to_w31,
)
from vllm.platforms import current_platform
@@ -230,49 +228,6 @@ def select_unquantized_moe_backend(
return _return_or_raise(requested_backend, moe_config, activation_format)
# Handle explicit FlashInfer FP16 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP16"):
if not envs.VLLM_USE_FLASHINFER_MOE_FP16:
if UnquantizedMoeBackend.FLASHINFER_TRTLLM in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_TRTLLM)
if UnquantizedMoeBackend.FLASHINFER_CUTLASS in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_CUTLASS)
elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
# If user is explicit about backend, validate it.
fi_backend = get_flashinfer_moe_backend()
if fi_backend == FlashinferMoeBackend.CUTLASS:
backend = UnquantizedMoeBackend.FLASHINFER_CUTLASS
elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
else:
raise ValueError(
f"FlashInfer MOE backend {fi_backend} "
"does not support unquantized MoE."
)
k_cls = backend_to_kernel_cls(backend)
return _return_or_raise(backend, moe_config, activation_format)
else:
# If the user is not explicit about the backend, try both.
for backend in [
UnquantizedMoeBackend.FLASHINFER_TRTLLM,
UnquantizedMoeBackend.FLASHINFER_CUTLASS,
]:
k_cls = backend_to_kernel_cls(backend)
supported, reason = k_cls.is_supported_config(
k_cls, moe_config, None, None, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
else:
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"Found VLLM_USE_FLASHINFER_MOE_FP16=1, but no "
"FlashInfer unquantized MoE backend supports the configuration."
)
# Handle explicit AITER FP8 configuration.
if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py
+5 -4
View File
@@ -6,7 +6,7 @@ import nixl_ep
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config
from vllm.distributed import get_ep_group
from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
from vllm.logger import init_logger
@@ -192,10 +192,11 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
x = x.view((-1, hidden_dim))
q_dtype = quant_config.quant_dtype
if envs.VLLM_FLASHINFER_MOE_BACKEND == "masked_gemm":
moe_backend = get_current_vllm_config().kernel_config.moe_backend
if moe_backend == "flashinfer_cutedsl":
logger.info_once(
"Skip quantization when using FlashInfer CUTEDSL(masked_gemm) "
"for ModelOptNvFp4FusedMoE."
"Skip quantization when using FlashInfer CUTEDSL "
"(--moe-backend flashinfer_cutedsl) for ModelOptNvFp4FusedMoE."
)
q_dtype = None
vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py
-15
View File
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
import torch
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_weights_for_fi,
@@ -15,10 +14,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
swizzle_blockscale,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
has_flashinfer_cutlass_fused_moe,
)
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -34,16 +29,6 @@ __all__ = [
]
def is_flashinfer_fp4_cutlass_moe_available() -> bool:
"""Return `True` when FlashInfer CUTLASS NV-FP4 kernels can be used."""
return (
envs.VLLM_USE_FLASHINFER_MOE_FP4
and has_flashinfer_cutlass_fused_moe()
and current_platform.is_cuda()
and current_platform.has_device_capability(100)
)
def reorder_w1w3_to_w3w1(
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
) -> tuple[torch.Tensor, torch.Tensor]:
vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
-30
View File
@@ -5,10 +5,8 @@ from typing import TYPE_CHECKING
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
if TYPE_CHECKING:
@@ -95,34 +93,6 @@ def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
)
def get_flashinfer_moe_backend() -> FlashinferMoeBackend:
backend_map = {
"throughput": FlashinferMoeBackend.CUTLASS,
"latency": FlashinferMoeBackend.TENSORRT_LLM,
"masked_gemm": FlashinferMoeBackend.CUTEDSL,
}
flashinfer_moe_backend = envs.VLLM_FLASHINFER_MOE_BACKEND
if flashinfer_moe_backend in backend_map:
if (
flashinfer_moe_backend == "latency"
and not current_platform.is_device_capability_family(100)
):
logger.info_once(
"Flashinfer TRTLLM MOE backend is only supported on "
"SM100 and later, using CUTLASS backend instead",
)
return FlashinferMoeBackend.CUTLASS
return backend_map[flashinfer_moe_backend]
elif current_platform.is_device_capability(90):
return FlashinferMoeBackend.CUTLASS
raise ValueError(
f"Unknown flashinfer moe backend: {flashinfer_moe_backend!r}. "
f"Expected one of {list(backend_map.keys())}."
)
def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool:
# TODO(shuw@nvidia): Update when new backends are added.
backends_supporting_global_sf = (