diff --git a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml index a87328fcdcc..164733cca6f 100644 --- a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml +++ b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml @@ -6,9 +6,7 @@ tasks: value: 0.7142 - name: "exact_match,flexible-extract" value: 0.4579 -env_vars: - VLLM_USE_FLASHINFER_MOE_FP8: "1" - VLLM_FLASHINFER_MOE_BACKEND: "throughput" +moe_backend: "flashinfer_cutlass" limit: 1319 num_fewshot: 5 max_model_len: 262144 diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py index d34e603b9e2..dd2fd5f05b4 100644 --- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py +++ b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py @@ -68,6 +68,10 @@ def launch_lm_eval(eval_config, tp_size): if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]: model_args += "attention_backend=TRITON_ATTN" + moe_backend = eval_config.get("moe_backend", None) + if moe_backend is not None: + model_args += f"moe_backend={moe_backend}," + env_vars = eval_config.get("env_vars", None) with scoped_env_vars(env_vars): results = lm_eval.simple_evaluate( diff --git a/docs/design/moe_kernel_features.md b/docs/design/moe_kernel_features.md index af7da63b550..279ab2d0d6f 100644 --- a/docs/design/moe_kernel_features.md +++ b/docs/design/moe_kernel_features.md @@ -42,7 +42,7 @@ th { 1. All types: mxfp4, nvfp4, int4, int8, fp8 2. A,T quantization occurs after dispatch. 3. All quantization happens after dispatch. - 4. Controlled by different env vars (`VLLM_FLASHINFER_MOE_BACKEND` "throughput" or "latency") + 4. Controlled by `--moe-backend` (`flashinfer_cutlass` or `flashinfer_trtllm`) 5. This is a no-op dispatcher that can be used to pair with any modular experts to produce a modular kernel that runs without dispatch or combine. These cannot be selected via environment variable. These are generally use for testing or adapting an expert subclass to the `fused_experts` API. 6. This depends on the experts implementation. diff --git a/docs/models/pooling_models/reward.md b/docs/models/pooling_models/reward.md index 4acacda5004..6049eb0a5f9 100644 --- a/docs/models/pooling_models/reward.md +++ b/docs/models/pooling_models/reward.md @@ -143,4 +143,4 @@ More examples can be found here: [examples/pooling/reward](../../../examples/poo ### `LLM.reward` -`llm.reward` api is deprecated and will be removed in v0.23. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead. +`llm.reward` API is deprecated and was removed in v0.24. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead. diff --git a/tests/compile/correctness_e2e/test_async_tp.py b/tests/compile/correctness_e2e/test_async_tp.py index 28c7eb6fbc2..e2d597bc7a3 100644 --- a/tests/compile/correctness_e2e/test_async_tp.py +++ b/tests/compile/correctness_e2e/test_async_tp.py @@ -102,7 +102,7 @@ def test_async_tp_pass_correctness( @create_new_process_for_each_test() -def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch): +def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int): if ( not current_platform.is_cuda() or not current_platform.is_device_capability_family(100) @@ -111,8 +111,6 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch): if not has_flashinfer(): pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path") - monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", "flashinfer-cutlass") - tp_size = 2 if num_gpus_available < tp_size: pytest.skip(f"Need at least {tp_size} GPUs") @@ -126,6 +124,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch): "8", "--load-format", "dummy", + "--linear-backend", + "flashinfer_cutlass", "--hf-overrides", json.dumps(NVFP4_HF_OVERRIDES), ] diff --git a/tests/conftest.py b/tests/conftest.py index ebf9608b01f..5db457e2939 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1226,10 +1226,6 @@ class VllmRunner: req_outputs = self.llm.encode(prompts, pooling_task="token_classify") return [req_output.outputs.data for req_output in req_outputs] - def reward(self, prompts: list[str]) -> list[list[float]]: - req_outputs = self.llm.encode(prompts, pooling_task="token_classify") - return [req_output.outputs.data for req_output in req_outputs] - def score( self, text_1: list[str] | str, diff --git a/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py b/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py index baaa112a4e6..551811e60e8 100644 --- a/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py +++ b/tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py @@ -37,6 +37,7 @@ class TestConfig: hidden_size: int intermediate_size: int num_tokens: int + moe_backend: str def make_fused_moe_layer( @@ -114,6 +115,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig): vllm_config = VllmConfig() vllm_config.parallel_config.data_parallel_size = world_size vllm_config.parallel_config.enable_expert_parallel = True + vllm_config.kernel_config.moe_backend = test_config.moe_backend with set_current_vllm_config(vllm_config): ensure_model_parallel_initialized( @@ -250,7 +252,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig): @pytest.mark.parametrize("hidden_size", [256]) @pytest.mark.parametrize("intermediate_size", [256]) @pytest.mark.parametrize("num_tokens", [256]) -@pytest.mark.parametrize("backend", ["latency", "throughput"]) +@pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"]) def test_eplb_fml( world_size: int, num_layers: int, @@ -258,12 +260,8 @@ def test_eplb_fml( hidden_size: int, intermediate_size: int, num_tokens: int, - backend: str, - monkeypatch, + moe_backend: str, ): - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1") - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", backend) - if torch.accelerator.device_count() < world_size: pytest.skip(f"Need at least {world_size} GPUs to run the test") @@ -278,6 +276,7 @@ def test_eplb_fml( hidden_size=hidden_size, intermediate_size=intermediate_size, num_tokens=num_tokens, + moe_backend=moe_backend, ) distributed_run( diff --git a/tests/entrypoints/pooling/reward/test_token_reward_offline.py b/tests/entrypoints/pooling/reward/test_token_reward_offline.py index b061b551451..50a4b54682b 100644 --- a/tests/entrypoints/pooling/reward/test_token_reward_offline.py +++ b/tests/entrypoints/pooling/reward/test_token_reward_offline.py @@ -45,9 +45,10 @@ def test_config(llm: LLM): def test_pooling_params(llm: LLM): def get_outputs(use_activation): - outputs = llm.reward( + outputs = llm.encode( prompts, pooling_params=PoolingParams(use_activation=use_activation), + pooling_task="token_classify", use_tqdm=False, ) return torch.cat([x.outputs.data for x in outputs]) diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml similarity index 68% rename from tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16.yaml rename to tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml index 952f7e87035..992cb3dfa49 100644 --- a/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16.yaml +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml @@ -3,6 +3,4 @@ model_name: "openai/gpt-oss-20b" metric_threshold: 0.568 reasoning_effort: "low" -server_args: "--tensor-parallel-size 2" -env: - VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: "1" +server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass" diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml new file mode 100644 index 00000000000..39b68930858 --- /dev/null +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +model_name: "openai/gpt-oss-20b" +metric_threshold: 0.568 +reasoning_effort: "low" +server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_trtllm" diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-marlin.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-marlin.yaml index 97e97fd19a6..99f10f4f31c 100644 --- a/tests/evals/gpt_oss/configs/gpt-oss-20b-marlin.yaml +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-marlin.yaml @@ -3,6 +3,4 @@ model_name: "openai/gpt-oss-20b" metric_threshold: 0.568 reasoning_effort: "low" -server_args: "--tensor-parallel-size 2" -env: - VLLM_MXFP4_USE_MARLIN: "1" +server_args: "--tensor-parallel-size 2 --moe-backend marlin --linear-backend marlin" diff --git a/tests/evals/gpt_oss/configs/models-b200.txt b/tests/evals/gpt_oss/configs/models-b200.txt index 8519109e192..4a7e80949ac 100644 --- a/tests/evals/gpt_oss/configs/models-b200.txt +++ b/tests/evals/gpt_oss/configs/models-b200.txt @@ -1,5 +1,5 @@ # B200 model configurations for GPQA evaluation # Tests different environment variable combinations -gpt-oss-20b-flashinfer-mxfp4-bf16.yaml +gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml \ No newline at end of file diff --git a/tests/evals/gpt_oss/configs/models-h100.txt b/tests/evals/gpt_oss/configs/models-h100.txt index 9577bac5f1d..05a35fdd8f1 100644 --- a/tests/evals/gpt_oss/configs/models-h100.txt +++ b/tests/evals/gpt_oss/configs/models-h100.txt @@ -1,5 +1,5 @@ # H100 model configurations for GPQA evaluation # Tests different environment variable combinations gpt-oss-20b-baseline.yaml -gpt-oss-20b-flashinfer-mxfp4-bf16.yaml +gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml gpt-oss-20b-marlin.yaml diff --git a/tests/evals/gsm8k/README.md b/tests/evals/gsm8k/README.md index dcbfd85bfee..db37d2e2243 100644 --- a/tests/evals/gsm8k/README.md +++ b/tests/evals/gsm8k/README.md @@ -30,9 +30,9 @@ model_name: "Qwen/Qwen2.5-1.5B-Instruct" accuracy_threshold: 0.54 # Minimum expected accuracy num_questions: 1319 # Number of questions (default: full test set) num_fewshot: 5 # Few-shot examples from train set -server_args: "--max-model-len 4096 --tensor-parallel-size 2" # Server arguments +server_args: "--max-model-len 4096 --tensor-parallel-size 2 --moe-backend flashinfer_cutlass" # Server arguments env: # Environment variables (optional) - VLLM_USE_FLASHINFER_MOE_FP4: "1" + VLLM_LOGGING_LEVEL: "DEBUG" ``` The `server_args` field accepts any arguments that can be passed to `vllm serve`. diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index 9a317d40fb2..45cd17b3b11 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -1585,7 +1585,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend( e: int, topk: int, dtype: torch.dtype, - monkeypatch, workspace_init, ): """ @@ -1593,8 +1592,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend( """ set_random_seed(7) - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1") - from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEParallelConfig, @@ -1626,6 +1623,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend( in_dtype=dtype, routing_method=RoutingMethodType.Renormalize, max_num_tokens=next_power_of_2(m), + moe_backend="flashinfer_trtllm", ) with set_current_vllm_config(vllm_config): diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index 5935c75a74f..d1bcd3241aa 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -1804,12 +1804,9 @@ def test_moe_layer( if os.environ.get("VLLM_LOGGING_LEVEL") is None: monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR") - # TODO - # VLLM_FLASHINFER_MOE_BACKEND=latency - # VLLM_USE_FLASHINFER_MOE_FP16=1 - # VLLM_USE_FLASHINFER_MOE_FP8 - # VLLM_USE_FLASHINFER_MOE_FP4 - # VLLM_USE_FLASHINFER_MOE_INT4 + # TODO: cover FlashInfer MoE backends via moe_backend, e.g. + # moe_backend=flashinfer_trtllm / flashinfer_cutlass / flashinfer_cutedsl + # (BF16, FP8 and NVFP4 paths), and VLLM_USE_FLASHINFER_MOE_INT4=1. parallel_config = ParallelConfig( pipeline_parallel_size=1, diff --git a/tests/kernels/moe/test_unquantized_backend_selection.py b/tests/kernels/moe/test_unquantized_backend_selection.py index bc322aed390..9e1afbbdff4 100644 --- a/tests/kernels/moe/test_unquantized_backend_selection.py +++ b/tests/kernels/moe/test_unquantized_backend_selection.py @@ -123,7 +123,7 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer): @pytest.mark.skipif( not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms." ) -def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeypatch): +def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm): """Test CUDA backend selection when FlashInfer TRTLLM is available and enabled.""" with ( patch.object(current_platform, "is_cuda", return_value=True), @@ -134,9 +134,8 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeyp patch.object(current_platform, "is_out_of_tree", return_value=False), patch.object(current_platform, "has_device_capability", return_value=True), ): - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1") - moe_config = make_dummy_moe_config() + moe_config.moe_backend = "flashinfer_trtllm" # TRTLLM requires EP and does not support DP moe_config.moe_parallel_config.use_ep = True moe_config.moe_parallel_config.use_dp = False @@ -168,7 +167,6 @@ def test_select_cuda_flashinfer_cutlass_backend( mock_has_flashinfer, mock_is_supported_trtllm, mock_is_supported_cutlass, - monkeypatch, ): """Test CUDA backend selection when FlashInfer TRTLLM is not available and FlashInfer CUTLASS is available.""" @@ -181,10 +179,9 @@ def test_select_cuda_flashinfer_cutlass_backend( patch.object(current_platform, "is_out_of_tree", return_value=False), patch.object(current_platform, "has_device_capability", return_value=True), ): - # Enable FlashInfer via env var - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1") - moe_config = make_dummy_moe_config() + # Select FlashInfer CUTLASS explicitly + moe_config.moe_backend = "flashinfer_cutlass" # CUTLASS requires EP and does not support DP moe_config.moe_parallel_config.use_ep = True moe_config.moe_parallel_config.use_dp = False @@ -241,37 +238,3 @@ def test_select_explicit_triton_backend(is_lora_enabled): assert selected_backend == UnquantizedMoeBackend.TRITON assert experts_cls is not None - - -@skipif_not_cuda_rocm -def test_select_explicit_triton_ignores_flashinfer_env(monkeypatch): - """Explicit triton backend should override FlashInfer env selection.""" - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1") - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput") - - moe_config = make_dummy_moe_config() - moe_config.is_lora_enabled = False - moe_config.moe_backend = "triton" - - selected_backend, experts_cls = select_unquantized_moe_backend( - moe_config=moe_config - ) - - assert selected_backend == UnquantizedMoeBackend.TRITON - assert experts_cls is not None - - -@skipif_not_cuda_rocm -def test_select_lora_ignores_flashinfer_env(monkeypatch): - """LoRA path should still choose Triton even if FlashInfer env is on.""" - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1") - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput") - - moe_config = make_dummy_moe_config() - moe_config.is_lora_enabled = True - selected_backend, experts_cls = select_unquantized_moe_backend( - moe_config=moe_config - ) - - assert selected_backend == UnquantizedMoeBackend.TRITON - assert experts_cls is not None diff --git a/tests/lora/test_gptoss_tp.py b/tests/lora/test_gptoss_tp.py index 70129671f0d..7aa8643cd9c 100644 --- a/tests/lora/test_gptoss_tp.py +++ b/tests/lora/test_gptoss_tp.py @@ -83,57 +83,55 @@ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None: @pytest.mark.parametrize("mxfp4_use_marlin", [True, False]) @pytest.mark.parametrize("specialize_active_lora", [True, False]) def test_gpt_oss_lora( - monkeypatch: pytest.MonkeyPatch, gptoss20b_lora_files, mxfp4_use_marlin, specialize_active_lora, ): - with monkeypatch.context() as m: - m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0") - llm = vllm.LLM( - MODEL_PATH, - max_model_len=1024, - enable_lora=True, - max_loras=4, - max_lora_rank=8, - max_num_seqs=2, - max_num_batched_tokens=2048, - specialize_active_lora=specialize_active_lora, - compilation_config=vllm.config.CompilationConfig( # Avoid OOM - cudagraph_specialize_lora=False, - ), - ) + llm = vllm.LLM( + MODEL_PATH, + max_model_len=1024, + enable_lora=True, + max_loras=4, + max_lora_rank=8, + max_num_seqs=2, + max_num_batched_tokens=2048, + specialize_active_lora=specialize_active_lora, + moe_backend="marlin" if mxfp4_use_marlin else "auto", + linear_backend="marlin" if mxfp4_use_marlin else "auto", + compilation_config=vllm.config.CompilationConfig( # Avoid OOM + cudagraph_specialize_lora=False, + ), + ) - generate_and_test(llm, gptoss20b_lora_files, lora_id=1) - generate_and_test(llm, gptoss20b_lora_files, lora_id=2) + generate_and_test(llm, gptoss20b_lora_files, lora_id=1) + generate_and_test(llm, gptoss20b_lora_files, lora_id=2) @multi_gpu_test(num_gpus=2) @pytest.mark.parametrize("fully_sharded_loras", [False, True]) @pytest.mark.parametrize("mxfp4_use_marlin", [True, False]) def test_gpt_oss_lora_tp2( - monkeypatch: pytest.MonkeyPatch, gptoss20b_lora_files, fully_sharded_loras, mxfp4_use_marlin, ): - with monkeypatch.context() as m: - m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0") - llm = vllm.LLM( - MODEL_PATH, - max_model_len=1024, - enable_lora=True, - max_loras=2, - max_num_seqs=2, - max_num_batched_tokens=2048, - tensor_parallel_size=2, - gpu_memory_utilization=0.8, - fully_sharded_loras=fully_sharded_loras, - enable_expert_parallel=not fully_sharded_loras, - compilation_config=vllm.config.CompilationConfig( - cudagraph_specialize_lora=False, - ), - ) + llm = vllm.LLM( + MODEL_PATH, + max_model_len=1024, + enable_lora=True, + max_loras=2, + max_num_seqs=2, + max_num_batched_tokens=2048, + tensor_parallel_size=2, + gpu_memory_utilization=0.8, + fully_sharded_loras=fully_sharded_loras, + enable_expert_parallel=not fully_sharded_loras, + moe_backend="marlin" if mxfp4_use_marlin else "auto", + linear_backend="marlin" if mxfp4_use_marlin else "auto", + compilation_config=vllm.config.CompilationConfig( + cudagraph_specialize_lora=False, + ), + ) - generate_and_test(llm, gptoss20b_lora_files, lora_id=1) - generate_and_test(llm, gptoss20b_lora_files, lora_id=2) + generate_and_test(llm, gptoss20b_lora_files, lora_id=1) + generate_and_test(llm, gptoss20b_lora_files, lora_id=2) diff --git a/tests/models/language/pooling/test_pooler_config_init_behaviour.py b/tests/models/language/pooling/test_pooler_config_init_behaviour.py index 2f6fb9c873f..f462e9673a9 100644 --- a/tests/models/language/pooling/test_pooler_config_init_behaviour.py +++ b/tests/models/language/pooling/test_pooler_config_init_behaviour.py @@ -106,7 +106,7 @@ def test_reward_models_using_activation( dtype=dtype, pooler_config=PoolerConfig(use_activation=False), ) as vllm_model: - wo_activation = vllm_model.reward(example_prompts) + wo_activation = vllm_model.token_classify(example_prompts) with vllm_runner( model, @@ -114,7 +114,7 @@ def test_reward_models_using_activation( dtype=dtype, pooler_config=PoolerConfig(use_activation=True), ) as vllm_model: - w_activation = vllm_model.reward(example_prompts) + w_activation = vllm_model.token_classify(example_prompts) for wo, w in zip(wo_activation, w_activation): wo = torch.tensor(wo) diff --git a/tests/models/language/pooling/test_reward.py b/tests/models/language/pooling/test_reward.py index 22e0539a989..1872ca4ae09 100644 --- a/tests/models/language/pooling/test_reward.py +++ b/tests/models/language/pooling/test_reward.py @@ -107,7 +107,7 @@ def test_prm_models( pytest.skip("CPU only supports V1") with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model: - vllm_outputs = vllm_model.reward(math_step_prompts) + vllm_outputs = vllm_model.token_classify(math_step_prompts) with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model: hf_model = step_reward_patch_hf_model(hf_model) @@ -146,7 +146,7 @@ def test_prm_models_with_golden_outputs( pytest.skip(f"No available golden outputs for {model}.") with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model: - vllm_outputs = vllm_model.reward(math_step_prompts) + vllm_outputs = vllm_model.token_classify(math_step_prompts) golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model]) diff --git a/tests/models/quantization/test_nvfp4.py b/tests/models/quantization/test_nvfp4.py index 5ca307a4b19..660643eeab5 100644 --- a/tests/models/quantization/test_nvfp4.py +++ b/tests/models/quantization/test_nvfp4.py @@ -133,11 +133,11 @@ def test_nvfp4(vllm_runner, model, eager, backend): not current_platform.is_rocm(), reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx", ) -def test_nvfp4_moe(vllm_runner, model, backend, monkeypatch): - monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend) +def test_nvfp4_moe(vllm_runner, model, backend): with vllm_runner( model, moe_backend=backend, + linear_backend=backend, load_format="dummy", hf_overrides={"num_hidden_layers": 2}, ) as llm: diff --git a/tests/quantization/test_blackwell_moe.py b/tests/quantization/test_blackwell_moe.py index 8c525149ca7..652748c668f 100644 --- a/tests/quantization/test_blackwell_moe.py +++ b/tests/quantization/test_blackwell_moe.py @@ -185,8 +185,11 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch): def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1") - can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT) + can_initialize( + "openai/gpt-oss-20b", + hf_overrides=HF_OVERRIDE_TEXT, + extra_args=["--moe-backend=flashinfer_trtllm"], + ) def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch): diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index 3dcff477c4e..abdcedd12be 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -1617,7 +1617,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser): "custom", "custom_audio", "custom_image", - "custom_mm", "prefix_repetition", "spec_bench", "speed_bench", @@ -2106,12 +2105,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]: no_oversample=args.no_oversample, ) - elif args.dataset_name in ("custom_image", "custom_mm"): - if args.dataset_name == "custom_mm": - logger.warning( - "Dataset name 'custom_mm' is deprecated and will be removed in v0.24. " - "Use '--dataset-name custom_image' instead." - ) + elif args.dataset_name == "custom_image": dataset = CustomImageDataset( dataset_path=args.dataset_path, disable_shuffle=args.disable_shuffle, diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index a191aca4f51..6b03c7adf1e 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -134,10 +134,6 @@ class PassConfig: """Enable async TP.""" fuse_allreduce_rms: bool = None # type: ignore[assignment] """Enable flashinfer allreduce fusion.""" - fuse_minimax_qk_norm: bool = None # type: ignore[assignment] - """Deprecated. The MiniMax QK norm fusion is now applied automatically at - runtime (see `MiniMaxText01RMSNormTP.forward_qkv`). This flag is kept for - backward compatibility and has no effect; it will be removed in v0.23.""" enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment] """Enable fused Q/K RMSNorm + RoPE pass.""" fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment] @@ -296,13 +292,6 @@ class PassConfig: "current platform is not CUDA or ROCm. The fusion will be disabled." ) self.fuse_rope_kvcache_cat_mla = False - if self.fuse_minimax_qk_norm is not None: - logger.warning_once( - "`fuse_minimax_qk_norm` is deprecated and has no effect; " - "the MiniMax QK norm fusion is now applied automatically at " - "runtime when its conditions are met. This flag will be " - "removed in v0.23." - ) def log_enabled_passes(self) -> None: """ diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index c5f44e1563d..7a393752f47 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -142,6 +142,7 @@ LinearBackend = Literal[ "flashinfer_cutlass", "flashinfer_trtllm", "flashinfer_cudnn", + "flashinfer_b12x", "marlin", "triton", "deep_gemm", @@ -197,6 +198,7 @@ class KernelConfig: - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels - "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels - "flashinfer_cudnn": Use FlashInfer with cuDNN kernels + - "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+) - "marlin": Use Marlin kernels - "triton": Use Triton-based kernels - "deep_gemm": Use DeepGEMM kernels diff --git a/vllm/entrypoints/pooling/offline.py b/vllm/entrypoints/pooling/offline.py index 0ab7e07c709..a005bb92b48 100644 --- a/vllm/entrypoints/pooling/offline.py +++ b/vllm/entrypoints/pooling/offline.py @@ -286,50 +286,6 @@ class PoolingOfflineMixin(OfflineInferenceMixin): return [ClassificationRequestOutput.from_base(item) for item in items] - def reward( - self, - prompts: PromptType | Sequence[PromptType], - /, - *, - pooling_params: PoolingParams | Sequence[PoolingParams] | None = None, - use_tqdm: bool | Callable[..., tqdm] = True, - lora_request: list[LoRARequest] | LoRARequest | None = None, - tokenization_kwargs: dict[str, Any] | None = None, - ) -> list[PoolingRequestOutput]: - """ - Generate rewards for each prompt. - - Args: - prompts: The prompts to the LLM. You may pass a sequence of prompts - for batch inference. See [PromptType][vllm.inputs.PromptType] - for more details about the format of each prompt. - pooling_params: The pooling parameters for pooling. If None, we - use the default pooling parameters. - use_tqdm: If `True`, shows a tqdm progress bar. - If a callable (e.g., `functools.partial(tqdm, leave=False)`), - it is used to create the progress bar. - If `False`, no progress bar is created. - lora_request: LoRA request to use for generation, if any. - tokenization_kwargs: Overrides for `tokenizer.encode`. - - Returns: - A list of `PoolingRequestOutput` objects containing the - pooled hidden states in the same order as the input prompts. - """ - logger.warning_once( - "`llm.reward` api is deprecated and will be removed in v0.23. " - 'Please use `LLM.encode` with `pooling_task="classify"` or ' - '`pooling_task="token_classify"` instead.' - ) - return self.encode( - prompts, - use_tqdm=use_tqdm, - lora_request=lora_request, - pooling_params=pooling_params, - pooling_task="token_classify", - tokenization_kwargs=tokenization_kwargs, - ) - def score( self, data_1: ScoreInput | list[ScoreInput], diff --git a/vllm/envs.py b/vllm/envs.py index 17c3ffc2a8d..d0133638f16 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -8,7 +8,6 @@ import os import sys import tempfile import uuid -import warnings from collections.abc import Callable from typing import TYPE_CHECKING, Any, Literal @@ -167,7 +166,6 @@ if TYPE_CHECKING: VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None VLLM_HUMMING_USE_F16_ACCUM: bool = False VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None - VLLM_MXFP4_USE_MARLIN: bool | None = None VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False VLLM_V1_USE_OUTLINES_CACHE: bool = False VLLM_TPU_BUCKET_PADDING_GAP: int = 0 @@ -184,13 +182,7 @@ if TYPE_CHECKING: ] = "relax" VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True - VLLM_USE_FLASHINFER_MOE_FP16: bool = False - VLLM_USE_FLASHINFER_MOE_FP8: bool = False - VLLM_USE_FLASHINFER_MOE_FP4: bool = False VLLM_USE_FLASHINFER_MOE_INT4: bool = False - VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency", "masked_gemm"] = ( - "latency" - ) VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto" VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024 @@ -212,7 +204,6 @@ if TYPE_CHECKING: VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None VLLM_COMPUTE_NANS_IN_LOGITS: bool = False - VLLM_USE_NVFP4_CT_EMULATIONS: bool = False VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[ "FP", "INT8", "INT6", "INT4", "NONE" ] = "NONE" @@ -225,12 +216,8 @@ if TYPE_CHECKING: VLLM_LOOPBACK_IP: str = "" VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True VLLM_ENABLE_RESPONSES_API_STORE: bool = False - VLLM_NVFP4_GEMM_BACKEND: str | None = None VLLM_HAS_FLASHINFER_CUBIN: bool = False - VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False - VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False - VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True VLLM_ALLREDUCE_USE_FLASHINFER: bool = False VLLM_TUNED_CONFIG_FOLDER: str | None = None @@ -257,7 +244,6 @@ if TYPE_CHECKING: VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True VLLM_USE_NCCL_SYMM_MEM: bool = False VLLM_NCCL_INCLUDE_PATH: str | None = None - VLLM_USE_FBGEMM: bool = False VLLM_GC_DEBUG: str = "" VLLM_DEBUG_WORKSPACE: bool = False VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False @@ -350,27 +336,6 @@ def use_mega_aot_artifact(): return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1" -def deprecated_env( - env_name: str, - removal_version: str, - replacement: str, - getter: Callable[[], Any], -) -> Callable[[], Any]: - """Wrap an env-var getter to emit a FutureWarning when the var is set.""" - - def _read() -> Any: - if env_name in os.environ: - warnings.warn( - f"{env_name} is deprecated and will be removed in " - f"{removal_version}. {replacement}", - FutureWarning, - stacklevel=2, - ) - return getter() - - return _read - - def env_with_choices( env_name: str, default: str | None, @@ -1371,15 +1336,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_MARLIN_USE_ATOMIC_ADD": lambda: ( os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1" ), - # Whether to use marlin kernel in mxfp4 quantization method - # Deprecated: use --moe-backend marlin (MoE) or --linear-backend marlin - # (linear) instead. - "VLLM_MXFP4_USE_MARLIN": deprecated_env( - "VLLM_MXFP4_USE_MARLIN", - "v0.23", - "Use --moe-backend marlin or --linear-backend marlin.", - lambda: maybe_convert_bool(os.environ.get("VLLM_MXFP4_USE_MARLIN", None)), - ), # The activation dtype for marlin kernel "VLLM_MARLIN_INPUT_DTYPE": env_with_choices( "VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"] @@ -1472,68 +1428,10 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool( int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1")) ), - # Allow use of FlashInfer BF16 MoE kernels for fused moe ops. - # Deprecated: use --moe-backend to select a kernel explicitly. - "VLLM_USE_FLASHINFER_MOE_FP16": deprecated_env( - "VLLM_USE_FLASHINFER_MOE_FP16", - "v0.23", - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).", - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP16", "0"))), - ), - # Allow use of FlashInfer FP8 MoE kernels for fused moe ops. - # Deprecated: use --moe-backend to select a kernel explicitly. - "VLLM_USE_FLASHINFER_MOE_FP8": deprecated_env( - "VLLM_USE_FLASHINFER_MOE_FP8", - "v0.23", - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).", - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP8", "0"))), - ), - # Allow use of FlashInfer NVFP4 MoE kernels for fused moe ops. - # Deprecated: use --moe-backend to select a kernel explicitly. - "VLLM_USE_FLASHINFER_MOE_FP4": deprecated_env( - "VLLM_USE_FLASHINFER_MOE_FP4", - "v0.23", - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass, " - "flashinfer_cutedsl).", - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP4", "0"))), - ), # Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops. "VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool( int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0")) ), - # If set to 1, use the FlashInfer - # MXFP8 (activation) x MXFP4 (weight) MoE backend. - # Deprecated: use --moe-backend flashinfer_trtllm combined with - # --quantization_config.moe.activation mxfp8. - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": deprecated_env( - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", - "v0.23", - "Use --moe-backend flashinfer_trtllm with " - "--quantization_config.moe.activation mxfp8.", - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))), - ), - # If set to 1, use the FlashInfer CUTLASS backend for - # MXFP8 (activation) x MXFP4 (weight) MoE. - # Deprecated: use --moe-backend flashinfer_cutlass combined with - # --quantization_config.moe.activation mxfp8. - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": deprecated_env( - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", - "v0.23", - "Use --moe-backend flashinfer_cutlass with " - "--quantization_config.moe.activation mxfp8.", - lambda: bool( - int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0")) - ), - ), - # If set to 1, use the FlashInfer - # BF16 (activation) x MXFP4 (weight) MoE backend. - # Deprecated: use --moe-backend to select a kernel explicitly. - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": deprecated_env( - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", - "v0.23", - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).", - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "0"))), - ), # Control the cache sized used by the xgrammar compiler. The default # of 512 MB should be enough for roughly 1000 JSON schemas. # It can be changed with this variable if needed for some reason. @@ -1585,25 +1483,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv( "MOONCAKE_REQUESTER_LOCAL_HOSTNAME" ), - # Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support. - # Both require compute capability 10.0 or above. - # Available options: - # - "throughput": [default] - # Uses CUTLASS kernels optimized for high-throughput batch inference. - # - "latency": - # Uses TensorRT-LLM kernels optimized for low-latency inference. - # Deprecated: pass --moe-backend flashinfer_{trtllm,cutlass,cutedsl} directly. - "VLLM_FLASHINFER_MOE_BACKEND": deprecated_env( - "VLLM_FLASHINFER_MOE_BACKEND", - "v0.23", - "Use --moe-backend flashinfer_trtllm, flashinfer_cutlass, or " - "flashinfer_cutedsl.", - env_with_choices( - "VLLM_FLASHINFER_MOE_BACKEND", - "latency", - ["throughput", "latency", "masked_gemm"], - ), - ), # Override the directory for the FlashInfer autotune config cache. "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv( "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None @@ -1681,16 +1560,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool( int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0")) ), - # Controls whether or not emulations are used for NVFP4 - # generations on machines < 100 for compressed-tensors - # models - # Deprecated: use --linear-backend emulation instead. - "VLLM_USE_NVFP4_CT_EMULATIONS": deprecated_env( - "VLLM_USE_NVFP4_CT_EMULATIONS", - "v0.23", - "Use --linear-backend emulation.", - lambda: bool(int(os.getenv("VLLM_USE_NVFP4_CT_EMULATIONS", "0"))), - ), # Timeout (in seconds) for MooncakeConnector in PD disaggregated setup. "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int( os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480") @@ -1700,35 +1569,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_HAS_FLASHINFER_CUBIN": lambda: bool( int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0")) ), - # Supported options: - # - "flashinfer-cudnn": use flashinfer cudnn GEMM backend - # - "flashinfer-trtllm": use flashinfer trtllm GEMM backend - # - "flashinfer-cutlass": use flashinfer cutlass GEMM backend - # - "marlin": use marlin GEMM backend (for GPUs without native FP4 support) - # - "emulation": - # use BF16/FP16 GEMM, dequantizing weights and running QDQ on activations. - # This is only meant for research purposes to run on devices where NVFP4 - # GEMM kernels are not available. - # - : automatically pick an available backend - # Deprecated: use --linear-backend instead. - "VLLM_NVFP4_GEMM_BACKEND": deprecated_env( - "VLLM_NVFP4_GEMM_BACKEND", - "v0.23", - "Use --linear-backend.", - env_with_choices( - "VLLM_NVFP4_GEMM_BACKEND", - None, - [ - "flashinfer-b12x", - "flashinfer-cudnn", - "flashinfer-trtllm", - "flashinfer-cutlass", - "cutlass", - "marlin", - "emulation", - ], - ), - ), # Controls garbage collection during CUDA graph capture. # If set to 0 (default), enables GC freezing to speed up capture time. # If set to 1, allows GC to run during capture. @@ -1892,14 +1732,6 @@ environment_variables: dict[str, Callable[[], Any]] = { ), # NCCL header path "VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None), - # Flag to enable FBGemm kernels on model execution - # Deprecated: use --linear-backend fbgemm instead. - "VLLM_USE_FBGEMM": deprecated_env( - "VLLM_USE_FBGEMM", - "v0.23", - "Use --linear-backend fbgemm.", - lambda: bool(int(os.getenv("VLLM_USE_FBGEMM", "0"))), - ), # GC debug config # - VLLM_GC_DEBUG=0: disable GC debugger # - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 8162acd5e8d..f9d2d9970de 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -212,6 +212,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { "flashinfer_cudnn": { FlashInferCudnnNvFp4LinearKernel, }, + "flashinfer_b12x": { + FlashInferB12xNvFp4LinearKernel, + }, "marlin": { MarlinFP8ScaledMMLinearKernel, MarlinLinearKernel, @@ -392,7 +395,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = { PlatformEnum.CUDA: [ # FlashInferB12xNvFp4LinearKernel excluded from auto-selection until # upstream CUTLASS SM121 MMA op guard is resolved; use - # VLLM_NVFP4_GEMM_BACKEND=flashinfer-b12x to opt in explicitly. + # --linear-backend flashinfer_b12x to opt in explicitly. FlashInferCutlassNvFp4LinearKernel, CutlassNvFp4LinearKernel, MarlinNvFp4LinearKernel, @@ -752,20 +755,6 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel: current platform.""" linear_backend = _get_linear_backend() - force_kernel: type[MxFp4LinearKernel] | None = None - if linear_backend == "auto" and envs.VLLM_MXFP4_USE_MARLIN: - force_kernel = MarlinMxFp4LinearKernel - - if force_kernel is not None: - is_supported, reason = force_kernel.is_supported() - if not is_supported: - raise ValueError( - f"Forced MXFP4 kernel {force_kernel.__name__} is not " - f"supported: {reason}" - ) - logger.info_once("Using %s for MXFP4 GEMM", force_kernel.__name__) - return force_kernel(MxFp4LinearLayerConfig()) - platform = current_platform._enum possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, [])) @@ -836,18 +825,6 @@ def init_wfp8_a16_linear_kernel( ) -# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes. -_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = { - "flashinfer-b12x": FlashInferB12xNvFp4LinearKernel, - "flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel, - "cutlass": CutlassNvFp4LinearKernel, - "marlin": MarlinNvFp4LinearKernel, - "flashinfer-trtllm": FlashInferTrtllmNvFp4LinearKernel, - "flashinfer-cudnn": FlashInferCudnnNvFp4LinearKernel, - "emulation": EmulationNvFp4LinearKernel, -} - - def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel: """Select and instantiate the best NVFP4 linear kernel for the current platform.""" @@ -855,8 +832,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel: # VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the # batch-invariant CUTLASS implementation when available, otherwise fall - # back to emulation. It overrides both --linear-backend and the deprecated - # env vars below. + # back to emulation. It overrides --linear-backend. force_kernel: type[NvFp4LinearKernel] | None = None linear_backend = _get_linear_backend() if envs.VLLM_BATCH_INVARIANT: @@ -888,24 +864,9 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel: reason, ) force_kernel = EmulationNvFp4LinearKernel - elif linear_backend == "auto": - # Deprecated env-var overrides — only honoured when --linear-backend - # is "auto". Deprecation warnings are emitted from vllm/envs.py. - if use_a16: # force a16 if running weight-only quantization - force_kernel = MarlinNvFp4LinearKernel - elif envs.VLLM_USE_FBGEMM: - force_kernel = FbgemmNvFp4LinearKernel - elif envs.VLLM_USE_NVFP4_CT_EMULATIONS: - force_kernel = EmulationNvFp4LinearKernel - elif envs.VLLM_NVFP4_GEMM_BACKEND is not None: - backend_name = envs.VLLM_NVFP4_GEMM_BACKEND - force_kernel = _NVFP4_BACKEND_TO_KERNEL.get(backend_name) - if force_kernel is None: - raise ValueError( - f"Unknown VLLM_NVFP4_GEMM_BACKEND={backend_name!r}. " - f"Valid choices: " - f"{list(_NVFP4_BACKEND_TO_KERNEL.keys())}" - ) + elif linear_backend == "auto" and use_a16: + # Force a16 (Marlin) when running weight-only quantization. + force_kernel = MarlinNvFp4LinearKernel if force_kernel is not None: is_supported, reason = force_kernel.is_supported() diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 1099368e107..3a65e7360f0 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -20,8 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import ( ) from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( - FlashinferMoeBackend, - get_flashinfer_moe_backend, prepare_fp8_moe_layer_for_fi, ) from vllm.model_executor.layers.quantization.utils.fp8_utils import ( @@ -321,54 +319,6 @@ def select_fp8_moe_backend( requested_backend, config, weight_key, activation_key, activation_format ) - # Handle explicit FlashInfer FP8 configuration. - if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP8"): - if not envs.VLLM_USE_FLASHINFER_MOE_FP8: - # If the user rejects FlashInfer remove those backends. - AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_TRTLLM) - AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_CUTLASS) - - elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"): - # If user is explicit about backend, validate it. - fi_backend = get_flashinfer_moe_backend() - if fi_backend == FlashinferMoeBackend.CUTLASS: - backend = Fp8MoeBackend.FLASHINFER_CUTLASS - elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM: - backend = Fp8MoeBackend.FLASHINFER_TRTLLM - else: - raise ValueError( - f"FlashInfer MOE backend {fi_backend} does not support FP8 MoE." - ) - k_cls = backend_to_kernel_cls(backend)[0] - return _return_or_raise( - backend, config, weight_key, activation_key, activation_format - ) - else: - # If the user is not explicit about the backend, try both. - for backend in [ - Fp8MoeBackend.FLASHINFER_TRTLLM, - Fp8MoeBackend.FLASHINFER_CUTLASS, - ]: - for k_cls in backend_to_kernel_cls(backend): - supported, reason = k_cls.is_supported_config( - k_cls, - config, - weight_key, - activation_key, - activation_format, - ) - - if supported: - logger.info_once(_make_log_backend(backend)) - return backend, k_cls - else: - logger.debug_once(_make_log_unsupported(backend, reason)) - - raise NotImplementedError( - "Found VLLM_USE_FLASHINFER_MOE_FP8=1, but no " - "FlashInfer FP8 MoE backend supports the configuration." - ) - # Handle explicit DeepGEMM FP8 configuration. if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"): if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 0b5ac873dec..87c44d92fd5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Literal, Union import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from vllm import envs from vllm.config import get_current_vllm_config from vllm.config.kernel import MoEBackend from vllm.config.quantization import QuantizationConfigArgs @@ -465,74 +464,6 @@ def select_mxfp4_moe_backend( _get_priority_backends_for_gpt_oss(), requested_activation_key ) - # Handle explicit FlashInfer MXFP4 BF16 configuration. - if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"): - if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: - for _b in ( - Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, - Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, - ): - if _b in AVAILABLE_BACKENDS: - AVAILABLE_BACKENDS.remove(_b) - else: - if current_platform.is_device_capability(90): - return _return_or_raise( - Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16, - config, - kMxfp4Static, - None, - activation_format, - ) - if current_platform.is_device_capability_family(100): - return _return_or_raise( - Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, - config, - kMxfp4Static, - None, - activation_format, - ) - raise ValueError( - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16=1 is set but the " - "current device capability is not supported. " - "Only SM90 (CUTLASS) and SM100+ (TRTLLM) are supported." - ) - - # Handle explicit FlashInfer MXFP4 MXFP8 TRTLLM configuration. - if ( - envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8") - and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8 - ): - return _return_or_raise( - Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8, - config, - kMxfp4Static, - kMxfp8Dynamic, - activation_format, - ) - - # Handle explicit FlashInfer MXFP4 MXFP8 CUTLASS configuration. - if ( - envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS") - and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS - ): - return _return_or_raise( - Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, - config, - kMxfp4Static, - kMxfp8Dynamic, - activation_format, - ) - - # Handle explicit Marlin MXFP4 configuration. - if envs.is_set("VLLM_MXFP4_USE_MARLIN") and envs.VLLM_MXFP4_USE_MARLIN: - return _return_or_raise( - Mxfp4MoeBackend.MARLIN, - config, - kMxfp4Static, - None, - activation_format, - ) - for backend in AVAILABLE_BACKENDS: # Use requested_activation_key if provided, otherwise use backend default act_key = ( diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index d5c55da96e6..93bc81c22be 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -22,10 +22,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import ( prepare_nvfp4_moe_layer_for_fi_or_cutlass, prepare_nvfp4_moe_layer_for_flashinfer_cutedsl, ) -from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( - FlashinferMoeBackend, - get_flashinfer_moe_backend, -) from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( prepare_nvfp4_moe_layer_for_marlin, ) @@ -58,12 +54,6 @@ FLASHINFER_NVFP4_MOE_BACKENDS = [ NvFp4MoeBackend.FLASHINFER_B12X, ] -fi_2_vllm_backend_map: dict[FlashinferMoeBackend, NvFp4MoeBackend] = { - FlashinferMoeBackend.CUTLASS: NvFp4MoeBackend.FLASHINFER_CUTLASS, - FlashinferMoeBackend.TENSORRT_LLM: NvFp4MoeBackend.FLASHINFER_TRTLLM, - FlashinferMoeBackend.CUTEDSL: NvFp4MoeBackend.FLASHINFER_CUTEDSL, -} - def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool: # Checks whether `backend` supports quantizing with scaling factors @@ -258,55 +248,6 @@ def select_nvfp4_moe_backend( requested_backend, config, weight_key, activation_key, activation_format ) - if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP4"): - if not envs.VLLM_USE_FLASHINFER_MOE_FP4: - # If the user rejects FlashInfer remove those backends. - for b in FLASHINFER_NVFP4_MOE_BACKENDS: - if b in AVAILABLE_BACKENDS: - AVAILABLE_BACKENDS.remove(b) - - elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"): - # If user is explicit about backend, validate it. - backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()] - if ( - config.swiglu_limit is not None - and backend not in NVFP4_BACKENDS_WITH_CLAMP - ): - raise ValueError( - f"Model sets swiglu_limit={config.swiglu_limit}, but the " - f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND " - f"({backend.value}) does not apply the SwiGLU clamp." - ) - return _return_or_raise( - backend, config, weight_key, activation_key, activation_format - ) - else: - # If the user is not explicit about the backend, try each. - fi_backends = [ - b - for b in FLASHINFER_NVFP4_MOE_BACKENDS - if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP - ] - for backend in fi_backends: - for k_cls in backend_to_kernel_cls(backend): - supported, reason = k_cls.is_supported_config( - k_cls, - config, - weight_key, - activation_key, - activation_format, - ) - if supported: - logger.info_once(_make_log_backend(backend)) - return backend, k_cls - else: - logger.debug_once(_make_log_unsupported(backend, reason)) - - raise NotImplementedError( - "Found VLLM_USE_FLASHINFER_MOE_FP4=1, but no " - "FlashInfer NVFP4 MoE backend supports the configuration." - ) - if envs.VLLM_TEST_FORCE_FP8_MARLIN: backend = NvFp4MoeBackend.MARLIN return _return_or_raise( diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 8e4012d3ec8..36129fab582 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -19,9 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, ) from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( - FlashinferMoeBackend, convert_moe_weights_to_flashinfer_trtllm_block_layout, - get_flashinfer_moe_backend, swap_w13_to_w31, ) from vllm.platforms import current_platform @@ -230,49 +228,6 @@ def select_unquantized_moe_backend( return _return_or_raise(requested_backend, moe_config, activation_format) - # Handle explicit FlashInfer FP16 configuration. - if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP16"): - if not envs.VLLM_USE_FLASHINFER_MOE_FP16: - if UnquantizedMoeBackend.FLASHINFER_TRTLLM in AVAILABLE_BACKENDS: - AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_TRTLLM) - if UnquantizedMoeBackend.FLASHINFER_CUTLASS in AVAILABLE_BACKENDS: - AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_CUTLASS) - - elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"): - # If user is explicit about backend, validate it. - fi_backend = get_flashinfer_moe_backend() - if fi_backend == FlashinferMoeBackend.CUTLASS: - backend = UnquantizedMoeBackend.FLASHINFER_CUTLASS - elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM: - backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM - else: - raise ValueError( - f"FlashInfer MOE backend {fi_backend} " - "does not support unquantized MoE." - ) - k_cls = backend_to_kernel_cls(backend) - return _return_or_raise(backend, moe_config, activation_format) - else: - # If the user is not explicit about the backend, try both. - for backend in [ - UnquantizedMoeBackend.FLASHINFER_TRTLLM, - UnquantizedMoeBackend.FLASHINFER_CUTLASS, - ]: - k_cls = backend_to_kernel_cls(backend) - supported, reason = k_cls.is_supported_config( - k_cls, moe_config, None, None, activation_format - ) - if supported: - logger.info_once(_make_log_backend(backend)) - return backend, k_cls - else: - logger.debug_once(_make_log_unsupported(backend, reason)) - - raise NotImplementedError( - "Found VLLM_USE_FLASHINFER_MOE_FP16=1, but no " - "FlashInfer unquantized MoE backend supports the configuration." - ) - # Handle explicit AITER FP8 configuration. if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"): if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE: diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py index 977d4556f13..850f54df4b4 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py @@ -6,7 +6,7 @@ import nixl_ep import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from vllm import envs +from vllm.config import get_current_vllm_config from vllm.distributed import get_ep_group from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager from vllm.logger import init_logger @@ -192,10 +192,11 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): x = x.view((-1, hidden_dim)) q_dtype = quant_config.quant_dtype - if envs.VLLM_FLASHINFER_MOE_BACKEND == "masked_gemm": + moe_backend = get_current_vllm_config().kernel_config.moe_backend + if moe_backend == "flashinfer_cutedsl": logger.info_once( - "Skip quantization when using FlashInfer CUTEDSL(masked_gemm) " - "for ModelOptNvFp4FusedMoE." + "Skip quantization when using FlashInfer CUTEDSL " + "(--moe-backend flashinfer_cutedsl) for ModelOptNvFp4FusedMoE." ) q_dtype = None diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py b/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py index 082e42f964f..23a7131a582 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING import torch -import vllm.envs as envs from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( align_fp4_moe_weights_for_fi, @@ -15,10 +14,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( swizzle_blockscale, ) -from vllm.platforms import current_platform -from vllm.utils.flashinfer import ( - has_flashinfer_cutlass_fused_moe, -) if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe import RoutedExperts @@ -34,16 +29,6 @@ __all__ = [ ] -def is_flashinfer_fp4_cutlass_moe_available() -> bool: - """Return `True` when FlashInfer CUTLASS NV-FP4 kernels can be used.""" - return ( - envs.VLLM_USE_FLASHINFER_MOE_FP4 - and has_flashinfer_cutlass_fused_moe() - and current_platform.is_cuda() - and current_platform.has_device_capability(100) - ) - - def reorder_w1w3_to_w3w1( weight: torch.Tensor, scale: torch.Tensor, dim: int = -2 ) -> tuple[torch.Tensor, torch.Tensor]: diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py index 973f759698f..61b52345ab8 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py @@ -5,10 +5,8 @@ from typing import TYPE_CHECKING import torch -from vllm import envs from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.activation import MoEActivation -from vllm.platforms import current_platform from vllm.utils.math_utils import round_up if TYPE_CHECKING: @@ -95,34 +93,6 @@ def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe( ) -def get_flashinfer_moe_backend() -> FlashinferMoeBackend: - backend_map = { - "throughput": FlashinferMoeBackend.CUTLASS, - "latency": FlashinferMoeBackend.TENSORRT_LLM, - "masked_gemm": FlashinferMoeBackend.CUTEDSL, - } - - flashinfer_moe_backend = envs.VLLM_FLASHINFER_MOE_BACKEND - if flashinfer_moe_backend in backend_map: - if ( - flashinfer_moe_backend == "latency" - and not current_platform.is_device_capability_family(100) - ): - logger.info_once( - "Flashinfer TRTLLM MOE backend is only supported on " - "SM100 and later, using CUTLASS backend instead", - ) - return FlashinferMoeBackend.CUTLASS - return backend_map[flashinfer_moe_backend] - elif current_platform.is_device_capability(90): - return FlashinferMoeBackend.CUTLASS - - raise ValueError( - f"Unknown flashinfer moe backend: {flashinfer_moe_backend!r}. " - f"Expected one of {list(backend_map.keys())}." - ) - - def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool: # TODO(shuw@nvidia): Update when new backends are added. backends_supporting_global_sf = (