diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index b2bf2b05237..91235d7d8eb 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -107,9 +107,9 @@ steps: commands: - pytest -s -v compile/passes --ignore compile/passes/distributed -- label: PyTorch Fullgraph Smoke Test +- label: PyTorch Fullgraph Test device: h200_35gb - key: pytorch-fullgraph-smoke-test + key: pytorch-fullgraph-test timeout_in_minutes: 90 source_file_dependencies: - vllm/__init__.py @@ -138,11 +138,11 @@ steps: - vllm/v1/ - tests/compile commands: - # Run smoke tests under fullgraph directory, except test_full_graph.py + # Run smoke tests under fullgraph directory, except test_full_cudagraph.py # as it is a heavy test that is covered in other steps. # Use `find` to launch multiple instances of pytest so that # they do not suffer from https://github.com/vllm-project/vllm/issues/28965 - - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" + - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'" # Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model # instances and cannot fit a 35GB MIG slice. L4 retains the original coverage: @@ -179,39 +179,6 @@ steps: commands: - pytest -s -v compile/fullgraph/test_full_cudagraph.py -- label: PyTorch Fullgraph - key: pytorch-fullgraph - timeout_in_minutes: 40 - device: h200_18gb - source_file_dependencies: - - vllm/__init__.py - - vllm/_aiter_ops.py - - vllm/_custom_ops.py - - vllm/compilation/ - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/env_override.py - - vllm/envs.py - - vllm/forward_context.py - - vllm/inputs/ - - vllm/ir/ - - vllm/kernels/ - - vllm/logger.py - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/platforms/ - - vllm/plugins/ - - vllm/sampling_params.py - - vllm/sequence.py - - vllm/transformers_utils/ - - vllm/triton_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/compile - commands: - - pytest -v -s compile/fullgraph/test_full_graph.py - - label: Pytorch Nightly Dependency Override Check # 2min key: pytorch-nightly-dependency-override-check # if this test fails, it means the nightly torch version is not compatible with some diff --git a/tests/compile/fullgraph/test_basic_correctness.py b/tests/compile/fullgraph/test_basic_correctness.py index 7471564340d..62e2250be37 100644 --- a/tests/compile/fullgraph/test_basic_correctness.py +++ b/tests/compile/fullgraph/test_basic_correctness.py @@ -36,21 +36,12 @@ class TestSetting: attn_backend=ATTN_BACKEND, method="generate", ), - # llama model with quantization - TestSetting( - model="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", - model_args=["--quantization", "gptq", "--max-model-len", "2048"], - pp_size=1, - tp_size=1, - attn_backend=ATTN_BACKEND, - method="generate", - ), # MoE model TestSetting( - model="ibm/PowerMoE-3b", + model="ibm-granite/granite-3.0-1b-a400m-instruct", model_args=["--max-model-len", "2048"], pp_size=1, - tp_size=2, + tp_size=1, attn_backend=ATTN_BACKEND, method="generate", ), @@ -72,30 +63,6 @@ class TestSetting: attn_backend=ATTN_BACKEND, method="encode", ), - pytest.param( - TestSetting( - model="BAAI/bge-base-en-v1.5", - model_args=["--runner", "pooling"], - pp_size=1, - tp_size=1, - attn_backend="FLASH_ATTN", - method="encode", - ), - marks=pytest.mark.skipif( - current_platform.is_rocm(), - reason="Encoder self-attention is not implemented for ROCm", - ), - ), - # vision language model - # See https://github.com/vllm-project/vllm/issues/26716. - # TestSetting( - # model="microsoft/Phi-3.5-vision-instruct", - # model_args=["--trust-remote-code", "--max-model-len", "2048"], - # pp_size=2, - # tp_size=1, - # attn_backend="FLASH_ATTN", - # method="generate_with_image", - # ), ], ) def test_compile_correctness( @@ -129,29 +96,29 @@ def test_compile_correctness( all_args: list[list[str]] = [] all_envs: list[dict[str, str] | None] = [] - for comp_mode in [ + # Test all compilation modes with inductor backend + for mode in [ + CompilationMode.NONE, CompilationMode.STOCK_TORCH_COMPILE, CompilationMode.DYNAMO_TRACE_ONCE, CompilationMode.VLLM_COMPILE, ]: - for mode in [CompilationMode.NONE, comp_mode]: - all_args.append( - final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"] - ) - all_envs.append({}) + all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"]) + all_envs.append({}) + # inductor will change the output, so we only compare if the output + # is close, not exactly the same. + compare_all_settings( + model, + all_args, + all_envs, + method=method if method != "generate" else "generate_close", + force_v1_runner=True, + ) - # inductor will change the output, so we only compare if the output - # is close, not exactly the same. - compare_all_settings( - model, - all_args, - all_envs, - method=method if method != "generate" else "generate_close", - force_v1_runner=True, - ) - all_envs.clear() - all_args.clear() + all_envs.clear() + all_args.clear() + # Test all compilation modes with eager backend for mode in [ CompilationMode.NONE, CompilationMode.STOCK_TORCH_COMPILE, @@ -160,5 +127,4 @@ def test_compile_correctness( ]: all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"]) all_envs.append({}) - compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True) diff --git a/tests/compile/fullgraph/test_full_graph.py b/tests/compile/fullgraph/test_full_graph.py index 3865fc4d030..4f0a78e25c8 100644 --- a/tests/compile/fullgraph/test_full_graph.py +++ b/tests/compile/fullgraph/test_full_graph.py @@ -21,28 +21,12 @@ def models_list(*, all: bool = True, keywords: list[str] | None = None): TEST_MODELS: list[tuple[str, dict[str, Any]]] = [ ("facebook/opt-125m", {}), ( - "neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic", + "RedHatAI/Llama-3.2-1B-Instruct-FP8-dynamic", {"dtype": torch.float16}, ), - ("meta-llama/Llama-3.2-1B-Instruct", {}), ] if all: - TEST_MODELS.extend( - [ - ("neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8", {}), - ( - "nm-testing/tinyllama-oneshot-w8w8-test-static-shape-change", - {"dtype": torch.float16}, - ), - ] - ) - - if is_quant_method_supported("gptq"): - TEST_MODELS.append( - ("TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", {"quantization": "gptq"}) - ) - if is_quant_method_supported("gptq_marlin"): TEST_MODELS.append( ( @@ -76,14 +60,6 @@ def test_full_graph( model_kwargs: dict[str, Any], compilation_mode: CompilationMode, ): - if ( - "w8a8" in model - or "w8w8" in model - and current_platform.has_device_capability((10, 0)) - ): - # int8 removed on Blackwell: - pytest.skip("int8 support removed on Blackwell") - with monkeypatch.context(): print(f"MODEL={model}") @@ -113,7 +89,7 @@ def test_full_graph( ), *model_info, ) - for model_info in models_list(keywords=["FP8-dynamic", "quantized.w8a8"]) + for model_info in models_list(keywords=["FP8-dynamic"]) ] + [ # Test depyf integration works @@ -164,14 +140,6 @@ def test_custom_compile_config( model: str, model_kwargs: dict[str, Any], ): - if ( - "w8a8" in model - or "w8w8" in model - and current_platform.has_device_capability((10, 0)) - ): - # int8 removed on Blackwell: - pytest.skip("int8 support removed on Blackwell") - if compilation_config.use_inductor_graph_partition and not is_torch_equal_or_newer( "2.9.0.dev" ):