[Model Runner v2] Force v1 runner for tests (#43233)

Signed-off-by: yewentao256 <[email protected]>

tests/compile/correctness_e2e/test_async_tp.py

@@ -92,7 +92,13 @@ def test_async_tp_pass_correctness(
         "mp",
     ]
 
-    compare_two_settings(model_id, async_tp_args, tp_args, method="generate")
+    compare_two_settings(
+        model_id,
+        async_tp_args,
+        tp_args,
+        method="generate",
+        force_v1_runner=True,
+    )
 
 
 @create_new_process_for_each_test()
@@ -154,4 +160,10 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
         "mp",
     ]
 
-    compare_two_settings(NVFP4_MODEL_ID, async_tp_args, tp_args, method="generate")
+    compare_two_settings(
+        NVFP4_MODEL_ID,
+        async_tp_args,
+        tp_args,
+        method="generate",
+        force_v1_runner=True,
+    )

tests/compile/correctness_e2e/test_sequence_parallel.py

@@ -294,7 +294,13 @@ def _compare_sp(
         "mp",
     ]
 
-    compare_two_settings(model_id, tp_sp_args, tp_args, method=method)
+    compare_two_settings(
+        model_id,
+        tp_sp_args,
+        tp_args,
+        method=method,
+        force_v1_runner=True,
+    )
 
 
 SP_TEXT_GENERATION_MODELS = {

tests/compile/fullgraph/test_basic_correctness.py

@@ -145,6 +145,7 @@ def test_compile_correctness(
             all_args,
             all_envs,
             method=method if method != "generate" else "generate_close",
+            force_v1_runner=True,
         )
         all_envs.clear()
         all_args.clear()
@@ -158,4 +159,4 @@ def test_compile_correctness(
         all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"])
         all_envs.append({})
 
-    compare_all_settings(model, all_args, all_envs, method=method)
+    compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True)

tests/distributed/test_pipeline_parallel.py

@@ -349,7 +349,14 @@ def _compare_tp(
         "mp",
     ]
 
-    compare_two_settings(model_id, pp_args, tp_args, pp_env, tp_env, method=method)
+    compare_two_settings(
+        model_id,
+        pp_args,
+        tp_args,
+        pp_env,
+        tp_env,
+        method=method,
+    )
 
 
 @pytest.mark.parametrize(

tests/models/quantization/test_bitsandbytes.py

@@ -137,7 +137,11 @@ def test_load_pp_4bit_bnb_model(model_name, description) -> None:
         "--pipeline-parallel-size",
         "2",
     ]
-    compare_two_settings(model_name, common_args, pp_args)
+    compare_two_settings(
+        model_name,
+        common_args,
+        pp_args,
+    )
 
 
 @pytest.mark.skipif(

tests/utils.py

@@ -1117,6 +1117,7 @@ def compare_two_settings(
     method: str = "generate",
     max_wait_seconds: float | None = None,
     include_seeded_sampling: bool = True,
+    force_v1_runner: bool = False,
 ) -> None:
     """
     Launch API server with two different sets of arguments/environments
@@ -1130,6 +1131,9 @@ def compare_two_settings(
         env2: The second set of environment variables to pass to the API server.
         include_seeded_sampling: Whether to include temperature=1.0 seeded
             sampling checks in the default generate comparison.
+        force_v1_runner: Whether to pin all compared settings to the v1 model
+            runner to avoid mixing model runner differences into correctness
+            tests.
     """
 
     compare_all_settings(
@@ -1139,6 +1143,7 @@ def compare_two_settings(
         method=method,
         max_wait_seconds=max_wait_seconds,
         include_seeded_sampling=include_seeded_sampling,
+        force_v1_runner=force_v1_runner,
     )
 
 
@@ -1150,6 +1155,7 @@ def compare_all_settings(
     method: str = "generate",
     max_wait_seconds: float | None = None,
     include_seeded_sampling: bool = True,
+    force_v1_runner: bool = False,
 ) -> None:
     """
     Launch API server with several different sets of arguments/environments
@@ -1160,8 +1166,16 @@ def compare_all_settings(
         all_envs: A list of environment dictionaries to pass to the API server.
         include_seeded_sampling: Whether to include temperature=1.0 seeded
             sampling checks in the default generate comparison.
+        force_v1_runner: Whether to pin all compared settings to the v1 model
+            runner to avoid mixing model runner differences into correctness
+            tests.
     """
 
+    if force_v1_runner:
+        all_envs = [
+            {"VLLM_USE_V2_MODEL_RUNNER": "0", **(env or {})} for env in all_envs
+        ]
+
     trust_remote_code = False
     for args in all_args:
         if "--trust-remote-code" in args: