From 63e78ce3652f4f94e9f484f40db71ca4cf019f21 Mon Sep 17 00:00:00 2001 From: Guan-Ming Chiu <105915352+guan404ming@users.noreply.github.com> Date: Sat, 1 Aug 2026 23:10:56 +0900 Subject: [PATCH] [Benchmark] Add probe requests to vllm bench serve (#49611) Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com> --- docs/benchmarking/cli.md | 25 ++++++ tests/benchmarks/test_skip_tokenizer_init.py | 1 + vllm/benchmarks/serve.py | 80 ++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index 23d37eb7268..476171f7322 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -622,6 +622,31 @@ The following arguments can be used to control the ramp-up: - `--ramp-up-start-rps`: The request rate at the beginning of the benchmark. - `--ramp-up-end-rps`: The request rate at the end of the benchmark. +#### Probe Requests + +The benchmark tool also supports sending probe requests alongside the main +workload. This can be useful for measuring how the main workload affects +unrelated traffic sharing the server, e.g. a few requests with large images +stalling a concurrent lightweight request while their multimodal preprocessing +occupies the frontend. + +Setting `--probe-request-rate` to a positive value sends single-token text-only +probe requests at that rate (requests per second) alongside the main workload. +Probes bypass `--max-concurrency` and their latency is reported separately, so +the probe percentiles directly measure the interference that the main workload +inflicts on unrelated requests. + +```bash +vllm bench serve \ + --model Qwen/Qwen2.5-VL-3B-Instruct \ + --backend openai-chat \ + --endpoint /v1/chat/completions \ + --dataset-name random-mm \ + --random-mm-bucket-config '{(2048, 2048, 1): 1.0}' \ + --request-rate 4 \ + --probe-request-rate 20 +``` + #### Load Pattern Configuration vLLM's benchmark serving script provides sophisticated load pattern simulation capabilities through three key parameters that control request generation and concurrency behavior: diff --git a/tests/benchmarks/test_skip_tokenizer_init.py b/tests/benchmarks/test_skip_tokenizer_init.py index e28320b6ae7..741db2e0d62 100644 --- a/tests/benchmarks/test_skip_tokenizer_init.py +++ b/tests/benchmarks/test_skip_tokenizer_init.py @@ -84,6 +84,7 @@ def _args(dataset_path: str) -> argparse.Namespace: request_rate=16.0, burstiness=1.0, max_concurrency=None, + probe_request_rate=0.0, # misc serve args that main_async reads before reaching get_samples plot_timeline=False, plot_dataset_stats=False, diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 996e3348604..e4ab5a583a0 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -795,6 +795,7 @@ async def benchmark( ready_check_timeout_sec: int = 600, ssl_context: ssl.SSLContext | bool | None = None, self_timed: bool = False, + probe_request_rate: float = 0.0, ): try: request_func = ASYNC_REQUEST_FUNCS[endpoint_type] @@ -971,6 +972,30 @@ async def benchmark( request_func_input=request_func_input, session=session, pbar=pbar ) + probe_outputs: list[RequestFuncOutput] = [] + probe_stop = asyncio.Event() + + async def probe_loop(): + probe_input = replace( + test_input, + prompt="Hi", + prompt_len=1, + output_len=1, + multi_modal_content=None, + chat_messages=None, + ) + interval = 1 / probe_request_rate + while not probe_stop.is_set(): + probe_outputs.append( + await request_func(request_func_input=probe_input, session=session) + ) + await asyncio.sleep(interval) + + probe_task: asyncio.Task | None = None + if probe_request_rate > 0: + print(f"Probe request rate: {probe_request_rate} req/s") + probe_task = asyncio.create_task(probe_loop()) + benchmark_start_time = time.perf_counter() tasks: list[asyncio.Task] = [] @@ -1042,6 +1067,10 @@ async def benchmark( ) outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks) + if probe_task is not None: + probe_stop.set() + await probe_task + if pbar is not None: pbar.close() @@ -1189,6 +1218,44 @@ async def benchmark( ) ) + probe_stats: dict[str, Any] | None = None + if probe_task is not None: + probe_lats = [o.latency for o in probe_outputs if o.success] + if probe_lats: + probe_stats = { + "probe_completed": len(probe_lats), + "probe_failed": len(probe_outputs) - len(probe_lats), + "probe_median_e2el_ms": float(np.median(probe_lats)) * 1000, + "probe_p99_e2el_ms": float(np.percentile(probe_lats, 99)) * 1000, + "probe_max_e2el_ms": float(max(probe_lats)) * 1000, + } + print("{s:{c}^{n}}".format(s="Probe Requests", n=50, c="-")) + print( + "{:<40} {:<10}".format( + "Probe requests completed:", probe_stats["probe_completed"] + ) + ) + print( + "{:<40} {:<10}".format( + "Probe requests failed:", probe_stats["probe_failed"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Median probe E2EL (ms):", probe_stats["probe_median_e2el_ms"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "P99 probe E2EL (ms):", probe_stats["probe_p99_e2el_ms"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Max probe E2EL (ms):", probe_stats["probe_max_e2el_ms"] + ) + ) + result: dict[str, Any] if isinstance(metrics, BenchmarkMetrics): result = { @@ -1223,6 +1290,9 @@ async def benchmark( "errors": [output.error for output in outputs], } + if probe_stats is not None: + result.update(probe_stats) + if rps_change_events: result["rps_change_events"] = rps_change_events @@ -1605,6 +1675,15 @@ def add_cli_args(parser: FlexibleArgumentParser): "bursty requests. A higher burstiness value (burstiness > 1) " "results in a more uniform arrival of requests.", ) + parser.add_argument( + "--probe-request-rate", + type=float, + default=0.0, + help="If positive, send single-token text-only probe requests at " + "this rate (req/s) alongside the main workload, bypassing " + "--max-concurrency, and report their latency separately. Useful " + "for measuring how the main workload stalls unrelated requests.", + ) parser.add_argument( "--disable-tqdm", action="store_true", @@ -2121,6 +2200,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]: ready_check_timeout_sec=args.ready_check_timeout_sec, ssl_context=ssl_context, self_timed=args.self_timed, + probe_request_rate=args.probe_request_rate, ) # Save config and results to json