From 6aabe221a56052965e6bb0a95e9ec682d046a6e7 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 29 May 2026 11:58:25 -0700 Subject: [PATCH] [CI] Make Model Executor test hangs fail fast with a traceback (#43971) Signed-off-by: khluu Co-authored-by: Claude --- .buildkite/test_areas/model_executor.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index c41ef8a7110..e34b7eadfac 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -14,5 +14,12 @@ steps: commands: - apt-get update && apt-get install -y curl libsodium23 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s model_executor -m '(not slow_test)' - - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py + # Dump tracebacks of all threads if a test hangs, so a wedged GPU/CUDA + # init surfaces a stack instead of silently stalling. + - export PYTHONFAULTHANDLER=1 + # Per-test watchdog: a single hung test (e.g. stuck during engine/CUDA + # init) fails fast with a traceback instead of running until the global + # build timeout. The `thread` method also handles hangs inside C/CUDA + # calls that the signal method cannot interrupt. + - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread