From 7b80cd8ac382851527225ed1f3475c138a4b7c01 Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Tue, 7 Apr 2026 19:02:26 -0700 Subject: [PATCH] [Docs] Add Phi-4-reasoning-vision to supported models + examples (#39232) Signed-off-by: Varun Sundar Rabindranath Co-authored-by: Varun Sundar Rabindranath --- docs/models/supported_models.md | 1 + examples/offline_inference/vision_language.py | 22 +++++++++++++++++++ .../vision_language_multi_image.py | 19 ++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index b86701c3ceb..b0b6c080ac4 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -601,6 +601,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + IE | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | ✅︎ | ✅︎ | | `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + IE+ | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ | | `Phi4MMForCausalLM` | Phi-4-multimodal | T + I+ / T + A+ / I+ + A+ | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ | +| `Phi4ForCausalLMV` | Phi-4-reasoning-vision | T + I+ | `microsoft/Phi-4-reasoning-vision-15B`, etc. | | ✅︎ | | `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I+ | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | ✅︎ | ✅︎ | | `QwenVLForConditionalGeneration`^ | Qwen-VL | T + IE+ | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ | | `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A+ | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ | diff --git a/examples/offline_inference/vision_language.py b/examples/offline_inference/vision_language.py index cc97126c2ac..d62f25c2285 100755 --- a/examples/offline_inference/vision_language.py +++ b/examples/offline_inference/vision_language.py @@ -1741,6 +1741,27 @@ def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData: ) +# Phi-4-reasoning-vision +def run_phi4siglip(questions: list[str], modality: str) -> ModelRequestData: + assert modality == "image" + model_name = "microsoft/Phi-4-reasoning-vision-15B" + prompts = [ + f"<|user|>\n\n{question}<|end|>\n<|assistant|>\n" + for question in questions + ] + engine_args = EngineArgs( + model=model_name, + trust_remote_code=True, + max_model_len=8192, + max_num_seqs=2, + limit_mm_per_prompt={modality: 1}, + ) + return ModelRequestData( + engine_args=engine_args, + prompts=prompts, + ) + + # Pixtral HF-format def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData: assert modality == "image" @@ -2222,6 +2243,7 @@ model_example_map = { "paligemma2": run_paligemma2, "phi3_v": run_phi3v, "phi4_mm": run_phi4mm, + "phi4_siglip": run_phi4siglip, "pixtral_hf": run_pixtral_hf, "qwen_vl": run_qwen_vl, "qwen2_vl": run_qwen2_vl, diff --git a/examples/offline_inference/vision_language_multi_image.py b/examples/offline_inference/vision_language_multi_image.py index 38a34a68ee5..1963ffff791 100755 --- a/examples/offline_inference/vision_language_multi_image.py +++ b/examples/offline_inference/vision_language_multi_image.py @@ -957,6 +957,24 @@ def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData: ) +def load_phi4siglip(question: str, image_urls: list[str]) -> ModelRequestData: + model_name = "microsoft/Phi-4-reasoning-vision-15B" + placeholders = "\n".join("" for _ in image_urls) + prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n" + engine_args = EngineArgs( + model=model_name, + trust_remote_code=True, + max_model_len=8192, + max_num_seqs=2, + limit_mm_per_prompt={"image": len(image_urls)}, + ) + return ModelRequestData( + engine_args=engine_args, + prompt=prompt, + image_data=[fetch_image(url) for url in image_urls], + ) + + def load_qwen_vl_chat(question: str, image_urls: list[str]) -> ModelRequestData: model_name = "Qwen/Qwen-VL-Chat" engine_args = EngineArgs( @@ -1455,6 +1473,7 @@ model_example_map = { "paddleocr_vl": load_paddleocr_vl, "phi3_v": load_phi3v, "phi4_mm": load_phi4mm, + "phi4_siglip": load_phi4siglip, "pixtral_hf": load_pixtral_hf, "qwen_vl_chat": load_qwen_vl_chat, "qwen2_vl": load_qwen2_vl,