A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
[Docs] Add Phi-4-reasoning-vision to supported models + examples (#39232)
Signed-off-by: Varun Sundar Rabindranath <vsundarr@redhat.com> Co-authored-by: Varun Sundar Rabindranath <vsundarr@redhat.com>
Varun Sundar Rabindranath committed
Apr 7, 2026 at 19:02 UTC
7b80cd8ac382851527225ed1f3475c138a4b7c01
3 files changed
+42
docs/models/supported_models.md
+1
@@ -601,6 +601,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
601
| `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + I<sup>E</sup> | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | ✅︎ | ✅︎ |
602
| `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + I<sup>E+</sup> | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
603
| `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
604
+| `Phi4ForCausalLMV` | Phi-4-reasoning-vision | T + I<sup>+</sup> | `microsoft/Phi-4-reasoning-vision-15B`, etc. | | ✅︎ |
605
| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | ✅︎ | ✅︎ |
606
| `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
607
| `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
examples/offline_inference/vision_language.py
+22
@@ -1741,6 +1741,27 @@ def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
1741
)
1742
1743
1744
+# Phi-4-reasoning-vision
1745
+def run_phi4siglip(questions: list[str], modality: str) -> ModelRequestData:
1746
+ assert modality == "image"
1747
+ model_name = "microsoft/Phi-4-reasoning-vision-15B"
1748
+ prompts = [
1749
+ f"<|user|>\n<image>\n{question}<|end|>\n<|assistant|>\n"
1750
+ for question in questions
1751
+ ]
1752
+ engine_args = EngineArgs(
1753
+ model=model_name,
1754
+ trust_remote_code=True,
1755
+ max_model_len=8192,
1756
+ max_num_seqs=2,
1757
+ limit_mm_per_prompt={modality: 1},
1758
+ )
1759
+ return ModelRequestData(
1760
+ engine_args=engine_args,
1761
+ prompts=prompts,
1762
+ )
1763
+
1764
+
1765
# Pixtral HF-format
1766
def run_pixtral_hf(questions: list[str], modality: str) -> ModelRequestData:
1767
assert modality == "image"
@@ -2222,6 +2243,7 @@ model_example_map = {
2243
"paligemma2": run_paligemma2,
2244
"phi3_v": run_phi3v,
2245
"phi4_mm": run_phi4mm,
2246
+ "phi4_siglip": run_phi4siglip,
2247
"pixtral_hf": run_pixtral_hf,
2248
"qwen_vl": run_qwen_vl,
2249
"qwen2_vl": run_qwen2_vl,
examples/offline_inference/vision_language_multi_image.py
+19
@@ -957,6 +957,24 @@ def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
957
)
958
959
960
+def load_phi4siglip(question: str, image_urls: list[str]) -> ModelRequestData:
961
+ model_name = "microsoft/Phi-4-reasoning-vision-15B"
962
+ placeholders = "\n".join("<image>" for _ in image_urls)
963
+ prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
964
+ engine_args = EngineArgs(
965
+ model=model_name,
966
+ trust_remote_code=True,
967
+ max_model_len=8192,
968
+ max_num_seqs=2,
969
+ limit_mm_per_prompt={"image": len(image_urls)},
970
+ )
971
+ return ModelRequestData(
972
+ engine_args=engine_args,
973
+ prompt=prompt,
974
+ image_data=[fetch_image(url) for url in image_urls],
975
+ )
976
+
977
+
978
def load_qwen_vl_chat(question: str, image_urls: list[str]) -> ModelRequestData:
979
model_name = "Qwen/Qwen-VL-Chat"
980
engine_args = EngineArgs(
@@ -1455,6 +1473,7 @@ model_example_map = {
1473
"paddleocr_vl": load_paddleocr_vl,
1474
"phi3_v": load_phi3v,
1475
"phi4_mm": load_phi4mm,
1476
+ "phi4_siglip": load_phi4siglip,
1477
"pixtral_hf": load_pixtral_hf,
1478
"qwen_vl_chat": load_qwen_vl_chat,
1479
"qwen2_vl": load_qwen2_vl,