A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Misc] Add Phi4-MM example (#14343)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Mar 8, 2025 at 01:28 UTC
952a0749807905b3514eca7d7dd365b791d6549b
4 files changed
+131
-7
examples/offline_inference/audio_language.py
+38
@@ -6,10 +6,14 @@ with the correct prompt format on audio language models.
6
For most models, the prompt format should follow corresponding examples
7
on HuggingFace model repository.
8
"""
9
+import os
10
+
11
+from huggingface_hub import snapshot_download
12
from transformers import AutoTokenizer
13
14
from vllm import LLM, SamplingParams
15
from vllm.assets.audio import AudioAsset
16
+from vllm.lora.request import LoRARequest
17
from vllm.utils import FlexibleArgumentParser
18
19
audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
@@ -51,6 +55,39 @@ def run_minicpmo(question: str, audio_count: int):
55
return llm, prompt, stop_token_ids
56
57
58
+# Phi-4-multimodal-instruct
59
+def run_phi4mm(questions: str, audio_count: int):
60
+ """
61
+ Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
62
+ show how to process audio inputs.
63
+ """
64
+ model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
65
+ # Since the vision-lora and speech-lora co-exist with the base model,
66
+ # we have to manually specify the path of the lora weights.
67
+ speech_lora_path = os.path.join(model_path, "speech-lora")
68
+ placeholders = "".join([f"<|audio_{i+1}|>" for i in range(audio_count)])
69
+
70
+ prompts = f"<|user|>{placeholders}{questions}<|end|><|assistant|>"
71
+
72
+ llm = LLM(
73
+ model=model_path,
74
+ trust_remote_code=True,
75
+ max_model_len=4096,
76
+ max_num_seqs=2,
77
+ enable_lora=True,
78
+ max_lora_rank=320,
79
+ lora_extra_vocab_size=0,
80
+ )
81
+ lora_request = LoRARequest("speech", 1, speech_lora_path)
82
+ # To maintain code compatibility in this script, we add LoRA here.
83
+ llm.llm_engine.add_lora(lora_request=lora_request)
84
+ # You can also add LoRA using:
85
+ # llm.generate(prompts, lora_request=lora_request,...)
86
+
87
+ stop_token_ids = None
88
+ return llm, prompts, stop_token_ids
89
+
90
+
91
# Qwen2-Audio
92
def run_qwen2_audio(question: str, audio_count: int):
93
model_name = "Qwen/Qwen2-Audio-7B-Instruct"
@@ -113,6 +150,7 @@ def run_whisper(question: str, audio_count: int):
150
151
model_example_map = {
152
"minicpmo": run_minicpmo,
153
+ "phi4_mm": run_phi4mm,
154
"qwen2_audio": run_qwen2_audio,
155
"ultravox": run_ultravox,
156
"whisper": run_whisper,
examples/offline_inference/vision_language.py
+38
@@ -6,13 +6,16 @@ the correct prompt format on vision language models for text generation.
6
For most models, the prompt format should follow corresponding examples
7
on HuggingFace model repository.
8
"""
9
+import os
10
import random
11
12
+from huggingface_hub import snapshot_download
13
from transformers import AutoTokenizer
14
15
from vllm import LLM, SamplingParams
16
from vllm.assets.image import ImageAsset
17
from vllm.assets.video import VideoAsset
18
+from vllm.lora.request import LoRARequest
19
from vllm.utils import FlexibleArgumentParser
20
21
# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
@@ -519,6 +522,40 @@ def run_phi3v(questions: list[str], modality: str):
522
return llm, prompts, stop_token_ids
523
524
525
+# Phi-4-multimodal-instruct
526
+def run_phi4mm(questions: list[str], modality: str):
527
+ """
528
+ Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
529
+ show how to process image inputs.
530
+ """
531
+ assert modality == "image"
532
+ model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
533
+ # Since the vision-lora and speech-lora co-exist with the base model,
534
+ # we have to manually specify the path of the lora weights.
535
+ vision_lora_path = os.path.join(model_path, "vision-lora")
536
+ prompts = [
537
+ f"<|user|><|image_1|>{question}<|end|><|assistant|>"
538
+ for question in questions
539
+ ]
540
+ llm = LLM(
541
+ model=model_path,
542
+ trust_remote_code=True,
543
+ max_model_len=4096,
544
+ max_num_seqs=2,
545
+ enable_lora=True,
546
+ max_lora_rank=320,
547
+ lora_extra_vocab_size=0,
548
+ )
549
+ lora_request = LoRARequest("vision", 1, vision_lora_path)
550
+ # To maintain code compatibility in this script, we add LoRA here.
551
+ llm.llm_engine.add_lora(lora_request=lora_request)
552
+ # You can also add LoRA using:
553
+ # llm.generate(prompts, lora_request=lora_request,...)
554
+
555
+ stop_token_ids = None
556
+ return llm, prompts, stop_token_ids
557
+
558
+
559
# Pixtral HF-format
560
def run_pixtral_hf(questions: list[str], modality: str):
561
assert modality == "image"
@@ -644,6 +681,7 @@ model_example_map = {
681
"paligemma": run_paligemma,
682
"paligemma2": run_paligemma2,
683
"phi3_v": run_phi3v,
684
+ "phi4_mm": run_phi4mm,
685
"pixtral_hf": run_pixtral_hf,
686
"qwen_vl": run_qwen_vl,
687
"qwen2_vl": run_qwen2_vl,
examples/offline_inference/vision_language_multi_image.py
+44
@@ -4,13 +4,16 @@ This example shows how to use vLLM for running offline inference with
4
multi-image input on vision language models for text generation,
5
using the chat template defined by the model.
6
"""
7
+import os
8
from argparse import Namespace
9
from typing import NamedTuple, Optional
10
11
+from huggingface_hub import snapshot_download
12
from PIL.Image import Image
13
from transformers import AutoProcessor, AutoTokenizer
14
15
from vllm import LLM, SamplingParams
16
+from vllm.lora.request import LoRARequest
17
from vllm.multimodal.utils import fetch_image
18
from vllm.utils import FlexibleArgumentParser
19
@@ -294,6 +297,46 @@ def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData:
297
)
298
299
300
+def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
301
+ """
302
+ Phi-4-multimodal-instruct supports both image and audio inputs. Here, we
303
+ show how to process multi images inputs.
304
+ """
305
+
306
+ model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
307
+ # Since the vision-lora and speech-lora co-exist with the base model,
308
+ # we have to manually specify the path of the lora weights.
309
+ vision_lora_path = os.path.join(model_path, "vision-lora")
310
+ llm = LLM(
311
+ model=model_path,
312
+ trust_remote_code=True,
313
+ max_model_len=10000,
314
+ max_num_seqs=2,
315
+ limit_mm_per_prompt={"image": len(image_urls)},
316
+ enable_lora=True,
317
+ max_lora_rank=320,
318
+ lora_extra_vocab_size=0,
319
+ )
320
+ lora_request = LoRARequest("vision", 1, vision_lora_path)
321
+ # To maintain code compatibility in this script, we add LoRA here.
322
+ llm.llm_engine.add_lora(lora_request=lora_request)
323
+ # You can also add LoRA using:
324
+ # llm.generate(prompts, lora_request=lora_request,...)
325
+
326
+ placeholders = "".join(f"<|image_{i}|>"
327
+ for i, _ in enumerate(image_urls, start=1))
328
+ prompt = f"<|user|>{placeholders}{question}<|end|><|assistant|>"
329
+ stop_token_ids = None
330
+
331
+ return ModelRequestData(
332
+ llm=llm,
333
+ prompt=prompt,
334
+ stop_token_ids=stop_token_ids,
335
+ image_data=[fetch_image(url) for url in image_urls],
336
+ chat_template=None,
337
+ )
338
+
339
+
340
def load_qwen_vl_chat(question: str,
341
image_urls: list[str]) -> ModelRequestData:
342
model_name = "Qwen/Qwen-VL-Chat"
@@ -459,6 +502,7 @@ model_example_map = {
502
"mllama": load_mllama,
503
"NVLM_D": load_nvlm_d,
504
"phi3_v": load_phi3v,
505
+ "phi4_mm": load_phi4mm,
506
"pixtral_hf": load_pixtral_hf,
507
"qwen_vl_chat": load_qwen_vl_chat,
508
"qwen2_vl": load_qwen2_vl,
vllm/model_executor/models/phi4mm.py
+11
-7
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
26
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
27
from vllm.model_executor.models.llama import LlamaModel
28
+from vllm.model_executor.models.module_mapping import MultiModelKeys
29
from vllm.model_executor.sampling_metadata import SamplingMetadata
30
from vllm.multimodal import MULTIMODAL_REGISTRY
31
from vllm.multimodal.inputs import MultiModalInputs, NestedTensors
@@ -1421,7 +1422,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1422
"""
1423
Implements the Phi-4-multimodal-instruct model in VLLM.
1424
"""
1424
- # LoRA specific attributes
1425
packed_modules_mapping = {
1426
"qkv_proj": [
1427
"qkv_proj",
@@ -1430,12 +1430,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1430
"gate_up_proj",
1431
],
1432
}
1433
- supported_lora_modules = [
1434
- "qkv_proj", "o_proj", "gate_up_proj", "down_proj"
1435
- ]
1436
- # Phi4MMForCausalLM does not apply LoRA to the embedding layer.
1437
- embedding_modules = {}
1438
- embedding_padding_modules = []
1433
1434
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1435
super().__init__()
@@ -1801,3 +1795,13 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1795
) -> Optional[SamplerOutput]:
1796
next_tokens = self.sampler(logits, sampling_metadata)
1797
return next_tokens
1798
+
1799
+ def get_mm_mapping(self) -> MultiModelKeys:
1800
+ """
1801
+ Get the module prefix in multimodal models
1802
+ """
1803
+ return MultiModelKeys.from_string_field(
1804
+ language_model="model.",
1805
+ connector=["audio_projection_for_vision", "audio_projection"],
1806
+ tower_model=["vision_encoder", "embed_tokens_extend"],
1807
+ )
\ No newline at end of file