A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Misc] Update Phi-3-Vision Example (#5981)
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Roger Wang committed
Jun 28, 2024 at 23:34 UTC
329df38f1a931215062d7b43660ceee1f83c0ab5
1 file changed
+5
-3
examples/phi3v_example.py
+5
-3
@@ -10,8 +10,10 @@ from vllm.multimodal.image import ImagePixelData
10
def run_phi3v():
11
model_path = "microsoft/Phi-3-vision-128k-instruct"
12
13
- # Note: The model has 128k context length by default which may cause OOM
14
- # In this example, we override max_model_len to 2048.
13
+ # Note: The default setting of max_num_seqs (256) and
14
+ # max_model_len (128k) for this model may cause OOM.
15
+ # In this example, we override max_num_seqs to 5 while
16
+ # keeping the original context length of 128k.
17
llm = LLM(
18
model=model_path,
19
trust_remote_code=True,
@@ -19,7 +21,7 @@ def run_phi3v():
21
image_token_id=32044,
22
image_input_shape="1,3,1008,1344",
23
image_feature_size=1921,
22
- max_model_len=2048,
24
+ max_num_seqs=5,
25
)
26
27
image = Image.open("images/cherry_blossom.jpg")