@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 329df38f1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Misc] Update Phi-3-Vision Example (#5981)

Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Roger Wang committed Jun 28, 2024 at 23:34 UTC 329df38f1a931215062d7b43660ceee1f83c0ab5
1 file changed +5 -3
examples/phi3v_example.py
+5 -3
@@ -10,8 +10,10 @@ from vllm.multimodal.image import ImagePixelData
10 def run_phi3v():
11 model_path = "microsoft/Phi-3-vision-128k-instruct"
12
13 - # Note: The model has 128k context length by default which may cause OOM
14 - # In this example, we override max_model_len to 2048.
13 + # Note: The default setting of max_num_seqs (256) and
14 + # max_model_len (128k) for this model may cause OOM.
15 + # In this example, we override max_num_seqs to 5 while
16 + # keeping the original context length of 128k.
17 llm = LLM(
18 model=model_path,
19 trust_remote_code=True,
@@ -19,7 +21,7 @@ def run_phi3v():
21 image_token_id=32044,
22 image_input_shape="1,3,1008,1344",
23 image_feature_size=1921,
22 - max_model_len=2048,
24 + max_num_seqs=5,
25 )
26
27 image = Image.open("images/cherry_blossom.jpg")