A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
[doc] fix multimodal example script (#18089)
Signed-off-by: David Xia <david@davidxia.com>
David Xia committed
May 16, 2025 at 02:05 UTC
5c04bb8b863bfdef8122b193631479315cc764f5
2 files changed
+40
-11
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+15
-11
@@ -1,5 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-"""An example showing how to use vLLM to serve multimodal models
2
+"""An example showing how to use vLLM to serve multimodal models
3
and run online serving with OpenAI client.
4
5
Launch the vLLM server with the following command:
@@ -12,12 +12,18 @@ vllm serve microsoft/Phi-3.5-vision-instruct --task generate \
12
--trust-remote-code --max-model-len 4096 --limit-mm-per-prompt '{"image":2}'
13
14
(audio inference with Ultravox)
15
-vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b --max-model-len 4096
15
+vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b \
16
+ --max-model-len 4096 --trust-remote-code
17
+
18
+run the script with
19
+python openai_chat_completion_client_for_multimodal.py --chat-type audio
20
"""
21
+
22
import base64
23
24
import requests
25
from openai import OpenAI
26
+from utils import get_first_model
27
28
from vllm.utils import FlexibleArgumentParser
29
@@ -31,9 +37,6 @@ client = OpenAI(
37
base_url=openai_api_base,
38
)
39
34
-models = client.models.list()
35
-model = models.data[0].id
36
-
40
41
def encode_base64_content_from_url(content_url: str) -> str:
42
"""Encode a content retrieved from a remote url to base64 format."""
@@ -46,7 +49,7 @@ def encode_base64_content_from_url(content_url: str) -> str:
49
50
51
# Text-only inference
49
-def run_text_only() -> None:
52
+def run_text_only(model: str) -> None:
53
chat_completion = client.chat.completions.create(
54
messages=[{
55
"role": "user",
@@ -61,7 +64,7 @@ def run_text_only() -> None:
64
65
66
# Single-image input inference
64
-def run_single_image() -> None:
67
+def run_single_image(model: str) -> None:
68
69
## Use image url in the payload
70
image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
@@ -117,7 +120,7 @@ def run_single_image() -> None:
120
121
122
# Multi-image input inference
120
-def run_multi_image() -> None:
123
+def run_multi_image(model: str) -> None:
124
image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
125
image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
126
chat_completion_from_url = client.chat.completions.create(
@@ -152,7 +155,7 @@ def run_multi_image() -> None:
155
156
157
# Video input inference
155
-def run_video() -> None:
158
+def run_video(model: str) -> None:
159
video_url = "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
160
video_base64 = encode_base64_content_from_url(video_url)
161
@@ -208,7 +211,7 @@ def run_video() -> None:
211
212
213
# Audio input inference
211
-def run_audio() -> None:
214
+def run_audio(model: str) -> None:
215
from vllm.assets.audio import AudioAsset
216
217
audio_url = AudioAsset("winning_call").url
@@ -318,7 +321,8 @@ def parse_args():
321
322
def main(args) -> None:
323
chat_type = args.chat_type
321
- example_function_map[chat_type]()
324
+ model = get_first_model(client)
325
+ example_function_map[chat_type](model)
326
327
328
if __name__ == "__main__":
examples/online_serving/utils.py
new
+25
@@ -0,0 +1,25 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from openai import APIConnectionError, OpenAI
3
+from openai.pagination import SyncPage
4
+from openai.types.model import Model
5
+
6
+
7
+def get_first_model(client: OpenAI) -> str:
8
+ """
9
+ Get the first model from the vLLM server.
10
+ """
11
+ try:
12
+ models: SyncPage[Model] = client.models.list()
13
+ except APIConnectionError as e:
14
+ raise RuntimeError(
15
+ "Failed to get the list of models from the vLLM server at "
16
+ f"{client.base_url} with API key {client.api_key}. Check\n"
17
+ "1. the server is running\n"
18
+ "2. the server URL is correct\n"
19
+ "3. the API key is correct") from e
20
+
21
+ if len(models.data) == 0:
22
+ raise RuntimeError(
23
+ f"No models found on the vLLM server at {client.base_url}")
24
+
25
+ return models.data[0].id