A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Update openai client example file for multimodal (#25795)
Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Roger Wang committed
Sep 27, 2025 at 00:57 UTC
23b8ee672d7ce4c383ed1527a7f268c0ca33c16c
1 file changed
+36
-27
examples/online_serving/openai_chat_completion_client_for_multimodal.py
+36
-27
@@ -38,11 +38,13 @@ client = OpenAI(
38
base_url=openai_api_base,
39
)
40
41
+headers = {"User-Agent": "vLLM Example Client"}
42
+
43
44
def encode_base64_content_from_url(content_url: str) -> str:
45
"""Encode a content retrieved from a remote url to base64 format."""
46
45
- with requests.get(content_url) as response:
47
+ with requests.get(content_url, headers=headers) as response:
48
response.raise_for_status()
49
result = base64.b64encode(response.content).decode("utf-8")
50
@@ -50,19 +52,19 @@ def encode_base64_content_from_url(content_url: str) -> str:
52
53
54
# Text-only inference
53
-def run_text_only(model: str) -> None:
55
+def run_text_only(model: str, max_completion_tokens: int) -> None:
56
chat_completion = client.chat.completions.create(
57
messages=[{"role": "user", "content": "What's the capital of France?"}],
58
model=model,
57
- max_completion_tokens=64,
59
+ max_completion_tokens=max_completion_tokens,
60
)
61
62
result = chat_completion.choices[0].message.content
61
- print("Chat completion output:", result)
63
+ print("Chat completion output:\n", result)
64
65
66
# Single-image input inference
65
-def run_single_image(model: str) -> None:
67
+def run_single_image(model: str, max_completion_tokens: int) -> None:
68
## Use image url in the payload
69
image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
70
chat_completion_from_url = client.chat.completions.create(
@@ -79,11 +81,11 @@ def run_single_image(model: str) -> None:
81
}
82
],
83
model=model,
82
- max_completion_tokens=64,
84
+ max_completion_tokens=max_completion_tokens,
85
)
86
87
result = chat_completion_from_url.choices[0].message.content
86
- print("Chat completion output from image url:", result)
88
+ print("Chat completion output from image url:\n", result)
89
90
## Use base64 encoded image in the payload
91
image_base64 = encode_base64_content_from_url(image_url)
@@ -101,7 +103,7 @@ def run_single_image(model: str) -> None:
103
}
104
],
105
model=model,
104
- max_completion_tokens=64,
106
+ max_completion_tokens=max_completion_tokens,
107
)
108
109
result = chat_completion_from_base64.choices[0].message.content
@@ -109,7 +111,7 @@ def run_single_image(model: str) -> None:
111
112
113
# Multi-image input inference
112
-def run_multi_image(model: str) -> None:
114
+def run_multi_image(model: str, max_completion_tokens: int) -> None:
115
image_url_duck = "https://upload.wikimedia.org/wikipedia/commons/d/da/2015_Kaczka_krzy%C5%BCowka_w_wodzie_%28samiec%29.jpg"
116
image_url_lion = "https://upload.wikimedia.org/wikipedia/commons/7/77/002_The_lion_king_Snyggve_in_the_Serengeti_National_Park_Photo_by_Giles_Laurent.jpg"
117
chat_completion_from_url = client.chat.completions.create(
@@ -130,15 +132,15 @@ def run_multi_image(model: str) -> None:
132
}
133
],
134
model=model,
133
- max_completion_tokens=64,
135
+ max_completion_tokens=max_completion_tokens,
136
)
137
138
result = chat_completion_from_url.choices[0].message.content
137
- print("Chat completion output:", result)
139
+ print("Chat completion output:\n", result)
140
141
142
# Video input inference
141
-def run_video(model: str) -> None:
143
+def run_video(model: str, max_completion_tokens: int) -> None:
144
video_url = "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
145
video_base64 = encode_base64_content_from_url(video_url)
146
@@ -157,11 +159,11 @@ def run_video(model: str) -> None:
159
}
160
],
161
model=model,
160
- max_completion_tokens=64,
162
+ max_completion_tokens=max_completion_tokens,
163
)
164
165
result = chat_completion_from_url.choices[0].message.content
164
- print("Chat completion output from image url:", result)
166
+ print("Chat completion output from video url:\n", result)
167
168
## Use base64 encoded video in the payload
169
chat_completion_from_base64 = client.chat.completions.create(
@@ -178,15 +180,15 @@ def run_video(model: str) -> None:
180
}
181
],
182
model=model,
181
- max_completion_tokens=64,
183
+ max_completion_tokens=max_completion_tokens,
184
)
185
186
result = chat_completion_from_base64.choices[0].message.content
185
- print("Chat completion output from base64 encoded image:", result)
187
+ print("Chat completion output from base64 encoded video:\n", result)
188
189
190
# Audio input inference
189
-def run_audio(model: str) -> None:
191
+def run_audio(model: str, max_completion_tokens: int) -> None:
192
from vllm.assets.audio import AudioAsset
193
194
audio_url = AudioAsset("winning_call").url
@@ -211,11 +213,11 @@ def run_audio(model: str) -> None:
213
}
214
],
215
model=model,
214
- max_completion_tokens=64,
216
+ max_completion_tokens=max_completion_tokens,
217
)
218
219
result = chat_completion_from_base64.choices[0].message.content
218
- print("Chat completion output from input audio:", result)
220
+ print("Chat completion output from input audio:\n", result)
221
222
# HTTP URL
223
chat_completion_from_url = client.chat.completions.create(
@@ -235,11 +237,11 @@ def run_audio(model: str) -> None:
237
}
238
],
239
model=model,
238
- max_completion_tokens=64,
240
+ max_completion_tokens=max_completion_tokens,
241
)
242
243
result = chat_completion_from_url.choices[0].message.content
242
- print("Chat completion output from audio url:", result)
244
+ print("Chat completion output from audio url:\n", result)
245
246
# base64 URL
247
chat_completion_from_base64 = client.chat.completions.create(
@@ -259,14 +261,14 @@ def run_audio(model: str) -> None:
261
}
262
],
263
model=model,
262
- max_completion_tokens=64,
264
+ max_completion_tokens=max_completion_tokens,
265
)
266
267
result = chat_completion_from_base64.choices[0].message.content
266
- print("Chat completion output from base64 encoded audio:", result)
268
+ print("Chat completion output from base64 encoded audio:\n", result)
269
270
269
-def run_multi_audio(model: str) -> None:
271
+def run_multi_audio(model: str, max_completion_tokens: int) -> None:
272
from vllm.assets.audio import AudioAsset
273
274
# Two different audios to showcase batched inference.
@@ -300,11 +302,11 @@ def run_multi_audio(model: str) -> None:
302
}
303
],
304
model=model,
303
- max_completion_tokens=64,
305
+ max_completion_tokens=max_completion_tokens,
306
)
307
308
result = chat_completion_from_base64.choices[0].message.content
307
- print("Chat completion output from input audio:", result)
309
+ print("Chat completion output from input audio:\n", result)
310
311
312
example_function_map = {
@@ -330,13 +332,20 @@ def parse_args():
332
choices=list(example_function_map.keys()),
333
help="Conversation type with multimodal data.",
334
)
335
+ parser.add_argument(
336
+ "--max-completion-tokens",
337
+ "-n",
338
+ type=int,
339
+ default=128,
340
+ help="Maximum number of tokens to generate for each completion.",
341
+ )
342
return parser.parse_args()
343
344
345
def main(args) -> None:
346
chat_type = args.chat_type
347
model = get_first_model(client)
339
- example_function_map[chat_type](model)
348
+ example_function_map[chat_type](model, args.max_completion_tokens)
349
350
351
if __name__ == "__main__":