@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 413 lines 13.8 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """An example showing how to use vLLM to serve multimodal models
4 and run online serving with OpenAI client.
5
6 Launch the vLLM server with the following command:
7
8 (single image inference with Llava)
9 vllm serve llava-hf/llava-1.5-7b-hf
10
11 (multi-image inference with Phi-3.5-vision-instruct)
12 vllm serve microsoft/Phi-3.5-vision-instruct --runner generate \
13 --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt.image 2
14
15 (audio inference with Ultravox)
16 vllm serve fixie-ai/ultravox-v0_5-llama-3_2-1b \
17 --max-model-len 4096 --trust-remote-code
18
19 run the script with
20 python openai_chat_completion_client_for_multimodal.py --chat-type audio
21 """
22
23 import os
24
25 import pybase64 as base64
26 import requests
27 from openai import OpenAI
28
29 from vllm.utils.argparse_utils import FlexibleArgumentParser
30
31 # Modify OpenAI's API key and API base to use vLLM's API server.
32 openai_api_key = "EMPTY"
33 openai_api_base = "http://localhost:8000/v1"
34
35 client = OpenAI(
36 # defaults to os.environ.get("OPENAI_API_KEY")
37 api_key=openai_api_key,
38 base_url=openai_api_base,
39 )
40
41 headers = {"User-Agent": "vLLM Example Client"}
42
43
44 def encode_base64_content_from_url(content_url: str) -> str:
45 """Encode a content retrieved from a remote url to base64 format."""
46 with requests.get(content_url, headers=headers) as response:
47 response.raise_for_status()
48 result = base64.b64encode(response.content).decode("utf-8")
49
50 return result
51
52
53 def encode_base64_content_from_file(file_path: str) -> str:
54 """Encode a local file content to base64 format."""
55 with open(file_path, "rb") as file:
56 file_content = file.read()
57 result = base64.b64encode(file_content).decode("utf-8")
58
59 return result
60
61
62 # Text-only inference
63 def run_text_only(model: str, max_completion_tokens: int) -> None:
64 chat_completion = client.chat.completions.create(
65 messages=[{"role": "user", "content": "What's the capital of France?"}],
66 model=model,
67 max_completion_tokens=max_completion_tokens,
68 )
69
70 result = chat_completion.choices[0].message.content
71 print("Chat completion output:\n", result)
72
73
74 # Single-image input inference
75 def run_single_image(model: str, max_completion_tokens: int) -> None:
76 ## Use image url in the payload
77 image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
78 image_file = "/path/to/image.jpg" # local file
79 chat_completion_from_url = client.chat.completions.create(
80 messages=[
81 {
82 "role": "user",
83 "content": [
84 {"type": "text", "text": "What's in this image?"},
85 {
86 "type": "image_url",
87 "image_url": {"url": image_url},
88 },
89 ],
90 }
91 ],
92 model=model,
93 max_completion_tokens=max_completion_tokens,
94 )
95
96 result = chat_completion_from_url.choices[0].message.content
97 print("Chat completion output from image url:\n", result)
98
99 ## Use local image url in the payload
100 # Launch the API server/engine with the --allowed-local-media-path argument.
101 if os.path.exists(image_file):
102 chat_completion_from_local_image_url = client.chat.completions.create(
103 messages=[
104 {
105 "role": "user",
106 "content": [
107 {"type": "text", "text": "What's in this image?"},
108 {
109 "type": "image_url",
110 "image_url": {"url": f"file://{image_file}"},
111 },
112 ],
113 }
114 ],
115 model=model,
116 max_completion_tokens=max_completion_tokens,
117 )
118 result = chat_completion_from_local_image_url.choices[0].message.content
119 print("Chat completion output from local image file:\n", result)
120 else:
121 print(f"Local image file not found at {image_file}, skipping local file test.")
122
123 ## Use base64 encoded image in the payload
124 image_base64 = encode_base64_content_from_url(image_url)
125 chat_completion_from_base64 = client.chat.completions.create(
126 messages=[
127 {
128 "role": "user",
129 "content": [
130 {"type": "text", "text": "What's in this image?"},
131 {
132 "type": "image_url",
133 "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"},
134 },
135 ],
136 }
137 ],
138 model=model,
139 max_completion_tokens=max_completion_tokens,
140 )
141
142 result = chat_completion_from_base64.choices[0].message.content
143 print("Chat completion output from base64 encoded image:", result)
144
145 ## Use base64 encoded local image in the payload
146 if os.path.exists(image_file):
147 local_image_base64 = encode_base64_content_from_file(image_file)
148 chat_completion_from_local_image_base64 = client.chat.completions.create(
149 messages=[
150 {
151 "role": "user",
152 "content": [
153 {"type": "text", "text": "What's in this image?"},
154 {
155 "type": "image_url",
156 "image_url": {
157 "url": f"data:image/jpeg;base64,{local_image_base64}"
158 },
159 },
160 ],
161 }
162 ],
163 model=model,
164 max_completion_tokens=max_completion_tokens,
165 )
166
167 result = chat_completion_from_local_image_base64.choices[0].message.content
168 print("Chat completion output from base64 encoded local image:", result)
169 else:
170 print(f"Local image file not found at {image_file}, skipping local file test.")
171
172
173 # Multi-image input inference
174 def run_multi_image(model: str, max_completion_tokens: int) -> None:
175 image_url_duck = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/duck.jpg"
176 image_url_lion = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/lion.jpg"
177 chat_completion_from_url = client.chat.completions.create(
178 messages=[
179 {
180 "role": "user",
181 "content": [
182 {"type": "text", "text": "What are the animals in these images?"},
183 {
184 "type": "image_url",
185 "image_url": {"url": image_url_duck},
186 },
187 {
188 "type": "image_url",
189 "image_url": {"url": image_url_lion},
190 },
191 ],
192 }
193 ],
194 model=model,
195 max_completion_tokens=max_completion_tokens,
196 )
197
198 result = chat_completion_from_url.choices[0].message.content
199 print("Chat completion output:\n", result)
200
201
202 # Video input inference
203 def run_video(model: str, max_completion_tokens: int) -> None:
204 video_url = "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
205 video_base64 = encode_base64_content_from_url(video_url)
206
207 ## Use video url in the payload
208 chat_completion_from_url = client.chat.completions.create(
209 messages=[
210 {
211 "role": "user",
212 "content": [
213 {"type": "text", "text": "What's in this video?"},
214 {
215 "type": "video_url",
216 "video_url": {"url": video_url},
217 },
218 ],
219 }
220 ],
221 model=model,
222 max_completion_tokens=max_completion_tokens,
223 )
224
225 result = chat_completion_from_url.choices[0].message.content
226 print("Chat completion output from video url:\n", result)
227
228 ## Use base64 encoded video in the payload
229 chat_completion_from_base64 = client.chat.completions.create(
230 messages=[
231 {
232 "role": "user",
233 "content": [
234 {"type": "text", "text": "What's in this video?"},
235 {
236 "type": "video_url",
237 "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
238 },
239 ],
240 }
241 ],
242 model=model,
243 max_completion_tokens=max_completion_tokens,
244 )
245
246 result = chat_completion_from_base64.choices[0].message.content
247 print("Chat completion output from base64 encoded video:\n", result)
248
249
250 # Audio input inference
251 def run_audio(model: str, max_completion_tokens: int) -> None:
252 from vllm.assets.audio import AudioAsset
253
254 audio_url = AudioAsset("winning_call").url
255 audio_base64 = encode_base64_content_from_url(audio_url)
256
257 # OpenAI-compatible schema (`input_audio`)
258 chat_completion_from_base64 = client.chat.completions.create(
259 messages=[
260 {
261 "role": "user",
262 "content": [
263 {"type": "text", "text": "What's in this audio?"},
264 {
265 "type": "input_audio",
266 "input_audio": {
267 # Any format supported by soundfile/PyAV is supported
268 "data": audio_base64,
269 "format": "wav",
270 },
271 },
272 ],
273 }
274 ],
275 model=model,
276 max_completion_tokens=max_completion_tokens,
277 )
278
279 result = chat_completion_from_base64.choices[0].message.content
280 print("Chat completion output from input audio:\n", result)
281
282 # HTTP URL
283 chat_completion_from_url = client.chat.completions.create(
284 messages=[
285 {
286 "role": "user",
287 "content": [
288 {"type": "text", "text": "What's in this audio?"},
289 {
290 "type": "audio_url",
291 "audio_url": {
292 # Any format supported by soundfile/PyAV is supported
293 "url": audio_url
294 },
295 },
296 ],
297 }
298 ],
299 model=model,
300 max_completion_tokens=max_completion_tokens,
301 )
302
303 result = chat_completion_from_url.choices[0].message.content
304 print("Chat completion output from audio url:\n", result)
305
306 # base64 URL
307 chat_completion_from_base64 = client.chat.completions.create(
308 messages=[
309 {
310 "role": "user",
311 "content": [
312 {"type": "text", "text": "What's in this audio?"},
313 {
314 "type": "audio_url",
315 "audio_url": {
316 # Any format supported by soundfile/PyAV is supported
317 "url": f"data:audio/ogg;base64,{audio_base64}"
318 },
319 },
320 ],
321 }
322 ],
323 model=model,
324 max_completion_tokens=max_completion_tokens,
325 )
326
327 result = chat_completion_from_base64.choices[0].message.content
328 print("Chat completion output from base64 encoded audio:\n", result)
329
330
331 def run_multi_audio(model: str, max_completion_tokens: int) -> None:
332 from vllm.assets.audio import AudioAsset
333
334 # Two different audios to showcase batched inference.
335 audio_url = AudioAsset("winning_call").url
336 audio_base64 = encode_base64_content_from_url(audio_url)
337 audio_url2 = AudioAsset("azacinto_foscolo").url
338 audio_base64_2 = encode_base64_content_from_url(audio_url2)
339
340 # OpenAI-compatible schema (`input_audio`)
341 chat_completion_from_base64 = client.chat.completions.create(
342 messages=[
343 {
344 "role": "user",
345 "content": [
346 {"type": "text", "text": "Are these two audios the same?"},
347 {
348 "type": "input_audio",
349 "input_audio": {
350 "data": audio_base64,
351 "format": "wav",
352 },
353 },
354 {
355 "type": "input_audio",
356 "input_audio": {
357 "data": audio_base64_2,
358 "format": "wav",
359 },
360 },
361 ],
362 }
363 ],
364 model=model,
365 max_completion_tokens=max_completion_tokens,
366 )
367
368 result = chat_completion_from_base64.choices[0].message.content
369 print("Chat completion output from input audio:\n", result)
370
371
372 example_function_map = {
373 "text-only": run_text_only,
374 "single-image": run_single_image,
375 "multi-image": run_multi_image,
376 "multi-audio": run_multi_audio,
377 "video": run_video,
378 "audio": run_audio,
379 }
380
381
382 def parse_args():
383 parser = FlexibleArgumentParser(
384 description="Demo on using OpenAI client for online serving with "
385 "multimodal language models served with vLLM."
386 )
387 parser.add_argument(
388 "--chat-type",
389 "-c",
390 type=str,
391 default="single-image",
392 choices=list(example_function_map.keys()),
393 help="Conversation type with multimodal data.",
394 )
395 parser.add_argument(
396 "--max-completion-tokens",
397 "-n",
398 type=int,
399 default=128,
400 help="Maximum number of tokens to generate for each completion.",
401 )
402 return parser.parse_args()
403
404
405 def main(args) -> None:
406 chat_type = args.chat_type
407 model = client.models.list().data[0].id
408 example_function_map[chat_type](model, args.max_completion_tokens)
409
410
411 if __name__ == "__main__":
412 args = parse_args()
413 main(args)