@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b85a977822
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Doc] Add video example to openai client for multimodal (#11521)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed Dec 27, 2024 at 01:31 UTC b85a977822c4216430c5a27a2fc47c93277e4b29
2 files changed +114 -11
docs/source/usage/multimodal_inputs.md
+49 -3
@@ -294,12 +294,58 @@ $ export VLLM_IMAGE_FETCH_TIMEOUT=<timeout>
294
295 ### Video
296
297 -Instead of {code}`image_url`, you can pass a video file via {code}`video_url`.
297 +Instead of {code}`image_url`, you can pass a video file via {code}`video_url`. Here is a simple example using [LLaVA-OneVision](https://huggingface.co/llava-hf/llava-onevision-qwen2-0.5b-ov-hf).
298
299 -You can use [these tests](gh-file:entrypoints/openai/test_video.py) as reference.
299 +First, launch the OpenAI-compatible server:
300 +
301 +```bash
302 +vllm serve llava-hf/llava-onevision-qwen2-0.5b-ov-hf --task generate --max-model-len 8192
303 +```
304 +
305 +Then, you can use the OpenAI client as follows:
306 +```python
307 +from openai import OpenAI
308 +
309 +openai_api_key = "EMPTY"
310 +openai_api_base = "http://localhost:8000/v1"
311 +
312 +client = OpenAI(
313 + api_key=openai_api_key,
314 + base_url=openai_api_base,
315 +)
316 +
317 +video_url = "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
318 +
319 +## Use video url in the payload
320 +chat_completion_from_url = client.chat.completions.create(
321 + messages=[{
322 + "role":
323 + "user",
324 + "content": [
325 + {
326 + "type": "text",
327 + "text": "What's in this video?"
328 + },
329 + {
330 + "type": "video_url",
331 + "video_url": {
332 + "url": video_url
333 + },
334 + },
335 + ],
336 + }],
337 + model=model,
338 + max_completion_tokens=64,
339 +)
340 +
341 +result = chat_completion_from_url.choices[0].message.content
342 +print("Chat completion output from image url:", result)
343 +```
344 +
345 +Full example: <gh-file:examples/openai_chat_completion_client_for_multimodal.py>
346
347 ````{note}
302 -By default, the timeout for fetching videos through HTTP URL url is `30` seconds.
348 +By default, the timeout for fetching videos through HTTP URL is `30` seconds.
349 You can override this by setting the environment variable:
350
351 ```console
examples/openai_chat_completion_client_for_multimodal.py
+65 -8
@@ -18,7 +18,6 @@ import base64
18 import requests
19 from openai import OpenAI
20
21 -from vllm.assets.audio import AudioAsset
21 from vllm.utils import FlexibleArgumentParser
22
23 # Modify OpenAI's API key and API base to use vLLM's API server.
@@ -151,8 +150,66 @@ def run_multi_image() -> None:
150 print("Chat completion output:", result)
151
152
153 +# Video input inference
154 +def run_video() -> None:
155 + video_url = "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
156 + video_base64 = encode_base64_content_from_url(video_url)
157 +
158 + ## Use video url in the payload
159 + chat_completion_from_url = client.chat.completions.create(
160 + messages=[{
161 + "role":
162 + "user",
163 + "content": [
164 + {
165 + "type": "text",
166 + "text": "What's in this video?"
167 + },
168 + {
169 + "type": "video_url",
170 + "video_url": {
171 + "url": video_url
172 + },
173 + },
174 + ],
175 + }],
176 + model=model,
177 + max_completion_tokens=64,
178 + )
179 +
180 + result = chat_completion_from_url.choices[0].message.content
181 + print("Chat completion output from image url:", result)
182 +
183 + ## Use base64 encoded video in the payload
184 + chat_completion_from_base64 = client.chat.completions.create(
185 + messages=[{
186 + "role":
187 + "user",
188 + "content": [
189 + {
190 + "type": "text",
191 + "text": "What's in this video?"
192 + },
193 + {
194 + "type": "video_url",
195 + "video_url": {
196 + "url": f"data:video/mp4;base64,{video_base64}"
197 + },
198 + },
199 + ],
200 + }],
201 + model=model,
202 + max_completion_tokens=64,
203 + )
204 +
205 + result = chat_completion_from_base64.choices[0].message.content
206 + print("Chat completion output from base64 encoded image:", result)
207 +
208 +
209 # Audio input inference
210 def run_audio() -> None:
211 + from vllm.assets.audio import AudioAsset
212 +
213 audio_url = AudioAsset("winning_call").url
214 audio_base64 = encode_base64_content_from_url(audio_url)
215
@@ -240,6 +297,7 @@ example_function_map = {
297 "text-only": run_text_only,
298 "single-image": run_single_image,
299 "multi-image": run_multi_image,
300 + "video": run_video,
301 "audio": run_audio,
302 }
303
@@ -253,12 +311,11 @@ if __name__ == "__main__":
311 parser = FlexibleArgumentParser(
312 description='Demo on using OpenAI client for online inference with '
313 'multimodal language models served with vLLM.')
256 - parser.add_argument(
257 - '--chat-type',
258 - '-c',
259 - type=str,
260 - default="single-image",
261 - choices=["text-only", "single-image", "multi-image", "audio"],
262 - help='Conversation type with multimodal data.')
314 + parser.add_argument('--chat-type',
315 + '-c',
316 + type=str,
317 + default="single-image",
318 + choices=list(example_function_map.keys()),
319 + help='Conversation type with multimodal data.')
320 args = parser.parse_args()
321 main(args)