@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28b2877d30
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

Online video support for VLMs (#10020)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: litianjian <litianjian@bytedance.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

litianjian committed Nov 8, 2024 at 04:25 UTC 28b2877d303caa6b2febc9d0b425f17828634a4c
12 files changed +598 -31
docs/source/conf.py
+1
@@ -116,6 +116,7 @@ autodoc_mock_imports = [
116 "soundfile",
117 "gguf",
118 "lark",
119 + "decord",
120 ]
121
122 for mock_target in autodoc_mock_imports:
requirements-test.in
+4 -2
@@ -8,6 +8,7 @@ pytest-shard
8
9 # testing utils
10 awscli
11 +decord # required for video tests
12 einops # required for MPT, qwen-vl and Mamba
13 httpx
14 librosa # required for audio tests
@@ -15,12 +16,13 @@ opencv-python # required for video tests
16 peft
17 requests
18 ray[adag]==2.35
18 -sentence-transformers # required for embedding
19 -soundfile # required for audio test
19 +sentence-transformers # required for embedding tests
20 +soundfile # required for audio tests
21 timm # required for internvl test
22 torch==2.5.1
23 transformers_stream_generator # required for qwen-vl test
24 matplotlib # required for qwen-vl test
25 +mistral_common[opencv] >= 1.4.4 # required for pixtral test
26 datamodel_code_generator # required for minicpm3 test
27 lm-eval[api]==0.4.4 # required for model evaluation test
28
requirements-test.txt
+50 -7
@@ -1,8 +1,8 @@
1 #
2 -# This file is autogenerated by pip-compile with Python 3.12
2 +# This file is autogenerated by pip-compile with Python 3.9
3 # by the following command:
4 #
5 -# pip-compile --output-file=requirements-test.txt requirements-test.in
5 +# pip-compile requirements-test.in
6 #
7 absl-py==2.1.0
8 # via rouge-score
@@ -28,6 +28,10 @@ anyio==4.6.2.post1
28 # via httpx
29 argcomplete==3.5.1
30 # via datamodel-code-generator
31 +async-timeout==4.0.3
32 + # via
33 + # aiohttp
34 + # redis
35 attrs==24.2.0
36 # via
37 # aiohttp
@@ -90,6 +94,8 @@ datasets==3.0.2
94 # lm-eval
95 decorator==5.1.1
96 # via librosa
97 +decord==0.6.0
98 + # via -r requirements-test.in
99 dill==0.3.8
100 # via
101 # datasets
@@ -106,6 +112,10 @@ email-validator==2.2.0
112 # via pydantic
113 evaluate==0.4.3
114 # via lm-eval
115 +exceptiongroup==1.2.2
116 + # via
117 + # anyio
118 + # pytest
119 fastrlock==0.8.2
120 # via cupy-cuda12x
121 filelock==3.16.1
@@ -156,6 +166,8 @@ idna==3.10
166 # httpx
167 # requests
168 # yarl
169 +importlib-resources==6.4.5
170 + # via matplotlib
171 inflect==5.6.2
172 # via datamodel-code-generator
173 iniconfig==2.0.0
@@ -178,7 +190,9 @@ joblib==1.4.2
190 jsonlines==4.0.0
191 # via lm-eval
192 jsonschema==4.23.0
181 - # via ray
193 + # via
194 + # mistral-common
195 + # ray
196 jsonschema-specifications==2024.10.1
197 # via jsonschema
198 kiwisolver==1.4.7
@@ -204,6 +218,10 @@ mbstrdecoder==1.1.3
218 # dataproperty
219 # pytablewriter
220 # typepy
221 +mistral-common[opencv]==1.4.4
222 + # via
223 + # -r requirements-test.in
224 + # mistral-common
225 more-itertools==10.5.0
226 # via lm-eval
227 mpmath==1.3.0
@@ -238,12 +256,15 @@ numpy==1.26.4
256 # contourpy
257 # cupy-cuda12x
258 # datasets
259 + # decord
260 # evaluate
261 # librosa
262 # matplotlib
263 + # mistral-common
264 # numba
265 # numexpr
266 # opencv-python
267 + # opencv-python-headless
268 # pandas
269 # peft
270 # rouge-score
@@ -288,6 +309,8 @@ nvidia-nvtx-cu12==12.4.127
309 # via torch
310 opencv-python==4.10.0.84
311 # via -r requirements-test.in
312 +opencv-python-headless==4.10.0.84
313 + # via mistral-common
314 packaging==24.1
315 # via
316 # accelerate
@@ -317,9 +340,10 @@ peft==0.13.2
340 # via
341 # -r requirements-test.in
342 # lm-eval
320 -pillow==11.0.0
343 +pillow==10.4.0
344 # via
345 # matplotlib
346 + # mistral-common
347 # sentence-transformers
348 # torchvision
349 platformdirs==4.3.6
@@ -354,7 +378,9 @@ pybind11==2.13.6
378 pycparser==2.22
379 # via cffi
380 pydantic[email]==2.9.2
357 - # via datamodel-code-generator
381 + # via
382 + # datamodel-code-generator
383 + # mistral-common
384 pydantic-core==2.23.4
385 # via pydantic
386 pyparsing==3.2.0
@@ -420,6 +446,7 @@ requests==2.32.3
446 # evaluate
447 # huggingface-hub
448 # lm-eval
449 + # mistral-common
450 # pooch
451 # ray
452 # tiktoken
@@ -456,6 +483,8 @@ scipy==1.13.1
483 # sentence-transformers
484 sentence-transformers==3.2.1
485 # via -r requirements-test.in
486 +sentencepiece==0.2.0
487 + # via mistral-common
488 six==1.16.0
489 # via
490 # python-dateutil
@@ -486,12 +515,20 @@ tensorizer==2.9.0
515 # via -r requirements-test.in
516 threadpoolctl==3.5.0
517 # via scikit-learn
489 -tiktoken==0.8.0
490 - # via lm-eval
518 +tiktoken==0.7.0
519 + # via
520 + # lm-eval
521 + # mistral-common
522 timm==1.0.11
523 # via -r requirements-test.in
524 tokenizers==0.20.1
525 # via transformers
526 +toml==0.10.2
527 + # via datamodel-code-generator
528 +tomli==2.0.2
529 + # via
530 + # black
531 + # pytest
532 torch==2.5.1
533 # via
534 # -r requirements-test.in
@@ -535,8 +572,12 @@ typepy[datetime]==1.3.2
572 # tabledata
573 typing-extensions==4.12.2
574 # via
575 + # anyio
576 + # black
577 # huggingface-hub
578 # librosa
579 + # mistral-common
580 + # multidict
581 # pydantic
582 # pydantic-core
583 # torch
@@ -554,6 +595,8 @@ xxhash==3.5.0
595 # evaluate
596 yarl==1.17.1
597 # via aiohttp
598 +zipp==3.20.2
599 + # via importlib-resources
600 zstandard==0.23.0
601 # via lm-eval
602
setup.py
+2 -1
@@ -554,7 +554,8 @@ setup(
554 ext_modules=ext_modules,
555 extras_require={
556 "tensorizer": ["tensorizer>=2.9.0"],
557 - "audio": ["librosa", "soundfile"] # Required for audio processing
557 + "audio": ["librosa", "soundfile"], # Required for audio processing
558 + "video": ["decord"] # Required for video processing
559 },
560 cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
561 package_data=package_data,
tests/entrypoints/openai/test_video.py new
+345
@@ -0,0 +1,345 @@
1 +from typing import Dict, List
2 +
3 +import openai
4 +import pytest
5 +import pytest_asyncio
6 +
7 +from vllm.multimodal.utils import encode_video_base64, fetch_video
8 +
9 +from ...utils import RemoteOpenAIServer
10 +
11 +MODEL_NAME = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
12 +MAXIMUM_VIDEOS = 4
13 +
14 +TEST_VIDEO_URLS = [
15 + "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4",
16 + "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4",
17 + "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4",
18 + "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4",
19 +]
20 +
21 +
22 +@pytest.fixture(scope="module")
23 +def server():
24 + args = [
25 + "--task",
26 + "generate",
27 + "--dtype",
28 + "bfloat16",
29 + "--max-model-len",
30 + "32768",
31 + "--max-num-seqs",
32 + "2",
33 + "--enforce-eager",
34 + "--trust-remote-code",
35 + "--limit-mm-per-prompt",
36 + f"video={MAXIMUM_VIDEOS}",
37 + ]
38 +
39 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
40 + yield remote_server
41 +
42 +
43 +@pytest_asyncio.fixture
44 +async def client(server):
45 + async with server.get_async_client() as async_client:
46 + yield async_client
47 +
48 +
49 +@pytest.fixture(scope="session")
50 +def base64_encoded_video() -> Dict[str, str]:
51 + return {
52 + video_url: encode_video_base64(fetch_video(video_url))
53 + for video_url in TEST_VIDEO_URLS
54 + }
55 +
56 +
57 +@pytest.mark.asyncio
58 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
59 +@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
60 +async def test_single_chat_session_video(client: openai.AsyncOpenAI,
61 + model_name: str, video_url: str):
62 + messages = [{
63 + "role":
64 + "user",
65 + "content": [
66 + {
67 + "type": "video_url",
68 + "video_url": {
69 + "url": video_url
70 + }
71 + },
72 + {
73 + "type": "text",
74 + "text": "What's in this video?"
75 + },
76 + ],
77 + }]
78 +
79 + # test single completion
80 + chat_completion = await client.chat.completions.create(
81 + model=model_name,
82 + messages=messages,
83 + max_completion_tokens=10,
84 + logprobs=True,
85 + top_logprobs=5)
86 + assert len(chat_completion.choices) == 1
87 +
88 + choice = chat_completion.choices[0]
89 + assert choice.finish_reason == "length"
90 + assert chat_completion.usage == openai.types.CompletionUsage(
91 + completion_tokens=10, prompt_tokens=6299, total_tokens=6309)
92 +
93 + message = choice.message
94 + message = chat_completion.choices[0].message
95 + assert message.content is not None and len(message.content) >= 10
96 + assert message.role == "assistant"
97 + messages.append({"role": "assistant", "content": message.content})
98 +
99 + # test multi-turn dialogue
100 + messages.append({"role": "user", "content": "express your result in json"})
101 + chat_completion = await client.chat.completions.create(
102 + model=model_name,
103 + messages=messages,
104 + max_completion_tokens=10,
105 + )
106 + message = chat_completion.choices[0].message
107 + assert message.content is not None and len(message.content) >= 0
108 +
109 +
110 +@pytest.mark.asyncio
111 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
112 +@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
113 +async def test_single_chat_session_video_beamsearch(client: openai.AsyncOpenAI,
114 + model_name: str,
115 + video_url: str):
116 + messages = [{
117 + "role":
118 + "user",
119 + "content": [
120 + {
121 + "type": "video_url",
122 + "video_url": {
123 + "url": video_url
124 + }
125 + },
126 + {
127 + "type": "text",
128 + "text": "What's in this video?"
129 + },
130 + ],
131 + }]
132 +
133 + chat_completion = await client.chat.completions.create(
134 + model=model_name,
135 + messages=messages,
136 + n=2,
137 + max_completion_tokens=10,
138 + logprobs=True,
139 + top_logprobs=5,
140 + extra_body=dict(use_beam_search=True))
141 + assert len(chat_completion.choices) == 2
142 + assert chat_completion.choices[
143 + 0].message.content != chat_completion.choices[1].message.content
144 +
145 +
146 +@pytest.mark.asyncio
147 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
148 +@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
149 +async def test_single_chat_session_video_base64encoded(
150 + client: openai.AsyncOpenAI, model_name: str, video_url: str,
151 + base64_encoded_video: Dict[str, str]):
152 +
153 + messages = [{
154 + "role":
155 + "user",
156 + "content": [
157 + {
158 + "type": "video_url",
159 + "video_url": {
160 + "url":
161 + f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"
162 + }
163 + },
164 + {
165 + "type": "text",
166 + "text": "What's in this video?"
167 + },
168 + ],
169 + }]
170 +
171 + # test single completion
172 + chat_completion = await client.chat.completions.create(
173 + model=model_name,
174 + messages=messages,
175 + max_completion_tokens=10,
176 + logprobs=True,
177 + top_logprobs=5)
178 + assert len(chat_completion.choices) == 1
179 +
180 + choice = chat_completion.choices[0]
181 + assert choice.finish_reason == "length"
182 + assert chat_completion.usage == openai.types.CompletionUsage(
183 + completion_tokens=10, prompt_tokens=6299, total_tokens=6309)
184 +
185 + message = choice.message
186 + message = chat_completion.choices[0].message
187 + assert message.content is not None and len(message.content) >= 10
188 + assert message.role == "assistant"
189 + messages.append({"role": "assistant", "content": message.content})
190 +
191 + # test multi-turn dialogue
192 + messages.append({"role": "user", "content": "express your result in json"})
193 + chat_completion = await client.chat.completions.create(
194 + model=model_name,
195 + messages=messages,
196 + max_completion_tokens=10,
197 + )
198 + message = chat_completion.choices[0].message
199 + assert message.content is not None and len(message.content) >= 0
200 +
201 +
202 +@pytest.mark.asyncio
203 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
204 +@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
205 +async def test_single_chat_session_video_base64encoded_beamsearch(
206 + client: openai.AsyncOpenAI, model_name: str, video_url: str,
207 + base64_encoded_video: Dict[str, str]):
208 +
209 + messages = [{
210 + "role":
211 + "user",
212 + "content": [
213 + {
214 + "type": "video_url",
215 + "video_url": {
216 + "url":
217 + f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"
218 + }
219 + },
220 + {
221 + "type": "text",
222 + "text": "What's in this video?"
223 + },
224 + ],
225 + }]
226 + chat_completion = await client.chat.completions.create(
227 + model=model_name,
228 + messages=messages,
229 + n=2,
230 + max_completion_tokens=10,
231 + extra_body=dict(use_beam_search=True))
232 + assert len(chat_completion.choices) == 2
233 + assert chat_completion.choices[
234 + 0].message.content != chat_completion.choices[1].message.content
235 +
236 +
237 +@pytest.mark.asyncio
238 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
239 +@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
240 +async def test_chat_streaming_video(client: openai.AsyncOpenAI,
241 + model_name: str, video_url: str):
242 + messages = [{
243 + "role":
244 + "user",
245 + "content": [
246 + {
247 + "type": "video_url",
248 + "video_url": {
249 + "url": video_url
250 + }
251 + },
252 + {
253 + "type": "text",
254 + "text": "What's in this video?"
255 + },
256 + ],
257 + }]
258 +
259 + # test single completion
260 + chat_completion = await client.chat.completions.create(
261 + model=model_name,
262 + messages=messages,
263 + max_completion_tokens=10,
264 + temperature=0.0,
265 + )
266 + output = chat_completion.choices[0].message.content
267 + stop_reason = chat_completion.choices[0].finish_reason
268 +
269 + # test streaming
270 + stream = await client.chat.completions.create(
271 + model=model_name,
272 + messages=messages,
273 + max_completion_tokens=10,
274 + temperature=0.0,
275 + stream=True,
276 + )
277 + chunks: List[str] = []
278 + finish_reason_count = 0
279 + async for chunk in stream:
280 + delta = chunk.choices[0].delta
281 + if delta.role:
282 + assert delta.role == "assistant"
283 + if delta.content:
284 + chunks.append(delta.content)
285 + if chunk.choices[0].finish_reason is not None:
286 + finish_reason_count += 1
287 + # finish reason should only return in last block
288 + assert finish_reason_count == 1
289 + assert chunk.choices[0].finish_reason == stop_reason
290 + assert delta.content
291 + assert "".join(chunks) == output
292 +
293 +
294 +@pytest.mark.asyncio
295 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
296 +@pytest.mark.parametrize(
297 + "video_urls",
298 + [TEST_VIDEO_URLS[:i] for i in range(2, len(TEST_VIDEO_URLS))])
299 +async def test_multi_video_input(client: openai.AsyncOpenAI, model_name: str,
300 + video_urls: List[str]):
301 +
302 + messages = [{
303 + "role":
304 + "user",
305 + "content": [
306 + *({
307 + "type": "video_url",
308 + "video_url": {
309 + "url": video_url
310 + }
311 + } for video_url in video_urls),
312 + {
313 + "type": "text",
314 + "text": "What's in this video?"
315 + },
316 + ],
317 + }]
318 +
319 + if len(video_urls) > MAXIMUM_VIDEOS:
320 + with pytest.raises(openai.BadRequestError): # test multi-video input
321 + await client.chat.completions.create(
322 + model=model_name,
323 + messages=messages,
324 + max_completion_tokens=10,
325 + temperature=0.0,
326 + )
327 +
328 + # the server should still work afterwards
329 + completion = await client.completions.create(
330 + model=model_name,
331 + prompt=[0, 0, 0, 0, 0],
332 + max_tokens=5,
333 + temperature=0.0,
334 + )
335 + completion = completion.choices[0].text
336 + assert completion is not None and len(completion) >= 0
337 + else:
338 + chat_completion = await client.chat.completions.create(
339 + model=model_name,
340 + messages=messages,
341 + max_completion_tokens=10,
342 + temperature=0.0,
343 + )
344 + message = chat_completion.choices[0].message
345 + assert message.content is not None and len(message.content) >= 0
vllm/assets/video.py
+2 -2
@@ -35,7 +35,7 @@ def download_video_asset(filename: str) -> str:
35
36
37 def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
38 - cv2 = try_import_video_packages()
38 + cv2, _ = try_import_video_packages()
39
40 cap = cv2.VideoCapture(path)
41 if not cap.isOpened():
@@ -59,7 +59,7 @@ def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
59
60 def video_to_pil_images_list(path: str,
61 num_frames: int = -1) -> List[Image.Image]:
62 - cv2 = try_import_video_packages()
62 + cv2, _ = try_import_video_packages()
63 frames = video_to_ndarrays(path, num_frames)
64 return [
65 Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
vllm/entrypoints/chat_utils.py
+63 -6
@@ -30,7 +30,9 @@ from vllm.logger import init_logger
30 from vllm.multimodal import MultiModalDataDict
31 from vllm.multimodal.utils import (async_get_and_parse_audio,
32 async_get_and_parse_image,
33 - get_and_parse_audio, get_and_parse_image)
33 + async_get_and_parse_video,
34 + get_and_parse_audio, get_and_parse_image,
35 + get_and_parse_video)
36 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
37 from vllm.utils import print_warning_once
38
@@ -51,6 +53,20 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
53 """The type of the content part."""
54
55
56 +class VideoURL(TypedDict, total=False):
57 + url: Required[str]
58 + """
59 + Either a URL of the video or a data URL with base64 encoded video data.
60 + """
61 +
62 +
63 +class ChatCompletionContentPartVideoParam(TypedDict, total=False):
64 + video_url: Required[VideoURL]
65 +
66 + type: Required[Literal["video_url"]]
67 + """The type of the content part."""
68 +
69 +
70 class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
71 """A simpler version of the param that only accepts a plain image_url.
72 This is supported by OpenAI API, although it is not documented.
@@ -74,11 +90,23 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
90 audio_url: Required[str]
91
92
93 +class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
94 + """A simpler version of the param that only accepts a plain audio_url.
95 +
96 + Example:
97 + {
98 + "video_url": "https://example.com/video.mp4"
99 + }
100 + """
101 + video_url: Required[str]
102 +
103 +
104 ChatCompletionContentPartParam: TypeAlias = Union[
105 OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
79 - ChatCompletionContentPartRefusalParam,
106 + ChatCompletionContentPartVideoParam, ChatCompletionContentPartRefusalParam,
107 CustomChatCompletionContentSimpleImageParam,
81 - CustomChatCompletionContentSimpleAudioParam, str]
108 + CustomChatCompletionContentSimpleAudioParam,
109 + CustomChatCompletionContentSimpleVideoParam, str]
110
111
112 class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -201,6 +229,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
229 elif modality == "video":
230 if model_type == "qwen2_vl":
231 return "<|vision_start|><|video_pad|><|vision_end|>"
232 + if model_type.startswith("llava"):
233 + return self._cached_token_str(self._tokenizer,
234 + hf_config.video_token_index)
235 raise TypeError(f"Unknown {modality} model type: {model_type}")
236 else:
237 raise TypeError(f"Unknown modality: {modality}")
@@ -291,6 +322,10 @@ class BaseMultiModalContentParser(ABC):
322 def parse_audio(self, audio_url: str) -> None:
323 raise NotImplementedError
324
325 + @abstractmethod
326 + def parse_video(self, video_url: str) -> None:
327 + raise NotImplementedError
328 +
329
330 class MultiModalContentParser(BaseMultiModalContentParser):
331
@@ -313,6 +348,12 @@ class MultiModalContentParser(BaseMultiModalContentParser):
348 placeholder = self._tracker.add("audio", audio)
349 self._add_placeholder(placeholder)
350
351 + def parse_video(self, video_url: str) -> None:
352 + video = get_and_parse_video(video_url)
353 +
354 + placeholder = self._tracker.add("video", video)
355 + self._add_placeholder(placeholder)
356 +
357
358 class AsyncMultiModalContentParser(BaseMultiModalContentParser):
359
@@ -336,6 +377,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
377 placeholder = self._tracker.add("audio", audio_coro)
378 self._add_placeholder(placeholder)
379
380 + def parse_video(self, video_url: str) -> None:
381 + video = async_get_and_parse_video(video_url)
382 +
383 + placeholder = self._tracker.add("video", video)
384 + self._add_placeholder(placeholder)
385 +
386
387 def validate_chat_template(chat_template: Optional[Union[Path, str]]):
388 """Raises if the provided chat template appears invalid."""
@@ -416,6 +463,7 @@ _TextParser = partial(cast, ChatCompletionContentPartTextParam)
463 _ImageParser = partial(cast, ChatCompletionContentPartImageParam)
464 _AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
465 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
466 +_VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
467 MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
468
469 # Define a mapping from part types to their corresponding parsing functions.
@@ -428,6 +476,8 @@ MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
476 lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
477 "refusal":
478 lambda part: _RefusalParser(part).get("refusal", ""),
479 + "video_url":
480 + lambda part: _VideoParser(part).get("video_url", {}).get("url", ""),
481 }
482
483
@@ -472,7 +522,10 @@ def _parse_chat_message_content_mm_part(
522 audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
523 part)
524 return "audio_url", audio_params.get("audio_url", "")
475 -
525 + if part.get("video_url") is not None:
526 + video_params = cast(CustomChatCompletionContentSimpleVideoParam,
527 + part)
528 + return "video_url", video_params.get("video_url", "")
529 # Raise an error if no 'type' or direct URL is found.
530 raise ValueError("Missing 'type' field in multimodal part.")
531
@@ -482,7 +535,7 @@ def _parse_chat_message_content_mm_part(
535
536
537 VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
485 - "audio_url")
538 + "audio_url", "video_url")
539
540
541 def _parse_chat_message_content_parts(
@@ -542,7 +595,7 @@ def _parse_chat_message_content_part(
595 # Handle structured dictionary parts
596 part_type, content = _parse_chat_message_content_mm_part(part)
597
545 - # if part_type is text/refusal/image_url/audio_url but
598 + # if part_type is text/refusal/image_url/audio_url/video_url but
599 # content is empty, log a warning and skip
600 if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
601 logger.warning(
@@ -561,6 +614,10 @@ def _parse_chat_message_content_part(
614 mm_parser.parse_audio(content)
615 return {'type': 'audio'} if wrap_dicts else None
616
617 + if part_type == "video_url":
618 + mm_parser.parse_video(content)
619 + return {'type': 'video'} if wrap_dicts else None
620 +
621 raise NotImplementedError(f"Unknown part type: {part_type}")
622
623
vllm/envs.py
+9 -3
@@ -49,7 +49,8 @@ if TYPE_CHECKING:
49 VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
50 VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
51 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
52 - VLLM_AUDIO_FETCH_TIMEOUT: int = 5
52 + VLLM_VIDEO_FETCH_TIMEOUT: int = 15
53 + VLLM_AUDIO_FETCH_TIMEOUT: int = 10
54 VLLM_TARGET_DEVICE: str = "cuda"
55 MAX_JOBS: Optional[str] = None
56 NVCC_THREADS: Optional[str] = None
@@ -376,10 +377,15 @@ environment_variables: Dict[str, Callable[[], Any]] = {
377 "VLLM_IMAGE_FETCH_TIMEOUT":
378 lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
379
380 + # Timeout for fetching videos when serving multimodal models
381 + # Default is 15 seconds
382 + "VLLM_VIDEO_FETCH_TIMEOUT":
383 + lambda: int(os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "15")),
384 +
385 # Timeout for fetching audio when serving multimodal models
380 - # Default is 5 seconds
386 + # Default is 10 seconds
387 "VLLM_AUDIO_FETCH_TIMEOUT":
382 - lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "5")),
388 + lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")),
389
390 # Path to the XLA persistent cache directory.
391 # Only used for XLA devices such as TPUs.
vllm/model_executor/models/llava_onevision.py
+3 -2
@@ -341,7 +341,7 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
341 get_llava_onevision_video_tokens(ctx, num_frames))
342
343 tokenizer = cached_get_tokenizer(model_config.tokenizer)
344 - new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
344 + new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
345 tokenizer,
346 inputs.get("prompt"),
347 inputs["prompt_token_ids"],
@@ -350,7 +350,8 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
350 )
351 return token_inputs(prompt_token_ids=new_token_ids,
352 prompt=new_prompt,
353 - multi_modal_data=multi_modal_data)
353 + multi_modal_data=multi_modal_data,
354 + multi_modal_placeholders={"video": ranges})
355 else:
356 raise TypeError(f"Invalid video type: {type(video_data)}")
357
vllm/multimodal/base.py
+3
@@ -136,6 +136,9 @@ class MultiModalDataBuiltins(TypedDict, total=False):
136 audio: MultiModalData[Tuple[np.ndarray, Union[int, float]]]
137 """The input audio item(s) and corresponding sampling rate(s)."""
138
139 + video: MultiModalData[Tuple[np.ndarray]]
140 + """The input video(s)."""
141 +
142
143 MultiModalDataDict = Union[MultiModalDataBuiltins,
144 Mapping[str, MultiModalData[object]]]
vllm/multimodal/utils.py
+114 -7
@@ -8,8 +8,8 @@ import numpy as np
8 import numpy.typing as npt
9 from PIL import Image
10
11 +import vllm.envs as envs
12 from vllm.connections import global_http_connection
12 -from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
13 from vllm.logger import init_logger
14 from vllm.multimodal.base import MultiModalDataDict, PlaceholderRange
15 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
@@ -80,7 +80,9 @@ def fetch_image(image_url: str,
80 """
81 if image_url.startswith('http'):
82 image_raw = global_http_connection.get_bytes(
83 - image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT)
83 + image_url,
84 + timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
85 + )
86 image = _load_image_from_bytes(image_raw)
87
88 elif image_url.startswith('data:image'):
@@ -105,7 +107,9 @@ async def async_fetch_image(image_url: str,
107 """
108 if image_url.startswith('http'):
109 image_raw = await global_http_connection.async_get_bytes(
108 - image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT)
110 + image_url,
111 + timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
112 + )
113 image = _load_image_from_bytes(image_raw)
114
115 elif image_url.startswith('data:image'):
@@ -119,6 +123,85 @@ async def async_fetch_image(image_url: str,
123 return image.convert(image_mode)
124
125
126 +def _load_video_frames_from_bytes(b: bytes):
127 + frame = Image.open(BytesIO(b))
128 + return np.array(frame)
129 +
130 +
131 +def load_video_frames_from_base64(frame: Union[bytes, str]):
132 + """Load frame from base64 format."""
133 + return _load_video_frames_from_bytes(base64.b64decode(frame))
134 +
135 +
136 +def _load_video_from_bytes(b: bytes, num_frames: int = 32):
137 + _, decord = try_import_video_packages()
138 +
139 + video_path = BytesIO(b)
140 + vr = decord.VideoReader(video_path, num_threads=1)
141 + total_frame_num = len(vr)
142 +
143 + if total_frame_num > num_frames:
144 + uniform_sampled_frames = np.linspace(0,
145 + total_frame_num - 1,
146 + num_frames,
147 + dtype=int)
148 + frame_idx = uniform_sampled_frames.tolist()
149 + else:
150 + frame_idx = [i for i in range(0, total_frame_num)]
151 + frames = vr.get_batch(frame_idx).asnumpy()
152 +
153 + return frames
154 +
155 +
156 +def _load_video_from_data_url(video_url: str):
157 + # Only split once and assume the second part is the base64 encoded image
158 + frames_base64 = video_url.split(",")[1:]
159 + return np.stack([
160 + load_video_frames_from_base64(frame_base64)
161 + for frame_base64 in frames_base64
162 + ])
163 +
164 +
165 +def fetch_video(video_url: str, *, num_frames: int = 32) -> npt.NDArray:
166 + """
167 + Load video from a HTTP or base64 data URL.
168 + """
169 + if video_url.startswith('http') or video_url.startswith('https'):
170 + video_raw = global_http_connection.get_bytes(
171 + video_url,
172 + timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
173 + )
174 + video = _load_video_from_bytes(video_raw, num_frames)
175 + elif video_url.startswith('data:video'):
176 + video = _load_video_from_data_url(video_url)
177 + else:
178 + raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
179 + "with either 'data:video' or 'http'.")
180 + return video
181 +
182 +
183 +async def async_fetch_video(video_url: str,
184 + *,
185 + num_frames: int = 32) -> npt.NDArray:
186 + """
187 + Asynchronously load video from a HTTP or base64 data URL.
188 +
189 + By default, the image is converted into RGB format.
190 + """
191 + if video_url.startswith('http') or video_url.startswith('https'):
192 + video_raw = await global_http_connection.async_get_bytes(
193 + video_url,
194 + timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
195 + )
196 + video = _load_video_from_bytes(video_raw, num_frames)
197 + elif video_url.startswith('data:video'):
198 + video = _load_video_from_data_url(video_url)
199 + else:
200 + raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
201 + "with either 'data:video' or 'http'.")
202 + return video
203 +
204 +
205 def try_import_audio_packages() -> Tuple[Any, Any]:
206 try:
207 import librosa
@@ -137,7 +220,9 @@ def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
220
221 if audio_url.startswith("http"):
222 audio_bytes = global_http_connection.get_bytes(
140 - audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
223 + audio_url,
224 + timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
225 + )
226 elif audio_url.startswith("data:audio"):
227 _, audio_base64 = audio_url.split(",", 1)
228 audio_bytes = base64.b64decode(audio_base64)
@@ -157,7 +242,9 @@ async def async_fetch_audio(
242
243 if audio_url.startswith("http"):
244 audio_bytes = await global_http_connection.async_get_bytes(
160 - audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
245 + audio_url,
246 + timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
247 + )
248 elif audio_url.startswith("data:audio"):
249 _, audio_base64 = audio_url.split(",", 1)
250 audio_bytes = base64.b64decode(audio_base64)
@@ -182,6 +269,11 @@ def get_and_parse_image(
269 return {"image": image}
270
271
272 +def get_and_parse_video(video_url: str) -> MultiModalDataDict:
273 + video = fetch_video(video_url)
274 + return {"video": video}
275 +
276 +
277 async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
278 audio, sr = await async_fetch_audio(audio_url)
279 return {"audio": (audio, sr)}
@@ -196,6 +288,11 @@ async def async_get_and_parse_image(
288 return {"image": image}
289
290
291 +async def async_get_and_parse_video(video_url: str) -> MultiModalDataDict:
292 + video = await async_fetch_video(video_url)
293 + return {"video": video}
294 +
295 +
296 def encode_audio_base64(
297 audio: np.ndarray,
298 sampling_rate: int,
@@ -246,14 +343,15 @@ def rescale_image_size(image: Image.Image,
343 def try_import_video_packages() -> Any:
344 try:
345 import cv2
346 + import decord
347 except ImportError:
348 raise ImportError(
349 "Please install vllm[video] for video support.") from None
252 - return cv2
350 + return cv2, decord
351
352
353 def resize_video(frames: npt.NDArray, size: Tuple[int, int]) -> npt.NDArray:
256 - cv2 = try_import_video_packages()
354 + cv2, _ = try_import_video_packages()
355
356 num_frames, _, _, channels = frames.shape
357 new_height, new_width = size
@@ -284,6 +382,15 @@ def sample_frames_from_video(frames: npt.NDArray,
382 return sampled_frames
383
384
385 +def encode_video_base64(frames: npt.NDArray):
386 + base64_frames = []
387 + frames_list = [frames[i] for i in range(frames.shape[0])]
388 + for frame in frames_list:
389 + img_base64 = encode_image_base64(Image.fromarray(frame))
390 + base64_frames.append(img_base64)
391 + return ",".join(base64_frames)
392 +
393 +
394 # Utilities for input processors
395 _T = TypeVar("_T", str, int)
396
vllm/multimodal/video.py
+2 -1
@@ -7,6 +7,7 @@ from vllm.inputs.registry import InputContext
7 from vllm.logger import init_logger
8 from vllm.transformers_utils.processor import get_video_processor
9 from vllm.transformers_utils.tokenizer import get_tokenizer
10 +from vllm.utils import is_list_of
11
12 from .base import MultiModalData, MultiModalInputs
13 from .image import ImagePlugin
@@ -60,7 +61,7 @@ class VideoPlugin(ImagePlugin):
61 if isinstance(data, list) and len(data) == 1:
62 data = data[0]
63
63 - if isinstance(data, np.ndarray):
64 + if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
65 video_processor = self._get_hf_video_processor(
66 model_config,
67 mm_processor_kwargs,