A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
Online video support for VLMs (#10020)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: litianjian <litianjian@bytedance.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
litianjian committed
Nov 8, 2024 at 04:25 UTC
28b2877d303caa6b2febc9d0b425f17828634a4c
12 files changed
+598
-31
docs/source/conf.py
+1
@@ -116,6 +116,7 @@ autodoc_mock_imports = [
116
"soundfile",
117
"gguf",
118
"lark",
119
+ "decord",
120
]
121
122
for mock_target in autodoc_mock_imports:
requirements-test.in
+4
-2
@@ -8,6 +8,7 @@ pytest-shard
8
9
# testing utils
10
awscli
11
+decord # required for video tests
12
einops # required for MPT, qwen-vl and Mamba
13
httpx
14
librosa # required for audio tests
@@ -15,12 +16,13 @@ opencv-python # required for video tests
16
peft
17
requests
18
ray[adag]==2.35
18
-sentence-transformers # required for embedding
19
-soundfile # required for audio test
19
+sentence-transformers # required for embedding tests
20
+soundfile # required for audio tests
21
timm # required for internvl test
22
torch==2.5.1
23
transformers_stream_generator # required for qwen-vl test
24
matplotlib # required for qwen-vl test
25
+mistral_common[opencv] >= 1.4.4 # required for pixtral test
26
datamodel_code_generator # required for minicpm3 test
27
lm-eval[api]==0.4.4 # required for model evaluation test
28
requirements-test.txt
+50
-7
@@ -1,8 +1,8 @@
1
#
2
-# This file is autogenerated by pip-compile with Python 3.12
2
+# This file is autogenerated by pip-compile with Python 3.9
3
# by the following command:
4
#
5
-# pip-compile --output-file=requirements-test.txt requirements-test.in
5
+# pip-compile requirements-test.in
6
#
7
absl-py==2.1.0
8
# via rouge-score
@@ -28,6 +28,10 @@ anyio==4.6.2.post1
28
# via httpx
29
argcomplete==3.5.1
30
# via datamodel-code-generator
31
+async-timeout==4.0.3
32
+ # via
33
+ # aiohttp
34
+ # redis
35
attrs==24.2.0
36
# via
37
# aiohttp
@@ -90,6 +94,8 @@ datasets==3.0.2
94
# lm-eval
95
decorator==5.1.1
96
# via librosa
97
+decord==0.6.0
98
+ # via -r requirements-test.in
99
dill==0.3.8
100
# via
101
# datasets
@@ -106,6 +112,10 @@ email-validator==2.2.0
112
# via pydantic
113
evaluate==0.4.3
114
# via lm-eval
115
+exceptiongroup==1.2.2
116
+ # via
117
+ # anyio
118
+ # pytest
119
fastrlock==0.8.2
120
# via cupy-cuda12x
121
filelock==3.16.1
@@ -156,6 +166,8 @@ idna==3.10
166
# httpx
167
# requests
168
# yarl
169
+importlib-resources==6.4.5
170
+ # via matplotlib
171
inflect==5.6.2
172
# via datamodel-code-generator
173
iniconfig==2.0.0
@@ -178,7 +190,9 @@ joblib==1.4.2
190
jsonlines==4.0.0
191
# via lm-eval
192
jsonschema==4.23.0
181
- # via ray
193
+ # via
194
+ # mistral-common
195
+ # ray
196
jsonschema-specifications==2024.10.1
197
# via jsonschema
198
kiwisolver==1.4.7
@@ -204,6 +218,10 @@ mbstrdecoder==1.1.3
218
# dataproperty
219
# pytablewriter
220
# typepy
221
+mistral-common[opencv]==1.4.4
222
+ # via
223
+ # -r requirements-test.in
224
+ # mistral-common
225
more-itertools==10.5.0
226
# via lm-eval
227
mpmath==1.3.0
@@ -238,12 +256,15 @@ numpy==1.26.4
256
# contourpy
257
# cupy-cuda12x
258
# datasets
259
+ # decord
260
# evaluate
261
# librosa
262
# matplotlib
263
+ # mistral-common
264
# numba
265
# numexpr
266
# opencv-python
267
+ # opencv-python-headless
268
# pandas
269
# peft
270
# rouge-score
@@ -288,6 +309,8 @@ nvidia-nvtx-cu12==12.4.127
309
# via torch
310
opencv-python==4.10.0.84
311
# via -r requirements-test.in
312
+opencv-python-headless==4.10.0.84
313
+ # via mistral-common
314
packaging==24.1
315
# via
316
# accelerate
@@ -317,9 +340,10 @@ peft==0.13.2
340
# via
341
# -r requirements-test.in
342
# lm-eval
320
-pillow==11.0.0
343
+pillow==10.4.0
344
# via
345
# matplotlib
346
+ # mistral-common
347
# sentence-transformers
348
# torchvision
349
platformdirs==4.3.6
@@ -354,7 +378,9 @@ pybind11==2.13.6
378
pycparser==2.22
379
# via cffi
380
pydantic[email]==2.9.2
357
- # via datamodel-code-generator
381
+ # via
382
+ # datamodel-code-generator
383
+ # mistral-common
384
pydantic-core==2.23.4
385
# via pydantic
386
pyparsing==3.2.0
@@ -420,6 +446,7 @@ requests==2.32.3
446
# evaluate
447
# huggingface-hub
448
# lm-eval
449
+ # mistral-common
450
# pooch
451
# ray
452
# tiktoken
@@ -456,6 +483,8 @@ scipy==1.13.1
483
# sentence-transformers
484
sentence-transformers==3.2.1
485
# via -r requirements-test.in
486
+sentencepiece==0.2.0
487
+ # via mistral-common
488
six==1.16.0
489
# via
490
# python-dateutil
@@ -486,12 +515,20 @@ tensorizer==2.9.0
515
# via -r requirements-test.in
516
threadpoolctl==3.5.0
517
# via scikit-learn
489
-tiktoken==0.8.0
490
- # via lm-eval
518
+tiktoken==0.7.0
519
+ # via
520
+ # lm-eval
521
+ # mistral-common
522
timm==1.0.11
523
# via -r requirements-test.in
524
tokenizers==0.20.1
525
# via transformers
526
+toml==0.10.2
527
+ # via datamodel-code-generator
528
+tomli==2.0.2
529
+ # via
530
+ # black
531
+ # pytest
532
torch==2.5.1
533
# via
534
# -r requirements-test.in
@@ -535,8 +572,12 @@ typepy[datetime]==1.3.2
572
# tabledata
573
typing-extensions==4.12.2
574
# via
575
+ # anyio
576
+ # black
577
# huggingface-hub
578
# librosa
579
+ # mistral-common
580
+ # multidict
581
# pydantic
582
# pydantic-core
583
# torch
@@ -554,6 +595,8 @@ xxhash==3.5.0
595
# evaluate
596
yarl==1.17.1
597
# via aiohttp
598
+zipp==3.20.2
599
+ # via importlib-resources
600
zstandard==0.23.0
601
# via lm-eval
602
setup.py
+2
-1
@@ -554,7 +554,8 @@ setup(
554
ext_modules=ext_modules,
555
extras_require={
556
"tensorizer": ["tensorizer>=2.9.0"],
557
- "audio": ["librosa", "soundfile"] # Required for audio processing
557
+ "audio": ["librosa", "soundfile"], # Required for audio processing
558
+ "video": ["decord"] # Required for video processing
559
},
560
cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
561
package_data=package_data,
tests/entrypoints/openai/test_video.py
new
+345
@@ -0,0 +1,345 @@
1
+from typing import Dict, List
2
+
3
+import openai
4
+import pytest
5
+import pytest_asyncio
6
+
7
+from vllm.multimodal.utils import encode_video_base64, fetch_video
8
+
9
+from ...utils import RemoteOpenAIServer
10
+
11
+MODEL_NAME = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
12
+MAXIMUM_VIDEOS = 4
13
+
14
+TEST_VIDEO_URLS = [
15
+ "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4",
16
+ "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4",
17
+ "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4",
18
+ "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4",
19
+]
20
+
21
+
22
+@pytest.fixture(scope="module")
23
+def server():
24
+ args = [
25
+ "--task",
26
+ "generate",
27
+ "--dtype",
28
+ "bfloat16",
29
+ "--max-model-len",
30
+ "32768",
31
+ "--max-num-seqs",
32
+ "2",
33
+ "--enforce-eager",
34
+ "--trust-remote-code",
35
+ "--limit-mm-per-prompt",
36
+ f"video={MAXIMUM_VIDEOS}",
37
+ ]
38
+
39
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
40
+ yield remote_server
41
+
42
+
43
+@pytest_asyncio.fixture
44
+async def client(server):
45
+ async with server.get_async_client() as async_client:
46
+ yield async_client
47
+
48
+
49
+@pytest.fixture(scope="session")
50
+def base64_encoded_video() -> Dict[str, str]:
51
+ return {
52
+ video_url: encode_video_base64(fetch_video(video_url))
53
+ for video_url in TEST_VIDEO_URLS
54
+ }
55
+
56
+
57
+@pytest.mark.asyncio
58
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
59
+@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
60
+async def test_single_chat_session_video(client: openai.AsyncOpenAI,
61
+ model_name: str, video_url: str):
62
+ messages = [{
63
+ "role":
64
+ "user",
65
+ "content": [
66
+ {
67
+ "type": "video_url",
68
+ "video_url": {
69
+ "url": video_url
70
+ }
71
+ },
72
+ {
73
+ "type": "text",
74
+ "text": "What's in this video?"
75
+ },
76
+ ],
77
+ }]
78
+
79
+ # test single completion
80
+ chat_completion = await client.chat.completions.create(
81
+ model=model_name,
82
+ messages=messages,
83
+ max_completion_tokens=10,
84
+ logprobs=True,
85
+ top_logprobs=5)
86
+ assert len(chat_completion.choices) == 1
87
+
88
+ choice = chat_completion.choices[0]
89
+ assert choice.finish_reason == "length"
90
+ assert chat_completion.usage == openai.types.CompletionUsage(
91
+ completion_tokens=10, prompt_tokens=6299, total_tokens=6309)
92
+
93
+ message = choice.message
94
+ message = chat_completion.choices[0].message
95
+ assert message.content is not None and len(message.content) >= 10
96
+ assert message.role == "assistant"
97
+ messages.append({"role": "assistant", "content": message.content})
98
+
99
+ # test multi-turn dialogue
100
+ messages.append({"role": "user", "content": "express your result in json"})
101
+ chat_completion = await client.chat.completions.create(
102
+ model=model_name,
103
+ messages=messages,
104
+ max_completion_tokens=10,
105
+ )
106
+ message = chat_completion.choices[0].message
107
+ assert message.content is not None and len(message.content) >= 0
108
+
109
+
110
+@pytest.mark.asyncio
111
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
112
+@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
113
+async def test_single_chat_session_video_beamsearch(client: openai.AsyncOpenAI,
114
+ model_name: str,
115
+ video_url: str):
116
+ messages = [{
117
+ "role":
118
+ "user",
119
+ "content": [
120
+ {
121
+ "type": "video_url",
122
+ "video_url": {
123
+ "url": video_url
124
+ }
125
+ },
126
+ {
127
+ "type": "text",
128
+ "text": "What's in this video?"
129
+ },
130
+ ],
131
+ }]
132
+
133
+ chat_completion = await client.chat.completions.create(
134
+ model=model_name,
135
+ messages=messages,
136
+ n=2,
137
+ max_completion_tokens=10,
138
+ logprobs=True,
139
+ top_logprobs=5,
140
+ extra_body=dict(use_beam_search=True))
141
+ assert len(chat_completion.choices) == 2
142
+ assert chat_completion.choices[
143
+ 0].message.content != chat_completion.choices[1].message.content
144
+
145
+
146
+@pytest.mark.asyncio
147
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
148
+@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
149
+async def test_single_chat_session_video_base64encoded(
150
+ client: openai.AsyncOpenAI, model_name: str, video_url: str,
151
+ base64_encoded_video: Dict[str, str]):
152
+
153
+ messages = [{
154
+ "role":
155
+ "user",
156
+ "content": [
157
+ {
158
+ "type": "video_url",
159
+ "video_url": {
160
+ "url":
161
+ f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"
162
+ }
163
+ },
164
+ {
165
+ "type": "text",
166
+ "text": "What's in this video?"
167
+ },
168
+ ],
169
+ }]
170
+
171
+ # test single completion
172
+ chat_completion = await client.chat.completions.create(
173
+ model=model_name,
174
+ messages=messages,
175
+ max_completion_tokens=10,
176
+ logprobs=True,
177
+ top_logprobs=5)
178
+ assert len(chat_completion.choices) == 1
179
+
180
+ choice = chat_completion.choices[0]
181
+ assert choice.finish_reason == "length"
182
+ assert chat_completion.usage == openai.types.CompletionUsage(
183
+ completion_tokens=10, prompt_tokens=6299, total_tokens=6309)
184
+
185
+ message = choice.message
186
+ message = chat_completion.choices[0].message
187
+ assert message.content is not None and len(message.content) >= 10
188
+ assert message.role == "assistant"
189
+ messages.append({"role": "assistant", "content": message.content})
190
+
191
+ # test multi-turn dialogue
192
+ messages.append({"role": "user", "content": "express your result in json"})
193
+ chat_completion = await client.chat.completions.create(
194
+ model=model_name,
195
+ messages=messages,
196
+ max_completion_tokens=10,
197
+ )
198
+ message = chat_completion.choices[0].message
199
+ assert message.content is not None and len(message.content) >= 0
200
+
201
+
202
+@pytest.mark.asyncio
203
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
204
+@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
205
+async def test_single_chat_session_video_base64encoded_beamsearch(
206
+ client: openai.AsyncOpenAI, model_name: str, video_url: str,
207
+ base64_encoded_video: Dict[str, str]):
208
+
209
+ messages = [{
210
+ "role":
211
+ "user",
212
+ "content": [
213
+ {
214
+ "type": "video_url",
215
+ "video_url": {
216
+ "url":
217
+ f"data:video/jpeg;base64,{base64_encoded_video[video_url]}"
218
+ }
219
+ },
220
+ {
221
+ "type": "text",
222
+ "text": "What's in this video?"
223
+ },
224
+ ],
225
+ }]
226
+ chat_completion = await client.chat.completions.create(
227
+ model=model_name,
228
+ messages=messages,
229
+ n=2,
230
+ max_completion_tokens=10,
231
+ extra_body=dict(use_beam_search=True))
232
+ assert len(chat_completion.choices) == 2
233
+ assert chat_completion.choices[
234
+ 0].message.content != chat_completion.choices[1].message.content
235
+
236
+
237
+@pytest.mark.asyncio
238
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
239
+@pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)
240
+async def test_chat_streaming_video(client: openai.AsyncOpenAI,
241
+ model_name: str, video_url: str):
242
+ messages = [{
243
+ "role":
244
+ "user",
245
+ "content": [
246
+ {
247
+ "type": "video_url",
248
+ "video_url": {
249
+ "url": video_url
250
+ }
251
+ },
252
+ {
253
+ "type": "text",
254
+ "text": "What's in this video?"
255
+ },
256
+ ],
257
+ }]
258
+
259
+ # test single completion
260
+ chat_completion = await client.chat.completions.create(
261
+ model=model_name,
262
+ messages=messages,
263
+ max_completion_tokens=10,
264
+ temperature=0.0,
265
+ )
266
+ output = chat_completion.choices[0].message.content
267
+ stop_reason = chat_completion.choices[0].finish_reason
268
+
269
+ # test streaming
270
+ stream = await client.chat.completions.create(
271
+ model=model_name,
272
+ messages=messages,
273
+ max_completion_tokens=10,
274
+ temperature=0.0,
275
+ stream=True,
276
+ )
277
+ chunks: List[str] = []
278
+ finish_reason_count = 0
279
+ async for chunk in stream:
280
+ delta = chunk.choices[0].delta
281
+ if delta.role:
282
+ assert delta.role == "assistant"
283
+ if delta.content:
284
+ chunks.append(delta.content)
285
+ if chunk.choices[0].finish_reason is not None:
286
+ finish_reason_count += 1
287
+ # finish reason should only return in last block
288
+ assert finish_reason_count == 1
289
+ assert chunk.choices[0].finish_reason == stop_reason
290
+ assert delta.content
291
+ assert "".join(chunks) == output
292
+
293
+
294
+@pytest.mark.asyncio
295
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
296
+@pytest.mark.parametrize(
297
+ "video_urls",
298
+ [TEST_VIDEO_URLS[:i] for i in range(2, len(TEST_VIDEO_URLS))])
299
+async def test_multi_video_input(client: openai.AsyncOpenAI, model_name: str,
300
+ video_urls: List[str]):
301
+
302
+ messages = [{
303
+ "role":
304
+ "user",
305
+ "content": [
306
+ *({
307
+ "type": "video_url",
308
+ "video_url": {
309
+ "url": video_url
310
+ }
311
+ } for video_url in video_urls),
312
+ {
313
+ "type": "text",
314
+ "text": "What's in this video?"
315
+ },
316
+ ],
317
+ }]
318
+
319
+ if len(video_urls) > MAXIMUM_VIDEOS:
320
+ with pytest.raises(openai.BadRequestError): # test multi-video input
321
+ await client.chat.completions.create(
322
+ model=model_name,
323
+ messages=messages,
324
+ max_completion_tokens=10,
325
+ temperature=0.0,
326
+ )
327
+
328
+ # the server should still work afterwards
329
+ completion = await client.completions.create(
330
+ model=model_name,
331
+ prompt=[0, 0, 0, 0, 0],
332
+ max_tokens=5,
333
+ temperature=0.0,
334
+ )
335
+ completion = completion.choices[0].text
336
+ assert completion is not None and len(completion) >= 0
337
+ else:
338
+ chat_completion = await client.chat.completions.create(
339
+ model=model_name,
340
+ messages=messages,
341
+ max_completion_tokens=10,
342
+ temperature=0.0,
343
+ )
344
+ message = chat_completion.choices[0].message
345
+ assert message.content is not None and len(message.content) >= 0
vllm/assets/video.py
+2
-2
@@ -35,7 +35,7 @@ def download_video_asset(filename: str) -> str:
35
36
37
def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
38
- cv2 = try_import_video_packages()
38
+ cv2, _ = try_import_video_packages()
39
40
cap = cv2.VideoCapture(path)
41
if not cap.isOpened():
@@ -59,7 +59,7 @@ def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
59
60
def video_to_pil_images_list(path: str,
61
num_frames: int = -1) -> List[Image.Image]:
62
- cv2 = try_import_video_packages()
62
+ cv2, _ = try_import_video_packages()
63
frames = video_to_ndarrays(path, num_frames)
64
return [
65
Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
vllm/entrypoints/chat_utils.py
+63
-6
@@ -30,7 +30,9 @@ from vllm.logger import init_logger
30
from vllm.multimodal import MultiModalDataDict
31
from vllm.multimodal.utils import (async_get_and_parse_audio,
32
async_get_and_parse_image,
33
- get_and_parse_audio, get_and_parse_image)
33
+ async_get_and_parse_video,
34
+ get_and_parse_audio, get_and_parse_image,
35
+ get_and_parse_video)
36
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
37
from vllm.utils import print_warning_once
38
@@ -51,6 +53,20 @@ class ChatCompletionContentPartAudioParam(TypedDict, total=False):
53
"""The type of the content part."""
54
55
56
+class VideoURL(TypedDict, total=False):
57
+ url: Required[str]
58
+ """
59
+ Either a URL of the video or a data URL with base64 encoded video data.
60
+ """
61
+
62
+
63
+class ChatCompletionContentPartVideoParam(TypedDict, total=False):
64
+ video_url: Required[VideoURL]
65
+
66
+ type: Required[Literal["video_url"]]
67
+ """The type of the content part."""
68
+
69
+
70
class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
71
"""A simpler version of the param that only accepts a plain image_url.
72
This is supported by OpenAI API, although it is not documented.
@@ -74,11 +90,23 @@ class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
90
audio_url: Required[str]
91
92
93
+class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
94
+ """A simpler version of the param that only accepts a plain audio_url.
95
+
96
+ Example:
97
+ {
98
+ "video_url": "https://example.com/video.mp4"
99
+ }
100
+ """
101
+ video_url: Required[str]
102
+
103
+
104
ChatCompletionContentPartParam: TypeAlias = Union[
105
OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
79
- ChatCompletionContentPartRefusalParam,
106
+ ChatCompletionContentPartVideoParam, ChatCompletionContentPartRefusalParam,
107
CustomChatCompletionContentSimpleImageParam,
81
- CustomChatCompletionContentSimpleAudioParam, str]
108
+ CustomChatCompletionContentSimpleAudioParam,
109
+ CustomChatCompletionContentSimpleVideoParam, str]
110
111
112
class CustomChatCompletionMessageParam(TypedDict, total=False):
@@ -201,6 +229,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
229
elif modality == "video":
230
if model_type == "qwen2_vl":
231
return "<|vision_start|><|video_pad|><|vision_end|>"
232
+ if model_type.startswith("llava"):
233
+ return self._cached_token_str(self._tokenizer,
234
+ hf_config.video_token_index)
235
raise TypeError(f"Unknown {modality} model type: {model_type}")
236
else:
237
raise TypeError(f"Unknown modality: {modality}")
@@ -291,6 +322,10 @@ class BaseMultiModalContentParser(ABC):
322
def parse_audio(self, audio_url: str) -> None:
323
raise NotImplementedError
324
325
+ @abstractmethod
326
+ def parse_video(self, video_url: str) -> None:
327
+ raise NotImplementedError
328
+
329
330
class MultiModalContentParser(BaseMultiModalContentParser):
331
@@ -313,6 +348,12 @@ class MultiModalContentParser(BaseMultiModalContentParser):
348
placeholder = self._tracker.add("audio", audio)
349
self._add_placeholder(placeholder)
350
351
+ def parse_video(self, video_url: str) -> None:
352
+ video = get_and_parse_video(video_url)
353
+
354
+ placeholder = self._tracker.add("video", video)
355
+ self._add_placeholder(placeholder)
356
+
357
358
class AsyncMultiModalContentParser(BaseMultiModalContentParser):
359
@@ -336,6 +377,12 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
377
placeholder = self._tracker.add("audio", audio_coro)
378
self._add_placeholder(placeholder)
379
380
+ def parse_video(self, video_url: str) -> None:
381
+ video = async_get_and_parse_video(video_url)
382
+
383
+ placeholder = self._tracker.add("video", video)
384
+ self._add_placeholder(placeholder)
385
+
386
387
def validate_chat_template(chat_template: Optional[Union[Path, str]]):
388
"""Raises if the provided chat template appears invalid."""
@@ -416,6 +463,7 @@ _TextParser = partial(cast, ChatCompletionContentPartTextParam)
463
_ImageParser = partial(cast, ChatCompletionContentPartImageParam)
464
_AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
465
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
466
+_VideoParser = partial(cast, ChatCompletionContentPartVideoParam)
467
MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
468
469
# Define a mapping from part types to their corresponding parsing functions.
@@ -428,6 +476,8 @@ MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
476
lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
477
"refusal":
478
lambda part: _RefusalParser(part).get("refusal", ""),
479
+ "video_url":
480
+ lambda part: _VideoParser(part).get("video_url", {}).get("url", ""),
481
}
482
483
@@ -472,7 +522,10 @@ def _parse_chat_message_content_mm_part(
522
audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
523
part)
524
return "audio_url", audio_params.get("audio_url", "")
475
-
525
+ if part.get("video_url") is not None:
526
+ video_params = cast(CustomChatCompletionContentSimpleVideoParam,
527
+ part)
528
+ return "video_url", video_params.get("video_url", "")
529
# Raise an error if no 'type' or direct URL is found.
530
raise ValueError("Missing 'type' field in multimodal part.")
531
@@ -482,7 +535,7 @@ def _parse_chat_message_content_mm_part(
535
536
537
VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
485
- "audio_url")
538
+ "audio_url", "video_url")
539
540
541
def _parse_chat_message_content_parts(
@@ -542,7 +595,7 @@ def _parse_chat_message_content_part(
595
# Handle structured dictionary parts
596
part_type, content = _parse_chat_message_content_mm_part(part)
597
545
- # if part_type is text/refusal/image_url/audio_url but
598
+ # if part_type is text/refusal/image_url/audio_url/video_url but
599
# content is empty, log a warning and skip
600
if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
601
logger.warning(
@@ -561,6 +614,10 @@ def _parse_chat_message_content_part(
614
mm_parser.parse_audio(content)
615
return {'type': 'audio'} if wrap_dicts else None
616
617
+ if part_type == "video_url":
618
+ mm_parser.parse_video(content)
619
+ return {'type': 'video'} if wrap_dicts else None
620
+
621
raise NotImplementedError(f"Unknown part type: {part_type}")
622
623
vllm/envs.py
+9
-3
@@ -49,7 +49,8 @@ if TYPE_CHECKING:
49
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
50
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
51
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
52
- VLLM_AUDIO_FETCH_TIMEOUT: int = 5
52
+ VLLM_VIDEO_FETCH_TIMEOUT: int = 15
53
+ VLLM_AUDIO_FETCH_TIMEOUT: int = 10
54
VLLM_TARGET_DEVICE: str = "cuda"
55
MAX_JOBS: Optional[str] = None
56
NVCC_THREADS: Optional[str] = None
@@ -376,10 +377,15 @@ environment_variables: Dict[str, Callable[[], Any]] = {
377
"VLLM_IMAGE_FETCH_TIMEOUT":
378
lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
379
380
+ # Timeout for fetching videos when serving multimodal models
381
+ # Default is 15 seconds
382
+ "VLLM_VIDEO_FETCH_TIMEOUT":
383
+ lambda: int(os.getenv("VLLM_VIDEO_FETCH_TIMEOUT", "15")),
384
+
385
# Timeout for fetching audio when serving multimodal models
380
- # Default is 5 seconds
386
+ # Default is 10 seconds
387
"VLLM_AUDIO_FETCH_TIMEOUT":
382
- lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "5")),
388
+ lambda: int(os.getenv("VLLM_AUDIO_FETCH_TIMEOUT", "10")),
389
390
# Path to the XLA persistent cache directory.
391
# Only used for XLA devices such as TPUs.
vllm/model_executor/models/llava_onevision.py
+3
-2
@@ -341,7 +341,7 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
341
get_llava_onevision_video_tokens(ctx, num_frames))
342
343
tokenizer = cached_get_tokenizer(model_config.tokenizer)
344
- new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
344
+ new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
345
tokenizer,
346
inputs.get("prompt"),
347
inputs["prompt_token_ids"],
@@ -350,7 +350,8 @@ def input_processor_when_multimodal_input_video(ctx: InputContext,
350
)
351
return token_inputs(prompt_token_ids=new_token_ids,
352
prompt=new_prompt,
353
- multi_modal_data=multi_modal_data)
353
+ multi_modal_data=multi_modal_data,
354
+ multi_modal_placeholders={"video": ranges})
355
else:
356
raise TypeError(f"Invalid video type: {type(video_data)}")
357
vllm/multimodal/base.py
+3
@@ -136,6 +136,9 @@ class MultiModalDataBuiltins(TypedDict, total=False):
136
audio: MultiModalData[Tuple[np.ndarray, Union[int, float]]]
137
"""The input audio item(s) and corresponding sampling rate(s)."""
138
139
+ video: MultiModalData[Tuple[np.ndarray]]
140
+ """The input video(s)."""
141
+
142
143
MultiModalDataDict = Union[MultiModalDataBuiltins,
144
Mapping[str, MultiModalData[object]]]
vllm/multimodal/utils.py
+114
-7
@@ -8,8 +8,8 @@ import numpy as np
8
import numpy.typing as npt
9
from PIL import Image
10
11
+import vllm.envs as envs
12
from vllm.connections import global_http_connection
12
-from vllm.envs import VLLM_AUDIO_FETCH_TIMEOUT, VLLM_IMAGE_FETCH_TIMEOUT
13
from vllm.logger import init_logger
14
from vllm.multimodal.base import MultiModalDataDict, PlaceholderRange
15
from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
@@ -80,7 +80,9 @@ def fetch_image(image_url: str,
80
"""
81
if image_url.startswith('http'):
82
image_raw = global_http_connection.get_bytes(
83
- image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT)
83
+ image_url,
84
+ timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
85
+ )
86
image = _load_image_from_bytes(image_raw)
87
88
elif image_url.startswith('data:image'):
@@ -105,7 +107,9 @@ async def async_fetch_image(image_url: str,
107
"""
108
if image_url.startswith('http'):
109
image_raw = await global_http_connection.async_get_bytes(
108
- image_url, timeout=VLLM_IMAGE_FETCH_TIMEOUT)
110
+ image_url,
111
+ timeout=envs.VLLM_IMAGE_FETCH_TIMEOUT,
112
+ )
113
image = _load_image_from_bytes(image_raw)
114
115
elif image_url.startswith('data:image'):
@@ -119,6 +123,85 @@ async def async_fetch_image(image_url: str,
123
return image.convert(image_mode)
124
125
126
+def _load_video_frames_from_bytes(b: bytes):
127
+ frame = Image.open(BytesIO(b))
128
+ return np.array(frame)
129
+
130
+
131
+def load_video_frames_from_base64(frame: Union[bytes, str]):
132
+ """Load frame from base64 format."""
133
+ return _load_video_frames_from_bytes(base64.b64decode(frame))
134
+
135
+
136
+def _load_video_from_bytes(b: bytes, num_frames: int = 32):
137
+ _, decord = try_import_video_packages()
138
+
139
+ video_path = BytesIO(b)
140
+ vr = decord.VideoReader(video_path, num_threads=1)
141
+ total_frame_num = len(vr)
142
+
143
+ if total_frame_num > num_frames:
144
+ uniform_sampled_frames = np.linspace(0,
145
+ total_frame_num - 1,
146
+ num_frames,
147
+ dtype=int)
148
+ frame_idx = uniform_sampled_frames.tolist()
149
+ else:
150
+ frame_idx = [i for i in range(0, total_frame_num)]
151
+ frames = vr.get_batch(frame_idx).asnumpy()
152
+
153
+ return frames
154
+
155
+
156
+def _load_video_from_data_url(video_url: str):
157
+ # Only split once and assume the second part is the base64 encoded image
158
+ frames_base64 = video_url.split(",")[1:]
159
+ return np.stack([
160
+ load_video_frames_from_base64(frame_base64)
161
+ for frame_base64 in frames_base64
162
+ ])
163
+
164
+
165
+def fetch_video(video_url: str, *, num_frames: int = 32) -> npt.NDArray:
166
+ """
167
+ Load video from a HTTP or base64 data URL.
168
+ """
169
+ if video_url.startswith('http') or video_url.startswith('https'):
170
+ video_raw = global_http_connection.get_bytes(
171
+ video_url,
172
+ timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
173
+ )
174
+ video = _load_video_from_bytes(video_raw, num_frames)
175
+ elif video_url.startswith('data:video'):
176
+ video = _load_video_from_data_url(video_url)
177
+ else:
178
+ raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
179
+ "with either 'data:video' or 'http'.")
180
+ return video
181
+
182
+
183
+async def async_fetch_video(video_url: str,
184
+ *,
185
+ num_frames: int = 32) -> npt.NDArray:
186
+ """
187
+ Asynchronously load video from a HTTP or base64 data URL.
188
+
189
+ By default, the image is converted into RGB format.
190
+ """
191
+ if video_url.startswith('http') or video_url.startswith('https'):
192
+ video_raw = await global_http_connection.async_get_bytes(
193
+ video_url,
194
+ timeout=envs.VLLM_VIDEO_FETCH_TIMEOUT,
195
+ )
196
+ video = _load_video_from_bytes(video_raw, num_frames)
197
+ elif video_url.startswith('data:video'):
198
+ video = _load_video_from_data_url(video_url)
199
+ else:
200
+ raise ValueError("Invalid 'video_url': A valid 'video_url' must start "
201
+ "with either 'data:video' or 'http'.")
202
+ return video
203
+
204
+
205
def try_import_audio_packages() -> Tuple[Any, Any]:
206
try:
207
import librosa
@@ -137,7 +220,9 @@ def fetch_audio(audio_url: str) -> Tuple[np.ndarray, Union[int, float]]:
220
221
if audio_url.startswith("http"):
222
audio_bytes = global_http_connection.get_bytes(
140
- audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
223
+ audio_url,
224
+ timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
225
+ )
226
elif audio_url.startswith("data:audio"):
227
_, audio_base64 = audio_url.split(",", 1)
228
audio_bytes = base64.b64decode(audio_base64)
@@ -157,7 +242,9 @@ async def async_fetch_audio(
242
243
if audio_url.startswith("http"):
244
audio_bytes = await global_http_connection.async_get_bytes(
160
- audio_url, timeout=VLLM_AUDIO_FETCH_TIMEOUT)
245
+ audio_url,
246
+ timeout=envs.VLLM_AUDIO_FETCH_TIMEOUT,
247
+ )
248
elif audio_url.startswith("data:audio"):
249
_, audio_base64 = audio_url.split(",", 1)
250
audio_bytes = base64.b64decode(audio_base64)
@@ -182,6 +269,11 @@ def get_and_parse_image(
269
return {"image": image}
270
271
272
+def get_and_parse_video(video_url: str) -> MultiModalDataDict:
273
+ video = fetch_video(video_url)
274
+ return {"video": video}
275
+
276
+
277
async def async_get_and_parse_audio(audio_url: str) -> MultiModalDataDict:
278
audio, sr = await async_fetch_audio(audio_url)
279
return {"audio": (audio, sr)}
@@ -196,6 +288,11 @@ async def async_get_and_parse_image(
288
return {"image": image}
289
290
291
+async def async_get_and_parse_video(video_url: str) -> MultiModalDataDict:
292
+ video = await async_fetch_video(video_url)
293
+ return {"video": video}
294
+
295
+
296
def encode_audio_base64(
297
audio: np.ndarray,
298
sampling_rate: int,
@@ -246,14 +343,15 @@ def rescale_image_size(image: Image.Image,
343
def try_import_video_packages() -> Any:
344
try:
345
import cv2
346
+ import decord
347
except ImportError:
348
raise ImportError(
349
"Please install vllm[video] for video support.") from None
252
- return cv2
350
+ return cv2, decord
351
352
353
def resize_video(frames: npt.NDArray, size: Tuple[int, int]) -> npt.NDArray:
256
- cv2 = try_import_video_packages()
354
+ cv2, _ = try_import_video_packages()
355
356
num_frames, _, _, channels = frames.shape
357
new_height, new_width = size
@@ -284,6 +382,15 @@ def sample_frames_from_video(frames: npt.NDArray,
382
return sampled_frames
383
384
385
+def encode_video_base64(frames: npt.NDArray):
386
+ base64_frames = []
387
+ frames_list = [frames[i] for i in range(frames.shape[0])]
388
+ for frame in frames_list:
389
+ img_base64 = encode_image_base64(Image.fromarray(frame))
390
+ base64_frames.append(img_base64)
391
+ return ",".join(base64_frames)
392
+
393
+
394
# Utilities for input processors
395
_T = TypeVar("_T", str, int)
396
vllm/multimodal/video.py
+2
-1
@@ -7,6 +7,7 @@ from vllm.inputs.registry import InputContext
7
from vllm.logger import init_logger
8
from vllm.transformers_utils.processor import get_video_processor
9
from vllm.transformers_utils.tokenizer import get_tokenizer
10
+from vllm.utils import is_list_of
11
12
from .base import MultiModalData, MultiModalInputs
13
from .image import ImagePlugin
@@ -60,7 +61,7 @@ class VideoPlugin(ImagePlugin):
61
if isinstance(data, list) and len(data) == 1:
62
data = data[0]
63
63
- if isinstance(data, np.ndarray):
64
+ if isinstance(data, np.ndarray) or is_list_of(data, np.ndarray):
65
video_processor = self._get_hf_video_processor(
66
model_config,
67
mm_processor_kwargs,