A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[model] Support for Llava-Next-Video model (#7559)
Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Yangshen⚡Deng committed
Sep 11, 2024 at 13:21 UTC
6a512a00dfa306762c2878bffc3a5664a758d105
21 files changed
+1083
-18
Dockerfile
+1
@@ -145,6 +145,7 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
145
&& echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
146
&& apt-get update -y \
147
&& apt-get install -y ccache software-properties-common git curl sudo vim python3-pip \
148
+ && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
149
&& add-apt-repository ppa:deadsnakes/ppa \
150
&& apt-get update -y \
151
&& apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
Dockerfile.cpu
+1
@@ -5,6 +5,7 @@ FROM ubuntu:22.04 AS cpu-test-1
5
RUN --mount=type=cache,target=/var/cache/apt \
6
apt-get update -y \
7
&& apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
8
+ && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
9
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
10
11
# https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html
Dockerfile.neuron
+3
-1
@@ -6,7 +6,9 @@ FROM $BASE_IMAGE
6
RUN echo "Base image is $BASE_IMAGE"
7
8
# Install some basic utilities
9
-RUN apt-get update && apt-get install python3 python3-pip -y
9
+RUN apt-get update \
10
+ && apt-get install python3 python3-pip -y \
11
+ && apt-get install -y ffmpeg libsm6 libxext6 libgl1
12
13
### Mount Point ###
14
# When launching the container, mount the code directory to /app
Dockerfile.openvino
+2
-1
@@ -4,7 +4,8 @@
4
FROM ubuntu:22.04 AS dev
5
6
RUN apt-get update -y && \
7
- apt-get install -y python3-pip git
7
+ apt-get install -y python3-pip git && \
8
+ apt-get install -y ffmpeg libsm6 libxext6 libgl1
9
WORKDIR /workspace
10
11
# copy requirements
Dockerfile.ppc64le
+1
-1
@@ -4,7 +4,7 @@ USER root
4
5
ENV PATH="/usr/local/cargo/bin:$PATH:/opt/conda/bin/"
6
7
-RUN apt-get update -y && apt-get install -y git wget curl vim libnuma-dev libsndfile-dev libprotobuf-dev build-essential
7
+RUN apt-get update -y && apt-get install -y git wget curl vim libnuma-dev libsndfile-dev libprotobuf-dev build-essential ffmpeg libsm6 libxext6 libgl1
8
9
# Some packages in requirements-cpu are installed here
10
# IBM provides optimized packages for ppc64le processors in the open-ce project for mamba
Dockerfile.tpu
+3
@@ -4,6 +4,9 @@ ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:night
4
FROM $BASE_IMAGE
5
WORKDIR /workspace
6
7
+# Install some basic utilities
8
+RUN apt-get update && apt-get install -y ffmpeg libsm6 libxext6 libgl1
9
+
10
# Install the TPU and Pallas dependencies.
11
RUN python3 -m pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
12
RUN python3 -m pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
Dockerfile.xpu
+1
-2
@@ -9,8 +9,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
9
chmod 644 /usr/share/keyrings/intel-graphics.gpg
10
11
RUN apt-get update -y \
12
-&& apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip
13
-
12
+&& apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip ffmpeg libsm6 libxext6 libgl1
13
COPY ./ /workspace/vllm
14
15
WORKDIR /workspace/vllm
docs/source/conf.py
+1
@@ -99,6 +99,7 @@ autodoc_mock_imports = [
99
"aiohttp",
100
"compressed_tensors",
101
"cpuinfo",
102
+ "cv2",
103
"torch",
104
"transformers",
105
"psutil",
docs/source/models/supported_models.rst
+14
@@ -227,6 +227,11 @@ Multimodal Language Models
227
- Image\ :sup:`E+`
228
- :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
229
-
230
+ * - :code:`LlavaNextVideoForConditionalGeneration`
231
+ - LLaVA-NeXT-Video
232
+ - Video
233
+ - :code:`llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. (see note)
234
+ -
235
* - :code:`MiniCPMV`
236
- MiniCPM-V
237
- Image\ :sup:`+`
@@ -260,6 +265,15 @@ Multimodal Language Models
265
For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
266
For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
267
268
+ For :code:`LLaVA-NeXT-Video`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
269
+ This can be installed by running the following command:
270
+
271
+
272
+ .. code-block:: bash
273
+
274
+ pip install git+https://github.com/huggingface/transformers.git@21fac7abba2a37fae86106f87fcf9974fd1e3830
275
+
276
+
277
----
278
279
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
examples/offline_inference_vision_language.py
+61
-9
@@ -9,12 +9,9 @@ from transformers import AutoTokenizer
9
10
from vllm import LLM, SamplingParams
11
from vllm.assets.image import ImageAsset
12
+from vllm.assets.video import VideoAsset
13
from vllm.utils import FlexibleArgumentParser
14
14
-# Input image and question
15
-image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
16
-question = "What is the content of this image?"
17
-
15
16
# LLaVA-1.5
17
def run_llava(question):
@@ -30,7 +27,16 @@ def run_llava(question):
27
def run_llava_next(question):
28
29
prompt = f"[INST] <image>\n{question} [/INST]"
33
- llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf")
30
+ llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
31
+ stop_token_ids = None
32
+ return llm, prompt, stop_token_ids
33
+
34
+
35
+# LlaVA-NeXT-Video
36
+# Currently only support for video input
37
+def run_llava_next_video(question):
38
+ prompt = f"USER: <video>\n{question} ASSISTANT:"
39
+ llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
40
stop_token_ids = None
41
return llm, prompt, stop_token_ids
42
@@ -176,6 +182,7 @@ def run_qwen_vl(question):
182
model_example_map = {
183
"llava": run_llava,
184
"llava-next": run_llava_next,
185
+ "llava-next-video": run_llava_next_video,
186
"fuyu": run_fuyu,
187
"phi3_v": run_phi3v,
188
"paligemma": run_paligemma,
@@ -187,11 +194,49 @@ model_example_map = {
194
}
195
196
197
+def get_multi_modal_input(args):
198
+ """
199
+ return {
200
+ "data": image or video,
201
+ "question": question,
202
+ }
203
+ """
204
+ if args.modality == "image":
205
+ # Input image and question
206
+ image = ImageAsset("cherry_blossom") \
207
+ .pil_image.convert("RGB")
208
+ img_question = "What is the content of this image?"
209
+
210
+ return {
211
+ "data": image,
212
+ "question": img_question,
213
+ }
214
+
215
+ if args.modality == "video":
216
+ # Input video and question
217
+ video = VideoAsset(name="sample_demo_1.mp4",
218
+ num_frames=args.num_frames).np_ndarrays
219
+ vid_question = "Why is this video funny?"
220
+
221
+ return {
222
+ "data": video,
223
+ "question": vid_question,
224
+ }
225
+
226
+ msg = f"Modality {args.modality} is not supported."
227
+ raise ValueError(msg)
228
+
229
+
230
def main(args):
231
model = args.model_type
232
if model not in model_example_map:
233
raise ValueError(f"Model type {model} is not supported.")
234
235
+ modality = args.modality
236
+ mm_input = get_multi_modal_input(args)
237
+ data = mm_input["data"]
238
+ question = mm_input["question"]
239
+
240
llm, prompt, stop_token_ids = model_example_map[model](question)
241
242
# We set temperature to 0.2 so that outputs can be different
@@ -206,7 +251,7 @@ def main(args):
251
inputs = {
252
"prompt": prompt,
253
"multi_modal_data": {
209
- "image": image
254
+ modality: data
255
},
256
}
257
@@ -215,7 +260,7 @@ def main(args):
260
inputs = [{
261
"prompt": prompt,
262
"multi_modal_data": {
218
- "image": image
263
+ modality: data
264
},
265
} for _ in range(args.num_prompts)]
266
@@ -238,8 +283,15 @@ if __name__ == "__main__":
283
help='Huggingface "model_type".')
284
parser.add_argument('--num-prompts',
285
type=int,
241
- default=1,
286
+ default=4,
287
help='Number of prompts to run.')
243
-
288
+ parser.add_argument('--modality',
289
+ type=str,
290
+ default="image",
291
+ help='Modality of the input.')
292
+ parser.add_argument('--num-frames',
293
+ type=int,
294
+ default=16,
295
+ help='Number of frames to extract from the video.')
296
args = parser.parse_args()
297
main(args)
requirements-test.txt
+1
@@ -11,6 +11,7 @@ awscli
11
einops # required for MPT, qwen-vl and Mamba
12
httpx
13
librosa # required for audio test
14
+opencv-python # required for video test
15
peft
16
requests
17
ray[adag]>=2.35
setup.py
+1
@@ -505,6 +505,7 @@ setup(
505
ext_modules=ext_modules,
506
extras_require={
507
"tensorizer": ["tensorizer>=2.9.0"],
508
+ "video": ["opencv-python"], # Required for video processing
509
"audio": ["librosa", "soundfile"] # Required for audio processing
510
},
511
cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
tests/conftest.py
+55
-1
@@ -21,6 +21,7 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
21
22
from vllm import LLM, SamplingParams
23
from vllm.assets.image import ImageAsset
24
+from vllm.assets.video import VideoAsset
25
from vllm.config import TokenizerPoolConfig
26
from vllm.connections import global_http_connection
27
from vllm.distributed import (destroy_distributed_environment,
@@ -44,6 +45,7 @@ _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
45
PromptImageInput = Union[List[Image.Image], List[List[Image.Image]]]
46
PromptAudioInput = Union[List[Tuple[np.ndarray, int]],
47
List[List[Tuple[np.ndarray, int]]]]
48
+PromptVideoInput = Union[List[np.ndarray], List[List[np.ndarray]]]
49
50
51
def _read_prompts(filename: str) -> List[str]:
@@ -85,8 +87,35 @@ class _ImageAssets(_ImageAssetsBase):
87
return [prompts["stop_sign"], prompts["cherry_blossom"]]
88
89
90
+class _VideoAssetPrompts(TypedDict):
91
+ sample_demo_1: str
92
+
93
+
94
+if sys.version_info < (3, 9):
95
+ # UserList cannot be subscripted
96
+ class _VideoAssetsBase(UserList):
97
+ pass
98
+else:
99
+
100
+ class _VideoAssetsBase(UserList[VideoAsset]):
101
+ pass
102
+
103
+
104
+class _VideoAssets(_VideoAssetsBase):
105
+
106
+ def __init__(self) -> None:
107
+ super().__init__([
108
+ VideoAsset("sample_demo_1.mp4"),
109
+ ])
110
+
111
+ def prompts(self, prompts: _VideoAssetPrompts) -> List[str]:
112
+ return [prompts["sample_demo_1"]]
113
+
114
+
115
IMAGE_ASSETS = _ImageAssets()
116
"""Singleton instance of :class:`_ImageAssets`."""
117
+VIDEO_ASSETS = _VideoAssets()
118
+"""Singleton instance of :class:`_VideoAssets`."""
119
120
121
@pytest.fixture(autouse=True)
@@ -202,6 +231,11 @@ def image_assets() -> _ImageAssets:
231
return IMAGE_ASSETS
232
233
234
+@pytest.fixture(scope="session")
235
+def video_assets() -> _VideoAssets:
236
+ return VIDEO_ASSETS
237
+
238
+
239
_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
240
241
@@ -279,6 +313,7 @@ class HfRunner:
313
self,
314
prompts: List[str],
315
images: Optional[PromptImageInput] = None,
316
+ videos: Optional[List[np.ndarray]] = None,
317
**kwargs: Any,
318
) -> List[Tuple[List[List[int]], List[str]]]:
319
if images:
@@ -292,6 +327,8 @@ class HfRunner:
327
}
328
if images is not None and images[i] is not None:
329
processor_kwargs["images"] = images[i]
330
+ if videos is not None and videos[i] is not None:
331
+ processor_kwargs["videos"] = videos[i]
332
333
inputs = self.processor(**processor_kwargs)
334
inputs = self.postprocess_inputs(inputs)
@@ -352,6 +389,7 @@ class HfRunner:
389
prompts: List[str],
390
max_tokens: int,
391
images: Optional[PromptImageInput] = None,
392
+ videos: Optional[List[np.ndarray]] = None,
393
**kwargs: Any,
394
) -> List[List[torch.Tensor]]:
395
all_logprobs: List[List[torch.Tensor]] = []
@@ -362,6 +400,8 @@ class HfRunner:
400
}
401
if images is not None and images[i] is not None:
402
processor_kwargs["images"] = images[i]
403
+ if videos is not None and videos[i] is not None:
404
+ processor_kwargs["videos"] = videos[i]
405
406
inputs = self.processor(**processor_kwargs)
407
inputs = self.postprocess_inputs(inputs)
@@ -435,6 +475,7 @@ class HfRunner:
475
num_logprobs: int,
476
images: Optional[PromptImageInput] = None,
477
audios: Optional[PromptAudioInput] = None,
478
+ videos: Optional[List[np.ndarray]] = None,
479
**kwargs: Any,
480
) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
481
all_logprobs: List[List[Dict[int, float]]] = []
@@ -454,6 +495,8 @@ class HfRunner:
495
processor_kwargs["audio"] = audio
496
processor_kwargs["sampling_rate"] = sr
497
498
+ if videos is not None:
499
+ processor_kwargs["videos"] = videos[i]
500
inputs = self.processor(**processor_kwargs)
501
inputs = self.postprocess_inputs(inputs)
502
@@ -634,12 +677,16 @@ class VllmRunner:
677
sampling_params: SamplingParams,
678
images: Optional[PromptImageInput] = None,
679
audios: Optional[PromptAudioInput] = None,
680
+ videos: Optional[PromptVideoInput] = None,
681
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
682
assert sampling_params.logprobs is not None
683
684
if images is not None:
685
assert len(prompts) == len(images)
686
687
+ if videos is not None:
688
+ assert len(prompts) == len(videos)
689
+
690
inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
691
if images is not None:
692
for i, image in enumerate(images):
@@ -649,6 +696,11 @@ class VllmRunner:
696
for i, audio in enumerate(audios):
697
inputs[i]["multi_modal_data"] = {"audio": audio}
698
699
+ if videos is not None:
700
+ for i, video in enumerate(videos):
701
+ inputs[i]["multi_modal_data"] = {"video": video}
702
+ print(f"[INPUTS!!!!]: {inputs}, {sampling_params}")
703
+
704
req_outputs = self.model.generate(inputs,
705
sampling_params=sampling_params)
706
return self._final_steps_generate_w_logprobs(req_outputs)
@@ -685,6 +737,7 @@ class VllmRunner:
737
num_logprobs: int,
738
images: Optional[PromptImageInput] = None,
739
audios: Optional[PromptAudioInput] = None,
740
+ videos: Optional[PromptVideoInput] = None,
741
stop_token_ids: Optional[List[int]] = None,
742
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
743
greedy_logprobs_params = SamplingParams(temperature=0.0,
@@ -694,7 +747,8 @@ class VllmRunner:
747
outputs = self.generate_w_logprobs(prompts,
748
greedy_logprobs_params,
749
images=images,
697
- audios=audios)
750
+ audios=audios,
751
+ videos=videos)
752
753
return [(output_ids, output_str, output_logprobs)
754
for output_ids, output_str, output_logprobs in outputs]
tests/models/test_llava_next_video.py
new
+236
@@ -0,0 +1,236 @@
1
+from typing import List, Optional, Tuple, Type, overload
2
+
3
+import pytest
4
+import transformers
5
+from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
6
+
7
+from vllm.multimodal.utils import (rescale_video_size, resize_video,
8
+ sample_frames_from_video)
9
+from vllm.sequence import SampleLogprobs
10
+
11
+from ..conftest import VIDEO_ASSETS, HfRunner, VllmRunner, _VideoAssets
12
+from .utils import check_logprobs_close
13
+
14
+pytestmark = pytest.mark.vlm
15
+
16
+_PREFACE = (
17
+ "A chat between a curious human and an artificial intelligence assistant. "
18
+ "The assistant gives helpful, detailed, and polite answers to the human's "
19
+ "questions.")
20
+
21
+HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
22
+ "sample_demo_1":
23
+ f"{_PREFACE}USER: <video>\nWhy is this video funny? ASSISTANT:"
24
+})
25
+
26
+models = ["llava-hf/LLaVA-NeXT-Video-7B-hf"]
27
+
28
+
29
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
30
+ Optional[SampleLogprobs]],
31
+ model: str):
32
+ """Sanitize vllm output to be comparable with hf output."""
33
+ output_ids, output_str, out_logprobs = vllm_output
34
+
35
+ config = AutoConfig.from_pretrained(model)
36
+ video_token_id = config.video_token_index
37
+
38
+ tokenizer = AutoTokenizer.from_pretrained(model)
39
+ eos_token_id = tokenizer.eos_token_id
40
+
41
+ hf_output_ids = [
42
+ token_id for idx, token_id in enumerate(output_ids)
43
+ if token_id != video_token_id or output_ids[idx - 1] != video_token_id
44
+ ]
45
+
46
+ assert output_str[0] == " "
47
+ hf_output_str = output_str[1:]
48
+ if hf_output_ids[-1] == eos_token_id:
49
+ hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
50
+
51
+ return hf_output_ids, hf_output_str, out_logprobs
52
+
53
+
54
+@overload
55
+def run_test(
56
+ hf_runner: Type[HfRunner],
57
+ vllm_runner: Type[VllmRunner],
58
+ video_assets: _VideoAssets,
59
+ model: str,
60
+ *,
61
+ size_factors: List[float],
62
+ dtype: str,
63
+ max_tokens: int,
64
+ num_logprobs: int,
65
+ num_frames: int,
66
+ tensor_parallel_size: int,
67
+ distributed_executor_backend: Optional[str] = None,
68
+):
69
+ ...
70
+
71
+
72
+@overload
73
+def run_test(
74
+ hf_runner: Type[HfRunner],
75
+ vllm_runner: Type[VllmRunner],
76
+ video_assets: _VideoAssets,
77
+ model: str,
78
+ *,
79
+ sizes: List[Tuple[int, int]],
80
+ dtype: str,
81
+ max_tokens: int,
82
+ num_logprobs: int,
83
+ num_frames: int,
84
+ tensor_parallel_size: int,
85
+ distributed_executor_backend: Optional[str] = None,
86
+):
87
+ ...
88
+
89
+
90
+def run_test(
91
+ hf_runner: Type[HfRunner],
92
+ vllm_runner: Type[VllmRunner],
93
+ video_assets: _VideoAssets,
94
+ model: str,
95
+ *,
96
+ size_factors: Optional[List[float]] = None,
97
+ sizes: Optional[List[Tuple[int, int]]] = None,
98
+ dtype: str,
99
+ max_tokens: int,
100
+ num_logprobs: int,
101
+ num_frames: int,
102
+ tensor_parallel_size: int,
103
+ distributed_executor_backend: Optional[str] = None,
104
+):
105
+ videos = [
106
+ sample_frames_from_video(asset.np_ndarrays, num_frames)
107
+ for asset in video_assets
108
+ ]
109
+
110
+ for video in videos:
111
+ print(video.shape)
112
+
113
+ if size_factors is not None:
114
+ inputs_per_video = [(
115
+ [prompt for _ in size_factors],
116
+ [rescale_video_size(video, factor) for factor in size_factors],
117
+ ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
118
+ elif sizes is not None:
119
+ inputs_per_video = [(
120
+ [prompt for _ in sizes],
121
+ [resize_video(video, size) for size in sizes],
122
+ ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
123
+ else:
124
+ raise ValueError("You must provide either `size_factors` or `sizes`")
125
+
126
+ # max_model_len should be greater than image_feature_size
127
+ with vllm_runner(model,
128
+ dtype=dtype,
129
+ max_model_len=4096,
130
+ tensor_parallel_size=tensor_parallel_size,
131
+ distributed_executor_backend=distributed_executor_backend,
132
+ enforce_eager=True) as vllm_model:
133
+ vllm_outputs_per_video = [
134
+ vllm_model.generate_greedy_logprobs(prompts,
135
+ max_tokens,
136
+ num_logprobs=num_logprobs,
137
+ videos=videos)
138
+ for prompts, videos in inputs_per_video
139
+ ]
140
+
141
+ with hf_runner(model, dtype=dtype,
142
+ auto_cls=AutoModelForVision2Seq) as hf_model:
143
+ hf_outputs_per_video = [
144
+ hf_model.generate_greedy_logprobs_limit(prompts,
145
+ max_tokens,
146
+ num_logprobs=num_logprobs,
147
+ videos=videos)
148
+ for prompts, videos in inputs_per_video
149
+ ]
150
+
151
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_video,
152
+ vllm_outputs_per_video):
153
+ # TODO: Check whether using original CLIPVisionModel can improve
154
+ # consistency against HF
155
+ check_logprobs_close(
156
+ outputs_0_lst=hf_outputs,
157
+ outputs_1_lst=[
158
+ vllm_to_hf_output(vllm_output, model)
159
+ for vllm_output in vllm_outputs
160
+ ],
161
+ name_0="hf",
162
+ name_1="vllm",
163
+ )
164
+
165
+
166
+@pytest.mark.skipif(transformers.__version__ < "4.45",
167
+ reason="Waiting for next transformers release")
168
+@pytest.mark.parametrize("model", models)
169
+@pytest.mark.parametrize(
170
+ "size_factors",
171
+ [
172
+ # No video
173
+ [],
174
+ # Single-scale
175
+ [1.0],
176
+ # Single-scale, batched
177
+ [1.0, 1.0, 1.0],
178
+ # Multi-scale
179
+ [0.25, 0.5, 1.0],
180
+ ],
181
+)
182
+@pytest.mark.parametrize("dtype", ["half"])
183
+@pytest.mark.parametrize("max_tokens", [128])
184
+@pytest.mark.parametrize("num_logprobs", [5])
185
+@pytest.mark.parametrize("num_frames", [16])
186
+def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
187
+ dtype, max_tokens, num_logprobs, num_frames) -> None:
188
+ """Inference result should be the same between hf and vllm.
189
+
190
+ All the image fixtures for the test is under tests/videos.
191
+ For huggingface runner, we provide the np.ndarray as input.
192
+ For vllm runner, we provide MultiModalDataDict objects
193
+ and corresponding MultiModalConfig as input.
194
+ Note, the text input is also adjusted to abide by vllm contract.
195
+ The text output is sanitized to be able to compare with hf.
196
+ """
197
+ run_test(
198
+ hf_runner,
199
+ vllm_runner,
200
+ video_assets,
201
+ model,
202
+ size_factors=size_factors,
203
+ dtype=dtype,
204
+ max_tokens=max_tokens,
205
+ num_logprobs=num_logprobs,
206
+ num_frames=num_frames,
207
+ tensor_parallel_size=1,
208
+ )
209
+
210
+
211
+@pytest.mark.skipif(transformers.__version__ < "4.45",
212
+ reason="Waiting for next transformers release")
213
+@pytest.mark.parametrize("model", models)
214
+@pytest.mark.parametrize(
215
+ "sizes",
216
+ [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
217
+)
218
+@pytest.mark.parametrize("dtype", ["half"])
219
+@pytest.mark.parametrize("max_tokens", [128])
220
+@pytest.mark.parametrize("num_logprobs", [5])
221
+@pytest.mark.parametrize("num_frames", [16])
222
+def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
223
+ dtype, max_tokens, num_logprobs,
224
+ num_frames) -> None:
225
+ run_test(
226
+ hf_runner,
227
+ vllm_runner,
228
+ video_assets,
229
+ model,
230
+ sizes=sizes,
231
+ dtype=dtype,
232
+ max_tokens=max_tokens,
233
+ num_logprobs=num_logprobs,
234
+ num_frames=num_frames,
235
+ tensor_parallel_size=1,
236
+ )
vllm/assets/video.py
new
+85
@@ -0,0 +1,85 @@
1
+from dataclasses import dataclass
2
+from functools import lru_cache
3
+from typing import List, Literal
4
+
5
+import numpy as np
6
+import numpy.typing as npt
7
+from huggingface_hub import hf_hub_download
8
+from PIL import Image
9
+
10
+from vllm.multimodal.utils import (sample_frames_from_video,
11
+ try_import_video_packages)
12
+
13
+from .base import get_cache_dir
14
+
15
+
16
+@lru_cache
17
+def download_video_asset(filename: str) -> str:
18
+ """
19
+ Download and open an image from huggingface
20
+ repo: raushan-testing-hf/videos-test
21
+ """
22
+ video_directory = get_cache_dir() / "video-eample-data"
23
+ video_directory.mkdir(parents=True, exist_ok=True)
24
+
25
+ video_path = video_directory / filename
26
+ video_path_str = str(video_path)
27
+ if not video_path.exists():
28
+ video_path_str = hf_hub_download(
29
+ repo_id="raushan-testing-hf/videos-test",
30
+ filename=filename,
31
+ repo_type="dataset",
32
+ cache_dir=video_directory,
33
+ )
34
+ return video_path_str
35
+
36
+
37
+def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
38
+ cv2 = try_import_video_packages()
39
+
40
+ cap = cv2.VideoCapture(path)
41
+ if not cap.isOpened():
42
+ raise ValueError(f"Could not open video file {path}")
43
+
44
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
45
+ frames = []
46
+ for i in range(total_frames):
47
+ ret, frame = cap.read()
48
+ if ret:
49
+ frames.append(frame)
50
+ cap.release()
51
+
52
+ frames = np.stack(frames)
53
+ frames = sample_frames_from_video(frames, num_frames)
54
+ if len(frames) < num_frames:
55
+ raise ValueError(f"Could not read enough frames from video file {path}"
56
+ f" (expected {num_frames} frames, got {len(frames)})")
57
+ return frames
58
+
59
+
60
+def video_to_pil_images_list(path: str,
61
+ num_frames: int = -1) -> List[Image.Image]:
62
+ cv2 = try_import_video_packages()
63
+ frames = video_to_ndarrays(path, num_frames)
64
+ return [
65
+ Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
66
+ for frame in frames
67
+ ]
68
+
69
+
70
+@dataclass(frozen=True)
71
+class VideoAsset:
72
+ name: Literal["sample_demo_1.mp4"]
73
+ num_frames: int = -1
74
+
75
+ @property
76
+ def pil_images(self) -> List[Image.Image]:
77
+ video_path = download_video_asset(self.name)
78
+ ret = video_to_pil_images_list(video_path, self.num_frames)
79
+ return ret
80
+
81
+ @property
82
+ def np_ndarrays(self) -> List[npt.NDArray]:
83
+ video_path = download_video_asset(self.name)
84
+ ret = video_to_ndarrays(video_path, self.num_frames)
85
+ return ret
vllm/model_executor/models/__init__.py
+4
-2
@@ -80,8 +80,10 @@ _MULTIMODAL_MODELS = {
80
"InternVLChatModel": ("internvl", "InternVLChatModel"),
81
"LlavaForConditionalGeneration":
82
("llava", "LlavaForConditionalGeneration"),
83
- "LlavaNextForConditionalGeneration":
84
- ("llava_next", "LlavaNextForConditionalGeneration"),
83
+ "LlavaNextForConditionalGeneration": ("llava_next",
84
+ "LlavaNextForConditionalGeneration"),
85
+ "LlavaNextVideoForConditionalGeneration":
86
+ ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
87
"MiniCPMV": ("minicpmv", "MiniCPMV"),
88
"PaliGemmaForConditionalGeneration": ("paligemma",
89
"PaliGemmaForConditionalGeneration"),
vllm/model_executor/models/llava_next_video.py
new
+471
@@ -0,0 +1,471 @@
1
+import itertools
2
+import math
3
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
4
+ TypedDict, Union)
5
+
6
+import numpy as np
7
+import torch
8
+import torch.nn as nn
9
+from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
10
+ SiglipVisionConfig)
11
+
12
+from vllm.attention import AttentionMetadata
13
+from vllm.config import CacheConfig, MultiModalConfig
14
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15
+from vllm.logger import init_logger
16
+from vllm.model_executor.layers.activation import get_act_fn
17
+from vllm.model_executor.layers.quantization.base_config import (
18
+ QuantizationConfig)
19
+from vllm.model_executor.layers.sampler import SamplerOutput
20
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
+from vllm.model_executor.models.clip import CLIPVisionModel
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
+from vllm.multimodal import MULTIMODAL_REGISTRY
24
+from vllm.multimodal.utils import (cached_get_tokenizer,
25
+ repeat_and_pad_placeholder_tokens)
26
+from vllm.sequence import IntermediateTensors
27
+from vllm.utils import is_list_of
28
+
29
+from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
30
+from .interfaces import SupportsMultiModal
31
+from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
32
+ dummy_seq_data_for_siglip)
33
+from .utils import (filter_weights, init_vllm_registered_model,
34
+ merge_multimodal_embeddings)
35
+
36
+logger = init_logger(__name__)
37
+
38
+# For profile run
39
+_MAX_FRAMES_PER_VIDEO = 32
40
+_MAX_NUM_VIDEOS = 1
41
+
42
+
43
+class LlavaNextVideoPixelInputs(TypedDict):
44
+ type: Literal["pixel_values_videos"]
45
+ data: Union[torch.Tensor, List[torch.Tensor]]
46
+ """
47
+ Shape: `(batch_size, num_frames, num_channels, height, width)`
48
+
49
+ Note that `num_frames` may be different for each batch, in which case
50
+ the data is passed as a list instead of a batched tensor.
51
+
52
+ Note that it only supports one video input for one batch.
53
+ """
54
+
55
+
56
+def get_llava_next_video_frame_feature_size(
57
+ hf_config: LlavaNextVideoConfig) -> int:
58
+ # Support both CLIPVisionConfig and SiglipVisionConfig
59
+ image_size = hf_config.vision_config.image_size
60
+ patch_size = hf_config.vision_config.patch_size
61
+ spatial_pool_stride = hf_config.spatial_pool_stride
62
+
63
+ return int((image_size / patch_size / spatial_pool_stride)**2)
64
+
65
+
66
+def _get_max_llm_tokens(ctx: InputContext) -> int:
67
+ """
68
+ Calculated from the maximum video frames under the context length
69
+ constraints of the language model.
70
+ """
71
+ hf_text_config = ctx.model_config.hf_text_config
72
+ model_config = ctx.model_config
73
+ max_tokens = model_config.max_model_len
74
+ rope_scaling = model_config.rope_scaling
75
+
76
+ if rope_scaling:
77
+ rope_scaling_factor = hf_text_config.rope_scaling["factor"]
78
+ else:
79
+ rope_scaling_factor = 1
80
+
81
+ max_tokens *= rope_scaling_factor
82
+
83
+ return max_tokens
84
+
85
+
86
+def get_max_llava_next_video_tokens(ctx: InputContext) -> int:
87
+ # Currently set to 32 frames
88
+ # TODO: max_tokens = _get_max_llm_tokens(ctx)
89
+ hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
90
+ tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
91
+ return _MAX_FRAMES_PER_VIDEO * tokens_per_frame
92
+
93
+
94
+def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
95
+ mm_counts: Mapping[str, int]):
96
+ hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
97
+ vision_config = hf_config.vision_config
98
+
99
+ # TODO: support multiple videos
100
+ num_videos = mm_counts["video"]
101
+ if num_videos != _MAX_NUM_VIDEOS:
102
+ raise NotImplementedError(
103
+ f"Only {_MAX_NUM_VIDEOS} videos are supported")
104
+
105
+ # TODO: support configuring the number of frames
106
+ frames_per_video = _MAX_FRAMES_PER_VIDEO
107
+ # num_images = num_videos * frames_per_video
108
+
109
+ # fills the sequence with as longer video data as possible
110
+ tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
111
+ video_feature_size = frames_per_video * tokens_per_frame
112
+
113
+ if isinstance(vision_config, CLIPVisionConfig):
114
+ seq_data = dummy_seq_data_for_clip(
115
+ vision_config,
116
+ seq_len,
117
+ num_videos,
118
+ image_token_id=hf_config.video_token_index,
119
+ image_feature_size_override=video_feature_size,
120
+ )
121
+
122
+ pil_frame = dummy_image_for_clip(vision_config, num_images=1)
123
+ np_frame = np.array(pil_frame["image"])
124
+ mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
125
+ mm_data = {"video": mm_data_per_video}
126
+ return seq_data, mm_data
127
+ elif isinstance(vision_config, SiglipVisionConfig):
128
+ seq_data = dummy_seq_data_for_siglip(
129
+ vision_config,
130
+ seq_len,
131
+ num_videos,
132
+ image_token_id=hf_config.video_token_index,
133
+ image_feature_size_override=video_feature_size,
134
+ )
135
+
136
+ pil_frame = dummy_image_for_siglip(vision_config, num_images=1)
137
+ np_frame = np.array(pil_frame["image"])
138
+ mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
139
+ mm_data = {"video": mm_data_per_video}
140
+ return seq_data, mm_data
141
+
142
+ msg = f"Unsupported vision config: {type(vision_config)}"
143
+ raise NotImplementedError(msg)
144
+
145
+
146
+def input_processor_for_llava_next_video(ctx: InputContext,
147
+ llm_inputs: LLMInputs):
148
+ multi_modal_data = llm_inputs.get("multi_modal_data")
149
+ if multi_modal_data is None or "video" not in multi_modal_data:
150
+ return llm_inputs
151
+ video_data = multi_modal_data["video"]
152
+
153
+ model_config = ctx.model_config
154
+ hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
155
+ vision_config = hf_config.vision_config
156
+
157
+ if isinstance(video_data, np.ndarray):
158
+ # Supports both CLIP and Siglip
159
+ num_frames = video_data.shape[0]
160
+ frame_feature_size = \
161
+ get_llava_next_video_frame_feature_size(hf_config)
162
+ video_feature_size = num_frames * frame_feature_size
163
+
164
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
165
+
166
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
167
+ tokenizer,
168
+ llm_inputs.get("prompt"),
169
+ llm_inputs["prompt_token_ids"],
170
+ placeholder_token_id=hf_config.video_token_index,
171
+ repeat_count=video_feature_size,
172
+ )
173
+
174
+ return LLMInputs(prompt_token_ids=new_token_ids,
175
+ prompt=new_prompt,
176
+ multi_modal_data=multi_modal_data)
177
+
178
+ elif is_list_of(video_data, np.ndarray):
179
+ raise NotImplementedError(
180
+ "Processing multiple videos is not supported")
181
+
182
+ msg = f"Unsupported vision config: {type(vision_config)}"
183
+ raise NotImplementedError(msg)
184
+
185
+
186
+def _init_vision_tower(hf_config: LlavaNextVideoConfig):
187
+ vision_config = hf_config.vision_config
188
+
189
+ # Initialize the vision tower only up to the required feature layer
190
+ vision_feature_layer = hf_config.vision_feature_layer
191
+ if vision_feature_layer < 0:
192
+ num_hidden_layers = hf_config.vision_config.num_hidden_layers \
193
+ + vision_feature_layer + 1
194
+ else:
195
+ num_hidden_layers = vision_feature_layer + 1
196
+
197
+ if isinstance(vision_config, CLIPVisionConfig):
198
+ return CLIPVisionModel(
199
+ vision_config,
200
+ num_hidden_layers_override=num_hidden_layers,
201
+ )
202
+ elif isinstance(vision_config, SiglipVisionConfig):
203
+ return SiglipVisionModel(
204
+ vision_config,
205
+ num_hidden_layers_override=num_hidden_layers,
206
+ )
207
+
208
+ msg = f"Unsupported vision config: {type(vision_config)}"
209
+ raise NotImplementedError(msg)
210
+
211
+
212
+# adopted from transformers modeling_llava_next_video.py
213
+class LlavaNextVideoPooler(nn.Module):
214
+
215
+ def __init__(self, config):
216
+ super().__init__()
217
+
218
+ mode = config.spatial_pool_mode
219
+ stride = config.spatial_pool_stride
220
+ image_size = config.vision_config.image_size
221
+ patch_size = config.vision_config.patch_size
222
+ self.image_size = image_size // patch_size**2
223
+
224
+ if mode == "average":
225
+ self.pool = nn.AvgPool2d(kernel_size=stride, stride=stride)
226
+ elif mode == "max":
227
+ self.pool = nn.MaxPool2d(kernel_size=stride, stride=stride)
228
+ else:
229
+ # TODO: Support Conv2d pooling layer, need to load weights
230
+ raise ValueError(
231
+ f"Unknown pooling mode: {mode}. Expected [`average`, `max`]")
232
+
233
+ def forward(self, image_features):
234
+ ori_width = int(
235
+ math.sqrt(image_features.shape[1] * self.image_size //
236
+ self.image_size))
237
+ ori_height = int(ori_width * self.image_size // self.image_size)
238
+
239
+ batch_size, _, dim = image_features.shape
240
+ image_features_spatial = image_features \
241
+ .view(batch_size, ori_height, ori_height, dim) \
242
+ .permute(0, 3, 1, 2)
243
+ image_features_spatial = self.pool(image_features_spatial)
244
+
245
+ return image_features_spatial.flatten(2).transpose(1, 2).contiguous()
246
+
247
+
248
+class LlavaNextMultiModalProjector(nn.Module):
249
+
250
+ def __init__(self, vision_hidden_size: int, text_hidden_size: int,
251
+ projector_hidden_act: str):
252
+ super().__init__()
253
+
254
+ self.linear_1 = nn.Linear(vision_hidden_size,
255
+ text_hidden_size,
256
+ bias=True)
257
+ self.act = get_act_fn(projector_hidden_act)
258
+ self.linear_2 = nn.Linear(text_hidden_size,
259
+ text_hidden_size,
260
+ bias=True)
261
+
262
+ def forward(self, image_features: torch.Tensor) -> torch.Tensor:
263
+ hidden_states = self.linear_1(image_features)
264
+ hidden_states = self.act(hidden_states)
265
+ hidden_states = self.linear_2(hidden_states)
266
+ return hidden_states
267
+
268
+
269
+@MULTIMODAL_REGISTRY.register_input_mapper("video")
270
+@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
271
+ "video", get_max_llava_next_video_tokens)
272
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next_video)
273
+@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next_video)
274
+class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal):
275
+
276
+ def __init__(self,
277
+ config: LlavaNextVideoConfig,
278
+ multimodal_config: MultiModalConfig,
279
+ cache_config: Optional[CacheConfig] = None,
280
+ quant_config: Optional[QuantizationConfig] = None) -> None:
281
+ super().__init__()
282
+
283
+ self.config = config
284
+ self.multimodal_config = multimodal_config
285
+
286
+ # Initialize the vision tower only up to the required feature layer
287
+ self.vision_tower = _init_vision_tower(config)
288
+ self.multi_modal_projector = LlavaNextMultiModalProjector(
289
+ vision_hidden_size=config.vision_config.hidden_size,
290
+ text_hidden_size=config.text_config.hidden_size,
291
+ projector_hidden_act=config.projector_hidden_act)
292
+ self.language_model = init_vllm_registered_model(
293
+ config.text_config, cache_config, quant_config)
294
+ self.vision_resampler = LlavaNextVideoPooler(config)
295
+
296
+ def _validate_video_pixel_values(
297
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
298
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
299
+
300
+ h = w = self.config.vision_config.image_size
301
+ expected_dims = (3, h, w)
302
+
303
+ def _validate_shape(d: torch.Tensor):
304
+ actual_dims = tuple(d.shape[2:])
305
+
306
+ if actual_dims != expected_dims:
307
+ expected_expr = ("num_frames", *map(str, expected_dims))
308
+ raise ValueError(
309
+ "The expected shape of pixel values in each video frame "
310
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
311
+
312
+ for d in data:
313
+ _validate_shape(d)
314
+
315
+ return data
316
+
317
+ def _parse_and_validate_video_input(
318
+ self, **kwargs: object) -> Optional[LlavaNextVideoPixelInputs]:
319
+ """
320
+ A legal video input should have the following dimensions:
321
+ {
322
+ "pixel_values_videos" :
323
+ List[b, Tensor(nb_frames, nb_channels, height, width)]
324
+ }
325
+ """
326
+ pixel_values = kwargs.pop("pixel_values_videos", None)
327
+
328
+ if pixel_values is None:
329
+ return None
330
+
331
+ if not (is_list_of(pixel_values,
332
+ (torch.Tensor)) # different shape videos
333
+ or isinstance(pixel_values,
334
+ torch.Tensor)): # same shape videos
335
+ raise ValueError("Incorrect type of pixel values. "
336
+ f"Got type: {type(pixel_values)}")
337
+
338
+ return LlavaNextVideoPixelInputs(
339
+ type="pixel_values_videos",
340
+ data=pixel_values,
341
+ )
342
+
343
+ def _select_image_features(self, image_features: torch.Tensor, *,
344
+ strategy: str) -> torch.Tensor:
345
+ if strategy == "default":
346
+ return image_features[:, 1:]
347
+ elif strategy == "full":
348
+ return image_features
349
+
350
+ raise ValueError(f"Unexpected select feature strategy: {strategy}")
351
+
352
+ def _video_pixels_to_features(
353
+ self,
354
+ vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
355
+ pixel_values: torch.Tensor,
356
+ ) -> torch.Tensor:
357
+
358
+ # NOTE: we skip the step to select the vision feature layer since
359
+ # this is already done inside the vision tower
360
+ image_features = vision_tower(pixel_values)
361
+ image_features = self._select_image_features(
362
+ image_features,
363
+ strategy=self.config.vision_feature_select_strategy,
364
+ )
365
+ image_features = self.vision_resampler(image_features)
366
+ image_features = self.multi_modal_projector(image_features)
367
+ return image_features
368
+
369
+ def _process_video_pixels(self, inputs: LlavaNextVideoPixelInputs):
370
+ assert self.vision_tower is not None
371
+
372
+ video_pixels = inputs["data"]
373
+
374
+ if isinstance(video_pixels, torch.Tensor):
375
+ # TODO: support multiple videos per input
376
+ b, num_videos, num_frames, c, h, w = video_pixels.shape
377
+ assert (num_videos == 1)
378
+ stacked_pixels = video_pixels.view(b * num_videos * num_frames, c,
379
+ h, w)
380
+ stacked_embeddings = self._video_pixels_to_features(
381
+ self.vision_tower, stacked_pixels)
382
+ return stacked_embeddings.view(b, num_frames,
383
+ *stacked_embeddings.shape[1:])
384
+
385
+ elif is_list_of(video_pixels, torch.Tensor):
386
+ frames_per_videos = [v.shape[0] for v in video_pixels]
387
+ stacked_pixels = torch.cat(video_pixels, dim=0)
388
+ stacked_embeddings = self._video_pixels_to_features(
389
+ self.vision_tower, stacked_pixels)
390
+ return torch.split(stacked_embeddings, frames_per_videos, dim=0)
391
+
392
+ else:
393
+ raise ValueError(
394
+ f"Unsupported type of video input {type(video_pixels)}")
395
+
396
+ def forward(
397
+ self,
398
+ input_ids: torch.Tensor,
399
+ positions: torch.Tensor,
400
+ kv_caches: List[torch.Tensor],
401
+ attn_metadata: AttentionMetadata,
402
+ intermediate_tensors: Optional[IntermediateTensors] = None,
403
+ **kwargs: object,
404
+ ) -> SamplerOutput:
405
+ """Run forward pass for LlaVA-NeXT-Video.
406
+ Args:
407
+ input_ids: Flattened (concatenated) input_ids corresponding to a
408
+ batch.
409
+ pixel_values_videos: Pixels in each frames for each input videos.
410
+ """
411
+ video_input = self._parse_and_validate_video_input(**kwargs)
412
+
413
+ # merge video embeddings into input embeddings
414
+ if video_input is not None:
415
+ video_embeddings = self._process_video_pixels(video_input)
416
+ inputs_embeds = self.language_model \
417
+ .model.get_input_embeddings(input_ids)
418
+
419
+ inputs_embeds = merge_multimodal_embeddings(
420
+ input_ids, inputs_embeds, video_embeddings,
421
+ self.config.video_token_index)
422
+
423
+ input_ids = None
424
+ else:
425
+ inputs_embeds = None
426
+
427
+ hidden_states = self.language_model.model(input_ids,
428
+ positions,
429
+ kv_caches,
430
+ attn_metadata,
431
+ None,
432
+ inputs_embeds=inputs_embeds)
433
+
434
+ return hidden_states
435
+
436
+ def compute_logits(
437
+ self,
438
+ hidden_states: torch.Tensor,
439
+ sampling_metadata: SamplingMetadata,
440
+ ) -> Optional[torch.Tensor]:
441
+ return self.language_model.compute_logits(hidden_states,
442
+ sampling_metadata)
443
+
444
+ def sample(
445
+ self,
446
+ logits: torch.Tensor,
447
+ sampling_metadata: SamplingMetadata,
448
+ ) -> Optional[SamplerOutput]:
449
+ return self.language_model.sample(logits, sampling_metadata)
450
+
451
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
452
+ # prepare weight iterators
453
+ vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
454
+ weights, 4)
455
+
456
+ # load vision encoder
457
+ vit_weights = filter_weights(vit_weights, "vision_tower")
458
+ self.vision_tower.load_weights(vit_weights)
459
+
460
+ # load mlp projector
461
+ mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
462
+ mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
463
+ for name, loaded_weight in mlp_weights:
464
+ param = mlp_params_dict[name]
465
+ weight_loader = getattr(param, "weight_loader",
466
+ default_weight_loader)
467
+ weight_loader(param, loaded_weight)
468
+
469
+ # load llm backbone
470
+ llm_weights = filter_weights(llm_weights, "language_model")
471
+ self.language_model.load_weights(llm_weights)
vllm/multimodal/registry.py
+2
-1
@@ -9,6 +9,7 @@ from .audio import AudioPlugin
9
from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
10
MultiModalPlugin, MultiModalTokensCalc, NestedTensors)
11
from .image import ImagePlugin
12
+from .video import VideoPlugin
13
14
logger = init_logger(__name__)
15
@@ -34,7 +35,7 @@ class MultiModalRegistry:
35
:class:`~vllm.multimodal.MultiModalPlugin` for each modality.
36
"""
37
37
- DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin())
38
+ DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin())
39
40
def __init__(
41
self,
vllm/multimodal/utils.py
+42
@@ -4,6 +4,7 @@ from io import BytesIO
4
from typing import Any, List, Optional, Tuple, TypeVar, Union
5
6
import numpy as np
7
+import numpy.typing as npt
8
from PIL import Image
9
10
from vllm.connections import global_http_connection
@@ -187,6 +188,47 @@ def rescale_image_size(image: Image.Image,
188
return image
189
190
191
+def try_import_video_packages() -> Any:
192
+ try:
193
+ import cv2
194
+ except ImportError:
195
+ raise ImportError(
196
+ "Please install vllm[video] for video support.") from None
197
+ return cv2
198
+
199
+
200
+def resize_video(frames: npt.NDArray, size: Tuple[int, int]) -> npt.NDArray:
201
+ cv2 = try_import_video_packages()
202
+
203
+ num_frames, _, _, channels = frames.shape
204
+ new_height, new_width = size
205
+ resized_frames = np.empty((num_frames, new_height, new_width, channels),
206
+ dtype=frames.dtype)
207
+ for i, frame in enumerate(frames):
208
+ resized_frame = cv2.resize(frame, (new_width, new_height))
209
+ resized_frames[i] = resized_frame
210
+ return resized_frames
211
+
212
+
213
+def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray:
214
+ _, height, width, _ = frames.shape
215
+ new_height = int(height * size_factor)
216
+ new_width = int(width * size_factor)
217
+
218
+ return resize_video(frames, (new_height, new_width))
219
+
220
+
221
+def sample_frames_from_video(frames: npt.NDArray,
222
+ num_frames: int) -> npt.NDArray:
223
+ total_frames = frames.shape[0]
224
+ if num_frames == -1:
225
+ return frames
226
+ else:
227
+ frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
228
+ sampled_frames = frames[frame_indices, ...]
229
+ return sampled_frames
230
+
231
+
232
# Utilities for input processors
233
_T = TypeVar("_T", str, int)
234
vllm/multimodal/video.py
new
+71
@@ -0,0 +1,71 @@
1
+from functools import lru_cache
2
+from typing import List, Union
3
+
4
+import numpy as np
5
+
6
+from vllm.config import ModelConfig
7
+from vllm.inputs.registry import InputContext
8
+from vllm.logger import init_logger
9
+from vllm.transformers_utils.image_processor import get_video_processor
10
+from vllm.transformers_utils.tokenizer import get_tokenizer
11
+from vllm.utils import is_list_of
12
+
13
+from .base import MultiModalData, MultiModalInputs
14
+from .image import ImagePlugin
15
+
16
+logger = init_logger(__name__)
17
+
18
+cached_get_video_processor = lru_cache(get_video_processor)
19
+cached_get_tokenizer = lru_cache(get_tokenizer)
20
+
21
+VideoInput = Union[
22
+ "np.ndarray", # single video input
23
+ List["np.ndarray"],
24
+ # TODO: support more types
25
+ # List[Image.Image], List[List[Image.Image]],
26
+ # "torch.Tensor",
27
+ # List["torch.Tensor"],
28
+ # List[List["np.ndarrray"]],
29
+ # List[List["torch.Tensor"]],
30
+]
31
+
32
+
33
+class VideoPlugin(ImagePlugin):
34
+ """Plugin for video data."""
35
+
36
+ def get_data_key(self) -> str:
37
+ return "video"
38
+
39
+ def _get_hf_video_processor(self, model_config: ModelConfig):
40
+ return cached_get_video_processor(
41
+ model_config.model,
42
+ trust_remote_code=model_config.trust_remote_code)
43
+
44
+ def _default_input_mapper(
45
+ self,
46
+ ctx: InputContext,
47
+ data: MultiModalData[object],
48
+ ) -> MultiModalInputs:
49
+ model_config = ctx.model_config
50
+
51
+ # single video input as np.ndarray
52
+ if isinstance(data, np.ndarray):
53
+ video_processor = self._get_hf_video_processor(model_config)
54
+ if video_processor is None:
55
+ raise RuntimeError("No HuggingFace processor is available "
56
+ "to process the image object")
57
+ try:
58
+ batch_data = video_processor(data, return_tensors="pt").data
59
+ except Exception:
60
+ logger.error("Failed to process image (%s)", data)
61
+ raise
62
+
63
+ return MultiModalInputs(batch_data)
64
+ elif is_list_of(data, np.ndarray):
65
+ raise NotImplementedError(
66
+ "Multi video for a prompt is not supported yet")
67
+
68
+ raise TypeError(f"Invalid video type: {type(data)}")
69
+
70
+ def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
71
+ return 4096
vllm/transformers_utils/image_processor.py
+27
@@ -1,6 +1,33 @@
1
from typing import cast
2
3
4
+def get_video_processor(
5
+ processor_name: str,
6
+ trust_remote_code: bool = False,
7
+):
8
+ """
9
+ Gets a processor for the given model name via HuggingFace.
10
+ """
11
+ from transformers import AutoProcessor
12
+
13
+ try:
14
+ processor = AutoProcessor.from_pretrained(processor_name)
15
+ video_processor = processor.video_processor
16
+
17
+ except ValueError as e:
18
+ if not trust_remote_code:
19
+ err_msg = (
20
+ "Failed to load the processor. If the processor is "
21
+ "a custom processor not yet available in the HuggingFace "
22
+ "transformers library, consider setting "
23
+ "`trust_remote_code=True` in LLM or using the "
24
+ "`--trust-remote-code` flag in the CLI.")
25
+ raise RuntimeError(err_msg) from e
26
+ else:
27
+ raise e
28
+ return video_processor
29
+
30
+
31
def get_image_processor(
32
processor_name: str,
33
*args,