@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6a512a00df
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[model] Support for Llava-Next-Video model (#7559)

Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Yangshen⚡Deng committed Sep 11, 2024 at 13:21 UTC 6a512a00dfa306762c2878bffc3a5664a758d105
21 files changed +1083 -18
Dockerfile
+1
@@ -145,6 +145,7 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
145 && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
146 && apt-get update -y \
147 && apt-get install -y ccache software-properties-common git curl sudo vim python3-pip \
148 + && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
149 && add-apt-repository ppa:deadsnakes/ppa \
150 && apt-get update -y \
151 && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libibverbs-dev \
Dockerfile.cpu
+1
@@ -5,6 +5,7 @@ FROM ubuntu:22.04 AS cpu-test-1
5 RUN --mount=type=cache,target=/var/cache/apt \
6 apt-get update -y \
7 && apt-get install -y curl ccache git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
8 + && apt-get install -y ffmpeg libsm6 libxext6 libgl1 \
9 && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
10
11 # https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html
Dockerfile.neuron
+3 -1
@@ -6,7 +6,9 @@ FROM $BASE_IMAGE
6 RUN echo "Base image is $BASE_IMAGE"
7
8 # Install some basic utilities
9 -RUN apt-get update && apt-get install python3 python3-pip -y
9 +RUN apt-get update \
10 + && apt-get install python3 python3-pip -y \
11 + && apt-get install -y ffmpeg libsm6 libxext6 libgl1
12
13 ### Mount Point ###
14 # When launching the container, mount the code directory to /app
Dockerfile.openvino
+2 -1
@@ -4,7 +4,8 @@
4 FROM ubuntu:22.04 AS dev
5
6 RUN apt-get update -y && \
7 - apt-get install -y python3-pip git
7 + apt-get install -y python3-pip git && \
8 + apt-get install -y ffmpeg libsm6 libxext6 libgl1
9 WORKDIR /workspace
10
11 # copy requirements
Dockerfile.ppc64le
+1 -1
@@ -4,7 +4,7 @@ USER root
4
5 ENV PATH="/usr/local/cargo/bin:$PATH:/opt/conda/bin/"
6
7 -RUN apt-get update -y && apt-get install -y git wget curl vim libnuma-dev libsndfile-dev libprotobuf-dev build-essential
7 +RUN apt-get update -y && apt-get install -y git wget curl vim libnuma-dev libsndfile-dev libprotobuf-dev build-essential ffmpeg libsm6 libxext6 libgl1
8
9 # Some packages in requirements-cpu are installed here
10 # IBM provides optimized packages for ppc64le processors in the open-ce project for mamba
Dockerfile.tpu
+3
@@ -4,6 +4,9 @@ ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:night
4 FROM $BASE_IMAGE
5 WORKDIR /workspace
6
7 +# Install some basic utilities
8 +RUN apt-get update && apt-get install -y ffmpeg libsm6 libxext6 libgl1
9 +
10 # Install the TPU and Pallas dependencies.
11 RUN python3 -m pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
12 RUN python3 -m pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
Dockerfile.xpu
+1 -2
@@ -9,8 +9,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
9 chmod 644 /usr/share/keyrings/intel-graphics.gpg
10
11 RUN apt-get update -y \
12 -&& apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip
13 -
12 +&& apt-get install -y curl libicu70 lsb-release git wget vim numactl python3 python3-pip ffmpeg libsm6 libxext6 libgl1
13 COPY ./ /workspace/vllm
14
15 WORKDIR /workspace/vllm
docs/source/conf.py
+1
@@ -99,6 +99,7 @@ autodoc_mock_imports = [
99 "aiohttp",
100 "compressed_tensors",
101 "cpuinfo",
102 + "cv2",
103 "torch",
104 "transformers",
105 "psutil",
docs/source/models/supported_models.rst
+14
@@ -227,6 +227,11 @@ Multimodal Language Models
227 - Image\ :sup:`E+`
228 - :code:`llava-hf/llava-v1.6-mistral-7b-hf`, :code:`llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
229 -
230 + * - :code:`LlavaNextVideoForConditionalGeneration`
231 + - LLaVA-NeXT-Video
232 + - Video
233 + - :code:`llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. (see note)
234 + -
235 * - :code:`MiniCPMV`
236 - MiniCPM-V
237 - Image\ :sup:`+`
@@ -260,6 +265,15 @@ Multimodal Language Models
265 For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
266 For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
267
268 + For :code:`LLaVA-NeXT-Video`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
269 + This can be installed by running the following command:
270 +
271 +
272 + .. code-block:: bash
273 +
274 + pip install git+https://github.com/huggingface/transformers.git@21fac7abba2a37fae86106f87fcf9974fd1e3830
275 +
276 +
277 ----
278
279 If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
examples/offline_inference_vision_language.py
+61 -9
@@ -9,12 +9,9 @@ from transformers import AutoTokenizer
9
10 from vllm import LLM, SamplingParams
11 from vllm.assets.image import ImageAsset
12 +from vllm.assets.video import VideoAsset
13 from vllm.utils import FlexibleArgumentParser
14
14 -# Input image and question
15 -image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
16 -question = "What is the content of this image?"
17 -
15
16 # LLaVA-1.5
17 def run_llava(question):
@@ -30,7 +27,16 @@ def run_llava(question):
27 def run_llava_next(question):
28
29 prompt = f"[INST] <image>\n{question} [/INST]"
33 - llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf")
30 + llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
31 + stop_token_ids = None
32 + return llm, prompt, stop_token_ids
33 +
34 +
35 +# LlaVA-NeXT-Video
36 +# Currently only support for video input
37 +def run_llava_next_video(question):
38 + prompt = f"USER: <video>\n{question} ASSISTANT:"
39 + llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
40 stop_token_ids = None
41 return llm, prompt, stop_token_ids
42
@@ -176,6 +182,7 @@ def run_qwen_vl(question):
182 model_example_map = {
183 "llava": run_llava,
184 "llava-next": run_llava_next,
185 + "llava-next-video": run_llava_next_video,
186 "fuyu": run_fuyu,
187 "phi3_v": run_phi3v,
188 "paligemma": run_paligemma,
@@ -187,11 +194,49 @@ model_example_map = {
194 }
195
196
197 +def get_multi_modal_input(args):
198 + """
199 + return {
200 + "data": image or video,
201 + "question": question,
202 + }
203 + """
204 + if args.modality == "image":
205 + # Input image and question
206 + image = ImageAsset("cherry_blossom") \
207 + .pil_image.convert("RGB")
208 + img_question = "What is the content of this image?"
209 +
210 + return {
211 + "data": image,
212 + "question": img_question,
213 + }
214 +
215 + if args.modality == "video":
216 + # Input video and question
217 + video = VideoAsset(name="sample_demo_1.mp4",
218 + num_frames=args.num_frames).np_ndarrays
219 + vid_question = "Why is this video funny?"
220 +
221 + return {
222 + "data": video,
223 + "question": vid_question,
224 + }
225 +
226 + msg = f"Modality {args.modality} is not supported."
227 + raise ValueError(msg)
228 +
229 +
230 def main(args):
231 model = args.model_type
232 if model not in model_example_map:
233 raise ValueError(f"Model type {model} is not supported.")
234
235 + modality = args.modality
236 + mm_input = get_multi_modal_input(args)
237 + data = mm_input["data"]
238 + question = mm_input["question"]
239 +
240 llm, prompt, stop_token_ids = model_example_map[model](question)
241
242 # We set temperature to 0.2 so that outputs can be different
@@ -206,7 +251,7 @@ def main(args):
251 inputs = {
252 "prompt": prompt,
253 "multi_modal_data": {
209 - "image": image
254 + modality: data
255 },
256 }
257
@@ -215,7 +260,7 @@ def main(args):
260 inputs = [{
261 "prompt": prompt,
262 "multi_modal_data": {
218 - "image": image
263 + modality: data
264 },
265 } for _ in range(args.num_prompts)]
266
@@ -238,8 +283,15 @@ if __name__ == "__main__":
283 help='Huggingface "model_type".')
284 parser.add_argument('--num-prompts',
285 type=int,
241 - default=1,
286 + default=4,
287 help='Number of prompts to run.')
243 -
288 + parser.add_argument('--modality',
289 + type=str,
290 + default="image",
291 + help='Modality of the input.')
292 + parser.add_argument('--num-frames',
293 + type=int,
294 + default=16,
295 + help='Number of frames to extract from the video.')
296 args = parser.parse_args()
297 main(args)
requirements-test.txt
+1
@@ -11,6 +11,7 @@ awscli
11 einops # required for MPT, qwen-vl and Mamba
12 httpx
13 librosa # required for audio test
14 +opencv-python # required for video test
15 peft
16 requests
17 ray[adag]>=2.35
setup.py
+1
@@ -505,6 +505,7 @@ setup(
505 ext_modules=ext_modules,
506 extras_require={
507 "tensorizer": ["tensorizer>=2.9.0"],
508 + "video": ["opencv-python"], # Required for video processing
509 "audio": ["librosa", "soundfile"] # Required for audio processing
510 },
511 cmdclass={"build_ext": cmake_build_ext} if len(ext_modules) > 0 else {},
tests/conftest.py
+55 -1
@@ -21,6 +21,7 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BatchEncoding,
21
22 from vllm import LLM, SamplingParams
23 from vllm.assets.image import ImageAsset
24 +from vllm.assets.video import VideoAsset
25 from vllm.config import TokenizerPoolConfig
26 from vllm.connections import global_http_connection
27 from vllm.distributed import (destroy_distributed_environment,
@@ -44,6 +45,7 @@ _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
45 PromptImageInput = Union[List[Image.Image], List[List[Image.Image]]]
46 PromptAudioInput = Union[List[Tuple[np.ndarray, int]],
47 List[List[Tuple[np.ndarray, int]]]]
48 +PromptVideoInput = Union[List[np.ndarray], List[List[np.ndarray]]]
49
50
51 def _read_prompts(filename: str) -> List[str]:
@@ -85,8 +87,35 @@ class _ImageAssets(_ImageAssetsBase):
87 return [prompts["stop_sign"], prompts["cherry_blossom"]]
88
89
90 +class _VideoAssetPrompts(TypedDict):
91 + sample_demo_1: str
92 +
93 +
94 +if sys.version_info < (3, 9):
95 + # UserList cannot be subscripted
96 + class _VideoAssetsBase(UserList):
97 + pass
98 +else:
99 +
100 + class _VideoAssetsBase(UserList[VideoAsset]):
101 + pass
102 +
103 +
104 +class _VideoAssets(_VideoAssetsBase):
105 +
106 + def __init__(self) -> None:
107 + super().__init__([
108 + VideoAsset("sample_demo_1.mp4"),
109 + ])
110 +
111 + def prompts(self, prompts: _VideoAssetPrompts) -> List[str]:
112 + return [prompts["sample_demo_1"]]
113 +
114 +
115 IMAGE_ASSETS = _ImageAssets()
116 """Singleton instance of :class:`_ImageAssets`."""
117 +VIDEO_ASSETS = _VideoAssets()
118 +"""Singleton instance of :class:`_VideoAssets`."""
119
120
121 @pytest.fixture(autouse=True)
@@ -202,6 +231,11 @@ def image_assets() -> _ImageAssets:
231 return IMAGE_ASSETS
232
233
234 +@pytest.fixture(scope="session")
235 +def video_assets() -> _VideoAssets:
236 + return VIDEO_ASSETS
237 +
238 +
239 _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
240
241
@@ -279,6 +313,7 @@ class HfRunner:
313 self,
314 prompts: List[str],
315 images: Optional[PromptImageInput] = None,
316 + videos: Optional[List[np.ndarray]] = None,
317 **kwargs: Any,
318 ) -> List[Tuple[List[List[int]], List[str]]]:
319 if images:
@@ -292,6 +327,8 @@ class HfRunner:
327 }
328 if images is not None and images[i] is not None:
329 processor_kwargs["images"] = images[i]
330 + if videos is not None and videos[i] is not None:
331 + processor_kwargs["videos"] = videos[i]
332
333 inputs = self.processor(**processor_kwargs)
334 inputs = self.postprocess_inputs(inputs)
@@ -352,6 +389,7 @@ class HfRunner:
389 prompts: List[str],
390 max_tokens: int,
391 images: Optional[PromptImageInput] = None,
392 + videos: Optional[List[np.ndarray]] = None,
393 **kwargs: Any,
394 ) -> List[List[torch.Tensor]]:
395 all_logprobs: List[List[torch.Tensor]] = []
@@ -362,6 +400,8 @@ class HfRunner:
400 }
401 if images is not None and images[i] is not None:
402 processor_kwargs["images"] = images[i]
403 + if videos is not None and videos[i] is not None:
404 + processor_kwargs["videos"] = videos[i]
405
406 inputs = self.processor(**processor_kwargs)
407 inputs = self.postprocess_inputs(inputs)
@@ -435,6 +475,7 @@ class HfRunner:
475 num_logprobs: int,
476 images: Optional[PromptImageInput] = None,
477 audios: Optional[PromptAudioInput] = None,
478 + videos: Optional[List[np.ndarray]] = None,
479 **kwargs: Any,
480 ) -> List[Tuple[List[int], str, List[Dict[int, float]]]]:
481 all_logprobs: List[List[Dict[int, float]]] = []
@@ -454,6 +495,8 @@ class HfRunner:
495 processor_kwargs["audio"] = audio
496 processor_kwargs["sampling_rate"] = sr
497
498 + if videos is not None:
499 + processor_kwargs["videos"] = videos[i]
500 inputs = self.processor(**processor_kwargs)
501 inputs = self.postprocess_inputs(inputs)
502
@@ -634,12 +677,16 @@ class VllmRunner:
677 sampling_params: SamplingParams,
678 images: Optional[PromptImageInput] = None,
679 audios: Optional[PromptAudioInput] = None,
680 + videos: Optional[PromptVideoInput] = None,
681 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
682 assert sampling_params.logprobs is not None
683
684 if images is not None:
685 assert len(prompts) == len(images)
686
687 + if videos is not None:
688 + assert len(prompts) == len(videos)
689 +
690 inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
691 if images is not None:
692 for i, image in enumerate(images):
@@ -649,6 +696,11 @@ class VllmRunner:
696 for i, audio in enumerate(audios):
697 inputs[i]["multi_modal_data"] = {"audio": audio}
698
699 + if videos is not None:
700 + for i, video in enumerate(videos):
701 + inputs[i]["multi_modal_data"] = {"video": video}
702 + print(f"[INPUTS!!!!]: {inputs}, {sampling_params}")
703 +
704 req_outputs = self.model.generate(inputs,
705 sampling_params=sampling_params)
706 return self._final_steps_generate_w_logprobs(req_outputs)
@@ -685,6 +737,7 @@ class VllmRunner:
737 num_logprobs: int,
738 images: Optional[PromptImageInput] = None,
739 audios: Optional[PromptAudioInput] = None,
740 + videos: Optional[PromptVideoInput] = None,
741 stop_token_ids: Optional[List[int]] = None,
742 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
743 greedy_logprobs_params = SamplingParams(temperature=0.0,
@@ -694,7 +747,8 @@ class VllmRunner:
747 outputs = self.generate_w_logprobs(prompts,
748 greedy_logprobs_params,
749 images=images,
697 - audios=audios)
750 + audios=audios,
751 + videos=videos)
752
753 return [(output_ids, output_str, output_logprobs)
754 for output_ids, output_str, output_logprobs in outputs]
tests/models/test_llava_next_video.py new
+236
@@ -0,0 +1,236 @@
1 +from typing import List, Optional, Tuple, Type, overload
2 +
3 +import pytest
4 +import transformers
5 +from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
6 +
7 +from vllm.multimodal.utils import (rescale_video_size, resize_video,
8 + sample_frames_from_video)
9 +from vllm.sequence import SampleLogprobs
10 +
11 +from ..conftest import VIDEO_ASSETS, HfRunner, VllmRunner, _VideoAssets
12 +from .utils import check_logprobs_close
13 +
14 +pytestmark = pytest.mark.vlm
15 +
16 +_PREFACE = (
17 + "A chat between a curious human and an artificial intelligence assistant. "
18 + "The assistant gives helpful, detailed, and polite answers to the human's "
19 + "questions.")
20 +
21 +HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
22 + "sample_demo_1":
23 + f"{_PREFACE}USER: <video>\nWhy is this video funny? ASSISTANT:"
24 +})
25 +
26 +models = ["llava-hf/LLaVA-NeXT-Video-7B-hf"]
27 +
28 +
29 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
30 + Optional[SampleLogprobs]],
31 + model: str):
32 + """Sanitize vllm output to be comparable with hf output."""
33 + output_ids, output_str, out_logprobs = vllm_output
34 +
35 + config = AutoConfig.from_pretrained(model)
36 + video_token_id = config.video_token_index
37 +
38 + tokenizer = AutoTokenizer.from_pretrained(model)
39 + eos_token_id = tokenizer.eos_token_id
40 +
41 + hf_output_ids = [
42 + token_id for idx, token_id in enumerate(output_ids)
43 + if token_id != video_token_id or output_ids[idx - 1] != video_token_id
44 + ]
45 +
46 + assert output_str[0] == " "
47 + hf_output_str = output_str[1:]
48 + if hf_output_ids[-1] == eos_token_id:
49 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
50 +
51 + return hf_output_ids, hf_output_str, out_logprobs
52 +
53 +
54 +@overload
55 +def run_test(
56 + hf_runner: Type[HfRunner],
57 + vllm_runner: Type[VllmRunner],
58 + video_assets: _VideoAssets,
59 + model: str,
60 + *,
61 + size_factors: List[float],
62 + dtype: str,
63 + max_tokens: int,
64 + num_logprobs: int,
65 + num_frames: int,
66 + tensor_parallel_size: int,
67 + distributed_executor_backend: Optional[str] = None,
68 +):
69 + ...
70 +
71 +
72 +@overload
73 +def run_test(
74 + hf_runner: Type[HfRunner],
75 + vllm_runner: Type[VllmRunner],
76 + video_assets: _VideoAssets,
77 + model: str,
78 + *,
79 + sizes: List[Tuple[int, int]],
80 + dtype: str,
81 + max_tokens: int,
82 + num_logprobs: int,
83 + num_frames: int,
84 + tensor_parallel_size: int,
85 + distributed_executor_backend: Optional[str] = None,
86 +):
87 + ...
88 +
89 +
90 +def run_test(
91 + hf_runner: Type[HfRunner],
92 + vllm_runner: Type[VllmRunner],
93 + video_assets: _VideoAssets,
94 + model: str,
95 + *,
96 + size_factors: Optional[List[float]] = None,
97 + sizes: Optional[List[Tuple[int, int]]] = None,
98 + dtype: str,
99 + max_tokens: int,
100 + num_logprobs: int,
101 + num_frames: int,
102 + tensor_parallel_size: int,
103 + distributed_executor_backend: Optional[str] = None,
104 +):
105 + videos = [
106 + sample_frames_from_video(asset.np_ndarrays, num_frames)
107 + for asset in video_assets
108 + ]
109 +
110 + for video in videos:
111 + print(video.shape)
112 +
113 + if size_factors is not None:
114 + inputs_per_video = [(
115 + [prompt for _ in size_factors],
116 + [rescale_video_size(video, factor) for factor in size_factors],
117 + ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
118 + elif sizes is not None:
119 + inputs_per_video = [(
120 + [prompt for _ in sizes],
121 + [resize_video(video, size) for size in sizes],
122 + ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
123 + else:
124 + raise ValueError("You must provide either `size_factors` or `sizes`")
125 +
126 + # max_model_len should be greater than image_feature_size
127 + with vllm_runner(model,
128 + dtype=dtype,
129 + max_model_len=4096,
130 + tensor_parallel_size=tensor_parallel_size,
131 + distributed_executor_backend=distributed_executor_backend,
132 + enforce_eager=True) as vllm_model:
133 + vllm_outputs_per_video = [
134 + vllm_model.generate_greedy_logprobs(prompts,
135 + max_tokens,
136 + num_logprobs=num_logprobs,
137 + videos=videos)
138 + for prompts, videos in inputs_per_video
139 + ]
140 +
141 + with hf_runner(model, dtype=dtype,
142 + auto_cls=AutoModelForVision2Seq) as hf_model:
143 + hf_outputs_per_video = [
144 + hf_model.generate_greedy_logprobs_limit(prompts,
145 + max_tokens,
146 + num_logprobs=num_logprobs,
147 + videos=videos)
148 + for prompts, videos in inputs_per_video
149 + ]
150 +
151 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_video,
152 + vllm_outputs_per_video):
153 + # TODO: Check whether using original CLIPVisionModel can improve
154 + # consistency against HF
155 + check_logprobs_close(
156 + outputs_0_lst=hf_outputs,
157 + outputs_1_lst=[
158 + vllm_to_hf_output(vllm_output, model)
159 + for vllm_output in vllm_outputs
160 + ],
161 + name_0="hf",
162 + name_1="vllm",
163 + )
164 +
165 +
166 +@pytest.mark.skipif(transformers.__version__ < "4.45",
167 + reason="Waiting for next transformers release")
168 +@pytest.mark.parametrize("model", models)
169 +@pytest.mark.parametrize(
170 + "size_factors",
171 + [
172 + # No video
173 + [],
174 + # Single-scale
175 + [1.0],
176 + # Single-scale, batched
177 + [1.0, 1.0, 1.0],
178 + # Multi-scale
179 + [0.25, 0.5, 1.0],
180 + ],
181 +)
182 +@pytest.mark.parametrize("dtype", ["half"])
183 +@pytest.mark.parametrize("max_tokens", [128])
184 +@pytest.mark.parametrize("num_logprobs", [5])
185 +@pytest.mark.parametrize("num_frames", [16])
186 +def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
187 + dtype, max_tokens, num_logprobs, num_frames) -> None:
188 + """Inference result should be the same between hf and vllm.
189 +
190 + All the image fixtures for the test is under tests/videos.
191 + For huggingface runner, we provide the np.ndarray as input.
192 + For vllm runner, we provide MultiModalDataDict objects
193 + and corresponding MultiModalConfig as input.
194 + Note, the text input is also adjusted to abide by vllm contract.
195 + The text output is sanitized to be able to compare with hf.
196 + """
197 + run_test(
198 + hf_runner,
199 + vllm_runner,
200 + video_assets,
201 + model,
202 + size_factors=size_factors,
203 + dtype=dtype,
204 + max_tokens=max_tokens,
205 + num_logprobs=num_logprobs,
206 + num_frames=num_frames,
207 + tensor_parallel_size=1,
208 + )
209 +
210 +
211 +@pytest.mark.skipif(transformers.__version__ < "4.45",
212 + reason="Waiting for next transformers release")
213 +@pytest.mark.parametrize("model", models)
214 +@pytest.mark.parametrize(
215 + "sizes",
216 + [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
217 +)
218 +@pytest.mark.parametrize("dtype", ["half"])
219 +@pytest.mark.parametrize("max_tokens", [128])
220 +@pytest.mark.parametrize("num_logprobs", [5])
221 +@pytest.mark.parametrize("num_frames", [16])
222 +def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
223 + dtype, max_tokens, num_logprobs,
224 + num_frames) -> None:
225 + run_test(
226 + hf_runner,
227 + vllm_runner,
228 + video_assets,
229 + model,
230 + sizes=sizes,
231 + dtype=dtype,
232 + max_tokens=max_tokens,
233 + num_logprobs=num_logprobs,
234 + num_frames=num_frames,
235 + tensor_parallel_size=1,
236 + )
vllm/assets/video.py new
+85
@@ -0,0 +1,85 @@
1 +from dataclasses import dataclass
2 +from functools import lru_cache
3 +from typing import List, Literal
4 +
5 +import numpy as np
6 +import numpy.typing as npt
7 +from huggingface_hub import hf_hub_download
8 +from PIL import Image
9 +
10 +from vllm.multimodal.utils import (sample_frames_from_video,
11 + try_import_video_packages)
12 +
13 +from .base import get_cache_dir
14 +
15 +
16 +@lru_cache
17 +def download_video_asset(filename: str) -> str:
18 + """
19 + Download and open an image from huggingface
20 + repo: raushan-testing-hf/videos-test
21 + """
22 + video_directory = get_cache_dir() / "video-eample-data"
23 + video_directory.mkdir(parents=True, exist_ok=True)
24 +
25 + video_path = video_directory / filename
26 + video_path_str = str(video_path)
27 + if not video_path.exists():
28 + video_path_str = hf_hub_download(
29 + repo_id="raushan-testing-hf/videos-test",
30 + filename=filename,
31 + repo_type="dataset",
32 + cache_dir=video_directory,
33 + )
34 + return video_path_str
35 +
36 +
37 +def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray:
38 + cv2 = try_import_video_packages()
39 +
40 + cap = cv2.VideoCapture(path)
41 + if not cap.isOpened():
42 + raise ValueError(f"Could not open video file {path}")
43 +
44 + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
45 + frames = []
46 + for i in range(total_frames):
47 + ret, frame = cap.read()
48 + if ret:
49 + frames.append(frame)
50 + cap.release()
51 +
52 + frames = np.stack(frames)
53 + frames = sample_frames_from_video(frames, num_frames)
54 + if len(frames) < num_frames:
55 + raise ValueError(f"Could not read enough frames from video file {path}"
56 + f" (expected {num_frames} frames, got {len(frames)})")
57 + return frames
58 +
59 +
60 +def video_to_pil_images_list(path: str,
61 + num_frames: int = -1) -> List[Image.Image]:
62 + cv2 = try_import_video_packages()
63 + frames = video_to_ndarrays(path, num_frames)
64 + return [
65 + Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
66 + for frame in frames
67 + ]
68 +
69 +
70 +@dataclass(frozen=True)
71 +class VideoAsset:
72 + name: Literal["sample_demo_1.mp4"]
73 + num_frames: int = -1
74 +
75 + @property
76 + def pil_images(self) -> List[Image.Image]:
77 + video_path = download_video_asset(self.name)
78 + ret = video_to_pil_images_list(video_path, self.num_frames)
79 + return ret
80 +
81 + @property
82 + def np_ndarrays(self) -> List[npt.NDArray]:
83 + video_path = download_video_asset(self.name)
84 + ret = video_to_ndarrays(video_path, self.num_frames)
85 + return ret
vllm/model_executor/models/__init__.py
+4 -2
@@ -80,8 +80,10 @@ _MULTIMODAL_MODELS = {
80 "InternVLChatModel": ("internvl", "InternVLChatModel"),
81 "LlavaForConditionalGeneration":
82 ("llava", "LlavaForConditionalGeneration"),
83 - "LlavaNextForConditionalGeneration":
84 - ("llava_next", "LlavaNextForConditionalGeneration"),
83 + "LlavaNextForConditionalGeneration": ("llava_next",
84 + "LlavaNextForConditionalGeneration"),
85 + "LlavaNextVideoForConditionalGeneration":
86 + ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
87 "MiniCPMV": ("minicpmv", "MiniCPMV"),
88 "PaliGemmaForConditionalGeneration": ("paligemma",
89 "PaliGemmaForConditionalGeneration"),
vllm/model_executor/models/llava_next_video.py new
+471
@@ -0,0 +1,471 @@
1 +import itertools
2 +import math
3 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
4 + TypedDict, Union)
5 +
6 +import numpy as np
7 +import torch
8 +import torch.nn as nn
9 +from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
10 + SiglipVisionConfig)
11 +
12 +from vllm.attention import AttentionMetadata
13 +from vllm.config import CacheConfig, MultiModalConfig
14 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
15 +from vllm.logger import init_logger
16 +from vllm.model_executor.layers.activation import get_act_fn
17 +from vllm.model_executor.layers.quantization.base_config import (
18 + QuantizationConfig)
19 +from vllm.model_executor.layers.sampler import SamplerOutput
20 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 +from vllm.model_executor.models.clip import CLIPVisionModel
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 +from vllm.multimodal import MULTIMODAL_REGISTRY
24 +from vllm.multimodal.utils import (cached_get_tokenizer,
25 + repeat_and_pad_placeholder_tokens)
26 +from vllm.sequence import IntermediateTensors
27 +from vllm.utils import is_list_of
28 +
29 +from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
30 +from .interfaces import SupportsMultiModal
31 +from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
32 + dummy_seq_data_for_siglip)
33 +from .utils import (filter_weights, init_vllm_registered_model,
34 + merge_multimodal_embeddings)
35 +
36 +logger = init_logger(__name__)
37 +
38 +# For profile run
39 +_MAX_FRAMES_PER_VIDEO = 32
40 +_MAX_NUM_VIDEOS = 1
41 +
42 +
43 +class LlavaNextVideoPixelInputs(TypedDict):
44 + type: Literal["pixel_values_videos"]
45 + data: Union[torch.Tensor, List[torch.Tensor]]
46 + """
47 + Shape: `(batch_size, num_frames, num_channels, height, width)`
48 +
49 + Note that `num_frames` may be different for each batch, in which case
50 + the data is passed as a list instead of a batched tensor.
51 +
52 + Note that it only supports one video input for one batch.
53 + """
54 +
55 +
56 +def get_llava_next_video_frame_feature_size(
57 + hf_config: LlavaNextVideoConfig) -> int:
58 + # Support both CLIPVisionConfig and SiglipVisionConfig
59 + image_size = hf_config.vision_config.image_size
60 + patch_size = hf_config.vision_config.patch_size
61 + spatial_pool_stride = hf_config.spatial_pool_stride
62 +
63 + return int((image_size / patch_size / spatial_pool_stride)**2)
64 +
65 +
66 +def _get_max_llm_tokens(ctx: InputContext) -> int:
67 + """
68 + Calculated from the maximum video frames under the context length
69 + constraints of the language model.
70 + """
71 + hf_text_config = ctx.model_config.hf_text_config
72 + model_config = ctx.model_config
73 + max_tokens = model_config.max_model_len
74 + rope_scaling = model_config.rope_scaling
75 +
76 + if rope_scaling:
77 + rope_scaling_factor = hf_text_config.rope_scaling["factor"]
78 + else:
79 + rope_scaling_factor = 1
80 +
81 + max_tokens *= rope_scaling_factor
82 +
83 + return max_tokens
84 +
85 +
86 +def get_max_llava_next_video_tokens(ctx: InputContext) -> int:
87 + # Currently set to 32 frames
88 + # TODO: max_tokens = _get_max_llm_tokens(ctx)
89 + hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
90 + tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
91 + return _MAX_FRAMES_PER_VIDEO * tokens_per_frame
92 +
93 +
94 +def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
95 + mm_counts: Mapping[str, int]):
96 + hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
97 + vision_config = hf_config.vision_config
98 +
99 + # TODO: support multiple videos
100 + num_videos = mm_counts["video"]
101 + if num_videos != _MAX_NUM_VIDEOS:
102 + raise NotImplementedError(
103 + f"Only {_MAX_NUM_VIDEOS} videos are supported")
104 +
105 + # TODO: support configuring the number of frames
106 + frames_per_video = _MAX_FRAMES_PER_VIDEO
107 + # num_images = num_videos * frames_per_video
108 +
109 + # fills the sequence with as longer video data as possible
110 + tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
111 + video_feature_size = frames_per_video * tokens_per_frame
112 +
113 + if isinstance(vision_config, CLIPVisionConfig):
114 + seq_data = dummy_seq_data_for_clip(
115 + vision_config,
116 + seq_len,
117 + num_videos,
118 + image_token_id=hf_config.video_token_index,
119 + image_feature_size_override=video_feature_size,
120 + )
121 +
122 + pil_frame = dummy_image_for_clip(vision_config, num_images=1)
123 + np_frame = np.array(pil_frame["image"])
124 + mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
125 + mm_data = {"video": mm_data_per_video}
126 + return seq_data, mm_data
127 + elif isinstance(vision_config, SiglipVisionConfig):
128 + seq_data = dummy_seq_data_for_siglip(
129 + vision_config,
130 + seq_len,
131 + num_videos,
132 + image_token_id=hf_config.video_token_index,
133 + image_feature_size_override=video_feature_size,
134 + )
135 +
136 + pil_frame = dummy_image_for_siglip(vision_config, num_images=1)
137 + np_frame = np.array(pil_frame["image"])
138 + mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
139 + mm_data = {"video": mm_data_per_video}
140 + return seq_data, mm_data
141 +
142 + msg = f"Unsupported vision config: {type(vision_config)}"
143 + raise NotImplementedError(msg)
144 +
145 +
146 +def input_processor_for_llava_next_video(ctx: InputContext,
147 + llm_inputs: LLMInputs):
148 + multi_modal_data = llm_inputs.get("multi_modal_data")
149 + if multi_modal_data is None or "video" not in multi_modal_data:
150 + return llm_inputs
151 + video_data = multi_modal_data["video"]
152 +
153 + model_config = ctx.model_config
154 + hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
155 + vision_config = hf_config.vision_config
156 +
157 + if isinstance(video_data, np.ndarray):
158 + # Supports both CLIP and Siglip
159 + num_frames = video_data.shape[0]
160 + frame_feature_size = \
161 + get_llava_next_video_frame_feature_size(hf_config)
162 + video_feature_size = num_frames * frame_feature_size
163 +
164 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
165 +
166 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
167 + tokenizer,
168 + llm_inputs.get("prompt"),
169 + llm_inputs["prompt_token_ids"],
170 + placeholder_token_id=hf_config.video_token_index,
171 + repeat_count=video_feature_size,
172 + )
173 +
174 + return LLMInputs(prompt_token_ids=new_token_ids,
175 + prompt=new_prompt,
176 + multi_modal_data=multi_modal_data)
177 +
178 + elif is_list_of(video_data, np.ndarray):
179 + raise NotImplementedError(
180 + "Processing multiple videos is not supported")
181 +
182 + msg = f"Unsupported vision config: {type(vision_config)}"
183 + raise NotImplementedError(msg)
184 +
185 +
186 +def _init_vision_tower(hf_config: LlavaNextVideoConfig):
187 + vision_config = hf_config.vision_config
188 +
189 + # Initialize the vision tower only up to the required feature layer
190 + vision_feature_layer = hf_config.vision_feature_layer
191 + if vision_feature_layer < 0:
192 + num_hidden_layers = hf_config.vision_config.num_hidden_layers \
193 + + vision_feature_layer + 1
194 + else:
195 + num_hidden_layers = vision_feature_layer + 1
196 +
197 + if isinstance(vision_config, CLIPVisionConfig):
198 + return CLIPVisionModel(
199 + vision_config,
200 + num_hidden_layers_override=num_hidden_layers,
201 + )
202 + elif isinstance(vision_config, SiglipVisionConfig):
203 + return SiglipVisionModel(
204 + vision_config,
205 + num_hidden_layers_override=num_hidden_layers,
206 + )
207 +
208 + msg = f"Unsupported vision config: {type(vision_config)}"
209 + raise NotImplementedError(msg)
210 +
211 +
212 +# adopted from transformers modeling_llava_next_video.py
213 +class LlavaNextVideoPooler(nn.Module):
214 +
215 + def __init__(self, config):
216 + super().__init__()
217 +
218 + mode = config.spatial_pool_mode
219 + stride = config.spatial_pool_stride
220 + image_size = config.vision_config.image_size
221 + patch_size = config.vision_config.patch_size
222 + self.image_size = image_size // patch_size**2
223 +
224 + if mode == "average":
225 + self.pool = nn.AvgPool2d(kernel_size=stride, stride=stride)
226 + elif mode == "max":
227 + self.pool = nn.MaxPool2d(kernel_size=stride, stride=stride)
228 + else:
229 + # TODO: Support Conv2d pooling layer, need to load weights
230 + raise ValueError(
231 + f"Unknown pooling mode: {mode}. Expected [`average`, `max`]")
232 +
233 + def forward(self, image_features):
234 + ori_width = int(
235 + math.sqrt(image_features.shape[1] * self.image_size //
236 + self.image_size))
237 + ori_height = int(ori_width * self.image_size // self.image_size)
238 +
239 + batch_size, _, dim = image_features.shape
240 + image_features_spatial = image_features \
241 + .view(batch_size, ori_height, ori_height, dim) \
242 + .permute(0, 3, 1, 2)
243 + image_features_spatial = self.pool(image_features_spatial)
244 +
245 + return image_features_spatial.flatten(2).transpose(1, 2).contiguous()
246 +
247 +
248 +class LlavaNextMultiModalProjector(nn.Module):
249 +
250 + def __init__(self, vision_hidden_size: int, text_hidden_size: int,
251 + projector_hidden_act: str):
252 + super().__init__()
253 +
254 + self.linear_1 = nn.Linear(vision_hidden_size,
255 + text_hidden_size,
256 + bias=True)
257 + self.act = get_act_fn(projector_hidden_act)
258 + self.linear_2 = nn.Linear(text_hidden_size,
259 + text_hidden_size,
260 + bias=True)
261 +
262 + def forward(self, image_features: torch.Tensor) -> torch.Tensor:
263 + hidden_states = self.linear_1(image_features)
264 + hidden_states = self.act(hidden_states)
265 + hidden_states = self.linear_2(hidden_states)
266 + return hidden_states
267 +
268 +
269 +@MULTIMODAL_REGISTRY.register_input_mapper("video")
270 +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
271 + "video", get_max_llava_next_video_tokens)
272 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next_video)
273 +@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next_video)
274 +class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal):
275 +
276 + def __init__(self,
277 + config: LlavaNextVideoConfig,
278 + multimodal_config: MultiModalConfig,
279 + cache_config: Optional[CacheConfig] = None,
280 + quant_config: Optional[QuantizationConfig] = None) -> None:
281 + super().__init__()
282 +
283 + self.config = config
284 + self.multimodal_config = multimodal_config
285 +
286 + # Initialize the vision tower only up to the required feature layer
287 + self.vision_tower = _init_vision_tower(config)
288 + self.multi_modal_projector = LlavaNextMultiModalProjector(
289 + vision_hidden_size=config.vision_config.hidden_size,
290 + text_hidden_size=config.text_config.hidden_size,
291 + projector_hidden_act=config.projector_hidden_act)
292 + self.language_model = init_vllm_registered_model(
293 + config.text_config, cache_config, quant_config)
294 + self.vision_resampler = LlavaNextVideoPooler(config)
295 +
296 + def _validate_video_pixel_values(
297 + self, data: Union[torch.Tensor, List[torch.Tensor]]
298 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
299 +
300 + h = w = self.config.vision_config.image_size
301 + expected_dims = (3, h, w)
302 +
303 + def _validate_shape(d: torch.Tensor):
304 + actual_dims = tuple(d.shape[2:])
305 +
306 + if actual_dims != expected_dims:
307 + expected_expr = ("num_frames", *map(str, expected_dims))
308 + raise ValueError(
309 + "The expected shape of pixel values in each video frame "
310 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
311 +
312 + for d in data:
313 + _validate_shape(d)
314 +
315 + return data
316 +
317 + def _parse_and_validate_video_input(
318 + self, **kwargs: object) -> Optional[LlavaNextVideoPixelInputs]:
319 + """
320 + A legal video input should have the following dimensions:
321 + {
322 + "pixel_values_videos" :
323 + List[b, Tensor(nb_frames, nb_channels, height, width)]
324 + }
325 + """
326 + pixel_values = kwargs.pop("pixel_values_videos", None)
327 +
328 + if pixel_values is None:
329 + return None
330 +
331 + if not (is_list_of(pixel_values,
332 + (torch.Tensor)) # different shape videos
333 + or isinstance(pixel_values,
334 + torch.Tensor)): # same shape videos
335 + raise ValueError("Incorrect type of pixel values. "
336 + f"Got type: {type(pixel_values)}")
337 +
338 + return LlavaNextVideoPixelInputs(
339 + type="pixel_values_videos",
340 + data=pixel_values,
341 + )
342 +
343 + def _select_image_features(self, image_features: torch.Tensor, *,
344 + strategy: str) -> torch.Tensor:
345 + if strategy == "default":
346 + return image_features[:, 1:]
347 + elif strategy == "full":
348 + return image_features
349 +
350 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
351 +
352 + def _video_pixels_to_features(
353 + self,
354 + vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
355 + pixel_values: torch.Tensor,
356 + ) -> torch.Tensor:
357 +
358 + # NOTE: we skip the step to select the vision feature layer since
359 + # this is already done inside the vision tower
360 + image_features = vision_tower(pixel_values)
361 + image_features = self._select_image_features(
362 + image_features,
363 + strategy=self.config.vision_feature_select_strategy,
364 + )
365 + image_features = self.vision_resampler(image_features)
366 + image_features = self.multi_modal_projector(image_features)
367 + return image_features
368 +
369 + def _process_video_pixels(self, inputs: LlavaNextVideoPixelInputs):
370 + assert self.vision_tower is not None
371 +
372 + video_pixels = inputs["data"]
373 +
374 + if isinstance(video_pixels, torch.Tensor):
375 + # TODO: support multiple videos per input
376 + b, num_videos, num_frames, c, h, w = video_pixels.shape
377 + assert (num_videos == 1)
378 + stacked_pixels = video_pixels.view(b * num_videos * num_frames, c,
379 + h, w)
380 + stacked_embeddings = self._video_pixels_to_features(
381 + self.vision_tower, stacked_pixels)
382 + return stacked_embeddings.view(b, num_frames,
383 + *stacked_embeddings.shape[1:])
384 +
385 + elif is_list_of(video_pixels, torch.Tensor):
386 + frames_per_videos = [v.shape[0] for v in video_pixels]
387 + stacked_pixels = torch.cat(video_pixels, dim=0)
388 + stacked_embeddings = self._video_pixels_to_features(
389 + self.vision_tower, stacked_pixels)
390 + return torch.split(stacked_embeddings, frames_per_videos, dim=0)
391 +
392 + else:
393 + raise ValueError(
394 + f"Unsupported type of video input {type(video_pixels)}")
395 +
396 + def forward(
397 + self,
398 + input_ids: torch.Tensor,
399 + positions: torch.Tensor,
400 + kv_caches: List[torch.Tensor],
401 + attn_metadata: AttentionMetadata,
402 + intermediate_tensors: Optional[IntermediateTensors] = None,
403 + **kwargs: object,
404 + ) -> SamplerOutput:
405 + """Run forward pass for LlaVA-NeXT-Video.
406 + Args:
407 + input_ids: Flattened (concatenated) input_ids corresponding to a
408 + batch.
409 + pixel_values_videos: Pixels in each frames for each input videos.
410 + """
411 + video_input = self._parse_and_validate_video_input(**kwargs)
412 +
413 + # merge video embeddings into input embeddings
414 + if video_input is not None:
415 + video_embeddings = self._process_video_pixels(video_input)
416 + inputs_embeds = self.language_model \
417 + .model.get_input_embeddings(input_ids)
418 +
419 + inputs_embeds = merge_multimodal_embeddings(
420 + input_ids, inputs_embeds, video_embeddings,
421 + self.config.video_token_index)
422 +
423 + input_ids = None
424 + else:
425 + inputs_embeds = None
426 +
427 + hidden_states = self.language_model.model(input_ids,
428 + positions,
429 + kv_caches,
430 + attn_metadata,
431 + None,
432 + inputs_embeds=inputs_embeds)
433 +
434 + return hidden_states
435 +
436 + def compute_logits(
437 + self,
438 + hidden_states: torch.Tensor,
439 + sampling_metadata: SamplingMetadata,
440 + ) -> Optional[torch.Tensor]:
441 + return self.language_model.compute_logits(hidden_states,
442 + sampling_metadata)
443 +
444 + def sample(
445 + self,
446 + logits: torch.Tensor,
447 + sampling_metadata: SamplingMetadata,
448 + ) -> Optional[SamplerOutput]:
449 + return self.language_model.sample(logits, sampling_metadata)
450 +
451 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
452 + # prepare weight iterators
453 + vit_weights, mlp_weights, newline_weights, llm_weights = itertools.tee(
454 + weights, 4)
455 +
456 + # load vision encoder
457 + vit_weights = filter_weights(vit_weights, "vision_tower")
458 + self.vision_tower.load_weights(vit_weights)
459 +
460 + # load mlp projector
461 + mlp_weights = filter_weights(mlp_weights, "multi_modal_projector")
462 + mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
463 + for name, loaded_weight in mlp_weights:
464 + param = mlp_params_dict[name]
465 + weight_loader = getattr(param, "weight_loader",
466 + default_weight_loader)
467 + weight_loader(param, loaded_weight)
468 +
469 + # load llm backbone
470 + llm_weights = filter_weights(llm_weights, "language_model")
471 + self.language_model.load_weights(llm_weights)
vllm/multimodal/registry.py
+2 -1
@@ -9,6 +9,7 @@ from .audio import AudioPlugin
9 from .base import (MultiModalDataDict, MultiModalInputMapper, MultiModalInputs,
10 MultiModalPlugin, MultiModalTokensCalc, NestedTensors)
11 from .image import ImagePlugin
12 +from .video import VideoPlugin
13
14 logger = init_logger(__name__)
15
@@ -34,7 +35,7 @@ class MultiModalRegistry:
35 :class:`~vllm.multimodal.MultiModalPlugin` for each modality.
36 """
37
37 - DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin())
38 + DEFAULT_PLUGINS = (ImagePlugin(), AudioPlugin(), VideoPlugin())
39
40 def __init__(
41 self,
vllm/multimodal/utils.py
+42
@@ -4,6 +4,7 @@ from io import BytesIO
4 from typing import Any, List, Optional, Tuple, TypeVar, Union
5
6 import numpy as np
7 +import numpy.typing as npt
8 from PIL import Image
9
10 from vllm.connections import global_http_connection
@@ -187,6 +188,47 @@ def rescale_image_size(image: Image.Image,
188 return image
189
190
191 +def try_import_video_packages() -> Any:
192 + try:
193 + import cv2
194 + except ImportError:
195 + raise ImportError(
196 + "Please install vllm[video] for video support.") from None
197 + return cv2
198 +
199 +
200 +def resize_video(frames: npt.NDArray, size: Tuple[int, int]) -> npt.NDArray:
201 + cv2 = try_import_video_packages()
202 +
203 + num_frames, _, _, channels = frames.shape
204 + new_height, new_width = size
205 + resized_frames = np.empty((num_frames, new_height, new_width, channels),
206 + dtype=frames.dtype)
207 + for i, frame in enumerate(frames):
208 + resized_frame = cv2.resize(frame, (new_width, new_height))
209 + resized_frames[i] = resized_frame
210 + return resized_frames
211 +
212 +
213 +def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray:
214 + _, height, width, _ = frames.shape
215 + new_height = int(height * size_factor)
216 + new_width = int(width * size_factor)
217 +
218 + return resize_video(frames, (new_height, new_width))
219 +
220 +
221 +def sample_frames_from_video(frames: npt.NDArray,
222 + num_frames: int) -> npt.NDArray:
223 + total_frames = frames.shape[0]
224 + if num_frames == -1:
225 + return frames
226 + else:
227 + frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int)
228 + sampled_frames = frames[frame_indices, ...]
229 + return sampled_frames
230 +
231 +
232 # Utilities for input processors
233 _T = TypeVar("_T", str, int)
234
vllm/multimodal/video.py new
+71
@@ -0,0 +1,71 @@
1 +from functools import lru_cache
2 +from typing import List, Union
3 +
4 +import numpy as np
5 +
6 +from vllm.config import ModelConfig
7 +from vllm.inputs.registry import InputContext
8 +from vllm.logger import init_logger
9 +from vllm.transformers_utils.image_processor import get_video_processor
10 +from vllm.transformers_utils.tokenizer import get_tokenizer
11 +from vllm.utils import is_list_of
12 +
13 +from .base import MultiModalData, MultiModalInputs
14 +from .image import ImagePlugin
15 +
16 +logger = init_logger(__name__)
17 +
18 +cached_get_video_processor = lru_cache(get_video_processor)
19 +cached_get_tokenizer = lru_cache(get_tokenizer)
20 +
21 +VideoInput = Union[
22 + "np.ndarray", # single video input
23 + List["np.ndarray"],
24 + # TODO: support more types
25 + # List[Image.Image], List[List[Image.Image]],
26 + # "torch.Tensor",
27 + # List["torch.Tensor"],
28 + # List[List["np.ndarrray"]],
29 + # List[List["torch.Tensor"]],
30 +]
31 +
32 +
33 +class VideoPlugin(ImagePlugin):
34 + """Plugin for video data."""
35 +
36 + def get_data_key(self) -> str:
37 + return "video"
38 +
39 + def _get_hf_video_processor(self, model_config: ModelConfig):
40 + return cached_get_video_processor(
41 + model_config.model,
42 + trust_remote_code=model_config.trust_remote_code)
43 +
44 + def _default_input_mapper(
45 + self,
46 + ctx: InputContext,
47 + data: MultiModalData[object],
48 + ) -> MultiModalInputs:
49 + model_config = ctx.model_config
50 +
51 + # single video input as np.ndarray
52 + if isinstance(data, np.ndarray):
53 + video_processor = self._get_hf_video_processor(model_config)
54 + if video_processor is None:
55 + raise RuntimeError("No HuggingFace processor is available "
56 + "to process the image object")
57 + try:
58 + batch_data = video_processor(data, return_tensors="pt").data
59 + except Exception:
60 + logger.error("Failed to process image (%s)", data)
61 + raise
62 +
63 + return MultiModalInputs(batch_data)
64 + elif is_list_of(data, np.ndarray):
65 + raise NotImplementedError(
66 + "Multi video for a prompt is not supported yet")
67 +
68 + raise TypeError(f"Invalid video type: {type(data)}")
69 +
70 + def _default_max_multimodal_tokens(self, ctx: InputContext) -> int:
71 + return 4096
vllm/transformers_utils/image_processor.py
+27
@@ -1,6 +1,33 @@
1 from typing import cast
2
3
4 +def get_video_processor(
5 + processor_name: str,
6 + trust_remote_code: bool = False,
7 +):
8 + """
9 + Gets a processor for the given model name via HuggingFace.
10 + """
11 + from transformers import AutoProcessor
12 +
13 + try:
14 + processor = AutoProcessor.from_pretrained(processor_name)
15 + video_processor = processor.video_processor
16 +
17 + except ValueError as e:
18 + if not trust_remote_code:
19 + err_msg = (
20 + "Failed to load the processor. If the processor is "
21 + "a custom processor not yet available in the HuggingFace "
22 + "transformers library, consider setting "
23 + "`trust_remote_code=True` in LLM or using the "
24 + "`--trust-remote-code` flag in the CLI.")
25 + raise RuntimeError(err_msg) from e
26 + else:
27 + raise e
28 + return video_processor
29 +
30 +
31 def get_image_processor(
32 processor_name: str,
33 *args,