@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b59532760
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Model][VLM] Add LLaVA-Onevision model support (#8486)

Co-authored-by: litianjian <litianjian@bytedance.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

litianjian committed Sep 23, 2024 at 01:51 UTC 5b59532760c82a9d91f65a3e227524da2af7d4ef
10 files changed +1330 -21
docs/source/models/supported_models.rst
+6 -1
@@ -244,6 +244,11 @@ Multimodal Language Models
244 - Video
245 - :code:`llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. (see note)
246 -
247 + * - :code:`LlavaOnevisionForConditionalGeneration`
248 + - LLaVA-Onevision
249 + - Image\ :sup:`+` / Video
250 + - :code:`llava-hf/llava-onevision-qwen2-7b-ov-hf`, :code:`llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. (see note)
251 + -
252 * - :code:`MiniCPMV`
253 - MiniCPM-V
254 - Image\ :sup:`+`
@@ -288,7 +293,7 @@ Multimodal Language Models
293 For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
294
295 .. note::
291 - For :code:`LLaVA-NeXT-Video` and :code:`Qwen2-VL`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
296 + For :code:`LLaVA-NeXT-Video`, :code:`LLaVA-Onevision` and :code:`Qwen2-VL`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
297 This can be installed by running the following command:
298
299 .. code-block:: bash
examples/offline_inference_vision_language.py
+47 -13
@@ -14,7 +14,8 @@ from vllm.utils import FlexibleArgumentParser
14
15
16 # LLaVA-1.5
17 -def run_llava(question):
17 +def run_llava(question, modality):
18 + assert modality == "image"
19
20 prompt = f"USER: <image>\n{question}\nASSISTANT:"
21
@@ -24,7 +25,8 @@ def run_llava(question):
25
26
27 # LLaVA-1.6/LLaVA-NeXT
27 -def run_llava_next(question):
28 +def run_llava_next(question, modality):
29 + assert modality == "image"
30
31 prompt = f"[INST] <image>\n{question} [/INST]"
32 llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
@@ -34,15 +36,35 @@ def run_llava_next(question):
36
37 # LlaVA-NeXT-Video
38 # Currently only support for video input
37 -def run_llava_next_video(question):
39 +def run_llava_next_video(question, modality):
40 + assert modality == "video"
41 +
42 prompt = f"USER: <video>\n{question} ASSISTANT:"
43 llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
44 stop_token_ids = None
45 return llm, prompt, stop_token_ids
46
47
48 +# LLaVA-OneVision
49 +def run_llava_onevision(question, modality):
50 +
51 + if modality == "video":
52 + prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
53 + <|im_start|>assistant\n"
54 +
55 + elif modality == "image":
56 + prompt = f"<|im_start|>user <image>\n{question}<|im_end|> \
57 + <|im_start|>assistant\n"
58 +
59 + llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
60 + max_model_len=32768)
61 + stop_token_ids = None
62 + return llm, prompt, stop_token_ids
63 +
64 +
65 # Fuyu
45 -def run_fuyu(question):
66 +def run_fuyu(question, modality):
67 + assert modality == "image"
68
69 prompt = f"{question}\n"
70 llm = LLM(model="adept/fuyu-8b")
@@ -51,7 +73,8 @@ def run_fuyu(question):
73
74
75 # Phi-3-Vision
54 -def run_phi3v(question):
76 +def run_phi3v(question, modality):
77 + assert modality == "image"
78
79 prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501
80 # Note: The default setting of max_num_seqs (256) and
@@ -70,7 +93,8 @@ def run_phi3v(question):
93
94
95 # PaliGemma
73 -def run_paligemma(question):
96 +def run_paligemma(question, modality):
97 + assert modality == "image"
98
99 # PaliGemma has special prompt format for VQA
100 prompt = "caption en"
@@ -80,7 +104,8 @@ def run_paligemma(question):
104
105
106 # Chameleon
83 -def run_chameleon(question):
107 +def run_chameleon(question, modality):
108 + assert modality == "image"
109
110 prompt = f"{question}<image>"
111 llm = LLM(model="facebook/chameleon-7b")
@@ -89,7 +114,8 @@ def run_chameleon(question):
114
115
116 # MiniCPM-V
92 -def run_minicpmv(question):
117 +def run_minicpmv(question, modality):
118 + assert modality == "image"
119
120 # 2.0
121 # The official repo doesn't work yet, so we need to use a fork for now
@@ -129,7 +155,9 @@ def run_minicpmv(question):
155
156
157 # InternVL
132 -def run_internvl(question):
158 +def run_internvl(question, modality):
159 + assert modality == "image"
160 +
161 model_name = "OpenGVLab/InternVL2-2B"
162
163 llm = LLM(
@@ -155,7 +183,8 @@ def run_internvl(question):
183
184
185 # BLIP-2
158 -def run_blip2(question):
186 +def run_blip2(question, modality):
187 + assert modality == "image"
188
189 # BLIP-2 prompt format is inaccurate on HuggingFace model repository.
190 # See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
@@ -166,7 +195,8 @@ def run_blip2(question):
195
196
197 # Qwen
169 -def run_qwen_vl(question):
198 +def run_qwen_vl(question, modality):
199 + assert modality == "image"
200
201 llm = LLM(
202 model="Qwen/Qwen-VL",
@@ -180,7 +210,9 @@ def run_qwen_vl(question):
210
211
212 # Qwen2-VL
183 -def run_qwen2_vl(question):
213 +def run_qwen2_vl(question, modality):
214 + assert modality == "image"
215 +
216 model_name = "Qwen/Qwen2-VL-7B-Instruct"
217
218 llm = LLM(
@@ -200,6 +232,7 @@ model_example_map = {
232 "llava": run_llava,
233 "llava-next": run_llava_next,
234 "llava-next-video": run_llava_next_video,
235 + "llava-onevision": run_llava_onevision,
236 "fuyu": run_fuyu,
237 "phi3_v": run_phi3v,
238 "paligemma": run_paligemma,
@@ -255,7 +288,7 @@ def main(args):
288 data = mm_input["data"]
289 question = mm_input["question"]
290
258 - llm, prompt, stop_token_ids = model_example_map[model](question)
291 + llm, prompt, stop_token_ids = model_example_map[model](question, modality)
292
293 # We set temperature to 0.2 so that outputs can be different
294 # even when all prompts are identical when running batch inference.
@@ -306,6 +339,7 @@ if __name__ == "__main__":
339 parser.add_argument('--modality',
340 type=str,
341 default="image",
342 + choices=['image', 'video'],
343 help='Modality of the input.')
344 parser.add_argument('--num-frames',
345 type=int,
tests/models/decoder_only/vision_language/test_llava_next_video.py
-3
@@ -105,9 +105,6 @@ def run_test(
105 for asset in video_assets
106 ]
107
108 - for video in videos:
109 - print(video.shape)
110 -
108 if size_factors is not None:
109 inputs_per_video = [(
110 [prompt for _ in size_factors],
tests/models/decoder_only/vision_language/test_llava_onevision.py new
+356
@@ -0,0 +1,356 @@
1 +from typing import List, Optional, Tuple, Type, overload
2 +
3 +import pytest
4 +import transformers
5 +from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6 + BatchEncoding)
7 +
8 +from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
9 + resize_video, sample_frames_from_video)
10 +from vllm.sequence import SampleLogprobs
11 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
12 +
13 +from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput, VllmRunner,
14 + _VideoAssets)
15 +from ...utils import check_logprobs_close
16 +
17 +# Video test
18 +HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
19 + "sample_demo_1":
20 + "<|im_start|>user <video>\nwhy is this video funny? \
21 + <|im_end|><|im_start|>assistant\n"
22 +})
23 +
24 +models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
25 +
26 +
27 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
28 + Optional[SampleLogprobs]],
29 + model: str):
30 + """Sanitize vllm output to be comparable with hf output."""
31 + output_ids, output_str, out_logprobs = vllm_output
32 +
33 + config = AutoConfig.from_pretrained(model)
34 + video_token_id = config.video_token_index
35 +
36 + tokenizer = AutoTokenizer.from_pretrained(model)
37 + eos_token_id = tokenizer.eos_token_id
38 +
39 + hf_output_ids = [
40 + token_id for idx, token_id in enumerate(output_ids)
41 + if token_id != video_token_id or output_ids[idx - 1] != video_token_id
42 + ]
43 +
44 + hf_output_str = output_str
45 + if hf_output_ids[-1] == eos_token_id:
46 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
47 +
48 + return hf_output_ids, hf_output_str, out_logprobs
49 +
50 +
51 +@overload
52 +def run_video_test(
53 + hf_runner: Type[HfRunner],
54 + vllm_runner: Type[VllmRunner],
55 + video_assets: _VideoAssets,
56 + model: str,
57 + *,
58 + size_factors: List[float],
59 + dtype: str,
60 + max_tokens: int,
61 + num_logprobs: int,
62 + num_frames: int,
63 + tensor_parallel_size: int,
64 + distributed_executor_backend: Optional[str] = None,
65 +):
66 + ...
67 +
68 +
69 +@overload
70 +def run_video_test(
71 + hf_runner: Type[HfRunner],
72 + vllm_runner: Type[VllmRunner],
73 + video_assets: _VideoAssets,
74 + model: str,
75 + *,
76 + sizes: List[Tuple[int, int]],
77 + dtype: str,
78 + max_tokens: int,
79 + num_logprobs: int,
80 + num_frames: int,
81 + tensor_parallel_size: int,
82 + distributed_executor_backend: Optional[str] = None,
83 +):
84 + ...
85 +
86 +
87 +def run_video_test(
88 + hf_runner: Type[HfRunner],
89 + vllm_runner: Type[VllmRunner],
90 + video_assets: _VideoAssets,
91 + model: str,
92 + *,
93 + size_factors: Optional[List[float]] = None,
94 + sizes: Optional[List[Tuple[int, int]]] = None,
95 + dtype: str,
96 + max_tokens: int,
97 + num_logprobs: int,
98 + num_frames: int,
99 + tensor_parallel_size: int,
100 + distributed_executor_backend: Optional[str] = None,
101 +):
102 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
103 +
104 + videos = [
105 + sample_frames_from_video(asset.np_ndarrays, num_frames)
106 + for asset in video_assets
107 + ]
108 +
109 + if size_factors is not None:
110 + inputs_per_video = [(
111 + [prompt for _ in size_factors],
112 + [rescale_video_size(video, factor) for factor in size_factors],
113 + ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
114 + elif sizes is not None:
115 + inputs_per_video = [(
116 + [prompt for _ in sizes],
117 + [resize_video(video, size) for size in sizes],
118 + ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
119 + else:
120 + raise ValueError("You must provide either `size_factors` or `sizes`")
121 +
122 + # max_model_len should be greater than image_feature_size
123 + with vllm_runner(model,
124 + dtype=dtype,
125 + max_model_len=4096,
126 + tensor_parallel_size=tensor_parallel_size,
127 + distributed_executor_backend=distributed_executor_backend,
128 + enforce_eager=True) as vllm_model:
129 + vllm_outputs_per_video = [
130 + vllm_model.generate_greedy_logprobs(prompts,
131 + max_tokens,
132 + num_logprobs=num_logprobs,
133 + videos=videos)
134 + for prompts, videos in inputs_per_video
135 + ]
136 +
137 + def process(hf_inputs: BatchEncoding):
138 + hf_inputs["pixel_values_videos"] = hf_inputs["pixel_values_videos"] \
139 + .to(torch_dtype) # type: ignore
140 + return hf_inputs
141 +
142 + with hf_runner(model,
143 + dtype=dtype,
144 + postprocess_inputs=process,
145 + auto_cls=AutoModelForVision2Seq) as hf_model:
146 + hf_outputs_per_video = [
147 + hf_model.generate_greedy_logprobs_limit(prompts,
148 + max_tokens,
149 + num_logprobs=num_logprobs,
150 + videos=videos)
151 + for prompts, videos in inputs_per_video
152 + ]
153 +
154 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_video,
155 + vllm_outputs_per_video):
156 + # TODO: Check whether using original CLIPVisionModel can improve
157 + # consistency against HF
158 + check_logprobs_close(
159 + outputs_0_lst=hf_outputs,
160 + outputs_1_lst=[
161 + vllm_to_hf_output(vllm_output, model)
162 + for vllm_output in vllm_outputs
163 + ],
164 + name_0="hf",
165 + name_1="vllm",
166 + )
167 +
168 +
169 +@pytest.mark.skipif(transformers.__version__ < "4.45",
170 + reason="Waiting for next transformers release")
171 +@pytest.mark.parametrize("model", models)
172 +@pytest.mark.parametrize(
173 + "size_factors",
174 + [
175 + # No video
176 + [],
177 + # Single-scale
178 + [1.0],
179 + # Single-scale, batched
180 + [1.0, 1.0, 1.0],
181 + # Multi-scale
182 + [0.25, 0.5, 1.0],
183 + ],
184 +)
185 +@pytest.mark.parametrize("dtype", ["half"])
186 +@pytest.mark.parametrize("max_tokens", [128])
187 +@pytest.mark.parametrize("num_logprobs", [5])
188 +@pytest.mark.parametrize("num_frames", [16])
189 +def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
190 + dtype, max_tokens, num_logprobs, num_frames) -> None:
191 + """Inference result should be the same between hf and vllm.
192 +
193 + All the image fixtures for the test is under tests/videos.
194 + For huggingface runner, we provide the np.ndarray as input.
195 + For vllm runner, we provide MultiModalDataDict objects
196 + and corresponding MultiModalConfig as input.
197 + Note, the text input is also adjusted to abide by vllm contract.
198 + The text output is sanitized to be able to compare with hf.
199 + """
200 + run_video_test(
201 + hf_runner,
202 + vllm_runner,
203 + video_assets,
204 + model,
205 + size_factors=size_factors,
206 + dtype=dtype,
207 + max_tokens=max_tokens,
208 + num_logprobs=num_logprobs,
209 + num_frames=num_frames,
210 + tensor_parallel_size=1,
211 + )
212 +
213 +
214 +@pytest.mark.skipif(transformers.__version__ < "4.45",
215 + reason="Waiting for next transformers release")
216 +@pytest.mark.parametrize("model", models)
217 +@pytest.mark.parametrize(
218 + "sizes",
219 + [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
220 +)
221 +@pytest.mark.parametrize("dtype", ["half"])
222 +@pytest.mark.parametrize("max_tokens", [128])
223 +@pytest.mark.parametrize("num_logprobs", [5])
224 +@pytest.mark.parametrize("num_frames", [16])
225 +def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
226 + dtype, max_tokens, num_logprobs,
227 + num_frames) -> None:
228 + run_video_test(
229 + hf_runner,
230 + vllm_runner,
231 + video_assets,
232 + model,
233 + sizes=sizes,
234 + dtype=dtype,
235 + max_tokens=max_tokens,
236 + num_logprobs=num_logprobs,
237 + num_frames=num_frames,
238 + tensor_parallel_size=1,
239 + )
240 +
241 +
242 +# Image test
243 +_LIMIT_IMAGE_PER_PROMPT = 4
244 +
245 +
246 +def run_image_test(
247 + hf_runner: Type[HfRunner],
248 + vllm_runner: Type[VllmRunner],
249 + inputs: List[Tuple[List[str], PromptImageInput]],
250 + model: str,
251 + dtype: str,
252 + max_tokens: int,
253 + num_logprobs: int,
254 + tensor_parallel_size: int,
255 + distributed_executor_backend: Optional[str] = None,
256 +):
257 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
258 +
259 + # max_model_len should be greater than image_feature_size
260 + with vllm_runner(model,
261 + dtype=dtype,
262 + max_model_len=32768,
263 + tensor_parallel_size=tensor_parallel_size,
264 + distributed_executor_backend=distributed_executor_backend,
265 + enforce_eager=True,
266 + limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
267 + }) as vllm_model:
268 + vllm_outputs_per_image = [
269 + vllm_model.generate_greedy_logprobs(prompts,
270 + max_tokens,
271 + num_logprobs=num_logprobs,
272 + images=images)
273 + for prompts, images in inputs
274 + ]
275 +
276 + def process(hf_inputs: BatchEncoding):
277 + hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
278 + .to(torch_dtype) # type: ignore
279 + return hf_inputs
280 +
281 + with hf_runner(model,
282 + dtype=dtype,
283 + postprocess_inputs=process,
284 + auto_cls=AutoModelForVision2Seq) as hf_model:
285 + hf_outputs_per_image = [
286 + hf_model.generate_greedy_logprobs_limit(prompts,
287 + max_tokens,
288 + num_logprobs=num_logprobs,
289 + images=images)
290 + for prompts, images in inputs
291 + ]
292 +
293 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
294 + vllm_outputs_per_image):
295 + # TODO: Check whether using original CLIPVisionModel can improve
296 + # consistency against HF
297 + check_logprobs_close(
298 + outputs_0_lst=hf_outputs,
299 + outputs_1_lst=[
300 + vllm_to_hf_output(vllm_output, model)
301 + for vllm_output in vllm_outputs
302 + ],
303 + name_0="hf",
304 + name_1="vllm",
305 + )
306 +
307 +
308 +@pytest.mark.skipif(transformers.__version__ < "4.45",
309 + reason="Waiting for next transformers release")
310 +@pytest.mark.parametrize("model", models)
311 +@pytest.mark.parametrize("dtype", ["half"])
312 +@pytest.mark.parametrize("max_tokens", [128])
313 +@pytest.mark.parametrize("num_logprobs", [5])
314 +def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
315 + model, dtype, max_tokens,
316 + num_logprobs) -> None:
317 + stop_sign = image_assets[0].pil_image
318 + cherry_blossom = image_assets[1].pil_image
319 +
320 + inputs = [(
321 + [
322 + "<|im_start|>user <image><image>\nDescribe 2 images. \
323 + <|im_end|><|im_start|>assistant\n",
324 + "<|im_start|>user <image><image>\nDescribe 2 images. \
325 + <|im_end|><|im_start|>assistant\n",
326 + "<|im_start|>user <image><image><image><image>\nDescribe 4 images. \
327 + <|im_end|><|im_start|>assistant\n",
328 + "<|im_start|>user <image>\nWhat is the season? \
329 + <|im_end|><|im_start|>assistant\n",
330 + ],
331 + [
332 + [stop_sign, cherry_blossom],
333 + # Images with different sizes and aspect-ratios
334 + [
335 + rescale_image_size(stop_sign, 0.1),
336 + stop_sign,
337 + ],
338 + [
339 + stop_sign,
340 + rescale_image_size(stop_sign, 0.25),
341 + cherry_blossom.resize((183, 488)),
342 + cherry_blossom.resize((488, 183))
343 + ],
344 + cherry_blossom,
345 + ])]
346 +
347 + run_image_test(
348 + hf_runner,
349 + vllm_runner,
350 + inputs,
351 + model,
352 + dtype=dtype,
353 + max_tokens=max_tokens,
354 + num_logprobs=num_logprobs,
355 + tensor_parallel_size=1,
356 + )
tests/models/test_registry.py
+2 -1
@@ -6,7 +6,8 @@ from vllm.model_executor.models import _MODELS, ModelRegistry
6
7 @pytest.mark.parametrize("model_cls", _MODELS)
8 def test_registry_imports(model_cls):
9 - if (model_cls == "Qwen2VLForConditionalGeneration"
9 + if (model_cls in ("LlavaOnevisionForConditionalGeneration",
10 + "Qwen2VLForConditionalGeneration")
11 and transformers.__version__ < "4.45"):
12 pytest.skip("Waiting for next transformers release")
13
vllm/assets/video.py
+1 -1
@@ -79,7 +79,7 @@ class VideoAsset:
79 return ret
80
81 @property
82 - def np_ndarrays(self) -> List[npt.NDArray]:
82 + def np_ndarrays(self) -> npt.NDArray:
83 video_path = download_video_asset(self.name)
84 ret = video_to_ndarrays(video_path, self.num_frames)
85 return ret
vllm/model_executor/models/__init__.py
+4 -2
@@ -83,12 +83,14 @@ _MULTIMODAL_MODELS = {
83 ("chameleon", "ChameleonForConditionalGeneration"),
84 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
85 "InternVLChatModel": ("internvl", "InternVLChatModel"),
86 - "LlavaForConditionalGeneration":
87 - ("llava", "LlavaForConditionalGeneration"),
86 + "LlavaForConditionalGeneration": ("llava",
87 + "LlavaForConditionalGeneration"),
88 "LlavaNextForConditionalGeneration": ("llava_next",
89 "LlavaNextForConditionalGeneration"),
90 "LlavaNextVideoForConditionalGeneration":
91 ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
92 + "LlavaOnevisionForConditionalGeneration":
93 + ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
94 "MiniCPMV": ("minicpmv", "MiniCPMV"),
95 "PaliGemmaForConditionalGeneration": ("paligemma",
96 "PaliGemmaForConditionalGeneration"),
vllm/model_executor/models/clip.py
+19
@@ -2,6 +2,7 @@
2 within a vision language model."""
3 from typing import Iterable, List, Optional, Tuple, Union
4
5 +import numpy as np
6 import torch
7 import torch.nn as nn
8 from PIL import Image
@@ -84,6 +85,24 @@ def dummy_image_for_clip(
85 return {"image": image if num_images == 1 else [image] * num_images}
86
87
88 +def dummy_video_for_clip(
89 + hf_config: CLIPVisionConfig,
90 + num_frames: int,
91 + *,
92 + image_width_override: Optional[int] = None,
93 + image_height_override: Optional[int] = None,
94 +):
95 + pil_frame = dummy_image_for_clip(
96 + hf_config,
97 + num_images=1,
98 + image_width_override=image_width_override,
99 + image_height_override=image_height_override)
100 + np_frame = np.array(pil_frame["image"])
101 + mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
102 + mm_data = {"video": mm_data_per_video}
103 + return mm_data
104 +
105 +
106 def input_processor_for_clip(
107 model_config: ModelConfig,
108 hf_config: CLIPVisionConfig,
vllm/model_executor/models/llava_onevision.py new
+876
@@ -0,0 +1,876 @@
1 +import math
2 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 + TypedDict, Union)
4 +
5 +import numpy as np
6 +import torch
7 +import torch.nn as nn
8 +from PIL import Image
9 +from transformers import (CLIPVisionConfig, LlavaOnevisionConfig,
10 + SiglipVisionConfig)
11 +from transformers.models.llava_onevision.modeling_llava_onevision import (
12 + get_anyres_image_grid_shape, unpad_image)
13 +from typing_extensions import NotRequired
14 +
15 +from vllm.attention import AttentionMetadata
16 +from vllm.config import CacheConfig, MultiModalConfig
17 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
18 +from vllm.logger import init_logger
19 +from vllm.model_executor.layers.activation import get_act_fn
20 +from vllm.model_executor.layers.quantization.base_config import (
21 + QuantizationConfig)
22 +from vllm.model_executor.layers.sampler import SamplerOutput
23 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 +from vllm.model_executor.sampling_metadata import SamplingMetadata
25 +from vllm.multimodal import MULTIMODAL_REGISTRY
26 +from vllm.multimodal.utils import (cached_get_tokenizer,
27 + repeat_and_pad_placeholder_tokens)
28 +from vllm.sequence import IntermediateTensors
29 +from vllm.utils import is_list_of
30 +
31 +from .clip import (CLIPVisionModel, dummy_seq_data_for_clip,
32 + dummy_video_for_clip, get_clip_image_feature_size,
33 + get_clip_patch_grid_length, input_processor_for_clip)
34 +from .interfaces import SupportsMultiModal
35 +from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
36 + dummy_video_for_siglip, get_siglip_image_feature_size,
37 + get_siglip_patch_grid_length, input_processor_for_siglip)
38 +from .utils import (flatten_bn, group_weights_with_prefix,
39 + init_vllm_registered_model, merge_multimodal_embeddings)
40 +
41 +logger = init_logger(__name__)
42 +
43 +# Result in the max possible feature size (2x2 grid of 336x336px tiles)
44 +MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
45 +
46 +# For profile run
47 +_MAX_FRAMES_PER_VIDEO = 16
48 +_MAX_NUM_VIDEOS = 1
49 +
50 +
51 +class LlavaOnevisionVideoPixelInputs(TypedDict):
52 + type: Literal["pixel_values_videos"]
53 + data: Union[torch.Tensor, List[torch.Tensor]]
54 + """
55 + Shape: `(batch_size, num_frames, num_channels, height, width)`
56 +
57 + Note that `num_frames` may be different for each batch, in which case
58 + the data is passed as a list instead of a batched tensor.
59 +
60 + Note that it only supports one video input for one batch.
61 + """
62 +
63 +
64 +class LlavaOnevisionImagePixelInputs(TypedDict):
65 + type: Literal["pixel_values"]
66 + data: Union[torch.Tensor, List[torch.Tensor]]
67 + """
68 + Shape:
69 + `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
70 +
71 + Note that `num_patches` may be different per batch and image,
72 + in which case the data is passed as a list instead of a batched tensor.
73 + """
74 +
75 + image_sizes: NotRequired[torch.Tensor]
76 + """
77 + Shape: `(batch_size * num_images, 2)`
78 +
79 + This should be in `(height, width)` format.
80 + """
81 +
82 +
83 +class LlavaOnevisionImageEmbeddingInputs(TypedDict):
84 + type: Literal["image_embeds"]
85 + data: torch.Tensor
86 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
87 +
88 + `hidden_size` must match the hidden size of language model backbone.
89 + """
90 +
91 +
92 +LlavaOnevisionImageInputs = Union[LlavaOnevisionImagePixelInputs,
93 + LlavaOnevisionImageEmbeddingInputs]
94 +
95 +LlavaOnevisionMultiInputs = Union[LlavaOnevisionImageInputs,
96 + LlavaOnevisionVideoPixelInputs]
97 +
98 +
99 +def _get_llava_onevision_image_unppaded_feature_size(height, width, patches,
100 + scale_height,
101 + scale_width):
102 + current_height = patches * scale_height
103 + current_width = patches * scale_width
104 +
105 + original_aspect_ratio = width / height
106 + current_aspect_ratio = current_width / current_height
107 + if original_aspect_ratio > current_aspect_ratio:
108 + new_height = int(height * (current_width / width))
109 + padding = (current_height - new_height) // 2
110 + current_height -= padding * 2
111 + else:
112 + new_width = int(width * (current_height / height))
113 + padding = (current_width - new_width) // 2
114 + current_width -= padding * 2
115 +
116 + unpadded_features = current_height * current_width
117 + newline_features = current_height
118 +
119 + ratio = math.sqrt(current_height * current_width / (9 * patches**2))
120 + if ratio > 1.1:
121 + unpadded_features = int(current_height // ratio) * int(
122 + current_width // ratio)
123 + newline_features = int(current_height // ratio)
124 +
125 + return (unpadded_features, newline_features)
126 +
127 +
128 +def get_llava_onevision_image_feature_size(
129 + hf_config: LlavaOnevisionConfig,
130 + *,
131 + input_height: int,
132 + input_width: int,
133 +) -> int:
134 + vision_config = hf_config.vision_config
135 +
136 + if isinstance(vision_config, CLIPVisionConfig):
137 + num_patches = get_clip_patch_grid_length(
138 + image_size=vision_config.image_size,
139 + patch_size=vision_config.patch_size,
140 + )
141 + base_feature_size = get_clip_image_feature_size(vision_config)
142 + elif isinstance(vision_config, SiglipVisionConfig):
143 + num_patches = get_siglip_patch_grid_length(
144 + image_size=vision_config.image_size,
145 + patch_size=vision_config.patch_size,
146 + )
147 + base_feature_size = get_siglip_image_feature_size(vision_config)
148 + else:
149 + msg = f"Unsupported vision config: {type(vision_config)}"
150 + raise NotImplementedError(msg)
151 +
152 + strategy = hf_config.vision_feature_select_strategy
153 + if strategy == "default":
154 + base_feature_size -= 1
155 + elif strategy == "full":
156 + pass
157 + else:
158 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
159 +
160 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
161 + image_size=(input_height, input_width),
162 + grid_pinpoints=hf_config.image_grid_pinpoints,
163 + patch_size=vision_config.image_size,
164 + )
165 +
166 + (
167 + unpadded_feature_size,
168 + newline_feature_size,
169 + ) = _get_llava_onevision_image_unppaded_feature_size(
170 + input_height, input_width, num_patches, num_patch_height,
171 + num_patch_width)
172 +
173 + return unpadded_feature_size + newline_feature_size + base_feature_size
174 +
175 +
176 +def get_max_llava_onevision_image_tokens(ctx: InputContext):
177 + return get_llava_onevision_image_feature_size(
178 + ctx.get_hf_config(LlavaOnevisionConfig),
179 + input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
180 + input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
181 + )
182 +
183 +
184 +def get_llava_onevision_video_frame_feature_size(
185 + hf_config: LlavaOnevisionConfig) -> int:
186 + # Support both CLIPVisionConfig and SiglipVisionConfig
187 + image_size = hf_config.vision_config.image_size
188 + patch_size = hf_config.vision_config.patch_size
189 + spatial_pool_stride = hf_config.spatial_pool_stride if hasattr(
190 + hf_config, "spatial_pool_stride") else 2
191 +
192 + height = width = image_size // patch_size
193 + return math.ceil(height / spatial_pool_stride) * math.ceil(
194 + width / spatial_pool_stride)
195 +
196 +
197 +def get_llava_onevision_video_tokens(ctx: InputContext,
198 + num_frames: int) -> int:
199 + hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
200 +
201 + # TODO: support configuring (not supported by HF right now)
202 + num_token_image_newline = 1
203 + tokens_per_frame = get_llava_onevision_video_frame_feature_size(hf_config)
204 + video_feature_size = num_frames * tokens_per_frame + num_token_image_newline
205 +
206 + return video_feature_size
207 +
208 +
209 +def get_max_llava_onevision_video_tokens(ctx: InputContext) -> int:
210 + return get_llava_onevision_video_tokens(ctx, _MAX_FRAMES_PER_VIDEO)
211 +
212 +
213 +def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
214 + mm_counts: Mapping[str, int]):
215 + hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
216 + vision_config = hf_config.vision_config
217 +
218 + # TODO: support multiple videos
219 + num_videos = mm_counts["video"]
220 + if num_videos > _MAX_NUM_VIDEOS:
221 + raise NotImplementedError(
222 + f"Only {_MAX_NUM_VIDEOS} videos are supported")
223 +
224 + # TODO: support configuring the number of frames
225 + num_frames = _MAX_FRAMES_PER_VIDEO
226 + video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
227 +
228 + if isinstance(vision_config, CLIPVisionConfig):
229 + seq_data = dummy_seq_data_for_clip(
230 + vision_config,
231 + seq_len,
232 + num_videos,
233 + image_token_id=hf_config.video_token_index,
234 + image_feature_size_override=video_feature_size,
235 + )
236 +
237 + mm_data = dummy_video_for_clip(vision_config, num_frames=num_frames)
238 + return seq_data, mm_data
239 + elif isinstance(vision_config, SiglipVisionConfig):
240 + seq_data = dummy_seq_data_for_siglip(
241 + vision_config,
242 + seq_len,
243 + num_videos,
244 + image_token_id=hf_config.video_token_index,
245 + image_feature_size_override=video_feature_size,
246 + )
247 +
248 + mm_data = dummy_video_for_siglip(vision_config, num_frames=num_frames)
249 + return seq_data, mm_data
250 +
251 + msg = f"Unsupported vision config: {type(vision_config)}"
252 + raise NotImplementedError(msg)
253 +
254 +
255 +def input_processor_when_multimodal_input_image(ctx: InputContext,
256 + llm_inputs: LLMInputs):
257 + multi_modal_data = llm_inputs.get("multi_modal_data")
258 + if multi_modal_data is None or "image" not in multi_modal_data:
259 + return llm_inputs
260 +
261 + model_config = ctx.model_config
262 + hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
263 + vision_config = hf_config.vision_config
264 +
265 + image_data = multi_modal_data["image"]
266 + if isinstance(image_data, Image.Image):
267 + width, height = image_data.size
268 +
269 + image_feature_size = get_llava_onevision_image_feature_size(
270 + hf_config,
271 + input_height=height,
272 + input_width=width,
273 + )
274 + elif is_list_of(image_data, Image.Image):
275 + image_feature_size = [
276 + get_llava_onevision_image_feature_size(hf_config,
277 + input_height=img.height,
278 + input_width=img.width)
279 + for img in image_data
280 + ]
281 + elif isinstance(image_data, torch.Tensor):
282 + num_images, image_feature_size, hidden_size = image_data.shape
283 + elif is_list_of(image_data, torch.Tensor):
284 + image_feature_size = [item.shape[1] for item in image_data]
285 + else:
286 + raise TypeError(f"Invalid image type: {type(image_data)}")
287 +
288 + vision_config = hf_config.vision_config
289 +
290 + if isinstance(vision_config, CLIPVisionConfig):
291 + return input_processor_for_clip(
292 + model_config,
293 + vision_config,
294 + llm_inputs,
295 + image_token_id=hf_config.image_token_index,
296 + image_feature_size_override=image_feature_size,
297 + )
298 + elif isinstance(vision_config, SiglipVisionConfig):
299 + return input_processor_for_siglip(
300 + model_config,
301 + vision_config,
302 + llm_inputs,
303 + image_token_id=hf_config.image_token_index,
304 + image_feature_size_override=image_feature_size,
305 + )
306 +
307 + msg = f"Unsupported vision config: {type(vision_config)}"
308 + raise NotImplementedError(msg)
309 +
310 +
311 +def input_processor_when_multimodal_input_video(ctx: InputContext,
312 + llm_inputs: LLMInputs):
313 + multi_modal_data = llm_inputs.get("multi_modal_data")
314 + if multi_modal_data is None or "video" not in multi_modal_data:
315 + return llm_inputs
316 + video_data = multi_modal_data["video"]
317 +
318 + model_config = ctx.model_config
319 + hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
320 + vision_config = hf_config.vision_config
321 +
322 + if isinstance(video_data, np.ndarray):
323 + # Supports both CLIP and Siglip
324 + num_frames = video_data.shape[0]
325 + video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
326 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
327 +
328 + new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
329 + tokenizer,
330 + llm_inputs.get("prompt"),
331 + llm_inputs["prompt_token_ids"],
332 + placeholder_token_id=hf_config.video_token_index,
333 + repeat_count=video_feature_size,
334 + )
335 +
336 + return LLMInputs(prompt_token_ids=new_token_ids,
337 + prompt=new_prompt,
338 + multi_modal_data=multi_modal_data)
339 +
340 + elif is_list_of(video_data, np.ndarray):
341 + raise NotImplementedError(
342 + "Processing multiple videos is not supported")
343 +
344 + msg = f"Unsupported vision config: {type(vision_config)}"
345 + raise NotImplementedError(msg)
346 +
347 +
348 +def input_processor_for_llava_onevision(ctx: InputContext,
349 + llm_inputs: LLMInputs):
350 + multi_modal_data = llm_inputs.get("multi_modal_data")
351 + if multi_modal_data is None or ("video" not in multi_modal_data
352 + and "image" not in multi_modal_data):
353 + return llm_inputs
354 + if "image" in multi_modal_data:
355 + return input_processor_when_multimodal_input_image(ctx, llm_inputs)
356 + if "video" in multi_modal_data:
357 + return input_processor_when_multimodal_input_video(ctx, llm_inputs)
358 +
359 + msg = "Unsupported multi data type"
360 + raise NotImplementedError(msg)
361 +
362 +
363 +def _init_vision_tower(hf_config: LlavaOnevisionConfig):
364 + vision_config = hf_config.vision_config
365 +
366 + # Initialize the vision tower only up to the required feature layer
367 + vision_feature_layer = hf_config.vision_feature_layer
368 + if vision_feature_layer < 0:
369 + num_hidden_layers = hf_config.vision_config.num_hidden_layers \
370 + + vision_feature_layer + 1
371 + else:
372 + num_hidden_layers = vision_feature_layer + 1
373 +
374 + if isinstance(vision_config, CLIPVisionConfig):
375 + return CLIPVisionModel(
376 + vision_config,
377 + num_hidden_layers_override=num_hidden_layers,
378 + )
379 + elif isinstance(vision_config, SiglipVisionConfig):
380 + return SiglipVisionModel(
381 + vision_config,
382 + num_hidden_layers_override=num_hidden_layers,
383 + )
384 +
385 + msg = f"Unsupported vision config: {type(vision_config)}"
386 + raise NotImplementedError(msg)
387 +
388 +
389 +class LlavaOnevisionMultiModalProjector(nn.Module):
390 +
391 + def __init__(self, config: LlavaOnevisionConfig):
392 + super().__init__()
393 +
394 + self.linear_1 = nn.Linear(config.vision_config.hidden_size,
395 + config.text_config.hidden_size,
396 + bias=True)
397 + self.act = get_act_fn(config.projector_hidden_act)
398 + self.linear_2 = nn.Linear(config.text_config.hidden_size,
399 + config.text_config.hidden_size,
400 + bias=True)
401 +
402 + def forward(self, image_features: torch.Tensor) -> torch.Tensor:
403 + hidden_states = self.linear_1(image_features)
404 + hidden_states = self.act(hidden_states)
405 + hidden_states = self.linear_2(hidden_states)
406 + return hidden_states
407 +
408 +
409 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
410 +@MULTIMODAL_REGISTRY.register_input_mapper("video")
411 +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
412 + "image", get_max_llava_onevision_image_tokens)
413 +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
414 + "video", get_max_llava_onevision_video_tokens)
415 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_onevision)
416 +@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_onevision)
417 +class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal):
418 +
419 + def __init__(self,
420 + config: LlavaOnevisionConfig,
421 + multimodal_config: MultiModalConfig,
422 + cache_config: Optional[CacheConfig] = None,
423 + quant_config: Optional[QuantizationConfig] = None) -> None:
424 + super().__init__()
425 +
426 + self.config = config
427 + self.multimodal_config = multimodal_config
428 +
429 + # Initialize the vision tower only up to the required feature layer
430 + self.vision_tower = _init_vision_tower(config)
431 + self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
432 + self.language_model = init_vllm_registered_model(
433 + config.text_config, cache_config, quant_config)
434 + self.image_newline = nn.Parameter(
435 + torch.empty(config.text_config.hidden_size))
436 +
437 + def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
438 + expected_dims = (2, )
439 +
440 + def _validate_shape(d: torch.Tensor):
441 + actual_dims = tuple(d.shape)
442 +
443 + if actual_dims != expected_dims:
444 + expected_expr = str(expected_dims)
445 + raise ValueError(
446 + f"The expected shape of image sizes per image per batch "
447 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
448 +
449 + for d in data:
450 + _validate_shape(d)
451 +
452 + return data
453 +
454 + def _validate_image_pixel_values(
455 + self, data: Union[torch.Tensor, List[torch.Tensor]]
456 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
457 +
458 + h = w = self.config.vision_config.image_size
459 + expected_dims = (3, h, w)
460 +
461 + def _validate_shape(d: torch.Tensor):
462 + actual_dims = tuple(d.shape[1:])
463 +
464 + if actual_dims != expected_dims:
465 + expected_expr = ("num_patches", *map(str, expected_dims))
466 + raise ValueError(
467 + "The expected shape of pixel values per image per batch "
468 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
469 +
470 + for d in data:
471 + _validate_shape(d)
472 +
473 + return data
474 +
475 + def _parse_and_validate_image_input(
476 + self, **kwargs: object) -> Optional[LlavaOnevisionImageInputs]:
477 + pixel_values = kwargs.pop("pixel_values", None)
478 + image_sizes = kwargs.pop("image_sizes", None)
479 + image_embeds = kwargs.pop("image_embeds", None)
480 +
481 + if pixel_values is None and image_embeds is None:
482 + return None
483 +
484 + if pixel_values is not None:
485 + if not isinstance(pixel_values, (torch.Tensor, list)):
486 + raise ValueError("Incorrect type of pixel values. "
487 + f"Got type: {type(pixel_values)}")
488 +
489 + if not isinstance(image_sizes, (torch.Tensor, list)):
490 + raise ValueError("Incorrect type of image sizes. "
491 + f"Got type: {type(image_sizes)}")
492 +
493 + return LlavaOnevisionImagePixelInputs(
494 + type="pixel_values",
495 + data=self._validate_image_pixel_values(
496 + flatten_bn(pixel_values)),
497 + image_sizes=self._validate_image_sizes(
498 + flatten_bn(image_sizes, concat=True)),
499 + )
500 +
501 + if image_embeds is not None:
502 + if not isinstance(image_embeds, torch.Tensor):
503 + raise ValueError("Incorrect type of image embeds. "
504 + f"Got type: {type(image_embeds)}")
505 +
506 + return LlavaOnevisionImageEmbeddingInputs(
507 + type="image_embeds",
508 + data=flatten_bn(image_embeds),
509 + )
510 +
511 + raise AssertionError("This line should be unreachable.")
512 +
513 + def _validate_video_pixel_values(
514 + self, data: Union[torch.Tensor, List[torch.Tensor]]
515 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
516 +
517 + h = w = self.config.vision_config.image_size
518 + expected_dims = (3, h, w)
519 +
520 + def _validate_shape(d: torch.Tensor):
521 + actual_dims = tuple(d.shape[2:])
522 +
523 + if actual_dims != expected_dims:
524 + expected_expr = ("num_frames", *map(str, expected_dims))
525 + raise ValueError(
526 + "The expected shape of pixel values in each video frame "
527 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
528 +
529 + for d in data:
530 + _validate_shape(d)
531 +
532 + return data
533 +
534 + def _parse_and_validate_video_input(
535 + self,
536 + **kwargs: object) -> Optional[LlavaOnevisionVideoPixelInputs]:
537 + """
538 + A legal video input should have the following dimensions:
539 + {
540 + "pixel_values_videos" :
541 + List[b, Tensor(nb_frames, nb_channels, height, width)]
542 + }
543 + """
544 + pixel_values = kwargs.pop("pixel_values_videos", None)
545 +
546 + if pixel_values is None:
547 + return None
548 +
549 + if not (is_list_of(pixel_values,
550 + (torch.Tensor)) # different shape videos
551 + or isinstance(pixel_values,
552 + torch.Tensor)): # same shape videos
553 + raise ValueError("Incorrect type of pixel values. "
554 + f"Got type: {type(pixel_values)}")
555 +
556 + return LlavaOnevisionVideoPixelInputs(
557 + type="pixel_values_videos",
558 + data=pixel_values,
559 + )
560 +
561 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
562 + modalities = {}
563 +
564 + if "pixel_values" in kwargs:
565 + modalities["images"] = self._parse_and_validate_image_input(
566 + **kwargs)
567 +
568 + if "pixel_values_videos" in kwargs:
569 + modalities["videos"] = self._parse_and_validate_video_input(
570 + **kwargs)
571 +
572 + return modalities
573 +
574 + def _select_image_features(self, image_features: torch.Tensor, *,
575 + strategy: str) -> torch.Tensor:
576 + if strategy == "default":
577 + return image_features[:, 1:]
578 + elif strategy == "full":
579 + return image_features
580 +
581 + raise ValueError(f"Unexpected select feature strategy: {strategy}")
582 +
583 + def _image_pixels_to_features(
584 + self,
585 + vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
586 + pixel_values: torch.Tensor,
587 + ) -> torch.Tensor:
588 +
589 + # NOTE: we skip the step to select the vision feature layer since
590 + # this is already done inside the vision tower
591 + image_features = vision_tower(pixel_values)
592 + return self._select_image_features(
593 + image_features,
594 + strategy=self.config.vision_feature_select_strategy,
595 + )
596 +
597 + # Based on: https://github.com/haotian-liu/LLaVA/blob/main/llava/model/llava_arch.py
598 + def _merge_image_patch_embeddings(self,
599 + image_size: torch.Tensor,
600 + patch_embeddings: torch.Tensor,
601 + *,
602 + image_newline=None,
603 + vision_aspect_ratio="anyres_max_9",
604 + strategy: str) -> torch.Tensor:
605 + if strategy == "flat":
606 + return patch_embeddings.flatten(0, 1)
607 +
608 + if strategy.startswith("spatial"):
609 + height = width = self.config.vision_config.image_size \
610 + // self.config.vision_config.patch_size
611 +
612 + base_patch_embeds = patch_embeddings[0]
613 + if height * width != base_patch_embeds.shape[0]:
614 + raise ValueError(
615 + "The number of patches is not consistent with the "
616 + "image size.")
617 +
618 + if patch_embeddings.shape[0] > 1:
619 + other_patch_embeds = patch_embeddings[1:]
620 +
621 + # Move to CPU to avoid floating-point errors
622 + orig_height, orig_width = image_size.tolist()
623 +
624 + # image_aspect_ratio == "anyres"
625 + num_patch_height, num_patch_width = get_anyres_image_grid_shape(
626 + (orig_height, orig_width),
627 + self.config.image_grid_pinpoints,
628 + self.config.vision_config.image_size,
629 + )
630 + num_patches = num_patch_height * num_patch_width
631 +
632 + # Image patches might be padded for batch processing
633 + other_patch_embeds = other_patch_embeds[:num_patches] \
634 + .view(num_patch_height, num_patch_width, height, width, -1)
635 +
636 + if "unpad" in strategy:
637 + other_patch_embeds = other_patch_embeds \
638 + .permute(4, 0, 2, 1, 3).contiguous() \
639 + .flatten(1, 2).flatten(2, 3)
640 + other_patch_embeds = unpad_image(other_patch_embeds,
641 + (orig_height, orig_width))
642 + max_num_patches = int(
643 + vision_aspect_ratio.removeprefix("anyres_max_"))
644 + channels, curr_height, curr_width = other_patch_embeds.shape
645 + ratio = math.sqrt(curr_height * curr_width /
646 + (max_num_patches * height**2))
647 + if ratio > 1.1:
648 + other_patch_embeds = other_patch_embeds[None]
649 + other_patch_embeds = nn.functional.interpolate(
650 + other_patch_embeds, [
651 + int(curr_height // ratio),
652 + int(curr_width // ratio)
653 + ],
654 + mode="bilinear")[0]
655 + if image_newline is not None:
656 + other_patch_embeds = torch.cat(
657 + (
658 + other_patch_embeds,
659 + image_newline[:, None, None] \
660 + .expand(*other_patch_embeds.shape[:-1], 1) \
661 + .to(other_patch_embeds.device),
662 + ),
663 + dim=-1)
664 + other_patch_embeds = other_patch_embeds \
665 + .flatten(1, 2).transpose(0, 1)
666 + else:
667 + other_patch_embeds = other_patch_embeds \
668 + .permute(0, 2, 1, 3, 4).contiguous() \
669 + .flatten(0, 3)
670 +
671 + merged_patch_embeddings = torch.cat(
672 + (base_patch_embeds, other_patch_embeds), dim=0)
673 + else:
674 + if "unpad" in strategy:
675 + merged_patch_embeddings = torch.cat(
676 + (base_patch_embeds,
677 + self.image_newline[None] \
678 + .to(base_patch_embeds.device)
679 + ), dim=0)
680 + else:
681 + merged_patch_embeddings = base_patch_embeds
682 +
683 + return merged_patch_embeddings
684 +
685 + raise ValueError(f"Unexpected patch merge strategy: {strategy}")
686 +
687 + def _process_image_pixels(
688 + self,
689 + inputs: LlavaOnevisionImagePixelInputs,
690 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
691 + assert self.vision_tower is not None
692 +
693 + pixel_values = inputs["data"]
694 +
695 + if isinstance(pixel_values, torch.Tensor):
696 + b, num_patches, c, h, w = pixel_values.shape
697 + stacked_pixel_values = pixel_values.view(b * num_patches, c, h, w)
698 + stacked_image_features = self._image_pixels_to_features(
699 + self.vision_tower, stacked_pixel_values)
700 + stacked_patch_embeddings = self.multi_modal_projector(
701 + stacked_image_features)
702 +
703 + return stacked_patch_embeddings.view(
704 + b, num_patches, *stacked_patch_embeddings.shape[1:])
705 +
706 + num_patches_per_batch = [v.shape[0] for v in pixel_values]
707 + stacked_pixel_values = torch.cat(pixel_values)
708 + stacked_image_features = self._image_pixels_to_features(
709 + self.vision_tower, stacked_pixel_values)
710 +
711 + return [
712 + self.multi_modal_projector(image_features) for image_features in
713 + torch.split(stacked_image_features, num_patches_per_batch)
714 + ]
715 +
716 + def _process_image_input(
717 + self,
718 + image_input: LlavaOnevisionImageInputs,
719 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
720 + if image_input["type"] == "image_embeds":
721 + return [image_input["data"]]
722 +
723 + patch_embeddings = self._process_image_pixels(image_input)
724 +
725 + image_sizes = image_input.get("image_sizes")
726 + if image_sizes is None:
727 + batch_size = len(image_input["data"])
728 + vision_config = self.config.vision_config
729 + default_height = default_width = vision_config.image_size
730 + image_sizes = torch.as_tensor([[default_height, default_width]
731 + for _ in range(batch_size)])
732 +
733 + return [
734 + self._merge_image_patch_embeddings(
735 + image_sizes[i],
736 + patch_features_batch,
737 + image_newline=self.image_newline,
738 + strategy="spatial_unpad")
739 + for i, patch_features_batch in enumerate(patch_embeddings)
740 + ]
741 +
742 + def _video_pixels_to_features(
743 + self,
744 + vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
745 + pixel_values: torch.Tensor,
746 + ) -> torch.Tensor:
747 +
748 + # NOTE: we skip the step to select the vision feature layer since
749 + # this is already done inside the vision tower
750 + b, num_videos, frames, c, h, w = pixel_values.shape
751 + assert (num_videos == _MAX_NUM_VIDEOS)
752 + pixel_values = pixel_values.reshape(b * num_videos * frames, c, h, w)
753 + video_features = vision_tower(pixel_values)
754 + video_features = self._select_image_features(
755 + video_features,
756 + strategy=self.config.vision_feature_select_strategy,
757 + )
758 + video_features = self.multi_modal_projector(video_features)
759 + video_features = self.apply_pooling(video_features)
760 + video_features = video_features.reshape(
761 + b, frames * video_features.shape[1], -1)
762 + image_newline = self.image_newline[None, None, :].repeat(b, 1, 1).to(
763 + video_features.device)
764 + video_features = torch.cat((video_features, image_newline), dim=1)
765 + video_features = video_features.flatten(0, 1)
766 +
767 + return video_features
768 +
769 + def _process_video_pixels(self, inputs: LlavaOnevisionVideoPixelInputs):
770 + assert self.vision_tower is not None
771 +
772 + video_pixels = inputs["data"]
773 +
774 + # TODO: support multiple videos per input
775 + if isinstance(video_pixels, torch.Tensor):
776 + stacked_embeddings = self._video_pixels_to_features(
777 + self.vision_tower, video_pixels)
778 + return stacked_embeddings
779 + else:
780 + raise ValueError(
781 + f"Unsupported type of video input {type(video_pixels)}")
782 +
783 + def apply_pooling(self, image_features, stride=2):
784 + vision_config = self.config.vision_config
785 + height = width = vision_config.image_size // vision_config.patch_size
786 + batch_frames, _, dim = image_features.shape
787 + image_features = image_features.view(batch_frames, height, width, -1)
788 + image_features = image_features.permute(0, 3, 1, 2)
789 +
790 + # TODO support other pooling types config
791 + height, width = image_features.shape[2:]
792 + scaled_shape = [math.ceil(height / stride), math.ceil(width / stride)]
793 + image_feature = nn.functional.interpolate(image_features,
794 + size=scaled_shape,
795 + mode='bilinear')
796 + image_feature = image_feature.permute(0, 2, 3, 1)
797 + image_feature = image_feature.view(batch_frames, -1, dim)
798 + return image_feature
799 +
800 + def forward(
801 + self,
802 + input_ids: torch.Tensor,
803 + positions: torch.Tensor,
804 + kv_caches: List[torch.Tensor],
805 + attn_metadata: AttentionMetadata,
806 + intermediate_tensors: Optional[IntermediateTensors] = None,
807 + **kwargs: object,
808 + ) -> SamplerOutput:
809 + """Run forward pass for LlaVA-Onevision.
810 + Args:
811 + input_ids: Flattened (concatenated) input_ids corresponding to a
812 + batch.
813 + pixel_values_videos: Pixels in each frames for each input videos.
814 + """
815 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
816 + # merge video embeddings into input embeddings
817 + if modalities:
818 + inputs_embeds = self.language_model.model.get_input_embeddings(
819 + input_ids)
820 + if "images" in modalities:
821 + image_input = modalities["images"]
822 + vision_embeddings = self._process_image_input(image_input)
823 + inputs_embeds = merge_multimodal_embeddings(
824 + input_ids, inputs_embeds, vision_embeddings,
825 + self.config.image_token_index)
826 + if "videos" in modalities:
827 + video_input = modalities["videos"]
828 + video_embeddings = self._process_video_pixels(video_input)
829 + inputs_embeds = merge_multimodal_embeddings(
830 + input_ids, inputs_embeds, video_embeddings,
831 + self.config.video_token_index)
832 + input_ids = None
833 + else:
834 + inputs_embeds = None
835 +
836 + hidden_states = self.language_model.model(input_ids,
837 + positions,
838 + kv_caches,
839 + attn_metadata,
840 + None,
841 + inputs_embeds=inputs_embeds)
842 +
843 + return hidden_states
844 +
845 + def compute_logits(
846 + self,
847 + hidden_states: torch.Tensor,
848 + sampling_metadata: SamplingMetadata,
849 + ) -> Optional[torch.Tensor]:
850 + return self.language_model.compute_logits(hidden_states,
851 + sampling_metadata)
852 +
853 + def sample(
854 + self,
855 + logits: torch.Tensor,
856 + sampling_metadata: SamplingMetadata,
857 + ) -> Optional[SamplerOutput]:
858 + return self.language_model.sample(logits, sampling_metadata)
859 +
860 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
861 + # prepare weight iterators for components
862 + weights_group = group_weights_with_prefix(weights)
863 +
864 + # load vision encoder
865 + self.vision_tower.load_weights(weights_group["vision_tower"])
866 +
867 + # load mlp projector
868 + mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
869 + for name, loaded_weight in weights_group["multi_modal_projector"]:
870 + param = mlp_params_dict[name]
871 + weight_loader = getattr(param, "weight_loader",
872 + default_weight_loader)
873 + weight_loader(param, loaded_weight)
874 +
875 + # load llm backbone
876 + self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/siglip.py
+19
@@ -4,6 +4,7 @@ within a vision language model."""
4 import math
5 from typing import Iterable, List, Optional, Tuple, Union
6
7 +import numpy as np
8 import torch
9 from PIL import Image
10 from torch import nn
@@ -89,6 +90,24 @@ def dummy_image_for_siglip(
90 return {"image": image if num_images == 1 else [image] * num_images}
91
92
93 +def dummy_video_for_siglip(
94 + hf_config: SiglipVisionConfig,
95 + num_frames: int,
96 + *,
97 + image_width_override: Optional[int] = None,
98 + image_height_override: Optional[int] = None,
99 +):
100 + pil_frame = dummy_image_for_siglip(
101 + hf_config,
102 + num_images=1,
103 + image_width_override=image_width_override,
104 + image_height_override=image_height_override)
105 + np_frame = np.array(pil_frame["image"])
106 + mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
107 + mm_data = {"video": mm_data_per_video}
108 + return mm_data
109 +
110 +
111 def input_processor_for_siglip(
112 model_config: ModelConfig,
113 hf_config: SiglipVisionConfig,