A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Model][VLM] Add LLaVA-Onevision model support (#8486)
Co-authored-by: litianjian <litianjian@bytedance.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
litianjian committed
Sep 23, 2024 at 01:51 UTC
5b59532760c82a9d91f65a3e227524da2af7d4ef
10 files changed
+1330
-21
docs/source/models/supported_models.rst
+6
-1
@@ -244,6 +244,11 @@ Multimodal Language Models
244
- Video
245
- :code:`llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. (see note)
246
-
247
+ * - :code:`LlavaOnevisionForConditionalGeneration`
248
+ - LLaVA-Onevision
249
+ - Image\ :sup:`+` / Video
250
+ - :code:`llava-hf/llava-onevision-qwen2-7b-ov-hf`, :code:`llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. (see note)
251
+ -
252
* - :code:`MiniCPMV`
253
- MiniCPM-V
254
- Image\ :sup:`+`
@@ -288,7 +293,7 @@ Multimodal Language Models
293
For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
294
295
.. note::
291
- For :code:`LLaVA-NeXT-Video` and :code:`Qwen2-VL`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
296
+ For :code:`LLaVA-NeXT-Video`, :code:`LLaVA-Onevision` and :code:`Qwen2-VL`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
297
This can be installed by running the following command:
298
299
.. code-block:: bash
examples/offline_inference_vision_language.py
+47
-13
@@ -14,7 +14,8 @@ from vllm.utils import FlexibleArgumentParser
14
15
16
# LLaVA-1.5
17
-def run_llava(question):
17
+def run_llava(question, modality):
18
+ assert modality == "image"
19
20
prompt = f"USER: <image>\n{question}\nASSISTANT:"
21
@@ -24,7 +25,8 @@ def run_llava(question):
25
26
27
# LLaVA-1.6/LLaVA-NeXT
27
-def run_llava_next(question):
28
+def run_llava_next(question, modality):
29
+ assert modality == "image"
30
31
prompt = f"[INST] <image>\n{question} [/INST]"
32
llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=8192)
@@ -34,15 +36,35 @@ def run_llava_next(question):
36
37
# LlaVA-NeXT-Video
38
# Currently only support for video input
37
-def run_llava_next_video(question):
39
+def run_llava_next_video(question, modality):
40
+ assert modality == "video"
41
+
42
prompt = f"USER: <video>\n{question} ASSISTANT:"
43
llm = LLM(model="llava-hf/LLaVA-NeXT-Video-7B-hf", max_model_len=8192)
44
stop_token_ids = None
45
return llm, prompt, stop_token_ids
46
47
48
+# LLaVA-OneVision
49
+def run_llava_onevision(question, modality):
50
+
51
+ if modality == "video":
52
+ prompt = f"<|im_start|>user <video>\n{question}<|im_end|> \
53
+ <|im_start|>assistant\n"
54
+
55
+ elif modality == "image":
56
+ prompt = f"<|im_start|>user <image>\n{question}<|im_end|> \
57
+ <|im_start|>assistant\n"
58
+
59
+ llm = LLM(model="llava-hf/llava-onevision-qwen2-7b-ov-hf",
60
+ max_model_len=32768)
61
+ stop_token_ids = None
62
+ return llm, prompt, stop_token_ids
63
+
64
+
65
# Fuyu
45
-def run_fuyu(question):
66
+def run_fuyu(question, modality):
67
+ assert modality == "image"
68
69
prompt = f"{question}\n"
70
llm = LLM(model="adept/fuyu-8b")
@@ -51,7 +73,8 @@ def run_fuyu(question):
73
74
75
# Phi-3-Vision
54
-def run_phi3v(question):
76
+def run_phi3v(question, modality):
77
+ assert modality == "image"
78
79
prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501
80
# Note: The default setting of max_num_seqs (256) and
@@ -70,7 +93,8 @@ def run_phi3v(question):
93
94
95
# PaliGemma
73
-def run_paligemma(question):
96
+def run_paligemma(question, modality):
97
+ assert modality == "image"
98
99
# PaliGemma has special prompt format for VQA
100
prompt = "caption en"
@@ -80,7 +104,8 @@ def run_paligemma(question):
104
105
106
# Chameleon
83
-def run_chameleon(question):
107
+def run_chameleon(question, modality):
108
+ assert modality == "image"
109
110
prompt = f"{question}<image>"
111
llm = LLM(model="facebook/chameleon-7b")
@@ -89,7 +114,8 @@ def run_chameleon(question):
114
115
116
# MiniCPM-V
92
-def run_minicpmv(question):
117
+def run_minicpmv(question, modality):
118
+ assert modality == "image"
119
120
# 2.0
121
# The official repo doesn't work yet, so we need to use a fork for now
@@ -129,7 +155,9 @@ def run_minicpmv(question):
155
156
157
# InternVL
132
-def run_internvl(question):
158
+def run_internvl(question, modality):
159
+ assert modality == "image"
160
+
161
model_name = "OpenGVLab/InternVL2-2B"
162
163
llm = LLM(
@@ -155,7 +183,8 @@ def run_internvl(question):
183
184
185
# BLIP-2
158
-def run_blip2(question):
186
+def run_blip2(question, modality):
187
+ assert modality == "image"
188
189
# BLIP-2 prompt format is inaccurate on HuggingFace model repository.
190
# See https://huggingface.co/Salesforce/blip2-opt-2.7b/discussions/15#64ff02f3f8cf9e4f5b038262 #noqa
@@ -166,7 +195,8 @@ def run_blip2(question):
195
196
197
# Qwen
169
-def run_qwen_vl(question):
198
+def run_qwen_vl(question, modality):
199
+ assert modality == "image"
200
201
llm = LLM(
202
model="Qwen/Qwen-VL",
@@ -180,7 +210,9 @@ def run_qwen_vl(question):
210
211
212
# Qwen2-VL
183
-def run_qwen2_vl(question):
213
+def run_qwen2_vl(question, modality):
214
+ assert modality == "image"
215
+
216
model_name = "Qwen/Qwen2-VL-7B-Instruct"
217
218
llm = LLM(
@@ -200,6 +232,7 @@ model_example_map = {
232
"llava": run_llava,
233
"llava-next": run_llava_next,
234
"llava-next-video": run_llava_next_video,
235
+ "llava-onevision": run_llava_onevision,
236
"fuyu": run_fuyu,
237
"phi3_v": run_phi3v,
238
"paligemma": run_paligemma,
@@ -255,7 +288,7 @@ def main(args):
288
data = mm_input["data"]
289
question = mm_input["question"]
290
258
- llm, prompt, stop_token_ids = model_example_map[model](question)
291
+ llm, prompt, stop_token_ids = model_example_map[model](question, modality)
292
293
# We set temperature to 0.2 so that outputs can be different
294
# even when all prompts are identical when running batch inference.
@@ -306,6 +339,7 @@ if __name__ == "__main__":
339
parser.add_argument('--modality',
340
type=str,
341
default="image",
342
+ choices=['image', 'video'],
343
help='Modality of the input.')
344
parser.add_argument('--num-frames',
345
type=int,
tests/models/decoder_only/vision_language/test_llava_next_video.py
-3
@@ -105,9 +105,6 @@ def run_test(
105
for asset in video_assets
106
]
107
108
- for video in videos:
109
- print(video.shape)
110
-
108
if size_factors is not None:
109
inputs_per_video = [(
110
[prompt for _ in size_factors],
tests/models/decoder_only/vision_language/test_llava_onevision.py
new
+356
@@ -0,0 +1,356 @@
1
+from typing import List, Optional, Tuple, Type, overload
2
+
3
+import pytest
4
+import transformers
5
+from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6
+ BatchEncoding)
7
+
8
+from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
9
+ resize_video, sample_frames_from_video)
10
+from vllm.sequence import SampleLogprobs
11
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
12
+
13
+from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput, VllmRunner,
14
+ _VideoAssets)
15
+from ...utils import check_logprobs_close
16
+
17
+# Video test
18
+HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
19
+ "sample_demo_1":
20
+ "<|im_start|>user <video>\nwhy is this video funny? \
21
+ <|im_end|><|im_start|>assistant\n"
22
+})
23
+
24
+models = ["llava-hf/llava-onevision-qwen2-7b-ov-hf"]
25
+
26
+
27
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
28
+ Optional[SampleLogprobs]],
29
+ model: str):
30
+ """Sanitize vllm output to be comparable with hf output."""
31
+ output_ids, output_str, out_logprobs = vllm_output
32
+
33
+ config = AutoConfig.from_pretrained(model)
34
+ video_token_id = config.video_token_index
35
+
36
+ tokenizer = AutoTokenizer.from_pretrained(model)
37
+ eos_token_id = tokenizer.eos_token_id
38
+
39
+ hf_output_ids = [
40
+ token_id for idx, token_id in enumerate(output_ids)
41
+ if token_id != video_token_id or output_ids[idx - 1] != video_token_id
42
+ ]
43
+
44
+ hf_output_str = output_str
45
+ if hf_output_ids[-1] == eos_token_id:
46
+ hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
47
+
48
+ return hf_output_ids, hf_output_str, out_logprobs
49
+
50
+
51
+@overload
52
+def run_video_test(
53
+ hf_runner: Type[HfRunner],
54
+ vllm_runner: Type[VllmRunner],
55
+ video_assets: _VideoAssets,
56
+ model: str,
57
+ *,
58
+ size_factors: List[float],
59
+ dtype: str,
60
+ max_tokens: int,
61
+ num_logprobs: int,
62
+ num_frames: int,
63
+ tensor_parallel_size: int,
64
+ distributed_executor_backend: Optional[str] = None,
65
+):
66
+ ...
67
+
68
+
69
+@overload
70
+def run_video_test(
71
+ hf_runner: Type[HfRunner],
72
+ vllm_runner: Type[VllmRunner],
73
+ video_assets: _VideoAssets,
74
+ model: str,
75
+ *,
76
+ sizes: List[Tuple[int, int]],
77
+ dtype: str,
78
+ max_tokens: int,
79
+ num_logprobs: int,
80
+ num_frames: int,
81
+ tensor_parallel_size: int,
82
+ distributed_executor_backend: Optional[str] = None,
83
+):
84
+ ...
85
+
86
+
87
+def run_video_test(
88
+ hf_runner: Type[HfRunner],
89
+ vllm_runner: Type[VllmRunner],
90
+ video_assets: _VideoAssets,
91
+ model: str,
92
+ *,
93
+ size_factors: Optional[List[float]] = None,
94
+ sizes: Optional[List[Tuple[int, int]]] = None,
95
+ dtype: str,
96
+ max_tokens: int,
97
+ num_logprobs: int,
98
+ num_frames: int,
99
+ tensor_parallel_size: int,
100
+ distributed_executor_backend: Optional[str] = None,
101
+):
102
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
103
+
104
+ videos = [
105
+ sample_frames_from_video(asset.np_ndarrays, num_frames)
106
+ for asset in video_assets
107
+ ]
108
+
109
+ if size_factors is not None:
110
+ inputs_per_video = [(
111
+ [prompt for _ in size_factors],
112
+ [rescale_video_size(video, factor) for factor in size_factors],
113
+ ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
114
+ elif sizes is not None:
115
+ inputs_per_video = [(
116
+ [prompt for _ in sizes],
117
+ [resize_video(video, size) for size in sizes],
118
+ ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
119
+ else:
120
+ raise ValueError("You must provide either `size_factors` or `sizes`")
121
+
122
+ # max_model_len should be greater than image_feature_size
123
+ with vllm_runner(model,
124
+ dtype=dtype,
125
+ max_model_len=4096,
126
+ tensor_parallel_size=tensor_parallel_size,
127
+ distributed_executor_backend=distributed_executor_backend,
128
+ enforce_eager=True) as vllm_model:
129
+ vllm_outputs_per_video = [
130
+ vllm_model.generate_greedy_logprobs(prompts,
131
+ max_tokens,
132
+ num_logprobs=num_logprobs,
133
+ videos=videos)
134
+ for prompts, videos in inputs_per_video
135
+ ]
136
+
137
+ def process(hf_inputs: BatchEncoding):
138
+ hf_inputs["pixel_values_videos"] = hf_inputs["pixel_values_videos"] \
139
+ .to(torch_dtype) # type: ignore
140
+ return hf_inputs
141
+
142
+ with hf_runner(model,
143
+ dtype=dtype,
144
+ postprocess_inputs=process,
145
+ auto_cls=AutoModelForVision2Seq) as hf_model:
146
+ hf_outputs_per_video = [
147
+ hf_model.generate_greedy_logprobs_limit(prompts,
148
+ max_tokens,
149
+ num_logprobs=num_logprobs,
150
+ videos=videos)
151
+ for prompts, videos in inputs_per_video
152
+ ]
153
+
154
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_video,
155
+ vllm_outputs_per_video):
156
+ # TODO: Check whether using original CLIPVisionModel can improve
157
+ # consistency against HF
158
+ check_logprobs_close(
159
+ outputs_0_lst=hf_outputs,
160
+ outputs_1_lst=[
161
+ vllm_to_hf_output(vllm_output, model)
162
+ for vllm_output in vllm_outputs
163
+ ],
164
+ name_0="hf",
165
+ name_1="vllm",
166
+ )
167
+
168
+
169
+@pytest.mark.skipif(transformers.__version__ < "4.45",
170
+ reason="Waiting for next transformers release")
171
+@pytest.mark.parametrize("model", models)
172
+@pytest.mark.parametrize(
173
+ "size_factors",
174
+ [
175
+ # No video
176
+ [],
177
+ # Single-scale
178
+ [1.0],
179
+ # Single-scale, batched
180
+ [1.0, 1.0, 1.0],
181
+ # Multi-scale
182
+ [0.25, 0.5, 1.0],
183
+ ],
184
+)
185
+@pytest.mark.parametrize("dtype", ["half"])
186
+@pytest.mark.parametrize("max_tokens", [128])
187
+@pytest.mark.parametrize("num_logprobs", [5])
188
+@pytest.mark.parametrize("num_frames", [16])
189
+def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
190
+ dtype, max_tokens, num_logprobs, num_frames) -> None:
191
+ """Inference result should be the same between hf and vllm.
192
+
193
+ All the image fixtures for the test is under tests/videos.
194
+ For huggingface runner, we provide the np.ndarray as input.
195
+ For vllm runner, we provide MultiModalDataDict objects
196
+ and corresponding MultiModalConfig as input.
197
+ Note, the text input is also adjusted to abide by vllm contract.
198
+ The text output is sanitized to be able to compare with hf.
199
+ """
200
+ run_video_test(
201
+ hf_runner,
202
+ vllm_runner,
203
+ video_assets,
204
+ model,
205
+ size_factors=size_factors,
206
+ dtype=dtype,
207
+ max_tokens=max_tokens,
208
+ num_logprobs=num_logprobs,
209
+ num_frames=num_frames,
210
+ tensor_parallel_size=1,
211
+ )
212
+
213
+
214
+@pytest.mark.skipif(transformers.__version__ < "4.45",
215
+ reason="Waiting for next transformers release")
216
+@pytest.mark.parametrize("model", models)
217
+@pytest.mark.parametrize(
218
+ "sizes",
219
+ [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
220
+)
221
+@pytest.mark.parametrize("dtype", ["half"])
222
+@pytest.mark.parametrize("max_tokens", [128])
223
+@pytest.mark.parametrize("num_logprobs", [5])
224
+@pytest.mark.parametrize("num_frames", [16])
225
+def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
226
+ dtype, max_tokens, num_logprobs,
227
+ num_frames) -> None:
228
+ run_video_test(
229
+ hf_runner,
230
+ vllm_runner,
231
+ video_assets,
232
+ model,
233
+ sizes=sizes,
234
+ dtype=dtype,
235
+ max_tokens=max_tokens,
236
+ num_logprobs=num_logprobs,
237
+ num_frames=num_frames,
238
+ tensor_parallel_size=1,
239
+ )
240
+
241
+
242
+# Image test
243
+_LIMIT_IMAGE_PER_PROMPT = 4
244
+
245
+
246
+def run_image_test(
247
+ hf_runner: Type[HfRunner],
248
+ vllm_runner: Type[VllmRunner],
249
+ inputs: List[Tuple[List[str], PromptImageInput]],
250
+ model: str,
251
+ dtype: str,
252
+ max_tokens: int,
253
+ num_logprobs: int,
254
+ tensor_parallel_size: int,
255
+ distributed_executor_backend: Optional[str] = None,
256
+):
257
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
258
+
259
+ # max_model_len should be greater than image_feature_size
260
+ with vllm_runner(model,
261
+ dtype=dtype,
262
+ max_model_len=32768,
263
+ tensor_parallel_size=tensor_parallel_size,
264
+ distributed_executor_backend=distributed_executor_backend,
265
+ enforce_eager=True,
266
+ limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
267
+ }) as vllm_model:
268
+ vllm_outputs_per_image = [
269
+ vllm_model.generate_greedy_logprobs(prompts,
270
+ max_tokens,
271
+ num_logprobs=num_logprobs,
272
+ images=images)
273
+ for prompts, images in inputs
274
+ ]
275
+
276
+ def process(hf_inputs: BatchEncoding):
277
+ hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
278
+ .to(torch_dtype) # type: ignore
279
+ return hf_inputs
280
+
281
+ with hf_runner(model,
282
+ dtype=dtype,
283
+ postprocess_inputs=process,
284
+ auto_cls=AutoModelForVision2Seq) as hf_model:
285
+ hf_outputs_per_image = [
286
+ hf_model.generate_greedy_logprobs_limit(prompts,
287
+ max_tokens,
288
+ num_logprobs=num_logprobs,
289
+ images=images)
290
+ for prompts, images in inputs
291
+ ]
292
+
293
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
294
+ vllm_outputs_per_image):
295
+ # TODO: Check whether using original CLIPVisionModel can improve
296
+ # consistency against HF
297
+ check_logprobs_close(
298
+ outputs_0_lst=hf_outputs,
299
+ outputs_1_lst=[
300
+ vllm_to_hf_output(vllm_output, model)
301
+ for vllm_output in vllm_outputs
302
+ ],
303
+ name_0="hf",
304
+ name_1="vllm",
305
+ )
306
+
307
+
308
+@pytest.mark.skipif(transformers.__version__ < "4.45",
309
+ reason="Waiting for next transformers release")
310
+@pytest.mark.parametrize("model", models)
311
+@pytest.mark.parametrize("dtype", ["half"])
312
+@pytest.mark.parametrize("max_tokens", [128])
313
+@pytest.mark.parametrize("num_logprobs", [5])
314
+def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
315
+ model, dtype, max_tokens,
316
+ num_logprobs) -> None:
317
+ stop_sign = image_assets[0].pil_image
318
+ cherry_blossom = image_assets[1].pil_image
319
+
320
+ inputs = [(
321
+ [
322
+ "<|im_start|>user <image><image>\nDescribe 2 images. \
323
+ <|im_end|><|im_start|>assistant\n",
324
+ "<|im_start|>user <image><image>\nDescribe 2 images. \
325
+ <|im_end|><|im_start|>assistant\n",
326
+ "<|im_start|>user <image><image><image><image>\nDescribe 4 images. \
327
+ <|im_end|><|im_start|>assistant\n",
328
+ "<|im_start|>user <image>\nWhat is the season? \
329
+ <|im_end|><|im_start|>assistant\n",
330
+ ],
331
+ [
332
+ [stop_sign, cherry_blossom],
333
+ # Images with different sizes and aspect-ratios
334
+ [
335
+ rescale_image_size(stop_sign, 0.1),
336
+ stop_sign,
337
+ ],
338
+ [
339
+ stop_sign,
340
+ rescale_image_size(stop_sign, 0.25),
341
+ cherry_blossom.resize((183, 488)),
342
+ cherry_blossom.resize((488, 183))
343
+ ],
344
+ cherry_blossom,
345
+ ])]
346
+
347
+ run_image_test(
348
+ hf_runner,
349
+ vllm_runner,
350
+ inputs,
351
+ model,
352
+ dtype=dtype,
353
+ max_tokens=max_tokens,
354
+ num_logprobs=num_logprobs,
355
+ tensor_parallel_size=1,
356
+ )
tests/models/test_registry.py
+2
-1
@@ -6,7 +6,8 @@ from vllm.model_executor.models import _MODELS, ModelRegistry
6
7
@pytest.mark.parametrize("model_cls", _MODELS)
8
def test_registry_imports(model_cls):
9
- if (model_cls == "Qwen2VLForConditionalGeneration"
9
+ if (model_cls in ("LlavaOnevisionForConditionalGeneration",
10
+ "Qwen2VLForConditionalGeneration")
11
and transformers.__version__ < "4.45"):
12
pytest.skip("Waiting for next transformers release")
13
vllm/assets/video.py
+1
-1
@@ -79,7 +79,7 @@ class VideoAsset:
79
return ret
80
81
@property
82
- def np_ndarrays(self) -> List[npt.NDArray]:
82
+ def np_ndarrays(self) -> npt.NDArray:
83
video_path = download_video_asset(self.name)
84
ret = video_to_ndarrays(video_path, self.num_frames)
85
return ret
vllm/model_executor/models/__init__.py
+4
-2
@@ -83,12 +83,14 @@ _MULTIMODAL_MODELS = {
83
("chameleon", "ChameleonForConditionalGeneration"),
84
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
85
"InternVLChatModel": ("internvl", "InternVLChatModel"),
86
- "LlavaForConditionalGeneration":
87
- ("llava", "LlavaForConditionalGeneration"),
86
+ "LlavaForConditionalGeneration": ("llava",
87
+ "LlavaForConditionalGeneration"),
88
"LlavaNextForConditionalGeneration": ("llava_next",
89
"LlavaNextForConditionalGeneration"),
90
"LlavaNextVideoForConditionalGeneration":
91
("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
92
+ "LlavaOnevisionForConditionalGeneration":
93
+ ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
94
"MiniCPMV": ("minicpmv", "MiniCPMV"),
95
"PaliGemmaForConditionalGeneration": ("paligemma",
96
"PaliGemmaForConditionalGeneration"),
vllm/model_executor/models/clip.py
+19
@@ -2,6 +2,7 @@
2
within a vision language model."""
3
from typing import Iterable, List, Optional, Tuple, Union
4
5
+import numpy as np
6
import torch
7
import torch.nn as nn
8
from PIL import Image
@@ -84,6 +85,24 @@ def dummy_image_for_clip(
85
return {"image": image if num_images == 1 else [image] * num_images}
86
87
88
+def dummy_video_for_clip(
89
+ hf_config: CLIPVisionConfig,
90
+ num_frames: int,
91
+ *,
92
+ image_width_override: Optional[int] = None,
93
+ image_height_override: Optional[int] = None,
94
+):
95
+ pil_frame = dummy_image_for_clip(
96
+ hf_config,
97
+ num_images=1,
98
+ image_width_override=image_width_override,
99
+ image_height_override=image_height_override)
100
+ np_frame = np.array(pil_frame["image"])
101
+ mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
102
+ mm_data = {"video": mm_data_per_video}
103
+ return mm_data
104
+
105
+
106
def input_processor_for_clip(
107
model_config: ModelConfig,
108
hf_config: CLIPVisionConfig,
vllm/model_executor/models/llava_onevision.py
new
+876
@@ -0,0 +1,876 @@
1
+import math
2
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
+ TypedDict, Union)
4
+
5
+import numpy as np
6
+import torch
7
+import torch.nn as nn
8
+from PIL import Image
9
+from transformers import (CLIPVisionConfig, LlavaOnevisionConfig,
10
+ SiglipVisionConfig)
11
+from transformers.models.llava_onevision.modeling_llava_onevision import (
12
+ get_anyres_image_grid_shape, unpad_image)
13
+from typing_extensions import NotRequired
14
+
15
+from vllm.attention import AttentionMetadata
16
+from vllm.config import CacheConfig, MultiModalConfig
17
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
18
+from vllm.logger import init_logger
19
+from vllm.model_executor.layers.activation import get_act_fn
20
+from vllm.model_executor.layers.quantization.base_config import (
21
+ QuantizationConfig)
22
+from vllm.model_executor.layers.sampler import SamplerOutput
23
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
+from vllm.model_executor.sampling_metadata import SamplingMetadata
25
+from vllm.multimodal import MULTIMODAL_REGISTRY
26
+from vllm.multimodal.utils import (cached_get_tokenizer,
27
+ repeat_and_pad_placeholder_tokens)
28
+from vllm.sequence import IntermediateTensors
29
+from vllm.utils import is_list_of
30
+
31
+from .clip import (CLIPVisionModel, dummy_seq_data_for_clip,
32
+ dummy_video_for_clip, get_clip_image_feature_size,
33
+ get_clip_patch_grid_length, input_processor_for_clip)
34
+from .interfaces import SupportsMultiModal
35
+from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
36
+ dummy_video_for_siglip, get_siglip_image_feature_size,
37
+ get_siglip_patch_grid_length, input_processor_for_siglip)
38
+from .utils import (flatten_bn, group_weights_with_prefix,
39
+ init_vllm_registered_model, merge_multimodal_embeddings)
40
+
41
+logger = init_logger(__name__)
42
+
43
+# Result in the max possible feature size (2x2 grid of 336x336px tiles)
44
+MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
45
+
46
+# For profile run
47
+_MAX_FRAMES_PER_VIDEO = 16
48
+_MAX_NUM_VIDEOS = 1
49
+
50
+
51
+class LlavaOnevisionVideoPixelInputs(TypedDict):
52
+ type: Literal["pixel_values_videos"]
53
+ data: Union[torch.Tensor, List[torch.Tensor]]
54
+ """
55
+ Shape: `(batch_size, num_frames, num_channels, height, width)`
56
+
57
+ Note that `num_frames` may be different for each batch, in which case
58
+ the data is passed as a list instead of a batched tensor.
59
+
60
+ Note that it only supports one video input for one batch.
61
+ """
62
+
63
+
64
+class LlavaOnevisionImagePixelInputs(TypedDict):
65
+ type: Literal["pixel_values"]
66
+ data: Union[torch.Tensor, List[torch.Tensor]]
67
+ """
68
+ Shape:
69
+ `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
70
+
71
+ Note that `num_patches` may be different per batch and image,
72
+ in which case the data is passed as a list instead of a batched tensor.
73
+ """
74
+
75
+ image_sizes: NotRequired[torch.Tensor]
76
+ """
77
+ Shape: `(batch_size * num_images, 2)`
78
+
79
+ This should be in `(height, width)` format.
80
+ """
81
+
82
+
83
+class LlavaOnevisionImageEmbeddingInputs(TypedDict):
84
+ type: Literal["image_embeds"]
85
+ data: torch.Tensor
86
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
87
+
88
+ `hidden_size` must match the hidden size of language model backbone.
89
+ """
90
+
91
+
92
+LlavaOnevisionImageInputs = Union[LlavaOnevisionImagePixelInputs,
93
+ LlavaOnevisionImageEmbeddingInputs]
94
+
95
+LlavaOnevisionMultiInputs = Union[LlavaOnevisionImageInputs,
96
+ LlavaOnevisionVideoPixelInputs]
97
+
98
+
99
+def _get_llava_onevision_image_unppaded_feature_size(height, width, patches,
100
+ scale_height,
101
+ scale_width):
102
+ current_height = patches * scale_height
103
+ current_width = patches * scale_width
104
+
105
+ original_aspect_ratio = width / height
106
+ current_aspect_ratio = current_width / current_height
107
+ if original_aspect_ratio > current_aspect_ratio:
108
+ new_height = int(height * (current_width / width))
109
+ padding = (current_height - new_height) // 2
110
+ current_height -= padding * 2
111
+ else:
112
+ new_width = int(width * (current_height / height))
113
+ padding = (current_width - new_width) // 2
114
+ current_width -= padding * 2
115
+
116
+ unpadded_features = current_height * current_width
117
+ newline_features = current_height
118
+
119
+ ratio = math.sqrt(current_height * current_width / (9 * patches**2))
120
+ if ratio > 1.1:
121
+ unpadded_features = int(current_height // ratio) * int(
122
+ current_width // ratio)
123
+ newline_features = int(current_height // ratio)
124
+
125
+ return (unpadded_features, newline_features)
126
+
127
+
128
+def get_llava_onevision_image_feature_size(
129
+ hf_config: LlavaOnevisionConfig,
130
+ *,
131
+ input_height: int,
132
+ input_width: int,
133
+) -> int:
134
+ vision_config = hf_config.vision_config
135
+
136
+ if isinstance(vision_config, CLIPVisionConfig):
137
+ num_patches = get_clip_patch_grid_length(
138
+ image_size=vision_config.image_size,
139
+ patch_size=vision_config.patch_size,
140
+ )
141
+ base_feature_size = get_clip_image_feature_size(vision_config)
142
+ elif isinstance(vision_config, SiglipVisionConfig):
143
+ num_patches = get_siglip_patch_grid_length(
144
+ image_size=vision_config.image_size,
145
+ patch_size=vision_config.patch_size,
146
+ )
147
+ base_feature_size = get_siglip_image_feature_size(vision_config)
148
+ else:
149
+ msg = f"Unsupported vision config: {type(vision_config)}"
150
+ raise NotImplementedError(msg)
151
+
152
+ strategy = hf_config.vision_feature_select_strategy
153
+ if strategy == "default":
154
+ base_feature_size -= 1
155
+ elif strategy == "full":
156
+ pass
157
+ else:
158
+ raise ValueError(f"Unexpected select feature strategy: {strategy}")
159
+
160
+ num_patch_height, num_patch_width = get_anyres_image_grid_shape(
161
+ image_size=(input_height, input_width),
162
+ grid_pinpoints=hf_config.image_grid_pinpoints,
163
+ patch_size=vision_config.image_size,
164
+ )
165
+
166
+ (
167
+ unpadded_feature_size,
168
+ newline_feature_size,
169
+ ) = _get_llava_onevision_image_unppaded_feature_size(
170
+ input_height, input_width, num_patches, num_patch_height,
171
+ num_patch_width)
172
+
173
+ return unpadded_feature_size + newline_feature_size + base_feature_size
174
+
175
+
176
+def get_max_llava_onevision_image_tokens(ctx: InputContext):
177
+ return get_llava_onevision_image_feature_size(
178
+ ctx.get_hf_config(LlavaOnevisionConfig),
179
+ input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
180
+ input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
181
+ )
182
+
183
+
184
+def get_llava_onevision_video_frame_feature_size(
185
+ hf_config: LlavaOnevisionConfig) -> int:
186
+ # Support both CLIPVisionConfig and SiglipVisionConfig
187
+ image_size = hf_config.vision_config.image_size
188
+ patch_size = hf_config.vision_config.patch_size
189
+ spatial_pool_stride = hf_config.spatial_pool_stride if hasattr(
190
+ hf_config, "spatial_pool_stride") else 2
191
+
192
+ height = width = image_size // patch_size
193
+ return math.ceil(height / spatial_pool_stride) * math.ceil(
194
+ width / spatial_pool_stride)
195
+
196
+
197
+def get_llava_onevision_video_tokens(ctx: InputContext,
198
+ num_frames: int) -> int:
199
+ hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
200
+
201
+ # TODO: support configuring (not supported by HF right now)
202
+ num_token_image_newline = 1
203
+ tokens_per_frame = get_llava_onevision_video_frame_feature_size(hf_config)
204
+ video_feature_size = num_frames * tokens_per_frame + num_token_image_newline
205
+
206
+ return video_feature_size
207
+
208
+
209
+def get_max_llava_onevision_video_tokens(ctx: InputContext) -> int:
210
+ return get_llava_onevision_video_tokens(ctx, _MAX_FRAMES_PER_VIDEO)
211
+
212
+
213
+def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
214
+ mm_counts: Mapping[str, int]):
215
+ hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
216
+ vision_config = hf_config.vision_config
217
+
218
+ # TODO: support multiple videos
219
+ num_videos = mm_counts["video"]
220
+ if num_videos > _MAX_NUM_VIDEOS:
221
+ raise NotImplementedError(
222
+ f"Only {_MAX_NUM_VIDEOS} videos are supported")
223
+
224
+ # TODO: support configuring the number of frames
225
+ num_frames = _MAX_FRAMES_PER_VIDEO
226
+ video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
227
+
228
+ if isinstance(vision_config, CLIPVisionConfig):
229
+ seq_data = dummy_seq_data_for_clip(
230
+ vision_config,
231
+ seq_len,
232
+ num_videos,
233
+ image_token_id=hf_config.video_token_index,
234
+ image_feature_size_override=video_feature_size,
235
+ )
236
+
237
+ mm_data = dummy_video_for_clip(vision_config, num_frames=num_frames)
238
+ return seq_data, mm_data
239
+ elif isinstance(vision_config, SiglipVisionConfig):
240
+ seq_data = dummy_seq_data_for_siglip(
241
+ vision_config,
242
+ seq_len,
243
+ num_videos,
244
+ image_token_id=hf_config.video_token_index,
245
+ image_feature_size_override=video_feature_size,
246
+ )
247
+
248
+ mm_data = dummy_video_for_siglip(vision_config, num_frames=num_frames)
249
+ return seq_data, mm_data
250
+
251
+ msg = f"Unsupported vision config: {type(vision_config)}"
252
+ raise NotImplementedError(msg)
253
+
254
+
255
+def input_processor_when_multimodal_input_image(ctx: InputContext,
256
+ llm_inputs: LLMInputs):
257
+ multi_modal_data = llm_inputs.get("multi_modal_data")
258
+ if multi_modal_data is None or "image" not in multi_modal_data:
259
+ return llm_inputs
260
+
261
+ model_config = ctx.model_config
262
+ hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
263
+ vision_config = hf_config.vision_config
264
+
265
+ image_data = multi_modal_data["image"]
266
+ if isinstance(image_data, Image.Image):
267
+ width, height = image_data.size
268
+
269
+ image_feature_size = get_llava_onevision_image_feature_size(
270
+ hf_config,
271
+ input_height=height,
272
+ input_width=width,
273
+ )
274
+ elif is_list_of(image_data, Image.Image):
275
+ image_feature_size = [
276
+ get_llava_onevision_image_feature_size(hf_config,
277
+ input_height=img.height,
278
+ input_width=img.width)
279
+ for img in image_data
280
+ ]
281
+ elif isinstance(image_data, torch.Tensor):
282
+ num_images, image_feature_size, hidden_size = image_data.shape
283
+ elif is_list_of(image_data, torch.Tensor):
284
+ image_feature_size = [item.shape[1] for item in image_data]
285
+ else:
286
+ raise TypeError(f"Invalid image type: {type(image_data)}")
287
+
288
+ vision_config = hf_config.vision_config
289
+
290
+ if isinstance(vision_config, CLIPVisionConfig):
291
+ return input_processor_for_clip(
292
+ model_config,
293
+ vision_config,
294
+ llm_inputs,
295
+ image_token_id=hf_config.image_token_index,
296
+ image_feature_size_override=image_feature_size,
297
+ )
298
+ elif isinstance(vision_config, SiglipVisionConfig):
299
+ return input_processor_for_siglip(
300
+ model_config,
301
+ vision_config,
302
+ llm_inputs,
303
+ image_token_id=hf_config.image_token_index,
304
+ image_feature_size_override=image_feature_size,
305
+ )
306
+
307
+ msg = f"Unsupported vision config: {type(vision_config)}"
308
+ raise NotImplementedError(msg)
309
+
310
+
311
+def input_processor_when_multimodal_input_video(ctx: InputContext,
312
+ llm_inputs: LLMInputs):
313
+ multi_modal_data = llm_inputs.get("multi_modal_data")
314
+ if multi_modal_data is None or "video" not in multi_modal_data:
315
+ return llm_inputs
316
+ video_data = multi_modal_data["video"]
317
+
318
+ model_config = ctx.model_config
319
+ hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
320
+ vision_config = hf_config.vision_config
321
+
322
+ if isinstance(video_data, np.ndarray):
323
+ # Supports both CLIP and Siglip
324
+ num_frames = video_data.shape[0]
325
+ video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
326
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
327
+
328
+ new_prompt, new_token_ids = repeat_and_pad_placeholder_tokens(
329
+ tokenizer,
330
+ llm_inputs.get("prompt"),
331
+ llm_inputs["prompt_token_ids"],
332
+ placeholder_token_id=hf_config.video_token_index,
333
+ repeat_count=video_feature_size,
334
+ )
335
+
336
+ return LLMInputs(prompt_token_ids=new_token_ids,
337
+ prompt=new_prompt,
338
+ multi_modal_data=multi_modal_data)
339
+
340
+ elif is_list_of(video_data, np.ndarray):
341
+ raise NotImplementedError(
342
+ "Processing multiple videos is not supported")
343
+
344
+ msg = f"Unsupported vision config: {type(vision_config)}"
345
+ raise NotImplementedError(msg)
346
+
347
+
348
+def input_processor_for_llava_onevision(ctx: InputContext,
349
+ llm_inputs: LLMInputs):
350
+ multi_modal_data = llm_inputs.get("multi_modal_data")
351
+ if multi_modal_data is None or ("video" not in multi_modal_data
352
+ and "image" not in multi_modal_data):
353
+ return llm_inputs
354
+ if "image" in multi_modal_data:
355
+ return input_processor_when_multimodal_input_image(ctx, llm_inputs)
356
+ if "video" in multi_modal_data:
357
+ return input_processor_when_multimodal_input_video(ctx, llm_inputs)
358
+
359
+ msg = "Unsupported multi data type"
360
+ raise NotImplementedError(msg)
361
+
362
+
363
+def _init_vision_tower(hf_config: LlavaOnevisionConfig):
364
+ vision_config = hf_config.vision_config
365
+
366
+ # Initialize the vision tower only up to the required feature layer
367
+ vision_feature_layer = hf_config.vision_feature_layer
368
+ if vision_feature_layer < 0:
369
+ num_hidden_layers = hf_config.vision_config.num_hidden_layers \
370
+ + vision_feature_layer + 1
371
+ else:
372
+ num_hidden_layers = vision_feature_layer + 1
373
+
374
+ if isinstance(vision_config, CLIPVisionConfig):
375
+ return CLIPVisionModel(
376
+ vision_config,
377
+ num_hidden_layers_override=num_hidden_layers,
378
+ )
379
+ elif isinstance(vision_config, SiglipVisionConfig):
380
+ return SiglipVisionModel(
381
+ vision_config,
382
+ num_hidden_layers_override=num_hidden_layers,
383
+ )
384
+
385
+ msg = f"Unsupported vision config: {type(vision_config)}"
386
+ raise NotImplementedError(msg)
387
+
388
+
389
+class LlavaOnevisionMultiModalProjector(nn.Module):
390
+
391
+ def __init__(self, config: LlavaOnevisionConfig):
392
+ super().__init__()
393
+
394
+ self.linear_1 = nn.Linear(config.vision_config.hidden_size,
395
+ config.text_config.hidden_size,
396
+ bias=True)
397
+ self.act = get_act_fn(config.projector_hidden_act)
398
+ self.linear_2 = nn.Linear(config.text_config.hidden_size,
399
+ config.text_config.hidden_size,
400
+ bias=True)
401
+
402
+ def forward(self, image_features: torch.Tensor) -> torch.Tensor:
403
+ hidden_states = self.linear_1(image_features)
404
+ hidden_states = self.act(hidden_states)
405
+ hidden_states = self.linear_2(hidden_states)
406
+ return hidden_states
407
+
408
+
409
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
410
+@MULTIMODAL_REGISTRY.register_input_mapper("video")
411
+@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
412
+ "image", get_max_llava_onevision_image_tokens)
413
+@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
414
+ "video", get_max_llava_onevision_video_tokens)
415
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_onevision)
416
+@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_onevision)
417
+class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal):
418
+
419
+ def __init__(self,
420
+ config: LlavaOnevisionConfig,
421
+ multimodal_config: MultiModalConfig,
422
+ cache_config: Optional[CacheConfig] = None,
423
+ quant_config: Optional[QuantizationConfig] = None) -> None:
424
+ super().__init__()
425
+
426
+ self.config = config
427
+ self.multimodal_config = multimodal_config
428
+
429
+ # Initialize the vision tower only up to the required feature layer
430
+ self.vision_tower = _init_vision_tower(config)
431
+ self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
432
+ self.language_model = init_vllm_registered_model(
433
+ config.text_config, cache_config, quant_config)
434
+ self.image_newline = nn.Parameter(
435
+ torch.empty(config.text_config.hidden_size))
436
+
437
+ def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
438
+ expected_dims = (2, )
439
+
440
+ def _validate_shape(d: torch.Tensor):
441
+ actual_dims = tuple(d.shape)
442
+
443
+ if actual_dims != expected_dims:
444
+ expected_expr = str(expected_dims)
445
+ raise ValueError(
446
+ f"The expected shape of image sizes per image per batch "
447
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
448
+
449
+ for d in data:
450
+ _validate_shape(d)
451
+
452
+ return data
453
+
454
+ def _validate_image_pixel_values(
455
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
456
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
457
+
458
+ h = w = self.config.vision_config.image_size
459
+ expected_dims = (3, h, w)
460
+
461
+ def _validate_shape(d: torch.Tensor):
462
+ actual_dims = tuple(d.shape[1:])
463
+
464
+ if actual_dims != expected_dims:
465
+ expected_expr = ("num_patches", *map(str, expected_dims))
466
+ raise ValueError(
467
+ "The expected shape of pixel values per image per batch "
468
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
469
+
470
+ for d in data:
471
+ _validate_shape(d)
472
+
473
+ return data
474
+
475
+ def _parse_and_validate_image_input(
476
+ self, **kwargs: object) -> Optional[LlavaOnevisionImageInputs]:
477
+ pixel_values = kwargs.pop("pixel_values", None)
478
+ image_sizes = kwargs.pop("image_sizes", None)
479
+ image_embeds = kwargs.pop("image_embeds", None)
480
+
481
+ if pixel_values is None and image_embeds is None:
482
+ return None
483
+
484
+ if pixel_values is not None:
485
+ if not isinstance(pixel_values, (torch.Tensor, list)):
486
+ raise ValueError("Incorrect type of pixel values. "
487
+ f"Got type: {type(pixel_values)}")
488
+
489
+ if not isinstance(image_sizes, (torch.Tensor, list)):
490
+ raise ValueError("Incorrect type of image sizes. "
491
+ f"Got type: {type(image_sizes)}")
492
+
493
+ return LlavaOnevisionImagePixelInputs(
494
+ type="pixel_values",
495
+ data=self._validate_image_pixel_values(
496
+ flatten_bn(pixel_values)),
497
+ image_sizes=self._validate_image_sizes(
498
+ flatten_bn(image_sizes, concat=True)),
499
+ )
500
+
501
+ if image_embeds is not None:
502
+ if not isinstance(image_embeds, torch.Tensor):
503
+ raise ValueError("Incorrect type of image embeds. "
504
+ f"Got type: {type(image_embeds)}")
505
+
506
+ return LlavaOnevisionImageEmbeddingInputs(
507
+ type="image_embeds",
508
+ data=flatten_bn(image_embeds),
509
+ )
510
+
511
+ raise AssertionError("This line should be unreachable.")
512
+
513
+ def _validate_video_pixel_values(
514
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
515
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
516
+
517
+ h = w = self.config.vision_config.image_size
518
+ expected_dims = (3, h, w)
519
+
520
+ def _validate_shape(d: torch.Tensor):
521
+ actual_dims = tuple(d.shape[2:])
522
+
523
+ if actual_dims != expected_dims:
524
+ expected_expr = ("num_frames", *map(str, expected_dims))
525
+ raise ValueError(
526
+ "The expected shape of pixel values in each video frame "
527
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
528
+
529
+ for d in data:
530
+ _validate_shape(d)
531
+
532
+ return data
533
+
534
+ def _parse_and_validate_video_input(
535
+ self,
536
+ **kwargs: object) -> Optional[LlavaOnevisionVideoPixelInputs]:
537
+ """
538
+ A legal video input should have the following dimensions:
539
+ {
540
+ "pixel_values_videos" :
541
+ List[b, Tensor(nb_frames, nb_channels, height, width)]
542
+ }
543
+ """
544
+ pixel_values = kwargs.pop("pixel_values_videos", None)
545
+
546
+ if pixel_values is None:
547
+ return None
548
+
549
+ if not (is_list_of(pixel_values,
550
+ (torch.Tensor)) # different shape videos
551
+ or isinstance(pixel_values,
552
+ torch.Tensor)): # same shape videos
553
+ raise ValueError("Incorrect type of pixel values. "
554
+ f"Got type: {type(pixel_values)}")
555
+
556
+ return LlavaOnevisionVideoPixelInputs(
557
+ type="pixel_values_videos",
558
+ data=pixel_values,
559
+ )
560
+
561
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
562
+ modalities = {}
563
+
564
+ if "pixel_values" in kwargs:
565
+ modalities["images"] = self._parse_and_validate_image_input(
566
+ **kwargs)
567
+
568
+ if "pixel_values_videos" in kwargs:
569
+ modalities["videos"] = self._parse_and_validate_video_input(
570
+ **kwargs)
571
+
572
+ return modalities
573
+
574
+ def _select_image_features(self, image_features: torch.Tensor, *,
575
+ strategy: str) -> torch.Tensor:
576
+ if strategy == "default":
577
+ return image_features[:, 1:]
578
+ elif strategy == "full":
579
+ return image_features
580
+
581
+ raise ValueError(f"Unexpected select feature strategy: {strategy}")
582
+
583
+ def _image_pixels_to_features(
584
+ self,
585
+ vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
586
+ pixel_values: torch.Tensor,
587
+ ) -> torch.Tensor:
588
+
589
+ # NOTE: we skip the step to select the vision feature layer since
590
+ # this is already done inside the vision tower
591
+ image_features = vision_tower(pixel_values)
592
+ return self._select_image_features(
593
+ image_features,
594
+ strategy=self.config.vision_feature_select_strategy,
595
+ )
596
+
597
+ # Based on: https://github.com/haotian-liu/LLaVA/blob/main/llava/model/llava_arch.py
598
+ def _merge_image_patch_embeddings(self,
599
+ image_size: torch.Tensor,
600
+ patch_embeddings: torch.Tensor,
601
+ *,
602
+ image_newline=None,
603
+ vision_aspect_ratio="anyres_max_9",
604
+ strategy: str) -> torch.Tensor:
605
+ if strategy == "flat":
606
+ return patch_embeddings.flatten(0, 1)
607
+
608
+ if strategy.startswith("spatial"):
609
+ height = width = self.config.vision_config.image_size \
610
+ // self.config.vision_config.patch_size
611
+
612
+ base_patch_embeds = patch_embeddings[0]
613
+ if height * width != base_patch_embeds.shape[0]:
614
+ raise ValueError(
615
+ "The number of patches is not consistent with the "
616
+ "image size.")
617
+
618
+ if patch_embeddings.shape[0] > 1:
619
+ other_patch_embeds = patch_embeddings[1:]
620
+
621
+ # Move to CPU to avoid floating-point errors
622
+ orig_height, orig_width = image_size.tolist()
623
+
624
+ # image_aspect_ratio == "anyres"
625
+ num_patch_height, num_patch_width = get_anyres_image_grid_shape(
626
+ (orig_height, orig_width),
627
+ self.config.image_grid_pinpoints,
628
+ self.config.vision_config.image_size,
629
+ )
630
+ num_patches = num_patch_height * num_patch_width
631
+
632
+ # Image patches might be padded for batch processing
633
+ other_patch_embeds = other_patch_embeds[:num_patches] \
634
+ .view(num_patch_height, num_patch_width, height, width, -1)
635
+
636
+ if "unpad" in strategy:
637
+ other_patch_embeds = other_patch_embeds \
638
+ .permute(4, 0, 2, 1, 3).contiguous() \
639
+ .flatten(1, 2).flatten(2, 3)
640
+ other_patch_embeds = unpad_image(other_patch_embeds,
641
+ (orig_height, orig_width))
642
+ max_num_patches = int(
643
+ vision_aspect_ratio.removeprefix("anyres_max_"))
644
+ channels, curr_height, curr_width = other_patch_embeds.shape
645
+ ratio = math.sqrt(curr_height * curr_width /
646
+ (max_num_patches * height**2))
647
+ if ratio > 1.1:
648
+ other_patch_embeds = other_patch_embeds[None]
649
+ other_patch_embeds = nn.functional.interpolate(
650
+ other_patch_embeds, [
651
+ int(curr_height // ratio),
652
+ int(curr_width // ratio)
653
+ ],
654
+ mode="bilinear")[0]
655
+ if image_newline is not None:
656
+ other_patch_embeds = torch.cat(
657
+ (
658
+ other_patch_embeds,
659
+ image_newline[:, None, None] \
660
+ .expand(*other_patch_embeds.shape[:-1], 1) \
661
+ .to(other_patch_embeds.device),
662
+ ),
663
+ dim=-1)
664
+ other_patch_embeds = other_patch_embeds \
665
+ .flatten(1, 2).transpose(0, 1)
666
+ else:
667
+ other_patch_embeds = other_patch_embeds \
668
+ .permute(0, 2, 1, 3, 4).contiguous() \
669
+ .flatten(0, 3)
670
+
671
+ merged_patch_embeddings = torch.cat(
672
+ (base_patch_embeds, other_patch_embeds), dim=0)
673
+ else:
674
+ if "unpad" in strategy:
675
+ merged_patch_embeddings = torch.cat(
676
+ (base_patch_embeds,
677
+ self.image_newline[None] \
678
+ .to(base_patch_embeds.device)
679
+ ), dim=0)
680
+ else:
681
+ merged_patch_embeddings = base_patch_embeds
682
+
683
+ return merged_patch_embeddings
684
+
685
+ raise ValueError(f"Unexpected patch merge strategy: {strategy}")
686
+
687
+ def _process_image_pixels(
688
+ self,
689
+ inputs: LlavaOnevisionImagePixelInputs,
690
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
691
+ assert self.vision_tower is not None
692
+
693
+ pixel_values = inputs["data"]
694
+
695
+ if isinstance(pixel_values, torch.Tensor):
696
+ b, num_patches, c, h, w = pixel_values.shape
697
+ stacked_pixel_values = pixel_values.view(b * num_patches, c, h, w)
698
+ stacked_image_features = self._image_pixels_to_features(
699
+ self.vision_tower, stacked_pixel_values)
700
+ stacked_patch_embeddings = self.multi_modal_projector(
701
+ stacked_image_features)
702
+
703
+ return stacked_patch_embeddings.view(
704
+ b, num_patches, *stacked_patch_embeddings.shape[1:])
705
+
706
+ num_patches_per_batch = [v.shape[0] for v in pixel_values]
707
+ stacked_pixel_values = torch.cat(pixel_values)
708
+ stacked_image_features = self._image_pixels_to_features(
709
+ self.vision_tower, stacked_pixel_values)
710
+
711
+ return [
712
+ self.multi_modal_projector(image_features) for image_features in
713
+ torch.split(stacked_image_features, num_patches_per_batch)
714
+ ]
715
+
716
+ def _process_image_input(
717
+ self,
718
+ image_input: LlavaOnevisionImageInputs,
719
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
720
+ if image_input["type"] == "image_embeds":
721
+ return [image_input["data"]]
722
+
723
+ patch_embeddings = self._process_image_pixels(image_input)
724
+
725
+ image_sizes = image_input.get("image_sizes")
726
+ if image_sizes is None:
727
+ batch_size = len(image_input["data"])
728
+ vision_config = self.config.vision_config
729
+ default_height = default_width = vision_config.image_size
730
+ image_sizes = torch.as_tensor([[default_height, default_width]
731
+ for _ in range(batch_size)])
732
+
733
+ return [
734
+ self._merge_image_patch_embeddings(
735
+ image_sizes[i],
736
+ patch_features_batch,
737
+ image_newline=self.image_newline,
738
+ strategy="spatial_unpad")
739
+ for i, patch_features_batch in enumerate(patch_embeddings)
740
+ ]
741
+
742
+ def _video_pixels_to_features(
743
+ self,
744
+ vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
745
+ pixel_values: torch.Tensor,
746
+ ) -> torch.Tensor:
747
+
748
+ # NOTE: we skip the step to select the vision feature layer since
749
+ # this is already done inside the vision tower
750
+ b, num_videos, frames, c, h, w = pixel_values.shape
751
+ assert (num_videos == _MAX_NUM_VIDEOS)
752
+ pixel_values = pixel_values.reshape(b * num_videos * frames, c, h, w)
753
+ video_features = vision_tower(pixel_values)
754
+ video_features = self._select_image_features(
755
+ video_features,
756
+ strategy=self.config.vision_feature_select_strategy,
757
+ )
758
+ video_features = self.multi_modal_projector(video_features)
759
+ video_features = self.apply_pooling(video_features)
760
+ video_features = video_features.reshape(
761
+ b, frames * video_features.shape[1], -1)
762
+ image_newline = self.image_newline[None, None, :].repeat(b, 1, 1).to(
763
+ video_features.device)
764
+ video_features = torch.cat((video_features, image_newline), dim=1)
765
+ video_features = video_features.flatten(0, 1)
766
+
767
+ return video_features
768
+
769
+ def _process_video_pixels(self, inputs: LlavaOnevisionVideoPixelInputs):
770
+ assert self.vision_tower is not None
771
+
772
+ video_pixels = inputs["data"]
773
+
774
+ # TODO: support multiple videos per input
775
+ if isinstance(video_pixels, torch.Tensor):
776
+ stacked_embeddings = self._video_pixels_to_features(
777
+ self.vision_tower, video_pixels)
778
+ return stacked_embeddings
779
+ else:
780
+ raise ValueError(
781
+ f"Unsupported type of video input {type(video_pixels)}")
782
+
783
+ def apply_pooling(self, image_features, stride=2):
784
+ vision_config = self.config.vision_config
785
+ height = width = vision_config.image_size // vision_config.patch_size
786
+ batch_frames, _, dim = image_features.shape
787
+ image_features = image_features.view(batch_frames, height, width, -1)
788
+ image_features = image_features.permute(0, 3, 1, 2)
789
+
790
+ # TODO support other pooling types config
791
+ height, width = image_features.shape[2:]
792
+ scaled_shape = [math.ceil(height / stride), math.ceil(width / stride)]
793
+ image_feature = nn.functional.interpolate(image_features,
794
+ size=scaled_shape,
795
+ mode='bilinear')
796
+ image_feature = image_feature.permute(0, 2, 3, 1)
797
+ image_feature = image_feature.view(batch_frames, -1, dim)
798
+ return image_feature
799
+
800
+ def forward(
801
+ self,
802
+ input_ids: torch.Tensor,
803
+ positions: torch.Tensor,
804
+ kv_caches: List[torch.Tensor],
805
+ attn_metadata: AttentionMetadata,
806
+ intermediate_tensors: Optional[IntermediateTensors] = None,
807
+ **kwargs: object,
808
+ ) -> SamplerOutput:
809
+ """Run forward pass for LlaVA-Onevision.
810
+ Args:
811
+ input_ids: Flattened (concatenated) input_ids corresponding to a
812
+ batch.
813
+ pixel_values_videos: Pixels in each frames for each input videos.
814
+ """
815
+ modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
816
+ # merge video embeddings into input embeddings
817
+ if modalities:
818
+ inputs_embeds = self.language_model.model.get_input_embeddings(
819
+ input_ids)
820
+ if "images" in modalities:
821
+ image_input = modalities["images"]
822
+ vision_embeddings = self._process_image_input(image_input)
823
+ inputs_embeds = merge_multimodal_embeddings(
824
+ input_ids, inputs_embeds, vision_embeddings,
825
+ self.config.image_token_index)
826
+ if "videos" in modalities:
827
+ video_input = modalities["videos"]
828
+ video_embeddings = self._process_video_pixels(video_input)
829
+ inputs_embeds = merge_multimodal_embeddings(
830
+ input_ids, inputs_embeds, video_embeddings,
831
+ self.config.video_token_index)
832
+ input_ids = None
833
+ else:
834
+ inputs_embeds = None
835
+
836
+ hidden_states = self.language_model.model(input_ids,
837
+ positions,
838
+ kv_caches,
839
+ attn_metadata,
840
+ None,
841
+ inputs_embeds=inputs_embeds)
842
+
843
+ return hidden_states
844
+
845
+ def compute_logits(
846
+ self,
847
+ hidden_states: torch.Tensor,
848
+ sampling_metadata: SamplingMetadata,
849
+ ) -> Optional[torch.Tensor]:
850
+ return self.language_model.compute_logits(hidden_states,
851
+ sampling_metadata)
852
+
853
+ def sample(
854
+ self,
855
+ logits: torch.Tensor,
856
+ sampling_metadata: SamplingMetadata,
857
+ ) -> Optional[SamplerOutput]:
858
+ return self.language_model.sample(logits, sampling_metadata)
859
+
860
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
861
+ # prepare weight iterators for components
862
+ weights_group = group_weights_with_prefix(weights)
863
+
864
+ # load vision encoder
865
+ self.vision_tower.load_weights(weights_group["vision_tower"])
866
+
867
+ # load mlp projector
868
+ mlp_params_dict = dict(self.multi_modal_projector.named_parameters())
869
+ for name, loaded_weight in weights_group["multi_modal_projector"]:
870
+ param = mlp_params_dict[name]
871
+ weight_loader = getattr(param, "weight_loader",
872
+ default_weight_loader)
873
+ weight_loader(param, loaded_weight)
874
+
875
+ # load llm backbone
876
+ self.language_model.load_weights(weights_group["language_model"])
vllm/model_executor/models/siglip.py
+19
@@ -4,6 +4,7 @@ within a vision language model."""
4
import math
5
from typing import Iterable, List, Optional, Tuple, Union
6
7
+import numpy as np
8
import torch
9
from PIL import Image
10
from torch import nn
@@ -89,6 +90,24 @@ def dummy_image_for_siglip(
90
return {"image": image if num_images == 1 else [image] * num_images}
91
92
93
+def dummy_video_for_siglip(
94
+ hf_config: SiglipVisionConfig,
95
+ num_frames: int,
96
+ *,
97
+ image_width_override: Optional[int] = None,
98
+ image_height_override: Optional[int] = None,
99
+):
100
+ pil_frame = dummy_image_for_siglip(
101
+ hf_config,
102
+ num_images=1,
103
+ image_width_override=image_width_override,
104
+ image_height_override=image_height_override)
105
+ np_frame = np.array(pil_frame["image"])
106
+ mm_data_per_video = np.repeat([np_frame], num_frames, axis=0)
107
+ mm_data = {"video": mm_data_per_video}
108
+ return mm_data
109
+
110
+
111
def input_processor_for_siglip(
112
model_config: ModelConfig,
113
hf_config: SiglipVisionConfig,