A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Model] Add support for the multi-modal Llama 3.2 model (#8811)
Co-authored-by: simon-mo <xmo@berkeley.edu> Co-authored-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: Simon Mo <simon.mo@hey.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Chen Zhang committed
Sep 25, 2024 at 13:29 UTC
770ec6024fc00cd696899f5c6fdc53b7148876e6
24 files changed
+1647
-45
docs/source/models/supported_models.rst
+5
@@ -254,6 +254,11 @@ Multimodal Language Models
254
- Image\ :sup:`+`
255
- :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
256
-
257
+ * - :code:`MllamaForConditionalGeneration`
258
+ - Llama 3.2
259
+ - Image
260
+ - :code:`meta-llama/Llama-3.2-90B-Vision-Instruct`, :code:`meta-llama/Llama-3.2-11B-Vision`, etc.
261
+ -
262
* - :code:`PaliGemmaForConditionalGeneration`
263
- PaliGemma
264
- Image\ :sup:`E`
examples/offline_inference_vision_language.py
+24
@@ -242,6 +242,29 @@ def run_qwen2_vl(question, modality):
242
return llm, prompt, stop_token_ids
243
244
245
+# LLama
246
+def run_mllama(question, modality):
247
+ assert modality == "image"
248
+
249
+ model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
250
+
251
+ # Note: The default setting of max_num_seqs (256) and
252
+ # max_model_len (131072) for this model may cause OOM.
253
+ # You may lower either to run this example on lower-end GPUs.
254
+
255
+ # The configuration below has been confirmed to launch on a
256
+ # single H100 GPU.
257
+ llm = LLM(
258
+ model=model_name,
259
+ max_num_seqs=16,
260
+ enforce_eager=True,
261
+ )
262
+
263
+ prompt = f"<|image|><|begin_of_text|>{question}"
264
+ stop_token_ids = None
265
+ return llm, prompt, stop_token_ids
266
+
267
+
268
model_example_map = {
269
"llava": run_llava,
270
"llava-next": run_llava_next,
@@ -256,6 +279,7 @@ model_example_map = {
279
"internvl_chat": run_internvl,
280
"qwen_vl": run_qwen_vl,
281
"qwen2_vl": run_qwen2_vl,
282
+ "mllama": run_mllama,
283
}
284
285
examples/openai_vision_api_client.py
+2
-2
@@ -38,7 +38,7 @@ chat_completion_from_url = client.chat.completions.create(
38
"content": [
39
{
40
"type": "text",
41
- "text": "What’s in this image?"
41
+ "text": "What's in this image?"
42
},
43
{
44
"type": "image_url",
@@ -75,7 +75,7 @@ chat_completion_from_base64 = client.chat.completions.create(
75
"content": [
76
{
77
"type": "text",
78
- "text": "What’s in this image?"
78
+ "text": "What's in this image?"
79
},
80
{
81
"type": "image_url",
requirements-common.txt
+1
-1
@@ -4,7 +4,7 @@ numpy < 2.0.0
4
requests
5
tqdm
6
py-cpuinfo
7
-transformers >= 4.43.2 # Required for Chameleon and Llama 3.1 hotfox.
7
+transformers >= 4.45.0 # Required for Llama 3.2.
8
tokenizers >= 0.19.1 # Required for Llama 3.
9
protobuf # Required by LlamaTokenizer.
10
fastapi < 0.113.0; python_version < '3.9'
tests/models/encoder_decoder/vision_language/__init__.py
tests/models/encoder_decoder/vision_language/test_mllama.py
new
+283
@@ -0,0 +1,283 @@
1
+from typing import List, Optional, Tuple, Type, overload
2
+
3
+import pytest
4
+from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5
+ BatchEncoding)
6
+
7
+from vllm.multimodal.utils import rescale_image_size
8
+from vllm.sequence import SampleLogprobs
9
+
10
+from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11
+ _ImageAssets)
12
+from ....utils import multi_gpu_test
13
+from ...utils import check_logprobs_close
14
+
15
+_LIMIT_IMAGE_PER_PROMPT = 1
16
+
17
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18
+ "stop_sign":
19
+ "<|image|><|begin_of_text|>The meaning of the image is",
20
+ "cherry_blossom":
21
+ "<|image|><|begin_of_text|>The city is",
22
+})
23
+
24
+text_only_prompts = [
25
+ "The color of the sky is blue but sometimes it can also be",
26
+]
27
+
28
+models = [
29
+ "meta-llama/Llama-3.2-11B-Vision-Instruct",
30
+]
31
+
32
+
33
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
34
+ Optional[SampleLogprobs]],
35
+ model: str):
36
+ """Sanitize vllm output to be comparable with hf output."""
37
+ output_ids, output_str, out_logprobs = vllm_output
38
+
39
+ config = AutoConfig.from_pretrained(model)
40
+ image_token_id = config.image_token_index
41
+
42
+ tokenizer = AutoTokenizer.from_pretrained(model)
43
+ eos_token_id = tokenizer.eos_token_id
44
+
45
+ hf_output_ids = [
46
+ token_id for idx, token_id in enumerate(output_ids)
47
+ if token_id != image_token_id or output_ids[idx - 1] != image_token_id
48
+ ]
49
+
50
+ assert output_str[0] == " "
51
+ hf_output_str = output_str[1:]
52
+ if hf_output_ids[-1] == eos_token_id:
53
+ hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
54
+
55
+ return hf_output_ids, hf_output_str, out_logprobs
56
+
57
+
58
+@overload
59
+def run_test(
60
+ hf_runner: Type[HfRunner],
61
+ vllm_runner: Type[VllmRunner],
62
+ image_assets: _ImageAssets,
63
+ model: str,
64
+ *,
65
+ size_factors: List[float],
66
+ dtype: str,
67
+ max_tokens: int,
68
+ num_logprobs: int,
69
+ tensor_parallel_size: int,
70
+ distributed_executor_backend: Optional[str] = None,
71
+):
72
+ ...
73
+
74
+
75
+@overload
76
+def run_test(
77
+ hf_runner: Type[HfRunner],
78
+ vllm_runner: Type[VllmRunner],
79
+ image_assets: _ImageAssets,
80
+ model: str,
81
+ *,
82
+ sizes: List[Tuple[int, int]],
83
+ dtype: str,
84
+ max_tokens: int,
85
+ num_logprobs: int,
86
+ tensor_parallel_size: int,
87
+ distributed_executor_backend: Optional[str] = None,
88
+):
89
+ ...
90
+
91
+
92
+def run_test(
93
+ hf_runner: Type[HfRunner],
94
+ vllm_runner: Type[VllmRunner],
95
+ image_assets: _ImageAssets,
96
+ model: str,
97
+ *,
98
+ size_factors: Optional[List[float]] = None,
99
+ sizes: Optional[List[Tuple[int, int]]] = None,
100
+ dtype: str,
101
+ max_tokens: int,
102
+ num_logprobs: int,
103
+ tensor_parallel_size: int,
104
+ distributed_executor_backend: Optional[str] = None,
105
+):
106
+ images = [asset.pil_image for asset in image_assets]
107
+
108
+ if size_factors is not None:
109
+ inputs_per_image = [(
110
+ [prompt for _ in size_factors],
111
+ [rescale_image_size(image, factor) for factor in size_factors],
112
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
113
+ elif sizes is not None:
114
+ inputs_per_image = [(
115
+ [
116
+ prompt if size is not None else text_only_prompts[0]
117
+ for size in sizes
118
+ ],
119
+ [
120
+ image.resize(size) if size is not None else None
121
+ for size in sizes
122
+ ],
123
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
124
+ if len(sizes) == 0:
125
+ inputs_per_image.append(
126
+ (text_only_prompts, [None] * len(text_only_prompts)))
127
+ else:
128
+ raise ValueError("You must provide either `size_factors` or `sizes`")
129
+
130
+ _run_test(hf_runner,
131
+ vllm_runner,
132
+ inputs_per_image,
133
+ model,
134
+ dtype=dtype,
135
+ max_tokens=max_tokens,
136
+ num_logprobs=num_logprobs,
137
+ tensor_parallel_size=tensor_parallel_size,
138
+ distributed_executor_backend=distributed_executor_backend)
139
+
140
+
141
+def _run_test(
142
+ hf_runner: Type[HfRunner],
143
+ vllm_runner: Type[VllmRunner],
144
+ inputs: List[Tuple[List[str], PromptImageInput]],
145
+ model: str,
146
+ *,
147
+ dtype: str,
148
+ max_tokens: int,
149
+ num_logprobs: int,
150
+ tensor_parallel_size: int,
151
+ distributed_executor_backend: Optional[str] = None,
152
+):
153
+ """Inference result should be the same between hf and vllm.
154
+
155
+ All the image fixtures for the test are from IMAGE_ASSETS.
156
+ For huggingface runner, we provide the PIL images as input.
157
+ For vllm runner, we provide MultiModalDataDict objects
158
+ and corresponding MultiModalConfig as input.
159
+ Note, the text input is also adjusted to abide by vllm contract.
160
+ The text output is sanitized to be able to compare with hf.
161
+ """
162
+ # NOTE: take care of the order. run vLLM first, and then run HF.
163
+ # vLLM needs a fresh new process without cuda initialization.
164
+ # if we run HF first, the cuda initialization will be done and it
165
+ # will hurt multiprocessing backend with fork method (the default method).
166
+
167
+ # max_model_len should be greater than image_feature_size
168
+ with vllm_runner(model,
169
+ dtype=dtype,
170
+ max_num_seqs=16,
171
+ max_model_len=4096,
172
+ tensor_parallel_size=tensor_parallel_size,
173
+ distributed_executor_backend=distributed_executor_backend,
174
+ enforce_eager=True,
175
+ limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
176
+ }) as vllm_model:
177
+ vllm_outputs_per_image = [
178
+ vllm_model.generate_greedy_logprobs(prompts,
179
+ max_tokens,
180
+ num_logprobs=num_logprobs,
181
+ images=images)
182
+ for prompts, images in inputs
183
+ ]
184
+
185
+ def process(hf_inputs: BatchEncoding):
186
+ return hf_inputs
187
+
188
+ from transformers import AutoConfig
189
+ from transformers.models.mllama import MllamaConfig as MllamaConfigHf
190
+
191
+ # use transformer's MllamaConfig for hf_runner
192
+ # and vllm's MllamaConfig for vllm_runner
193
+ AutoConfig.register("mllama", MllamaConfigHf, exist_ok=True)
194
+ with hf_runner(model,
195
+ dtype=dtype,
196
+ postprocess_inputs=process,
197
+ auto_cls=AutoModelForVision2Seq) as hf_model:
198
+ hf_outputs_per_image = [
199
+ hf_model.generate_greedy_logprobs_limit(prompts,
200
+ max_tokens,
201
+ num_logprobs=num_logprobs,
202
+ images=images)
203
+ for prompts, images in inputs
204
+ ]
205
+
206
+ from vllm.transformers_utils.configs.mllama import MllamaConfig
207
+ AutoConfig.register("mllama", MllamaConfig, exist_ok=True)
208
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
209
+ vllm_outputs_per_image):
210
+ check_logprobs_close(
211
+ outputs_0_lst=hf_outputs,
212
+ outputs_1_lst=[
213
+ vllm_to_hf_output(vllm_output, model)
214
+ for vllm_output in vllm_outputs
215
+ ],
216
+ name_0="hf",
217
+ name_1="vllm",
218
+ )
219
+
220
+
221
+@pytest.mark.parametrize("model", models)
222
+@pytest.mark.parametrize(
223
+ "sizes",
224
+ [
225
+ # Text only
226
+ [],
227
+ # Single-size
228
+ [(512, 512)],
229
+ # Single-size, batched
230
+ [(512, 512), (512, 512), (512, 512)],
231
+ # Multi-size, batched
232
+ [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
233
+ (1024, 1024), (512, 1536), (512, 2028)],
234
+ # Multi-size, batched, including text only
235
+ [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
236
+ (1024, 1024), (512, 1536), (512, 2028), None],
237
+ # mllama has 8 possible aspect ratios, carefully set the sizes
238
+ # to cover all of them
239
+ ],
240
+)
241
+@pytest.mark.parametrize("dtype", ["bfloat16"])
242
+@pytest.mark.parametrize("max_tokens", [128])
243
+@pytest.mark.parametrize("num_logprobs", [5])
244
+def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
245
+ max_tokens, num_logprobs) -> None:
246
+ run_test(
247
+ hf_runner,
248
+ vllm_runner,
249
+ image_assets,
250
+ model,
251
+ sizes=sizes,
252
+ dtype=dtype,
253
+ max_tokens=max_tokens,
254
+ num_logprobs=num_logprobs,
255
+ tensor_parallel_size=1,
256
+ )
257
+
258
+
259
+@multi_gpu_test(num_gpus=2)
260
+@pytest.mark.parametrize("model", models)
261
+@pytest.mark.parametrize(
262
+ "sizes",
263
+ [
264
+ [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
265
+ (1024, 1024), (512, 1536), (512, 2028), None],
266
+ ],
267
+)
268
+@pytest.mark.parametrize("dtype", ["bfloat16"])
269
+@pytest.mark.parametrize("max_tokens", [128])
270
+@pytest.mark.parametrize("num_logprobs", [5])
271
+def test_models_distributed(hf_runner, vllm_runner, image_assets, model, sizes,
272
+ dtype, max_tokens, num_logprobs) -> None:
273
+ run_test(
274
+ hf_runner,
275
+ vllm_runner,
276
+ image_assets,
277
+ model,
278
+ sizes=sizes,
279
+ dtype=dtype,
280
+ max_tokens=max_tokens,
281
+ num_logprobs=num_logprobs,
282
+ tensor_parallel_size=2,
283
+ )
vllm/config.py
+3
-1
@@ -576,7 +576,9 @@ class ModelConfig:
576
@property
577
def is_encoder_decoder_model(self) -> bool:
578
"""Extract the HF encoder/decoder model flag."""
579
- return getattr(self.hf_config, "is_encoder_decoder", False)
579
+ return getattr(self.hf_config, "is_encoder_decoder", False) or (
580
+ (hasattr(self.hf_config, "text_config") and getattr(
581
+ self.hf_config.text_config, "is_encoder_decoder", False)))
582
583
@property
584
def is_embedding_model(self) -> bool:
vllm/engine/llm_engine.py
+5
-1
@@ -1734,7 +1734,11 @@ class LLMEngine:
1734
1735
def _validate_model_inputs(self, inputs: Union[LLMInputs,
1736
EncoderDecoderLLMInputs]):
1737
- if self.is_encoder_decoder_model():
1737
+ if self.model_config.is_multimodal_model:
1738
+ # For encoder-decoder multimodal models, the max_prompt_len
1739
+ # restricts the decoder prompt length
1740
+ prompt_ids = inputs.get("prompt_token_ids")
1741
+ elif self.is_encoder_decoder_model():
1742
prompt_ids = inputs.get("encoder_prompt_token_ids")
1743
else:
1744
prompt_ids = inputs.get("prompt_token_ids")
vllm/entrypoints/chat_utils.py
+22
-6
@@ -159,6 +159,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
159
hf_config.image_token_index)
160
if model_type in ("chameleon", "internvl_chat"):
161
return "<image>"
162
+ if model_type == "mllama":
163
+ return "<|image|>"
164
if model_type == "qwen2_vl":
165
return "<|vision_start|><|image_pad|><|vision_end|>"
166
@@ -358,6 +360,7 @@ _TextParser = partial(cast, ChatCompletionContentPartTextParam)
360
_ImageParser = partial(cast, ChatCompletionContentPartImageParam)
361
_AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
362
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
363
+MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
364
365
366
def _parse_chat_message_content_parts(
@@ -368,7 +371,11 @@ def _parse_chat_message_content_parts(
371
texts: List[str] = []
372
373
mm_parser = mm_tracker.create_parser()
374
+ keep_multimodal_content = \
375
+ mm_tracker._model_config.hf_config.model_type in \
376
+ MODEL_KEEP_MULTI_MODAL_CONTENT
377
378
+ has_image = False
379
for part in parts:
380
part_type = part["type"]
381
if part_type == "text":
@@ -383,6 +390,7 @@ def _parse_chat_message_content_parts(
390
"will be ignored.")
391
392
mm_parser.parse_image(image_url["url"])
393
+ has_image = True
394
elif part_type == "audio_url":
395
audio_url = _AudioParser(part)["audio_url"]
396
@@ -394,12 +402,20 @@ def _parse_chat_message_content_parts(
402
raise NotImplementedError(f"Unknown part type: {part_type}")
403
404
text_prompt = "\n".join(texts)
397
- mm_placeholder_counts = mm_parser.mm_placeholder_counts()
398
- if mm_placeholder_counts:
399
- text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
400
- text_prompt)
401
-
402
- return [ConversationMessage(role=role, content=text_prompt)]
405
+ if keep_multimodal_content:
406
+ text_prompt = "\n".join(texts)
407
+ role_content = [{'type': 'text', 'text': text_prompt}]
408
+
409
+ if has_image:
410
+ role_content = [{'type': 'image'}] + role_content
411
+ return [ConversationMessage(role=role,
412
+ content=role_content)] # type: ignore
413
+ else:
414
+ mm_placeholder_counts = mm_parser.mm_placeholder_counts()
415
+ if mm_placeholder_counts:
416
+ text_prompt = _get_full_multimodal_text_prompt(
417
+ mm_placeholder_counts, text_prompt)
418
+ return [ConversationMessage(role=role, content=text_prompt)]
419
420
421
# No need to validate using Pydantic again
vllm/entrypoints/openai/serving_chat.py
+2
@@ -309,6 +309,8 @@ class OpenAIServingChat(OpenAIServing):
309
async for res in result_generator:
310
if res.prompt_token_ids is not None:
311
num_prompt_tokens = len(res.prompt_token_ids)
312
+ if res.encoder_prompt_token_ids is not None:
313
+ num_prompt_tokens += len(res.encoder_prompt_token_ids)
314
315
# We need to do it here, because if there are exceptions in
316
# the result_generator, it needs to be sent as the FIRST
vllm/inputs/data.py
+6
@@ -139,6 +139,12 @@ class EncoderDecoderLLMInputs(LLMInputs):
139
available.
140
"""
141
142
+ encoder_multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
143
+ """
144
+ Optional multi-modal data to pass to the encoder model,
145
+ if the model supports it.
146
+ """
147
+
148
149
_T1 = TypeVar("_T1",
150
bound=SingletonPromptInputs,
vllm/inputs/preprocess.py
+15
-7
@@ -128,6 +128,7 @@ class InputPreprocessor:
128
def _prepare_decoder_input_ids_for_generation(
129
self,
130
decoder_input_ids: Optional[List[int]],
131
+ force_bos: bool = True,
132
) -> List[int]:
133
"""
134
Prepares `decoder_input_ids` for generation with encoder-decoder models.
@@ -157,8 +158,8 @@ class InputPreprocessor:
158
# use decoder_start_token_id as decoder_input_ids
159
decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
160
160
- if (len(decoder_input_ids) == 0
161
- or decoder_input_ids[0] != decoder_start_token_id):
161
+ if force_bos and (len(decoder_input_ids) == 0
162
+ or decoder_input_ids[0] != decoder_start_token_id):
163
decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
164
165
return decoder_input_ids
@@ -295,18 +296,25 @@ class InputPreprocessor:
296
encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
297
decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
298
298
- if encoder_mm_data is not None or decoder_mm_data is not None:
299
- raise ValueError("Multi-modal encoder-decoder models are "
300
- "not supported yet")
299
+ if decoder_mm_data is not None:
300
+ raise ValueError(
301
+ "Multi-modality decoder inputs of encoder-decoder models are "
302
+ "not supported yet")
303
302
- decoder_prompt_ids = (
303
- self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
304
+ # For Multi-Modal models (e.g., mllama), the text input can be
305
+ # <|image|><|begin_of_text|>hello world. And we should not add
306
+ # another <|begin_of_text|> to the beginning.
307
+ decoder_prompt_ids = (self._prepare_decoder_input_ids_for_generation(
308
+ decoder_prompt_ids,
309
+ force_bos=(encoder_mm_data is None and decoder_mm_data is None)))
310
311
return EncoderDecoderLLMInputs(
312
prompt_token_ids=decoder_prompt_ids,
313
prompt=decoder_prompt,
314
+ multi_modal_data=decoder_mm_data,
315
encoder_prompt_token_ids=encoder_prompt_ids,
316
encoder_prompt=encoder_prompt,
317
+ encoder_multi_modal_data=encoder_mm_data,
318
)
319
320
def _process_encoder_decoder_prompt(
vllm/inputs/registry.py
+48
-6
@@ -112,6 +112,8 @@ class InputRegistry:
112
def __init__(self) -> None:
113
self._dummy_factories_by_model_type: Dict[Type[nn.Module],
114
DummyDataFactory] = {}
115
+ self._dummy_encoder_factories_by_model_type: Dict[
116
+ Type[nn.Module], DummyDataFactory] = {}
117
self._input_processors_by_model_type: Dict[Type[nn.Module],
118
InputProcessor] = {}
119
@@ -162,11 +164,44 @@ class InputRegistry:
164
return self._dummy_factories_by_model_type \
165
.get(model_cls, self._default_dummy_data_factory)
166
167
+ def register_dummy_encoder_data(self, factory: DummyDataFactory):
168
+ """
169
+ Register a dummy encoder data factory to a model class
170
+
171
+ This is similar to :meth:`~register_dummy_data`, but for encoder input.
172
+ """
173
+
174
+ def wrapper(model_cls: N) -> N:
175
+ if model_cls in self._dummy_encoder_factories_by_model_type:
176
+ logger.warning(
177
+ "Model class %s already has dummy encoder data "
178
+ "registered to %s. It is overwritten by the new one.",
179
+ model_cls, self)
180
+
181
+ self._dummy_encoder_factories_by_model_type[model_cls] = factory
182
+
183
+ return model_cls
184
+
185
+ return wrapper
186
+
187
+ def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
188
+ if model_cls in self._dummy_encoder_factories_by_model_type:
189
+ dummy_factory = self._dummy_encoder_factories_by_model_type[
190
+ model_cls]
191
+ else:
192
+ logger.warning(
193
+ "No dummy encoder data factory registered to %s. "
194
+ "Using the dummy data factory for the model instead.",
195
+ model_cls)
196
+ dummy_factory = self._get_dummy_data_factory(model_cls)
197
+ return dummy_factory
198
+
199
def dummy_data_for_profiling(
200
self,
201
model_config: "ModelConfig",
202
seq_len: int,
203
mm_registry: "MultiModalRegistry",
204
+ is_encoder_data: bool = False,
205
) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
206
"""
207
Create dummy data for profiling the memory usage of a model.
@@ -184,8 +219,10 @@ class InputRegistry:
219
from vllm.model_executor.model_loader import get_model_architecture
220
221
model_cls, _ = get_model_architecture(model_config)
187
- dummy_factory = self._get_dummy_data_factory(model_cls)
188
-
222
+ if is_encoder_data:
223
+ dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
224
+ else:
225
+ dummy_factory = self._get_dummy_data_factory(model_cls)
226
mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
227
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
228
dummy_factory, overrides=model_config.mm_processor_kwargs)
@@ -196,10 +233,15 @@ class InputRegistry:
233
234
# Having more tokens is over-conservative but otherwise fine
235
num_tokens = seq_data.prompt_token_ids
199
- assert len(num_tokens) >= seq_len, (
200
- f"Expected at least {seq_len} dummy tokens for profiling, "
201
- f"but found {len(num_tokens)} tokens instead.")
202
-
236
+ if len(num_tokens) < seq_len:
237
+ if is_encoder_data:
238
+ logger.warning(
239
+ "Expected at least %d dummy encoder tokens for profiling, "
240
+ "but found %d tokens instead.", seq_len, len(num_tokens))
241
+ else:
242
+ raise AssertionError(
243
+ f"Expected at least {seq_len} dummy tokens for profiling, "
244
+ f"but found {len(num_tokens)} tokens instead.")
245
if mm_data is not None:
246
for k, v in mm_data.items():
247
num_items = len(v) if isinstance(v, list) else 1
vllm/model_executor/models/__init__.py
+2
@@ -101,6 +101,8 @@ _MULTIMODAL_MODELS = {
101
"Qwen2VLForConditionalGeneration": ("qwen2_vl",
102
"Qwen2VLForConditionalGeneration"),
103
"UltravoxModel": ("ultravox", "UltravoxModel"),
104
+ "MllamaForConditionalGeneration": ("mllama",
105
+ "MllamaForConditionalGeneration"),
106
}
107
_CONDITIONAL_GENERATION_MODELS = {
108
"BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/mllama.py
new
+1135
@@ -0,0 +1,1135 @@
1
+# coding=utf-8
2
+# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+# http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+"""PyTorch Mllama model."""
16
+import math
17
+from array import array
18
+from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19
+ TypedDict, Union)
20
+
21
+import torch
22
+import torch.nn.functional as F
23
+import torch.utils.checkpoint
24
+import transformers.models.mllama.configuration_mllama as config_mllama
25
+from PIL import Image
26
+from torch import nn
27
+from transformers.modeling_outputs import (BaseModelOutput,
28
+ CausalLMOutputWithPast)
29
+from transformers.models.mllama.image_processing_mllama import (
30
+ get_optimal_tiled_canvas)
31
+
32
+import vllm.distributed.parallel_state as ps
33
+from vllm.attention import Attention, AttentionMetadata, AttentionType
34
+from vllm.config import CacheConfig, MultiModalConfig
35
+from vllm.distributed import get_tensor_model_parallel_world_size
36
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
37
+from vllm.logger import init_logger
38
+from vllm.model_executor.layers.layernorm import RMSNorm
39
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
40
+ QKVParallelLinear,
41
+ RowParallelLinear)
42
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
43
+from vllm.model_executor.layers.quantization import QuantizationConfig
44
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45
+from vllm.model_executor.layers.vocab_parallel_embedding import (
46
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
47
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48
+from vllm.model_executor.sampling_metadata import SamplingMetadata
49
+from vllm.multimodal import MULTIMODAL_REGISTRY
50
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
51
+
52
+from .clip import CLIPMLP
53
+from .interfaces import SupportsMultiModal
54
+from .llama import LlamaDecoderLayer, LlamaMLP
55
+
56
+logger = init_logger(__name__)
57
+MLLAMA_IMAGE_TOKEN_ID = 128256
58
+MLLAMA_IMAGE_TOKEN = "<|image|>"
59
+
60
+
61
+class MllamaImagePixelInputs(TypedDict):
62
+ type: Literal["pixel_values"]
63
+ data: torch.Tensor
64
+ """Shape: """
65
+ """(batch_size, max_num_image, max_num_chunk, num_channel, height, width)"""
66
+ aspect_ratio_ids: torch.Tensor
67
+ """Shape: `(batch_size, max_num_image)`"""
68
+ aspect_ratio_mask: torch.Tensor
69
+ """Shape: `(batch_size, max_num_image, max_num_tiles)`"""
70
+
71
+
72
+# TODO: support LlamaImageEmbeddingInputs
73
+
74
+
75
+def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
76
+ # move encoder_prompt to prompt
77
+ if llm_inputs.get("prompt") is None:
78
+ llm_inputs["prompt"] = llm_inputs["encoder_prompt"]
79
+ llm_inputs["prompt_token_ids"] = llm_inputs["encoder_prompt_token_ids"]
80
+
81
+ # process multi-modal data
82
+ assert "decoder_multi_modal_data" not in llm_inputs, \
83
+ "multi-modal data should be put in encoder message of mllama"
84
+ multi_modal_data = llm_inputs.get("encoder_multi_modal_data")
85
+
86
+ if multi_modal_data is None or "image" not in multi_modal_data \
87
+ or multi_modal_data["image"] is None:
88
+ # text-only
89
+ llm_inputs["encoder_prompt"] = ""
90
+ llm_inputs["encoder_prompt_token_ids"] = []
91
+ llm_inputs["encoder_multi_modal_data"] = {}
92
+ return llm_inputs
93
+
94
+ # get num_tiles
95
+ if isinstance(multi_modal_data['image'], Image.Image):
96
+ multi_modal_data['image'] = [multi_modal_data['image']]
97
+ hf_config = ctx.model_config.hf_config
98
+ num_tiles = 0
99
+ for image in multi_modal_data["image"]:
100
+ width, height = image.size
101
+ tile_size = hf_config.vision_config.image_size
102
+ canvas_height, canvas_width = get_optimal_tiled_canvas(
103
+ image_height=height,
104
+ image_width=width,
105
+ max_image_tiles=hf_config.vision_config.max_num_tiles,
106
+ tile_size=tile_size,
107
+ )
108
+ num_tiles_height = canvas_height // tile_size
109
+ num_tiles_width = canvas_width // tile_size
110
+ num_tiles += num_tiles_height * num_tiles_width
111
+
112
+ # set encoder prompt based on num_tiles
113
+ assert hf_config.vision_config.image_size % 14 == 0, \
114
+ "chunk size should be multiple of 14"
115
+ token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
116
+ num_tokens = num_tiles * token_per_chunk
117
+ llm_inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
118
+ llm_inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID
119
+ ] * num_tokens
120
+
121
+ return llm_inputs
122
+
123
+
124
+def get_max_mllama_image_tokens(ctx: InputContext) -> int:
125
+ hf_config = ctx.model_config.hf_config
126
+ token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
127
+ return hf_config.vision_config.max_num_tiles * token_per_chunk
128
+
129
+
130
+def dummy_decoder_seq_data(seq_len: int, num_images: int):
131
+ # <|image|> * num_images + 0 * (seq_len - num_images)
132
+ assert seq_len >= num_images, \
133
+ "seq_len should be greater than or equal to num_images"
134
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
135
+ [MLLAMA_IMAGE_TOKEN_ID]) * num_images
136
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * (seq_len - num_images)
137
+ return SequenceData(token_ids)
138
+
139
+
140
+def dummy_encoder_seq_data(ctx: InputContext, num_images: int):
141
+ num_tokens = get_max_mllama_image_tokens(ctx) * num_images
142
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
143
+ [MLLAMA_IMAGE_TOKEN_ID]) * num_tokens
144
+ return SequenceData(token_ids)
145
+
146
+
147
+def dummy_image(num_images: int, ):
148
+ width = height = 1024
149
+ image = Image.new("RGB", (width, height), color=0)
150
+ return {"image": image if num_images == 1 else [image] * num_images}
151
+
152
+
153
+def dummy_decoder_data_for_mllama(ctx: InputContext, seq_len: int,
154
+ mm_counts: Mapping[str, int]):
155
+ num_images = mm_counts["image"]
156
+ return dummy_decoder_seq_data(seq_len, num_images), None
157
+
158
+
159
+def dummy_encoder_data_for_mllama(ctx: InputContext, seq_len: int,
160
+ mm_counts: Mapping[str, int]):
161
+ num_images = mm_counts["image"]
162
+ return dummy_encoder_seq_data(ctx, num_images), dummy_image(num_images)
163
+
164
+
165
+def _prepare_aspect_ratio_attention_mask(
166
+ aspect_ratio_mask: torch.Tensor,
167
+ num_patches: int,
168
+ target_length: int,
169
+ dtype: torch.dtype,
170
+) -> torch.Tensor:
171
+ # Expand aspect ratio mask to target_length
172
+ batch_size, max_num_tiles = aspect_ratio_mask.shape
173
+ attention_mask = aspect_ratio_mask.view(batch_size, max_num_tiles, 1,
174
+ 1).to(dtype)
175
+ attention_mask = attention_mask.repeat(1, 1, target_length, 1)
176
+
177
+ # Mask padding patches
178
+ pad_patches = target_length - num_patches
179
+ attention_mask[:, :, -pad_patches:] = 0
180
+
181
+ # Invert the mask (0 -> 1, 1 -> 0)
182
+ attention_mask = 1 - attention_mask
183
+
184
+ # Reshape to 2D and create 4D attention mask
185
+ # (batch_size, 1, max_num_tiles*target_length, max_num_tiles*target_length)
186
+ attention_mask = attention_mask.reshape(batch_size,
187
+ max_num_tiles * target_length, 1)
188
+ attention_mask = attention_mask @ attention_mask.transpose(
189
+ -1, -2) * torch.finfo(dtype).min
190
+ attention_mask = attention_mask.unsqueeze(1)
191
+
192
+ return attention_mask
193
+
194
+
195
+class ColumnParallelConv2dPatch(torch.nn.Module):
196
+ """Conv2D Patching layer with model parallelism.
197
+ Column parallel over unfolded input.
198
+ Arguments:
199
+ in_channels: Input channels.
200
+ out_channels: Output channels.
201
+ kernel_size: Size of convolution kernel.
202
+ stride (default 1): Stride for convolution.
203
+ bias (default False): Use bias in Conv2d.
204
+ Input: (bsz, in_channels, width, height)
205
+ Output: (bsz, num_tokens, out_channels)
206
+ """
207
+
208
+ def __init__(
209
+ self,
210
+ in_channels: int,
211
+ out_channels: int,
212
+ kernel_size: Union[int, Tuple[int, int]],
213
+ stride: Union[int, Tuple[int, int]],
214
+ bias: bool = False,
215
+ ) -> None:
216
+ super().__init__()
217
+ if isinstance(kernel_size, int):
218
+ kernel_size = (kernel_size, kernel_size)
219
+ self._unfold = torch.nn.Unfold(kernel_size=kernel_size, stride=stride)
220
+ self._linear = ColumnParallelLinear(
221
+ in_channels * kernel_size[0] * kernel_size[1],
222
+ out_channels,
223
+ bias=bias,
224
+ )
225
+
226
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
227
+ x = self._unfold(x)
228
+ x = x.permute(0, 2, 1)
229
+ x, _ = self._linear(x)
230
+ return x
231
+
232
+
233
+class MllamaPrecomputedAspectRatioEmbedding(nn.Module):
234
+
235
+ def __init__(self,
236
+ config: config_mllama.MllamaVisionConfig,
237
+ is_gated: bool = True):
238
+ super().__init__()
239
+ self.max_num_tiles = config.max_num_tiles
240
+ self.hidden_size = config.hidden_size
241
+ self.max_aspect_ratio_id = config.max_aspect_ratio_id
242
+ self.is_gated = is_gated
243
+
244
+ self.embedding = nn.Embedding(self.max_aspect_ratio_id + 1,
245
+ self.max_num_tiles * self.hidden_size)
246
+ if is_gated:
247
+ self.gate = nn.Parameter(torch.zeros(1))
248
+
249
+ def forward(self, hidden_state: torch.Tensor,
250
+ aspect_ratio_ids: torch.Tensor) -> torch.Tensor:
251
+ embeddings = self.embedding(aspect_ratio_ids)
252
+ embeddings = embeddings.reshape(-1, self.max_num_tiles, 1,
253
+ self.hidden_size)
254
+
255
+ if self.is_gated:
256
+ embeddings = embeddings * self.gate.tanh()
257
+
258
+ hidden_state = hidden_state + embeddings
259
+ return hidden_state
260
+
261
+
262
+class MllamaPrecomputedPositionEmbedding(nn.Module):
263
+
264
+ def __init__(self, config: config_mllama.MllamaVisionConfig):
265
+ super().__init__()
266
+ self.max_num_tiles = config.max_num_tiles
267
+ self.max_aspect_ratio_id = config.max_aspect_ratio_id
268
+ self.num_patches = (config.image_size // config.patch_size)**2 + 1
269
+ self.hidden_size = config.hidden_size
270
+ self.scale = config.hidden_size**-0.5
271
+
272
+ self.gate = nn.Parameter(torch.zeros(1))
273
+
274
+ # position embedding
275
+ position_embedding = torch.randn(self.num_patches, self.hidden_size)
276
+ self.embedding = nn.Parameter(self.scale * position_embedding)
277
+
278
+ # tile position embedding
279
+ self.tile_embedding = nn.Embedding(
280
+ self.max_aspect_ratio_id + 1,
281
+ self.max_num_tiles * self.num_patches * self.hidden_size)
282
+
283
+ def forward(self, hidden_state: torch.Tensor,
284
+ aspect_ratio_ids: torch.Tensor) -> torch.Tensor:
285
+ # position embeddings
286
+ gated_position_embedding = (1 - self.gate.tanh()) * self.embedding
287
+ hidden_state = hidden_state + gated_position_embedding.view(
288
+ 1, 1, self.num_patches, self.hidden_size)
289
+
290
+ # precomputed tile position embeddings
291
+ tile_position_embedding = self.tile_embedding(aspect_ratio_ids)
292
+ batch_size = hidden_state.shape[0]
293
+ tile_position_embedding = tile_position_embedding.reshape(
294
+ batch_size, self.max_num_tiles, self.num_patches, self.hidden_size)
295
+ gated_tile_position_embedding = self.gate.tanh(
296
+ ) * tile_position_embedding
297
+ hidden_state = hidden_state + gated_tile_position_embedding
298
+
299
+ return hidden_state
300
+
301
+
302
+# TODO: support other attention backends for attention in vision model
303
+class MllamaVisionSdpaAttention(nn.Module):
304
+
305
+ def __init__(self, config: config_mllama.MllamaVisionConfig):
306
+ super().__init__()
307
+
308
+ model_parallel_size = get_tensor_model_parallel_world_size()
309
+ self.embed_dim = config.hidden_size
310
+ self.num_heads = config.attention_heads
311
+ self.head_dim = config.hidden_size // config.attention_heads
312
+ self.num_local_heads = self.num_heads // model_parallel_size
313
+ self.q_size = self.num_local_heads * self.head_dim
314
+ self.kv_size = self.num_local_heads * self.head_dim
315
+
316
+ self.qkv_proj = QKVParallelLinear(
317
+ self.embed_dim,
318
+ self.head_dim,
319
+ self.num_heads,
320
+ bias=False,
321
+ )
322
+ self.o_proj = RowParallelLinear(
323
+ self.num_heads * self.head_dim,
324
+ self.embed_dim,
325
+ bias=False,
326
+ input_is_parallel=True,
327
+ )
328
+
329
+ def forward(
330
+ self,
331
+ hidden_state: torch.Tensor,
332
+ attention_mask: Optional[torch.Tensor] = None,
333
+ ) -> torch.Tensor:
334
+ qkv, _ = self.qkv_proj(hidden_state)
335
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
336
+ q = q.view(q.shape[0], q.shape[1], self.num_local_heads,
337
+ self.head_dim).transpose(1, 2)
338
+ k = k.view(k.shape[0], k.shape[1], self.num_local_heads,
339
+ self.head_dim).transpose(1, 2)
340
+ v = v.view(v.shape[0], v.shape[1], self.num_local_heads,
341
+ self.head_dim).transpose(1, 2)
342
+
343
+ # TODO: remove padding in image encoder
344
+ attn_output = F.scaled_dot_product_attention(q,
345
+ k,
346
+ v,
347
+ attn_mask=attention_mask,
348
+ dropout_p=0.0)
349
+
350
+ attn_output = attn_output.transpose(1, 2).contiguous()
351
+ attn_output = attn_output.reshape(attn_output.shape[0],
352
+ attn_output.shape[1], -1)
353
+ output, _ = self.o_proj(attn_output)
354
+ return output
355
+
356
+
357
+class MllamaVisionEncoderLayer(nn.Module):
358
+
359
+ def __init__(self,
360
+ config: config_mllama.MllamaVisionConfig,
361
+ is_gated: bool = False):
362
+ super().__init__()
363
+
364
+ self.hidden_size = config.hidden_size
365
+ self.num_attention_heads = config.attention_heads
366
+ self.is_gated = is_gated
367
+ self.intermediate_size = config.intermediate_size
368
+
369
+ self.self_attn = MllamaVisionSdpaAttention(config)
370
+ self.mlp = CLIPMLP(config)
371
+
372
+ self.input_layernorm = nn.LayerNorm(self.hidden_size,
373
+ eps=config.norm_eps)
374
+ self.post_attention_layernorm = nn.LayerNorm(self.hidden_size,
375
+ eps=config.norm_eps)
376
+
377
+ # there used to be an if else here, no code path
378
+ if is_gated:
379
+ self.gate_attn = nn.Parameter(torch.ones(1) * math.pi / 4)
380
+ self.gate_ffn = nn.Parameter(torch.ones(1) * math.pi / 4)
381
+
382
+ def forward(
383
+ self,
384
+ hidden_state: torch.Tensor,
385
+ attention_mask: Optional[torch.Tensor] = None,
386
+ ):
387
+ # Self Attention
388
+ residual = hidden_state
389
+ hidden_state = self.input_layernorm(hidden_state)
390
+ hidden_state = self.self_attn(hidden_state,
391
+ attention_mask=attention_mask)
392
+ gate_attn = 1 if not self.is_gated else self.gate_attn.tanh()
393
+ hidden_state = residual + gate_attn * hidden_state
394
+
395
+ # Feed forward
396
+ residual = hidden_state
397
+ hidden_state = self.post_attention_layernorm(hidden_state)
398
+ hidden_state = self.mlp(hidden_state)
399
+ gate_ffn = 1 if not self.is_gated else self.gate_ffn.tanh()
400
+ hidden_state = residual + gate_ffn * hidden_state
401
+
402
+ return hidden_state
403
+
404
+
405
+class MllamaVisionEncoder(nn.Module):
406
+
407
+ def __init__(self,
408
+ config: config_mllama.MllamaVisionConfig,
409
+ num_layers=32,
410
+ is_gated=False,
411
+ output_hidden_states=None):
412
+ super().__init__()
413
+ self.config = config
414
+ self.layers = nn.ModuleList([
415
+ MllamaVisionEncoderLayer(config, is_gated)
416
+ for _ in range(num_layers)
417
+ ])
418
+ self.output_hidden_states = output_hidden_states or []
419
+
420
+ def forward(
421
+ self,
422
+ hidden_states: torch.Tensor,
423
+ attention_mask: Optional[torch.Tensor] = None,
424
+ ) -> Union[Tuple, BaseModelOutput]:
425
+ encoder_states = ()
426
+
427
+ for i, encoder_layer in enumerate(self.layers):
428
+ if i in self.output_hidden_states:
429
+ encoder_states = encoder_states + (hidden_states, )
430
+ hidden_states = encoder_layer(
431
+ hidden_states,
432
+ attention_mask,
433
+ )
434
+
435
+ if len(self.layers) - 1 in self.output_hidden_states:
436
+ encoder_states = encoder_states + (hidden_states, )
437
+
438
+ return hidden_states, encoder_states
439
+
440
+
441
+class MllamaVisionModel(nn.Module):
442
+
443
+ def __init__(self, config: config_mllama.MllamaVisionConfig):
444
+ super().__init__()
445
+ self.image_size = config.image_size
446
+ self.patch_size = config.patch_size
447
+ self.max_num_tiles = config.max_num_tiles
448
+ self.hidden_size = config.hidden_size
449
+ self.in_channels = config.num_channels
450
+ self.intermediate_layers_indices = config.intermediate_layers_indices
451
+
452
+ self.num_patches = (self.image_size // self.patch_size)**2 + 1
453
+ self.scale = config.hidden_size**-0.5
454
+
455
+ self.patch_embedding = ColumnParallelConv2dPatch(
456
+ in_channels=config.num_channels,
457
+ out_channels=self.hidden_size,
458
+ kernel_size=self.patch_size,
459
+ stride=self.patch_size,
460
+ bias=False,
461
+ )
462
+
463
+ self.class_embedding = nn.Parameter(self.scale *
464
+ torch.randn(self.hidden_size))
465
+ self.gated_positional_embedding = MllamaPrecomputedPositionEmbedding(
466
+ config)
467
+
468
+ self.pre_tile_positional_embedding = \
469
+ MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True)
470
+ self.post_tile_positional_embedding = \
471
+ MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True)
472
+
473
+ # layer norms
474
+ self.layernorm_pre = nn.LayerNorm(self.hidden_size)
475
+ self.layernorm_post = nn.LayerNorm(self.hidden_size)
476
+
477
+ # encoders
478
+ self.transformer = MllamaVisionEncoder(
479
+ config,
480
+ config.num_hidden_layers,
481
+ is_gated=False,
482
+ output_hidden_states=config.intermediate_layers_indices)
483
+ self.global_transformer = MllamaVisionEncoder(config,
484
+ config.num_global_layers,
485
+ is_gated=True)
486
+
487
+ def apply_class_embedding(self,
488
+ hidden_state: torch.Tensor) -> torch.Tensor:
489
+ batch_size, _, hidden_size = hidden_state.shape
490
+ class_embedding = self.class_embedding.expand(batch_size, 1,
491
+ hidden_size)
492
+ hidden_state = torch.cat([class_embedding, hidden_state], dim=1)
493
+ return hidden_state
494
+
495
+ def forward(self, pixel_values: torch.Tensor,
496
+ aspect_ratio_ids: torch.Tensor,
497
+ aspect_ratio_mask: torch.Tensor) -> torch.Tensor:
498
+ batch_size, num_concurrent_media, num_tiles, num_channels, \
499
+ height, width = pixel_values.shape
500
+
501
+ pixel_values = pixel_values.reshape(
502
+ batch_size * num_concurrent_media * num_tiles, num_channels,
503
+ height, width)
504
+ aspect_ratio_ids = aspect_ratio_ids.reshape(
505
+ batch_size * num_concurrent_media, -1)
506
+
507
+ # patch embedding
508
+ patch_embeds = self.patch_embedding(
509
+ pixel_values.to(self.layernorm_pre.weight.dtype))
510
+ hidden_state = patch_embeds
511
+ hidden_state = ps.get_tp_group().all_gather(hidden_state)
512
+
513
+ # tile embeddings
514
+ _, num_patches, dim = hidden_state.shape
515
+ hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
516
+ num_tiles, -1, dim)
517
+ hidden_state = self.pre_tile_positional_embedding(
518
+ hidden_state, aspect_ratio_ids)
519
+
520
+ # apply cls token
521
+ hidden_state = hidden_state.reshape(
522
+ batch_size * num_concurrent_media * num_tiles, num_patches, dim)
523
+ hidden_state = self.apply_class_embedding(hidden_state)
524
+ num_patches += 1
525
+
526
+ # apply position embeddings
527
+ hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
528
+ num_tiles, num_patches, dim)
529
+ hidden_state = self.gated_positional_embedding(hidden_state,
530
+ aspect_ratio_ids)
531
+
532
+ # apply encoder
533
+ hidden_state = self.layernorm_pre(hidden_state)
534
+
535
+ # Compute the number of tokens to pad
536
+ num_padding_patches = (8 - (hidden_state.shape[-2] % 8)) % 8
537
+ # Compute padding tuple for pad function
538
+ padding = (
539
+ 0, 0, 0, num_padding_patches
540
+ ) # (pad_left, pad_right, pad_left for dim -2, pad_right for dim -2)
541
+ # Pad the tensor
542
+ hidden_state = F.pad(hidden_state, padding, mode="constant", value=0)
543
+ slice_index = -num_padding_patches if num_padding_patches > 0 else None
544
+
545
+ attention_mask = aspect_ratio_mask.reshape(
546
+ batch_size * num_concurrent_media, -1)
547
+ attention_mask = _prepare_aspect_ratio_attention_mask(
548
+ aspect_ratio_mask=attention_mask,
549
+ num_patches=self.num_patches,
550
+ target_length=hidden_state.shape[2],
551
+ dtype=self.layernorm_pre.weight.dtype,
552
+ )
553
+
554
+ hidden_state = hidden_state.view(batch_size * num_concurrent_media, -1,
555
+ dim)
556
+ output = self.transformer(
557
+ hidden_state,
558
+ attention_mask=attention_mask,
559
+ )
560
+ hidden_state, intermediate_hidden_states = output[0], output[1]
561
+ intermediate_hidden_states = torch.stack(intermediate_hidden_states,
562
+ dim=-1)
563
+
564
+ # apply global encoder
565
+ hidden_state = self.layernorm_post(hidden_state)
566
+ hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
567
+ num_tiles,
568
+ num_patches + num_padding_patches,
569
+ dim)
570
+ hidden_state = self.post_tile_positional_embedding(
571
+ hidden_state, aspect_ratio_ids)
572
+ hidden_state = hidden_state.reshape(
573
+ batch_size * num_concurrent_media,
574
+ num_tiles * (num_patches + num_padding_patches), dim)
575
+ hidden_state = self.global_transformer(
576
+ hidden_state, attention_mask=attention_mask)[0]
577
+ hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
578
+ num_tiles,
579
+ num_patches + num_padding_patches,
580
+ dim)
581
+ hidden_state = hidden_state[:, :, :slice_index]
582
+
583
+ # adding intermediate layer outputs
584
+ hidden_state = hidden_state.reshape(batch_size, num_concurrent_media,
585
+ num_tiles, num_patches, dim)
586
+ intermediate_hidden_states = intermediate_hidden_states.reshape(
587
+ batch_size * num_concurrent_media, num_tiles,
588
+ num_patches + num_padding_patches, -1)
589
+ intermediate_hidden_states = intermediate_hidden_states[:, :, :
590
+ slice_index]
591
+ intermediate_hidden_states = intermediate_hidden_states.reshape(
592
+ batch_size, num_concurrent_media, num_tiles, num_patches, -1)
593
+ hidden_state = torch.cat([hidden_state, intermediate_hidden_states],
594
+ dim=-1)
595
+ return hidden_state
596
+
597
+
598
+class MllamaTextRMSNorm(nn.Module):
599
+
600
+ def __init__(self, hidden_size, eps=1e-6):
601
+ """
602
+ MllamaTextRMSNorm is equivalent to T5LayerNorm
603
+ """
604
+ super().__init__()
605
+ self.weight = nn.Parameter(torch.ones(hidden_size))
606
+ self.variance_epsilon = eps
607
+
608
+ def forward(self, hidden_states):
609
+ input_dtype = hidden_states.dtype
610
+ hidden_states = hidden_states.to(torch.float32)
611
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
612
+ hidden_states = hidden_states * torch.rsqrt(variance +
613
+ self.variance_epsilon)
614
+ return self.weight * hidden_states.to(input_dtype)
615
+
616
+ def extra_repr(self):
617
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
618
+
619
+
620
+class MllamaTextCrossAttention(nn.Module):
621
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
622
+
623
+ def __init__(
624
+ self,
625
+ config: Optional[config_mllama.MllamaTextConfig] = None,
626
+ layer_idx: Optional[int] = None,
627
+ ):
628
+ super().__init__()
629
+ self.config = config
630
+ self.model_parallel_size = get_tensor_model_parallel_world_size()
631
+ self.num_heads = self.config.num_attention_heads
632
+ self.num_local_heads = self.num_heads // self.model_parallel_size
633
+ self.num_key_value_heads = self.config.num_key_value_heads
634
+ self.num_local_key_value_heads = \
635
+ self.num_key_value_heads // self.model_parallel_size
636
+ self.dropout = config.dropout
637
+ self.hidden_size = config.hidden_size
638
+ self.head_dim = config.hidden_size // self.num_heads
639
+ self.layer_idx = layer_idx
640
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
641
+ self.q_local_size = self.num_local_heads * self.head_dim
642
+ self.kv_local_size = self.num_local_key_value_heads * self.head_dim
643
+
644
+ # TODO: change to Q/KV separate linear after #7448 is merged
645
+ self.qkv_proj = QKVParallelLinear(
646
+ self.hidden_size,
647
+ self.head_dim,
648
+ self.num_heads,
649
+ self.num_key_value_heads,
650
+ bias=False,
651
+ )
652
+ self.o_proj = RowParallelLinear(
653
+ self.num_heads * self.head_dim,
654
+ self.hidden_size,
655
+ bias=False,
656
+ input_is_parallel=True,
657
+ )
658
+ # vllm.model_executor.layers.layernorm.RMSNorm has precision issue,
659
+ # use huggingface's instead
660
+ self.q_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
661
+ self.k_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
662
+ self.scaling = self.head_dim**-0.5
663
+
664
+ self.attn = Attention(
665
+ self.num_local_heads,
666
+ self.head_dim,
667
+ self.scaling,
668
+ self.num_local_key_value_heads,
669
+ )
670
+
671
+ def forward(
672
+ self,
673
+ hidden_states: torch.Tensor,
674
+ attention_mask: Optional[torch.Tensor],
675
+ cross_attention_states: Optional[torch.Tensor],
676
+ kv_cache: torch.Tensor,
677
+ attn_metadata: AttentionMetadata,
678
+ ) -> torch.Tensor:
679
+ qkv_dec, _ = self.qkv_proj(hidden_states)
680
+ q, _, _ = qkv_dec.split(
681
+ [self.q_local_size, self.kv_local_size, self.kv_local_size],
682
+ dim=-1)
683
+ if cross_attention_states is None:
684
+ k = None
685
+ v = None
686
+ else:
687
+ qkv_enc, _ = self.qkv_proj(cross_attention_states)
688
+ _, k, v = qkv_enc.split(
689
+ [self.q_local_size, self.kv_local_size, self.kv_local_size],
690
+ dim=-1)
691
+ k = k.view(-1, self.num_local_key_value_heads, self.head_dim)
692
+ v = v.view(-1, self.num_local_key_value_heads, self.head_dim)
693
+ k = self.k_norm(k)
694
+ q = q.view(-1, self.num_local_heads, self.head_dim)
695
+ q = self.q_norm(q)
696
+
697
+ output = self.attn(q,
698
+ k,
699
+ v,
700
+ kv_cache,
701
+ attn_metadata,
702
+ attn_type=AttentionType.ENCODER_DECODER)
703
+ out, _ = self.o_proj(output)
704
+ return out
705
+
706
+
707
+class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
708
+ """Cross-attention transformer block with tanh-gated attention
709
+ and feedforward."""
710
+
711
+ def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int) \
712
+ -> None:
713
+ super().__init__()
714
+ self.layer_idx = layer_idx
715
+ self.cross_attn = MllamaTextCrossAttention(
716
+ config=config,
717
+ layer_idx=layer_idx,
718
+ )
719
+
720
+ self.input_layernorm = RMSNorm(config.hidden_size,
721
+ eps=config.rms_norm_eps)
722
+ self.cross_attn_attn_gate = torch.nn.Parameter(torch.zeros(1))
723
+
724
+ self.mlp = LlamaMLP(
725
+ hidden_size=config.hidden_size,
726
+ intermediate_size=config.intermediate_size,
727
+ hidden_act=config.hidden_act,
728
+ )
729
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
730
+ eps=config.rms_norm_eps)
731
+ self.cross_attn_mlp_gate = torch.nn.Parameter(torch.zeros(1))
732
+
733
+ def forward(
734
+ self,
735
+ hidden_states: torch.Tensor,
736
+ cross_attention_states: torch.Tensor,
737
+ cross_attention_mask: torch.Tensor,
738
+ full_text_row_masked_out_mask: torch.Tensor,
739
+ kv_cache: List[torch.Tensor],
740
+ attn_metadata: AttentionMetadata,
741
+ ) -> torch.Tensor:
742
+ residual = hidden_states
743
+ hidden_states = self.input_layernorm(hidden_states)
744
+
745
+ hidden_states = self.cross_attn(
746
+ hidden_states=hidden_states,
747
+ attention_mask=cross_attention_mask,
748
+ cross_attention_states=cross_attention_states,
749
+ kv_cache=kv_cache,
750
+ attn_metadata=attn_metadata,
751
+ )
752
+ hidden_states = full_text_row_masked_out_mask * hidden_states
753
+ hidden_states = residual + self.cross_attn_attn_gate.tanh(
754
+ ) * hidden_states
755
+
756
+ residual = hidden_states
757
+ hidden_states = self.post_attention_layernorm(hidden_states)
758
+ hidden_states = self.mlp(hidden_states)
759
+ hidden_states = full_text_row_masked_out_mask * hidden_states
760
+ hidden_states = residual + self.cross_attn_mlp_gate.tanh(
761
+ ) * hidden_states
762
+ return hidden_states
763
+
764
+
765
+class MllamaTextModel(nn.Module):
766
+ config_class = config_mllama.MllamaTextConfig
767
+ base_model_prefix = "model"
768
+
769
+ def __init__(self, config: config_mllama.MllamaTextConfig,
770
+ cache_config: Optional[CacheConfig],
771
+ quant_config: Optional[QuantizationConfig]):
772
+ super().__init__()
773
+ self.padding_idx = config.pad_token_id
774
+ self.vocab_size = config.vocab_size
775
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8,
776
+ config.hidden_size)
777
+ self.cross_attention_layers = config.cross_attention_layers
778
+
779
+ layers = []
780
+ for layer_idx in range(config.num_hidden_layers):
781
+ if layer_idx in self.cross_attention_layers:
782
+ layers.append(
783
+ MllamaCrossAttentionDecoderLayer(config, layer_idx))
784
+ else:
785
+ # TODO: force LlamaDecoderLayer to config.attention_bias=False
786
+ layers.append(
787
+ LlamaDecoderLayer(config,
788
+ cache_config=cache_config,
789
+ quant_config=quant_config))
790
+
791
+ self.layers = nn.ModuleList(layers)
792
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
793
+
794
+ def forward(
795
+ self,
796
+ input_ids: torch.LongTensor,
797
+ positions: Optional[torch.LongTensor],
798
+ cross_attention_states: Optional[torch.LongTensor],
799
+ cross_attention_mask: Optional[torch.LongTensor],
800
+ full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
801
+ torch.Tensor]],
802
+ kv_caches: List[torch.Tensor],
803
+ attn_metadata: AttentionMetadata,
804
+ skip_cross_attention: bool,
805
+ ) -> torch.Tensor:
806
+ inputs_embeds = self.embed_tokens(input_ids)
807
+ hidden_states = inputs_embeds
808
+
809
+ for idx, decoder_layer in enumerate(self.layers):
810
+ if isinstance(decoder_layer, MllamaCrossAttentionDecoderLayer):
811
+ if not skip_cross_attention:
812
+ hidden_states = decoder_layer(
813
+ hidden_states=hidden_states,
814
+ cross_attention_states=cross_attention_states,
815
+ cross_attention_mask=cross_attention_mask,
816
+ full_text_row_masked_out_mask=
817
+ full_text_row_masked_out_mask,
818
+ kv_cache=kv_caches[idx],
819
+ attn_metadata=attn_metadata,
820
+ )
821
+ elif isinstance(decoder_layer, LlamaDecoderLayer):
822
+ hidden_states, residual = decoder_layer(
823
+ positions=positions,
824
+ hidden_states=hidden_states,
825
+ kv_cache=kv_caches[idx],
826
+ attn_metadata=attn_metadata,
827
+ residual=None,
828
+ )
829
+ hidden_states = hidden_states + residual
830
+ else:
831
+ raise ValueError(
832
+ f"Unknown decoder layer type {type(decoder_layer)}")
833
+ hidden_states = self.norm(hidden_states)
834
+ return hidden_states
835
+
836
+
837
+class MllamaForCausalLM(nn.Module):
838
+ config_class = config_mllama.MllamaTextConfig
839
+ base_model_prefix = "language_model"
840
+ _no_split_modules = [
841
+ "MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"
842
+ ]
843
+
844
+ def __init__(self, config: config_mllama.MllamaTextConfig,
845
+ cache_config: Optional[CacheConfig],
846
+ quant_config: Optional[QuantizationConfig]):
847
+ super().__init__()
848
+ self.vocab_size = config.vocab_size
849
+ self.model = MllamaTextModel(config, cache_config, quant_config)
850
+ self.lm_head = ParallelLMHead(
851
+ config.vocab_size,
852
+ config.hidden_size,
853
+ org_num_embeddings=config.vocab_size,
854
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE,
855
+ quant_config=quant_config,
856
+ )
857
+
858
+ def forward(
859
+ self,
860
+ input_ids: torch.LongTensor,
861
+ positions: Optional[torch.LongTensor],
862
+ cross_attention_states: Optional[torch.LongTensor],
863
+ cross_attention_mask: Optional[torch.LongTensor],
864
+ full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
865
+ torch.Tensor]],
866
+ kv_caches: List[torch.Tensor],
867
+ attn_metadata: AttentionMetadata,
868
+ skip_cross_attention: bool,
869
+ ) -> torch.Tensor:
870
+ hidden_states = self.model(
871
+ input_ids=input_ids,
872
+ positions=positions,
873
+ cross_attention_states=cross_attention_states,
874
+ cross_attention_mask=cross_attention_mask,
875
+ full_text_row_masked_out_mask=full_text_row_masked_out_mask,
876
+ kv_caches=kv_caches,
877
+ attn_metadata=attn_metadata,
878
+ skip_cross_attention=skip_cross_attention,
879
+ )
880
+ return hidden_states
881
+
882
+
883
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
884
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_mllama_image_tokens)
885
+@INPUT_REGISTRY.register_dummy_data(dummy_decoder_data_for_mllama)
886
+@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
887
+@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
888
+class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
889
+
890
+ def __init__(self,
891
+ config: config_mllama.MllamaConfig,
892
+ multimodal_config: MultiModalConfig,
893
+ cache_config: Optional[CacheConfig] = None,
894
+ quant_config: Optional[QuantizationConfig] = None):
895
+ super().__init__()
896
+ self.vocab_size = config.text_config.vocab_size
897
+ self.hidden_size = config.text_config.hidden_size
898
+ self.max_num_tiles = config.vision_config.max_num_tiles
899
+ self.vision_output_dim = config.vision_config.vision_output_dim
900
+ self.pad_token_id = \
901
+ config.pad_token_id if config.pad_token_id is not None else -1
902
+ self.image_size = config.vision_config.image_size
903
+
904
+ self.vision_model = MllamaVisionModel(config.vision_config)
905
+ self.language_model = MllamaForCausalLM(
906
+ config.text_config,
907
+ cache_config=cache_config,
908
+ quant_config=quant_config,
909
+ )
910
+ self.multi_modal_projector = nn.Linear(
911
+ config.vision_config.vision_output_dim,
912
+ config.text_config.hidden_size,
913
+ bias=True,
914
+ )
915
+ self.logits_processor = LogitsProcessor(config.output_hidden_states,
916
+ config.text_config.vocab_size)
917
+ self.sampler = Sampler()
918
+
919
+ def compute_logits(
920
+ self,
921
+ hidden_states: torch.Tensor,
922
+ sampling_metadata: SamplingMetadata,
923
+ ) -> Optional[torch.Tensor]:
924
+ logits = self.logits_processor(self.language_model.lm_head,
925
+ hidden_states, sampling_metadata)
926
+ return logits
927
+
928
+ def sample(
929
+ self,
930
+ logits: torch.Tensor,
931
+ sampling_metadata: SamplingMetadata,
932
+ ) -> Optional[SamplerOutput]:
933
+ next_tokens = self.sampler(logits, sampling_metadata)
934
+ return next_tokens
935
+
936
+ def _parse_and_validate_image_input(self, **kwargs: object):
937
+ # tensor with the same shape will be batched together by
938
+ # MultiModalInputs.batch, so pixel_values here can be:
939
+ # - List[List[torch.Tensor]]:
940
+ # with shape (num_tiles, 3, image_res, image_res)
941
+ # - List[torch.Tensor]:
942
+ # with shape (num_image, num_tiles, 3, image_res, image_res)
943
+ # - torch.Tensor:
944
+ # with shape (bs, num_image, num_tiles, 3, image_res, image_res)
945
+ pixel_values: Optional[Union[List[List[torch.Tensor]],
946
+ List[torch.Tensor],
947
+ torch.Tensor]] = kwargs.pop(
948
+ "pixel_values", None)
949
+ image_embeds: Optional[Union[List[List[torch.Tensor]],
950
+ List[torch.Tensor],
951
+ torch.Tensor]] = kwargs.pop(
952
+ "image_embeds", None)
953
+ aspect_ratio_ids: Optional[Union[List[List[torch.Tensor]],
954
+ List[torch.Tensor],
955
+ torch.Tensor]] = kwargs.pop(
956
+ "aspect_ratio_ids", None)
957
+ aspect_ratio_mask: Optional[Union[List[List[torch.Tensor]],
958
+ List[torch.Tensor],
959
+ torch.Tensor]] = kwargs.pop(
960
+ "aspect_ratio_mask", None)
961
+
962
+ if pixel_values is None and image_embeds is None:
963
+ return None
964
+
965
+ if pixel_values is not None and image_embeds is not None:
966
+ raise ValueError(
967
+ "Both pixel values and image embeds are provided.")
968
+
969
+ if pixel_values is not None:
970
+ assert aspect_ratio_ids is not None
971
+ assert aspect_ratio_mask is not None
972
+ max_num_images = max([len(x[0]) for x in pixel_values])
973
+ if max_num_images == 0:
974
+ raise ValueError("No images provided.")
975
+ max_num_tiles = max(
976
+ max([len(x) for x in y[0]]) for y in pixel_values)
977
+ device = self.multi_modal_projector.weight.device
978
+ bsz = len(pixel_values)
979
+ out_num_tiles = []
980
+ out_images = torch.zeros(
981
+ bsz,
982
+ max_num_images,
983
+ max_num_tiles,
984
+ 3,
985
+ self.image_size,
986
+ self.image_size,
987
+ dtype=torch.float32,
988
+ device=device,
989
+ )
990
+ out_ar_ids = torch.ones(bsz,
991
+ max_num_images,
992
+ dtype=torch.int64,
993
+ device=device)
994
+ out_ar_mask = torch.zeros(bsz,
995
+ max_num_images,
996
+ max_num_tiles,
997
+ dtype=torch.int64,
998
+ device=device)
999
+ for b in range(len(pixel_values)):
1000
+ _num_tiles = []
1001
+ for i in range(len(pixel_values[b][0])):
1002
+ img = pixel_values[b][0][i]
1003
+ out_images[b, i, :img.shape[0]] = img
1004
+ out_ar_ids[b, i] = aspect_ratio_ids[b][0][i]
1005
+ out_ar_mask[b, i] = aspect_ratio_mask[b][0][i]
1006
+ _num_tiles.append(img.shape[0])
1007
+ out_num_tiles.append(_num_tiles)
1008
+
1009
+ return MllamaImagePixelInputs(
1010
+ type="pixel_values",
1011
+ data=out_images,
1012
+ aspect_ratio_ids=out_ar_ids,
1013
+ aspect_ratio_mask=out_ar_mask,
1014
+ )
1015
+
1016
+ if image_embeds is not None:
1017
+ raise NotImplementedError
1018
+
1019
+ raise AssertionError("This line should be unreachable.")
1020
+
1021
+ def flat_encoder_result(self, cross_attention_states: torch.Tensor,
1022
+ attn_metadata: AttentionMetadata):
1023
+
1024
+ cross_attention_states_flat = torch.zeros(
1025
+ sum(attn_metadata.encoder_seq_lens),
1026
+ cross_attention_states.shape[-1],
1027
+ device=cross_attention_states.device,
1028
+ dtype=cross_attention_states.dtype)
1029
+ start_pos = 0
1030
+ for seq_len, vision_token_in_batch in zip(
1031
+ attn_metadata.encoder_seq_lens, cross_attention_states):
1032
+ end_pos = start_pos + seq_len
1033
+ cross_attention_states_flat[
1034
+ start_pos:end_pos] = vision_token_in_batch[:seq_len]
1035
+ start_pos = end_pos
1036
+ cross_attention_states = cross_attention_states_flat
1037
+
1038
+ full_text_row_masked_out_mask = torch.ones(
1039
+ (attn_metadata.num_prefill_tokens, 1), dtype=torch.bool)
1040
+ start_pos = 0
1041
+ for seq_len, encoder_seq_len in zip(
1042
+ attn_metadata.seq_lens_tensor.cpu(),
1043
+ attn_metadata.encoder_seq_lens):
1044
+ if encoder_seq_len == 0:
1045
+ full_text_row_masked_out_mask[start_pos:start_pos +
1046
+ seq_len] = False
1047
+ start_pos += seq_len
1048
+ full_text_row_masked_out_mask = full_text_row_masked_out_mask.to(
1049
+ cross_attention_states.device)
1050
+
1051
+ return cross_attention_states, full_text_row_masked_out_mask
1052
+
1053
+ def forward(
1054
+ self,
1055
+ input_ids: torch.Tensor,
1056
+ positions: torch.Tensor,
1057
+ kv_caches: List[torch.Tensor],
1058
+ attn_metadata: AttentionMetadata,
1059
+ **kwargs: object,
1060
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1061
+ if attn_metadata.num_prefill_tokens > 0 and \
1062
+ attn_metadata.num_decode_tokens > 0:
1063
+ raise ValueError("Chunk prefill not supported")
1064
+ image_inputs = self._parse_and_validate_image_input(**kwargs)
1065
+ if image_inputs is None:
1066
+ cross_attention_mask = None
1067
+ full_text_row_masked_out_mask = (
1068
+ attn_metadata.encoder_seq_lens_tensor != 0).reshape(-1, 1).to(
1069
+ input_ids.device)
1070
+ cross_attention_states = None
1071
+ skip_cross_attention = max(attn_metadata.encoder_seq_lens) == 0
1072
+ else:
1073
+ # NOTE: llama's reference implementation runs vision model on CPU
1074
+ pixel_values = image_inputs['data']
1075
+ aspect_ratio_ids = image_inputs['aspect_ratio_ids']
1076
+ aspect_ratio_mask = image_inputs['aspect_ratio_mask']
1077
+ cross_attention_states = self.vision_model(pixel_values,
1078
+ aspect_ratio_ids,
1079
+ aspect_ratio_mask)
1080
+ cross_attention_states = self.multi_modal_projector(
1081
+ cross_attention_states)
1082
+
1083
+ bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
1084
+ cross_attention_states = cross_attention_states.view(
1085
+ bsz, -1, image_token_dim)
1086
+
1087
+ cross_attention_states, full_text_row_masked_out_mask = \
1088
+ self.flat_encoder_result(cross_attention_states, attn_metadata)
1089
+ skip_cross_attention = False
1090
+ # TODO: support multi-image by this mask
1091
+ cross_attention_mask = None
1092
+
1093
+ outputs = self.language_model(
1094
+ input_ids=input_ids,
1095
+ positions=positions,
1096
+ cross_attention_states=cross_attention_states,
1097
+ cross_attention_mask=cross_attention_mask,
1098
+ full_text_row_masked_out_mask=full_text_row_masked_out_mask,
1099
+ kv_caches=kv_caches,
1100
+ attn_metadata=attn_metadata,
1101
+ skip_cross_attention=skip_cross_attention,
1102
+ )
1103
+
1104
+ return outputs
1105
+
1106
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1107
+ stacked_params_mapping = [
1108
+ # (param_name, shard_name, shard_id)
1109
+ (".qkv_proj", ".q_proj", "q"),
1110
+ (".qkv_proj", ".k_proj", "k"),
1111
+ (".qkv_proj", ".v_proj", "v"),
1112
+ (".gate_up_proj", ".gate_proj", 0),
1113
+ (".gate_up_proj", ".up_proj", 1),
1114
+ ]
1115
+ params_dict = dict(self.named_parameters())
1116
+ updated_params = set()
1117
+ for name, loaded_weight in weights:
1118
+ if 'patch_embedding.weight' in name:
1119
+ name = name.replace('patch_embedding.weight',
1120
+ 'patch_embedding._linear.weight')
1121
+ loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1122
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
1123
+ if weight_name not in name:
1124
+ continue
1125
+ name = name.replace(weight_name, param_name)
1126
+ param = params_dict[name]
1127
+ updated_params.add(name)
1128
+ weight_loader = param.weight_loader
1129
+ weight_loader(param, loaded_weight, shard_id)
1130
+ break
1131
+ else:
1132
+ param = params_dict.pop(name)
1133
+ weight_loader = getattr(param, "weight_loader",
1134
+ default_weight_loader)
1135
+ weight_loader(param, loaded_weight)
vllm/multimodal/base.py
+6
@@ -54,6 +54,12 @@ class MultiModalInputs(_MultiModalInputsBase):
54
if isinstance(nested_tensors, torch.Tensor):
55
return nested_tensors
56
57
+ if isinstance(nested_tensors, np.ndarray):
58
+ return torch.from_numpy(nested_tensors)
59
+
60
+ if isinstance(nested_tensors, (int, float)):
61
+ return torch.tensor(nested_tensors)
62
+
63
stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
64
if not is_list_of(stacked, torch.Tensor, check="all"):
65
# Only tensors (not lists) can be stacked.
vllm/multimodal/image.py
+5
@@ -2,6 +2,7 @@ from functools import lru_cache
2
3
import torch
4
from PIL import Image
5
+from transformers.image_processing_base import BatchFeature
6
7
from vllm.config import ModelConfig
8
from vllm.inputs.registry import InputContext
@@ -39,6 +40,10 @@ class ImagePlugin(MultiModalPlugin):
40
) -> MultiModalInputs:
41
model_config = ctx.model_config
42
43
+ # Processed by input processor
44
+ if isinstance(data, BatchFeature):
45
+ return MultiModalInputs(data.data)
46
+
47
# PIL image
48
if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
49
image_processor = self._get_hf_image_processor(model_config)
vllm/sequence.py
+10
-2
@@ -13,6 +13,7 @@ from typing import Set, Tuple, Union, cast
13
import msgspec
14
import torch
15
16
+from vllm.inputs import EncoderDecoderLLMInputs, LLMInputs
17
from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
18
from vllm.lora.request import LoRARequest
19
from vllm.pooling_params import PoolingParams
@@ -21,7 +22,6 @@ from vllm.sampling_params import SamplingParams
22
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
23
24
if TYPE_CHECKING:
24
- from vllm.inputs import LLMInputs
25
from vllm.multimodal.base import MultiModalDataDict
26
27
VLLM_TOKEN_ID_ARRAY_TYPE = "l"
@@ -471,7 +471,15 @@ class Sequence:
471
472
@property
473
def multi_modal_data(self) -> "MultiModalDataDict":
474
- return self.inputs.get("multi_modal_data") or {}
474
+ if self.inputs.get("multi_modal_data") and self.inputs.get(
475
+ "encoder_multi_modal_data"):
476
+ raise ValueError(
477
+ "Multi-modal data in both encoder and decoder is not supported."
478
+ )
479
+ inputs = self.inputs
480
+ return self.inputs.get("multi_modal_data") or (cast(
481
+ EncoderDecoderLLMInputs,
482
+ inputs).get("encoder_multi_modal_data")) or {}
483
484
@property
485
def lora_int_id(self) -> int:
vllm/transformers_utils/config.py
+13
-4
@@ -22,9 +22,10 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
22
EAGLEConfig, ExaoneConfig,
23
GraniteConfig, InternVLChatConfig,
24
JAISConfig, MedusaConfig,
25
- MLPSpeculatorConfig, MPTConfig,
26
- NemotronConfig, RWConfig,
27
- SolarConfig, UltravoxConfig)
25
+ MllamaConfig, MLPSpeculatorConfig,
26
+ MPTConfig, NemotronConfig,
27
+ RWConfig, SolarConfig,
28
+ UltravoxConfig)
29
# yapf: enable
30
from vllm.transformers_utils.utils import check_gguf_file
31
@@ -37,6 +38,10 @@ MISTRAL_CONFIG_NAME = "params.json"
38
39
logger = init_logger(__name__)
40
41
+_CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
42
+ "mllama": MllamaConfig
43
+}
44
+
45
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
46
"chatglm": ChatGLMConfig,
47
"dbrx": DbrxConfig,
@@ -55,11 +60,15 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
60
# Granite can be removed from here once we have upgraded to
61
# transformers 4.45+
62
"granite": GraniteConfig,
63
+ **_CONFIG_REGISTRY_OVERRIDE_HF
64
}
65
66
for name, cls in _CONFIG_REGISTRY.items():
67
with contextlib.suppress(ValueError):
62
- AutoConfig.register(name, cls)
68
+ if name in _CONFIG_REGISTRY_OVERRIDE_HF:
69
+ AutoConfig.register(name, cls, exist_ok=True)
70
+ else:
71
+ AutoConfig.register(name, cls)
72
73
74
class ConfigFormat(str, enum.Enum):
vllm/transformers_utils/configs/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.transformers_utils.configs.granite import GraniteConfig
10
from vllm.transformers_utils.configs.internvl import InternVLChatConfig
11
from vllm.transformers_utils.configs.jais import JAISConfig
12
from vllm.transformers_utils.configs.medusa import MedusaConfig
13
+from vllm.transformers_utils.configs.mllama import MllamaConfig
14
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
15
from vllm.transformers_utils.configs.mpt import MPTConfig
16
from vllm.transformers_utils.configs.nemotron import NemotronConfig
@@ -26,6 +27,7 @@ __all__ = [
27
"MedusaConfig",
28
"EAGLEConfig",
29
"ExaoneConfig",
30
+ "MllamaConfig",
31
"MLPSpeculatorConfig",
32
"NemotronConfig",
33
"SolarConfig",
vllm/transformers_utils/configs/mllama.py
new
+28
@@ -0,0 +1,28 @@
1
+from transformers.models.mllama import configuration_mllama as mllama_hf_config
2
+
3
+
4
+class MllamaTextConfig(mllama_hf_config.MllamaTextConfig):
5
+ '''
6
+ Use this class to override is_encoder_decoder:
7
+ - transformers regards mllama as is_encoder_decoder=False
8
+ - vllm needs is_encoder_decoder=True to enable cross-attention
9
+ '''
10
+
11
+ def __init__(
12
+ self,
13
+ **kwargs,
14
+ ):
15
+ super().__init__(**kwargs)
16
+ self.is_encoder_decoder = True
17
+
18
+
19
+class MllamaConfig(mllama_hf_config.MllamaConfig):
20
+
21
+ def __init__(
22
+ self,
23
+ text_config=None,
24
+ **kwargs,
25
+ ):
26
+ if isinstance(text_config, dict):
27
+ text_config = MllamaTextConfig(**text_config)
28
+ super().__init__(text_config=text_config, **kwargs)
vllm/transformers_utils/tokenizer.py
-1
@@ -111,7 +111,6 @@ def get_tokenizer(
111
'encoding and decoding.',
112
FutureWarning,
113
stacklevel=2)
114
-
114
if tokenizer_mode == "mistral":
115
tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
116
revision=revision)
vllm/worker/enc_dec_model_runner.py
+30
-10
@@ -18,7 +18,8 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry
18
from vllm.logger import init_logger
19
from vllm.model_executor import SamplingMetadata
20
from vllm.model_executor.layers.sampler import SamplerOutput
21
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
21
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
22
+ MultiModalRegistry)
23
from vllm.sampling_params import SamplingParams
24
from vllm.sequence import (IntermediateTensors, PoolerOutput,
25
SequenceGroupMetadata)
@@ -52,6 +53,7 @@ class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
53
"virtual_engine": self.virtual_engine,
54
"request_ids_to_seq_ids": self.request_ids_to_seq_ids,
55
"finished_requests_ids": self.finished_requests_ids,
56
+ "multi_modal_kwargs": self.multi_modal_kwargs,
57
}
58
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
59
_add_sampling_metadata_broadcastable_dict(tensor_dict,
@@ -194,6 +196,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
196
"finished_requests_ids": model_input.finished_requests_ids,
197
"request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
198
} if self.has_seqlen_agnostic else {}
199
+
200
+ multi_modal_kwargs = model_input.multi_modal_kwargs or {}
201
hidden_or_intermediate_states = model_executable(
202
input_ids=model_input.input_tokens,
203
positions=model_input.input_positions,
@@ -202,6 +206,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
206
kv_caches=kv_caches,
207
attn_metadata=model_input.attn_metadata,
208
intermediate_tensors=intermediate_tensors,
209
+ **MultiModalInputs.as_kwargs(multi_modal_kwargs,
210
+ device=self.device),
211
**seqlen_agnostic_kwargs)
212
213
logits = self.model.compute_logits(hidden_or_intermediate_states,
@@ -288,8 +294,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
294
max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
295
self.model_config)
296
if max_mm_tokens > 0:
291
- raise NotImplementedError(
292
- "Multi-modal encoder-decoder models are not supported yet")
297
+ logger.info("Starting profile run for multi-modal models.")
298
299
batch_size = 0
300
for group_id in range(max_num_seqs):
@@ -297,24 +302,39 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
302
(group_id < max_num_batched_tokens % max_num_seqs))
303
batch_size += seq_len
304
300
- seq_data, _ = self.input_registry \
301
- .dummy_data_for_profiling(self.model_config,
305
+ decoder_seq_data, decoder_dummy_multi_modal_data \
306
+ = self.input_registry.dummy_data_for_profiling(
307
+ self.model_config,
308
seq_len,
303
- self.mm_registry)
309
+ self.mm_registry,
310
+ is_encoder_data=False)
311
+ encoder_seq_data, encoder_dummy_multi_modal_data \
312
+ = self.input_registry.dummy_data_for_profiling(
313
+ self.model_config,
314
+ seq_len,
315
+ self.mm_registry,
316
+ is_encoder_data=True)
317
318
# Having more tokens is over-conservative but otherwise fine
306
- assert len(seq_data.prompt_token_ids) >= seq_len, (
319
+ assert len(decoder_seq_data.prompt_token_ids) >= seq_len, (
320
f"Expected at least {seq_len} dummy tokens for profiling, "
308
- f"but got: {len(seq_data.prompt_token_ids)}")
321
+ f"but got: {len(decoder_seq_data.prompt_token_ids)}")
322
+
323
+ assert decoder_dummy_multi_modal_data is None or \
324
+ encoder_dummy_multi_modal_data is None, (
325
+ "Multi-modal data can't be provided in both encoder and decoder"
326
+ )
327
328
seq = SequenceGroupMetadata(
329
request_id=str(group_id),
330
is_prompt=True,
313
- seq_data={group_id: seq_data},
331
+ seq_data={group_id: decoder_seq_data},
332
sampling_params=sampling_params,
333
block_tables=None,
316
- encoder_seq_data=seq_data,
334
+ encoder_seq_data=encoder_seq_data,
335
cross_block_table=None,
336
+ multi_modal_data=decoder_dummy_multi_modal_data
337
+ or encoder_dummy_multi_modal_data,
338
)
339
seqs.append(seq)
340
vllm/worker/utils.py
-4
@@ -39,10 +39,6 @@ def assert_enc_dec_mr_supported_scenario(
39
raise NotImplementedError(
40
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
41
42
- if enc_dec_mr.model_config.is_multimodal_model:
43
- raise NotImplementedError(
44
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_MM'])
45
-
42
if enc_dec_mr.scheduler_config.num_lookahead_slots > 0:
43
raise NotImplementedError(
44
STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SPEC_DEC'])