@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 770ec6024f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Model] Add support for the multi-modal Llama 3.2 model (#8811)

Co-authored-by: simon-mo <xmo@berkeley.edu> Co-authored-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: Simon Mo <simon.mo@hey.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Chen Zhang committed Sep 25, 2024 at 13:29 UTC 770ec6024fc00cd696899f5c6fdc53b7148876e6
24 files changed +1647 -45
docs/source/models/supported_models.rst
+5
@@ -254,6 +254,11 @@ Multimodal Language Models
254 - Image\ :sup:`+`
255 - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
256 -
257 + * - :code:`MllamaForConditionalGeneration`
258 + - Llama 3.2
259 + - Image
260 + - :code:`meta-llama/Llama-3.2-90B-Vision-Instruct`, :code:`meta-llama/Llama-3.2-11B-Vision`, etc.
261 + -
262 * - :code:`PaliGemmaForConditionalGeneration`
263 - PaliGemma
264 - Image\ :sup:`E`
examples/offline_inference_vision_language.py
+24
@@ -242,6 +242,29 @@ def run_qwen2_vl(question, modality):
242 return llm, prompt, stop_token_ids
243
244
245 +# LLama
246 +def run_mllama(question, modality):
247 + assert modality == "image"
248 +
249 + model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
250 +
251 + # Note: The default setting of max_num_seqs (256) and
252 + # max_model_len (131072) for this model may cause OOM.
253 + # You may lower either to run this example on lower-end GPUs.
254 +
255 + # The configuration below has been confirmed to launch on a
256 + # single H100 GPU.
257 + llm = LLM(
258 + model=model_name,
259 + max_num_seqs=16,
260 + enforce_eager=True,
261 + )
262 +
263 + prompt = f"<|image|><|begin_of_text|>{question}"
264 + stop_token_ids = None
265 + return llm, prompt, stop_token_ids
266 +
267 +
268 model_example_map = {
269 "llava": run_llava,
270 "llava-next": run_llava_next,
@@ -256,6 +279,7 @@ model_example_map = {
279 "internvl_chat": run_internvl,
280 "qwen_vl": run_qwen_vl,
281 "qwen2_vl": run_qwen2_vl,
282 + "mllama": run_mllama,
283 }
284
285
examples/openai_vision_api_client.py
+2 -2
@@ -38,7 +38,7 @@ chat_completion_from_url = client.chat.completions.create(
38 "content": [
39 {
40 "type": "text",
41 - "text": "What’s in this image?"
41 + "text": "What's in this image?"
42 },
43 {
44 "type": "image_url",
@@ -75,7 +75,7 @@ chat_completion_from_base64 = client.chat.completions.create(
75 "content": [
76 {
77 "type": "text",
78 - "text": "What’s in this image?"
78 + "text": "What's in this image?"
79 },
80 {
81 "type": "image_url",
requirements-common.txt
+1 -1
@@ -4,7 +4,7 @@ numpy < 2.0.0
4 requests
5 tqdm
6 py-cpuinfo
7 -transformers >= 4.43.2 # Required for Chameleon and Llama 3.1 hotfox.
7 +transformers >= 4.45.0 # Required for Llama 3.2.
8 tokenizers >= 0.19.1 # Required for Llama 3.
9 protobuf # Required by LlamaTokenizer.
10 fastapi < 0.113.0; python_version < '3.9'
tests/models/encoder_decoder/vision_language/__init__.py
tests/models/encoder_decoder/vision_language/test_mllama.py new
+283
@@ -0,0 +1,283 @@
1 +from typing import List, Optional, Tuple, Type, overload
2 +
3 +import pytest
4 +from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5 + BatchEncoding)
6 +
7 +from vllm.multimodal.utils import rescale_image_size
8 +from vllm.sequence import SampleLogprobs
9 +
10 +from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11 + _ImageAssets)
12 +from ....utils import multi_gpu_test
13 +from ...utils import check_logprobs_close
14 +
15 +_LIMIT_IMAGE_PER_PROMPT = 1
16 +
17 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18 + "stop_sign":
19 + "<|image|><|begin_of_text|>The meaning of the image is",
20 + "cherry_blossom":
21 + "<|image|><|begin_of_text|>The city is",
22 +})
23 +
24 +text_only_prompts = [
25 + "The color of the sky is blue but sometimes it can also be",
26 +]
27 +
28 +models = [
29 + "meta-llama/Llama-3.2-11B-Vision-Instruct",
30 +]
31 +
32 +
33 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
34 + Optional[SampleLogprobs]],
35 + model: str):
36 + """Sanitize vllm output to be comparable with hf output."""
37 + output_ids, output_str, out_logprobs = vllm_output
38 +
39 + config = AutoConfig.from_pretrained(model)
40 + image_token_id = config.image_token_index
41 +
42 + tokenizer = AutoTokenizer.from_pretrained(model)
43 + eos_token_id = tokenizer.eos_token_id
44 +
45 + hf_output_ids = [
46 + token_id for idx, token_id in enumerate(output_ids)
47 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
48 + ]
49 +
50 + assert output_str[0] == " "
51 + hf_output_str = output_str[1:]
52 + if hf_output_ids[-1] == eos_token_id:
53 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
54 +
55 + return hf_output_ids, hf_output_str, out_logprobs
56 +
57 +
58 +@overload
59 +def run_test(
60 + hf_runner: Type[HfRunner],
61 + vllm_runner: Type[VllmRunner],
62 + image_assets: _ImageAssets,
63 + model: str,
64 + *,
65 + size_factors: List[float],
66 + dtype: str,
67 + max_tokens: int,
68 + num_logprobs: int,
69 + tensor_parallel_size: int,
70 + distributed_executor_backend: Optional[str] = None,
71 +):
72 + ...
73 +
74 +
75 +@overload
76 +def run_test(
77 + hf_runner: Type[HfRunner],
78 + vllm_runner: Type[VllmRunner],
79 + image_assets: _ImageAssets,
80 + model: str,
81 + *,
82 + sizes: List[Tuple[int, int]],
83 + dtype: str,
84 + max_tokens: int,
85 + num_logprobs: int,
86 + tensor_parallel_size: int,
87 + distributed_executor_backend: Optional[str] = None,
88 +):
89 + ...
90 +
91 +
92 +def run_test(
93 + hf_runner: Type[HfRunner],
94 + vllm_runner: Type[VllmRunner],
95 + image_assets: _ImageAssets,
96 + model: str,
97 + *,
98 + size_factors: Optional[List[float]] = None,
99 + sizes: Optional[List[Tuple[int, int]]] = None,
100 + dtype: str,
101 + max_tokens: int,
102 + num_logprobs: int,
103 + tensor_parallel_size: int,
104 + distributed_executor_backend: Optional[str] = None,
105 +):
106 + images = [asset.pil_image for asset in image_assets]
107 +
108 + if size_factors is not None:
109 + inputs_per_image = [(
110 + [prompt for _ in size_factors],
111 + [rescale_image_size(image, factor) for factor in size_factors],
112 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
113 + elif sizes is not None:
114 + inputs_per_image = [(
115 + [
116 + prompt if size is not None else text_only_prompts[0]
117 + for size in sizes
118 + ],
119 + [
120 + image.resize(size) if size is not None else None
121 + for size in sizes
122 + ],
123 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
124 + if len(sizes) == 0:
125 + inputs_per_image.append(
126 + (text_only_prompts, [None] * len(text_only_prompts)))
127 + else:
128 + raise ValueError("You must provide either `size_factors` or `sizes`")
129 +
130 + _run_test(hf_runner,
131 + vllm_runner,
132 + inputs_per_image,
133 + model,
134 + dtype=dtype,
135 + max_tokens=max_tokens,
136 + num_logprobs=num_logprobs,
137 + tensor_parallel_size=tensor_parallel_size,
138 + distributed_executor_backend=distributed_executor_backend)
139 +
140 +
141 +def _run_test(
142 + hf_runner: Type[HfRunner],
143 + vllm_runner: Type[VllmRunner],
144 + inputs: List[Tuple[List[str], PromptImageInput]],
145 + model: str,
146 + *,
147 + dtype: str,
148 + max_tokens: int,
149 + num_logprobs: int,
150 + tensor_parallel_size: int,
151 + distributed_executor_backend: Optional[str] = None,
152 +):
153 + """Inference result should be the same between hf and vllm.
154 +
155 + All the image fixtures for the test are from IMAGE_ASSETS.
156 + For huggingface runner, we provide the PIL images as input.
157 + For vllm runner, we provide MultiModalDataDict objects
158 + and corresponding MultiModalConfig as input.
159 + Note, the text input is also adjusted to abide by vllm contract.
160 + The text output is sanitized to be able to compare with hf.
161 + """
162 + # NOTE: take care of the order. run vLLM first, and then run HF.
163 + # vLLM needs a fresh new process without cuda initialization.
164 + # if we run HF first, the cuda initialization will be done and it
165 + # will hurt multiprocessing backend with fork method (the default method).
166 +
167 + # max_model_len should be greater than image_feature_size
168 + with vllm_runner(model,
169 + dtype=dtype,
170 + max_num_seqs=16,
171 + max_model_len=4096,
172 + tensor_parallel_size=tensor_parallel_size,
173 + distributed_executor_backend=distributed_executor_backend,
174 + enforce_eager=True,
175 + limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
176 + }) as vllm_model:
177 + vllm_outputs_per_image = [
178 + vllm_model.generate_greedy_logprobs(prompts,
179 + max_tokens,
180 + num_logprobs=num_logprobs,
181 + images=images)
182 + for prompts, images in inputs
183 + ]
184 +
185 + def process(hf_inputs: BatchEncoding):
186 + return hf_inputs
187 +
188 + from transformers import AutoConfig
189 + from transformers.models.mllama import MllamaConfig as MllamaConfigHf
190 +
191 + # use transformer's MllamaConfig for hf_runner
192 + # and vllm's MllamaConfig for vllm_runner
193 + AutoConfig.register("mllama", MllamaConfigHf, exist_ok=True)
194 + with hf_runner(model,
195 + dtype=dtype,
196 + postprocess_inputs=process,
197 + auto_cls=AutoModelForVision2Seq) as hf_model:
198 + hf_outputs_per_image = [
199 + hf_model.generate_greedy_logprobs_limit(prompts,
200 + max_tokens,
201 + num_logprobs=num_logprobs,
202 + images=images)
203 + for prompts, images in inputs
204 + ]
205 +
206 + from vllm.transformers_utils.configs.mllama import MllamaConfig
207 + AutoConfig.register("mllama", MllamaConfig, exist_ok=True)
208 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
209 + vllm_outputs_per_image):
210 + check_logprobs_close(
211 + outputs_0_lst=hf_outputs,
212 + outputs_1_lst=[
213 + vllm_to_hf_output(vllm_output, model)
214 + for vllm_output in vllm_outputs
215 + ],
216 + name_0="hf",
217 + name_1="vllm",
218 + )
219 +
220 +
221 +@pytest.mark.parametrize("model", models)
222 +@pytest.mark.parametrize(
223 + "sizes",
224 + [
225 + # Text only
226 + [],
227 + # Single-size
228 + [(512, 512)],
229 + # Single-size, batched
230 + [(512, 512), (512, 512), (512, 512)],
231 + # Multi-size, batched
232 + [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
233 + (1024, 1024), (512, 1536), (512, 2028)],
234 + # Multi-size, batched, including text only
235 + [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
236 + (1024, 1024), (512, 1536), (512, 2028), None],
237 + # mllama has 8 possible aspect ratios, carefully set the sizes
238 + # to cover all of them
239 + ],
240 +)
241 +@pytest.mark.parametrize("dtype", ["bfloat16"])
242 +@pytest.mark.parametrize("max_tokens", [128])
243 +@pytest.mark.parametrize("num_logprobs", [5])
244 +def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
245 + max_tokens, num_logprobs) -> None:
246 + run_test(
247 + hf_runner,
248 + vllm_runner,
249 + image_assets,
250 + model,
251 + sizes=sizes,
252 + dtype=dtype,
253 + max_tokens=max_tokens,
254 + num_logprobs=num_logprobs,
255 + tensor_parallel_size=1,
256 + )
257 +
258 +
259 +@multi_gpu_test(num_gpus=2)
260 +@pytest.mark.parametrize("model", models)
261 +@pytest.mark.parametrize(
262 + "sizes",
263 + [
264 + [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
265 + (1024, 1024), (512, 1536), (512, 2028), None],
266 + ],
267 +)
268 +@pytest.mark.parametrize("dtype", ["bfloat16"])
269 +@pytest.mark.parametrize("max_tokens", [128])
270 +@pytest.mark.parametrize("num_logprobs", [5])
271 +def test_models_distributed(hf_runner, vllm_runner, image_assets, model, sizes,
272 + dtype, max_tokens, num_logprobs) -> None:
273 + run_test(
274 + hf_runner,
275 + vllm_runner,
276 + image_assets,
277 + model,
278 + sizes=sizes,
279 + dtype=dtype,
280 + max_tokens=max_tokens,
281 + num_logprobs=num_logprobs,
282 + tensor_parallel_size=2,
283 + )
vllm/config.py
+3 -1
@@ -576,7 +576,9 @@ class ModelConfig:
576 @property
577 def is_encoder_decoder_model(self) -> bool:
578 """Extract the HF encoder/decoder model flag."""
579 - return getattr(self.hf_config, "is_encoder_decoder", False)
579 + return getattr(self.hf_config, "is_encoder_decoder", False) or (
580 + (hasattr(self.hf_config, "text_config") and getattr(
581 + self.hf_config.text_config, "is_encoder_decoder", False)))
582
583 @property
584 def is_embedding_model(self) -> bool:
vllm/engine/llm_engine.py
+5 -1
@@ -1734,7 +1734,11 @@ class LLMEngine:
1734
1735 def _validate_model_inputs(self, inputs: Union[LLMInputs,
1736 EncoderDecoderLLMInputs]):
1737 - if self.is_encoder_decoder_model():
1737 + if self.model_config.is_multimodal_model:
1738 + # For encoder-decoder multimodal models, the max_prompt_len
1739 + # restricts the decoder prompt length
1740 + prompt_ids = inputs.get("prompt_token_ids")
1741 + elif self.is_encoder_decoder_model():
1742 prompt_ids = inputs.get("encoder_prompt_token_ids")
1743 else:
1744 prompt_ids = inputs.get("prompt_token_ids")
vllm/entrypoints/chat_utils.py
+22 -6
@@ -159,6 +159,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
159 hf_config.image_token_index)
160 if model_type in ("chameleon", "internvl_chat"):
161 return "<image>"
162 + if model_type == "mllama":
163 + return "<|image|>"
164 if model_type == "qwen2_vl":
165 return "<|vision_start|><|image_pad|><|vision_end|>"
166
@@ -358,6 +360,7 @@ _TextParser = partial(cast, ChatCompletionContentPartTextParam)
360 _ImageParser = partial(cast, ChatCompletionContentPartImageParam)
361 _AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
362 _RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
363 +MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
364
365
366 def _parse_chat_message_content_parts(
@@ -368,7 +371,11 @@ def _parse_chat_message_content_parts(
371 texts: List[str] = []
372
373 mm_parser = mm_tracker.create_parser()
374 + keep_multimodal_content = \
375 + mm_tracker._model_config.hf_config.model_type in \
376 + MODEL_KEEP_MULTI_MODAL_CONTENT
377
378 + has_image = False
379 for part in parts:
380 part_type = part["type"]
381 if part_type == "text":
@@ -383,6 +390,7 @@ def _parse_chat_message_content_parts(
390 "will be ignored.")
391
392 mm_parser.parse_image(image_url["url"])
393 + has_image = True
394 elif part_type == "audio_url":
395 audio_url = _AudioParser(part)["audio_url"]
396
@@ -394,12 +402,20 @@ def _parse_chat_message_content_parts(
402 raise NotImplementedError(f"Unknown part type: {part_type}")
403
404 text_prompt = "\n".join(texts)
397 - mm_placeholder_counts = mm_parser.mm_placeholder_counts()
398 - if mm_placeholder_counts:
399 - text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
400 - text_prompt)
401 -
402 - return [ConversationMessage(role=role, content=text_prompt)]
405 + if keep_multimodal_content:
406 + text_prompt = "\n".join(texts)
407 + role_content = [{'type': 'text', 'text': text_prompt}]
408 +
409 + if has_image:
410 + role_content = [{'type': 'image'}] + role_content
411 + return [ConversationMessage(role=role,
412 + content=role_content)] # type: ignore
413 + else:
414 + mm_placeholder_counts = mm_parser.mm_placeholder_counts()
415 + if mm_placeholder_counts:
416 + text_prompt = _get_full_multimodal_text_prompt(
417 + mm_placeholder_counts, text_prompt)
418 + return [ConversationMessage(role=role, content=text_prompt)]
419
420
421 # No need to validate using Pydantic again
vllm/entrypoints/openai/serving_chat.py
+2
@@ -309,6 +309,8 @@ class OpenAIServingChat(OpenAIServing):
309 async for res in result_generator:
310 if res.prompt_token_ids is not None:
311 num_prompt_tokens = len(res.prompt_token_ids)
312 + if res.encoder_prompt_token_ids is not None:
313 + num_prompt_tokens += len(res.encoder_prompt_token_ids)
314
315 # We need to do it here, because if there are exceptions in
316 # the result_generator, it needs to be sent as the FIRST
vllm/inputs/data.py
+6
@@ -139,6 +139,12 @@ class EncoderDecoderLLMInputs(LLMInputs):
139 available.
140 """
141
142 + encoder_multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
143 + """
144 + Optional multi-modal data to pass to the encoder model,
145 + if the model supports it.
146 + """
147 +
148
149 _T1 = TypeVar("_T1",
150 bound=SingletonPromptInputs,
vllm/inputs/preprocess.py
+15 -7
@@ -128,6 +128,7 @@ class InputPreprocessor:
128 def _prepare_decoder_input_ids_for_generation(
129 self,
130 decoder_input_ids: Optional[List[int]],
131 + force_bos: bool = True,
132 ) -> List[int]:
133 """
134 Prepares `decoder_input_ids` for generation with encoder-decoder models.
@@ -157,8 +158,8 @@ class InputPreprocessor:
158 # use decoder_start_token_id as decoder_input_ids
159 decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
160
160 - if (len(decoder_input_ids) == 0
161 - or decoder_input_ids[0] != decoder_start_token_id):
161 + if force_bos and (len(decoder_input_ids) == 0
162 + or decoder_input_ids[0] != decoder_start_token_id):
163 decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
164
165 return decoder_input_ids
@@ -295,18 +296,25 @@ class InputPreprocessor:
296 encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
297 decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
298
298 - if encoder_mm_data is not None or decoder_mm_data is not None:
299 - raise ValueError("Multi-modal encoder-decoder models are "
300 - "not supported yet")
299 + if decoder_mm_data is not None:
300 + raise ValueError(
301 + "Multi-modality decoder inputs of encoder-decoder models are "
302 + "not supported yet")
303
302 - decoder_prompt_ids = (
303 - self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
304 + # For Multi-Modal models (e.g., mllama), the text input can be
305 + # <|image|><|begin_of_text|>hello world. And we should not add
306 + # another <|begin_of_text|> to the beginning.
307 + decoder_prompt_ids = (self._prepare_decoder_input_ids_for_generation(
308 + decoder_prompt_ids,
309 + force_bos=(encoder_mm_data is None and decoder_mm_data is None)))
310
311 return EncoderDecoderLLMInputs(
312 prompt_token_ids=decoder_prompt_ids,
313 prompt=decoder_prompt,
314 + multi_modal_data=decoder_mm_data,
315 encoder_prompt_token_ids=encoder_prompt_ids,
316 encoder_prompt=encoder_prompt,
317 + encoder_multi_modal_data=encoder_mm_data,
318 )
319
320 def _process_encoder_decoder_prompt(
vllm/inputs/registry.py
+48 -6
@@ -112,6 +112,8 @@ class InputRegistry:
112 def __init__(self) -> None:
113 self._dummy_factories_by_model_type: Dict[Type[nn.Module],
114 DummyDataFactory] = {}
115 + self._dummy_encoder_factories_by_model_type: Dict[
116 + Type[nn.Module], DummyDataFactory] = {}
117 self._input_processors_by_model_type: Dict[Type[nn.Module],
118 InputProcessor] = {}
119
@@ -162,11 +164,44 @@ class InputRegistry:
164 return self._dummy_factories_by_model_type \
165 .get(model_cls, self._default_dummy_data_factory)
166
167 + def register_dummy_encoder_data(self, factory: DummyDataFactory):
168 + """
169 + Register a dummy encoder data factory to a model class
170 +
171 + This is similar to :meth:`~register_dummy_data`, but for encoder input.
172 + """
173 +
174 + def wrapper(model_cls: N) -> N:
175 + if model_cls in self._dummy_encoder_factories_by_model_type:
176 + logger.warning(
177 + "Model class %s already has dummy encoder data "
178 + "registered to %s. It is overwritten by the new one.",
179 + model_cls, self)
180 +
181 + self._dummy_encoder_factories_by_model_type[model_cls] = factory
182 +
183 + return model_cls
184 +
185 + return wrapper
186 +
187 + def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
188 + if model_cls in self._dummy_encoder_factories_by_model_type:
189 + dummy_factory = self._dummy_encoder_factories_by_model_type[
190 + model_cls]
191 + else:
192 + logger.warning(
193 + "No dummy encoder data factory registered to %s. "
194 + "Using the dummy data factory for the model instead.",
195 + model_cls)
196 + dummy_factory = self._get_dummy_data_factory(model_cls)
197 + return dummy_factory
198 +
199 def dummy_data_for_profiling(
200 self,
201 model_config: "ModelConfig",
202 seq_len: int,
203 mm_registry: "MultiModalRegistry",
204 + is_encoder_data: bool = False,
205 ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
206 """
207 Create dummy data for profiling the memory usage of a model.
@@ -184,8 +219,10 @@ class InputRegistry:
219 from vllm.model_executor.model_loader import get_model_architecture
220
221 model_cls, _ = get_model_architecture(model_config)
187 - dummy_factory = self._get_dummy_data_factory(model_cls)
188 -
222 + if is_encoder_data:
223 + dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
224 + else:
225 + dummy_factory = self._get_dummy_data_factory(model_cls)
226 mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
227 mm_processor_kwargs = get_allowed_kwarg_only_overrides(
228 dummy_factory, overrides=model_config.mm_processor_kwargs)
@@ -196,10 +233,15 @@ class InputRegistry:
233
234 # Having more tokens is over-conservative but otherwise fine
235 num_tokens = seq_data.prompt_token_ids
199 - assert len(num_tokens) >= seq_len, (
200 - f"Expected at least {seq_len} dummy tokens for profiling, "
201 - f"but found {len(num_tokens)} tokens instead.")
202 -
236 + if len(num_tokens) < seq_len:
237 + if is_encoder_data:
238 + logger.warning(
239 + "Expected at least %d dummy encoder tokens for profiling, "
240 + "but found %d tokens instead.", seq_len, len(num_tokens))
241 + else:
242 + raise AssertionError(
243 + f"Expected at least {seq_len} dummy tokens for profiling, "
244 + f"but found {len(num_tokens)} tokens instead.")
245 if mm_data is not None:
246 for k, v in mm_data.items():
247 num_items = len(v) if isinstance(v, list) else 1
vllm/model_executor/models/__init__.py
+2
@@ -101,6 +101,8 @@ _MULTIMODAL_MODELS = {
101 "Qwen2VLForConditionalGeneration": ("qwen2_vl",
102 "Qwen2VLForConditionalGeneration"),
103 "UltravoxModel": ("ultravox", "UltravoxModel"),
104 + "MllamaForConditionalGeneration": ("mllama",
105 + "MllamaForConditionalGeneration"),
106 }
107 _CONDITIONAL_GENERATION_MODELS = {
108 "BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/mllama.py new
+1135
@@ -0,0 +1,1135 @@
1 +# coding=utf-8
2 +# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
3 +#
4 +# Licensed under the Apache License, Version 2.0 (the "License");
5 +# you may not use this file except in compliance with the License.
6 +# You may obtain a copy of the License at
7 +#
8 +# http://www.apache.org/licenses/LICENSE-2.0
9 +#
10 +# Unless required by applicable law or agreed to in writing, software
11 +# distributed under the License is distributed on an "AS IS" BASIS,
12 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 +# See the License for the specific language governing permissions and
14 +# limitations under the License.
15 +"""PyTorch Mllama model."""
16 +import math
17 +from array import array
18 +from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19 + TypedDict, Union)
20 +
21 +import torch
22 +import torch.nn.functional as F
23 +import torch.utils.checkpoint
24 +import transformers.models.mllama.configuration_mllama as config_mllama
25 +from PIL import Image
26 +from torch import nn
27 +from transformers.modeling_outputs import (BaseModelOutput,
28 + CausalLMOutputWithPast)
29 +from transformers.models.mllama.image_processing_mllama import (
30 + get_optimal_tiled_canvas)
31 +
32 +import vllm.distributed.parallel_state as ps
33 +from vllm.attention import Attention, AttentionMetadata, AttentionType
34 +from vllm.config import CacheConfig, MultiModalConfig
35 +from vllm.distributed import get_tensor_model_parallel_world_size
36 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
37 +from vllm.logger import init_logger
38 +from vllm.model_executor.layers.layernorm import RMSNorm
39 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
40 + QKVParallelLinear,
41 + RowParallelLinear)
42 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 +from vllm.model_executor.layers.quantization import QuantizationConfig
44 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
45 +from vllm.model_executor.layers.vocab_parallel_embedding import (
46 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
47 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
48 +from vllm.model_executor.sampling_metadata import SamplingMetadata
49 +from vllm.multimodal import MULTIMODAL_REGISTRY
50 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
51 +
52 +from .clip import CLIPMLP
53 +from .interfaces import SupportsMultiModal
54 +from .llama import LlamaDecoderLayer, LlamaMLP
55 +
56 +logger = init_logger(__name__)
57 +MLLAMA_IMAGE_TOKEN_ID = 128256
58 +MLLAMA_IMAGE_TOKEN = "<|image|>"
59 +
60 +
61 +class MllamaImagePixelInputs(TypedDict):
62 + type: Literal["pixel_values"]
63 + data: torch.Tensor
64 + """Shape: """
65 + """(batch_size, max_num_image, max_num_chunk, num_channel, height, width)"""
66 + aspect_ratio_ids: torch.Tensor
67 + """Shape: `(batch_size, max_num_image)`"""
68 + aspect_ratio_mask: torch.Tensor
69 + """Shape: `(batch_size, max_num_image, max_num_tiles)`"""
70 +
71 +
72 +# TODO: support LlamaImageEmbeddingInputs
73 +
74 +
75 +def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
76 + # move encoder_prompt to prompt
77 + if llm_inputs.get("prompt") is None:
78 + llm_inputs["prompt"] = llm_inputs["encoder_prompt"]
79 + llm_inputs["prompt_token_ids"] = llm_inputs["encoder_prompt_token_ids"]
80 +
81 + # process multi-modal data
82 + assert "decoder_multi_modal_data" not in llm_inputs, \
83 + "multi-modal data should be put in encoder message of mllama"
84 + multi_modal_data = llm_inputs.get("encoder_multi_modal_data")
85 +
86 + if multi_modal_data is None or "image" not in multi_modal_data \
87 + or multi_modal_data["image"] is None:
88 + # text-only
89 + llm_inputs["encoder_prompt"] = ""
90 + llm_inputs["encoder_prompt_token_ids"] = []
91 + llm_inputs["encoder_multi_modal_data"] = {}
92 + return llm_inputs
93 +
94 + # get num_tiles
95 + if isinstance(multi_modal_data['image'], Image.Image):
96 + multi_modal_data['image'] = [multi_modal_data['image']]
97 + hf_config = ctx.model_config.hf_config
98 + num_tiles = 0
99 + for image in multi_modal_data["image"]:
100 + width, height = image.size
101 + tile_size = hf_config.vision_config.image_size
102 + canvas_height, canvas_width = get_optimal_tiled_canvas(
103 + image_height=height,
104 + image_width=width,
105 + max_image_tiles=hf_config.vision_config.max_num_tiles,
106 + tile_size=tile_size,
107 + )
108 + num_tiles_height = canvas_height // tile_size
109 + num_tiles_width = canvas_width // tile_size
110 + num_tiles += num_tiles_height * num_tiles_width
111 +
112 + # set encoder prompt based on num_tiles
113 + assert hf_config.vision_config.image_size % 14 == 0, \
114 + "chunk size should be multiple of 14"
115 + token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
116 + num_tokens = num_tiles * token_per_chunk
117 + llm_inputs["encoder_prompt"] = MLLAMA_IMAGE_TOKEN * num_tokens
118 + llm_inputs["encoder_prompt_token_ids"] = [MLLAMA_IMAGE_TOKEN_ID
119 + ] * num_tokens
120 +
121 + return llm_inputs
122 +
123 +
124 +def get_max_mllama_image_tokens(ctx: InputContext) -> int:
125 + hf_config = ctx.model_config.hf_config
126 + token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
127 + return hf_config.vision_config.max_num_tiles * token_per_chunk
128 +
129 +
130 +def dummy_decoder_seq_data(seq_len: int, num_images: int):
131 + # <|image|> * num_images + 0 * (seq_len - num_images)
132 + assert seq_len >= num_images, \
133 + "seq_len should be greater than or equal to num_images"
134 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
135 + [MLLAMA_IMAGE_TOKEN_ID]) * num_images
136 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * (seq_len - num_images)
137 + return SequenceData(token_ids)
138 +
139 +
140 +def dummy_encoder_seq_data(ctx: InputContext, num_images: int):
141 + num_tokens = get_max_mllama_image_tokens(ctx) * num_images
142 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
143 + [MLLAMA_IMAGE_TOKEN_ID]) * num_tokens
144 + return SequenceData(token_ids)
145 +
146 +
147 +def dummy_image(num_images: int, ):
148 + width = height = 1024
149 + image = Image.new("RGB", (width, height), color=0)
150 + return {"image": image if num_images == 1 else [image] * num_images}
151 +
152 +
153 +def dummy_decoder_data_for_mllama(ctx: InputContext, seq_len: int,
154 + mm_counts: Mapping[str, int]):
155 + num_images = mm_counts["image"]
156 + return dummy_decoder_seq_data(seq_len, num_images), None
157 +
158 +
159 +def dummy_encoder_data_for_mllama(ctx: InputContext, seq_len: int,
160 + mm_counts: Mapping[str, int]):
161 + num_images = mm_counts["image"]
162 + return dummy_encoder_seq_data(ctx, num_images), dummy_image(num_images)
163 +
164 +
165 +def _prepare_aspect_ratio_attention_mask(
166 + aspect_ratio_mask: torch.Tensor,
167 + num_patches: int,
168 + target_length: int,
169 + dtype: torch.dtype,
170 +) -> torch.Tensor:
171 + # Expand aspect ratio mask to target_length
172 + batch_size, max_num_tiles = aspect_ratio_mask.shape
173 + attention_mask = aspect_ratio_mask.view(batch_size, max_num_tiles, 1,
174 + 1).to(dtype)
175 + attention_mask = attention_mask.repeat(1, 1, target_length, 1)
176 +
177 + # Mask padding patches
178 + pad_patches = target_length - num_patches
179 + attention_mask[:, :, -pad_patches:] = 0
180 +
181 + # Invert the mask (0 -> 1, 1 -> 0)
182 + attention_mask = 1 - attention_mask
183 +
184 + # Reshape to 2D and create 4D attention mask
185 + # (batch_size, 1, max_num_tiles*target_length, max_num_tiles*target_length)
186 + attention_mask = attention_mask.reshape(batch_size,
187 + max_num_tiles * target_length, 1)
188 + attention_mask = attention_mask @ attention_mask.transpose(
189 + -1, -2) * torch.finfo(dtype).min
190 + attention_mask = attention_mask.unsqueeze(1)
191 +
192 + return attention_mask
193 +
194 +
195 +class ColumnParallelConv2dPatch(torch.nn.Module):
196 + """Conv2D Patching layer with model parallelism.
197 + Column parallel over unfolded input.
198 + Arguments:
199 + in_channels: Input channels.
200 + out_channels: Output channels.
201 + kernel_size: Size of convolution kernel.
202 + stride (default 1): Stride for convolution.
203 + bias (default False): Use bias in Conv2d.
204 + Input: (bsz, in_channels, width, height)
205 + Output: (bsz, num_tokens, out_channels)
206 + """
207 +
208 + def __init__(
209 + self,
210 + in_channels: int,
211 + out_channels: int,
212 + kernel_size: Union[int, Tuple[int, int]],
213 + stride: Union[int, Tuple[int, int]],
214 + bias: bool = False,
215 + ) -> None:
216 + super().__init__()
217 + if isinstance(kernel_size, int):
218 + kernel_size = (kernel_size, kernel_size)
219 + self._unfold = torch.nn.Unfold(kernel_size=kernel_size, stride=stride)
220 + self._linear = ColumnParallelLinear(
221 + in_channels * kernel_size[0] * kernel_size[1],
222 + out_channels,
223 + bias=bias,
224 + )
225 +
226 + def forward(self, x: torch.Tensor) -> torch.Tensor:
227 + x = self._unfold(x)
228 + x = x.permute(0, 2, 1)
229 + x, _ = self._linear(x)
230 + return x
231 +
232 +
233 +class MllamaPrecomputedAspectRatioEmbedding(nn.Module):
234 +
235 + def __init__(self,
236 + config: config_mllama.MllamaVisionConfig,
237 + is_gated: bool = True):
238 + super().__init__()
239 + self.max_num_tiles = config.max_num_tiles
240 + self.hidden_size = config.hidden_size
241 + self.max_aspect_ratio_id = config.max_aspect_ratio_id
242 + self.is_gated = is_gated
243 +
244 + self.embedding = nn.Embedding(self.max_aspect_ratio_id + 1,
245 + self.max_num_tiles * self.hidden_size)
246 + if is_gated:
247 + self.gate = nn.Parameter(torch.zeros(1))
248 +
249 + def forward(self, hidden_state: torch.Tensor,
250 + aspect_ratio_ids: torch.Tensor) -> torch.Tensor:
251 + embeddings = self.embedding(aspect_ratio_ids)
252 + embeddings = embeddings.reshape(-1, self.max_num_tiles, 1,
253 + self.hidden_size)
254 +
255 + if self.is_gated:
256 + embeddings = embeddings * self.gate.tanh()
257 +
258 + hidden_state = hidden_state + embeddings
259 + return hidden_state
260 +
261 +
262 +class MllamaPrecomputedPositionEmbedding(nn.Module):
263 +
264 + def __init__(self, config: config_mllama.MllamaVisionConfig):
265 + super().__init__()
266 + self.max_num_tiles = config.max_num_tiles
267 + self.max_aspect_ratio_id = config.max_aspect_ratio_id
268 + self.num_patches = (config.image_size // config.patch_size)**2 + 1
269 + self.hidden_size = config.hidden_size
270 + self.scale = config.hidden_size**-0.5
271 +
272 + self.gate = nn.Parameter(torch.zeros(1))
273 +
274 + # position embedding
275 + position_embedding = torch.randn(self.num_patches, self.hidden_size)
276 + self.embedding = nn.Parameter(self.scale * position_embedding)
277 +
278 + # tile position embedding
279 + self.tile_embedding = nn.Embedding(
280 + self.max_aspect_ratio_id + 1,
281 + self.max_num_tiles * self.num_patches * self.hidden_size)
282 +
283 + def forward(self, hidden_state: torch.Tensor,
284 + aspect_ratio_ids: torch.Tensor) -> torch.Tensor:
285 + # position embeddings
286 + gated_position_embedding = (1 - self.gate.tanh()) * self.embedding
287 + hidden_state = hidden_state + gated_position_embedding.view(
288 + 1, 1, self.num_patches, self.hidden_size)
289 +
290 + # precomputed tile position embeddings
291 + tile_position_embedding = self.tile_embedding(aspect_ratio_ids)
292 + batch_size = hidden_state.shape[0]
293 + tile_position_embedding = tile_position_embedding.reshape(
294 + batch_size, self.max_num_tiles, self.num_patches, self.hidden_size)
295 + gated_tile_position_embedding = self.gate.tanh(
296 + ) * tile_position_embedding
297 + hidden_state = hidden_state + gated_tile_position_embedding
298 +
299 + return hidden_state
300 +
301 +
302 +# TODO: support other attention backends for attention in vision model
303 +class MllamaVisionSdpaAttention(nn.Module):
304 +
305 + def __init__(self, config: config_mllama.MllamaVisionConfig):
306 + super().__init__()
307 +
308 + model_parallel_size = get_tensor_model_parallel_world_size()
309 + self.embed_dim = config.hidden_size
310 + self.num_heads = config.attention_heads
311 + self.head_dim = config.hidden_size // config.attention_heads
312 + self.num_local_heads = self.num_heads // model_parallel_size
313 + self.q_size = self.num_local_heads * self.head_dim
314 + self.kv_size = self.num_local_heads * self.head_dim
315 +
316 + self.qkv_proj = QKVParallelLinear(
317 + self.embed_dim,
318 + self.head_dim,
319 + self.num_heads,
320 + bias=False,
321 + )
322 + self.o_proj = RowParallelLinear(
323 + self.num_heads * self.head_dim,
324 + self.embed_dim,
325 + bias=False,
326 + input_is_parallel=True,
327 + )
328 +
329 + def forward(
330 + self,
331 + hidden_state: torch.Tensor,
332 + attention_mask: Optional[torch.Tensor] = None,
333 + ) -> torch.Tensor:
334 + qkv, _ = self.qkv_proj(hidden_state)
335 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
336 + q = q.view(q.shape[0], q.shape[1], self.num_local_heads,
337 + self.head_dim).transpose(1, 2)
338 + k = k.view(k.shape[0], k.shape[1], self.num_local_heads,
339 + self.head_dim).transpose(1, 2)
340 + v = v.view(v.shape[0], v.shape[1], self.num_local_heads,
341 + self.head_dim).transpose(1, 2)
342 +
343 + # TODO: remove padding in image encoder
344 + attn_output = F.scaled_dot_product_attention(q,
345 + k,
346 + v,
347 + attn_mask=attention_mask,
348 + dropout_p=0.0)
349 +
350 + attn_output = attn_output.transpose(1, 2).contiguous()
351 + attn_output = attn_output.reshape(attn_output.shape[0],
352 + attn_output.shape[1], -1)
353 + output, _ = self.o_proj(attn_output)
354 + return output
355 +
356 +
357 +class MllamaVisionEncoderLayer(nn.Module):
358 +
359 + def __init__(self,
360 + config: config_mllama.MllamaVisionConfig,
361 + is_gated: bool = False):
362 + super().__init__()
363 +
364 + self.hidden_size = config.hidden_size
365 + self.num_attention_heads = config.attention_heads
366 + self.is_gated = is_gated
367 + self.intermediate_size = config.intermediate_size
368 +
369 + self.self_attn = MllamaVisionSdpaAttention(config)
370 + self.mlp = CLIPMLP(config)
371 +
372 + self.input_layernorm = nn.LayerNorm(self.hidden_size,
373 + eps=config.norm_eps)
374 + self.post_attention_layernorm = nn.LayerNorm(self.hidden_size,
375 + eps=config.norm_eps)
376 +
377 + # there used to be an if else here, no code path
378 + if is_gated:
379 + self.gate_attn = nn.Parameter(torch.ones(1) * math.pi / 4)
380 + self.gate_ffn = nn.Parameter(torch.ones(1) * math.pi / 4)
381 +
382 + def forward(
383 + self,
384 + hidden_state: torch.Tensor,
385 + attention_mask: Optional[torch.Tensor] = None,
386 + ):
387 + # Self Attention
388 + residual = hidden_state
389 + hidden_state = self.input_layernorm(hidden_state)
390 + hidden_state = self.self_attn(hidden_state,
391 + attention_mask=attention_mask)
392 + gate_attn = 1 if not self.is_gated else self.gate_attn.tanh()
393 + hidden_state = residual + gate_attn * hidden_state
394 +
395 + # Feed forward
396 + residual = hidden_state
397 + hidden_state = self.post_attention_layernorm(hidden_state)
398 + hidden_state = self.mlp(hidden_state)
399 + gate_ffn = 1 if not self.is_gated else self.gate_ffn.tanh()
400 + hidden_state = residual + gate_ffn * hidden_state
401 +
402 + return hidden_state
403 +
404 +
405 +class MllamaVisionEncoder(nn.Module):
406 +
407 + def __init__(self,
408 + config: config_mllama.MllamaVisionConfig,
409 + num_layers=32,
410 + is_gated=False,
411 + output_hidden_states=None):
412 + super().__init__()
413 + self.config = config
414 + self.layers = nn.ModuleList([
415 + MllamaVisionEncoderLayer(config, is_gated)
416 + for _ in range(num_layers)
417 + ])
418 + self.output_hidden_states = output_hidden_states or []
419 +
420 + def forward(
421 + self,
422 + hidden_states: torch.Tensor,
423 + attention_mask: Optional[torch.Tensor] = None,
424 + ) -> Union[Tuple, BaseModelOutput]:
425 + encoder_states = ()
426 +
427 + for i, encoder_layer in enumerate(self.layers):
428 + if i in self.output_hidden_states:
429 + encoder_states = encoder_states + (hidden_states, )
430 + hidden_states = encoder_layer(
431 + hidden_states,
432 + attention_mask,
433 + )
434 +
435 + if len(self.layers) - 1 in self.output_hidden_states:
436 + encoder_states = encoder_states + (hidden_states, )
437 +
438 + return hidden_states, encoder_states
439 +
440 +
441 +class MllamaVisionModel(nn.Module):
442 +
443 + def __init__(self, config: config_mllama.MllamaVisionConfig):
444 + super().__init__()
445 + self.image_size = config.image_size
446 + self.patch_size = config.patch_size
447 + self.max_num_tiles = config.max_num_tiles
448 + self.hidden_size = config.hidden_size
449 + self.in_channels = config.num_channels
450 + self.intermediate_layers_indices = config.intermediate_layers_indices
451 +
452 + self.num_patches = (self.image_size // self.patch_size)**2 + 1
453 + self.scale = config.hidden_size**-0.5
454 +
455 + self.patch_embedding = ColumnParallelConv2dPatch(
456 + in_channels=config.num_channels,
457 + out_channels=self.hidden_size,
458 + kernel_size=self.patch_size,
459 + stride=self.patch_size,
460 + bias=False,
461 + )
462 +
463 + self.class_embedding = nn.Parameter(self.scale *
464 + torch.randn(self.hidden_size))
465 + self.gated_positional_embedding = MllamaPrecomputedPositionEmbedding(
466 + config)
467 +
468 + self.pre_tile_positional_embedding = \
469 + MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True)
470 + self.post_tile_positional_embedding = \
471 + MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True)
472 +
473 + # layer norms
474 + self.layernorm_pre = nn.LayerNorm(self.hidden_size)
475 + self.layernorm_post = nn.LayerNorm(self.hidden_size)
476 +
477 + # encoders
478 + self.transformer = MllamaVisionEncoder(
479 + config,
480 + config.num_hidden_layers,
481 + is_gated=False,
482 + output_hidden_states=config.intermediate_layers_indices)
483 + self.global_transformer = MllamaVisionEncoder(config,
484 + config.num_global_layers,
485 + is_gated=True)
486 +
487 + def apply_class_embedding(self,
488 + hidden_state: torch.Tensor) -> torch.Tensor:
489 + batch_size, _, hidden_size = hidden_state.shape
490 + class_embedding = self.class_embedding.expand(batch_size, 1,
491 + hidden_size)
492 + hidden_state = torch.cat([class_embedding, hidden_state], dim=1)
493 + return hidden_state
494 +
495 + def forward(self, pixel_values: torch.Tensor,
496 + aspect_ratio_ids: torch.Tensor,
497 + aspect_ratio_mask: torch.Tensor) -> torch.Tensor:
498 + batch_size, num_concurrent_media, num_tiles, num_channels, \
499 + height, width = pixel_values.shape
500 +
501 + pixel_values = pixel_values.reshape(
502 + batch_size * num_concurrent_media * num_tiles, num_channels,
503 + height, width)
504 + aspect_ratio_ids = aspect_ratio_ids.reshape(
505 + batch_size * num_concurrent_media, -1)
506 +
507 + # patch embedding
508 + patch_embeds = self.patch_embedding(
509 + pixel_values.to(self.layernorm_pre.weight.dtype))
510 + hidden_state = patch_embeds
511 + hidden_state = ps.get_tp_group().all_gather(hidden_state)
512 +
513 + # tile embeddings
514 + _, num_patches, dim = hidden_state.shape
515 + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
516 + num_tiles, -1, dim)
517 + hidden_state = self.pre_tile_positional_embedding(
518 + hidden_state, aspect_ratio_ids)
519 +
520 + # apply cls token
521 + hidden_state = hidden_state.reshape(
522 + batch_size * num_concurrent_media * num_tiles, num_patches, dim)
523 + hidden_state = self.apply_class_embedding(hidden_state)
524 + num_patches += 1
525 +
526 + # apply position embeddings
527 + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
528 + num_tiles, num_patches, dim)
529 + hidden_state = self.gated_positional_embedding(hidden_state,
530 + aspect_ratio_ids)
531 +
532 + # apply encoder
533 + hidden_state = self.layernorm_pre(hidden_state)
534 +
535 + # Compute the number of tokens to pad
536 + num_padding_patches = (8 - (hidden_state.shape[-2] % 8)) % 8
537 + # Compute padding tuple for pad function
538 + padding = (
539 + 0, 0, 0, num_padding_patches
540 + ) # (pad_left, pad_right, pad_left for dim -2, pad_right for dim -2)
541 + # Pad the tensor
542 + hidden_state = F.pad(hidden_state, padding, mode="constant", value=0)
543 + slice_index = -num_padding_patches if num_padding_patches > 0 else None
544 +
545 + attention_mask = aspect_ratio_mask.reshape(
546 + batch_size * num_concurrent_media, -1)
547 + attention_mask = _prepare_aspect_ratio_attention_mask(
548 + aspect_ratio_mask=attention_mask,
549 + num_patches=self.num_patches,
550 + target_length=hidden_state.shape[2],
551 + dtype=self.layernorm_pre.weight.dtype,
552 + )
553 +
554 + hidden_state = hidden_state.view(batch_size * num_concurrent_media, -1,
555 + dim)
556 + output = self.transformer(
557 + hidden_state,
558 + attention_mask=attention_mask,
559 + )
560 + hidden_state, intermediate_hidden_states = output[0], output[1]
561 + intermediate_hidden_states = torch.stack(intermediate_hidden_states,
562 + dim=-1)
563 +
564 + # apply global encoder
565 + hidden_state = self.layernorm_post(hidden_state)
566 + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
567 + num_tiles,
568 + num_patches + num_padding_patches,
569 + dim)
570 + hidden_state = self.post_tile_positional_embedding(
571 + hidden_state, aspect_ratio_ids)
572 + hidden_state = hidden_state.reshape(
573 + batch_size * num_concurrent_media,
574 + num_tiles * (num_patches + num_padding_patches), dim)
575 + hidden_state = self.global_transformer(
576 + hidden_state, attention_mask=attention_mask)[0]
577 + hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
578 + num_tiles,
579 + num_patches + num_padding_patches,
580 + dim)
581 + hidden_state = hidden_state[:, :, :slice_index]
582 +
583 + # adding intermediate layer outputs
584 + hidden_state = hidden_state.reshape(batch_size, num_concurrent_media,
585 + num_tiles, num_patches, dim)
586 + intermediate_hidden_states = intermediate_hidden_states.reshape(
587 + batch_size * num_concurrent_media, num_tiles,
588 + num_patches + num_padding_patches, -1)
589 + intermediate_hidden_states = intermediate_hidden_states[:, :, :
590 + slice_index]
591 + intermediate_hidden_states = intermediate_hidden_states.reshape(
592 + batch_size, num_concurrent_media, num_tiles, num_patches, -1)
593 + hidden_state = torch.cat([hidden_state, intermediate_hidden_states],
594 + dim=-1)
595 + return hidden_state
596 +
597 +
598 +class MllamaTextRMSNorm(nn.Module):
599 +
600 + def __init__(self, hidden_size, eps=1e-6):
601 + """
602 + MllamaTextRMSNorm is equivalent to T5LayerNorm
603 + """
604 + super().__init__()
605 + self.weight = nn.Parameter(torch.ones(hidden_size))
606 + self.variance_epsilon = eps
607 +
608 + def forward(self, hidden_states):
609 + input_dtype = hidden_states.dtype
610 + hidden_states = hidden_states.to(torch.float32)
611 + variance = hidden_states.pow(2).mean(-1, keepdim=True)
612 + hidden_states = hidden_states * torch.rsqrt(variance +
613 + self.variance_epsilon)
614 + return self.weight * hidden_states.to(input_dtype)
615 +
616 + def extra_repr(self):
617 + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
618 +
619 +
620 +class MllamaTextCrossAttention(nn.Module):
621 + """Multi-headed attention from 'Attention Is All You Need' paper"""
622 +
623 + def __init__(
624 + self,
625 + config: Optional[config_mllama.MllamaTextConfig] = None,
626 + layer_idx: Optional[int] = None,
627 + ):
628 + super().__init__()
629 + self.config = config
630 + self.model_parallel_size = get_tensor_model_parallel_world_size()
631 + self.num_heads = self.config.num_attention_heads
632 + self.num_local_heads = self.num_heads // self.model_parallel_size
633 + self.num_key_value_heads = self.config.num_key_value_heads
634 + self.num_local_key_value_heads = \
635 + self.num_key_value_heads // self.model_parallel_size
636 + self.dropout = config.dropout
637 + self.hidden_size = config.hidden_size
638 + self.head_dim = config.hidden_size // self.num_heads
639 + self.layer_idx = layer_idx
640 + self.num_key_value_groups = self.num_heads // self.num_key_value_heads
641 + self.q_local_size = self.num_local_heads * self.head_dim
642 + self.kv_local_size = self.num_local_key_value_heads * self.head_dim
643 +
644 + # TODO: change to Q/KV separate linear after #7448 is merged
645 + self.qkv_proj = QKVParallelLinear(
646 + self.hidden_size,
647 + self.head_dim,
648 + self.num_heads,
649 + self.num_key_value_heads,
650 + bias=False,
651 + )
652 + self.o_proj = RowParallelLinear(
653 + self.num_heads * self.head_dim,
654 + self.hidden_size,
655 + bias=False,
656 + input_is_parallel=True,
657 + )
658 + # vllm.model_executor.layers.layernorm.RMSNorm has precision issue,
659 + # use huggingface's instead
660 + self.q_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
661 + self.k_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
662 + self.scaling = self.head_dim**-0.5
663 +
664 + self.attn = Attention(
665 + self.num_local_heads,
666 + self.head_dim,
667 + self.scaling,
668 + self.num_local_key_value_heads,
669 + )
670 +
671 + def forward(
672 + self,
673 + hidden_states: torch.Tensor,
674 + attention_mask: Optional[torch.Tensor],
675 + cross_attention_states: Optional[torch.Tensor],
676 + kv_cache: torch.Tensor,
677 + attn_metadata: AttentionMetadata,
678 + ) -> torch.Tensor:
679 + qkv_dec, _ = self.qkv_proj(hidden_states)
680 + q, _, _ = qkv_dec.split(
681 + [self.q_local_size, self.kv_local_size, self.kv_local_size],
682 + dim=-1)
683 + if cross_attention_states is None:
684 + k = None
685 + v = None
686 + else:
687 + qkv_enc, _ = self.qkv_proj(cross_attention_states)
688 + _, k, v = qkv_enc.split(
689 + [self.q_local_size, self.kv_local_size, self.kv_local_size],
690 + dim=-1)
691 + k = k.view(-1, self.num_local_key_value_heads, self.head_dim)
692 + v = v.view(-1, self.num_local_key_value_heads, self.head_dim)
693 + k = self.k_norm(k)
694 + q = q.view(-1, self.num_local_heads, self.head_dim)
695 + q = self.q_norm(q)
696 +
697 + output = self.attn(q,
698 + k,
699 + v,
700 + kv_cache,
701 + attn_metadata,
702 + attn_type=AttentionType.ENCODER_DECODER)
703 + out, _ = self.o_proj(output)
704 + return out
705 +
706 +
707 +class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
708 + """Cross-attention transformer block with tanh-gated attention
709 + and feedforward."""
710 +
711 + def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int) \
712 + -> None:
713 + super().__init__()
714 + self.layer_idx = layer_idx
715 + self.cross_attn = MllamaTextCrossAttention(
716 + config=config,
717 + layer_idx=layer_idx,
718 + )
719 +
720 + self.input_layernorm = RMSNorm(config.hidden_size,
721 + eps=config.rms_norm_eps)
722 + self.cross_attn_attn_gate = torch.nn.Parameter(torch.zeros(1))
723 +
724 + self.mlp = LlamaMLP(
725 + hidden_size=config.hidden_size,
726 + intermediate_size=config.intermediate_size,
727 + hidden_act=config.hidden_act,
728 + )
729 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
730 + eps=config.rms_norm_eps)
731 + self.cross_attn_mlp_gate = torch.nn.Parameter(torch.zeros(1))
732 +
733 + def forward(
734 + self,
735 + hidden_states: torch.Tensor,
736 + cross_attention_states: torch.Tensor,
737 + cross_attention_mask: torch.Tensor,
738 + full_text_row_masked_out_mask: torch.Tensor,
739 + kv_cache: List[torch.Tensor],
740 + attn_metadata: AttentionMetadata,
741 + ) -> torch.Tensor:
742 + residual = hidden_states
743 + hidden_states = self.input_layernorm(hidden_states)
744 +
745 + hidden_states = self.cross_attn(
746 + hidden_states=hidden_states,
747 + attention_mask=cross_attention_mask,
748 + cross_attention_states=cross_attention_states,
749 + kv_cache=kv_cache,
750 + attn_metadata=attn_metadata,
751 + )
752 + hidden_states = full_text_row_masked_out_mask * hidden_states
753 + hidden_states = residual + self.cross_attn_attn_gate.tanh(
754 + ) * hidden_states
755 +
756 + residual = hidden_states
757 + hidden_states = self.post_attention_layernorm(hidden_states)
758 + hidden_states = self.mlp(hidden_states)
759 + hidden_states = full_text_row_masked_out_mask * hidden_states
760 + hidden_states = residual + self.cross_attn_mlp_gate.tanh(
761 + ) * hidden_states
762 + return hidden_states
763 +
764 +
765 +class MllamaTextModel(nn.Module):
766 + config_class = config_mllama.MllamaTextConfig
767 + base_model_prefix = "model"
768 +
769 + def __init__(self, config: config_mllama.MllamaTextConfig,
770 + cache_config: Optional[CacheConfig],
771 + quant_config: Optional[QuantizationConfig]):
772 + super().__init__()
773 + self.padding_idx = config.pad_token_id
774 + self.vocab_size = config.vocab_size
775 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8,
776 + config.hidden_size)
777 + self.cross_attention_layers = config.cross_attention_layers
778 +
779 + layers = []
780 + for layer_idx in range(config.num_hidden_layers):
781 + if layer_idx in self.cross_attention_layers:
782 + layers.append(
783 + MllamaCrossAttentionDecoderLayer(config, layer_idx))
784 + else:
785 + # TODO: force LlamaDecoderLayer to config.attention_bias=False
786 + layers.append(
787 + LlamaDecoderLayer(config,
788 + cache_config=cache_config,
789 + quant_config=quant_config))
790 +
791 + self.layers = nn.ModuleList(layers)
792 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
793 +
794 + def forward(
795 + self,
796 + input_ids: torch.LongTensor,
797 + positions: Optional[torch.LongTensor],
798 + cross_attention_states: Optional[torch.LongTensor],
799 + cross_attention_mask: Optional[torch.LongTensor],
800 + full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
801 + torch.Tensor]],
802 + kv_caches: List[torch.Tensor],
803 + attn_metadata: AttentionMetadata,
804 + skip_cross_attention: bool,
805 + ) -> torch.Tensor:
806 + inputs_embeds = self.embed_tokens(input_ids)
807 + hidden_states = inputs_embeds
808 +
809 + for idx, decoder_layer in enumerate(self.layers):
810 + if isinstance(decoder_layer, MllamaCrossAttentionDecoderLayer):
811 + if not skip_cross_attention:
812 + hidden_states = decoder_layer(
813 + hidden_states=hidden_states,
814 + cross_attention_states=cross_attention_states,
815 + cross_attention_mask=cross_attention_mask,
816 + full_text_row_masked_out_mask=
817 + full_text_row_masked_out_mask,
818 + kv_cache=kv_caches[idx],
819 + attn_metadata=attn_metadata,
820 + )
821 + elif isinstance(decoder_layer, LlamaDecoderLayer):
822 + hidden_states, residual = decoder_layer(
823 + positions=positions,
824 + hidden_states=hidden_states,
825 + kv_cache=kv_caches[idx],
826 + attn_metadata=attn_metadata,
827 + residual=None,
828 + )
829 + hidden_states = hidden_states + residual
830 + else:
831 + raise ValueError(
832 + f"Unknown decoder layer type {type(decoder_layer)}")
833 + hidden_states = self.norm(hidden_states)
834 + return hidden_states
835 +
836 +
837 +class MllamaForCausalLM(nn.Module):
838 + config_class = config_mllama.MllamaTextConfig
839 + base_model_prefix = "language_model"
840 + _no_split_modules = [
841 + "MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"
842 + ]
843 +
844 + def __init__(self, config: config_mllama.MllamaTextConfig,
845 + cache_config: Optional[CacheConfig],
846 + quant_config: Optional[QuantizationConfig]):
847 + super().__init__()
848 + self.vocab_size = config.vocab_size
849 + self.model = MllamaTextModel(config, cache_config, quant_config)
850 + self.lm_head = ParallelLMHead(
851 + config.vocab_size,
852 + config.hidden_size,
853 + org_num_embeddings=config.vocab_size,
854 + padding_size=DEFAULT_VOCAB_PADDING_SIZE,
855 + quant_config=quant_config,
856 + )
857 +
858 + def forward(
859 + self,
860 + input_ids: torch.LongTensor,
861 + positions: Optional[torch.LongTensor],
862 + cross_attention_states: Optional[torch.LongTensor],
863 + cross_attention_mask: Optional[torch.LongTensor],
864 + full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
865 + torch.Tensor]],
866 + kv_caches: List[torch.Tensor],
867 + attn_metadata: AttentionMetadata,
868 + skip_cross_attention: bool,
869 + ) -> torch.Tensor:
870 + hidden_states = self.model(
871 + input_ids=input_ids,
872 + positions=positions,
873 + cross_attention_states=cross_attention_states,
874 + cross_attention_mask=cross_attention_mask,
875 + full_text_row_masked_out_mask=full_text_row_masked_out_mask,
876 + kv_caches=kv_caches,
877 + attn_metadata=attn_metadata,
878 + skip_cross_attention=skip_cross_attention,
879 + )
880 + return hidden_states
881 +
882 +
883 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
884 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_mllama_image_tokens)
885 +@INPUT_REGISTRY.register_dummy_data(dummy_decoder_data_for_mllama)
886 +@INPUT_REGISTRY.register_dummy_encoder_data(dummy_encoder_data_for_mllama)
887 +@INPUT_REGISTRY.register_input_processor(input_processor_for_mllama)
888 +class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
889 +
890 + def __init__(self,
891 + config: config_mllama.MllamaConfig,
892 + multimodal_config: MultiModalConfig,
893 + cache_config: Optional[CacheConfig] = None,
894 + quant_config: Optional[QuantizationConfig] = None):
895 + super().__init__()
896 + self.vocab_size = config.text_config.vocab_size
897 + self.hidden_size = config.text_config.hidden_size
898 + self.max_num_tiles = config.vision_config.max_num_tiles
899 + self.vision_output_dim = config.vision_config.vision_output_dim
900 + self.pad_token_id = \
901 + config.pad_token_id if config.pad_token_id is not None else -1
902 + self.image_size = config.vision_config.image_size
903 +
904 + self.vision_model = MllamaVisionModel(config.vision_config)
905 + self.language_model = MllamaForCausalLM(
906 + config.text_config,
907 + cache_config=cache_config,
908 + quant_config=quant_config,
909 + )
910 + self.multi_modal_projector = nn.Linear(
911 + config.vision_config.vision_output_dim,
912 + config.text_config.hidden_size,
913 + bias=True,
914 + )
915 + self.logits_processor = LogitsProcessor(config.output_hidden_states,
916 + config.text_config.vocab_size)
917 + self.sampler = Sampler()
918 +
919 + def compute_logits(
920 + self,
921 + hidden_states: torch.Tensor,
922 + sampling_metadata: SamplingMetadata,
923 + ) -> Optional[torch.Tensor]:
924 + logits = self.logits_processor(self.language_model.lm_head,
925 + hidden_states, sampling_metadata)
926 + return logits
927 +
928 + def sample(
929 + self,
930 + logits: torch.Tensor,
931 + sampling_metadata: SamplingMetadata,
932 + ) -> Optional[SamplerOutput]:
933 + next_tokens = self.sampler(logits, sampling_metadata)
934 + return next_tokens
935 +
936 + def _parse_and_validate_image_input(self, **kwargs: object):
937 + # tensor with the same shape will be batched together by
938 + # MultiModalInputs.batch, so pixel_values here can be:
939 + # - List[List[torch.Tensor]]:
940 + # with shape (num_tiles, 3, image_res, image_res)
941 + # - List[torch.Tensor]:
942 + # with shape (num_image, num_tiles, 3, image_res, image_res)
943 + # - torch.Tensor:
944 + # with shape (bs, num_image, num_tiles, 3, image_res, image_res)
945 + pixel_values: Optional[Union[List[List[torch.Tensor]],
946 + List[torch.Tensor],
947 + torch.Tensor]] = kwargs.pop(
948 + "pixel_values", None)
949 + image_embeds: Optional[Union[List[List[torch.Tensor]],
950 + List[torch.Tensor],
951 + torch.Tensor]] = kwargs.pop(
952 + "image_embeds", None)
953 + aspect_ratio_ids: Optional[Union[List[List[torch.Tensor]],
954 + List[torch.Tensor],
955 + torch.Tensor]] = kwargs.pop(
956 + "aspect_ratio_ids", None)
957 + aspect_ratio_mask: Optional[Union[List[List[torch.Tensor]],
958 + List[torch.Tensor],
959 + torch.Tensor]] = kwargs.pop(
960 + "aspect_ratio_mask", None)
961 +
962 + if pixel_values is None and image_embeds is None:
963 + return None
964 +
965 + if pixel_values is not None and image_embeds is not None:
966 + raise ValueError(
967 + "Both pixel values and image embeds are provided.")
968 +
969 + if pixel_values is not None:
970 + assert aspect_ratio_ids is not None
971 + assert aspect_ratio_mask is not None
972 + max_num_images = max([len(x[0]) for x in pixel_values])
973 + if max_num_images == 0:
974 + raise ValueError("No images provided.")
975 + max_num_tiles = max(
976 + max([len(x) for x in y[0]]) for y in pixel_values)
977 + device = self.multi_modal_projector.weight.device
978 + bsz = len(pixel_values)
979 + out_num_tiles = []
980 + out_images = torch.zeros(
981 + bsz,
982 + max_num_images,
983 + max_num_tiles,
984 + 3,
985 + self.image_size,
986 + self.image_size,
987 + dtype=torch.float32,
988 + device=device,
989 + )
990 + out_ar_ids = torch.ones(bsz,
991 + max_num_images,
992 + dtype=torch.int64,
993 + device=device)
994 + out_ar_mask = torch.zeros(bsz,
995 + max_num_images,
996 + max_num_tiles,
997 + dtype=torch.int64,
998 + device=device)
999 + for b in range(len(pixel_values)):
1000 + _num_tiles = []
1001 + for i in range(len(pixel_values[b][0])):
1002 + img = pixel_values[b][0][i]
1003 + out_images[b, i, :img.shape[0]] = img
1004 + out_ar_ids[b, i] = aspect_ratio_ids[b][0][i]
1005 + out_ar_mask[b, i] = aspect_ratio_mask[b][0][i]
1006 + _num_tiles.append(img.shape[0])
1007 + out_num_tiles.append(_num_tiles)
1008 +
1009 + return MllamaImagePixelInputs(
1010 + type="pixel_values",
1011 + data=out_images,
1012 + aspect_ratio_ids=out_ar_ids,
1013 + aspect_ratio_mask=out_ar_mask,
1014 + )
1015 +
1016 + if image_embeds is not None:
1017 + raise NotImplementedError
1018 +
1019 + raise AssertionError("This line should be unreachable.")
1020 +
1021 + def flat_encoder_result(self, cross_attention_states: torch.Tensor,
1022 + attn_metadata: AttentionMetadata):
1023 +
1024 + cross_attention_states_flat = torch.zeros(
1025 + sum(attn_metadata.encoder_seq_lens),
1026 + cross_attention_states.shape[-1],
1027 + device=cross_attention_states.device,
1028 + dtype=cross_attention_states.dtype)
1029 + start_pos = 0
1030 + for seq_len, vision_token_in_batch in zip(
1031 + attn_metadata.encoder_seq_lens, cross_attention_states):
1032 + end_pos = start_pos + seq_len
1033 + cross_attention_states_flat[
1034 + start_pos:end_pos] = vision_token_in_batch[:seq_len]
1035 + start_pos = end_pos
1036 + cross_attention_states = cross_attention_states_flat
1037 +
1038 + full_text_row_masked_out_mask = torch.ones(
1039 + (attn_metadata.num_prefill_tokens, 1), dtype=torch.bool)
1040 + start_pos = 0
1041 + for seq_len, encoder_seq_len in zip(
1042 + attn_metadata.seq_lens_tensor.cpu(),
1043 + attn_metadata.encoder_seq_lens):
1044 + if encoder_seq_len == 0:
1045 + full_text_row_masked_out_mask[start_pos:start_pos +
1046 + seq_len] = False
1047 + start_pos += seq_len
1048 + full_text_row_masked_out_mask = full_text_row_masked_out_mask.to(
1049 + cross_attention_states.device)
1050 +
1051 + return cross_attention_states, full_text_row_masked_out_mask
1052 +
1053 + def forward(
1054 + self,
1055 + input_ids: torch.Tensor,
1056 + positions: torch.Tensor,
1057 + kv_caches: List[torch.Tensor],
1058 + attn_metadata: AttentionMetadata,
1059 + **kwargs: object,
1060 + ) -> Union[Tuple, CausalLMOutputWithPast]:
1061 + if attn_metadata.num_prefill_tokens > 0 and \
1062 + attn_metadata.num_decode_tokens > 0:
1063 + raise ValueError("Chunk prefill not supported")
1064 + image_inputs = self._parse_and_validate_image_input(**kwargs)
1065 + if image_inputs is None:
1066 + cross_attention_mask = None
1067 + full_text_row_masked_out_mask = (
1068 + attn_metadata.encoder_seq_lens_tensor != 0).reshape(-1, 1).to(
1069 + input_ids.device)
1070 + cross_attention_states = None
1071 + skip_cross_attention = max(attn_metadata.encoder_seq_lens) == 0
1072 + else:
1073 + # NOTE: llama's reference implementation runs vision model on CPU
1074 + pixel_values = image_inputs['data']
1075 + aspect_ratio_ids = image_inputs['aspect_ratio_ids']
1076 + aspect_ratio_mask = image_inputs['aspect_ratio_mask']
1077 + cross_attention_states = self.vision_model(pixel_values,
1078 + aspect_ratio_ids,
1079 + aspect_ratio_mask)
1080 + cross_attention_states = self.multi_modal_projector(
1081 + cross_attention_states)
1082 +
1083 + bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
1084 + cross_attention_states = cross_attention_states.view(
1085 + bsz, -1, image_token_dim)
1086 +
1087 + cross_attention_states, full_text_row_masked_out_mask = \
1088 + self.flat_encoder_result(cross_attention_states, attn_metadata)
1089 + skip_cross_attention = False
1090 + # TODO: support multi-image by this mask
1091 + cross_attention_mask = None
1092 +
1093 + outputs = self.language_model(
1094 + input_ids=input_ids,
1095 + positions=positions,
1096 + cross_attention_states=cross_attention_states,
1097 + cross_attention_mask=cross_attention_mask,
1098 + full_text_row_masked_out_mask=full_text_row_masked_out_mask,
1099 + kv_caches=kv_caches,
1100 + attn_metadata=attn_metadata,
1101 + skip_cross_attention=skip_cross_attention,
1102 + )
1103 +
1104 + return outputs
1105 +
1106 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1107 + stacked_params_mapping = [
1108 + # (param_name, shard_name, shard_id)
1109 + (".qkv_proj", ".q_proj", "q"),
1110 + (".qkv_proj", ".k_proj", "k"),
1111 + (".qkv_proj", ".v_proj", "v"),
1112 + (".gate_up_proj", ".gate_proj", 0),
1113 + (".gate_up_proj", ".up_proj", 1),
1114 + ]
1115 + params_dict = dict(self.named_parameters())
1116 + updated_params = set()
1117 + for name, loaded_weight in weights:
1118 + if 'patch_embedding.weight' in name:
1119 + name = name.replace('patch_embedding.weight',
1120 + 'patch_embedding._linear.weight')
1121 + loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1122 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
1123 + if weight_name not in name:
1124 + continue
1125 + name = name.replace(weight_name, param_name)
1126 + param = params_dict[name]
1127 + updated_params.add(name)
1128 + weight_loader = param.weight_loader
1129 + weight_loader(param, loaded_weight, shard_id)
1130 + break
1131 + else:
1132 + param = params_dict.pop(name)
1133 + weight_loader = getattr(param, "weight_loader",
1134 + default_weight_loader)
1135 + weight_loader(param, loaded_weight)
vllm/multimodal/base.py
+6
@@ -54,6 +54,12 @@ class MultiModalInputs(_MultiModalInputsBase):
54 if isinstance(nested_tensors, torch.Tensor):
55 return nested_tensors
56
57 + if isinstance(nested_tensors, np.ndarray):
58 + return torch.from_numpy(nested_tensors)
59 +
60 + if isinstance(nested_tensors, (int, float)):
61 + return torch.tensor(nested_tensors)
62 +
63 stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
64 if not is_list_of(stacked, torch.Tensor, check="all"):
65 # Only tensors (not lists) can be stacked.
vllm/multimodal/image.py
+5
@@ -2,6 +2,7 @@ from functools import lru_cache
2
3 import torch
4 from PIL import Image
5 +from transformers.image_processing_base import BatchFeature
6
7 from vllm.config import ModelConfig
8 from vllm.inputs.registry import InputContext
@@ -39,6 +40,10 @@ class ImagePlugin(MultiModalPlugin):
40 ) -> MultiModalInputs:
41 model_config = ctx.model_config
42
43 + # Processed by input processor
44 + if isinstance(data, BatchFeature):
45 + return MultiModalInputs(data.data)
46 +
47 # PIL image
48 if isinstance(data, Image.Image) or is_list_of(data, Image.Image):
49 image_processor = self._get_hf_image_processor(model_config)
vllm/sequence.py
+10 -2
@@ -13,6 +13,7 @@ from typing import Set, Tuple, Union, cast
13 import msgspec
14 import torch
15
16 +from vllm.inputs import EncoderDecoderLLMInputs, LLMInputs
17 from vllm.inputs.parse import is_valid_encoder_decoder_llm_inputs
18 from vllm.lora.request import LoRARequest
19 from vllm.pooling_params import PoolingParams
@@ -21,7 +22,6 @@ from vllm.sampling_params import SamplingParams
22 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
23
24 if TYPE_CHECKING:
24 - from vllm.inputs import LLMInputs
25 from vllm.multimodal.base import MultiModalDataDict
26
27 VLLM_TOKEN_ID_ARRAY_TYPE = "l"
@@ -471,7 +471,15 @@ class Sequence:
471
472 @property
473 def multi_modal_data(self) -> "MultiModalDataDict":
474 - return self.inputs.get("multi_modal_data") or {}
474 + if self.inputs.get("multi_modal_data") and self.inputs.get(
475 + "encoder_multi_modal_data"):
476 + raise ValueError(
477 + "Multi-modal data in both encoder and decoder is not supported."
478 + )
479 + inputs = self.inputs
480 + return self.inputs.get("multi_modal_data") or (cast(
481 + EncoderDecoderLLMInputs,
482 + inputs).get("encoder_multi_modal_data")) or {}
483
484 @property
485 def lora_int_id(self) -> int:
vllm/transformers_utils/config.py
+13 -4
@@ -22,9 +22,10 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
22 EAGLEConfig, ExaoneConfig,
23 GraniteConfig, InternVLChatConfig,
24 JAISConfig, MedusaConfig,
25 - MLPSpeculatorConfig, MPTConfig,
26 - NemotronConfig, RWConfig,
27 - SolarConfig, UltravoxConfig)
25 + MllamaConfig, MLPSpeculatorConfig,
26 + MPTConfig, NemotronConfig,
27 + RWConfig, SolarConfig,
28 + UltravoxConfig)
29 # yapf: enable
30 from vllm.transformers_utils.utils import check_gguf_file
31
@@ -37,6 +38,10 @@ MISTRAL_CONFIG_NAME = "params.json"
38
39 logger = init_logger(__name__)
40
41 +_CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
42 + "mllama": MllamaConfig
43 +}
44 +
45 _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
46 "chatglm": ChatGLMConfig,
47 "dbrx": DbrxConfig,
@@ -55,11 +60,15 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
60 # Granite can be removed from here once we have upgraded to
61 # transformers 4.45+
62 "granite": GraniteConfig,
63 + **_CONFIG_REGISTRY_OVERRIDE_HF
64 }
65
66 for name, cls in _CONFIG_REGISTRY.items():
67 with contextlib.suppress(ValueError):
62 - AutoConfig.register(name, cls)
68 + if name in _CONFIG_REGISTRY_OVERRIDE_HF:
69 + AutoConfig.register(name, cls, exist_ok=True)
70 + else:
71 + AutoConfig.register(name, cls)
72
73
74 class ConfigFormat(str, enum.Enum):
vllm/transformers_utils/configs/__init__.py
+2
@@ -10,6 +10,7 @@ from vllm.transformers_utils.configs.granite import GraniteConfig
10 from vllm.transformers_utils.configs.internvl import InternVLChatConfig
11 from vllm.transformers_utils.configs.jais import JAISConfig
12 from vllm.transformers_utils.configs.medusa import MedusaConfig
13 +from vllm.transformers_utils.configs.mllama import MllamaConfig
14 from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
15 from vllm.transformers_utils.configs.mpt import MPTConfig
16 from vllm.transformers_utils.configs.nemotron import NemotronConfig
@@ -26,6 +27,7 @@ __all__ = [
27 "MedusaConfig",
28 "EAGLEConfig",
29 "ExaoneConfig",
30 + "MllamaConfig",
31 "MLPSpeculatorConfig",
32 "NemotronConfig",
33 "SolarConfig",
vllm/transformers_utils/configs/mllama.py new
+28
@@ -0,0 +1,28 @@
1 +from transformers.models.mllama import configuration_mllama as mllama_hf_config
2 +
3 +
4 +class MllamaTextConfig(mllama_hf_config.MllamaTextConfig):
5 + '''
6 + Use this class to override is_encoder_decoder:
7 + - transformers regards mllama as is_encoder_decoder=False
8 + - vllm needs is_encoder_decoder=True to enable cross-attention
9 + '''
10 +
11 + def __init__(
12 + self,
13 + **kwargs,
14 + ):
15 + super().__init__(**kwargs)
16 + self.is_encoder_decoder = True
17 +
18 +
19 +class MllamaConfig(mllama_hf_config.MllamaConfig):
20 +
21 + def __init__(
22 + self,
23 + text_config=None,
24 + **kwargs,
25 + ):
26 + if isinstance(text_config, dict):
27 + text_config = MllamaTextConfig(**text_config)
28 + super().__init__(text_config=text_config, **kwargs)
vllm/transformers_utils/tokenizer.py
-1
@@ -111,7 +111,6 @@ def get_tokenizer(
111 'encoding and decoding.',
112 FutureWarning,
113 stacklevel=2)
114 -
114 if tokenizer_mode == "mistral":
115 tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
116 revision=revision)
vllm/worker/enc_dec_model_runner.py
+30 -10
@@ -18,7 +18,8 @@ from vllm.inputs import INPUT_REGISTRY, InputRegistry
18 from vllm.logger import init_logger
19 from vllm.model_executor import SamplingMetadata
20 from vllm.model_executor.layers.sampler import SamplerOutput
21 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
21 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalInputs,
22 + MultiModalRegistry)
23 from vllm.sampling_params import SamplingParams
24 from vllm.sequence import (IntermediateTensors, PoolerOutput,
25 SequenceGroupMetadata)
@@ -52,6 +53,7 @@ class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
53 "virtual_engine": self.virtual_engine,
54 "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
55 "finished_requests_ids": self.finished_requests_ids,
56 + "multi_modal_kwargs": self.multi_modal_kwargs,
57 }
58 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
59 _add_sampling_metadata_broadcastable_dict(tensor_dict,
@@ -194,6 +196,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
196 "finished_requests_ids": model_input.finished_requests_ids,
197 "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
198 } if self.has_seqlen_agnostic else {}
199 +
200 + multi_modal_kwargs = model_input.multi_modal_kwargs or {}
201 hidden_or_intermediate_states = model_executable(
202 input_ids=model_input.input_tokens,
203 positions=model_input.input_positions,
@@ -202,6 +206,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
206 kv_caches=kv_caches,
207 attn_metadata=model_input.attn_metadata,
208 intermediate_tensors=intermediate_tensors,
209 + **MultiModalInputs.as_kwargs(multi_modal_kwargs,
210 + device=self.device),
211 **seqlen_agnostic_kwargs)
212
213 logits = self.model.compute_logits(hidden_or_intermediate_states,
@@ -288,8 +294,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
294 max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
295 self.model_config)
296 if max_mm_tokens > 0:
291 - raise NotImplementedError(
292 - "Multi-modal encoder-decoder models are not supported yet")
297 + logger.info("Starting profile run for multi-modal models.")
298
299 batch_size = 0
300 for group_id in range(max_num_seqs):
@@ -297,24 +302,39 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
302 (group_id < max_num_batched_tokens % max_num_seqs))
303 batch_size += seq_len
304
300 - seq_data, _ = self.input_registry \
301 - .dummy_data_for_profiling(self.model_config,
305 + decoder_seq_data, decoder_dummy_multi_modal_data \
306 + = self.input_registry.dummy_data_for_profiling(
307 + self.model_config,
308 seq_len,
303 - self.mm_registry)
309 + self.mm_registry,
310 + is_encoder_data=False)
311 + encoder_seq_data, encoder_dummy_multi_modal_data \
312 + = self.input_registry.dummy_data_for_profiling(
313 + self.model_config,
314 + seq_len,
315 + self.mm_registry,
316 + is_encoder_data=True)
317
318 # Having more tokens is over-conservative but otherwise fine
306 - assert len(seq_data.prompt_token_ids) >= seq_len, (
319 + assert len(decoder_seq_data.prompt_token_ids) >= seq_len, (
320 f"Expected at least {seq_len} dummy tokens for profiling, "
308 - f"but got: {len(seq_data.prompt_token_ids)}")
321 + f"but got: {len(decoder_seq_data.prompt_token_ids)}")
322 +
323 + assert decoder_dummy_multi_modal_data is None or \
324 + encoder_dummy_multi_modal_data is None, (
325 + "Multi-modal data can't be provided in both encoder and decoder"
326 + )
327
328 seq = SequenceGroupMetadata(
329 request_id=str(group_id),
330 is_prompt=True,
313 - seq_data={group_id: seq_data},
331 + seq_data={group_id: decoder_seq_data},
332 sampling_params=sampling_params,
333 block_tables=None,
316 - encoder_seq_data=seq_data,
334 + encoder_seq_data=encoder_seq_data,
335 cross_block_table=None,
336 + multi_modal_data=decoder_dummy_multi_modal_data
337 + or encoder_dummy_multi_modal_data,
338 )
339 seqs.append(seq)
340
vllm/worker/utils.py
-4
@@ -39,10 +39,6 @@ def assert_enc_dec_mr_supported_scenario(
39 raise NotImplementedError(
40 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
41
42 - if enc_dec_mr.model_config.is_multimodal_model:
43 - raise NotImplementedError(
44 - STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_MM'])
45 -
42 if enc_dec_mr.scheduler_config.num_lookahead_slots > 0:
43 raise NotImplementedError(
44 STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SPEC_DEC'])