A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
Pixtral (#8377)
Co-authored-by: Roger Wang <ywang@roblox.com>
Patrick von Platen committed
Sep 11, 2024 at 23:41 UTC
d394787e5268903a705850413e494ebf2ddcefb5
8 files changed
+807
-9
docs/source/models/supported_models.rst
+5
@@ -247,6 +247,11 @@ Multimodal Language Models
247
- Image\ :sup:`E+`
248
- :code:`microsoft/Phi-3-vision-128k-instruct`, :code:`microsoft/Phi-3.5-vision-instruct` etc.
249
-
250
+ * - :code:`PixtralForConditionalGeneration`
251
+ - Pixtral
252
+ - Image\ :sup:`+`
253
+ - :code:`mistralai/Pixtral-12B-2409`
254
+ -
255
* - :code:`QWenLMHeadModel`
256
- Qwen-VL
257
- Image\ :sup:`E`
examples/offline_inference_pixtral.py
new
+164
@@ -0,0 +1,164 @@
1
+# ruff: noqa
2
+import argparse
3
+
4
+from vllm import LLM
5
+from vllm.sampling_params import SamplingParams
6
+
7
+# This script is an offline demo for running Pixtral.
8
+#
9
+# If you want to run a server/client setup, please follow this code:
10
+#
11
+# - Server:
12
+#
13
+# ```bash
14
+# vllm serve mistralai/Pixtral-12B-2409 --tokenizer_mode mistral --limit_mm_per_prompt 'image=4' --max_num_batched_tokens 16384
15
+# ```
16
+#
17
+# - Client:
18
+#
19
+# ```bash
20
+# curl --location 'http://<your-node-url>:8000/v1/chat/completions' \
21
+# --header 'Content-Type: application/json' \
22
+# --header 'Authorization: Bearer token' \
23
+# --data '{
24
+# "model": "mistralai/Pixtral-12B-2409",
25
+# "messages": [
26
+# {
27
+# "role": "user",
28
+# "content": [
29
+# {"type" : "text", "text": "Describe this image in detail please."},
30
+# {"type": "image_url", "image_url": {"url": "https://s3.amazonaws.com/cms.ipressroom.com/338/files/201808/5b894ee1a138352221103195_A680%7Ejogging-edit/A680%7Ejogging-edit_hero.jpg"}},
31
+# {"type" : "text", "text": "and this one as well. Answer in French."},
32
+# {"type": "image_url", "image_url": {"url": "https://www.wolframcloud.com/obj/resourcesystem/images/a0e/a0ee3983-46c6-4c92-b85d-059044639928/6af8cfb971db031b.png"}}
33
+# ]
34
+# }
35
+# ]
36
+# }'
37
+# ```
38
+#
39
+# Usage:
40
+# python demo.py simple
41
+# python demo.py advanced
42
+
43
+
44
+def run_simple_demo():
45
+ model_name = "mistralai/Pixtral-12B-2409"
46
+ sampling_params = SamplingParams(max_tokens=8192)
47
+
48
+ llm = LLM(model=model_name, tokenizer_mode="mistral")
49
+
50
+ prompt = "Describe this image in one sentence."
51
+ image_url = "https://picsum.photos/id/237/200/300"
52
+
53
+ messages = [
54
+ {
55
+ "role":
56
+ "user",
57
+ "content": [
58
+ {
59
+ "type": "text",
60
+ "text": prompt
61
+ },
62
+ {
63
+ "type": "image_url",
64
+ "image_url": {
65
+ "url": image_url
66
+ }
67
+ },
68
+ ],
69
+ },
70
+ ]
71
+ outputs = llm.chat(messages, sampling_params=sampling_params)
72
+
73
+ print(outputs[0].outputs[0].text)
74
+
75
+
76
+def run_advanced_demo():
77
+ model_name = "mistralai/Pixtral-12B-2409"
78
+ max_img_per_msg = 5
79
+ max_tokens_per_img = 4096
80
+
81
+ sampling_params = SamplingParams(max_tokens=8192, temperature=0.7)
82
+ llm = LLM(
83
+ model=model_name,
84
+ tokenizer_mode="mistral",
85
+ limit_mm_per_prompt={"image": max_img_per_msg},
86
+ max_num_batched_tokens=max_img_per_msg * max_tokens_per_img,
87
+ )
88
+
89
+ prompt = "Describe the following image."
90
+
91
+ url_1 = "https://huggingface.co/datasets/patrickvonplaten/random_img/resolve/main/yosemite.png"
92
+ url_2 = "https://picsum.photos/seed/picsum/200/300"
93
+ url_3 = "https://picsum.photos/id/32/512/512"
94
+
95
+ messages = [
96
+ {
97
+ "role":
98
+ "user",
99
+ "content": [
100
+ {
101
+ "type": "text",
102
+ "text": prompt
103
+ },
104
+ {
105
+ "type": "image_url",
106
+ "image_url": {
107
+ "url": url_1
108
+ }
109
+ },
110
+ {
111
+ "type": "image_url",
112
+ "image_url": {
113
+ "url": url_2
114
+ }
115
+ },
116
+ ],
117
+ },
118
+ {
119
+ "role": "assistant",
120
+ "content": "The images show nature.",
121
+ },
122
+ {
123
+ "role": "user",
124
+ "content": "More details please and answer only in French!.",
125
+ },
126
+ {
127
+ "role": "user",
128
+ "content": [
129
+ {
130
+ "type": "image_url",
131
+ "image_url": {
132
+ "url": url_3
133
+ }
134
+ },
135
+ ],
136
+ },
137
+ ]
138
+
139
+ outputs = llm.chat(messages=messages, sampling_params=sampling_params)
140
+ print(outputs[0].outputs[0].text)
141
+
142
+
143
+def main():
144
+ parser = argparse.ArgumentParser(
145
+ description="Run a demo in simple or advanced mode.")
146
+
147
+ parser.add_argument(
148
+ "mode",
149
+ choices=["simple", "advanced"],
150
+ help="Specify the demo mode: 'simple' or 'advanced'",
151
+ )
152
+
153
+ args = parser.parse_args()
154
+
155
+ if args.mode == "simple":
156
+ print("Running simple demo...")
157
+ run_simple_demo()
158
+ elif args.mode == "advanced":
159
+ print("Running advanced demo...")
160
+ run_advanced_demo()
161
+
162
+
163
+if __name__ == "__main__":
164
+ main()
requirements-common.txt
+1
-1
@@ -25,7 +25,7 @@ pyzmq
25
msgspec
26
gguf == 0.9.1
27
importlib_metadata
28
-mistral_common >= 1.3.4
28
+mistral_common >= 1.4.0
29
pyyaml
30
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
31
einops # Required for Qwen2-VL.
tests/models/test_pixtral.py
new
+64
@@ -0,0 +1,64 @@
1
+"""Compare the outputs of HF and vLLM for Mistral models using greedy sampling.
2
+
3
+Run `pytest tests/models/test_mistral.py`.
4
+"""
5
+import pytest
6
+
7
+from vllm.sampling_params import SamplingParams
8
+
9
+pytestmark = pytest.mark.vlm
10
+
11
+MODELS = ["mistralai/Pixtral-12B-2409"]
12
+
13
+
14
+@pytest.mark.skip(
15
+ reason=
16
+ "Model is too big, test passed on A100 locally but will OOM on CI machine."
17
+)
18
+@pytest.mark.parametrize("model", MODELS)
19
+@pytest.mark.parametrize("dtype", ["bfloat16"])
20
+@pytest.mark.parametrize("max_tokens", [64])
21
+@pytest.mark.parametrize("num_logprobs", [5])
22
+def test_models(
23
+ vllm_runner,
24
+ example_prompts,
25
+ model: str,
26
+ dtype: str,
27
+ max_tokens: int,
28
+ num_logprobs: int,
29
+) -> None:
30
+ image_urls = [
31
+ "https://picsum.photos/id/237/200/300",
32
+ "https://picsum.photos/seed/picsum/200/300"
33
+ ]
34
+ expected = [
35
+ "The image depicts a black dog lying on a wooden surface, looking directly at the camera with a calm expression.", # noqa
36
+ "The image depicts a serene landscape with a snow-covered mountain under a pastel-colored sky during sunset." # noqa
37
+ ]
38
+ prompt = "Describe the image in one short sentence."
39
+
40
+ sampling_params = SamplingParams(max_tokens=512, temperature=0.0)
41
+
42
+ with vllm_runner(model, dtype=dtype,
43
+ tokenizer_mode="mistral") as vllm_model:
44
+
45
+ for i, image_url in enumerate(image_urls):
46
+ messages = [
47
+ {
48
+ "role":
49
+ "user",
50
+ "content": [{
51
+ "type": "text",
52
+ "text": prompt
53
+ }, {
54
+ "type": "image_url",
55
+ "image_url": {
56
+ "url": image_url
57
+ }
58
+ }]
59
+ },
60
+ ]
61
+
62
+ outputs = vllm_model.model.chat(messages,
63
+ sampling_params=sampling_params)
64
+ assert outputs[0].outputs[0].text == expected[i]
vllm/entrypoints/chat_utils.py
+2
-1
@@ -148,7 +148,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
148
return f"<|image_{current_count}|>"
149
if model_type == "minicpmv":
150
return "(<image>./</image>)"
151
- if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
151
+ if model_type in ("blip-2", "chatglm", "fuyu", "paligemma",
152
+ "pixtral"):
153
# These models do not use image tokens in the prompt
154
return None
155
if model_type == "qwen":
vllm/model_executor/models/__init__.py
+2
@@ -92,6 +92,8 @@ _MULTIMODAL_MODELS = {
92
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
93
"UltravoxModel": ("ultravox", "UltravoxModel"),
94
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
95
+ "PixtralForConditionalGeneration": ("pixtral",
96
+ "PixtralForConditionalGeneration"),
97
"Qwen2VLForConditionalGeneration": ("qwen2_vl",
98
"Qwen2VLForConditionalGeneration"),
99
}
vllm/model_executor/models/pixtral.py
new
+551
@@ -0,0 +1,551 @@
1
+import math
2
+from array import array
3
+from dataclasses import dataclass, fields
4
+from itertools import tee
5
+from typing import Iterable, List, Mapping, Optional, Tuple, Union
6
+
7
+import torch
8
+import torch.nn as nn
9
+import torch.nn.functional as F
10
+from mistral_common.protocol.instruct.messages import ImageChunk
11
+from PIL import Image
12
+from transformers import PretrainedConfig
13
+from xformers.ops.fmha import memory_efficient_attention
14
+from xformers.ops.fmha.attn_bias import BlockDiagonalMask
15
+
16
+from vllm.attention import AttentionMetadata
17
+from vllm.config import CacheConfig, MultiModalConfig
18
+from vllm.inputs import INPUT_REGISTRY, InputContext
19
+from vllm.model_executor.layers.layernorm import RMSNorm
20
+from vllm.model_executor.layers.quantization import QuantizationConfig
21
+from vllm.model_executor.layers.sampler import SamplerOutput
22
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
+from vllm.model_executor.sampling_metadata import SamplingMetadata
24
+from vllm.multimodal import MULTIMODAL_REGISTRY
25
+from vllm.multimodal.base import MultiModalInputs
26
+from vllm.multimodal.utils import cached_get_tokenizer
27
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
28
+ SequenceData)
29
+
30
+from .interfaces import SupportsMultiModal
31
+from .utils import init_vllm_registered_model
32
+
33
+
34
+def get_max_pixtral_image_tokens(ctx: InputContext):
35
+ tokenizer = cached_get_tokenizer(
36
+ ctx.model_config.tokenizer,
37
+ tokenizer_mode=ctx.model_config.tokenizer_mode)
38
+ mm_encoder = tokenizer.instruct.mm_encoder
39
+
40
+ max_image_size = mm_encoder.mm_config.max_image_size
41
+ image_patch_size = mm_encoder.mm_config.image_patch_size
42
+
43
+ return ((max_image_size // image_patch_size)**2)
44
+
45
+
46
+def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
47
+ mm_counts: Mapping[str, int]):
48
+ tokenizer = cached_get_tokenizer(
49
+ ctx.model_config.tokenizer,
50
+ tokenizer_mode=ctx.model_config.tokenizer_mode)
51
+ mm_encoder = tokenizer.instruct.mm_encoder
52
+
53
+ mm_config = ctx.model_config.multimodal_config
54
+ max_num_images_per_request = mm_config.limit_per_prompt.get("image", 1)
55
+
56
+ # approximate image size
57
+ size = int(math.sqrt(seq_len) * mm_encoder.mm_config.image_patch_size)
58
+
59
+ image = Image.new("RGB", (size, size), color=0)
60
+ img_chunk = ImageChunk(image=image)
61
+
62
+ tokens = mm_encoder(img_chunk).tokens
63
+ token_ids = max_num_images_per_request * array(VLLM_TOKEN_ID_ARRAY_TYPE,
64
+ tokens)
65
+
66
+ seq_data = SequenceData(token_ids)
67
+ mm_data = {"image": max_num_images_per_request * [image]}
68
+ return seq_data, mm_data
69
+
70
+
71
+def input_mapper_for_pixtral(ctx: InputContext,
72
+ data: object) -> MultiModalInputs:
73
+ """Maps the input data to its MultiModalInputs (if any).
74
+
75
+ Args:
76
+ ctx: Context of the loaded model.
77
+ data: data potentially containing image/image embeddings to be mapped
78
+ to pixel_values in .forward() for a visual QWenLMHeadModel model.
79
+
80
+ Returns:
81
+ MultiModalInputs containing the stacked normalized images tensor or
82
+ image embeddings.
83
+ """
84
+ # Early exit if we have provided an image to a language only Qwen model
85
+ model_config = ctx.model_config
86
+ tokenizer = cached_get_tokenizer(
87
+ model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
88
+
89
+ data_list = data if isinstance(data, list) else [data]
90
+
91
+ images = []
92
+ for image_data in data_list:
93
+ image = ImageChunk(image=image_data)
94
+ encoding = tokenizer.instruct.mm_encoder(image)
95
+ image = torch.from_numpy(encoding.image).to(device="cuda",
96
+ dtype=torch.float16)
97
+ images.append(image)
98
+
99
+ return MultiModalInputs({"images": images})
100
+
101
+
102
+def merge_multimodal_embeddings(input_ids: torch.Tensor,
103
+ inputs_embeds: torch.Tensor,
104
+ image_features: Optional[List[torch.Tensor]],
105
+ image_id: int) -> torch.Tensor:
106
+ text_locations = input_ids != image_id
107
+ image_locations = input_ids == image_id
108
+
109
+ seq_len = input_ids.shape[0]
110
+
111
+ N_txt = text_locations.sum().item()
112
+ _, D_txt = inputs_embeds.shape
113
+ N_img, D_img = image_features.shape
114
+
115
+ assert (D_txt == D_img), (f"Text features dim {D_txt} should be equal "
116
+ "to image features dim {D_img}")
117
+ assert (seq_len == N_txt +
118
+ N_img), (f"seq_len {seq_len} should be equal to N_txt + N_img "
119
+ f"{(N_txt, N_img, image_locations.sum().item())}")
120
+
121
+ inputs_embeds[image_locations, :] = image_features
122
+ return inputs_embeds
123
+
124
+
125
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral)
126
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_pixtral_image_tokens)
127
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_pixtral)
128
+class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal):
129
+
130
+ def __init__(self,
131
+ config: PretrainedConfig,
132
+ multimodal_config: MultiModalConfig,
133
+ cache_config: Optional[CacheConfig] = None,
134
+ quant_config: Optional[QuantizationConfig] = None) -> None:
135
+ super().__init__()
136
+
137
+ self.config = config
138
+ self.multimodal_config = multimodal_config
139
+
140
+ dataclass_fields = {field.name for field in fields(VisionEncoderArgs)}
141
+ vision_args = {
142
+ key: value
143
+ for key, value in self.config.vision_config.to_dict().items()
144
+ if key in dataclass_fields
145
+ }
146
+
147
+ self.vision_args = VisionEncoderArgs(**vision_args)
148
+
149
+ # init MistralForCausalLM
150
+ self.language_model = init_vllm_registered_model(
151
+ config.text_config, cache_config, quant_config)
152
+
153
+ self.vision_encoder = VisionTransformer(self.vision_args)
154
+ self.vision_language_adapter = VisionLanguageAdapter(
155
+ self.vision_args, dim=config.text_config.hidden_size)
156
+
157
+ def forward(
158
+ self,
159
+ input_ids: torch.Tensor,
160
+ positions: torch.Tensor,
161
+ kv_caches: List[torch.Tensor],
162
+ attn_metadata: AttentionMetadata,
163
+ intermediate_tensors: Optional[IntermediateTensors] = None,
164
+ **kwargs: object,
165
+ ) -> SamplerOutput:
166
+ """Run forward pass for pixtral.
167
+
168
+ TODO
169
+
170
+ """
171
+ image_input = self._parse_and_validate_image_input(**kwargs)
172
+
173
+ if image_input is not None:
174
+ vision_embeddings = self._process_image_input(image_input)
175
+ inputs_embeds = self.language_model.model.get_input_embeddings(
176
+ input_ids)
177
+
178
+ inputs_embeds = merge_multimodal_embeddings(
179
+ input_ids, inputs_embeds, vision_embeddings,
180
+ self.vision_args.image_token_id)
181
+
182
+ input_ids = None
183
+ else:
184
+ inputs_embeds = None
185
+
186
+ hidden_states = self.language_model.model(input_ids,
187
+ positions,
188
+ kv_caches,
189
+ attn_metadata,
190
+ None,
191
+ inputs_embeds=inputs_embeds)
192
+
193
+ return hidden_states
194
+
195
+ def _parse_and_validate_image_input(
196
+ self,
197
+ images: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor],
198
+ torch.Tensor]] = None
199
+ ) -> Optional[List[torch.Tensor]]:
200
+ if images is None:
201
+ return None
202
+
203
+ if isinstance(images, torch.Tensor):
204
+ # always take last images
205
+ images = [images[-1][i] for i in range(images.size(1))]
206
+ elif isinstance(images, list):
207
+ # always take last images
208
+ images = [images[-1][i] for i in range(len(images[0]))]
209
+
210
+ return images
211
+
212
+ def _process_image_input(self,
213
+ image_input: List[torch.Tensor]) -> torch.Tensor:
214
+ return self.vision_language_adapter(self.vision_encoder(image_input))
215
+
216
+ def compute_logits(
217
+ self,
218
+ hidden_states: torch.Tensor,
219
+ sampling_metadata: SamplingMetadata,
220
+ ) -> Optional[torch.Tensor]:
221
+ return self.language_model.compute_logits(hidden_states,
222
+ sampling_metadata)
223
+
224
+ def sample(
225
+ self,
226
+ logits: torch.Tensor,
227
+ sampling_metadata: SamplingMetadata,
228
+ ) -> Optional[SamplerOutput]:
229
+ return self.language_model.sample(logits, sampling_metadata)
230
+
231
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
232
+
233
+ def is_vision_encoder_weights(weight: Tuple[str, torch.Tensor]):
234
+ return weight[0].startswith("vision_encoder")
235
+
236
+ def is_vision_lang_adapter_weights(weight: Tuple[str, torch.Tensor]):
237
+ return weight[0].startswith("vision_language_adapter")
238
+
239
+ def is_vision_weights(weight: Tuple[str, torch.Tensor]):
240
+ return is_vision_encoder_weights(
241
+ weight) or is_vision_lang_adapter_weights(weight)
242
+
243
+ llm_weights, vision_encoder_weights, vision_lang_adapter_weights = tee(
244
+ weights, 3)
245
+
246
+ # llm
247
+ llm_weights = filter(lambda x: not is_vision_weights(x), llm_weights)
248
+ self.language_model.load_weights(llm_weights)
249
+
250
+ # vision encoder
251
+ vision_encoder_weights = filter(is_vision_encoder_weights,
252
+ vision_encoder_weights)
253
+ vision_encoder_dict = dict(self.vision_encoder.named_parameters())
254
+ for name, loaded_weight in vision_encoder_weights:
255
+ # cut 'vision_encoder.'
256
+ name = '.'.join(name.split(".")[1:])
257
+ param = vision_encoder_dict[name]
258
+
259
+ default_weight_loader(param, loaded_weight)
260
+
261
+ # adapter
262
+ vision_lang_adapter_weights = filter(is_vision_lang_adapter_weights,
263
+ vision_lang_adapter_weights)
264
+ vision_lang_adpter_dict = dict(
265
+ self.vision_language_adapter.named_parameters())
266
+ for name, loaded_weight in vision_lang_adapter_weights:
267
+ # cut 'vision_language_adapter.'
268
+ name = '.'.join(name.split(".")[1:])
269
+ param = vision_lang_adpter_dict[name]
270
+ default_weight_loader(param, loaded_weight)
271
+
272
+
273
+# Vision encoder
274
+@dataclass
275
+class VisionEncoderArgs:
276
+ hidden_size: int
277
+ num_channels: int
278
+ image_size: int
279
+ patch_size: int
280
+ intermediate_size: int
281
+ num_hidden_layers: int
282
+ num_attention_heads: int
283
+ rope_theta: float # for rope-2D
284
+ image_token_id: int
285
+
286
+
287
+def _reshape_for_broadcast(freqs_cis: torch.Tensor,
288
+ x: torch.Tensor) -> torch.Tensor:
289
+ """
290
+ freqs_cis: complex - (seq_len, head_dim / 2)
291
+ x: complex - (bsz, seq_len, head_dim / 2)
292
+ """
293
+ ndim = x.ndim
294
+ assert ndim > 1
295
+ assert freqs_cis.shape == (x.shape[1], x.shape[-1]), (
296
+ freqs_cis.shape,
297
+ (x.shape[1], x.shape[-1]),
298
+ )
299
+ shape = [
300
+ d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)
301
+ ]
302
+ return freqs_cis.view(*shape)
303
+
304
+
305
+def precompute_freqs_cis_2d(
306
+ dim: int,
307
+ height: int,
308
+ width: int,
309
+ theta: float,
310
+) -> torch.Tensor:
311
+ """
312
+ freqs_cis: 2D complex tensor of shape (height, width, dim // 2)
313
+ to be indexed by (height, width) position tuples
314
+ """
315
+ # (dim / 2) frequency bases
316
+ freqs = 1.0 / (theta**(torch.arange(0, dim, 2).float() / dim))
317
+
318
+ h = torch.arange(height, device=freqs.device)
319
+ w = torch.arange(width, device=freqs.device)
320
+
321
+ freqs_h = torch.outer(h, freqs[::2]).float()
322
+ freqs_w = torch.outer(w, freqs[1::2]).float()
323
+ freqs_2d = torch.cat(
324
+ [
325
+ freqs_h[:, None, :].repeat(1, width, 1),
326
+ freqs_w[None, :, :].repeat(height, 1, 1),
327
+ ],
328
+ dim=-1,
329
+ )
330
+ return torch.polar(torch.ones_like(freqs_2d), freqs_2d)
331
+
332
+
333
+def apply_rotary_emb_vit(
334
+ xq: torch.Tensor,
335
+ xk: torch.Tensor,
336
+ freqs_cis: torch.Tensor,
337
+) -> Tuple[torch.Tensor, torch.Tensor]:
338
+ xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
339
+ xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
340
+ assert freqs_cis.dtype == torch.complex64
341
+ freqs_cis = _reshape_for_broadcast(freqs_cis, xq_)
342
+ xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
343
+ xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)
344
+ return xq_out.type_as(xq), xk_out.type_as(xk)
345
+
346
+
347
+class FeedForward(nn.Module):
348
+
349
+ def __init__(self, args: VisionEncoderArgs):
350
+ super().__init__()
351
+ assert args.intermediate_size is not None
352
+ self.w1 = nn.Linear(args.hidden_size,
353
+ args.intermediate_size,
354
+ bias=False)
355
+ self.w2 = nn.Linear(args.intermediate_size,
356
+ args.hidden_size,
357
+ bias=False)
358
+ self.w3 = nn.Linear(args.hidden_size,
359
+ args.intermediate_size,
360
+ bias=False)
361
+
362
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
363
+ return self.w2(F.silu(self.w1(x)) * self.w3(x))
364
+
365
+
366
+class Attention(nn.Module):
367
+
368
+ def __init__(self, args: VisionEncoderArgs):
369
+ super().__init__()
370
+ self.args = args
371
+ assert not args.hidden_size % args.num_attention_heads
372
+ self.n_heads = args.num_attention_heads
373
+ self.head_dim = args.hidden_size // args.num_attention_heads
374
+
375
+ self.wq = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
376
+ self.wk = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
377
+ self.wv = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
378
+ self.wo = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
379
+
380
+ def forward(
381
+ self,
382
+ x: torch.Tensor,
383
+ mask: BlockDiagonalMask,
384
+ freqs_cis: torch.Tensor,
385
+ ) -> torch.Tensor:
386
+ batch, patches, _ = x.shape
387
+
388
+ q, k, v = self.wq(x), self.wk(x), self.wv(x)
389
+ q = q.reshape(batch, patches, self.n_heads, self.head_dim)
390
+ k = k.reshape(batch, patches, self.n_heads, self.head_dim)
391
+ v = v.reshape(batch, patches, self.n_heads, self.head_dim)
392
+
393
+ q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis)
394
+ out = memory_efficient_attention(q, k, v, attn_bias=mask)
395
+ out = out.reshape(batch, patches, self.n_heads * self.head_dim)
396
+ return self.wo(out)
397
+
398
+
399
+class TransformerBlock(nn.Module):
400
+
401
+ def __init__(self, args: VisionEncoderArgs):
402
+ super().__init__()
403
+ self.attention = Attention(args)
404
+ self.feed_forward = FeedForward(args)
405
+ self.attention_norm = RMSNorm(args.hidden_size, eps=1e-5)
406
+ self.ffn_norm = RMSNorm(args.hidden_size, eps=1e-5)
407
+
408
+ def forward(
409
+ self,
410
+ x: torch.Tensor,
411
+ mask: BlockDiagonalMask,
412
+ freqs_cis: torch.Tensor,
413
+ ) -> torch.Tensor:
414
+ r = self.attention.forward(self.attention_norm(x),
415
+ mask=mask,
416
+ freqs_cis=freqs_cis)
417
+ h = x + r
418
+ r = self.feed_forward.forward(self.ffn_norm(h))
419
+ out = h + r
420
+ return out
421
+
422
+
423
+class Transformer(nn.Module):
424
+
425
+ def __init__(self, args: VisionEncoderArgs):
426
+ super().__init__()
427
+ self.layers = torch.nn.ModuleList()
428
+ for _ in range(args.num_hidden_layers):
429
+ self.layers.append(TransformerBlock(args))
430
+
431
+ def forward(
432
+ self,
433
+ x: torch.Tensor,
434
+ mask: BlockDiagonalMask,
435
+ freqs_cis: Optional[torch.Tensor],
436
+ ) -> torch.Tensor:
437
+ for layer in self.layers:
438
+ x = layer(x, mask=mask, freqs_cis=freqs_cis)
439
+ return x
440
+
441
+
442
+def position_meshgrid(patch_embeds_list: list[torch.Tensor], ) -> torch.Tensor:
443
+ positions = torch.cat([
444
+ torch.stack(
445
+ torch.meshgrid(
446
+ torch.arange(p.shape[-2]),
447
+ torch.arange(p.shape[-1]),
448
+ indexing="ij",
449
+ ),
450
+ dim=-1,
451
+ ).reshape(-1, 2) for p in patch_embeds_list
452
+ ])
453
+ return positions
454
+
455
+
456
+class VisionTransformer(nn.Module):
457
+
458
+ def __init__(self, args: VisionEncoderArgs):
459
+ super().__init__()
460
+ self.args = args
461
+ self.patch_conv = nn.Conv2d(
462
+ in_channels=args.num_channels,
463
+ out_channels=args.hidden_size,
464
+ kernel_size=args.patch_size,
465
+ stride=args.patch_size,
466
+ bias=False,
467
+ )
468
+ self.ln_pre = RMSNorm(args.hidden_size, eps=1e-5)
469
+ self.transformer = Transformer(args)
470
+
471
+ head_dim = self.args.hidden_size // self.args.num_attention_heads
472
+ assert head_dim % 2 == 0, "ROPE requires even head_dim"
473
+ self._freqs_cis: Optional[torch.Tensor] = None
474
+
475
+ @property
476
+ def max_patches_per_side(self) -> int:
477
+ return self.args.image_size // self.args.patch_size
478
+
479
+ @property
480
+ def device(self) -> torch.device:
481
+ return next(self.parameters()).device
482
+
483
+ @property
484
+ def dtype(self) -> torch.device:
485
+ return next(self.parameters()).dtype
486
+
487
+ @property
488
+ def freqs_cis(self) -> torch.Tensor:
489
+ if self._freqs_cis is None:
490
+ self._freqs_cis = precompute_freqs_cis_2d(
491
+ dim=self.args.hidden_size // self.args.num_attention_heads,
492
+ height=self.max_patches_per_side,
493
+ width=self.max_patches_per_side,
494
+ theta=self.args.rope_theta,
495
+ )
496
+
497
+ if self._freqs_cis.device != self.device:
498
+ self._freqs_cis = self._freqs_cis.to(device=self.device)
499
+
500
+ return self._freqs_cis
501
+
502
+ def forward(
503
+ self,
504
+ images: List[torch.Tensor],
505
+ ) -> torch.Tensor:
506
+ """
507
+ Args:
508
+ images: list of N_img images of variable sizes,
509
+ each of shape (C, H, W)
510
+ Returns:
511
+ image_features: tensor of token features for
512
+ all tokens of all images of shape (N_toks, D)
513
+ """
514
+ # pass images through initial convolution independently
515
+ patch_embeds_list = [
516
+ self.patch_conv(img.unsqueeze(0).to(self.dtype)) for img in images
517
+ ]
518
+
519
+ # flatten to a single sequence
520
+ patch_embeds = torch.cat(
521
+ [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1)
522
+ patch_embeds = self.ln_pre(patch_embeds)
523
+
524
+ # positional embeddings
525
+ positions = position_meshgrid(patch_embeds_list).to(self.device)
526
+ freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]]
527
+
528
+ # pass through Transformer with a block diagonal mask delimiting images
529
+ mask = BlockDiagonalMask.from_seqlens(
530
+ [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
531
+ out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis)
532
+
533
+ # remove batch dimension of the single sequence
534
+ return out.squeeze(0)
535
+
536
+
537
+class VisionLanguageAdapter(nn.Module):
538
+
539
+ def __init__(self, args: VisionEncoderArgs, dim: int):
540
+ super().__init__()
541
+ assert isinstance(args, VisionEncoderArgs)
542
+ self.w_in = nn.Linear(
543
+ args.hidden_size,
544
+ dim,
545
+ bias=True,
546
+ )
547
+ self.gelu = nn.GELU()
548
+ self.w_out = nn.Linear(dim, dim, bias=True)
549
+
550
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
551
+ return self.w_out(self.gelu(self.w_in(x)))
vllm/transformers_utils/config.py
+18
-7
@@ -70,7 +70,7 @@ def file_or_path_exists(model: Union[str, Path], config_name, revision,
70
if Path(model).exists():
71
return (Path(model) / config_name).is_file()
72
73
- return file_exists(model, HF_CONFIG_NAME, revision=revision, token=token)
73
+ return file_exists(model, config_name, revision=revision, token=token)
74
75
76
def get_config(
@@ -205,14 +205,25 @@ def load_params_config(model, revision) -> PretrainedConfig:
205
config_dict["hidden_act"] = config_dict.get("activation", "silu")
206
config_dict["tie_word_embeddings"] = config_dict.get(
207
"tie_embeddings", False)
208
+ config_dict["max_seq_len"] = config_dict.get("max_seq_len", 128_000)
209
209
- if config_dict["model_type"] == "transformer":
210
- if "moe" in config_dict:
211
- config_dict["architectures"] = ["MixtralForCausalLM"]
212
- else:
213
- config_dict["architectures"] = ["MistralForCausalLM"]
210
+ if config_dict.get("moe") is not None:
211
+ config_dict["architectures"] = ["MixtralForCausalLM"]
212
+ else:
213
+ config_dict["architectures"] = ["MistralForCausalLM"]
214
+
215
+ if config_dict.get("vision_encoder") is not None:
216
+ multimodal_config = config_dict.pop("vision_encoder")
217
215
- return recurse_elems(config_dict)
218
+ config_dict = {
219
+ "text_config": config_dict,
220
+ "vision_config": multimodal_config
221
+ }
222
+ config_dict["architectures"] = ["PixtralForConditionalGeneration"]
223
+ config_dict["model_type"] = "pixtral"
224
+
225
+ config = recurse_elems(config_dict)
226
+ return config
227
228
229
def get_hf_image_processor_config(