@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d394787e52
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

Pixtral (#8377)

Co-authored-by: Roger Wang <ywang@roblox.com>

Patrick von Platen committed Sep 11, 2024 at 23:41 UTC d394787e5268903a705850413e494ebf2ddcefb5
8 files changed +807 -9
docs/source/models/supported_models.rst
+5
@@ -247,6 +247,11 @@ Multimodal Language Models
247 - Image\ :sup:`E+`
248 - :code:`microsoft/Phi-3-vision-128k-instruct`, :code:`microsoft/Phi-3.5-vision-instruct` etc.
249 -
250 + * - :code:`PixtralForConditionalGeneration`
251 + - Pixtral
252 + - Image\ :sup:`+`
253 + - :code:`mistralai/Pixtral-12B-2409`
254 + -
255 * - :code:`QWenLMHeadModel`
256 - Qwen-VL
257 - Image\ :sup:`E`
examples/offline_inference_pixtral.py new
+164
@@ -0,0 +1,164 @@
1 +# ruff: noqa
2 +import argparse
3 +
4 +from vllm import LLM
5 +from vllm.sampling_params import SamplingParams
6 +
7 +# This script is an offline demo for running Pixtral.
8 +#
9 +# If you want to run a server/client setup, please follow this code:
10 +#
11 +# - Server:
12 +#
13 +# ```bash
14 +# vllm serve mistralai/Pixtral-12B-2409 --tokenizer_mode mistral --limit_mm_per_prompt 'image=4' --max_num_batched_tokens 16384
15 +# ```
16 +#
17 +# - Client:
18 +#
19 +# ```bash
20 +# curl --location 'http://<your-node-url>:8000/v1/chat/completions' \
21 +# --header 'Content-Type: application/json' \
22 +# --header 'Authorization: Bearer token' \
23 +# --data '{
24 +# "model": "mistralai/Pixtral-12B-2409",
25 +# "messages": [
26 +# {
27 +# "role": "user",
28 +# "content": [
29 +# {"type" : "text", "text": "Describe this image in detail please."},
30 +# {"type": "image_url", "image_url": {"url": "https://s3.amazonaws.com/cms.ipressroom.com/338/files/201808/5b894ee1a138352221103195_A680%7Ejogging-edit/A680%7Ejogging-edit_hero.jpg"}},
31 +# {"type" : "text", "text": "and this one as well. Answer in French."},
32 +# {"type": "image_url", "image_url": {"url": "https://www.wolframcloud.com/obj/resourcesystem/images/a0e/a0ee3983-46c6-4c92-b85d-059044639928/6af8cfb971db031b.png"}}
33 +# ]
34 +# }
35 +# ]
36 +# }'
37 +# ```
38 +#
39 +# Usage:
40 +# python demo.py simple
41 +# python demo.py advanced
42 +
43 +
44 +def run_simple_demo():
45 + model_name = "mistralai/Pixtral-12B-2409"
46 + sampling_params = SamplingParams(max_tokens=8192)
47 +
48 + llm = LLM(model=model_name, tokenizer_mode="mistral")
49 +
50 + prompt = "Describe this image in one sentence."
51 + image_url = "https://picsum.photos/id/237/200/300"
52 +
53 + messages = [
54 + {
55 + "role":
56 + "user",
57 + "content": [
58 + {
59 + "type": "text",
60 + "text": prompt
61 + },
62 + {
63 + "type": "image_url",
64 + "image_url": {
65 + "url": image_url
66 + }
67 + },
68 + ],
69 + },
70 + ]
71 + outputs = llm.chat(messages, sampling_params=sampling_params)
72 +
73 + print(outputs[0].outputs[0].text)
74 +
75 +
76 +def run_advanced_demo():
77 + model_name = "mistralai/Pixtral-12B-2409"
78 + max_img_per_msg = 5
79 + max_tokens_per_img = 4096
80 +
81 + sampling_params = SamplingParams(max_tokens=8192, temperature=0.7)
82 + llm = LLM(
83 + model=model_name,
84 + tokenizer_mode="mistral",
85 + limit_mm_per_prompt={"image": max_img_per_msg},
86 + max_num_batched_tokens=max_img_per_msg * max_tokens_per_img,
87 + )
88 +
89 + prompt = "Describe the following image."
90 +
91 + url_1 = "https://huggingface.co/datasets/patrickvonplaten/random_img/resolve/main/yosemite.png"
92 + url_2 = "https://picsum.photos/seed/picsum/200/300"
93 + url_3 = "https://picsum.photos/id/32/512/512"
94 +
95 + messages = [
96 + {
97 + "role":
98 + "user",
99 + "content": [
100 + {
101 + "type": "text",
102 + "text": prompt
103 + },
104 + {
105 + "type": "image_url",
106 + "image_url": {
107 + "url": url_1
108 + }
109 + },
110 + {
111 + "type": "image_url",
112 + "image_url": {
113 + "url": url_2
114 + }
115 + },
116 + ],
117 + },
118 + {
119 + "role": "assistant",
120 + "content": "The images show nature.",
121 + },
122 + {
123 + "role": "user",
124 + "content": "More details please and answer only in French!.",
125 + },
126 + {
127 + "role": "user",
128 + "content": [
129 + {
130 + "type": "image_url",
131 + "image_url": {
132 + "url": url_3
133 + }
134 + },
135 + ],
136 + },
137 + ]
138 +
139 + outputs = llm.chat(messages=messages, sampling_params=sampling_params)
140 + print(outputs[0].outputs[0].text)
141 +
142 +
143 +def main():
144 + parser = argparse.ArgumentParser(
145 + description="Run a demo in simple or advanced mode.")
146 +
147 + parser.add_argument(
148 + "mode",
149 + choices=["simple", "advanced"],
150 + help="Specify the demo mode: 'simple' or 'advanced'",
151 + )
152 +
153 + args = parser.parse_args()
154 +
155 + if args.mode == "simple":
156 + print("Running simple demo...")
157 + run_simple_demo()
158 + elif args.mode == "advanced":
159 + print("Running advanced demo...")
160 + run_advanced_demo()
161 +
162 +
163 +if __name__ == "__main__":
164 + main()
requirements-common.txt
+1 -1
@@ -25,7 +25,7 @@ pyzmq
25 msgspec
26 gguf == 0.9.1
27 importlib_metadata
28 -mistral_common >= 1.3.4
28 +mistral_common >= 1.4.0
29 pyyaml
30 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
31 einops # Required for Qwen2-VL.
tests/models/test_pixtral.py new
+64
@@ -0,0 +1,64 @@
1 +"""Compare the outputs of HF and vLLM for Mistral models using greedy sampling.
2 +
3 +Run `pytest tests/models/test_mistral.py`.
4 +"""
5 +import pytest
6 +
7 +from vllm.sampling_params import SamplingParams
8 +
9 +pytestmark = pytest.mark.vlm
10 +
11 +MODELS = ["mistralai/Pixtral-12B-2409"]
12 +
13 +
14 +@pytest.mark.skip(
15 + reason=
16 + "Model is too big, test passed on A100 locally but will OOM on CI machine."
17 +)
18 +@pytest.mark.parametrize("model", MODELS)
19 +@pytest.mark.parametrize("dtype", ["bfloat16"])
20 +@pytest.mark.parametrize("max_tokens", [64])
21 +@pytest.mark.parametrize("num_logprobs", [5])
22 +def test_models(
23 + vllm_runner,
24 + example_prompts,
25 + model: str,
26 + dtype: str,
27 + max_tokens: int,
28 + num_logprobs: int,
29 +) -> None:
30 + image_urls = [
31 + "https://picsum.photos/id/237/200/300",
32 + "https://picsum.photos/seed/picsum/200/300"
33 + ]
34 + expected = [
35 + "The image depicts a black dog lying on a wooden surface, looking directly at the camera with a calm expression.", # noqa
36 + "The image depicts a serene landscape with a snow-covered mountain under a pastel-colored sky during sunset." # noqa
37 + ]
38 + prompt = "Describe the image in one short sentence."
39 +
40 + sampling_params = SamplingParams(max_tokens=512, temperature=0.0)
41 +
42 + with vllm_runner(model, dtype=dtype,
43 + tokenizer_mode="mistral") as vllm_model:
44 +
45 + for i, image_url in enumerate(image_urls):
46 + messages = [
47 + {
48 + "role":
49 + "user",
50 + "content": [{
51 + "type": "text",
52 + "text": prompt
53 + }, {
54 + "type": "image_url",
55 + "image_url": {
56 + "url": image_url
57 + }
58 + }]
59 + },
60 + ]
61 +
62 + outputs = vllm_model.model.chat(messages,
63 + sampling_params=sampling_params)
64 + assert outputs[0].outputs[0].text == expected[i]
vllm/entrypoints/chat_utils.py
+2 -1
@@ -148,7 +148,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
148 return f"<|image_{current_count}|>"
149 if model_type == "minicpmv":
150 return "(<image>./</image>)"
151 - if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
151 + if model_type in ("blip-2", "chatglm", "fuyu", "paligemma",
152 + "pixtral"):
153 # These models do not use image tokens in the prompt
154 return None
155 if model_type == "qwen":
vllm/model_executor/models/__init__.py
+2
@@ -92,6 +92,8 @@ _MULTIMODAL_MODELS = {
92 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
93 "UltravoxModel": ("ultravox", "UltravoxModel"),
94 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
95 + "PixtralForConditionalGeneration": ("pixtral",
96 + "PixtralForConditionalGeneration"),
97 "Qwen2VLForConditionalGeneration": ("qwen2_vl",
98 "Qwen2VLForConditionalGeneration"),
99 }
vllm/model_executor/models/pixtral.py new
+551
@@ -0,0 +1,551 @@
1 +import math
2 +from array import array
3 +from dataclasses import dataclass, fields
4 +from itertools import tee
5 +from typing import Iterable, List, Mapping, Optional, Tuple, Union
6 +
7 +import torch
8 +import torch.nn as nn
9 +import torch.nn.functional as F
10 +from mistral_common.protocol.instruct.messages import ImageChunk
11 +from PIL import Image
12 +from transformers import PretrainedConfig
13 +from xformers.ops.fmha import memory_efficient_attention
14 +from xformers.ops.fmha.attn_bias import BlockDiagonalMask
15 +
16 +from vllm.attention import AttentionMetadata
17 +from vllm.config import CacheConfig, MultiModalConfig
18 +from vllm.inputs import INPUT_REGISTRY, InputContext
19 +from vllm.model_executor.layers.layernorm import RMSNorm
20 +from vllm.model_executor.layers.quantization import QuantizationConfig
21 +from vllm.model_executor.layers.sampler import SamplerOutput
22 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 +from vllm.model_executor.sampling_metadata import SamplingMetadata
24 +from vllm.multimodal import MULTIMODAL_REGISTRY
25 +from vllm.multimodal.base import MultiModalInputs
26 +from vllm.multimodal.utils import cached_get_tokenizer
27 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
28 + SequenceData)
29 +
30 +from .interfaces import SupportsMultiModal
31 +from .utils import init_vllm_registered_model
32 +
33 +
34 +def get_max_pixtral_image_tokens(ctx: InputContext):
35 + tokenizer = cached_get_tokenizer(
36 + ctx.model_config.tokenizer,
37 + tokenizer_mode=ctx.model_config.tokenizer_mode)
38 + mm_encoder = tokenizer.instruct.mm_encoder
39 +
40 + max_image_size = mm_encoder.mm_config.max_image_size
41 + image_patch_size = mm_encoder.mm_config.image_patch_size
42 +
43 + return ((max_image_size // image_patch_size)**2)
44 +
45 +
46 +def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
47 + mm_counts: Mapping[str, int]):
48 + tokenizer = cached_get_tokenizer(
49 + ctx.model_config.tokenizer,
50 + tokenizer_mode=ctx.model_config.tokenizer_mode)
51 + mm_encoder = tokenizer.instruct.mm_encoder
52 +
53 + mm_config = ctx.model_config.multimodal_config
54 + max_num_images_per_request = mm_config.limit_per_prompt.get("image", 1)
55 +
56 + # approximate image size
57 + size = int(math.sqrt(seq_len) * mm_encoder.mm_config.image_patch_size)
58 +
59 + image = Image.new("RGB", (size, size), color=0)
60 + img_chunk = ImageChunk(image=image)
61 +
62 + tokens = mm_encoder(img_chunk).tokens
63 + token_ids = max_num_images_per_request * array(VLLM_TOKEN_ID_ARRAY_TYPE,
64 + tokens)
65 +
66 + seq_data = SequenceData(token_ids)
67 + mm_data = {"image": max_num_images_per_request * [image]}
68 + return seq_data, mm_data
69 +
70 +
71 +def input_mapper_for_pixtral(ctx: InputContext,
72 + data: object) -> MultiModalInputs:
73 + """Maps the input data to its MultiModalInputs (if any).
74 +
75 + Args:
76 + ctx: Context of the loaded model.
77 + data: data potentially containing image/image embeddings to be mapped
78 + to pixel_values in .forward() for a visual QWenLMHeadModel model.
79 +
80 + Returns:
81 + MultiModalInputs containing the stacked normalized images tensor or
82 + image embeddings.
83 + """
84 + # Early exit if we have provided an image to a language only Qwen model
85 + model_config = ctx.model_config
86 + tokenizer = cached_get_tokenizer(
87 + model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
88 +
89 + data_list = data if isinstance(data, list) else [data]
90 +
91 + images = []
92 + for image_data in data_list:
93 + image = ImageChunk(image=image_data)
94 + encoding = tokenizer.instruct.mm_encoder(image)
95 + image = torch.from_numpy(encoding.image).to(device="cuda",
96 + dtype=torch.float16)
97 + images.append(image)
98 +
99 + return MultiModalInputs({"images": images})
100 +
101 +
102 +def merge_multimodal_embeddings(input_ids: torch.Tensor,
103 + inputs_embeds: torch.Tensor,
104 + image_features: Optional[List[torch.Tensor]],
105 + image_id: int) -> torch.Tensor:
106 + text_locations = input_ids != image_id
107 + image_locations = input_ids == image_id
108 +
109 + seq_len = input_ids.shape[0]
110 +
111 + N_txt = text_locations.sum().item()
112 + _, D_txt = inputs_embeds.shape
113 + N_img, D_img = image_features.shape
114 +
115 + assert (D_txt == D_img), (f"Text features dim {D_txt} should be equal "
116 + "to image features dim {D_img}")
117 + assert (seq_len == N_txt +
118 + N_img), (f"seq_len {seq_len} should be equal to N_txt + N_img "
119 + f"{(N_txt, N_img, image_locations.sum().item())}")
120 +
121 + inputs_embeds[image_locations, :] = image_features
122 + return inputs_embeds
123 +
124 +
125 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_pixtral)
126 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_pixtral_image_tokens)
127 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_pixtral)
128 +class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal):
129 +
130 + def __init__(self,
131 + config: PretrainedConfig,
132 + multimodal_config: MultiModalConfig,
133 + cache_config: Optional[CacheConfig] = None,
134 + quant_config: Optional[QuantizationConfig] = None) -> None:
135 + super().__init__()
136 +
137 + self.config = config
138 + self.multimodal_config = multimodal_config
139 +
140 + dataclass_fields = {field.name for field in fields(VisionEncoderArgs)}
141 + vision_args = {
142 + key: value
143 + for key, value in self.config.vision_config.to_dict().items()
144 + if key in dataclass_fields
145 + }
146 +
147 + self.vision_args = VisionEncoderArgs(**vision_args)
148 +
149 + # init MistralForCausalLM
150 + self.language_model = init_vllm_registered_model(
151 + config.text_config, cache_config, quant_config)
152 +
153 + self.vision_encoder = VisionTransformer(self.vision_args)
154 + self.vision_language_adapter = VisionLanguageAdapter(
155 + self.vision_args, dim=config.text_config.hidden_size)
156 +
157 + def forward(
158 + self,
159 + input_ids: torch.Tensor,
160 + positions: torch.Tensor,
161 + kv_caches: List[torch.Tensor],
162 + attn_metadata: AttentionMetadata,
163 + intermediate_tensors: Optional[IntermediateTensors] = None,
164 + **kwargs: object,
165 + ) -> SamplerOutput:
166 + """Run forward pass for pixtral.
167 +
168 + TODO
169 +
170 + """
171 + image_input = self._parse_and_validate_image_input(**kwargs)
172 +
173 + if image_input is not None:
174 + vision_embeddings = self._process_image_input(image_input)
175 + inputs_embeds = self.language_model.model.get_input_embeddings(
176 + input_ids)
177 +
178 + inputs_embeds = merge_multimodal_embeddings(
179 + input_ids, inputs_embeds, vision_embeddings,
180 + self.vision_args.image_token_id)
181 +
182 + input_ids = None
183 + else:
184 + inputs_embeds = None
185 +
186 + hidden_states = self.language_model.model(input_ids,
187 + positions,
188 + kv_caches,
189 + attn_metadata,
190 + None,
191 + inputs_embeds=inputs_embeds)
192 +
193 + return hidden_states
194 +
195 + def _parse_and_validate_image_input(
196 + self,
197 + images: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor],
198 + torch.Tensor]] = None
199 + ) -> Optional[List[torch.Tensor]]:
200 + if images is None:
201 + return None
202 +
203 + if isinstance(images, torch.Tensor):
204 + # always take last images
205 + images = [images[-1][i] for i in range(images.size(1))]
206 + elif isinstance(images, list):
207 + # always take last images
208 + images = [images[-1][i] for i in range(len(images[0]))]
209 +
210 + return images
211 +
212 + def _process_image_input(self,
213 + image_input: List[torch.Tensor]) -> torch.Tensor:
214 + return self.vision_language_adapter(self.vision_encoder(image_input))
215 +
216 + def compute_logits(
217 + self,
218 + hidden_states: torch.Tensor,
219 + sampling_metadata: SamplingMetadata,
220 + ) -> Optional[torch.Tensor]:
221 + return self.language_model.compute_logits(hidden_states,
222 + sampling_metadata)
223 +
224 + def sample(
225 + self,
226 + logits: torch.Tensor,
227 + sampling_metadata: SamplingMetadata,
228 + ) -> Optional[SamplerOutput]:
229 + return self.language_model.sample(logits, sampling_metadata)
230 +
231 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
232 +
233 + def is_vision_encoder_weights(weight: Tuple[str, torch.Tensor]):
234 + return weight[0].startswith("vision_encoder")
235 +
236 + def is_vision_lang_adapter_weights(weight: Tuple[str, torch.Tensor]):
237 + return weight[0].startswith("vision_language_adapter")
238 +
239 + def is_vision_weights(weight: Tuple[str, torch.Tensor]):
240 + return is_vision_encoder_weights(
241 + weight) or is_vision_lang_adapter_weights(weight)
242 +
243 + llm_weights, vision_encoder_weights, vision_lang_adapter_weights = tee(
244 + weights, 3)
245 +
246 + # llm
247 + llm_weights = filter(lambda x: not is_vision_weights(x), llm_weights)
248 + self.language_model.load_weights(llm_weights)
249 +
250 + # vision encoder
251 + vision_encoder_weights = filter(is_vision_encoder_weights,
252 + vision_encoder_weights)
253 + vision_encoder_dict = dict(self.vision_encoder.named_parameters())
254 + for name, loaded_weight in vision_encoder_weights:
255 + # cut 'vision_encoder.'
256 + name = '.'.join(name.split(".")[1:])
257 + param = vision_encoder_dict[name]
258 +
259 + default_weight_loader(param, loaded_weight)
260 +
261 + # adapter
262 + vision_lang_adapter_weights = filter(is_vision_lang_adapter_weights,
263 + vision_lang_adapter_weights)
264 + vision_lang_adpter_dict = dict(
265 + self.vision_language_adapter.named_parameters())
266 + for name, loaded_weight in vision_lang_adapter_weights:
267 + # cut 'vision_language_adapter.'
268 + name = '.'.join(name.split(".")[1:])
269 + param = vision_lang_adpter_dict[name]
270 + default_weight_loader(param, loaded_weight)
271 +
272 +
273 +# Vision encoder
274 +@dataclass
275 +class VisionEncoderArgs:
276 + hidden_size: int
277 + num_channels: int
278 + image_size: int
279 + patch_size: int
280 + intermediate_size: int
281 + num_hidden_layers: int
282 + num_attention_heads: int
283 + rope_theta: float # for rope-2D
284 + image_token_id: int
285 +
286 +
287 +def _reshape_for_broadcast(freqs_cis: torch.Tensor,
288 + x: torch.Tensor) -> torch.Tensor:
289 + """
290 + freqs_cis: complex - (seq_len, head_dim / 2)
291 + x: complex - (bsz, seq_len, head_dim / 2)
292 + """
293 + ndim = x.ndim
294 + assert ndim > 1
295 + assert freqs_cis.shape == (x.shape[1], x.shape[-1]), (
296 + freqs_cis.shape,
297 + (x.shape[1], x.shape[-1]),
298 + )
299 + shape = [
300 + d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)
301 + ]
302 + return freqs_cis.view(*shape)
303 +
304 +
305 +def precompute_freqs_cis_2d(
306 + dim: int,
307 + height: int,
308 + width: int,
309 + theta: float,
310 +) -> torch.Tensor:
311 + """
312 + freqs_cis: 2D complex tensor of shape (height, width, dim // 2)
313 + to be indexed by (height, width) position tuples
314 + """
315 + # (dim / 2) frequency bases
316 + freqs = 1.0 / (theta**(torch.arange(0, dim, 2).float() / dim))
317 +
318 + h = torch.arange(height, device=freqs.device)
319 + w = torch.arange(width, device=freqs.device)
320 +
321 + freqs_h = torch.outer(h, freqs[::2]).float()
322 + freqs_w = torch.outer(w, freqs[1::2]).float()
323 + freqs_2d = torch.cat(
324 + [
325 + freqs_h[:, None, :].repeat(1, width, 1),
326 + freqs_w[None, :, :].repeat(height, 1, 1),
327 + ],
328 + dim=-1,
329 + )
330 + return torch.polar(torch.ones_like(freqs_2d), freqs_2d)
331 +
332 +
333 +def apply_rotary_emb_vit(
334 + xq: torch.Tensor,
335 + xk: torch.Tensor,
336 + freqs_cis: torch.Tensor,
337 +) -> Tuple[torch.Tensor, torch.Tensor]:
338 + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2))
339 + xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2))
340 + assert freqs_cis.dtype == torch.complex64
341 + freqs_cis = _reshape_for_broadcast(freqs_cis, xq_)
342 + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3)
343 + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3)
344 + return xq_out.type_as(xq), xk_out.type_as(xk)
345 +
346 +
347 +class FeedForward(nn.Module):
348 +
349 + def __init__(self, args: VisionEncoderArgs):
350 + super().__init__()
351 + assert args.intermediate_size is not None
352 + self.w1 = nn.Linear(args.hidden_size,
353 + args.intermediate_size,
354 + bias=False)
355 + self.w2 = nn.Linear(args.intermediate_size,
356 + args.hidden_size,
357 + bias=False)
358 + self.w3 = nn.Linear(args.hidden_size,
359 + args.intermediate_size,
360 + bias=False)
361 +
362 + def forward(self, x: torch.Tensor) -> torch.Tensor:
363 + return self.w2(F.silu(self.w1(x)) * self.w3(x))
364 +
365 +
366 +class Attention(nn.Module):
367 +
368 + def __init__(self, args: VisionEncoderArgs):
369 + super().__init__()
370 + self.args = args
371 + assert not args.hidden_size % args.num_attention_heads
372 + self.n_heads = args.num_attention_heads
373 + self.head_dim = args.hidden_size // args.num_attention_heads
374 +
375 + self.wq = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
376 + self.wk = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
377 + self.wv = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
378 + self.wo = nn.Linear(args.hidden_size, args.hidden_size, bias=False)
379 +
380 + def forward(
381 + self,
382 + x: torch.Tensor,
383 + mask: BlockDiagonalMask,
384 + freqs_cis: torch.Tensor,
385 + ) -> torch.Tensor:
386 + batch, patches, _ = x.shape
387 +
388 + q, k, v = self.wq(x), self.wk(x), self.wv(x)
389 + q = q.reshape(batch, patches, self.n_heads, self.head_dim)
390 + k = k.reshape(batch, patches, self.n_heads, self.head_dim)
391 + v = v.reshape(batch, patches, self.n_heads, self.head_dim)
392 +
393 + q, k = apply_rotary_emb_vit(q, k, freqs_cis=freqs_cis)
394 + out = memory_efficient_attention(q, k, v, attn_bias=mask)
395 + out = out.reshape(batch, patches, self.n_heads * self.head_dim)
396 + return self.wo(out)
397 +
398 +
399 +class TransformerBlock(nn.Module):
400 +
401 + def __init__(self, args: VisionEncoderArgs):
402 + super().__init__()
403 + self.attention = Attention(args)
404 + self.feed_forward = FeedForward(args)
405 + self.attention_norm = RMSNorm(args.hidden_size, eps=1e-5)
406 + self.ffn_norm = RMSNorm(args.hidden_size, eps=1e-5)
407 +
408 + def forward(
409 + self,
410 + x: torch.Tensor,
411 + mask: BlockDiagonalMask,
412 + freqs_cis: torch.Tensor,
413 + ) -> torch.Tensor:
414 + r = self.attention.forward(self.attention_norm(x),
415 + mask=mask,
416 + freqs_cis=freqs_cis)
417 + h = x + r
418 + r = self.feed_forward.forward(self.ffn_norm(h))
419 + out = h + r
420 + return out
421 +
422 +
423 +class Transformer(nn.Module):
424 +
425 + def __init__(self, args: VisionEncoderArgs):
426 + super().__init__()
427 + self.layers = torch.nn.ModuleList()
428 + for _ in range(args.num_hidden_layers):
429 + self.layers.append(TransformerBlock(args))
430 +
431 + def forward(
432 + self,
433 + x: torch.Tensor,
434 + mask: BlockDiagonalMask,
435 + freqs_cis: Optional[torch.Tensor],
436 + ) -> torch.Tensor:
437 + for layer in self.layers:
438 + x = layer(x, mask=mask, freqs_cis=freqs_cis)
439 + return x
440 +
441 +
442 +def position_meshgrid(patch_embeds_list: list[torch.Tensor], ) -> torch.Tensor:
443 + positions = torch.cat([
444 + torch.stack(
445 + torch.meshgrid(
446 + torch.arange(p.shape[-2]),
447 + torch.arange(p.shape[-1]),
448 + indexing="ij",
449 + ),
450 + dim=-1,
451 + ).reshape(-1, 2) for p in patch_embeds_list
452 + ])
453 + return positions
454 +
455 +
456 +class VisionTransformer(nn.Module):
457 +
458 + def __init__(self, args: VisionEncoderArgs):
459 + super().__init__()
460 + self.args = args
461 + self.patch_conv = nn.Conv2d(
462 + in_channels=args.num_channels,
463 + out_channels=args.hidden_size,
464 + kernel_size=args.patch_size,
465 + stride=args.patch_size,
466 + bias=False,
467 + )
468 + self.ln_pre = RMSNorm(args.hidden_size, eps=1e-5)
469 + self.transformer = Transformer(args)
470 +
471 + head_dim = self.args.hidden_size // self.args.num_attention_heads
472 + assert head_dim % 2 == 0, "ROPE requires even head_dim"
473 + self._freqs_cis: Optional[torch.Tensor] = None
474 +
475 + @property
476 + def max_patches_per_side(self) -> int:
477 + return self.args.image_size // self.args.patch_size
478 +
479 + @property
480 + def device(self) -> torch.device:
481 + return next(self.parameters()).device
482 +
483 + @property
484 + def dtype(self) -> torch.device:
485 + return next(self.parameters()).dtype
486 +
487 + @property
488 + def freqs_cis(self) -> torch.Tensor:
489 + if self._freqs_cis is None:
490 + self._freqs_cis = precompute_freqs_cis_2d(
491 + dim=self.args.hidden_size // self.args.num_attention_heads,
492 + height=self.max_patches_per_side,
493 + width=self.max_patches_per_side,
494 + theta=self.args.rope_theta,
495 + )
496 +
497 + if self._freqs_cis.device != self.device:
498 + self._freqs_cis = self._freqs_cis.to(device=self.device)
499 +
500 + return self._freqs_cis
501 +
502 + def forward(
503 + self,
504 + images: List[torch.Tensor],
505 + ) -> torch.Tensor:
506 + """
507 + Args:
508 + images: list of N_img images of variable sizes,
509 + each of shape (C, H, W)
510 + Returns:
511 + image_features: tensor of token features for
512 + all tokens of all images of shape (N_toks, D)
513 + """
514 + # pass images through initial convolution independently
515 + patch_embeds_list = [
516 + self.patch_conv(img.unsqueeze(0).to(self.dtype)) for img in images
517 + ]
518 +
519 + # flatten to a single sequence
520 + patch_embeds = torch.cat(
521 + [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1)
522 + patch_embeds = self.ln_pre(patch_embeds)
523 +
524 + # positional embeddings
525 + positions = position_meshgrid(patch_embeds_list).to(self.device)
526 + freqs_cis = self.freqs_cis[positions[:, 0], positions[:, 1]]
527 +
528 + # pass through Transformer with a block diagonal mask delimiting images
529 + mask = BlockDiagonalMask.from_seqlens(
530 + [p.shape[-2] * p.shape[-1] for p in patch_embeds_list], )
531 + out = self.transformer(patch_embeds, mask=mask, freqs_cis=freqs_cis)
532 +
533 + # remove batch dimension of the single sequence
534 + return out.squeeze(0)
535 +
536 +
537 +class VisionLanguageAdapter(nn.Module):
538 +
539 + def __init__(self, args: VisionEncoderArgs, dim: int):
540 + super().__init__()
541 + assert isinstance(args, VisionEncoderArgs)
542 + self.w_in = nn.Linear(
543 + args.hidden_size,
544 + dim,
545 + bias=True,
546 + )
547 + self.gelu = nn.GELU()
548 + self.w_out = nn.Linear(dim, dim, bias=True)
549 +
550 + def forward(self, x: torch.Tensor) -> torch.Tensor:
551 + return self.w_out(self.gelu(self.w_in(x)))
vllm/transformers_utils/config.py
+18 -7
@@ -70,7 +70,7 @@ def file_or_path_exists(model: Union[str, Path], config_name, revision,
70 if Path(model).exists():
71 return (Path(model) / config_name).is_file()
72
73 - return file_exists(model, HF_CONFIG_NAME, revision=revision, token=token)
73 + return file_exists(model, config_name, revision=revision, token=token)
74
75
76 def get_config(
@@ -205,14 +205,25 @@ def load_params_config(model, revision) -> PretrainedConfig:
205 config_dict["hidden_act"] = config_dict.get("activation", "silu")
206 config_dict["tie_word_embeddings"] = config_dict.get(
207 "tie_embeddings", False)
208 + config_dict["max_seq_len"] = config_dict.get("max_seq_len", 128_000)
209
209 - if config_dict["model_type"] == "transformer":
210 - if "moe" in config_dict:
211 - config_dict["architectures"] = ["MixtralForCausalLM"]
212 - else:
213 - config_dict["architectures"] = ["MistralForCausalLM"]
210 + if config_dict.get("moe") is not None:
211 + config_dict["architectures"] = ["MixtralForCausalLM"]
212 + else:
213 + config_dict["architectures"] = ["MistralForCausalLM"]
214 +
215 + if config_dict.get("vision_encoder") is not None:
216 + multimodal_config = config_dict.pop("vision_encoder")
217
215 - return recurse_elems(config_dict)
218 + config_dict = {
219 + "text_config": config_dict,
220 + "vision_config": multimodal_config
221 + }
222 + config_dict["architectures"] = ["PixtralForConditionalGeneration"]
223 + config_dict["model_type"] = "pixtral"
224 +
225 + config = recurse_elems(config_dict)
226 + return config
227
228
229 def get_hf_image_processor_config(