@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c0d8f1636c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] SiglipVisionModel ported from transformers (#6942)

Co-authored-by: Roger Wang <ywang@roblox.com>

Jungho Christopher Cho committed Aug 5, 2024 at 15:22 UTC c0d8f1636c58f5464e512eaabfed5aa29f2c5b7d
3 files changed +650 -53
examples/offline_inference_vision_language.py
+2 -1
@@ -65,7 +65,8 @@ def run_phi3v(question):
65 # PaliGemma
66 def run_paligemma(question):
67
68 - prompt = question
68 + # PaliGemma has special prompt format for VQA
69 + prompt = "caption en"
70 llm = LLM(model="google/paligemma-3b-mix-224")
71
72 return llm, prompt
vllm/model_executor/models/paligemma.py
+27 -52
@@ -1,9 +1,8 @@
1 from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2
3 import torch
4 -from PIL import Image
4 from torch import nn
6 -from transformers import PaliGemmaConfig, SiglipVisionConfig, SiglipVisionModel
5 +from transformers import PaliGemmaConfig
6
7 from vllm.attention import AttentionMetadata
8 from vllm.config import CacheConfig, MultiModalConfig
@@ -18,9 +17,11 @@ from vllm.model_executor.models.gemma import GemmaModel
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 from vllm.multimodal.image import cached_get_tokenizer
21 -from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
20 +from vllm.sequence import IntermediateTensors, SamplerOutput
21
22 from .interfaces import SupportsVision
23 +from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24 + dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
25 from .utils import merge_vision_embeddings
26
27 logger = init_logger(__name__)
@@ -32,55 +33,22 @@ _KEYS_TO_MODIFY_MAPPING = {
33
34 def get_max_paligemma_image_tokens(ctx: InputContext):
35 hf_config = ctx.get_hf_config(PaliGemmaConfig)
35 - text_config = hf_config.text_config
36 -
37 - return text_config.num_image_tokens
38 -
39 -
40 -def dummy_seq_data_for_paligemma(
41 - hf_config: PaliGemmaConfig,
42 - seq_len: int,
43 - *,
44 - image_token_id: int,
45 - image_feature_size_override: Optional[int] = None,
46 -):
47 - if image_feature_size_override is None:
48 - image_feature_size = hf_config.text_config.num_image_tokens
49 - else:
50 - image_feature_size = image_feature_size_override
51 -
52 - token_ids = [image_token_id] * image_feature_size
53 - token_ids += [0] * (seq_len - image_feature_size)
54 - return SequenceData(token_ids)
55 -
56 -
57 -def dummy_image_for_paligemma(
58 - hf_config: SiglipVisionConfig,
59 - *,
60 - image_width_override: Optional[int] = None,
61 - image_height_override: Optional[int] = None,
62 -):
63 - width = height = hf_config.image_size
64 - if image_width_override is not None:
65 - width = image_width_override
66 - if image_height_override is not None:
67 - height = image_height_override
36 + vision_config = hf_config.vision_config
37
69 - image = Image.new("RGB", (width, height), color=0)
70 - return {"image": image}
38 + return get_max_siglip_image_tokens(vision_config)
39
40
41 def dummy_data_for_paligemma(ctx: InputContext, seq_len: int):
42 hf_config = ctx.get_hf_config(PaliGemmaConfig)
43 vision_config = hf_config.vision_config
44
77 - seq_data = dummy_seq_data_for_paligemma(
78 - hf_config,
45 + seq_data = dummy_seq_data_for_siglip(
46 + vision_config,
47 seq_len,
48 image_token_id=hf_config.image_token_index,
49 )
50
83 - mm_data = dummy_image_for_paligemma(vision_config)
51 + mm_data = dummy_image_for_siglip(vision_config)
52 return seq_data, mm_data
53
54
@@ -208,30 +176,37 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
176 data=self._validate_pixel_values(pixel_values),
177 )
178
211 - def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
212 - pixel_values: torch.Tensor) -> torch.Tensor:
179 + def _image_pixels_to_features(
180 + self,
181 + vision_tower: SiglipVisionModel,
182 + pixel_values: torch.Tensor,
183 + ) -> torch.Tensor:
184
185 target_dtype = vision_tower.get_input_embeddings().weight.dtype
215 - image_outputs = vision_tower(pixel_values.to(dtype=target_dtype),
216 - output_hidden_states=True)
217 -
218 - selected_image_features = image_outputs.last_hidden_state
186 + image_features = vision_tower(pixel_values.to(dtype=target_dtype))
187
220 - return selected_image_features
188 + return image_features
189
190 def _process_image_pixels(
223 - self, inputs: PaliGemmaImagePixelInputs) -> torch.Tensor:
191 + self,
192 + inputs: PaliGemmaImagePixelInputs,
193 + ) -> torch.Tensor:
194 assert self.vision_tower is not None
195
196 pixel_values = inputs["data"]
197
228 - return self._image_pixels_to_features(self.vision_tower, pixel_values)
198 + return self._image_pixels_to_features(
199 + self.vision_tower,
200 + pixel_values,
201 + )
202
203 def _process_image_input(
231 - self, image_input: PaliGemmaImageInputs) -> torch.Tensor:
204 + self,
205 + image_input: PaliGemmaImageInputs,
206 + ) -> torch.Tensor:
207
208 assert self.vision_tower is not None
234 - image_features = self._process_image_pixels(image_input)
209 + image_features = self._process_image_pixels(image_input, )
210
211 return self.multi_modal_projector(image_features)
212
vllm/model_executor/models/siglip.py new
+621
@@ -0,0 +1,621 @@
1 +"""Implementation of SiglipVisionModel intended to be only used
2 +within a vision language model."""
3 +
4 +import math
5 +from typing import Optional, Tuple
6 +
7 +import torch
8 +from PIL import Image
9 +from torch import nn
10 +from transformers import SiglipConfig, SiglipVisionConfig
11 +from transformers.models.siglip.modeling_siglip import SiglipAttention
12 +from vllm_flash_attn import flash_attn_func
13 +from xformers.ops import memory_efficient_attention
14 +
15 +from vllm.config import ModelConfig
16 +from vllm.distributed import get_tensor_model_parallel_world_size
17 +from vllm.inputs import LLMInputs
18 +from vllm.model_executor.layers.activation import get_act_fn
19 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20 + QKVParallelLinear,
21 + RowParallelLinear)
22 +from vllm.model_executor.layers.quantization import QuantizationConfig
23 +from vllm.model_executor.layers.vocab_parallel_embedding import (
24 + VocabParallelEmbedding)
25 +from vllm.multimodal.image import (cached_get_tokenizer,
26 + repeat_and_pad_image_tokens)
27 +from vllm.sequence import SequenceData
28 +
29 +
30 +def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
31 + assert image_size % patch_size == 0
32 + return image_size // patch_size
33 +
34 +
35 +def get_siglip_num_patches(*, image_size: int, patch_size: int) -> int:
36 + grid_length = get_siglip_patch_grid_length(image_size=image_size,
37 + patch_size=patch_size)
38 + return grid_length * grid_length
39 +
40 +
41 +def get_siglip_image_feature_size(hf_config: SiglipVisionConfig) -> int:
42 + return get_siglip_num_patches(image_size=hf_config.image_size,
43 + patch_size=hf_config.patch_size)
44 +
45 +
46 +def get_max_siglip_image_tokens(hf_config: SiglipVisionConfig) -> int:
47 + return get_siglip_image_feature_size(hf_config)
48 +
49 +
50 +def dummy_seq_data_for_siglip(
51 + hf_config: SiglipVisionConfig,
52 + seq_len: int,
53 + *,
54 + image_token_id: int,
55 + image_feature_size_override: Optional[int] = None,
56 +):
57 + if image_feature_size_override is None:
58 + image_feature_size = get_siglip_image_feature_size(hf_config)
59 + else:
60 + image_feature_size = image_feature_size_override
61 +
62 + token_ids = [image_token_id] * image_feature_size
63 + token_ids += [0] * (seq_len - image_feature_size)
64 + return SequenceData(token_ids)
65 +
66 +
67 +def dummy_image_for_siglip(
68 + hf_config: SiglipVisionConfig,
69 + *,
70 + image_width_override: Optional[int] = None,
71 + image_height_override: Optional[int] = None,
72 +):
73 + width = height = hf_config.image_size
74 + if image_width_override is not None:
75 + width = image_width_override
76 + if image_height_override is not None:
77 + height = image_height_override
78 +
79 + image = Image.new("RGB", (width, height), color=0)
80 + return {"image": image}
81 +
82 +
83 +def input_processor_for_siglip(
84 + model_config: ModelConfig,
85 + hf_config: SiglipVisionConfig,
86 + llm_inputs: LLMInputs,
87 + *,
88 + image_token_id: int,
89 + image_feature_size_override: Optional[int] = None,
90 +):
91 + multi_modal_data = llm_inputs.get("multi_modal_data")
92 + if multi_modal_data is None or "image" not in multi_modal_data:
93 + return llm_inputs
94 +
95 + tokenizer = cached_get_tokenizer(model_config.tokenizer)
96 +
97 + if image_feature_size_override is None:
98 + image_feature_size = get_siglip_image_feature_size(hf_config)
99 + else:
100 + image_feature_size = image_feature_size_override
101 +
102 + new_prompt, new_token_ids = repeat_and_pad_image_tokens(
103 + tokenizer,
104 + llm_inputs.get("prompt"),
105 + llm_inputs["prompt_token_ids"],
106 + image_token_id=image_token_id,
107 + repeat_count=image_feature_size,
108 + )
109 +
110 + # NOTE: Create a defensive copy of the original inputs
111 + return LLMInputs(
112 + prompt_token_ids=new_token_ids,
113 + prompt=new_prompt,
114 + multi_modal_data=multi_modal_data,
115 + )
116 +
117 +
118 +# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
119 +class SiglipVisionEmbeddings(nn.Module):
120 +
121 + def __init__(self, config: SiglipVisionConfig):
122 + super().__init__()
123 + self.config = config
124 + self.embed_dim = config.hidden_size
125 + self.image_size = config.image_size
126 + self.patch_size = config.patch_size
127 +
128 + self.patch_embedding = nn.Conv2d(
129 + in_channels=config.num_channels,
130 + out_channels=self.embed_dim,
131 + kernel_size=self.patch_size,
132 + stride=self.patch_size,
133 + padding="valid",
134 + )
135 +
136 + self.num_patches = (self.image_size // self.patch_size)**2
137 + self.num_positions = self.num_patches
138 + self.position_embedding = VocabParallelEmbedding(
139 + self.num_positions, self.embed_dim)
140 + self.register_buffer(
141 + "position_ids",
142 + torch.arange(self.num_positions, dtype=torch.int64).expand(
143 + (1, -1)),
144 + persistent=False,
145 + )
146 +
147 + def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int,
148 + width: int) -> torch.Tensor:
149 + """
150 + This method is an adapted method for SigLIP (due to SigLIP not having
151 + class embedding unlike other ViTs) that allows the model to interpolate
152 + the pre-trained position encodings such that it can be usable on higher
153 + resolution images.
154 +
155 + Source:
156 + https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174
157 + """
158 + position_embeddings = self.position_embedding.weight.unsqueeze(0)
159 + num_patches = embeddings.shape[1]
160 + num_positions = position_embeddings.shape[1]
161 + if num_patches == num_positions and height == width:
162 + return position_embeddings
163 +
164 + dim = embeddings.shape[-1]
165 + height = height // self.patch_size
166 + width = width // self.patch_size
167 + # we add a small number to avoid floating point error
168 + # in the interpolation
169 + # see discussion at https://github.com/facebookresearch/dino/issues/8
170 + height, width = height + 0.1, width + 0.1
171 +
172 + patch_pos_embed = position_embeddings.reshape(
173 + 1, int(math.sqrt(num_positions)), int(math.sqrt(num_positions)),
174 + dim)
175 + patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)
176 + patch_pos_embed = nn.functional.interpolate(
177 + patch_pos_embed,
178 + scale_factor=(
179 + height / math.sqrt(num_positions),
180 + width / math.sqrt(num_positions),
181 + ),
182 + mode="bicubic",
183 + align_corners=False,
184 + )
185 + if (int(height) != patch_pos_embed.shape[-2]
186 + or int(width) != patch_pos_embed.shape[-1]):
187 + raise ValueError("Width or height does not match with "
188 + "the interpolated position embeddings")
189 +
190 + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
191 + return patch_pos_embed
192 +
193 + def forward(self,
194 + pixel_values: torch.Tensor,
195 + interpolate_pos_encoding: bool = False) -> torch.Tensor:
196 + _, _, height, width = pixel_values.shape
197 + target_dtype = self.patch_embedding.weight.dtype
198 + patch_embeds = self.patch_embedding(pixel_values.to(
199 + dtype=target_dtype)) # shape = [*, width, grid, grid]
200 + embeddings = patch_embeds.flatten(2).transpose(1, 2)
201 +
202 + if interpolate_pos_encoding:
203 + embeddings = embeddings + self.interpolate_pos_encoding(
204 + embeddings, height, width)
205 + else:
206 + embeddings = embeddings + self.position_embedding(
207 + self.position_ids)
208 + return embeddings
209 +
210 +
211 +# NOTE: Not used - kept for later when we TP the ViT
212 +# TODO(ChristopherCho): Implement TP version of Attention
213 +class SiglipTPAttention(nn.Module):
214 +
215 + def __init__(
216 + self,
217 + config,
218 + quant_config: Optional[QuantizationConfig] = None,
219 + ):
220 + super().__init__()
221 + self.config = config
222 + self.embed_dim = config.hidden_size
223 +
224 + tp_size = get_tensor_model_parallel_world_size()
225 + self.total_num_heads = config.num_attention_heads
226 + if self.total_num_heads % tp_size != 0:
227 + raise ValueError(
228 + f"Number of attention heads ({self.total_num_heads}) "
229 + "must be divisible by the tensor model parallel size"
230 + f" ({tp_size}).")
231 +
232 + self.num_heads = self.total_num_heads // tp_size
233 + self.head_dim = self.embed_dim // self.total_num_heads
234 + if self.head_dim * self.total_num_heads != self.embed_dim:
235 + raise ValueError(f"embed_dim must be divisible by num_heads (got "
236 + "`embed_dim`: {self.embed_dim} and `num_heads`:"
237 + f" {self.num_heads}).")
238 + self.qkv_size = self.num_heads * self.head_dim
239 + self.scale = self.head_dim**-0.5
240 + self.dropout = config.attention_dropout
241 +
242 + self.qkv_proj = QKVParallelLinear(
243 + hidden_size=self.embed_dim,
244 + head_size=self.head_dim,
245 + total_num_heads=self.total_num_heads,
246 + quant_config=quant_config,
247 + )
248 + self.out_proj = RowParallelLinear(
249 + input_size=self.embed_dim,
250 + output_size=self.embed_dim,
251 + quant_config=quant_config,
252 + )
253 +
254 + self.attn_fn = self._basic_attention_forward
255 +
256 + def forward(
257 + self,
258 + hidden_states: torch.Tensor,
259 + ) -> torch.Tensor:
260 + """Input shape: Batch x Time x Channel"""
261 + batch_size, q_len, _ = hidden_states.size()
262 +
263 + qkv_states, _ = self.qkv_proj(hidden_states)
264 + query_states, key_states, value_states = qkv_states.split(
265 + [self.qkv_size] * 3, dim=-1)
266 +
267 + attn_output = self.attn_fn(
268 + q=query_states,
269 + k=key_states,
270 + v=value_states,
271 + batch_size=batch_size,
272 + q_len=q_len,
273 + )
274 +
275 + attn_output, _ = self.out_proj(attn_output)
276 + return attn_output
277 +
278 + def _basic_attention_forward(self, q, k, v, batch_size, q_len):
279 + q = q.view(batch_size, q_len, self.num_heads,
280 + self.head_dim).transpose(1, 2)
281 + k = k.view(batch_size, q_len, self.num_heads,
282 + self.head_dim).transpose(1, 2)
283 + v = v.view(batch_size, q_len, self.num_heads,
284 + self.head_dim).transpose(1, 2)
285 +
286 + k_v_seq_len = k.shape[-2]
287 + attn_weights = torch.matmul(q, k.transpose(2, 3)) * self.scale
288 +
289 + if attn_weights.size() != (
290 + batch_size,
291 + self.num_heads,
292 + q_len,
293 + k_v_seq_len,
294 + ):
295 + raise ValueError(
296 + "Attention weights should be of size "
297 + f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
298 + f" {attn_weights.size()}")
299 +
300 + # upcast attention to fp32
301 + attn_weights = nn.functional.softmax(attn_weights,
302 + dim=-1,
303 + dtype=torch.float32).to(q.dtype)
304 + attn_weights = nn.functional.dropout(attn_weights,
305 + p=self.dropout,
306 + training=self.training)
307 + attn_output = torch.matmul(attn_weights, v)
308 +
309 + if attn_output.size() != (
310 + batch_size,
311 + self.num_heads,
312 + q_len,
313 + self.head_dim,
314 + ):
315 + raise ValueError(
316 + "`attn_output` should be of size "
317 + f"{(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
318 + f" {attn_output.size()}")
319 +
320 + attn_output = attn_output.transpose(1, 2).contiguous()
321 + attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
322 +
323 + return attn_output
324 +
325 +
326 +# NOTE: Not used - kept for later when we TP the ViT
327 +# TODO(ChristopherCho): flash_attn_func is not working properly.
328 +# It constantly throws a CUDA error.
329 +class SiglipFlashAttention2(SiglipTPAttention):
330 +
331 + def __init__(self, *args, **kwargs):
332 + super().__init__(*args, **kwargs)
333 + self.attn_fn = self._flash_attention_forward
334 +
335 + # Ported from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L449
336 + # and https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/modeling_flash_attention_utils.py#L133
337 + def _flash_attention_forward(self, q, k, v, batch_size, q_len, *args,
338 + **kwargs):
339 + """Implements the multihead softmax attention.
340 + Arguments
341 + ---------
342 + q, k, v: The tensor containing the
343 + query, key, and value. (B, S, H, D)
344 + """
345 +
346 + q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
347 + k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
348 + v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
349 +
350 + attn_output = flash_attn_func(
351 + q,
352 + k,
353 + v,
354 + dropout_p=self.dropout,
355 + causal=False,
356 + )
357 +
358 + attn_output = attn_output.reshape(batch_size, q_len,
359 + self.embed_dim).contiguous()
360 +
361 + return attn_output
362 +
363 +
364 +# NOTE: Not used - kept for later when we TP the ViT
365 +class SiglipSdpaAttention(SiglipTPAttention):
366 +
367 + def __init__(self, *args, **kwargs):
368 + super().__init__(*args, **kwargs)
369 + self.is_causal = False
370 + self.attn_fn = self._sdpa_attention_forward
371 +
372 + def _sdpa_attention_forward(self, q, k, v, batch_size, q_len):
373 + q = q.view(batch_size, q_len, self.num_heads,
374 + self.head_dim).transpose(1, 2)
375 + k = k.view(batch_size, q_len, self.num_heads,
376 + self.head_dim).transpose(1, 2)
377 + v = v.view(batch_size, q_len, self.num_heads,
378 + self.head_dim).transpose(1, 2)
379 +
380 + attn_output = torch.nn.functional.scaled_dot_product_attention(
381 + q, k, v, dropout_p=self.dropout, is_causal=False, scale=self.scale)
382 +
383 + attn_output = attn_output.transpose(1, 2).contiguous()
384 + attn_output = attn_output.view(batch_size, q_len, self.embed_dim)
385 +
386 + return attn_output
387 +
388 +
389 +# NOTE: Not used - kept for later when we TP the ViT
390 +class SiglipxFormersAttention(SiglipTPAttention):
391 +
392 + def __init__(self, *args, **kwargs):
393 + super().__init__(*args, **kwargs)
394 + self.attn_fn = self._xformers_attention_forward
395 +
396 + def _xformers_attention_forward(self, q, k, v, batch_size, q_len):
397 + q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
398 + k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
399 + v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
400 +
401 + attn_output = memory_efficient_attention(q,
402 + k,
403 + v,
404 + p=0.0,
405 + scale=self.scale)
406 + attn_output = attn_output.reshape(batch_size, q_len,
407 + self.embed_dim).contiguous()
408 +
409 + return attn_output
410 +
411 +
412 +# NOTE: Not used - kept for later when we TP the ViT
413 +SIGLIP_ATTENTION_CLASSES = {
414 + "eager": SiglipTPAttention,
415 + "flash_attention_2": SiglipFlashAttention2,
416 + "sdpa": SiglipSdpaAttention,
417 + "xformers": SiglipxFormersAttention,
418 +}
419 +
420 +
421 +class SiglipMLP(nn.Module):
422 +
423 + def __init__(
424 + self,
425 + config,
426 + quant_config: Optional[QuantizationConfig] = None,
427 + ):
428 + super().__init__()
429 + self.config = config
430 + self.activation_fn = get_act_fn(config.hidden_act)
431 +
432 + # For quantization, we require the hidden size to be a multiple of 64
433 + quantizable = (config.hidden_size % 64 == 0
434 + and config.intermediate_size % 64 == 0)
435 + self.fc1 = ColumnParallelLinear(
436 + config.hidden_size,
437 + config.intermediate_size,
438 + quant_config=quant_config if quantizable else None,
439 + )
440 + self.fc2 = RowParallelLinear(
441 + config.intermediate_size,
442 + config.hidden_size,
443 + quant_config=quant_config if quantizable else None,
444 + )
445 +
446 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
447 + hidden_states, _ = self.fc1(hidden_states)
448 + hidden_states = self.activation_fn(hidden_states)
449 + hidden_states, _ = self.fc2(hidden_states)
450 + return hidden_states
451 +
452 +
453 +class SiglipEncoderLayer(nn.Module):
454 +
455 + def __init__(
456 + self,
457 + config: SiglipConfig,
458 + quant_config: Optional[QuantizationConfig] = None,
459 + ):
460 + super().__init__()
461 + self.embed_dim = config.hidden_size
462 +
463 + # TODO(ChristopherCho): use TP'ed Attention block
464 + self.self_attn = SiglipAttention(config)
465 + self.layer_norm1 = nn.LayerNorm(self.embed_dim,
466 + eps=config.layer_norm_eps)
467 + self.mlp = SiglipMLP(
468 + config,
469 + quant_config=quant_config,
470 + )
471 + self.layer_norm2 = nn.LayerNorm(self.embed_dim,
472 + eps=config.layer_norm_eps)
473 +
474 + def forward(
475 + self,
476 + hidden_states: torch.Tensor,
477 + ) -> Tuple[torch.Tensor]:
478 + residual = hidden_states
479 +
480 + hidden_states = self.layer_norm1(hidden_states)
481 + hidden_states, _ = self.self_attn(hidden_states=hidden_states)
482 + hidden_states = residual + hidden_states
483 +
484 + residual = hidden_states
485 + hidden_states = self.layer_norm2(hidden_states)
486 + hidden_states = self.mlp(hidden_states)
487 + hidden_states = residual + hidden_states
488 +
489 + return hidden_states, None
490 +
491 +
492 +class SiglipEncoder(nn.Module):
493 +
494 + def __init__(
495 + self,
496 + config: SiglipConfig,
497 + quant_config: Optional[QuantizationConfig] = None,
498 + ):
499 + super().__init__()
500 + self.config = config
501 + self.layers = nn.ModuleList([
502 + SiglipEncoderLayer(
503 + config,
504 + quant_config=quant_config,
505 + ) for _ in range(config.num_hidden_layers)
506 + ])
507 +
508 + def forward(
509 + self,
510 + inputs_embeds: torch.Tensor,
511 + ) -> Tuple:
512 + hidden_states = inputs_embeds
513 + for encoder_layer in self.layers:
514 + hidden_states, _ = encoder_layer(hidden_states)
515 +
516 + return hidden_states
517 +
518 +
519 +class SiglipMultiheadAttentionPoolingHead(nn.Module):
520 + """Multihead Attention Pooling."""
521 +
522 + def __init__(
523 + self,
524 + config: SiglipVisionConfig,
525 + quant_config: Optional[QuantizationConfig] = None,
526 + ):
527 + super().__init__()
528 +
529 + self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
530 + # TODO(ChristopherCho): Implement vLLM version of MultiheadAttention
531 + self.attention = torch.nn.MultiheadAttention(
532 + config.hidden_size, config.num_attention_heads, batch_first=True)
533 + self.layernorm = nn.LayerNorm(config.hidden_size,
534 + eps=config.layer_norm_eps)
535 + self.mlp = SiglipMLP(config=config, quant_config=quant_config)
536 +
537 + def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
538 + batch_size = hidden_state.shape[0]
539 + probe = self.probe.repeat(batch_size, 1, 1)
540 +
541 + hidden_state = self.attention(probe, hidden_state, hidden_state)[0]
542 +
543 + residual = hidden_state
544 + hidden_state = self.layernorm(hidden_state)
545 + hidden_state = residual + self.mlp(hidden_state)
546 +
547 + return hidden_state[:, 0]
548 +
549 +
550 +class SiglipVisionTransformer(nn.Module):
551 +
552 + def __init__(
553 + self,
554 + config: SiglipVisionConfig,
555 + quant_config: Optional[QuantizationConfig] = None,
556 + ):
557 + super().__init__()
558 + self.config = config
559 + embed_dim = config.hidden_size
560 +
561 + self.embeddings = SiglipVisionEmbeddings(config)
562 + self.encoder = SiglipEncoder(
563 + config,
564 + quant_config=quant_config,
565 + )
566 + self.post_layernorm = nn.LayerNorm(embed_dim,
567 + eps=config.layer_norm_eps)
568 + self.use_head = (True if not hasattr(config, "vision_use_head") else
569 + config.vision_use_head)
570 + if self.use_head:
571 + self.head = SiglipMultiheadAttentionPoolingHead(
572 + config=config, quant_config=quant_config)
573 +
574 + def forward(
575 + self,
576 + pixel_values: torch.Tensor,
577 + interpolate_pos_encoding: bool = True,
578 + ) -> torch.Tensor:
579 + hidden_states = self.embeddings(
580 + pixel_values,
581 + interpolate_pos_encoding=interpolate_pos_encoding,
582 + )
583 +
584 + encoder_outputs = self.encoder(inputs_embeds=hidden_states)
585 +
586 + last_hidden_state = self.post_layernorm(encoder_outputs)
587 +
588 + # TODO: add this back when pooled_output is used in inference
589 + # if self.use_head:
590 + # pooled_output = self.head(last_hidden_state)
591 +
592 + return last_hidden_state
593 +
594 +
595 +class SiglipVisionModel(nn.Module):
596 + config_class = SiglipVisionConfig
597 + main_input_name = "pixel_values"
598 +
599 + def __init__(
600 + self,
601 + config: SiglipVisionConfig,
602 + quant_config: Optional[QuantizationConfig] = None,
603 + ):
604 + super().__init__()
605 + self.vision_model = SiglipVisionTransformer(
606 + config,
607 + quant_config,
608 + )
609 +
610 + def get_input_embeddings(self) -> nn.Module:
611 + return self.vision_model.embeddings.patch_embedding
612 +
613 + def forward(
614 + self,
615 + pixel_values: torch.Tensor,
616 + interpolate_pos_encoding: bool = False,
617 + ) -> torch.Tensor:
618 + return self.vision_model(
619 + pixel_values=pixel_values,
620 + interpolate_pos_encoding=interpolate_pos_encoding,
621 + )