A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Model] SiglipVisionModel ported from transformers (#6942)
Co-authored-by: Roger Wang <ywang@roblox.com>
Jungho Christopher Cho committed
Aug 5, 2024 at 15:22 UTC
c0d8f1636c58f5464e512eaabfed5aa29f2c5b7d
3 files changed
+650
-53
examples/offline_inference_vision_language.py
+2
-1
@@ -65,7 +65,8 @@ def run_phi3v(question):
65
# PaliGemma
66
def run_paligemma(question):
67
68
- prompt = question
68
+ # PaliGemma has special prompt format for VQA
69
+ prompt = "caption en"
70
llm = LLM(model="google/paligemma-3b-mix-224")
71
72
return llm, prompt
vllm/model_executor/models/paligemma.py
+27
-52
@@ -1,9 +1,8 @@
1
from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2
3
import torch
4
-from PIL import Image
4
from torch import nn
6
-from transformers import PaliGemmaConfig, SiglipVisionConfig, SiglipVisionModel
5
+from transformers import PaliGemmaConfig
6
7
from vllm.attention import AttentionMetadata
8
from vllm.config import CacheConfig, MultiModalConfig
@@ -18,9 +17,11 @@ from vllm.model_executor.models.gemma import GemmaModel
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.multimodal.image import cached_get_tokenizer
21
-from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
20
+from vllm.sequence import IntermediateTensors, SamplerOutput
21
22
from .interfaces import SupportsVision
23
+from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
24
+ dummy_seq_data_for_siglip, get_max_siglip_image_tokens)
25
from .utils import merge_vision_embeddings
26
27
logger = init_logger(__name__)
@@ -32,55 +33,22 @@ _KEYS_TO_MODIFY_MAPPING = {
33
34
def get_max_paligemma_image_tokens(ctx: InputContext):
35
hf_config = ctx.get_hf_config(PaliGemmaConfig)
35
- text_config = hf_config.text_config
36
-
37
- return text_config.num_image_tokens
38
-
39
-
40
-def dummy_seq_data_for_paligemma(
41
- hf_config: PaliGemmaConfig,
42
- seq_len: int,
43
- *,
44
- image_token_id: int,
45
- image_feature_size_override: Optional[int] = None,
46
-):
47
- if image_feature_size_override is None:
48
- image_feature_size = hf_config.text_config.num_image_tokens
49
- else:
50
- image_feature_size = image_feature_size_override
51
-
52
- token_ids = [image_token_id] * image_feature_size
53
- token_ids += [0] * (seq_len - image_feature_size)
54
- return SequenceData(token_ids)
55
-
56
-
57
-def dummy_image_for_paligemma(
58
- hf_config: SiglipVisionConfig,
59
- *,
60
- image_width_override: Optional[int] = None,
61
- image_height_override: Optional[int] = None,
62
-):
63
- width = height = hf_config.image_size
64
- if image_width_override is not None:
65
- width = image_width_override
66
- if image_height_override is not None:
67
- height = image_height_override
36
+ vision_config = hf_config.vision_config
37
69
- image = Image.new("RGB", (width, height), color=0)
70
- return {"image": image}
38
+ return get_max_siglip_image_tokens(vision_config)
39
40
41
def dummy_data_for_paligemma(ctx: InputContext, seq_len: int):
42
hf_config = ctx.get_hf_config(PaliGemmaConfig)
43
vision_config = hf_config.vision_config
44
77
- seq_data = dummy_seq_data_for_paligemma(
78
- hf_config,
45
+ seq_data = dummy_seq_data_for_siglip(
46
+ vision_config,
47
seq_len,
48
image_token_id=hf_config.image_token_index,
49
)
50
83
- mm_data = dummy_image_for_paligemma(vision_config)
51
+ mm_data = dummy_image_for_siglip(vision_config)
52
return seq_data, mm_data
53
54
@@ -208,30 +176,37 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsVision):
176
data=self._validate_pixel_values(pixel_values),
177
)
178
211
- def _image_pixels_to_features(self, vision_tower: SiglipVisionModel,
212
- pixel_values: torch.Tensor) -> torch.Tensor:
179
+ def _image_pixels_to_features(
180
+ self,
181
+ vision_tower: SiglipVisionModel,
182
+ pixel_values: torch.Tensor,
183
+ ) -> torch.Tensor:
184
185
target_dtype = vision_tower.get_input_embeddings().weight.dtype
215
- image_outputs = vision_tower(pixel_values.to(dtype=target_dtype),
216
- output_hidden_states=True)
217
-
218
- selected_image_features = image_outputs.last_hidden_state
186
+ image_features = vision_tower(pixel_values.to(dtype=target_dtype))
187
220
- return selected_image_features
188
+ return image_features
189
190
def _process_image_pixels(
223
- self, inputs: PaliGemmaImagePixelInputs) -> torch.Tensor:
191
+ self,
192
+ inputs: PaliGemmaImagePixelInputs,
193
+ ) -> torch.Tensor:
194
assert self.vision_tower is not None
195
196
pixel_values = inputs["data"]
197
228
- return self._image_pixels_to_features(self.vision_tower, pixel_values)
198
+ return self._image_pixels_to_features(
199
+ self.vision_tower,
200
+ pixel_values,
201
+ )
202
203
def _process_image_input(
231
- self, image_input: PaliGemmaImageInputs) -> torch.Tensor:
204
+ self,
205
+ image_input: PaliGemmaImageInputs,
206
+ ) -> torch.Tensor:
207
208
assert self.vision_tower is not None
234
- image_features = self._process_image_pixels(image_input)
209
+ image_features = self._process_image_pixels(image_input, )
210
211
return self.multi_modal_projector(image_features)
212
vllm/model_executor/models/siglip.py
new
+621
@@ -0,0 +1,621 @@
1
+"""Implementation of SiglipVisionModel intended to be only used
2
+within a vision language model."""
3
+
4
+import math
5
+from typing import Optional, Tuple
6
+
7
+import torch
8
+from PIL import Image
9
+from torch import nn
10
+from transformers import SiglipConfig, SiglipVisionConfig
11
+from transformers.models.siglip.modeling_siglip import SiglipAttention
12
+from vllm_flash_attn import flash_attn_func
13
+from xformers.ops import memory_efficient_attention
14
+
15
+from vllm.config import ModelConfig
16
+from vllm.distributed import get_tensor_model_parallel_world_size
17
+from vllm.inputs import LLMInputs
18
+from vllm.model_executor.layers.activation import get_act_fn
19
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
+ QKVParallelLinear,
21
+ RowParallelLinear)
22
+from vllm.model_executor.layers.quantization import QuantizationConfig
23
+from vllm.model_executor.layers.vocab_parallel_embedding import (
24
+ VocabParallelEmbedding)
25
+from vllm.multimodal.image import (cached_get_tokenizer,
26
+ repeat_and_pad_image_tokens)
27
+from vllm.sequence import SequenceData
28
+
29
+
30
+def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
31
+ assert image_size % patch_size == 0
32
+ return image_size // patch_size
33
+
34
+
35
+def get_siglip_num_patches(*, image_size: int, patch_size: int) -> int:
36
+ grid_length = get_siglip_patch_grid_length(image_size=image_size,
37
+ patch_size=patch_size)
38
+ return grid_length * grid_length
39
+
40
+
41
+def get_siglip_image_feature_size(hf_config: SiglipVisionConfig) -> int:
42
+ return get_siglip_num_patches(image_size=hf_config.image_size,
43
+ patch_size=hf_config.patch_size)
44
+
45
+
46
+def get_max_siglip_image_tokens(hf_config: SiglipVisionConfig) -> int:
47
+ return get_siglip_image_feature_size(hf_config)
48
+
49
+
50
+def dummy_seq_data_for_siglip(
51
+ hf_config: SiglipVisionConfig,
52
+ seq_len: int,
53
+ *,
54
+ image_token_id: int,
55
+ image_feature_size_override: Optional[int] = None,
56
+):
57
+ if image_feature_size_override is None:
58
+ image_feature_size = get_siglip_image_feature_size(hf_config)
59
+ else:
60
+ image_feature_size = image_feature_size_override
61
+
62
+ token_ids = [image_token_id] * image_feature_size
63
+ token_ids += [0] * (seq_len - image_feature_size)
64
+ return SequenceData(token_ids)
65
+
66
+
67
+def dummy_image_for_siglip(
68
+ hf_config: SiglipVisionConfig,
69
+ *,
70
+ image_width_override: Optional[int] = None,
71
+ image_height_override: Optional[int] = None,
72
+):
73
+ width = height = hf_config.image_size
74
+ if image_width_override is not None:
75
+ width = image_width_override
76
+ if image_height_override is not None:
77
+ height = image_height_override
78
+
79
+ image = Image.new("RGB", (width, height), color=0)
80
+ return {"image": image}
81
+
82
+
83
+def input_processor_for_siglip(
84
+ model_config: ModelConfig,
85
+ hf_config: SiglipVisionConfig,
86
+ llm_inputs: LLMInputs,
87
+ *,
88
+ image_token_id: int,
89
+ image_feature_size_override: Optional[int] = None,
90
+):
91
+ multi_modal_data = llm_inputs.get("multi_modal_data")
92
+ if multi_modal_data is None or "image" not in multi_modal_data:
93
+ return llm_inputs
94
+
95
+ tokenizer = cached_get_tokenizer(model_config.tokenizer)
96
+
97
+ if image_feature_size_override is None:
98
+ image_feature_size = get_siglip_image_feature_size(hf_config)
99
+ else:
100
+ image_feature_size = image_feature_size_override
101
+
102
+ new_prompt, new_token_ids = repeat_and_pad_image_tokens(
103
+ tokenizer,
104
+ llm_inputs.get("prompt"),
105
+ llm_inputs["prompt_token_ids"],
106
+ image_token_id=image_token_id,
107
+ repeat_count=image_feature_size,
108
+ )
109
+
110
+ # NOTE: Create a defensive copy of the original inputs
111
+ return LLMInputs(
112
+ prompt_token_ids=new_token_ids,
113
+ prompt=new_prompt,
114
+ multi_modal_data=multi_modal_data,
115
+ )
116
+
117
+
118
+# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
119
+class SiglipVisionEmbeddings(nn.Module):
120
+
121
+ def __init__(self, config: SiglipVisionConfig):
122
+ super().__init__()
123
+ self.config = config
124
+ self.embed_dim = config.hidden_size
125
+ self.image_size = config.image_size
126
+ self.patch_size = config.patch_size
127
+
128
+ self.patch_embedding = nn.Conv2d(
129
+ in_channels=config.num_channels,
130
+ out_channels=self.embed_dim,
131
+ kernel_size=self.patch_size,
132
+ stride=self.patch_size,
133
+ padding="valid",
134
+ )
135
+
136
+ self.num_patches = (self.image_size // self.patch_size)**2
137
+ self.num_positions = self.num_patches
138
+ self.position_embedding = VocabParallelEmbedding(
139
+ self.num_positions, self.embed_dim)
140
+ self.register_buffer(
141
+ "position_ids",
142
+ torch.arange(self.num_positions, dtype=torch.int64).expand(
143
+ (1, -1)),
144
+ persistent=False,
145
+ )
146
+
147
+ def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int,
148
+ width: int) -> torch.Tensor:
149
+ """
150
+ This method is an adapted method for SigLIP (due to SigLIP not having
151
+ class embedding unlike other ViTs) that allows the model to interpolate
152
+ the pre-trained position encodings such that it can be usable on higher
153
+ resolution images.
154
+
155
+ Source:
156
+ https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174
157
+ """
158
+ position_embeddings = self.position_embedding.weight.unsqueeze(0)
159
+ num_patches = embeddings.shape[1]
160
+ num_positions = position_embeddings.shape[1]
161
+ if num_patches == num_positions and height == width:
162
+ return position_embeddings
163
+
164
+ dim = embeddings.shape[-1]
165
+ height = height // self.patch_size
166
+ width = width // self.patch_size
167
+ # we add a small number to avoid floating point error
168
+ # in the interpolation
169
+ # see discussion at https://github.com/facebookresearch/dino/issues/8
170
+ height, width = height + 0.1, width + 0.1
171
+
172
+ patch_pos_embed = position_embeddings.reshape(
173
+ 1, int(math.sqrt(num_positions)), int(math.sqrt(num_positions)),
174
+ dim)
175
+ patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)
176
+ patch_pos_embed = nn.functional.interpolate(
177
+ patch_pos_embed,
178
+ scale_factor=(
179
+ height / math.sqrt(num_positions),
180
+ width / math.sqrt(num_positions),
181
+ ),
182
+ mode="bicubic",
183
+ align_corners=False,
184
+ )
185
+ if (int(height) != patch_pos_embed.shape[-2]
186
+ or int(width) != patch_pos_embed.shape[-1]):
187
+ raise ValueError("Width or height does not match with "
188
+ "the interpolated position embeddings")
189
+
190
+ patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
191
+ return patch_pos_embed
192
+
193
+ def forward(self,
194
+ pixel_values: torch.Tensor,
195
+ interpolate_pos_encoding: bool = False) -> torch.Tensor:
196
+ _, _, height, width = pixel_values.shape
197
+ target_dtype = self.patch_embedding.weight.dtype
198
+ patch_embeds = self.patch_embedding(pixel_values.to(
199
+ dtype=target_dtype)) # shape = [*, width, grid, grid]
200
+ embeddings = patch_embeds.flatten(2).transpose(1, 2)
201
+
202
+ if interpolate_pos_encoding:
203
+ embeddings = embeddings + self.interpolate_pos_encoding(
204
+ embeddings, height, width)
205
+ else:
206
+ embeddings = embeddings + self.position_embedding(
207
+ self.position_ids)
208
+ return embeddings
209
+
210
+
211
+# NOTE: Not used - kept for later when we TP the ViT
212
+# TODO(ChristopherCho): Implement TP version of Attention
213
+class SiglipTPAttention(nn.Module):
214
+
215
+ def __init__(
216
+ self,
217
+ config,
218
+ quant_config: Optional[QuantizationConfig] = None,
219
+ ):
220
+ super().__init__()
221
+ self.config = config
222
+ self.embed_dim = config.hidden_size
223
+
224
+ tp_size = get_tensor_model_parallel_world_size()
225
+ self.total_num_heads = config.num_attention_heads
226
+ if self.total_num_heads % tp_size != 0:
227
+ raise ValueError(
228
+ f"Number of attention heads ({self.total_num_heads}) "
229
+ "must be divisible by the tensor model parallel size"
230
+ f" ({tp_size}).")
231
+
232
+ self.num_heads = self.total_num_heads // tp_size
233
+ self.head_dim = self.embed_dim // self.total_num_heads
234
+ if self.head_dim * self.total_num_heads != self.embed_dim:
235
+ raise ValueError(f"embed_dim must be divisible by num_heads (got "
236
+ "`embed_dim`: {self.embed_dim} and `num_heads`:"
237
+ f" {self.num_heads}).")
238
+ self.qkv_size = self.num_heads * self.head_dim
239
+ self.scale = self.head_dim**-0.5
240
+ self.dropout = config.attention_dropout
241
+
242
+ self.qkv_proj = QKVParallelLinear(
243
+ hidden_size=self.embed_dim,
244
+ head_size=self.head_dim,
245
+ total_num_heads=self.total_num_heads,
246
+ quant_config=quant_config,
247
+ )
248
+ self.out_proj = RowParallelLinear(
249
+ input_size=self.embed_dim,
250
+ output_size=self.embed_dim,
251
+ quant_config=quant_config,
252
+ )
253
+
254
+ self.attn_fn = self._basic_attention_forward
255
+
256
+ def forward(
257
+ self,
258
+ hidden_states: torch.Tensor,
259
+ ) -> torch.Tensor:
260
+ """Input shape: Batch x Time x Channel"""
261
+ batch_size, q_len, _ = hidden_states.size()
262
+
263
+ qkv_states, _ = self.qkv_proj(hidden_states)
264
+ query_states, key_states, value_states = qkv_states.split(
265
+ [self.qkv_size] * 3, dim=-1)
266
+
267
+ attn_output = self.attn_fn(
268
+ q=query_states,
269
+ k=key_states,
270
+ v=value_states,
271
+ batch_size=batch_size,
272
+ q_len=q_len,
273
+ )
274
+
275
+ attn_output, _ = self.out_proj(attn_output)
276
+ return attn_output
277
+
278
+ def _basic_attention_forward(self, q, k, v, batch_size, q_len):
279
+ q = q.view(batch_size, q_len, self.num_heads,
280
+ self.head_dim).transpose(1, 2)
281
+ k = k.view(batch_size, q_len, self.num_heads,
282
+ self.head_dim).transpose(1, 2)
283
+ v = v.view(batch_size, q_len, self.num_heads,
284
+ self.head_dim).transpose(1, 2)
285
+
286
+ k_v_seq_len = k.shape[-2]
287
+ attn_weights = torch.matmul(q, k.transpose(2, 3)) * self.scale
288
+
289
+ if attn_weights.size() != (
290
+ batch_size,
291
+ self.num_heads,
292
+ q_len,
293
+ k_v_seq_len,
294
+ ):
295
+ raise ValueError(
296
+ "Attention weights should be of size "
297
+ f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
298
+ f" {attn_weights.size()}")
299
+
300
+ # upcast attention to fp32
301
+ attn_weights = nn.functional.softmax(attn_weights,
302
+ dim=-1,
303
+ dtype=torch.float32).to(q.dtype)
304
+ attn_weights = nn.functional.dropout(attn_weights,
305
+ p=self.dropout,
306
+ training=self.training)
307
+ attn_output = torch.matmul(attn_weights, v)
308
+
309
+ if attn_output.size() != (
310
+ batch_size,
311
+ self.num_heads,
312
+ q_len,
313
+ self.head_dim,
314
+ ):
315
+ raise ValueError(
316
+ "`attn_output` should be of size "
317
+ f"{(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
318
+ f" {attn_output.size()}")
319
+
320
+ attn_output = attn_output.transpose(1, 2).contiguous()
321
+ attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
322
+
323
+ return attn_output
324
+
325
+
326
+# NOTE: Not used - kept for later when we TP the ViT
327
+# TODO(ChristopherCho): flash_attn_func is not working properly.
328
+# It constantly throws a CUDA error.
329
+class SiglipFlashAttention2(SiglipTPAttention):
330
+
331
+ def __init__(self, *args, **kwargs):
332
+ super().__init__(*args, **kwargs)
333
+ self.attn_fn = self._flash_attention_forward
334
+
335
+ # Ported from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L449
336
+ # and https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/modeling_flash_attention_utils.py#L133
337
+ def _flash_attention_forward(self, q, k, v, batch_size, q_len, *args,
338
+ **kwargs):
339
+ """Implements the multihead softmax attention.
340
+ Arguments
341
+ ---------
342
+ q, k, v: The tensor containing the
343
+ query, key, and value. (B, S, H, D)
344
+ """
345
+
346
+ q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
347
+ k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
348
+ v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
349
+
350
+ attn_output = flash_attn_func(
351
+ q,
352
+ k,
353
+ v,
354
+ dropout_p=self.dropout,
355
+ causal=False,
356
+ )
357
+
358
+ attn_output = attn_output.reshape(batch_size, q_len,
359
+ self.embed_dim).contiguous()
360
+
361
+ return attn_output
362
+
363
+
364
+# NOTE: Not used - kept for later when we TP the ViT
365
+class SiglipSdpaAttention(SiglipTPAttention):
366
+
367
+ def __init__(self, *args, **kwargs):
368
+ super().__init__(*args, **kwargs)
369
+ self.is_causal = False
370
+ self.attn_fn = self._sdpa_attention_forward
371
+
372
+ def _sdpa_attention_forward(self, q, k, v, batch_size, q_len):
373
+ q = q.view(batch_size, q_len, self.num_heads,
374
+ self.head_dim).transpose(1, 2)
375
+ k = k.view(batch_size, q_len, self.num_heads,
376
+ self.head_dim).transpose(1, 2)
377
+ v = v.view(batch_size, q_len, self.num_heads,
378
+ self.head_dim).transpose(1, 2)
379
+
380
+ attn_output = torch.nn.functional.scaled_dot_product_attention(
381
+ q, k, v, dropout_p=self.dropout, is_causal=False, scale=self.scale)
382
+
383
+ attn_output = attn_output.transpose(1, 2).contiguous()
384
+ attn_output = attn_output.view(batch_size, q_len, self.embed_dim)
385
+
386
+ return attn_output
387
+
388
+
389
+# NOTE: Not used - kept for later when we TP the ViT
390
+class SiglipxFormersAttention(SiglipTPAttention):
391
+
392
+ def __init__(self, *args, **kwargs):
393
+ super().__init__(*args, **kwargs)
394
+ self.attn_fn = self._xformers_attention_forward
395
+
396
+ def _xformers_attention_forward(self, q, k, v, batch_size, q_len):
397
+ q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
398
+ k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
399
+ v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
400
+
401
+ attn_output = memory_efficient_attention(q,
402
+ k,
403
+ v,
404
+ p=0.0,
405
+ scale=self.scale)
406
+ attn_output = attn_output.reshape(batch_size, q_len,
407
+ self.embed_dim).contiguous()
408
+
409
+ return attn_output
410
+
411
+
412
+# NOTE: Not used - kept for later when we TP the ViT
413
+SIGLIP_ATTENTION_CLASSES = {
414
+ "eager": SiglipTPAttention,
415
+ "flash_attention_2": SiglipFlashAttention2,
416
+ "sdpa": SiglipSdpaAttention,
417
+ "xformers": SiglipxFormersAttention,
418
+}
419
+
420
+
421
+class SiglipMLP(nn.Module):
422
+
423
+ def __init__(
424
+ self,
425
+ config,
426
+ quant_config: Optional[QuantizationConfig] = None,
427
+ ):
428
+ super().__init__()
429
+ self.config = config
430
+ self.activation_fn = get_act_fn(config.hidden_act)
431
+
432
+ # For quantization, we require the hidden size to be a multiple of 64
433
+ quantizable = (config.hidden_size % 64 == 0
434
+ and config.intermediate_size % 64 == 0)
435
+ self.fc1 = ColumnParallelLinear(
436
+ config.hidden_size,
437
+ config.intermediate_size,
438
+ quant_config=quant_config if quantizable else None,
439
+ )
440
+ self.fc2 = RowParallelLinear(
441
+ config.intermediate_size,
442
+ config.hidden_size,
443
+ quant_config=quant_config if quantizable else None,
444
+ )
445
+
446
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
447
+ hidden_states, _ = self.fc1(hidden_states)
448
+ hidden_states = self.activation_fn(hidden_states)
449
+ hidden_states, _ = self.fc2(hidden_states)
450
+ return hidden_states
451
+
452
+
453
+class SiglipEncoderLayer(nn.Module):
454
+
455
+ def __init__(
456
+ self,
457
+ config: SiglipConfig,
458
+ quant_config: Optional[QuantizationConfig] = None,
459
+ ):
460
+ super().__init__()
461
+ self.embed_dim = config.hidden_size
462
+
463
+ # TODO(ChristopherCho): use TP'ed Attention block
464
+ self.self_attn = SiglipAttention(config)
465
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim,
466
+ eps=config.layer_norm_eps)
467
+ self.mlp = SiglipMLP(
468
+ config,
469
+ quant_config=quant_config,
470
+ )
471
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim,
472
+ eps=config.layer_norm_eps)
473
+
474
+ def forward(
475
+ self,
476
+ hidden_states: torch.Tensor,
477
+ ) -> Tuple[torch.Tensor]:
478
+ residual = hidden_states
479
+
480
+ hidden_states = self.layer_norm1(hidden_states)
481
+ hidden_states, _ = self.self_attn(hidden_states=hidden_states)
482
+ hidden_states = residual + hidden_states
483
+
484
+ residual = hidden_states
485
+ hidden_states = self.layer_norm2(hidden_states)
486
+ hidden_states = self.mlp(hidden_states)
487
+ hidden_states = residual + hidden_states
488
+
489
+ return hidden_states, None
490
+
491
+
492
+class SiglipEncoder(nn.Module):
493
+
494
+ def __init__(
495
+ self,
496
+ config: SiglipConfig,
497
+ quant_config: Optional[QuantizationConfig] = None,
498
+ ):
499
+ super().__init__()
500
+ self.config = config
501
+ self.layers = nn.ModuleList([
502
+ SiglipEncoderLayer(
503
+ config,
504
+ quant_config=quant_config,
505
+ ) for _ in range(config.num_hidden_layers)
506
+ ])
507
+
508
+ def forward(
509
+ self,
510
+ inputs_embeds: torch.Tensor,
511
+ ) -> Tuple:
512
+ hidden_states = inputs_embeds
513
+ for encoder_layer in self.layers:
514
+ hidden_states, _ = encoder_layer(hidden_states)
515
+
516
+ return hidden_states
517
+
518
+
519
+class SiglipMultiheadAttentionPoolingHead(nn.Module):
520
+ """Multihead Attention Pooling."""
521
+
522
+ def __init__(
523
+ self,
524
+ config: SiglipVisionConfig,
525
+ quant_config: Optional[QuantizationConfig] = None,
526
+ ):
527
+ super().__init__()
528
+
529
+ self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
530
+ # TODO(ChristopherCho): Implement vLLM version of MultiheadAttention
531
+ self.attention = torch.nn.MultiheadAttention(
532
+ config.hidden_size, config.num_attention_heads, batch_first=True)
533
+ self.layernorm = nn.LayerNorm(config.hidden_size,
534
+ eps=config.layer_norm_eps)
535
+ self.mlp = SiglipMLP(config=config, quant_config=quant_config)
536
+
537
+ def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
538
+ batch_size = hidden_state.shape[0]
539
+ probe = self.probe.repeat(batch_size, 1, 1)
540
+
541
+ hidden_state = self.attention(probe, hidden_state, hidden_state)[0]
542
+
543
+ residual = hidden_state
544
+ hidden_state = self.layernorm(hidden_state)
545
+ hidden_state = residual + self.mlp(hidden_state)
546
+
547
+ return hidden_state[:, 0]
548
+
549
+
550
+class SiglipVisionTransformer(nn.Module):
551
+
552
+ def __init__(
553
+ self,
554
+ config: SiglipVisionConfig,
555
+ quant_config: Optional[QuantizationConfig] = None,
556
+ ):
557
+ super().__init__()
558
+ self.config = config
559
+ embed_dim = config.hidden_size
560
+
561
+ self.embeddings = SiglipVisionEmbeddings(config)
562
+ self.encoder = SiglipEncoder(
563
+ config,
564
+ quant_config=quant_config,
565
+ )
566
+ self.post_layernorm = nn.LayerNorm(embed_dim,
567
+ eps=config.layer_norm_eps)
568
+ self.use_head = (True if not hasattr(config, "vision_use_head") else
569
+ config.vision_use_head)
570
+ if self.use_head:
571
+ self.head = SiglipMultiheadAttentionPoolingHead(
572
+ config=config, quant_config=quant_config)
573
+
574
+ def forward(
575
+ self,
576
+ pixel_values: torch.Tensor,
577
+ interpolate_pos_encoding: bool = True,
578
+ ) -> torch.Tensor:
579
+ hidden_states = self.embeddings(
580
+ pixel_values,
581
+ interpolate_pos_encoding=interpolate_pos_encoding,
582
+ )
583
+
584
+ encoder_outputs = self.encoder(inputs_embeds=hidden_states)
585
+
586
+ last_hidden_state = self.post_layernorm(encoder_outputs)
587
+
588
+ # TODO: add this back when pooled_output is used in inference
589
+ # if self.use_head:
590
+ # pooled_output = self.head(last_hidden_state)
591
+
592
+ return last_hidden_state
593
+
594
+
595
+class SiglipVisionModel(nn.Module):
596
+ config_class = SiglipVisionConfig
597
+ main_input_name = "pixel_values"
598
+
599
+ def __init__(
600
+ self,
601
+ config: SiglipVisionConfig,
602
+ quant_config: Optional[QuantizationConfig] = None,
603
+ ):
604
+ super().__init__()
605
+ self.vision_model = SiglipVisionTransformer(
606
+ config,
607
+ quant_config,
608
+ )
609
+
610
+ def get_input_embeddings(self) -> nn.Module:
611
+ return self.vision_model.embeddings.patch_embedding
612
+
613
+ def forward(
614
+ self,
615
+ pixel_values: torch.Tensor,
616
+ interpolate_pos_encoding: bool = False,
617
+ ) -> torch.Tensor:
618
+ return self.vision_model(
619
+ pixel_values=pixel_values,
620
+ interpolate_pos_encoding=interpolate_pos_encoding,
621
+ )