A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[VLM] Clean up Phi-4-MM ViT implementation (#14812)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Isotr0py committed
Mar 16, 2025 at 09:53 UTC
def232e122624504e49f1e5ff0ae01a7285de1a3
7 files changed
+316
-1988
requirements/test.in
+1
@@ -8,6 +8,7 @@ pytest-shard
8
9
# testing utils
10
awscli
11
+backoff # required for phi4mm test
12
decord # required for video tests
13
einops # required for MPT, qwen-vl and Mamba
14
httpx
requirements/test.txt
+2
@@ -33,6 +33,8 @@ audioread==3.0.1
33
# via librosa
34
awscli==1.35.23
35
# via -r requirements/test.in
36
+backoff==2.2.1
37
+ # via -r requirements/test.in
38
bitsandbytes==0.45.3
39
# via -r requirements/test.in
40
black==24.10.0
tests/models/decoder_only/vision_language/test_phi4mm.py
new
+229
@@ -0,0 +1,229 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import os
4
+import re
5
+from typing import Optional
6
+
7
+import pytest
8
+from huggingface_hub import snapshot_download
9
+from transformers import AutoTokenizer
10
+
11
+from vllm.lora.request import LoRARequest
12
+from vllm.multimodal.image import rescale_image_size
13
+from vllm.platforms import current_platform
14
+from vllm.sequence import SampleLogprobs
15
+
16
+from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
17
+from ....utils import large_gpu_test
18
+from ...utils import check_logprobs_close
19
+
20
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21
+ "stop_sign":
22
+ "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
23
+ "cherry_blossom":
24
+ "<|user|>\n<|image_1|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
25
+})
26
+HF_MULTIIMAGE_IMAGE_PROMPT = "<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
27
+
28
+model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct")
29
+# Since the vision-lora and speech-lora co-exist with the base model,
30
+# we have to manually specify the path of the lora weights.
31
+vision_lora_path = os.path.join(model_path, "vision-lora")
32
+models = [model_path]
33
+
34
+
35
+def vllm_to_hf_output(vllm_output: tuple[list[int], str,
36
+ Optional[SampleLogprobs]],
37
+ model: str):
38
+ """Sanitize vllm output to be comparable with hf output."""
39
+ _, output_str, out_logprobs = vllm_output
40
+
41
+ output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
42
+ assert output_str_without_image[0] == " "
43
+ output_str_without_image = output_str_without_image[1:]
44
+
45
+ hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
46
+
47
+ tokenizer = AutoTokenizer.from_pretrained(model)
48
+ hf_output_ids = tokenizer.encode(output_str_without_image)
49
+ assert hf_output_ids[0] == 1
50
+ hf_output_ids = hf_output_ids[1:]
51
+
52
+ return hf_output_ids, hf_output_str, out_logprobs
53
+
54
+
55
+target_dtype = "half"
56
+
57
+# ROCm Triton FA can run into shared memory issues with these models,
58
+# use other backends in the meantime
59
+# FIXME (mattwong, gshtrasb, hongxiayan)
60
+if current_platform.is_rocm():
61
+ os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
62
+
63
+
64
+def run_test(
65
+ hf_runner: type[HfRunner],
66
+ vllm_runner: type[VllmRunner],
67
+ inputs: list[tuple[list[str], PromptImageInput]],
68
+ model: str,
69
+ *,
70
+ max_model_len: int,
71
+ dtype: str,
72
+ max_tokens: int,
73
+ num_logprobs: int,
74
+ mm_limit: int,
75
+ tensor_parallel_size: int,
76
+ distributed_executor_backend: Optional[str] = None,
77
+):
78
+ """Inference result should be the same between hf and vllm.
79
+
80
+ All the image fixtures for the test are from IMAGE_ASSETS.
81
+ For huggingface runner, we provide the PIL images as input.
82
+ For vllm runner, we provide MultiModalDataDict objects
83
+ and corresponding MultiModalConfig as input.
84
+ Note, the text input is also adjusted to abide by vllm contract.
85
+ The text output is sanitized to be able to compare with hf.
86
+ """
87
+ # NOTE: take care of the order. run vLLM first, and then run HF.
88
+ # vLLM needs a fresh new process without cuda initialization.
89
+ # if we run HF first, the cuda initialization will be done and it
90
+ # will hurt multiprocessing backend with fork method (the default method).
91
+ # max_model_len should be greater than image_feature_size
92
+ with vllm_runner(
93
+ model,
94
+ task="generate",
95
+ max_model_len=max_model_len,
96
+ max_num_seqs=2,
97
+ dtype=dtype,
98
+ limit_mm_per_prompt={"image": mm_limit},
99
+ tensor_parallel_size=tensor_parallel_size,
100
+ distributed_executor_backend=distributed_executor_backend,
101
+ enable_lora=True,
102
+ max_lora_rank=320,
103
+ lora_extra_vocab_size=0,
104
+ gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
105
+ enforce_eager=True,
106
+ ) as vllm_model:
107
+ lora_request = LoRARequest("vision", 1, vision_lora_path)
108
+ vllm_model.model.llm_engine.add_lora(lora_request=lora_request)
109
+ vllm_outputs_per_case = [
110
+ vllm_model.generate_greedy_logprobs(prompts,
111
+ max_tokens,
112
+ num_logprobs=num_logprobs,
113
+ images=images)
114
+ for prompts, images in inputs
115
+ ]
116
+
117
+ # use eager mode for hf runner, since phi3_v didn't work with flash_attn
118
+ hf_model_kwargs = {"_attn_implementation": "eager"}
119
+ with hf_runner(model, dtype=dtype,
120
+ model_kwargs=hf_model_kwargs) as hf_model:
121
+ eos_token_id = hf_model.processor.tokenizer.eos_token_id
122
+ hf_outputs_per_case = [
123
+ hf_model.generate_greedy_logprobs_limit(prompts,
124
+ max_tokens,
125
+ num_logprobs=num_logprobs,
126
+ images=images,
127
+ eos_token_id=eos_token_id,
128
+ num_logits_to_keep=0)
129
+ for prompts, images in inputs
130
+ ]
131
+
132
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
133
+ vllm_outputs_per_case):
134
+ check_logprobs_close(
135
+ outputs_0_lst=hf_outputs,
136
+ outputs_1_lst=vllm_outputs,
137
+ name_0="hf",
138
+ name_1="vllm",
139
+ )
140
+
141
+
142
+# Since we use _attn_implementation="eager" for hf_runner, there is more
143
+# significant numerical difference. The basic `logprobs=5` fails to pass.
144
+@pytest.mark.parametrize("model", models)
145
+@pytest.mark.parametrize(
146
+ "size_factors",
147
+ [
148
+ # No image
149
+ [],
150
+ # Single-scale
151
+ [1.0],
152
+ # Single-scale, batched
153
+ [1.0, 1.0, 1.0],
154
+ # Multi-scale
155
+ [0.7, 0.75, 1.0],
156
+ ],
157
+)
158
+@pytest.mark.parametrize("dtype", [target_dtype])
159
+@pytest.mark.parametrize("max_model_len", [4096])
160
+@pytest.mark.parametrize("max_tokens", [128])
161
+@pytest.mark.parametrize("num_logprobs", [10])
162
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
163
+ dtype: str, max_model_len: int, max_tokens: int,
164
+ num_logprobs: int) -> None:
165
+ images = [asset.pil_image for asset in image_assets]
166
+
167
+ inputs_per_image = [(
168
+ [prompt for _ in size_factors],
169
+ [rescale_image_size(image, factor) for factor in size_factors],
170
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
171
+
172
+ run_test(
173
+ hf_runner,
174
+ vllm_runner,
175
+ inputs_per_image,
176
+ model,
177
+ dtype=dtype,
178
+ max_model_len=max_model_len,
179
+ max_tokens=max_tokens,
180
+ num_logprobs=num_logprobs,
181
+ mm_limit=1,
182
+ tensor_parallel_size=1,
183
+ )
184
+
185
+
186
+@large_gpu_test(min_gb=48)
187
+@pytest.mark.parametrize("model", models)
188
+@pytest.mark.parametrize(
189
+ "size_factors",
190
+ [
191
+ # No image
192
+ # [],
193
+ # Single-scale
194
+ [1.0],
195
+ # Single-scale, batched
196
+ [1.0, 1.0, 1.0],
197
+ # Multi-scale
198
+ [0.25, 0.5, 1.0],
199
+ ],
200
+)
201
+@pytest.mark.parametrize("dtype", [target_dtype])
202
+@pytest.mark.parametrize("max_model_len", [10000])
203
+@pytest.mark.parametrize("max_tokens", [128])
204
+@pytest.mark.parametrize("num_logprobs", [10])
205
+@pytest.mark.xfail(
206
+ reason="Phi-4-MM multi-image inference is divergent with hf model.")
207
+def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
208
+ size_factors, dtype: str, max_model_len: int,
209
+ max_tokens: int, num_logprobs: int) -> None:
210
+ images = [asset.pil_image for asset in image_assets]
211
+
212
+ inputs_per_case = [
213
+ ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
214
+ [[rescale_image_size(image, factor) for image in images]
215
+ for factor in size_factors])
216
+ ]
217
+
218
+ run_test(
219
+ hf_runner,
220
+ vllm_runner,
221
+ inputs_per_case,
222
+ model,
223
+ dtype=dtype,
224
+ max_model_len=max_model_len,
225
+ max_tokens=max_tokens,
226
+ num_logprobs=num_logprobs,
227
+ mm_limit=2,
228
+ tensor_parallel_size=1,
229
+ )
vllm/model_executor/models/aria.py
+2
-2
@@ -60,7 +60,7 @@ class AriaVisionTransformer(Idefics3VisionTransformer, SupportsQuant):
60
quant_config: Optional[QuantizationConfig] = None,
61
prefix: str = "",
62
) -> None:
63
- super().__init__(config, quant_config, prefix)
63
+ super().__init__(config, quant_config=quant_config, prefix=prefix)
64
# Unlike Idefics3VisionTransformer which uses LayerNorm after the
65
# final layer, Aria omits this normalization, so we replace it with an
66
# Identity layer
@@ -512,7 +512,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
512
self.config = config
513
self.vision_tower = AriaVisionTransformer(
514
config.vision_config,
515
- quant_config,
515
+ quant_config=quant_config,
516
prefix=f"{prefix}.vision_tower",
517
)
518
self.multi_modal_projector = AriaProjector(config)
vllm/model_executor/models/idefics2_vision_model.py
+49
-8
@@ -113,7 +113,7 @@ class Idefics2VisionAttention(nn.Module):
113
114
def __init__(
115
self,
116
- config: Idefics2Config,
116
+ config: Idefics2VisionConfig,
117
quant_config: Optional[QuantizationConfig] = None,
118
prefix: str = "",
119
) -> None:
@@ -164,7 +164,7 @@ class Idefics2VisionMLP(nn.Module):
164
165
def __init__(
166
self,
167
- config: Idefics2Config,
167
+ config: Idefics2VisionConfig,
168
quant_config: Optional[QuantizationConfig] = None,
169
prefix: str = "",
170
) -> None:
@@ -249,16 +249,24 @@ class Idefics2Encoder(nn.Module):
249
self,
250
config: Idefics2Config,
251
quant_config: Optional[QuantizationConfig] = None,
252
+ *,
253
+ num_hidden_layers_override: Optional[int] = None,
254
prefix: str = "",
255
) -> None:
256
super().__init__()
257
258
self.config = config
259
+
260
+ if num_hidden_layers_override is None:
261
+ num_hidden_layers = config.num_hidden_layers
262
+ else:
263
+ num_hidden_layers = num_hidden_layers_override
264
+
265
self.layers = nn.ModuleList([
266
Idefics2EncoderLayer(config,
267
quant_config=quant_config,
268
prefix=f"{prefix}.layers.{layer_idx}")
261
- for layer_idx in range(config.num_hidden_layers)
269
+ for layer_idx in range(num_hidden_layers)
270
])
271
272
def forward(
@@ -287,6 +295,9 @@ class Idefics2VisionTransformer(nn.Module):
295
self,
296
config: Idefics2VisionConfig,
297
quant_config: Optional[QuantizationConfig] = None,
298
+ *,
299
+ num_hidden_layers_override: Optional[int] = None,
300
+ require_post_norm: bool = True,
301
prefix: str = "",
302
) -> None:
303
super().__init__()
@@ -294,11 +305,24 @@ class Idefics2VisionTransformer(nn.Module):
305
embed_dim = config.hidden_size
306
self.config = config
307
self.embeddings = Idefics2VisionEmbeddings(config)
297
- self.encoder = Idefics2Encoder(config,
298
- quant_config=quant_config,
299
- prefix=f"{prefix}.encoder")
300
- self.post_layernorm = nn.LayerNorm(embed_dim,
301
- eps=config.layer_norm_eps)
308
+ self.encoder = Idefics2Encoder(
309
+ config,
310
+ quant_config=quant_config,
311
+ num_hidden_layers_override=num_hidden_layers_override,
312
+ prefix=f"{prefix}.encoder")
313
+
314
+ num_hidden_layers = config.num_hidden_layers
315
+ if len(self.encoder.layers) > config.num_hidden_layers:
316
+ raise ValueError(
317
+ f"The original encoder only has {num_hidden_layers} "
318
+ f"layers, but you requested {len(self.encoder.layers)} layers."
319
+ )
320
+
321
+ self.require_post_norm = require_post_norm
322
+ self.post_layernorm = nn.LayerNorm(
323
+ embed_dim,
324
+ eps=config.layer_norm_eps,
325
+ ) if require_post_norm else nn.Identity()
326
327
def get_input_embeddings(self):
328
return self.embeddings
@@ -328,7 +352,24 @@ class Idefics2VisionTransformer(nn.Module):
352
]
353
params_dict = dict(self.named_parameters())
354
loaded_params: Set[str] = set()
355
+ layer_count = len(self.encoder.layers)
356
+
357
for name, loaded_weight in weights:
358
+ # skip pooling header
359
+ if name.startswith("head."):
360
+ continue
361
+
362
+ # post_layernorm is optional
363
+ if (name.startswith("post_layernorm.")
364
+ and not self.require_post_norm):
365
+ continue
366
+
367
+ # omit layers when num_hidden_layers_override is set
368
+ if name.startswith("encoder.layers."):
369
+ layer_idx = int(name.split(".")[2])
370
+ if layer_idx >= layer_count:
371
+ continue
372
+
373
for param_name, weight_name, shard_id in stacked_params_mapping:
374
if weight_name not in name:
375
continue
vllm/model_executor/models/phi4mm.py
+33
-12
@@ -11,7 +11,7 @@ import torch
11
import torch.nn as nn
12
import torchvision.transforms as T
13
from PIL import Image
14
-from transformers import PretrainedConfig
14
+from transformers import PretrainedConfig, SiglipVisionConfig
15
from transformers.utils import logging
16
17
from vllm.config import VllmConfig
@@ -32,10 +32,10 @@ from vllm.multimodal.inputs import MultiModalInputs, NestedTensors
32
from vllm.sequence import IntermediateTensors, SequenceData
33
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
34
35
+from .idefics2_vision_model import Idefics2VisionTransformer
36
from .interfaces import SupportsLoRA, SupportsMultiModal
37
from .phi4mm_audio import AudioEmbedding
38
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
38
-from .vision_siglip_navit import get_siglip_vision_model
39
40
# <|endoftext10|> (see vocab.json in hf model)
41
_IMAGE_PLACEHOLDER_TOKEN_ID = 200010
@@ -339,6 +339,33 @@ def preprocess(images, dynamic_hd_size, vit_resolution, vit_patch_size):
339
return data
340
341
342
+def get_navit_vision_model(layer_idx: int = -1, **kwargs):
343
+ vision_config = {
344
+ "hidden_size": 1152,
345
+ "image_size": 448,
346
+ "intermediate_size": 4304,
347
+ "model_type": "siglip_vision_model",
348
+ "num_attention_heads": 16,
349
+ "num_hidden_layers": 27,
350
+ "patch_size": 14,
351
+ }
352
+
353
+ model_config = SiglipVisionConfig(**vision_config, **kwargs)
354
+ if layer_idx < 0:
355
+ num_hidden_layers = model_config.num_hidden_layers \
356
+ + layer_idx + 1
357
+ else:
358
+ num_hidden_layers = layer_idx + 1
359
+
360
+ vision_model = Idefics2VisionTransformer(
361
+ config=model_config,
362
+ require_post_norm=False,
363
+ num_hidden_layers_override=num_hidden_layers,
364
+ )
365
+
366
+ return vision_model
367
+
368
+
369
class Phi4MMImageEncoder(nn.Module):
370
"""Image embedding."""
371
@@ -362,8 +389,7 @@ class Phi4MMImageEncoder(nn.Module):
389
self.layer_idx = -2
390
self.type_feature = 'patch'
391
365
- self.img_processor = get_siglip_vision_model(
366
- _flash_attn_2_enabled=True)
392
+ self.img_processor = get_navit_vision_model(layer_idx=self.layer_idx)
393
394
pe_weight = self.img_processor.embeddings.position_embedding.weight
395
L, D = pe_weight.size()
@@ -430,16 +456,11 @@ class Phi4MMImageEncoder(nn.Module):
456
def get_img_features(self,
457
img_embeds: torch.FloatTensor,
458
attention_mask=None) -> torch.FloatTensor:
433
- LAYER_IDX = self.layer_idx
434
- TYPE_FEATURE = self.type_feature
459
436
- img_processor_output = self.img_processor(
437
- img_embeds,
438
- output_hidden_states=True,
439
- patch_attention_mask=attention_mask)
440
- img_feature = img_processor_output.hidden_states[LAYER_IDX]
460
+ img_feature = self.img_processor(img_embeds,
461
+ patch_attention_mask=attention_mask)
462
442
- if TYPE_FEATURE == "patch":
463
+ if self.type_feature == "patch":
464
patch_feature = img_feature
465
466
use_token_compression = self.image_token_compression is not None
vllm/model_executor/models/vision_siglip_navit.py
deleted
-1966
@@ -1,1966 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
3
-#
4
-# Licensed under the Apache License, Version 2.0 (the "License");
5
-# you may not use this file except in compliance with the License.
6
-# You may obtain a copy of the License at
7
-#
8
-# http://www.apache.org/licenses/LICENSE-2.0
9
-#
10
-# Unless required by applicable law or agreed to in writing, software
11
-# distributed under the License is distributed on an "AS IS" BASIS,
12
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
-# See the License for the specific language governing permissions and
14
-# limitations under the License.
15
-""" Siglip model configuration"""
16
-
17
-import math
18
-import os
19
-import warnings
20
-from dataclasses import dataclass
21
-from typing import Any, Optional, Tuple, Union
22
-
23
-import numpy as np
24
-import torch
25
-import torch.nn.functional as F
26
-import torch.utils.checkpoint
27
-from torch import nn
28
-from torch.nn.init import _calculate_fan_in_and_fan_out
29
-from transformers.activations import ACT2FN
30
-from transformers.configuration_utils import PretrainedConfig
31
-from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
32
-from transformers.modeling_outputs import (BaseModelOutput,
33
- BaseModelOutputWithPooling)
34
-from transformers.modeling_utils import PreTrainedModel
35
-from transformers.utils import (ModelOutput, add_start_docstrings,
36
- add_start_docstrings_to_model_forward, logging,
37
- replace_return_docstrings)
38
-
39
-from vllm.platforms import _Backend
40
-
41
-from .vision import get_vit_attn_backend
42
-
43
-logger = logging.get_logger(__name__)
44
-
45
-SIGLIP_PRETRAINED_CONFIG_ARCHIVE_MAP = {
46
- "google/siglip-base-patch16-224":
47
- "https://huggingface.co/google/siglip-base-patch16-224/"\
48
- "resolve/main/config.json",
49
-}
50
-
51
-
52
-class SiglipTextConfig(PretrainedConfig):
53
- r"""
54
- This is the configuration class to store the configuration of a
55
- [`SiglipTextModel`]. It is used to instantiate a Siglip text encoder
56
- according to the specified arguments, defining the model architecture.
57
- Instantiating a configuration with the defaults will yield a similar
58
- configuration to that of the text encoder of the Siglip [google/
59
- siglip-base-patch16-224](https://huggingface.co/google/siglip-base
60
- -patch16-224) architecture.
61
- Configuration objects inherit from [`PretrainedConfig`] and can be used
62
- to control the model outputs. Read the documentation from
63
- [`PretrainedConfig`] for more information.
64
- Args:
65
- vocab_size (`int`, *optional*, defaults to 32000):
66
- Vocabulary size of the Siglip text model. Defines the number of
67
- different tokens that can be represented by the `inputs_ids`
68
- passed when calling [`SiglipModel`].
69
- hidden_size (`int`, *optional*, defaults to 768):
70
- Dimensionality of the encoder layers and the pooler layer.
71
- intermediate_size (`int`, *optional*, defaults to 3072):
72
- Dimensionality of the "intermediate" (i.e., feed-forward) layer
73
- in the Transformer encoder.
74
- num_hidden_layers (`int`, *optional*, defaults to 12):
75
- Number of hidden layers in the Transformer encoder.
76
- num_attention_heads (`int`, *optional*, defaults to 12):
77
- Number of attention heads for each attention layer in the
78
- Transformer encoder.
79
- max_position_embeddings (`int`, *optional*, defaults to 64):
80
- The maximum sequence length that this model might ever be used
81
- with. Typically set this to something large
82
- just in case (e.g., 512 or 1024 or 2048).
83
- hidden_act (`str` or `function`, *optional*, defaults to
84
- `"gelu_pytorch_tanh"`):
85
- The non-linear activation function (function or string) in the
86
- encoder and pooler. If string, `"gelu"`,
87
- `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
88
- layer_norm_eps (`float`, *optional*, defaults to 1e-06):
89
- The epsilon used by the layer normalization layers.
90
- attention_dropout (`float`, *optional*, defaults to 0.0):
91
- The dropout ratio for the attention probabilities.
92
- pad_token_id (`int`, *optional*, defaults to 1):
93
- The id of the padding token in the vocabulary.
94
- bos_token_id (`int`, *optional*, defaults to 49406):
95
- The id of the beginning-of-sequence token in the vocabulary.
96
- eos_token_id (`int`, *optional*, defaults to 49407):
97
- The id of the end-of-sequence token in the vocabulary.
98
- Example:
99
- ```python
100
- >>> from transformers import SiglipTextConfig, SiglipTextModel
101
- >>> # Initializing a SiglipTextConfig with google/siglip-base-patch16-224
102
- style configuration
103
- >>> configuration = SiglipTextConfig()
104
- >>> # Initializing a SiglipTextModel (with random weights) from the
105
- google/siglip-base-patch16-224 style configuration
106
- >>> model = SiglipTextModel(configuration)
107
- >>> # Accessing the model configuration
108
- >>> configuration = model.config
109
- ```"""
110
-
111
- model_type = "siglip_text_model"
112
-
113
- def __init__(
114
- self,
115
- vocab_size=32000,
116
- hidden_size=768,
117
- intermediate_size=3072,
118
- num_hidden_layers=12,
119
- num_attention_heads=12,
120
- max_position_embeddings=64,
121
- hidden_act="gelu_pytorch_tanh",
122
- layer_norm_eps=1e-6,
123
- attention_dropout=0.0,
124
- # This differs from `CLIPTokenizer`'s default and from openai/siglip
125
- # See https://github.com/huggingface/transformers/pull/24773#
126
- # issuecomment-1632287538
127
- pad_token_id=1,
128
- bos_token_id=49406,
129
- eos_token_id=49407,
130
- _flash_attn_2_enabled=True,
131
- **kwargs,
132
- ):
133
- super().__init__(pad_token_id=pad_token_id,
134
- bos_token_id=bos_token_id,
135
- eos_token_id=eos_token_id,
136
- **kwargs)
137
-
138
- self.vocab_size = vocab_size
139
- self.hidden_size = hidden_size
140
- self.intermediate_size = intermediate_size
141
- self.num_hidden_layers = num_hidden_layers
142
- self.num_attention_heads = num_attention_heads
143
- self.max_position_embeddings = max_position_embeddings
144
- self.layer_norm_eps = layer_norm_eps
145
- self.hidden_act = hidden_act
146
- self.attention_dropout = attention_dropout
147
- self._flash_attn_2_enabled = _flash_attn_2_enabled
148
-
149
- @classmethod
150
- def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
151
- os.PathLike],
152
- **kwargs) -> "PretrainedConfig":
153
- cls._set_token_in_kwargs(kwargs)
154
-
155
- config_dict, kwargs = cls.get_config_dict(
156
- pretrained_model_name_or_path, **kwargs)
157
-
158
- # get the text config dict if we are loading from SiglipConfig
159
- if config_dict.get("model_type") == "siglip":
160
- config_dict = config_dict["text_config"]
161
-
162
- if "model_type" in config_dict and hasattr(
163
- cls,
164
- "model_type") and config_dict["model_type"] != cls.model_type:
165
- logger.warning(
166
- "You are using a model of type %s to instantiate a model of "
167
- "type %s. This is not supported for all configurations of "
168
- "models and can yield errors.", config_dict['model_type'],
169
- cls.model_type)
170
-
171
- return cls.from_dict(config_dict, **kwargs)
172
-
173
-
174
-class SiglipVisionConfig(PretrainedConfig):
175
- r"""
176
- This is the configuration class to store the configuration of a
177
- [`SiglipVisionModel`]. It is used to instantiate a
178
- Siglip vision encoder according to the specified arguments, defining the
179
- model architecture. Instantiating a configuration with the defaults will
180
- yield a similar configuration to that of the vision encoder of the Siglip
181
- [google/siglip-base-patch16-224](https://huggingface.co/google/
182
- siglip-base-patch16-224) architecture.
183
- Configuration objects inherit from [`PretrainedConfig`] and can be used
184
- to control the model outputs. Read the
185
- documentation from [`PretrainedConfig`] for more information.
186
- Args:
187
- hidden_size (`int`, *optional*, defaults to 768):
188
- Dimensionality of the encoder layers and the pooler layer.
189
- intermediate_size (`int`, *optional*, defaults to 3072):
190
- Dimensionality of the "intermediate" (i.e., feed-forward) layer
191
- in the Transformer encoder.
192
- num_hidden_layers (`int`, *optional*, defaults to 12):
193
- Number of hidden layers in the Transformer encoder.
194
- num_attention_heads (`int`, *optional*, defaults to 12):
195
- Number of attention heads for each attention layer in the
196
- Transformer encoder.
197
- num_channels (`int`, *optional*, defaults to 3):
198
- Number of channels in the input images.
199
- image_size (`int`, *optional*, defaults to 224):
200
- The size (resolution) of each image.
201
- patch_size (`int`, *optional*, defaults to 16):
202
- The size (resolution) of each patch.
203
- hidden_act (`str` or `function`, *optional*, defaults to
204
- `"gelu_pytorch_tanh"`):
205
- The non-linear activation function (function or string) in the
206
- encoder and pooler. If string, `"gelu"`, `"relu"`, `"selu"` and
207
- `"gelu_new"` ``"quick_gelu"` are supported.
208
- layer_norm_eps (`float`, *optional*, defaults to 1e-06):
209
- The epsilon used by the layer normalization layers.
210
- attention_dropout (`float`, *optional*, defaults to 0.0):
211
- The dropout ratio for the attention probabilities.
212
- Example:
213
- ```python
214
- >>> from transformers import SiglipVisionConfig, SiglipVisionModel
215
- >>> # Initializing a SiglipVisionConfig with google/siglip-base-patch16-224
216
- style configuration
217
- >>> configuration = SiglipVisionConfig()
218
- >>> # Initializing a SiglipVisionModel (with random weights) from the
219
- google/siglip-base-patch16-224 style configuration
220
- >>> model = SiglipVisionModel(configuration)
221
- >>> # Accessing the model configuration
222
- >>> configuration = model.config
223
- ```"""
224
-
225
- model_type = "siglip_vision_model"
226
-
227
- def __init__(
228
- self,
229
- hidden_size=768,
230
- intermediate_size=3072,
231
- num_hidden_layers=12,
232
- num_attention_heads=12,
233
- num_channels=3,
234
- image_size=224,
235
- patch_size=16,
236
- hidden_act="gelu_pytorch_tanh",
237
- layer_norm_eps=1e-6,
238
- attention_dropout=0.0,
239
- _flash_attn_2_enabled=True,
240
- **kwargs,
241
- ):
242
- super().__init__(**kwargs)
243
-
244
- self.hidden_size = hidden_size
245
- self.intermediate_size = intermediate_size
246
- self.num_hidden_layers = num_hidden_layers
247
- self.num_attention_heads = num_attention_heads
248
- self.num_channels = num_channels
249
- self.patch_size = patch_size
250
- self.image_size = image_size
251
- self.attention_dropout = attention_dropout
252
- self.layer_norm_eps = layer_norm_eps
253
- self.hidden_act = hidden_act
254
- self._flash_attn_2_enabled = _flash_attn_2_enabled
255
-
256
- @classmethod
257
- def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
258
- os.PathLike],
259
- **kwargs) -> "PretrainedConfig":
260
- cls._set_token_in_kwargs(kwargs)
261
-
262
- config_dict, kwargs = cls.get_config_dict(
263
- pretrained_model_name_or_path, **kwargs)
264
-
265
- # get the vision config dict if we are loading from SiglipConfig
266
- if config_dict.get("model_type") == "siglip":
267
- config_dict = config_dict["vision_config"]
268
-
269
- if "model_type" in config_dict and hasattr(
270
- cls,
271
- "model_type") and config_dict["model_type"] != cls.model_type:
272
- logger.warning(
273
- "You are using a model of type %s to "
274
- "instantiate a model of type %s. This is not"
275
- " supported for all configurations of models and can yield"
276
- " errors.", config_dict['model_type'], cls.model_type)
277
-
278
- return cls.from_dict(config_dict, **kwargs)
279
-
280
-
281
-class SiglipConfig(PretrainedConfig):
282
- r"""
283
- [`SiglipConfig`] is the configuration class to store the configuration of a
284
- [`SiglipModel`]. It is used to instantiate a Siglip model according to the
285
- specified arguments, defining the text model and vision model configs.
286
- Instantiating a configuration with the defaults will yield a similar
287
- configuration to that of the Siglip [google/siglip-base-patch16-224](
288
- https://huggingface.co/google/siglip-base-patch16-224) architecture.
289
- Configuration objects inherit from [`PretrainedConfig`] and can be used to
290
- control the model outputs. Read the documentation from
291
- [`PretrainedConfig`] for more information.
292
- Args:
293
- text_config (`dict`, *optional*):
294
- Dictionary of configuration options used to initialize
295
- [`SiglipTextConfig`].
296
- vision_config (`dict`, *optional*):
297
- Dictionary of configuration options used to initialize
298
- [`SiglipVisionConfig`].
299
- kwargs (*optional*):
300
- Dictionary of keyword arguments.
301
- Example:
302
- ```python
303
- >>> from transformers import SiglipConfig, SiglipModel
304
- >>> # Initializing a SiglipConfig with google/siglip-base-patch16-224
305
- style configuration
306
- >>> configuration = SiglipConfig()
307
- >>> # Initializing a SiglipModel (with random weights) from the
308
- google/siglip-base-patch16-224 style configuration
309
- >>> model = SiglipModel(configuration)
310
- >>> # Accessing the model configuration
311
- >>> configuration = model.config
312
- >>> # We can also initialize a SiglipConfig from a SiglipTextConfig
313
- and a SiglipVisionConfig
314
- >>> from transformers import SiglipTextConfig, SiglipVisionConfig
315
- >>> # Initializing a SiglipText and SiglipVision configuration
316
- >>> config_text = SiglipTextConfig()
317
- >>> config_vision = SiglipVisionConfig()
318
- >>> config = SiglipConfig.from_text_vision_configs(config_text,
319
- config_vision)
320
- ```"""
321
-
322
- model_type = "siglip"
323
-
324
- def __init__(self, text_config=None, vision_config=None, **kwargs):
325
- super().__init__(**kwargs)
326
-
327
- if text_config is None:
328
- text_config = {}
329
- logger.info(
330
- "`text_config` is `None`. Initializing the `SiglipTextConfig`"
331
- " with default values.")
332
-
333
- if vision_config is None:
334
- vision_config = {}
335
- logger.info("`vision_config` is `None`. initializing the "
336
- "`SiglipVisionConfig` with default values.")
337
-
338
- self.text_config = SiglipTextConfig(**text_config)
339
- self.vision_config = SiglipVisionConfig(**vision_config)
340
-
341
- self.initializer_factor = 1.0
342
-
343
- @classmethod
344
- def from_text_vision_configs(cls, text_config: SiglipTextConfig,
345
- vision_config: SiglipVisionConfig, **kwargs):
346
- r"""
347
- Instantiate a [`SiglipConfig`] (or a derived class) from siglip text
348
- model configuration and siglip vision
349
- model configuration.
350
- Returns:
351
- [`SiglipConfig`]: An instance of a configuration object
352
- """
353
-
354
- return cls(text_config=text_config.to_dict(),
355
- vision_config=vision_config.to_dict(),
356
- **kwargs)
357
-
358
-
359
-# coding=utf-8
360
-# Copyright 2024 Google AI and The HuggingFace Team. All rights reserved.
361
-#
362
-# Licensed under the Apache License, Version 2.0 (the "License");
363
-# you may not use this file except in compliance with the License.
364
-# You may obtain a copy of the License at
365
-#
366
-# http://www.apache.org/licenses/LICENSE-2.0
367
-#
368
-# Unless required by applicable law or agreed to in writing, software
369
-# distributed under the License is distributed on an "AS IS" BASIS,
370
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
371
-# See the License for the specific language governing permissions and
372
-# limitations under the License.
373
-""" PyTorch Siglip model."""
374
-
375
-_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
376
-
377
-SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
378
- "google/siglip-base-patch16-224",
379
- # See all SigLIP models at https://huggingface.co/models?filter=siglip
380
-]
381
-
382
-
383
-# Copied from transformers.models.llama.modeling_llama._get_unpad_data
384
-def _get_unpad_data(attention_mask):
385
- seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
386
- indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
387
- max_seqlen_in_batch = seqlens_in_batch.max().item()
388
- cu_seqlens = F.pad(
389
- torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
390
- return (
391
- indices,
392
- cu_seqlens,
393
- max_seqlen_in_batch,
394
- )
395
-
396
-
397
-def _trunc_normal_(tensor, mean, std, a, b):
398
- # Cut & paste from PyTorch official master until it's in a few official
399
- # releases - RW
400
- # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/
401
- # truncated_normal.pdf
402
- def norm_cdf(x):
403
- # Computes standard normal cumulative distribution function
404
- return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
405
-
406
- if (mean < a - 2 * std) or (mean > b + 2 * std):
407
- warnings.warn(
408
- "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
409
- "The distribution of values may be incorrect.",
410
- stacklevel=2,
411
- )
412
-
413
- # Values are generated by using a truncated uniform distribution and
414
- # then using the inverse CDF for the normal distribution.
415
- # Get upper and lower cdf values
416
- l = norm_cdf((a - mean) / std) # noqa
417
- u = norm_cdf((b - mean) / std) # noqa
418
-
419
- # Uniformly fill tensor with values from [l, u], then translate to
420
- # [2l-1, 2u-1].
421
- tensor.uniform_(2 * l - 1, 2 * u - 1)
422
-
423
- # Use inverse cdf transform for normal distribution to get truncated
424
- # standard normal
425
- if tensor.dtype in [torch.float16, torch.bfloat16]:
426
- # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
427
- og_dtype = tensor.dtype
428
- tensor = tensor.to(torch.float32)
429
- tensor.erfinv_()
430
- tensor = tensor.to(og_dtype)
431
- else:
432
- tensor.erfinv_()
433
-
434
- # Transform to proper mean, std
435
- tensor.mul_(std * math.sqrt(2.0))
436
- tensor.add_(mean)
437
-
438
- # Clamp to ensure it's in the proper range
439
- if tensor.dtype == torch.float16:
440
- # The `clamp_` op is not (yet?) defined in float16+cpu
441
- tensor = tensor.to(torch.float32)
442
- tensor.clamp_(min=a, max=b)
443
- tensor = tensor.to(torch.float16)
444
- else:
445
- tensor.clamp_(min=a, max=b)
446
-
447
-
448
-def trunc_normal_tf_(tensor: torch.Tensor,
449
- mean: float = 0.0,
450
- std: float = 1.0,
451
- a: float = -2.0,
452
- b: float = 2.0) -> torch.Tensor:
453
- """Fills the input Tensor with values drawn from a truncated
454
- normal distribution. The values are effectively drawn from the
455
- normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
456
- with values outside :math:`[a, b]` redrawn until they are within
457
- the bounds. The method used for generating the random values works
458
- best when :math:`a \\leq \text{mean} \\leq b`.
459
- NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where
460
- the bounds [a, b] are applied when sampling the normal distribution with
461
- mean=0, std=1.0 and the result is subsequently scaled and shifted by the
462
- mean and std args.
463
- Args:
464
- tensor: an n-dimensional `torch.Tensor`
465
- mean: the mean of the normal distribution
466
- std: the standard deviation of the normal distribution
467
- a: the minimum cutoff value
468
- b: the maximum cutoff value
469
- """
470
- with torch.no_grad():
471
- _trunc_normal_(tensor, 0, 1.0, a, b)
472
- tensor.mul_(std).add_(mean)
473
-
474
-
475
-def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
476
- fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
477
- if mode == "fan_in":
478
- denom = fan_in
479
- elif mode == "fan_out":
480
- denom = fan_out
481
- elif mode == "fan_avg":
482
- denom = (fan_in + fan_out) / 2
483
-
484
- variance = scale / denom
485
-
486
- if distribution == "truncated_normal":
487
- # constant is stddev of standard normal truncated to (-2, 2)
488
- trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
489
- elif distribution == "normal":
490
- with torch.no_grad():
491
- tensor.normal_(std=math.sqrt(variance))
492
- elif distribution == "uniform":
493
- bound = math.sqrt(3 * variance)
494
- with torch.no_grad():
495
- tensor.uniform_(-bound, bound)
496
- else:
497
- raise ValueError(f"invalid distribution {distribution}")
498
-
499
-
500
-def lecun_normal_(tensor):
501
- variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
502
-
503
-
504
-def default_flax_embed_init(tensor):
505
- variance_scaling_(tensor, mode="fan_in", distribution="normal")
506
-
507
-
508
-@dataclass
509
-# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with
510
-# CLIP->Siglip
511
-class SiglipVisionModelOutput(ModelOutput):
512
- """
513
- Base class for vision model's outputs that also contains image embeddings
514
- of the pooling of the last hidden states.
515
- Args:
516
- image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`
517
- *optional* returned when model is initialized with
518
- `with_projection=True`):
519
- The image embeddings obtained by applying the projection layer to
520
- the pooler_output.
521
- last_hidden_state (`torch.FloatTensor` of shape `(batch_size,
522
- sequence_length, hidden_size)`):
523
- Sequence of hidden-states at the output of the last layer of the
524
- model.
525
- hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when
526
- `output_hidden_states=True` is passed or when
527
- `config.output_hidden_states=True`):
528
- Tuple of `torch.FloatTensor` (one for the output of the embeddings,
529
- if the model has an embedding layer, + one for the output of each
530
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
531
- Hidden-states of the model at the output of each layer plus the
532
- optional initial embedding outputs.
533
- attentions (`tuple(torch.FloatTensor)`, *optional*, returned when
534
- `output_attentions=True` is passed or when
535
- `config.output_attentions=True`):
536
- Tuple of `torch.FloatTensor` (one for each layer) of shape
537
- `(batch_size, num_heads, sequence_length, sequence_length)`.
538
- Attentions weights after the attention softmax, used to compute the
539
- weighted average in the self-attention heads.
540
- """
541
-
542
- image_embeds: Optional[torch.FloatTensor] = None
543
- last_hidden_state: torch.FloatTensor = None
544
- hidden_states: Optional[Tuple[torch.FloatTensor]] = None
545
- attentions: Optional[Tuple[torch.FloatTensor]] = None
546
-
547
-
548
-@dataclass
549
-# Copied from transformers.models.clip.modeling_clip.CLIPTextModelOutput with
550
-# CLIP->Siglip
551
-class SiglipTextModelOutput(ModelOutput):
552
- """
553
- Base class for text model's outputs that also contains a pooling of the
554
- last hidden states.
555
- Args:
556
- text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`
557
- *optional* returned when model is initialized with
558
- `with_projection=True`):
559
- The text embeddings obtained by applying the projection layer to
560
- model.
561
- the pooler_output.
562
- last_hidden_state (`torch.FloatTensor` of shape `(batch_size,
563
- sequence_length, hidden_size)`):
564
- Sequence of hidden-states at the output of the last layer of the
565
- hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when
566
- `output_hidden_states=True` is passed or when
567
- `config.output_hidden_states=True`):
568
- Tuple of `torch.FloatTensor` (one for the output of the
569
- embeddings, if the model has an embedding layer, + one for the
570
- output of each layer) of shape `(batch_size, sequence_length,
571
- hidden_size)`.
572
- Hidden-states of the model at the output of each layer plus the
573
- optional initial embedding outputs.
574
- attentions (`tuple(torch.FloatTensor)`, *optional*, returned when
575
- `output_attentions=True` is passed or when
576
- `config.output_attentions=True`):
577
- Tuple of `torch.FloatTensor` (one for each layer) of shape
578
- `(batch_size, num_heads, sequence_length, sequence_length)`.
579
- Attentions weights after the attention softmax, used to compute
580
- the weighted average in the self-attention heads.
581
- """
582
-
583
- text_embeds: Optional[torch.FloatTensor] = None
584
- last_hidden_state: torch.FloatTensor = None
585
- hidden_states: Optional[Tuple[torch.FloatTensor]] = None
586
- attentions: Optional[Tuple[torch.FloatTensor]] = None
587
-
588
-
589
-@dataclass
590
-# Copied from transformers.models.clip.modeling_clip.CLIPOutput with
591
-# CLIP->Siglip
592
-class SiglipOutput(ModelOutput):
593
- """
594
- Args:
595
- loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when
596
- `return_loss` is `True`):
597
- Contrastive loss for image-text similarity.
598
- logits_per_image:(`torch.FloatTensor` of shape `(image_batch_size,
599
- text_batch_size)`):
600
- The scaled dot product scores between `image_embeds` and
601
- `text_embeds`. This represents the image-text similarity scores.
602
- logits_per_text:(`torch.FloatTensor` of shape `(text_batch_size,
603
- image_batch_size)`):
604
- The scaled dot product scores between `text_embeds` and
605
- `image_embeds`. This represents the text-image similarity scores.
606
- text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
607
- The text embeddings obtained by applying the projection layer to
608
- the pooled output of [`SiglipTextModel`].
609
- image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
610
- The image embeddings obtained by applying the projection layer to
611
- the pooled output of [`SiglipVisionModel`].
612
- text_model_output(`BaseModelOutputWithPooling`):
613
- The output of the [`SiglipTextModel`].
614
- vision_model_output(`BaseModelOutputWithPooling`):
615
- The output of the [`SiglipVisionModel`].
616
- """
617
-
618
- loss: Optional[torch.FloatTensor] = None
619
- logits_per_image: torch.FloatTensor = None
620
- logits_per_text: torch.FloatTensor = None
621
- text_embeds: torch.FloatTensor = None
622
- image_embeds: torch.FloatTensor = None
623
- text_model_output: BaseModelOutputWithPooling = None
624
- vision_model_output: BaseModelOutputWithPooling = None
625
-
626
- def to_tuple(self) -> Tuple[Any]:
627
- return tuple(
628
- self[k] if k not in ["text_model_output", "vision_model_output"
629
- ] else getattr(self, k).to_tuple()
630
- for k in self.keys())
631
-
632
-
633
-class SiglipVisionEmbeddings(nn.Module):
634
-
635
- def __init__(self, config: SiglipVisionConfig):
636
- super().__init__()
637
- self.config = config
638
- self.embed_dim = config.hidden_size
639
- self.image_size = config.image_size
640
- self.patch_size = config.patch_size
641
-
642
- self.patch_embedding = nn.Conv2d(
643
- in_channels=config.num_channels,
644
- out_channels=self.embed_dim,
645
- kernel_size=self.patch_size,
646
- stride=self.patch_size,
647
- padding="valid",
648
- )
649
-
650
- self.num_patches_per_side = self.image_size // self.patch_size
651
- self.num_patches = self.num_patches_per_side**2
652
- self.num_positions = self.num_patches
653
- self.position_embedding = nn.Embedding(self.num_positions,
654
- self.embed_dim)
655
-
656
- def forward(self, pixel_values: torch.FloatTensor,
657
- patch_attention_mask: torch.BoolTensor) -> torch.Tensor:
658
- batch_size = pixel_values.size(0)
659
-
660
- patch_embeds = self.patch_embedding(pixel_values)
661
- embeddings = patch_embeds.flatten(2).transpose(1, 2)
662
-
663
- max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
664
- max_nb_patches_h, max_nb_patches_w = max_im_h // self.patch_size, \
665
- max_im_w // self.patch_size
666
- boundaries = torch.arange(1 / self.num_patches_per_side, 1.0,
667
- 1 / self.num_patches_per_side)
668
- position_ids = torch.full(
669
- size=(
670
- batch_size,
671
- max_nb_patches_h * max_nb_patches_w,
672
- ),
673
- fill_value=0,
674
- )
675
-
676
- for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
677
- nb_patches_h = p_attn_mask[:, 0].sum()
678
- nb_patches_w = p_attn_mask[0].sum()
679
-
680
- fractional_coords_h = torch.linspace(0, 1 - 1 / nb_patches_h,
681
- nb_patches_h)
682
- fractional_coords_w = torch.linspace(0, 1 - 1 / nb_patches_w,
683
- nb_patches_w)
684
-
685
- bucket_coords_h = torch.bucketize(fractional_coords_h,
686
- boundaries,
687
- right=True)
688
- bucket_coords_w = torch.bucketize(fractional_coords_w,
689
- boundaries,
690
- right=True)
691
-
692
- pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side +
693
- bucket_coords_w).flatten()
694
- position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
695
-
696
- position_ids = position_ids.to(self.position_embedding.weight.device)
697
-
698
- embeddings = embeddings + self.position_embedding(position_ids)
699
- return embeddings
700
-
701
-
702
-# Copied from transformers.models.clip.modeling_clip.CLIPTextEmbeddings with
703
-# CLIP->Siglip
704
-class SiglipTextEmbeddings(nn.Module):
705
-
706
- def __init__(self, config: SiglipTextConfig):
707
- super().__init__()
708
- embed_dim = config.hidden_size
709
-
710
- self.token_embedding = nn.Embedding(config.vocab_size, embed_dim)
711
- self.position_embedding = nn.Embedding(config.max_position_embeddings,
712
- embed_dim)
713
-
714
- # position_ids (1, len position emb) is contiguous in memory and
715
- # exported when serialized
716
- self.register_buffer(
717
- "position_ids",
718
- torch.arange(config.max_position_embeddings).expand((1, -1)),
719
- persistent=False)
720
-
721
- def forward(
722
- self,
723
- input_ids: Optional[torch.LongTensor] = None,
724
- position_ids: Optional[torch.LongTensor] = None,
725
- inputs_embeds: Optional[torch.FloatTensor] = None,
726
- ) -> torch.Tensor:
727
- seq_length = input_ids.shape[
728
- -1] if input_ids is not None else inputs_embeds.shape[-2]
729
-
730
- if position_ids is None:
731
- position_ids = self.position_ids[:, :seq_length]
732
-
733
- if inputs_embeds is None:
734
- inputs_embeds = self.token_embedding(input_ids)
735
-
736
- position_embeddings = self.position_embedding(position_ids)
737
- embeddings = inputs_embeds + position_embeddings
738
-
739
- return embeddings
740
-
741
-
742
-class SiglipAttention(nn.Module):
743
- """Multi-headed attention from 'Attention Is All You Need' paper"""
744
-
745
- # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
746
- def __init__(self, config):
747
- super().__init__()
748
- self.config = config
749
- self.embed_dim = config.hidden_size
750
- self.num_heads = config.num_attention_heads
751
- self.head_dim = self.embed_dim // self.num_heads
752
- if self.head_dim * self.num_heads != self.embed_dim:
753
- raise ValueError(
754
- f"embed_dim must be divisible by num_heads (got `embed_dim`:"
755
- f" {self.embed_dim} and `num_heads`: {self.num_heads}).")
756
- self.scale = self.head_dim**-0.5
757
- self.dropout = config.attention_dropout
758
-
759
- self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
760
- self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
761
- self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
762
- self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
763
-
764
- def forward(
765
- self,
766
- hidden_states: torch.Tensor,
767
- attention_mask: Optional[torch.Tensor] = None,
768
- output_attentions: Optional[bool] = False,
769
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
770
- Optional[Tuple[torch.Tensor]]]:
771
- """Input shape: Batch x Time x Channel"""
772
-
773
- batch_size, q_len, _ = hidden_states.size()
774
-
775
- query_states = self.q_proj(hidden_states)
776
- key_states = self.k_proj(hidden_states)
777
- value_states = self.v_proj(hidden_states)
778
-
779
- query_states = query_states.view(batch_size, q_len, self.num_heads,
780
- self.head_dim).transpose(1, 2)
781
- key_states = key_states.view(batch_size, q_len, self.num_heads,
782
- self.head_dim).transpose(1, 2)
783
- value_states = value_states.view(batch_size, q_len, self.num_heads,
784
- self.head_dim).transpose(1, 2)
785
-
786
- k_v_seq_len = key_states.shape[-2]
787
- attn_weights = torch.matmul(query_states, key_states.transpose(
788
- 2, 3)) * self.scale
789
-
790
- if attn_weights.size() != (batch_size, self.num_heads, q_len,
791
- k_v_seq_len):
792
- raise ValueError(
793
- f"Attention weights should be of size "
794
- f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
795
- f" {attn_weights.size()}")
796
-
797
- if attention_mask is not None:
798
- if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
799
- raise ValueError(f"Attention mask should be of size "
800
- f"{(batch_size, 1, q_len, k_v_seq_len)}, "
801
- f"but is {attention_mask.size()}")
802
- attn_weights = attn_weights + attention_mask
803
-
804
- # upcast attention to fp32
805
- attn_weights = nn.functional.softmax(attn_weights,
806
- dim=-1,
807
- dtype=torch.float32).to(
808
- query_states.dtype)
809
- attn_weights = nn.functional.dropout(attn_weights,
810
- p=self.dropout,
811
- training=self.training)
812
- attn_output = torch.matmul(attn_weights, value_states)
813
-
814
- if attn_output.size() != (batch_size, self.num_heads, q_len,
815
- self.head_dim):
816
- raise ValueError(
817
- f"`attn_output` should be of size "
818
- f"{(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
819
- f" {attn_output.size()}")
820
-
821
- attn_output = attn_output.transpose(1, 2).contiguous()
822
- attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
823
-
824
- attn_output = self.out_proj(attn_output)
825
-
826
- return attn_output, attn_weights
827
-
828
-
829
-class SiglipFlashAttention2(SiglipAttention):
830
- """
831
- Llama flash attention module. This module inherits from `LlamaAttention` as
832
- the weights of the module stays untouched. The only required change would
833
- be on the forward pass where it needs to correctly call the public API of
834
- flash attention and deal with padding tokens in case the input contains any
835
- of them.
836
- """
837
-
838
- def __init__(self, *args, **kwargs):
839
- super().__init__(*args, **kwargs)
840
- self.is_causal = False # Hack to make sure we don't use a causal mask
841
-
842
- def forward(
843
- self,
844
- hidden_states: torch.Tensor,
845
- attention_mask: Optional[torch.LongTensor] = None,
846
- position_ids: Optional[torch.LongTensor] = None,
847
- past_key_value: Optional[Tuple[torch.Tensor]] = None,
848
- output_attentions: bool = False,
849
- use_cache: bool = False,
850
- **kwargs,
851
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
852
- Optional[Tuple[torch.Tensor]]]:
853
- output_attentions = False
854
-
855
- bsz, q_len, _ = hidden_states.size()
856
-
857
- query_states = self.q_proj(hidden_states)
858
- key_states = self.k_proj(hidden_states)
859
- value_states = self.v_proj(hidden_states)
860
-
861
- # Flash attention requires the input to have the shape
862
- # batch_size x seq_length x head_dim x hidden_dim
863
- # therefore we just need to keep the original shape
864
- query_states = query_states.view(bsz, q_len, self.num_heads,
865
- self.head_dim).transpose(1, 2)
866
- key_states = key_states.view(bsz, q_len, self.num_heads,
867
- self.head_dim).transpose(1, 2)
868
- value_states = value_states.view(bsz, q_len, self.num_heads,
869
- self.head_dim).transpose(1, 2)
870
-
871
- kv_seq_len = key_states.shape[-2]
872
- if past_key_value is not None:
873
- kv_seq_len += past_key_value.get_usable_length(
874
- kv_seq_len, self.layer_idx)
875
-
876
- # TODO: These transpose are quite inefficient but Flash Attention
877
- # requires the layout [batch_size, sequence_length, num_heads,
878
- # head_dim]. We would need to refactor the KV cache
879
- # to be able to avoid many of these transpose/reshape/view.
880
- query_states = query_states.transpose(1, 2)
881
- key_states = key_states.transpose(1, 2)
882
- value_states = value_states.transpose(1, 2)
883
-
884
- dropout_rate = self.dropout if self.training else 0.0
885
-
886
- # In PEFT, usually we cast the layer norms in float32 for training
887
- # stability reasons therefore the input hidden states gets silently
888
- # casted in float32. Hence, we need cast them back in the correct
889
- # dtype just to be sure everything works as expected.
890
- # This might slowdown training & inference so it is recommended to
891
- # not cast the LayerNorms in fp32. (LlamaRMSNorm handles it correctly)
892
-
893
- input_dtype = query_states.dtype
894
- if input_dtype == torch.float32:
895
- if torch.is_autocast_enabled():
896
- target_dtype = torch.get_autocast_gpu_dtype()
897
- # Handle the case where the model is quantized
898
- elif hasattr(self.config, "_pre_quantization_dtype"):
899
- target_dtype = self.config._pre_quantization_dtype
900
- else:
901
- target_dtype = self.q_proj.weight.dtype
902
-
903
- logger.warning_once(
904
- "The input hidden states seems to be silently casted in "
905
- "float32, this might be related to the fact you have upcasted "
906
- "embedding or layer norm layers in float32. We will cast "
907
- f"back the input in {target_dtype}.")
908
-
909
- query_states = query_states.to(target_dtype)
910
- key_states = key_states.to(target_dtype)
911
- value_states = value_states.to(target_dtype)
912
-
913
- attn_output = self._flash_attention_forward(query_states,
914
- key_states,
915
- value_states,
916
- attention_mask,
917
- q_len,
918
- dropout=dropout_rate)
919
-
920
- attn_output = attn_output.reshape(bsz, q_len,
921
- self.embed_dim).contiguous()
922
- attn_output = self.out_proj(attn_output)
923
-
924
- if not output_attentions:
925
- attn_weights = None
926
-
927
- return attn_output, attn_weights
928
-
929
- def _flash_attention_forward(self,
930
- query_states,
931
- key_states,
932
- value_states,
933
- attention_mask,
934
- query_length,
935
- dropout=0.0,
936
- softmax_scale=None):
937
- """
938
- Calls the forward method of Flash Attention - if the input hidden
939
- states contain at least one padding token first unpad the input,
940
- then computes the attention scores and pad the final attention
941
- scores.
942
- Args:
943
- query_states (`torch.Tensor`):
944
- Input query states to be passed to Flash Attention API
945
- key_states (`torch.Tensor`):
946
- Input key states to be passed to Flash Attention API
947
- value_states (`torch.Tensor`):
948
- Input value states to be passed to Flash Attention API
949
- attention_mask (`torch.Tensor`):
950
- The padding mask - corresponds to a tensor of size
951
- `(batch_size, seq_len)` where 0 stands for the position
952
- of padding tokens and 1 for the position of non-padding
953
- tokens.
954
- dropout (`int`, *optional*):
955
- Attention dropout
956
- softmax_scale (`float`, *optional*):
957
- The scaling of QK^T before applying softmax. Default to 1 /
958
- sqrt(head_dim)
959
- """
960
- from flash_attn import flash_attn_func, flash_attn_varlen_func
961
- from flash_attn.bert_padding import pad_input # noqa
962
-
963
- # TODO: Remove the `query_length != 1` check once Flash Attention for
964
- # RoCm is bumped to 2.1. For details, please see the comment in
965
- # LlamaFlashAttention2 __init__.
966
- causal = self.is_causal and query_length != 1
967
-
968
- # Contains at least one padding token in the sequence
969
- if attention_mask is not None:
970
- batch_size = query_states.shape[0]
971
- query_states, key_states, value_states, indices_q, cu_seq_lens, \
972
- max_seq_lens = self._upad_input(
973
- query_states, key_states, value_states, attention_mask,
974
- query_length)
975
-
976
- cu_seqlens_q, cu_seqlens_k = cu_seq_lens
977
- max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
978
-
979
- attn_output_unpad = flash_attn_varlen_func(
980
- query_states,
981
- key_states,
982
- value_states,
983
- cu_seqlens_q=cu_seqlens_q,
984
- cu_seqlens_k=cu_seqlens_k,
985
- max_seqlen_q=max_seqlen_in_batch_q,
986
- max_seqlen_k=max_seqlen_in_batch_k,
987
- dropout_p=dropout,
988
- softmax_scale=softmax_scale,
989
- causal=causal,
990
- )
991
-
992
- attn_output = pad_input(attn_output_unpad, indices_q, batch_size,
993
- query_length)
994
- else:
995
- attn_output = flash_attn_func(query_states,
996
- key_states,
997
- value_states,
998
- dropout,
999
- softmax_scale=softmax_scale,
1000
- causal=causal)
1001
-
1002
- return attn_output
1003
-
1004
- def _upad_input(self, query_layer, key_layer, value_layer, attention_mask,
1005
- query_length):
1006
- from flash_attn.bert_padding import index_first_axis, unpad_input
1007
- indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(
1008
- attention_mask)
1009
- batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
1010
-
1011
- key_layer = index_first_axis(
1012
- key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
1013
- head_dim), indices_k)
1014
- value_layer = index_first_axis(
1015
- value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
1016
- head_dim), indices_k)
1017
- if query_length == kv_seq_len:
1018
- query_layer = index_first_axis(
1019
- query_layer.reshape(batch_size * kv_seq_len, self.num_heads,
1020
- head_dim), indices_k)
1021
- cu_seqlens_q = cu_seqlens_k
1022
- max_seqlen_in_batch_q = max_seqlen_in_batch_k
1023
- indices_q = indices_k
1024
- elif query_length == 1:
1025
- max_seqlen_in_batch_q = 1
1026
- cu_seqlens_q = torch.arange(
1027
- batch_size + 1, dtype=torch.int32, device=query_layer.device
1028
- ) # There is a memcpy here, that is very bad.
1029
- indices_q = cu_seqlens_q[:-1]
1030
- query_layer = query_layer.squeeze(1)
1031
- else:
1032
- # The -q_len: slice assumes left padding.
1033
- attention_mask = attention_mask[:, -query_length:]
1034
- query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = \
1035
- unpad_input(query_layer, attention_mask)
1036
-
1037
- return (
1038
- query_layer,
1039
- key_layer,
1040
- value_layer,
1041
- indices_q,
1042
- (cu_seqlens_q, cu_seqlens_k),
1043
- (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
1044
- )
1045
-
1046
-
1047
-# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
1048
-class SiglipMLP(nn.Module):
1049
-
1050
- def __init__(self, config):
1051
- super().__init__()
1052
- self.config = config
1053
- self.activation_fn = ACT2FN[config.hidden_act]
1054
- self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
1055
- self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
1056
-
1057
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
1058
- hidden_states = self.fc1(hidden_states)
1059
- hidden_states = self.activation_fn(hidden_states)
1060
- hidden_states = self.fc2(hidden_states)
1061
- return hidden_states
1062
-
1063
-
1064
-# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with
1065
-# CLIP->Siglip
1066
-class SiglipEncoderLayer(nn.Module):
1067
-
1068
- def __init__(self, config: SiglipConfig):
1069
- super().__init__()
1070
- self.embed_dim = config.hidden_size
1071
- self.self_attn = (SiglipAttention(config) if
1072
- not getattr(config, "_flash_attn_2_enabled", False)
1073
- else SiglipFlashAttention2(config))
1074
- self.layer_norm1 = nn.LayerNorm(self.embed_dim,
1075
- eps=config.layer_norm_eps)
1076
- self.mlp = SiglipMLP(config)
1077
- self.layer_norm2 = nn.LayerNorm(self.embed_dim,
1078
- eps=config.layer_norm_eps)
1079
-
1080
- def forward(
1081
- self,
1082
- hidden_states: torch.Tensor,
1083
- attention_mask: torch.Tensor,
1084
- output_attentions: Optional[bool] = False,
1085
- ) -> Tuple[torch.FloatTensor]:
1086
- """
1087
- Args:
1088
- hidden_states (`torch.FloatTensor`):
1089
- Input to the layer of shape `(batch, seq_len, embed_dim)`.
1090
- attention_mask (`torch.FloatTensor`):
1091
- Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where
1092
- padding elements are indicated by very large negative values.
1093
- output_attentions (`bool`, *optional*, defaults to `False`):
1094
- Whether or not to return the attentions tensors of all
1095
- attention layers. See `attentions` under returned tensors for
1096
- more detail.
1097
- """
1098
- residual = hidden_states
1099
-
1100
- hidden_states = self.layer_norm1(hidden_states)
1101
- hidden_states, attn_weights = self.self_attn(
1102
- hidden_states=hidden_states,
1103
- attention_mask=attention_mask,
1104
- output_attentions=output_attentions,
1105
- )
1106
- hidden_states = residual + hidden_states
1107
-
1108
- residual = hidden_states
1109
- hidden_states = self.layer_norm2(hidden_states)
1110
- hidden_states = self.mlp(hidden_states)
1111
- hidden_states = residual + hidden_states
1112
-
1113
- outputs = (hidden_states, )
1114
-
1115
- if output_attentions:
1116
- outputs += (attn_weights, )
1117
-
1118
- return outputs
1119
-
1120
-
1121
-class SiglipPreTrainedModel(PreTrainedModel):
1122
- """
1123
- An abstract class to handle weights initialization and a simple interface
1124
- for downloading and loading pretrained models.
1125
- """
1126
-
1127
- config_class = SiglipConfig
1128
- base_model_prefix = "siglip"
1129
- supports_gradient_checkpointing = True
1130
-
1131
- def _init_weights(self, module):
1132
- """Initialize the weights"""
1133
-
1134
- if isinstance(module, SiglipVisionEmbeddings):
1135
- width = (self.config.vision_config.hidden_size if isinstance(
1136
- self.config, SiglipConfig) else self.config.hidden_size)
1137
- nn.init.normal_(module.position_embedding.weight,
1138
- std=1 / np.sqrt(width))
1139
- elif isinstance(module, nn.Embedding):
1140
- default_flax_embed_init(module.weight)
1141
- elif isinstance(module, SiglipAttention):
1142
- nn.init.normal_(module.q_proj.weight)
1143
- nn.init.normal_(module.k_proj.weight)
1144
- nn.init.normal_(module.v_proj.weight)
1145
- nn.init.normal_(module.out_proj.weight)
1146
- nn.init.zeros_(module.q_proj.bias)
1147
- nn.init.zeros_(module.k_proj.bias)
1148
- nn.init.zeros_(module.v_proj.bias)
1149
- nn.init.zeros_(module.out_proj.bias)
1150
- elif isinstance(module, SiglipMLP):
1151
- nn.init.normal_(module.fc1.weight)
1152
- nn.init.normal_(module.fc2.weight)
1153
- nn.init.normal_(module.fc1.bias, std=1e-6)
1154
- nn.init.normal_(module.fc2.bias, std=1e-6)
1155
- elif isinstance(module, SiglipMultiheadAttentionPoolingHead):
1156
- nn.init.normal_(module.probe.data)
1157
- nn.init.normal_(module.attention.in_proj_weight.data)
1158
- nn.init.zeros_(module.attention.in_proj_bias.data)
1159
- elif isinstance(module, SiglipModel):
1160
- logit_scale_init = torch.tensor(0.0)
1161
- module.logit_scale.data.fill_(logit_scale_init)
1162
- module.logit_bias.data.zero_()
1163
- elif isinstance(module, (nn.Linear, nn.Conv2d)):
1164
- lecun_normal_(module.weight)
1165
- if module.bias is not None:
1166
- nn.init.zeros_(module.bias)
1167
- elif isinstance(module, nn.LayerNorm):
1168
- module.bias.data.zero_()
1169
- module.weight.data.fill_(1.0)
1170
-
1171
-
1172
-SIGLIP_START_DOCSTRING = r"""
1173
- This model inherits from [`PreTrainedModel`]. Check the superclass
1174
- documentation for the generic methods the library implements for all
1175
- its model (such as downloading or saving, resizing the input embeddings,
1176
- pruning heads etc.)
1177
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/
1178
- stable/nn.html#torch.nn.Module) subclass.
1179
- Use it as a regular PyTorch Module and refer to the PyTorch documentation
1180
- for all matter related to general usage and behavior.
1181
- Parameters:
1182
- config ([`SiglipConfig`]): Model configuration class with all the
1183
- parameters of the model.
1184
- Initializing with a config file does not load the weights
1185
- associated with the model, only the configuration. Check out
1186
- the [`~PreTrainedModel.from_pretrained`] method to load the
1187
- model weights.
1188
-"""
1189
-
1190
-SIGLIP_TEXT_INPUTS_DOCSTRING = r"""
1191
- Args:
1192
- input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)
1193
- `):
1194
- Indices of input sequence tokens in the vocabulary. Padding will
1195
- be ignored by default should you provide it.
1196
- Indices can be obtained using [`AutoTokenizer`]. See
1197
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
1198
- for details. [What are input IDs?](../glossary#input-ids)
1199
- attention_mask (`torch.Tensor` of shape `(batch_size,
1200
- sequence_length)`, *optional*):
1201
- Mask to avoid performing attention on padding token indices. Mask
1202
- values selected in `[0, 1]`:
1203
- - 1 for tokens that are **not masked**,
1204
- - 0 for tokens that are **masked**.
1205
- [What are attention masks?](../glossary#attention-mask)
1206
- position_ids (`torch.LongTensor` of shape `(batch_size,
1207
- sequence_length)`, *optional*):
1208
- Indices of positions of each input sequence tokens in the position
1209
- embeddings. Selected in the range `[0,
1210
- config.max_position_embeddings - 1]`.
1211
- [What are position IDs?](../glossary#position-ids)
1212
- output_attentions (`bool`, *optional*):
1213
- Whether or not to return the attentions tensors of all attention
1214
- layers. See `attentions` under returned tensors for more detail.
1215
- output_hidden_states (`bool`, *optional*):
1216
- Whether or not to return the hidden states of all layers. See
1217
- `hidden_states` under returned tensors for more detail.
1218
- return_dict (`bool`, *optional*):
1219
- Whether or not to return a [`~utils.ModelOutput`] instead of a
1220
- plain tuple.
1221
-"""
1222
-
1223
-SIGLIP_VISION_INPUTS_DOCSTRING = r"""
1224
- Args:
1225
- pixel_values (`torch.FloatTensor` of shape `(batch_size,
1226
- num_channels, height, width)`):
1227
- Pixel values. Padding will be ignored by default should you
1228
- provide it. Pixel values can be obtained using
1229
- [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`]
1230
- for details.
1231
- output_attentions (`bool`, *optional*):
1232
- Whether or not to return the attentions tensors of all attention
1233
- layers. See `attentions` under returned
1234
- tensors for more detail.
1235
- output_hidden_states (`bool`, *optional*):
1236
- Whether or not to return the hidden states of all layers. See
1237
- `hidden_states` under returned tensors for more detail.
1238
- return_dict (`bool`, *optional*):
1239
- Whether or not to return a [`~utils.ModelOutput`] instead of a
1240
- plain tuple.
1241
-"""
1242
-
1243
-SIGLIP_INPUTS_DOCSTRING = r"""
1244
- Args:
1245
- input_ids (`torch.LongTensor` of shape `(batch_size,
1246
- sequence_length)`):
1247
- Indices of input sequence tokens in the vocabulary. Padding
1248
- will be ignored by default should you provide it.
1249
- Indices can be obtained using [`AutoTokenizer`]. See
1250
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
1251
- for details. [What are input IDs?](../glossary#input-ids)
1252
- attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`
1253
- , *optional*):
1254
- Mask to avoid performing attention on padding token indices. Mask
1255
- values selected in `[0, 1]`:
1256
- - 1 for tokens that are **not masked**,
1257
- - 0 for tokens that are **masked**.
1258
- [What are attention masks?](../glossary#attention-mask)
1259
- position_ids (`torch.LongTensor` of shape `(batch_size,
1260
- sequence_length)`, *optional*):
1261
- Indices of positions of each input sequence tokens in the position
1262
- embeddings. Selected in the range `[0,
1263
- config.max_position_embeddings - 1]`.
1264
- [What are position IDs?](../glossary#position-ids)
1265
- pixel_values (`torch.FloatTensor` of shape `(batch_size,
1266
- num_channels, height, width)`):
1267
- Pixel values. Padding will be ignored by default should you
1268
- provide it. Pixel values can be obtained using
1269
- [`AutoImageProcessor`]. See [`CLIPImageProcessor.__call__`]
1270
- for details.
1271
- return_loss (`bool`, *optional*):
1272
- Whether or not to return the contrastive loss.
1273
- output_attentions (`bool`, *optional*):
1274
- Whether or not to return the attentions tensors of all attention
1275
- layers. See `attentions` under returned tensors for more detail.
1276
- output_hidden_states (`bool`, *optional*):
1277
- Whether or not to return the hidden states of all layers. See
1278
- `hidden_states` under returned tensors for
1279
- more detail.
1280
- return_dict (`bool`, *optional*):
1281
- Whether or not to return a [`~utils.ModelOutput`] instead of a
1282
- plain tuple.
1283
-"""
1284
-
1285
-
1286
-# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with
1287
-# CLIP->Siglip
1288
-class SiglipEncoder(nn.Module):
1289
- """
1290
- Transformer encoder consisting of `config.num_hidden_layers`
1291
- self attention layers. Each layer is a [`SiglipEncoderLayer`].
1292
- Args:
1293
- config: SiglipConfig
1294
- """
1295
-
1296
- def __init__(self, config: SiglipConfig):
1297
- super().__init__()
1298
- self.config = config
1299
- self.layers = nn.ModuleList([
1300
- SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)
1301
- ])
1302
- self.gradient_checkpointing = False
1303
-
1304
- # Ignore copy
1305
- def forward(
1306
- self,
1307
- inputs_embeds,
1308
- attention_mask: Optional[torch.Tensor] = None,
1309
- output_attentions: Optional[bool] = None,
1310
- output_hidden_states: Optional[bool] = None,
1311
- return_dict: Optional[bool] = None,
1312
- ) -> Union[Tuple, BaseModelOutput]:
1313
- r"""
1314
- Args:
1315
- inputs_embeds (`torch.FloatTensor` of shape `(batch_size,
1316
- sequence_length, hidden_size)`):
1317
- Optionally, instead of passing `input_ids` you can choose to
1318
- directly pass an embedded representation.
1319
- This is useful if you want more control over how to convert
1320
- `input_ids` indices into associated vectors
1321
- than the model's internal embedding lookup matrix.
1322
- attention_mask (`torch.Tensor` of shape `(batch_size,
1323
- sequence_length)`, *optional*):
1324
- Mask to avoid performing attention on padding token indices.
1325
- Mask values selected in `[0, 1]`:
1326
- - 1 for tokens that are **not masked**,
1327
- - 0 for tokens that are **masked**.
1328
- [What are attention masks?](../glossary#attention-mask)
1329
- output_attentions (`bool`, *optional*):
1330
- Whether or not to return the attentions tensors of all
1331
- attention layers. See `attentions` under returned tensors for
1332
- more detail.
1333
- output_hidden_states (`bool`, *optional*):
1334
- Whether or not to return the hidden states of all layers. See
1335
- `hidden_states` under returned tensors for more detail.
1336
- return_dict (`bool`, *optional*):
1337
- Whether or not to return a [`~utils.ModelOutput`] instead of a
1338
- plain tuple.
1339
- """
1340
- output_attentions = output_attentions if output_attentions \
1341
- is not None else self.config.output_attentions
1342
- output_hidden_states = (output_hidden_states
1343
- if output_hidden_states is not None else
1344
- self.config.output_hidden_states)
1345
- return_dict = return_dict if return_dict is not None else \
1346
- self.config.use_return_dict
1347
-
1348
- encoder_states = () if output_hidden_states else None
1349
- all_attentions = () if output_attentions else None
1350
-
1351
- hidden_states = inputs_embeds
1352
- for encoder_layer in self.layers:
1353
- if output_hidden_states:
1354
- encoder_states = encoder_states + (hidden_states, )
1355
- if self.gradient_checkpointing and self.training:
1356
- layer_outputs = self._gradient_checkpointing_func(
1357
- encoder_layer.__call__,
1358
- hidden_states,
1359
- attention_mask,
1360
- output_attentions,
1361
- )
1362
- else:
1363
- layer_outputs = encoder_layer(
1364
- hidden_states,
1365
- attention_mask,
1366
- output_attentions=output_attentions,
1367
- )
1368
-
1369
- hidden_states = layer_outputs[0]
1370
-
1371
- if output_attentions:
1372
- all_attentions = all_attentions + (layer_outputs[1], )
1373
-
1374
- if output_hidden_states:
1375
- encoder_states = encoder_states + (hidden_states, )
1376
-
1377
- if not return_dict:
1378
- return tuple(
1379
- v for v in [hidden_states, encoder_states, all_attentions]
1380
- if v is not None)
1381
- return BaseModelOutput(last_hidden_state=hidden_states,
1382
- hidden_states=encoder_states,
1383
- attentions=all_attentions)
1384
-
1385
-
1386
-class SiglipTextTransformer(nn.Module):
1387
-
1388
- def __init__(self, config: SiglipTextConfig):
1389
- super().__init__()
1390
- self.config = config
1391
- embed_dim = config.hidden_size
1392
- self.embeddings = SiglipTextEmbeddings(config)
1393
- self.encoder = SiglipEncoder(config)
1394
- self.final_layer_norm = nn.LayerNorm(embed_dim,
1395
- eps=config.layer_norm_eps)
1396
-
1397
- self.head = nn.Linear(embed_dim, embed_dim)
1398
-
1399
- @add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING)
1400
- @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
1401
- config_class=SiglipTextConfig)
1402
- def forward(
1403
- self,
1404
- input_ids: Optional[torch.Tensor] = None,
1405
- attention_mask: Optional[torch.Tensor] = None,
1406
- position_ids: Optional[torch.Tensor] = None,
1407
- output_attentions: Optional[bool] = None,
1408
- output_hidden_states: Optional[bool] = None,
1409
- return_dict: Optional[bool] = None,
1410
- ) -> Union[Tuple, BaseModelOutputWithPooling]:
1411
- r"""
1412
- Returns:
1413
- """
1414
- output_attentions = output_attentions if output_attentions \
1415
- is not None else self.config.output_attentions
1416
- output_hidden_states = (output_hidden_states
1417
- if output_hidden_states \
1418
- is not None else
1419
- self.config.output_hidden_states)
1420
- return_dict = return_dict if return_dict is not None else \
1421
- self.config.use_return_dict
1422
-
1423
- if input_ids is None:
1424
- raise ValueError("You have to specify input_ids")
1425
-
1426
- input_shape = input_ids.size()
1427
- input_ids = input_ids.view(-1, input_shape[-1])
1428
-
1429
- hidden_states = self.embeddings(input_ids=input_ids,
1430
- position_ids=position_ids)
1431
-
1432
- # note: SigLIP's text model does not use a causal mask, unlike the
1433
- # original CLIP model.
1434
- # expand attention_mask
1435
- if attention_mask is not None:
1436
- # [batch_size, seq_len] ->
1437
- # [batch_size, 1, tgt_seq_len, src_seq_len]
1438
- attention_mask = _prepare_4d_attention_mask(
1439
- attention_mask, hidden_states.dtype)
1440
-
1441
- encoder_outputs = self.encoder(
1442
- inputs_embeds=hidden_states,
1443
- attention_mask=attention_mask,
1444
- output_attentions=output_attentions,
1445
- output_hidden_states=output_hidden_states,
1446
- return_dict=return_dict,
1447
- )
1448
-
1449
- last_hidden_state = encoder_outputs[0]
1450
- last_hidden_state = self.final_layer_norm(last_hidden_state)
1451
-
1452
- # Assuming "sticky" EOS tokenization, last token is always EOS.
1453
- pooled_output = last_hidden_state[:, -1, :]
1454
- pooled_output = self.head(pooled_output)
1455
-
1456
- if not return_dict:
1457
- return (last_hidden_state, pooled_output) + encoder_outputs[1:]
1458
-
1459
- return BaseModelOutputWithPooling(
1460
- last_hidden_state=last_hidden_state,
1461
- pooler_output=pooled_output,
1462
- hidden_states=encoder_outputs.hidden_states,
1463
- attentions=encoder_outputs.attentions,
1464
- )
1465
-
1466
-
1467
-@add_start_docstrings(
1468
- """The text model from SigLIP without any head or projection on top.""",
1469
- SIGLIP_START_DOCSTRING,
1470
-)
1471
-class SiglipTextModel(SiglipPreTrainedModel):
1472
- config_class = SiglipTextConfig
1473
-
1474
- _no_split_modules = ["SiglipTextEmbeddings", "SiglipEncoderLayer"]
1475
-
1476
- def __init__(self, config: SiglipTextConfig):
1477
- super().__init__(config)
1478
- self.text_model = SiglipTextTransformer(config)
1479
- # Initialize weights and apply final processing
1480
- self.post_init()
1481
-
1482
- def get_input_embeddings(self) -> nn.Module:
1483
- return self.text_model.embeddings.token_embedding
1484
-
1485
- def set_input_embeddings(self, value):
1486
- self.text_model.embeddings.token_embedding = value
1487
-
1488
- @add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING)
1489
- @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
1490
- config_class=SiglipTextConfig)
1491
- def forward(
1492
- self,
1493
- input_ids: Optional[torch.Tensor] = None,
1494
- attention_mask: Optional[torch.Tensor] = None,
1495
- position_ids: Optional[torch.Tensor] = None,
1496
- output_attentions: Optional[bool] = None,
1497
- output_hidden_states: Optional[bool] = None,
1498
- return_dict: Optional[bool] = None,
1499
- ) -> Union[Tuple, BaseModelOutputWithPooling]:
1500
- r"""
1501
- Returns:
1502
- Examples:
1503
- ```python
1504
- >>> from transformers import AutoTokenizer, SiglipTextModel
1505
- >>> model = SiglipTextModel.
1506
- from_pretrained("google/siglip-base-patch16-224")
1507
- >>> tokenizer = AutoTokenizer.
1508
- from_pretrained("google/siglip-base-patch16-224")
1509
- >>> # important: make sure to set padding="max_length"
1510
- as that's how the model was trained
1511
- >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"],
1512
- padding="max_length", return_tensors="pt")
1513
- >>> outputs = model(**inputs)
1514
- >>> last_hidden_state = outputs.last_hidden_state
1515
- >>> pooled_output = outputs.pooler_output # pooled (EOS token)
1516
- states
1517
- ```"""
1518
- return_dict = return_dict if return_dict is not None else \
1519
- self.config.use_return_dict
1520
-
1521
- return self.text_model(
1522
- input_ids=input_ids,
1523
- attention_mask=attention_mask,
1524
- position_ids=position_ids,
1525
- output_attentions=output_attentions,
1526
- output_hidden_states=output_hidden_states,
1527
- return_dict=return_dict,
1528
- )
1529
-
1530
-
1531
-class SiglipVisionTransformer(nn.Module):
1532
-
1533
- def __init__(self, config: SiglipVisionConfig):
1534
- super().__init__()
1535
- self.config = config
1536
- embed_dim = config.hidden_size
1537
-
1538
- self.embeddings = SiglipVisionEmbeddings(config)
1539
- self.encoder = SiglipEncoder(config)
1540
- self.post_layernorm = nn.LayerNorm(embed_dim,
1541
- eps=config.layer_norm_eps)
1542
- self.head = SiglipMultiheadAttentionPoolingHead(config)
1543
-
1544
- @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
1545
- @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
1546
- config_class=SiglipVisionConfig)
1547
- def forward(
1548
- self,
1549
- pixel_values,
1550
- patch_attention_mask: Optional[torch.BoolTensor] = None,
1551
- output_attentions: Optional[bool] = None,
1552
- output_hidden_states: Optional[bool] = None,
1553
- return_dict: Optional[bool] = None,
1554
- ) -> Union[Tuple, BaseModelOutputWithPooling]:
1555
- r"""
1556
- Returns:
1557
- """
1558
- output_attentions = output_attentions if output_attentions is not None\
1559
- else self.config.output_attentions
1560
- output_hidden_states = (output_hidden_states
1561
- if output_hidden_states is not None else
1562
- self.config.output_hidden_states)
1563
- return_dict = return_dict if return_dict is not None \
1564
- else self.config.use_return_dict
1565
-
1566
- batch_size = pixel_values.size(0)
1567
- if patch_attention_mask is None:
1568
- patch_attention_mask = torch.ones(
1569
- size=(
1570
- batch_size,
1571
- pixel_values.size(2) // self.config.patch_size,
1572
- pixel_values.size(3) // self.config.patch_size,
1573
- ),
1574
- dtype=torch.bool,
1575
- device=pixel_values.device,
1576
- )
1577
-
1578
- hidden_states = self.embeddings(
1579
- pixel_values=pixel_values,
1580
- patch_attention_mask=patch_attention_mask)
1581
-
1582
- patch_attention_mask = patch_attention_mask.view(batch_size, -1)
1583
- # The call to `_upad_input` in `_flash_attention_forward` is expensive
1584
- # So when the `patch_attention_mask` is full of 1s (i.e. attending
1585
- # to the whole sequence), avoiding passing the attention_mask, which
1586
- # is equivalent to attending to the full sequence
1587
- if not torch.any(~patch_attention_mask):
1588
- attention_mask = None
1589
- else:
1590
- attention_mask = (_prepare_4d_attention_mask(
1591
- patch_attention_mask, hidden_states.dtype)
1592
- if not self.config._flash_attn_2_enabled else
1593
- patch_attention_mask)
1594
-
1595
- encoder_outputs = self.encoder(
1596
- inputs_embeds=hidden_states,
1597
- attention_mask=attention_mask,
1598
- output_attentions=output_attentions,
1599
- output_hidden_states=output_hidden_states,
1600
- return_dict=return_dict,
1601
- )
1602
-
1603
- last_hidden_state = encoder_outputs[0]
1604
- last_hidden_state = self.post_layernorm(last_hidden_state)
1605
-
1606
- pooled_output = self.head(
1607
- hidden_state=last_hidden_state,
1608
- attention_mask=patch_attention_mask,
1609
- )
1610
-
1611
- if not return_dict:
1612
- return (last_hidden_state, pooled_output) + encoder_outputs[1:]
1613
-
1614
- return BaseModelOutputWithPooling(
1615
- last_hidden_state=last_hidden_state,
1616
- pooler_output=pooled_output,
1617
- hidden_states=encoder_outputs.hidden_states,
1618
- attentions=encoder_outputs.attentions,
1619
- )
1620
-
1621
-
1622
-class SiglipMultiheadAttentionPoolingHead(nn.Module):
1623
- """Multihead Attention Pooling."""
1624
-
1625
- def __init__(self, config: SiglipVisionConfig):
1626
- super().__init__()
1627
-
1628
- self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
1629
- self.attention = torch.nn.MultiheadAttention(
1630
- config.hidden_size, config.num_attention_heads, batch_first=True)
1631
- self.layernorm = nn.LayerNorm(config.hidden_size,
1632
- eps=config.layer_norm_eps)
1633
- self.mlp = SiglipMLP(config)
1634
-
1635
- def forward(self, hidden_state, attention_mask):
1636
- batch_size = hidden_state.shape[0]
1637
- probe = self.probe.repeat(batch_size, 1, 1)
1638
-
1639
- hidden_state = self.attention(query=probe,
1640
- key=hidden_state,
1641
- value=hidden_state,
1642
- key_padding_mask=~attention_mask)[0]
1643
-
1644
- residual = hidden_state
1645
- hidden_state = self.layernorm(hidden_state)
1646
- hidden_state = residual + self.mlp(hidden_state)
1647
-
1648
- return hidden_state[:, 0]
1649
-
1650
-
1651
-@add_start_docstrings(
1652
- """The vision model from SigLIP without any head or projection on top.""",
1653
- SIGLIP_START_DOCSTRING,
1654
-)
1655
-class SiglipVisionModel(SiglipPreTrainedModel):
1656
- config_class = SiglipVisionConfig
1657
- main_input_name = "pixel_values"
1658
-
1659
- def __init__(self, config: SiglipVisionConfig):
1660
- super().__init__(config)
1661
-
1662
- self.vision_model = SiglipVisionTransformer(config)
1663
-
1664
- # Initialize weights and apply final processing
1665
- self.post_init()
1666
-
1667
- def get_input_embeddings(self) -> nn.Module:
1668
- return self.vision_model.embeddings.patch_embedding
1669
-
1670
- @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
1671
- @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
1672
- config_class=SiglipVisionConfig)
1673
- def forward(
1674
- self,
1675
- pixel_values,
1676
- patch_attention_mask: Optional[torch.BoolTensor] = None,
1677
- output_attentions: Optional[bool] = None,
1678
- output_hidden_states: Optional[bool] = None,
1679
- return_dict: Optional[bool] = None,
1680
- ) -> Union[Tuple, BaseModelOutputWithPooling]:
1681
- r"""
1682
- Returns:
1683
- Examples:
1684
- ```python
1685
- >>> from PIL import Image
1686
- >>> import requests
1687
- >>> from transformers import AutoProcessor, SiglipVisionModel
1688
- >>> model = SiglipVisionModel.from_pretrained(
1689
- "google/siglip-base-patch16-224")
1690
- >>> processor = AutoProcessor.from_pretrained(
1691
- "google/siglip-base-patch16-224")
1692
- >>> url =
1693
- "http://images.cocodataset.org/val2017/000000039769.jpg"
1694
- >>> image = Image.open(requests.get(url, stream=True).raw)
1695
- >>> inputs = processor(images=image, return_tensors="pt")
1696
- >>> outputs = model(**inputs)
1697
- >>> last_hidden_state = outputs.last_hidden_state
1698
- >>> pooled_output = outputs.pooler_output # pooled features
1699
- ```"""
1700
- return_dict = return_dict if return_dict is not None \
1701
- else self.config.use_return_dict
1702
-
1703
- return self.vision_model(
1704
- pixel_values=pixel_values,
1705
- patch_attention_mask=patch_attention_mask,
1706
- output_attentions=output_attentions,
1707
- output_hidden_states=output_hidden_states,
1708
- return_dict=return_dict,
1709
- )
1710
-
1711
-
1712
-@add_start_docstrings(SIGLIP_START_DOCSTRING)
1713
-class SiglipModel(SiglipPreTrainedModel):
1714
- config_class = SiglipConfig
1715
-
1716
- def __init__(self, config: SiglipConfig):
1717
- super().__init__(config)
1718
-
1719
- if not isinstance(config.text_config, SiglipTextConfig):
1720
- raise ValueError("config.text_config is expected to be of type "
1721
- f"SiglipTextConfig but is of type"
1722
- f" {type(config.text_config)}.")
1723
-
1724
- if not isinstance(config.vision_config, SiglipVisionConfig):
1725
- raise ValueError("config.vision_config is expected to be of type "
1726
- "SiglipVisionConfig but is of type"
1727
- f" {type(config.vision_config)}.")
1728
-
1729
- text_config = config.text_config
1730
- vision_config = config.vision_config
1731
-
1732
- self.text_model = SiglipTextTransformer(text_config)
1733
- self.vision_model = SiglipVisionTransformer(vision_config)
1734
-
1735
- self.logit_scale = nn.Parameter(torch.randn(1))
1736
- self.logit_bias = nn.Parameter(torch.randn(1))
1737
-
1738
- # Initialize weights and apply final processing
1739
- self.post_init()
1740
-
1741
- @add_start_docstrings_to_model_forward(SIGLIP_TEXT_INPUTS_DOCSTRING)
1742
- def get_text_features(
1743
- self,
1744
- input_ids: Optional[torch.Tensor] = None,
1745
- attention_mask: Optional[torch.Tensor] = None,
1746
- position_ids: Optional[torch.Tensor] = None,
1747
- output_attentions: Optional[bool] = None,
1748
- output_hidden_states: Optional[bool] = None,
1749
- return_dict: Optional[bool] = None,
1750
- ) -> torch.FloatTensor:
1751
- r"""
1752
- Returns:
1753
- text_features (`torch.FloatTensor` of shape `(batch_size,
1754
- output_dim`): The text embeddings obtained by
1755
- applying the projection layer to the pooled output
1756
- of [`SiglipTextModel`].
1757
- Examples:
1758
- ```python
1759
- >>> from transformers import AutoTokenizer, AutoModel
1760
- >>> import torch
1761
- >>> model = AutoModel.from_pretrained(
1762
- "google/siglip-base-patch16-224")
1763
- >>> tokenizer = AutoTokenizer.from_pretrained(
1764
- "google/siglip-base-patch16-224")
1765
- >>> # important: make sure to set padding="max_length" as that's
1766
- how the model was trained
1767
- >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"],
1768
- padding="max_length", return_tensors="pt")
1769
- >>> with torch.no_grad():
1770
- ... text_features = model.get_text_features(**inputs)
1771
- ```"""
1772
- # Use SigLIP model's config for some fields (if specified) instead
1773
- # of those of vision & text components.
1774
- output_attentions = output_attentions if output_attentions is not None\
1775
- else self.config.output_attentions
1776
- output_hidden_states = (output_hidden_states
1777
- if output_hidden_states is not None else
1778
- self.config.output_hidden_states)
1779
- return_dict = return_dict if return_dict is not None \
1780
- else self.config.use_return_dict
1781
-
1782
- text_outputs = self.text_model(
1783
- input_ids=input_ids,
1784
- attention_mask=attention_mask,
1785
- position_ids=position_ids,
1786
- output_attentions=output_attentions,
1787
- output_hidden_states=output_hidden_states,
1788
- return_dict=return_dict,
1789
- )
1790
-
1791
- pooled_output = text_outputs[1]
1792
-
1793
- return pooled_output
1794
-
1795
- @add_start_docstrings_to_model_forward(SIGLIP_VISION_INPUTS_DOCSTRING)
1796
- def get_image_features(
1797
- self,
1798
- pixel_values: Optional[torch.FloatTensor] = None,
1799
- output_attentions: Optional[bool] = None,
1800
- output_hidden_states: Optional[bool] = None,
1801
- return_dict: Optional[bool] = None,
1802
- ) -> torch.FloatTensor:
1803
- r"""
1804
- Returns:
1805
- image_features (`torch.FloatTensor` of shape `(batch_size,
1806
- output_dim`): The image embeddings obtained by applying the
1807
- projection layer to the pooled output of [`SiglipVisionModel`].
1808
- Examples:
1809
- ```python
1810
- >>> from PIL import Image
1811
- >>> import requests
1812
- >>> from transformers import AutoProcessor, AutoModel
1813
- >>> import torch
1814
- >>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
1815
- >>> processor = AutoProcessor.from_pretrained(
1816
- "google/siglip-base-patch16-224")
1817
- >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
1818
- >>> image = Image.open(requests.get(url, stream=True).raw)
1819
- >>> inputs = processor(images=image, return_tensors="pt")
1820
- >>> with torch.no_grad():
1821
- ... image_features = model.get_image_features(**inputs)
1822
- ```"""
1823
- # Use SiglipModel's config for some fields (if specified) instead
1824
- # of those of vision & text components.
1825
- output_attentions = output_attentions if output_attentions \
1826
- is not None else self.config.output_attentions
1827
- output_hidden_states = (output_hidden_states
1828
- if output_hidden_states is not None else
1829
- self.config.output_hidden_states)
1830
- return_dict = return_dict if return_dict is not None else \
1831
- self.config.use_return_dict
1832
-
1833
- vision_outputs = self.vision_model(
1834
- pixel_values=pixel_values,
1835
- output_attentions=output_attentions,
1836
- output_hidden_states=output_hidden_states,
1837
- return_dict=return_dict,
1838
- )
1839
-
1840
- pooled_output = vision_outputs[1]
1841
-
1842
- return pooled_output
1843
-
1844
- @add_start_docstrings_to_model_forward(SIGLIP_INPUTS_DOCSTRING)
1845
- @replace_return_docstrings(output_type=SiglipOutput,
1846
- config_class=SiglipConfig)
1847
- def forward(
1848
- self,
1849
- input_ids: Optional[torch.LongTensor] = None,
1850
- pixel_values: Optional[torch.FloatTensor] = None,
1851
- attention_mask: Optional[torch.Tensor] = None,
1852
- position_ids: Optional[torch.LongTensor] = None,
1853
- return_loss: Optional[bool] = None,
1854
- output_attentions: Optional[bool] = None,
1855
- output_hidden_states: Optional[bool] = None,
1856
- return_dict: Optional[bool] = None,
1857
- ) -> Union[Tuple, SiglipOutput]:
1858
- r"""
1859
- Returns:
1860
- Examples:
1861
- ```python
1862
- >>> from PIL import Image
1863
- >>> import requests
1864
- >>> from transformers import AutoProcessor, AutoModel
1865
- >>> import torch
1866
- >>> model = AutoModel.from_pretrained("google/siglip-base-patch16-224")
1867
- >>> processor = AutoProcessor.from_pretrained(
1868
- "google/siglip-base-patch16-224")
1869
- >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
1870
- >>> image = Image.open(requests.get(url, stream=True).raw)
1871
- >>> texts = ["a photo of 2 cats", "a photo of 2 dogs"]
1872
- >>> # important: we pass `padding=max_length` since the model was
1873
- trained with this
1874
- >>> inputs = processor(text=texts, images=image,
1875
- padding="max_length", return_tensors="pt")
1876
- >>> with torch.no_grad():
1877
- ... outputs = model(**inputs)
1878
- >>> logits_per_image = outputs.logits_per_image
1879
- >>> probs = torch.sigmoid(logits_per_image) # these are the
1880
- probabilities
1881
- >>> print(f"{probs[0][0]:.1%} that image 0 is '{texts[0]}'")
1882
- 31.9% that image 0 is 'a photo of 2 cats'
1883
- ```"""
1884
- # Use SigLIP model's config for some fields (if specified) instead of
1885
- # those of vision & text components.
1886
- output_attentions = output_attentions if output_attentions \
1887
- is not None else self.config.output_attentions
1888
- output_hidden_states = (output_hidden_states
1889
- if output_hidden_states is not None else
1890
- self.config.output_hidden_states)
1891
- return_dict = return_dict if return_dict is not None else \
1892
- self.config.use_return_dict
1893
-
1894
- vision_outputs = self.vision_model(
1895
- pixel_values=pixel_values,
1896
- output_attentions=output_attentions,
1897
- output_hidden_states=output_hidden_states,
1898
- return_dict=return_dict,
1899
- )
1900
-
1901
- text_outputs = self.text_model(
1902
- input_ids=input_ids,
1903
- attention_mask=attention_mask,
1904
- position_ids=position_ids,
1905
- output_attentions=output_attentions,
1906
- output_hidden_states=output_hidden_states,
1907
- return_dict=return_dict,
1908
- )
1909
-
1910
- image_embeds = vision_outputs[1]
1911
- text_embeds = text_outputs[1]
1912
-
1913
- # normalized features
1914
- image_embeds = image_embeds / image_embeds.norm(
1915
- p=2, dim=-1, keepdim=True)
1916
- text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True)
1917
-
1918
- # cosine similarity as logits
1919
- logits_per_text = torch.matmul(text_embeds, image_embeds.t(
1920
- )) * self.logit_scale.exp() + self.logit_bias
1921
- logits_per_image = logits_per_text.t()
1922
-
1923
- loss = None
1924
- if return_loss:
1925
- raise NotImplementedError("SigLIP loss to be implemented")
1926
-
1927
- if not return_dict:
1928
- output = (logits_per_image, logits_per_text, text_embeds,
1929
- image_embeds, text_outputs, vision_outputs)
1930
- return ((loss, ) + output) if loss is not None else output
1931
-
1932
- return SiglipOutput(
1933
- loss=loss,
1934
- logits_per_image=logits_per_image,
1935
- logits_per_text=logits_per_text,
1936
- text_embeds=text_embeds,
1937
- image_embeds=image_embeds,
1938
- text_model_output=text_outputs,
1939
- vision_model_output=vision_outputs,
1940
- )
1941
-
1942
-
1943
-def get_siglip_vision_model(_flash_attn_2_enabled=True, **kwargs):
1944
- siglip_vision_config = {
1945
- "hidden_size": 1152,
1946
- "image_size": 448,
1947
- "intermediate_size": 4304,
1948
- "model_type": "siglip_vision_model",
1949
- "num_attention_heads": 16,
1950
- "num_hidden_layers": 27,
1951
- "patch_size": 14,
1952
- }
1953
-
1954
- # Detect attention implementation.
1955
- attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
1956
- if attn_backend != _Backend.FLASH_ATTN:
1957
- _flash_attn_2_enabled = False
1958
-
1959
- model_config = SiglipVisionConfig(
1960
- **siglip_vision_config,
1961
- _flash_attn_2_enabled=_flash_attn_2_enabled,
1962
- **kwargs)
1963
-
1964
- vision_model = SiglipVisionModel(model_config).vision_model
1965
-
1966
- return vision_model