A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Siglip2 Model Support (#27566)
Signed-off-by: piood <2477084691@qq.com>
Yu Jiaqi committed
Oct 27, 2025 at 21:57 UTC
4f882be4a0d319551ce2a0eadcace0e76f3432cd
4 files changed
+18
-6
docs/models/supported_models.md
+1
-1
@@ -775,7 +775,7 @@ The following table lists those that are tested in vLLM.
775
| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
776
| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
777
| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
778
-| `SiglipModel` | SigLIP | T / I | `google/siglip-base-patch16-224` | | |
778
+| `SiglipModel` | SigLIP, SigLIP2 | T / I | `google/siglip-base-patch16-224`, `google/siglip2-base-patch16-224` | | |
779
| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
780
781
<sup>C</sup> Automatically converted into an embedding model via `--convert embed`. ([details](./pooling_models.md#model-conversion))
tests/models/multimodal/pooling/test_siglip.py
+1
-1
@@ -19,7 +19,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
19
}
20
)
21
22
-MODELS = ["google/siglip-base-patch16-224"]
22
+MODELS = ["google/siglip-base-patch16-224", "google/siglip2-base-patch16-224"]
23
24
25
def _run_test(
vllm/model_executor/models/siglip.py
+5
-3
@@ -174,9 +174,11 @@ class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
174
@cached_property
175
def image_token_id(self) -> int:
176
tokenizer = self.info.get_tokenizer()
177
- dummy_token_id = 0
178
-
179
- assert dummy_token_id not in tokenizer.all_special_ids
177
+ dummy_token_id = next(
178
+ token_id
179
+ for token_id in range(tokenizer.vocab_size)
180
+ if token_id not in tokenizer.all_special_ids
181
+ )
182
183
return dummy_token_id
184
vllm/transformers_utils/config.py
+11
-1
@@ -26,7 +26,10 @@ from huggingface_hub.utils import (
26
)
27
from transformers import GenerationConfig, PretrainedConfig
28
from transformers.models.auto.image_processing_auto import get_image_processor_config
29
-from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
29
+from transformers.models.auto.modeling_auto import (
30
+ MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
31
+ MODEL_MAPPING_NAMES,
32
+)
33
from transformers.models.auto.tokenization_auto import get_tokenizer_config
34
from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
35
@@ -616,6 +619,13 @@ def get_config(
619
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
620
config.update({"architectures": [model_type]})
621
622
+ # Architecture mapping for models without explicit architectures field
623
+ if not config.architectures:
624
+ if config.model_type not in MODEL_MAPPING_NAMES:
625
+ raise ValueError(f"Cannot find architecture name for {config.model_type}")
626
+ model_type = MODEL_MAPPING_NAMES[config.model_type]
627
+ config.update({"architectures": [model_type]})
628
+
629
# ModelOpt 0.31.0 and after saves the quantization config in the model
630
# config file.
631
quantization_config = config_dict.get("quantization_config", None)