@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0552cfb195
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Model] Siglip Embedding Support (#27324)

Signed-off-by: piood <2477084691@qq.com>

Yu Jiaqi committed Oct 24, 2025 at 04:19 UTC 0552cfb195b24a490f253d485de11f2b94b0c6b8
8 files changed +870 -86
docs/models/supported_models.md
+7 -6
@@ -800,12 +800,13 @@ These models primarily support the [`LLM.embed`](./pooling_models.md#llmembed) A
800
801 The following table lists those that are tested in vLLM.
802
803 -| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
804 -|--------------|--------|--------|-------------------|----------------------|---------------------------|
805 -| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
806 -| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
807 -| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
808 -| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
803 +| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | [V1](gh-issue:8779) |
804 +|--------------|--------|--------|-------------------|----------------------|---------------------------|---------------------|
805 +| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | | ✅︎ |
806 +| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ | ✅︎ |
807 +| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ | ✅︎ |
808 +| `SiglipModel` | SigLIP | T / I | `google/siglip-base-patch16-224` | | | ✅︎ |
809 +| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* | \* |
810
811 <sup>C</sup> Automatically converted into an embedding model via `--convert embed`. ([details](./pooling_models.md#model-conversion))
812 \* Feature support is the same as that of the original model.
examples/offline_inference/vision_language_pooling.py
+49 -24
@@ -110,6 +110,53 @@ def run_e5_v(query: Query) -> ModelRequestData:
110 )
111
112
113 +def run_jinavl_reranker(query: Query) -> ModelRequestData:
114 + if query["modality"] != "text+images":
115 + raise ValueError(f"Unsupported query modality: '{query['modality']}'")
116 +
117 + engine_args = EngineArgs(
118 + model="jinaai/jina-reranker-m0",
119 + runner="pooling",
120 + max_model_len=32768,
121 + trust_remote_code=True,
122 + mm_processor_kwargs={
123 + "min_pixels": 3136,
124 + "max_pixels": 602112,
125 + },
126 + limit_mm_per_prompt={"image": 1},
127 + )
128 +
129 + return ModelRequestData(
130 + engine_args=engine_args,
131 + query=query["text"],
132 + documents=query["image"],
133 + )
134 +
135 +
136 +def run_siglip(query: Query) -> ModelRequestData:
137 + if query["modality"] == "text":
138 + prompt = query["text"]
139 + image = None
140 + elif query["modality"] == "image":
141 + prompt = "" # For image input, make sure that the prompt text is empty
142 + image = query["image"]
143 + else:
144 + modality = query["modality"]
145 + raise ValueError(f"Unsupported query modality: '{modality}'")
146 +
147 + engine_args = EngineArgs(
148 + model="google/siglip-base-patch16-224",
149 + runner="pooling",
150 + limit_mm_per_prompt={"image": 1},
151 + )
152 +
153 + return ModelRequestData(
154 + engine_args=engine_args,
155 + prompt=prompt,
156 + image=image,
157 + )
158 +
159 +
160 def _get_vlm2vec_prompt_image(query: Query, image_token: str):
161 if query["modality"] == "text":
162 text = query["text"]
@@ -211,29 +258,6 @@ def run_vlm2vec_qwen2vl(query: Query) -> ModelRequestData:
258 )
259
260
214 -def run_jinavl_reranker(query: Query) -> ModelRequestData:
215 - if query["modality"] != "text+images":
216 - raise ValueError(f"Unsupported query modality: '{query['modality']}'")
217 -
218 - engine_args = EngineArgs(
219 - model="jinaai/jina-reranker-m0",
220 - runner="pooling",
221 - max_model_len=32768,
222 - trust_remote_code=True,
223 - mm_processor_kwargs={
224 - "min_pixels": 3136,
225 - "max_pixels": 602112,
226 - },
227 - limit_mm_per_prompt={"image": 1},
228 - )
229 -
230 - return ModelRequestData(
231 - engine_args=engine_args,
232 - query=query["text"],
233 - documents=query["image"],
234 - )
235 -
236 -
261 def get_query(modality: QueryModality):
262 if modality == "text":
263 return TextQuery(modality="text", text="A dog sitting in the grass")
@@ -328,9 +352,10 @@ def run_score(model: str, modality: QueryModality, seed: int | None):
352 model_example_map = {
353 "clip": run_clip,
354 "e5_v": run_e5_v,
355 + "jinavl_reranker": run_jinavl_reranker,
356 + "siglip": run_siglip,
357 "vlm2vec_phi3v": run_vlm2vec_phi3v,
358 "vlm2vec_qwen2vl": run_vlm2vec_qwen2vl,
333 - "jinavl_reranker": run_jinavl_reranker,
359 }
360
361
examples/online_serving/pooling/openai_chat_embedding_client_for_multimodal.py
+78 -36
@@ -83,25 +83,29 @@ def run_clip(client: OpenAI, model: str):
83 print("Text embedding output:", response.data[0].embedding)
84
85
86 -def run_vlm2vec(client: OpenAI, model: str):
86 +def run_dse_qwen2_vl(client: OpenAI, model: str):
87 """
88 Start the server using:
89
90 - vllm serve TIGER-Lab/VLM2Vec-Full \
90 + vllm serve MrLight/dse-qwen2-2b-mrl-v1 \
91 --runner pooling \
92 --trust-remote-code \
93 - --max-model-len 4096 \
94 - --chat-template examples/template_vlm2vec_phi3v.jinja
93 + --max-model-len 8192 \
94 + --chat-template examples/template_dse_qwen2_vl.jinja
95 """
96 -
96 response = create_chat_embeddings(
97 client,
98 messages=[
99 {
100 "role": "user",
101 "content": [
103 - {"type": "image_url", "image_url": {"url": image_url}},
104 - {"type": "text", "text": "Represent the given image."},
102 + {
103 + "type": "image_url",
104 + "image_url": {
105 + "url": image_url,
106 + },
107 + },
108 + {"type": "text", "text": "What is shown in this image?"},
109 ],
110 }
111 ],
@@ -111,17 +115,26 @@ def run_vlm2vec(client: OpenAI, model: str):
115
116 print("Image embedding output:", response.data[0].embedding)
117
118 + # MrLight/dse-qwen2-2b-mrl-v1 requires a placeholder image
119 + # of the minimum input size
120 + buffer = io.BytesIO()
121 + image_placeholder = Image.new("RGB", (56, 56))
122 + image_placeholder.save(buffer, "png")
123 + buffer.seek(0)
124 + image_placeholder = base64.b64encode(buffer.read()).decode("utf-8")
125 response = create_chat_embeddings(
126 client,
127 messages=[
128 {
129 "role": "user",
130 "content": [
120 - {"type": "image_url", "image_url": {"url": image_url}},
131 {
122 - "type": "text",
123 - "text": "Represent the given image with the following question: What is in the image.",
132 + "type": "image_url",
133 + "image_url": {
134 + "url": f"data:image/jpeg;base64,{image_placeholder}",
135 + },
136 },
137 + {"type": "text", "text": "Query: What is the weather like today?"},
138 ],
139 }
140 ],
@@ -129,7 +142,16 @@ def run_vlm2vec(client: OpenAI, model: str):
142 encoding_format="float",
143 )
144
132 - print("Image+Text embedding output:", response.data[0].embedding)
145 + print("Text embedding output:", response.data[0].embedding)
146 +
147 +
148 +def run_siglip(client: OpenAI, model: str):
149 + """
150 + Start the server using:
151 +
152 + vllm serve google/siglip-base-patch16-224 \
153 + --runner pooling
154 + """
155
156 response = create_chat_embeddings(
157 client,
@@ -137,7 +159,23 @@ def run_vlm2vec(client: OpenAI, model: str):
159 {
160 "role": "user",
161 "content": [
140 - {"type": "text", "text": "A cat and a dog"},
162 + {"type": "image_url", "image_url": {"url": image_url}},
163 + ],
164 + }
165 + ],
166 + model=model,
167 + encoding_format="float",
168 + )
169 +
170 + print("Image embedding output:", response.data[0].embedding)
171 +
172 + response = create_chat_embeddings(
173 + client,
174 + messages=[
175 + {
176 + "role": "user",
177 + "content": [
178 + {"type": "text", "text": "a photo of a cat"},
179 ],
180 }
181 ],
@@ -148,29 +186,25 @@ def run_vlm2vec(client: OpenAI, model: str):
186 print("Text embedding output:", response.data[0].embedding)
187
188
151 -def run_dse_qwen2_vl(client: OpenAI, model: str):
189 +def run_vlm2vec(client: OpenAI, model: str):
190 """
191 Start the server using:
192
155 - vllm serve MrLight/dse-qwen2-2b-mrl-v1 \
193 + vllm serve TIGER-Lab/VLM2Vec-Full \
194 --runner pooling \
195 --trust-remote-code \
158 - --max-model-len 8192 \
159 - --chat-template examples/template_dse_qwen2_vl.jinja
196 + --max-model-len 4096 \
197 + --chat-template examples/template_vlm2vec_phi3v.jinja
198 """
199 +
200 response = create_chat_embeddings(
201 client,
202 messages=[
203 {
204 "role": "user",
205 "content": [
167 - {
168 - "type": "image_url",
169 - "image_url": {
170 - "url": image_url,
171 - },
172 - },
173 - {"type": "text", "text": "What is shown in this image?"},
206 + {"type": "image_url", "image_url": {"url": image_url}},
207 + {"type": "text", "text": "Represent the given image."},
208 ],
209 }
210 ],
@@ -180,26 +214,33 @@ def run_dse_qwen2_vl(client: OpenAI, model: str):
214
215 print("Image embedding output:", response.data[0].embedding)
216
183 - # MrLight/dse-qwen2-2b-mrl-v1 requires a placeholder image
184 - # of the minimum input size
185 - buffer = io.BytesIO()
186 - image_placeholder = Image.new("RGB", (56, 56))
187 - image_placeholder.save(buffer, "png")
188 - buffer.seek(0)
189 - image_placeholder = base64.b64encode(buffer.read()).decode("utf-8")
217 response = create_chat_embeddings(
218 client,
219 messages=[
220 {
221 "role": "user",
222 "content": [
223 + {"type": "image_url", "image_url": {"url": image_url}},
224 {
197 - "type": "image_url",
198 - "image_url": {
199 - "url": f"data:image/jpeg;base64,{image_placeholder}",
200 - },
225 + "type": "text",
226 + "text": "Represent the given image with the following question: What is in the image.",
227 },
202 - {"type": "text", "text": "Query: What is the weather like today?"},
228 + ],
229 + }
230 + ],
231 + model=model,
232 + encoding_format="float",
233 + )
234 +
235 + print("Image+Text embedding output:", response.data[0].embedding)
236 +
237 + response = create_chat_embeddings(
238 + client,
239 + messages=[
240 + {
241 + "role": "user",
242 + "content": [
243 + {"type": "text", "text": "A cat and a dog"},
244 ],
245 }
246 ],
@@ -212,8 +253,9 @@ def run_dse_qwen2_vl(client: OpenAI, model: str):
253
254 model_example_map = {
255 "clip": run_clip,
215 - "vlm2vec": run_vlm2vec,
256 "dse_qwen2_vl": run_dse_qwen2_vl,
257 + "siglip": run_siglip,
258 + "vlm2vec": run_vlm2vec,
259 }
260
261
tests/models/multimodal/pooling/test_siglip.py new
+137
@@ -0,0 +1,137 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +from transformers import SiglipModel
6 +
7 +from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
8 +from ...utils import check_embeddings_close
9 +
10 +HF_TEXT_PROMPTS = [
11 + "a photo of a stop sign",
12 + "a photo of a cherry blossom",
13 +]
14 +
15 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
16 + {
17 + "stop_sign": "",
18 + "cherry_blossom": "",
19 + }
20 +)
21 +
22 +MODELS = ["google/siglip-base-patch16-224"]
23 +
24 +
25 +def _run_test(
26 + hf_runner: type[HfRunner],
27 + vllm_runner: type[VllmRunner],
28 + input_texts: list[str],
29 + input_images: PromptImageInput,
30 + model: str,
31 + *,
32 + dtype: str,
33 +) -> None:
34 + with vllm_runner(
35 + model, runner="pooling", dtype=dtype, enforce_eager=True, max_model_len=64
36 + ) as vllm_model:
37 + vllm_outputs = vllm_model.embed(input_texts, images=input_images)
38 +
39 + with hf_runner(model, dtype=dtype, auto_cls=SiglipModel) as hf_model:
40 + all_inputs = hf_model.get_inputs(input_texts, images=input_images)
41 +
42 + all_outputs = []
43 + for inputs in all_inputs:
44 + inputs = hf_model.wrap_device(inputs)
45 +
46 + if "pixel_values" in inputs:
47 + pooled_output = hf_model.model.get_image_features(
48 + pixel_values=inputs.pixel_values,
49 + ).squeeze(0)
50 + else:
51 + pooled_output = hf_model.model.get_text_features(
52 + input_ids=inputs.input_ids,
53 + ).squeeze(0)
54 +
55 + all_outputs.append(pooled_output.tolist())
56 +
57 + hf_outputs = all_outputs
58 +
59 + check_embeddings_close(
60 + embeddings_0_lst=hf_outputs,
61 + embeddings_1_lst=vllm_outputs,
62 + name_0="hf",
63 + name_1="vllm",
64 + )
65 +
66 +
67 +@pytest.mark.parametrize("model", MODELS)
68 +@pytest.mark.parametrize("dtype", ["float"])
69 +def test_models_text(
70 + hf_runner,
71 + vllm_runner,
72 + image_assets,
73 + model: str,
74 + dtype: str,
75 +) -> None:
76 + input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
77 + input_texts = [text for text, _ in input_texts_images]
78 + input_images = [image for _, image in input_texts_images]
79 +
80 + _run_test(
81 + hf_runner,
82 + vllm_runner,
83 + input_texts,
84 + input_images, # type: ignore
85 + model,
86 + dtype=dtype,
87 + )
88 +
89 +
90 +@pytest.mark.parametrize("model", MODELS)
91 +@pytest.mark.parametrize("dtype", ["float"])
92 +def test_models_image(
93 + hf_runner,
94 + vllm_runner,
95 + image_assets,
96 + model: str,
97 + dtype: str,
98 +) -> None:
99 + input_texts_images = [
100 + (text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
101 + ]
102 + input_texts = [text for text, _ in input_texts_images]
103 + input_images = [image for _, image in input_texts_images]
104 +
105 + _run_test(
106 + hf_runner,
107 + vllm_runner,
108 + input_texts,
109 + input_images,
110 + model,
111 + dtype=dtype,
112 + )
113 +
114 +
115 +@pytest.mark.parametrize("model", MODELS)
116 +@pytest.mark.parametrize("dtype", ["float"])
117 +def test_models_text_image_no_crash(
118 + vllm_runner,
119 + image_assets,
120 + model: str,
121 + dtype: str,
122 +) -> None:
123 + texts = [HF_TEXT_PROMPTS[0]]
124 + images = [image_assets[0].pil_image]
125 +
126 + with vllm_runner(
127 + model,
128 + runner="pooling",
129 + dtype=dtype,
130 + enforce_eager=True,
131 + max_model_len=64,
132 + ) as vllm_model:
133 + with pytest.raises(ValueError, match="not both"):
134 + vllm_model.embed(texts, images=images)
135 +
136 + vllm_model.embed(texts)
137 + vllm_model.embed([""], images=images)
tests/models/registry.py
+1
@@ -471,6 +471,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
471 "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True
472 ),
473 "Qwen2VLForConditionalGeneration": _HfExamplesInfo("MrLight/dse-qwen2-2b-mrl-v1"),
474 + "SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"),
475 "PrithviGeoSpatialMAE": _HfExamplesInfo(
476 "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
477 dtype="float16",
vllm/model_executor/models/registry.py
+1
@@ -209,6 +209,7 @@ _EMBEDDING_MODELS = {
209 ),
210 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
211 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
212 + "SiglipModel": ("siglip", "SiglipEmbeddingModel"),
213 # Technically Terratorch models work on images, both in
214 # input and output. I am adding it here because it piggy-backs on embedding
215 # models for the time being.
vllm/model_executor/models/siglip.py
+594 -18
@@ -4,13 +4,23 @@
4 within a vision language model."""
5
6 import math
7 -from collections.abc import Iterable
7 +from collections.abc import Iterable, Mapping
8 +from functools import cached_property
9 +from typing import Annotated, Literal
10
11 import torch
12 from torch import nn
11 -from transformers import SiglipVisionConfig
13 +from transformers import (
14 + BatchFeature,
15 + SiglipConfig,
16 + SiglipProcessor,
17 + SiglipTextConfig,
18 + SiglipVisionConfig,
19 +)
20
21 from vllm.attention.layer import MultiHeadAttention
22 +from vllm.config import VllmConfig
23 +from vllm.config.multimodal import BaseDummyOptions
24 from vllm.distributed import divide, get_tensor_model_parallel_world_size
25 from vllm.model_executor.layers.activation import get_act_fn
26 from vllm.model_executor.layers.linear import (
@@ -18,20 +28,232 @@ from vllm.model_executor.layers.linear import (
28 QKVParallelLinear,
29 RowParallelLinear,
30 )
31 +from vllm.model_executor.layers.pooler import DispatchPooler, Pooler
32 from vllm.model_executor.layers.quantization import QuantizationConfig
33 from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
34 from vllm.model_executor.model_loader.weight_utils import (
35 default_weight_loader,
36 maybe_remap_kv_scale_name,
37 )
38 +from vllm.multimodal import MULTIMODAL_REGISTRY
39 +from vllm.multimodal.inputs import (
40 + MultiModalDataDict,
41 + MultiModalFieldConfig,
42 + MultiModalInputs,
43 + MultiModalKwargsItems,
44 + MultiModalUUIDDict,
45 +)
46 +from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
47 +from vllm.multimodal.processing import (
48 + BaseMultiModalProcessor,
49 + BaseProcessingInfo,
50 + PromptIndexTargets,
51 + PromptReplacement,
52 + PromptUpdate,
53 +)
54 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
55 +from vllm.sequence import IntermediateTensors
56 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
57
58 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
59 +from .interfaces_base import default_pooling_type
60 +from .utils import AutoWeightsLoader, maybe_prefix
61 from .vision import (
62 VisionEncoderInfo,
63 VisionFeatureSelectStrategy,
64 + VisionFeatureSelectStrategyStr,
65 + get_num_selected_vision_tokens,
66 resolve_visual_encoder_outputs,
67 )
68
69
70 +class SiglipImagePixelInputs(TensorSchema):
71 + """
72 + Dimensions:
73 + - bn: Batch size * number of images
74 + - c: Number of channels (3)
75 + - h: Height of each image
76 + - w: Width of each image
77 + """
78 +
79 + type: Literal["pixel_values"]
80 + data: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
81 +
82 +
83 +_POOLING_TYPE_TO_STRATEGY: dict[str, VisionFeatureSelectStrategyStr] = {
84 + "MEAN": "full",
85 + "ALL": "full",
86 + "CLS": "class",
87 +}
88 +
89 +
90 +def _get_vision_feature_select_strategy(
91 + pooling_type: str,
92 +) -> VisionFeatureSelectStrategyStr:
93 + try:
94 + return _POOLING_TYPE_TO_STRATEGY[pooling_type]
95 + except KeyError:
96 + raise ValueError(
97 + f"No feature selection strategy is defined for "
98 + f"pooling_type: {pooling_type!r}"
99 + ) from None
100 +
101 +
102 +class SiglipProcessingInfo(BaseProcessingInfo):
103 + def get_hf_config(self):
104 + return self.ctx.get_hf_config(SiglipConfig)
105 +
106 + def get_vision_encoder_info(self):
107 + return SiglipEncoderInfo(self.get_hf_config())
108 +
109 + def get_hf_processor(self, **kwargs: object):
110 + return self.ctx.get_hf_processor(SiglipProcessor, **kwargs)
111 +
112 + def get_supported_mm_limits(self) -> Mapping[str, int | None]:
113 + return {"image": 1}
114 +
115 + def get_num_image_tokens(
116 + self,
117 + *,
118 + image_width: int,
119 + image_height: int,
120 + ) -> int:
121 + vision_encoder_info = self.get_vision_encoder_info()
122 +
123 + pooler_config = self.ctx.model_config.pooler_config
124 + assert pooler_config is not None
125 +
126 + return get_num_selected_vision_tokens(
127 + vision_encoder_info.get_num_image_tokens(
128 + image_width=image_width,
129 + image_height=image_height,
130 + ),
131 + _get_vision_feature_select_strategy(pooler_config.pooling_type),
132 + )
133 +
134 + def get_image_size_with_most_features(self) -> ImageSize:
135 + vision_encoder_info = self.get_vision_encoder_info()
136 + width = height = vision_encoder_info.get_image_size()
137 + return ImageSize(width=width, height=height)
138 +
139 + def get_max_image_tokens(self) -> int:
140 + target_width, target_height = self.get_image_size_with_most_features()
141 +
142 + return self.get_num_image_tokens(
143 + image_width=target_width, image_height=target_height
144 + )
145 +
146 +
147 +class SiglipDummyInputsBuilder(BaseDummyInputsBuilder[SiglipProcessingInfo]):
148 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
149 + return ""
150 +
151 + def get_dummy_mm_data(
152 + self,
153 + seq_len: int,
154 + mm_counts: Mapping[str, int],
155 + mm_options: Mapping[str, BaseDummyOptions] | None = None,
156 + ) -> MultiModalDataDict:
157 + num_images = mm_counts.get("image", 0)
158 +
159 + target_width, target_height = self.info.get_image_size_with_most_features()
160 +
161 + image_overrides = mm_options.get("image") if mm_options else None
162 +
163 + return {
164 + "image": self._get_dummy_images(
165 + width=target_width,
166 + height=target_height,
167 + num_images=num_images,
168 + overrides=image_overrides,
169 + )
170 + }
171 +
172 +
173 +class SiglipMultiModalProcessor(BaseMultiModalProcessor[SiglipProcessingInfo]):
174 + @cached_property
175 + def image_token_id(self) -> int:
176 + tokenizer = self.info.get_tokenizer()
177 + dummy_token_id = 0
178 +
179 + assert dummy_token_id not in tokenizer.all_special_ids
180 +
181 + return dummy_token_id
182 +
183 + def apply(
184 + self,
185 + prompt: str | list[int],
186 + mm_data: MultiModalDataDict,
187 + hf_processor_mm_kwargs: Mapping[str, object],
188 + tokenization_kwargs: Mapping[str, object] | None = None,
189 + *,
190 + mm_uuids: MultiModalUUIDDict | None = None,
191 + ) -> MultiModalInputs:
192 + if prompt and mm_data:
193 + raise ValueError(
194 + "Siglip accepts text-only or image-only inputs, not both! "
195 + "Image-only inputs means passing an image with an empty text "
196 + "prompt."
197 + )
198 +
199 + if mm_data:
200 + # For multi-modal data, the prompt after processing should
201 + # only contain the image token
202 + tokenization_kwargs = {
203 + **(tokenization_kwargs or {}),
204 + "add_special_tokens": False,
205 + }
206 +
207 + return super().apply(
208 + prompt=prompt,
209 + mm_data=mm_data,
210 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
211 + tokenization_kwargs=tokenization_kwargs,
212 + mm_uuids=mm_uuids,
213 + )
214 +
215 + def _hf_processor_applies_updates(
216 + self,
217 + prompt_text: str,
218 + mm_items: MultiModalDataItems,
219 + hf_processor_mm_kwargs: Mapping[str, object],
220 + tokenization_kwargs: Mapping[str, object],
221 + ) -> bool:
222 + return False
223 +
224 + def _get_mm_fields_config(
225 + self,
226 + hf_inputs: BatchFeature,
227 + hf_processor_mm_kwargs: Mapping[str, object],
228 + ) -> Mapping[str, MultiModalFieldConfig]:
229 + return dict(pixel_values=MultiModalFieldConfig.batched("image"))
230 +
231 + def _get_prompt_updates(
232 + self,
233 + mm_items: MultiModalDataItems,
234 + hf_processor_mm_kwargs: Mapping[str, object],
235 + out_mm_kwargs: MultiModalKwargsItems,
236 + ) -> list[PromptUpdate]:
237 + image_token_id = self.image_token_id
238 +
239 + def get_replacement(item_idx: int):
240 + images = mm_items.get_items("image", ImageProcessorItems)
241 + image_size = images.get_image_size(item_idx)
242 +
243 + num_image_tokens = self.info.get_num_image_tokens(
244 + image_width=image_size.width, image_height=image_size.height
245 + )
246 + return [image_token_id] * num_image_tokens
247 +
248 + return [
249 + PromptReplacement(
250 + modality="image",
251 + target=PromptIndexTargets.start(),
252 + replacement=get_replacement,
253 + ),
254 + ]
255 +
256 +
257 class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
258 def get_num_image_tokens(
259 self,
@@ -151,8 +373,9 @@ class SiglipVisionEmbeddings(nn.Module):
373 class SiglipAttention(nn.Module):
374 def __init__(
375 self,
154 - config: SiglipVisionConfig,
376 + config: SiglipVisionConfig | SiglipTextConfig,
377 quant_config: QuantizationConfig | None = None,
378 + *,
379 prefix: str = "",
380 ) -> None:
381 super().__init__()
@@ -195,12 +418,29 @@ class SiglipAttention(nn.Module):
418 def forward(
419 self,
420 hidden_states: torch.Tensor,
198 - ) -> torch.Tensor:
421 + ) -> tuple[torch.Tensor, None]:
422 """Input shape: Batch x Time x Channel"""
423 qkv_states, _ = self.qkv_proj(hidden_states)
424 query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
425
426 + needs_unsqueeze = query_states.ndim == 2
427 + if needs_unsqueeze:
428 + query_states, key_states, value_states = (
429 + query_states.unsqueeze(0),
430 + key_states.unsqueeze(0),
431 + value_states.unsqueeze(0),
432 + )
433 +
434 out = self.attn(query_states, key_states, value_states)
435 +
436 + if needs_unsqueeze:
437 + out, query_states, key_states, value_states = (
438 + out.squeeze(0),
439 + query_states.squeeze(0),
440 + key_states.squeeze(0),
441 + value_states.squeeze(0),
442 + )
443 +
444 attn_output, _ = self.out_proj(out)
445
446 return attn_output, None
@@ -209,7 +449,7 @@ class SiglipAttention(nn.Module):
449 class SiglipMLP(nn.Module):
450 def __init__(
451 self,
212 - config: SiglipVisionConfig,
452 + config: SiglipVisionConfig | SiglipTextConfig,
453 quant_config: QuantizationConfig | None = None,
454 prefix: str = "",
455 ) -> None:
@@ -249,8 +489,9 @@ class SiglipMLP(nn.Module):
489 class SiglipEncoderLayer(nn.Module):
490 def __init__(
491 self,
252 - config: SiglipVisionConfig,
492 + config: SiglipVisionConfig | SiglipTextConfig,
493 quant_config: QuantizationConfig | None = None,
494 + *,
495 prefix: str = "",
496 ) -> None:
497 super().__init__()
@@ -291,9 +532,10 @@ class SiglipEncoderLayer(nn.Module):
532 class SiglipEncoder(nn.Module):
533 def __init__(
534 self,
294 - config: SiglipVisionConfig,
535 + config: SiglipVisionConfig | SiglipTextConfig,
536 quant_config: QuantizationConfig | None = None,
537 num_hidden_layers_override: int | None = None,
538 + *,
539 prefix: str = "",
540 ) -> None:
541 super().__init__()
@@ -335,6 +577,76 @@ class SiglipEncoder(nn.Module):
577 return hidden_states
578
579
580 +class SiglipTextTransformer(nn.Module):
581 + def __init__(
582 + self,
583 + config: SiglipTextConfig,
584 + quant_config: QuantizationConfig | None = None,
585 + *,
586 + prefix: str = "",
587 + ) -> None:
588 + super().__init__()
589 +
590 + self.config = config
591 + embed_dim = config.hidden_size
592 +
593 + self.embeddings = SiglipTextEmbeddings(config)
594 +
595 + self.encoder = SiglipEncoder(
596 + config=config,
597 + quant_config=quant_config,
598 + prefix=f"{prefix}.encoder",
599 + )
600 +
601 + self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
602 + self.head = nn.Linear(embed_dim, config.projection_size)
603 +
604 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
605 + return self.embeddings.token_embedding(input_ids)
606 +
607 + def forward(
608 + self,
609 + input_ids: torch.Tensor | None,
610 + position_ids: torch.Tensor,
611 + inputs_embeds: torch.Tensor | None = None,
612 + ) -> torch.Tensor:
613 + hidden_states = self.embeddings(input_ids, position_ids, inputs_embeds)
614 +
615 + last_hidden_state = self.encoder(
616 + inputs_embeds=hidden_states, return_all_hidden_states=False
617 + )
618 +
619 + last_hidden_state = self.final_layer_norm(last_hidden_state)
620 +
621 + return last_hidden_state
622 +
623 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
624 + stacked_params_mapping = [
625 + # (param_name, shard_name, shard_id)
626 + ("qkv_proj", "q_proj", "q"),
627 + ("qkv_proj", "k_proj", "k"),
628 + ("qkv_proj", "v_proj", "v"),
629 + ]
630 + params_dict = dict(self.named_parameters())
631 + loaded_params: set[str] = set()
632 +
633 + for name, loaded_weight in weights:
634 + for param_name, weight_name, shard_id in stacked_params_mapping:
635 + if weight_name not in name:
636 + continue
637 + name = name.replace(weight_name, param_name)
638 + param = params_dict[name]
639 + weight_loader = param.weight_loader
640 + weight_loader(param, loaded_weight, shard_id)
641 + break
642 + else:
643 + param = params_dict[name]
644 + weight_loader = getattr(param, "weight_loader", default_weight_loader)
645 + weight_loader(param, loaded_weight)
646 + loaded_params.add(name)
647 + return loaded_params
648 +
649 +
650 class SiglipMultiheadAttentionPoolingHead(nn.Module):
651 """Multihead Attention Pooling."""
652
@@ -357,8 +669,9 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
669 )
670
671 def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
360 - batch_size = hidden_state.shape[0]
361 - probe = self.probe.repeat(batch_size, 1, 1)
672 + batch_size = hidden_state.size(0)
673 +
674 + probe = self.probe.expand(batch_size, -1, -1)
675
676 hidden_state = self.attention(probe, hidden_state, hidden_state)[0]
677
@@ -367,7 +680,9 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
680 hidden_state = self.mlp(hidden_state)
681 hidden_state += residual
682
370 - return hidden_state[:, 0]
683 + pooled = hidden_state[:, 0]
684 +
685 + return pooled.unsqueeze(1)
686
687
688 class SiglipVisionTransformer(nn.Module):
@@ -420,6 +735,14 @@ class SiglipVisionTransformer(nn.Module):
735 prefix=f"{prefix}.head",
736 )
737
738 + @property
739 + def dtype(self):
740 + return next(self.parameters()).dtype
741 +
742 + @property
743 + def device(self):
744 + return next(self.parameters()).device
745 +
746 def forward(
747 self,
748 pixel_values: torch.Tensor,
@@ -432,7 +755,6 @@ class SiglipVisionTransformer(nn.Module):
755 pixel_values,
756 interpolate_pos_encoding=interpolate_pos_encoding,
757 )
435 -
758 # Produces either the last layer output or all of the hidden states,
759 # depending on if we have select_layers or not
760 encoder_outputs = self.encoder(
@@ -440,21 +762,60 @@ class SiglipVisionTransformer(nn.Module):
762 return_all_hidden_states=select_layers is not None,
763 )
764
443 - # Handle post-norm (if applicable) and stacks feature layers if needed
765 + if self.post_layernorm is not None:
766 + encoder_outputs = self.post_layernorm(encoder_outputs)
767 +
768 + if self.use_head:
769 + encoder_outputs = self.head(encoder_outputs)
770 +
771 + # stacks feature layers if needed
772 encoder_outputs = resolve_visual_encoder_outputs(
773 encoder_outputs,
446 - self.post_layernorm,
774 + None,
775 select_layers=select_layers,
776 max_possible_layers=self.config.num_hidden_layers,
777 feature_select_strategy=feature_select_strategy,
778 )
779
452 - # TODO: add this back when pooled_output is used in inference.
453 - # if self.use_head:
454 - # pooled_output = self.head(encoder_outputs)
455 -
780 return encoder_outputs
781
782 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
783 + stacked_params_mapping = [
784 + # (param_name, shard_name, shard_id)
785 + ("qkv_proj", "q_proj", "q"),
786 + ("qkv_proj", "k_proj", "k"),
787 + ("qkv_proj", "v_proj", "v"),
788 + ]
789 + params_dict = dict(self.named_parameters())
790 + loaded_params: set[str] = set()
791 + layer_count = len(self.encoder.layers)
792 +
793 + for name, loaded_weight in weights:
794 + # post_layernorm is not needed in SiglipVisionTransformer
795 + if name.startswith("post_layernorm") and self.post_layernorm is None:
796 + continue
797 +
798 + # omit layers when num_hidden_layers_override is set
799 + if name.startswith("encoder.layers"):
800 + layer_idx = int(name.split(".")[2])
801 + if layer_idx >= layer_count:
802 + continue
803 +
804 + for param_name, weight_name, shard_id in stacked_params_mapping:
805 + if weight_name not in name:
806 + continue
807 + name = name.replace(weight_name, param_name)
808 + param = params_dict[name]
809 + weight_loader = param.weight_loader
810 + weight_loader(param, loaded_weight, shard_id)
811 + break
812 + else:
813 + param = params_dict[name]
814 + weight_loader = getattr(param, "weight_loader", default_weight_loader)
815 + weight_loader(param, loaded_weight)
816 + loaded_params.add(name)
817 + return loaded_params
818 +
819
820 class SiglipVisionModel(nn.Module):
821 config_class = SiglipVisionConfig
@@ -484,7 +845,11 @@ class SiglipVisionModel(nn.Module):
845
846 @property
847 def dtype(self):
487 - return self.get_input_embeddings().weight.dtype
848 + return self.vision_model.dtype
849 +
850 + @property
851 + def device(self):
852 + return self.vision_model.device
853
854 def forward(
855 self,
@@ -555,3 +920,214 @@ class SiglipVisionModel(nn.Module):
920 weight_loader(param, loaded_weight)
921 loaded_params.add(name)
922 return loaded_params
923 +
924 +
925 +# Adapted from: https://github.com/huggingface/transformers/blob/v4.54.1/src/transformers/models/siglip/modeling_siglip.py#L200
926 +class SiglipTextEmbeddings(nn.Module):
927 + def __init__(self, config: SiglipTextConfig):
928 + super().__init__()
929 + self.config = config
930 +
931 + self.token_embedding = VocabParallelEmbedding(
932 + config.vocab_size, config.hidden_size
933 + )
934 +
935 + self.position_embedding = VocabParallelEmbedding(
936 + config.max_position_embeddings, config.hidden_size
937 + )
938 +
939 + self.register_buffer(
940 + "position_ids",
941 + torch.arange(config.max_position_embeddings).expand((1, -1)),
942 + persistent=False,
943 + )
944 +
945 + def forward(
946 + self,
947 + input_ids: torch.Tensor | None,
948 + position_ids: torch.Tensor,
949 + inputs_embeds: torch.Tensor | None = None,
950 + ) -> torch.Tensor:
951 + if inputs_embeds is None:
952 + inputs_embeds = self.token_embedding(input_ids)
953 +
954 + position_embeddings = self.position_embedding(position_ids)
955 + embeddings = inputs_embeds + position_embeddings
956 + return embeddings
957 +
958 +
959 +# Assume EOS token corresponds to CLS token in text model
960 +@default_pooling_type("CLS")
961 +@MULTIMODAL_REGISTRY.register_processor(
962 + SiglipMultiModalProcessor,
963 + info=SiglipProcessingInfo,
964 + dummy_inputs=SiglipDummyInputsBuilder,
965 +)
966 +class SiglipEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
967 + is_pooling_model = True
968 +
969 + packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
970 + merge_by_field_config = True
971 +
972 + @classmethod
973 + def get_placeholder_str(cls, modality: str, i: int) -> str | None:
974 + if modality.startswith("image"):
975 + return None
976 +
977 + raise ValueError("Only image modality is supported")
978 +
979 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
980 + super().__init__()
981 +
982 + config: SiglipConfig = vllm_config.model_config.hf_config
983 + quant_config = vllm_config.quant_config
984 + multimodal_config = vllm_config.model_config.multimodal_config
985 + self.config = config
986 + self.multimodal_config = multimodal_config
987 +
988 + if hasattr(config, "num_labels"):
989 + config.num_labels = 0
990 +
991 + text_config = config.text_config
992 + vision_config = config.vision_config
993 +
994 + self.text_embed_dim = text_config.hidden_size
995 + self.vision_embed_dim = vision_config.hidden_size
996 +
997 + self.text_model = SiglipTextTransformer(
998 + text_config,
999 + quant_config=quant_config,
1000 + prefix=maybe_prefix(prefix, "text_model"),
1001 + )
1002 + self.vision_model = SiglipVisionTransformer(
1003 + vision_config,
1004 + quant_config=quant_config,
1005 + prefix=maybe_prefix(prefix, "vision_model"),
1006 + )
1007 +
1008 + self.text_projection_size = text_config.projection_size
1009 +
1010 + pooler_config = vllm_config.model_config.pooler_config
1011 + assert pooler_config is not None
1012 + self.pooler_config = pooler_config
1013 +
1014 + self.pooler = DispatchPooler(
1015 + {
1016 + "token_embed": Pooler.for_token_embed(pooler_config),
1017 + "embed": Pooler.for_embed(pooler_config),
1018 + }
1019 + )
1020 +
1021 + self._is_text_input = True
1022 +
1023 + def get_text_features(
1024 + self,
1025 + input_ids: torch.Tensor | None,
1026 + position_ids: torch.Tensor,
1027 + inputs_embeds: torch.Tensor | None = None,
1028 + ) -> torch.Tensor:
1029 + last_hidden_state = self.text_model(
1030 + input_ids=input_ids,
1031 + position_ids=position_ids,
1032 + inputs_embeds=inputs_embeds,
1033 + )
1034 + text_features = self.text_model.head(last_hidden_state)
1035 + # Flip to extract CLS token (first token after reversal) for pooling
1036 + text_features = text_features.flip(0)
1037 + return text_features
1038 +
1039 + def get_image_features(
1040 + self,
1041 + pixel_values: torch.Tensor,
1042 + feature_select_strategy: VisionFeatureSelectStrategy | None = None,
1043 + ) -> torch.Tensor:
1044 + if feature_select_strategy is None:
1045 + feature_select_strategy = _get_vision_feature_select_strategy(
1046 + self.pooler_config.pooling_type
1047 + )
1048 +
1049 + pooled_output = self.vision_model(
1050 + pixel_values=pixel_values,
1051 + select_layers=None,
1052 + feature_select_strategy=feature_select_strategy,
1053 + )
1054 +
1055 + return pooled_output
1056 +
1057 + def _parse_and_validate_image_input(
1058 + self, **kwargs: object
1059 + ) -> SiglipImagePixelInputs | None:
1060 + pixel_values = kwargs.pop("pixel_values", None)
1061 + if pixel_values is None:
1062 + return None
1063 +
1064 + expected_h = expected_w = self.config.vision_config.image_size
1065 + return SiglipImagePixelInputs(
1066 + type="pixel_values",
1067 + data=pixel_values,
1068 + resolve_bindings={"h": expected_h, "w": expected_w},
1069 + )
1070 +
1071 + def _process_image_inputs(self, inputs: SiglipImagePixelInputs) -> torch.Tensor:
1072 + pixel_values = inputs["data"]
1073 +
1074 + return self.get_image_features(pixel_values)
1075 +
1076 + def get_language_model(self) -> torch.nn.Module:
1077 + return self.text_model
1078 +
1079 + def get_input_embeddings(
1080 + self,
1081 + input_ids: torch.Tensor,
1082 + multimodal_embeddings: MultiModalEmbeddings | None = None,
1083 + *,
1084 + is_multimodal: torch.Tensor | None = None,
1085 + handle_oov_mm_token: bool = False,
1086 + ) -> torch.Tensor:
1087 + self._is_text_input = (
1088 + multimodal_embeddings is None or len(multimodal_embeddings) == 0
1089 + )
1090 +
1091 + if multimodal_embeddings is None or is_multimodal is None:
1092 + return super().get_input_embeddings(input_ids)
1093 +
1094 + return super().get_input_embeddings(
1095 + input_ids,
1096 + multimodal_embeddings=multimodal_embeddings,
1097 + is_multimodal=is_multimodal,
1098 + handle_oov_mm_token=handle_oov_mm_token,
1099 + )
1100 +
1101 + def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
1102 + image_input = self._parse_and_validate_image_input(**kwargs)
1103 + if image_input is None:
1104 + return []
1105 +
1106 + vision_embeddings = self._process_image_inputs(image_input)
1107 + return vision_embeddings
1108 +
1109 + def forward(
1110 + self,
1111 + input_ids: torch.Tensor | None,
1112 + positions: torch.Tensor,
1113 + intermediate_tensors: IntermediateTensors | None = None,
1114 + inputs_embeds: torch.Tensor | None = None,
1115 + **kwargs: object,
1116 + ) -> torch.Tensor:
1117 + if intermediate_tensors is not None:
1118 + raise RuntimeError("PP is not supported for this model")
1119 +
1120 + # Multimodal inputs (image embeddings)
1121 + if not self._is_text_input:
1122 + return inputs_embeds
1123 +
1124 + return self.get_text_features(input_ids, positions, inputs_embeds)
1125 +
1126 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
1127 + loader = AutoWeightsLoader(
1128 + self,
1129 + skip_substrs=[".position_ids"],
1130 + ignore_unexpected_prefixes=["logit_scale.", "logit_bias."],
1131 + )
1132 +
1133 + return loader.load_weights(weights)
vllm/transformers_utils/chat_templates/registry.py
+3 -2
@@ -31,14 +31,15 @@ def _get_minicpmv_chat_template_fallback(tokenizer_name_or_path: str) -> Path |
31
32 _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
33 "blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
34 - "clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
34 "chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
36 - "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
35 + "clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
36 "deepseek_ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
37 + "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
38 "fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
39 "minicpmv": _get_minicpmv_chat_template_fallback,
40 "paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
41 "qwen": _get_qwen_chat_template_fallback,
42 + "siglip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
43 }
44
45