@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4570535ec4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Model] CLIP Embedding Support (#26010)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Oct 4, 2025 at 21:21 UTC 4570535ec41e9e6f808d4cd3a9a06c6928652dea
11 files changed +850 -78
docs/models/supported_models.md
+1
@@ -829,6 +829,7 @@ The following table lists those that are tested in vLLM.
829
830 | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | [V1](gh-issue:8779) |
831 |--------------|--------|--------|-------------------|----------------------|---------------------------|---------------------|
832 +| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | | ✅︎ |
833 | `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ | ✅︎ |
834 | `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ | ✅︎ |
835 | `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* | \* |
examples/offline_inference/vision_language_pooling.py
+31 -3
@@ -58,6 +58,30 @@ class ModelRequestData(NamedTuple):
58 documents: Optional[ScoreMultiModalParam] = None
59
60
61 +def run_clip(query: Query) -> ModelRequestData:
62 + if query["modality"] == "text":
63 + prompt = query["text"]
64 + image = None
65 + elif query["modality"] == "image":
66 + prompt = "" # For image input, make sure that the prompt text is empty
67 + image = query["image"]
68 + else:
69 + modality = query["modality"]
70 + raise ValueError(f"Unsupported query modality: '{modality}'")
71 +
72 + engine_args = EngineArgs(
73 + model="openai/clip-vit-base-patch32",
74 + runner="pooling",
75 + limit_mm_per_prompt={"image": 1},
76 + )
77 +
78 + return ModelRequestData(
79 + engine_args=engine_args,
80 + prompt=prompt,
81 + image=image,
82 + )
83 +
84 +
85 def run_e5_v(query: Query) -> ModelRequestData:
86 llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n" # noqa: E501
87
@@ -146,7 +170,8 @@ def run_vlm2vec_qwen2vl(query: Query) -> ModelRequestData:
170
171 processor = AutoProcessor.from_pretrained(
172 model_id,
149 - # `min_pixels` and `max_pixels` are deprecated
173 + # `min_pixels` and `max_pixels` are deprecated for
174 + # transformers `preprocessor_config.json`
175 size={"shortest_edge": 3136, "longest_edge": 12845056},
176 )
177 processor.chat_template = load_chat_template(
@@ -172,8 +197,10 @@ def run_vlm2vec_qwen2vl(query: Query) -> ModelRequestData:
197 model=merged_path,
198 runner="pooling",
199 max_model_len=4096,
175 - trust_remote_code=True,
176 - mm_processor_kwargs={"num_crops": 4},
200 + mm_processor_kwargs={
201 + "min_pixels": 3136,
202 + "max_pixels": 12845056,
203 + },
204 limit_mm_per_prompt={"image": 1},
205 )
206
@@ -299,6 +326,7 @@ def run_score(model: str, modality: QueryModality, seed: Optional[int]):
326
327
328 model_example_map = {
329 + "clip": run_clip,
330 "e5_v": run_e5_v,
331 "vlm2vec_phi3v": run_vlm2vec_phi3v,
332 "vlm2vec_qwen2vl": run_vlm2vec_qwen2vl,
examples/online_serving/pooling/openai_chat_embedding_client_for_multimodal.py
+64 -8
@@ -1,14 +1,9 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 # ruff: noqa: E501
4 -"""Example Python client for multimodal embedding API using vLLM API server
5 -NOTE:
6 - start a supported multimodal embeddings model server with `vllm serve`, e.g.
7 - vllm serve TIGER-Lab/VLM2Vec-Full \
8 - --runner pooling \
9 - --trust-remote-code \
10 - --max-model-len 4096 \
11 - --chat-template examples/template_vlm2vec_phi3v.jinja
4 +"""Example Python client for multimodal embedding API using vLLM API server.
5 +
6 +Refer to each `run_*` function for the command to run the server for that model.
7 """
8
9 import argparse
@@ -47,7 +42,58 @@ def create_chat_embeddings(
42 )
43
44
45 +def run_clip(client: OpenAI, model: str):
46 + """
47 + Start the server using:
48 +
49 + vllm serve openai/clip-vit-base-patch32 \
50 + --runner pooling
51 + """
52 +
53 + response = create_chat_embeddings(
54 + client,
55 + messages=[
56 + {
57 + "role": "user",
58 + "content": [
59 + {"type": "image_url", "image_url": {"url": image_url}},
60 + ],
61 + }
62 + ],
63 + model=model,
64 + encoding_format="float",
65 + )
66 +
67 + print("Image embedding output:", response.data[0].embedding)
68 +
69 + response = create_chat_embeddings(
70 + client,
71 + messages=[
72 + {
73 + "role": "user",
74 + "content": [
75 + {"type": "text", "text": "a photo of a cat"},
76 + ],
77 + }
78 + ],
79 + model=model,
80 + encoding_format="float",
81 + )
82 +
83 + print("Text embedding output:", response.data[0].embedding)
84 +
85 +
86 def run_vlm2vec(client: OpenAI, model: str):
87 + """
88 + Start the server using:
89 +
90 + vllm serve TIGER-Lab/VLM2Vec-Full \
91 + --runner pooling \
92 + --trust-remote-code \
93 + --max-model-len 4096 \
94 + --chat-template examples/template_vlm2vec_phi3v.jinja
95 + """
96 +
97 response = create_chat_embeddings(
98 client,
99 messages=[
@@ -103,6 +149,15 @@ def run_vlm2vec(client: OpenAI, model: str):
149
150
151 def run_dse_qwen2_vl(client: OpenAI, model: str):
152 + """
153 + Start the server using:
154 +
155 + vllm serve MrLight/dse-qwen2-2b-mrl-v1 \
156 + --runner pooling \
157 + --trust-remote-code \
158 + --max-model-len 8192 \
159 + --chat-template examples/template_dse_qwen2_vl.jinja
160 + """
161 response = create_chat_embeddings(
162 client,
163 messages=[
@@ -156,6 +211,7 @@ def run_dse_qwen2_vl(client: OpenAI, model: str):
211
212
213 model_example_map = {
214 + "clip": run_clip,
215 "vlm2vec": run_vlm2vec,
216 "dse_qwen2_vl": run_dse_qwen2_vl,
217 }
tests/models/multimodal/pooling/test_clip.py new
+138
@@ -0,0 +1,138 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +from transformers import CLIPModel
6 +
7 +from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
8 +from ...utils import check_embeddings_close
9 +
10 +HF_TEXT_PROMPTS = [
11 + "a photo of a stop sign",
12 + "a photo of a cherry blossom",
13 +]
14 +
15 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
16 + "stop_sign": "",
17 + "cherry_blossom": "",
18 +})
19 +
20 +MODELS = ["openai/clip-vit-base-patch32"]
21 +
22 +
23 +def _run_test(
24 + hf_runner: type[HfRunner],
25 + vllm_runner: type[VllmRunner],
26 + input_texts: list[str],
27 + input_images: PromptImageInput,
28 + model: str,
29 + *,
30 + dtype: str,
31 +) -> None:
32 + # NOTE: take care of the order. run vLLM first, and then run HF.
33 + # vLLM needs a fresh new process without cuda initialization.
34 + # if we run HF first, the cuda initialization will be done and it
35 + # will hurt multiprocessing backend with fork method (the default method).
36 + with vllm_runner(model,
37 + runner="pooling",
38 + dtype=dtype,
39 + enforce_eager=True,
40 + max_model_len=77) as vllm_model:
41 + vllm_outputs = vllm_model.embed(input_texts, images=input_images)
42 +
43 + with hf_runner(model, dtype=dtype, auto_cls=CLIPModel) as hf_model:
44 + all_inputs = hf_model.get_inputs(input_texts, images=input_images)
45 +
46 + all_outputs = []
47 + for inputs in all_inputs:
48 + if "pixel_values" in inputs:
49 + inputs.pop("input_ids")
50 + pooled_output = hf_model.model.get_image_features(
51 + **hf_model.wrap_device(inputs)).squeeze(0)
52 + else:
53 + pooled_output = hf_model.model.get_text_features(
54 + **hf_model.wrap_device(inputs)).squeeze(0)
55 +
56 + all_outputs.append(pooled_output.tolist())
57 +
58 + hf_outputs = all_outputs
59 +
60 + check_embeddings_close(
61 + embeddings_0_lst=hf_outputs,
62 + embeddings_1_lst=vllm_outputs,
63 + name_0="hf",
64 + name_1="vllm",
65 + )
66 +
67 +
68 +@pytest.mark.parametrize("model", MODELS)
69 +@pytest.mark.parametrize("dtype", ["float"])
70 +def test_models_text(
71 + hf_runner,
72 + vllm_runner,
73 + image_assets,
74 + model: str,
75 + dtype: str,
76 +) -> None:
77 + input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
78 + input_texts = [text for text, _ in input_texts_images]
79 + input_images = [image for _, image in input_texts_images]
80 +
81 + _run_test(
82 + hf_runner,
83 + vllm_runner,
84 + input_texts,
85 + input_images, # type: ignore
86 + model,
87 + dtype=dtype,
88 + )
89 +
90 +
91 +@pytest.mark.parametrize("model", MODELS)
92 +@pytest.mark.parametrize("dtype", ["float"])
93 +def test_models_image(
94 + hf_runner,
95 + vllm_runner,
96 + image_assets,
97 + model: str,
98 + dtype: str,
99 +) -> None:
100 + input_texts_images = [
101 + (text, asset.pil_image)
102 + for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
103 + ]
104 + input_texts = [text for text, _ in input_texts_images]
105 + input_images = [image for _, image in input_texts_images]
106 +
107 + _run_test(
108 + hf_runner,
109 + vllm_runner,
110 + input_texts,
111 + input_images,
112 + model,
113 + dtype=dtype,
114 + )
115 +
116 +
117 +@pytest.mark.parametrize("model", MODELS)
118 +@pytest.mark.parametrize("dtype", ["float"])
119 +def test_models_text_image_no_crash(
120 + vllm_runner,
121 + image_assets,
122 + model: str,
123 + dtype: str,
124 +) -> None:
125 + texts = [HF_TEXT_PROMPTS[0]]
126 + images = [image_assets[0].pil_image]
127 +
128 + with vllm_runner(model,
129 + runner="pooling",
130 + dtype=dtype,
131 + enforce_eager=True,
132 + max_model_len=77) as vllm_model:
133 + with pytest.raises(ValueError, match="not both"):
134 + vllm_model.embed(texts, images=images)
135 +
136 + # Should still be able to run subsequent requests
137 + vllm_model.embed(texts)
138 + vllm_model.embed([""], images=images)
tests/models/registry.py
+8 -2
@@ -389,6 +389,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
389 "RobertaForMaskedLM": _HfExamplesInfo("sentence-transformers/all-roberta-large-v1"), # noqa: E501
390 "XLMRobertaModel": _HfExamplesInfo("intfloat/multilingual-e5-small"), # noqa: E501
391 # [Multimodal]
392 + "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"),
393 "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"),
394 "Phi3VForCausalLM": _HfExamplesInfo("TIGER-Lab/VLM2Vec-Full",
395 trust_remote_code=True),
@@ -687,7 +688,11 @@ class HfExampleModels:
688 return self.hf_models.keys()
689
690 def get_hf_info(self, model_arch: str) -> _HfExamplesInfo:
690 - return self.hf_models[model_arch]
691 + try:
692 + return self.hf_models[model_arch]
693 + except KeyError:
694 + raise ValueError(f"No example model defined for {model_arch}; "
695 + f"please update this file.") from None
696
697 def find_hf_info(self, model_id: str) -> _HfExamplesInfo:
698 for info in self.hf_models.values():
@@ -699,7 +704,8 @@ class HfExampleModels:
704 if any(extra == model_id for extra in info.extras.values()):
705 return info
706
702 - raise ValueError(f"No example model defined for {model_id}")
707 + raise ValueError(f"No example model defined for {model_id}; "
708 + f"please update this file.")
709
710
711 HF_EXAMPLE_MODELS = HfExampleModels(_EXAMPLE_MODELS)
vllm/attention/layer.py
+5 -1
@@ -417,12 +417,16 @@ class MultiHeadAttention(nn.Module):
417 head_size: int,
418 scale: float,
419 num_kv_heads: Optional[int] = None,
420 - ):
420 + # This has no effect, it is only here to make it easier to swap
421 + # between Attention and MultiHeadAttention
422 + prefix: str = "",
423 + ) -> None:
424 super().__init__()
425 self.num_heads = num_heads
426 self.head_size = head_size
427 self.scale = scale
428 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
429 + self.layer_name = prefix
430
431 assert self.num_heads % self.num_kv_heads == 0, \
432 f"num_heads ({self.num_heads}) is not " \
vllm/model_executor/models/bert.py
+1 -1
@@ -351,7 +351,7 @@ class BertModel(nn.Module, SupportsQuant):
351 prefix=f"{prefix}.encoder")
352
353 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
354 - return self.embeddings(input_ids)
354 + return self.embeddings.word_embeddings(input_ids)
355
356 def forward(
357 self,
vllm/model_executor/models/clip.py
+596 -61
@@ -1,28 +1,63 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Minimal implementation of CLIPVisionModel intended to be only used
4 -within a vision language model."""
5 -from collections.abc import Iterable
6 -from typing import Optional, Union
3 +from collections.abc import Iterable, Mapping, Sequence
4 +from functools import cached_property
5 +from typing import Annotated, Literal, Optional, Union
6
7 import torch
8 import torch.nn as nn
10 -from transformers import CLIPVisionConfig
9 +from transformers import (BatchFeature, CLIPConfig, CLIPProcessor,
10 + CLIPTextConfig, CLIPVisionConfig)
11
12 +from vllm.attention import Attention
13 from vllm.attention.layer import MultiHeadAttention
14 +from vllm.config import VllmConfig
15 +from vllm.config.multimodal import BaseDummyOptions
16 from vllm.distributed import divide, get_tensor_model_parallel_world_size
17 from vllm.model_executor.layers.activation import get_act_fn
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 QKVParallelLinear,
20 RowParallelLinear)
21 +from vllm.model_executor.layers.pooler import DispatchPooler, Pooler
22 from vllm.model_executor.layers.quantization import QuantizationConfig
23 +from vllm.model_executor.layers.vocab_parallel_embedding import (
24 + VocabParallelEmbedding)
25 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26 from vllm.model_executor.models.interfaces import SupportsQuant
21 -
27 +from vllm.multimodal import MULTIMODAL_REGISTRY
28 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
29 + MultiModalInputs, MultiModalKwargsItems,
30 + MultiModalUUIDDict)
31 +from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
32 + MultiModalDataItems)
33 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 + BaseProcessingInfo, PromptIndexTargets,
35 + PromptReplacement, PromptUpdate)
36 +from vllm.multimodal.profiling import BaseDummyInputsBuilder
37 +from vllm.sequence import IntermediateTensors
38 +from vllm.utils.tensor_schema import TensorSchema, TensorShape
39 +
40 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal
41 +from .interfaces_base import default_pooling_type
42 +from .utils import AutoWeightsLoader, maybe_prefix
43 from .vision import (VisionEncoderInfo, VisionFeatureSelectStrategy,
44 + VisionFeatureSelectStrategyStr,
45 + get_num_selected_vision_tokens,
46 resolve_visual_encoder_outputs)
47
48
49 +class CLIPImagePixelInputs(TensorSchema):
50 + """
51 + Dimensions:
52 + - bn: Batch size * number of images
53 + - c: Number of channels (3)
54 + - h: Height of each image
55 + - w: Width of each image
56 + """
57 + type: Literal["pixel_values"]
58 + data: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
59 +
60 +
61 class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
62
63 def get_num_image_tokens(
@@ -45,7 +80,214 @@ class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
80 return image_size // patch_size
81
82
48 -# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
83 +_POOLING_TYPE_TO_STRATEGY: dict[str, VisionFeatureSelectStrategyStr] = {
84 + "MEAN": "full",
85 + "ALL": "full",
86 + "CLS": "class",
87 + # This lets us use the same pooling type for both text and image
88 + "LAST": "class",
89 +}
90 +
91 +
92 +def _get_vision_feature_select_strategy(pooling_type: str):
93 + try:
94 + return _POOLING_TYPE_TO_STRATEGY[pooling_type]
95 + except KeyError:
96 + raise ValueError(f"No feature selection strategy is defined for "
97 + f"pooling_type: {pooling_type!r}") from None
98 +
99 +
100 +class CLIPProcessingInfo(BaseProcessingInfo):
101 +
102 + def get_hf_config(self):
103 + return self.ctx.get_hf_config(CLIPConfig)
104 +
105 + def get_vision_encoder_info(self):
106 + return CLIPEncoderInfo(self.get_hf_config())
107 +
108 + def get_hf_processor(self, **kwargs: object):
109 + return self.ctx.get_hf_processor(CLIPProcessor, **kwargs)
110 +
111 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
112 + return {"image": 1}
113 +
114 + def get_num_image_tokens(
115 + self,
116 + *,
117 + image_width: int,
118 + image_height: int,
119 + ) -> int:
120 + vision_encoder_info = self.get_vision_encoder_info()
121 +
122 + pooler_config = self.ctx.model_config.pooler_config
123 + assert pooler_config is not None
124 +
125 + return get_num_selected_vision_tokens(
126 + vision_encoder_info.get_num_image_tokens(
127 + image_width=image_width,
128 + image_height=image_height,
129 + ),
130 + _get_vision_feature_select_strategy(pooler_config.pooling_type),
131 + )
132 +
133 + def get_image_size_with_most_features(self) -> ImageSize:
134 + vision_encoder_info = self.get_vision_encoder_info()
135 + width = height = vision_encoder_info.get_image_size()
136 + return ImageSize(width=width, height=height)
137 +
138 + def get_max_image_tokens(self) -> int:
139 + target_width, target_height = self.get_image_size_with_most_features()
140 +
141 + return self.get_num_image_tokens(
142 + image_width=target_width,
143 + image_height=target_height,
144 + )
145 +
146 +
147 +class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]):
148 +
149 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
150 + return ""
151 +
152 + def get_dummy_mm_data(
153 + self,
154 + seq_len: int,
155 + mm_counts: Mapping[str, int],
156 + mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
157 + ) -> MultiModalDataDict:
158 + num_images = mm_counts.get("image", 0)
159 +
160 + target_width, target_height = \
161 + self.info.get_image_size_with_most_features()
162 +
163 + image_overrides = mm_options.get("image") if mm_options else None
164 +
165 + return {
166 + "image":
167 + self._get_dummy_images(width=target_width,
168 + height=target_height,
169 + num_images=num_images,
170 + overrides=image_overrides)
171 + }
172 +
173 +
174 +class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
175 +
176 + @cached_property
177 + def image_token_id(self) -> int:
178 + tokenizer = self.info.get_tokenizer()
179 + dummy_token_id = 0
180 +
181 + assert dummy_token_id not in tokenizer.all_special_ids
182 +
183 + return dummy_token_id
184 +
185 + def apply(
186 + self,
187 + prompt: Union[str, list[int]],
188 + mm_data: MultiModalDataDict,
189 + hf_processor_mm_kwargs: Mapping[str, object],
190 + tokenization_kwargs: Optional[Mapping[str, object]] = None,
191 + *,
192 + mm_uuids: Optional[MultiModalUUIDDict] = None,
193 + ) -> MultiModalInputs:
194 + if prompt and mm_data:
195 + raise ValueError(
196 + "CLIP accepts text-only or image-only inputs, not both! "
197 + "Image-only inputs means passing an image with an empty text "
198 + "prompt.")
199 +
200 + if mm_data:
201 + # For multi-modal data, the prompt after processing should
202 + # only contain the dummy image tokens
203 + tokenization_kwargs = {
204 + **(tokenization_kwargs or {}),
205 + "add_special_tokens": False,
206 + }
207 +
208 + return super().apply(
209 + prompt=prompt,
210 + mm_data=mm_data,
211 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
212 + tokenization_kwargs=tokenization_kwargs,
213 + mm_uuids=mm_uuids,
214 + )
215 +
216 + def _hf_processor_applies_updates(
217 + self,
218 + prompt_text: str,
219 + mm_items: MultiModalDataItems,
220 + hf_processor_mm_kwargs: Mapping[str, object],
221 + tokenization_kwargs: Mapping[str, object],
222 + ) -> bool:
223 + return False
224 +
225 + def _get_mm_fields_config(
226 + self,
227 + hf_inputs: BatchFeature,
228 + hf_processor_mm_kwargs: Mapping[str, object],
229 + ) -> Mapping[str, MultiModalFieldConfig]:
230 + return dict(pixel_values=MultiModalFieldConfig.batched("image"))
231 +
232 + def _get_prompt_updates(
233 + self,
234 + mm_items: MultiModalDataItems,
235 + hf_processor_mm_kwargs: Mapping[str, object],
236 + out_mm_kwargs: MultiModalKwargsItems,
237 + ) -> Sequence[PromptUpdate]:
238 + image_token_id = self.image_token_id
239 +
240 + def get_replacement(item_idx: int):
241 + images = mm_items.get_items("image", ImageProcessorItems)
242 + image_size = images.get_image_size(item_idx)
243 +
244 + num_image_tokens = self.info.get_num_image_tokens(
245 + image_width=image_size.width,
246 + image_height=image_size.height,
247 + )
248 + return [image_token_id] * num_image_tokens
249 +
250 + return [
251 + PromptReplacement(
252 + modality="image",
253 + target=PromptIndexTargets.start(),
254 + replacement=get_replacement,
255 + ),
256 + ]
257 +
258 +
259 +# Adapted from: https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/models/clip/modeling_clip.py
260 +class CLIPTextEmbeddings(nn.Module):
261 +
262 + def __init__(self, config: CLIPTextConfig):
263 + super().__init__()
264 +
265 + embed_dim = config.hidden_size
266 +
267 + self.token_embedding = VocabParallelEmbedding(config.vocab_size,
268 + embed_dim)
269 + self.position_embedding = VocabParallelEmbedding(
270 + config.max_position_embeddings, embed_dim)
271 +
272 + def forward(
273 + self,
274 + input_ids: Optional[torch.Tensor],
275 + position_ids: torch.Tensor,
276 + inputs_embeds: Optional[torch.Tensor] = None,
277 + ) -> torch.Tensor:
278 + if inputs_embeds is None:
279 + if input_ids is None:
280 + raise ValueError(
281 + "Either `input_ids` or `input_embeds` must be provided")
282 +
283 + inputs_embeds = self.token_embedding(input_ids)
284 +
285 + position_embeddings = self.position_embedding(position_ids)
286 + embeddings = inputs_embeds + position_embeddings
287 +
288 + return embeddings
289 +
290 +
291 class CLIPVisionEmbeddings(nn.Module):
292
293 def __init__(self, config: CLIPVisionConfig):
@@ -89,15 +331,17 @@ class CLIPVisionEmbeddings(nn.Module):
331
332
333 class CLIPAttention(nn.Module):
92 - """Multi-headed attention from 'Attention Is All You Need' paper"""
334
335 def __init__(
336 self,
96 - config: CLIPVisionConfig,
337 + config: Union[CLIPTextConfig, CLIPVisionConfig],
338 quant_config: Optional[QuantizationConfig] = None,
339 + *,
340 prefix: str = "",
99 - ):
341 + attn_cls: Union[type[Attention], type[MultiHeadAttention]],
342 + ) -> None:
343 super().__init__()
344 +
345 self.config = config
346 self.embed_dim = config.hidden_size
347 self.num_heads = config.num_attention_heads
@@ -127,8 +371,12 @@ class CLIPAttention(nn.Module):
371 self.tp_size = get_tensor_model_parallel_world_size()
372 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
373
130 - self.attn = MultiHeadAttention(self.num_heads_per_partition,
131 - self.head_dim, self.scale)
374 + self.attn = attn_cls(
375 + self.num_heads_per_partition,
376 + self.head_dim,
377 + self.scale,
378 + prefix=f"{prefix}.attn",
379 + )
380
381 def forward(
382 self,
@@ -148,7 +396,7 @@ class CLIPMLP(nn.Module):
396
397 def __init__(
398 self,
151 - config: CLIPVisionConfig,
399 + config: Union[CLIPTextConfig, CLIPVisionConfig],
400 quant_config: Optional[QuantizationConfig] = None,
401 prefix: str = "",
402 ) -> None:
@@ -178,15 +426,18 @@ class CLIPEncoderLayer(nn.Module):
426
427 def __init__(
428 self,
181 - config: CLIPVisionConfig,
429 + config: Union[CLIPTextConfig, CLIPVisionConfig],
430 quant_config: Optional[QuantizationConfig] = None,
431 + *,
432 prefix: str = "",
433 + attn_cls: Union[type[Attention], type[MultiHeadAttention]],
434 ) -> None:
435 super().__init__()
436 self.self_attn = CLIPAttention(
437 config,
438 quant_config=quant_config,
439 prefix=f"{prefix}.self_attn",
440 + attn_cls=attn_cls,
441 )
442 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
443 eps=config.layer_norm_eps)
@@ -223,10 +474,12 @@ class CLIPEncoder(nn.Module):
474
475 def __init__(
476 self,
226 - config: CLIPVisionConfig,
477 + config: Union[CLIPTextConfig, CLIPVisionConfig],
478 quant_config: Optional[QuantizationConfig] = None,
479 num_hidden_layers_override: Optional[int] = None,
480 + *,
481 prefix: str = "",
482 + attn_cls: Union[type[Attention], type[MultiHeadAttention]],
483 ) -> None:
484 super().__init__()
485
@@ -239,12 +492,15 @@ class CLIPEncoder(nn.Module):
492 self.layers = nn.ModuleList([
493 CLIPEncoderLayer(config=config,
494 quant_config=quant_config,
242 - prefix=f"{prefix}.layers.{layer_idx}")
495 + prefix=f"{prefix}.layers.{layer_idx}",
496 + attn_cls=attn_cls)
497 for layer_idx in range(num_hidden_layers)
498 ])
499
500 def forward(
247 - self, inputs_embeds: torch.Tensor, return_all_hidden_states: bool
501 + self,
502 + inputs_embeds: torch.Tensor,
503 + return_all_hidden_states: bool,
504 ) -> Union[torch.Tensor, list[torch.Tensor]]:
505 hidden_states_pool = [inputs_embeds]
506 hidden_states = inputs_embeds
@@ -260,6 +516,87 @@ class CLIPEncoder(nn.Module):
516 return hidden_states
517
518
519 +class CLIPTextTransformer(nn.Module):
520 +
521 + def __init__(
522 + self,
523 + config: CLIPTextConfig,
524 + quant_config: Optional[QuantizationConfig] = None,
525 + *,
526 + prefix: str = "",
527 + ) -> None:
528 + super().__init__()
529 +
530 + self.config = config
531 + embed_dim = config.hidden_size
532 +
533 + self.embeddings = CLIPTextEmbeddings(config)
534 +
535 + self.encoder = CLIPEncoder(
536 + config=config,
537 + quant_config=quant_config,
538 + prefix=f"{prefix}.encoder",
539 + attn_cls=Attention,
540 + )
541 +
542 + self.final_layer_norm = nn.LayerNorm(
543 + embed_dim,
544 + eps=config.layer_norm_eps,
545 + )
546 +
547 + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
548 + return self.embeddings.token_embedding(input_ids)
549 +
550 + def forward(
551 + self,
552 + input_ids: Optional[torch.Tensor],
553 + position_ids: torch.Tensor,
554 + inputs_embeds: Optional[torch.Tensor] = None,
555 + ) -> torch.Tensor:
556 + hidden_states = self.embeddings(
557 + input_ids=input_ids,
558 + position_ids=position_ids,
559 + inputs_embeds=inputs_embeds,
560 + )
561 +
562 + last_hidden_state = self.encoder(
563 + inputs_embeds=hidden_states,
564 + return_all_hidden_states=False,
565 + )
566 + last_hidden_state = self.final_layer_norm(last_hidden_state)
567 +
568 + return last_hidden_state
569 +
570 + def load_weights(self, weights: Iterable[tuple[str,
571 + torch.Tensor]]) -> set[str]:
572 + stacked_params_mapping = [
573 + # (param_name, shard_name, shard_id)
574 + ("qkv_proj", "q_proj", "q"),
575 + ("qkv_proj", "k_proj", "k"),
576 + ("qkv_proj", "v_proj", "v"),
577 + ]
578 + params_dict = dict(self.named_parameters())
579 + loaded_params: set[str] = set()
580 +
581 + for name, loaded_weight in weights:
582 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
583 + if weight_name not in name:
584 + continue
585 + name = name.replace(weight_name, param_name)
586 +
587 + param = params_dict[name]
588 + weight_loader = param.weight_loader
589 + weight_loader(param, loaded_weight, shard_id)
590 + break
591 + else:
592 + param = params_dict[name]
593 + weight_loader = getattr(param, "weight_loader",
594 + default_weight_loader)
595 + weight_loader(param, loaded_weight)
596 + loaded_params.add(name)
597 + return loaded_params
598 +
599 +
600 class CLIPVisionTransformer(nn.Module):
601
602 def __init__(
@@ -287,6 +624,7 @@ class CLIPVisionTransformer(nn.Module):
624 quant_config=quant_config,
625 num_hidden_layers_override=num_hidden_layers_override,
626 prefix=f"{prefix}.encoder",
627 + attn_cls=MultiHeadAttention,
628 )
629
630 num_hidden_layers = config.num_hidden_layers
@@ -306,6 +644,14 @@ class CLIPVisionTransformer(nn.Module):
644 else:
645 self.post_layernorm = None
646
647 + @property
648 + def dtype(self):
649 + return next(self.parameters()).dtype
650 +
651 + @property
652 + def device(self):
653 + return next(self.parameters()).device
654 +
655 def forward(
656 self,
657 pixel_values: torch.Tensor,
@@ -335,11 +681,49 @@ class CLIPVisionTransformer(nn.Module):
681
682 return encoder_outputs
683
684 + def load_weights(self, weights: Iterable[tuple[str,
685 + torch.Tensor]]) -> set[str]:
686 + stacked_params_mapping = [
687 + # (param_name, shard_name, shard_id)
688 + ("qkv_proj", "q_proj", "q"),
689 + ("qkv_proj", "k_proj", "k"),
690 + ("qkv_proj", "v_proj", "v"),
691 + ]
692 + params_dict = dict(self.named_parameters())
693 + loaded_params: set[str] = set()
694 + layer_count = len(self.encoder.layers)
695
339 -class CLIPVisionModel(nn.Module, SupportsQuant):
340 - config_class = CLIPVisionConfig
341 - main_input_name = "pixel_values"
342 - packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
696 + for name, loaded_weight in weights:
697 + # post_layernorm is not needed in CLIPVisionModel
698 + if (name.startswith("post_layernorm")
699 + and self.post_layernorm is None):
700 + continue
701 +
702 + # omit layers when num_hidden_layers_override is set
703 + if name.startswith("encoder.layers"):
704 + layer_idx = int(name.split(".")[2])
705 + if layer_idx >= layer_count:
706 + continue
707 +
708 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
709 + if weight_name not in name:
710 + continue
711 + name = name.replace(weight_name, param_name)
712 +
713 + param = params_dict[name]
714 + weight_loader = param.weight_loader
715 + weight_loader(param, loaded_weight, shard_id)
716 + break
717 + else:
718 + param = params_dict[name]
719 + weight_loader = getattr(param, "weight_loader",
720 + default_weight_loader)
721 + weight_loader(param, loaded_weight)
722 + loaded_params.add(name)
723 + return loaded_params
724 +
725 +
726 +class CLIPVisionModel(nn.Module):
727
728 def __init__(
729 self,
@@ -351,12 +735,14 @@ class CLIPVisionModel(nn.Module, SupportsQuant):
735 prefix: str = "",
736 ) -> None:
737 super().__init__()
738 +
739 self.vision_model = CLIPVisionTransformer(
740 config=config,
741 quant_config=quant_config,
742 num_hidden_layers_override=num_hidden_layers_override,
743 require_post_norm=require_post_norm,
359 - prefix=f"{prefix}.vision_model")
744 + prefix=f"{prefix}.vision_model",
745 + )
746
747 def forward(
748 self,
@@ -370,49 +756,198 @@ class CLIPVisionModel(nn.Module, SupportsQuant):
756 feature_select_strategy=feature_select_strategy,
757 )
758
759 + @property
760 + def dtype(self):
761 + return self.vision_model.dtype
762 +
763 @property
764 def device(self):
375 - return next(self.parameters()).device
765 + return self.vision_model.device
766
377 - # (TODO) Add prefix argument for filtering out weights to be loaded
378 - # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
379 - def load_weights(self, weights: Iterable[tuple[str,
380 - torch.Tensor]]) -> set[str]:
381 - stacked_params_mapping = [
382 - # (param_name, shard_name, shard_id)
383 - ("qkv_proj", "q_proj", "q"),
384 - ("qkv_proj", "k_proj", "k"),
385 - ("qkv_proj", "v_proj", "v"),
386 - ]
387 - params_dict = dict(self.named_parameters())
388 - loaded_params: set[str] = set()
389 - layer_count = len(self.vision_model.encoder.layers)
767
391 - for name, loaded_weight in weights:
392 - # post_layernorm is not needed in CLIPVisionModel
393 - if (name.startswith("vision_model.post_layernorm")
394 - and self.vision_model.post_layernorm is None):
395 - continue
768 +# Assume EOS token corresponds to LAST token in text model
769 +@default_pooling_type("LAST")
770 +@MULTIMODAL_REGISTRY.register_processor(CLIPMultiModalProcessor,
771 + info=CLIPProcessingInfo,
772 + dummy_inputs=CLIPDummyInputsBuilder)
773 +class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
774
397 - # omit layers when num_hidden_layers_override is set
398 - if name.startswith("vision_model.encoder.layers"):
399 - layer_idx = int(name.split(".")[3])
400 - if layer_idx >= layer_count:
401 - continue
775 + is_pooling_model = True
776
403 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
404 - if weight_name not in name:
405 - continue
406 - name = name.replace(weight_name, param_name)
777 + packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
778 + merge_by_field_config = True
779
408 - param = params_dict[name]
409 - weight_loader = param.weight_loader
410 - weight_loader(param, loaded_weight, shard_id)
411 - break
412 - else:
413 - param = params_dict[name]
414 - weight_loader = getattr(param, "weight_loader",
415 - default_weight_loader)
416 - weight_loader(param, loaded_weight)
417 - loaded_params.add(name)
418 - return loaded_params
780 + @classmethod
781 + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
782 + if modality.startswith("image"):
783 + return None
784 +
785 + raise ValueError("Only image modality is supported")
786 +
787 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
788 + super().__init__()
789 +
790 + config: CLIPConfig = vllm_config.model_config.hf_config
791 + quant_config = vllm_config.quant_config
792 + multimodal_config = vllm_config.model_config.multimodal_config
793 + self.config = config
794 + self.multimodal_config = multimodal_config
795 +
796 + text_config = config.text_config
797 + vision_config = config.vision_config
798 +
799 + self.projection_dim = config.projection_dim
800 + self.text_embed_dim = text_config.hidden_size
801 + self.vision_embed_dim = vision_config.hidden_size
802 +
803 + self.text_model = CLIPTextTransformer(
804 + text_config,
805 + quant_config=quant_config,
806 + prefix=maybe_prefix(prefix, "text_model"),
807 + )
808 + self.vision_model = CLIPVisionTransformer(
809 + vision_config,
810 + quant_config=quant_config,
811 + prefix=maybe_prefix(prefix, "vision_model"),
812 + )
813 +
814 + self.visual_projection = nn.Linear(
815 + self.vision_embed_dim,
816 + self.projection_dim,
817 + bias=False,
818 + )
819 + self.text_projection = nn.Linear(
820 + self.text_embed_dim,
821 + self.projection_dim,
822 + bias=False,
823 + )
824 +
825 + pooler_config = vllm_config.model_config.pooler_config
826 + assert pooler_config is not None
827 + self.pooler_config = pooler_config
828 +
829 + self.pooler = DispatchPooler({
830 + "encode": Pooler.for_encode(pooler_config),
831 + "embed": Pooler.for_embed(pooler_config),
832 + })
833 +
834 + # Assumes that self.forward is called after self.get_input_embeddings
835 + self._is_text_input = True
836 +
837 + def get_text_features(
838 + self,
839 + input_ids: Optional[torch.Tensor],
840 + position_ids: torch.Tensor,
841 + inputs_embeds: Optional[torch.Tensor] = None,
842 + ) -> torch.Tensor:
843 + pooled_output = self.text_model(
844 + input_ids=input_ids,
845 + position_ids=position_ids,
846 + inputs_embeds=inputs_embeds,
847 + )
848 +
849 + text_features = self.text_projection(pooled_output)
850 +
851 + return text_features
852 +
853 + def get_image_features(
854 + self,
855 + pixel_values: torch.Tensor,
856 + feature_select_strategy: Optional[VisionFeatureSelectStrategy] = None,
857 + ) -> torch.Tensor:
858 + if feature_select_strategy is None:
859 + feature_select_strategy = _get_vision_feature_select_strategy(
860 + self.pooler_config.pooling_type)
861 +
862 + pooled_output = self.vision_model(
863 + pixel_values=pixel_values,
864 + select_layers=None,
865 + feature_select_strategy=feature_select_strategy,
866 + )
867 +
868 + image_features = self.visual_projection(pooled_output)
869 +
870 + return image_features
871 +
872 + def _parse_and_validate_image_input(
873 + self, **kwargs: object) -> Optional[CLIPImagePixelInputs]:
874 + pixel_values = kwargs.pop("pixel_values", None)
875 + if pixel_values is None:
876 + return None
877 +
878 + expected_h = expected_w = self.config.vision_config.image_size
879 + return CLIPImagePixelInputs(type="pixel_values",
880 + data=pixel_values,
881 + resolve_bindings={
882 + "h": expected_h,
883 + "w": expected_w
884 + })
885 +
886 + def _process_image_inputs(self,
887 + inputs: CLIPImagePixelInputs) -> torch.Tensor:
888 + pixel_values = inputs["data"]
889 +
890 + return self.get_image_features(pixel_values)
891 +
892 + def get_language_model(self) -> torch.nn.Module:
893 + return self.text_model
894 +
895 + def get_input_embeddings(
896 + self,
897 + input_ids: torch.Tensor,
898 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
899 + *,
900 + is_multimodal: Optional[torch.Tensor] = None,
901 + handle_oov_mm_token: bool = False,
902 + ) -> torch.Tensor:
903 + self._is_text_input = (multimodal_embeddings is None
904 + or len(multimodal_embeddings) == 0)
905 +
906 + # This is to satisfy the type checker for each overload
907 + if multimodal_embeddings is None or is_multimodal is None:
908 + return super().get_input_embeddings(input_ids)
909 +
910 + return super().get_input_embeddings(
911 + input_ids,
912 + multimodal_embeddings=multimodal_embeddings,
913 + is_multimodal=is_multimodal,
914 + handle_oov_mm_token=handle_oov_mm_token,
915 + )
916 +
917 + def get_multimodal_embeddings(self,
918 + **kwargs: object) -> MultiModalEmbeddings:
919 + image_input = self._parse_and_validate_image_input(**kwargs)
920 + if image_input is None:
921 + return []
922 +
923 + vision_embeddings = self._process_image_inputs(image_input)
924 + return vision_embeddings
925 +
926 + def forward(
927 + self,
928 + input_ids: Optional[torch.Tensor],
929 + positions: torch.Tensor,
930 + intermediate_tensors: Optional[IntermediateTensors] = None,
931 + inputs_embeds: Optional[torch.Tensor] = None,
932 + **kwargs: object,
933 + ) -> torch.Tensor:
934 + if intermediate_tensors is not None:
935 + raise RuntimeError("PP is not supported for this model")
936 +
937 + # Multimodal inputs
938 + if not self._is_text_input:
939 + return inputs_embeds
940 +
941 + # Text inputs
942 + return self.get_text_features(input_ids=input_ids,
943 + position_ids=positions,
944 + inputs_embeds=inputs_embeds)
945 +
946 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
947 + loader = AutoWeightsLoader(
948 + self,
949 + skip_substrs=[".position_ids"],
950 + ignore_unexpected_prefixes=["logit_scale."],
951 + )
952 +
953 + return loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -187,6 +187,7 @@ _EMBEDDING_MODELS = {
187 "TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
188 "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
189 # [Multimodal]
190 + "CLIPModel": ("clip", "CLIPEmbeddingModel"),
191 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
192 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
193 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
vllm/model_executor/models/vision.py
+4 -2
@@ -92,8 +92,10 @@ def get_vit_attn_backend(head_size: int, dtype: torch.dtype) -> _Backend:
92 return current_platform.get_vit_attn_backend(head_size, dtype)
93
94
95 +VisionFeatureSelectStrategyStr = Literal["class", "default", "full"]
96 +
97 VisionFeatureSelectStrategy = Union[
96 - Literal["class", "default", "full"],
98 + VisionFeatureSelectStrategyStr,
99 Callable[[torch.Tensor], torch.Tensor],
100 ]
101
@@ -106,7 +108,7 @@ def _get_vision_feature_selector(
108
109 # https://github.com/huggingface/transformers/blob/cd74917ffc3e8f84e4a886052c5ab32b7ac623cc/src/transformers/models/clip/modeling_clip.py#L762
110 if strategy == "class":
109 - return lambda feats: feats[:, 0, :]
111 + return lambda feats: feats[:, :1, :]
112
113 # https://github.com/huggingface/transformers/blob/4a02bc7004285bdb12cc033e87ad2578ce2fa900/src/transformers/models/llava/modeling_llava.py#L196
114 if strategy == "default":
vllm/transformers_utils/chat_templates/registry.py
+1
@@ -33,6 +33,7 @@ def _get_minicpmv_chat_template_fallback(
33 # yapf: disable
34 _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
35 "blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
36 + "clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
37 "chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
38 "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
39 "fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",