@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 10f08dedfa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 19:15 UTC

[Model] Add ColPali late interaction model for multi-modal retrieval (#36818)

Signed-off-by: Nikita Sukharev <kaonael@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Nikita committed Mar 13, 2026 at 03:18 UTC 10f08dedfa1636dd477f1ac970827c64f234aad4
9 files changed +634
docs/models/supported_models.md
+1
@@ -828,6 +828,7 @@ The following table lists those that are tested in vLLM.
828 | ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |
829 | `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
830 | `ColModernVBertForRetrieval` | ColModernVBERT | T / I | `ModernVBERT/colmodernvbert-merged` | | |
831 +| `ColPaliForRetrieval` | ColPali | T / I | `vidore/colpali-v1.3-hf` | | |
832 | `LlamaNemotronVLModel` | Llama Nemotron Embedding + SigLIP | T + I | `nvidia/llama-nemotron-embed-vl-1b-v2` | | |
833 | `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
834 | `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
tests/models/multimodal/pooling/test_colpali.py new
+323
@@ -0,0 +1,323 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Tests for ColPali late interaction model for multi-modal retrieval.
4 +
5 +ColPali is a multi-vector retrieval model based on PaliGemma backbone
6 +(SigLIP + Gemma) with ColBERT-style late interaction scoring (MaxSim).
7 +It produces per-token embeddings for both text and image inputs.
8 +"""
9 +
10 +import base64
11 +from io import BytesIO
12 +
13 +import pytest
14 +import torch
15 +from PIL import Image
16 +
17 +from vllm.entrypoints.chat_utils import (
18 + ChatCompletionContentPartImageParam,
19 + ChatCompletionContentPartTextParam,
20 +)
21 +from vllm.entrypoints.pooling.score.utils import ScoreMultiModalParam
22 +
23 +from ....conftest import VllmRunner
24 +
25 +MODELS = [
26 + "vidore/colpali-v1.3-hf",
27 +]
28 +
29 +EMBED_DIMS = {
30 + "vidore/colpali-v1.3-hf": 128,
31 +}
32 +
33 +TEXT_QUERIES = [
34 + "What is the capital of France?",
35 + "Describe the contents of the document.",
36 +]
37 +
38 +TEXT_DOCUMENTS = [
39 + "The capital of France is Paris.",
40 + "This document contains important financial data.",
41 +]
42 +
43 +DTYPE = "half"
44 +GPU_MEMORY_UTILIZATION = 0.7
45 +
46 +
47 +def _make_base64_image(
48 + width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
49 +) -> str:
50 + """Create a small solid-color PNG image and return its base64 data URI."""
51 + img = Image.new("RGB", (width, height), color)
52 + buf = BytesIO()
53 + img.save(buf, format="PNG")
54 + b64 = base64.b64encode(buf.getvalue()).decode()
55 + return f"data:image/png;base64,{b64}"
56 +
57 +
58 +def _make_image_mm_param(
59 + image_uri: str,
60 + text: str | None = None,
61 +) -> ScoreMultiModalParam:
62 + """Build a ScoreMultiModalParam containing an image (and optional text)."""
63 + content: list = [
64 + ChatCompletionContentPartImageParam(
65 + type="image_url",
66 + image_url={"url": image_uri},
67 + ),
68 + ]
69 + if text is not None:
70 + content.append(
71 + ChatCompletionContentPartTextParam(type="text", text=text),
72 + )
73 + return ScoreMultiModalParam(content=content)
74 +
75 +
76 +def _run_token_embed_test(
77 + vllm_runner: type[VllmRunner],
78 + model: str,
79 + *,
80 + dtype: str,
81 +) -> None:
82 + """Verify per-token embedding shape and L2 normalization."""
83 + with vllm_runner(
84 + model,
85 + runner="pooling",
86 + dtype=dtype,
87 + max_model_len=4096,
88 + enforce_eager=True,
89 + gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
90 + ) as vllm_model:
91 + outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
92 +
93 + assert len(outputs) == 1
94 + emb = torch.tensor(outputs[0])
95 + # Token embeddings should be 2D: [num_tokens, embed_dim]
96 + assert emb.dim() == 2
97 + assert emb.shape[1] == EMBED_DIMS[model]
98 + assert emb.shape[0] > 1
99 +
100 + # Verify L2 normalization
101 + norms = torch.norm(emb, p=2, dim=-1)
102 + torch.testing.assert_close(
103 + norms,
104 + torch.ones_like(norms),
105 + rtol=1e-2,
106 + atol=1e-2,
107 + )
108 +
109 +
110 +def _run_late_interaction_test(
111 + vllm_runner: type[VllmRunner],
112 + model: str,
113 + *,
114 + dtype: str,
115 +) -> None:
116 + """Verify MaxSim scoring matches manual computation."""
117 + from vllm.entrypoints.pooling.score.utils import compute_maxsim_score
118 +
119 + with vllm_runner(
120 + model,
121 + runner="pooling",
122 + dtype=dtype,
123 + max_model_len=4096,
124 + enforce_eager=True,
125 + gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
126 + ) as vllm_model:
127 + q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
128 + d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
129 +
130 + q_emb = torch.tensor(q_outputs[0])
131 + d_emb = torch.tensor(d_outputs[0])
132 +
133 + manual_score = compute_maxsim_score(q_emb, d_emb).item()
134 +
135 + vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
136 +
137 + assert len(vllm_scores) == 1
138 + assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
139 +
140 +
141 +def _run_relevance_test(
142 + vllm_runner: type[VllmRunner],
143 + model: str,
144 + *,
145 + dtype: str,
146 +) -> None:
147 + """Verify that relevant documents score higher than irrelevant ones."""
148 + query = "What is machine learning?"
149 + documents = [
150 + "Machine learning is a subset of artificial intelligence.",
151 + "The weather forecast shows rain tomorrow.",
152 + "Deep learning uses neural networks for complex tasks.",
153 + ]
154 +
155 + with vllm_runner(
156 + model,
157 + runner="pooling",
158 + dtype=dtype,
159 + max_model_len=4096,
160 + enforce_eager=True,
161 + gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
162 + ) as vllm_model:
163 + scores = vllm_model.score(query, documents)
164 +
165 + assert len(scores) == 3
166 + assert scores[0] > scores[1], "ML doc should score higher than weather doc"
167 + assert scores[2] > scores[1], "DL doc should score higher than weather doc"
168 +
169 +
170 +@pytest.mark.parametrize("model", MODELS)
171 +@pytest.mark.parametrize("dtype", [DTYPE])
172 +def test_colpali_token_embed(
173 + vllm_runner,
174 + model: str,
175 + dtype: str,
176 +) -> None:
177 + _run_token_embed_test(vllm_runner, model, dtype=dtype)
178 +
179 +
180 +@pytest.mark.parametrize("model", MODELS)
181 +@pytest.mark.parametrize("dtype", [DTYPE])
182 +def test_colpali_late_interaction_scoring(
183 + vllm_runner,
184 + model: str,
185 + dtype: str,
186 +) -> None:
187 + _run_late_interaction_test(vllm_runner, model, dtype=dtype)
188 +
189 +
190 +@pytest.mark.parametrize("model", MODELS)
191 +@pytest.mark.parametrize("dtype", [DTYPE])
192 +def test_colpali_relevance_ordering(
193 + vllm_runner,
194 + model: str,
195 + dtype: str,
196 +) -> None:
197 + _run_relevance_test(vllm_runner, model, dtype=dtype)
198 +
199 +
200 +# ── Multimodal scoring tests ────────────────────────────────
201 +
202 +
203 +def _run_multimodal_text_query_image_docs_test(
204 + vllm_runner: type[VllmRunner],
205 + model: str,
206 + *,
207 + dtype: str,
208 +) -> None:
209 + """Score a text query against image documents via the multimodal path."""
210 + red_image = _make_base64_image(64, 64, color=(255, 0, 0))
211 + blue_image = _make_base64_image(64, 64, color=(0, 0, 255))
212 +
213 + query = "Describe the red object"
214 + image_docs = [
215 + _make_image_mm_param(red_image),
216 + _make_image_mm_param(blue_image),
217 + ]
218 +
219 + with vllm_runner(
220 + model,
221 + runner="pooling",
222 + dtype=dtype,
223 + max_model_len=4096,
224 + enforce_eager=True,
225 + gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
226 + ) as vllm_model:
227 + scores = vllm_model.llm.score(query, image_docs)
228 +
229 + assert len(scores) == 2
230 + for s in scores:
231 + assert isinstance(s.outputs.score, float)
232 +
233 +
234 +def _run_multimodal_mixed_docs_test(
235 + vllm_runner: type[VllmRunner],
236 + model: str,
237 + *,
238 + dtype: str,
239 +) -> None:
240 + """Score a text query against a mix of text and image documents."""
241 + red_image = _make_base64_image(64, 64, color=(255, 0, 0))
242 +
243 + query = "What is the capital of France?"
244 + documents: list = [
245 + "The capital of France is Paris.",
246 + _make_image_mm_param(red_image),
247 + ]
248 +
249 + with vllm_runner(
250 + model,
251 + runner="pooling",
252 + dtype=dtype,
253 + max_model_len=4096,
254 + enforce_eager=True,
255 + gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
256 + ) as vllm_model:
257 + scores = vllm_model.llm.score(query, documents)
258 +
259 + assert len(scores) == 2
260 + for s in scores:
261 + assert isinstance(s.outputs.score, float)
262 + # Text document about France should score higher than a random image
263 + assert scores[0].outputs.score > scores[1].outputs.score
264 +
265 +
266 +def _run_multimodal_image_query_text_docs_test(
267 + vllm_runner: type[VllmRunner],
268 + model: str,
269 + *,
270 + dtype: str,
271 +) -> None:
272 + """Score an image query against text documents."""
273 + red_image = _make_base64_image(64, 64, color=(255, 0, 0))
274 + image_query = _make_image_mm_param(red_image, text="red color")
275 +
276 + documents = [
277 + "A bright red sports car.",
278 + "The weather forecast shows rain tomorrow.",
279 + ]
280 +
281 + with vllm_runner(
282 + model,
283 + runner="pooling",
284 + dtype=dtype,
285 + max_model_len=4096,
286 + enforce_eager=True,
287 + gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
288 + ) as vllm_model:
289 + scores = vllm_model.llm.score(image_query, documents)
290 +
291 + assert len(scores) == 2
292 + for s in scores:
293 + assert isinstance(s.outputs.score, float)
294 +
295 +
296 +@pytest.mark.parametrize("model", MODELS)
297 +@pytest.mark.parametrize("dtype", [DTYPE])
298 +def test_colpali_multimodal_text_query_image_docs(
299 + vllm_runner,
300 + model: str,
301 + dtype: str,
302 +) -> None:
303 + _run_multimodal_text_query_image_docs_test(vllm_runner, model, dtype=dtype)
304 +
305 +
306 +@pytest.mark.parametrize("model", MODELS)
307 +@pytest.mark.parametrize("dtype", [DTYPE])
308 +def test_colpali_multimodal_mixed_docs(
309 + vllm_runner,
310 + model: str,
311 + dtype: str,
312 +) -> None:
313 + _run_multimodal_mixed_docs_test(vllm_runner, model, dtype=dtype)
314 +
315 +
316 +@pytest.mark.parametrize("model", MODELS)
317 +@pytest.mark.parametrize("dtype", [DTYPE])
318 +def test_colpali_multimodal_image_query_text_docs(
319 + vllm_runner,
320 + model: str,
321 + dtype: str,
322 +) -> None:
323 + _run_multimodal_image_query_text_docs_test(vllm_runner, model, dtype=dtype)
tests/models/registry.py
+1
@@ -631,6 +631,7 @@ _LATE_INTERACTION_EXAMPLE_MODELS = {
631 "ColModernVBertForRetrieval": _HfExamplesInfo(
632 "ModernVBERT/colmodernvbert-merged",
633 ),
634 + "ColPaliForRetrieval": _HfExamplesInfo("vidore/colpali-v1.3-hf"),
635 "ColQwen3": _HfExamplesInfo(
636 "TomoroAI/tomoro-colqwen3-embed-4b", trust_remote_code=True
637 ),
vllm/model_executor/models/colpali.py new
+245
@@ -0,0 +1,245 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +ColPali late interaction model for multi-modal retrieval and reranking.
5 +
6 +ColPali extends PaliGemma with a ColBERT-style late interaction head,
7 +producing per-token embeddings for both text and image inputs. It uses
8 +MaxSim scoring for retrieval/reranking tasks.
9 +
10 +This model supports the "token_embed" pooling task and is designed for
11 +multi-vector retrieval of documents containing both text and images.
12 +
13 +Reference: https://arxiv.org/abs/2407.01449 (ColPali)
14 +Based on: PaliGemma backbone (SigLIP + Gemma) with custom text projection
15 +
16 +Target models:
17 +- vidore/colpali-v1.3-hf
18 +"""
19 +
20 +from collections.abc import Iterable, Mapping
21 +
22 +import torch
23 +import torch.nn as nn
24 +from transformers import BatchFeature, PaliGemmaProcessor
25 +
26 +from vllm.config import VllmConfig
27 +from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
28 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29 +from vllm.multimodal import MULTIMODAL_REGISTRY
30 +
31 +from .interfaces import SupportsLateInteraction
32 +from .interfaces_base import default_pooling_type
33 +from .paligemma import (
34 + PaliGemmaDummyInputsBuilder,
35 + PaliGemmaForConditionalGeneration,
36 + PaliGemmaMultiModalProcessor,
37 + PaliGemmaProcessingInfo,
38 +)
39 +from .utils import AutoWeightsLoader, WeightsMapper
40 +
41 +
42 +class ColPaliProcessingInfo(PaliGemmaProcessingInfo):
43 + """Processing info for ColPali models.
44 +
45 + ColPali models use a custom HuggingFace config (ColPaliConfig) that is
46 + not an instance of PaliGemmaConfig. We override get_hf_config() and
47 + get_hf_processor() to skip the strict type check.
48 + """
49 +
50 + def get_hf_config(self):
51 + return self.ctx.get_hf_config()
52 +
53 + def get_hf_processor(self, **kwargs: object) -> PaliGemmaProcessor:
54 + # Force standard PaliGemmaProcessor even when trust_remote_code=True.
55 + return self.ctx.get_hf_processor(PaliGemmaProcessor, **kwargs)
56 +
57 +
58 +class ColPaliMultiModalProcessor(PaliGemmaMultiModalProcessor):
59 + """Multimodal processor for ColPali."""
60 +
61 + def _call_hf_processor(
62 + self,
63 + prompt: str,
64 + mm_data: Mapping[str, object],
65 + mm_kwargs: Mapping[str, object],
66 + tok_kwargs: Mapping[str, object],
67 + ) -> BatchFeature:
68 + if mm_data:
69 + # The ColPali tokenizer_config.json ships with a small default
70 + # max_length (50) that truncates the 1024 image tokens inserted
71 + # by PaliGemmaProcessor, causing a token-count mismatch.
72 + # vLLM enforces its own max_model_len, so we disable HF
73 + # truncation to keep all image + text tokens intact.
74 + tok_kwargs = dict(tok_kwargs, truncation=False)
75 + return super()._call_hf_processor(
76 + prompt=prompt,
77 + mm_data=mm_data,
78 + mm_kwargs=mm_kwargs,
79 + tok_kwargs=tok_kwargs,
80 + )
81 +
82 +
83 +@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
84 +@MULTIMODAL_REGISTRY.register_processor(
85 + ColPaliMultiModalProcessor,
86 + info=ColPaliProcessingInfo,
87 + dummy_inputs=PaliGemmaDummyInputsBuilder,
88 +)
89 +class ColPaliModel(
90 + PaliGemmaForConditionalGeneration,
91 + SupportsLateInteraction,
92 +):
93 + """ColPali late interaction model for multi-modal retrieval/reranking.
94 +
95 + This model extends PaliGemmaForConditionalGeneration with a ColBERT-style
96 + linear projection layer for per-token embeddings. It supports:
97 + - "token_embed" task: Per-token embeddings for late interaction scoring
98 +
99 + The model produces L2-normalized per-token embeddings by:
100 + 1. Running the PaliGemma backbone (vision + language) to get hidden states
101 + 2. Projecting hidden states through a linear layer (hidden_size -> embed_dim)
102 + 3. L2-normalizing the projected embeddings
103 + """
104 +
105 + # Mark this as a pooling model so vLLM routes to pooler path
106 + is_pooling_model = True
107 +
108 + # Override hf_to_vllm_mapper to handle ColPali weight naming.
109 + hf_to_vllm_mapper = WeightsMapper(
110 + orig_to_new_prefix={
111 + # HF transformers checkpoint (vidore/colpali-v1.3-hf)
112 + # Weights: vlm.vision_tower.*, vlm.language_model.*,
113 + # vlm.multi_modal_projector.*
114 + "vlm.vision_tower.": "vision_tower.",
115 + "vlm.language_model.": "language_model.",
116 + "vlm.multi_modal_projector.": "multi_modal_projector.",
117 + # colpali-engine checkpoint naming
118 + "model.vision_tower.": "vision_tower.",
119 + "model.language_model.": "language_model.",
120 + "model.multi_modal_projector.": "multi_modal_projector.",
121 + "lm_head.": "language_model.lm_head.",
122 + }
123 + )
124 +
125 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
126 + super().__init__(vllm_config=vllm_config, prefix=prefix)
127 +
128 + config = vllm_config.model_config.hf_config
129 + head_dtype = vllm_config.model_config.head_dtype
130 +
131 + hidden_size = getattr(config, "hidden_size", None)
132 + if hidden_size is None and hasattr(config, "text_config"):
133 + hidden_size = config.text_config.hidden_size
134 + if hidden_size is None:
135 + raise ValueError(
136 + "Unable to determine text hidden size from config. "
137 + "Expected 'hidden_size' or 'text_config.hidden_size'."
138 + )
139 + self._proj_hidden_size = hidden_size
140 +
141 + # ColPali uses embedding_dim=128, but also check other naming variants
142 + self.embed_dim: int | None = (
143 + getattr(config, "embedding_dim", None)
144 + or getattr(config, "embed_dim", None)
145 + or getattr(config, "dim", None)
146 + or getattr(config, "projection_dim", None)
147 + or getattr(config, "colbert_dim", None)
148 + )
149 +
150 + # Build the projection layer if embed_dim is known
151 + if self.embed_dim is not None:
152 + self.custom_text_proj = nn.Linear(
153 + hidden_size,
154 + self.embed_dim,
155 + bias=False,
156 + dtype=head_dtype,
157 + )
158 + else:
159 + # Will be created during load_weights when dim is inferred
160 + self.custom_text_proj = None
161 +
162 + pooler_config = vllm_config.model_config.pooler_config
163 + assert pooler_config is not None
164 + self.pooler = pooler_for_token_embed(
165 + pooler_config,
166 + projector=self.custom_text_proj,
167 + )
168 +
169 + def forward(
170 + self,
171 + input_ids: torch.Tensor | None,
172 + positions: torch.Tensor,
173 + intermediate_tensors=None,
174 + inputs_embeds: torch.Tensor | None = None,
175 + **kwargs: object,
176 + ) -> torch.Tensor:
177 + return super().forward(
178 + input_ids=input_ids,
179 + positions=positions,
180 + intermediate_tensors=intermediate_tensors,
181 + inputs_embeds=inputs_embeds,
182 + **kwargs,
183 + )
184 +
185 + # Names used for the projection layer across different ColPali variants
186 + _PROJ_LAYER_NAMES = {
187 + "custom_text_proj", # vLLM internal naming
188 + "embedding_proj_layer", # colpali-engine / HF naming
189 + }
190 +
191 + def _is_proj_weight(self, name: str) -> bool:
192 + """Check if a weight name belongs to the projection layer."""
193 + return any(proj_name in name for proj_name in self._PROJ_LAYER_NAMES)
194 +
195 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
196 + """Load weights with special handling for ColPali projection layer."""
197 + weights_list = list(weights)
198 + proj_weights: list[tuple[str, torch.Tensor]] = []
199 + model_weights: list[tuple[str, torch.Tensor]] = []
200 +
201 + for name, weight in weights_list:
202 + if self._is_proj_weight(name):
203 + proj_weights.append((name, weight))
204 + else:
205 + model_weights.append((name, weight))
206 +
207 + loader = AutoWeightsLoader(self)
208 + loaded = loader.load_weights(model_weights, mapper=self.hf_to_vllm_mapper)
209 +
210 + if proj_weights:
211 + model_dtype = next(self.language_model.parameters()).dtype
212 + model_device = next(self.language_model.parameters()).device
213 +
214 + for name, weight in proj_weights:
215 + if self.embed_dim is None and "weight" in name:
216 + self.embed_dim = weight.shape[0]
217 + has_bias = any("bias" in n for n, _ in proj_weights)
218 + self.custom_text_proj = nn.Linear(
219 + self._proj_hidden_size,
220 + self.embed_dim,
221 + bias=has_bias,
222 + dtype=model_dtype,
223 + )
224 + self.custom_text_proj.to(model_device)
225 +
226 + if self.custom_text_proj is not None:
227 + param_name = name.split(".")[-1]
228 + param = getattr(self.custom_text_proj, param_name, None)
229 + if param is not None:
230 + weight = weight.to(device=param.device, dtype=param.dtype)
231 + default_weight_loader(param, weight)
232 + loaded.add(f"custom_text_proj.{param_name}")
233 +
234 + # Update pooler projector for the lazy-creation path
235 + self.pooler.head.projector = self.custom_text_proj
236 +
237 + # Mark pooler projector params as loaded
238 + if hasattr(self, "pooler") and hasattr(self.pooler, "head"):
239 + head = self.pooler.head
240 + projector = getattr(head, "projector", None)
241 + if projector is not None and isinstance(projector, nn.Module):
242 + for pname, _ in projector.named_parameters():
243 + loaded.add(f"pooler.head.projector.{pname}")
244 +
245 + return loaded
vllm/model_executor/models/registry.py
+1
@@ -247,6 +247,7 @@ _EMBEDDING_MODELS = {
247 "XLMRobertaModel": ("roberta", "RobertaEmbeddingModel"),
248 # [Multimodal]
249 "CLIPModel": ("clip", "CLIPEmbeddingModel"),
250 + "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
251 "LlavaNextForConditionalGeneration": (
252 "llava_next",
253 "LlavaNextForConditionalGeneration",
vllm/transformers_utils/chat_templates/registry.py
+1
@@ -33,6 +33,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
33 "blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
34 "chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
35 "clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
36 + "colpali": CHAT_TEMPLATES_DIR / "template_basic.jinja",
37 "deepseek_ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
38 "deepseek_ocr2": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
39 "deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
vllm/transformers_utils/config.py
+1
@@ -78,6 +78,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
78 bagel="BagelConfig",
79 chatglm="ChatGLMConfig",
80 colmodernvbert="ColModernVBertConfig",
81 + colpali="ColPaliConfig",
82 colqwen3="ColQwen3Config",
83 ops_colqwen3="OpsColQwen3Config",
84 qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
vllm/transformers_utils/configs/__init__.py
+2
@@ -20,6 +20,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
20 "BagelConfig": "vllm.transformers_utils.configs.bagel",
21 "ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
22 "ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
23 + "ColPaliConfig": "vllm.transformers_utils.configs.colpali",
24 "ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
25 "OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
26 "Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
@@ -76,6 +77,7 @@ __all__ = [
77 "BagelConfig",
78 "ChatGLMConfig",
79 "ColModernVBertConfig",
80 + "ColPaliConfig",
81 "ColQwen3Config",
82 "OpsColQwen3Config",
83 "Qwen3VLNemotronEmbedConfig",
vllm/transformers_utils/configs/colpali.py new
+59
@@ -0,0 +1,59 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +ColPali configuration that extends PaliGemmaConfig with embedding projection
5 +fields. This allows ColPali models to be loaded without trust_remote_code
6 +by mapping their custom model_type (colpali) to a standard config class
7 +that vLLM understands.
8 +
9 +Supported model_types:
10 +- colpali (vidore/colpali-v1.3-hf)
11 +"""
12 +
13 +from transformers import PaliGemmaConfig
14 +
15 +
16 +class ColPaliConfig(PaliGemmaConfig):
17 + """Configuration class for ColPali models.
18 +
19 + Extends PaliGemmaConfig with additional fields used by ColPali variants
20 + for the embedding projection layer.
21 + """
22 +
23 + model_type = "colpali"
24 +
25 + def __init__(
26 + self,
27 + embedding_dim: int | None = None,
28 + embed_dim: int | None = None,
29 + dim: int | None = None,
30 + projection_dim: int | None = None,
31 + colbert_dim: int | None = None,
32 + pooling: str | None = None,
33 + vlm_config: dict | None = None,
34 + **kwargs,
35 + ):
36 + # Store embedding projection config fields
37 + self.embedding_dim = embedding_dim
38 + self.embed_dim = embed_dim
39 + self.dim = dim
40 + self.projection_dim = projection_dim
41 + self.colbert_dim = colbert_dim
42 + self.pooling = pooling
43 +
44 + # The HF checkpoint nests PaliGemma config inside "vlm_config".
45 + # Flatten it so PaliGemmaConfig receives vision_config, text_config,
46 + # image_token_index, etc. directly.
47 + # Use setdefault to avoid overwriting keys already set (e.g.
48 + # model_type="colpali" would be clobbered by "paligemma" from
49 + # vlm_config).
50 + if vlm_config is not None:
51 + vlm_dict = (
52 + vlm_config if isinstance(vlm_config, dict) else vlm_config.to_dict()
53 + )
54 + _conflicting = {"model_type", "_name_or_path"}
55 + for key, value in vlm_dict.items():
56 + if key not in _conflicting:
57 + kwargs.setdefault(key, value)
58 +
59 + super().__init__(**kwargs)