A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 13:15 UTC
[Model] Add ColQwen3.5 4.5B support (#36887)
Signed-off-by: Athrael Soju <athrael.soju@gmail.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
Athrael Soju committed
Mar 17, 2026 at 21:17 UTC
c0745a851a4f6d9a3651d768abb1c14ab8353827
8 files changed
+579
docs/models/pooling_models.md
+40
@@ -625,6 +625,46 @@ curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
625
}'
626
```
627
628
+### ColQwen3.5 Multi-Modal Late Interaction Models
629
+
630
+ColQwen3.5 is based on [ColPali](https://arxiv.org/abs/2407.01449), extending ColBERT's late interaction approach to **multi-modal** inputs. It uses the Qwen3.5 hybrid backbone (linear + full attention) and produces per-token L2-normalized vectors for MaxSim scoring.
631
+
632
+| Architecture | Backbone | Example HF Models |
633
+| - | - | - |
634
+| `ColQwen3_5` | Qwen3.5 | `athrael-soju/colqwen3.5-4.5B` |
635
+
636
+Start the server:
637
+
638
+```shell
639
+vllm serve athrael-soju/colqwen3.5-4.5B --max-model-len 4096
640
+```
641
+
642
+Then you can use the rerank endpoint:
643
+
644
+```shell
645
+curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
646
+ "model": "athrael-soju/colqwen3.5-4.5B",
647
+ "query": "What is machine learning?",
648
+ "documents": [
649
+ "Machine learning is a subset of artificial intelligence.",
650
+ "Python is a programming language.",
651
+ "Deep learning uses neural networks."
652
+ ]
653
+}'
654
+```
655
+
656
+Or the score endpoint:
657
+
658
+```shell
659
+curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
660
+ "model": "athrael-soju/colqwen3.5-4.5B",
661
+ "text_1": "What is the capital of France?",
662
+ "text_2": ["The capital of France is Paris.", "Python is a programming language."]
663
+}'
664
+```
665
+
666
+An example can be found here: [examples/pooling/score/colqwen3_5_rerank_online.py](../../examples/pooling/score/colqwen3_5_rerank_online.py)
667
+
668
### BAAI/bge-m3
669
670
The `BAAI/bge-m3` model comes with extra weights for sparse and colbert embeddings but unfortunately in its `config.json`
docs/models/supported_models.md
+1
@@ -834,6 +834,7 @@ The following table lists those that are tested in vLLM.
834
| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
835
| `ColModernVBertForRetrieval` | ColModernVBERT | T / I | `ModernVBERT/colmodernvbert-merged` | | |
836
| `ColPaliForRetrieval` | ColPali | T / I | `vidore/colpali-v1.3-hf` | | |
837
+| `ColQwen3_5` | ColQwen3.5 | T + I + V | `athrael-soju/colqwen3.5-4.5B-v3` | | |
838
| `LlamaNemotronVLModel` | Llama Nemotron Embedding + SigLIP | T + I | `nvidia/llama-nemotron-embed-vl-1b-v2` | | |
839
| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
840
| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
examples/pooling/score/colqwen3_5_rerank_online.py
new
+130
@@ -0,0 +1,130 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Example of using ColQwen3.5 late interaction model for reranking.
5
+
6
+ColQwen3.5 is a multi-modal ColBERT-style model based on Qwen3.5.
7
+It produces per-token embeddings and uses MaxSim scoring for retrieval
8
+and reranking. Supports both text and image inputs.
9
+
10
+Start the server with:
11
+ vllm serve athrael-soju/colqwen3.5-4.5B --max-model-len 4096
12
+
13
+Then run this script:
14
+ python colqwen3_5_rerank_online.py
15
+"""
16
+
17
+import requests
18
+
19
+MODEL = "athrael-soju/colqwen3.5-4.5B"
20
+BASE_URL = "http://127.0.0.1:8000"
21
+
22
+headers = {"accept": "application/json", "Content-Type": "application/json"}
23
+
24
+
25
+def rerank_text():
26
+ """Text-only reranking via /rerank endpoint."""
27
+ print("=" * 60)
28
+ print("1. Text reranking (/rerank)")
29
+ print("=" * 60)
30
+
31
+ data = {
32
+ "model": MODEL,
33
+ "query": "What is machine learning?",
34
+ "documents": [
35
+ "Machine learning is a subset of artificial intelligence.",
36
+ "Python is a programming language.",
37
+ "Deep learning uses neural networks for complex tasks.",
38
+ "The weather today is sunny.",
39
+ ],
40
+ }
41
+
42
+ response = requests.post(f"{BASE_URL}/rerank", headers=headers, json=data)
43
+
44
+ if response.status_code == 200:
45
+ result = response.json()
46
+ print("\n Ranked documents (most relevant first):")
47
+ for item in result["results"]:
48
+ doc_idx = item["index"]
49
+ score = item["relevance_score"]
50
+ print(f" [{score:.4f}] {data['documents'][doc_idx]}")
51
+ else:
52
+ print(f" Request failed: {response.status_code}")
53
+ print(f" {response.text[:300]}")
54
+
55
+
56
+def score_text():
57
+ """Text-only scoring via /score endpoint."""
58
+ print()
59
+ print("=" * 60)
60
+ print("2. Text scoring (/score)")
61
+ print("=" * 60)
62
+
63
+ query = "What is the capital of France?"
64
+ documents = [
65
+ "The capital of France is Paris.",
66
+ "Berlin is the capital of Germany.",
67
+ "Python is a programming language.",
68
+ ]
69
+
70
+ data = {
71
+ "model": MODEL,
72
+ "text_1": query,
73
+ "text_2": documents,
74
+ }
75
+
76
+ response = requests.post(f"{BASE_URL}/score", headers=headers, json=data)
77
+
78
+ if response.status_code == 200:
79
+ result = response.json()
80
+ print(f"\n Query: {query}\n")
81
+ for item in result["data"]:
82
+ idx = item["index"]
83
+ score = item["score"]
84
+ print(f" Doc {idx} (score={score:.4f}): {documents[idx]}")
85
+ else:
86
+ print(f" Request failed: {response.status_code}")
87
+ print(f" {response.text[:300]}")
88
+
89
+
90
+def score_text_top_n():
91
+ """Text reranking with top_n filtering via /rerank endpoint."""
92
+ print()
93
+ print("=" * 60)
94
+ print("3. Text reranking with top_n=2 (/rerank)")
95
+ print("=" * 60)
96
+
97
+ data = {
98
+ "model": MODEL,
99
+ "query": "What is the capital of France?",
100
+ "documents": [
101
+ "The capital of France is Paris.",
102
+ "Berlin is the capital of Germany.",
103
+ "Python is a programming language.",
104
+ "The Eiffel Tower is in Paris.",
105
+ ],
106
+ "top_n": 2,
107
+ }
108
+
109
+ response = requests.post(f"{BASE_URL}/rerank", headers=headers, json=data)
110
+
111
+ if response.status_code == 200:
112
+ result = response.json()
113
+ print(f"\n Top {data['top_n']} results:")
114
+ for item in result["results"]:
115
+ doc_idx = item["index"]
116
+ score = item["relevance_score"]
117
+ print(f" [{score:.4f}] {data['documents'][doc_idx]}")
118
+ else:
119
+ print(f" Request failed: {response.status_code}")
120
+ print(f" {response.text[:300]}")
121
+
122
+
123
+def main():
124
+ rerank_text()
125
+ score_text()
126
+ score_text_top_n()
127
+
128
+
129
+if __name__ == "__main__":
130
+ main()
tests/models/multimodal/pooling/test_colqwen3_5.py
new
+154
@@ -0,0 +1,154 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Tests for ColQwen3.5 late interaction model for multi-modal retrieval.
4
+
5
+ColQwen3.5 is a multi-vector retrieval model based on Qwen3.5 backbone with
6
+ColBERT-style late interaction scoring (MaxSim). It produces per-token
7
+embeddings for both text and image inputs.
8
+"""
9
+
10
+import pytest
11
+import torch
12
+
13
+from ....conftest import VllmRunner
14
+
15
+MODELS = [
16
+ "athrael-soju/colqwen3.5-4.5B-v3",
17
+]
18
+
19
+EMBED_DIMS = {
20
+ "athrael-soju/colqwen3.5-4.5B-v3": 320,
21
+}
22
+
23
+TEXT_QUERIES = [
24
+ "What is the capital of France?",
25
+ "Describe the contents of the document.",
26
+]
27
+
28
+TEXT_DOCUMENTS = [
29
+ "The capital of France is Paris.",
30
+ "This document contains important financial data.",
31
+]
32
+
33
+DTYPE = "half"
34
+
35
+
36
+def _run_token_embed_test(
37
+ vllm_runner: type[VllmRunner],
38
+ model: str,
39
+ *,
40
+ dtype: str,
41
+) -> None:
42
+ """Verify per-token embedding shape and L2 normalization."""
43
+ with vllm_runner(
44
+ model,
45
+ runner="pooling",
46
+ dtype=dtype,
47
+ max_model_len=4096,
48
+ enforce_eager=True,
49
+ ) as vllm_model:
50
+ outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
51
+
52
+ assert len(outputs) == 1
53
+ emb = torch.tensor(outputs[0])
54
+ # Token embeddings should be 2D: [num_tokens, embed_dim]
55
+ assert emb.dim() == 2
56
+ assert emb.shape[1] == EMBED_DIMS[model]
57
+ assert emb.shape[0] > 1
58
+
59
+ # Verify L2 normalization
60
+ norms = torch.norm(emb, p=2, dim=-1)
61
+ torch.testing.assert_close(
62
+ norms,
63
+ torch.ones_like(norms),
64
+ rtol=1e-2,
65
+ atol=1e-2,
66
+ )
67
+
68
+
69
+def _run_late_interaction_test(
70
+ vllm_runner: type[VllmRunner],
71
+ model: str,
72
+ *,
73
+ dtype: str,
74
+) -> None:
75
+ """Verify MaxSim scoring matches manual computation."""
76
+ from vllm.entrypoints.pooling.score.utils import compute_maxsim_score
77
+
78
+ with vllm_runner(
79
+ model,
80
+ runner="pooling",
81
+ dtype=dtype,
82
+ max_model_len=4096,
83
+ enforce_eager=True,
84
+ ) as vllm_model:
85
+ q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
86
+ d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
87
+
88
+ q_emb = torch.tensor(q_outputs[0])
89
+ d_emb = torch.tensor(d_outputs[0])
90
+
91
+ manual_score = compute_maxsim_score(q_emb, d_emb).item()
92
+
93
+ vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
94
+
95
+ assert len(vllm_scores) == 1
96
+ assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
97
+
98
+
99
+def _run_relevance_test(
100
+ vllm_runner: type[VllmRunner],
101
+ model: str,
102
+ *,
103
+ dtype: str,
104
+) -> None:
105
+ """Verify that relevant documents score higher than irrelevant ones."""
106
+ query = "What is machine learning?"
107
+ documents = [
108
+ "Machine learning is a subset of artificial intelligence.",
109
+ "The weather forecast shows rain tomorrow.",
110
+ "Deep learning uses neural networks for complex tasks.",
111
+ ]
112
+
113
+ with vllm_runner(
114
+ model,
115
+ runner="pooling",
116
+ dtype=dtype,
117
+ max_model_len=4096,
118
+ enforce_eager=True,
119
+ ) as vllm_model:
120
+ scores = vllm_model.score(query, documents)
121
+
122
+ assert len(scores) == 3
123
+ assert scores[0] > scores[1], "ML doc should score higher than weather doc"
124
+ assert scores[2] > scores[1], "DL doc should score higher than weather doc"
125
+
126
+
127
+@pytest.mark.parametrize("model", MODELS)
128
+@pytest.mark.parametrize("dtype", [DTYPE])
129
+def test_colqwen3_5_token_embed(
130
+ vllm_runner,
131
+ model: str,
132
+ dtype: str,
133
+) -> None:
134
+ _run_token_embed_test(vllm_runner, model, dtype=dtype)
135
+
136
+
137
+@pytest.mark.parametrize("model", MODELS)
138
+@pytest.mark.parametrize("dtype", [DTYPE])
139
+def test_colqwen3_5_late_interaction_scoring(
140
+ vllm_runner,
141
+ model: str,
142
+ dtype: str,
143
+) -> None:
144
+ _run_late_interaction_test(vllm_runner, model, dtype=dtype)
145
+
146
+
147
+@pytest.mark.parametrize("model", MODELS)
148
+@pytest.mark.parametrize("dtype", [DTYPE])
149
+def test_colqwen3_5_relevance_ordering(
150
+ vllm_runner,
151
+ model: str,
152
+ dtype: str,
153
+) -> None:
154
+ _run_relevance_test(vllm_runner, model, dtype=dtype)
tests/models/registry.py
+5
@@ -639,6 +639,11 @@ _LATE_INTERACTION_EXAMPLE_MODELS = {
639
"OpsColQwen3Model": _HfExamplesInfo(
640
"OpenSearch-AI/Ops-Colqwen3-4B", trust_remote_code=True
641
),
642
+ "ColQwen3_5": _HfExamplesInfo(
643
+ "athrael-soju/colqwen3.5-4.5B-v3",
644
+ trust_remote_code=True,
645
+ max_model_len=4096,
646
+ ),
647
"Qwen3VLNemotronEmbedModel": _HfExamplesInfo(
648
"nvidia/nemotron-colembed-vl-4b-v2",
649
),
vllm/model_executor/models/colqwen3_5.py
new
+246
@@ -0,0 +1,246 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+ColQwen3.5 late interaction model for multi-modal retrieval and reranking.
5
+
6
+ColQwen3.5 extends Qwen3.5 with a ColBERT-style late interaction head,
7
+producing per-token embeddings for both text and image inputs. It uses
8
+MaxSim scoring for retrieval/reranking tasks.
9
+
10
+This model supports the "token_embed" pooling task and is designed for
11
+multi-vector retrieval of documents containing both text and images.
12
+
13
+Reference: https://arxiv.org/abs/2407.01449 (ColPali)
14
+Based on: Qwen3.5 backbone with custom text projection
15
+
16
+Target models:
17
+- athrael-soju/colqwen3.5-4.5B-v3
18
+"""
19
+
20
+from collections.abc import Iterable, Mapping
21
+
22
+import torch
23
+import torch.nn as nn
24
+from transformers.models.qwen3_vl import Qwen3VLProcessor
25
+
26
+from vllm.config import VllmConfig
27
+from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
28
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29
+from vllm.multimodal import MULTIMODAL_REGISTRY
30
+
31
+from .interfaces import SupportsLateInteraction
32
+from .interfaces_base import default_pooling_type
33
+from .qwen2_vl import Qwen2VLMultiModalDataParser
34
+from .qwen3_5 import (
35
+ Qwen3_5ForConditionalGeneration,
36
+ Qwen3_5ProcessingInfo,
37
+)
38
+from .qwen3_vl import (
39
+ Qwen3VLDummyInputsBuilder,
40
+ Qwen3VLMultiModalProcessor,
41
+)
42
+from .utils import AutoWeightsLoader, WeightsMapper
43
+
44
+
45
+class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
46
+ """Processing info for ColQwen3.5 models.
47
+
48
+ ColQwen3.5 models use custom HuggingFace processors (e.g.
49
+ ColQwen3_5Processor) that are incompatible with vLLM's
50
+ Qwen3VLMultiModalProcessor. We override get_hf_config() and
51
+ get_hf_processor() to skip the strict type check and force the
52
+ standard Qwen3VLProcessor.
53
+ """
54
+
55
+ def get_hf_config(self):
56
+ return self.ctx.get_hf_config()
57
+
58
+ def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:
59
+ return self.ctx.get_hf_processor(
60
+ Qwen3VLProcessor,
61
+ use_fast=kwargs.pop("use_fast", True),
62
+ **kwargs,
63
+ )
64
+
65
+ @property
66
+ def _supports_video(self) -> bool:
67
+ """Check if the HF processor supports video inputs."""
68
+ return hasattr(self.get_hf_processor(), "video_processor")
69
+
70
+ def get_video_processor(self, **kwargs: object):
71
+ if not self._supports_video:
72
+ raise AttributeError(
73
+ f"The processor for {self.ctx.model_config.model} does not "
74
+ "support video inputs (no video_processor attribute)."
75
+ )
76
+ return self.get_hf_processor(**kwargs).video_processor # type: ignore[attr-defined]
77
+
78
+ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
79
+ limits: dict[str, int | None] = {"image": None}
80
+ if self._supports_video:
81
+ limits["video"] = None
82
+ return limits
83
+
84
+ def get_mm_max_tokens_per_item(
85
+ self,
86
+ seq_len: int,
87
+ mm_counts: Mapping[str, int],
88
+ ) -> Mapping[str, int]:
89
+ max_image_tokens = self.get_max_image_tokens()
90
+ result: dict[str, int] = {"image": max_image_tokens}
91
+ if self._supports_video:
92
+ max_video_tokens = self.get_max_video_tokens(seq_len, mm_counts)
93
+ result["video"] = max_video_tokens
94
+ return result
95
+
96
+ def get_data_parser(self):
97
+ hf_config = self.get_hf_config()
98
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
99
+ return Qwen2VLMultiModalDataParser(
100
+ spatial_merge_size,
101
+ video_needs_metadata=self._supports_video,
102
+ expected_hidden_size=self._get_expected_hidden_size(),
103
+ )
104
+
105
+
106
+@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
107
+@MULTIMODAL_REGISTRY.register_processor(
108
+ Qwen3VLMultiModalProcessor,
109
+ info=ColQwen3_5ProcessingInfo,
110
+ dummy_inputs=Qwen3VLDummyInputsBuilder,
111
+)
112
+class ColQwen3_5Model(
113
+ Qwen3_5ForConditionalGeneration,
114
+ SupportsLateInteraction,
115
+):
116
+ """ColQwen3.5 late interaction model for multi-modal retrieval/reranking.
117
+
118
+ This model extends Qwen3_5ForConditionalGeneration with a ColBERT-style
119
+ linear projection layer for per-token embeddings. It supports:
120
+ - "token_embed" task: Per-token embeddings for late interaction scoring
121
+
122
+ The model produces per-token embeddings by:
123
+ 1. Running the Qwen3.5 backbone (vision + language) to get hidden states
124
+ 2. Projecting hidden states through a linear layer (hidden_size -> embed_dim)
125
+ 3. L2 normalization is handled by the pooler via PoolerNormalize
126
+
127
+ Attributes:
128
+ custom_text_proj: Linear projection from hidden_size to embed_dim
129
+ """
130
+
131
+ # Mark this as a pooling model so vLLM routes to pooler path
132
+ is_pooling_model = True
133
+
134
+ # Override hf_to_vllm_mapper to handle ColQwen3.5 weight naming.
135
+ # ColPali saves weights as "language_model.*" but vLLM's
136
+ # Qwen3_5ForCausalLM has them under "language_model.model.*".
137
+ # Visual weights ("visual.*") already match the vLLM module path.
138
+ hf_to_vllm_mapper = WeightsMapper(
139
+ orig_to_new_prefix={
140
+ "language_model.": "language_model.model.",
141
+ }
142
+ )
143
+
144
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
145
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
146
+
147
+ config = vllm_config.model_config.hf_config
148
+ head_dtype = vllm_config.model_config.head_dtype
149
+
150
+ hidden_size = getattr(config, "hidden_size", None)
151
+ if hidden_size is None and hasattr(config, "text_config"):
152
+ hidden_size = config.text_config.hidden_size
153
+ if hidden_size is None:
154
+ raise ValueError(
155
+ "Unable to determine text hidden size from config. "
156
+ "Expected 'hidden_size' or 'text_config.hidden_size'."
157
+ )
158
+
159
+ # (ColPali: dim, projection_dim, colbert_dim)
160
+ self.embed_dim: int = (
161
+ getattr(config, "embed_dim", None)
162
+ or getattr(config, "dims", None)
163
+ or getattr(config, "dim", None)
164
+ or getattr(config, "projection_dim", None)
165
+ or getattr(config, "colbert_dim", None)
166
+ or 128 # default from reference implementation
167
+ )
168
+
169
+ self.custom_text_proj = nn.Linear(
170
+ hidden_size,
171
+ self.embed_dim,
172
+ bias=False,
173
+ dtype=head_dtype,
174
+ )
175
+
176
+ pooler_config = vllm_config.model_config.pooler_config
177
+ assert pooler_config is not None
178
+ self.pooler = pooler_for_token_embed(
179
+ pooler_config,
180
+ projector=None,
181
+ )
182
+
183
+ def forward(
184
+ self,
185
+ input_ids: torch.Tensor | None,
186
+ positions: torch.Tensor,
187
+ intermediate_tensors=None,
188
+ inputs_embeds: torch.Tensor | None = None,
189
+ **kwargs: object,
190
+ ) -> torch.Tensor:
191
+ """Run forward pass producing per-token embeddings."""
192
+ hidden_states = super().forward(
193
+ input_ids=input_ids,
194
+ positions=positions,
195
+ intermediate_tensors=intermediate_tensors,
196
+ inputs_embeds=inputs_embeds,
197
+ **kwargs,
198
+ )
199
+
200
+ if not isinstance(hidden_states, torch.Tensor):
201
+ return hidden_states # type: ignore
202
+
203
+ proj_dtype = self.custom_text_proj.weight.dtype
204
+ if hidden_states.dtype != proj_dtype:
205
+ hidden_states = hidden_states.to(proj_dtype)
206
+
207
+ # Project to embedding dimension (normalization handled by pooler)
208
+ return self.custom_text_proj(hidden_states)
209
+
210
+ # Names used for the projection layer across different ColQwen3.5 variants
211
+ _PROJ_LAYER_NAMES = {
212
+ "custom_text_proj", # ColPali naming
213
+ "embedding_proj_layer", # Alternative naming
214
+ }
215
+
216
+ def _is_proj_weight(self, name: str) -> bool:
217
+ """Check if a weight name belongs to the projection layer."""
218
+ return any(proj_name in name for proj_name in self._PROJ_LAYER_NAMES)
219
+
220
+ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
221
+ """Load weights with special handling for projection layer."""
222
+ weights_list = list(weights)
223
+ proj_weights: list[tuple[str, torch.Tensor]] = []
224
+ model_weights: list[tuple[str, torch.Tensor]] = []
225
+
226
+ for name, weight in weights_list:
227
+ if self._is_proj_weight(name):
228
+ proj_weights.append((name, weight))
229
+ else:
230
+ model_weights.append((name, weight))
231
+
232
+ loader = AutoWeightsLoader(
233
+ self,
234
+ skip_prefixes=["mtp."],
235
+ )
236
+ loaded = loader.load_weights(model_weights, mapper=self.hf_to_vllm_mapper)
237
+
238
+ for name, weight in proj_weights:
239
+ param_name = name.split(".")[-1]
240
+ param = getattr(self.custom_text_proj, param_name, None)
241
+ if param is not None:
242
+ weight = weight.to(device=param.device, dtype=param.dtype)
243
+ default_weight_loader(param, weight)
244
+ loaded.add(f"custom_text_proj.{param_name}")
245
+
246
+ return loaded
vllm/model_executor/models/config.py
+1
@@ -647,6 +647,7 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
647
648
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
649
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
650
+ "ColQwen3_5": Qwen3_5ForConditionalGenerationConfig,
651
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
652
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
653
"FalconMambaForCausalLM": MambaModelConfig,
vllm/model_executor/models/registry.py
+2
@@ -274,8 +274,10 @@ _LATE_INTERACTION_MODELS = {
274
"ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
275
# [Multimodal]
276
"ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
277
+ "ColPaliForRetrieval": ("colpali", "ColPaliModel"),
278
"ColQwen3": ("colqwen3", "ColQwen3Model"),
279
"OpsColQwen3Model": ("colqwen3", "ColQwen3Model"),
280
+ "ColQwen3_5": ("colqwen3_5", "ColQwen3_5Model"),
281
"Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"),
282
}
283