A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 12:45 UTC
[Models] Allow converting Qwen3-VL into Reranker model (#31890)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jan 8, 2026 at 16:10 UTC
eac3b96ec04d07a987823504671650a0bcad5a10
8 files changed
+287
-13
docs/models/supported_models.md
+9
@@ -800,6 +800,7 @@ The following table lists those that are tested in vLLM.
800
| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
801
| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
802
| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
803
+| `Qwen3VLForConditionalGeneration`<sup>C</sup> | Qwen3-VL | T + I + V | `Qwen/Qwen3-VL-Embedding-2B`, etc. | ✅︎ | ✅︎ |
804
| `SiglipModel` | SigLIP, SigLIP2 | T / I | `google/siglip-base-patch16-224`, `google/siglip2-base-patch16-224` | | |
805
| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
806
@@ -816,10 +817,18 @@ These models primarily support the [`LLM.score`](./pooling_models.md#llmscore) A
817
| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
818
|--------------|--------|--------|-------------------|----------------------|---------------------------|
819
| `JinaVLForSequenceClassification` | JinaVL-based | T + I<sup>E+</sup> | `jinaai/jina-reranker-m0`, etc. | ✅︎ | ✅︎ |
820
+| `Qwen3VLForSequenceClassification` | Qwen3-VL-Reranker | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-Reranker-2B`(see note), etc. | ✅︎ | ✅︎ |
821
822
<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./pooling_models.md#model-conversion))
823
\* Feature support is the same as that of the original model.
824
825
+!!! note
826
+ Similar to Qwen3-Reranker, you need to use the following `--hf_overrides` to load the official original `Qwen3-VL-Reranker`.
827
+
828
+ ```bash
829
+ vllm serve Qwen/Qwen3-VL-Reranker-2B --hf_overrides '{"architectures": ["Qwen3VLForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}'
830
+ ```
831
+
832
## Model Support Policy
833
834
At vLLM, we are committed to facilitating the integration and support of third-party models within our ecosystem. Our approach is designed to balance the need for robustness and the practical limitations of supporting a wide range of models. Here’s how we manage third-party model support:
examples/pooling/pooling/vision_language_pooling.py
+31
@@ -133,6 +133,36 @@ def run_jinavl_reranker(query: Query) -> ModelRequestData:
133
)
134
135
136
+def run_qwen3_vl(query: Query) -> ModelRequestData:
137
+ image_placeholder = "<vision_start><|image_pad|><vision_end>"
138
+ if query["modality"] == "text":
139
+ prompt = query["text"]
140
+ image = None
141
+ elif query["modality"] == "image":
142
+ prompt = image_placeholder
143
+ image = query["image"]
144
+ elif query["modality"] == "text+image":
145
+ text = query["text"]
146
+ prompt = f"{image_placeholder}\n{text}"
147
+ image = query["image"]
148
+ else:
149
+ modality = query["modality"]
150
+ raise ValueError(f"Unsupported query modality: '{modality}'")
151
+
152
+ engine_args = EngineArgs(
153
+ model="Qwen/Qwen3-VL-Embedding-2B",
154
+ runner="pooling",
155
+ max_model_len=8192,
156
+ limit_mm_per_prompt={"image": 1},
157
+ )
158
+
159
+ return ModelRequestData(
160
+ engine_args=engine_args,
161
+ prompt=prompt,
162
+ image=image,
163
+ )
164
+
165
+
166
def run_siglip(query: Query) -> ModelRequestData:
167
if query["modality"] == "text":
168
prompt = query["text"]
@@ -353,6 +383,7 @@ model_example_map = {
383
"clip": run_clip,
384
"e5_v": run_e5_v,
385
"jinavl_reranker": run_jinavl_reranker,
386
+ "qwen3_vl": run_qwen3_vl,
387
"siglip": run_siglip,
388
"vlm2vec_phi3v": run_vlm2vec_phi3v,
389
"vlm2vec_qwen2vl": run_vlm2vec_qwen2vl,
examples/pooling/score/template/qwen3_vl_reranker.jinja
new
+23
@@ -0,0 +1,23 @@
1
+<|im_start|>system
2
+Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>
3
+<|im_start|>user
4
+<Instruct>: {{
5
+ messages
6
+ | selectattr("role", "eq", "system")
7
+ | map(attribute="content")
8
+ | first
9
+ | default("Given a search query, retrieve relevant candidates that answer the query.")
10
+}}<Query>:{{
11
+ messages
12
+ | selectattr("role", "eq", "query")
13
+ | map(attribute="content")
14
+ | first
15
+}}
16
+<Document>:{{
17
+ messages
18
+ | selectattr("role", "eq", "document")
19
+ | map(attribute="content")
20
+ | first
21
+}}<|im_end|>
22
+<|im_start|>assistant
23
+
examples/pooling/score/vision_language_reranker.py
new
+172
@@ -0,0 +1,172 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+This example shows how to use vLLM for running offline inference with
5
+vision language reranker models for multimodal scoring tasks.
6
+
7
+Vision language rerankers score the relevance between a text query and
8
+multimodal documents (text + images/videos).
9
+"""
10
+
11
+from argparse import Namespace
12
+from collections.abc import Callable
13
+from dataclasses import asdict
14
+from pathlib import Path
15
+from typing import NamedTuple
16
+
17
+from vllm import LLM, EngineArgs
18
+from vllm.entrypoints.score_utils import ScoreMultiModalParam
19
+from vllm.utils.argparse_utils import FlexibleArgumentParser
20
+
21
+TEMPLATE_HOME = Path(__file__).parent / "template"
22
+
23
+
24
+class RerankModelData(NamedTuple):
25
+ engine_args: EngineArgs
26
+ chat_template: str | None = None
27
+
28
+
29
+def run_jinavl_reranker(modality: str) -> RerankModelData:
30
+ assert modality == "image"
31
+
32
+ engine_args = EngineArgs(
33
+ model="jinaai/jina-reranker-m0",
34
+ runner="pooling",
35
+ max_model_len=32768,
36
+ trust_remote_code=True,
37
+ mm_processor_kwargs={
38
+ "min_pixels": 3136,
39
+ "max_pixels": 602112,
40
+ },
41
+ limit_mm_per_prompt={modality: 1},
42
+ )
43
+ return RerankModelData(
44
+ engine_args=engine_args,
45
+ )
46
+
47
+
48
+def run_qwen3_vl_reranker(modality: str) -> RerankModelData:
49
+ engine_args = EngineArgs(
50
+ model="Qwen/Qwen3-VL-Reranker-2B",
51
+ runner="pooling",
52
+ max_model_len=16384,
53
+ limit_mm_per_prompt={modality: 1},
54
+ # HuggingFace model configuration overrides required for compatibility
55
+ hf_overrides={
56
+ # Manually route to sequence classification architecture
57
+ # This tells vLLM to use Qwen3VLForSequenceClassification instead of
58
+ # the default Qwen3VLForConditionalGeneration
59
+ "architectures": ["Qwen3VLForSequenceClassification"],
60
+ # Specify which token logits to extract from the language model head
61
+ # The original reranker uses "no" and "yes" token logits for scoring
62
+ "classifier_from_token": ["no", "yes"],
63
+ # Enable special handling for original Qwen3-Reranker models
64
+ # This flag triggers conversion logic that transforms the two token
65
+ # vectors into a single classification vector
66
+ "is_original_qwen3_reranker": True,
67
+ },
68
+ )
69
+ chat_template_path = "qwen3_vl_reranker.jinja"
70
+ chat_template = (TEMPLATE_HOME / chat_template_path).read_text()
71
+ return RerankModelData(
72
+ engine_args=engine_args,
73
+ chat_template=chat_template,
74
+ )
75
+
76
+
77
+model_example_map: dict[str, Callable[[str], RerankModelData]] = {
78
+ "jinavl_reranker": run_jinavl_reranker,
79
+ "qwen3_vl_reranker": run_qwen3_vl_reranker,
80
+}
81
+
82
+
83
+def parse_args():
84
+ parser = FlexibleArgumentParser(
85
+ description="Demo on using vLLM for offline inference with "
86
+ "vision language reranker models for multimodal scoring tasks."
87
+ )
88
+ parser.add_argument(
89
+ "--model-name",
90
+ "-m",
91
+ type=str,
92
+ default="jinavl_reranker",
93
+ choices=model_example_map.keys(),
94
+ help="The name of the reranker model.",
95
+ )
96
+ parser.add_argument(
97
+ "--modality",
98
+ type=str,
99
+ default="image",
100
+ choices=["image", "video"],
101
+ help="Modality of the multimodal input (image or video).",
102
+ )
103
+ return parser.parse_args()
104
+
105
+
106
+def get_multi_modal_input(modality: str) -> tuple[str, ScoreMultiModalParam]:
107
+ # Sample query for testing the reranker
108
+ if modality == "image":
109
+ query = "A woman playing with her dog on a beach at sunset."
110
+ # Sample multimodal documents to be scored against the query
111
+ # Each document contains an image URL that will be fetched and processed
112
+ documents: ScoreMultiModalParam = {
113
+ "content": [
114
+ {
115
+ "type": "text",
116
+ "text": (
117
+ "A woman shares a joyful moment with her golden retriever on a sun-drenched beach at sunset, " # noqa: E501
118
+ "as the dog offers its paw in a heartwarming display of companionship and trust." # noqa: E501
119
+ ),
120
+ },
121
+ {
122
+ "type": "image_url",
123
+ "image_url": {
124
+ "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
125
+ },
126
+ },
127
+ ]
128
+ }
129
+ elif modality == "video":
130
+ query = "A girl is drawing pictures on an ipad."
131
+ # Sample video documents to be scored against the query
132
+ documents: ScoreMultiModalParam = {
133
+ "content": [
134
+ {
135
+ "type": "text",
136
+ "text": "A girl is drawing a guitar on her ipad with Apple Pencil.",
137
+ },
138
+ {
139
+ "type": "video_url",
140
+ "video_url": {
141
+ "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4"
142
+ },
143
+ },
144
+ ]
145
+ }
146
+ else:
147
+ raise ValueError(f"Unsupported modality: {modality}")
148
+ return query, documents
149
+
150
+
151
+def main(args: Namespace):
152
+ # Run the selected reranker model
153
+ modality = args.modality
154
+ model_request = model_example_map[args.model_name](modality)
155
+ engine_args = model_request.engine_args
156
+
157
+ llm = LLM(**asdict(engine_args))
158
+
159
+ query, documents = get_multi_modal_input(modality)
160
+ outputs = llm.score(query, documents, chat_template=model_request.chat_template)
161
+
162
+ print("-" * 50)
163
+ print(f"Model: {engine_args.model}")
164
+ print(f"Modality: {modality}")
165
+ print(f"Query: {query}")
166
+ print("Relevance scores:", [output.outputs.score for output in outputs])
167
+ print("-" * 50)
168
+
169
+
170
+if __name__ == "__main__":
171
+ args = parse_args()
172
+ main(args)
tests/models/registry.py
+9
@@ -612,6 +612,15 @@ _AUTOMATIC_CONVERTED_MODELS = {
612
"tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
613
),
614
"Qwen3ForTokenClassification": _HfExamplesInfo("bd2lcco/Qwen3-0.6B-finetuned"),
615
+ "Qwen3VLForSequenceClassification": _HfExamplesInfo(
616
+ "Qwen/Qwen3-VL-Reranker-2B",
617
+ is_available_online=False,
618
+ hf_overrides={
619
+ "architectures": ["Qwen3VLForSequenceClassification"],
620
+ "classifier_from_token": ["no", "yes"],
621
+ "is_original_qwen3_reranker": True,
622
+ },
623
+ ),
624
}
625
626
_MULTIMODAL_EXAMPLE_MODELS = {
vllm/entrypoints/score_utils.py
+2
@@ -11,6 +11,7 @@ from vllm.entrypoints.chat_utils import (
11
ChatCompletionContentPartImageEmbedsParam,
12
ChatCompletionContentPartImageParam,
13
ChatCompletionContentPartTextParam,
14
+ ChatCompletionContentPartVideoParam,
15
ChatTemplateResolutionError,
16
MultiModalItemTracker,
17
_ContentPart,
@@ -27,6 +28,7 @@ ScoreContentPartParam: TypeAlias = (
28
ChatCompletionContentPartImageParam
29
| ChatCompletionContentPartImageEmbedsParam
30
| ChatCompletionContentPartTextParam
31
+ | ChatCompletionContentPartVideoParam
32
)
33
34
vllm/model_executor/models/adapters.py
+36
-13
@@ -333,9 +333,14 @@ def as_seq_cls_model(cls: _T) -> _T:
333
)
334
335
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
336
- text_config = self.config.get_text_config()
337
- tokens = getattr(text_config, "classifier_from_token", None)
338
- method = getattr(text_config, "method", None)
336
+ hf_config = self.config
337
+ text_config = hf_config.get_text_config()
338
+ tokens = getattr(
339
+ hf_config,
340
+ "classifier_from_token",
341
+ getattr(text_config, "classifier_from_token", None),
342
+ )
343
+ method = getattr(hf_config, "method", getattr(text_config, "method", None))
344
345
def auto_set_score_bias(weights):
346
for name, weight in weights:
@@ -366,9 +371,14 @@ def as_seq_cls_model(cls: _T) -> _T:
371
class SequenceClassificationConfig(VerifyAndUpdateConfig):
372
@staticmethod
373
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
369
- text_config = vllm_config.model_config.hf_config.get_text_config()
370
- method = getattr(text_config, "method", None)
371
- tokens = getattr(text_config, "classifier_from_token", None)
374
+ hf_config = vllm_config.model_config.hf_config
375
+ text_config = hf_config.get_text_config()
376
+ method = getattr(hf_config, "method", getattr(text_config, "method", None))
377
+ tokens = getattr(
378
+ hf_config,
379
+ "classifier_from_token",
380
+ getattr(text_config, "classifier_from_token", None),
381
+ )
382
383
if method is None:
384
return
@@ -378,8 +388,10 @@ class SequenceClassificationConfig(VerifyAndUpdateConfig):
388
389
if method == "from_2_way_softmax":
390
assert len(tokens) == 2
391
+ hf_config.num_labels = 1
392
text_config.num_labels = 1
393
else:
394
+ hf_config.num_labels = len(tokens)
395
text_config.num_labels = len(tokens)
396
397
# `llm as reranker` defaults to not using separating token.
@@ -396,9 +408,14 @@ def load_weights_using_from_2_way_softmax(
408
409
model_config = model.vllm_config.model_config
410
quant_config = model.vllm_config.quant_config
399
- text_config = model.config.get_text_config()
411
+ hf_config = model.config
412
+ text_config = hf_config.get_text_config()
413
401
- tokens = getattr(text_config, "classifier_from_token", [])
414
+ tokens = getattr(
415
+ hf_config,
416
+ "classifier_from_token",
417
+ getattr(text_config, "classifier_from_token", []),
418
+ )
419
tokens = cast(list[int], tokens)
420
assert len(tokens) == 2
421
@@ -409,10 +426,15 @@ def load_weights_using_from_2_way_softmax(
426
# embed_tokens is the assumed name for input embeddings. If the model does not
427
# have this attribute, we fall back to get_input_embeddings(), which is used by
428
# the Transformers modeling backend.
429
+ text_backbone = (
430
+ model.get_language_model().model
431
+ if hasattr(model, "get_language_model")
432
+ else model.model
433
+ )
434
embed_tokens = (
413
- model.model.embed_tokens
414
- if hasattr(model.model, "embed_tokens")
415
- else model.model.get_input_embeddings()
435
+ text_backbone.embed_tokens
436
+ if hasattr(text_backbone, "embed_tokens")
437
+ else text_backbone.get_input_embeddings()
438
)
439
model.lm_head = model.lm_head.tie_weights(embed_tokens)
440
@@ -516,8 +538,9 @@ def seq_cls_model_loader(model, weights: Iterable[tuple[str, torch.Tensor]]):
538
# - GemmaForCausalLM
539
# - bge-reranker-v2-gemma
540
519
- text_config = model.vllm_config.model_config.hf_config.get_text_config()
520
- method = getattr(text_config, "method", None)
541
+ hf_config = model.vllm_config.model_config.hf_config
542
+ text_config = hf_config.get_text_config()
543
+ method = getattr(hf_config, "method", getattr(text_config, "method", None))
544
assert method in SEQ_CLS_LOAD_METHODS, f"method {method} not supported"
545
return SEQ_CLS_LOAD_METHODS[method](model, weights)
546
vllm/model_executor/models/config.py
+5
@@ -256,6 +256,10 @@ class Qwen3ForSequenceClassificationConfig(VerifyAndUpdateConfig):
256
model_config.hf_config.method = "from_2_way_softmax"
257
258
259
+class Qwen3VLForSequenceClassificationConfig(Qwen3ForSequenceClassificationConfig):
260
+ pass
261
+
262
+
263
class JinaVLForSequenceClassificationConfig(VerifyAndUpdateConfig):
264
@staticmethod
265
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
@@ -551,6 +555,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
555
"Qwen2ForProcessRewardModel": Qwen2ForProcessRewardModelConfig,
556
"Qwen2ForRewardModel": Qwen2ForRewardModelConfig,
557
"Qwen3ForSequenceClassification": Qwen3ForSequenceClassificationConfig,
558
+ "Qwen3VLForSequenceClassification": Qwen3VLForSequenceClassificationConfig,
559
"XLMRobertaModel": JinaRobertaModelConfig,
560
"JinaVLForRanking": JinaVLForSequenceClassificationConfig,
561
"JambaForSequenceClassification": JambaForSequenceClassificationConfig,