A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 10:00 UTC
[Bugfix] Fix Qwen3-VL-Reranker load. (#33298)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
wang.yuqi committed
Jan 29, 2026 at 16:42 UTC
abb34ac43a5a14dc0a918687e08e51abe73f74a5
6 files changed
+234
-112
examples/pooling/score/vision_rerank_api_online.py
+46
-49
@@ -18,48 +18,32 @@ e.g.
18
"""
19
20
import argparse
21
-import base64
22
-import json
21
+import pprint
22
23
import requests
24
26
-
27
-def encode_base64_content_from_url(content_url: str) -> dict[str, str]:
28
- """Encode a content retrieved from a remote url to base64 format."""
29
-
30
- with requests.get(content_url, headers=headers) as response:
31
- response.raise_for_status()
32
- result = base64.b64encode(response.content).decode("utf-8")
33
-
34
- return {"url": f"data:image/jpeg;base64,{result}"}
35
-
36
-
37
-headers = {"accept": "application/json", "Content-Type": "application/json"}
25
+from vllm.multimodal.utils import encode_image_url, fetch_image
26
27
query = "A woman playing with her dog on a beach at sunset."
40
-documents = {
41
- "content": [
42
- {
43
- "type": "text",
44
- "text": (
45
- "A woman shares a joyful moment with her golden retriever on a sun-drenched beach at sunset, "
46
- "as the dog offers its paw in a heartwarming display of companionship and trust."
47
- ),
48
- },
49
- {
50
- "type": "image_url",
51
- "image_url": {
52
- "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
53
- },
54
- },
55
- {
56
- "type": "image_url",
57
- "image_url": encode_base64_content_from_url(
58
- "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
59
- ),
60
- },
61
- ]
62
-}
28
+document = (
29
+ "A woman shares a joyful moment with her golden retriever on a sun-drenched beach at sunset, "
30
+ "as the dog offers its paw in a heartwarming display of companionship and trust."
31
+)
32
+image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
33
+documents = [
34
+ {
35
+ "type": "text",
36
+ "text": document,
37
+ },
38
+ {
39
+ "type": "image_url",
40
+ "image_url": {"url": image_url},
41
+ },
42
+ {
43
+ "type": "image_url",
44
+ "image_url": {"url": encode_image_url(fetch_image(image_url))},
45
+ },
46
+]
47
48
49
def parse_args():
@@ -74,23 +58,36 @@ def main(args):
58
models_url = base_url + "/v1/models"
59
rerank_url = base_url + "/rerank"
60
77
- response = requests.get(models_url, headers=headers)
61
+ response = requests.get(models_url)
62
model = response.json()["data"][0]["id"]
63
80
- data = {
64
+ print("Query: string & Document: list of string")
65
+ prompt = {"model": model, "query": query, "documents": [document]}
66
+ response = requests.post(rerank_url, json=prompt)
67
+ pprint.pprint(response.json())
68
+
69
+ print("Query: string & Document: text")
70
+ prompt = {"model": model, "query": query, "documents": {"content": [documents[0]]}}
71
+ response = requests.post(rerank_url, json=prompt)
72
+ pprint.pprint(response.json())
73
+
74
+ print("Query: string & Document: image url")
75
+ prompt = {
76
+ "model": model,
77
+ "query": query,
78
+ "documents": {"content": [documents[1]]},
79
+ }
80
+ response = requests.post(rerank_url, json=prompt)
81
+ pprint.pprint(response.json())
82
+
83
+ print("Query: string & Document: image base64")
84
+ prompt = {
85
"model": model,
86
"query": query,
83
- "documents": documents,
87
+ "documents": {"content": [documents[2]]},
88
}
85
- response = requests.post(rerank_url, headers=headers, json=data)
86
-
87
- # Check the response
88
- if response.status_code == 200:
89
- print("Request successful!")
90
- print(json.dumps(response.json(), indent=2))
91
- else:
92
- print(f"Request failed with status code: {response.status_code}")
93
- print(response.text)
89
+ response = requests.post(rerank_url, json=prompt)
90
+ pprint.pprint(response.json())
91
92
93
if __name__ == "__main__":
examples/pooling/score/vision_score_api_online.py
+54
-45
@@ -17,48 +17,32 @@ e.g.
17
"""
18
19
import argparse
20
-import base64
21
-import json
20
import pprint
21
22
import requests
23
26
-
27
-def encode_base64_content_from_url(content_url: str) -> dict[str, str]:
28
- """Encode a content retrieved from a remote url to base64 format."""
29
-
30
- with requests.get(content_url, headers=headers) as response:
31
- response.raise_for_status()
32
- result = base64.b64encode(response.content).decode("utf-8")
33
-
34
- return {"url": f"data:image/jpeg;base64,{result}"}
35
-
36
-
37
-headers = {"accept": "application/json", "Content-Type": "application/json"}
38
-
39
-queries = "slm markdown"
40
-documents = {
41
- "content": [
42
- {
43
- "type": "image_url",
44
- "image_url": {
45
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
46
- },
47
- },
48
- {
49
- "type": "image_url",
50
- "image_url": {
51
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
52
- },
53
- },
54
- {
55
- "type": "image_url",
56
- "image_url": encode_base64_content_from_url(
57
- "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
58
- ),
59
- },
60
- ]
61
-}
24
+from vllm.multimodal.utils import encode_image_url, fetch_image
25
+
26
+query = "A woman playing with her dog on a beach at sunset."
27
+document = (
28
+ "A woman shares a joyful moment with her golden retriever on a sun-drenched beach at sunset, "
29
+ "as the dog offers its paw in a heartwarming display of companionship and trust."
30
+)
31
+image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
32
+documents = [
33
+ {
34
+ "type": "text",
35
+ "text": document,
36
+ },
37
+ {
38
+ "type": "image_url",
39
+ "image_url": {"url": image_url},
40
+ },
41
+ {
42
+ "type": "image_url",
43
+ "image_url": {"url": encode_image_url(fetch_image(image_url))},
44
+ },
45
+]
46
47
48
def parse_args():
@@ -73,15 +57,40 @@ def main(args):
57
models_url = base_url + "/v1/models"
58
score_url = base_url + "/score"
59
76
- response = requests.get(models_url, headers=headers)
60
+ response = requests.get(models_url)
61
model = response.json()["data"][0]["id"]
62
79
- prompt = {"model": model, "queries": queries, "documents": documents}
80
- response = requests.post(score_url, headers=headers, json=prompt)
81
- print("\nPrompt when queries is string and documents is a image list:")
82
- pprint.pprint(prompt)
83
- print("\nScore Response:")
84
- print(json.dumps(response.json(), indent=2))
63
+ print("Query: string & Document: string")
64
+ prompt = {"model": model, "queries": query, "documents": document}
65
+ response = requests.post(score_url, json=prompt)
66
+ pprint.pprint(response.json())
67
+
68
+ print("Query: string & Document: text")
69
+ prompt = {
70
+ "model": model,
71
+ "queries": query,
72
+ "documents": {"content": [documents[0]]},
73
+ }
74
+ response = requests.post(score_url, json=prompt)
75
+ pprint.pprint(response.json())
76
+
77
+ print("Query: string & Document: image url")
78
+ prompt = {
79
+ "model": model,
80
+ "queries": query,
81
+ "documents": {"content": [documents[1]]},
82
+ }
83
+ response = requests.post(score_url, json=prompt)
84
+ pprint.pprint(response.json())
85
+
86
+ print("Query: string & Document: image base64")
87
+ prompt = {
88
+ "model": model,
89
+ "queries": query,
90
+ "documents": {"content": [documents[2]]},
91
+ }
92
+ response = requests.post(score_url, json=prompt)
93
+ pprint.pprint(response.json())
94
95
96
if __name__ == "__main__":
tests/entrypoints/pooling/classify/test_online_vision.py
+2
-2
@@ -5,9 +5,9 @@ import json
5
import pytest
6
import requests
7
8
-from tests.entrypoints.test_utils import encode_base64_content_from_url
8
from tests.utils import RemoteOpenAIServer
9
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
10
+from vllm.multimodal.utils import encode_image_url, fetch_image
11
12
MODEL_NAME = "muziyongshixin/Qwen2.5-VL-7B-for-VideoCls"
13
MAXIMUM_VIDEOS = 1
@@ -19,7 +19,7 @@ HF_OVERRIDES = {
19
}
20
input_text = "This product was excellent and exceeded my expectations"
21
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
22
-image_base64 = encode_base64_content_from_url(image_url)
22
+image_base64 = {"url": encode_image_url(fetch_image(image_url))}
23
video_url = "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4"
24
25
tests/entrypoints/pooling/score/test_online_score_vision.py
new
+122
@@ -0,0 +1,122 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import pytest
5
+import requests
6
+
7
+from tests.utils import VLLM_PATH, RemoteOpenAIServer
8
+from vllm.entrypoints.pooling.score.protocol import ScoreResponse
9
+from vllm.multimodal.utils import encode_image_url, fetch_image
10
+
11
+MODEL_NAME = "Qwen/Qwen3-VL-Reranker-2B"
12
+HF_OVERRIDES = {
13
+ "architectures": ["Qwen3VLForSequenceClassification"],
14
+ "classifier_from_token": ["no", "yes"],
15
+ "is_original_qwen3_reranker": True,
16
+}
17
+
18
+query = "A cat standing in the snow."
19
+image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
20
+documents = [
21
+ {
22
+ "type": "text",
23
+ "text": query,
24
+ },
25
+ {
26
+ "type": "image_url",
27
+ "image_url": {"url": image_url},
28
+ },
29
+ {
30
+ "type": "image_url",
31
+ "image_url": {"url": encode_image_url(fetch_image(image_url))},
32
+ },
33
+]
34
+
35
+
36
+@pytest.fixture(scope="module")
37
+def server():
38
+ args = [
39
+ "--enforce-eager",
40
+ "--max-model-len",
41
+ "8192",
42
+ "--chat-template",
43
+ str(VLLM_PATH / "examples/pooling/score/template/qwen3_vl_reranker.jinja"),
44
+ ]
45
+
46
+ with RemoteOpenAIServer(
47
+ MODEL_NAME, args, override_hf_configs=HF_OVERRIDES
48
+ ) as remote_server:
49
+ yield remote_server
50
+
51
+
52
+def test_score_api_queries_str_documents_str(server: RemoteOpenAIServer):
53
+ queries = "What is the capital of France?"
54
+ documents = "The capital of France is Paris."
55
+
56
+ score_response = requests.post(
57
+ server.url_for("score"),
58
+ json={
59
+ "model": MODEL_NAME,
60
+ "queries": queries,
61
+ "documents": documents,
62
+ },
63
+ )
64
+ score_response.raise_for_status()
65
+ score = ScoreResponse.model_validate(score_response.json())
66
+
67
+ assert score.id is not None
68
+ assert score.data is not None
69
+ assert len(score.data) == 1
70
+
71
+
72
+def test_score_api_queries_str_documents_text_content(server: RemoteOpenAIServer):
73
+ score_response = requests.post(
74
+ server.url_for("score"),
75
+ json={
76
+ "model": MODEL_NAME,
77
+ "queries": query,
78
+ "documents": {"content": [documents[0]]},
79
+ },
80
+ )
81
+ score_response.raise_for_status()
82
+ score = ScoreResponse.model_validate(score_response.json())
83
+
84
+ assert score.id is not None
85
+ assert score.data is not None
86
+ assert len(score.data) == 1
87
+
88
+
89
+def test_score_api_queries_str_documents_image_url_content(server: RemoteOpenAIServer):
90
+ score_response = requests.post(
91
+ server.url_for("score"),
92
+ json={
93
+ "model": MODEL_NAME,
94
+ "queries": query,
95
+ "documents": {"content": [documents[1]]},
96
+ },
97
+ )
98
+ score_response.raise_for_status()
99
+ score = ScoreResponse.model_validate(score_response.json())
100
+
101
+ assert score.id is not None
102
+ assert score.data is not None
103
+ assert len(score.data) == 1
104
+
105
+
106
+def test_score_api_queries_str_documents_image_base64_content(
107
+ server: RemoteOpenAIServer,
108
+):
109
+ score_response = requests.post(
110
+ server.url_for("score"),
111
+ json={
112
+ "model": MODEL_NAME,
113
+ "queries": query,
114
+ "documents": {"content": [documents[2]]},
115
+ },
116
+ )
117
+ score_response.raise_for_status()
118
+ score = ScoreResponse.model_validate(score_response.json())
119
+
120
+ assert score.id is not None
121
+ assert score.data is not None
122
+ assert len(score.data) == 1
tests/entrypoints/test_utils.py
-12
@@ -1,9 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import base64
4
-
5
-import requests
6
-
3
from vllm.entrypoints.utils import sanitize_message
4
5
@@ -12,11 +8,3 @@ def test_sanitize_message():
8
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
9
== "<_io.BytesIO object>"
10
)
15
-
16
-
17
-def encode_base64_content_from_url(content_url: str) -> dict[str, str]:
18
- with requests.get(content_url) as response:
19
- response.raise_for_status()
20
- result = base64.b64encode(response.content).decode("utf-8")
21
-
22
- return {"url": f"data:image/jpeg;base64,{result}"}
vllm/model_executor/models/adapters.py
+10
-4
@@ -466,6 +466,7 @@ def load_weights_using_from_2_way_softmax(
466
467
language_model = _get_language_model_for_seq_cls(model)
468
is_vlm = language_model is not model
469
+ using_vlm_head = is_vlm and hasattr(language_model, "score")
470
471
language_model.lm_head = ParallelLMHead(
472
text_config.vocab_size, text_config.hidden_size, quant_config=quant_config
@@ -506,14 +507,16 @@ def load_weights_using_from_2_way_softmax(
507
torch.float32
508
) - lm_head_weight.data[[false_id]].to(torch.float32)
509
509
- score_layer = language_model.score if is_vlm else model.score
510
+ score_layer = language_model.score if using_vlm_head else model.score
511
param = score_layer.weight
512
weight_loader = getattr(param, "weight_loader", default_weight_loader)
513
weight_loader(param, score_weight)
514
515
del language_model.lm_head
516
516
- score_weight_name = "language_model.score.weight" if is_vlm else "score.weight"
517
+ score_weight_name = (
518
+ "language_model.score.weight" if using_vlm_head else "score.weight"
519
+ )
520
loaded_weights.add(score_weight_name)
521
522
lm_head_name = "lm_head.weight"
@@ -537,6 +540,7 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
540
541
language_model = _get_language_model_for_seq_cls(model)
542
is_vlm = language_model is not model
543
+ using_vlm_head = is_vlm and hasattr(language_model, "score")
544
545
language_model.lm_head = ParallelLMHead(
546
text_config.vocab_size, text_config.hidden_size, quant_config=quant_config
@@ -572,14 +576,16 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te
576
token_ids = [tokenizer.convert_tokens_to_ids(t) for t in tokens]
577
score_weight = language_model.lm_head.weight.data[token_ids]
578
575
- score_layer = language_model.score if is_vlm else model.score
579
+ score_layer = language_model.score if using_vlm_head else model.score
580
param = score_layer.weight
581
weight_loader = getattr(param, "weight_loader", default_weight_loader)
582
weight_loader(param, score_weight)
583
584
del language_model.lm_head
585
582
- score_weight_name = "language_model.score.weight" if is_vlm else "score.weight"
586
+ score_weight_name = (
587
+ "language_model.score.weight" if using_vlm_head else "score.weight"
588
+ )
589
loaded_weights.add(score_weight_name)
590
591
lm_head_name = "lm_head.weight"