A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 07:45 UTC
[Model] Improve multimodal pooling examples (#32085)
Signed-off-by: wang.yuqi <noooop@126.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jan 12, 2026 at 15:54 UTC
60446cd684d55c0918295bb5e61da19330a0fb35
10 files changed
+381
-69
docs/serving/openai_compatible_server.md
+7
-4
@@ -362,7 +362,7 @@ and passing a list of `messages` in the request. Refer to the examples below for
362
`MrLight/dse-qwen2-2b-mrl-v1` requires a placeholder image of the minimum image size for text query embeddings. See the full code
363
example below for details.
364
365
-Full example: [examples/pooling/embed/openai_chat_embedding_client_for_multimodal.py](../../examples/pooling/embed/openai_chat_embedding_client_for_multimodal.py)
365
+Full example: [examples/pooling/embed/vision_embedding_online.py](../../examples/pooling/embed/vision_embedding_online.py)
366
367
#### Extra parameters
368
@@ -667,7 +667,7 @@ Usually, the score for a sentence pair refers to the similarity between two sent
667
668
You can find the documentation for cross encoder models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
669
670
-Code example: [examples/pooling/score/openai_cross_encoder_score.py](../../examples/pooling/score/openai_cross_encoder_score.py)
670
+Code example: [examples/pooling/score/score_api_online.py](../../examples/pooling/score/score_api_online.py)
671
672
#### Score Template
673
@@ -863,7 +863,10 @@ You can pass multi-modal inputs to scoring models by passing `content` including
863
print("Scoring output:", response_json["data"][0]["score"])
864
print("Scoring output:", response_json["data"][1]["score"])
865
```
866
-Full example: [examples/pooling/score/openai_cross_encoder_score_for_multimodal.py](../../examples/pooling/score/openai_cross_encoder_score_for_multimodal.py)
866
+Full example:
867
+
868
+- [examples/pooling/score/vision_score_api_online.py](../../examples/pooling/score/vision_score_api_online.py)
869
+- examples/pooling/score/vision_rerank_api_online.py](../../examples/pooling/score/vision_rerank_api_online.py)
870
871
#### Extra parameters
872
@@ -893,7 +896,7 @@ endpoints are compatible with both [Jina AI's re-rank API interface](https://jin
896
[Cohere's re-rank API interface](https://docs.cohere.com/v2/reference/rerank) to ensure compatibility with
897
popular open-source tools.
898
896
-Code example: [examples/pooling/score/openai_reranker.py](../../examples/pooling/score/openai_reranker.py)
899
+Code example: [examples/pooling/score/rerank_api_online.py](../../examples/pooling/score/rerank_api_online.py)
900
901
#### Example Request
902
examples/pooling/embed/vision_embedding_offline.py
new
+93
@@ -0,0 +1,93 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa: E501
4
+"""
5
+This example shows how to use vLLM for running offline inference with
6
+the correct prompt format on vision language models for multimodal embedding.
7
+
8
+For most models, the prompt format should follow corresponding examples
9
+on HuggingFace model repository.
10
+"""
11
+
12
+import argparse
13
+from dataclasses import asdict
14
+
15
+from vllm import LLM, EngineArgs
16
+from vllm.multimodal.utils import fetch_image
17
+
18
+image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
19
+text = "A cat standing in the snow."
20
+multi_modal_data = {"image": fetch_image(image_url)}
21
+
22
+
23
+def print_embeddings(embeds):
24
+ embeds_trimmed = (str(embeds[:4])[:-1] + ", ...]") if len(embeds) > 4 else embeds
25
+ print(f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
26
+
27
+
28
+def run_qwen3_vl():
29
+ engine_args = EngineArgs(
30
+ model="Qwen/Qwen3-VL-Embedding-2B",
31
+ runner="pooling",
32
+ max_model_len=8192,
33
+ limit_mm_per_prompt={"image": 1},
34
+ )
35
+ default_instruction = "Represent the user's input."
36
+ image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
37
+ text_prompt = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{text}<|im_end|>\n<|im_start|>assistant\n"
38
+ image_prompt = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}<|im_end|>\n<|im_start|>assistant\n"
39
+ image_text_prompt = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}{text}<|im_end|>\n<|im_start|>assistant\n"
40
+
41
+ llm = LLM(**asdict(engine_args))
42
+
43
+ print("Text embedding output:")
44
+ outputs = llm.embed(text_prompt, use_tqdm=False)
45
+ print_embeddings(outputs[0].outputs.embedding)
46
+
47
+ print("Image embedding output:")
48
+ outputs = llm.embed(
49
+ {
50
+ "prompt": image_prompt,
51
+ "multi_modal_data": multi_modal_data,
52
+ },
53
+ use_tqdm=False,
54
+ )
55
+ print_embeddings(outputs[0].outputs.embedding)
56
+
57
+ print("Image+Text embedding output:")
58
+ outputs = llm.embed(
59
+ {
60
+ "prompt": image_text_prompt,
61
+ "multi_modal_data": multi_modal_data,
62
+ },
63
+ use_tqdm=False,
64
+ )
65
+ print_embeddings(outputs[0].outputs.embedding)
66
+
67
+
68
+model_example_map = {
69
+ "qwen3_vl": run_qwen3_vl,
70
+}
71
+
72
+
73
+def parse_args():
74
+ parser = argparse.ArgumentParser(
75
+ "Script to run a specified VLM through vLLM offline api."
76
+ )
77
+ parser.add_argument(
78
+ "--model",
79
+ type=str,
80
+ choices=model_example_map.keys(),
81
+ required=True,
82
+ help="The name of the embedding model.",
83
+ )
84
+ return parser.parse_args()
85
+
86
+
87
+def main(args):
88
+ model_example_map[args.model]()
89
+
90
+
91
+if __name__ == "__main__":
92
+ args = parse_args()
93
+ main(args)
examples/pooling/embed/vision_embedding_online.py
renamed
+130
-5
@@ -21,7 +21,8 @@ from PIL import Image
21
openai_api_key = "EMPTY"
22
openai_api_base = "http://localhost:8000/v1"
23
24
-image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
24
+image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
25
+text = "A cat standing in the snow."
26
27
28
def create_chat_embeddings(
@@ -30,6 +31,8 @@ def create_chat_embeddings(
31
messages: list[ChatCompletionMessageParam],
32
model: str,
33
encoding_format: Literal["base64", "float"] | NotGiven = NOT_GIVEN,
34
+ continue_final_message: bool = False,
35
+ add_special_tokens: bool = False,
36
) -> CreateEmbeddingResponse:
37
"""
38
Convenience function for accessing vLLM's Chat Embeddings API,
@@ -38,10 +41,21 @@ def create_chat_embeddings(
41
return client.post(
42
"/embeddings",
43
cast_to=CreateEmbeddingResponse,
41
- body={"messages": messages, "model": model, "encoding_format": encoding_format},
44
+ body={
45
+ "messages": messages,
46
+ "model": model,
47
+ "encoding_format": encoding_format,
48
+ "continue_final_message": continue_final_message,
49
+ "add_special_tokens": add_special_tokens,
50
+ },
51
)
52
53
54
+def print_embeddings(embeds):
55
+ embeds_trimmed = (str(embeds[:4])[:-1] + ", ...]") if len(embeds) > 4 else embeds
56
+ print(f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
57
+
58
+
59
def run_clip(client: OpenAI, model: str):
60
"""
61
Start the server using:
@@ -145,6 +159,113 @@ def run_dse_qwen2_vl(client: OpenAI, model: str):
159
print("Text embedding output:", response.data[0].embedding)
160
161
162
+def run_qwen3_vl(client: OpenAI, model: str):
163
+ """
164
+ Start the server using:
165
+
166
+ vllm serve Qwen/Qwen3-VL-Embedding-2B \
167
+ --runner pooling \
168
+ --max-model-len 8192
169
+ """
170
+
171
+ default_instruction = "Represent the user's input."
172
+
173
+ print("Text embedding output:")
174
+ response = create_chat_embeddings(
175
+ client,
176
+ messages=[
177
+ {
178
+ "role": "system",
179
+ "content": [
180
+ {"type": "text", "text": default_instruction},
181
+ ],
182
+ },
183
+ {
184
+ "role": "user",
185
+ "content": [
186
+ {"type": "text", "text": text},
187
+ ],
188
+ },
189
+ {
190
+ "role": "assistant",
191
+ "content": [
192
+ {"type": "text", "text": ""},
193
+ ],
194
+ },
195
+ ],
196
+ model=model,
197
+ encoding_format="float",
198
+ continue_final_message=True,
199
+ add_special_tokens=True,
200
+ )
201
+ print_embeddings(response.data[0].embedding)
202
+
203
+ print("Image embedding output:")
204
+ response = create_chat_embeddings(
205
+ client,
206
+ messages=[
207
+ {
208
+ "role": "system",
209
+ "content": [
210
+ {"type": "text", "text": default_instruction},
211
+ ],
212
+ },
213
+ {
214
+ "role": "user",
215
+ "content": [
216
+ {"type": "image_url", "image_url": {"url": image_url}},
217
+ {"type": "text", "text": ""},
218
+ ],
219
+ },
220
+ {
221
+ "role": "assistant",
222
+ "content": [
223
+ {"type": "text", "text": ""},
224
+ ],
225
+ },
226
+ ],
227
+ model=model,
228
+ encoding_format="float",
229
+ continue_final_message=True,
230
+ add_special_tokens=True,
231
+ )
232
+ print_embeddings(response.data[0].embedding)
233
+
234
+ print("Image+Text embedding output:")
235
+ response = create_chat_embeddings(
236
+ client,
237
+ messages=[
238
+ {
239
+ "role": "system",
240
+ "content": [
241
+ {"type": "text", "text": default_instruction},
242
+ ],
243
+ },
244
+ {
245
+ "role": "user",
246
+ "content": [
247
+ {"type": "image_url", "image_url": {"url": image_url}},
248
+ {
249
+ "type": "text",
250
+ "text": f"{text}",
251
+ },
252
+ ],
253
+ },
254
+ {
255
+ "role": "assistant",
256
+ "content": [
257
+ {"type": "text", "text": ""},
258
+ ],
259
+ },
260
+ ],
261
+ model=model,
262
+ encoding_format="float",
263
+ continue_final_message=True,
264
+ add_special_tokens=True,
265
+ )
266
+ print_embeddings(response.data[0].embedding)
267
+
268
+
269
def run_siglip(client: OpenAI, model: str):
270
"""
271
Start the server using:
@@ -213,7 +334,8 @@ def run_vlm2vec(client: OpenAI, model: str):
334
encoding_format="float",
335
)
336
216
- print("Image embedding output:", response.data[0].embedding)
337
+ print("Image embedding output:")
338
+ print_embeddings(response.data[0].embedding)
339
340
response = create_chat_embeddings(
341
client,
@@ -233,7 +355,8 @@ def run_vlm2vec(client: OpenAI, model: str):
355
encoding_format="float",
356
)
357
236
- print("Image+Text embedding output:", response.data[0].embedding)
358
+ print("Image+Text embedding output:")
359
+ print_embeddings(response.data[0].embedding)
360
361
response = create_chat_embeddings(
362
client,
@@ -249,11 +372,13 @@ def run_vlm2vec(client: OpenAI, model: str):
372
encoding_format="float",
373
)
374
252
- print("Text embedding output:", response.data[0].embedding)
375
+ print("Text embedding output:")
376
+ print_embeddings(response.data[0].embedding)
377
378
379
model_example_map = {
380
"clip": run_clip,
381
+ "qwen3_vl": run_qwen3_vl,
382
"dse_qwen2_vl": run_dse_qwen2_vl,
383
"siglip": run_siglip,
384
"vlm2vec": run_vlm2vec,
examples/pooling/score/cohere_rerank_online.py
renamed
examples/pooling/score/openai_cross_encoder_score_for_multimodal.py
deleted
-60
@@ -1,60 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-Example online usage of Score API.
5
-
6
-Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
7
-"""
8
-
9
-import argparse
10
-import pprint
11
-
12
-import requests
13
-
14
-
15
-def post_http_request(prompt: dict, api_url: str) -> requests.Response:
16
- headers = {"User-Agent": "Test Client"}
17
- response = requests.post(api_url, headers=headers, json=prompt)
18
- return response
19
-
20
-
21
-def parse_args():
22
- parser = argparse.ArgumentParser()
23
- parser.add_argument("--host", type=str, default="localhost")
24
- parser.add_argument("--port", type=int, default=8000)
25
- parser.add_argument("--model", type=str, default="jinaai/jina-reranker-m0")
26
- return parser.parse_args()
27
-
28
-
29
-def main(args):
30
- api_url = f"http://{args.host}:{args.port}/score"
31
- model_name = args.model
32
-
33
- text_1 = "slm markdown"
34
- text_2 = {
35
- "content": [
36
- {
37
- "type": "image_url",
38
- "image_url": {
39
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
40
- },
41
- },
42
- {
43
- "type": "image_url",
44
- "image_url": {
45
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
46
- },
47
- },
48
- ]
49
- }
50
- prompt = {"model": model_name, "text_1": text_1, "text_2": text_2}
51
- score_response = post_http_request(prompt=prompt, api_url=api_url)
52
- print("\nPrompt when text_1 is string and text_2 is a image list:")
53
- pprint.pprint(prompt)
54
- print("\nScore Response:")
55
- pprint.pprint(score_response.json())
56
-
57
-
58
-if __name__ == "__main__":
59
- args = parse_args()
60
- main(args)
examples/pooling/score/rerank_api_online.py
renamed
examples/pooling/score/score_api_online.py
renamed
examples/pooling/score/vision_rerank_api_online.py
new
+80
@@ -0,0 +1,80 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa: E501
4
+
5
+"""
6
+Example Python client for multimodal rerank API which is compatible with
7
+Jina and Cohere https://jina.ai/reranker
8
+
9
+Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
10
+e.g.
11
+ vllm serve jinaai/jina-reranker-m0 --runner pooling
12
+
13
+ vllm serve Qwen/Qwen3-VL-Reranker-2B \
14
+ --runner pooling \
15
+ --max-model-len 4096 \
16
+ --hf_overrides '{"architectures": ["Qwen3VLForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}' \
17
+ --chat-template examples/pooling/score/template/qwen3_vl_reranker.jinja
18
+"""
19
+
20
+import argparse
21
+import json
22
+
23
+import requests
24
+
25
+headers = {"accept": "application/json", "Content-Type": "application/json"}
26
+
27
+query = "A woman playing with her dog on a beach at sunset."
28
+documents = {
29
+ "content": [
30
+ {
31
+ "type": "text",
32
+ "text": (
33
+ "A woman shares a joyful moment with her golden retriever on a sun-drenched beach at sunset, " # noqa: E501
34
+ "as the dog offers its paw in a heartwarming display of companionship and trust." # noqa: E501
35
+ ),
36
+ },
37
+ {
38
+ "type": "image_url",
39
+ "image_url": {
40
+ "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
41
+ },
42
+ },
43
+ ]
44
+}
45
+
46
+
47
+def parse_args():
48
+ parser = argparse.ArgumentParser()
49
+ parser.add_argument("--host", type=str, default="localhost")
50
+ parser.add_argument("--port", type=int, default=8000)
51
+ return parser.parse_args()
52
+
53
+
54
+def main(args):
55
+ base_url = f"http://{args.host}:{args.port}"
56
+ models_url = base_url + "/v1/models"
57
+ rerank_url = base_url + "/rerank"
58
+
59
+ response = requests.get(models_url, headers=headers)
60
+ model = response.json()["data"][0]["id"]
61
+
62
+ data = {
63
+ "model": model,
64
+ "query": query,
65
+ "documents": documents,
66
+ }
67
+ response = requests.post(rerank_url, headers=headers, json=data)
68
+
69
+ # Check the response
70
+ if response.status_code == 200:
71
+ print("Request successful!")
72
+ print(json.dumps(response.json(), indent=2))
73
+ else:
74
+ print(f"Request failed with status code: {response.status_code}")
75
+ print(response.text)
76
+
77
+
78
+if __name__ == "__main__":
79
+ args = parse_args()
80
+ main(args)
examples/pooling/score/vision_reranker_offline.py
renamed
examples/pooling/score/vision_score_api_online.py
new
+71
@@ -0,0 +1,71 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa: E501
4
+
5
+"""
6
+Example online usage of Score API.
7
+
8
+Run `vllm serve <model> --runner pooling` to start up the server in vLLM.
9
+e.g.
10
+ vllm serve jinaai/jina-reranker-m0 --runner pooling
11
+
12
+ vllm serve Qwen/Qwen3-VL-Reranker-2B \
13
+ --runner pooling \
14
+ --max-model-len 4096 \
15
+ --hf_overrides '{"architectures": ["Qwen3VLForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}' \
16
+ --chat-template examples/pooling/score/template/qwen3_vl_reranker.jinja
17
+"""
18
+
19
+import argparse
20
+import json
21
+import pprint
22
+
23
+import requests
24
+
25
+headers = {"accept": "application/json", "Content-Type": "application/json"}
26
+
27
+text_1 = "slm markdown"
28
+text_2 = {
29
+ "content": [
30
+ {
31
+ "type": "image_url",
32
+ "image_url": {
33
+ "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
34
+ },
35
+ },
36
+ {
37
+ "type": "image_url",
38
+ "image_url": {
39
+ "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
40
+ },
41
+ },
42
+ ]
43
+}
44
+
45
+
46
+def parse_args():
47
+ parser = argparse.ArgumentParser()
48
+ parser.add_argument("--host", type=str, default="localhost")
49
+ parser.add_argument("--port", type=int, default=8000)
50
+ return parser.parse_args()
51
+
52
+
53
+def main(args):
54
+ base_url = f"http://{args.host}:{args.port}"
55
+ models_url = base_url + "/v1/models"
56
+ score_url = base_url + "/score"
57
+
58
+ response = requests.get(models_url, headers=headers)
59
+ model = response.json()["data"][0]["id"]
60
+
61
+ prompt = {"model": model, "text_1": text_1, "text_2": text_2}
62
+ response = requests.post(score_url, headers=headers, json=prompt)
63
+ print("\nPrompt when text_1 is string and text_2 is a image list:")
64
+ pprint.pprint(prompt)
65
+ print("\nScore Response:")
66
+ print(json.dumps(response.json(), indent=2))
67
+
68
+
69
+if __name__ == "__main__":
70
+ args = parse_args()
71
+ main(args)