A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 16:30 UTC
[Frontend][last/5] Make pooling entrypoints request schema consensus. (#31127)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Feb 9, 2026 at 14:42 UTC
22b64948f6f4381bce7ac8ec0487020f0129e1cb
24 files changed
+658
-612
.buildkite/test-amd.yaml
+1
-1
@@ -514,7 +514,7 @@ steps:
514
- python3 offline_inference/vision_language_multi_image.py --seed 0
515
- python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
516
# for pooling models
517
- - python3 pooling/pooling/vision_language_pooling.py --seed 0
517
+ - python3 pooling/embed/vision_embedding_offline.py --seed 0
518
# for features demo
519
- python3 offline_inference/prefix_caching.py
520
- python3 offline_inference/llm_engine_example.py
.buildkite/test-pipeline.yaml
+1
-1
@@ -453,7 +453,7 @@ steps:
453
- python3 offline_inference/vision_language_multi_image.py --seed 0
454
- python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
455
# for pooling models
456
- - python3 pooling/pooling/vision_language_pooling.py --seed 0
456
+ - python3 pooling/embed/vision_embedding_offline.py --seed 0
457
# for features demo
458
- python3 offline_inference/prefix_caching.py
459
- python3 offline_inference/llm_engine_example.py
.buildkite/test_areas/misc.yaml
+1
-1
@@ -72,7 +72,7 @@ steps:
72
- python3 offline_inference/vision_language_multi_image.py --seed 0
73
- python3 offline_inference/encoder_decoder_multimodal.py --model-type whisper --seed 0
74
# for pooling models
75
- - python3 pooling/pooling/vision_language_pooling.py --seed 0
75
+ - python3 pooling/embed/vision_embedding_offline.py --seed 0
76
# for features demo
77
- python3 offline_inference/prefix_caching.py
78
- python3 offline_inference/llm_engine_example.py
docs/features/multimodal_inputs.md
+1
-1
@@ -510,7 +510,7 @@ Our OpenAI-compatible server accepts multi-modal data via the [Chat Completions
510
If no fallback is available, an error is raised and you have to provide the chat template manually via the `--chat-template` argument.
511
512
For certain models, we provide alternative chat templates inside [examples](../../examples).
513
- For example, VLM2Vec uses [examples/template_vlm2vec_phi3v.jinja](../../examples/template_vlm2vec_phi3v.jinja) which is different from the default one for Phi-3-Vision.
513
+ For example, VLM2Vec uses [examples/pooling/embed/template/vlm2vec_phi3v.jinja](../../examples/pooling/embed/template/vlm2vec_phi3v.jinja) which is different from the default one for Phi-3-Vision.
514
515
### Image Inputs
516
docs/serving/openai_compatible_server.md
+31
-29
@@ -311,7 +311,7 @@ and passing a list of `messages` in the request. Refer to the examples below for
311
vllm serve TIGER-Lab/VLM2Vec-Full --runner pooling \
312
--trust-remote-code \
313
--max-model-len 4096 \
314
- --chat-template examples/template_vlm2vec_phi3v.jinja
314
+ --chat-template examples/pooling/embed/template/vlm2vec_phi3v.jinja
315
```
316
317
!!! important
@@ -319,7 +319,7 @@ and passing a list of `messages` in the request. Refer to the examples below for
319
to run this model in embedding mode instead of text generation mode.
320
321
The custom chat template is completely different from the original one for this model,
322
- and can be found here: [examples/template_vlm2vec_phi3v.jinja](../../examples/template_vlm2vec_phi3v.jinja)
322
+ and can be found here: [examples/pooling/embed/template/vlm2vec_phi3v.jinja](../../examples/pooling/embed/template/vlm2vec_phi3v.jinja)
323
324
Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level `requests` library:
325
@@ -359,14 +359,14 @@ and passing a list of `messages` in the request. Refer to the examples below for
359
vllm serve MrLight/dse-qwen2-2b-mrl-v1 --runner pooling \
360
--trust-remote-code \
361
--max-model-len 8192 \
362
- --chat-template examples/template_dse_qwen2_vl.jinja
362
+ --chat-template examples/pooling/embed/template/dse_qwen2_vl.jinja
363
```
364
365
!!! important
366
Like with VLM2Vec, we have to explicitly pass `--runner pooling`.
367
368
Additionally, `MrLight/dse-qwen2-2b-mrl-v1` requires an EOS token for embeddings, which is handled
369
- by a custom chat template: [examples/template_dse_qwen2_vl.jinja](../../examples/template_dse_qwen2_vl.jinja)
369
+ by a custom chat template: [examples/pooling/embed/template/dse_qwen2_vl.jinja](../../examples/pooling/embed/template/dse_qwen2_vl.jinja)
370
371
!!! important
372
`MrLight/dse-qwen2-2b-mrl-v1` requires a placeholder image of the minimum image size for text query embeddings. See the full code
@@ -532,7 +532,7 @@ The following [sampling parameters](../api/README.md#inference-parameters) are s
532
??? code
533
534
```python
535
- --8<-- "vllm/entrypoints/openai/protocol.py:transcription-sampling-params"
535
+ --8<-- "vllm/entrypoints/openai/speech_to_text/protocol.py:transcription-sampling-params"
536
```
537
538
The following extra parameters are supported:
@@ -540,7 +540,7 @@ The following extra parameters are supported:
540
??? code
541
542
```python
543
- --8<-- "vllm/entrypoints/openai/protocol.py:transcription-extra-params"
543
+ --8<-- "vllm/entrypoints/openai/speech_to_text/protocol.py:transcription-extra-params"
544
```
545
546
### Translations API
@@ -560,13 +560,13 @@ Code example: [examples/online_serving/openai_translation_client.py](../../examp
560
The following [sampling parameters](../api/README.md#inference-parameters) are supported.
561
562
```python
563
---8<-- "vllm/entrypoints/openai/protocol.py:translation-sampling-params"
563
+--8<-- "vllm/entrypoints/openai/speech_to_text/protocol.py:translation-sampling-params"
564
```
565
566
The following extra parameters are supported:
567
568
```python
569
---8<-- "vllm/entrypoints/openai/protocol.py:translation-extra-params"
569
+--8<-- "vllm/entrypoints/openai/speech_to_text/protocol.py:translation-extra-params"
570
```
571
572
### Realtime API
@@ -954,28 +954,34 @@ You can pass multi-modal inputs to scoring models by passing `content` including
954
955
```python
956
import requests
957
-
957
+
958
response = requests.post(
959
"http://localhost:8000/v1/score",
960
json={
961
"model": "jinaai/jina-reranker-m0",
962
"queries": "slm markdown",
963
- "documents": {
964
- "content": [
965
- {
966
- "type": "image_url",
967
- "image_url": {
968
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
969
- },
970
- },
971
- {
972
- "type": "image_url",
973
- "image_url": {
974
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
975
- },
976
- },
977
- ],
978
- },
963
+ "documents": [
964
+ {
965
+ "content": [
966
+ {
967
+ "type": "image_url",
968
+ "image_url": {
969
+ "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
970
+ },
971
+ }
972
+ ],
973
+ },
974
+ {
975
+ "content": [
976
+ {
977
+ "type": "image_url",
978
+ "image_url": {
979
+ "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
980
+ },
981
+ }
982
+ ]
983
+ },
984
+ ],
985
},
986
)
987
response.raise_for_status()
@@ -1001,7 +1007,6 @@ The following Score API parameters are supported:
1007
1008
```python
1009
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
1004
---8<-- "vllm/entrypoints/pooling/score/protocol.py:score-extra-params"
1010
```
1011
1012
The following extra parameters are supported:
@@ -1009,7 +1014,6 @@ The following extra parameters are supported:
1014
```python
1015
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
1016
--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
1012
---8<-- "vllm/entrypoints/pooling/score/protocol.py:score-extra-params"
1017
```
1018
1019
### Re-rank API
@@ -1092,7 +1096,6 @@ The following Re-rank API parameters are supported:
1096
```python
1097
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
1098
--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
1095
---8<-- "vllm/entrypoints/pooling/score/protocol.py:score-extra-params"
1099
```
1100
1101
The following extra parameters are supported:
@@ -1100,7 +1103,6 @@ The following extra parameters are supported:
1103
```python
1104
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
1105
--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
1103
---8<-- "vllm/entrypoints/pooling/score/protocol.py:rerank-extra-params"
1106
```
1107
1108
## Ray Serve LLM
examples/pooling/classify/vision_classification_online.py
new
+110
@@ -0,0 +1,110 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# ruff: noqa: E501
4
+"""Example Python client for multimodal classification API using vLLM API server
5
+NOTE:
6
+ start a supported multimodal classification model server with `vllm serve`, e.g.
7
+ vllm serve muziyongshixin/Qwen2.5-VL-7B-for-VideoCls \
8
+ --runner pooling \
9
+ --max-model-len 5000 \
10
+ --limit-mm-per-prompt '{"video": 1}' \
11
+ --hf-overrides '{"text_config": {"architectures": ["Qwen2_5_VLForSequenceClassification"]}}'
12
+"""
13
+
14
+import argparse
15
+import pprint
16
+
17
+import requests
18
+
19
+from vllm.multimodal.utils import encode_image_url, fetch_image
20
+
21
+input_text = "This product was excellent and exceeded my expectations"
22
+image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
23
+image_base64 = {"url": encode_image_url(fetch_image(image_url))}
24
+video_url = "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4"
25
+
26
+
27
+def parse_args():
28
+ parse = argparse.ArgumentParser()
29
+ parse.add_argument("--host", type=str, default="localhost")
30
+ parse.add_argument("--port", type=int, default=8000)
31
+ return parse.parse_args()
32
+
33
+
34
+def main(args):
35
+ base_url = f"http://{args.host}:{args.port}"
36
+ models_url = base_url + "/v1/models"
37
+ classify_url = base_url + "/classify"
38
+
39
+ response = requests.get(models_url)
40
+ model_name = response.json()["data"][0]["id"]
41
+
42
+ print("Text classification output:")
43
+ messages = [
44
+ {
45
+ "role": "assistant",
46
+ "content": "Please classify this text request.",
47
+ },
48
+ {
49
+ "role": "user",
50
+ "content": input_text,
51
+ },
52
+ ]
53
+ response = requests.post(
54
+ classify_url,
55
+ json={"model": model_name, "messages": messages},
56
+ )
57
+ pprint.pprint(response.json())
58
+
59
+ print("Image url classification output:")
60
+ messages = [
61
+ {
62
+ "role": "user",
63
+ "content": [
64
+ {"type": "text", "text": "Please classify this image."},
65
+ {"type": "image_url", "image_url": {"url": image_url}},
66
+ ],
67
+ }
68
+ ]
69
+ response = requests.post(
70
+ classify_url,
71
+ json={"model": model_name, "messages": messages},
72
+ )
73
+ pprint.pprint(response.json())
74
+
75
+ print("Image base64 classification output:")
76
+ messages = [
77
+ {
78
+ "role": "user",
79
+ "content": [
80
+ {"type": "text", "text": "Please classify this image."},
81
+ {"type": "image_url", "image_url": image_base64},
82
+ ],
83
+ }
84
+ ]
85
+ response = requests.post(
86
+ classify_url,
87
+ json={"model": model_name, "messages": messages},
88
+ )
89
+ pprint.pprint(response.json())
90
+
91
+ print("Video url classification output:")
92
+ messages = [
93
+ {
94
+ "role": "user",
95
+ "content": [
96
+ {"type": "text", "text": "Please classify this video."},
97
+ {"type": "video_url", "video_url": {"url": video_url}},
98
+ ],
99
+ }
100
+ ]
101
+ response = requests.post(
102
+ classify_url,
103
+ json={"model": model_name, "messages": messages},
104
+ )
105
+ pprint.pprint(response.json())
106
+
107
+
108
+if __name__ == "__main__":
109
+ args = parse_args()
110
+ main(args)
examples/pooling/embed/template/dse_qwen2_vl.jinja
renamed
examples/pooling/embed/template/vlm2vec_phi3v.jinja
renamed
examples/pooling/embed/template/vlm2vec_qwen2vl.jinja
renamed
examples/pooling/embed/vision_embedding_offline.py
+236
-13
@@ -11,23 +11,79 @@ on HuggingFace model repository.
11
12
import argparse
13
from dataclasses import asdict
14
+from pathlib import Path
15
16
from PIL.Image import Image
17
18
from vllm import LLM, EngineArgs
19
from vllm.multimodal.utils import fetch_image
20
+from vllm.utils.print_utils import print_embeddings
21
+
22
+ROOT_DIR = Path(__file__).parent.parent.parent
23
+EMBED_TEMPLATE_DIR = ROOT_DIR / "pooling/embed/template/"
24
25
image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
26
text = "A cat standing in the snow."
27
multi_modal_data = {"image": fetch_image(image_url)}
28
29
25
-def print_embeddings(embeds: list[float]):
26
- embeds_trimmed = (str(embeds[:4])[:-1] + ", ...]") if len(embeds) > 4 else embeds
27
- print(f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
30
+def run_clip(seed: int):
31
+ engine_args = EngineArgs(
32
+ model="openai/clip-vit-base-patch32",
33
+ runner="pooling",
34
+ limit_mm_per_prompt={"image": 1},
35
+ )
36
+
37
+ llm = LLM(**asdict(engine_args) | {"seed": seed})
38
+
39
+ print("Text embedding output:")
40
+ outputs = llm.embed(text, use_tqdm=False)
41
+ print_embeddings(outputs[0].outputs.embedding)
42
43
+ print("Image embedding output:")
44
+ prompt = "" # For image input, make sure that the prompt text is empty
45
+ outputs = llm.embed(
46
+ {
47
+ "prompt": prompt,
48
+ "multi_modal_data": multi_modal_data,
49
+ },
50
+ use_tqdm=False,
51
+ )
52
+ print_embeddings(outputs[0].outputs.embedding)
53
30
-def run_qwen3_vl():
54
+
55
+def run_e5_v(seed: int):
56
+ engine_args = EngineArgs(
57
+ model="royokong/e5-v",
58
+ runner="pooling",
59
+ max_model_len=4096,
60
+ limit_mm_per_prompt={"image": 1},
61
+ )
62
+
63
+ llm = LLM(**asdict(engine_args) | {"seed": seed})
64
+
65
+ llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n" # noqa: E501
66
+
67
+ print("Text embedding output:")
68
+ prompt_text = llama3_template.format(
69
+ f"{text}\nSummary above sentence in one word: "
70
+ )
71
+ outputs = llm.embed(prompt_text, use_tqdm=False)
72
+ print_embeddings(outputs[0].outputs.embedding)
73
+
74
+ print("Image embedding output:")
75
+ prompt_image = llama3_template.format("<image>\nSummary above image in one word: ")
76
+ outputs = llm.embed(
77
+ {
78
+ "prompt": prompt_image,
79
+ "multi_modal_data": multi_modal_data,
80
+ },
81
+ use_tqdm=False,
82
+ )
83
+ print_embeddings(outputs[0].outputs.embedding)
84
+
85
+
86
+def run_qwen3_vl(seed: int):
87
try:
88
from qwen_vl_utils import smart_resize
89
except ModuleNotFoundError:
@@ -61,20 +117,85 @@ def run_qwen3_vl():
117
)
118
default_instruction = "Represent the user's input."
119
image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
64
- text_prompt = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{text}<|im_end|>\n<|im_start|>assistant\n"
65
- image_prompt = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}<|im_end|>\n<|im_start|>assistant\n"
66
- image_text_prompt = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}{text}<|im_end|>\n<|im_start|>assistant\n"
120
+ prompt_text = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{text}<|im_end|>\n<|im_start|>assistant\n"
121
+ prompt_image = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}<|im_end|>\n<|im_start|>assistant\n"
122
+ prompt_image_text = f"<|im_start|>system\n{default_instruction}<|im_end|>\n<|im_start|>user\n{image_placeholder}{text}<|im_end|>\n<|im_start|>assistant\n"
123
+
124
+ llm = LLM(**asdict(engine_args) | {"seed": seed})
125
+
126
+ print("Text embedding output:")
127
+ outputs = llm.embed(prompt_text, use_tqdm=False)
128
+ print_embeddings(outputs[0].outputs.embedding)
129
+
130
+ print("Image embedding output:")
131
+ outputs = llm.embed(
132
+ {
133
+ "prompt": prompt_image,
134
+ "multi_modal_data": multi_modal_data,
135
+ },
136
+ use_tqdm=False,
137
+ )
138
+ print_embeddings(outputs[0].outputs.embedding)
139
+
140
+ print("Image+Text embedding output:")
141
+ outputs = llm.embed(
142
+ {
143
+ "prompt": prompt_image_text,
144
+ "multi_modal_data": multi_modal_data,
145
+ },
146
+ use_tqdm=False,
147
+ )
148
+ print_embeddings(outputs[0].outputs.embedding)
149
+
150
+
151
+def run_siglip(seed: int):
152
+ engine_args = EngineArgs(
153
+ model="google/siglip-base-patch16-224",
154
+ runner="pooling",
155
+ limit_mm_per_prompt={"image": 1},
156
+ )
157
68
- llm = LLM(**asdict(engine_args))
158
+ llm = LLM(**asdict(engine_args) | {"seed": seed})
159
160
print("Text embedding output:")
71
- outputs = llm.embed(text_prompt, use_tqdm=False)
161
+ outputs = llm.embed(text, use_tqdm=False)
162
print_embeddings(outputs[0].outputs.embedding)
163
164
print("Image embedding output:")
165
+ prompt = "" # For image input, make sure that the prompt text is empty
166
outputs = llm.embed(
167
{
77
- "prompt": image_prompt,
168
+ "prompt": prompt,
169
+ "multi_modal_data": multi_modal_data,
170
+ },
171
+ use_tqdm=False,
172
+ )
173
+ print_embeddings(outputs[0].outputs.embedding)
174
+
175
+
176
+def run_vlm2vec_phi3v(seed: int):
177
+ engine_args = EngineArgs(
178
+ model="TIGER-Lab/VLM2Vec-Full",
179
+ runner="pooling",
180
+ max_model_len=4096,
181
+ trust_remote_code=True,
182
+ mm_processor_kwargs={"num_crops": 4},
183
+ limit_mm_per_prompt={"image": 1},
184
+ )
185
+
186
+ llm = LLM(**asdict(engine_args) | {"seed": seed})
187
+ image_token = "<|image_1|>"
188
+
189
+ print("Text embedding output:")
190
+ prompt_text = f"Find me an everyday image that matches the given caption: {text}"
191
+ outputs = llm.embed(prompt_text, use_tqdm=False)
192
+ print_embeddings(outputs[0].outputs.embedding)
193
+
194
+ print("Image embedding output:")
195
+ prompt_image = f"{image_token} Find a day-to-day image that looks similar to the provided image." # noqa: E501
196
+ outputs = llm.embed(
197
+ {
198
+ "prompt": prompt_image,
199
"multi_modal_data": multi_modal_data,
200
},
201
use_tqdm=False,
@@ -82,9 +203,99 @@ def run_qwen3_vl():
203
print_embeddings(outputs[0].outputs.embedding)
204
205
print("Image+Text embedding output:")
206
+ prompt_image_text = (
207
+ f"{image_token} Represent the given image with the following question: {text}" # noqa: E501
208
+ )
209
outputs = llm.embed(
210
{
87
- "prompt": image_text_prompt,
211
+ "prompt": prompt_image_text,
212
+ "multi_modal_data": multi_modal_data,
213
+ },
214
+ use_tqdm=False,
215
+ )
216
+ print_embeddings(outputs[0].outputs.embedding)
217
+
218
+
219
+def run_vlm2vec_qwen2vl(seed: int):
220
+ # vLLM does not support LoRA adapters on multi-modal encoder,
221
+ # so we merge the weights first
222
+ from huggingface_hub.constants import HF_HUB_CACHE
223
+ from peft import PeftConfig, PeftModel
224
+ from transformers import AutoModelForImageTextToText, AutoProcessor
225
+
226
+ from vllm.entrypoints.chat_utils import load_chat_template
227
+
228
+ model_id = "TIGER-Lab/VLM2Vec-Qwen2VL-2B"
229
+
230
+ base_model = AutoModelForImageTextToText.from_pretrained(model_id)
231
+ lora_model = PeftModel.from_pretrained(
232
+ base_model,
233
+ model_id,
234
+ config=PeftConfig.from_pretrained(model_id),
235
+ )
236
+ model = lora_model.merge_and_unload().to(dtype=base_model.dtype)
237
+ model._hf_peft_config_loaded = False # Needed to save the merged model
238
+
239
+ processor = AutoProcessor.from_pretrained(
240
+ model_id,
241
+ # `min_pixels` and `max_pixels` are deprecated for
242
+ # transformers `preprocessor_config.json`
243
+ size={"shortest_edge": 3136, "longest_edge": 12845056},
244
+ )
245
+ processor.chat_template = load_chat_template(
246
+ # The original chat template is not correct
247
+ EMBED_TEMPLATE_DIR / "vlm2vec_qwen2vl.jinja",
248
+ )
249
+
250
+ merged_path = str(
251
+ Path(HF_HUB_CACHE) / ("models--" + model_id.replace("/", "--") + "-vllm")
252
+ )
253
+ print(f"Saving merged model to {merged_path}...")
254
+ print(
255
+ "NOTE: This directory is not tracked by `huggingface_hub` "
256
+ "so you have to delete this manually if you don't want it anymore."
257
+ )
258
+ model.save_pretrained(merged_path)
259
+ processor.save_pretrained(merged_path)
260
+ print("Done!")
261
+
262
+ engine_args = EngineArgs(
263
+ model=merged_path,
264
+ runner="pooling",
265
+ max_model_len=4096,
266
+ mm_processor_kwargs={
267
+ "min_pixels": 3136,
268
+ "max_pixels": 12845056,
269
+ },
270
+ limit_mm_per_prompt={"image": 1},
271
+ )
272
+
273
+ llm = LLM(**asdict(engine_args) | {"seed": seed})
274
+ image_token = "<|image_pad|>"
275
+
276
+ print("Text embedding output:")
277
+ prompt_text = f"Find me an everyday image that matches the given caption: {text}"
278
+ outputs = llm.embed(prompt_text, use_tqdm=False)
279
+ print_embeddings(outputs[0].outputs.embedding)
280
+
281
+ print("Image embedding output:")
282
+ prompt_image = f"{image_token} Find a day-to-day image that looks similar to the provided image." # noqa: E501
283
+ outputs = llm.embed(
284
+ {
285
+ "prompt": prompt_image,
286
+ "multi_modal_data": multi_modal_data,
287
+ },
288
+ use_tqdm=False,
289
+ )
290
+ print_embeddings(outputs[0].outputs.embedding)
291
+
292
+ print("Image+Text embedding output:")
293
+ prompt_image_text = (
294
+ f"{image_token} Represent the given image with the following question: {text}" # noqa: E501
295
+ )
296
+ outputs = llm.embed(
297
+ {
298
+ "prompt": prompt_image_text,
299
"multi_modal_data": multi_modal_data,
300
},
301
use_tqdm=False,
@@ -93,7 +304,12 @@ def run_qwen3_vl():
304
305
306
model_example_map = {
307
+ "clip": run_clip,
308
+ "e5_v": run_e5_v,
309
"qwen3_vl": run_qwen3_vl,
310
+ "siglip": run_siglip,
311
+ "vlm2vec_phi3v": run_vlm2vec_phi3v,
312
+ "vlm2vec_qwen2vl": run_vlm2vec_qwen2vl,
313
}
314
315
@@ -103,16 +319,23 @@ def parse_args():
319
)
320
parser.add_argument(
321
"--model",
322
+ "-m",
323
type=str,
324
+ default="vlm2vec_phi3v",
325
choices=model_example_map.keys(),
108
- required=True,
326
help="The name of the embedding model.",
327
)
328
+ parser.add_argument(
329
+ "--seed",
330
+ type=int,
331
+ default=0,
332
+ help="Set the seed when initializing `vllm.LLM`.",
333
+ )
334
return parser.parse_args()
335
336
337
def main(args):
115
- model_example_map[args.model]()
338
+ model_example_map[args.model](args.seed)
339
340
341
if __name__ == "__main__":
examples/pooling/embed/vision_embedding_online.py
+4
-7
@@ -17,6 +17,8 @@ from openai.types.chat import ChatCompletionMessageParam
17
from openai.types.create_embedding_response import CreateEmbeddingResponse
18
from PIL import Image
19
20
+from vllm.utils.print_utils import print_embeddings
21
+
22
# Modify OpenAI's API key and API base to use vLLM's API server.
23
openai_api_key = "EMPTY"
24
openai_api_base = "http://localhost:8000/v1"
@@ -51,11 +53,6 @@ def create_chat_embeddings(
53
)
54
55
54
-def print_embeddings(embeds):
55
- embeds_trimmed = (str(embeds[:4])[:-1] + ", ...]") if len(embeds) > 4 else embeds
56
- print(f"Embeddings: {embeds_trimmed} (size={len(embeds)})")
57
-
58
-
56
def run_clip(client: OpenAI, model: str):
57
"""
58
Start the server using:
@@ -105,7 +102,7 @@ def run_dse_qwen2_vl(client: OpenAI, model: str):
102
--runner pooling \
103
--trust-remote-code \
104
--max-model-len 8192 \
108
- --chat-template examples/template_dse_qwen2_vl.jinja
105
+ --chat-template examples/pooling/embed/template/dse_qwen2_vl.jinja
106
"""
107
response = create_chat_embeddings(
108
client,
@@ -316,7 +313,7 @@ def run_vlm2vec(client: OpenAI, model: str):
313
--runner pooling \
314
--trust-remote-code \
315
--max-model-len 4096 \
319
- --chat-template examples/template_vlm2vec_phi3v.jinja
316
+ --chat-template examples/pooling/embed/template/vlm2vec_phi3v.jinja
317
"""
318
319
response = create_chat_embeddings(
examples/pooling/pooling/vision_language_pooling.py
deleted
-441
@@ -1,441 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-This example shows how to use vLLM for running offline inference with
5
-the correct prompt format on vision language models for multimodal pooling.
6
-
7
-For most models, the prompt format should follow corresponding examples
8
-on HuggingFace model repository.
9
-"""
10
-
11
-from argparse import Namespace
12
-from dataclasses import asdict
13
-from pathlib import Path
14
-from typing import Literal, NamedTuple, TypeAlias, TypedDict, get_args
15
-
16
-from PIL.Image import Image
17
-
18
-from vllm import LLM, EngineArgs
19
-from vllm.entrypoints.pooling.score.utils import ScoreMultiModalParam
20
-from vllm.multimodal.utils import fetch_image
21
-from vllm.utils.argparse_utils import FlexibleArgumentParser
22
-
23
-ROOT_DIR = Path(__file__).parent.parent.parent
24
-EXAMPLES_DIR = ROOT_DIR / "examples"
25
-
26
-
27
-class TextQuery(TypedDict):
28
- modality: Literal["text"]
29
- text: str
30
-
31
-
32
-class ImageQuery(TypedDict):
33
- modality: Literal["image"]
34
- image: Image
35
-
36
-
37
-class TextImageQuery(TypedDict):
38
- modality: Literal["text+image"]
39
- text: str
40
- image: Image
41
-
42
-
43
-class TextImagesQuery(TypedDict):
44
- modality: Literal["text+images"]
45
- text: str
46
- image: ScoreMultiModalParam
47
-
48
-
49
-QueryModality = Literal["text", "image", "text+image", "text+images"]
50
-Query: TypeAlias = TextQuery | ImageQuery | TextImageQuery | TextImagesQuery
51
-
52
-
53
-class ModelRequestData(NamedTuple):
54
- engine_args: EngineArgs
55
- prompt: str | None = None
56
- image: Image | None = None
57
- query: str | None = None
58
- documents: ScoreMultiModalParam | None = None
59
-
60
-
61
-def run_clip(query: Query) -> ModelRequestData:
62
- if query["modality"] == "text":
63
- prompt = query["text"]
64
- image = None
65
- elif query["modality"] == "image":
66
- prompt = "" # For image input, make sure that the prompt text is empty
67
- image = query["image"]
68
- else:
69
- modality = query["modality"]
70
- raise ValueError(f"Unsupported query modality: '{modality}'")
71
-
72
- engine_args = EngineArgs(
73
- model="openai/clip-vit-base-patch32",
74
- runner="pooling",
75
- limit_mm_per_prompt={"image": 1},
76
- )
77
-
78
- return ModelRequestData(
79
- engine_args=engine_args,
80
- prompt=prompt,
81
- image=image,
82
- )
83
-
84
-
85
-def run_e5_v(query: Query) -> ModelRequestData:
86
- llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n" # noqa: E501
87
-
88
- if query["modality"] == "text":
89
- text = query["text"]
90
- prompt = llama3_template.format(f"{text}\nSummary above sentence in one word: ")
91
- image = None
92
- elif query["modality"] == "image":
93
- prompt = llama3_template.format("<image>\nSummary above image in one word: ")
94
- image = query["image"]
95
- else:
96
- modality = query["modality"]
97
- raise ValueError(f"Unsupported query modality: '{modality}'")
98
-
99
- engine_args = EngineArgs(
100
- model="royokong/e5-v",
101
- runner="pooling",
102
- max_model_len=4096,
103
- limit_mm_per_prompt={"image": 1},
104
- )
105
-
106
- return ModelRequestData(
107
- engine_args=engine_args,
108
- prompt=prompt,
109
- image=image,
110
- )
111
-
112
-
113
-def run_jinavl_reranker(query: Query) -> ModelRequestData:
114
- if query["modality"] != "text+images":
115
- raise ValueError(f"Unsupported query modality: '{query['modality']}'")
116
-
117
- engine_args = EngineArgs(
118
- model="jinaai/jina-reranker-m0",
119
- runner="pooling",
120
- max_model_len=32768,
121
- trust_remote_code=True,
122
- mm_processor_kwargs={
123
- "min_pixels": 3136,
124
- "max_pixels": 602112,
125
- },
126
- limit_mm_per_prompt={"image": 1},
127
- )
128
-
129
- return ModelRequestData(
130
- engine_args=engine_args,
131
- query=query["text"],
132
- documents=query["image"],
133
- )
134
-
135
-
136
-def run_qwen3_vl(query: Query) -> ModelRequestData:
137
- image_placeholder = "<vision_start><|image_pad|><vision_end>"
138
- if query["modality"] == "text":
139
- prompt = query["text"]
140
- image = None
141
- elif query["modality"] == "image":
142
- prompt = image_placeholder
143
- image = query["image"]
144
- elif query["modality"] == "text+image":
145
- text = query["text"]
146
- prompt = f"{image_placeholder}\n{text}"
147
- image = query["image"]
148
- else:
149
- modality = query["modality"]
150
- raise ValueError(f"Unsupported query modality: '{modality}'")
151
-
152
- engine_args = EngineArgs(
153
- model="Qwen/Qwen3-VL-Embedding-2B",
154
- runner="pooling",
155
- max_model_len=8192,
156
- limit_mm_per_prompt={"image": 1},
157
- )
158
-
159
- return ModelRequestData(
160
- engine_args=engine_args,
161
- prompt=prompt,
162
- image=image,
163
- )
164
-
165
-
166
-def run_siglip(query: Query) -> ModelRequestData:
167
- if query["modality"] == "text":
168
- prompt = query["text"]
169
- image = None
170
- elif query["modality"] == "image":
171
- prompt = "" # For image input, make sure that the prompt text is empty
172
- image = query["image"]
173
- else:
174
- modality = query["modality"]
175
- raise ValueError(f"Unsupported query modality: '{modality}'")
176
-
177
- engine_args = EngineArgs(
178
- model="google/siglip-base-patch16-224",
179
- runner="pooling",
180
- limit_mm_per_prompt={"image": 1},
181
- )
182
-
183
- return ModelRequestData(
184
- engine_args=engine_args,
185
- prompt=prompt,
186
- image=image,
187
- )
188
-
189
-
190
-def _get_vlm2vec_prompt_image(query: Query, image_token: str):
191
- if query["modality"] == "text":
192
- text = query["text"]
193
- prompt = f"Find me an everyday image that matches the given caption: {text}"
194
- image = None
195
- elif query["modality"] == "image":
196
- prompt = f"{image_token} Find a day-to-day image that looks similar to the provided image." # noqa: E501
197
- image = query["image"]
198
- elif query["modality"] == "text+image":
199
- text = query["text"]
200
- prompt = f"{image_token} Represent the given image with the following question: {text}" # noqa: E501
201
- image = query["image"]
202
- else:
203
- modality = query["modality"]
204
- raise ValueError(f"Unsupported query modality: {modality!r}")
205
-
206
- return prompt, image
207
-
208
-
209
-def run_vlm2vec_phi3v(query: Query) -> ModelRequestData:
210
- prompt, image = _get_vlm2vec_prompt_image(query, "<|image_1|>")
211
-
212
- engine_args = EngineArgs(
213
- model="TIGER-Lab/VLM2Vec-Full",
214
- runner="pooling",
215
- max_model_len=4096,
216
- trust_remote_code=True,
217
- mm_processor_kwargs={"num_crops": 4},
218
- limit_mm_per_prompt={"image": 1},
219
- )
220
-
221
- return ModelRequestData(
222
- engine_args=engine_args,
223
- prompt=prompt,
224
- image=image,
225
- )
226
-
227
-
228
-def run_vlm2vec_qwen2vl(query: Query) -> ModelRequestData:
229
- # vLLM does not support LoRA adapters on multi-modal encoder,
230
- # so we merge the weights first
231
- from huggingface_hub.constants import HF_HUB_CACHE
232
- from peft import PeftConfig, PeftModel
233
- from transformers import AutoModelForImageTextToText, AutoProcessor
234
-
235
- from vllm.entrypoints.chat_utils import load_chat_template
236
-
237
- model_id = "TIGER-Lab/VLM2Vec-Qwen2VL-2B"
238
-
239
- base_model = AutoModelForImageTextToText.from_pretrained(model_id)
240
- lora_model = PeftModel.from_pretrained(
241
- base_model,
242
- model_id,
243
- config=PeftConfig.from_pretrained(model_id),
244
- )
245
- model = lora_model.merge_and_unload().to(dtype=base_model.dtype)
246
- model._hf_peft_config_loaded = False # Needed to save the merged model
247
-
248
- processor = AutoProcessor.from_pretrained(
249
- model_id,
250
- # `min_pixels` and `max_pixels` are deprecated for
251
- # transformers `preprocessor_config.json`
252
- size={"shortest_edge": 3136, "longest_edge": 12845056},
253
- )
254
- processor.chat_template = load_chat_template(
255
- # The original chat template is not correct
256
- EXAMPLES_DIR / "template_vlm2vec_qwen2vl.jinja",
257
- )
258
-
259
- merged_path = str(
260
- Path(HF_HUB_CACHE) / ("models--" + model_id.replace("/", "--") + "-vllm")
261
- )
262
- print(f"Saving merged model to {merged_path}...")
263
- print(
264
- "NOTE: This directory is not tracked by `huggingface_hub` "
265
- "so you have to delete this manually if you don't want it anymore."
266
- )
267
- model.save_pretrained(merged_path)
268
- processor.save_pretrained(merged_path)
269
- print("Done!")
270
-
271
- prompt, image = _get_vlm2vec_prompt_image(query, "<|image_pad|>")
272
-
273
- engine_args = EngineArgs(
274
- model=merged_path,
275
- runner="pooling",
276
- max_model_len=4096,
277
- mm_processor_kwargs={
278
- "min_pixels": 3136,
279
- "max_pixels": 12845056,
280
- },
281
- limit_mm_per_prompt={"image": 1},
282
- )
283
-
284
- return ModelRequestData(
285
- engine_args=engine_args,
286
- prompt=prompt,
287
- image=image,
288
- )
289
-
290
-
291
-def get_query(modality: QueryModality):
292
- if modality == "text":
293
- return TextQuery(modality="text", text="A dog sitting in the grass")
294
-
295
- if modality == "image":
296
- return ImageQuery(
297
- modality="image",
298
- image=fetch_image(
299
- "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/eskimo.jpg" # noqa: E501
300
- ),
301
- )
302
-
303
- if modality == "text+image":
304
- return TextImageQuery(
305
- modality="text+image",
306
- text="A cat standing in the snow.",
307
- image=fetch_image(
308
- "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg" # noqa: E501
309
- ),
310
- )
311
-
312
- if modality == "text+images":
313
- return TextImagesQuery(
314
- modality="text+images",
315
- text="slm markdown",
316
- image={
317
- "content": [
318
- {
319
- "type": "image_url",
320
- "image_url": {
321
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
322
- },
323
- },
324
- {
325
- "type": "image_url",
326
- "image_url": {
327
- "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/paper-11.png"
328
- },
329
- },
330
- ]
331
- },
332
- )
333
-
334
- msg = f"Modality {modality} is not supported."
335
- raise ValueError(msg)
336
-
337
-
338
-def run_encode(model: str, modality: QueryModality, seed: int):
339
- query = get_query(modality)
340
- req_data = model_example_map[model](query)
341
-
342
- # Disable other modalities to save memory
343
- default_limits = {"image": 0, "video": 0, "audio": 0}
344
- req_data.engine_args.limit_mm_per_prompt = default_limits | dict(
345
- req_data.engine_args.limit_mm_per_prompt or {}
346
- )
347
-
348
- engine_args = asdict(req_data.engine_args) | {"seed": seed}
349
- llm = LLM(**engine_args)
350
-
351
- mm_data = {}
352
- if req_data.image is not None:
353
- mm_data["image"] = req_data.image
354
-
355
- outputs = llm.embed(
356
- {
357
- "prompt": req_data.prompt,
358
- "multi_modal_data": mm_data,
359
- }
360
- )
361
-
362
- print("-" * 50)
363
- for output in outputs:
364
- print(output.outputs.embedding)
365
- print("-" * 50)
366
-
367
-
368
-def run_score(model: str, modality: QueryModality, seed: int):
369
- query = get_query(modality)
370
- req_data = model_example_map[model](query)
371
-
372
- engine_args = asdict(req_data.engine_args) | {"seed": seed}
373
- llm = LLM(**engine_args)
374
-
375
- outputs = llm.score(req_data.query, req_data.documents)
376
-
377
- print("-" * 30)
378
- print([output.outputs.score for output in outputs])
379
- print("-" * 30)
380
-
381
-
382
-model_example_map = {
383
- "clip": run_clip,
384
- "e5_v": run_e5_v,
385
- "jinavl_reranker": run_jinavl_reranker,
386
- "qwen3_vl": run_qwen3_vl,
387
- "siglip": run_siglip,
388
- "vlm2vec_phi3v": run_vlm2vec_phi3v,
389
- "vlm2vec_qwen2vl": run_vlm2vec_qwen2vl,
390
-}
391
-
392
-
393
-def parse_args():
394
- parser = FlexibleArgumentParser(
395
- description="Demo on using vLLM for offline inference with "
396
- "vision language models for multimodal pooling tasks."
397
- )
398
- parser.add_argument(
399
- "--model-name",
400
- "-m",
401
- type=str,
402
- default="vlm2vec_phi3v",
403
- choices=model_example_map.keys(),
404
- help="The name of the embedding model.",
405
- )
406
- parser.add_argument(
407
- "--task",
408
- "-t",
409
- type=str,
410
- default="embedding",
411
- choices=["embedding", "scoring"],
412
- help="The task type.",
413
- )
414
- parser.add_argument(
415
- "--modality",
416
- type=str,
417
- default="image",
418
- choices=get_args(QueryModality),
419
- help="Modality of the input.",
420
- )
421
- parser.add_argument(
422
- "--seed",
423
- type=int,
424
- default=0,
425
- help="Set the seed when initializing `vllm.LLM`.",
426
- )
427
- return parser.parse_args()
428
-
429
-
430
-def main(args: Namespace):
431
- if args.task == "embedding":
432
- run_encode(args.model_name, args.modality, args.seed)
433
- elif args.task == "scoring":
434
- run_score(args.model_name, args.modality, args.seed)
435
- else:
436
- raise ValueError(f"Unsupported task: {args.task}")
437
-
438
-
439
-if __name__ == "__main__":
440
- args = parse_args()
441
- main(args)
examples/pooling/score/vision_rerank_api_online.py
+14
@@ -30,6 +30,7 @@ document = (
30
"as the dog offers its paw in a heartwarming display of companionship and trust."
31
)
32
image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
33
+video_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4"
34
documents = [
35
{
36
"type": "text",
@@ -43,6 +44,10 @@ documents = [
44
"type": "image_url",
45
"image_url": {"url": encode_image_url(fetch_image(image_url))},
46
},
47
+ {
48
+ "type": "video_url",
49
+ "video_url": {"url": video_url},
50
+ },
51
]
52
53
@@ -89,6 +94,15 @@ def main(args):
94
response = requests.post(rerank_url, json=prompt)
95
pprint.pprint(response.json())
96
97
+ print("Query: string & Document: video url")
98
+ prompt = {
99
+ "model": model,
100
+ "query": query,
101
+ "documents": {"content": [documents[3]]},
102
+ }
103
+ response = requests.post(rerank_url, json=prompt)
104
+ pprint.pprint(response.json())
105
+
106
print("Query: string & Document: text + image url")
107
prompt = {
108
"model": model,
examples/pooling/score/vision_reranker_offline.py
+85
-72
@@ -15,20 +15,47 @@ from pathlib import Path
15
from typing import NamedTuple
16
17
from vllm import LLM, EngineArgs
18
-from vllm.entrypoints.pooling.score.utils import ScoreMultiModalParam
18
+from vllm.multimodal.utils import encode_image_url, fetch_image
19
from vllm.utils.argparse_utils import FlexibleArgumentParser
20
21
TEMPLATE_HOME = Path(__file__).parent / "template"
22
23
24
+query = "A woman playing with her dog on a beach at sunset."
25
+document = (
26
+ "A woman shares a joyful moment with her golden retriever on a sun-drenched "
27
+ "beach at sunset, as the dog offers its paw in a heartwarming display of "
28
+ "companionship and trust."
29
+)
30
+image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
31
+video_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4"
32
+documents = [
33
+ {
34
+ "type": "text",
35
+ "text": document,
36
+ },
37
+ {
38
+ "type": "image_url",
39
+ "image_url": {"url": image_url},
40
+ },
41
+ {
42
+ "type": "image_url",
43
+ "image_url": {"url": encode_image_url(fetch_image(image_url))},
44
+ },
45
+ {
46
+ "type": "video_url",
47
+ "video_url": {"url": video_url},
48
+ },
49
+]
50
+
51
+
52
class RerankModelData(NamedTuple):
53
engine_args: EngineArgs
54
chat_template: str | None = None
55
+ modality: set[str] = {}
56
57
29
-def run_jinavl_reranker(modality: str) -> RerankModelData:
30
- assert modality == "image"
31
-
58
+def run_jinavl_reranker() -> RerankModelData:
59
engine_args = EngineArgs(
60
model="jinaai/jina-reranker-m0",
61
runner="pooling",
@@ -38,19 +65,15 @@ def run_jinavl_reranker(modality: str) -> RerankModelData:
65
"min_pixels": 3136,
66
"max_pixels": 602112,
67
},
41
- limit_mm_per_prompt={modality: 1},
42
- )
43
- return RerankModelData(
44
- engine_args=engine_args,
68
)
69
+ return RerankModelData(engine_args=engine_args, modality={"image"})
70
71
48
-def run_qwen3_vl_reranker(modality: str) -> RerankModelData:
72
+def run_qwen3_vl_reranker() -> RerankModelData:
73
engine_args = EngineArgs(
74
model="Qwen/Qwen3-VL-Reranker-2B",
75
runner="pooling",
76
max_model_len=16384,
53
- limit_mm_per_prompt={modality: 1},
77
# HuggingFace model configuration overrides required for compatibility
78
hf_overrides={
79
# Manually route to sequence classification architecture
@@ -71,10 +94,11 @@ def run_qwen3_vl_reranker(modality: str) -> RerankModelData:
94
return RerankModelData(
95
engine_args=engine_args,
96
chat_template=chat_template,
97
+ modality={"image", "video"},
98
)
99
100
77
-model_example_map: dict[str, Callable[[str], RerankModelData]] = {
101
+model_example_map: dict[str, Callable[[], RerankModelData]] = {
102
"jinavl_reranker": run_jinavl_reranker,
103
"qwen3_vl_reranker": run_qwen3_vl_reranker,
104
}
@@ -93,78 +117,67 @@ def parse_args():
117
choices=model_example_map.keys(),
118
help="The name of the reranker model.",
119
)
96
- parser.add_argument(
97
- "--modality",
98
- type=str,
99
- default="image",
100
- choices=["image", "video"],
101
- help="Modality of the multimodal input (image or video).",
102
- )
120
return parser.parse_args()
121
122
106
-def get_multi_modal_input(modality: str) -> tuple[str, ScoreMultiModalParam]:
107
- # Sample query for testing the reranker
108
- if modality == "image":
109
- query = "A woman playing with her dog on a beach at sunset."
110
- # Sample multimodal documents to be scored against the query
111
- # Each document contains an image URL that will be fetched and processed
112
- documents: ScoreMultiModalParam = {
113
- "content": [
114
- {
115
- "type": "text",
116
- "text": (
117
- "A woman shares a joyful moment with her golden retriever on a sun-drenched beach at sunset, " # noqa: E501
118
- "as the dog offers its paw in a heartwarming display of companionship and trust." # noqa: E501
119
- ),
120
- },
121
- {
122
- "type": "image_url",
123
- "image_url": {
124
- "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
125
- },
126
- },
127
- ]
128
- }
129
- elif modality == "video":
130
- query = "A girl is drawing pictures on an ipad."
131
- # Sample video documents to be scored against the query
132
- documents: ScoreMultiModalParam = {
133
- "content": [
134
- {
135
- "type": "text",
136
- "text": "A girl is drawing a guitar on her ipad with Apple Pencil.",
137
- },
138
- {
139
- "type": "video_url",
140
- "video_url": {
141
- "url": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4"
142
- },
143
- },
144
- ]
145
- }
146
- else:
147
- raise ValueError(f"Unsupported modality: {modality}")
148
- return query, documents
149
-
150
-
123
def main(args: Namespace):
124
# Run the selected reranker model
153
- modality = args.modality
154
- model_request = model_example_map[args.model_name](modality)
125
+ model_request = model_example_map[args.model_name]()
126
engine_args = model_request.engine_args
127
128
llm = LLM(**asdict(engine_args))
129
159
- query, documents = get_multi_modal_input(modality)
160
- outputs = llm.score(query, documents, chat_template=model_request.chat_template)
130
+ print("Query: string & Document: string")
131
+ outputs = llm.score(query, document)
132
+ print("Relevance scores:", [output.outputs.score for output in outputs])
133
162
- print("-" * 50)
163
- print(f"Model: {engine_args.model}")
164
- print(f"Modality: {modality}")
165
- print(f"Query: {query}")
134
+ print("Query: string & Document: text")
135
+ outputs = llm.score(
136
+ query, {"content": [documents[0]]}, chat_template=model_request.chat_template
137
+ )
138
+ print("Relevance scores:", [output.outputs.score for output in outputs])
139
+
140
+ print("Query: string & Document: image url")
141
+ outputs = llm.score(
142
+ query, {"content": [documents[1]]}, chat_template=model_request.chat_template
143
+ )
144
+ print("Relevance scores:", [output.outputs.score for output in outputs])
145
+
146
+ print("Query: string & Document: image base64")
147
+ outputs = llm.score(
148
+ query, {"content": [documents[2]]}, chat_template=model_request.chat_template
149
+ )
150
+ print("Relevance scores:", [output.outputs.score for output in outputs])
151
+
152
+ if "video" in model_request.modality:
153
+ print("Query: string & Document: video url")
154
+ outputs = llm.score(
155
+ query,
156
+ {"content": [documents[3]]},
157
+ chat_template=model_request.chat_template,
158
+ )
159
+ print("Relevance scores:", [output.outputs.score for output in outputs])
160
+
161
+ print("Query: string & Document: text + image url")
162
+ outputs = llm.score(
163
+ query,
164
+ {"content": [documents[0], documents[1]]},
165
+ chat_template=model_request.chat_template,
166
+ )
167
+ print("Relevance scores:", [output.outputs.score for output in outputs])
168
+
169
+ print("Query: string & Document: list")
170
+ outputs = llm.score(
171
+ query,
172
+ [
173
+ document,
174
+ {"content": [documents[0]]},
175
+ {"content": [documents[1]]},
176
+ {"content": [documents[0], documents[1]]},
177
+ ],
178
+ chat_template=model_request.chat_template,
179
+ )
180
print("Relevance scores:", [output.outputs.score for output in outputs])
167
- print("-" * 50)
181
182
183
if __name__ == "__main__":
examples/pooling/score/vision_score_api_online.py
+14
@@ -29,6 +29,7 @@ document = (
29
"as the dog offers its paw in a heartwarming display of companionship and trust."
30
)
31
image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
32
+video_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4"
33
documents = [
34
{
35
"type": "text",
@@ -42,6 +43,10 @@ documents = [
43
"type": "image_url",
44
"image_url": {"url": encode_image_url(fetch_image(image_url))},
45
},
46
+ {
47
+ "type": "video_url",
48
+ "video_url": {"url": video_url},
49
+ },
50
]
51
52
@@ -92,6 +97,15 @@ def main(args):
97
response = requests.post(score_url, json=prompt)
98
pprint.pprint(response.json())
99
100
+ print("Query: string & Document: video url")
101
+ prompt = {
102
+ "model": model,
103
+ "queries": query,
104
+ "documents": {"content": [documents[3]]},
105
+ }
106
+ response = requests.post(score_url, json=prompt)
107
+ pprint.pprint(response.json())
108
+
109
print("Query: string & Document: text + image url")
110
prompt = {
111
"model": model,
tests/entrypoints/pooling/classify/test_offline.py
+50
-7
@@ -7,12 +7,15 @@ import pytest
7
import torch
8
9
from tests.models.utils import softmax
10
-from vllm import LLM, PoolingParams
10
+from vllm import LLM, ClassificationRequestOutput, PoolingParams, PoolingRequestOutput
11
from vllm.distributed import cleanup_dist_env_and_memory
12
+from vllm.tasks import PoolingTask
13
14
MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
15
15
-prompts = ["The chef prepared a delicious meal."]
16
+prompt = "The chef prepared a delicious meal."
17
+prompt_token_ids = [785, 29706, 10030, 264, 17923, 15145, 13]
18
+num_labels = 2
19
20
21
@pytest.fixture(scope="module")
@@ -35,11 +38,48 @@ def llm():
38
cleanup_dist_env_and_memory()
39
40
41
+@pytest.mark.skip_global_cleanup
42
+def test_str_prompts(llm: LLM):
43
+ outputs = llm.classify(prompt, use_tqdm=False)
44
+ assert len(outputs) == 1
45
+ assert isinstance(outputs[0], ClassificationRequestOutput)
46
+ assert outputs[0].prompt_token_ids == prompt_token_ids
47
+ assert len(outputs[0].outputs.probs) == num_labels
48
+
49
+
50
+@pytest.mark.skip_global_cleanup
51
+def test_token_ids_prompts(llm: LLM):
52
+ outputs = llm.classify([prompt_token_ids], use_tqdm=False)
53
+ assert len(outputs) == 1
54
+ assert isinstance(outputs[0], ClassificationRequestOutput)
55
+ assert outputs[0].prompt_token_ids == prompt_token_ids
56
+ assert len(outputs[0].outputs.probs) == num_labels
57
+
58
+
59
+@pytest.mark.skip_global_cleanup
60
+def test_list_prompts(llm: LLM):
61
+ outputs = llm.classify([prompt, prompt_token_ids], use_tqdm=False)
62
+ assert len(outputs) == 2
63
+ for i in range(len(outputs)):
64
+ assert isinstance(outputs[i], ClassificationRequestOutput)
65
+ assert outputs[i].prompt_token_ids == prompt_token_ids
66
+ assert len(outputs[i].outputs.probs) == num_labels
67
+
68
+
69
+@pytest.mark.skip_global_cleanup
70
+def test_token_classify(llm: LLM):
71
+ outputs = llm.encode(prompt, pooling_task="token_classify", use_tqdm=False)
72
+ assert len(outputs) == 1
73
+ assert isinstance(outputs[0], PoolingRequestOutput)
74
+ assert outputs[0].prompt_token_ids == prompt_token_ids
75
+ assert outputs[0].outputs.data.shape == (len(prompt_token_ids), num_labels)
76
+
77
+
78
@pytest.mark.skip_global_cleanup
79
def test_pooling_params(llm: LLM):
80
def get_outputs(use_activation):
81
outputs = llm.classify(
42
- prompts,
82
+ prompt,
83
pooling_params=PoolingParams(use_activation=use_activation),
84
use_tqdm=False,
85
)
@@ -61,11 +101,14 @@ def test_pooling_params(llm: LLM):
101
102
103
@pytest.mark.skip_global_cleanup
64
-def test_token_classify(llm: LLM):
65
- llm.encode(prompts, pooling_task="token_classify", use_tqdm=False)
66
-
67
-
104
def test_score_api(llm: LLM):
105
err_msg = "Score API is only enabled for num_labels == 1."
106
with pytest.raises(ValueError, match=err_msg):
107
llm.score("ping", "pong", use_tqdm=False)
108
+
109
+
110
+@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"])
111
+def test_unsupported_tasks(llm: LLM, task: PoolingTask):
112
+ err_msg = f"Unsupported task: '{task}' Supported tasks.+"
113
+ with pytest.raises(ValueError, match=err_msg):
114
+ llm.encode(prompt, pooling_task=task, use_tqdm=False)
tests/entrypoints/pooling/embed/test_online_vision.py
+81
-2
@@ -10,12 +10,12 @@ from transformers import AutoProcessor
10
from tests.utils import VLLM_PATH, RemoteOpenAIServer
11
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
12
from vllm.multimodal.media import MediaWithBytes
13
-from vllm.multimodal.utils import fetch_image
13
+from vllm.multimodal.utils import encode_image_url, fetch_image
14
15
MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"
16
MAXIMUM_IMAGES = 2
17
18
-vlm2vec_jinja_path = VLLM_PATH / "examples/template_vlm2vec_phi3v.jinja"
18
+vlm2vec_jinja_path = VLLM_PATH / "examples/pooling/embed/template/vlm2vec_phi3v.jinja"
19
assert vlm2vec_jinja_path.exists()
20
21
# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
@@ -26,6 +26,10 @@ TEST_IMAGE_ASSETS = [
26
"RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
27
]
28
29
+input_text = "The best thing about vLLM is that it supports many different models"
30
+image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
31
+image_base64 = {"url": encode_image_url(fetch_image(image_url))}
32
+
33
34
@pytest.fixture(scope="module")
35
def server():
@@ -48,6 +52,81 @@ def server():
52
yield remote_server
53
54
55
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
56
+def test_chat_text_request(server: RemoteOpenAIServer, model_name: str):
57
+ messages = [
58
+ {
59
+ "role": "user",
60
+ "content": input_text,
61
+ },
62
+ ]
63
+
64
+ # note: vlm2vec_phi3v.jinja
65
+ # Embedding models should only embed one message at a time.
66
+
67
+ response = requests.post(
68
+ server.url_for("v1/embeddings"),
69
+ json={"model": model_name, "messages": messages},
70
+ )
71
+ response.raise_for_status()
72
+
73
+ output = EmbeddingResponse.model_validate(response.json())
74
+ assert len(output.data) == 1
75
+ assert output.model == MODEL_NAME
76
+ assert len(output.data[0].embedding) == 3072
77
+ assert output.usage.prompt_tokens == 14
78
+
79
+
80
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
81
+def test_chat_image_url_request(server: RemoteOpenAIServer, model_name: str):
82
+ messages = [
83
+ {
84
+ "role": "user",
85
+ "content": [
86
+ {"type": "text", "text": "Represent the user's input."},
87
+ {"type": "image_url", "image_url": {"url": image_url}},
88
+ ],
89
+ }
90
+ ]
91
+
92
+ response = requests.post(
93
+ server.url_for("v1/embeddings"),
94
+ json={"model": model_name, "messages": messages},
95
+ )
96
+ response.raise_for_status()
97
+
98
+ output = EmbeddingResponse.model_validate(response.json())
99
+ assert len(output.data) == 1
100
+ assert output.model == MODEL_NAME
101
+ assert len(output.data[0].embedding) == 3072
102
+ assert output.usage.prompt_tokens == 767
103
+
104
+
105
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
106
+def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
107
+ messages = [
108
+ {
109
+ "role": "user",
110
+ "content": [
111
+ {"type": "text", "text": "Represent the user's input."},
112
+ {"type": "image_url", "image_url": image_base64},
113
+ ],
114
+ }
115
+ ]
116
+
117
+ response = requests.post(
118
+ server.url_for("v1/embeddings"),
119
+ json={"model": model_name, "messages": messages},
120
+ )
121
+ response.raise_for_status()
122
+
123
+ output = EmbeddingResponse.model_validate(response.json())
124
+ assert len(output.data) == 1
125
+ assert output.model == MODEL_NAME
126
+ assert len(output.data[0].embedding) == 3072
127
+ assert output.usage.prompt_tokens == 767
128
+
129
+
130
def get_hf_prompt_tokens(model_name, content, image_url):
131
processor = AutoProcessor.from_pretrained(
132
model_name, trust_remote_code=True, num_crops=4
tests/renderers/test_hf.py
+3
-3
@@ -428,13 +428,13 @@ def test_resolve_content_format_fallbacks(model, expected_format):
428
("template_chatglm.jinja", "string"),
429
("template_chatglm2.jinja", "string"),
430
("template_chatml.jinja", "string"),
431
- ("template_dse_qwen2_vl.jinja", "openai"),
431
("template_falcon_180b.jinja", "string"),
432
("template_falcon.jinja", "string"),
433
("template_inkbot.jinja", "string"),
434
("template_teleflm.jinja", "string"),
436
- ("template_vlm2vec_phi3v.jinja", "openai"),
437
- ("template_vlm2vec_qwen2vl.jinja", "openai"),
435
+ ("pooling/embed/template/dse_qwen2_vl.jinja", "openai"),
436
+ ("pooling/embed/template/vlm2vec_phi3v.jinja", "openai"),
437
+ ("pooling/embed/template/vlm2vec_qwen2vl.jinja", "openai"),
438
("tool_chat_template_granite_20b_fc.jinja", "string"),
439
("tool_chat_template_hermes.jinja", "string"),
440
("tool_chat_template_internlm2_tool.jinja", "string"),
vllm/entrypoints/pooling/base/protocol.py
+15
@@ -40,6 +40,21 @@ class PoolingBasicRequestMixin(OpenAIBaseModel):
40
"if the served model does not use priority scheduling."
41
),
42
)
43
+ mm_processor_kwargs: dict[str, Any] | None = Field(
44
+ default=None,
45
+ description="Additional kwargs to pass to the HF processor.",
46
+ )
47
+ cache_salt: str | None = Field(
48
+ default=None,
49
+ description=(
50
+ "If specified, the prefix cache will be salted with the provided "
51
+ "string to prevent an attacker to guess prompts in multi-user "
52
+ "environments. The salt should be random, protected from "
53
+ "access by 3rd parties, and long enough to be "
54
+ "unpredictable (e.g., 43 characters base64-encoded, corresponding "
55
+ "to 256 bit)."
56
+ ),
57
+ )
58
# --8<-- [end:pooling-common-extra-params]
59
60
vllm/entrypoints/pooling/classify/protocol.py
+1
-7
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import time
5
-from typing import Any, TypeAlias
5
+from typing import TypeAlias
6
7
from pydantic import Field
8
@@ -48,12 +48,6 @@ class ClassificationCompletionRequest(
48
class ClassificationChatRequest(
49
PoolingBasicRequestMixin, ChatRequestMixin, ClassifyRequestMixin
50
):
51
- # --8<-- [start:chat-classification-extra-params]
52
- mm_processor_kwargs: dict[str, Any] | None = Field(
53
- default=None,
54
- description=("Additional kwargs to pass to the HF processor."),
55
- )
56
-
51
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
52
encoder_config = model_config.encoder_config or {}
53
vllm/entrypoints/pooling/embed/protocol.py
+1
-6
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
-from typing import Any, TypeAlias
4
+from typing import TypeAlias
5
6
from pydantic import Field
7
@@ -78,11 +78,6 @@ class EmbeddingCompletionRequest(
78
class EmbeddingChatRequest(
79
PoolingBasicRequestMixin, ChatRequestMixin, EmbedRequestMixin
80
):
81
- mm_processor_kwargs: dict[str, Any] | None = Field(
82
- default=None,
83
- description=("Additional kwargs to pass to the HF processor."),
84
- )
85
-
81
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
82
encoder_config = model_config.encoder_config or {}
83
vllm/entrypoints/pooling/pooling/protocol.py
+1
-6
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
-from typing import Any, Generic, TypeAlias, TypeVar
4
+from typing import Generic, TypeAlias, TypeVar
5
6
from pydantic import Field
7
@@ -65,11 +65,6 @@ class PoolingChatRequest(
65
):
66
task: PoolingTask | None = None
67
68
- mm_processor_kwargs: dict[str, Any] | None = Field(
69
- default=None,
70
- description=("Additional kwargs to pass to the HF processor."),
71
- )
72
-
68
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
69
encoder_config = model_config.encoder_config or {}
70
vllm/entrypoints/pooling/score/protocol.py
+1
-15
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
-from typing import Any, TypeAlias
4
+from typing import TypeAlias
5
6
from pydantic import BaseModel, Field
7
@@ -23,13 +23,6 @@ from vllm.utils import random_uuid
23
24
25
class ScoreRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
26
- # --8<-- [start:score-extra-params]
27
- mm_processor_kwargs: dict[str, Any] | None = Field(
28
- default=None,
29
- description=("Additional kwargs to pass to the HF processor."),
30
- )
31
- # --8<-- [end:score-extra-params]
32
-
26
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
27
encoder_config = model_config.encoder_config or {}
28
@@ -106,13 +99,6 @@ class RerankRequest(PoolingBasicRequestMixin, ClassifyRequestMixin):
99
documents: ScoreInputs
100
top_n: int = Field(default_factory=lambda: 0)
101
109
- # --8<-- [start:rerank-extra-params]
110
- mm_processor_kwargs: dict[str, Any] | None = Field(
111
- default=None,
112
- description=("Additional kwargs to pass to the HF processor."),
113
- )
114
- # --8<-- [end:rerank-extra-params]
115
-
102
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
103
encoder_config = model_config.encoder_config or {}
104
vllm/utils/print_utils.py
new
+7
@@ -0,0 +1,7 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+def print_embeddings(embeds: list[float]):
6
+ embeds_trimmed = (str(embeds[:4])[:-1] + ", ...]") if len(embeds) > 4 else embeds
7
+ print(f"Embeddings: {embeds_trimmed} (size={len(embeds)})")